1#![doc = include_str!("../README.md")]
2
3use std::{
4 any::{type_name, Any, TypeId},
5 collections::{BTreeMap, HashMap},
6 future::Future,
7 io::{self, Read},
8 pin::Pin,
9 sync::{
10 atomic::{AtomicBool, Ordering},
11 Arc, Mutex, OnceLock,
12 },
13 task::{Context as TaskContext, Poll},
14 time::{Duration, Instant, SystemTime, UNIX_EPOCH},
15};
16
17use apache_avro::{from_avro_datum, types::Value as AvroDatum, Schema};
18use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
19use chrono::DateTime;
20use futures_util::{future::OptionFuture, task::noop_waker_ref};
21use serde::{
22 de::DeserializeOwned,
23 ser::{SerializeMap, SerializeSeq},
24 Deserialize, Deserializer, Serialize, Serializer,
25};
26pub use serde_json::{json, Value};
27use sha2::{Digest, Sha256};
28use thiserror::Error;
29pub use uuid::Uuid;
30
31pub const WORKER_PROTOCOL_VERSION: &str = "1.16";
32pub const CONTROL_PLANE_VERSION: &str = "2";
33pub const DEFAULT_CODEC: &str = "avro";
34pub const SDK_VERSION: &str = concat!("durable-workflow-rust/", env!("CARGO_PKG_VERSION"));
35pub const QUERY_TASKS_CAPABILITY: &str = "query_tasks";
37pub const WORKFLOW_UPDATES_CAPABILITY: &str = "workflow_updates";
39pub const QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
41pub const SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
43pub const CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.9";
45
46const MAX_LONG_POLL_TIMEOUT_SECONDS: u64 = 60;
47const WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE: &str =
48 "Workflow task waiting for scheduled history.";
49const WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE: &str = "WorkflowTaskWaitingForHistory";
50const MISSING_TASK_PAYLOAD_CODEC: &str = "\0missing-task-payload-codec";
51const NULL_TASK_PAYLOAD_CODEC: &str = "\0null-task-payload-codec";
52const NON_STRING_TASK_PAYLOAD_CODEC: &str = "\0non-string-task-payload-codec";
53const MAX_MEMO_ENTRIES: usize = 100;
54const MAX_MEMO_VALUE_SIZE_BYTES: usize = 10_240;
55const MAX_MEMO_TOTAL_SIZE_BYTES: usize = 65_536;
56
57const QUERY_TASK_FINAL_REJECTION_REASONS: &[&str] = &[
58 "lease_expired",
59 "query_task_not_found",
60 "query_task_not_leased",
61 "query_task_timed_out",
62];
63
64pub const AVRO_VALUE_SCHEMA_JSON: &str =
66 include_str!("../schema/durable_workflow.protocol.Value.v1.avsc");
67pub const AVRO_VALUE_SCHEMA_FINGERPRINT_HEX: &str = "e2a33dff55802237";
68pub const AVRO_VALUE_SCHEMA_FINGERPRINT: [u8; 8] = [0xe2, 0xa3, 0x3d, 0xff, 0x55, 0x80, 0x22, 0x37];
69const AVRO_SINGLE_OBJECT_MAGIC: [u8; 2] = [0xc3, 0x01];
70
71static AVRO_VALUE_SCHEMA: OnceLock<std::result::Result<Schema, String>> = OnceLock::new();
72
73#[derive(Clone, Copy)]
74enum RequestProtocol {
75 ControlPlane,
76 Worker(&'static str),
77}
78
79pub type Result<T> = std::result::Result<T, Error>;
80
81#[derive(Debug, Error)]
82pub enum Error {
83 #[error("transport error: {0}")]
84 Transport(#[from] reqwest::Error),
85 #[error(
86 "invalid Durable Workflow base URL: omit the SDK-owned /api suffix and pass the Server or Cloud runtime base URL; the SDK appends /api automatically"
87 )]
88 InvalidBaseUrl,
89 #[error("json error: {0}")]
90 Json(#[from] serde_json::Error),
91 #[error("http {status}: {body}")]
92 Http {
93 status: reqwest::StatusCode,
94 body: String,
95 },
96 #[error("codec error: {0}")]
97 Codec(String),
98 #[error(transparent)]
99 QueryFailed(QueryFailure),
100 #[error(transparent)]
101 Protocol(ProtocolFailure),
102 #[error(transparent)]
103 NonDeterministicReplay(ReplayFailure),
104 #[error(transparent)]
105 ChildWorkflowFailed(ChildWorkflowFailure),
106 #[error(transparent)]
107 ActivityFailed(ActivityFailure),
108 #[error(transparent)]
109 ParallelFailed(ParallelFailure),
110 #[error(transparent)]
111 SagaCompensationFailed(SagaCompensationFailure),
112 #[error(transparent)]
113 InvalidParallelGroup(ParallelGroupError),
114 #[error(transparent)]
115 WorkflowCancellationRequested(WorkflowCancellationRequested),
116 #[error(transparent)]
117 WorkflowCommandRejected(WorkflowCommandRejection),
118 #[error(transparent)]
119 WorkflowFailed(WorkflowTerminalOutcome),
120 #[error(transparent)]
121 WorkflowCancelled(WorkflowTerminalOutcome),
122 #[error(transparent)]
123 WorkflowTerminated(WorkflowTerminalOutcome),
124 #[error(transparent)]
125 WorkflowTimedOut(WorkflowTerminalOutcome),
126 #[error(transparent)]
127 ActivityTaskRejected(ActivityTaskRejection),
128 #[error("workflow handler {0:?} is not registered")]
129 WorkflowNotRegistered(String),
130 #[error("activity handler {0:?} is not registered")]
131 ActivityNotRegistered(String),
132 #[error(
133 "{handler_kind} handler {handler_name:?} {value_kind} type {rust_type} is incompatible with the fixed Avro Value codec: {message}"
134 )]
135 HandlerType {
136 handler_kind: HandlerKind,
137 handler_name: String,
138 value_kind: HandlerValueKind,
139 rust_type: &'static str,
140 message: String,
141 },
142 #[error("workflow future yielded without emitting a durable command")]
143 WorkflowYieldedWithoutCommand,
144 #[error(
145 "workflow_stream_command_identity_missing: workflow stream authoring requires a non-empty server-provided workflow_command_id"
146 )]
147 MissingWorkflowCommandIdentity,
148 #[error("workflow state lock is poisoned")]
149 WorkflowStatePoisoned,
150 #[error("timer duration is too large for the worker protocol")]
151 TimerDurationOverflow,
152 #[error(transparent)]
153 InvalidConditionWaitOptions(#[from] ConditionWaitOptionsError),
154 #[error(transparent)]
155 InvalidSearchAttributeUpdate(#[from] SearchAttributeUpdateError),
156 #[error("operation timed out")]
157 Timeout,
158 #[error(
159 "missing {role}-plane credentials: configure ClientBuilder::{role}_token or ClientBuilder::token; a {opposite_role}-plane token cannot authorize this request"
160 )]
161 MissingRoleCredentials {
162 role: &'static str,
163 opposite_role: &'static str,
164 },
165 #[error("worker loop error: {0}")]
166 WorkerLoop(String),
167 #[error(
168 "workflow command contract for {workflow_type:?} declares update validators, but this Rust SDK cannot execute synchronous pre-accept update validation"
169 )]
170 UnsupportedUpdateValidators { workflow_type: String },
171 #[error("{primary}; worker deregistration also failed: {deregistration}")]
172 WorkerShutdown {
173 primary: Box<Error>,
174 deregistration: Box<Error>,
175 },
176 #[error("invalid child workflow options: {0}")]
177 InvalidChildWorkflowOptions(String),
178 #[error("invalid workflow memo update: {0}")]
179 InvalidMemoUpdate(String),
180 #[error(
181 "workflow_memo_updates_unavailable: the connected runtime did not advertise workflow memo update support"
182 )]
183 WorkflowMemoUpdatesUnavailable,
184 #[error(transparent)]
185 InvalidActivityOptions(ActivityOptionsError),
186 #[error(transparent)]
187 InvalidContinueAsNewOptions(#[from] ContinueAsNewOptionsError),
188 #[doc(hidden)]
189 #[error("workflow requested continue as new")]
190 ContinueAsNew(ContinueAsNewRequest),
191}
192
193#[derive(Clone, Debug, Error, PartialEq, Eq)]
195pub enum ConditionWaitOptionsError {
196 #[error("condition_key must be non-empty")]
197 EmptyKey,
198 #[error("condition_definition_fingerprint must be non-empty")]
199 EmptyPredicateIdentity,
200 #[error("condition timeout is too large for the worker protocol")]
201 TimeoutOverflow,
202}
203
204#[derive(Clone, Debug, PartialEq, Eq)]
211pub struct ConditionWaitOptions {
212 condition_key: String,
213 predicate_identity: String,
214 timeout: Option<Duration>,
215}
216
217impl ConditionWaitOptions {
218 pub fn new(condition_key: impl Into<String>, predicate_identity: impl Into<String>) -> Self {
219 Self {
220 condition_key: condition_key.into(),
221 predicate_identity: predicate_identity.into(),
222 timeout: None,
223 }
224 }
225
226 pub fn timeout(mut self, timeout: Duration) -> Self {
227 self.timeout = Some(timeout);
228 self
229 }
230
231 fn validate(
232 &self,
233 ) -> std::result::Result<ValidatedConditionWaitOptions, ConditionWaitOptionsError> {
234 let condition_key = self.condition_key.trim();
235 if condition_key.is_empty() {
236 return Err(ConditionWaitOptionsError::EmptyKey);
237 }
238 let predicate_identity = self.predicate_identity.trim();
239 if predicate_identity.is_empty() {
240 return Err(ConditionWaitOptionsError::EmptyPredicateIdentity);
241 }
242 let timeout_seconds = self
243 .timeout
244 .map(|timeout| {
245 timeout
246 .as_secs()
247 .checked_add(u64::from(timeout.subsec_nanos() > 0))
248 .ok_or(ConditionWaitOptionsError::TimeoutOverflow)
249 })
250 .transpose()?;
251
252 Ok(ValidatedConditionWaitOptions {
253 condition_key: condition_key.to_string(),
254 predicate_identity: predicate_identity.to_string(),
255 timeout_seconds,
256 })
257 }
258}
259
260#[derive(Clone, Debug, PartialEq, Eq)]
261struct ValidatedConditionWaitOptions {
262 condition_key: String,
263 predicate_identity: String,
264 timeout_seconds: Option<u64>,
265}
266
267#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
269#[serde(rename_all = "snake_case")]
270pub enum ConditionWaitResult {
271 Satisfied,
272 TimedOut,
273}
274
275impl ConditionWaitResult {
276 pub fn is_satisfied(self) -> bool {
277 self == Self::Satisfied
278 }
279
280 pub fn is_timed_out(self) -> bool {
281 self == Self::TimedOut
282 }
283}
284
285#[doc(hidden)]
287pub fn __condition_definition_fingerprint(source: &str) -> String {
288 let mut digest = Sha256::new();
289 digest.update(b"durable-workflow-rust.wait-condition.v1\0");
290 digest.update(source.as_bytes());
291 format!("sha256:{:x}", digest.finalize())
292}
293
294#[macro_export]
300macro_rules! wait_condition {
301 ($ctx:expr, $key:expr, timeout: $timeout:expr, $predicate:expr $(,)?) => {{
302 $ctx.wait_condition(
303 $crate::ConditionWaitOptions::new(
304 $key,
305 $crate::__condition_definition_fingerprint(concat!(
306 module_path!(),
307 "\0",
308 stringify!($predicate)
309 )),
310 )
311 .timeout($timeout),
312 $predicate,
313 )
314 }};
315 ($ctx:expr, $key:expr, $predicate:expr $(,)?) => {{
316 $ctx.wait_condition(
317 $crate::ConditionWaitOptions::new(
318 $key,
319 $crate::__condition_definition_fingerprint(concat!(
320 module_path!(),
321 "\0",
322 stringify!($predicate)
323 )),
324 ),
325 $predicate,
326 )
327 }};
328}
329
330const MAX_SEARCH_ATTRIBUTES_PER_UPDATE: usize = 100;
331const MAX_SEARCH_ATTRIBUTE_KEY_LENGTH: usize = 64;
332const MAX_SEARCH_ATTRIBUTE_STRING_LENGTH: usize = 255;
333const MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH: usize = 255;
334const MAX_SEARCH_ATTRIBUTE_UPDATE_BYTES: usize = 65_536;
335
336#[derive(Clone, Debug, Error, PartialEq, Eq)]
338pub enum SearchAttributeUpdateError {
339 #[error("search-attribute update requires at least one attribute")]
340 Empty,
341 #[error("search attribute key {0:?} must be 1-64 URL-safe ASCII characters")]
342 InvalidKey(String),
343 #[error("search-attribute update exceeds the limit of 100 attributes")]
344 TooManyAttributes,
345 #[error("search attribute {key:?} {kind} value exceeds {limit} bytes")]
346 ValueTooLong {
347 key: String,
348 kind: &'static str,
349 limit: usize,
350 },
351 #[error(
352 "search attribute {0:?} must not contain an empty string value; use delete() to remove it"
353 )]
354 EmptyString(String),
355 #[error("search attribute {0:?} has a non-finite float value")]
356 NonFiniteFloat(String),
357 #[error("search attribute {0:?} must use an RFC 3339 datetime with an explicit timezone")]
358 InvalidDateTime(String),
359 #[error("search-attribute update exceeds the 65536-byte protocol limit")]
360 PayloadTooLarge,
361}
362
363#[derive(Clone, Debug, PartialEq)]
365pub enum SearchAttributeValue {
366 String(String),
367 Keyword(String),
368 KeywordList(Vec<String>),
369 Int(i64),
370 Float(f64),
371 Bool(bool),
372 DateTime(String),
373 Delete,
374}
375
376impl SearchAttributeValue {
377 fn type_name(&self) -> Option<&'static str> {
378 match self {
379 Self::String(_) => Some("string"),
380 Self::Keyword(_) => Some("keyword"),
381 Self::KeywordList(_) => Some("keyword_list"),
382 Self::Int(_) => Some("int"),
383 Self::Float(_) => Some("float"),
384 Self::Bool(_) => Some("bool"),
385 Self::DateTime(_) => Some("datetime"),
386 Self::Delete => None,
387 }
388 }
389
390 fn normalized(self, key: &str) -> std::result::Result<Self, SearchAttributeUpdateError> {
391 let normalize_string = |value: String, kind: &'static str, limit: usize| {
392 let value = value.trim().to_string();
393 if value.is_empty() {
394 return Err(SearchAttributeUpdateError::EmptyString(key.to_string()));
395 }
396 if value.len() > limit {
397 return Err(SearchAttributeUpdateError::ValueTooLong {
398 key: key.to_string(),
399 kind,
400 limit,
401 });
402 }
403 Ok(value)
404 };
405
406 match self {
407 Self::String(value) => Ok(Self::String(normalize_string(
408 value,
409 "string",
410 MAX_SEARCH_ATTRIBUTE_STRING_LENGTH,
411 )?)),
412 Self::Keyword(value) => Ok(Self::Keyword(normalize_string(
413 value,
414 "keyword",
415 MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH,
416 )?)),
417 Self::KeywordList(values) => {
418 let values = values
419 .into_iter()
420 .map(|value| {
421 let value = value.trim().to_string();
422 if value.len() > MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH {
423 return Err(SearchAttributeUpdateError::ValueTooLong {
424 key: key.to_string(),
425 kind: "keyword-list entry",
426 limit: MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH,
427 });
428 }
429 Ok(value)
430 })
431 .collect::<std::result::Result<Vec<_>, _>>()?;
432 Ok(Self::KeywordList(values))
433 }
434 Self::Float(value) if !value.is_finite() => {
435 Err(SearchAttributeUpdateError::NonFiniteFloat(key.to_string()))
436 }
437 Self::DateTime(value) => {
438 let value =
439 normalize_string(value, "datetime", MAX_SEARCH_ATTRIBUTE_STRING_LENGTH)?;
440 if DateTime::parse_from_rfc3339(&value).is_err() {
441 return Err(SearchAttributeUpdateError::InvalidDateTime(key.to_string()));
442 }
443 Ok(Self::DateTime(value))
444 }
445 value => Ok(value),
446 }
447 }
448
449 fn into_json(self) -> Value {
450 match self {
451 Self::String(value) | Self::Keyword(value) | Self::DateTime(value) => {
452 Value::String(value)
453 }
454 Self::KeywordList(values) => {
455 Value::Array(values.into_iter().map(Value::String).collect())
456 }
457 Self::Int(value) => json!(value),
458 Self::Float(value) => json!(value),
459 Self::Bool(value) => json!(value),
460 Self::Delete => Value::Null,
461 }
462 }
463}
464
465#[derive(Clone, Debug, Default, PartialEq)]
467pub struct SearchAttributeUpdate {
468 attributes: BTreeMap<String, SearchAttributeValue>,
469}
470
471impl SearchAttributeUpdate {
472 pub fn new() -> Self {
473 Self::default()
474 }
475
476 pub fn set(
477 mut self,
478 key: impl Into<String>,
479 value: SearchAttributeValue,
480 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
481 let key = key.into();
482 validate_search_attribute_key(&key)?;
483 if !self.attributes.contains_key(&key)
484 && self.attributes.len() >= MAX_SEARCH_ATTRIBUTES_PER_UPDATE
485 {
486 return Err(SearchAttributeUpdateError::TooManyAttributes);
487 }
488 self.attributes.insert(key.clone(), value.normalized(&key)?);
489 self.validate_size()?;
490 Ok(self)
491 }
492
493 pub fn string(
494 self,
495 key: impl Into<String>,
496 value: impl Into<String>,
497 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
498 self.set(key, SearchAttributeValue::String(value.into()))
499 }
500
501 pub fn keyword(
502 self,
503 key: impl Into<String>,
504 value: impl Into<String>,
505 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
506 self.set(key, SearchAttributeValue::Keyword(value.into()))
507 }
508
509 pub fn keyword_list<I, V>(
510 self,
511 key: impl Into<String>,
512 values: I,
513 ) -> std::result::Result<Self, SearchAttributeUpdateError>
514 where
515 I: IntoIterator<Item = V>,
516 V: Into<String>,
517 {
518 self.set(
519 key,
520 SearchAttributeValue::KeywordList(values.into_iter().map(Into::into).collect()),
521 )
522 }
523
524 pub fn int(
525 self,
526 key: impl Into<String>,
527 value: i64,
528 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
529 self.set(key, SearchAttributeValue::Int(value))
530 }
531
532 pub fn float(
533 self,
534 key: impl Into<String>,
535 value: f64,
536 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
537 self.set(key, SearchAttributeValue::Float(value))
538 }
539
540 pub fn bool(
541 self,
542 key: impl Into<String>,
543 value: bool,
544 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
545 self.set(key, SearchAttributeValue::Bool(value))
546 }
547
548 pub fn datetime(
549 self,
550 key: impl Into<String>,
551 value: impl Into<String>,
552 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
553 self.set(key, SearchAttributeValue::DateTime(value.into()))
554 }
555
556 pub fn delete(
557 self,
558 key: impl Into<String>,
559 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
560 self.set(key, SearchAttributeValue::Delete)
561 }
562
563 fn validate_size(&self) -> std::result::Result<(), SearchAttributeUpdateError> {
564 let (attributes, _) = self.clone().into_wire_parts();
565 if serde_json::to_vec(&attributes)
566 .map(|payload| payload.len() > MAX_SEARCH_ATTRIBUTE_UPDATE_BYTES)
567 .unwrap_or(true)
568 {
569 return Err(SearchAttributeUpdateError::PayloadTooLarge);
570 }
571 Ok(())
572 }
573
574 fn into_wire_parts(self) -> (Value, BTreeMap<String, String>) {
575 let mut attributes = serde_json::Map::new();
576 let mut attribute_types = BTreeMap::new();
577 for (key, value) in self.attributes {
578 if let Some(type_name) = value.type_name() {
579 attribute_types.insert(key.clone(), type_name.to_string());
580 }
581 attributes.insert(key, value.into_json());
582 }
583 (Value::Object(attributes), attribute_types)
584 }
585
586 fn validate(&self) -> std::result::Result<(), SearchAttributeUpdateError> {
587 if self.attributes.is_empty() {
588 return Err(SearchAttributeUpdateError::Empty);
589 }
590 self.validate_size()
591 }
592}
593
594fn validate_search_attribute_key(key: &str) -> std::result::Result<(), SearchAttributeUpdateError> {
595 let valid = !key.is_empty()
596 && key.len() <= MAX_SEARCH_ATTRIBUTE_KEY_LENGTH
597 && key
598 .bytes()
599 .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-' | b':'));
600 if valid {
601 Ok(())
602 } else {
603 Err(SearchAttributeUpdateError::InvalidKey(key.to_string()))
604 }
605}
606
607#[derive(Clone, Copy, Debug, PartialEq, Eq)]
609pub enum HandlerKind {
610 Workflow,
611 Activity,
612}
613
614impl std::fmt::Display for HandlerKind {
615 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
616 formatter.write_str(match self {
617 Self::Workflow => "workflow",
618 Self::Activity => "activity",
619 })
620 }
621}
622
623#[derive(Clone, Copy, Debug, PartialEq, Eq)]
625pub enum HandlerValueKind {
626 Input,
627 Result,
628}
629
630impl std::fmt::Display for HandlerValueKind {
631 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
632 formatter.write_str(match self {
633 Self::Input => "input",
634 Self::Result => "result",
635 })
636 }
637}
638
639#[derive(Clone, Copy, Debug, PartialEq, Eq)]
641pub enum WorkflowCommandKind {
642 Cancel,
643 Terminate,
644}
645
646impl WorkflowCommandKind {
647 fn as_str(self) -> &'static str {
648 match self {
649 Self::Cancel => "cancel",
650 Self::Terminate => "terminate",
651 }
652 }
653}
654
655#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
657pub struct WorkflowCommandOptions {
658 #[serde(skip_serializing_if = "Option::is_none")]
659 pub reason: Option<String>,
660 #[serde(skip_serializing_if = "Option::is_none")]
661 pub request_id: Option<String>,
662}
663
664#[derive(Clone, Debug, PartialEq, Eq)]
671pub struct WorkflowStartOptions {
672 pub execution_timeout_seconds: u64,
673 pub run_timeout_seconds: u64,
674}
675
676impl Default for WorkflowStartOptions {
677 fn default() -> Self {
678 Self {
679 execution_timeout_seconds: 3600,
680 run_timeout_seconds: 600,
681 }
682 }
683}
684
685impl WorkflowStartOptions {
686 pub fn new() -> Self {
687 Self::default()
688 }
689
690 pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
691 self.execution_timeout_seconds = seconds;
692 self
693 }
694
695 pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
696 self.run_timeout_seconds = seconds;
697 self
698 }
699
700 fn validate(&self) -> Result<()> {
701 if self.execution_timeout_seconds == 0 {
702 return Err(Error::Codec(
703 "execution_timeout_seconds must be at least 1".to_string(),
704 ));
705 }
706 if self.run_timeout_seconds == 0 {
707 return Err(Error::Codec(
708 "run_timeout_seconds must be at least 1".to_string(),
709 ));
710 }
711 if self.run_timeout_seconds > self.execution_timeout_seconds {
712 return Err(Error::Codec(
713 "run_timeout_seconds cannot exceed execution_timeout_seconds".to_string(),
714 ));
715 }
716
717 Ok(())
718 }
719}
720
721#[derive(Clone, Debug, Default, PartialEq, Eq)]
726pub struct ContinueAsNewOptions {
727 pub workflow_type: Option<String>,
728 pub task_queue: Option<String>,
729}
730
731impl ContinueAsNewOptions {
732 pub fn new() -> Self {
733 Self::default()
734 }
735
736 pub fn workflow_type(mut self, workflow_type: impl Into<String>) -> Self {
737 self.workflow_type = Some(workflow_type.into());
738 self
739 }
740
741 pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
742 self.task_queue = Some(task_queue.into());
743 self
744 }
745
746 fn validate(&self) -> std::result::Result<(), ContinueAsNewOptionsError> {
747 for (field, value) in [
748 ("workflow_type", self.workflow_type.as_deref()),
749 ("task_queue", self.task_queue.as_deref()),
750 ] {
751 if value.is_some_and(|value| value.trim().is_empty()) {
752 return Err(ContinueAsNewOptionsError {
753 field,
754 message: format!("{field} must not be empty"),
755 });
756 }
757 }
758 Ok(())
759 }
760}
761
762#[derive(Clone, Debug, Error, PartialEq, Eq)]
764#[error("invalid continue-as-new option {field}: {message}")]
765pub struct ContinueAsNewOptionsError {
766 pub field: &'static str,
767 pub message: String,
768}
769
770#[derive(Clone, Debug, Default, PartialEq, Eq)]
772pub struct WorkflowHistoryBudget {
773 pub event_count: u64,
774 pub size_bytes: Option<u64>,
775 pub continue_as_new_recommended: bool,
776 pub pressure: Option<String>,
777}
778
779#[doc(hidden)]
780#[derive(Clone, Debug)]
781pub struct ContinueAsNewRequest {
782 arguments: AvroValue,
783 options: ContinueAsNewOptions,
784}
785
786impl WorkflowCommandOptions {
787 pub fn new() -> Self {
788 Self::default()
789 }
790
791 pub fn reason(mut self, reason: impl Into<String>) -> Self {
792 self.reason = Some(reason.into());
793 self
794 }
795
796 pub fn request_id(mut self, request_id: impl Into<String>) -> Self {
797 self.request_id = Some(request_id.into());
798 self
799 }
800}
801
802#[derive(Clone, Debug, PartialEq)]
804pub struct WorkflowCommandResult {
805 pub command: WorkflowCommandKind,
806 pub workflow_id: String,
807 pub run_id: Option<String>,
808 pub outcome: Option<String>,
809 pub reason: Option<String>,
810 pub command_status: Option<String>,
811 pub raw: Value,
812}
813
814#[derive(Clone, Debug, Error)]
816#[error("workflow {command:?} rejected ({reason}, HTTP {status}): {message}")]
817pub struct WorkflowCommandRejection {
818 pub command: WorkflowCommandKind,
819 pub status: u16,
820 pub reason: String,
821 pub message: String,
822 pub workflow_id: String,
823 pub run_id: Option<String>,
824 pub target_scope: Option<String>,
825 pub body: Value,
826}
827
828#[derive(Clone, Copy, Debug, PartialEq, Eq)]
830pub enum WorkflowTerminalKind {
831 Failed,
832 Cancelled,
833 Terminated,
834 TimedOut,
835}
836
837#[derive(Clone, Debug, Error)]
843#[error("workflow {workflow_id} run {run_id:?} ended as {kind:?} ({reason})")]
844pub struct WorkflowTerminalOutcome {
845 pub kind: WorkflowTerminalKind,
846 pub workflow_id: String,
847 pub run_id: Option<String>,
848 pub reason: String,
849 pub failure_category: Option<String>,
850 pub failure_id: Option<String>,
851 pub exception_type: Option<String>,
852 pub exception_class: Option<String>,
853 pub non_retryable: Option<bool>,
854 pub message: Option<String>,
855 pub exception: Option<Value>,
856 pub raw: Value,
857}
858
859#[derive(Clone, Debug, Error)]
861#[error("activity task {operation} rejected ({reason}, HTTP {status})")]
862pub struct ActivityTaskRejection {
863 pub operation: String,
864 pub status: u16,
865 pub reason: String,
866 pub task_id: String,
867 pub activity_attempt_id: String,
868 pub cancel_requested: bool,
869 pub can_continue: Option<bool>,
870 pub run_closed_reason: Option<String>,
871 pub body: Value,
872}
873
874#[derive(Clone, Copy, Debug, PartialEq, Eq)]
876pub enum ActivityOptionsErrorKind {
877 EmptyTaskQueue,
878 EmptyRetryPolicy,
879 InvalidMaxAttempts,
880 BackoffWithoutRetryBudget,
881 TooManyBackoffIntervals,
882 InvalidBackoffCoefficient,
883 BackoffGenerationTooLarge,
884 BackoffOverflow,
885 EmptyNonRetryableErrorType,
886 TimeoutNotPositive,
887 TimeoutOverflow,
888 TimeoutOrder,
889}
890
891#[derive(Clone, Debug, Error, PartialEq, Eq)]
893#[error("invalid activity options ({kind:?}, {field:?}): {message}")]
894pub struct ActivityOptionsError {
895 pub kind: ActivityOptionsErrorKind,
896 pub field: Option<&'static str>,
897 pub message: String,
898}
899
900impl ActivityOptionsError {
901 fn new(
902 kind: ActivityOptionsErrorKind,
903 field: Option<&'static str>,
904 message: impl Into<String>,
905 ) -> Self {
906 Self {
907 kind,
908 field,
909 message: message.into(),
910 }
911 }
912}
913
914#[derive(Clone, Copy, Debug, PartialEq, Eq)]
916pub enum ActivityFailureKind {
917 Failed,
918 Cancelled,
919 TimedOut,
920}
921
922#[derive(Clone, Debug, Error)]
927#[error("activity failed ({reason}): {message}")]
928pub struct ActivityFailure {
929 pub kind: ActivityFailureKind,
930 pub reason: String,
931 pub message: String,
932 pub activity_execution_id: Option<String>,
933 pub activity_attempt_id: Option<String>,
934 pub activity_type: Option<String>,
935 pub activity_class: Option<String>,
936 pub attempt_number: Option<u64>,
937 pub failure_id: Option<String>,
938 pub failure_category: Option<String>,
939 pub timeout_kind: Option<String>,
940 pub non_retryable: bool,
941 pub exception_type: Option<String>,
942 pub exception_class: Option<String>,
943 pub code: Option<Value>,
944 pub exception: Option<Value>,
945}
946
947#[derive(Clone, Copy, Debug, PartialEq, Eq)]
949pub enum ChildWorkflowFailureKind {
950 Failed,
951 Cancelled,
952 Terminated,
953}
954
955#[derive(Clone, Debug, Error)]
961#[error("child workflow failed ({reason}): {message}")]
962pub struct ChildWorkflowFailure {
963 pub kind: ChildWorkflowFailureKind,
964 pub reason: String,
965 pub message: String,
966 pub parent_workflow_id: Option<String>,
967 pub parent_workflow_run_id: Option<String>,
968 pub child_workflow_id: Option<String>,
969 pub child_workflow_run_id: Option<String>,
970 pub child_workflow_type: Option<String>,
971 pub failure_id: Option<String>,
972 pub failure_category: Option<String>,
973 pub exception_type: Option<String>,
974 pub exception_class: Option<String>,
975 pub non_retryable: bool,
976 pub code: Option<Value>,
977 pub exception: Option<Value>,
978}
979
980#[derive(Clone, Debug, PartialEq, Eq)]
982pub struct WorkflowIdentity {
983 pub workflow_id: Option<String>,
984 pub run_id: Option<String>,
985}
986
987#[derive(Clone, Debug, PartialEq)]
989pub struct ChildWorkflowResult {
990 pub parent: WorkflowIdentity,
991 pub child: WorkflowIdentity,
992 pub child_workflow_type: Option<String>,
993 pub result: Value,
994}
995
996#[derive(Clone, Debug, PartialEq)]
998pub struct ChildWorkflowAvroResult {
999 pub parent: WorkflowIdentity,
1000 pub child: WorkflowIdentity,
1001 pub child_workflow_type: Option<String>,
1002 pub result: AvroValue,
1003}
1004
1005#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
1011pub struct ParallelGroupMetadata {
1012 pub parallel_group_id: String,
1013 pub parallel_group_kind: String,
1014 pub parallel_group_base_sequence: u64,
1015 pub parallel_group_size: usize,
1016 pub parallel_group_index: usize,
1017}
1018
1019#[derive(Clone, Debug, PartialEq)]
1021pub enum ParallelResult {
1022 Activity(Value),
1023 ChildWorkflow(ChildWorkflowResult),
1024 Timer,
1025 Group(Vec<ParallelResult>),
1026}
1027
1028#[derive(Clone, Debug, PartialEq)]
1030pub enum ParallelAvroResult {
1031 Activity(AvroValue),
1032 ChildWorkflow(ChildWorkflowAvroResult),
1033 Timer,
1034 Group(Vec<ParallelAvroResult>),
1035}
1036
1037impl ParallelAvroResult {
1038 fn into_json_result(self) -> Result<ParallelResult> {
1039 match self {
1040 Self::Activity(value) => Ok(ParallelResult::Activity(value.into_json()?)),
1041 Self::ChildWorkflow(result) => Ok(ParallelResult::ChildWorkflow(ChildWorkflowResult {
1042 parent: result.parent,
1043 child: result.child,
1044 child_workflow_type: result.child_workflow_type,
1045 result: result.result.into_json()?,
1046 })),
1047 Self::Timer => Ok(ParallelResult::Timer),
1048 Self::Group(results) => Ok(ParallelResult::Group(
1049 results
1050 .into_iter()
1051 .map(Self::into_json_result)
1052 .collect::<Result<Vec<_>>>()?,
1053 )),
1054 }
1055 }
1056}
1057
1058#[derive(Clone, Debug, PartialEq)]
1060pub struct ParallelCompletion {
1061 pub member_path: Vec<usize>,
1062 pub result: ParallelResult,
1063}
1064
1065#[derive(Debug, Error)]
1073#[error("parallel group {group_id} member {member_path:?} failed: {cause}")]
1074pub struct ParallelFailure {
1075 pub group_id: String,
1076 pub member_path: Vec<usize>,
1077 pub group_path: Vec<ParallelGroupMetadata>,
1078 pub completed: Vec<ParallelCompletion>,
1079 #[source]
1080 pub cause: Box<Error>,
1081}
1082
1083#[derive(Clone, Debug, Error, PartialEq, Eq)]
1085#[error("invalid deterministic parallel group ({reason}): {message}")]
1086pub struct ParallelGroupError {
1087 pub reason: &'static str,
1088 pub member_path: Vec<usize>,
1089 pub message: String,
1090}
1091
1092#[derive(Clone, Debug, Error, PartialEq, Eq)]
1094#[error("workflow cancellation was requested")]
1095pub struct WorkflowCancellationRequested;
1096
1097#[derive(Debug, Error)]
1099#[error(
1100 "saga forward execution failed; compensation activity {compensation_activity_type} (registration {compensation_registration_order}) also failed: {compensation_failure}"
1101)]
1102pub struct SagaCompensationFailure {
1103 pub initiating_failure: Box<Error>,
1104 pub compensation_failure: Box<Error>,
1105 pub compensation_activity_type: String,
1106 pub compensation_registration_order: usize,
1107}
1108
1109#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
1111pub enum ParentClosePolicy {
1112 #[default]
1113 Abandon,
1114 RequestCancel,
1115 Terminate,
1116}
1117
1118impl ParentClosePolicy {
1119 fn as_str(self) -> &'static str {
1120 match self {
1121 Self::Abandon => "abandon",
1122 Self::RequestCancel => "request_cancel",
1123 Self::Terminate => "terminate",
1124 }
1125 }
1126}
1127
1128#[derive(Clone, Debug, Default, PartialEq, Eq)]
1130pub struct ChildWorkflowRetryPolicy {
1131 pub max_attempts: Option<u32>,
1132 pub backoff_seconds: Vec<u64>,
1133 pub non_retryable_error_types: Vec<String>,
1134}
1135
1136#[derive(Clone, Debug, PartialEq, Eq)]
1140pub struct ChildWorkflowOptions {
1141 pub task_queue: String,
1142 pub parent_close_policy: ParentClosePolicy,
1143 pub retry_policy: Option<ChildWorkflowRetryPolicy>,
1144 pub execution_timeout_seconds: Option<u64>,
1145 pub run_timeout_seconds: Option<u64>,
1146}
1147
1148impl ChildWorkflowOptions {
1149 pub fn new(task_queue: impl Into<String>) -> Self {
1150 Self {
1151 task_queue: task_queue.into(),
1152 parent_close_policy: ParentClosePolicy::Abandon,
1153 retry_policy: None,
1154 execution_timeout_seconds: None,
1155 run_timeout_seconds: None,
1156 }
1157 }
1158
1159 pub fn parent_close_policy(mut self, policy: ParentClosePolicy) -> Self {
1160 self.parent_close_policy = policy;
1161 self
1162 }
1163
1164 pub fn retry_policy(mut self, policy: ChildWorkflowRetryPolicy) -> Self {
1165 self.retry_policy = Some(policy);
1166 self
1167 }
1168
1169 pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
1170 self.execution_timeout_seconds = Some(seconds);
1171 self
1172 }
1173
1174 pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
1175 self.run_timeout_seconds = Some(seconds);
1176 self
1177 }
1178}
1179
1180#[derive(Clone, Debug, PartialEq, Eq)]
1182pub enum ActivityBackoff {
1183 Explicit(Vec<Duration>),
1186 Exponential {
1188 initial_interval: Duration,
1189 coefficient: u32,
1190 maximum_interval: Option<Duration>,
1191 },
1192}
1193
1194#[derive(Clone, Debug, Default, PartialEq, Eq)]
1196pub struct ActivityRetryPolicy {
1197 pub max_attempts: Option<u32>,
1198 pub backoff: Option<ActivityBackoff>,
1199 pub non_retryable_error_types: Vec<String>,
1200}
1201
1202impl ActivityRetryPolicy {
1203 pub fn new(max_attempts: u32) -> Self {
1205 Self {
1206 max_attempts: Some(max_attempts),
1207 ..Self::default()
1208 }
1209 }
1210
1211 pub fn backoff_intervals(mut self, intervals: impl IntoIterator<Item = Duration>) -> Self {
1212 self.backoff = Some(ActivityBackoff::Explicit(intervals.into_iter().collect()));
1213 self
1214 }
1215
1216 pub fn exponential_backoff(
1217 mut self,
1218 initial_interval: Duration,
1219 coefficient: u32,
1220 maximum_interval: Option<Duration>,
1221 ) -> Self {
1222 self.backoff = Some(ActivityBackoff::Exponential {
1223 initial_interval,
1224 coefficient,
1225 maximum_interval,
1226 });
1227 self
1228 }
1229
1230 pub fn non_retryable_error_type(mut self, error_type: impl Into<String>) -> Self {
1231 self.non_retryable_error_types.push(error_type.into());
1232 self
1233 }
1234
1235 pub fn non_retryable_error_types(
1236 mut self,
1237 error_types: impl IntoIterator<Item = impl Into<String>>,
1238 ) -> Self {
1239 self.non_retryable_error_types
1240 .extend(error_types.into_iter().map(Into::into));
1241 self
1242 }
1243}
1244
1245#[derive(Clone, Debug, Default, PartialEq, Eq)]
1250pub struct ActivityOptions {
1251 pub task_queue: Option<String>,
1252 pub retry_policy: Option<ActivityRetryPolicy>,
1253 pub start_to_close_timeout: Option<Duration>,
1254 pub schedule_to_start_timeout: Option<Duration>,
1255 pub schedule_to_close_timeout: Option<Duration>,
1256 pub heartbeat_timeout: Option<Duration>,
1257}
1258
1259impl ActivityOptions {
1260 pub fn new() -> Self {
1261 Self::default()
1262 }
1263
1264 pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
1265 self.task_queue = Some(task_queue.into());
1266 self
1267 }
1268
1269 pub fn retry_policy(mut self, policy: ActivityRetryPolicy) -> Self {
1270 self.retry_policy = Some(policy);
1271 self
1272 }
1273
1274 pub fn start_to_close_timeout(mut self, timeout: Duration) -> Self {
1275 self.start_to_close_timeout = Some(timeout);
1276 self
1277 }
1278
1279 pub fn schedule_to_start_timeout(mut self, timeout: Duration) -> Self {
1280 self.schedule_to_start_timeout = Some(timeout);
1281 self
1282 }
1283
1284 pub fn schedule_to_close_timeout(mut self, timeout: Duration) -> Self {
1285 self.schedule_to_close_timeout = Some(timeout);
1286 self
1287 }
1288
1289 pub fn heartbeat_timeout(mut self, timeout: Duration) -> Self {
1290 self.heartbeat_timeout = Some(timeout);
1291 self
1292 }
1293
1294 fn validate(&self) -> std::result::Result<ValidatedActivityOptions, ActivityOptionsError> {
1295 if self
1296 .task_queue
1297 .as_deref()
1298 .is_some_and(|queue| queue.trim().is_empty())
1299 {
1300 return Err(ActivityOptionsError::new(
1301 ActivityOptionsErrorKind::EmptyTaskQueue,
1302 Some("task_queue"),
1303 "task_queue must not be empty",
1304 ));
1305 }
1306
1307 for (field, value) in [
1308 ("start_to_close_timeout", self.start_to_close_timeout),
1309 ("schedule_to_start_timeout", self.schedule_to_start_timeout),
1310 ("schedule_to_close_timeout", self.schedule_to_close_timeout),
1311 ("heartbeat_timeout", self.heartbeat_timeout),
1312 ] {
1313 if value.is_some_and(|value| value.is_zero()) {
1314 return Err(ActivityOptionsError::new(
1315 ActivityOptionsErrorKind::TimeoutNotPositive,
1316 Some(field),
1317 format!("{field} must be positive"),
1318 ));
1319 }
1320 }
1321
1322 validate_timeout_order(
1323 "heartbeat_timeout",
1324 self.heartbeat_timeout,
1325 "start_to_close_timeout",
1326 self.start_to_close_timeout,
1327 )?;
1328 validate_timeout_order(
1329 "start_to_close_timeout",
1330 self.start_to_close_timeout,
1331 "schedule_to_close_timeout",
1332 self.schedule_to_close_timeout,
1333 )?;
1334 validate_timeout_order(
1335 "schedule_to_start_timeout",
1336 self.schedule_to_start_timeout,
1337 "schedule_to_close_timeout",
1338 self.schedule_to_close_timeout,
1339 )?;
1340
1341 Ok(ValidatedActivityOptions {
1342 task_queue: self.task_queue.clone(),
1343 retry_policy: self
1344 .retry_policy
1345 .as_ref()
1346 .map(validate_activity_retry_policy)
1347 .transpose()?,
1348 start_to_close_timeout: timeout_seconds(
1349 "start_to_close_timeout",
1350 self.start_to_close_timeout,
1351 )?,
1352 schedule_to_start_timeout: timeout_seconds(
1353 "schedule_to_start_timeout",
1354 self.schedule_to_start_timeout,
1355 )?,
1356 schedule_to_close_timeout: timeout_seconds(
1357 "schedule_to_close_timeout",
1358 self.schedule_to_close_timeout,
1359 )?,
1360 heartbeat_timeout: timeout_seconds("heartbeat_timeout", self.heartbeat_timeout)?,
1361 })
1362 }
1363}
1364
1365pub enum ParallelOperation {
1371 Activity {
1372 activity_type: String,
1373 options: ActivityOptions,
1374 arguments: Result<AvroValue>,
1375 },
1376 ChildWorkflow {
1377 workflow_type: String,
1378 options: ChildWorkflowOptions,
1379 arguments: Result<AvroValue>,
1380 },
1381 Timer(Duration),
1382 Group(Vec<ParallelOperation>),
1383}
1384
1385impl ParallelOperation {
1386 pub fn activity<T: Serialize>(activity_type: impl Into<String>, args: T) -> Self {
1387 Self::activity_with_options(activity_type, ActivityOptions::new(), args)
1388 }
1389
1390 pub fn activity_with_options<T: Serialize>(
1391 activity_type: impl Into<String>,
1392 options: ActivityOptions,
1393 args: T,
1394 ) -> Self {
1395 Self::Activity {
1396 activity_type: activity_type.into(),
1397 options,
1398 arguments: AvroValue::from_serialize(&args),
1399 }
1400 }
1401
1402 pub fn child_workflow<T: Serialize>(
1403 workflow_type: impl Into<String>,
1404 options: ChildWorkflowOptions,
1405 args: T,
1406 ) -> Self {
1407 Self::ChildWorkflow {
1408 workflow_type: workflow_type.into(),
1409 options,
1410 arguments: AvroValue::from_serialize(&args),
1411 }
1412 }
1413
1414 pub fn timer(duration: Duration) -> Self {
1415 Self::Timer(duration)
1416 }
1417
1418 pub fn group(operations: Vec<ParallelOperation>) -> Self {
1419 Self::Group(operations)
1420 }
1421}
1422
1423#[derive(Clone, Debug)]
1424struct ValidatedActivityOptions {
1425 task_queue: Option<String>,
1426 retry_policy: Option<Value>,
1427 start_to_close_timeout: Option<u64>,
1428 schedule_to_start_timeout: Option<u64>,
1429 schedule_to_close_timeout: Option<u64>,
1430 heartbeat_timeout: Option<u64>,
1431}
1432
1433fn validate_timeout_order(
1434 smaller_name: &'static str,
1435 smaller: Option<Duration>,
1436 larger_name: &'static str,
1437 larger: Option<Duration>,
1438) -> std::result::Result<(), ActivityOptionsError> {
1439 if matches!((smaller, larger), (Some(smaller), Some(larger)) if smaller > larger) {
1440 return Err(ActivityOptionsError::new(
1441 ActivityOptionsErrorKind::TimeoutOrder,
1442 Some(smaller_name),
1443 format!("{smaller_name} must be <= {larger_name}"),
1444 ));
1445 }
1446 Ok(())
1447}
1448
1449fn timeout_seconds(
1450 field: &'static str,
1451 value: Option<Duration>,
1452) -> std::result::Result<Option<u64>, ActivityOptionsError> {
1453 value
1454 .map(|value| {
1455 activity_protocol_seconds(value).ok_or_else(|| {
1456 ActivityOptionsError::new(
1457 ActivityOptionsErrorKind::TimeoutOverflow,
1458 Some(field),
1459 format!("{field} is too large for the worker protocol"),
1460 )
1461 })
1462 })
1463 .transpose()
1464}
1465
1466fn duration_seconds_ceil(value: Duration) -> Option<u64> {
1467 value
1468 .as_secs()
1469 .checked_add(u64::from(value.subsec_nanos() > 0))
1470}
1471
1472fn activity_protocol_seconds(value: Duration) -> Option<u64> {
1473 duration_seconds_ceil(value).filter(|seconds| *seconds <= i64::MAX as u64)
1474}
1475
1476fn validate_activity_retry_policy(
1477 policy: &ActivityRetryPolicy,
1478) -> std::result::Result<Value, ActivityOptionsError> {
1479 if policy.max_attempts.is_none()
1480 && policy.backoff.is_none()
1481 && policy.non_retryable_error_types.is_empty()
1482 {
1483 return Err(ActivityOptionsError::new(
1484 ActivityOptionsErrorKind::EmptyRetryPolicy,
1485 Some("retry_policy"),
1486 "retry_policy must configure at least one field",
1487 ));
1488 }
1489 if policy.max_attempts == Some(0) {
1490 return Err(ActivityOptionsError::new(
1491 ActivityOptionsErrorKind::InvalidMaxAttempts,
1492 Some("retry_policy.max_attempts"),
1493 "max_attempts must be >= 1",
1494 ));
1495 }
1496 if policy
1497 .non_retryable_error_types
1498 .iter()
1499 .any(|error_type| error_type.trim().is_empty())
1500 {
1501 return Err(ActivityOptionsError::new(
1502 ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
1503 Some("retry_policy.non_retryable_error_types"),
1504 "non_retryable_error_types must not contain empty values",
1505 ));
1506 }
1507
1508 let backoff_seconds = match &policy.backoff {
1509 None => None,
1510 Some(backoff) => {
1511 let max_attempts = policy.max_attempts.ok_or_else(|| {
1512 ActivityOptionsError::new(
1513 ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
1514 Some("retry_policy.backoff"),
1515 "backoff requires max_attempts",
1516 )
1517 })?;
1518 let retry_count = max_attempts.saturating_sub(1) as usize;
1519 let intervals = match backoff {
1520 ActivityBackoff::Explicit(intervals) => {
1521 if intervals.len() > retry_count {
1522 return Err(ActivityOptionsError::new(
1523 ActivityOptionsErrorKind::TooManyBackoffIntervals,
1524 Some("retry_policy.backoff"),
1525 "backoff interval count must not exceed max_attempts - 1",
1526 ));
1527 }
1528 intervals.clone()
1529 }
1530 ActivityBackoff::Exponential {
1531 initial_interval,
1532 coefficient,
1533 maximum_interval,
1534 } => {
1535 if *coefficient < 1 {
1536 return Err(ActivityOptionsError::new(
1537 ActivityOptionsErrorKind::InvalidBackoffCoefficient,
1538 Some("retry_policy.backoff.coefficient"),
1539 "backoff coefficient must be >= 1",
1540 ));
1541 }
1542 if retry_count > 10_000 {
1543 return Err(ActivityOptionsError::new(
1544 ActivityOptionsErrorKind::BackoffGenerationTooLarge,
1545 Some("retry_policy.max_attempts"),
1546 "generated backoff supports at most 10000 retry intervals",
1547 ));
1548 }
1549 let mut current = *initial_interval;
1550 let mut intervals = Vec::with_capacity(retry_count);
1551 for _ in 0..retry_count {
1552 let interval = maximum_interval
1553 .map(|maximum| current.min(maximum))
1554 .unwrap_or(current);
1555 intervals.push(interval);
1556 if maximum_interval.is_some_and(|maximum| interval == maximum) {
1557 break;
1558 }
1559 current = current.checked_mul(*coefficient).ok_or_else(|| {
1560 ActivityOptionsError::new(
1561 ActivityOptionsErrorKind::BackoffOverflow,
1562 Some("retry_policy.backoff"),
1563 "generated backoff interval overflowed",
1564 )
1565 })?;
1566 }
1567 intervals
1568 }
1569 };
1570 Some(
1571 intervals
1572 .into_iter()
1573 .map(|interval| {
1574 activity_protocol_seconds(interval).ok_or_else(|| {
1575 ActivityOptionsError::new(
1576 ActivityOptionsErrorKind::BackoffOverflow,
1577 Some("retry_policy.backoff"),
1578 "backoff interval is too large for the worker protocol",
1579 )
1580 })
1581 })
1582 .collect::<std::result::Result<Vec<_>, _>>()?,
1583 )
1584 }
1585 };
1586
1587 let mut encoded = serde_json::Map::new();
1588 if let Some(max_attempts) = policy.max_attempts {
1589 encoded.insert("max_attempts".to_string(), json!(max_attempts));
1590 }
1591 if let Some(backoff_seconds) = backoff_seconds {
1592 encoded.insert("backoff_seconds".to_string(), json!(backoff_seconds));
1593 }
1594 if !policy.non_retryable_error_types.is_empty() {
1595 let mut canonical_error_types = Vec::new();
1596 for error_type in policy
1597 .non_retryable_error_types
1598 .iter()
1599 .map(|error_type| error_type.trim())
1600 {
1601 if !canonical_error_types.contains(&error_type) {
1602 canonical_error_types.push(error_type);
1603 }
1604 }
1605 encoded.insert(
1606 "non_retryable_error_types".to_string(),
1607 json!(canonical_error_types),
1608 );
1609 }
1610 Ok(Value::Object(encoded))
1611}
1612
1613#[derive(Clone, Debug, Error)]
1616#[error("non-deterministic workflow replay ({reason}) at sequence {sequence:?}: {message}")]
1617pub struct ReplayFailure {
1618 pub reason: String,
1619 pub sequence: Option<u64>,
1620 pub expected: Option<String>,
1621 pub actual: Option<String>,
1622 pub message: String,
1623}
1624
1625impl ReplayFailure {
1626 fn new(
1627 reason: impl Into<String>,
1628 sequence: Option<u64>,
1629 expected: Option<String>,
1630 actual: Option<String>,
1631 message: impl Into<String>,
1632 ) -> Self {
1633 Self {
1634 reason: reason.into(),
1635 sequence,
1636 expected,
1637 actual,
1638 message: message.into(),
1639 }
1640 }
1641}
1642
1643#[derive(Clone, Debug, Error)]
1645#[error("query failed ({reason}, HTTP {status}): {message}")]
1646pub struct QueryFailure {
1647 pub status: u16,
1648 pub reason: String,
1649 pub message: String,
1650 pub body: Value,
1651}
1652
1653#[derive(Clone, Debug, Error)]
1655#[error("protocol rejected ({reason}, HTTP {status}): {message}")]
1656pub struct ProtocolFailure {
1657 pub status: u16,
1658 pub reason: String,
1659 pub message: String,
1660 pub supported_version: Option<String>,
1661 pub requested_version: Option<String>,
1662 pub body: Value,
1663}
1664
1665#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
1666pub struct PayloadEnvelope {
1667 pub codec: String,
1668 pub blob: String,
1669}
1670
1671impl PayloadEnvelope {
1672 pub fn avro<T: Serialize>(value: &T) -> Result<Self> {
1673 encode_payload(value, DEFAULT_CODEC)
1674 }
1675
1676 pub fn avro_value(value: &AvroValue) -> Result<Self> {
1679 encode_avro_value(value)
1680 }
1681}
1682
1683#[derive(Clone, Debug)]
1685pub enum AvroValue {
1686 Null,
1687 Boolean(bool),
1688 Long(i64),
1689 Double(f64),
1690 Bytes(Vec<u8>),
1691 String(String),
1692 Array(Vec<AvroValue>),
1693 Map(BTreeMap<String, AvroValue>),
1694}
1695
1696impl PartialEq for AvroValue {
1697 fn eq(&self, other: &Self) -> bool {
1698 match (self, other) {
1699 (Self::Null, Self::Null) => true,
1700 (Self::Boolean(left), Self::Boolean(right)) => left == right,
1701 (Self::Long(left), Self::Long(right)) => left == right,
1702 (Self::Double(left), Self::Double(right)) => left.to_bits() == right.to_bits(),
1703 (Self::Bytes(left), Self::Bytes(right)) => left == right,
1704 (Self::String(left), Self::String(right)) => left == right,
1705 (Self::Array(left), Self::Array(right)) => left == right,
1706 (Self::Map(left), Self::Map(right)) => left == right,
1707 _ => false,
1708 }
1709 }
1710}
1711
1712impl AvroValue {
1713 fn from_serialize<T: Serialize>(value: &T) -> Result<Self> {
1714 Self::from_serde_value(
1715 serde_value::to_value(value).map_err(|error| {
1716 Error::Codec(format!("could not adapt value for Avro: {error}"))
1717 })?,
1718 )
1719 }
1720
1721 fn from_serde_value(value: serde_value::Value) -> Result<Self> {
1722 use serde_value::Value as SerdeValue;
1723
1724 match value {
1725 SerdeValue::Unit => Ok(Self::Null),
1726 SerdeValue::Bool(value) => Ok(Self::Boolean(value)),
1727 SerdeValue::I8(value) => Ok(Self::Long(i64::from(value))),
1728 SerdeValue::I16(value) => Ok(Self::Long(i64::from(value))),
1729 SerdeValue::I32(value) => Ok(Self::Long(i64::from(value))),
1730 SerdeValue::I64(value) => Ok(Self::Long(value)),
1731 SerdeValue::U8(value) => Ok(Self::Long(i64::from(value))),
1732 SerdeValue::U16(value) => Ok(Self::Long(i64::from(value))),
1733 SerdeValue::U32(value) => Ok(Self::Long(i64::from(value))),
1734 SerdeValue::U64(value) => i64::try_from(value).map(Self::Long).map_err(|_| {
1735 Error::Codec(
1736 "integer_overflow: Avro Value long must be within signed 64-bit range"
1737 .to_string(),
1738 )
1739 }),
1740 SerdeValue::F32(value) => Self::finite_double(f64::from(value)),
1741 SerdeValue::F64(value) => Self::finite_double(value),
1742 SerdeValue::Char(value) => Ok(Self::String(value.to_string())),
1743 SerdeValue::String(value) => Ok(Self::String(value)),
1744 SerdeValue::Bytes(value) => Ok(Self::Bytes(value)),
1745 SerdeValue::Option(None) => Ok(Self::Null),
1746 SerdeValue::Option(Some(value)) | SerdeValue::Newtype(value) => {
1747 Self::from_serde_value(*value)
1748 }
1749 SerdeValue::Seq(values) => values
1750 .into_iter()
1751 .map(Self::from_serde_value)
1752 .collect::<Result<Vec<_>>>()
1753 .map(Self::Array),
1754 SerdeValue::Map(values) => values
1755 .into_iter()
1756 .map(|(key, value)| {
1757 let SerdeValue::String(key) = key else {
1758 return Err(Error::Codec(
1759 "invalid_map_key: Avro Value map keys must be strings".to_string(),
1760 ));
1761 };
1762
1763 Ok((key, Self::from_serde_value(value)?))
1764 })
1765 .collect::<Result<BTreeMap<_, _>>>()
1766 .map(Self::Map),
1767 }
1768 }
1769
1770 fn finite_double(value: f64) -> Result<Self> {
1771 if !value.is_finite() {
1772 return Err(Error::Codec(
1773 "non_finite_float: Avro Value doubles must be finite".to_string(),
1774 ));
1775 }
1776
1777 Ok(Self::Double(value))
1778 }
1779
1780 fn into_json(self) -> Result<Value> {
1781 match self {
1782 Self::Null => Ok(Value::Null),
1783 Self::Boolean(value) => Ok(Value::Bool(value)),
1784 Self::Long(value) => Ok(Value::Number(value.into())),
1785 Self::Double(value) => serde_json::Number::from_f64(value)
1786 .map(Value::Number)
1787 .ok_or_else(|| {
1788 Error::Codec(
1789 "non_finite_float: decoded Avro Value double is not finite".to_string(),
1790 )
1791 }),
1792 Self::Bytes(value) => Ok(json!({
1793 "$type": "bytes",
1794 "base64": BASE64.encode(value),
1795 })),
1796 Self::String(value) => Ok(Value::String(value)),
1797 Self::Array(values) => values
1798 .into_iter()
1799 .map(Self::into_json)
1800 .collect::<Result<Vec<_>>>()
1801 .map(Value::Array),
1802 Self::Map(values) => values
1803 .into_iter()
1804 .map(|(key, value)| Ok((key, value.into_json()?)))
1805 .collect::<Result<serde_json::Map<_, _>>>()
1806 .map(Value::Object),
1807 }
1808 }
1809
1810 fn into_serde_value(self) -> serde_value::Value {
1811 use serde_value::Value as SerdeValue;
1812
1813 match self {
1814 Self::Null => SerdeValue::Unit,
1815 Self::Boolean(value) => SerdeValue::Bool(value),
1816 Self::Long(value) => SerdeValue::I64(value),
1817 Self::Double(value) => SerdeValue::F64(value),
1818 Self::Bytes(value) => SerdeValue::Bytes(value),
1819 Self::String(value) => SerdeValue::String(value),
1820 Self::Array(values) => {
1821 SerdeValue::Seq(values.into_iter().map(Self::into_serde_value).collect())
1822 }
1823 Self::Map(values) => SerdeValue::Map(
1824 values
1825 .into_iter()
1826 .map(|(key, value)| (SerdeValue::String(key), value.into_serde_value()))
1827 .collect(),
1828 ),
1829 }
1830 }
1831
1832 pub fn deserialize<T: DeserializeOwned>(self) -> Result<T> {
1833 self.into_serde_value().deserialize_into().map_err(|error| {
1834 Error::Codec(format!(
1835 "avro_value_type_mismatch: could not adapt decoded value: {error}"
1836 ))
1837 })
1838 }
1839}
1840
1841impl Serialize for AvroValue {
1842 fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
1843 where
1844 S: Serializer,
1845 {
1846 match self {
1847 Self::Null => serializer.serialize_unit(),
1848 Self::Boolean(value) => serializer.serialize_bool(*value),
1849 Self::Long(value) => serializer.serialize_i64(*value),
1850 Self::Double(value) => serializer.serialize_f64(*value),
1851 Self::Bytes(value) => serializer.serialize_bytes(value),
1852 Self::String(value) => serializer.serialize_str(value),
1853 Self::Array(values) => {
1854 let mut sequence = serializer.serialize_seq(Some(values.len()))?;
1855 for value in values {
1856 sequence.serialize_element(value)?;
1857 }
1858 sequence.end()
1859 }
1860 Self::Map(values) => {
1861 let mut map = serializer.serialize_map(Some(values.len()))?;
1862 for (key, value) in values {
1863 map.serialize_entry(key, value)?;
1864 }
1865 map.end()
1866 }
1867 }
1868 }
1869}
1870
1871pub fn encode_avro_value(value: &AvroValue) -> Result<PayloadEnvelope> {
1872 let mut bytes = Vec::new();
1873 bytes.extend_from_slice(&AVRO_SINGLE_OBJECT_MAGIC);
1874 bytes.extend_from_slice(&AVRO_VALUE_SCHEMA_FINGERPRINT);
1875 encode_avro_value_datum(&mut bytes, value)?;
1876 Ok(PayloadEnvelope {
1877 codec: DEFAULT_CODEC.to_string(),
1878 blob: BASE64.encode(bytes),
1879 })
1880}
1881
1882pub fn decode_avro_value(envelope: &PayloadEnvelope) -> Result<AvroValue> {
1883 if envelope.codec != DEFAULT_CODEC {
1884 return Err(unsupported_payload_codec(&envelope.codec));
1885 }
1886 decode_avro_value_blob(&envelope.blob)
1887}
1888
1889pub fn encode_payload<T: Serialize>(value: &T, codec: &str) -> Result<PayloadEnvelope> {
1890 let blob = match codec {
1891 DEFAULT_CODEC => encode_avro_value(&AvroValue::from_serialize(value)?)?.blob,
1892 other => return Err(unsupported_payload_codec(other)),
1893 };
1894
1895 Ok(PayloadEnvelope {
1896 codec: codec.to_string(),
1897 blob,
1898 })
1899}
1900
1901pub fn decode_payload<T: DeserializeOwned>(envelope: &PayloadEnvelope) -> Result<T> {
1902 match envelope.codec.as_str() {
1903 DEFAULT_CODEC => decode_avro_value(envelope)?.deserialize(),
1904 other => Err(unsupported_payload_codec(other)),
1905 }
1906}
1907
1908fn handler_type_error<T>(
1909 handler_kind: HandlerKind,
1910 handler_name: &str,
1911 value_kind: HandlerValueKind,
1912 message: impl Into<String>,
1913) -> Error {
1914 Error::HandlerType {
1915 handler_kind,
1916 handler_name: handler_name.to_string(),
1917 value_kind,
1918 rust_type: type_name::<T>(),
1919 message: message.into(),
1920 }
1921}
1922
1923fn decode_handler_input<T: DeserializeOwned>(
1924 arguments: AvroValue,
1925 handler_kind: HandlerKind,
1926 handler_name: &str,
1927) -> Result<T> {
1928 let argument = match arguments {
1929 AvroValue::Array(mut arguments) if arguments.len() == 1 => {
1930 arguments.pop().expect("one typed handler argument")
1931 }
1932 AvroValue::Array(arguments) if arguments.is_empty() => AvroValue::Null,
1933 AvroValue::Array(arguments) => {
1934 return Err(handler_type_error::<T>(
1935 handler_kind,
1936 handler_name,
1937 HandlerValueKind::Input,
1938 format!(
1939 "typed handlers accept one request value, but the task carried {} arguments",
1940 arguments.len()
1941 ),
1942 ));
1943 }
1944 argument => argument,
1945 };
1946
1947 argument.deserialize().map_err(|error| {
1948 handler_type_error::<T>(
1949 handler_kind,
1950 handler_name,
1951 HandlerValueKind::Input,
1952 error.to_string(),
1953 )
1954 })
1955}
1956
1957fn encode_handler_result<T: Serialize>(
1958 result: &T,
1959 handler_kind: HandlerKind,
1960 handler_name: &str,
1961) -> Result<AvroValue> {
1962 AvroValue::from_serialize(result).map_err(|error| {
1963 handler_type_error::<T>(
1964 handler_kind,
1965 handler_name,
1966 HandlerValueKind::Result,
1967 error.to_string(),
1968 )
1969 })
1970}
1971
1972fn decode_handler_result<T: DeserializeOwned>(
1973 result: AvroValue,
1974 handler_kind: HandlerKind,
1975 handler_name: &str,
1976) -> Result<T> {
1977 result.deserialize().map_err(|error| {
1978 handler_type_error::<T>(
1979 handler_kind,
1980 handler_name,
1981 HandlerValueKind::Result,
1982 error.to_string(),
1983 )
1984 })
1985}
1986
1987#[cfg(test)]
1988fn encode_value_envelope(value: &Value, codec: &str) -> Result<Value> {
1989 Ok(serde_json::to_value(encode_payload(value, codec)?)?)
1990}
1991
1992fn decode_wire_value(value: &Value, fallback_codec: &str) -> Result<Value> {
1993 validate_payload_codec(fallback_codec)?;
1994
1995 if value.is_null() {
1996 return Ok(Value::Null);
1997 }
1998
1999 if let Some((codec, blob)) = payload_envelope_parts(value)? {
2000 return decode_blob(blob, codec);
2001 }
2002
2003 if let Some(blob) = value.as_str() {
2004 return decode_blob(blob, fallback_codec);
2005 }
2006
2007 Err(untagged_payload_value())
2008}
2009
2010fn encode_typed_envelope(value: &AvroValue, codec: &str) -> Result<Value> {
2011 let envelope = match codec {
2012 DEFAULT_CODEC => encode_avro_value(value)?,
2013 other => return Err(unsupported_payload_codec(other)),
2014 };
2015 Ok(serde_json::to_value(envelope)?)
2016}
2017
2018fn decode_wire_avro_value(value: &Value, fallback_codec: &str) -> Result<AvroValue> {
2019 validate_payload_codec(fallback_codec)?;
2020
2021 if value.is_null() {
2022 return Ok(AvroValue::Null);
2023 }
2024
2025 if let Some((codec, blob)) = payload_envelope_parts(value)? {
2026 validate_payload_codec(codec)?;
2027 return decode_avro_value_blob(blob);
2028 }
2029
2030 if let Some(blob) = value.as_str() {
2031 return match fallback_codec {
2032 DEFAULT_CODEC => decode_avro_value_blob(blob),
2033 other => Err(unsupported_payload_codec(other)),
2034 };
2035 }
2036
2037 Err(untagged_payload_value())
2038}
2039
2040fn normalize_avro_arguments(value: AvroValue) -> AvroValue {
2041 match value {
2042 AvroValue::Null => AvroValue::Array(Vec::new()),
2043 AvroValue::Array(_) => value,
2044 other => AvroValue::Array(vec![other]),
2045 }
2046}
2047
2048fn decode_blob(blob: &str, codec: &str) -> Result<Value> {
2049 match codec {
2050 DEFAULT_CODEC => decode_avro_value_blob(blob)?.into_json(),
2051 other => Err(unsupported_payload_codec(other)),
2052 }
2053}
2054
2055fn validate_payload_codec(codec: &str) -> Result<()> {
2056 match codec {
2057 DEFAULT_CODEC => Ok(()),
2058 MISSING_TASK_PAYLOAD_CODEC => {
2059 Err(invalid_task_payload_codec("task payload_codec is missing"))
2060 }
2061 NULL_TASK_PAYLOAD_CODEC => Err(invalid_task_payload_codec("task payload_codec is null")),
2062 NON_STRING_TASK_PAYLOAD_CODEC => Err(invalid_task_payload_codec(
2063 "task payload_codec must be a string",
2064 )),
2065 other => Err(unsupported_payload_codec(other)),
2066 }
2067}
2068
2069fn invalid_task_payload_codec(reason: &str) -> Error {
2070 Error::Codec(format!(
2071 "unsupported_payload_codec: {reason}; Durable Workflow 2.0 requires an explicit string payload_codec=\"avro\" before worker task execution"
2072 ))
2073}
2074
2075fn payload_envelope_parts(value: &Value) -> Result<Option<(&str, &str)>> {
2076 let Some(object) = value.as_object() else {
2077 return Ok(None);
2078 };
2079 if !object.contains_key("codec") && !object.contains_key("blob") {
2080 return Ok(None);
2081 }
2082
2083 let codec = object
2084 .get("codec")
2085 .and_then(Value::as_str)
2086 .ok_or_else(invalid_payload_envelope)?;
2087 validate_payload_codec(codec)?;
2088 let blob = object
2089 .get("blob")
2090 .and_then(Value::as_str)
2091 .ok_or_else(invalid_payload_envelope)?;
2092 Ok(Some((codec, blob)))
2093}
2094
2095fn invalid_payload_envelope() -> Error {
2096 Error::Codec(
2097 "invalid_payload_envelope: durable payloads must use an object with string codec=\"avro\" and blob fields"
2098 .to_string(),
2099 )
2100}
2101
2102fn validate_workflow_task_commands(commands: &[Value]) -> Result<()> {
2103 for command in commands {
2104 let Some(command) = command.as_object() else {
2105 continue;
2106 };
2107 let Some(command_type) = command.get("type").and_then(Value::as_str) else {
2108 continue;
2109 };
2110 let Some(payload_field) = workflow_command_payload_field(command_type) else {
2111 continue;
2112 };
2113
2114 if let Some(codec) = command.get("payload_codec") {
2115 let codec = codec.as_str().ok_or_else(invalid_payload_envelope)?;
2116 validate_payload_codec(codec)?;
2117 }
2118
2119 let payload = command
2120 .get(payload_field)
2121 .ok_or_else(invalid_payload_envelope)?;
2122 validate_outbound_payload_envelope(payload)?;
2123 }
2124 Ok(())
2125}
2126
2127fn workflow_completion_protocol_version(commands: &[Value]) -> &'static str {
2128 if commands
2129 .iter()
2130 .any(|command| command.get("type").and_then(Value::as_str) == Some("open_condition_wait"))
2131 {
2132 CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION
2133 } else if commands.iter().any(|command| {
2134 command.get("type").and_then(Value::as_str) == Some("upsert_search_attributes")
2135 }) {
2136 SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION
2137 } else {
2138 WORKER_PROTOCOL_VERSION
2139 }
2140}
2141
2142fn workflow_command_payload_field(command_type: &str) -> Option<&'static str> {
2143 match command_type {
2144 "complete_workflow" | "complete_update" | "record_side_effect" => Some("result"),
2145 "schedule_activity" | "start_child_workflow" | "continue_as_new" => Some("arguments"),
2146 "start_service_operation" => Some("request_payload"),
2147 "upsert_memo" => Some("entries"),
2148 _ => None,
2149 }
2150}
2151
2152fn validate_outbound_payload_envelope(value: &Value) -> Result<()> {
2153 let Some((codec, blob)) = payload_envelope_parts(value)? else {
2154 return Err(untagged_payload_value());
2155 };
2156 validate_payload_codec(codec)?;
2157 decode_avro_value_blob(blob)?;
2158 Ok(())
2159}
2160
2161fn unsupported_payload_codec(codec: &str) -> Error {
2162 Error::Codec(format!(
2163 "unsupported_payload_codec: workflow payload codec {codec:?} is not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec"
2164 ))
2165}
2166
2167fn untagged_payload_value() -> Error {
2168 Error::Codec(
2169 "unsupported_payload_codec: untagged durable payload values are not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec"
2170 .to_string(),
2171 )
2172}
2173
2174fn decode_avro_value_blob(blob: &str) -> Result<AvroValue> {
2175 let bytes = BASE64.decode(blob).map_err(|err| {
2176 Error::Codec(format!(
2177 "invalid_payload_framing: expected strict base64 Avro single-object bytes: {err}"
2178 ))
2179 })?;
2180
2181 if serde_json::from_slice::<Value>(&bytes).is_ok() {
2182 return Err(unsupported_payload_codec("json"));
2183 }
2184
2185 if bytes.len() < 10 || bytes[..2] != AVRO_SINGLE_OBJECT_MAGIC {
2186 return Err(Error::Codec(
2187 "invalid_payload_framing: expected Avro single-object magic c301".to_string(),
2188 ));
2189 }
2190
2191 let fingerprint: [u8; 8] = bytes[2..10]
2192 .try_into()
2193 .map_err(|_| Error::Codec("invalid Avro fingerprint length".to_string()))?;
2194 if fingerprint != AVRO_VALUE_SCHEMA_FINGERPRINT {
2195 return Err(Error::Codec(format!(
2196 "unsupported_payload_schema: unknown CRC-64-AVRO fingerprint {}",
2197 fingerprint
2198 .iter()
2199 .map(|byte| format!("{byte:02x}"))
2200 .collect::<String>()
2201 )));
2202 }
2203
2204 let mut datum_reader = StrictAvroDatumReader::new(&bytes[10..]);
2205 let datum = from_avro_datum(avro_value_schema()?, &mut datum_reader, None);
2209 if datum_reader.truncated {
2210 return Err(Error::Codec(
2211 "invalid_payload_framing: truncated Avro Value datum".to_string(),
2212 ));
2213 }
2214 let datum = datum.map_err(|err| {
2215 Error::Codec(format!(
2216 "invalid_payload_framing: malformed Avro Value datum: {err}"
2217 ))
2218 })?;
2219 if datum_reader.remaining() != 0 {
2220 return Err(Error::Codec(format!(
2221 "invalid_payload_framing: {} trailing bytes after Avro Value datum",
2222 datum_reader.remaining()
2223 )));
2224 }
2225 avro_value_from_datum(datum)
2226}
2227
2228struct StrictAvroDatumReader<'a> {
2229 bytes: &'a [u8],
2230 offset: usize,
2231 truncated: bool,
2232}
2233
2234impl<'a> StrictAvroDatumReader<'a> {
2235 fn new(bytes: &'a [u8]) -> Self {
2236 Self {
2237 bytes,
2238 offset: 0,
2239 truncated: false,
2240 }
2241 }
2242
2243 fn remaining(&self) -> usize {
2244 self.bytes.len() - self.offset
2245 }
2246}
2247
2248impl Read for StrictAvroDatumReader<'_> {
2249 fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
2250 let count = buffer.len().min(self.remaining());
2251 buffer[..count].copy_from_slice(&self.bytes[self.offset..self.offset + count]);
2252 self.offset += count;
2253 if count < buffer.len() {
2254 self.truncated = true;
2255 }
2256
2257 Ok(count)
2258 }
2259}
2260
2261fn encode_avro_long(bytes: &mut Vec<u8>, value: i64) {
2262 let mut value = ((value as u64) << 1) ^ ((value >> 63) as u64);
2263 loop {
2264 if value & !0x7f == 0 {
2265 bytes.push(value as u8);
2266 break;
2267 }
2268 bytes.push(((value & 0x7f) | 0x80) as u8);
2269 value >>= 7;
2270 }
2271}
2272
2273fn encode_avro_size(bytes: &mut Vec<u8>, size: usize) -> Result<()> {
2274 let size = i64::try_from(size)
2275 .map_err(|_| Error::Codec("avro_value_encode_failed: collection too large".to_string()))?;
2276 encode_avro_long(bytes, size);
2277 Ok(())
2278}
2279
2280fn encode_avro_bytes(bytes: &mut Vec<u8>, value: &[u8]) -> Result<()> {
2281 encode_avro_size(bytes, value.len())?;
2282 bytes.extend_from_slice(value);
2283 Ok(())
2284}
2285
2286fn encode_avro_string(bytes: &mut Vec<u8>, value: &str) -> Result<()> {
2287 encode_avro_bytes(bytes, value.as_bytes())
2288}
2289
2290fn encode_avro_value_datum(bytes: &mut Vec<u8>, value: &AvroValue) -> Result<()> {
2291 match value {
2292 AvroValue::Null => encode_avro_long(bytes, 0),
2293 AvroValue::Boolean(value) => {
2294 encode_avro_long(bytes, 1);
2295 bytes.push(u8::from(*value));
2296 }
2297 AvroValue::Long(value) => {
2298 encode_avro_long(bytes, 2);
2299 encode_avro_long(bytes, *value);
2300 }
2301 AvroValue::Double(value) => {
2302 if !value.is_finite() {
2303 return Err(Error::Codec(
2304 "non_finite_float: Avro Value doubles must be finite".to_string(),
2305 ));
2306 }
2307 encode_avro_long(bytes, 3);
2308 bytes.extend_from_slice(&value.to_le_bytes());
2309 }
2310 AvroValue::Bytes(value) => {
2311 encode_avro_long(bytes, 4);
2312 encode_avro_bytes(bytes, value)?;
2313 }
2314 AvroValue::String(value) => {
2315 encode_avro_long(bytes, 5);
2316 encode_avro_string(bytes, value)?;
2317 }
2318 AvroValue::Array(values) => {
2319 encode_avro_long(bytes, 6);
2320 if !values.is_empty() {
2321 encode_avro_size(bytes, values.len())?;
2322 for value in values {
2323 encode_avro_value_datum(bytes, value)?;
2324 }
2325 }
2326 encode_avro_long(bytes, 0);
2327 }
2328 AvroValue::Map(values) => {
2329 encode_avro_long(bytes, 7);
2330 if !values.is_empty() {
2331 encode_avro_size(bytes, values.len())?;
2332 for (key, value) in values {
2333 encode_avro_string(bytes, key)?;
2334 encode_avro_value_datum(bytes, value)?;
2335 }
2336 }
2337 encode_avro_long(bytes, 0);
2338 }
2339 }
2340 Ok(())
2341}
2342
2343fn avro_value_from_datum(datum: AvroDatum) -> Result<AvroValue> {
2344 let AvroDatum::Record(mut outer) = datum else {
2345 return Err(Error::Codec(
2346 "invalid_payload_framing: datum is not a Value record".to_string(),
2347 ));
2348 };
2349 let (_, branch) = outer
2350 .pop()
2351 .filter(|(name, _)| name == "value")
2352 .ok_or_else(|| Error::Codec("invalid_payload_framing: Value field missing".to_string()))?;
2353 let AvroDatum::Union(_, branch) = branch else {
2354 return Err(Error::Codec(
2355 "invalid_payload_framing: invalid Value union".to_string(),
2356 ));
2357 };
2358 match *branch {
2359 AvroDatum::Null => Ok(AvroValue::Null),
2360 AvroDatum::Record(mut fields) => {
2361 let (name, value) = fields.pop().ok_or_else(|| {
2362 Error::Codec("invalid_payload_framing: empty Value branch".to_string())
2363 })?;
2364 match (name.as_str(), value) {
2365 ("boolean", AvroDatum::Boolean(value)) => Ok(AvroValue::Boolean(value)),
2366 ("long", AvroDatum::Long(value)) => Ok(AvroValue::Long(value)),
2367 ("double", AvroDatum::Double(value)) if value.is_finite() => {
2368 Ok(AvroValue::Double(value))
2369 }
2370 ("bytes", AvroDatum::Bytes(value)) => Ok(AvroValue::Bytes(value)),
2371 ("string", AvroDatum::String(value)) => Ok(AvroValue::String(value)),
2372 ("items", AvroDatum::Array(values)) => values
2373 .into_iter()
2374 .map(avro_value_from_datum)
2375 .collect::<Result<Vec<_>>>()
2376 .map(AvroValue::Array),
2377 ("entries", AvroDatum::Map(values)) => values
2378 .into_iter()
2379 .map(|(key, value)| Ok((key, avro_value_from_datum(value)?)))
2380 .collect::<Result<BTreeMap<_, _>>>()
2381 .map(AvroValue::Map),
2382 _ => Err(Error::Codec(
2383 "invalid_payload_framing: unknown Value branch".to_string(),
2384 )),
2385 }
2386 }
2387 _ => Err(Error::Codec(
2388 "invalid_payload_framing: invalid Value branch".to_string(),
2389 )),
2390 }
2391}
2392
2393fn avro_value_schema() -> Result<&'static Schema> {
2394 match AVRO_VALUE_SCHEMA.get_or_init(|| {
2395 Schema::parse_str(AVRO_VALUE_SCHEMA_JSON)
2396 .map_err(|err| format!("could not parse Avro Value schema: {err}"))
2397 }) {
2398 Ok(schema) => Ok(schema),
2399 Err(message) => Err(Error::Codec(message.clone())),
2400 }
2401}
2402
2403#[derive(Clone, Debug)]
2404pub struct Client {
2405 http: reqwest::Client,
2406 base_url: String,
2407 token: Option<String>,
2408 control_token: Option<String>,
2409 worker_token: Option<String>,
2410 namespace: String,
2411}
2412
2413impl Client {
2414 pub fn new(base_url: impl Into<String>) -> Result<Self> {
2415 Self::builder(base_url).build()
2416 }
2417
2418 pub fn builder(base_url: impl Into<String>) -> ClientBuilder {
2419 ClientBuilder {
2420 base_url: base_url.into(),
2421 token: None,
2422 control_token: None,
2423 worker_token: None,
2424 namespace: "default".to_string(),
2425 timeout: Duration::from_secs(60),
2426 }
2427 }
2428
2429 pub async fn health(&self) -> Result<Value> {
2430 self.request_json(
2431 reqwest::Method::GET,
2432 "/health",
2433 RequestProtocol::ControlPlane,
2434 Option::<&Value>::None,
2435 )
2436 .await
2437 }
2438
2439 pub async fn cluster_info(&self) -> Result<Value> {
2440 self.request_json(
2441 reqwest::Method::GET,
2442 "/cluster/info",
2443 RequestProtocol::ControlPlane,
2444 Option::<&Value>::None,
2445 )
2446 .await
2447 }
2448
2449 pub async fn start_workflow<T: Serialize>(
2450 &self,
2451 workflow_type: &str,
2452 task_queue: &str,
2453 workflow_id: &str,
2454 input: T,
2455 ) -> Result<WorkflowHandle> {
2456 self.start_workflow_with_options(
2457 workflow_type,
2458 task_queue,
2459 workflow_id,
2460 WorkflowStartOptions::default(),
2461 input,
2462 )
2463 .await
2464 }
2465
2466 pub async fn start_workflow_with_options<T: Serialize>(
2469 &self,
2470 workflow_type: &str,
2471 task_queue: &str,
2472 workflow_id: &str,
2473 options: WorkflowStartOptions,
2474 input: T,
2475 ) -> Result<WorkflowHandle> {
2476 options.validate()?;
2477 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2478 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2479 let body = json!({
2480 "workflow_id": workflow_id,
2481 "workflow_type": workflow_type,
2482 "task_queue": task_queue,
2483 "input": input_envelope,
2484 "execution_timeout_seconds": options.execution_timeout_seconds,
2485 "run_timeout_seconds": options.run_timeout_seconds
2486 });
2487
2488 let data: Value = self
2489 .request_json(
2490 reqwest::Method::POST,
2491 "/workflows",
2492 RequestProtocol::ControlPlane,
2493 Some(&body),
2494 )
2495 .await?;
2496
2497 Ok(WorkflowHandle {
2498 client: self.clone(),
2499 workflow_id: data
2500 .get("workflow_id")
2501 .and_then(Value::as_str)
2502 .unwrap_or(workflow_id)
2503 .to_string(),
2504 run_id: data
2505 .get("run_id")
2506 .and_then(Value::as_str)
2507 .map(str::to_string),
2508 workflow_type: data
2509 .get("workflow_type")
2510 .and_then(Value::as_str)
2511 .unwrap_or(workflow_type)
2512 .to_string(),
2513 })
2514 }
2515
2516 pub async fn signal_workflow<T: Serialize>(
2517 &self,
2518 workflow_id: &str,
2519 signal_name: &str,
2520 input: T,
2521 ) -> Result<Value> {
2522 self.signal_workflow_target(workflow_id, None, signal_name, input)
2523 .await
2524 }
2525
2526 pub async fn signal_workflow_run<T: Serialize>(
2528 &self,
2529 workflow_id: &str,
2530 run_id: &str,
2531 signal_name: &str,
2532 input: T,
2533 ) -> Result<Value> {
2534 self.signal_workflow_target(workflow_id, Some(run_id), signal_name, input)
2535 .await
2536 }
2537
2538 async fn signal_workflow_target<T: Serialize>(
2539 &self,
2540 workflow_id: &str,
2541 run_id: Option<&str>,
2542 signal_name: &str,
2543 input: T,
2544 ) -> Result<Value> {
2545 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2546 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2547 let body = json!({
2548 "input": input_envelope
2549 });
2550 let path = match run_id {
2551 Some(run_id) => {
2552 format!("/workflows/{workflow_id}/runs/{run_id}/signal/{signal_name}")
2553 }
2554 None => format!("/workflows/{workflow_id}/signal/{signal_name}"),
2555 };
2556 self.request_json(
2557 reqwest::Method::POST,
2558 &path,
2559 RequestProtocol::ControlPlane,
2560 Some(&body),
2561 )
2562 .await
2563 }
2564
2565 pub async fn cancel_workflow(
2567 &self,
2568 workflow_id: &str,
2569 options: WorkflowCommandOptions,
2570 ) -> Result<WorkflowCommandResult> {
2571 self.workflow_command(workflow_id, None, WorkflowCommandKind::Cancel, options)
2572 .await
2573 }
2574
2575 pub async fn cancel_workflow_run(
2577 &self,
2578 workflow_id: &str,
2579 run_id: &str,
2580 options: WorkflowCommandOptions,
2581 ) -> Result<WorkflowCommandResult> {
2582 self.workflow_command(
2583 workflow_id,
2584 Some(run_id),
2585 WorkflowCommandKind::Cancel,
2586 options,
2587 )
2588 .await
2589 }
2590
2591 pub async fn terminate_workflow(
2593 &self,
2594 workflow_id: &str,
2595 options: WorkflowCommandOptions,
2596 ) -> Result<WorkflowCommandResult> {
2597 self.workflow_command(workflow_id, None, WorkflowCommandKind::Terminate, options)
2598 .await
2599 }
2600
2601 pub async fn terminate_workflow_run(
2603 &self,
2604 workflow_id: &str,
2605 run_id: &str,
2606 options: WorkflowCommandOptions,
2607 ) -> Result<WorkflowCommandResult> {
2608 self.workflow_command(
2609 workflow_id,
2610 Some(run_id),
2611 WorkflowCommandKind::Terminate,
2612 options,
2613 )
2614 .await
2615 }
2616
2617 async fn workflow_command(
2618 &self,
2619 workflow_id: &str,
2620 run_id: Option<&str>,
2621 command: WorkflowCommandKind,
2622 options: WorkflowCommandOptions,
2623 ) -> Result<WorkflowCommandResult> {
2624 let path = match run_id {
2625 Some(run_id) => format!(
2626 "/workflows/{workflow_id}/runs/{run_id}/{}",
2627 command.as_str()
2628 ),
2629 None => format!("/workflows/{workflow_id}/{}", command.as_str()),
2630 };
2631 let data = match self
2632 .request_json(
2633 reqwest::Method::POST,
2634 &path,
2635 RequestProtocol::ControlPlane,
2636 Some(&options),
2637 )
2638 .await
2639 {
2640 Ok(data) => data,
2641 Err(Error::Http { status, body }) => {
2642 return Err(Error::WorkflowCommandRejected(workflow_command_rejection(
2643 command,
2644 status,
2645 body,
2646 workflow_id,
2647 run_id,
2648 )));
2649 }
2650 Err(error) => return Err(error),
2651 };
2652
2653 Ok(workflow_command_result(command, data, workflow_id, run_id))
2654 }
2655
2656 pub async fn query_workflow<T: Serialize>(
2662 &self,
2663 workflow_id: &str,
2664 query_name: &str,
2665 input: T,
2666 ) -> Result<Value> {
2667 self.query_workflow_target(workflow_id, None, query_name, input)
2668 .await
2669 }
2670
2671 pub async fn query_workflow_run<T: Serialize>(
2673 &self,
2674 workflow_id: &str,
2675 run_id: &str,
2676 query_name: &str,
2677 input: T,
2678 ) -> Result<Value> {
2679 self.query_workflow_target(workflow_id, Some(run_id), query_name, input)
2680 .await
2681 }
2682
2683 pub async fn query_workflow_avro_value<T: Serialize>(
2685 &self,
2686 workflow_id: &str,
2687 query_name: &str,
2688 input: T,
2689 ) -> Result<AvroValue> {
2690 self.query_workflow_avro_value_target(workflow_id, None, query_name, input)
2691 .await
2692 }
2693
2694 pub async fn query_workflow_run_avro_value<T: Serialize>(
2696 &self,
2697 workflow_id: &str,
2698 run_id: &str,
2699 query_name: &str,
2700 input: T,
2701 ) -> Result<AvroValue> {
2702 self.query_workflow_avro_value_target(workflow_id, Some(run_id), query_name, input)
2703 .await
2704 }
2705
2706 async fn query_workflow_avro_value_target<T: Serialize>(
2707 &self,
2708 workflow_id: &str,
2709 run_id: Option<&str>,
2710 query_name: &str,
2711 input: T,
2712 ) -> Result<AvroValue> {
2713 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2714 let body = json!({"input": encode_typed_envelope(&input, DEFAULT_CODEC)?});
2715 let path = match run_id {
2716 Some(run_id) => {
2717 format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
2718 }
2719 None => format!("/workflows/{workflow_id}/query/{query_name}"),
2720 };
2721 let response: Value = match self
2722 .request_json(
2723 reqwest::Method::POST,
2724 &path,
2725 RequestProtocol::ControlPlane,
2726 Some(&body),
2727 )
2728 .await
2729 {
2730 Ok(response) => response,
2731 Err(Error::Http { status, body }) => {
2732 return Err(Error::QueryFailed(query_failure(status, body)));
2733 }
2734 Err(error) => return Err(error),
2735 };
2736
2737 let envelope = response
2738 .get("result_envelope")
2739 .filter(|envelope| !envelope.is_null())
2740 .ok_or_else(|| {
2741 Error::Codec(
2742 "missing_payload_envelope: typed query result requires result_envelope"
2743 .to_string(),
2744 )
2745 })?;
2746 decode_wire_avro_value(envelope, DEFAULT_CODEC)
2747 }
2748
2749 async fn query_workflow_target<T: Serialize>(
2750 &self,
2751 workflow_id: &str,
2752 run_id: Option<&str>,
2753 query_name: &str,
2754 input: T,
2755 ) -> Result<Value> {
2756 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2757 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2758 let body = json!({
2759 "input": input_envelope
2760 });
2761 let path = match run_id {
2762 Some(run_id) => {
2763 format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
2764 }
2765 None => format!("/workflows/{workflow_id}/query/{query_name}"),
2766 };
2767 let response: Value = match self
2768 .request_json(
2769 reqwest::Method::POST,
2770 &path,
2771 RequestProtocol::ControlPlane,
2772 Some(&body),
2773 )
2774 .await
2775 {
2776 Ok(response) => response,
2777 Err(Error::Http { status, body }) => {
2778 return Err(Error::QueryFailed(query_failure(status, body)));
2779 }
2780 Err(error) => return Err(error),
2781 };
2782
2783 if let Some(envelope) = response
2784 .get("result_envelope")
2785 .filter(|envelope| !envelope.is_null())
2786 {
2787 return decode_wire_value(envelope, DEFAULT_CODEC);
2788 }
2789
2790 Ok(response.get("result").cloned().unwrap_or(Value::Null))
2791 }
2792
2793 pub async fn update_workflow<T: Serialize>(
2795 &self,
2796 workflow_id: &str,
2797 update_name: &str,
2798 input: T,
2799 request_id: Option<&str>,
2800 ) -> Result<Value> {
2801 let response = self
2802 .update_workflow_response(workflow_id, update_name, input, request_id)
2803 .await?;
2804 if let Some(envelope) = response
2805 .get("result_envelope")
2806 .filter(|envelope| !envelope.is_null())
2807 {
2808 return decode_wire_value(envelope, DEFAULT_CODEC);
2809 }
2810 Ok(response.get("result").cloned().unwrap_or(response))
2811 }
2812
2813 pub async fn update_workflow_avro_value<T: Serialize>(
2815 &self,
2816 workflow_id: &str,
2817 update_name: &str,
2818 input: T,
2819 request_id: Option<&str>,
2820 ) -> Result<AvroValue> {
2821 let response = self
2822 .update_workflow_response(workflow_id, update_name, input, request_id)
2823 .await?;
2824 let envelope = response
2825 .get("result_envelope")
2826 .filter(|envelope| !envelope.is_null())
2827 .ok_or_else(|| {
2828 Error::Codec(
2829 "missing_payload_envelope: typed update result requires result_envelope"
2830 .to_string(),
2831 )
2832 })?;
2833 decode_wire_avro_value(envelope, DEFAULT_CODEC)
2834 }
2835
2836 async fn update_workflow_response<T: Serialize>(
2837 &self,
2838 workflow_id: &str,
2839 update_name: &str,
2840 input: T,
2841 request_id: Option<&str>,
2842 ) -> Result<Value> {
2843 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2844 let mut body = json!({
2845 "input": encode_typed_envelope(&input, DEFAULT_CODEC)?,
2846 "wait_for": "completed",
2847 });
2848 if let Some(request_id) = request_id {
2849 body["request_id"] = json!(request_id);
2850 }
2851 self.request_json(
2852 reqwest::Method::POST,
2853 &format!("/workflows/{workflow_id}/update/{update_name}"),
2854 RequestProtocol::ControlPlane,
2855 Some(&body),
2856 )
2857 .await
2858 }
2859
2860 pub async fn describe_workflow(&self, workflow_id: &str) -> Result<WorkflowDescription> {
2861 let path = format!("/workflows/{workflow_id}");
2862 let mut data: WorkflowDescription = self
2863 .request_json(
2864 reqwest::Method::GET,
2865 &path,
2866 RequestProtocol::ControlPlane,
2867 Option::<&Value>::None,
2868 )
2869 .await?;
2870 data.decode_payloads()?;
2871 Ok(data)
2872 }
2873
2874 pub async fn describe_workflow_run(
2876 &self,
2877 workflow_id: &str,
2878 run_id: &str,
2879 ) -> Result<WorkflowDescription> {
2880 let path = format!("/workflows/{workflow_id}/runs/{run_id}");
2881 let mut data: WorkflowDescription = self
2882 .request_json(
2883 reqwest::Method::GET,
2884 &path,
2885 RequestProtocol::ControlPlane,
2886 Option::<&Value>::None,
2887 )
2888 .await?;
2889 data.decode_payloads()?;
2890 Ok(data)
2891 }
2892
2893 fn workflow_stream_path(workflow_id: &str, run_id: &str, stream_name: Option<&str>) -> String {
2894 let mut path = format!(
2895 "/workflows/{}/runs/{}/streams",
2896 percent_encode_path_segment(workflow_id),
2897 percent_encode_path_segment(run_id),
2898 );
2899 if let Some(stream_name) = stream_name {
2900 path.push('/');
2901 path.push_str(&percent_encode_path_segment(stream_name));
2902 }
2903 path
2904 }
2905
2906 pub async fn list_workflow_streams(
2908 &self,
2909 workflow_id: &str,
2910 run_id: &str,
2911 ) -> Result<Vec<WorkflowStreamDescription>> {
2912 let response: WorkflowStreamListResponse = self
2913 .request_json(
2914 reqwest::Method::GET,
2915 &Self::workflow_stream_path(workflow_id, run_id, None),
2916 RequestProtocol::ControlPlane,
2917 Option::<&Value>::None,
2918 )
2919 .await?;
2920 Ok(response.streams)
2921 }
2922
2923 pub async fn describe_workflow_stream(
2925 &self,
2926 workflow_id: &str,
2927 run_id: &str,
2928 stream_name: &str,
2929 ) -> Result<WorkflowStreamDescription> {
2930 let response: WorkflowStreamDescriptionResponse = self
2931 .request_json(
2932 reqwest::Method::GET,
2933 &Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
2934 RequestProtocol::ControlPlane,
2935 Option::<&Value>::None,
2936 )
2937 .await?;
2938 Ok(response.stream)
2939 }
2940
2941 pub async fn subscribe_workflow_stream(
2948 &self,
2949 workflow_id: &str,
2950 run_id: &str,
2951 stream_name: &str,
2952 from_offset: u64,
2953 max_items: usize,
2954 wait: Duration,
2955 ) -> Result<WorkflowStreamPage> {
2956 let max_items = max_items.clamp(1, 500);
2957 let wait_seconds = wait.as_secs().min(MAX_LONG_POLL_TIMEOUT_SECONDS);
2958 let path = format!(
2959 "{}/items?from={from_offset}&max_items={max_items}&wait_seconds={wait_seconds}",
2960 Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
2961 );
2962 let response: WorkflowStreamPageResponse = self
2963 .request_json_with_timeout(
2964 reqwest::Method::GET,
2965 &path,
2966 RequestProtocol::ControlPlane,
2967 Option::<&Value>::None,
2968 Duration::from_secs(wait_seconds.saturating_add(5).max(5)),
2969 )
2970 .await?;
2971
2972 let items = response
2973 .items
2974 .into_iter()
2975 .map(|raw| {
2976 let offset = raw.get("offset").and_then(Value::as_u64).unwrap_or(0);
2977 let envelope = raw.get("payload").cloned();
2978 let payload = envelope
2979 .as_ref()
2980 .filter(|value| value.get("blob").is_some())
2981 .map(|value| decode_wire_avro_value(value, DEFAULT_CODEC))
2982 .transpose()?
2983 .map(AvroValue::into_json)
2984 .transpose()?;
2985 Ok(WorkflowStreamItem {
2986 offset,
2987 payload,
2988 payload_envelope: envelope,
2989 payload_reference: raw
2990 .get("payload_reference")
2991 .and_then(Value::as_str)
2992 .map(str::to_string),
2993 payload_codec: raw
2994 .get("payload_codec")
2995 .and_then(Value::as_str)
2996 .map(str::to_string),
2997 idempotency_key: raw
2998 .get("idempotency_key")
2999 .and_then(Value::as_str)
3000 .map(str::to_string),
3001 item_type: raw
3002 .get("item_type")
3003 .and_then(Value::as_str)
3004 .map(str::to_string),
3005 content_type: raw
3006 .get("content_type")
3007 .and_then(Value::as_str)
3008 .map(str::to_string),
3009 origin: raw
3010 .get("origin")
3011 .and_then(Value::as_str)
3012 .map(str::to_string),
3013 origin_reference: raw
3014 .get("origin_reference")
3015 .and_then(Value::as_str)
3016 .map(str::to_string),
3017 emitted_at: raw
3018 .get("emitted_at")
3019 .and_then(Value::as_str)
3020 .map(str::to_string),
3021 raw,
3022 })
3023 })
3024 .collect::<Result<Vec<_>>>()?;
3025 Ok(WorkflowStreamPage {
3026 stream: response.stream,
3027 items,
3028 next_offset: response.next_offset,
3029 terminal: response.terminal,
3030 })
3031 }
3032
3033 pub async fn append_workflow_stream(
3035 &self,
3036 workflow_id: &str,
3037 run_id: &str,
3038 stream_name: &str,
3039 items: &[WorkflowStreamAppendItem],
3040 max_pending_items: Option<u64>,
3041 ) -> Result<WorkflowStreamAppendResult> {
3042 if items.is_empty() {
3043 return Err(Error::Codec(
3044 "workflow_stream_items_empty: append requires at least one item".to_string(),
3045 ));
3046 }
3047 let mut body = json!({
3048 "items": items
3049 .iter()
3050 .map(|item| item.wire_value(None))
3051 .collect::<Vec<_>>(),
3052 });
3053 if let Some(max_pending_items) = max_pending_items {
3054 if max_pending_items == 0 {
3055 return Err(Error::Codec(
3056 "workflow_stream_pending_limit_invalid: max_pending_items must be positive"
3057 .to_string(),
3058 ));
3059 }
3060 body["max_pending_items"] = json!(max_pending_items);
3061 }
3062 let response: WorkflowStreamAppendResponse = self
3063 .request_json(
3064 reqwest::Method::POST,
3065 &format!(
3066 "{}/items",
3067 Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3068 ),
3069 RequestProtocol::ControlPlane,
3070 Some(&body),
3071 )
3072 .await?;
3073 Ok(WorkflowStreamAppendResult {
3074 stream: response.stream,
3075 accepted_offsets: response.accepted_offsets,
3076 accepted: response.accepted,
3077 deduped: response.deduped,
3078 })
3079 }
3080
3081 pub async fn close_workflow_stream(
3083 &self,
3084 workflow_id: &str,
3085 run_id: &str,
3086 stream_name: &str,
3087 error_reason: Option<&str>,
3088 retention_seconds: Option<u64>,
3089 ) -> Result<WorkflowStreamDescription> {
3090 let mut body = json!({});
3091 if let Some(error_reason) = error_reason {
3092 body["error_reason"] = json!(error_reason);
3093 }
3094 if let Some(retention_seconds) = retention_seconds {
3095 if retention_seconds == 0 {
3096 return Err(Error::Codec(
3097 "workflow_stream_retention_invalid: retention_seconds must be positive"
3098 .to_string(),
3099 ));
3100 }
3101 body["retention_seconds"] = json!(retention_seconds);
3102 }
3103 let response: WorkflowStreamDescriptionResponse = self
3104 .request_json(
3105 reqwest::Method::POST,
3106 &format!(
3107 "{}/close",
3108 Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3109 ),
3110 RequestProtocol::ControlPlane,
3111 Some(&body),
3112 )
3113 .await?;
3114 Ok(response.stream)
3115 }
3116
3117 pub async fn register_worker(
3118 &self,
3119 worker_id: &str,
3120 task_queue: &str,
3121 supported_workflow_types: Vec<String>,
3122 supported_activity_types: Vec<String>,
3123 max_concurrent_workflow_tasks: usize,
3124 max_concurrent_activity_tasks: usize,
3125 ) -> Result<RegisterWorkerResponse> {
3126 self.register_worker_with_capabilities(
3127 worker_id,
3128 task_queue,
3129 supported_workflow_types,
3130 supported_activity_types,
3131 max_concurrent_workflow_tasks,
3132 max_concurrent_activity_tasks,
3133 Vec::new(),
3134 )
3135 .await
3136 }
3137
3138 pub async fn register_worker_with_capabilities(
3140 &self,
3141 worker_id: &str,
3142 task_queue: &str,
3143 supported_workflow_types: Vec<String>,
3144 supported_activity_types: Vec<String>,
3145 max_concurrent_workflow_tasks: usize,
3146 max_concurrent_activity_tasks: usize,
3147 capabilities: Vec<String>,
3148 ) -> Result<RegisterWorkerResponse> {
3149 self.register_worker_with_command_contracts(
3150 worker_id,
3151 task_queue,
3152 supported_workflow_types,
3153 supported_activity_types,
3154 max_concurrent_workflow_tasks,
3155 max_concurrent_activity_tasks,
3156 capabilities,
3157 Value::Object(serde_json::Map::new()),
3158 )
3159 .await
3160 }
3161
3162 #[allow(clippy::too_many_arguments)]
3169 pub async fn register_worker_with_command_contracts(
3170 &self,
3171 worker_id: &str,
3172 task_queue: &str,
3173 supported_workflow_types: Vec<String>,
3174 supported_activity_types: Vec<String>,
3175 max_concurrent_workflow_tasks: usize,
3176 max_concurrent_activity_tasks: usize,
3177 capabilities: Vec<String>,
3178 workflow_command_contracts: Value,
3179 ) -> Result<RegisterWorkerResponse> {
3180 if let Some(contracts) = workflow_command_contracts.as_object() {
3181 for (workflow_type, contract) in contracts {
3182 let Some(update_validators) = contract.get("update_validators") else {
3183 continue;
3184 };
3185 if !update_validators
3186 .as_array()
3187 .is_some_and(|validators| validators.is_empty())
3188 {
3189 return Err(Error::UnsupportedUpdateValidators {
3190 workflow_type: workflow_type.clone(),
3191 });
3192 }
3193 }
3194 }
3195
3196 let mut body = json!({
3197 "worker_id": worker_id,
3198 "task_queue": task_queue,
3199 "runtime": "rust",
3200 "sdk_version": SDK_VERSION,
3201 "supported_workflow_types": supported_workflow_types,
3202 "supported_activity_types": supported_activity_types,
3203 "capabilities": capabilities,
3204 "max_concurrent_workflow_tasks": max_concurrent_workflow_tasks,
3205 "max_concurrent_activity_tasks": max_concurrent_activity_tasks
3206 });
3207 if workflow_command_contracts
3208 .as_object()
3209 .is_some_and(|contracts| !contracts.is_empty())
3210 {
3211 body["workflow_command_contracts"] = workflow_command_contracts;
3212 }
3213
3214 self.request_json(
3215 reqwest::Method::POST,
3216 "/worker/register",
3217 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3218 Some(&body),
3219 )
3220 .await
3221 }
3222
3223 pub async fn deregister_worker_registration(
3229 &self,
3230 worker_id: &str,
3231 ) -> Result<WorkerDeregistrationEnvelope> {
3232 let path = format!(
3233 "/worker/registrations/{}",
3234 percent_encode_path_segment(worker_id)
3235 );
3236 self.request_json(
3237 reqwest::Method::DELETE,
3238 &path,
3239 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3240 Option::<&Value>::None,
3241 )
3242 .await
3243 }
3244
3245 pub async fn poll_query_task(
3247 &self,
3248 worker_id: &str,
3249 task_queue: &str,
3250 timeout: Duration,
3251 ) -> Result<Option<QueryTask>> {
3252 Ok(self
3253 .poll_query_task_response(worker_id, task_queue, timeout)
3254 .await?
3255 .task)
3256 }
3257
3258 pub async fn poll_query_task_response(
3260 &self,
3261 worker_id: &str,
3262 task_queue: &str,
3263 timeout: Duration,
3264 ) -> Result<PollQueryTaskResponse> {
3265 let poll_request_id = unique_request_id("rust-query-poll");
3266 self.poll_query_task_response_with_request_id(
3267 worker_id,
3268 task_queue,
3269 timeout,
3270 &poll_request_id,
3271 1,
3272 )
3273 .await
3274 }
3275
3276 async fn poll_query_task_response_with_request_id(
3277 &self,
3278 worker_id: &str,
3279 task_queue: &str,
3280 timeout: Duration,
3281 poll_request_id: &str,
3282 transport_retries: usize,
3283 ) -> Result<PollQueryTaskResponse> {
3284 let timeout_seconds = long_poll_timeout_seconds(timeout);
3285 let body = json!({
3286 "worker_id": worker_id,
3287 "task_queue": task_queue,
3288 "poll_request_id": poll_request_id,
3289 "timeout_seconds": timeout_seconds,
3290 });
3291 self.poll_request_json(
3292 "/worker/query-tasks/poll",
3293 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3294 &body,
3295 timeout + Duration::from_secs(5),
3296 transport_retries,
3297 )
3298 .await
3299 }
3300
3301 pub async fn complete_query_task<T: Serialize>(
3303 &self,
3304 query_task_id: &str,
3305 lease_owner: &str,
3306 query_task_attempt: u64,
3307 result: T,
3308 codec: &str,
3309 ) -> Result<Value> {
3310 let typed_result = AvroValue::from_serialize(&result)?;
3311 let result_envelope = encode_typed_envelope(&typed_result, codec)?;
3312 self.complete_query_task_with_envelope(
3313 query_task_id,
3314 lease_owner,
3315 query_task_attempt,
3316 typed_result.into_json()?,
3317 result_envelope,
3318 )
3319 .await
3320 }
3321
3322 async fn complete_query_task_with_envelope(
3323 &self,
3324 query_task_id: &str,
3325 lease_owner: &str,
3326 query_task_attempt: u64,
3327 result: Value,
3328 result_envelope: Value,
3329 ) -> Result<Value> {
3330 let body = json!({
3331 "lease_owner": lease_owner,
3332 "query_task_attempt": query_task_attempt,
3333 "result": result,
3334 "result_envelope": result_envelope,
3335 });
3336 let path = format!("/worker/query-tasks/{query_task_id}/complete");
3337 let response = self
3338 .request_json(
3339 reqwest::Method::POST,
3340 &path,
3341 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3342 Some(&body),
3343 )
3344 .await;
3345 query_task_response(response)
3346 }
3347
3348 pub async fn fail_query_task(
3350 &self,
3351 query_task_id: &str,
3352 lease_owner: &str,
3353 query_task_attempt: u64,
3354 message: impl Into<String>,
3355 reason: impl Into<String>,
3356 failure_type: impl Into<String>,
3357 ) -> Result<Value> {
3358 let body = json!({
3359 "lease_owner": lease_owner,
3360 "query_task_attempt": query_task_attempt,
3361 "failure": {
3362 "message": message.into(),
3363 "reason": reason.into(),
3364 "type": failure_type.into(),
3365 }
3366 });
3367 let path = format!("/worker/query-tasks/{query_task_id}/fail");
3368 let response = self
3369 .request_json(
3370 reqwest::Method::POST,
3371 &path,
3372 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3373 Some(&body),
3374 )
3375 .await;
3376 query_task_response(response)
3377 }
3378
3379 pub async fn heartbeat_worker(
3380 &self,
3381 worker_id: &str,
3382 workflow_available: usize,
3383 activity_available: usize,
3384 ) -> Result<Value> {
3385 let body = json!({
3386 "worker_id": worker_id,
3387 "task_slots": {
3388 "workflow_available": workflow_available,
3389 "activity_available": activity_available
3390 },
3391 "process_metrics": {
3392 "process_id": std::process::id(),
3393 "process_uptime_seconds": 0
3394 }
3395 });
3396
3397 self.request_json(
3398 reqwest::Method::POST,
3399 "/worker/heartbeat",
3400 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3401 Some(&body),
3402 )
3403 .await
3404 }
3405
3406 pub async fn poll_workflow_task(
3407 &self,
3408 worker_id: &str,
3409 task_queue: &str,
3410 timeout: Duration,
3411 ) -> Result<Option<WorkflowTask>> {
3412 Ok(self
3413 .poll_workflow_task_response(worker_id, task_queue, timeout)
3414 .await?
3415 .task)
3416 }
3417
3418 pub async fn poll_workflow_task_response(
3419 &self,
3420 worker_id: &str,
3421 task_queue: &str,
3422 timeout: Duration,
3423 ) -> Result<PollWorkflowTaskResponse> {
3424 let poll_request_id = unique_request_id("rust-workflow-poll");
3425 self.poll_workflow_task_response_with_request_id(
3426 worker_id,
3427 task_queue,
3428 timeout,
3429 &poll_request_id,
3430 1,
3431 )
3432 .await
3433 }
3434
3435 async fn poll_workflow_task_response_with_request_id(
3436 &self,
3437 worker_id: &str,
3438 task_queue: &str,
3439 timeout: Duration,
3440 poll_request_id: &str,
3441 transport_retries: usize,
3442 ) -> Result<PollWorkflowTaskResponse> {
3443 let body = json!({
3444 "worker_id": worker_id,
3445 "task_queue": task_queue,
3446 "poll_request_id": poll_request_id,
3447 "timeout_seconds": long_poll_timeout_seconds(timeout),
3448 });
3449 let mut data: PollWorkflowTaskResponse = self
3450 .poll_request_json(
3451 "/worker/workflow-tasks/poll",
3452 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3453 &body,
3454 timeout + Duration::from_secs(5),
3455 transport_retries,
3456 )
3457 .await?;
3458
3459 if let Some(task) = data.task.as_mut() {
3460 self.fetch_remaining_workflow_history(worker_id, task)
3461 .await?;
3462 }
3463
3464 Ok(data)
3465 }
3466
3467 async fn fetch_remaining_workflow_history(
3468 &self,
3469 worker_id: &str,
3470 task: &mut WorkflowTask,
3471 ) -> Result<()> {
3472 let mut next_token = task.next_history_page_token.clone();
3473
3474 while let Some(token) = next_token.take().filter(|token| !token.is_empty()) {
3475 let lease_owner = task
3476 .lease_owner
3477 .clone()
3478 .unwrap_or_else(|| worker_id.to_string());
3479 let page = self
3480 .workflow_task_history_page(
3481 &task.task_id,
3482 &lease_owner,
3483 task.workflow_task_attempt,
3484 &token,
3485 )
3486 .await?;
3487
3488 task.append_history_page(page);
3489
3490 if task.next_history_page_token.as_deref() == Some(token.as_str()) {
3491 return Err(Error::Codec(
3492 "workflow history pagination returned the same page token".to_string(),
3493 ));
3494 }
3495
3496 next_token = task.next_history_page_token.clone();
3497 }
3498
3499 Ok(())
3500 }
3501
3502 async fn workflow_task_history_page(
3503 &self,
3504 task_id: &str,
3505 lease_owner: &str,
3506 workflow_task_attempt: u64,
3507 next_history_page_token: &str,
3508 ) -> Result<WorkflowTaskHistoryPage> {
3509 let body = json!({
3510 "lease_owner": lease_owner,
3511 "workflow_task_attempt": workflow_task_attempt,
3512 "next_history_page_token": next_history_page_token
3513 });
3514 let path = format!("/worker/workflow-tasks/{task_id}/history");
3515
3516 self.request_json(
3517 reqwest::Method::POST,
3518 &path,
3519 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3520 Some(&body),
3521 )
3522 .await
3523 }
3524
3525 pub async fn complete_workflow_task(
3526 &self,
3527 task_id: &str,
3528 lease_owner: &str,
3529 workflow_task_attempt: u64,
3530 commands: Vec<Value>,
3531 ) -> Result<Value> {
3532 validate_workflow_task_commands(&commands)?;
3533 let protocol_version = workflow_completion_protocol_version(&commands);
3534 let body = json!({
3535 "lease_owner": lease_owner,
3536 "workflow_task_attempt": workflow_task_attempt,
3537 "commands": commands
3538 });
3539 let path = format!("/worker/workflow-tasks/{task_id}/complete");
3540 self.request_json(
3541 reqwest::Method::POST,
3542 &path,
3543 RequestProtocol::Worker(protocol_version),
3544 Some(&body),
3545 )
3546 .await
3547 }
3548
3549 pub async fn fail_workflow_task(
3550 &self,
3551 task_id: &str,
3552 lease_owner: &str,
3553 workflow_task_attempt: u64,
3554 message: impl Into<String>,
3555 ) -> Result<Value> {
3556 self.fail_workflow_task_with_type(
3557 task_id,
3558 lease_owner,
3559 workflow_task_attempt,
3560 message,
3561 "RustWorkflowTaskFailure",
3562 )
3563 .await
3564 }
3565
3566 async fn fail_workflow_task_with_type(
3567 &self,
3568 task_id: &str,
3569 lease_owner: &str,
3570 workflow_task_attempt: u64,
3571 message: impl Into<String>,
3572 failure_type: &str,
3573 ) -> Result<Value> {
3574 let body = json!({
3575 "lease_owner": lease_owner,
3576 "workflow_task_attempt": workflow_task_attempt,
3577 "failure": {
3578 "message": message.into(),
3579 "type": failure_type
3580 }
3581 });
3582 let path = format!("/worker/workflow-tasks/{task_id}/fail");
3583 self.request_json(
3584 reqwest::Method::POST,
3585 &path,
3586 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3587 Some(&body),
3588 )
3589 .await
3590 }
3591
3592 pub async fn poll_activity_task(
3593 &self,
3594 worker_id: &str,
3595 task_queue: &str,
3596 timeout: Duration,
3597 ) -> Result<Option<ActivityTask>> {
3598 Ok(self
3599 .poll_activity_task_response(worker_id, task_queue, timeout)
3600 .await?
3601 .task)
3602 }
3603
3604 pub async fn poll_activity_task_response(
3606 &self,
3607 worker_id: &str,
3608 task_queue: &str,
3609 timeout: Duration,
3610 ) -> Result<PollActivityTaskResponse> {
3611 let poll_request_id = unique_request_id("rust-activity-poll");
3612 self.poll_activity_task_response_with_request_id(
3613 worker_id,
3614 task_queue,
3615 timeout,
3616 &poll_request_id,
3617 1,
3618 )
3619 .await
3620 }
3621
3622 async fn poll_activity_task_response_with_request_id(
3623 &self,
3624 worker_id: &str,
3625 task_queue: &str,
3626 timeout: Duration,
3627 poll_request_id: &str,
3628 transport_retries: usize,
3629 ) -> Result<PollActivityTaskResponse> {
3630 let body = json!({
3631 "worker_id": worker_id,
3632 "task_queue": task_queue,
3633 "poll_request_id": poll_request_id,
3634 "timeout_seconds": long_poll_timeout_seconds(timeout),
3635 });
3636 let data: PollActivityTaskResponse = self
3637 .poll_request_json(
3638 "/worker/activity-tasks/poll",
3639 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3640 &body,
3641 timeout + Duration::from_secs(5),
3642 transport_retries,
3643 )
3644 .await?;
3645 Ok(data)
3646 }
3647
3648 pub async fn complete_activity_task<T: Serialize>(
3649 &self,
3650 task_id: &str,
3651 activity_attempt_id: &str,
3652 lease_owner: &str,
3653 result: T,
3654 codec: &str,
3655 ) -> Result<Value> {
3656 let result = encode_typed_envelope(&AvroValue::from_serialize(&result)?, codec)?;
3657 let body = json!({
3658 "activity_attempt_id": activity_attempt_id,
3659 "lease_owner": lease_owner,
3660 "result": result
3661 });
3662 let path = format!("/worker/activity-tasks/{task_id}/complete");
3663 activity_task_response(
3664 self.request_json(
3665 reqwest::Method::POST,
3666 &path,
3667 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3668 Some(&body),
3669 )
3670 .await,
3671 "complete",
3672 task_id,
3673 activity_attempt_id,
3674 )
3675 }
3676
3677 pub async fn fail_activity_task(
3678 &self,
3679 task_id: &str,
3680 activity_attempt_id: &str,
3681 lease_owner: &str,
3682 message: impl Into<String>,
3683 non_retryable: bool,
3684 ) -> Result<Value> {
3685 let body = json!({
3686 "activity_attempt_id": activity_attempt_id,
3687 "lease_owner": lease_owner,
3688 "failure": {
3689 "message": message.into(),
3690 "type": "RustActivityFailure",
3691 "non_retryable": non_retryable
3692 }
3693 });
3694 let path = format!("/worker/activity-tasks/{task_id}/fail");
3695 activity_task_response(
3696 self.request_json(
3697 reqwest::Method::POST,
3698 &path,
3699 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3700 Some(&body),
3701 )
3702 .await,
3703 "fail",
3704 task_id,
3705 activity_attempt_id,
3706 )
3707 }
3708
3709 pub async fn heartbeat_activity_task<T: Serialize>(
3710 &self,
3711 task_id: &str,
3712 activity_attempt_id: &str,
3713 lease_owner: &str,
3714 details: T,
3715 ) -> Result<ActivityHeartbeatResponse> {
3716 let details = encode_typed_envelope(&AvroValue::from_serialize(&details)?, DEFAULT_CODEC)?;
3717 let body = json!({
3718 "activity_attempt_id": activity_attempt_id,
3719 "lease_owner": lease_owner,
3720 "details": details
3721 });
3722 let path = format!("/worker/activity-tasks/{task_id}/heartbeat");
3723 activity_task_response(
3724 self.request_json(
3725 reqwest::Method::POST,
3726 &path,
3727 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3728 Some(&body),
3729 )
3730 .await,
3731 "heartbeat",
3732 task_id,
3733 activity_attempt_id,
3734 )
3735 }
3736
3737 async fn request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
3738 &self,
3739 method: reqwest::Method,
3740 path: &str,
3741 protocol: RequestProtocol,
3742 body: Option<&B>,
3743 ) -> Result<T> {
3744 self.request_json_with_timeout(method, path, protocol, body, Duration::from_secs(60))
3745 .await
3746 }
3747
3748 async fn request_json_with_timeout<T: DeserializeOwned, B: Serialize + ?Sized>(
3749 &self,
3750 method: reqwest::Method,
3751 path: &str,
3752 protocol: RequestProtocol,
3753 body: Option<&B>,
3754 timeout: Duration,
3755 ) -> Result<T> {
3756 let auth_token = self.auth_token(protocol)?;
3757 let mut request = self
3758 .http
3759 .request(method, format!("{}/api{}", self.base_url, path))
3760 .timeout(timeout)
3761 .header(reqwest::header::ACCEPT, "application/json")
3762 .header(reqwest::header::CONTENT_TYPE, "application/json")
3763 .header("X-Namespace", &self.namespace);
3764
3765 match protocol {
3766 RequestProtocol::Worker(version) => {
3767 request = request.header("X-Durable-Workflow-Protocol-Version", version);
3768 }
3769 RequestProtocol::ControlPlane => {
3770 request = request.header(
3771 "X-Durable-Workflow-Control-Plane-Version",
3772 CONTROL_PLANE_VERSION,
3773 );
3774 }
3775 }
3776
3777 if let Some(token) = auth_token {
3778 request = request.bearer_auth(token);
3779 }
3780
3781 if let Some(body) = body {
3782 request = request.json(body);
3783 }
3784
3785 let response = request.send().await?;
3786 let status = response.status();
3787 let bytes = response.bytes().await?;
3788
3789 if !status.is_success() {
3790 let body = String::from_utf8_lossy(&bytes).to_string();
3791 if let Some(protocol) = protocol_failure(status, &body) {
3792 return Err(Error::Protocol(protocol));
3793 }
3794 return Err(Error::Http { status, body });
3795 }
3796
3797 if bytes.is_empty() {
3798 return Ok(serde_json::from_value(Value::Null)?);
3799 }
3800
3801 Ok(serde_json::from_slice(&bytes)?)
3802 }
3803
3804 async fn poll_request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
3805 &self,
3806 path: &str,
3807 protocol: RequestProtocol,
3808 body: &B,
3809 timeout: Duration,
3810 max_retries: usize,
3811 ) -> Result<T> {
3812 let mut retries = 0;
3813
3814 loop {
3815 let response = self
3816 .request_json_with_timeout(
3817 reqwest::Method::POST,
3818 path,
3819 protocol,
3820 Some(body),
3821 timeout,
3822 )
3823 .await;
3824
3825 match response {
3826 Err(Error::Transport(_)) if retries < max_retries => retries += 1,
3827 response => return worker_poll_response(response),
3828 }
3829 }
3830 }
3831
3832 fn auth_token(&self, protocol: RequestProtocol) -> Result<Option<&str>> {
3833 match protocol {
3834 RequestProtocol::Worker(_) => {
3835 if let Some(token) = self.worker_token.as_deref().or(self.token.as_deref()) {
3836 return Ok(Some(token));
3837 }
3838 if self.control_token.is_some() {
3839 return Err(Error::MissingRoleCredentials {
3840 role: "worker",
3841 opposite_role: "control",
3842 });
3843 }
3844 Ok(None)
3845 }
3846 RequestProtocol::ControlPlane => {
3847 if let Some(token) = self.control_token.as_deref().or(self.token.as_deref()) {
3848 return Ok(Some(token));
3849 }
3850 if self.worker_token.is_some() {
3851 return Err(Error::MissingRoleCredentials {
3852 role: "control",
3853 opposite_role: "worker",
3854 });
3855 }
3856 Ok(None)
3857 }
3858 }
3859 }
3860}
3861
3862fn query_failure(status: reqwest::StatusCode, raw_body: String) -> QueryFailure {
3863 let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
3864 let reason = body
3865 .get("reason")
3866 .and_then(Value::as_str)
3867 .unwrap_or("query_rejected")
3868 .to_string();
3869 let message = body
3870 .get("message")
3871 .or_else(|| body.get("error"))
3872 .and_then(Value::as_str)
3873 .unwrap_or("workflow query was rejected")
3874 .to_string();
3875
3876 QueryFailure {
3877 status: status.as_u16(),
3878 reason,
3879 message,
3880 body,
3881 }
3882}
3883
3884fn workflow_command_result(
3885 command: WorkflowCommandKind,
3886 data: Value,
3887 workflow_id: &str,
3888 run_id: Option<&str>,
3889) -> WorkflowCommandResult {
3890 WorkflowCommandResult {
3891 command,
3892 workflow_id: data
3893 .get("workflow_id")
3894 .and_then(Value::as_str)
3895 .unwrap_or(workflow_id)
3896 .to_string(),
3897 run_id: data
3898 .get("run_id")
3899 .and_then(Value::as_str)
3900 .or(run_id)
3901 .map(str::to_string),
3902 outcome: data
3903 .get("outcome")
3904 .and_then(Value::as_str)
3905 .map(str::to_string),
3906 reason: data
3907 .get("reason")
3908 .and_then(Value::as_str)
3909 .map(str::to_string),
3910 command_status: data
3911 .get("command_status")
3912 .and_then(Value::as_str)
3913 .map(str::to_string),
3914 raw: data,
3915 }
3916}
3917
3918fn workflow_command_rejection(
3919 command: WorkflowCommandKind,
3920 status: reqwest::StatusCode,
3921 raw_body: String,
3922 workflow_id: &str,
3923 run_id: Option<&str>,
3924) -> WorkflowCommandRejection {
3925 let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
3926 WorkflowCommandRejection {
3927 command,
3928 status: status.as_u16(),
3929 reason: body
3930 .get("reason")
3931 .and_then(Value::as_str)
3932 .unwrap_or("workflow_command_rejected")
3933 .to_string(),
3934 message: body
3935 .get("message")
3936 .or_else(|| body.get("error"))
3937 .and_then(Value::as_str)
3938 .unwrap_or("workflow lifecycle command was rejected")
3939 .to_string(),
3940 workflow_id: body
3941 .get("workflow_id")
3942 .and_then(Value::as_str)
3943 .unwrap_or(workflow_id)
3944 .to_string(),
3945 run_id: body
3946 .get("run_id")
3947 .and_then(Value::as_str)
3948 .or(run_id)
3949 .map(str::to_string),
3950 target_scope: body
3951 .get("target_scope")
3952 .and_then(Value::as_str)
3953 .map(str::to_string),
3954 body,
3955 }
3956}
3957
3958fn query_task_response(response: Result<Value>) -> Result<Value> {
3959 match response {
3960 Err(Error::Http { status, body }) => Err(Error::QueryFailed(query_failure(status, body))),
3961 response => response,
3962 }
3963}
3964
3965fn worker_poll_response<T: DeserializeOwned>(response: Result<T>) -> Result<T> {
3966 match response {
3967 Err(Error::Http { status, body })
3968 if status == reqwest::StatusCode::CONFLICT && worker_poll_body_is_stop(&body) =>
3969 {
3970 Ok(serde_json::from_str(&body)?)
3971 }
3972 response => response,
3973 }
3974}
3975
3976fn worker_poll_body_is_stop(body: &str) -> bool {
3977 serde_json::from_str::<Value>(body)
3978 .ok()
3979 .is_some_and(|body| {
3980 worker_poll_is_stop(
3981 body.get("poll_status").and_then(Value::as_str),
3982 body.get("reason").and_then(Value::as_str),
3983 )
3984 })
3985}
3986
3987fn worker_poll_is_stop(poll_status: Option<&str>, reason: Option<&str>) -> bool {
3988 matches!(poll_status, Some("draining" | "stopped"))
3989 || matches!(reason, Some("worker_draining" | "worker_stopped"))
3990}
3991
3992fn query_task_rejection_is_final(error: &Error) -> bool {
3993 matches!(
3994 error,
3995 Error::QueryFailed(failure)
3996 if QUERY_TASK_FINAL_REJECTION_REASONS.contains(&failure.reason.as_str())
3997 )
3998}
3999
4000fn activity_task_response<T>(
4001 response: Result<T>,
4002 operation: &str,
4003 task_id: &str,
4004 activity_attempt_id: &str,
4005) -> Result<T> {
4006 match response {
4007 Err(Error::Http { status, body }) => {
4008 let body = serde_json::from_str(&body).unwrap_or_else(|_| json!({"message": body}));
4009 Err(Error::ActivityTaskRejected(ActivityTaskRejection {
4010 operation: operation.to_string(),
4011 status: status.as_u16(),
4012 reason: body
4013 .get("reason")
4014 .and_then(Value::as_str)
4015 .unwrap_or("activity_task_rejected")
4016 .to_string(),
4017 task_id: body
4018 .get("task_id")
4019 .and_then(Value::as_str)
4020 .unwrap_or(task_id)
4021 .to_string(),
4022 activity_attempt_id: body
4023 .get("activity_attempt_id")
4024 .and_then(Value::as_str)
4025 .unwrap_or(activity_attempt_id)
4026 .to_string(),
4027 cancel_requested: body
4028 .get("cancel_requested")
4029 .and_then(Value::as_bool)
4030 .unwrap_or(false),
4031 can_continue: body.get("can_continue").and_then(Value::as_bool),
4032 run_closed_reason: body
4033 .get("run_closed_reason")
4034 .and_then(Value::as_str)
4035 .map(str::to_string),
4036 body,
4037 }))
4038 }
4039 response => response,
4040 }
4041}
4042
4043fn activity_task_rejection_is_final(error: &Error) -> bool {
4044 matches!(
4045 error,
4046 Error::ActivityTaskRejected(rejection)
4047 if matches!(
4048 rejection.reason.as_str(),
4049 "run_cancelled"
4050 | "run_terminated"
4051 | "attempt_closed"
4052 | "stale_attempt"
4053 | "activity_cancelled"
4054 | "task_cancelled"
4055 | "run_closed"
4056 | "activity_not_running"
4057 | "attempt_not_found"
4058 )
4059 )
4060}
4061
4062fn workflow_task_completion_is_terminal_timeout(
4063 error: &Error,
4064 task_id: &str,
4065 workflow_task_attempt: u64,
4066 run_id: Option<&str>,
4067) -> bool {
4068 let Error::Http { status, body } = error else {
4069 return false;
4070 };
4071 if *status != reqwest::StatusCode::CONFLICT {
4072 return false;
4073 }
4074
4075 let Some(run_id) = run_id else {
4076 return false;
4077 };
4078 let Ok(body) = serde_json::from_str::<Value>(body) else {
4079 return false;
4080 };
4081
4082 body.get("recorded").and_then(Value::as_bool) == Some(false)
4083 && body.get("reason").and_then(Value::as_str) == Some("run_timed_out")
4084 && body.get("run_status").and_then(Value::as_str) == Some("failed")
4085 && body.get("run_id").and_then(Value::as_str) == Some(run_id)
4086 && body.get("task_id").and_then(Value::as_str) == Some(task_id)
4087 && body.get("workflow_task_attempt").and_then(Value::as_u64) == Some(workflow_task_attempt)
4088}
4089
4090fn protocol_failure(status: reqwest::StatusCode, raw_body: &str) -> Option<ProtocolFailure> {
4091 let body: Value = serde_json::from_str(raw_body).ok()?;
4092 let reason = body.get("reason")?.as_str()?;
4093 if !matches!(
4094 reason,
4095 "missing_protocol_version"
4096 | "unsupported_protocol_version"
4097 | "missing_control_plane_version"
4098 | "unsupported_control_plane_version"
4099 ) {
4100 return None;
4101 }
4102
4103 Some(ProtocolFailure {
4104 status: status.as_u16(),
4105 reason: reason.to_string(),
4106 message: body
4107 .get("message")
4108 .or_else(|| body.get("error"))
4109 .and_then(Value::as_str)
4110 .unwrap_or("protocol version rejected")
4111 .to_string(),
4112 supported_version: body
4113 .get("supported_version")
4114 .and_then(Value::as_str)
4115 .map(str::to_string),
4116 requested_version: body
4117 .get("requested_version")
4118 .and_then(Value::as_str)
4119 .map(str::to_string),
4120 body,
4121 })
4122}
4123
4124fn long_poll_timeout_seconds(timeout: Duration) -> u64 {
4125 timeout
4126 .as_secs()
4127 .saturating_add(u64::from(timeout.subsec_nanos() > 0))
4128 .min(MAX_LONG_POLL_TIMEOUT_SECONDS)
4129}
4130
4131fn worker_operation_is_retryable(error: &Error) -> bool {
4132 match error {
4133 Error::Transport(error) => {
4134 error.is_timeout() || error.is_connect() || error.is_request() || error.is_body()
4135 }
4136 Error::Http { status, .. } => {
4137 matches!(
4138 *status,
4139 reqwest::StatusCode::REQUEST_TIMEOUT | reqwest::StatusCode::TOO_MANY_REQUESTS
4140 ) || status.is_server_error()
4141 }
4142 _ => false,
4143 }
4144}
4145
4146fn worker_retry_delay(policy: WorkerRetryPolicy, retry: usize) -> Duration {
4147 let exponent = retry.saturating_sub(1).min(31) as u32;
4148 policy
4149 .initial_backoff
4150 .saturating_mul(1_u32 << exponent)
4151 .min(policy.max_backoff)
4152}
4153
4154#[derive(Debug)]
4155pub struct ClientBuilder {
4156 base_url: String,
4157 token: Option<String>,
4158 control_token: Option<String>,
4159 worker_token: Option<String>,
4160 namespace: String,
4161 timeout: Duration,
4162}
4163
4164impl ClientBuilder {
4165 pub fn token(mut self, token: Option<String>) -> Self {
4166 self.token = token;
4167 self
4168 }
4169
4170 pub fn control_token(mut self, token: Option<String>) -> Self {
4171 self.control_token = token;
4172 self
4173 }
4174
4175 pub fn worker_token(mut self, token: Option<String>) -> Self {
4176 self.worker_token = token;
4177 self
4178 }
4179
4180 pub fn namespace(mut self, namespace: impl Into<String>) -> Self {
4181 self.namespace = namespace.into();
4182 self
4183 }
4184
4185 pub fn timeout(mut self, timeout: Duration) -> Self {
4186 self.timeout = timeout;
4187 self
4188 }
4189
4190 pub fn build(self) -> Result<Client> {
4191 let base_url = self.base_url.trim_end_matches('/').to_string();
4192 let has_sdk_api_suffix = reqwest::Url::parse(&base_url)
4193 .map(|url| url.path().trim_end_matches('/').ends_with("/api"))
4194 .unwrap_or_else(|_| base_url.ends_with("/api"));
4195
4196 if has_sdk_api_suffix {
4197 return Err(Error::InvalidBaseUrl);
4198 }
4199
4200 Ok(Client {
4201 http: reqwest::Client::builder().timeout(self.timeout).build()?,
4202 base_url,
4203 token: self.token,
4204 control_token: self.control_token,
4205 worker_token: self.worker_token,
4206 namespace: self.namespace,
4207 })
4208 }
4209}
4210
4211#[derive(Clone, Debug)]
4212pub struct WorkflowHandle {
4213 client: Client,
4214 pub workflow_id: String,
4215 pub run_id: Option<String>,
4216 pub workflow_type: String,
4217}
4218
4219impl WorkflowHandle {
4220 pub async fn describe(&self) -> Result<WorkflowDescription> {
4222 self.client.describe_workflow(&self.workflow_id).await
4223 }
4224
4225 pub async fn describe_selected_run(&self) -> Result<WorkflowDescription> {
4227 let run_id = self.run_id.as_deref().ok_or_else(|| {
4228 Error::Codec("run_id is required for selected-run description".to_string())
4229 })?;
4230 self.client
4231 .describe_workflow_run(&self.workflow_id, run_id)
4232 .await
4233 }
4234
4235 pub async fn signal<T: Serialize>(&self, signal_name: &str, input: T) -> Result<Value> {
4236 self.client
4237 .signal_workflow(&self.workflow_id, signal_name, input)
4238 .await
4239 }
4240
4241 pub async fn signal_selected_run<T: Serialize>(
4243 &self,
4244 signal_name: &str,
4245 input: T,
4246 ) -> Result<Value> {
4247 let run_id = self.run_id.as_deref().ok_or_else(|| {
4248 Error::Codec("run_id is required for selected-run signaling".to_string())
4249 })?;
4250 self.client
4251 .signal_workflow_run(&self.workflow_id, run_id, signal_name, input)
4252 .await
4253 }
4254
4255 pub async fn cancel(&self, options: WorkflowCommandOptions) -> Result<WorkflowCommandResult> {
4257 self.client
4258 .cancel_workflow(&self.workflow_id, options)
4259 .await
4260 }
4261
4262 pub async fn cancel_selected_run(
4264 &self,
4265 options: WorkflowCommandOptions,
4266 ) -> Result<WorkflowCommandResult> {
4267 let run_id = self.run_id.as_deref().ok_or_else(|| {
4268 Error::Codec("run_id is required for selected-run cancellation".to_string())
4269 })?;
4270 self.client
4271 .cancel_workflow_run(&self.workflow_id, run_id, options)
4272 .await
4273 }
4274
4275 pub async fn terminate(
4277 &self,
4278 options: WorkflowCommandOptions,
4279 ) -> Result<WorkflowCommandResult> {
4280 self.client
4281 .terminate_workflow(&self.workflow_id, options)
4282 .await
4283 }
4284
4285 pub async fn terminate_selected_run(
4287 &self,
4288 options: WorkflowCommandOptions,
4289 ) -> Result<WorkflowCommandResult> {
4290 let run_id = self.run_id.as_deref().ok_or_else(|| {
4291 Error::Codec("run_id is required for selected-run termination".to_string())
4292 })?;
4293 self.client
4294 .terminate_workflow_run(&self.workflow_id, run_id, options)
4295 .await
4296 }
4297
4298 pub async fn query<T: Serialize>(&self, query_name: &str, input: T) -> Result<Value> {
4300 self.client
4301 .query_workflow(&self.workflow_id, query_name, input)
4302 .await
4303 }
4304
4305 pub async fn query_avro_value<T: Serialize>(
4306 &self,
4307 query_name: &str,
4308 input: T,
4309 ) -> Result<AvroValue> {
4310 self.client
4311 .query_workflow_avro_value(&self.workflow_id, query_name, input)
4312 .await
4313 }
4314
4315 pub async fn update<T: Serialize>(
4316 &self,
4317 update_name: &str,
4318 input: T,
4319 request_id: Option<&str>,
4320 ) -> Result<Value> {
4321 self.client
4322 .update_workflow(&self.workflow_id, update_name, input, request_id)
4323 .await
4324 }
4325
4326 pub async fn update_avro_value<T: Serialize>(
4327 &self,
4328 update_name: &str,
4329 input: T,
4330 request_id: Option<&str>,
4331 ) -> Result<AvroValue> {
4332 self.client
4333 .update_workflow_avro_value(&self.workflow_id, update_name, input, request_id)
4334 .await
4335 }
4336
4337 pub async fn query_selected_run<T: Serialize>(
4339 &self,
4340 query_name: &str,
4341 input: T,
4342 ) -> Result<Value> {
4343 let run_id = self
4344 .run_id
4345 .as_deref()
4346 .ok_or_else(|| Error::Codec("run_id is required for selected-run query".to_string()))?;
4347 self.client
4348 .query_workflow_run(&self.workflow_id, run_id, query_name, input)
4349 .await
4350 }
4351
4352 pub async fn result(&self, options: WorkflowResultOptions) -> Result<Value> {
4354 self.result_target(options, None).await
4355 }
4356
4357 pub async fn result_avro_value(&self, options: WorkflowResultOptions) -> Result<AvroValue> {
4359 self.result_avro_value_target(options, None).await
4360 }
4361
4362 pub async fn result_typed<T: DeserializeOwned>(
4364 &self,
4365 options: WorkflowResultOptions,
4366 ) -> Result<T> {
4367 let result = self.result_avro_value(options).await?;
4368 decode_handler_result(result, HandlerKind::Workflow, &self.workflow_type)
4369 }
4370
4371 pub async fn result_selected_run(&self, options: WorkflowResultOptions) -> Result<Value> {
4373 let run_id = self.run_id.as_deref().ok_or_else(|| {
4374 Error::Codec("run_id is required for selected-run result".to_string())
4375 })?;
4376 self.result_target(options, Some(run_id)).await
4377 }
4378
4379 pub async fn result_selected_run_avro_value(
4381 &self,
4382 options: WorkflowResultOptions,
4383 ) -> Result<AvroValue> {
4384 let run_id = self.run_id.as_deref().ok_or_else(|| {
4385 Error::Codec("run_id is required for selected-run result".to_string())
4386 })?;
4387 self.result_avro_value_target(options, Some(run_id)).await
4388 }
4389
4390 pub async fn result_selected_run_typed<T: DeserializeOwned>(
4392 &self,
4393 options: WorkflowResultOptions,
4394 ) -> Result<T> {
4395 let result = self.result_selected_run_avro_value(options).await?;
4396 decode_handler_result(result, HandlerKind::Workflow, &self.workflow_type)
4397 }
4398
4399 async fn result_avro_value_target(
4400 &self,
4401 options: WorkflowResultOptions,
4402 selected_run_id: Option<&str>,
4403 ) -> Result<AvroValue> {
4404 let started = Instant::now();
4405
4406 loop {
4407 let description = match selected_run_id {
4408 Some(run_id) => {
4409 self.client
4410 .describe_workflow_run(&self.workflow_id, run_id)
4411 .await?
4412 }
4413 None => self.describe().await?,
4414 };
4415 if description.is_completed() {
4416 return description.output_avro_value.ok_or_else(|| {
4417 Error::Codec(
4418 "missing_payload_envelope: typed workflow result requires output_envelope"
4419 .to_string(),
4420 )
4421 });
4422 }
4423 if description.is_terminal() {
4424 let outcome =
4425 workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
4426 return Err(match outcome.kind {
4427 WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
4428 WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
4429 WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
4430 WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
4431 });
4432 }
4433 if started.elapsed() >= options.timeout {
4434 return Err(Error::Timeout);
4435 }
4436 tokio::time::sleep(options.poll_interval).await;
4437 }
4438 }
4439
4440 async fn result_target(
4441 &self,
4442 options: WorkflowResultOptions,
4443 selected_run_id: Option<&str>,
4444 ) -> Result<Value> {
4445 let started = Instant::now();
4446
4447 loop {
4448 let description = match selected_run_id {
4449 Some(run_id) => {
4450 self.client
4451 .describe_workflow_run(&self.workflow_id, run_id)
4452 .await?
4453 }
4454 None => self.describe().await?,
4455 };
4456 if description.is_completed() {
4457 return Ok(description.output.unwrap_or(Value::Null));
4458 }
4459
4460 if description.is_terminal() {
4461 let outcome =
4462 workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
4463 return Err(match outcome.kind {
4464 WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
4465 WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
4466 WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
4467 WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
4468 });
4469 }
4470
4471 if started.elapsed() >= options.timeout {
4472 return Err(Error::WorkflowTimedOut(WorkflowTerminalOutcome {
4473 kind: WorkflowTerminalKind::TimedOut,
4474 workflow_id: description
4475 .workflow_id
4476 .clone()
4477 .unwrap_or_else(|| self.workflow_id.clone()),
4478 run_id: description
4479 .run_id
4480 .clone()
4481 .or_else(|| selected_run_id.map(str::to_string)),
4482 reason: "result_wait_timeout".to_string(),
4483 failure_category: Some("client_timeout".to_string()),
4484 failure_id: None,
4485 exception_type: None,
4486 exception_class: None,
4487 non_retryable: None,
4488 message: Some(format!(
4489 "workflow result was not terminal within {:?}",
4490 options.timeout
4491 )),
4492 exception: None,
4493 raw: description.raw_value(),
4494 }));
4495 }
4496
4497 tokio::time::sleep(options.poll_interval).await;
4498 }
4499 }
4500}
4501
4502#[derive(Clone, Copy, Debug)]
4503pub struct WorkflowResultOptions {
4504 pub poll_interval: Duration,
4505 pub timeout: Duration,
4506}
4507
4508impl Default for WorkflowResultOptions {
4509 fn default() -> Self {
4510 Self {
4511 poll_interval: Duration::from_millis(500),
4512 timeout: Duration::from_secs(30),
4513 }
4514 }
4515}
4516
4517#[derive(Clone, Debug, Deserialize)]
4518pub struct WorkflowDescription {
4519 pub workflow_id: Option<String>,
4520 pub run_id: Option<String>,
4521 pub workflow_type: Option<String>,
4522 pub status: Option<String>,
4523 #[serde(default)]
4524 pub closed_reason: Option<String>,
4525 #[serde(default)]
4526 pub error: Option<String>,
4527 #[serde(default)]
4528 pub failure: Option<Value>,
4529 #[serde(default)]
4530 pub exception: Option<Value>,
4531 #[serde(default)]
4532 pub failures: Vec<Value>,
4533 #[serde(default)]
4534 pub output: Option<Value>,
4535 #[serde(default)]
4536 pub output_envelope: Option<Value>,
4537 #[serde(skip)]
4538 pub output_avro_value: Option<AvroValue>,
4539 #[serde(flatten)]
4540 pub raw: HashMap<String, Value>,
4541}
4542
4543#[derive(Clone, Debug, Deserialize)]
4545pub struct WorkflowStreamDescription {
4546 pub stream_name: String,
4547 pub status: String,
4548 pub last_offset: i64,
4549 pub total_items: u64,
4550 pub pending_items: u64,
4551 #[serde(default)]
4552 pub opened_at: Option<String>,
4553 #[serde(default)]
4554 pub last_appended_at: Option<String>,
4555 #[serde(default)]
4556 pub closed_at: Option<String>,
4557 #[serde(default)]
4558 pub error_reason: Option<String>,
4559 #[serde(default)]
4560 pub retention_seconds: Option<u64>,
4561 #[serde(flatten)]
4562 pub raw: HashMap<String, Value>,
4563}
4564
4565impl WorkflowStreamDescription {
4566 pub fn is_terminal(&self) -> bool {
4567 matches!(self.status.as_str(), "closed" | "errored")
4568 }
4569}
4570
4571#[derive(Clone, Debug, Default)]
4573pub struct WorkflowStreamAppendItem {
4574 pub payload_envelope: Option<Value>,
4575 pub payload_reference: Option<String>,
4576 pub item_type: Option<String>,
4577 pub content_type: Option<String>,
4578 pub idempotency_key: Option<String>,
4579}
4580
4581impl WorkflowStreamAppendItem {
4582 pub fn new<T: Serialize>(payload: T) -> Result<Self> {
4584 let value = AvroValue::from_serialize(&payload)?;
4585 Ok(Self {
4586 payload_envelope: Some(encode_typed_envelope(&value, DEFAULT_CODEC)?),
4587 ..Self::default()
4588 })
4589 }
4590
4591 pub fn from_reference(reference: impl Into<String>) -> Self {
4593 Self {
4594 payload_reference: Some(reference.into()),
4595 ..Self::default()
4596 }
4597 }
4598
4599 pub fn item_type(mut self, item_type: impl Into<String>) -> Self {
4600 self.item_type = Some(item_type.into());
4601 self
4602 }
4603
4604 pub fn content_type(mut self, content_type: impl Into<String>) -> Self {
4605 self.content_type = Some(content_type.into());
4606 self
4607 }
4608
4609 pub fn idempotency_key(mut self, idempotency_key: impl Into<String>) -> Self {
4610 self.idempotency_key = Some(idempotency_key.into());
4611 self
4612 }
4613
4614 fn wire_value(&self, derived_idempotency_key: Option<String>) -> Value {
4615 let mut item = serde_json::Map::new();
4616 if let Some(payload) = &self.payload_envelope {
4617 item.insert("payload".to_string(), payload.clone());
4618 item.insert("payload_codec".to_string(), json!(DEFAULT_CODEC));
4619 }
4620 if let Some(reference) = &self.payload_reference {
4621 item.insert("payload_reference".to_string(), json!(reference));
4622 }
4623 if let Some(item_type) = &self.item_type {
4624 item.insert("item_type".to_string(), json!(item_type));
4625 }
4626 if let Some(content_type) = &self.content_type {
4627 item.insert("content_type".to_string(), json!(content_type));
4628 }
4629 if let Some(key) = derived_idempotency_key
4630 .as_ref()
4631 .or(self.idempotency_key.as_ref())
4632 {
4633 item.insert("idempotency_key".to_string(), json!(key));
4634 }
4635 Value::Object(item)
4636 }
4637}
4638
4639#[derive(Clone, Debug)]
4641pub struct WorkflowStreamItem {
4642 pub offset: u64,
4643 pub payload: Option<Value>,
4644 pub payload_envelope: Option<Value>,
4645 pub payload_reference: Option<String>,
4646 pub payload_codec: Option<String>,
4647 pub idempotency_key: Option<String>,
4648 pub item_type: Option<String>,
4649 pub content_type: Option<String>,
4650 pub origin: Option<String>,
4651 pub origin_reference: Option<String>,
4652 pub emitted_at: Option<String>,
4653 pub raw: Value,
4654}
4655
4656#[derive(Clone, Debug)]
4658pub struct WorkflowStreamPage {
4659 pub stream: WorkflowStreamDescription,
4660 pub items: Vec<WorkflowStreamItem>,
4661 pub next_offset: u64,
4662 pub terminal: bool,
4663}
4664
4665#[derive(Clone, Debug)]
4667pub struct WorkflowStreamAppendResult {
4668 pub stream: WorkflowStreamDescription,
4669 pub accepted_offsets: Vec<u64>,
4670 pub accepted: u64,
4671 pub deduped: u64,
4672}
4673
4674#[derive(Deserialize)]
4675struct WorkflowStreamListResponse {
4676 #[serde(default)]
4677 streams: Vec<WorkflowStreamDescription>,
4678}
4679
4680#[derive(Deserialize)]
4681struct WorkflowStreamDescriptionResponse {
4682 stream: WorkflowStreamDescription,
4683}
4684
4685#[derive(Deserialize)]
4686struct WorkflowStreamPageResponse {
4687 stream: WorkflowStreamDescription,
4688 #[serde(default)]
4689 items: Vec<Value>,
4690 next_offset: u64,
4691 terminal: bool,
4692}
4693
4694#[derive(Deserialize)]
4695struct WorkflowStreamAppendResponse {
4696 stream: WorkflowStreamDescription,
4697 #[serde(default)]
4698 accepted_offsets: Vec<u64>,
4699 accepted: u64,
4700 deduped: u64,
4701}
4702
4703impl WorkflowDescription {
4704 pub fn is_completed(&self) -> bool {
4705 matches!(self.status.as_deref(), Some("completed" | "Completed"))
4706 }
4707
4708 pub fn is_terminal(&self) -> bool {
4709 matches!(
4710 self.status.as_deref(),
4711 Some(
4712 "completed"
4713 | "Completed"
4714 | "failed"
4715 | "Failed"
4716 | "cancelled"
4717 | "Cancelled"
4718 | "terminated"
4719 | "Terminated"
4720 | "timed_out"
4721 | "TimedOut",
4722 )
4723 )
4724 }
4725
4726 fn decode_payloads(&mut self) -> Result<()> {
4727 if let Some(envelope) = &self.output_envelope {
4728 let value = decode_wire_avro_value(envelope, DEFAULT_CODEC)?;
4729 self.output = Some(value.clone().into_json()?);
4730 self.output_avro_value = Some(value);
4731 }
4732
4733 Ok(())
4734 }
4735
4736 fn raw_value(&self) -> Value {
4737 let mut data = self.raw.clone();
4738 data.insert(
4739 "workflow_id".to_string(),
4740 self.workflow_id
4741 .clone()
4742 .map(Value::String)
4743 .unwrap_or(Value::Null),
4744 );
4745 data.insert(
4746 "run_id".to_string(),
4747 self.run_id
4748 .clone()
4749 .map(Value::String)
4750 .unwrap_or(Value::Null),
4751 );
4752 data.insert(
4753 "workflow_type".to_string(),
4754 self.workflow_type
4755 .clone()
4756 .map(Value::String)
4757 .unwrap_or(Value::Null),
4758 );
4759 data.insert(
4760 "status".to_string(),
4761 self.status
4762 .clone()
4763 .map(Value::String)
4764 .unwrap_or(Value::Null),
4765 );
4766 data.insert(
4767 "closed_reason".to_string(),
4768 self.closed_reason
4769 .clone()
4770 .map(Value::String)
4771 .unwrap_or(Value::Null),
4772 );
4773 if let Some(failure) = &self.failure {
4774 data.insert("failure".to_string(), failure.clone());
4775 }
4776 if let Some(exception) = &self.exception {
4777 data.insert("exception".to_string(), exception.clone());
4778 }
4779 Value::Object(data.into_iter().collect())
4780 }
4781}
4782
4783fn workflow_terminal_outcome(
4784 description: &WorkflowDescription,
4785 workflow_id: &str,
4786 run_id: Option<&str>,
4787) -> WorkflowTerminalOutcome {
4788 let terminal_kind = description
4789 .closed_reason
4790 .as_deref()
4791 .or(description.status.as_deref())
4792 .unwrap_or("failed")
4793 .to_ascii_lowercase();
4794 let kind = match terminal_kind.as_str() {
4795 "cancelled" | "canceled" => WorkflowTerminalKind::Cancelled,
4796 "terminated" => WorkflowTerminalKind::Terminated,
4797 "timed_out" | "timedout" => WorkflowTerminalKind::TimedOut,
4798 _ => WorkflowTerminalKind::Failed,
4799 };
4800 let default_reason = match kind {
4801 WorkflowTerminalKind::Failed => "workflow_failed",
4802 WorkflowTerminalKind::Cancelled => "cancelled",
4803 WorkflowTerminalKind::Terminated => "terminated",
4804 WorkflowTerminalKind::TimedOut => "timed_out",
4805 };
4806 let failure = description
4807 .failure
4808 .as_ref()
4809 .filter(|value| value.is_object());
4810 let nested_failure = failure
4811 .and_then(|value| value.get("failures"))
4812 .and_then(Value::as_array)
4813 .and_then(|failures| failures.last())
4814 .or_else(|| description.failures.last());
4815 let exception = description
4816 .exception
4817 .clone()
4818 .or_else(|| failure.and_then(|value| value.get("exception")).cloned())
4819 .or_else(|| {
4820 nested_failure
4821 .and_then(|value| value.get("exception_payload"))
4822 .cloned()
4823 });
4824 let string_field = |name: &str| {
4825 failure
4826 .and_then(|value| value.get(name))
4827 .and_then(Value::as_str)
4828 .or_else(|| {
4829 nested_failure
4830 .and_then(|value| value.get(name))
4831 .and_then(Value::as_str)
4832 })
4833 .map(str::to_string)
4834 };
4835 let exception_field = |name: &str| {
4836 exception
4837 .as_ref()
4838 .and_then(|value| value.get(name))
4839 .and_then(Value::as_str)
4840 .map(str::to_string)
4841 };
4842 let message = description
4843 .error
4844 .clone()
4845 .or_else(|| string_field("message"))
4846 .or_else(|| exception_field("message"));
4847 let reason = description
4848 .raw
4849 .get("reason")
4850 .and_then(Value::as_str)
4851 .map(str::to_string)
4852 .or_else(|| {
4853 failure
4854 .and_then(|value| value.get("reason"))
4855 .and_then(Value::as_str)
4856 .map(str::to_string)
4857 })
4858 .or_else(|| description.closed_reason.clone())
4859 .unwrap_or_else(|| default_reason.to_string());
4860 let failure_id = string_field("failure_id").or_else(|| {
4861 nested_failure
4862 .and_then(|value| value.get("id"))
4863 .and_then(Value::as_str)
4864 .map(str::to_string)
4865 });
4866
4867 WorkflowTerminalOutcome {
4868 kind,
4869 workflow_id: description
4870 .workflow_id
4871 .clone()
4872 .unwrap_or_else(|| workflow_id.to_string()),
4873 run_id: description
4874 .run_id
4875 .clone()
4876 .or_else(|| run_id.map(str::to_string)),
4877 reason,
4878 failure_category: string_field("failure_category")
4879 .or_else(|| Some(default_reason.to_string())),
4880 failure_id,
4881 exception_type: string_field("exception_type").or_else(|| exception_field("type")),
4882 exception_class: string_field("exception_class").or_else(|| exception_field("class")),
4883 non_retryable: failure
4884 .and_then(|value| value.get("non_retryable"))
4885 .and_then(Value::as_bool)
4886 .or_else(|| {
4887 nested_failure
4888 .and_then(|value| value.get("non_retryable"))
4889 .and_then(Value::as_bool)
4890 }),
4891 message,
4892 exception,
4893 raw: description.raw_value(),
4894 }
4895}
4896
4897#[derive(Clone, Debug, Deserialize)]
4898pub struct RegisterWorkerResponse {
4899 pub worker_id: String,
4900 pub registered: bool,
4901 #[serde(default)]
4902 pub heartbeat_interval_seconds: Option<u64>,
4903 #[serde(default)]
4904 pub protocol_version: Option<String>,
4905 #[serde(default)]
4906 pub server_capabilities: Option<Value>,
4907}
4908
4909#[derive(Clone, Debug, Deserialize, PartialEq, Eq)]
4911pub struct WorkerDeregistrationEnvelope {
4912 pub worker_id: String,
4913 pub outcome: String,
4914 pub recovered_workflow_task_count: u64,
4915}
4916
4917#[derive(Clone, Debug, Deserialize)]
4918pub struct PollWorkflowTaskResponse {
4919 #[serde(default)]
4920 pub task: Option<WorkflowTask>,
4921 #[serde(default)]
4922 pub poll_status: Option<String>,
4923 #[serde(default)]
4924 pub reason: Option<String>,
4925 #[serde(default)]
4926 pub protocol_version: Option<String>,
4927 #[serde(default)]
4928 pub server_capabilities: Option<Value>,
4929}
4930
4931impl PollWorkflowTaskResponse {
4932 pub fn outcome(&self) -> WorkerPollOutcome {
4934 worker_poll_outcome(
4935 self.task.is_some(),
4936 self.poll_status.as_deref(),
4937 self.reason.as_deref(),
4938 )
4939 }
4940}
4941
4942fn runtime_supports_workflow_memo_updates(capabilities: Option<&Value>) -> bool {
4943 let Some(capabilities) = capabilities.and_then(Value::as_object) else {
4944 return false;
4945 };
4946 let supported = capabilities
4947 .get("workflow_memo_updates")
4948 .and_then(Value::as_object)
4949 .and_then(|memo| memo.get("supported"))
4950 .and_then(Value::as_bool)
4951 == Some(true);
4952 let command_advertised = capabilities
4953 .get("supported_workflow_task_commands")
4954 .and_then(Value::as_array)
4955 .is_some_and(|commands| {
4956 commands
4957 .iter()
4958 .any(|command| command.as_str() == Some("upsert_memo"))
4959 });
4960 supported && command_advertised
4961}
4962
4963fn commands_use_workflow_memo_updates(commands: &[Value]) -> bool {
4964 commands
4965 .iter()
4966 .any(|command| command.get("type").and_then(Value::as_str) == Some("upsert_memo"))
4967}
4968
4969#[derive(Clone, Debug, Deserialize)]
4970pub struct PollActivityTaskResponse {
4971 #[serde(default)]
4972 pub task: Option<ActivityTask>,
4973 #[serde(default)]
4974 pub poll_status: Option<String>,
4975 #[serde(default)]
4976 pub reason: Option<String>,
4977}
4978
4979impl PollActivityTaskResponse {
4980 pub fn outcome(&self) -> WorkerPollOutcome {
4982 worker_poll_outcome(
4983 self.task.is_some(),
4984 self.poll_status.as_deref(),
4985 self.reason.as_deref(),
4986 )
4987 }
4988}
4989
4990#[derive(Clone, Debug, Deserialize)]
4991pub struct PollQueryTaskResponse {
4992 #[serde(default)]
4993 pub task: Option<QueryTask>,
4994 #[serde(default)]
4995 pub poll_status: Option<String>,
4996 #[serde(default)]
4997 pub reason: Option<String>,
4998}
4999
5000impl PollQueryTaskResponse {
5001 pub fn outcome(&self) -> WorkerPollOutcome {
5003 worker_poll_outcome(
5004 self.task.is_some(),
5005 self.poll_status.as_deref(),
5006 self.reason.as_deref(),
5007 )
5008 }
5009}
5010
5011#[derive(Clone, Debug, PartialEq, Eq)]
5013pub enum WorkerPollOutcome {
5014 Task,
5016 Idle {
5018 poll_status: Option<String>,
5019 reason: Option<String>,
5020 },
5021 Stop {
5023 poll_status: Option<String>,
5024 reason: Option<String>,
5025 },
5026}
5027
5028impl WorkerPollOutcome {
5029 pub fn should_stop(&self) -> bool {
5030 matches!(self, Self::Stop { .. })
5031 }
5032}
5033
5034fn worker_poll_outcome(
5035 has_task: bool,
5036 poll_status: Option<&str>,
5037 reason: Option<&str>,
5038) -> WorkerPollOutcome {
5039 if worker_poll_is_stop(poll_status, reason) {
5040 return WorkerPollOutcome::Stop {
5041 poll_status: poll_status.map(str::to_string),
5042 reason: reason.map(str::to_string),
5043 };
5044 }
5045
5046 if has_task {
5047 WorkerPollOutcome::Task
5048 } else {
5049 WorkerPollOutcome::Idle {
5050 poll_status: poll_status.map(str::to_string),
5051 reason: reason.map(str::to_string),
5052 }
5053 }
5054}
5055
5056#[derive(Clone, Debug, Deserialize)]
5058pub struct QueryTask {
5059 pub query_task_id: String,
5060 #[serde(default = "default_workflow_task_attempt")]
5061 pub query_task_attempt: u64,
5062 #[serde(default)]
5063 pub lease_owner: Option<String>,
5064 #[serde(default)]
5065 pub workflow_id: Option<String>,
5066 #[serde(default)]
5067 pub run_id: Option<String>,
5068 pub workflow_type: String,
5069 pub query_name: String,
5070 #[serde(
5071 default = "missing_task_payload_codec",
5072 deserialize_with = "deserialize_task_payload_codec"
5073 )]
5074 pub payload_codec: String,
5075 #[serde(default)]
5076 pub workflow_arguments: Option<Value>,
5077 #[serde(default)]
5078 pub query_arguments: Option<Value>,
5079 #[serde(default)]
5080 pub history_events: Vec<HistoryEvent>,
5081 #[serde(default)]
5082 pub history_export: Option<Value>,
5083 #[serde(default)]
5084 pub run_status: Option<String>,
5085}
5086
5087#[derive(Clone, Debug, Deserialize)]
5088pub struct WorkflowTask {
5089 pub task_id: String,
5090 #[serde(default)]
5091 pub workflow_command_id: Option<String>,
5092 #[serde(default)]
5093 pub workflow_id: Option<String>,
5094 #[serde(default)]
5095 pub run_id: Option<String>,
5096 pub workflow_type: String,
5097 #[serde(default)]
5098 pub cancel_requested: bool,
5099 #[serde(
5100 default = "missing_task_payload_codec",
5101 deserialize_with = "deserialize_task_payload_codec"
5102 )]
5103 pub payload_codec: String,
5104 #[serde(default)]
5105 pub arguments: Option<Value>,
5106 #[serde(default)]
5107 pub history_events: Vec<HistoryEvent>,
5108 #[serde(default)]
5109 pub total_history_events: Option<u64>,
5110 #[serde(default)]
5111 pub history_size_bytes: Option<u64>,
5112 #[serde(default)]
5113 pub continue_as_new_recommended: Option<bool>,
5114 #[serde(default)]
5115 pub history_budget_pressure: Option<String>,
5116 #[serde(default)]
5117 pub next_history_page_token: Option<String>,
5118 #[serde(default = "default_workflow_task_attempt")]
5119 pub workflow_task_attempt: u64,
5120 #[serde(default)]
5121 pub workflow_signal_id: Option<String>,
5122 #[serde(default)]
5123 pub signal_name: Option<String>,
5124 #[serde(default)]
5125 pub signal_arguments: Option<Value>,
5126 #[serde(default)]
5127 pub workflow_update_id: Option<String>,
5128 #[serde(default)]
5129 pub update_name: Option<String>,
5130 #[serde(default)]
5131 pub lease_owner: Option<String>,
5132}
5133
5134impl WorkflowTask {
5135 fn append_history_page(&mut self, page: WorkflowTaskHistoryPage) {
5136 self.history_events.extend(page.history_events);
5137
5138 if page.total_history_events.is_some() {
5139 self.total_history_events = page.total_history_events;
5140 }
5141
5142 self.next_history_page_token = page
5143 .next_history_page_token
5144 .filter(|token| !token.is_empty());
5145 }
5146}
5147
5148#[derive(Clone, Debug, Deserialize)]
5149struct WorkflowTaskHistoryPage {
5150 #[serde(default)]
5151 history_events: Vec<HistoryEvent>,
5152 #[serde(default)]
5153 total_history_events: Option<u64>,
5154 #[serde(default)]
5155 next_history_page_token: Option<String>,
5156}
5157
5158#[derive(Clone, Debug, Deserialize)]
5159pub struct ActivityTask {
5160 pub task_id: String,
5161 #[serde(default)]
5162 pub activity_attempt_id: Option<String>,
5163 #[serde(default)]
5164 pub attempt_id: Option<String>,
5165 pub activity_type: String,
5166 #[serde(
5167 default = "missing_task_payload_codec",
5168 deserialize_with = "deserialize_task_payload_codec"
5169 )]
5170 pub payload_codec: String,
5171 #[serde(default)]
5172 pub arguments: Option<Value>,
5173 #[serde(default = "default_attempt_number")]
5174 pub attempt_number: u64,
5175 #[serde(default)]
5176 pub lease_owner: Option<String>,
5177}
5178
5179#[derive(Clone, Debug, Deserialize)]
5180pub struct HistoryEvent {
5181 #[serde(alias = "type")]
5182 pub event_type: String,
5183 #[serde(default)]
5184 pub payload: Value,
5185 #[serde(flatten)]
5186 pub raw: HashMap<String, Value>,
5187}
5188
5189#[derive(Clone, Debug, PartialEq)]
5191pub struct QuerySignal {
5192 pub id: Option<String>,
5193 pub name: String,
5194 pub arguments: Vec<Value>,
5195 avro_arguments: Vec<AvroValue>,
5196 pub workflow_sequence: Option<u64>,
5197}
5198
5199impl QuerySignal {
5200 pub fn arguments_avro_value(&self) -> &[AvroValue] {
5202 &self.avro_arguments
5203 }
5204}
5205
5206#[derive(Clone, Debug)]
5212pub struct QueryContext {
5213 pub workflow_id: Option<String>,
5214 pub run_id: Option<String>,
5215 pub workflow_type: String,
5216 pub run_status: Option<String>,
5217 workflow_input: Value,
5218 workflow_input_avro_value: AvroValue,
5219 history_events: Arc<Vec<HistoryEvent>>,
5220 signal_events: Arc<Vec<QuerySignal>>,
5221}
5222
5223impl QueryContext {
5224 pub fn workflow_input(&self) -> &Value {
5226 &self.workflow_input
5227 }
5228
5229 pub fn workflow_input_avro_value(&self) -> &AvroValue {
5231 &self.workflow_input_avro_value
5232 }
5233
5234 pub fn history_events(&self) -> &[HistoryEvent] {
5236 self.history_events.as_slice()
5237 }
5238
5239 pub fn signal_events(&self) -> &[QuerySignal] {
5241 self.signal_events.as_slice()
5242 }
5243
5244 pub fn signals(&self, signal_name: &str) -> Vec<Vec<Value>> {
5246 self.signal_events
5247 .iter()
5248 .filter(|signal| signal.name == signal_name)
5249 .map(|signal| signal.arguments.clone())
5250 .collect()
5251 }
5252
5253 pub fn signals_avro_value(&self, signal_name: &str) -> Vec<Vec<AvroValue>> {
5255 self.signal_events
5256 .iter()
5257 .filter(|signal| signal.name == signal_name)
5258 .map(|signal| signal.avro_arguments.clone())
5259 .collect()
5260 }
5261}
5262
5263#[derive(Clone, Debug, Deserialize)]
5264pub struct ActivityHeartbeatResponse {
5265 #[serde(default)]
5266 pub cancel_requested: bool,
5267 #[serde(default)]
5268 pub heartbeat_recorded: bool,
5269 #[serde(default)]
5270 pub can_continue: Option<bool>,
5271 #[serde(default)]
5272 pub reason: Option<String>,
5273 #[serde(default)]
5274 pub run_closed_reason: Option<String>,
5275 #[serde(default)]
5276 pub run_closed_at: Option<String>,
5277 #[serde(default)]
5278 pub lease_expires_at: Option<String>,
5279 #[serde(default)]
5280 pub last_heartbeat_at: Option<String>,
5281}
5282
5283impl ActivityHeartbeatResponse {
5284 pub fn should_stop(&self) -> bool {
5286 self.cancel_requested || self.can_continue == Some(false)
5287 }
5288}
5289
5290fn missing_task_payload_codec() -> String {
5291 MISSING_TASK_PAYLOAD_CODEC.to_string()
5292}
5293
5294fn deserialize_task_payload_codec<'de, D>(deserializer: D) -> std::result::Result<String, D::Error>
5295where
5296 D: Deserializer<'de>,
5297{
5298 Ok(match Value::deserialize(deserializer)? {
5299 Value::String(codec) => codec,
5300 Value::Null => NULL_TASK_PAYLOAD_CODEC.to_string(),
5301 _ => NON_STRING_TASK_PAYLOAD_CODEC.to_string(),
5302 })
5303}
5304
5305fn default_workflow_task_attempt() -> u64 {
5306 1
5307}
5308
5309fn default_attempt_number() -> u64 {
5310 1
5311}
5312
5313type WorkflowFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
5314type WorkflowHandler = Arc<dyn Fn(WorkflowContext, AvroValue) -> WorkflowFuture + Send + Sync>;
5315type ErasedWorkflowState = Arc<dyn Any + Send + Sync>;
5316type WorkflowStateSnapshot = Arc<dyn Fn() -> Result<ErasedWorkflowState> + Send + Sync>;
5317type ReplayedWorkflowHandler =
5318 Arc<dyn Fn(WorkflowContext, AvroValue) -> ReplayedWorkflowInvocation + Send + Sync>;
5319type ActivityFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
5320type ActivityHandler = Arc<dyn Fn(ActivityContext, AvroValue) -> ActivityFuture + Send + Sync>;
5321type QueryFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
5322type QueryHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
5323type UpdateHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
5324type ReplayedQueryHandler = Arc<
5325 dyn Fn(QueryContext, ErasedWorkflowState, AvroValue) -> std::result::Result<QueryFuture, String>
5326 + Send
5327 + Sync,
5328>;
5329type WorkerHeartbeatObserver = Arc<dyn Fn(&WorkerHeartbeatObservation) + Send + Sync>;
5330
5331struct ReplayedWorkflowInvocation {
5332 future: WorkflowFuture,
5333 snapshot: WorkflowStateSnapshot,
5334}
5335
5336#[derive(Clone)]
5337struct RegisteredWorkflow {
5338 execute: WorkflowHandler,
5339 replay: Option<ReplayedWorkflowHandler>,
5340 state_type: Option<TypeId>,
5341}
5342
5343#[derive(Clone)]
5344enum RegisteredQuery {
5345 Snapshot(QueryHandler),
5346 Replayed {
5347 state_type: TypeId,
5348 handler: ReplayedQueryHandler,
5349 },
5350}
5351
5352#[derive(Clone, Debug)]
5353pub struct WorkerHeartbeatObservation {
5354 pub worker_id: String,
5355 pub task_queue: String,
5356 pub acknowledged_at_unix_millis: u64,
5357 pub acknowledgement: Value,
5358}
5359
5360#[derive(Clone, Copy, Debug)]
5367pub struct WorkerRetryPolicy {
5368 pub max_retries: usize,
5370 pub initial_backoff: Duration,
5372 pub max_backoff: Duration,
5374}
5375
5376impl Default for WorkerRetryPolicy {
5377 fn default() -> Self {
5378 Self {
5379 max_retries: 5,
5380 initial_backoff: Duration::from_millis(100),
5381 max_backoff: Duration::from_secs(5),
5382 }
5383 }
5384}
5385
5386#[derive(Clone, Copy, Debug, PartialEq, Eq)]
5387enum ManagedPollOutcome {
5388 Idle,
5389 Handled,
5390 Stop,
5391}
5392
5393#[derive(Clone)]
5394pub struct Worker {
5395 client: Client,
5396 worker_id: String,
5397 task_queue: String,
5398 workflows: HashMap<String, RegisteredWorkflow>,
5399 activities: HashMap<String, ActivityHandler>,
5400 queries: HashMap<String, HashMap<String, RegisteredQuery>>,
5401 updates: HashMap<String, HashMap<String, UpdateHandler>>,
5402 max_concurrent_workflow_tasks: usize,
5403 max_concurrent_activity_tasks: usize,
5404 poll_timeout: Duration,
5405 heartbeat_interval: Duration,
5406 retry_policy: WorkerRetryPolicy,
5407 heartbeat_observer: Option<WorkerHeartbeatObserver>,
5408}
5409
5410impl Worker {
5411 pub fn new(client: Client, task_queue: impl Into<String>) -> Self {
5412 Self {
5413 client,
5414 worker_id: default_worker_id(),
5415 task_queue: task_queue.into(),
5416 workflows: HashMap::new(),
5417 activities: HashMap::new(),
5418 queries: HashMap::new(),
5419 updates: HashMap::new(),
5420 max_concurrent_workflow_tasks: 10,
5421 max_concurrent_activity_tasks: 10,
5422 poll_timeout: Duration::from_secs(30),
5423 heartbeat_interval: Duration::from_secs(60),
5424 retry_policy: WorkerRetryPolicy::default(),
5425 heartbeat_observer: None,
5426 }
5427 }
5428
5429 pub fn worker_id(mut self, worker_id: impl Into<String>) -> Self {
5430 self.worker_id = worker_id.into();
5431 self
5432 }
5433
5434 pub fn poll_timeout(mut self, timeout: Duration) -> Self {
5435 self.poll_timeout = timeout;
5436 self
5437 }
5438
5439 pub fn heartbeat_interval(mut self, interval: Duration) -> Self {
5440 self.heartbeat_interval = interval;
5441 self
5442 }
5443
5444 pub fn retry_policy(mut self, policy: WorkerRetryPolicy) -> Self {
5446 self.retry_policy = policy;
5447 self
5448 }
5449
5450 pub fn on_worker_heartbeat<F>(mut self, observer: F) -> Self
5451 where
5452 F: Fn(&WorkerHeartbeatObservation) + Send + Sync + 'static,
5453 {
5454 self.heartbeat_observer = Some(Arc::new(observer));
5455 self
5456 }
5457
5458 pub fn max_concurrent_workflow_tasks(mut self, count: usize) -> Self {
5459 self.max_concurrent_workflow_tasks = count.max(1);
5460 self
5461 }
5462
5463 pub fn max_concurrent_activity_tasks(mut self, count: usize) -> Self {
5464 self.max_concurrent_activity_tasks = count.max(1);
5465 self
5466 }
5467
5468 pub fn register_workflow<F, Fut>(&mut self, workflow_type: impl Into<String>, handler: F)
5475 where
5476 F: Fn(WorkflowContext, Value) -> Fut + Send + Sync + 'static,
5477 Fut: Future<Output = Result<Value>> + Send + 'static,
5478 {
5479 let handler = Arc::new(handler);
5480 self.workflows.insert(
5481 workflow_type.into(),
5482 RegisteredWorkflow {
5483 execute: Arc::new(move |ctx, input| {
5484 let handler = Arc::clone(&handler);
5485 Box::pin(async move {
5486 let result = handler(ctx, input.into_json()?).await?;
5487 AvroValue::from_serialize(&result)
5488 })
5489 }),
5490 replay: None,
5491 state_type: None,
5492 },
5493 );
5494 }
5495
5496 pub fn register_typed_workflow<I, O, F, Fut>(
5507 &mut self,
5508 workflow_type: impl Into<String>,
5509 handler: F,
5510 ) where
5511 I: DeserializeOwned + Send + 'static,
5512 O: Serialize + Send + 'static,
5513 F: Fn(WorkflowContext, I) -> Fut + Send + Sync + 'static,
5514 Fut: Future<Output = Result<O>> + Send + 'static,
5515 {
5516 let workflow_type = workflow_type.into();
5517 let handler_name = workflow_type.clone();
5518 let handler = Arc::new(handler);
5519 self.workflows.insert(
5520 workflow_type,
5521 RegisteredWorkflow {
5522 execute: Arc::new(move |ctx, input| {
5523 let handler = Arc::clone(&handler);
5524 let handler_name = handler_name.clone();
5525 Box::pin(async move {
5526 let input =
5527 decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
5528 let result = handler(ctx, input).await?;
5529 encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
5530 })
5531 }),
5532 replay: None,
5533 state_type: None,
5534 },
5535 );
5536 }
5537
5538 pub fn register_workflow_avro_value<F, Fut>(
5540 &mut self,
5541 workflow_type: impl Into<String>,
5542 handler: F,
5543 ) where
5544 F: Fn(WorkflowContext, AvroValue) -> Fut + Send + Sync + 'static,
5545 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
5546 {
5547 self.workflows.insert(
5548 workflow_type.into(),
5549 RegisteredWorkflow {
5550 execute: Arc::new(move |ctx, input| Box::pin(handler(ctx, input))),
5551 replay: None,
5552 state_type: None,
5553 },
5554 );
5555 }
5556
5557 pub fn register_replayed_workflow<S, Factory, F, Fut>(
5565 &mut self,
5566 workflow_type: impl Into<String>,
5567 state_factory: Factory,
5568 handler: F,
5569 ) where
5570 S: Clone + Send + Sync + 'static,
5571 Factory: Fn() -> S + Send + Sync + 'static,
5572 F: Fn(WorkflowContext, Value, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
5573 Fut: Future<Output = Result<Value>> + Send + 'static,
5574 {
5575 let state_factory = Arc::new(state_factory);
5576 let handler = Arc::new(handler);
5577
5578 let execute_factory = Arc::clone(&state_factory);
5579 let execute_handler = Arc::clone(&handler);
5580 let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
5581 let state = WorkflowInstance::new(execute_factory());
5582 let handler = Arc::clone(&execute_handler);
5583 Box::pin(async move {
5584 let result = handler(ctx, input.into_json()?, state).await?;
5585 AvroValue::from_serialize(&result)
5586 }) as WorkflowFuture
5587 });
5588
5589 let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
5590 let state = WorkflowInstance::new(state_factory());
5591 let snapshot_state = state.clone();
5592 let snapshot: WorkflowStateSnapshot =
5593 Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
5594 let replay_handler = Arc::clone(&handler);
5595 let future = async move {
5596 let result = replay_handler(ctx, input.into_json()?, state).await?;
5597 AvroValue::from_serialize(&result)
5598 };
5599 ReplayedWorkflowInvocation {
5600 future: Box::pin(future),
5601 snapshot,
5602 }
5603 });
5604
5605 self.workflows.insert(
5606 workflow_type.into(),
5607 RegisteredWorkflow {
5608 execute,
5609 replay: Some(replay),
5610 state_type: Some(TypeId::of::<S>()),
5611 },
5612 );
5613 }
5614
5615 pub fn register_typed_replayed_workflow<I, O, S, Factory, F, Fut>(
5621 &mut self,
5622 workflow_type: impl Into<String>,
5623 state_factory: Factory,
5624 handler: F,
5625 ) where
5626 I: DeserializeOwned + Send + 'static,
5627 O: Serialize + Send + 'static,
5628 S: Clone + Send + Sync + 'static,
5629 Factory: Fn() -> S + Send + Sync + 'static,
5630 F: Fn(WorkflowContext, I, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
5631 Fut: Future<Output = Result<O>> + Send + 'static,
5632 {
5633 let workflow_type = workflow_type.into();
5634 let state_factory = Arc::new(state_factory);
5635 let handler = Arc::new(handler);
5636
5637 let execute_name = workflow_type.clone();
5638 let execute_factory = Arc::clone(&state_factory);
5639 let execute_handler = Arc::clone(&handler);
5640 let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
5641 let state = WorkflowInstance::new(execute_factory());
5642 let handler = Arc::clone(&execute_handler);
5643 let handler_name = execute_name.clone();
5644 Box::pin(async move {
5645 let input = decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
5646 let result = handler(ctx, input, state).await?;
5647 encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
5648 }) as WorkflowFuture
5649 });
5650
5651 let replay_name = workflow_type.clone();
5652 let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
5653 let state = WorkflowInstance::new(state_factory());
5654 let snapshot_state = state.clone();
5655 let snapshot: WorkflowStateSnapshot =
5656 Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
5657 let handler = Arc::clone(&handler);
5658 let handler_name = replay_name.clone();
5659 let future = async move {
5660 let input = decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
5661 let result = handler(ctx, input, state).await?;
5662 encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
5663 };
5664 ReplayedWorkflowInvocation {
5665 future: Box::pin(future),
5666 snapshot,
5667 }
5668 });
5669
5670 self.workflows.insert(
5671 workflow_type,
5672 RegisteredWorkflow {
5673 execute,
5674 replay: Some(replay),
5675 state_type: Some(TypeId::of::<S>()),
5676 },
5677 );
5678 }
5679
5680 pub fn register_replayed_workflow_avro_value<S, Factory, F, Fut>(
5682 &mut self,
5683 workflow_type: impl Into<String>,
5684 state_factory: Factory,
5685 handler: F,
5686 ) where
5687 S: Clone + Send + Sync + 'static,
5688 Factory: Fn() -> S + Send + Sync + 'static,
5689 F: Fn(WorkflowContext, AvroValue, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
5690 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
5691 {
5692 let state_factory = Arc::new(state_factory);
5693 let handler = Arc::new(handler);
5694
5695 let execute_factory = Arc::clone(&state_factory);
5696 let execute_handler = Arc::clone(&handler);
5697 let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
5698 let state = WorkflowInstance::new(execute_factory());
5699 Box::pin(execute_handler(ctx, input, state)) as WorkflowFuture
5700 });
5701
5702 let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
5703 let state = WorkflowInstance::new(state_factory());
5704 let snapshot_state = state.clone();
5705 let snapshot: WorkflowStateSnapshot =
5706 Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
5707 ReplayedWorkflowInvocation {
5708 future: Box::pin(handler(ctx, input, state)),
5709 snapshot,
5710 }
5711 });
5712
5713 self.workflows.insert(
5714 workflow_type.into(),
5715 RegisteredWorkflow {
5716 execute,
5717 replay: Some(replay),
5718 state_type: Some(TypeId::of::<S>()),
5719 },
5720 );
5721 }
5722
5723 pub fn register_activity<F, Fut>(&mut self, activity_type: impl Into<String>, handler: F)
5724 where
5725 F: Fn(ActivityContext, Value) -> Fut + Send + Sync + 'static,
5726 Fut: Future<Output = Result<Value>> + Send + 'static,
5727 {
5728 let handler = Arc::new(handler);
5729 self.activities.insert(
5730 activity_type.into(),
5731 Arc::new(move |ctx, args| {
5732 let handler = Arc::clone(&handler);
5733 Box::pin(async move {
5734 let result = handler(ctx, args.into_json()?).await?;
5735 AvroValue::from_serialize(&result)
5736 })
5737 }),
5738 );
5739 }
5740
5741 pub fn register_typed_activity<I, O, F, Fut>(
5747 &mut self,
5748 activity_type: impl Into<String>,
5749 handler: F,
5750 ) where
5751 I: DeserializeOwned + Send + 'static,
5752 O: Serialize + Send + 'static,
5753 F: Fn(ActivityContext, I) -> Fut + Send + Sync + 'static,
5754 Fut: Future<Output = Result<O>> + Send + 'static,
5755 {
5756 let activity_type = activity_type.into();
5757 let handler_name = activity_type.clone();
5758 let handler = Arc::new(handler);
5759 self.activities.insert(
5760 activity_type,
5761 Arc::new(move |ctx, input| {
5762 let handler = Arc::clone(&handler);
5763 let handler_name = handler_name.clone();
5764 Box::pin(async move {
5765 let input =
5766 decode_handler_input::<I>(input, HandlerKind::Activity, &handler_name)?;
5767 let result = handler(ctx, input).await?;
5768 encode_handler_result(&result, HandlerKind::Activity, &handler_name)
5769 })
5770 }),
5771 );
5772 }
5773
5774 pub fn register_activity_avro_value<F, Fut>(
5776 &mut self,
5777 activity_type: impl Into<String>,
5778 handler: F,
5779 ) where
5780 F: Fn(ActivityContext, AvroValue) -> Fut + Send + Sync + 'static,
5781 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
5782 {
5783 self.activities.insert(
5784 activity_type.into(),
5785 Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
5786 );
5787 }
5788
5789 pub fn register_query<F, Fut>(
5795 &mut self,
5796 workflow_type: impl Into<String>,
5797 query_name: impl Into<String>,
5798 handler: F,
5799 ) where
5800 F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
5801 Fut: Future<Output = Result<Value>> + Send + 'static,
5802 {
5803 let handler = Arc::new(handler);
5804 self.queries
5805 .entry(workflow_type.into())
5806 .or_default()
5807 .insert(
5808 query_name.into(),
5809 RegisteredQuery::Snapshot(Arc::new(move |ctx, args| {
5810 let handler = Arc::clone(&handler);
5811 Box::pin(async move {
5812 let result = handler(ctx, args.into_json()?).await?;
5813 AvroValue::from_serialize(&result)
5814 })
5815 })),
5816 );
5817 }
5818
5819 pub fn register_query_avro_value<F, Fut>(
5821 &mut self,
5822 workflow_type: impl Into<String>,
5823 query_name: impl Into<String>,
5824 handler: F,
5825 ) where
5826 F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
5827 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
5828 {
5829 self.queries
5830 .entry(workflow_type.into())
5831 .or_default()
5832 .insert(
5833 query_name.into(),
5834 RegisteredQuery::Snapshot(Arc::new(move |ctx, args| Box::pin(handler(ctx, args)))),
5835 );
5836 }
5837
5838 pub fn register_replayed_query<S, F, Fut>(
5845 &mut self,
5846 workflow_type: impl Into<String>,
5847 query_name: impl Into<String>,
5848 handler: F,
5849 ) where
5850 S: Clone + Send + Sync + 'static,
5851 F: Fn(QueryContext, Arc<S>, Value) -> Fut + Send + Sync + 'static,
5852 Fut: Future<Output = Result<Value>> + Send + 'static,
5853 {
5854 let handler = Arc::new(handler);
5855 let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
5856 let state = state.downcast::<S>().map_err(|_| {
5857 "registered query state type does not match the replayed workflow state".to_string()
5858 })?;
5859 let handler = Arc::clone(&handler);
5860 Ok(Box::pin(async move {
5861 let result = handler(ctx, state, args.into_json()?).await?;
5862 AvroValue::from_serialize(&result)
5863 }))
5864 });
5865
5866 self.queries
5867 .entry(workflow_type.into())
5868 .or_default()
5869 .insert(
5870 query_name.into(),
5871 RegisteredQuery::Replayed {
5872 state_type: TypeId::of::<S>(),
5873 handler: erased_handler,
5874 },
5875 );
5876 }
5877
5878 pub fn register_replayed_query_avro_value<S, F, Fut>(
5880 &mut self,
5881 workflow_type: impl Into<String>,
5882 query_name: impl Into<String>,
5883 handler: F,
5884 ) where
5885 S: Clone + Send + Sync + 'static,
5886 F: Fn(QueryContext, Arc<S>, AvroValue) -> Fut + Send + Sync + 'static,
5887 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
5888 {
5889 let handler = Arc::new(handler);
5890 let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
5891 let state = state.downcast::<S>().map_err(|_| {
5892 "registered query state type does not match the replayed workflow state".to_string()
5893 })?;
5894 Ok(Box::pin(handler(ctx, state, args)))
5895 });
5896
5897 self.queries
5898 .entry(workflow_type.into())
5899 .or_default()
5900 .insert(
5901 query_name.into(),
5902 RegisteredQuery::Replayed {
5903 state_type: TypeId::of::<S>(),
5904 handler: erased_handler,
5905 },
5906 );
5907 }
5908
5909 pub fn register_update<F, Fut>(
5911 &mut self,
5912 workflow_type: impl Into<String>,
5913 update_name: impl Into<String>,
5914 handler: F,
5915 ) where
5916 F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
5917 Fut: Future<Output = Result<Value>> + Send + 'static,
5918 {
5919 let handler = Arc::new(handler);
5920 self.updates
5921 .entry(workflow_type.into())
5922 .or_default()
5923 .insert(
5924 update_name.into(),
5925 Arc::new(move |ctx, args| {
5926 let handler = Arc::clone(&handler);
5927 Box::pin(async move {
5928 let result = handler(ctx, args.into_json()?).await?;
5929 AvroValue::from_serialize(&result)
5930 })
5931 }),
5932 );
5933 }
5934
5935 pub fn register_update_avro_value<F, Fut>(
5937 &mut self,
5938 workflow_type: impl Into<String>,
5939 update_name: impl Into<String>,
5940 handler: F,
5941 ) where
5942 F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
5943 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
5944 {
5945 self.updates
5946 .entry(workflow_type.into())
5947 .or_default()
5948 .insert(
5949 update_name.into(),
5950 Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
5951 );
5952 }
5953
5954 pub async fn register(&self) -> Result<RegisterWorkerResponse> {
5955 let mut command_contracts = serde_json::Map::new();
5956 for workflow_type in self.workflows.keys() {
5957 let mut queries = self
5958 .queries
5959 .get(workflow_type)
5960 .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
5961 .unwrap_or_default();
5962 queries.sort();
5963 let mut updates = self
5964 .updates
5965 .get(workflow_type)
5966 .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
5967 .unwrap_or_default();
5968 updates.sort();
5969 if !queries.is_empty() || !updates.is_empty() {
5970 command_contracts.insert(
5971 workflow_type.clone(),
5972 json!({
5973 "queries": queries,
5974 "updates": updates,
5975 "update_validators": [],
5976 }),
5977 );
5978 }
5979 }
5980
5981 self.client
5982 .register_worker_with_command_contracts(
5983 &self.worker_id,
5984 &self.task_queue,
5985 self.workflows.keys().cloned().collect(),
5986 self.activities.keys().cloned().collect(),
5987 self.max_concurrent_workflow_tasks,
5988 self.max_concurrent_activity_tasks,
5989 [
5990 (!self.queries.is_empty()).then(|| QUERY_TASKS_CAPABILITY.to_string()),
5991 (!self.updates.is_empty()).then(|| WORKFLOW_UPDATES_CAPABILITY.to_string()),
5992 ]
5993 .into_iter()
5994 .flatten()
5995 .collect(),
5996 Value::Object(command_contracts),
5997 )
5998 .await
5999 }
6000
6001 pub async fn run(&self) -> Result<()> {
6007 self.run_until(std::future::pending::<()>()).await
6008 }
6009
6010 pub async fn run_until<F>(&self, shutdown: F) -> Result<()>
6014 where
6015 F: Future<Output = ()>,
6016 {
6017 let registration = self.register().await?;
6018 if !registration.registered {
6019 return Err(Error::WorkerLoop(format!(
6020 "worker registration for {:?} was not accepted",
6021 self.worker_id
6022 )));
6023 }
6024 let registered_worker_id = registration.worker_id.clone();
6025 let primary = self.run_registered_until(shutdown, registration).await;
6026 let deregistration = self
6027 .client
6028 .deregister_worker_registration(®istered_worker_id)
6029 .await;
6030
6031 match (primary, deregistration) {
6032 (Ok(()), Ok(_)) => Ok(()),
6033 (Ok(()), Err(deregistration)) => Err(deregistration),
6034 (Err(primary), Ok(_)) => Err(primary),
6035 (Err(primary), Err(deregistration)) => Err(Error::WorkerShutdown {
6036 primary: Box::new(primary),
6037 deregistration: Box::new(deregistration),
6038 }),
6039 }
6040 }
6041
6042 async fn run_registered_until<F>(
6043 &self,
6044 shutdown: F,
6045 registration: RegisterWorkerResponse,
6046 ) -> Result<()>
6047 where
6048 F: Future<Output = ()>,
6049 {
6050 let heartbeat_interval = Duration::from_secs(
6051 registration
6052 .heartbeat_interval_seconds
6053 .unwrap_or(self.heartbeat_interval.as_secs().max(1)),
6054 );
6055 let heartbeat = tokio::time::sleep(Duration::ZERO);
6061 tokio::pin!(heartbeat);
6062 tokio::pin!(shutdown);
6063 let stop = Arc::new(AtomicBool::new(false));
6064 let mut workflow_poller = (!self.workflows.is_empty()).then(|| {
6068 let worker = self.clone();
6069 let stop = Arc::clone(&stop);
6070 tokio::spawn(async move { worker.poll_workflows_until_stopped(stop).await })
6071 });
6072 let mut activity_poller = (!self.activities.is_empty()).then(|| {
6073 let worker = self.clone();
6074 let stop = Arc::clone(&stop);
6075 tokio::spawn(async move { worker.poll_activities_until_stopped(stop).await })
6076 });
6077 let mut query_poller = (!self.queries.is_empty()).then(|| {
6078 let worker = self.clone();
6079 let stop = Arc::clone(&stop);
6080 tokio::spawn(async move { worker.poll_queries_until_stopped(stop).await })
6081 });
6082
6083 loop {
6084 tokio::select! {
6085 _ = &mut shutdown => {
6086 stop.store(true, Ordering::SeqCst);
6087 break;
6088 }
6089 _ = &mut heartbeat => {
6090 let result = self.retry_worker_operation(|| {
6091 self.client.heartbeat_worker(
6092 &self.worker_id,
6093 self.max_concurrent_workflow_tasks,
6094 self.max_concurrent_activity_tasks,
6095 )
6096 }).await;
6097 heartbeat
6098 .as_mut()
6099 .reset(tokio::time::Instant::now() + heartbeat_interval);
6100 match result {
6101 Ok(acknowledgement) => {
6102 if let Some(observer) = &self.heartbeat_observer {
6103 observer(&WorkerHeartbeatObservation {
6104 worker_id: self.worker_id.clone(),
6105 task_queue: self.task_queue.clone(),
6106 acknowledged_at_unix_millis: SystemTime::now()
6107 .duration_since(UNIX_EPOCH)
6108 .unwrap_or_default()
6109 .as_millis()
6110 .min(u64::MAX as u128)
6111 as u64,
6112 acknowledgement,
6113 });
6114 }
6115 }
6116 Err(error) => {
6117 stop.store(true, Ordering::SeqCst);
6118 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await?;
6119 return Err(error);
6120 }
6121 }
6122 }
6123 result = OptionFuture::from(workflow_poller.as_mut()), if workflow_poller.is_some() => {
6124 workflow_poller = None;
6125 let stopped_by_server = stop.load(Ordering::SeqCst);
6126 stop.store(true, Ordering::SeqCst);
6127 let poller_result = optional_poller_result("workflow", result);
6128 let join_result =
6129 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6130 poller_result?;
6131 join_result?;
6132 if stopped_by_server {
6133 return Ok(());
6134 }
6135 return Err(Error::WorkerLoop(
6136 "workflow poller stopped unexpectedly".to_string(),
6137 ));
6138 }
6139 result = OptionFuture::from(activity_poller.as_mut()), if activity_poller.is_some() => {
6140 activity_poller = None;
6141 let stopped_by_server = stop.load(Ordering::SeqCst);
6142 stop.store(true, Ordering::SeqCst);
6143 let poller_result = optional_poller_result("activity", result);
6144 let join_result =
6145 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6146 poller_result?;
6147 join_result?;
6148 if stopped_by_server {
6149 return Ok(());
6150 }
6151 return Err(Error::WorkerLoop(
6152 "activity poller stopped unexpectedly".to_string(),
6153 ));
6154 }
6155 result = OptionFuture::from(query_poller.as_mut()), if query_poller.is_some() => {
6156 query_poller = None;
6157 let stopped_by_server = stop.load(Ordering::SeqCst);
6158 stop.store(true, Ordering::SeqCst);
6159 let poller_result = optional_poller_result("query", result);
6160 let join_result =
6161 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6162 poller_result?;
6163 join_result?;
6164 if stopped_by_server {
6165 return Ok(());
6166 }
6167 return Err(Error::WorkerLoop(
6168 "query poller stopped unexpectedly".to_string(),
6169 ));
6170 }
6171 }
6172 }
6173
6174 join_pollers(
6175 workflow_poller.take(),
6176 activity_poller.take(),
6177 query_poller.take(),
6178 )
6179 .await
6180 }
6181
6182 pub async fn run_once(&self) -> Result<usize> {
6196 let mut handled = 0;
6197 match self.poll_workflow_once().await? {
6198 ManagedPollOutcome::Handled => handled += 1,
6199 ManagedPollOutcome::Stop => return Ok(handled),
6200 ManagedPollOutcome::Idle => {}
6201 }
6202 match self.poll_activity_once().await? {
6203 ManagedPollOutcome::Handled => handled += 1,
6204 ManagedPollOutcome::Stop => return Ok(handled),
6205 ManagedPollOutcome::Idle => {}
6206 }
6207 if !self.queries.is_empty() {
6208 match self.poll_query_once().await? {
6209 ManagedPollOutcome::Handled => handled += 1,
6210 ManagedPollOutcome::Stop => return Ok(handled),
6211 ManagedPollOutcome::Idle => {}
6212 }
6213 }
6214 Ok(handled)
6215 }
6216
6217 async fn poll_workflow_once(&self) -> Result<ManagedPollOutcome> {
6218 let poll_request_id = unique_request_id("rust-workflow-poll");
6219 let response = self
6220 .retry_worker_operation(|| {
6221 self.client.poll_workflow_task_response_with_request_id(
6222 &self.worker_id,
6223 &self.task_queue,
6224 self.poll_timeout,
6225 &poll_request_id,
6226 0,
6227 )
6228 })
6229 .await?;
6230 if response.outcome().should_stop() {
6231 return Ok(ManagedPollOutcome::Stop);
6232 }
6233 let memo_updates_supported =
6234 runtime_supports_workflow_memo_updates(response.server_capabilities.as_ref());
6235 let Some(task) = response.task else {
6236 return Ok(ManagedPollOutcome::Idle);
6237 };
6238
6239 let task_id = task.task_id.clone();
6240 let attempt = task.workflow_task_attempt;
6241 let run_id = task.run_id.clone();
6242 let lease_owner = task
6243 .lease_owner
6244 .clone()
6245 .unwrap_or_else(|| self.worker_id.clone());
6246
6247 match self.execute_workflow_task(task) {
6248 Ok(commands)
6249 if commands_use_workflow_memo_updates(&commands) && !memo_updates_supported =>
6250 {
6251 self.client
6252 .fail_workflow_task(
6253 &task_id,
6254 &lease_owner,
6255 attempt,
6256 Error::WorkflowMemoUpdatesUnavailable.to_string(),
6257 )
6258 .await?;
6259 }
6260 Ok(commands) if commands.is_empty() => {
6261 self.client
6267 .fail_workflow_task_with_type(
6268 &task_id,
6269 &lease_owner,
6270 attempt,
6271 WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE,
6272 WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE,
6273 )
6274 .await?;
6275 }
6276 Ok(commands) => {
6277 let completion = self
6278 .client
6279 .complete_workflow_task(&task_id, &lease_owner, attempt, commands)
6280 .await;
6281 if let Err(error) = completion {
6282 if !workflow_task_completion_is_terminal_timeout(
6283 &error,
6284 &task_id,
6285 attempt,
6286 run_id.as_deref(),
6287 ) {
6288 return Err(error);
6289 }
6290 }
6291 }
6292 Err(error) => {
6293 self.client
6294 .fail_workflow_task(&task_id, &lease_owner, attempt, error.to_string())
6295 .await?;
6296 }
6297 }
6298
6299 Ok(ManagedPollOutcome::Handled)
6300 }
6301
6302 async fn poll_workflows_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
6303 while !stop.load(Ordering::SeqCst) {
6304 if self.poll_workflow_once().await? == ManagedPollOutcome::Stop {
6305 stop.store(true, Ordering::SeqCst);
6306 break;
6307 }
6308 }
6309
6310 Ok(())
6311 }
6312
6313 async fn poll_activity_once(&self) -> Result<ManagedPollOutcome> {
6314 let poll_request_id = unique_request_id("rust-activity-poll");
6315 let response = self
6316 .retry_worker_operation(|| {
6317 self.client.poll_activity_task_response_with_request_id(
6318 &self.worker_id,
6319 &self.task_queue,
6320 self.poll_timeout,
6321 &poll_request_id,
6322 0,
6323 )
6324 })
6325 .await?;
6326 if response.outcome().should_stop() {
6327 return Ok(ManagedPollOutcome::Stop);
6328 }
6329 let Some(task) = response.task else {
6330 return Ok(ManagedPollOutcome::Idle);
6331 };
6332
6333 let task_id = task.task_id.clone();
6334 let attempt_id = task
6335 .activity_attempt_id
6336 .clone()
6337 .or(task.attempt_id.clone())
6338 .unwrap_or_default();
6339 let lease_owner = task
6340 .lease_owner
6341 .clone()
6342 .unwrap_or_else(|| self.worker_id.clone());
6343 let codec = task.payload_codec.clone();
6344 let result = self.execute_activity_task(task).await;
6345 match result {
6346 Ok(value) => {
6347 let completion = self
6348 .client
6349 .complete_activity_task(&task_id, &attempt_id, &lease_owner, value, &codec)
6350 .await;
6351 if let Err(error) = completion {
6352 if !activity_task_rejection_is_final(&error) {
6353 return Err(error);
6354 }
6355 }
6356 }
6357 Err(error) => {
6358 let failure = self
6359 .client
6360 .fail_activity_task(
6361 &task_id,
6362 &attempt_id,
6363 &lease_owner,
6364 error.to_string(),
6365 false,
6366 )
6367 .await;
6368 if let Err(error) = failure {
6369 if !activity_task_rejection_is_final(&error) {
6370 return Err(error);
6371 }
6372 }
6373 }
6374 }
6375
6376 Ok(ManagedPollOutcome::Handled)
6377 }
6378
6379 async fn poll_activities_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
6380 while !stop.load(Ordering::SeqCst) {
6381 if self.poll_activity_once().await? == ManagedPollOutcome::Stop {
6382 stop.store(true, Ordering::SeqCst);
6383 break;
6384 }
6385 }
6386
6387 Ok(())
6388 }
6389
6390 async fn poll_query_once(&self) -> Result<ManagedPollOutcome> {
6391 let poll_request_id = unique_request_id("rust-query-poll");
6392 let response = self
6393 .retry_worker_operation(|| {
6394 self.client.poll_query_task_response_with_request_id(
6395 &self.worker_id,
6396 &self.task_queue,
6397 self.poll_timeout,
6398 &poll_request_id,
6399 0,
6400 )
6401 })
6402 .await?;
6403 if response.outcome().should_stop() {
6404 return Ok(ManagedPollOutcome::Stop);
6405 }
6406 let Some(task) = response.task else {
6407 return Ok(ManagedPollOutcome::Idle);
6408 };
6409
6410 let query_task_id = task.query_task_id.clone();
6411 let attempt = task.query_task_attempt;
6412 let lease_owner = task
6413 .lease_owner
6414 .clone()
6415 .unwrap_or_else(|| self.worker_id.clone());
6416 let codec = task.payload_codec.clone();
6417
6418 match self.execute_query_task(task).await {
6419 Ok(value) => {
6420 let result_envelope = match encode_typed_envelope(&value, &codec) {
6421 Ok(result_envelope) => result_envelope,
6422 Err(error) => {
6423 let failure = self
6424 .client
6425 .fail_query_task(
6426 &query_task_id,
6427 &lease_owner,
6428 attempt,
6429 error.to_string(),
6430 "query_result_encode_failed",
6431 "QueryResultEncodeFailed",
6432 )
6433 .await;
6434 if let Err(error) = failure {
6435 if !query_task_rejection_is_final(&error) {
6436 return Err(error);
6437 }
6438 }
6439 return Ok(ManagedPollOutcome::Handled);
6440 }
6441 };
6442
6443 if let Err(error) = self
6444 .client
6445 .complete_query_task_with_envelope(
6446 &query_task_id,
6447 &lease_owner,
6448 attempt,
6449 value.clone().into_json()?,
6450 result_envelope,
6451 )
6452 .await
6453 {
6454 if !query_task_rejection_is_final(&error) {
6455 return Err(error);
6456 }
6457 }
6458 }
6459 Err(failure) => {
6460 let result = self
6461 .client
6462 .fail_query_task(
6463 &query_task_id,
6464 &lease_owner,
6465 attempt,
6466 failure.message,
6467 failure.reason,
6468 failure.failure_type,
6469 )
6470 .await;
6471 if let Err(error) = result {
6472 if !query_task_rejection_is_final(&error) {
6473 return Err(error);
6474 }
6475 }
6476 }
6477 }
6478
6479 Ok(ManagedPollOutcome::Handled)
6480 }
6481
6482 async fn poll_queries_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
6483 while !stop.load(Ordering::SeqCst) {
6484 if self.poll_query_once().await? == ManagedPollOutcome::Stop {
6485 stop.store(true, Ordering::SeqCst);
6486 break;
6487 }
6488 }
6489
6490 Ok(())
6491 }
6492
6493 async fn retry_worker_operation<T, F, Fut>(&self, mut operation: F) -> Result<T>
6494 where
6495 F: FnMut() -> Fut,
6496 Fut: Future<Output = Result<T>>,
6497 {
6498 let mut retries = 0;
6499
6500 loop {
6501 match operation().await {
6502 Err(error)
6503 if worker_operation_is_retryable(&error)
6504 && retries < self.retry_policy.max_retries =>
6505 {
6506 retries += 1;
6507 tokio::time::sleep(worker_retry_delay(self.retry_policy, retries)).await;
6508 }
6509 result => return result,
6510 }
6511 }
6512 }
6513
6514 async fn execute_query_task(
6515 &self,
6516 mut task: QueryTask,
6517 ) -> std::result::Result<AvroValue, QueryTaskExecutionFailure> {
6518 validate_query_task_payloads(&task).map_err(|error| {
6519 QueryTaskExecutionFailure::new(
6520 "query_payload_decode_failed",
6521 error.to_string(),
6522 "QueryPayloadDecodeFailed",
6523 )
6524 })?;
6525
6526 if !self.workflows.contains_key(&task.workflow_type) {
6527 return Err(QueryTaskExecutionFailure::new(
6528 "query_workflow_type_not_registered",
6529 format!("no workflow registered for type {:?}", task.workflow_type),
6530 "WorkflowTypeNotRegistered",
6531 ));
6532 }
6533
6534 let Some(handlers) = self.queries.get(&task.workflow_type) else {
6535 return Err(QueryTaskExecutionFailure::new(
6536 "query_handler_unavailable",
6537 format!(
6538 "query handlers are unavailable for workflow type {:?}",
6539 task.workflow_type
6540 ),
6541 "QueryHandlerUnavailable",
6542 ));
6543 };
6544 let Some(query) = handlers.get(&task.query_name) else {
6545 return Err(QueryTaskExecutionFailure::new(
6546 "rejected_unknown_query",
6547 format!("unknown query {:?}", task.query_name),
6548 "QueryFailed",
6549 ));
6550 };
6551
6552 let args = decode_task_avro_arguments(task.query_arguments.as_ref(), &task.payload_codec)
6553 .map_err(|error| {
6554 QueryTaskExecutionFailure::new(
6555 "query_payload_decode_failed",
6556 format!("cannot decode query arguments: {error}"),
6557 "QueryPayloadDecodeFailed",
6558 )
6559 })?;
6560 let workflow_input_typed =
6561 decode_task_avro_arguments(task.workflow_arguments.as_ref(), &task.payload_codec)
6562 .map_err(|error| {
6563 QueryTaskExecutionFailure::new(
6564 "query_workflow_state_unavailable",
6565 format!("cannot decode workflow start input: {error}"),
6566 "QueryWorkflowStateUnavailable",
6567 )
6568 })?;
6569 let workflow_input = workflow_input_typed.clone().into_json().map_err(|error| {
6570 QueryTaskExecutionFailure::new(
6571 "query_workflow_state_unavailable",
6572 format!("cannot project workflow start input: {error}"),
6573 "QueryWorkflowStateUnavailable",
6574 )
6575 })?;
6576 hydrate_query_history_from_export(&mut task).map_err(|error| {
6577 QueryTaskExecutionFailure::new(
6578 "query_workflow_state_unavailable",
6579 format!("cannot restore query history snapshot: {error}"),
6580 "QueryWorkflowStateUnavailable",
6581 )
6582 })?;
6583 enrich_query_history_from_export(&mut task).map_err(|error| {
6584 QueryTaskExecutionFailure::new(
6585 "query_workflow_state_unavailable",
6586 format!("cannot restore compact query history payloads: {error}"),
6587 "QueryWorkflowStateUnavailable",
6588 )
6589 })?;
6590 let signal_events = query_signal_events(&task).map_err(|error| {
6591 QueryTaskExecutionFailure::new(
6592 "query_workflow_state_unavailable",
6593 format!("cannot decode committed workflow signals: {error}"),
6594 "QueryWorkflowStateUnavailable",
6595 )
6596 })?;
6597 let history_events = Arc::new(std::mem::take(&mut task.history_events));
6598 let context = QueryContext {
6599 workflow_id: task.workflow_id,
6600 run_id: task.run_id,
6601 workflow_type: task.workflow_type.clone(),
6602 run_status: task.run_status,
6603 workflow_input,
6604 workflow_input_avro_value: workflow_input_typed.clone(),
6605 history_events: Arc::clone(&history_events),
6606 signal_events: Arc::new(signal_events),
6607 };
6608
6609 let future = match query {
6610 RegisteredQuery::Snapshot(handler) => handler(context, args),
6611 RegisteredQuery::Replayed {
6612 state_type,
6613 handler,
6614 } => {
6615 let workflow = self
6616 .workflows
6617 .get(&task.workflow_type)
6618 .expect("workflow registration was checked above");
6619 if workflow.state_type != Some(*state_type) {
6620 return Err(QueryTaskExecutionFailure::new(
6621 "query_workflow_state_unavailable",
6622 "replayed query state type does not match its workflow registration",
6623 "QueryWorkflowStateUnavailable",
6624 ));
6625 }
6626 let replay = workflow.replay.as_ref().ok_or_else(|| {
6627 QueryTaskExecutionFailure::new(
6628 "query_workflow_state_unavailable",
6629 format!(
6630 "workflow type {:?} is not registered for instance-state replay",
6631 task.workflow_type
6632 ),
6633 "QueryWorkflowStateUnavailable",
6634 )
6635 })?;
6636 let workflow_state = Arc::new(Mutex::new(
6637 WorkflowState::new_with_identity(
6638 history_events.as_ref().clone(),
6639 context.workflow_id.clone(),
6640 context.run_id.clone(),
6641 self.task_queue.clone(),
6642 task.payload_codec,
6643 None,
6644 )
6645 .map_err(|error| {
6646 QueryTaskExecutionFailure::new(
6647 "query_workflow_state_unavailable",
6648 format!("workflow replay failed before query: {error}"),
6649 "QueryWorkflowStateUnavailable",
6650 )
6651 })?,
6652 ));
6653 let workflow_context = WorkflowContext {
6654 state: workflow_state,
6655 };
6656 let mut invocation = replay(workflow_context.clone(), workflow_input_typed.clone());
6657 let mut cx = TaskContext::from_waker(noop_waker_ref());
6658 match invocation.future.as_mut().poll(&mut cx) {
6659 Poll::Ready(Ok(_)) => {
6660 workflow_context
6661 .ensure_history_consumed()
6662 .map_err(|error| {
6663 QueryTaskExecutionFailure::new(
6664 "query_workflow_state_unavailable",
6665 format!("workflow replay failed before query: {error}"),
6666 "QueryWorkflowStateUnavailable",
6667 )
6668 })?;
6669 }
6670 Poll::Ready(Err(error)) => {
6671 return Err(QueryTaskExecutionFailure::new(
6672 "query_workflow_state_unavailable",
6673 format!("workflow replay failed before query: {error}"),
6674 "QueryWorkflowStateUnavailable",
6675 ));
6676 }
6677 Poll::Pending => {
6678 let commands = workflow_context.take_commands().map_err(|error| {
6679 QueryTaskExecutionFailure::new(
6680 "query_workflow_state_unavailable",
6681 format!("workflow replay failed before query: {error}"),
6682 "QueryWorkflowStateUnavailable",
6683 )
6684 })?;
6685 if commands.is_empty()
6686 && !workflow_context
6687 .matched_recorded_pending()
6688 .map_err(|error| {
6689 QueryTaskExecutionFailure::new(
6690 "query_workflow_state_unavailable",
6691 format!("workflow replay failed before query: {error}"),
6692 "QueryWorkflowStateUnavailable",
6693 )
6694 })?
6695 {
6696 return Err(QueryTaskExecutionFailure::new(
6697 "query_workflow_state_unavailable",
6698 "workflow replay yielded without a durable command",
6699 "QueryWorkflowStateUnavailable",
6700 ));
6701 }
6702 }
6703 }
6704 let state = (invocation.snapshot)().map_err(|error| {
6705 QueryTaskExecutionFailure::new(
6706 "query_workflow_state_unavailable",
6707 format!("cannot snapshot replayed workflow state: {error}"),
6708 "QueryWorkflowStateUnavailable",
6709 )
6710 })?;
6711 handler(context, state, args).map_err(|message| {
6712 QueryTaskExecutionFailure::new(
6713 "query_workflow_state_unavailable",
6714 message,
6715 "QueryWorkflowStateUnavailable",
6716 )
6717 })?
6718 }
6719 };
6720
6721 future.await.map_err(|error| {
6722 QueryTaskExecutionFailure::new("query_rejected", error.to_string(), "QueryFailed")
6723 })
6724 }
6725
6726 fn execute_workflow_task(&self, task: WorkflowTask) -> Result<Vec<Value>> {
6727 validate_workflow_task_payloads(&task)?;
6728
6729 if let Some(update_id) = task
6730 .workflow_update_id
6731 .as_deref()
6732 .filter(|update_id| !update_id.is_empty())
6733 {
6734 return self.execute_update_task(&task, update_id);
6735 }
6736
6737 let workflow = self
6738 .workflows
6739 .get(&task.workflow_type)
6740 .ok_or_else(|| Error::WorkflowNotRegistered(task.workflow_type.clone()))?;
6741 let input = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
6742 let resume_signal = decode_resume_signal(&task)?;
6743 let history_budget = WorkflowHistoryBudget {
6744 event_count: task
6745 .total_history_events
6746 .unwrap_or_else(|| u64::try_from(task.history_events.len()).unwrap_or(u64::MAX)),
6747 size_bytes: task.history_size_bytes,
6748 continue_as_new_recommended: task.continue_as_new_recommended.unwrap_or(false),
6749 pressure: task.history_budget_pressure.clone(),
6750 };
6751 let workflow_command_identity = task
6752 .workflow_command_id
6753 .clone()
6754 .filter(|identity| !identity.is_empty())
6755 .unwrap_or_default();
6756 let mut workflow_state = WorkflowState::new_with_identity(
6757 task.history_events,
6758 task.workflow_id,
6759 task.run_id,
6760 self.task_queue.clone(),
6761 task.payload_codec.clone(),
6762 resume_signal,
6763 )?;
6764 workflow_state.history_budget = history_budget;
6765 workflow_state.workflow_command_identity = workflow_command_identity;
6766 workflow_state.cancel_requested = task.cancel_requested;
6767 let state = Arc::new(Mutex::new(workflow_state));
6768 let ctx = WorkflowContext { state };
6769 let mut future = (workflow.execute)(ctx.clone(), input);
6770 let mut cx = TaskContext::from_waker(noop_waker_ref());
6771
6772 match future.as_mut().poll(&mut cx) {
6773 Poll::Ready(Ok(result)) => {
6774 ctx.ensure_history_consumed()?;
6775 let result = encode_typed_envelope(&result, &task.payload_codec)?;
6776 let mut commands = ctx.take_commands()?;
6777 commands.push(json!({
6778 "type": "complete_workflow",
6779 "result": result
6780 }));
6781 Ok(commands)
6782 }
6783 Poll::Ready(Err(error)) => {
6784 if let Error::ContinueAsNew(request) = error {
6785 let mut commands = ctx.take_commands()?;
6786 if let Some(command) = ctx.continue_as_new_command(request)? {
6787 commands.push(command);
6788 }
6789 ctx.ensure_history_consumed()?;
6790 return Ok(commands);
6791 }
6792 if workflow_task_integrity_error(&error) {
6793 return Err(error);
6798 }
6799 ctx.ensure_history_consumed()?;
6802 let mut commands = ctx.take_commands()?;
6803 commands.push(workflow_failure_command(&error));
6804 Ok(commands)
6805 }
6806 Poll::Pending => {
6807 let commands = ctx.take_commands()?;
6808 if commands.is_empty() && !ctx.matched_recorded_pending()? {
6809 Err(Error::WorkflowYieldedWithoutCommand)
6810 } else {
6811 Ok(commands)
6812 }
6813 }
6814 }
6815 }
6816
6817 fn execute_update_task(&self, task: &WorkflowTask, update_id: &str) -> Result<Vec<Value>> {
6818 if !self.workflows.contains_key(&task.workflow_type) {
6819 return Err(Error::WorkflowNotRegistered(task.workflow_type.clone()));
6820 }
6821
6822 let accepted = task.history_events.iter().rev().find_map(|event| {
6823 (event.event_type == "UpdateAccepted"
6824 && event.payload.get("update_id").and_then(Value::as_str) == Some(update_id))
6825 .then_some(&event.payload)
6826 });
6827 let update_name = accepted
6828 .and_then(|payload| payload.get("update_name"))
6829 .and_then(Value::as_str)
6830 .or(task.update_name.as_deref())
6831 .unwrap_or_default();
6832 let Some(handler) = self
6833 .updates
6834 .get(&task.workflow_type)
6835 .and_then(|handlers| handlers.get(update_name))
6836 else {
6837 return Ok(vec![json!({
6838 "type": "fail_update",
6839 "update_id": update_id,
6840 "message": format!(
6841 "no update handler is registered for {}.{update_name}",
6842 task.workflow_type
6843 ),
6844 "exception_type": "UnknownUpdate",
6845 "non_retryable": true,
6846 })]);
6847 };
6848 let arguments = accepted
6849 .and_then(|payload| payload.get("arguments"))
6850 .or(task.arguments.as_ref());
6851 let arguments = decode_task_avro_arguments(arguments, &task.payload_codec)?;
6852 let context = QueryContext {
6853 workflow_id: task.workflow_id.clone(),
6854 run_id: task.run_id.clone(),
6855 workflow_type: task.workflow_type.clone(),
6856 run_status: Some("running".to_string()),
6857 workflow_input: Value::Null,
6858 workflow_input_avro_value: AvroValue::Null,
6859 history_events: Arc::new(task.history_events.clone()),
6860 signal_events: Arc::new(Vec::new()),
6861 };
6862 let mut future = handler(context, arguments);
6863 let mut cx = TaskContext::from_waker(noop_waker_ref());
6864
6865 match future.as_mut().poll(&mut cx) {
6866 Poll::Ready(Ok(result)) => Ok(vec![json!({
6867 "type": "complete_update",
6868 "update_id": update_id,
6869 "result": encode_typed_envelope(&result, &task.payload_codec)?,
6870 })]),
6871 Poll::Ready(Err(error)) => Ok(vec![json!({
6872 "type": "fail_update",
6873 "update_id": update_id,
6874 "message": error.to_string(),
6875 "exception_type": "UpdateFailed",
6876 "non_retryable": true,
6877 })]),
6878 Poll::Pending => Err(Error::WorkflowYieldedWithoutCommand),
6879 }
6880 }
6881
6882 async fn execute_activity_task(&self, task: ActivityTask) -> Result<AvroValue> {
6883 validate_activity_task_payloads(&task)?;
6884
6885 let handler = self
6886 .activities
6887 .get(&task.activity_type)
6888 .ok_or_else(|| Error::ActivityNotRegistered(task.activity_type.clone()))?;
6889 let args = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
6890 let attempt_id = task
6891 .activity_attempt_id
6892 .clone()
6893 .or(task.attempt_id.clone())
6894 .unwrap_or_default();
6895 let lease_owner = task
6896 .lease_owner
6897 .clone()
6898 .unwrap_or_else(|| self.worker_id.clone());
6899 let ctx = ActivityContext {
6900 client: self.client.clone(),
6901 task_id: task.task_id,
6902 activity_attempt_id: attempt_id,
6903 lease_owner,
6904 activity_type: task.activity_type,
6905 attempt_number: task.attempt_number,
6906 task_queue: self.task_queue.clone(),
6907 worker_id: self.worker_id.clone(),
6908 };
6909
6910 handler(ctx, args).await
6911 }
6912}
6913
6914fn poller_result(
6915 kind: &str,
6916 result: std::result::Result<Result<()>, tokio::task::JoinError>,
6917) -> Result<()> {
6918 match result {
6919 Ok(result) => result,
6920 Err(error) => Err(Error::WorkerLoop(format!(
6921 "{kind} poller join error: {error}"
6922 ))),
6923 }
6924}
6925
6926fn optional_poller_result(
6927 kind: &str,
6928 result: Option<std::result::Result<Result<()>, tokio::task::JoinError>>,
6929) -> Result<()> {
6930 match result {
6931 Some(result) => poller_result(kind, result),
6932 None => Ok(()),
6933 }
6934}
6935
6936async fn join_pollers(
6937 workflow_poller: Option<tokio::task::JoinHandle<Result<()>>>,
6938 activity_poller: Option<tokio::task::JoinHandle<Result<()>>>,
6939 query_poller: Option<tokio::task::JoinHandle<Result<()>>>,
6940) -> Result<()> {
6941 let mut first_error = None;
6942
6943 if let Some(handle) = workflow_poller {
6944 if let Err(error) = poller_result("workflow", handle.await) {
6945 first_error.get_or_insert(error);
6946 }
6947 }
6948
6949 if let Some(handle) = activity_poller {
6950 if let Err(error) = poller_result("activity", handle.await) {
6951 first_error.get_or_insert(error);
6952 }
6953 }
6954
6955 if let Some(handle) = query_poller {
6956 if let Err(error) = poller_result("query", handle.await) {
6957 first_error.get_or_insert(error);
6958 }
6959 }
6960
6961 if let Some(error) = first_error {
6962 Err(error)
6963 } else {
6964 Ok(())
6965 }
6966}
6967
6968fn default_worker_id() -> String {
6969 let millis = SystemTime::now()
6970 .duration_since(UNIX_EPOCH)
6971 .unwrap_or_default()
6972 .as_millis();
6973 format!("rust-worker-{}-{millis}", std::process::id())
6974}
6975
6976fn percent_encode_path_segment(segment: &str) -> String {
6977 const HEX: &[u8; 16] = b"0123456789ABCDEF";
6978 let mut encoded = String::with_capacity(segment.len());
6979
6980 for byte in segment.bytes() {
6981 if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') {
6982 encoded.push(char::from(byte));
6983 } else {
6984 encoded.push('%');
6985 encoded.push(char::from(HEX[(byte >> 4) as usize]));
6986 encoded.push(char::from(HEX[(byte & 0x0f) as usize]));
6987 }
6988 }
6989
6990 encoded
6991}
6992
6993fn unique_request_id(prefix: &str) -> String {
6994 let nanos = SystemTime::now()
6995 .duration_since(UNIX_EPOCH)
6996 .unwrap_or_default()
6997 .as_nanos();
6998 format!("{prefix}-{}-{nanos}", std::process::id())
6999}
7000
7001#[derive(Debug)]
7002struct QueryTaskExecutionFailure {
7003 reason: String,
7004 message: String,
7005 failure_type: String,
7006}
7007
7008impl QueryTaskExecutionFailure {
7009 fn new(
7010 reason: impl Into<String>,
7011 message: impl Into<String>,
7012 failure_type: impl Into<String>,
7013 ) -> Self {
7014 Self {
7015 reason: reason.into(),
7016 message: message.into(),
7017 failure_type: failure_type.into(),
7018 }
7019 }
7020}
7021
7022#[derive(Clone, Debug)]
7028pub struct WorkflowInstance<S> {
7029 state: Arc<Mutex<S>>,
7030}
7031
7032impl<S> WorkflowInstance<S> {
7033 fn new(state: S) -> Self {
7034 Self {
7035 state: Arc::new(Mutex::new(state)),
7036 }
7037 }
7038
7039 pub fn read<R>(&self, reader: impl FnOnce(&S) -> R) -> Result<R> {
7041 let state = self
7042 .state
7043 .lock()
7044 .map_err(|_| Error::WorkflowStatePoisoned)?;
7045 Ok(reader(&state))
7046 }
7047
7048 pub fn update<R>(&self, transition: impl FnOnce(&mut S) -> R) -> Result<R> {
7050 let mut state = self
7051 .state
7052 .lock()
7053 .map_err(|_| Error::WorkflowStatePoisoned)?;
7054 Ok(transition(&mut state))
7055 }
7056}
7057
7058impl<S: Clone> WorkflowInstance<S> {
7059 fn snapshot(&self) -> Result<S> {
7060 self.read(Clone::clone)
7061 }
7062}
7063
7064#[derive(Clone, Debug)]
7065pub struct WorkflowContext {
7066 state: Arc<Mutex<WorkflowState>>,
7067}
7068
7069fn valid_memo_key(key: &str) -> bool {
7070 let numeric_candidate = key.strip_prefix('-').unwrap_or(key);
7071
7072 !key.is_empty()
7073 && key.len() <= 64
7074 && (numeric_candidate.is_empty()
7075 || !numeric_candidate.bytes().all(|byte| byte.is_ascii_digit()))
7076 && key
7077 .bytes()
7078 .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'.' | b':' | b'-'))
7079}
7080
7081fn avro_encoded_size(value: &AvroValue) -> Result<usize> {
7082 BASE64
7083 .decode(encode_avro_value(value)?.blob)
7084 .map(|bytes| bytes.len())
7085 .map_err(|error| Error::Codec(format!("memo Avro encoding was not strict base64: {error}")))
7086}
7087
7088fn canonical_memo_entries(value: AvroValue, require_entries: bool) -> Result<AvroValue> {
7089 let AvroValue::Map(entries) = value else {
7090 return Err(Error::InvalidMemoUpdate(
7091 "entries must serialize to an Avro string-keyed map".to_string(),
7092 ));
7093 };
7094 if require_entries && entries.is_empty() {
7095 return Err(Error::InvalidMemoUpdate(
7096 "at least one entry is required".to_string(),
7097 ));
7098 }
7099 if entries.len() > MAX_MEMO_ENTRIES {
7100 return Err(Error::InvalidMemoUpdate(format!(
7101 "at most {MAX_MEMO_ENTRIES} entries are allowed"
7102 )));
7103 }
7104
7105 for (key, value) in &entries {
7106 if !valid_memo_key(&key) {
7107 return Err(Error::InvalidMemoUpdate(
7108 "keys must match ^(?!-?[0-9]+$)[A-Za-z0-9_.:-]{1,64}$".to_string(),
7109 ));
7110 }
7111 if avro_encoded_size(value)? > MAX_MEMO_VALUE_SIZE_BYTES {
7112 return Err(Error::InvalidMemoUpdate(format!(
7113 "value {key:?} exceeds the {MAX_MEMO_VALUE_SIZE_BYTES}-byte limit"
7114 )));
7115 }
7116 }
7117
7118 let value = AvroValue::Map(entries);
7119 if avro_encoded_size(&value)? > MAX_MEMO_TOTAL_SIZE_BYTES {
7120 return Err(Error::InvalidMemoUpdate(format!(
7121 "update exceeds the {MAX_MEMO_TOTAL_SIZE_BYTES}-byte total limit"
7122 )));
7123 }
7124 Ok(value)
7125}
7126
7127fn decode_memo_history_map(envelope: &Value, require_entries: bool) -> Result<AvroValue> {
7128 let object = envelope.as_object().ok_or_else(|| {
7129 Error::InvalidMemoUpdate(
7130 "history field must use the public {codec, blob} payload envelope".to_string(),
7131 )
7132 })?;
7133 if object.len() != 2 || !object.contains_key("codec") || !object.contains_key("blob") {
7134 return Err(Error::InvalidMemoUpdate(
7135 "history field must use exactly the public {codec, blob} payload envelope".to_string(),
7136 ));
7137 }
7138
7139 canonical_memo_entries(
7140 decode_wire_avro_value(envelope, DEFAULT_CODEC)?,
7141 require_entries,
7142 )
7143}
7144
7145impl WorkflowContext {
7146 pub fn workflow_identity(&self) -> Result<WorkflowIdentity> {
7148 let state = self
7149 .state
7150 .lock()
7151 .map_err(|_| Error::WorkflowStatePoisoned)?;
7152 Ok(WorkflowIdentity {
7153 workflow_id: state.workflow_id.clone(),
7154 run_id: state.run_id.clone(),
7155 })
7156 }
7157
7158 pub fn history_budget(&self) -> Result<WorkflowHistoryBudget> {
7160 let state = self
7161 .state
7162 .lock()
7163 .map_err(|_| Error::WorkflowStatePoisoned)?;
7164 Ok(state.history_budget.clone())
7165 }
7166
7167 pub fn continue_as_new<T: Serialize>(&self, args: T) -> Result<Value> {
7173 self.continue_as_new_with_options(ContinueAsNewOptions::new(), args)
7174 }
7175
7176 pub fn continue_as_new_with_options<T: Serialize>(
7178 &self,
7179 options: ContinueAsNewOptions,
7180 args: T,
7181 ) -> Result<Value> {
7182 options.validate()?;
7183 Err(Error::ContinueAsNew(ContinueAsNewRequest {
7184 arguments: normalize_avro_arguments(AvroValue::from_serialize(&args)?),
7185 options,
7186 }))
7187 }
7188
7189 pub fn activity<T: Serialize>(
7190 &self,
7191 activity_type: impl Into<String>,
7192 args: T,
7193 ) -> ActivityCall {
7194 self.activity_with_options(activity_type, ActivityOptions::new(), args)
7195 }
7196
7197 pub fn activity_on_queue<T, Q>(
7198 &self,
7199 activity_type: impl Into<String>,
7200 task_queue: Option<Q>,
7201 args: T,
7202 ) -> ActivityCall
7203 where
7204 T: Serialize,
7205 Q: Into<String>,
7206 {
7207 let mut options = ActivityOptions::new();
7208 options.task_queue = task_queue.map(Into::into);
7209 self.activity_with_options(activity_type, options, args)
7210 }
7211
7212 pub fn activity_with_options<T: Serialize>(
7250 &self,
7251 activity_type: impl Into<String>,
7252 options: ActivityOptions,
7253 args: T,
7254 ) -> ActivityCall {
7255 ActivityCall {
7256 ctx: self.clone(),
7257 activity_type: activity_type.into(),
7258 options,
7259 args: Some(AvroValue::from_serialize(&args)),
7260 scheduled: false,
7261 parallel_group_path: Vec::new(),
7262 }
7263 }
7264
7265 pub async fn activity_avro_value<T: Serialize>(
7266 &self,
7267 activity_type: impl Into<String>,
7268 args: T,
7269 ) -> Result<AvroValue> {
7270 let mut call = self.activity(activity_type, args);
7271 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
7272 }
7273
7274 pub async fn activity_avro_value_with_options<T: Serialize>(
7275 &self,
7276 activity_type: impl Into<String>,
7277 options: ActivityOptions,
7278 args: T,
7279 ) -> Result<AvroValue> {
7280 let mut call = self.activity_with_options(activity_type, options, args);
7281 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
7282 }
7283
7284 pub async fn activity_typed<I, O>(&self, activity_type: impl Into<String>, args: I) -> Result<O>
7286 where
7287 I: Serialize,
7288 O: DeserializeOwned,
7289 {
7290 self.activity_typed_with_options(activity_type, ActivityOptions::new(), args)
7291 .await
7292 }
7293
7294 pub async fn activity_typed_with_options<I, O>(
7300 &self,
7301 activity_type: impl Into<String>,
7302 options: ActivityOptions,
7303 args: I,
7304 ) -> Result<O>
7305 where
7306 I: Serialize,
7307 O: DeserializeOwned,
7308 {
7309 let activity_type = activity_type.into();
7310 let encoded = AvroValue::from_serialize(&args).map_err(|error| {
7311 handler_type_error::<I>(
7312 HandlerKind::Activity,
7313 &activity_type,
7314 HandlerValueKind::Input,
7315 error.to_string(),
7316 )
7317 });
7318 let mut call = ActivityCall {
7319 ctx: self.clone(),
7320 activity_type: activity_type.clone(),
7321 options,
7322 args: Some(encoded),
7323 scheduled: false,
7324 parallel_group_path: Vec::new(),
7325 };
7326 let result = std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await?;
7327 decode_handler_result(result, HandlerKind::Activity, &activity_type)
7328 }
7329
7330 pub fn parallel(&self, operations: Vec<ParallelOperation>) -> ParallelCall {
7337 ParallelCall::new(self.clone(), operations)
7338 }
7339
7340 pub fn join(&self, operations: Vec<ParallelOperation>) -> ParallelCall {
7342 self.parallel(operations)
7343 }
7344
7345 pub async fn parallel_avro_value(
7347 &self,
7348 operations: Vec<ParallelOperation>,
7349 ) -> Result<Vec<ParallelAvroResult>> {
7350 let mut call = self.parallel(operations);
7351 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
7352 }
7353
7354 pub fn saga(&self) -> Saga {
7356 Saga::new(self.clone())
7357 }
7358
7359 pub fn is_cancellation_requested(&self) -> Result<bool> {
7361 let state = self
7362 .state
7363 .lock()
7364 .map_err(|_| Error::WorkflowStatePoisoned)?;
7365 Ok(state.cancel_requested)
7366 }
7367
7368 pub fn throw_if_cancellation_requested(&self) -> Result<()> {
7373 if self.is_cancellation_requested()? {
7374 return Err(Error::WorkflowCancellationRequested(
7375 WorkflowCancellationRequested,
7376 ));
7377 }
7378 Ok(())
7379 }
7380
7381 pub fn wait_signal(&self, signal_name: impl Into<String>) -> SignalCall {
7382 SignalCall {
7383 ctx: self.clone(),
7384 signal_name: signal_name.into(),
7385 opened_wait: false,
7386 matched_pending: false,
7387 }
7388 }
7389
7390 pub async fn wait_signal_avro_value(
7391 &self,
7392 signal_name: impl Into<String>,
7393 ) -> Result<Vec<AvroValue>> {
7394 let mut call = self.wait_signal(signal_name);
7395 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
7396 }
7397
7398 pub fn signals(&self, signal_name: &str) -> Result<Vec<Vec<Value>>> {
7404 self.signals_avro_value(signal_name)?
7405 .into_iter()
7406 .map(|arguments| {
7407 arguments
7408 .into_iter()
7409 .map(AvroValue::into_json)
7410 .collect::<Result<Vec<_>>>()
7411 })
7412 .collect()
7413 }
7414
7415 pub fn signals_avro_value(&self, signal_name: &str) -> Result<Vec<Vec<AvroValue>>> {
7417 let state = self
7418 .state
7419 .lock()
7420 .map_err(|_| Error::WorkflowStatePoisoned)?;
7421 state
7422 .history_events
7423 .iter()
7424 .filter(|event| {
7425 event.event_type == "SignalReceived"
7426 && event.payload.get("signal_name").and_then(Value::as_str) == Some(signal_name)
7427 })
7428 .map(|event| decode_signal_event_arguments(event, &state.payload_codec))
7429 .collect()
7430 }
7431
7432 pub fn updates(&self, update_name: &str) -> Result<Vec<Vec<Value>>> {
7438 self.updates_avro_value(update_name)?
7439 .into_iter()
7440 .map(|arguments| {
7441 arguments
7442 .into_iter()
7443 .map(AvroValue::into_json)
7444 .collect::<Result<Vec<_>>>()
7445 })
7446 .collect()
7447 }
7448
7449 pub fn updates_avro_value(&self, update_name: &str) -> Result<Vec<Vec<AvroValue>>> {
7451 let state = self
7452 .state
7453 .lock()
7454 .map_err(|_| Error::WorkflowStatePoisoned)?;
7455 let mut seen = Vec::new();
7456 let mut updates = Vec::new();
7457 for event in state.history_events.iter() {
7458 if !matches!(
7459 event.event_type.as_str(),
7460 "UpdateAccepted" | "UpdateApplied"
7461 ) || event.payload.get("update_name").and_then(Value::as_str) != Some(update_name)
7462 || event.payload.get("arguments").is_none()
7463 {
7464 continue;
7465 }
7466 if let Some(update_id) = event.payload.get("update_id").and_then(Value::as_str) {
7467 if seen.iter().any(|recorded| recorded == update_id) {
7468 continue;
7469 }
7470 seen.push(update_id.to_string());
7471 }
7472 updates.push(decode_update_event_arguments(event, &state.payload_codec)?);
7473 }
7474 Ok(updates)
7475 }
7476
7477 pub fn wait_condition<F>(
7485 &self,
7486 options: ConditionWaitOptions,
7487 predicate: F,
7488 ) -> ConditionWaitCall
7489 where
7490 F: Fn() -> Result<bool> + Send + 'static,
7491 {
7492 ConditionWaitCall {
7493 ctx: self.clone(),
7494 options,
7495 predicate: Box::new(predicate),
7496 opened_wait: false,
7497 }
7498 }
7499
7500 pub fn sleep(&self, duration: Duration) -> TimerCall {
7521 let delay_seconds = duration
7522 .as_secs()
7523 .checked_add(u64::from(duration.subsec_nanos() > 0));
7524 TimerCall {
7525 ctx: self.clone(),
7526 delay_seconds,
7527 scheduled: false,
7528 matched_pending: false,
7529 parallel_group_path: Vec::new(),
7530 }
7531 }
7532
7533 pub fn start_timer(&self, duration: Duration) -> TimerCall {
7535 self.sleep(duration)
7536 }
7537
7538 pub fn side_effect<T, F>(&self, callback: F) -> Result<T>
7545 where
7546 T: Serialize + DeserializeOwned,
7547 F: FnOnce() -> T,
7548 {
7549 {
7550 let mut state = self
7551 .state
7552 .lock()
7553 .map_err(|_| Error::WorkflowStatePoisoned)?;
7554 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
7555 return match recorded {
7556 RecordedCommand::SideEffect { sequence, value } => {
7557 state.command_cursor += 1;
7558 value.deserialize().map_err(|error| {
7559 Error::NonDeterministicReplay(ReplayFailure::new(
7560 "side_effect_type_mismatch",
7561 Some(sequence),
7562 Some(std::any::type_name::<T>().to_string()),
7563 Some(error.to_string()),
7564 "recorded side-effect value is incompatible with the requested Rust type",
7565 ))
7566 })
7567 }
7568 other => Err(command_mismatch(&other, "side effect")),
7569 };
7570 }
7571 }
7572
7573 let value = callback();
7574 let avro_value = AvroValue::from_serialize(&value)?;
7575 let mut state = self
7576 .state
7577 .lock()
7578 .map_err(|_| Error::WorkflowStatePoisoned)?;
7579 let result = encode_typed_envelope(&avro_value, &state.payload_codec)?;
7580 state.commands.push(json!({
7581 "type": "record_side_effect",
7582 "result": result,
7583 }));
7584 Ok(value)
7585 }
7586
7587 pub fn side_effect_avro_value<F>(&self, callback: F) -> Result<AvroValue>
7589 where
7590 F: FnOnce() -> AvroValue,
7591 {
7592 {
7593 let mut state = self
7594 .state
7595 .lock()
7596 .map_err(|_| Error::WorkflowStatePoisoned)?;
7597 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
7598 return match recorded {
7599 RecordedCommand::SideEffect { value, .. } => {
7600 state.command_cursor += 1;
7601 Ok(value)
7602 }
7603 other => Err(command_mismatch(&other, "side effect")),
7604 };
7605 }
7606 }
7607
7608 let value = callback();
7609 let mut state = self
7610 .state
7611 .lock()
7612 .map_err(|_| Error::WorkflowStatePoisoned)?;
7613 let result = encode_typed_envelope(&value, &state.payload_codec)?;
7614 state.commands.push(json!({
7615 "type": "record_side_effect",
7616 "result": result,
7617 }));
7618 Ok(value)
7619 }
7620
7621 pub fn append_workflow_stream(
7627 &self,
7628 stream_name: impl Into<String>,
7629 items: &[WorkflowStreamAppendItem],
7630 max_pending_items: Option<u64>,
7631 ) -> Result<()> {
7632 if items.is_empty() {
7633 return Err(Error::Codec(
7634 "workflow_stream_items_empty: append requires at least one item".to_string(),
7635 ));
7636 }
7637 if max_pending_items == Some(0) {
7638 return Err(Error::Codec(
7639 "workflow_stream_pending_limit_invalid: max_pending_items must be positive"
7640 .to_string(),
7641 ));
7642 }
7643 let stream_name = stream_name.into();
7644 if stream_name.is_empty() {
7645 return Err(Error::Codec(
7646 "workflow_stream_name_invalid: stream name must not be empty".to_string(),
7647 ));
7648 }
7649
7650 let mut state = self
7651 .state
7652 .lock()
7653 .map_err(|_| Error::WorkflowStatePoisoned)?;
7654 let command_ordinal = state.workflow_stream_command_counter;
7655 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
7656 state.workflow_stream_command_counter += 1;
7657 return match recorded {
7658 RecordedCommand::SideEffect { .. } => {
7659 state.command_cursor += 1;
7660 Ok(())
7661 }
7662 other => Err(command_mismatch(&other, "workflow stream append")),
7663 };
7664 }
7665
7666 let identity = Self::workflow_stream_command_identity(&state)?.to_string();
7667 state.workflow_stream_command_counter += 1;
7668 let wire_items = items
7669 .iter()
7670 .enumerate()
7671 .map(|(item_index, item)| {
7672 item.wire_value(Some(format!(
7673 "dw-stream:{identity}:{command_ordinal}:{item_index}"
7674 )))
7675 })
7676 .collect::<Vec<_>>();
7677 let mut directive = json!({
7678 "operation": "append",
7679 "stream_name": stream_name,
7680 "command_identity": identity,
7681 "command_ordinal": command_ordinal,
7682 "items": wire_items,
7683 });
7684 if let Some(max_pending_items) = max_pending_items {
7685 directive["max_pending_items"] = json!(max_pending_items);
7686 }
7687 let result = encode_typed_envelope(&AvroValue::Null, &state.payload_codec)?;
7688 state.commands.push(json!({
7689 "type": "record_side_effect",
7690 "result": result,
7691 "workflow_stream": directive,
7692 }));
7693 Ok(())
7694 }
7695
7696 pub fn close_workflow_stream(
7698 &self,
7699 stream_name: impl Into<String>,
7700 retention_seconds: Option<u64>,
7701 ) -> Result<()> {
7702 self.finish_workflow_stream(stream_name.into(), None, retention_seconds)
7703 }
7704
7705 pub fn error_workflow_stream(
7707 &self,
7708 stream_name: impl Into<String>,
7709 error_reason: impl Into<String>,
7710 retention_seconds: Option<u64>,
7711 ) -> Result<()> {
7712 let error_reason = error_reason.into();
7713 if error_reason.is_empty() {
7714 return Err(Error::Codec(
7715 "workflow_stream_error_invalid: error reason must not be empty".to_string(),
7716 ));
7717 }
7718 self.finish_workflow_stream(stream_name.into(), Some(error_reason), retention_seconds)
7719 }
7720
7721 fn finish_workflow_stream(
7722 &self,
7723 stream_name: String,
7724 error_reason: Option<String>,
7725 retention_seconds: Option<u64>,
7726 ) -> Result<()> {
7727 if stream_name.is_empty() {
7728 return Err(Error::Codec(
7729 "workflow_stream_name_invalid: stream name must not be empty".to_string(),
7730 ));
7731 }
7732 if retention_seconds == Some(0) {
7733 return Err(Error::Codec(
7734 "workflow_stream_retention_invalid: retention_seconds must be positive".to_string(),
7735 ));
7736 }
7737 let mut state = self
7738 .state
7739 .lock()
7740 .map_err(|_| Error::WorkflowStatePoisoned)?;
7741 let command_ordinal = state.workflow_stream_command_counter;
7742 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
7743 state.workflow_stream_command_counter += 1;
7744 return match recorded {
7745 RecordedCommand::SideEffect { .. } => {
7746 state.command_cursor += 1;
7747 Ok(())
7748 }
7749 other => Err(command_mismatch(&other, "workflow stream close")),
7750 };
7751 }
7752 let identity = Self::workflow_stream_command_identity(&state)?.to_string();
7753 state.workflow_stream_command_counter += 1;
7754 let mut directive = json!({
7755 "operation": if error_reason.is_some() { "error" } else { "close" },
7756 "stream_name": stream_name,
7757 "command_identity": identity,
7758 "command_ordinal": command_ordinal,
7759 });
7760 if let Some(error_reason) = error_reason {
7761 directive["error_reason"] = json!(error_reason);
7762 }
7763 if let Some(retention_seconds) = retention_seconds {
7764 directive["retention_seconds"] = json!(retention_seconds);
7765 }
7766 let result = encode_typed_envelope(&AvroValue::Null, &state.payload_codec)?;
7767 state.commands.push(json!({
7768 "type": "record_side_effect",
7769 "result": result,
7770 "workflow_stream": directive,
7771 }));
7772 Ok(())
7773 }
7774
7775 fn workflow_stream_command_identity(state: &WorkflowState) -> Result<&str> {
7776 let identity = state.workflow_command_identity.as_str();
7777 if identity.is_empty() {
7778 return Err(Error::MissingWorkflowCommandIdentity);
7779 }
7780 Ok(identity)
7781 }
7782
7783 pub fn upsert_search_attributes(&self, update: SearchAttributeUpdate) -> Result<()> {
7789 update.validate()?;
7790 let (attributes, attribute_types) = update.into_wire_parts();
7791 let mut state = self
7792 .state
7793 .lock()
7794 .map_err(|_| Error::WorkflowStatePoisoned)?;
7795
7796 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
7797 return match recorded {
7798 RecordedCommand::SearchAttributes {
7799 sequence,
7800 attributes: recorded_attributes,
7801 attribute_types: recorded_attribute_types,
7802 } => {
7803 if recorded_attributes != attributes {
7804 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
7805 "search_attribute_value_mismatch",
7806 Some(sequence),
7807 Some(recorded_attributes.to_string()),
7808 Some(attributes.to_string()),
7809 "search-attribute values differ from the recorded durable command",
7810 )));
7811 }
7812 if let RecordedSnapshotValue::Known(recorded_types) = recorded_attribute_types {
7813 if recorded_types != attribute_types {
7814 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
7815 "search_attribute_type_mismatch",
7816 Some(sequence),
7817 Some(json!(recorded_types).to_string()),
7818 Some(json!(attribute_types).to_string()),
7819 "search-attribute declared types differ from the recorded durable command",
7820 )));
7821 }
7822 }
7823 state.command_cursor += 1;
7824 Ok(())
7825 }
7826 other => Err(command_mismatch(&other, "search-attribute update")),
7827 };
7828 }
7829
7830 let mut command = serde_json::Map::from_iter([
7831 ("type".to_string(), json!("upsert_search_attributes")),
7832 ("attributes".to_string(), attributes),
7833 ]);
7834 if !attribute_types.is_empty() {
7835 command.insert("attribute_types".to_string(), json!(attribute_types));
7836 }
7837 state.commands.push(Value::Object(command));
7838 Ok(())
7839 }
7840
7841 pub fn uuid_v4(&self) -> Result<Uuid> {
7843 self.side_effect(Uuid::new_v4)
7844 }
7845
7846 pub fn get_version(
7849 &self,
7850 change_id: impl Into<String>,
7851 min_supported: i32,
7852 max_supported: i32,
7853 ) -> Result<i32> {
7854 let change_id = change_id.into();
7855 if change_id.trim().is_empty() {
7856 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
7857 "version_change_id_invalid",
7858 None,
7859 Some("non-empty change ID".to_string()),
7860 Some(change_id),
7861 "version markers require a stable non-empty change ID",
7862 )));
7863 }
7864 if min_supported > max_supported {
7865 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
7866 "version_range_invalid",
7867 None,
7868 Some("min_supported <= max_supported".to_string()),
7869 Some(format!("{min_supported}..={max_supported}")),
7870 "version marker supported range is invalid",
7871 )));
7872 }
7873
7874 let mut state = self
7875 .state
7876 .lock()
7877 .map_err(|_| Error::WorkflowStatePoisoned)?;
7878 if let Some((version, sequence)) = state.version_markers.get(&change_id).copied() {
7879 ensure_version_supported(&change_id, version, min_supported, max_supported, sequence)?;
7880 return Ok(version);
7881 }
7882
7883 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
7884 return match recorded {
7885 RecordedCommand::VersionMarker {
7886 sequence,
7887 change_id: recorded_change_id,
7888 version,
7889 ..
7890 } => {
7891 if recorded_change_id != change_id {
7892 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
7893 "version_change_id_mismatch",
7894 Some(sequence),
7895 Some(recorded_change_id),
7896 Some(change_id),
7897 "recorded version marker change ID differs from current workflow code",
7898 )));
7899 }
7900 ensure_version_supported(
7901 &change_id,
7902 version,
7903 min_supported,
7904 max_supported,
7905 sequence,
7906 )?;
7907 state.command_cursor += 1;
7908 state.version_markers.insert(change_id, (version, sequence));
7909 Ok(version)
7910 }
7911 other => Err(command_mismatch(
7912 &other,
7913 format!("version marker:{change_id}"),
7914 )),
7915 };
7916 }
7917
7918 let version = max_supported;
7919 state.commands.push(json!({
7920 "type": "record_version_marker",
7921 "change_id": change_id,
7922 "version": version,
7923 "min_supported": min_supported,
7924 "max_supported": max_supported,
7925 }));
7926 state.version_markers.insert(change_id, (version, 0));
7929 Ok(version)
7930 }
7931
7932 pub fn patched(&self, change_id: impl Into<String>) -> Result<bool> {
7934 Ok(self.get_version(change_id, -1, 1)? == 1)
7935 }
7936
7937 pub fn deprecate_patch(&self, change_id: impl Into<String>) -> Result<()> {
7939 self.get_version(change_id, -1, 1).map(|_| ())
7940 }
7941
7942 pub fn upsert_memo<T: Serialize>(&self, entries: T) -> Result<()> {
7947 let entries = canonical_memo_entries(AvroValue::from_serialize(&entries)?, true)?;
7948 let mut state = self
7949 .state
7950 .lock()
7951 .map_err(|_| Error::WorkflowStatePoisoned)?;
7952
7953 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
7954 return match recorded {
7955 RecordedCommand::Memo {
7956 sequence,
7957 entries: recorded_entries,
7958 } => {
7959 if recorded_entries != entries {
7960 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
7961 "memo_update_mismatch",
7962 Some(sequence),
7963 Some(format!("{recorded_entries:?}")),
7964 Some(format!("{entries:?}")),
7965 "recorded memo entries differ from the current workflow update",
7966 )));
7967 }
7968 state.command_cursor += 1;
7969 Ok(())
7970 }
7971 other => Err(command_mismatch(&other, "memo upsert")),
7972 };
7973 }
7974
7975 let entries_envelope = encode_typed_envelope(&entries, DEFAULT_CODEC)?;
7976 state.commands.push(json!({
7977 "type": "upsert_memo",
7978 "entries": entries_envelope,
7979 }));
7980 Ok(())
7981 }
7982
7983 pub fn start_child_workflow<T: Serialize>(
8010 &self,
8011 workflow_type: impl Into<String>,
8012 options: ChildWorkflowOptions,
8013 args: T,
8014 ) -> ChildWorkflowCall {
8015 ChildWorkflowCall {
8016 ctx: self.clone(),
8017 workflow_type: workflow_type.into(),
8018 options,
8019 args: Some(AvroValue::from_serialize(&args)),
8020 scheduled: false,
8021 matched_pending: false,
8022 parallel_group_path: Vec::new(),
8023 }
8024 }
8025
8026 pub async fn start_child_workflow_avro_value<T: Serialize>(
8027 &self,
8028 workflow_type: impl Into<String>,
8029 options: ChildWorkflowOptions,
8030 args: T,
8031 ) -> Result<ChildWorkflowAvroResult> {
8032 let mut call = self.start_child_workflow(workflow_type, options, args);
8033 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8034 }
8035
8036 fn take_commands(&self) -> Result<Vec<Value>> {
8037 let mut state = self
8038 .state
8039 .lock()
8040 .map_err(|_| Error::WorkflowStatePoisoned)?;
8041 Ok(std::mem::take(&mut state.commands))
8042 }
8043
8044 fn continue_as_new_command(&self, request: ContinueAsNewRequest) -> Result<Option<Value>> {
8045 let mut state = self
8046 .state
8047 .lock()
8048 .map_err(|_| Error::WorkflowStatePoisoned)?;
8049
8050 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8051 return Err(command_mismatch(&recorded, "continue as new"));
8052 }
8053 if state.recorded_continue_as_new_sequence.is_some() {
8054 state.continue_as_new_consumed = true;
8055 return Ok(None);
8056 }
8057
8058 let arguments = encode_typed_envelope(&request.arguments, &state.payload_codec)?;
8059 let mut command = serde_json::Map::from_iter([
8060 ("type".to_string(), json!("continue_as_new")),
8061 ("arguments".to_string(), arguments),
8062 ("queue".to_string(), json!(state.task_queue.clone())),
8063 ]);
8064 if let Some(workflow_type) = request.options.workflow_type {
8065 command.insert("workflow_type".to_string(), json!(workflow_type));
8066 }
8067 if let Some(task_queue) = request.options.task_queue {
8068 command.insert("queue".to_string(), json!(task_queue));
8069 }
8070 Ok(Some(Value::Object(command)))
8071 }
8072
8073 fn matched_recorded_pending(&self) -> Result<bool> {
8074 let state = self
8075 .state
8076 .lock()
8077 .map_err(|_| Error::WorkflowStatePoisoned)?;
8078 Ok(state.matched_recorded_pending)
8079 }
8080
8081 fn ensure_history_consumed(&self) -> Result<()> {
8082 let state = self
8083 .state
8084 .lock()
8085 .map_err(|_| Error::WorkflowStatePoisoned)?;
8086 if let Some(command) = state.recorded_commands.get(state.command_cursor) {
8087 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8088 "recorded_commands_unconsumed",
8089 Some(command.sequence()),
8090 Some(command.shape().to_string()),
8091 Some("workflow completion".to_string()),
8092 "workflow completed before consuming all recorded durable commands",
8093 )));
8094 }
8095 if let Some(sequence) = state
8096 .recorded_continue_as_new_sequence
8097 .filter(|_| !state.continue_as_new_consumed)
8098 {
8099 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8100 "recorded_continue_as_new_unconsumed",
8101 Some(sequence),
8102 Some("continue as new".to_string()),
8103 Some("workflow completion".to_string()),
8104 "workflow completed without consuming its recorded continue-as-new transition",
8105 )));
8106 }
8107 Ok(())
8108 }
8109}
8110
8111#[derive(Debug)]
8112struct WorkflowState {
8113 workflow_id: Option<String>,
8114 run_id: Option<String>,
8115 task_queue: String,
8116 payload_codec: String,
8117 history_events: Arc<Vec<HistoryEvent>>,
8118 history_budget: WorkflowHistoryBudget,
8119 cancel_requested: bool,
8120 resume_signal: Option<ResumeSignal>,
8121 recorded_commands: Vec<RecordedCommand>,
8122 recorded_continue_as_new_sequence: Option<u64>,
8123 continue_as_new_consumed: bool,
8124 command_cursor: usize,
8125 matched_recorded_pending: bool,
8126 version_markers: HashMap<String, (i32, u64)>,
8127 workflow_command_identity: String,
8128 workflow_stream_command_counter: u64,
8129 commands: Vec<Value>,
8130}
8131
8132impl WorkflowState {
8133 #[cfg(test)]
8134 fn new(
8135 history: Vec<HistoryEvent>,
8136 task_queue: String,
8137 payload_codec: String,
8138 resume_signal: Option<ResumeSignal>,
8139 ) -> Result<Self> {
8140 Self::new_with_identity(
8141 history,
8142 None,
8143 None,
8144 task_queue,
8145 payload_codec,
8146 resume_signal,
8147 )
8148 }
8149
8150 fn new_with_identity(
8151 history: Vec<HistoryEvent>,
8152 workflow_id: Option<String>,
8153 run_id: Option<String>,
8154 task_queue: String,
8155 payload_codec: String,
8156 resume_signal: Option<ResumeSignal>,
8157 ) -> Result<Self> {
8158 let recorded_commands = recorded_commands(
8159 &history,
8160 &payload_codec,
8161 WorkflowIdentity {
8162 workflow_id: workflow_id.clone(),
8163 run_id: run_id.clone(),
8164 },
8165 )?;
8166 let recorded_continue_as_new = history
8167 .iter()
8168 .filter(|event| event.event_type == "WorkflowContinuedAsNew")
8169 .collect::<Vec<_>>();
8170 if recorded_continue_as_new.len() > 1 {
8171 return Err(invalid_recorded_history(
8172 "duplicate_continue_as_new_transition",
8173 recorded_continue_as_new
8174 .last()
8175 .and_then(|event| durable_event_sequence(event))
8176 .unwrap_or(0),
8177 "one WorkflowContinuedAsNew event",
8178 &format!(
8179 "{} WorkflowContinuedAsNew events",
8180 recorded_continue_as_new.len()
8181 ),
8182 "workflow history records one continue-as-new transition more than once",
8183 ));
8184 }
8185 let recorded_continue_as_new_sequence = recorded_continue_as_new
8186 .first()
8187 .map(|event| {
8188 durable_event_sequence(event).ok_or_else(|| {
8189 Error::NonDeterministicReplay(ReplayFailure::new(
8190 "continue_as_new_sequence_missing",
8191 None,
8192 Some("recorded transition sequence".to_string()),
8193 Some("missing sequence".to_string()),
8194 "WorkflowContinuedAsNew history is missing its recorded sequence",
8195 ))
8196 })
8197 })
8198 .transpose()?;
8199 let event_count = u64::try_from(history.len()).unwrap_or(u64::MAX);
8200 let cancel_requested = history.iter().any(|event| {
8201 matches!(
8202 event.event_type.as_str(),
8203 "WorkflowCancellationRequested" | "WorkflowCancelRequested"
8204 )
8205 });
8206 Ok(Self {
8207 workflow_command_identity: String::new(),
8208 workflow_stream_command_counter: 0,
8209 workflow_id,
8210 run_id,
8211 task_queue,
8212 payload_codec,
8213 history_events: Arc::new(history),
8214 history_budget: WorkflowHistoryBudget {
8215 event_count,
8216 ..WorkflowHistoryBudget::default()
8217 },
8218 cancel_requested,
8219 resume_signal,
8220 recorded_commands,
8221 recorded_continue_as_new_sequence,
8222 continue_as_new_consumed: false,
8223 command_cursor: 0,
8224 matched_recorded_pending: false,
8225 version_markers: HashMap::new(),
8226 commands: Vec::new(),
8227 })
8228 }
8229}
8230
8231#[derive(Clone, Debug)]
8232enum RecordedCommand {
8233 Activity {
8234 sequence: u64,
8235 activity_type: Option<String>,
8236 options: Option<RecordedActivityOptions>,
8237 outcome: Option<ActivityOutcome>,
8238 parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
8239 },
8240 Timer {
8241 sequence: u64,
8242 delay_seconds: u64,
8243 fired: bool,
8244 parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
8245 },
8246 ChildWorkflow {
8247 sequence: u64,
8248 workflow_type: Option<String>,
8249 outcome: Option<ChildWorkflowOutcome>,
8250 parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
8251 },
8252 SignalWait {
8253 sequence: u64,
8254 signal_name: String,
8255 value: Option<Vec<AvroValue>>,
8256 },
8257 ConditionWait {
8258 sequence: u64,
8259 condition_key: Option<String>,
8260 predicate_identity: String,
8261 timeout_seconds: Option<u64>,
8262 result: Option<ConditionWaitResult>,
8263 },
8264 SearchAttributes {
8265 sequence: u64,
8266 attributes: Value,
8267 attribute_types: RecordedSnapshotValue<BTreeMap<String, String>>,
8268 },
8269 SideEffect {
8270 sequence: u64,
8271 value: AvroValue,
8272 },
8273 VersionMarker {
8274 sequence: u64,
8275 change_id: String,
8276 version: i32,
8277 },
8278 Memo {
8279 sequence: u64,
8280 entries: AvroValue,
8281 },
8282}
8283
8284#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
8285struct RecordedActivityOptions {
8286 task_queue: RecordedSnapshotValue<Option<String>>,
8287 execution_mode: RecordedSnapshotValue<Option<String>>,
8288 retry_policy: ActivityRetrySnapshot,
8289}
8290
8291#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
8292enum RecordedSnapshotValue<T> {
8293 Unknown,
8295 Known(T),
8296}
8297
8298impl<T: PartialEq> RecordedSnapshotValue<T> {
8299 fn matches_current(&self, current: &Self) -> bool {
8300 match self {
8301 Self::Unknown => true,
8302 Self::Known(recorded) => matches!(current, Self::Known(value) if value == recorded),
8303 }
8304 }
8305}
8306
8307#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
8308struct ActivityRetrySnapshot {
8309 snapshot_version: RecordedSnapshotValue<Option<u64>>,
8310 max_attempts: RecordedSnapshotValue<Option<u64>>,
8311 backoff_seconds: RecordedSnapshotValue<Vec<u64>>,
8312 start_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
8313 schedule_to_start_timeout: RecordedSnapshotValue<Option<u64>>,
8314 schedule_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
8315 heartbeat_timeout: RecordedSnapshotValue<Option<u64>>,
8316 non_retryable_error_types: RecordedSnapshotValue<Vec<String>>,
8317}
8318
8319impl ActivityRetrySnapshot {
8320 fn matches_current(&self, current: &Self) -> bool {
8321 self.snapshot_version
8322 .matches_current(¤t.snapshot_version)
8323 && self.max_attempts.matches_current(¤t.max_attempts)
8324 && self
8325 .backoff_seconds
8326 .matches_current(¤t.backoff_seconds)
8327 && self
8328 .start_to_close_timeout
8329 .matches_current(¤t.start_to_close_timeout)
8330 && self
8331 .schedule_to_start_timeout
8332 .matches_current(¤t.schedule_to_start_timeout)
8333 && self
8334 .schedule_to_close_timeout
8335 .matches_current(¤t.schedule_to_close_timeout)
8336 && self
8337 .heartbeat_timeout
8338 .matches_current(¤t.heartbeat_timeout)
8339 && self
8340 .non_retryable_error_types
8341 .matches_current(¤t.non_retryable_error_types)
8342 }
8343}
8344
8345fn recorded_optional_u64(
8346 object: Option<&serde_json::Map<String, Value>>,
8347 field: &str,
8348) -> RecordedSnapshotValue<Option<u64>> {
8349 match object.and_then(|object| object.get(field)) {
8350 None => RecordedSnapshotValue::Unknown,
8351 Some(Value::Null) => RecordedSnapshotValue::Known(None),
8352 Some(value) => RecordedSnapshotValue::Known(value_as_u64(value)),
8353 }
8354}
8355
8356fn recorded_optional_string(
8357 object: &serde_json::Map<String, Value>,
8358 field: &str,
8359) -> RecordedSnapshotValue<Option<String>> {
8360 match object.get(field) {
8361 None => RecordedSnapshotValue::Unknown,
8362 Some(Value::Null) => RecordedSnapshotValue::Known(None),
8363 Some(value) => RecordedSnapshotValue::Known(value.as_str().map(str::to_string)),
8364 }
8365}
8366
8367fn recorded_activity_retry_snapshot(policy: Option<&Value>) -> ActivityRetrySnapshot {
8368 let policy = policy.and_then(Value::as_object);
8369 let backoff_seconds = policy
8370 .and_then(|policy| policy.get("backoff_seconds"))
8371 .and_then(Value::as_array)
8372 .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
8373 .map_or(RecordedSnapshotValue::Unknown, RecordedSnapshotValue::Known);
8374 let mut non_retryable_error_types = Vec::new();
8375 for error_type in policy
8376 .and_then(|policy| policy.get("non_retryable_error_types"))
8377 .and_then(Value::as_array)
8378 .into_iter()
8379 .flatten()
8380 .filter_map(Value::as_str)
8381 .map(str::trim)
8382 .filter(|error_type| !error_type.is_empty())
8383 {
8384 if !non_retryable_error_types
8385 .iter()
8386 .any(|recorded| recorded == error_type)
8387 {
8388 non_retryable_error_types.push(error_type.to_string());
8389 }
8390 }
8391
8392 ActivityRetrySnapshot {
8393 snapshot_version: recorded_optional_u64(policy, "snapshot_version"),
8394 max_attempts: recorded_optional_u64(policy, "max_attempts"),
8395 backoff_seconds,
8396 start_to_close_timeout: recorded_optional_u64(policy, "start_to_close_timeout"),
8397 schedule_to_start_timeout: recorded_optional_u64(policy, "schedule_to_start_timeout"),
8398 schedule_to_close_timeout: recorded_optional_u64(policy, "schedule_to_close_timeout"),
8399 heartbeat_timeout: recorded_optional_u64(policy, "heartbeat_timeout"),
8400 non_retryable_error_types: if policy
8401 .is_some_and(|policy| policy.contains_key("non_retryable_error_types"))
8402 {
8403 RecordedSnapshotValue::Known(non_retryable_error_types)
8404 } else {
8405 RecordedSnapshotValue::Unknown
8406 },
8407 }
8408}
8409
8410fn current_activity_retry_snapshot(options: &ValidatedActivityOptions) -> ActivityRetrySnapshot {
8411 let policy = options.retry_policy.as_ref();
8412 let max_attempts = match policy.and_then(|policy| policy.get("max_attempts")) {
8413 Some(Value::Null) => None,
8414 Some(value) => value_as_u64(value),
8415 None => Some(1),
8416 };
8417 let backoff_seconds = policy
8418 .and_then(|policy| policy.get("backoff_seconds"))
8419 .and_then(Value::as_array)
8420 .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
8421 .unwrap_or_default();
8422 let non_retryable_error_types = policy
8423 .and_then(|policy| policy.get("non_retryable_error_types"))
8424 .and_then(Value::as_array)
8425 .into_iter()
8426 .flatten()
8427 .filter_map(Value::as_str)
8428 .map(str::to_string)
8429 .collect();
8430
8431 ActivityRetrySnapshot {
8432 snapshot_version: RecordedSnapshotValue::Known(Some(1)),
8433 max_attempts: RecordedSnapshotValue::Known(max_attempts),
8434 backoff_seconds: RecordedSnapshotValue::Known(backoff_seconds),
8435 start_to_close_timeout: RecordedSnapshotValue::Known(options.start_to_close_timeout),
8436 schedule_to_start_timeout: RecordedSnapshotValue::Known(options.schedule_to_start_timeout),
8437 schedule_to_close_timeout: RecordedSnapshotValue::Known(options.schedule_to_close_timeout),
8438 heartbeat_timeout: RecordedSnapshotValue::Known(options.heartbeat_timeout),
8439 non_retryable_error_types: RecordedSnapshotValue::Known(non_retryable_error_types),
8440 }
8441}
8442
8443fn activity_options_description(options: &RecordedActivityOptions) -> String {
8444 serde_json::to_string(options).unwrap_or_else(|_| format!("{options:?}"))
8445}
8446
8447impl RecordedCommand {
8448 fn sequence(&self) -> u64 {
8449 match self {
8450 Self::Activity { sequence, .. }
8451 | Self::Timer { sequence, .. }
8452 | Self::ChildWorkflow { sequence, .. }
8453 | Self::SignalWait { sequence, .. }
8454 | Self::ConditionWait { sequence, .. }
8455 | Self::SearchAttributes { sequence, .. }
8456 | Self::SideEffect { sequence, .. }
8457 | Self::VersionMarker { sequence, .. }
8458 | Self::Memo { sequence, .. } => *sequence,
8459 }
8460 }
8461
8462 fn shape(&self) -> &'static str {
8463 match self {
8464 Self::Activity { .. } => "activity",
8465 Self::Timer { .. } => "timer",
8466 Self::ChildWorkflow { .. } => "child workflow",
8467 Self::SignalWait { .. } => "signal wait",
8468 Self::ConditionWait { .. } => "condition wait",
8469 Self::SearchAttributes { .. } => "search-attribute update",
8470 Self::SideEffect { .. } => "side effect",
8471 Self::VersionMarker { .. } => "version marker",
8472 Self::Memo { .. } => "memo upsert",
8473 }
8474 }
8475}
8476
8477fn ensure_version_supported(
8478 change_id: &str,
8479 version: i32,
8480 min_supported: i32,
8481 max_supported: i32,
8482 sequence: u64,
8483) -> Result<()> {
8484 if (min_supported..=max_supported).contains(&version) {
8485 return Ok(());
8486 }
8487 Err(Error::NonDeterministicReplay(ReplayFailure::new(
8488 "version_marker_incompatible_range",
8489 (sequence != 0).then_some(sequence),
8490 Some(format!("{min_supported}..={max_supported}")),
8491 Some(format!("{change_id}:{version}")),
8492 "recorded workflow version is outside the range supported by current code",
8493 )))
8494}
8495
8496#[derive(Clone, Debug)]
8497struct ResumeSignal {
8498 signal_name: String,
8499 arguments: Vec<AvroValue>,
8500}
8501
8502const MAX_PARALLEL_OPERATIONS: usize = 1000;
8503
8504fn parallel_group_prefix(kind: &str) -> &'static str {
8505 match kind {
8506 "activity" => "parallel-activities",
8507 "child" => "parallel-children",
8508 "timer" => "parallel-timers",
8509 _ => "parallel-calls",
8510 }
8511}
8512
8513fn parallel_group_entry(
8514 base_sequence: u64,
8515 size: usize,
8516 index: usize,
8517 kind: &str,
8518) -> ParallelGroupMetadata {
8519 ParallelGroupMetadata {
8520 parallel_group_id: format!("{}:{base_sequence}:{size}", parallel_group_prefix(kind)),
8521 parallel_group_kind: kind.to_string(),
8522 parallel_group_base_sequence: base_sequence,
8523 parallel_group_size: size,
8524 parallel_group_index: index,
8525 }
8526}
8527
8528fn apply_parallel_group_path(
8529 command: &mut serde_json::Map<String, Value>,
8530 path: &[ParallelGroupMetadata],
8531) {
8532 let Some(inner) = path.last() else {
8533 return;
8534 };
8535 command.insert(
8536 "parallel_group_id".to_string(),
8537 json!(inner.parallel_group_id),
8538 );
8539 command.insert(
8540 "parallel_group_kind".to_string(),
8541 json!(inner.parallel_group_kind),
8542 );
8543 command.insert(
8544 "parallel_group_base_sequence".to_string(),
8545 json!(inner.parallel_group_base_sequence),
8546 );
8547 command.insert(
8548 "parallel_group_size".to_string(),
8549 json!(inner.parallel_group_size),
8550 );
8551 command.insert(
8552 "parallel_group_index".to_string(),
8553 json!(inner.parallel_group_index),
8554 );
8555 command.insert("parallel_group_path".to_string(), json!(path));
8556}
8557
8558fn ensure_parallel_path_matches(
8559 sequence: u64,
8560 recorded: Option<&[ParallelGroupMetadata]>,
8561 expected: &[ParallelGroupMetadata],
8562) -> Result<()> {
8563 match (recorded, expected.is_empty()) {
8564 (None, true) => Ok(()),
8565 (Some(recorded), false) if recorded == expected => Ok(()),
8566 (None, false) => Err(invalid_recorded_history(
8567 "parallel_group_metadata_missing",
8568 sequence,
8569 &serde_json::to_string(expected).unwrap_or_default(),
8570 "<missing>",
8571 "recorded parallel member is missing its durable group path",
8572 )),
8573 (Some(recorded), true) => Err(invalid_recorded_history(
8574 "parallel_group_shape_mismatch",
8575 sequence,
8576 "sequential command",
8577 &serde_json::to_string(recorded).unwrap_or_default(),
8578 "recorded command belonged to a parallel group but current code schedules it sequentially",
8579 )),
8580 (Some(recorded), false) => Err(invalid_recorded_history(
8581 "parallel_group_shape_mismatch",
8582 sequence,
8583 &serde_json::to_string(recorded).unwrap_or_default(),
8584 &serde_json::to_string(expected).unwrap_or_default(),
8585 "recorded parallel-group identity or path changed during replay",
8586 )),
8587 }
8588}
8589
8590#[derive(Clone, Debug)]
8591enum ParallelShape {
8592 Leaf,
8593 Group(Vec<ParallelShape>),
8594}
8595
8596struct ParallelDescriptor {
8597 operation: ParallelOperation,
8598 offset: usize,
8599 member_path: Vec<usize>,
8600 group_path: Vec<ParallelGroupMetadata>,
8601}
8602
8603fn parallel_leaf_count(operations: &[ParallelOperation]) -> usize {
8604 operations
8605 .iter()
8606 .map(|operation| match operation {
8607 ParallelOperation::Group(children) => parallel_leaf_count(children),
8608 _ => 1,
8609 })
8610 .sum()
8611}
8612
8613fn parallel_operation_kind(operation: &ParallelOperation) -> Option<&'static str> {
8614 match operation {
8615 ParallelOperation::Activity { .. } => Some("activity"),
8616 ParallelOperation::ChildWorkflow { .. } => Some("child"),
8617 ParallelOperation::Timer(_) => Some("timer"),
8618 ParallelOperation::Group(children) => parallel_group_kind(children),
8619 }
8620}
8621
8622fn parallel_group_kind(operations: &[ParallelOperation]) -> Option<&'static str> {
8623 let mut kind = None;
8624 for operation in operations {
8625 let Some(operation_kind) = parallel_operation_kind(operation) else {
8626 continue;
8627 };
8628 match kind {
8629 None => kind = Some(operation_kind),
8630 Some(current) if current == operation_kind => {}
8631 Some(_) => return Some("mixed"),
8632 }
8633 }
8634 kind
8635}
8636
8637fn validate_parallel_operations(
8638 operations: &[ParallelOperation],
8639 member_path: &mut Vec<usize>,
8640 root: bool,
8641) -> Result<()> {
8642 let leaves = parallel_leaf_count(operations);
8643 if leaves > MAX_PARALLEL_OPERATIONS {
8644 return Err(Error::InvalidParallelGroup(ParallelGroupError {
8645 reason: "fan_out_limit_exceeded",
8646 member_path: member_path.clone(),
8647 message: format!(
8648 "group contains {leaves} durable leaves; the limit is {MAX_PARALLEL_OPERATIONS}"
8649 ),
8650 }));
8651 }
8652 if !root && operations.is_empty() {
8653 return Err(Error::InvalidParallelGroup(ParallelGroupError {
8654 reason: "nested_group_empty",
8655 member_path: member_path.clone(),
8656 message: "a nested group must contain at least one durable leaf".to_string(),
8657 }));
8658 }
8659
8660 for (index, operation) in operations.iter().enumerate() {
8661 member_path.push(index);
8662 match operation {
8663 ParallelOperation::Activity {
8664 options, arguments, ..
8665 } => {
8666 options
8667 .validate()
8668 .map_err(|error| Error::InvalidActivityOptions(error))?;
8669 if let Err(error) = arguments {
8670 return Err(Error::InvalidParallelGroup(ParallelGroupError {
8671 reason: "arguments_invalid",
8672 member_path: member_path.clone(),
8673 message: error.to_string(),
8674 }));
8675 }
8676 }
8677 ParallelOperation::ChildWorkflow {
8678 options, arguments, ..
8679 } => {
8680 validate_parallel_child_options(options)?;
8681 if let Err(error) = arguments {
8682 return Err(Error::InvalidParallelGroup(ParallelGroupError {
8683 reason: "arguments_invalid",
8684 member_path: member_path.clone(),
8685 message: error.to_string(),
8686 }));
8687 }
8688 }
8689 ParallelOperation::Timer(duration)
8690 if duration.as_secs() == u64::MAX && duration.subsec_nanos() > 0 =>
8691 {
8692 return Err(Error::TimerDurationOverflow);
8693 }
8694 ParallelOperation::Timer(_) => {}
8695 ParallelOperation::Group(children) => {
8696 validate_parallel_operations(children, member_path, false)?;
8697 }
8698 }
8699 member_path.pop();
8700 }
8701 Ok(())
8702}
8703
8704fn validate_parallel_child_options(options: &ChildWorkflowOptions) -> Result<()> {
8705 if options.task_queue.trim().is_empty() {
8706 return Err(Error::InvalidChildWorkflowOptions(
8707 "task_queue must not be empty".to_string(),
8708 ));
8709 }
8710 for (name, value) in [
8711 (
8712 "execution_timeout_seconds",
8713 options.execution_timeout_seconds,
8714 ),
8715 ("run_timeout_seconds", options.run_timeout_seconds),
8716 ] {
8717 if value == Some(0) {
8718 return Err(Error::InvalidChildWorkflowOptions(format!(
8719 "{name} must be at least 1"
8720 )));
8721 }
8722 }
8723 if options
8724 .retry_policy
8725 .as_ref()
8726 .is_some_and(|policy| policy.max_attempts == Some(0))
8727 {
8728 return Err(Error::InvalidChildWorkflowOptions(
8729 "retry_policy.max_attempts must be at least 1".to_string(),
8730 ));
8731 }
8732 Ok(())
8733}
8734
8735fn parallel_shape(operations: &[ParallelOperation]) -> ParallelShape {
8736 ParallelShape::Group(
8737 operations
8738 .iter()
8739 .map(|operation| match operation {
8740 ParallelOperation::Group(children) => parallel_shape(children),
8741 _ => ParallelShape::Leaf,
8742 })
8743 .collect(),
8744 )
8745}
8746
8747fn parallel_descriptors(
8748 operations: Vec<ParallelOperation>,
8749 base_sequence: u64,
8750) -> Result<Vec<ParallelDescriptor>> {
8751 let size = parallel_leaf_count(&operations);
8752 let kind = parallel_group_kind(&operations).unwrap_or("activity");
8753 let mut descriptors = Vec::with_capacity(size);
8754 let mut cursor = 0;
8755
8756 for (index, operation) in operations.into_iter().enumerate() {
8757 match operation {
8758 ParallelOperation::Group(children) => {
8759 let child_base = base_sequence
8760 .checked_add(u64::try_from(cursor).unwrap_or(u64::MAX))
8761 .ok_or(Error::TimerDurationOverflow)?;
8762 for mut descriptor in parallel_descriptors(children, child_base)? {
8763 let outer_index = cursor + descriptor.offset;
8764 descriptor.group_path.insert(
8765 0,
8766 parallel_group_entry(base_sequence, size, outer_index, kind),
8767 );
8768 descriptor.member_path.insert(0, index);
8769 descriptor.offset = outer_index;
8770 descriptors.push(descriptor);
8771 }
8772 cursor = descriptors.len();
8773 }
8774 operation => {
8775 descriptors.push(ParallelDescriptor {
8776 operation,
8777 offset: cursor,
8778 member_path: vec![index],
8779 group_path: vec![parallel_group_entry(base_sequence, size, cursor, kind)],
8780 });
8781 cursor += 1;
8782 }
8783 }
8784 }
8785 Ok(descriptors)
8786}
8787
8788enum ParallelLeafCall {
8789 Activity(ActivityCall),
8790 ChildWorkflow(ChildWorkflowCall),
8791 Timer(TimerCall),
8792}
8793
8794impl ParallelLeafCall {
8795 fn poll_avro_value(&mut self, cx: &mut TaskContext<'_>) -> Poll<Result<ParallelAvroResult>> {
8796 match self {
8797 Self::Activity(call) => Pin::new(call)
8798 .poll_avro_value(cx)
8799 .map_ok(ParallelAvroResult::Activity),
8800 Self::ChildWorkflow(call) => Pin::new(call)
8801 .poll_avro_value(cx)
8802 .map_ok(ParallelAvroResult::ChildWorkflow),
8803 Self::Timer(call) => Pin::new(call)
8804 .poll(cx)
8805 .map_ok(|()| ParallelAvroResult::Timer),
8806 }
8807 }
8808}
8809
8810struct ParallelLeaf {
8811 call: ParallelLeafCall,
8812 member_path: Vec<usize>,
8813 group_path: Vec<ParallelGroupMetadata>,
8814 result: Option<ParallelAvroResult>,
8815}
8816
8817pub struct ParallelCall {
8819 ctx: WorkflowContext,
8820 operations: Option<Vec<ParallelOperation>>,
8821 shape: Option<ParallelShape>,
8822 leaves: Vec<ParallelLeaf>,
8823}
8824
8825impl ParallelCall {
8826 fn new(ctx: WorkflowContext, operations: Vec<ParallelOperation>) -> Self {
8827 Self {
8828 ctx,
8829 operations: Some(operations),
8830 shape: None,
8831 leaves: Vec::new(),
8832 }
8833 }
8834
8835 fn initialize(&mut self) -> Result<()> {
8836 let operations = self.operations.take().unwrap_or_default();
8837 validate_parallel_operations(&operations, &mut Vec::new(), true)?;
8838 self.shape = Some(parallel_shape(&operations));
8839 if operations.is_empty() {
8840 return Ok(());
8841 }
8842
8843 let base_sequence = {
8844 let state = self
8845 .ctx
8846 .state
8847 .lock()
8848 .map_err(|_| Error::WorkflowStatePoisoned)?;
8849 if let Some(recorded) = state.recorded_commands.get(state.command_cursor) {
8850 recorded.sequence()
8851 } else {
8852 let last = state
8853 .recorded_commands
8854 .last()
8855 .map(RecordedCommand::sequence)
8856 .unwrap_or(0);
8857 last.checked_add(u64::try_from(state.commands.len()).unwrap_or(u64::MAX))
8858 .and_then(|sequence| sequence.checked_add(1))
8859 .ok_or_else(|| {
8860 Error::InvalidParallelGroup(ParallelGroupError {
8861 reason: "sequence_overflow",
8862 member_path: Vec::new(),
8863 message: "parallel group sequence identity overflowed u64".to_string(),
8864 })
8865 })?
8866 }
8867 };
8868
8869 self.leaves = parallel_descriptors(operations, base_sequence)?
8870 .into_iter()
8871 .map(|descriptor| {
8872 let path = descriptor.group_path.clone();
8873 let call = match descriptor.operation {
8874 ParallelOperation::Activity {
8875 activity_type,
8876 options,
8877 arguments,
8878 } => ParallelLeafCall::Activity(ActivityCall {
8879 ctx: self.ctx.clone(),
8880 activity_type,
8881 options,
8882 args: Some(arguments),
8883 scheduled: false,
8884 parallel_group_path: path,
8885 }),
8886 ParallelOperation::ChildWorkflow {
8887 workflow_type,
8888 options,
8889 arguments,
8890 } => ParallelLeafCall::ChildWorkflow(ChildWorkflowCall {
8891 ctx: self.ctx.clone(),
8892 workflow_type,
8893 options,
8894 args: Some(arguments),
8895 scheduled: false,
8896 matched_pending: false,
8897 parallel_group_path: path,
8898 }),
8899 ParallelOperation::Timer(duration) => {
8900 let delay_seconds = duration
8901 .as_secs()
8902 .checked_add(u64::from(duration.subsec_nanos() > 0));
8903 ParallelLeafCall::Timer(TimerCall {
8904 ctx: self.ctx.clone(),
8905 delay_seconds,
8906 scheduled: false,
8907 matched_pending: false,
8908 parallel_group_path: path,
8909 })
8910 }
8911 ParallelOperation::Group(_) => {
8912 unreachable!("parallel descriptors contain only durable leaves")
8913 }
8914 };
8915 ParallelLeaf {
8916 call,
8917 member_path: descriptor.member_path,
8918 group_path: descriptor.group_path,
8919 result: None,
8920 }
8921 })
8922 .collect();
8923 Ok(())
8924 }
8925
8926 fn poll_avro_value(
8927 mut self: Pin<&mut Self>,
8928 cx: &mut TaskContext<'_>,
8929 ) -> Poll<Result<Vec<ParallelAvroResult>>> {
8930 if self.operations.is_some() {
8931 if let Err(error) = self.initialize() {
8932 return Poll::Ready(Err(error));
8933 }
8934 }
8935 if self.leaves.is_empty() {
8936 return Poll::Ready(Ok(Vec::new()));
8937 }
8938
8939 let mut failures = Vec::new();
8940 let mut pending = false;
8941 for (index, leaf) in self.leaves.iter_mut().enumerate() {
8942 if leaf.result.is_some() {
8943 continue;
8944 }
8945 match leaf.call.poll_avro_value(cx) {
8946 Poll::Ready(Ok(result)) => leaf.result = Some(result),
8947 Poll::Ready(Err(error)) => failures.push((index, error)),
8948 Poll::Pending => pending = true,
8949 }
8950 }
8951
8952 if !failures.is_empty() {
8953 if let Some(position) = failures
8954 .iter()
8955 .position(|(_, error)| workflow_task_integrity_error(error))
8956 {
8957 return Poll::Ready(Err(failures.remove(position).1));
8958 }
8959 failures.sort_by_key(|(index, _)| *index);
8960 let (failed_index, cause) = failures.remove(0);
8961 let failed = &self.leaves[failed_index];
8962 let completed = self
8963 .leaves
8964 .iter()
8965 .filter_map(|leaf| {
8966 leaf.result
8967 .clone()
8968 .and_then(|result| result.into_json_result().ok())
8969 .map(|result| ParallelCompletion {
8970 member_path: leaf.member_path.clone(),
8971 result,
8972 })
8973 })
8974 .collect();
8975 let group_id = failed
8976 .group_path
8977 .first()
8978 .map(|entry| entry.parallel_group_id.clone())
8979 .unwrap_or_default();
8980 return Poll::Ready(Err(Error::ParallelFailed(ParallelFailure {
8981 group_id,
8982 member_path: failed.member_path.clone(),
8983 group_path: failed.group_path.clone(),
8984 completed,
8985 cause: Box::new(cause),
8986 })));
8987 }
8988 if pending {
8989 return Poll::Pending;
8990 }
8991
8992 let mut flat_results = self
8993 .leaves
8994 .iter_mut()
8995 .map(|leaf| leaf.result.take().expect("completed parallel leaf"))
8996 .collect::<Vec<_>>()
8997 .into_iter();
8998 let results = parallel_results_for_shape(
8999 self.shape.as_ref().expect("initialized parallel shape"),
9000 &mut flat_results,
9001 );
9002 Poll::Ready(Ok(match results {
9003 ParallelAvroResult::Group(results) => results,
9004 ParallelAvroResult::Activity(_)
9005 | ParallelAvroResult::ChildWorkflow(_)
9006 | ParallelAvroResult::Timer => unreachable!("root parallel shape is a group"),
9007 }))
9008 }
9009}
9010
9011fn parallel_results_for_shape(
9012 shape: &ParallelShape,
9013 flat_results: &mut impl Iterator<Item = ParallelAvroResult>,
9014) -> ParallelAvroResult {
9015 match shape {
9016 ParallelShape::Leaf => flat_results.next().expect("one result per parallel leaf"),
9017 ParallelShape::Group(children) => ParallelAvroResult::Group(
9018 children
9019 .iter()
9020 .map(|child| parallel_results_for_shape(child, flat_results))
9021 .collect(),
9022 ),
9023 }
9024}
9025
9026impl Future for ParallelCall {
9027 type Output = Result<Vec<ParallelResult>>;
9028
9029 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
9030 self.poll_avro_value(cx)
9031 .map_ok(|results| {
9032 results
9033 .into_iter()
9034 .map(ParallelAvroResult::into_json_result)
9035 .collect::<Result<Vec<_>>>()
9036 })
9037 .map_ok(|result| result)
9038 .flatten_result()
9039 }
9040}
9041
9042trait PollNestedResultExt<T> {
9043 fn flatten_result(self) -> Poll<Result<T>>;
9044}
9045
9046impl<T> PollNestedResultExt<T> for Poll<Result<Result<T>>> {
9047 fn flatten_result(self) -> Poll<Result<T>> {
9048 match self {
9049 Poll::Ready(Ok(result)) => Poll::Ready(result),
9050 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
9051 Poll::Pending => Poll::Pending,
9052 }
9053 }
9054}
9055
9056struct SagaCompensation {
9057 activity_type: String,
9058 options: ActivityOptions,
9059 arguments: AvroValue,
9060 registration_order: usize,
9061}
9062
9063pub struct Saga {
9071 ctx: WorkflowContext,
9072 compensations: Vec<SagaCompensation>,
9073}
9074
9075impl Saga {
9076 fn new(ctx: WorkflowContext) -> Self {
9077 Self {
9078 ctx,
9079 compensations: Vec::new(),
9080 }
9081 }
9082
9083 pub fn add_compensation<T: Serialize>(
9084 &mut self,
9085 activity_type: impl Into<String>,
9086 args: T,
9087 ) -> Result<&mut Self> {
9088 self.add_compensation_with_options(activity_type, ActivityOptions::new(), args)
9089 }
9090
9091 pub fn add_compensation_with_options<T: Serialize>(
9092 &mut self,
9093 activity_type: impl Into<String>,
9094 options: ActivityOptions,
9095 args: T,
9096 ) -> Result<&mut Self> {
9097 let activity_type = activity_type.into();
9098 if activity_type.trim().is_empty() || activity_type.trim() != activity_type {
9099 return Err(Error::Codec(
9100 "saga compensation activity type must be non-empty without surrounding whitespace"
9101 .to_string(),
9102 ));
9103 }
9104 options.validate().map_err(Error::InvalidActivityOptions)?;
9105 let arguments = AvroValue::from_serialize(&args)?;
9106 let registration_order = self.compensations.len() + 1;
9107 self.compensations.push(SagaCompensation {
9108 activity_type,
9109 options,
9110 arguments,
9111 registration_order,
9112 });
9113 Ok(self)
9114 }
9115
9116 pub async fn compensate(mut self, initiating_failure: Error) -> Error {
9118 while let Some(compensation) = self.compensations.pop() {
9119 if let Err(compensation_failure) = self
9120 .ctx
9121 .activity_with_options(
9122 compensation.activity_type.clone(),
9123 compensation.options,
9124 compensation.arguments,
9125 )
9126 .await
9127 {
9128 if workflow_task_integrity_error(&compensation_failure) {
9129 return compensation_failure;
9130 }
9131 return Error::SagaCompensationFailed(SagaCompensationFailure {
9132 initiating_failure: Box::new(initiating_failure),
9133 compensation_failure: Box::new(compensation_failure),
9134 compensation_activity_type: compensation.activity_type,
9135 compensation_registration_order: compensation.registration_order,
9136 });
9137 }
9138 }
9139 initiating_failure
9140 }
9141
9142 pub async fn finish<T>(self, outcome: Result<T>) -> Result<T> {
9144 match outcome {
9145 Ok(value) => Ok(value),
9146 Err(error) => Err(self.compensate(error).await),
9147 }
9148 }
9149}
9150
9151pub struct ActivityCall {
9152 ctx: WorkflowContext,
9153 activity_type: String,
9154 options: ActivityOptions,
9155 args: Option<Result<AvroValue>>,
9156 scheduled: bool,
9157 parallel_group_path: Vec<ParallelGroupMetadata>,
9158}
9159
9160impl ActivityCall {
9161 fn poll_avro_value(
9162 mut self: Pin<&mut Self>,
9163 _cx: &mut TaskContext<'_>,
9164 ) -> Poll<Result<AvroValue>> {
9165 let ctx = self.ctx.clone();
9166 let mut state = match ctx.state.lock() {
9167 Ok(state) => state,
9168 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
9169 };
9170
9171 if self.scheduled {
9172 return Poll::Pending;
9173 }
9174
9175 let options = match self.options.validate() {
9176 Ok(options) => options,
9177 Err(error) => {
9178 return Poll::Ready(Err(Error::InvalidActivityOptions(error)));
9179 }
9180 };
9181 let task_queue = options
9182 .task_queue
9183 .clone()
9184 .unwrap_or_else(|| state.task_queue.clone());
9185 let current_recorded_options = RecordedActivityOptions {
9186 task_queue: RecordedSnapshotValue::Known(Some(task_queue.clone())),
9187 execution_mode: RecordedSnapshotValue::Known(None),
9190 retry_policy: current_activity_retry_snapshot(&options),
9191 };
9192
9193 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
9194 let sequence = recorded.sequence();
9195 match recorded {
9196 RecordedCommand::Activity {
9197 activity_type,
9198 options: recorded_options,
9199 outcome,
9200 parallel_group_path,
9201 ..
9202 } => {
9203 if let Err(error) = ensure_parallel_path_matches(
9204 sequence,
9205 parallel_group_path.as_deref(),
9206 &self.parallel_group_path,
9207 ) {
9208 return Poll::Ready(Err(error));
9209 }
9210 if let Some(recorded_type) = activity_type {
9211 if recorded_type != self.activity_type {
9212 return Poll::Ready(Err(Error::NonDeterministicReplay(
9213 ReplayFailure::new(
9214 "recorded_command_detail_mismatch",
9215 Some(sequence),
9216 Some(format!("activity:{recorded_type}")),
9217 Some(format!("activity:{}", self.activity_type)),
9218 "recorded activity type differs from the current workflow command",
9219 ),
9220 )));
9221 }
9222 }
9223 if let Some(recorded_options) = recorded_options {
9224 if !recorded_options
9225 .task_queue
9226 .matches_current(¤t_recorded_options.task_queue)
9227 {
9228 return Poll::Ready(Err(Error::NonDeterministicReplay(
9229 ReplayFailure::new(
9230 "activity_task_queue_mismatch",
9231 Some(sequence),
9232 Some(activity_options_description(&recorded_options)),
9233 Some(activity_options_description(¤t_recorded_options)),
9234 "recorded activity task queue differs from the current workflow command",
9235 ),
9236 )));
9237 }
9238 if !recorded_options
9239 .execution_mode
9240 .matches_current(¤t_recorded_options.execution_mode)
9241 {
9242 return Poll::Ready(Err(Error::NonDeterministicReplay(
9243 ReplayFailure::new(
9244 "activity_execution_mode_mismatch",
9245 Some(sequence),
9246 Some(activity_options_description(&recorded_options)),
9247 Some(activity_options_description(¤t_recorded_options)),
9248 "recorded activity execution mode differs from the current workflow command",
9249 ),
9250 )));
9251 }
9252 if !recorded_options
9253 .retry_policy
9254 .matches_current(¤t_recorded_options.retry_policy)
9255 {
9256 return Poll::Ready(Err(Error::NonDeterministicReplay(
9257 ReplayFailure::new(
9258 "activity_retry_policy_mismatch",
9259 Some(sequence),
9260 Some(activity_options_description(&recorded_options)),
9261 Some(activity_options_description(¤t_recorded_options)),
9262 "recorded activity retry policy differs from the current workflow command",
9263 ),
9264 )));
9265 }
9266 }
9267 state.command_cursor += 1;
9268 if let Some(outcome) = outcome {
9269 return Poll::Ready(outcome.map_err(Error::ActivityFailed));
9270 }
9271 state.matched_recorded_pending = true;
9272 self.scheduled = true;
9273 return Poll::Pending;
9274 }
9275 other => {
9276 return Poll::Ready(Err(command_mismatch(
9277 &other,
9278 format!("activity:{}", self.activity_type),
9279 )));
9280 }
9281 }
9282 }
9283
9284 if !self.scheduled {
9285 let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
9286 Ok(args) => args,
9287 Err(error) => return Poll::Ready(Err(error)),
9288 };
9289 let arguments = normalize_avro_arguments(args);
9290 let envelope = match encode_typed_envelope(&arguments, &state.payload_codec) {
9291 Ok(envelope) => envelope,
9292 Err(error) => return Poll::Ready(Err(error)),
9293 };
9294
9295 let mut command = serde_json::Map::from_iter([
9296 ("type".to_string(), json!("schedule_activity")),
9297 (
9298 "activity_type".to_string(),
9299 json!(self.activity_type.clone()),
9300 ),
9301 ("queue".to_string(), json!(task_queue)),
9302 ("arguments".to_string(), envelope),
9303 ]);
9304 for (field, value) in [
9305 ("start_to_close_timeout", options.start_to_close_timeout),
9306 (
9307 "schedule_to_start_timeout",
9308 options.schedule_to_start_timeout,
9309 ),
9310 (
9311 "schedule_to_close_timeout",
9312 options.schedule_to_close_timeout,
9313 ),
9314 ("heartbeat_timeout", options.heartbeat_timeout),
9315 ] {
9316 if let Some(value) = value {
9317 command.insert(field.to_string(), json!(value));
9318 }
9319 }
9320 if let Some(retry_policy) = options.retry_policy {
9321 command.insert("retry_policy".to_string(), retry_policy);
9322 }
9323 apply_parallel_group_path(&mut command, &self.parallel_group_path);
9324 state.commands.push(Value::Object(command));
9325 self.scheduled = true;
9326 }
9327
9328 Poll::Pending
9329 }
9330}
9331
9332impl Future for ActivityCall {
9333 type Output = Result<Value>;
9334
9335 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
9336 match self.poll_avro_value(cx) {
9337 Poll::Ready(Ok(value)) => Poll::Ready(value.into_json()),
9338 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
9339 Poll::Pending => Poll::Pending,
9340 }
9341 }
9342}
9343
9344pub struct TimerCall {
9346 ctx: WorkflowContext,
9347 delay_seconds: Option<u64>,
9348 scheduled: bool,
9349 matched_pending: bool,
9350 parallel_group_path: Vec<ParallelGroupMetadata>,
9351}
9352
9353impl Future for TimerCall {
9354 type Output = Result<()>;
9355
9356 fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
9357 if self.matched_pending {
9358 return Poll::Pending;
9359 }
9360
9361 let ctx = self.ctx.clone();
9362 let Some(requested_delay) = self.delay_seconds else {
9363 return Poll::Ready(Err(Error::TimerDurationOverflow));
9364 };
9365 let mut state = match ctx.state.lock() {
9366 Ok(state) => state,
9367 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
9368 };
9369
9370 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
9371 match recorded {
9372 RecordedCommand::Timer {
9373 sequence,
9374 delay_seconds,
9375 fired,
9376 parallel_group_path,
9377 ..
9378 } => {
9379 if let Err(error) = ensure_parallel_path_matches(
9380 sequence,
9381 parallel_group_path.as_deref(),
9382 &self.parallel_group_path,
9383 ) {
9384 return Poll::Ready(Err(error));
9385 }
9386 if delay_seconds != requested_delay {
9387 return Poll::Ready(Err(Error::NonDeterministicReplay(
9388 ReplayFailure::new(
9389 "timer_delay_mismatch",
9390 Some(sequence),
9391 Some(format!("timer:{delay_seconds}s")),
9392 Some(format!("timer:{requested_delay}s")),
9393 "recorded timer delay differs from the current workflow command",
9394 ),
9395 )));
9396 }
9397 state.command_cursor += 1;
9398 if fired {
9399 return Poll::Ready(Ok(()));
9400 }
9401 state.matched_recorded_pending = true;
9402 self.scheduled = true;
9403 self.matched_pending = true;
9404 return Poll::Pending;
9405 }
9406 other => return Poll::Ready(Err(command_mismatch(&other, "timer"))),
9407 }
9408 }
9409
9410 if !self.scheduled {
9411 let mut command = serde_json::Map::from_iter([
9412 ("type".to_string(), json!("start_timer")),
9413 ("delay_seconds".to_string(), json!(requested_delay)),
9414 ]);
9415 apply_parallel_group_path(&mut command, &self.parallel_group_path);
9416 state.commands.push(Value::Object(command));
9417 self.scheduled = true;
9418 }
9419
9420 Poll::Pending
9421 }
9422}
9423
9424pub struct ConditionWaitCall {
9426 ctx: WorkflowContext,
9427 options: ConditionWaitOptions,
9428 predicate: Box<dyn Fn() -> Result<bool> + Send + 'static>,
9429 opened_wait: bool,
9430}
9431
9432impl Future for ConditionWaitCall {
9433 type Output = Result<ConditionWaitResult>;
9434
9435 fn poll(self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
9436 if self.opened_wait {
9437 return Poll::Pending;
9438 }
9439
9440 let options = match self.options.validate() {
9441 Ok(options) => options,
9442 Err(error) => return Poll::Ready(Err(Error::InvalidConditionWaitOptions(error))),
9443 };
9444 let ctx = self.ctx.clone();
9445
9446 let recorded_result = {
9447 let mut state = match ctx.state.lock() {
9448 Ok(state) => state,
9449 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
9450 };
9451 let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() else {
9452 drop(state);
9453 return self.poll_new_condition(options);
9454 };
9455 if !matches!(recorded, RecordedCommand::ConditionWait { .. }) {
9456 return Poll::Ready(Err(command_mismatch(&recorded, "condition wait")));
9457 }
9458
9459 let mut cursor = state.command_cursor;
9460 let mut result = None;
9461 loop {
9462 let Some(RecordedCommand::ConditionWait {
9463 sequence,
9464 condition_key,
9465 predicate_identity,
9466 timeout_seconds,
9467 result: recorded_result,
9468 ..
9469 }) = state.recorded_commands.get(cursor)
9470 else {
9471 break;
9472 };
9473
9474 if cursor > state.command_cursor
9475 && !same_logical_condition_wait(
9476 condition_key.as_deref(),
9477 predicate_identity,
9478 &options,
9479 )
9480 {
9481 break;
9482 }
9483 if let Err(error) = validate_recorded_condition_wait(
9484 *sequence,
9485 condition_key.as_deref(),
9486 predicate_identity,
9487 *timeout_seconds,
9488 &options,
9489 ) {
9490 return Poll::Ready(Err(error));
9491 }
9492 if result == Some(ConditionWaitResult::TimedOut) {
9493 return Poll::Ready(Err(Error::NonDeterministicReplay(ReplayFailure::new(
9494 "condition_wait_reopened_after_timeout",
9495 Some(*sequence),
9496 Some("timed-out condition is terminal".to_string()),
9497 Some("another physical wait-open".to_string()),
9498 "condition history reopened one logical wait after its durable timeout",
9499 ))));
9500 }
9501 result = *recorded_result;
9502 cursor += 1;
9503 }
9504 state.command_cursor = cursor;
9505 result
9506 };
9507
9508 if let Some(result) = recorded_result {
9509 return Poll::Ready(Ok(result));
9510 }
9511
9512 self.poll_open_condition(options)
9513 }
9514}
9515
9516impl ConditionWaitCall {
9517 fn poll_new_condition(
9518 self: Pin<&mut Self>,
9519 options: ValidatedConditionWaitOptions,
9520 ) -> Poll<Result<ConditionWaitResult>> {
9521 self.poll_open_condition(options)
9522 }
9523
9524 fn poll_open_condition(
9525 mut self: Pin<&mut Self>,
9526 options: ValidatedConditionWaitOptions,
9527 ) -> Poll<Result<ConditionWaitResult>> {
9528 match (self.predicate)() {
9529 Ok(true) => return Poll::Ready(Ok(ConditionWaitResult::Satisfied)),
9530 Ok(false) => {}
9531 Err(error) => return Poll::Ready(Err(error)),
9532 }
9533 if options.timeout_seconds == Some(0) {
9534 return Poll::Ready(Ok(ConditionWaitResult::TimedOut));
9535 }
9536
9537 let ctx = self.ctx.clone();
9538 let mut state = match ctx.state.lock() {
9539 Ok(state) => state,
9540 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
9541 };
9542 let mut command = serde_json::Map::from_iter([
9543 ("type".to_string(), json!("open_condition_wait")),
9544 ("condition_key".to_string(), json!(options.condition_key)),
9545 (
9546 "condition_definition_fingerprint".to_string(),
9547 json!(options.predicate_identity),
9548 ),
9549 ]);
9550 if let Some(timeout_seconds) = options.timeout_seconds {
9551 command.insert("timeout_seconds".to_string(), json!(timeout_seconds));
9552 }
9553 state.commands.push(Value::Object(command));
9554 drop(state);
9555 self.opened_wait = true;
9556 Poll::Pending
9557 }
9558}
9559
9560fn same_logical_condition_wait(
9561 recorded_key: Option<&str>,
9562 recorded_predicate_identity: &str,
9563 current: &ValidatedConditionWaitOptions,
9564) -> bool {
9565 recorded_key == Some(current.condition_key.as_str())
9566 || recorded_predicate_identity == current.predicate_identity
9567}
9568
9569fn validate_recorded_condition_wait(
9570 sequence: u64,
9571 recorded_key: Option<&str>,
9572 recorded_predicate_identity: &str,
9573 recorded_timeout_seconds: Option<u64>,
9574 current: &ValidatedConditionWaitOptions,
9575) -> Result<()> {
9576 if recorded_key != Some(current.condition_key.as_str()) {
9577 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
9578 "condition_wait_key_mismatch",
9579 Some(sequence),
9580 recorded_key.map(str::to_string),
9581 Some(current.condition_key.clone()),
9582 "recorded condition identity differs from the current workflow wait",
9583 )));
9584 }
9585 if recorded_predicate_identity != current.predicate_identity {
9586 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
9587 "condition_wait_predicate_mismatch",
9588 Some(sequence),
9589 Some(recorded_predicate_identity.to_string()),
9590 Some(current.predicate_identity.clone()),
9591 "recorded condition predicate behavior differs from current workflow code",
9592 )));
9593 }
9594 if recorded_timeout_seconds != current.timeout_seconds {
9595 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
9596 "condition_wait_timeout_mismatch",
9597 Some(sequence),
9598 recorded_timeout_seconds.map(|seconds| format!("{seconds}s")),
9599 current.timeout_seconds.map(|seconds| format!("{seconds}s")),
9600 "recorded condition timeout differs from the current workflow wait",
9601 )));
9602 }
9603 Ok(())
9604}
9605
9606pub struct ChildWorkflowCall {
9608 ctx: WorkflowContext,
9609 workflow_type: String,
9610 options: ChildWorkflowOptions,
9611 args: Option<Result<AvroValue>>,
9612 scheduled: bool,
9613 matched_pending: bool,
9614 parallel_group_path: Vec<ParallelGroupMetadata>,
9615}
9616
9617impl ChildWorkflowCall {
9618 fn poll_avro_value(
9619 mut self: Pin<&mut Self>,
9620 _cx: &mut TaskContext<'_>,
9621 ) -> Poll<Result<ChildWorkflowAvroResult>> {
9622 if self.matched_pending {
9623 return Poll::Pending;
9624 }
9625
9626 let ctx = self.ctx.clone();
9627 let mut state = match ctx.state.lock() {
9628 Ok(state) => state,
9629 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
9630 };
9631
9632 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
9633 let sequence = recorded.sequence();
9634 match recorded {
9635 RecordedCommand::ChildWorkflow {
9636 workflow_type,
9637 outcome,
9638 parallel_group_path,
9639 ..
9640 } => {
9641 if let Err(error) = ensure_parallel_path_matches(
9642 sequence,
9643 parallel_group_path.as_deref(),
9644 &self.parallel_group_path,
9645 ) {
9646 return Poll::Ready(Err(error));
9647 }
9648 if let Some(recorded_type) = workflow_type {
9649 if recorded_type != self.workflow_type {
9650 return Poll::Ready(Err(Error::NonDeterministicReplay(
9651 ReplayFailure::new(
9652 "recorded_command_detail_mismatch",
9653 Some(sequence),
9654 Some(format!("child workflow:{recorded_type}")),
9655 Some(format!("child workflow:{}", self.workflow_type)),
9656 "recorded child workflow type differs from the current workflow command",
9657 ),
9658 )));
9659 }
9660 }
9661 state.command_cursor += 1;
9662 if let Some(outcome) = outcome {
9663 return Poll::Ready(outcome.map_err(Error::ChildWorkflowFailed));
9664 }
9665 state.matched_recorded_pending = true;
9666 self.scheduled = true;
9667 self.matched_pending = true;
9668 return Poll::Pending;
9669 }
9670 other => {
9671 return Poll::Ready(Err(command_mismatch(
9672 &other,
9673 format!("child workflow:{}", self.workflow_type),
9674 )));
9675 }
9676 }
9677 }
9678
9679 if !self.scheduled {
9680 if self.options.task_queue.trim().is_empty() {
9681 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
9682 "task_queue must not be empty".to_string(),
9683 )));
9684 }
9685 for (name, value) in [
9686 (
9687 "execution_timeout_seconds",
9688 self.options.execution_timeout_seconds,
9689 ),
9690 ("run_timeout_seconds", self.options.run_timeout_seconds),
9691 ] {
9692 if value == Some(0) {
9693 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(format!(
9694 "{name} must be at least 1"
9695 ))));
9696 }
9697 }
9698
9699 let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
9700 Ok(args) => args,
9701 Err(error) => return Poll::Ready(Err(error)),
9702 };
9703 let arguments = match encode_typed_envelope(
9704 &normalize_avro_arguments(args),
9705 &state.payload_codec,
9706 ) {
9707 Ok(arguments) => arguments,
9708 Err(error) => return Poll::Ready(Err(error)),
9709 };
9710 let mut command = json!({
9711 "type": "start_child_workflow",
9712 "workflow_type": self.workflow_type,
9713 "queue": self.options.task_queue,
9714 "parent_close_policy": self.options.parent_close_policy.as_str(),
9715 "arguments": arguments,
9716 });
9717 let object = command
9718 .as_object_mut()
9719 .expect("child workflow command is always an object");
9720 if let Some(policy) = &self.options.retry_policy {
9721 let mut retry_policy = serde_json::Map::new();
9722 if let Some(max_attempts) = policy.max_attempts {
9723 if max_attempts == 0 {
9724 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
9725 "retry_policy.max_attempts must be at least 1".to_string(),
9726 )));
9727 }
9728 retry_policy.insert("max_attempts".to_string(), json!(max_attempts));
9729 }
9730 if !policy.backoff_seconds.is_empty() {
9731 retry_policy
9732 .insert("backoff_seconds".to_string(), json!(policy.backoff_seconds));
9733 }
9734 if !policy.non_retryable_error_types.is_empty() {
9735 retry_policy.insert(
9736 "non_retryable_error_types".to_string(),
9737 json!(policy.non_retryable_error_types),
9738 );
9739 }
9740 if retry_policy.is_empty() {
9741 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
9742 "retry_policy must configure at least one field".to_string(),
9743 )));
9744 }
9745 object.insert("retry_policy".to_string(), Value::Object(retry_policy));
9746 }
9747 if let Some(seconds) = self.options.execution_timeout_seconds {
9748 object.insert("execution_timeout_seconds".to_string(), json!(seconds));
9749 }
9750 if let Some(seconds) = self.options.run_timeout_seconds {
9751 object.insert("run_timeout_seconds".to_string(), json!(seconds));
9752 }
9753 apply_parallel_group_path(object, &self.parallel_group_path);
9754 state.commands.push(command);
9755 self.scheduled = true;
9756 }
9757
9758 Poll::Pending
9759 }
9760}
9761
9762impl Future for ChildWorkflowCall {
9763 type Output = Result<ChildWorkflowResult>;
9764
9765 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
9766 match self.poll_avro_value(cx) {
9767 Poll::Ready(Ok(result)) => match result.result.into_json() {
9768 Ok(projected) => Poll::Ready(Ok(ChildWorkflowResult {
9769 parent: result.parent,
9770 child: result.child,
9771 child_workflow_type: result.child_workflow_type,
9772 result: projected,
9773 })),
9774 Err(error) => Poll::Ready(Err(error)),
9775 },
9776 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
9777 Poll::Pending => Poll::Pending,
9778 }
9779 }
9780}
9781
9782fn command_mismatch(recorded: &RecordedCommand, actual: impl Into<String>) -> Error {
9783 Error::NonDeterministicReplay(ReplayFailure::new(
9784 "recorded_command_mismatch",
9785 Some(recorded.sequence()),
9786 Some(recorded.shape().to_string()),
9787 Some(actual.into()),
9788 "current workflow command does not match the recorded durable command sequence",
9789 ))
9790}
9791
9792pub struct SignalCall {
9793 ctx: WorkflowContext,
9794 signal_name: String,
9795 opened_wait: bool,
9796 matched_pending: bool,
9797}
9798
9799impl SignalCall {
9800 fn poll_avro_value(
9801 mut self: Pin<&mut Self>,
9802 _cx: &mut TaskContext<'_>,
9803 ) -> Poll<Result<Vec<AvroValue>>> {
9804 if self.matched_pending {
9805 return Poll::Pending;
9806 }
9807
9808 let ctx = self.ctx.clone();
9809 let mut state = match ctx.state.lock() {
9810 Ok(state) => state,
9811 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
9812 };
9813
9814 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
9815 match recorded {
9816 RecordedCommand::SignalWait {
9817 sequence,
9818 signal_name,
9819 value,
9820 } => {
9821 if signal_name != self.signal_name {
9822 return Poll::Ready(Err(Error::NonDeterministicReplay(
9823 ReplayFailure::new(
9824 "recorded_command_detail_mismatch",
9825 Some(sequence),
9826 Some(format!("signal wait:{signal_name}")),
9827 Some(format!("signal wait:{}", self.signal_name)),
9828 "recorded signal name differs from the current workflow command",
9829 ),
9830 )));
9831 }
9832
9833 state.command_cursor += 1;
9834 if let Some(value) = value {
9835 return Poll::Ready(Ok(value));
9836 }
9837 if state
9838 .resume_signal
9839 .as_ref()
9840 .is_some_and(|signal| signal.signal_name == self.signal_name)
9841 {
9842 let signal = state
9843 .resume_signal
9844 .take()
9845 .expect("matching resume signal is present");
9846 return Poll::Ready(Ok(signal.arguments));
9847 }
9848
9849 state.matched_recorded_pending = true;
9850 self.opened_wait = true;
9851 self.matched_pending = true;
9852 return Poll::Pending;
9853 }
9854 other => {
9855 return Poll::Ready(Err(command_mismatch(
9856 &other,
9857 format!("signal wait:{}", self.signal_name),
9858 )));
9859 }
9860 }
9861 }
9862
9863 if state
9864 .resume_signal
9865 .as_ref()
9866 .is_some_and(|signal| signal.signal_name == self.signal_name)
9867 {
9868 let signal = state
9869 .resume_signal
9870 .take()
9871 .expect("matching resume signal is present");
9872 return Poll::Ready(Ok(signal.arguments));
9873 }
9874
9875 if !self.opened_wait {
9876 state.commands.push(json!({
9877 "type": "open_signal_wait",
9878 "signal_name": self.signal_name
9879 }));
9880 self.opened_wait = true;
9881 }
9882
9883 Poll::Pending
9884 }
9885}
9886
9887impl Future for SignalCall {
9888 type Output = Result<Vec<Value>>;
9889
9890 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
9891 match self.poll_avro_value(cx) {
9892 Poll::Ready(Ok(values)) => Poll::Ready(
9893 values
9894 .into_iter()
9895 .map(AvroValue::into_json)
9896 .collect::<Result<Vec<_>>>(),
9897 ),
9898 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
9899 Poll::Pending => Poll::Pending,
9900 }
9901 }
9902}
9903
9904#[derive(Clone, Debug)]
9905pub struct ActivityContext {
9906 client: Client,
9907 pub task_id: String,
9908 pub activity_attempt_id: String,
9909 pub lease_owner: String,
9910 pub activity_type: String,
9911 pub attempt_number: u64,
9912 pub task_queue: String,
9913 pub worker_id: String,
9914}
9915
9916impl ActivityContext {
9917 pub async fn heartbeat<T: Serialize>(&self, details: T) -> Result<ActivityHeartbeatResponse> {
9918 self.client
9919 .heartbeat_activity_task(
9920 &self.task_id,
9921 &self.activity_attempt_id,
9922 &self.lease_owner,
9923 details,
9924 )
9925 .await
9926 }
9927}
9928
9929fn decode_task_avro_arguments(value: Option<&Value>, codec: &str) -> Result<AvroValue> {
9930 validate_payload_codec(codec)?;
9931 match value {
9932 Some(value) => Ok(normalize_avro_arguments(decode_wire_avro_value(
9933 value, codec,
9934 )?)),
9935 None => Ok(AvroValue::Array(Vec::new())),
9936 }
9937}
9938
9939fn decode_resume_signal(task: &WorkflowTask) -> Result<Option<ResumeSignal>> {
9940 let Some(signal_name) = task
9941 .signal_name
9942 .as_deref()
9943 .filter(|value| !value.is_empty())
9944 else {
9945 return Ok(None);
9946 };
9947 let decoded = decode_task_avro_arguments(task.signal_arguments.as_ref(), &task.payload_codec)?;
9948 let AvroValue::Array(arguments) = decoded else {
9949 unreachable!("normalize_avro_arguments always returns an array");
9950 };
9951
9952 Ok(Some(ResumeSignal {
9953 signal_name: signal_name.to_string(),
9954 arguments,
9955 }))
9956}
9957
9958fn validate_workflow_task_payloads(task: &WorkflowTask) -> Result<()> {
9959 validate_payload_codec(&task.payload_codec)?;
9960 validate_optional_inbound_payload(task.arguments.as_ref(), &task.payload_codec)?;
9961 validate_optional_inbound_payload(task.signal_arguments.as_ref(), &task.payload_codec)?;
9962 for event in &task.history_events {
9963 validate_history_event_payloads(event, &task.payload_codec)?;
9964 }
9965 Ok(())
9966}
9967
9968fn validate_activity_task_payloads(task: &ActivityTask) -> Result<()> {
9969 validate_payload_codec(&task.payload_codec)?;
9970 validate_optional_inbound_payload(task.arguments.as_ref(), &task.payload_codec)
9971}
9972
9973fn validate_query_task_payloads(task: &QueryTask) -> Result<()> {
9974 validate_payload_codec(&task.payload_codec)?;
9975 validate_optional_inbound_payload(task.workflow_arguments.as_ref(), &task.payload_codec)?;
9976 validate_optional_inbound_payload(task.query_arguments.as_ref(), &task.payload_codec)?;
9977 for event in &task.history_events {
9978 validate_history_event_payloads(event, &task.payload_codec)?;
9979 }
9980
9981 let Some(export) = task.history_export.as_ref() else {
9982 return Ok(());
9983 };
9984 let export_codec = match export.get("payloads") {
9985 Some(payloads) => declared_payload_codec(payloads, "codec")?,
9986 None => None,
9987 }
9988 .unwrap_or(&task.payload_codec);
9989 validate_payload_codec(export_codec)?;
9990
9991 if let Some(events) = export.get("history_events").and_then(Value::as_array) {
9992 for event in events {
9993 let event_type = event
9994 .get("event_type")
9995 .or_else(|| event.get("type"))
9996 .and_then(Value::as_str)
9997 .unwrap_or_default();
9998 if let Some(payload) = event.get("payload") {
9999 validate_history_payloads(event_type, payload, export_codec)?;
10000 }
10001 }
10002 }
10003 for signal in export
10004 .get("signals")
10005 .and_then(Value::as_array)
10006 .into_iter()
10007 .flatten()
10008 {
10009 let codec = declared_payload_codec(signal, "payload_codec")?.unwrap_or(export_codec);
10010 validate_payload_codec(codec)?;
10011 validate_optional_inbound_payload(signal.get("arguments"), codec)?;
10012 }
10013 for activity in export
10014 .get("activities")
10015 .and_then(Value::as_array)
10016 .into_iter()
10017 .flatten()
10018 {
10019 let codec = declared_payload_codec(activity, "payload_codec")?.unwrap_or(export_codec);
10020 validate_payload_codec(codec)?;
10021 validate_optional_inbound_payload(activity.get("arguments"), codec)?;
10022 validate_optional_inbound_payload(activity.get("result"), codec)?;
10023 }
10024 Ok(())
10025}
10026
10027fn validate_history_event_payloads(event: &HistoryEvent, fallback_codec: &str) -> Result<()> {
10028 validate_history_payloads(&event.event_type, &event.payload, fallback_codec)
10029}
10030
10031fn validate_history_payloads(
10032 event_type: &str,
10033 payload: &Value,
10034 fallback_codec: &str,
10035) -> Result<()> {
10036 let codec = declared_payload_codec(payload, "payload_codec")?.unwrap_or(fallback_codec);
10037 validate_payload_codec(codec)?;
10038 for field in history_payload_fields(event_type) {
10039 validate_optional_inbound_payload(payload.get(*field), codec)?;
10040 }
10041 Ok(())
10042}
10043
10044const SIGNAL_HISTORY_PAYLOAD_FIELDS: &[&str] = &["value", "input", "arguments"];
10045
10046fn history_payload_fields(event_type: &str) -> &'static [&'static str] {
10047 match event_type {
10048 "ActivityCompleted" => &["result"],
10049 "SignalReceived" | "SignalApplied" => SIGNAL_HISTORY_PAYLOAD_FIELDS,
10050 "UpdateAccepted" | "UpdateRejected" | "UpdateApplied" => &["arguments"],
10051 "UpdateCompleted" | "SideEffectRecorded" => &["result"],
10052 "ChildRunCompleted" => &["result", "output"],
10053 "WorkflowCompleted" => &["output"],
10054 "ServiceCallStarted"
10055 | "ServiceCallCompleted"
10056 | "ServiceCallFailed"
10057 | "ServiceCallCancelled" => &["request_payload", "response_payload"],
10058 _ => &[],
10059 }
10060}
10061
10062fn signal_history_payload(payload: &Value) -> Option<&Value> {
10063 SIGNAL_HISTORY_PAYLOAD_FIELDS
10064 .iter()
10065 .find_map(|field| payload.get(*field))
10066}
10067
10068fn declared_payload_codec<'a>(value: &'a Value, field: &str) -> Result<Option<&'a str>> {
10069 match value.get(field) {
10070 None => Ok(None),
10071 Some(Value::String(codec)) => Ok(Some(codec)),
10072 Some(_) => Err(invalid_payload_envelope()),
10073 }
10074}
10075
10076fn validate_optional_inbound_payload(value: Option<&Value>, codec: &str) -> Result<()> {
10077 validate_payload_codec(codec)?;
10078 if let Some(value) = value.filter(|value| !value.is_null()) {
10079 decode_wire_avro_value(value, codec)?;
10080 }
10081 Ok(())
10082}
10083
10084fn recorded_parallel_group_entry(payload: &Value, sequence: u64) -> Result<ParallelGroupMetadata> {
10085 let group_id = payload_string(payload, "parallel_group_id").ok_or_else(|| {
10086 invalid_recorded_history(
10087 "parallel_group_metadata_invalid",
10088 sequence,
10089 "non-empty parallel_group_id",
10090 &payload.to_string(),
10091 "parallel-group history is missing its stable identity",
10092 )
10093 })?;
10094 let kind = payload_string(payload, "parallel_group_kind").ok_or_else(|| {
10095 invalid_recorded_history(
10096 "parallel_group_metadata_invalid",
10097 sequence,
10098 "activity, child, timer, or mixed group kind",
10099 &payload.to_string(),
10100 "parallel-group history is missing its group kind",
10101 )
10102 })?;
10103 if !matches!(kind.as_str(), "activity" | "child" | "timer" | "mixed") {
10104 return Err(invalid_recorded_history(
10105 "parallel_group_metadata_invalid",
10106 sequence,
10107 "activity, child, timer, or mixed group kind",
10108 &kind,
10109 "parallel-group history contains an unsupported group kind",
10110 ));
10111 }
10112 let base_sequence = payload
10113 .get("parallel_group_base_sequence")
10114 .and_then(value_as_u64)
10115 .filter(|value| *value > 0)
10116 .ok_or_else(|| {
10117 invalid_recorded_history(
10118 "parallel_group_metadata_invalid",
10119 sequence,
10120 "positive parallel_group_base_sequence",
10121 &payload.to_string(),
10122 "parallel-group history contains an invalid base sequence",
10123 )
10124 })?;
10125 let size = payload
10126 .get("parallel_group_size")
10127 .and_then(value_as_u64)
10128 .and_then(|value| usize::try_from(value).ok())
10129 .filter(|value| (1..=MAX_PARALLEL_OPERATIONS).contains(value))
10130 .ok_or_else(|| {
10131 invalid_recorded_history(
10132 "parallel_group_metadata_invalid",
10133 sequence,
10134 "bounded positive parallel_group_size",
10135 &payload.to_string(),
10136 "parallel-group history contains an invalid group size",
10137 )
10138 })?;
10139 let index = payload
10140 .get("parallel_group_index")
10141 .and_then(value_as_u64)
10142 .and_then(|value| usize::try_from(value).ok())
10143 .filter(|value| *value < size)
10144 .ok_or_else(|| {
10145 invalid_recorded_history(
10146 "parallel_group_metadata_invalid",
10147 sequence,
10148 "parallel_group_index within group bounds",
10149 &payload.to_string(),
10150 "parallel-group history contains an invalid member index",
10151 )
10152 })?;
10153 if base_sequence.checked_add(u64::try_from(index).unwrap_or(u64::MAX)) != Some(sequence) {
10154 return Err(invalid_recorded_history(
10155 "parallel_group_metadata_invalid",
10156 sequence,
10157 "base sequence plus member index equals workflow sequence",
10158 &payload.to_string(),
10159 "parallel-group path does not preserve durable workflow position",
10160 ));
10161 }
10162 let expected_id = format!("{}:{base_sequence}:{size}", parallel_group_prefix(&kind));
10163 if group_id != expected_id {
10164 return Err(invalid_recorded_history(
10165 "parallel_group_metadata_invalid",
10166 sequence,
10167 &expected_id,
10168 &group_id,
10169 "parallel-group history contains an incompatible stable group ID",
10170 ));
10171 }
10172 Ok(ParallelGroupMetadata {
10173 parallel_group_id: group_id,
10174 parallel_group_kind: kind,
10175 parallel_group_base_sequence: base_sequence,
10176 parallel_group_size: size,
10177 parallel_group_index: index,
10178 })
10179}
10180
10181fn recorded_parallel_group_path(
10182 events: &[&HistoryEvent],
10183 sequence: u64,
10184) -> Result<Option<Vec<ParallelGroupMetadata>>> {
10185 let mut recorded: Option<Vec<ParallelGroupMetadata>> = None;
10186 for event in events {
10187 let payload = &event.payload;
10188 let has_metadata = payload.get("parallel_group_path").is_some()
10189 || payload.get("parallel_group_id").is_some()
10190 || payload.get("parallel_group_kind").is_some()
10191 || payload.get("parallel_group_base_sequence").is_some()
10192 || payload.get("parallel_group_size").is_some()
10193 || payload.get("parallel_group_index").is_some();
10194 if !has_metadata {
10195 continue;
10196 }
10197
10198 let top_level = recorded_parallel_group_entry(payload, sequence)?;
10199 let path = match payload.get("parallel_group_path") {
10200 None => vec![top_level.clone()],
10201 Some(Value::Array(entries)) if !entries.is_empty() => entries
10202 .iter()
10203 .map(|entry| recorded_parallel_group_entry(entry, sequence))
10204 .collect::<Result<Vec<_>>>()?,
10205 Some(value) => {
10206 return Err(invalid_recorded_history(
10207 "parallel_group_metadata_invalid",
10208 sequence,
10209 "non-empty parallel_group_path list",
10210 &value.to_string(),
10211 "parallel-group history contains an invalid group path",
10212 ));
10213 }
10214 };
10215 if path.last() != Some(&top_level) {
10216 return Err(invalid_recorded_history(
10217 "parallel_group_metadata_invalid",
10218 sequence,
10219 &serde_json::to_string(&path.last()).unwrap_or_default(),
10220 &serde_json::to_string(&top_level).unwrap_or_default(),
10221 "parallel-group top-level fields do not match the innermost path entry",
10222 ));
10223 }
10224 if recorded.as_ref().is_some_and(|existing| existing != &path) {
10225 return Err(invalid_recorded_history(
10226 "parallel_group_history_conflict",
10227 sequence,
10228 &serde_json::to_string(&recorded.as_ref()).unwrap_or_default(),
10229 &serde_json::to_string(&path).unwrap_or_default(),
10230 "parallel-group metadata changed between scheduling and resolution history",
10231 ));
10232 }
10233 recorded = Some(path);
10234 }
10235 Ok(recorded)
10236}
10237
10238fn recorded_commands(
10239 events: &[HistoryEvent],
10240 fallback_codec: &str,
10241 parent: WorkflowIdentity,
10242) -> Result<Vec<RecordedCommand>> {
10243 let mut events_by_sequence: BTreeMap<u64, Vec<&HistoryEvent>> = BTreeMap::new();
10244 let mut last_new_sequence = None;
10245
10246 for event in events {
10247 let is_activity = matches!(
10248 event.event_type.as_str(),
10249 "ActivityScheduled"
10250 | "ActivityStarted"
10251 | "ActivityHeartbeatRecorded"
10252 | "ActivityRetryScheduled"
10253 | "ActivityCompleted"
10254 | "ActivityFailed"
10255 | "ActivityCancelled"
10256 | "ActivityTimedOut"
10257 );
10258 let is_workflow_timer = matches!(
10259 event.event_type.as_str(),
10260 "TimerScheduled" | "TimerCancelled" | "TimerFired"
10261 ) && !is_internal_timer_event(event);
10262 let is_child_workflow = matches!(
10263 event.event_type.as_str(),
10264 "ChildWorkflowScheduled"
10265 | "ChildRunCompleted"
10266 | "ChildRunFailed"
10267 | "ChildRunCancelled"
10268 | "ChildRunTerminated"
10269 );
10270 let is_signal_wait = is_recorded_signal_wait_event(event);
10271 let is_condition_wait = is_recorded_condition_wait_event(event);
10272 let is_search_attributes = event.event_type == "SearchAttributesUpserted";
10273 let is_side_effect = event.event_type == "SideEffectRecorded";
10274 let is_version_marker = event.event_type == "VersionMarkerRecorded";
10275 let is_memo = event.event_type == "MemoUpserted";
10276 if !is_activity
10277 && !is_workflow_timer
10278 && !is_child_workflow
10279 && !is_signal_wait
10280 && !is_condition_wait
10281 && !is_search_attributes
10282 && !is_side_effect
10283 && !is_version_marker
10284 && !is_memo
10285 {
10286 continue;
10287 }
10288
10289 let sequence = durable_event_sequence(event).ok_or_else(|| {
10290 Error::NonDeterministicReplay(ReplayFailure::new(
10291 "durable_command_sequence_missing",
10292 None,
10293 Some("positive workflow sequence".to_string()),
10294 Some(event.event_type.clone()),
10295 "durable command history event has no workflow sequence",
10296 ))
10297 })?;
10298 if sequence == 0 {
10299 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
10300 "durable_command_sequence_invalid",
10301 Some(sequence),
10302 Some("positive workflow sequence".to_string()),
10303 Some(sequence.to_string()),
10304 "durable command history uses an invalid workflow sequence",
10305 )));
10306 }
10307 if !events_by_sequence.contains_key(&sequence) {
10308 if let Some(previous) = last_new_sequence {
10309 if sequence < previous {
10310 return Err(invalid_recorded_history(
10311 "durable_command_sequence_mismatch",
10312 sequence,
10313 &format!("workflow sequence greater than {previous}"),
10314 &sequence.to_string(),
10315 "durable commands are not strictly ordered by their recorded workflow sequence",
10316 ));
10317 }
10318 }
10319 last_new_sequence = Some(sequence);
10320 }
10321 events_by_sequence.entry(sequence).or_default().push(event);
10322 }
10323
10324 let commands: Vec<RecordedCommand> = events_by_sequence
10325 .into_iter()
10326 .map(|(sequence, sequence_events)| {
10327 let activity_events: Vec<_> = sequence_events
10328 .iter()
10329 .copied()
10330 .filter(|event| event.event_type.starts_with("Activity"))
10331 .collect();
10332 let timer_events: Vec<_> = sequence_events
10333 .iter()
10334 .copied()
10335 .filter(|event| event.event_type.starts_with("Timer"))
10336 .collect();
10337 let child_events: Vec<_> = sequence_events
10338 .iter()
10339 .copied()
10340 .filter(|event| {
10341 event.event_type == "ChildWorkflowScheduled"
10342 || event.event_type.starts_with("ChildRun")
10343 })
10344 .collect();
10345 let signal_wait_events: Vec<_> = sequence_events
10346 .iter()
10347 .copied()
10348 .filter(|event| is_recorded_signal_wait_event(event))
10349 .collect();
10350 let condition_wait_events: Vec<_> = sequence_events
10351 .iter()
10352 .copied()
10353 .filter(|event| is_recorded_condition_wait_event(event))
10354 .collect();
10355 let search_attribute_events: Vec<_> = sequence_events
10356 .iter()
10357 .copied()
10358 .filter(|event| event.event_type == "SearchAttributesUpserted")
10359 .collect();
10360 let side_effect_events: Vec<_> = sequence_events
10361 .iter()
10362 .copied()
10363 .filter(|event| event.event_type == "SideEffectRecorded")
10364 .collect();
10365 let version_marker_events: Vec<_> = sequence_events
10366 .iter()
10367 .copied()
10368 .filter(|event| event.event_type == "VersionMarkerRecorded")
10369 .collect();
10370 let memo_events: Vec<_> = sequence_events
10371 .iter()
10372 .copied()
10373 .filter(|event| event.event_type == "MemoUpserted")
10374 .collect();
10375
10376 let command_kind_count = usize::from(!activity_events.is_empty())
10377 + usize::from(!timer_events.is_empty())
10378 + usize::from(!child_events.is_empty())
10379 + usize::from(!signal_wait_events.is_empty())
10380 + usize::from(!condition_wait_events.is_empty())
10381 + usize::from(!search_attribute_events.is_empty())
10382 + usize::from(!side_effect_events.is_empty())
10383 + usize::from(!version_marker_events.is_empty())
10384 + usize::from(!memo_events.is_empty());
10385 if command_kind_count > 1 {
10386 let actual = [
10387 (!activity_events.is_empty()).then_some("activity"),
10388 (!timer_events.is_empty()).then_some("timer"),
10389 (!child_events.is_empty()).then_some("child workflow"),
10390 (!signal_wait_events.is_empty()).then_some("signal wait"),
10391 (!condition_wait_events.is_empty()).then_some("condition wait"),
10392 (!search_attribute_events.is_empty()).then_some("search-attribute update"),
10393 (!side_effect_events.is_empty()).then_some("side effect"),
10394 (!version_marker_events.is_empty()).then_some("version marker"),
10395 (!memo_events.is_empty()).then_some("memo upsert"),
10396 ]
10397 .into_iter()
10398 .flatten()
10399 .collect::<Vec<_>>()
10400 .join(" and ");
10401 return Err(invalid_recorded_history(
10402 "durable_command_sequence_collision",
10403 sequence,
10404 "one durable command kind",
10405 &actual,
10406 "one workflow sequence records more than one durable command kind",
10407 ));
10408 }
10409
10410 if !activity_events.is_empty() {
10411 let parallel_group_path =
10412 recorded_parallel_group_path(&activity_events, sequence)?;
10413 let scheduled_count = activity_events
10414 .iter()
10415 .filter(|event| event.event_type == "ActivityScheduled")
10416 .count();
10417 if scheduled_count > 1 {
10418 return Err(invalid_recorded_history(
10419 "duplicate_activity_schedule",
10420 sequence,
10421 "at most one ActivityScheduled event",
10422 "multiple ActivityScheduled events",
10423 "activity history schedules more than one command at one workflow sequence",
10424 ));
10425 }
10426 let activity_type = activity_events.iter().find_map(|event| {
10427 event
10428 .payload
10429 .get("activity_type")
10430 .or_else(|| event.payload.get("activity_name"))
10431 .and_then(Value::as_str)
10432 .map(str::to_string)
10433 });
10434 if activity_events.iter().filter_map(|event| {
10435 event
10436 .payload
10437 .get("activity_type")
10438 .or_else(|| event.payload.get("activity_name"))
10439 .and_then(Value::as_str)
10440 }).any(|candidate| Some(candidate) != activity_type.as_deref()) {
10441 return Err(invalid_recorded_history(
10442 "activity_identity_mismatch",
10443 sequence,
10444 activity_type.as_deref().unwrap_or("one activity identity"),
10445 "conflicting activity identities",
10446 "activity lifecycle events at one workflow sequence disagree on identity",
10447 ));
10448 }
10449 let terminal: Vec<_> = activity_events
10450 .iter()
10451 .copied()
10452 .filter(|event| {
10453 matches!(
10454 event.event_type.as_str(),
10455 "ActivityCompleted"
10456 | "ActivityFailed"
10457 | "ActivityCancelled"
10458 | "ActivityTimedOut"
10459 )
10460 })
10461 .collect();
10462 let duplicate_delivery = terminal.first().is_some_and(|first| {
10463 terminal.iter().all(|event| {
10464 event.event_type == first.event_type && event.payload == first.payload
10465 })
10466 });
10467 if terminal.len() > 1 && !duplicate_delivery {
10468 return Err(invalid_recorded_history(
10469 "duplicate_activity_terminal_event",
10470 sequence,
10471 "at most one terminal activity event",
10472 "multiple terminal activity events",
10473 "activity history settles one command more than once",
10474 ));
10475 }
10476 let outcome = terminal
10477 .first()
10478 .map(|event| activity_outcome(event, fallback_codec, activity_type.clone()))
10479 .transpose()?;
10480 let options = activity_events
10481 .iter()
10482 .find(|event| event.event_type == "ActivityScheduled")
10483 .and_then(|event| event.payload.get("activity"))
10484 .and_then(Value::as_object)
10485 .map(|activity| RecordedActivityOptions {
10486 task_queue: recorded_optional_string(activity, "queue"),
10487 execution_mode: recorded_optional_string(activity, "execution_mode"),
10488 retry_policy: recorded_activity_retry_snapshot(
10489 activity.get("retry_policy"),
10490 ),
10491 });
10492 return Ok(RecordedCommand::Activity {
10493 sequence,
10494 activity_type,
10495 options,
10496 outcome,
10497 parallel_group_path,
10498 });
10499 }
10500
10501 if !child_events.is_empty() {
10502 let parallel_group_path = recorded_parallel_group_path(&child_events, sequence)?;
10503 let scheduled: Vec<_> = child_events
10504 .iter()
10505 .copied()
10506 .filter(|event| event.event_type == "ChildWorkflowScheduled")
10507 .collect();
10508 if scheduled.len() != 1 {
10509 return Err(invalid_recorded_history(
10510 "child_workflow_schedule_missing_or_duplicate",
10511 sequence,
10512 "one ChildWorkflowScheduled event",
10513 &format!("{} ChildWorkflowScheduled events", scheduled.len()),
10514 "child workflow replay requires exactly one recorded schedule event",
10515 ));
10516 }
10517 let workflow_type = child_events.iter().find_map(|event| {
10518 event
10519 .payload
10520 .get("child_workflow_type")
10521 .or_else(|| event.payload.get("workflow_type"))
10522 .and_then(Value::as_str)
10523 .filter(|value| !value.is_empty())
10524 .map(str::to_string)
10525 });
10526 if child_events
10527 .iter()
10528 .filter_map(|event| {
10529 event
10530 .payload
10531 .get("child_workflow_type")
10532 .or_else(|| event.payload.get("workflow_type"))
10533 .and_then(Value::as_str)
10534 })
10535 .any(|candidate| Some(candidate) != workflow_type.as_deref())
10536 {
10537 return Err(invalid_recorded_history(
10538 "child_workflow_identity_mismatch",
10539 sequence,
10540 workflow_type
10541 .as_deref()
10542 .unwrap_or("one child workflow type"),
10543 "conflicting child workflow types",
10544 "child workflow lifecycle events at one sequence disagree on type",
10545 ));
10546 }
10547 let mut outcomes = child_workflow_outcomes(
10548 &child_events.iter().map(|event| (*event).clone()).collect::<Vec<_>>(),
10549 fallback_codec,
10550 parent.clone(),
10551 )?;
10552 let terminal_events = child_events
10553 .iter()
10554 .copied()
10555 .filter(|event| event.event_type.starts_with("ChildRun"))
10556 .collect::<Vec<_>>();
10557 let duplicate_delivery = terminal_events.first().is_some_and(|first| {
10558 terminal_events.iter().all(|event| {
10559 event.event_type == first.event_type && event.payload == first.payload
10560 })
10561 });
10562 if outcomes.len() > 1 && !duplicate_delivery {
10563 return Err(invalid_recorded_history(
10564 "duplicate_child_workflow_terminal_event",
10565 sequence,
10566 "at most one terminal child event",
10567 "multiple terminal child events",
10568 "child workflow history settles one command more than once",
10569 ));
10570 }
10571 return Ok(RecordedCommand::ChildWorkflow {
10572 sequence,
10573 workflow_type,
10574 outcome: outcomes.pop(),
10575 parallel_group_path,
10576 });
10577 }
10578
10579 if !signal_wait_events.is_empty() {
10580 let opened: Vec<_> = signal_wait_events
10581 .iter()
10582 .copied()
10583 .filter(|event| event.event_type == "SignalWaitOpened")
10584 .collect();
10585 if opened.len() != 1 {
10586 return Err(invalid_recorded_history(
10587 "signal_wait_open_missing_or_duplicate",
10588 sequence,
10589 "one SignalWaitOpened event",
10590 &format!("{} SignalWaitOpened events", opened.len()),
10591 "signal replay requires exactly one canonical wait-open event",
10592 ));
10593 }
10594
10595 let applied: Vec<_> = signal_wait_events
10596 .iter()
10597 .copied()
10598 .filter(|event| event.event_type == "SignalApplied")
10599 .collect();
10600 if applied.len() > 1 {
10601 return Err(invalid_recorded_history(
10602 "duplicate_signal_wait_apply",
10603 sequence,
10604 "at most one SignalApplied event",
10605 "multiple SignalApplied events",
10606 "signal history applies one durable wait more than once",
10607 ));
10608 }
10609
10610 let signal_names = signal_wait_events
10611 .iter()
10612 .map(|event| required_signal_wait_name(event, sequence))
10613 .collect::<Result<Vec<_>>>()?;
10614 let signal_name = signal_names
10615 .first()
10616 .expect("signal wait events are not empty")
10617 .clone();
10618 if signal_names.iter().any(|candidate| candidate != &signal_name) {
10619 return Err(invalid_recorded_history(
10620 "signal_wait_identity_mismatch",
10621 sequence,
10622 &signal_name,
10623 "conflicting signal names",
10624 "signal wait lifecycle events at one workflow sequence disagree on identity",
10625 ));
10626 }
10627 let value = applied
10628 .first()
10629 .map(|event| decode_signal_event_arguments(event, fallback_codec))
10630 .transpose()?;
10631 return Ok(RecordedCommand::SignalWait {
10632 sequence,
10633 signal_name,
10634 value,
10635 });
10636 }
10637
10638 if !condition_wait_events.is_empty() {
10639 return recorded_condition_wait(
10640 sequence,
10641 &condition_wait_events,
10642 events,
10643 );
10644 }
10645
10646 if !search_attribute_events.is_empty() {
10647 if search_attribute_events.len() != 1 {
10648 return Err(invalid_recorded_history(
10649 "duplicate_search_attribute_update",
10650 sequence,
10651 "one SearchAttributesUpserted event",
10652 &format!(
10653 "{} SearchAttributesUpserted events",
10654 search_attribute_events.len()
10655 ),
10656 "search-attribute history records one workflow command more than once",
10657 ));
10658 }
10659 let payload = &search_attribute_events[0].payload;
10660 let attributes = payload
10661 .get("attributes")
10662 .filter(|value| value.as_object().is_some_and(|values| !values.is_empty()))
10663 .cloned()
10664 .ok_or_else(|| {
10665 invalid_recorded_history(
10666 "search_attribute_update_missing",
10667 sequence,
10668 "non-empty attributes object",
10669 "missing or invalid attributes",
10670 "search-attribute history is missing its recorded mutation",
10671 )
10672 })?;
10673 let attribute_types =
10674 recorded_search_attribute_types(payload, &attributes, sequence)?;
10675 return Ok(RecordedCommand::SearchAttributes {
10676 sequence,
10677 attributes,
10678 attribute_types,
10679 });
10680 }
10681
10682 if !side_effect_events.is_empty() {
10683 if side_effect_events.len() != 1 {
10684 return Err(invalid_recorded_history(
10685 "duplicate_side_effect_record",
10686 sequence,
10687 "one SideEffectRecorded event",
10688 &format!("{} SideEffectRecorded events", side_effect_events.len()),
10689 "side-effect history records one workflow command more than once",
10690 ));
10691 }
10692 let event = side_effect_events[0];
10693 let result = event.payload.get("result").ok_or_else(|| {
10694 invalid_recorded_history(
10695 "side_effect_result_missing",
10696 sequence,
10697 "recorded result payload",
10698 "missing result",
10699 "side-effect history is missing its recorded value",
10700 )
10701 })?;
10702 let has_published_envelope = result.as_str().is_some()
10703 || result.as_object().is_some_and(|envelope| {
10704 envelope.get("codec").and_then(Value::as_str).is_some()
10705 && envelope.get("blob").and_then(Value::as_str).is_some()
10706 });
10707 if !has_published_envelope {
10708 return Err(invalid_recorded_history(
10709 "side_effect_payload_malformed",
10710 sequence,
10711 "payload blob or {codec, blob} envelope",
10712 &result.to_string(),
10713 "side-effect history result does not use a published payload envelope",
10714 ));
10715 }
10716 let codec = event
10717 .payload
10718 .get("payload_codec")
10719 .and_then(Value::as_str)
10720 .unwrap_or(fallback_codec);
10721 let value = decode_wire_avro_value(result, codec).map_err(|error| {
10722 if error.to_string().contains("unsupported_payload_codec") {
10723 return error;
10724 }
10725
10726 invalid_recorded_history(
10727 "side_effect_payload_incompatible",
10728 sequence,
10729 &format!("valid {codec} payload envelope"),
10730 &error.to_string(),
10731 "side-effect history payload cannot be decoded with its recorded codec",
10732 )
10733 })?;
10734 return Ok(RecordedCommand::SideEffect { sequence, value });
10735 }
10736
10737 if !version_marker_events.is_empty() {
10738 if version_marker_events.len() != 1 {
10739 return Err(invalid_recorded_history(
10740 "duplicate_version_marker_record",
10741 sequence,
10742 "one VersionMarkerRecorded event",
10743 &format!("{} VersionMarkerRecorded events", version_marker_events.len()),
10744 "version-marker history records one workflow command more than once",
10745 ));
10746 }
10747 let payload = &version_marker_events[0].payload;
10748 let change_id = payload
10749 .get("change_id")
10750 .and_then(Value::as_str)
10751 .filter(|value| !value.is_empty())
10752 .map(str::to_string)
10753 .ok_or_else(|| {
10754 invalid_recorded_history(
10755 "version_marker_field_missing",
10756 sequence,
10757 "non-empty change_id",
10758 "missing or invalid change_id",
10759 "version-marker history is missing its stable change ID",
10760 )
10761 })?;
10762 let version = required_version_i32(payload, "version", sequence)?;
10763 let min_supported = required_version_i32(payload, "min_supported", sequence)?;
10764 let max_supported = required_version_i32(payload, "max_supported", sequence)?;
10765 if min_supported > max_supported || version < min_supported || version > max_supported {
10766 return Err(invalid_recorded_history(
10767 "version_marker_history_range_invalid",
10768 sequence,
10769 "min_supported <= version <= max_supported",
10770 &format!("{min_supported} <= {version} <= {max_supported}"),
10771 "recorded version marker contains an internally incompatible range",
10772 ));
10773 }
10774 return Ok(RecordedCommand::VersionMarker {
10775 sequence,
10776 change_id,
10777 version,
10778 });
10779 }
10780
10781 if !memo_events.is_empty() {
10782 if memo_events.len() != 1 {
10783 return Err(invalid_recorded_history(
10784 "duplicate_memo_upsert_record",
10785 sequence,
10786 "one MemoUpserted event",
10787 &format!("{} MemoUpserted events", memo_events.len()),
10788 "memo history records one workflow update more than once",
10789 ));
10790 }
10791 let payload = &memo_events[0].payload;
10792 let entries = payload.get("entries").cloned().ok_or_else(|| {
10793 invalid_recorded_history(
10794 "memo_entries_missing",
10795 sequence,
10796 "memo entries object",
10797 "missing entries",
10798 "MemoUpserted history is missing replay identity entries",
10799 )
10800 })?;
10801 let entries = decode_memo_history_map(&entries, true).map_err(|error| {
10802 invalid_recorded_history(
10803 "memo_entries_invalid",
10804 sequence,
10805 "valid canonical memo entries",
10806 &error.to_string(),
10807 "MemoUpserted history contains invalid replay identity entries",
10808 )
10809 })?;
10810 let merged = payload.get("merged").cloned().ok_or_else(|| {
10811 invalid_recorded_history(
10812 "memo_merged_projection_missing",
10813 sequence,
10814 "merged memo projection",
10815 "missing merged",
10816 "MemoUpserted history is missing its merged projection",
10817 )
10818 })?;
10819 decode_memo_history_map(&merged, false).map_err(|error| {
10820 invalid_recorded_history(
10821 "memo_merged_projection_invalid",
10822 sequence,
10823 "valid merged memo projection",
10824 &error.to_string(),
10825 "MemoUpserted history contains an invalid merged projection",
10826 )
10827 })?;
10828
10829 return Ok(RecordedCommand::Memo { sequence, entries });
10830 }
10831 let scheduled: Vec<_> = timer_events
10832 .iter()
10833 .copied()
10834 .filter(|event| event.event_type == "TimerScheduled")
10835 .collect();
10836 let fired: Vec<_> = timer_events
10837 .iter()
10838 .copied()
10839 .filter(|event| event.event_type == "TimerFired")
10840 .collect();
10841 if scheduled.len() != 1 {
10842 return Err(invalid_recorded_history(
10843 "timer_schedule_missing_or_duplicate",
10844 sequence,
10845 "one TimerScheduled event",
10846 &format!("{} TimerScheduled events", scheduled.len()),
10847 "timer replay requires exactly one recorded schedule event",
10848 ));
10849 }
10850 if fired.len() > 1 {
10851 return Err(invalid_recorded_history(
10852 "duplicate_timer_fire",
10853 sequence,
10854 "at most one TimerFired event",
10855 "multiple TimerFired events",
10856 "timer history contains more than one fire event for a workflow sequence",
10857 ));
10858 }
10859
10860 let scheduled = scheduled[0];
10861 let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
10862 let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
10863 if let Some(fired) = fired.first() {
10864 let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
10865 if fired_timer_id != timer_id {
10866 return Err(invalid_recorded_history(
10867 "timer_identity_mismatch",
10868 sequence,
10869 &timer_id,
10870 &fired_timer_id,
10871 "TimerFired does not correspond to the recorded TimerScheduled event",
10872 ));
10873 }
10874 let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
10875 if fired_delay != delay_seconds {
10876 return Err(invalid_recorded_history(
10877 "timer_history_delay_mismatch",
10878 sequence,
10879 &delay_seconds.to_string(),
10880 &fired_delay.to_string(),
10881 "TimerScheduled and TimerFired record different delays",
10882 ));
10883 }
10884 }
10885
10886 Ok(RecordedCommand::Timer {
10887 sequence,
10888 delay_seconds,
10889 fired: !fired.is_empty(),
10890 parallel_group_path: recorded_parallel_group_path(&timer_events, sequence)?,
10891 })
10892 })
10893 .collect::<Result<_>>()?;
10894
10895 let mut marker_sequences = HashMap::new();
10896 for command in &commands {
10897 if let RecordedCommand::VersionMarker {
10898 sequence,
10899 change_id,
10900 ..
10901 } = command
10902 {
10903 if let Some(first_sequence) = marker_sequences.insert(change_id.clone(), *sequence) {
10904 return Err(invalid_recorded_history(
10905 "duplicate_version_marker",
10906 *sequence,
10907 &format!("one marker for change ID {change_id:?}"),
10908 &format!("markers at sequences {first_sequence} and {sequence}"),
10909 "workflow history contains duplicate markers for one stable change ID",
10910 ));
10911 }
10912 }
10913 }
10914
10915 Ok(commands)
10916}
10917
10918fn required_version_i32(payload: &Value, field: &str, sequence: u64) -> Result<i32> {
10919 payload
10920 .get(field)
10921 .and_then(Value::as_i64)
10922 .and_then(|value| i32::try_from(value).ok())
10923 .ok_or_else(|| {
10924 invalid_recorded_history(
10925 "version_marker_field_missing",
10926 sequence,
10927 &format!("integer {field}"),
10928 "missing or out-of-range integer",
10929 "version-marker history is missing a required integer field",
10930 )
10931 })
10932}
10933
10934fn durable_event_sequence(event: &HistoryEvent) -> Option<u64> {
10935 event
10936 .payload
10937 .get("sequence")
10938 .or_else(|| event.payload.get("workflow_sequence"))
10939 .or_else(|| event.raw.get("sequence"))
10940 .or_else(|| event.raw.get("workflow_sequence"))
10941 .and_then(value_as_u64)
10942}
10943
10944fn is_internal_timer_event(event: &HistoryEvent) -> bool {
10945 matches!(
10946 event
10947 .payload
10948 .get("timer_kind")
10949 .or_else(|| event.raw.get("timer_kind"))
10950 .and_then(Value::as_str),
10951 Some("condition_timeout" | "signal_timeout")
10952 )
10953}
10954
10955fn is_recorded_condition_wait_event(event: &HistoryEvent) -> bool {
10956 matches!(
10957 event.event_type.as_str(),
10958 "ConditionWaitOpened" | "ConditionWaitSatisfied" | "ConditionWaitTimedOut"
10959 )
10960}
10961
10962fn recorded_condition_wait(
10963 sequence: u64,
10964 condition_events: &[&HistoryEvent],
10965 all_events: &[HistoryEvent],
10966) -> Result<RecordedCommand> {
10967 let opened = condition_events
10968 .iter()
10969 .copied()
10970 .filter(|event| event.event_type == "ConditionWaitOpened")
10971 .collect::<Vec<_>>();
10972 if opened.len() != 1 {
10973 return Err(invalid_recorded_history(
10974 "condition_wait_open_missing_or_duplicate",
10975 sequence,
10976 "one ConditionWaitOpened event",
10977 &format!("{} ConditionWaitOpened events", opened.len()),
10978 "condition replay requires exactly one canonical wait-open event",
10979 ));
10980 }
10981 let terminal = condition_events
10982 .iter()
10983 .copied()
10984 .filter(|event| {
10985 matches!(
10986 event.event_type.as_str(),
10987 "ConditionWaitSatisfied" | "ConditionWaitTimedOut"
10988 )
10989 })
10990 .collect::<Vec<_>>();
10991 if terminal.len() > 1 {
10992 return Err(invalid_recorded_history(
10993 "duplicate_condition_wait_terminal_event",
10994 sequence,
10995 "at most one condition terminal event",
10996 "multiple condition terminal events",
10997 "condition history settles one durable wait more than once",
10998 ));
10999 }
11000
11001 let opened = opened[0];
11002 let condition_wait_id = required_condition_wait_id(opened, sequence)?;
11003 for event in condition_events
11004 .iter()
11005 .copied()
11006 .filter(|event| !std::ptr::eq(*event, opened))
11007 {
11008 let event_wait_id = required_condition_wait_id(event, sequence)?;
11009 if event_wait_id != condition_wait_id {
11010 return Err(invalid_recorded_history(
11011 "condition_wait_id_mismatch",
11012 sequence,
11013 &condition_wait_id,
11014 &event_wait_id,
11015 "condition lifecycle events at one sequence disagree on wait identity",
11016 ));
11017 }
11018 }
11019
11020 let condition_key = optional_non_empty_history_string(opened, "condition_key");
11021 let predicate_identity = opened
11022 .payload
11023 .get("condition_definition_fingerprint")
11024 .and_then(Value::as_str)
11025 .filter(|value| !value.is_empty())
11026 .map(str::to_string)
11027 .ok_or_else(|| {
11028 invalid_recorded_history(
11029 "condition_wait_predicate_fingerprint_missing",
11030 sequence,
11031 "non-empty condition_definition_fingerprint",
11032 &opened.event_type,
11033 "canonical condition history is missing its predicate identity",
11034 )
11035 })?;
11036 let timeout_seconds = optional_history_u64(opened, "timeout_seconds", sequence)?;
11037 for event in condition_events
11038 .iter()
11039 .copied()
11040 .filter(|event| !std::ptr::eq(*event, opened))
11041 {
11042 for (field, opened_value) in [
11043 ("condition_key", condition_key.as_deref()),
11044 (
11045 "condition_definition_fingerprint",
11046 Some(predicate_identity.as_str()),
11047 ),
11048 ] {
11049 if let Some(value) = optional_non_empty_history_string(event, field) {
11050 if opened_value.is_some_and(|opened_value| opened_value != value) {
11051 return Err(invalid_recorded_history(
11052 "condition_wait_definition_history_mismatch",
11053 sequence,
11054 opened_value.unwrap_or_default(),
11055 &value,
11056 "condition lifecycle events disagree on the recorded definition",
11057 ));
11058 }
11059 }
11060 }
11061 if let Some(event_timeout) = optional_history_u64(event, "timeout_seconds", sequence)? {
11062 if timeout_seconds.is_some_and(|opened_timeout| opened_timeout != event_timeout) {
11063 return Err(invalid_recorded_history(
11064 "condition_wait_definition_history_mismatch",
11065 sequence,
11066 &format!("{}s", timeout_seconds.unwrap_or_default()),
11067 &format!("{event_timeout}s"),
11068 "condition lifecycle events disagree on the recorded timeout",
11069 ));
11070 }
11071 }
11072 }
11073
11074 let timeout_timer_events = all_events
11075 .iter()
11076 .filter(|event| {
11077 matches!(
11078 event.event_type.as_str(),
11079 "TimerScheduled" | "TimerCancelled" | "TimerFired"
11080 ) && event.payload.get("timer_kind").and_then(Value::as_str)
11081 == Some("condition_timeout")
11082 && event
11083 .payload
11084 .get("condition_wait_id")
11085 .and_then(Value::as_str)
11086 == Some(condition_wait_id.as_str())
11087 })
11088 .collect::<Vec<_>>();
11089 let scheduled = timeout_timer_events
11090 .iter()
11091 .copied()
11092 .filter(|event| event.event_type == "TimerScheduled")
11093 .collect::<Vec<_>>();
11094 let fired = timeout_timer_events
11095 .iter()
11096 .copied()
11097 .filter(|event| event.event_type == "TimerFired")
11098 .collect::<Vec<_>>();
11099 if scheduled.len() > 1 || fired.len() > 1 || (!fired.is_empty() && scheduled.len() != 1) {
11100 return Err(invalid_recorded_history(
11101 "condition_wait_timeout_history_invalid",
11102 sequence,
11103 "one timeout schedule and at most one fire",
11104 &format!("{} schedules and {} fires", scheduled.len(), fired.len()),
11105 "condition timeout history has a missing or duplicate lifecycle event",
11106 ));
11107 }
11108 if let Some(scheduled) = scheduled.first() {
11109 let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
11110 let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
11111 if timeout_seconds.is_some_and(|timeout| timeout != delay_seconds) {
11112 return Err(invalid_recorded_history(
11113 "condition_wait_timeout_delay_mismatch",
11114 sequence,
11115 &format!("{}s", timeout_seconds.unwrap_or_default()),
11116 &format!("{delay_seconds}s"),
11117 "condition timeout timer differs from the wait definition",
11118 ));
11119 }
11120 if let Some(fired) = fired.first() {
11121 let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
11122 let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
11123 if fired_timer_id != timer_id || fired_delay != delay_seconds {
11124 return Err(invalid_recorded_history(
11125 "condition_wait_timeout_identity_mismatch",
11126 sequence,
11127 &format!("{timer_id}:{delay_seconds}s"),
11128 &format!("{fired_timer_id}:{fired_delay}s"),
11129 "condition timeout fire does not match its durable schedule",
11130 ));
11131 }
11132 }
11133 }
11134
11135 let result = terminal.first().map(|event| {
11136 if event.event_type == "ConditionWaitTimedOut" {
11137 ConditionWaitResult::TimedOut
11138 } else {
11139 ConditionWaitResult::Satisfied
11140 }
11141 });
11142 let result = if !fired.is_empty() {
11143 if result == Some(ConditionWaitResult::Satisfied) {
11144 return Err(invalid_recorded_history(
11145 "condition_wait_terminal_conflict",
11146 sequence,
11147 "one satisfied or timed-out outcome",
11148 "satisfied event and fired timeout",
11149 "condition history records conflicting terminal outcomes",
11150 ));
11151 }
11152 Some(ConditionWaitResult::TimedOut)
11153 } else {
11154 result
11155 };
11156
11157 Ok(RecordedCommand::ConditionWait {
11158 sequence,
11159 condition_key,
11160 predicate_identity,
11161 timeout_seconds,
11162 result,
11163 })
11164}
11165
11166fn required_condition_wait_id(event: &HistoryEvent, sequence: u64) -> Result<String> {
11167 event
11168 .payload
11169 .get("condition_wait_id")
11170 .and_then(Value::as_str)
11171 .filter(|value| !value.is_empty())
11172 .map(str::to_string)
11173 .ok_or_else(|| {
11174 invalid_recorded_history(
11175 "condition_wait_id_missing",
11176 sequence,
11177 "non-empty condition_wait_id",
11178 &event.event_type,
11179 "canonical condition history is missing its durable wait identity",
11180 )
11181 })
11182}
11183
11184fn optional_non_empty_history_string(event: &HistoryEvent, field: &str) -> Option<String> {
11185 event
11186 .payload
11187 .get(field)
11188 .and_then(Value::as_str)
11189 .filter(|value| !value.is_empty())
11190 .map(str::to_string)
11191}
11192
11193fn optional_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<Option<u64>> {
11194 match event.payload.get(field) {
11195 None | Some(Value::Null) => Ok(None),
11196 Some(value) => value_as_u64(value).map(Some).ok_or_else(|| {
11197 invalid_recorded_history(
11198 "condition_wait_definition_invalid",
11199 sequence,
11200 &format!("non-negative integer {field}"),
11201 &value.to_string(),
11202 "condition history contains an invalid numeric definition field",
11203 )
11204 }),
11205 }
11206}
11207
11208fn required_signal_wait_name(event: &HistoryEvent, sequence: u64) -> Result<String> {
11209 event
11210 .payload
11211 .get("signal_name")
11212 .or_else(|| event.raw.get("signal_name"))
11213 .and_then(Value::as_str)
11214 .filter(|value| !value.is_empty())
11215 .map(str::to_string)
11216 .ok_or_else(|| {
11217 invalid_recorded_history(
11218 "signal_wait_name_missing",
11219 sequence,
11220 "non-empty signal_name",
11221 &event.event_type,
11222 "canonical signal-wait history is missing its signal identity",
11223 )
11224 })
11225}
11226
11227fn is_recorded_signal_wait_event(event: &HistoryEvent) -> bool {
11228 matches!(
11229 event.event_type.as_str(),
11230 "SignalWaitOpened" | "SignalApplied"
11231 )
11232}
11233
11234fn required_history_string(event: &HistoryEvent, field: &str, sequence: u64) -> Result<String> {
11235 event
11236 .payload
11237 .get(field)
11238 .and_then(Value::as_str)
11239 .filter(|value| !value.is_empty())
11240 .map(str::to_string)
11241 .ok_or_else(|| {
11242 invalid_recorded_history(
11243 "timer_history_field_missing",
11244 sequence,
11245 field,
11246 &event.event_type,
11247 "timer history is missing a required identity field",
11248 )
11249 })
11250}
11251
11252fn required_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<u64> {
11253 event
11254 .payload
11255 .get(field)
11256 .and_then(value_as_u64)
11257 .ok_or_else(|| {
11258 invalid_recorded_history(
11259 "timer_history_field_missing",
11260 sequence,
11261 field,
11262 &event.event_type,
11263 "timer history is missing a required numeric field",
11264 )
11265 })
11266}
11267
11268fn recorded_search_attribute_types(
11269 payload: &Value,
11270 attributes: &Value,
11271 sequence: u64,
11272) -> Result<RecordedSnapshotValue<BTreeMap<String, String>>> {
11273 let Some(raw_types) = payload.get("attribute_types") else {
11274 return Ok(RecordedSnapshotValue::Unknown);
11278 };
11279 let Some(raw_types) = raw_types.as_object() else {
11280 return Err(invalid_recorded_history(
11281 "search_attribute_types_malformed",
11282 sequence,
11283 "canonical attribute type map",
11284 &raw_types.to_string(),
11285 "search-attribute history contains malformed type identity",
11286 ));
11287 };
11288 let attribute_keys = attributes
11289 .as_object()
11290 .expect("recorded search attributes were validated as an object");
11291 let mut types = BTreeMap::new();
11292 for (key, value) in raw_types {
11293 let Some(attribute_type) = value.as_str() else {
11294 return Err(invalid_recorded_history(
11295 "search_attribute_types_malformed",
11296 sequence,
11297 "canonical string type name",
11298 &value.to_string(),
11299 "search-attribute history contains a non-string type identity",
11300 ));
11301 };
11302 if !attribute_keys.contains_key(key)
11303 || !matches!(
11304 attribute_type,
11305 "string" | "keyword" | "keyword_list" | "int" | "float" | "bool" | "datetime"
11306 )
11307 {
11308 return Err(invalid_recorded_history(
11309 "search_attribute_types_malformed",
11310 sequence,
11311 "canonical types for keys present in attributes",
11312 &format!("{key}:{attribute_type}"),
11313 "search-attribute history contains unsupported or orphaned type identity",
11314 ));
11315 }
11316 types.insert(key.clone(), attribute_type.to_string());
11317 }
11318 Ok(RecordedSnapshotValue::Known(types))
11319}
11320
11321fn invalid_recorded_history(
11322 reason: &str,
11323 sequence: u64,
11324 expected: &str,
11325 actual: &str,
11326 message: &str,
11327) -> Error {
11328 Error::NonDeterministicReplay(ReplayFailure::new(
11329 reason,
11330 Some(sequence),
11331 Some(expected.to_string()),
11332 Some(actual.to_string()),
11333 message,
11334 ))
11335}
11336
11337type ActivityOutcome = std::result::Result<AvroValue, ActivityFailure>;
11338
11339fn activity_outcome(
11340 event: &HistoryEvent,
11341 fallback_codec: &str,
11342 recorded_activity_type: Option<String>,
11343) -> Result<ActivityOutcome> {
11344 if event.event_type == "ActivityCompleted" {
11345 let codec = event
11346 .payload
11347 .get("payload_codec")
11348 .and_then(Value::as_str)
11349 .unwrap_or(fallback_codec);
11350 return Ok(Ok(decode_wire_avro_value(
11351 event.payload.get("result").unwrap_or(&Value::Null),
11352 codec,
11353 )?));
11354 }
11355
11356 let payload = &event.payload;
11357 let (kind, fallback_reason, fallback_message) = match event.event_type.as_str() {
11358 "ActivityFailed" => (ActivityFailureKind::Failed, "activity", "activity failed"),
11359 "ActivityCancelled" => (
11360 ActivityFailureKind::Cancelled,
11361 "cancelled",
11362 "activity was cancelled",
11363 ),
11364 "ActivityTimedOut" => (
11365 ActivityFailureKind::TimedOut,
11366 "timeout",
11367 "activity timed out",
11368 ),
11369 _ => unreachable!("activity_outcome is called only for terminal activity events"),
11370 };
11371 let exception = payload
11372 .get("exception")
11373 .filter(|value| !value.is_null())
11374 .cloned();
11375 let failure_category = payload_string(payload, "failure_category");
11376 let timeout_kind = payload_string(payload, "timeout_kind");
11377 let reason = payload_string(payload, "reason").unwrap_or_else(|| match kind {
11378 ActivityFailureKind::Failed => failure_category
11379 .clone()
11380 .unwrap_or_else(|| fallback_reason.to_string()),
11381 ActivityFailureKind::Cancelled => fallback_reason.to_string(),
11382 ActivityFailureKind::TimedOut => timeout_kind
11383 .clone()
11384 .unwrap_or_else(|| fallback_reason.to_string()),
11385 });
11386 let message = payload_string(payload, "message")
11387 .or_else(|| {
11388 exception
11389 .as_ref()
11390 .and_then(|value| payload_string(value, "message"))
11391 })
11392 .unwrap_or_else(|| fallback_message.to_string());
11393
11394 Ok(Err(ActivityFailure {
11395 kind,
11396 reason,
11397 message,
11398 activity_execution_id: payload_string(payload, "activity_execution_id"),
11399 activity_attempt_id: payload_string(payload, "activity_attempt_id"),
11400 activity_type: payload_string(payload, "activity_type")
11401 .or_else(|| payload_string(payload, "activity_name"))
11402 .or(recorded_activity_type),
11403 activity_class: payload_string(payload, "activity_class"),
11404 attempt_number: payload.get("attempt_number").and_then(value_as_u64),
11405 failure_id: payload_string(payload, "failure_id"),
11406 failure_category,
11407 timeout_kind,
11408 non_retryable: payload
11409 .get("non_retryable")
11410 .and_then(Value::as_bool)
11411 .unwrap_or(false),
11412 exception_type: payload_string(payload, "exception_type").or_else(|| {
11413 exception
11414 .as_ref()
11415 .and_then(|value| payload_string(value, "type"))
11416 }),
11417 exception_class: payload_string(payload, "exception_class").or_else(|| {
11418 exception
11419 .as_ref()
11420 .and_then(|value| payload_string(value, "class"))
11421 }),
11422 code: payload
11423 .get("code")
11424 .filter(|value| !value.is_null())
11425 .cloned(),
11426 exception,
11427 }))
11428}
11429
11430type ChildWorkflowOutcome = std::result::Result<ChildWorkflowAvroResult, ChildWorkflowFailure>;
11431
11432fn child_workflow_outcomes(
11433 events: &[HistoryEvent],
11434 fallback_codec: &str,
11435 parent: WorkflowIdentity,
11436) -> Result<Vec<ChildWorkflowOutcome>> {
11437 let mut outcomes = Vec::new();
11438
11439 for event in events {
11440 let kind = match event.event_type.as_str() {
11441 "ChildRunCompleted" => None,
11442 "ChildRunFailed" => Some((
11443 ChildWorkflowFailureKind::Failed,
11444 "child_workflow",
11445 "child workflow failed",
11446 )),
11447 "ChildRunCancelled" => Some((
11448 ChildWorkflowFailureKind::Cancelled,
11449 "cancelled",
11450 "child workflow was cancelled",
11451 )),
11452 "ChildRunTerminated" => Some((
11453 ChildWorkflowFailureKind::Terminated,
11454 "terminated",
11455 "child workflow was terminated",
11456 )),
11457 _ => continue,
11458 };
11459 let payload = &event.payload;
11460 let child_workflow_id = payload_string(payload, "child_workflow_instance_id");
11461 let child_workflow_run_id = payload_string(payload, "child_workflow_run_id");
11462 let child_workflow_type = payload_string(payload, "child_workflow_type");
11463
11464 if let Some((kind, reason, fallback_message)) = kind {
11465 let exception = payload
11466 .get("exception")
11467 .filter(|value| !value.is_null())
11468 .cloned();
11469 let message = payload_string(payload, "message")
11470 .or_else(|| {
11471 exception
11472 .as_ref()
11473 .and_then(|value| payload_string(value, "message"))
11474 })
11475 .unwrap_or_else(|| fallback_message.to_string());
11476 let exception_type = payload_string(payload, "exception_type").or_else(|| {
11477 exception
11478 .as_ref()
11479 .and_then(|value| payload_string(value, "type"))
11480 });
11481 let exception_class = payload_string(payload, "exception_class").or_else(|| {
11482 exception
11483 .as_ref()
11484 .and_then(|value| payload_string(value, "class"))
11485 });
11486 outcomes.push(Err(ChildWorkflowFailure {
11487 kind,
11488 reason: reason.to_string(),
11489 message,
11490 parent_workflow_id: parent.workflow_id.clone(),
11491 parent_workflow_run_id: parent.run_id.clone(),
11492 child_workflow_id,
11493 child_workflow_run_id,
11494 child_workflow_type,
11495 failure_id: payload_string(payload, "failure_id"),
11496 failure_category: payload_string(payload, "failure_category"),
11497 exception_type,
11498 exception_class,
11499 non_retryable: payload
11500 .get("non_retryable")
11501 .and_then(Value::as_bool)
11502 .unwrap_or(false),
11503 code: payload
11504 .get("code")
11505 .filter(|value| !value.is_null())
11506 .cloned(),
11507 exception,
11508 }));
11509 continue;
11510 }
11511
11512 let codec = payload
11513 .get("payload_codec")
11514 .and_then(Value::as_str)
11515 .unwrap_or(fallback_codec);
11516 let result = payload
11517 .get("result")
11518 .or_else(|| payload.get("output"))
11519 .unwrap_or(&Value::Null);
11520 outcomes.push(Ok(ChildWorkflowAvroResult {
11521 parent: parent.clone(),
11522 child: WorkflowIdentity {
11523 workflow_id: child_workflow_id,
11524 run_id: child_workflow_run_id,
11525 },
11526 child_workflow_type,
11527 result: decode_wire_avro_value(result, codec)?,
11528 }));
11529 }
11530
11531 Ok(outcomes)
11532}
11533
11534fn payload_string(payload: &Value, key: &str) -> Option<String> {
11535 payload
11536 .get(key)
11537 .and_then(Value::as_str)
11538 .filter(|value| !value.is_empty())
11539 .map(str::to_string)
11540}
11541
11542fn workflow_failure_command(error: &Error) -> Value {
11543 let (exception_type, exception_class, properties) = match error {
11544 Error::ActivityFailed(failure) => (
11545 match failure.kind {
11546 ActivityFailureKind::Failed => "ActivityFailed",
11547 ActivityFailureKind::Cancelled => "ActivityCancelled",
11548 ActivityFailureKind::TimedOut => "ActivityTimedOut",
11549 },
11550 "durable_workflow::ActivityFailure",
11551 json!({
11552 "reason": failure.reason,
11553 "activity_execution_id": failure.activity_execution_id,
11554 "activity_attempt_id": failure.activity_attempt_id,
11555 "activity_type": failure.activity_type,
11556 "activity_class": failure.activity_class,
11557 "attempt_number": failure.attempt_number,
11558 "failure_id": failure.failure_id,
11559 "failure_category": failure.failure_category,
11560 "timeout_kind": failure.timeout_kind,
11561 "activity_non_retryable": failure.non_retryable,
11562 "activity_exception_type": failure.exception_type,
11563 "activity_exception_class": failure.exception_class,
11564 "activity_code": failure.code,
11565 "activity_exception": failure.exception,
11566 }),
11567 ),
11568 Error::ChildWorkflowFailed(failure) => (
11569 match failure.kind {
11570 ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
11571 ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
11572 ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
11573 },
11574 "durable_workflow::ChildWorkflowFailure",
11575 json!({
11576 "reason": failure.reason,
11577 "parent_workflow_id": failure.parent_workflow_id,
11578 "parent_workflow_run_id": failure.parent_workflow_run_id,
11579 "child_workflow_id": failure.child_workflow_id,
11580 "child_workflow_run_id": failure.child_workflow_run_id,
11581 "child_workflow_type": failure.child_workflow_type,
11582 "failure_id": failure.failure_id,
11583 "failure_category": failure.failure_category,
11584 "child_exception_type": failure.exception_type,
11585 "child_exception_class": failure.exception_class,
11586 "child_non_retryable": failure.non_retryable,
11587 "child_code": failure.code,
11588 "child_exception": failure.exception,
11589 }),
11590 ),
11591 Error::ParallelFailed(failure) => (
11592 "ParallelFailed",
11593 "durable_workflow::ParallelFailure",
11594 json!({
11595 "parallel_group_id": failure.group_id,
11596 "parallel_member_path": failure.member_path,
11597 "parallel_group_path": failure.group_path,
11598 "completed_members": failure.completed.iter().map(|completion| &completion.member_path).collect::<Vec<_>>(),
11599 "cause_type": workflow_error_type(&failure.cause),
11600 "cause_message": failure.cause.to_string(),
11601 }),
11602 ),
11603 Error::SagaCompensationFailed(failure) => (
11604 "SagaCompensationFailed",
11605 "durable_workflow::SagaCompensationFailure",
11606 json!({
11607 "initiating_failure_type": workflow_error_type(&failure.initiating_failure),
11608 "initiating_failure_message": failure.initiating_failure.to_string(),
11609 "compensation_activity_type": failure.compensation_activity_type,
11610 "compensation_registration_order": failure.compensation_registration_order,
11611 "compensation_failure_type": workflow_error_type(&failure.compensation_failure),
11612 "compensation_failure_message": failure.compensation_failure.to_string(),
11613 }),
11614 ),
11615 Error::WorkflowCancellationRequested(_) => (
11616 "WorkflowCancellationRequested",
11617 "durable_workflow::WorkflowCancellationRequested",
11618 json!({"reason": "cancelled"}),
11619 ),
11620 Error::NonDeterministicReplay(_) => (
11621 "NonDeterministicReplay",
11622 "durable_workflow::Error",
11623 Value::Null,
11624 ),
11625 _ => ("RustWorkflowError", "durable_workflow::Error", Value::Null),
11626 };
11627 let non_retryable = match error {
11628 Error::ActivityFailed(failure) => failure.non_retryable,
11629 Error::ChildWorkflowFailed(failure) => failure.non_retryable,
11630 Error::ParallelFailed(failure) => workflow_error_non_retryable(&failure.cause),
11631 Error::SagaCompensationFailed(failure) => {
11632 workflow_error_non_retryable(&failure.compensation_failure)
11633 }
11634 Error::WorkflowCancellationRequested(_) => true,
11635 Error::NonDeterministicReplay(_) => true,
11636 _ => false,
11637 };
11638
11639 json!({
11640 "type": "fail_workflow",
11641 "message": error.to_string(),
11642 "exception_type": exception_type,
11643 "exception_class": exception_class,
11644 "non_retryable": non_retryable,
11645 "exception": {
11646 "type": exception_type,
11647 "class": exception_class,
11648 "message": error.to_string(),
11649 "properties": properties,
11650 }
11651 })
11652}
11653
11654fn workflow_error_type(error: &Error) -> &'static str {
11655 match error {
11656 Error::ActivityFailed(failure) => match failure.kind {
11657 ActivityFailureKind::Failed => "ActivityFailed",
11658 ActivityFailureKind::Cancelled => "ActivityCancelled",
11659 ActivityFailureKind::TimedOut => "ActivityTimedOut",
11660 },
11661 Error::ChildWorkflowFailed(failure) => match failure.kind {
11662 ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
11663 ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
11664 ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
11665 },
11666 Error::ParallelFailed(_) => "ParallelFailed",
11667 Error::SagaCompensationFailed(_) => "SagaCompensationFailed",
11668 Error::WorkflowCancellationRequested(_) => "WorkflowCancellationRequested",
11669 Error::NonDeterministicReplay(_) => "NonDeterministicReplay",
11670 _ => "RustWorkflowError",
11671 }
11672}
11673
11674fn workflow_error_non_retryable(error: &Error) -> bool {
11675 match error {
11676 Error::ActivityFailed(failure) => failure.non_retryable,
11677 Error::ChildWorkflowFailed(failure) => failure.non_retryable,
11678 Error::ParallelFailed(failure) => workflow_error_non_retryable(&failure.cause),
11679 Error::SagaCompensationFailed(failure) => {
11680 workflow_error_non_retryable(&failure.compensation_failure)
11681 }
11682 Error::WorkflowCancellationRequested(_) | Error::NonDeterministicReplay(_) => true,
11683 _ => false,
11684 }
11685}
11686
11687fn workflow_task_integrity_error(error: &Error) -> bool {
11688 matches!(
11689 error,
11690 Error::NonDeterministicReplay(_)
11691 | Error::Protocol(_)
11692 | Error::MissingWorkflowCommandIdentity
11693 | Error::WorkflowStatePoisoned
11694 )
11695}
11696
11697fn decode_signal_event_arguments(
11698 event: &HistoryEvent,
11699 fallback_codec: &str,
11700) -> Result<Vec<AvroValue>> {
11701 let codec = declared_payload_codec(&event.payload, "payload_codec")?.unwrap_or(fallback_codec);
11702 validate_payload_codec(codec)?;
11703 let raw = signal_history_payload(&event.payload);
11704 let decoded = match raw.filter(|value| !value.is_null()) {
11705 Some(value) => decode_wire_avro_value(value, codec)?,
11706 None => AvroValue::Array(Vec::new()),
11707 };
11708 let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
11709 unreachable!("normalize_avro_arguments always returns an array");
11710 };
11711 Ok(arguments)
11712}
11713
11714fn decode_update_event_arguments(
11715 event: &HistoryEvent,
11716 fallback_codec: &str,
11717) -> Result<Vec<AvroValue>> {
11718 let codec = declared_payload_codec(&event.payload, "payload_codec")?.unwrap_or(fallback_codec);
11719 validate_payload_codec(codec)?;
11720 let decoded = match event
11721 .payload
11722 .get("arguments")
11723 .filter(|value| !value.is_null())
11724 {
11725 Some(value) => decode_wire_avro_value(value, codec)?,
11726 None => AvroValue::Array(Vec::new()),
11727 };
11728 let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
11729 unreachable!("normalize_avro_arguments always returns an array");
11730 };
11731 Ok(arguments)
11732}
11733
11734fn hydrate_query_history_from_export(task: &mut QueryTask) -> Result<()> {
11735 let Some(export_events) = task
11736 .history_export
11737 .as_ref()
11738 .and_then(|export| export.get("history_events"))
11739 .and_then(Value::as_array)
11740 else {
11741 return Ok(());
11742 };
11743
11744 if export_events.len() > task.history_events.len() {
11745 task.history_events = serde_json::from_value(Value::Array(export_events.clone()))?;
11746 }
11747
11748 Ok(())
11749}
11750
11751fn enrich_query_history_from_export(task: &mut QueryTask) -> Result<()> {
11752 let Some(export) = task.history_export.as_ref() else {
11753 return Ok(());
11754 };
11755 let signals = export
11756 .get("signals")
11757 .and_then(Value::as_array)
11758 .cloned()
11759 .unwrap_or_default();
11760 let activities = export
11761 .get("activities")
11762 .and_then(Value::as_array)
11763 .cloned()
11764 .unwrap_or_default();
11765 let export_codec = export
11766 .get("payloads")
11767 .and_then(|payloads| payloads.get("codec"))
11768 .and_then(Value::as_str)
11769 .unwrap_or(&task.payload_codec)
11770 .to_string();
11771 let mut signal_name_offsets: HashMap<String, usize> = HashMap::new();
11772
11773 for event in &mut task.history_events {
11774 if event.event_type == "ActivityCompleted" {
11775 let sequence = event
11776 .payload
11777 .get("sequence")
11778 .or_else(|| event.payload.get("workflow_sequence"))
11779 .and_then(value_as_u64);
11780 let Some(activity) = sequence.and_then(|sequence| {
11781 activities.iter().find(|activity| {
11782 activity.get("sequence").and_then(value_as_u64) == Some(sequence)
11783 })
11784 }) else {
11785 continue;
11786 };
11787 let Some(payload) = event.payload.as_object_mut() else {
11788 continue;
11789 };
11790 if missing_payload(payload.get("result")) {
11791 if let Some(result) = activity
11792 .get("result")
11793 .filter(|value| !missing_payload(Some(value)))
11794 {
11795 payload.insert("result".to_string(), result.clone());
11796 }
11797 }
11798 for field in ["payload_codec", "activity_type"] {
11799 if payload
11800 .get(field)
11801 .and_then(Value::as_str)
11802 .unwrap_or_default()
11803 .is_empty()
11804 {
11805 if let Some(value) = activity.get(field) {
11806 payload.insert(field.to_string(), value.clone());
11807 }
11808 }
11809 }
11810 continue;
11811 }
11812
11813 if event.event_type != "SignalReceived" && event.event_type != "SignalApplied" {
11814 continue;
11815 }
11816 let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
11817 let command_id = event
11818 .payload
11819 .get("workflow_command_id")
11820 .or_else(|| event.raw.get("workflow_command_id"))
11821 .and_then(Value::as_str);
11822 let signal_name = event
11823 .payload
11824 .get("signal_name")
11825 .and_then(Value::as_str)
11826 .unwrap_or_default()
11827 .to_string();
11828 let matched = signals
11829 .iter()
11830 .find(|signal| {
11831 signal_id.is_some() && signal.get("id").and_then(Value::as_str) == signal_id
11832 })
11833 .or_else(|| {
11834 signals.iter().find(|signal| {
11835 command_id.is_some()
11836 && signal.get("command_id").and_then(Value::as_str) == command_id
11837 })
11838 })
11839 .or_else(|| {
11840 let offset = signal_name_offsets.entry(signal_name.clone()).or_default();
11841 let signal = signals
11842 .iter()
11843 .filter(|signal| {
11844 signal.get("name").and_then(Value::as_str) == Some(signal_name.as_str())
11845 })
11846 .nth(*offset);
11847 if signal.is_some() {
11848 *offset += 1;
11849 }
11850 signal
11851 });
11852 let Some(signal) = matched else {
11853 continue;
11854 };
11855 let signal_codec = signal
11856 .get("payload_codec")
11857 .and_then(Value::as_str)
11858 .unwrap_or(&export_codec);
11859 let Some(payload) = event.payload.as_object_mut() else {
11860 continue;
11861 };
11862 if missing_payload(payload.get("arguments")) {
11863 if let Some(arguments) = signal
11864 .get("arguments")
11865 .filter(|value| !missing_payload(Some(value)))
11866 {
11867 let envelope = match arguments {
11868 Value::String(blob) => json!({"codec": signal_codec, "blob": blob}),
11869 other => other.clone(),
11870 };
11871 payload.insert("arguments".to_string(), envelope);
11872 }
11873 }
11874 if payload
11875 .get("payload_codec")
11876 .and_then(Value::as_str)
11877 .unwrap_or_default()
11878 .is_empty()
11879 {
11880 payload.insert("payload_codec".to_string(), json!(signal_codec));
11881 }
11882 }
11883
11884 Ok(())
11885}
11886
11887fn missing_payload(value: Option<&Value>) -> bool {
11888 match value {
11889 None | Some(Value::Null) => true,
11890 Some(Value::String(value)) => value.is_empty(),
11891 Some(_) => false,
11892 }
11893}
11894
11895fn query_signal_events(task: &QueryTask) -> Result<Vec<QuerySignal>> {
11896 let export_signals = task
11897 .history_export
11898 .as_ref()
11899 .and_then(|export| export.get("signals"))
11900 .and_then(Value::as_array)
11901 .cloned()
11902 .unwrap_or_default();
11903 let export_codec = task
11904 .history_export
11905 .as_ref()
11906 .and_then(|export| export.get("payloads"))
11907 .and_then(|payloads| payloads.get("codec"))
11908 .and_then(Value::as_str)
11909 .unwrap_or(&task.payload_codec);
11910 let mut name_offsets: HashMap<String, usize> = HashMap::new();
11911 let mut signals = Vec::new();
11912
11913 for event in &task.history_events {
11914 if event.event_type != "SignalApplied" && event.event_type != "SignalReceived" {
11915 continue;
11916 }
11917
11918 let name = event
11919 .payload
11920 .get("signal_name")
11921 .and_then(Value::as_str)
11922 .unwrap_or_default();
11923 if name.is_empty() {
11924 continue;
11925 }
11926 let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
11927 let command_id = event
11928 .payload
11929 .get("workflow_command_id")
11930 .or_else(|| event.raw.get("workflow_command_id"))
11931 .and_then(Value::as_str);
11932 let matched_export = export_signals
11933 .iter()
11934 .find(|candidate| {
11935 signal_id.is_some() && candidate.get("id").and_then(Value::as_str) == signal_id
11936 })
11937 .or_else(|| {
11938 export_signals.iter().find(|candidate| {
11939 command_id.is_some()
11940 && candidate.get("command_id").and_then(Value::as_str) == command_id
11941 })
11942 })
11943 .or_else(|| {
11944 let offset = name_offsets.entry(name.to_string()).or_default();
11945 let candidate = export_signals
11946 .iter()
11947 .filter(|candidate| candidate.get("name").and_then(Value::as_str) == Some(name))
11948 .nth(*offset);
11949 if candidate.is_some() {
11950 *offset += 1;
11951 }
11952 candidate
11953 });
11954 let codec = event
11955 .payload
11956 .get("payload_codec")
11957 .and_then(Value::as_str)
11958 .or_else(|| {
11959 matched_export
11960 .and_then(|signal| signal.get("payload_codec"))
11961 .and_then(Value::as_str)
11962 })
11963 .unwrap_or(export_codec);
11964 let raw_arguments = signal_history_payload(&event.payload)
11965 .filter(|value| !value.is_null())
11966 .or_else(|| matched_export.and_then(|signal| signal.get("arguments")));
11967 let (arguments, avro_arguments) = decode_query_signal_arguments(raw_arguments, codec)?;
11968 let workflow_sequence = event
11969 .payload
11970 .get("workflow_sequence")
11971 .and_then(value_as_u64)
11972 .or_else(|| {
11973 matched_export
11974 .and_then(|signal| signal.get("workflow_sequence"))
11975 .and_then(value_as_u64)
11976 });
11977
11978 signals.push(QuerySignal {
11979 id: signal_id.map(str::to_string).or_else(|| {
11980 matched_export
11981 .and_then(|signal| signal.get("id"))
11982 .and_then(Value::as_str)
11983 .map(str::to_string)
11984 }),
11985 name: name.to_string(),
11986 arguments,
11987 avro_arguments,
11988 workflow_sequence,
11989 });
11990 }
11991
11992 if signals.is_empty() {
11993 for signal in export_signals {
11994 if signal.get("status").and_then(Value::as_str) == Some("rejected") {
11995 continue;
11996 }
11997 let Some(name) = signal.get("name").and_then(Value::as_str) else {
11998 continue;
11999 };
12000 let codec = signal
12001 .get("payload_codec")
12002 .and_then(Value::as_str)
12003 .unwrap_or(export_codec);
12004 let (arguments, avro_arguments) =
12005 decode_query_signal_arguments(signal.get("arguments"), codec)?;
12006 signals.push(QuerySignal {
12007 id: signal.get("id").and_then(Value::as_str).map(str::to_string),
12008 name: name.to_string(),
12009 arguments,
12010 avro_arguments,
12011 workflow_sequence: signal.get("workflow_sequence").and_then(value_as_u64),
12012 });
12013 }
12014 signals.sort_by_key(|signal| signal.workflow_sequence.unwrap_or(u64::MAX));
12015 }
12016
12017 Ok(signals)
12018}
12019
12020fn decode_query_signal_arguments(
12021 raw: Option<&Value>,
12022 codec: &str,
12023) -> Result<(Vec<Value>, Vec<AvroValue>)> {
12024 validate_payload_codec(codec)?;
12025 let decoded = match raw.filter(|value| !value.is_null()) {
12026 Some(value) => decode_wire_avro_value(value, codec)?,
12027 None => AvroValue::Array(Vec::new()),
12028 };
12029 let AvroValue::Array(avro_arguments) = normalize_avro_arguments(decoded) else {
12030 unreachable!("normalize_avro_arguments always returns an array");
12031 };
12032 let arguments = avro_arguments
12033 .iter()
12034 .cloned()
12035 .map(AvroValue::into_json)
12036 .collect::<Result<Vec<_>>>()?;
12037 Ok((arguments, avro_arguments))
12038}
12039
12040fn value_as_u64(value: &Value) -> Option<u64> {
12041 value
12042 .as_u64()
12043 .or_else(|| value.as_str().and_then(|value| value.parse().ok()))
12044}
12045
12046#[cfg(test)]
12047mod tests {
12048 use super::*;
12049 use std::{
12050 io::{Read, Write},
12051 net::{SocketAddr, TcpListener, TcpStream},
12052 sync::atomic::AtomicUsize,
12053 thread,
12054 };
12055
12056 #[derive(Clone, Copy, Debug)]
12057 enum InvalidTaskPayloadCodec {
12058 Missing,
12059 Null,
12060 NonString,
12061 }
12062
12063 impl InvalidTaskPayloadCodec {
12064 fn label(self) -> &'static str {
12065 match self {
12066 Self::Missing => "missing",
12067 Self::Null => "null",
12068 Self::NonString => "non-string",
12069 }
12070 }
12071
12072 fn apply(self, task: &mut Value) {
12073 let task = task.as_object_mut().expect("task fixture object");
12074 match self {
12075 Self::Missing => {
12076 task.remove("payload_codec");
12077 }
12078 Self::Null => {
12079 task.insert("payload_codec".to_string(), Value::Null);
12080 }
12081 Self::NonString => {
12082 task.insert("payload_codec".to_string(), json!(42));
12083 }
12084 }
12085 }
12086 }
12087
12088 fn fixture_envelope(value: Value) -> Value {
12089 encode_value_envelope(&value, DEFAULT_CODEC).expect("encode Avro test fixture")
12090 }
12091
12092 fn fixture_blob(value: Value) -> String {
12093 encode_payload(&value, DEFAULT_CODEC)
12094 .expect("encode Avro test fixture")
12095 .blob
12096 }
12097
12098 #[test]
12099 fn client_builder_rejects_the_sdk_owned_api_suffix() {
12100 for base_url in [
12101 "http://127.0.0.1:8080/api",
12102 "http://localhost:8080/api/",
12103 "https://runtime.example.test/namespaces/orders/api",
12104 ] {
12105 let error = Client::builder(base_url)
12106 .build()
12107 .expect_err("SDK-owned /api suffix must be rejected during build");
12108
12109 assert!(matches!(error, Error::InvalidBaseUrl), "{base_url}");
12110 assert!(
12111 error.to_string().contains("SDK appends /api automatically"),
12112 "the validation error must explain how to fix the endpoint"
12113 );
12114 }
12115 }
12116
12117 #[test]
12118 fn client_builder_preserves_self_hosted_and_managed_runtime_prefixes() {
12119 for (base_url, expected) in [
12120 ("http://127.0.0.1:8080", "http://127.0.0.1:8080"),
12121 (
12122 "http://localhost:8080/durable-workflow/",
12123 "http://localhost:8080/durable-workflow",
12124 ),
12125 (
12126 "https://runtime.example.test/namespaces/orders",
12127 "https://runtime.example.test/namespaces/orders",
12128 ),
12129 (
12130 "https://runtime.example.test/gateway/api/namespaces/orders",
12131 "https://runtime.example.test/gateway/api/namespaces/orders",
12132 ),
12133 (
12134 "https://api.example.test/runtime/orders/",
12135 "https://api.example.test/runtime/orders",
12136 ),
12137 ] {
12138 let client = Client::builder(base_url)
12139 .build()
12140 .expect("Server and Cloud runtime base URL must remain valid");
12141
12142 assert_eq!(client.base_url, expected);
12143 }
12144 }
12145
12146 #[test]
12147 fn workflow_completion_uses_the_additive_command_protocol_floor() {
12148 assert_eq!(
12149 workflow_completion_protocol_version(&[json!({"type": "complete_workflow"})]),
12150 WORKER_PROTOCOL_VERSION
12151 );
12152 assert_eq!(
12153 workflow_completion_protocol_version(&[json!({
12154 "type": "upsert_search_attributes",
12155 "attributes": {"OrderStatus": "waiting"},
12156 })]),
12157 SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION
12158 );
12159 assert_eq!(
12160 workflow_completion_protocol_version(&[
12161 json!({"type": "upsert_search_attributes", "attributes": {"State": "waiting"}}),
12162 json!({"type": "open_condition_wait", "condition_key": "ready"}),
12163 ]),
12164 CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION
12165 );
12166 }
12167
12168 fn typed_fidelity_probe() -> AvroValue {
12169 AvroValue::Map(BTreeMap::from([
12170 ("bytes".to_string(), AvroValue::Bytes(vec![0, 0xff])),
12171 ("empty".to_string(), AvroValue::Map(BTreeMap::new())),
12172 (
12173 "numeric".to_string(),
12174 AvroValue::Map(BTreeMap::from([
12175 ("0".to_string(), AvroValue::String("zero".to_string())),
12176 ("1".to_string(), AvroValue::String("one".to_string())),
12177 ])),
12178 ),
12179 (
12180 "nested".to_string(),
12181 AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([(
12182 "enabled".to_string(),
12183 AvroValue::Boolean(true),
12184 )]))]),
12185 ),
12186 (
12187 "projection_collisions".to_string(),
12188 AvroValue::Array(projection_collision_probe()),
12189 ),
12190 ]))
12191 }
12192
12193 fn projection_collision_probe() -> Vec<AvroValue> {
12194 vec![
12195 AvroValue::Map(BTreeMap::from([
12196 ("$type".to_string(), AvroValue::String("bytes".to_string())),
12197 (
12198 "base64".to_string(),
12199 AvroValue::String("ordinary user text".to_string()),
12200 ),
12201 ])),
12202 AvroValue::Map(BTreeMap::from([
12203 ("$type".to_string(), AvroValue::String("map".to_string())),
12204 (
12205 "entries".to_string(),
12206 AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([
12207 ("key".to_string(), AvroValue::String("ordinary".to_string())),
12208 (
12209 "value".to_string(),
12210 AvroValue::String("user map".to_string()),
12211 ),
12212 ]))]),
12213 ),
12214 ])),
12215 ]
12216 }
12217
12218 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
12219 struct TypedContract {
12220 nested: TypedNested,
12221 mode: TypedMode,
12222 optional: Option<String>,
12223 absent: Option<String>,
12224 items: Vec<i64>,
12225 labels: BTreeMap<String, String>,
12226 bytes: serde_bytes::ByteBuf,
12227 signed: i64,
12228 finite: f64,
12229 }
12230
12231 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
12232 struct TypedNested {
12233 enabled: bool,
12234 }
12235
12236 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
12237 enum TypedMode {
12238 Detailed { label: String },
12239 }
12240
12241 fn typed_contract() -> TypedContract {
12242 TypedContract {
12243 nested: TypedNested { enabled: true },
12244 mode: TypedMode::Detailed {
12245 label: "compiler-checked".to_string(),
12246 },
12247 optional: Some("present".to_string()),
12248 absent: None,
12249 items: vec![i64::MIN, 0, i64::MAX],
12250 labels: BTreeMap::from([
12251 ("language".to_string(), "rust".to_string()),
12252 ("wire".to_string(), "avro".to_string()),
12253 ]),
12254 bytes: serde_bytes::ByteBuf::from(vec![0, 0xff, 7]),
12255 signed: -9_223_372_036_854_775_000,
12256 finite: 12.5,
12257 }
12258 }
12259
12260 #[derive(Clone, Debug, Default, PartialEq)]
12261 struct ReplayCounterState {
12262 loaded: Option<String>,
12263 count: i64,
12264 finished: bool,
12265 }
12266
12267 fn replay_counter_worker() -> Worker {
12268 let client = Client::new("http://127.0.0.1:8080").expect("client");
12269 let mut worker = Worker::new(client, "rust-workers");
12270 worker.register_replayed_workflow(
12271 "replay-counter",
12272 ReplayCounterState::default,
12273 |ctx, _input, state| async move {
12274 let loaded = ctx.activity("load-counter", json!([])).await?;
12275 state.update(|current| {
12276 current.loaded = loaded.as_str().map(str::to_string);
12277 })?;
12278 for _ in 0..2 {
12279 let signal = ctx.wait_signal("increment").await?;
12280 let amount = signal.first().and_then(Value::as_i64).unwrap_or_default();
12281 state.update(|current| current.count += amount)?;
12282 }
12283 state.update(|current| current.finished = true)?;
12284 state.read(|current| Ok(json!(current.count)))?
12285 },
12286 );
12287 worker.register_replayed_query::<ReplayCounterState, _, _>(
12288 "replay-counter",
12289 "current",
12290 |_ctx, state, _args| async move {
12291 Ok(json!({
12292 "loaded": state.loaded,
12293 "count": state.count,
12294 "finished": state.finished,
12295 }))
12296 },
12297 );
12298 worker.register_replayed_query::<ReplayCounterState, _, _>(
12299 "replay-counter",
12300 "detached-mutation",
12301 |_ctx, state, _args| async move {
12302 let mut detached = (*state).clone();
12303 detached.count = 999;
12304 Ok(json!(detached.count))
12305 },
12306 );
12307 worker.register_replayed_query::<ReplayCounterState, _, _>(
12308 "replay-counter",
12309 "failed-mutation",
12310 |_ctx, state, _args| async move {
12311 let mut detached = (*state).clone();
12312 detached.count = 999;
12313 Err(Error::WorkerLoop("query refused".to_string()))
12314 },
12315 );
12316 worker
12317 }
12318
12319 fn replay_counter_query(
12320 query_name: &str,
12321 history_events: Value,
12322 run_status: &str,
12323 ) -> QueryTask {
12324 let arguments = fixture_envelope(json!([]));
12325 serde_json::from_value(json!({
12326 "query_task_id": format!("query-{query_name}"),
12327 "workflow_type": "replay-counter",
12328 "query_name": query_name,
12329 "payload_codec": DEFAULT_CODEC,
12330 "workflow_arguments": arguments.clone(),
12331 "query_arguments": arguments,
12332 "history_events": history_events,
12333 "run_status": run_status,
12334 }))
12335 .expect("query task")
12336 }
12337
12338 fn workflow_context(history: Vec<HistoryEvent>) -> WorkflowContext {
12339 workflow_context_with_codec(history, DEFAULT_CODEC)
12340 }
12341
12342 fn workflow_context_with_codec(
12343 history: Vec<HistoryEvent>,
12344 payload_codec: &str,
12345 ) -> WorkflowContext {
12346 WorkflowContext {
12347 state: Arc::new(Mutex::new(
12348 WorkflowState::new_with_identity(
12349 history,
12350 None,
12351 None,
12352 "rust-workers".to_string(),
12353 payload_codec.to_string(),
12354 None,
12355 )
12356 .expect("valid workflow history"),
12357 )),
12358 }
12359 }
12360
12361 fn history_event(event_type: &str, payload: Value) -> HistoryEvent {
12362 HistoryEvent {
12363 event_type: event_type.to_string(),
12364 payload,
12365 raw: HashMap::new(),
12366 }
12367 }
12368
12369 fn parallel_path_entry(
12370 kind: &str,
12371 base: u64,
12372 size: usize,
12373 index: usize,
12374 ) -> ParallelGroupMetadata {
12375 parallel_group_entry(base, size, index, kind)
12376 }
12377
12378 fn parallel_history_event(
12379 event_type: &str,
12380 sequence: u64,
12381 identity_field: &str,
12382 identity: &str,
12383 path: Vec<ParallelGroupMetadata>,
12384 result: Option<Value>,
12385 ) -> HistoryEvent {
12386 let mut payload = serde_json::Map::from_iter([
12387 ("sequence".to_string(), json!(sequence)),
12388 (identity_field.to_string(), json!(identity)),
12389 ]);
12390 let inner = path.last().expect("parallel history path");
12391 apply_parallel_group_path(&mut payload, std::slice::from_ref(inner));
12392 payload.insert("parallel_group_path".to_string(), json!(path));
12393 if let Some(result) = result {
12394 let field = if event_type == "ChildRunCompleted" {
12395 "result"
12396 } else {
12397 "result"
12398 };
12399 payload.insert(field.to_string(), fixture_envelope(result));
12400 payload.insert("payload_codec".to_string(), json!(DEFAULT_CODEC));
12401 }
12402 history_event(event_type, Value::Object(payload))
12403 }
12404
12405 fn nested_parallel_operations() -> Vec<ParallelOperation> {
12406 vec![
12407 ParallelOperation::activity("first", json!([])),
12408 ParallelOperation::group(vec![
12409 ParallelOperation::child_workflow(
12410 "second",
12411 ChildWorkflowOptions::new("child-workers"),
12412 json!([]),
12413 ),
12414 ParallelOperation::activity("third", json!([])),
12415 ]),
12416 ]
12417 }
12418
12419 fn nested_parallel_paths() -> [Vec<ParallelGroupMetadata>; 3] {
12420 let outer = [
12421 parallel_path_entry("mixed", 1, 3, 0),
12422 parallel_path_entry("mixed", 1, 3, 1),
12423 parallel_path_entry("mixed", 1, 3, 2),
12424 ];
12425 [
12426 vec![outer[0].clone()],
12427 vec![outer[1].clone(), parallel_path_entry("mixed", 2, 2, 0)],
12428 vec![outer[2].clone(), parallel_path_entry("mixed", 2, 2, 1)],
12429 ]
12430 }
12431
12432 #[test]
12433 fn parallel_schedules_every_nested_mixed_leaf_with_stable_metadata() {
12434 let ctx = workflow_context(Vec::new());
12435 let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
12436 let mut task_context = TaskContext::from_waker(noop_waker_ref());
12437
12438 assert!(matches!(
12439 call.as_mut().poll(&mut task_context),
12440 Poll::Pending
12441 ));
12442 let commands = ctx.take_commands().expect("parallel commands");
12443 assert_eq!(
12444 commands
12445 .iter()
12446 .map(|command| command["type"].as_str().unwrap_or_default())
12447 .collect::<Vec<_>>(),
12448 [
12449 "schedule_activity",
12450 "start_child_workflow",
12451 "schedule_activity"
12452 ]
12453 );
12454 let paths = nested_parallel_paths();
12455 for (command, path) in commands.iter().zip(paths) {
12456 assert_eq!(command["parallel_group_path"], json!(path));
12457 assert_eq!(
12458 command["parallel_group_id"],
12459 json!(path.last().expect("inner group").parallel_group_id)
12460 );
12461 }
12462 }
12463
12464 fn completed_nested_parallel_history() -> Vec<HistoryEvent> {
12465 let paths = nested_parallel_paths();
12466 let third = parallel_history_event(
12467 "ActivityCompleted",
12468 3,
12469 "activity_type",
12470 "third",
12471 paths[2].clone(),
12472 Some(json!("three")),
12473 );
12474 vec![
12475 parallel_history_event(
12476 "ActivityCompleted",
12477 1,
12478 "activity_type",
12479 "first",
12480 paths[0].clone(),
12481 Some(json!("one")),
12482 ),
12483 parallel_history_event(
12484 "ChildWorkflowScheduled",
12485 2,
12486 "child_workflow_type",
12487 "second",
12488 paths[1].clone(),
12489 None,
12490 ),
12491 parallel_history_event(
12492 "ChildRunCompleted",
12493 2,
12494 "child_workflow_type",
12495 "second",
12496 paths[1].clone(),
12497 Some(json!("two")),
12498 ),
12499 third.clone(),
12500 third,
12501 ]
12502 }
12503
12504 #[test]
12505 fn parallel_replay_rebuilds_input_order_and_tolerates_duplicate_delivery() {
12506 for _restart_or_completed_replay in 0..2 {
12507 let ctx = workflow_context(completed_nested_parallel_history());
12508 let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
12509 let mut task_context = TaskContext::from_waker(noop_waker_ref());
12510 let Poll::Ready(Ok(results)) = call.as_mut().poll(&mut task_context) else {
12511 panic!("completed nested parallel history must replay");
12512 };
12513 assert_eq!(
12514 results,
12515 vec![
12516 ParallelResult::Activity(json!("one")),
12517 ParallelResult::Group(vec![
12518 ParallelResult::ChildWorkflow(ChildWorkflowResult {
12519 parent: WorkflowIdentity {
12520 workflow_id: None,
12521 run_id: None,
12522 },
12523 child: WorkflowIdentity {
12524 workflow_id: None,
12525 run_id: None,
12526 },
12527 child_workflow_type: Some("second".to_string()),
12528 result: json!("two"),
12529 }),
12530 ParallelResult::Activity(json!("three")),
12531 ]),
12532 ]
12533 );
12534 assert!(ctx.take_commands().expect("commands").is_empty());
12535 ctx.ensure_history_consumed().expect("history consumed");
12536 }
12537 }
12538
12539 #[test]
12540 fn parallel_failure_keeps_typed_cause_path_and_late_completions() {
12541 let paths = nested_parallel_paths();
12542 let history = vec![
12543 parallel_history_event(
12544 "ActivityCompleted",
12545 1,
12546 "activity_type",
12547 "first",
12548 paths[0].clone(),
12549 Some(json!("one")),
12550 ),
12551 parallel_history_event(
12552 "ChildWorkflowScheduled",
12553 2,
12554 "child_workflow_type",
12555 "second",
12556 paths[1].clone(),
12557 None,
12558 ),
12559 parallel_history_event(
12560 "ChildRunFailed",
12561 2,
12562 "child_workflow_type",
12563 "second",
12564 paths[1].clone(),
12565 None,
12566 ),
12567 parallel_history_event(
12568 "ActivityCompleted",
12569 3,
12570 "activity_type",
12571 "third",
12572 paths[2].clone(),
12573 Some(json!("late")),
12574 ),
12575 ];
12576 let ctx = workflow_context(history);
12577 let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
12578 let mut task_context = TaskContext::from_waker(noop_waker_ref());
12579 let outcome = call.as_mut().poll(&mut task_context);
12580 let Poll::Ready(Err(Error::ParallelFailed(failure))) = outcome else {
12581 panic!("one failed child must return a typed partial failure: {outcome:?}");
12582 };
12583 assert_eq!(failure.member_path, [1, 0]);
12584 assert_eq!(failure.group_id, "parallel-calls:1:3");
12585 assert!(matches!(*failure.cause, Error::ChildWorkflowFailed(_)));
12586 assert_eq!(
12587 failure
12588 .completed
12589 .iter()
12590 .map(|completion| completion.member_path.clone())
12591 .collect::<Vec<_>>(),
12592 [vec![0], vec![1, 1]]
12593 );
12594 }
12595
12596 #[test]
12597 fn pending_parallel_history_restarts_without_rescheduling_any_leaf() {
12598 let paths = nested_parallel_paths();
12599 let history = vec![
12600 parallel_history_event(
12601 "ActivityScheduled",
12602 1,
12603 "activity_type",
12604 "first",
12605 paths[0].clone(),
12606 None,
12607 ),
12608 parallel_history_event(
12609 "ChildWorkflowScheduled",
12610 2,
12611 "child_workflow_type",
12612 "second",
12613 paths[1].clone(),
12614 None,
12615 ),
12616 parallel_history_event(
12617 "ActivityScheduled",
12618 3,
12619 "activity_type",
12620 "third",
12621 paths[2].clone(),
12622 None,
12623 ),
12624 ];
12625 for _restart in 0..2 {
12626 let ctx = workflow_context(history.clone());
12627 let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
12628 let mut task_context = TaskContext::from_waker(noop_waker_ref());
12629 let outcome = call.as_mut().poll(&mut task_context);
12630 assert!(matches!(outcome, Poll::Pending), "{outcome:?}");
12631 assert!(ctx.take_commands().expect("commands").is_empty());
12632 }
12633 }
12634
12635 async fn trip_saga(ctx: WorkflowContext) -> Result<Value> {
12636 let mut saga = ctx.saga();
12637 let outcome = async {
12638 let flight = ctx.activity("trip.reserve-flight", json!([])).await?;
12639 saga.add_compensation("trip.cancel-flight", json!([flight]))?;
12640 let hotel = ctx.activity("trip.reserve-hotel", json!([])).await?;
12641 saga.add_compensation("trip.cancel-hotel", json!([hotel]))?;
12642 ctx.activity("trip.charge", json!([])).await?;
12643 Ok(json!({"status": "booked"}))
12644 }
12645 .await;
12646 saga.finish(outcome).await
12647 }
12648
12649 fn saga_activity(
12650 event_type: &str,
12651 sequence: u64,
12652 activity_type: &str,
12653 result: Option<Value>,
12654 ) -> HistoryEvent {
12655 let mut payload = json!({
12656 "sequence": sequence,
12657 "activity_type": activity_type,
12658 "message": format!("{activity_type} failed"),
12659 "exception_type": "PlannedFailure",
12660 "non_retryable": true,
12661 });
12662 if let Some(result) = result {
12663 payload["result"] = fixture_envelope(result);
12664 }
12665 history_event(event_type, payload)
12666 }
12667
12668 #[test]
12669 fn saga_replays_reverse_compensation_across_restart_and_duplicate_delivery() {
12670 let completed_hotel_compensation = saga_activity(
12671 "ActivityCompleted",
12672 4,
12673 "trip.cancel-hotel",
12674 Some(Value::Null),
12675 );
12676 let history = vec![
12677 saga_activity(
12678 "ActivityCompleted",
12679 1,
12680 "trip.reserve-flight",
12681 Some(json!("flight-1")),
12682 ),
12683 saga_activity(
12684 "ActivityCompleted",
12685 2,
12686 "trip.reserve-hotel",
12687 Some(json!("hotel-1")),
12688 ),
12689 saga_activity("ActivityFailed", 3, "trip.charge", None),
12690 completed_hotel_compensation.clone(),
12691 completed_hotel_compensation,
12692 ];
12693
12694 for _restart in 0..2 {
12695 let ctx = workflow_context(history.clone());
12696 let mut future = Box::pin(trip_saga(ctx.clone()));
12697 let mut task_context = TaskContext::from_waker(noop_waker_ref());
12698 assert!(matches!(
12699 future.as_mut().poll(&mut task_context),
12700 Poll::Pending
12701 ));
12702 let commands = ctx.take_commands().expect("compensation command");
12703 assert_eq!(commands.len(), 1);
12704 assert_eq!(commands[0]["activity_type"], "trip.cancel-flight");
12705 }
12706 }
12707
12708 #[test]
12709 fn saga_compensation_failure_preserves_both_typed_failures() {
12710 let history = vec![
12711 saga_activity(
12712 "ActivityCompleted",
12713 1,
12714 "trip.reserve-flight",
12715 Some(json!("flight-1")),
12716 ),
12717 saga_activity(
12718 "ActivityCompleted",
12719 2,
12720 "trip.reserve-hotel",
12721 Some(json!("hotel-1")),
12722 ),
12723 saga_activity("ActivityFailed", 3, "trip.charge", None),
12724 saga_activity("ActivityFailed", 4, "trip.cancel-hotel", None),
12725 ];
12726 let ctx = workflow_context(history);
12727 let mut future = Box::pin(trip_saga(ctx));
12728 let mut task_context = TaskContext::from_waker(noop_waker_ref());
12729 let Poll::Ready(Err(Error::SagaCompensationFailed(failure))) =
12730 future.as_mut().poll(&mut task_context)
12731 else {
12732 panic!("compensation failure must remain structured");
12733 };
12734 assert!(matches!(
12735 *failure.initiating_failure,
12736 Error::ActivityFailed(_)
12737 ));
12738 assert!(matches!(
12739 *failure.compensation_failure,
12740 Error::ActivityFailed(_)
12741 ));
12742 assert_eq!(failure.compensation_activity_type, "trip.cancel-hotel");
12743 assert_eq!(failure.compensation_registration_order, 2);
12744 }
12745
12746 #[test]
12747 fn saga_compensates_cooperative_cancellation() {
12748 let ctx = workflow_context(vec![saga_activity(
12749 "ActivityCompleted",
12750 1,
12751 "trip.reserve-flight",
12752 Some(json!("flight-1")),
12753 )]);
12754 ctx.state.lock().expect("state").cancel_requested = true;
12755 let run = {
12756 let ctx = ctx.clone();
12757 async move {
12758 let mut saga = ctx.saga();
12759 let outcome = async {
12760 let flight = ctx.activity("trip.reserve-flight", json!([])).await?;
12761 saga.add_compensation("trip.cancel-flight", json!([flight]))?;
12762 ctx.throw_if_cancellation_requested()?;
12763 Ok(json!("unexpected"))
12764 }
12765 .await;
12766 saga.finish(outcome).await
12767 }
12768 };
12769 let mut future = Box::pin(run);
12770 let mut task_context = TaskContext::from_waker(noop_waker_ref());
12771 assert!(matches!(
12772 future.as_mut().poll(&mut task_context),
12773 Poll::Pending
12774 ));
12775 let commands = ctx.take_commands().expect("cancellation compensation");
12776 assert_eq!(commands[0]["activity_type"], "trip.cancel-flight");
12777 }
12778
12779 fn workflow_task(
12780 workflow_type: &str,
12781 history_events: Vec<HistoryEvent>,
12782 payload_codec: &str,
12783 ) -> WorkflowTask {
12784 WorkflowTask {
12785 task_id: format!("wft-{workflow_type}"),
12786 workflow_command_id: None,
12787 workflow_id: Some(format!("wf-{workflow_type}")),
12788 run_id: Some(format!("run-{workflow_type}")),
12789 workflow_type: workflow_type.to_string(),
12790 cancel_requested: false,
12791 payload_codec: payload_codec.to_string(),
12792 arguments: Some(
12793 encode_value_envelope(&json!([]), payload_codec).expect("workflow arguments"),
12794 ),
12795 total_history_events: Some(history_events.len() as u64),
12796 history_size_bytes: None,
12797 continue_as_new_recommended: None,
12798 history_budget_pressure: None,
12799 history_events,
12800 next_history_page_token: None,
12801 workflow_task_attempt: 1,
12802 workflow_signal_id: None,
12803 signal_name: None,
12804 signal_arguments: None,
12805 workflow_update_id: None,
12806 update_name: None,
12807 lease_owner: Some("rust-worker".to_string()),
12808 }
12809 }
12810
12811 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
12812 struct SideEffectProbe {
12813 request_id: String,
12814 attempt: u32,
12815 }
12816
12817 #[test]
12818 fn typed_side_effect_runs_callback_once_and_replay_skips_it() {
12819 let calls = AtomicUsize::new(0);
12820 let ctx = workflow_context(Vec::new());
12821 let value = ctx
12822 .side_effect(|| {
12823 calls.fetch_add(1, Ordering::SeqCst);
12824 SideEffectProbe {
12825 request_id: "request-42".to_string(),
12826 attempt: 3,
12827 }
12828 })
12829 .expect("first side effect");
12830 assert_eq!(value.attempt, 3);
12831 assert_eq!(calls.load(Ordering::SeqCst), 1);
12832 let commands = ctx.take_commands().expect("commands");
12833 assert_eq!(commands.len(), 1);
12834 assert_eq!(commands[0]["type"], "record_side_effect");
12835 assert_eq!(
12836 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
12837 serde_json::to_value(&value).expect("value")
12838 );
12839
12840 let replay = workflow_context(vec![history_event(
12841 "SideEffectRecorded",
12842 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
12843 )]);
12844 let replayed: SideEffectProbe = replay
12845 .side_effect(|| {
12846 calls.fetch_add(1, Ordering::SeqCst);
12847 panic!("committed side-effect callbacks must not run during replay")
12848 })
12849 .expect("replayed side effect");
12850 assert_eq!(replayed, value);
12851 assert_eq!(calls.load(Ordering::SeqCst), 1);
12852 assert!(replay.take_commands().expect("commands").is_empty());
12853 replay.ensure_history_consumed().expect("history consumed");
12854 }
12855
12856 #[test]
12857 fn side_effect_uses_avro_envelope_and_uuid_is_replay_stable() {
12858 let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
12859 let value = ctx
12860 .side_effect(|| SideEffectProbe {
12861 request_id: "avro-request".to_string(),
12862 attempt: 1,
12863 })
12864 .expect("Avro side effect");
12865 let uuid = ctx.uuid_v4().expect("deterministic UUID");
12866 let commands = ctx.take_commands().expect("commands");
12867 assert_eq!(commands.len(), 2);
12868 assert_eq!(commands[0]["result"]["codec"], DEFAULT_CODEC);
12869 assert_eq!(commands[1]["result"]["codec"], DEFAULT_CODEC);
12870 assert_eq!(
12871 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
12872 serde_json::to_value(&value).expect("value")
12873 );
12874
12875 let replay = workflow_context_with_codec(
12876 vec![
12877 history_event(
12878 "SideEffectRecorded",
12879 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
12880 ),
12881 history_event(
12882 "SideEffectRecorded",
12883 json!({"sequence": 2, "result": commands[1]["result"].clone()}),
12884 ),
12885 ],
12886 DEFAULT_CODEC,
12887 );
12888 let replayed: SideEffectProbe = replay
12889 .side_effect(|| panic!("Avro callback must not run"))
12890 .expect("replayed Avro value");
12891 let replayed_uuid = replay.uuid_v4().expect("replayed UUID");
12892 assert_eq!(replayed, value);
12893 assert_eq!(replayed_uuid, uuid);
12894 assert!(replay.take_commands().expect("commands").is_empty());
12895 }
12896
12897 #[test]
12898 fn typed_side_effect_replay_preserves_bytes_and_maps() {
12899 let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
12900 let value = ctx
12901 .side_effect_avro_value(typed_fidelity_probe)
12902 .expect("typed side effect");
12903 let commands = ctx.take_commands().expect("side-effect command");
12904 assert_eq!(
12905 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
12906 .expect("recorded side effect"),
12907 value
12908 );
12909
12910 let replay = workflow_context_with_codec(
12911 vec![history_event(
12912 "SideEffectRecorded",
12913 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
12914 )],
12915 DEFAULT_CODEC,
12916 );
12917 assert_eq!(
12918 replay
12919 .side_effect_avro_value(|| panic!("replay must not invoke callback"))
12920 .expect("replayed typed side effect"),
12921 value
12922 );
12923 }
12924
12925 #[test]
12926 fn ordered_side_effects_share_the_durable_command_stream() {
12927 let first = encode_value_envelope(&json!("first"), DEFAULT_CODEC).expect("first");
12928 let second = encode_value_envelope(&json!(29), DEFAULT_CODEC).expect("second");
12929 let ctx = workflow_context(vec![
12930 history_event(
12931 "SideEffectRecorded",
12932 json!({"sequence": 1, "result": first}),
12933 ),
12934 history_event(
12935 "SideEffectRecorded",
12936 json!({"sequence": 2, "result": second}),
12937 ),
12938 ]);
12939 let first: String = ctx
12940 .side_effect(|| panic!("first callback must not run"))
12941 .expect("first replay");
12942 let second: i32 = ctx
12943 .side_effect(|| panic!("second callback must not run"))
12944 .expect("second replay");
12945 assert_eq!(first, "first");
12946 assert_eq!(second, 29);
12947 ctx.ensure_history_consumed().expect("ordered history");
12948
12949 let reordered = workflow_context(vec![history_event(
12950 "VersionMarkerRecorded",
12951 json!({
12952 "sequence": 1,
12953 "change_id": "before-side-effect",
12954 "version": 1,
12955 "min_supported": 1,
12956 "max_supported": 1,
12957 }),
12958 )]);
12959 let error = reordered
12960 .side_effect(|| "new".to_string())
12961 .expect_err("command reordering must fail");
12962 assert!(matches!(
12963 error,
12964 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
12965 if reason == "recorded_command_mismatch"
12966 ));
12967 }
12968
12969 #[test]
12970 fn version_markers_replay_across_upgrades_and_do_not_duplicate() {
12971 let ctx = workflow_context(Vec::new());
12972 assert_eq!(ctx.get_version("checkout-v2", 1, 2).expect("version"), 2);
12973 assert_eq!(ctx.get_version("checkout-v2", 1, 3).expect("cached"), 2);
12974 assert!(ctx.patched("new-search").expect("patch"));
12975 ctx.deprecate_patch("new-search").expect("deprecate patch");
12976 let commands = ctx.take_commands().expect("commands");
12977 assert_eq!(commands.len(), 2);
12978 assert_eq!(commands[0]["type"], "record_version_marker");
12979 assert_eq!(commands[0]["version"], 2);
12980 assert_eq!(commands[1]["change_id"], "new-search");
12981
12982 let replay = workflow_context(vec![history_event(
12983 "VersionMarkerRecorded",
12984 json!({
12985 "sequence": 1,
12986 "change_id": "checkout-v2",
12987 "version": 2,
12988 "min_supported": 1,
12989 "max_supported": 2,
12990 }),
12991 )]);
12992 assert_eq!(replay.get_version("checkout-v2", 1, 4).expect("upgrade"), 2);
12993 assert_eq!(replay.get_version("checkout-v2", 2, 5).expect("repeat"), 2);
12994 assert!(replay.take_commands().expect("commands").is_empty());
12995 replay.ensure_history_consumed().expect("history consumed");
12996 }
12997
12998 #[test]
12999 fn version_markers_reject_incompatible_or_malformed_history() {
13000 let incompatible = workflow_context(vec![history_event(
13001 "VersionMarkerRecorded",
13002 json!({
13003 "sequence": 1,
13004 "change_id": "checkout-v2",
13005 "version": 1,
13006 "min_supported": 1,
13007 "max_supported": 2,
13008 }),
13009 )]);
13010 let error = incompatible
13011 .get_version("checkout-v2", 2, 3)
13012 .expect_err("old version is unsupported");
13013 assert!(matches!(
13014 error,
13015 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
13016 if reason == "version_marker_incompatible_range"
13017 ));
13018
13019 for (history, reason) in [
13020 (
13021 vec![history_event("SideEffectRecorded", json!({"sequence": 1}))],
13022 "side_effect_result_missing",
13023 ),
13024 (
13025 vec![history_event(
13026 "SideEffectRecorded",
13027 json!({
13028 "sequence": 1,
13029 "result": {"codec": "avro", "blob": "not-base64"},
13030 }),
13031 )],
13032 "side_effect_payload_incompatible",
13033 ),
13034 (
13035 vec![history_event(
13036 "SideEffectRecorded",
13037 json!({"sequence": 1, "result": {"unwrapped": true}}),
13038 )],
13039 "side_effect_payload_malformed",
13040 ),
13041 (
13042 vec![history_event(
13043 "VersionMarkerRecorded",
13044 json!({
13045 "sequence": 1,
13046 "change_id": "change",
13047 "version": 1,
13048 "min_supported": 2,
13049 "max_supported": 1,
13050 }),
13051 )],
13052 "version_marker_history_range_invalid",
13053 ),
13054 ] {
13055 let error = WorkflowState::new(
13056 history,
13057 "rust-workers".to_string(),
13058 DEFAULT_CODEC.to_string(),
13059 None,
13060 )
13061 .expect_err("malformed history must fail");
13062 assert!(matches!(
13063 error,
13064 Error::NonDeterministicReplay(ReplayFailure { reason: actual, .. })
13065 if actual == reason
13066 ));
13067 }
13068 }
13069
13070 #[test]
13071 fn typed_search_attributes_replay_value_and_type_identity_after_restart() {
13072 let history = vec![history_event(
13073 "SearchAttributesUpserted",
13074 json!({
13075 "sequence": 1,
13076 "attributes": {"customer_tier": "gold"},
13077 "attribute_types": {"customer_tier": "keyword"},
13078 "merged": {"customer_tier": "gold"}
13079 }),
13080 )];
13081
13082 let matching = workflow_context(history.clone());
13083 matching
13084 .upsert_search_attributes(
13085 SearchAttributeUpdate::new()
13086 .keyword("customer_tier", "gold")
13087 .expect("keyword update"),
13088 )
13089 .expect("matching typed update must replay");
13090 matching
13091 .ensure_history_consumed()
13092 .expect("history consumed");
13093
13094 let changed_type = workflow_context(history.clone());
13095 let error = changed_type
13096 .upsert_search_attributes(
13097 SearchAttributeUpdate::new()
13098 .string("customer_tier", "gold")
13099 .expect("string update"),
13100 )
13101 .expect_err("same JSON value with a different declaration must be nondeterministic");
13102 let Error::NonDeterministicReplay(failure) = error else {
13103 panic!("typed identity drift must be a replay failure");
13104 };
13105 assert_eq!(failure.reason, "search_attribute_type_mismatch");
13106 assert_eq!(failure.sequence, Some(1));
13107
13108 let changed_value = workflow_context(history);
13109 let error = changed_value
13110 .upsert_search_attributes(
13111 SearchAttributeUpdate::new()
13112 .keyword("customer_tier", "platinum")
13113 .expect("keyword update"),
13114 )
13115 .expect_err("changed values must be nondeterministic");
13116 let Error::NonDeterministicReplay(failure) = error else {
13117 panic!("value drift must be a replay failure");
13118 };
13119 assert_eq!(failure.reason, "search_attribute_value_mismatch");
13120 }
13121
13122 #[test]
13123 fn legacy_search_attribute_history_keeps_type_identity_unknown() {
13124 let history = vec![history_event(
13125 "SearchAttributesUpserted",
13126 json!({
13127 "sequence": 1,
13128 "attributes": {"customer_tier": "gold"},
13129 "merged": {"customer_tier": "gold"}
13130 }),
13131 )];
13132
13133 for update in [
13134 SearchAttributeUpdate::new()
13135 .keyword("customer_tier", "gold")
13136 .expect("keyword update"),
13137 SearchAttributeUpdate::new()
13138 .string("customer_tier", "gold")
13139 .expect("string update"),
13140 ] {
13141 let restarted = workflow_context(history.clone());
13142 restarted
13143 .upsert_search_attributes(update)
13144 .expect("legacy history constrains values but has unknown type identity");
13145 restarted
13146 .ensure_history_consumed()
13147 .expect("history consumed");
13148 }
13149 }
13150
13151 #[test]
13152 fn search_attribute_command_emits_canonical_types() {
13153 let ctx = workflow_context(Vec::new());
13154 ctx.upsert_search_attributes(
13155 SearchAttributeUpdate::new()
13156 .keyword("customer_tier", "gold")
13157 .expect("keyword update")
13158 .int("attempts", 3)
13159 .expect("int update")
13160 .delete("obsolete")
13161 .expect("delete update"),
13162 )
13163 .expect("valid search attributes");
13164
13165 assert_eq!(
13166 ctx.take_commands().expect("commands"),
13167 vec![json!({
13168 "type": "upsert_search_attributes",
13169 "attributes": {
13170 "attempts": 3,
13171 "customer_tier": "gold",
13172 "obsolete": null
13173 },
13174 "attribute_types": {
13175 "attempts": "int",
13176 "customer_tier": "keyword"
13177 }
13178 })]
13179 );
13180 }
13181
13182 #[test]
13183 fn duplicate_side_effects_and_version_markers_are_rejected() {
13184 let duplicate_side_effect = WorkflowState::new(
13185 vec![
13186 history_event(
13187 "SideEffectRecorded",
13188 json!({"sequence": 1, "result": fixture_envelope(json!(1))}),
13189 ),
13190 history_event(
13191 "SideEffectRecorded",
13192 json!({"sequence": 1, "result": fixture_envelope(json!(2))}),
13193 ),
13194 ],
13195 "rust-workers".to_string(),
13196 DEFAULT_CODEC.to_string(),
13197 None,
13198 )
13199 .expect_err("duplicate side effect");
13200 assert!(matches!(
13201 duplicate_side_effect,
13202 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
13203 if reason == "duplicate_side_effect_record"
13204 ));
13205
13206 let marker = |sequence| {
13207 history_event(
13208 "VersionMarkerRecorded",
13209 json!({
13210 "sequence": sequence,
13211 "change_id": "same-change",
13212 "version": 1,
13213 "min_supported": 1,
13214 "max_supported": 1,
13215 }),
13216 )
13217 };
13218 let duplicate_marker = WorkflowState::new(
13219 vec![marker(1), marker(3)],
13220 "rust-workers".to_string(),
13221 DEFAULT_CODEC.to_string(),
13222 None,
13223 )
13224 .expect_err("duplicate marker");
13225 assert!(matches!(
13226 duplicate_marker,
13227 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
13228 if reason == "duplicate_version_marker"
13229 ));
13230 }
13231
13232 #[test]
13233 fn workflow_stream_authoring_derives_identity_and_replay_skips_duplicate_append() {
13234 let mut state = WorkflowState::new(
13235 Vec::new(),
13236 "rust-workers".to_string(),
13237 DEFAULT_CODEC.to_string(),
13238 None,
13239 )
13240 .expect("workflow state");
13241 state.workflow_command_identity = "command-7".to_string();
13242 let context = WorkflowContext {
13243 state: Arc::new(Mutex::new(state)),
13244 };
13245 let item =
13246 WorkflowStreamAppendItem::from_reference("s3://bucket/item.avro").item_type("receipt");
13247
13248 context
13249 .append_workflow_stream("output", &[item], Some(10))
13250 .expect("append command");
13251 context
13252 .error_workflow_stream("output", "producer failed", None)
13253 .expect("error command");
13254 let commands = context.take_commands().expect("commands");
13255
13256 assert_eq!(commands[0]["type"], "record_side_effect");
13257 assert_eq!(
13258 commands[0]["workflow_stream"]["command_identity"],
13259 "command-7"
13260 );
13261 assert_eq!(commands[0]["workflow_stream"]["command_ordinal"], 0);
13262 assert_eq!(
13263 commands[0]["workflow_stream"]["items"][0]["idempotency_key"],
13264 "dw-stream:command-7:0:0"
13265 );
13266 assert_eq!(commands[1]["workflow_stream"]["operation"], "error");
13267
13268 let recorded = history_event(
13269 "SideEffectRecorded",
13270 json!({"sequence": 1, "result": fixture_envelope(Value::Null)}),
13271 );
13272 let mut replay_state = WorkflowState::new(
13273 vec![recorded],
13274 "rust-workers".to_string(),
13275 DEFAULT_CODEC.to_string(),
13276 None,
13277 )
13278 .expect("replay state");
13279 replay_state.workflow_command_identity = "command-7".to_string();
13280 let replay_context = WorkflowContext {
13281 state: Arc::new(Mutex::new(replay_state)),
13282 };
13283 replay_context
13284 .append_workflow_stream(
13285 "output",
13286 &[WorkflowStreamAppendItem::from_reference(
13287 "s3://bucket/item.avro",
13288 )],
13289 Some(10),
13290 )
13291 .expect("replayed append");
13292 assert!(replay_context
13293 .take_commands()
13294 .expect("replayed commands")
13295 .is_empty());
13296 }
13297
13298 #[test]
13299 fn workflow_stream_authoring_requires_server_durable_command_identity() {
13300 let context = workflow_context(Vec::new());
13301 let error = context
13302 .append_workflow_stream(
13303 "output",
13304 &[WorkflowStreamAppendItem::from_reference(
13305 "s3://bucket/item.avro",
13306 )],
13307 None,
13308 )
13309 .expect_err("stream append without durable command identity must fail closed");
13310
13311 assert!(matches!(error, Error::MissingWorkflowCommandIdentity));
13312 assert!(context.take_commands().expect("commands").is_empty());
13313 }
13314
13315 #[test]
13316 fn cold_worker_replay_does_not_repeat_committed_side_effects_or_markers() {
13317 fn worker(calls: Arc<AtomicUsize>) -> Worker {
13318 let client = Client::new("http://127.0.0.1:8080").expect("client");
13319 let mut worker = Worker::new(client, "rust-workers");
13320 worker.register_workflow("rust.side-effect-version", move |ctx, _input| {
13321 let calls = Arc::clone(&calls);
13322 async move {
13323 let captured = ctx.side_effect(|| {
13324 calls.fetch_add(1, Ordering::SeqCst);
13325 "captured-once".to_string()
13326 })?;
13327 let version = ctx.get_version("cold-restart", 1, 2)?;
13328 Ok(json!({"captured": captured, "version": version}))
13329 }
13330 });
13331 worker
13332 }
13333
13334 fn task(history_events: Vec<HistoryEvent>) -> WorkflowTask {
13335 WorkflowTask {
13336 task_id: "wft-side-effect-version".to_string(),
13337 workflow_command_id: None,
13338 workflow_id: Some("wf-side-effect-version".to_string()),
13339 run_id: Some("run-side-effect-version".to_string()),
13340 workflow_type: "rust.side-effect-version".to_string(),
13341 cancel_requested: false,
13342 payload_codec: DEFAULT_CODEC.to_string(),
13343 arguments: Some(
13344 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("arguments"),
13345 ),
13346 history_events,
13347 total_history_events: None,
13348 history_size_bytes: None,
13349 continue_as_new_recommended: None,
13350 history_budget_pressure: None,
13351 next_history_page_token: None,
13352 workflow_task_attempt: 1,
13353 workflow_signal_id: None,
13354 signal_name: None,
13355 signal_arguments: None,
13356 workflow_update_id: None,
13357 update_name: None,
13358 lease_owner: Some("rust-worker".to_string()),
13359 }
13360 }
13361
13362 let calls = Arc::new(AtomicUsize::new(0));
13363 let initial = worker(Arc::clone(&calls))
13364 .execute_workflow_task(task(Vec::new()))
13365 .expect("initial execution");
13366 assert_eq!(
13367 initial
13368 .iter()
13369 .map(|command| &command["type"])
13370 .collect::<Vec<_>>(),
13371 vec![
13372 "record_side_effect",
13373 "record_version_marker",
13374 "complete_workflow"
13375 ]
13376 );
13377 assert_eq!(calls.load(Ordering::SeqCst), 1);
13378
13379 let restarted = worker(Arc::clone(&calls));
13380 let replayed = restarted
13381 .execute_workflow_task(task(vec![
13382 history_event(
13383 "SideEffectRecorded",
13384 json!({"sequence": 1, "result": initial[0]["result"].clone()}),
13385 ),
13386 history_event(
13387 "VersionMarkerRecorded",
13388 json!({
13389 "sequence": 2,
13390 "change_id": "cold-restart",
13391 "version": 2,
13392 "min_supported": 1,
13393 "max_supported": 2,
13394 }),
13395 ),
13396 ]))
13397 .expect("cold replay");
13398 assert_eq!(replayed.len(), 1);
13399 assert_eq!(replayed[0]["type"], "complete_workflow");
13400 assert_eq!(calls.load(Ordering::SeqCst), 1);
13401 }
13402
13403 #[test]
13404 fn side_effect_replay_rejects_changed_rust_value_type() {
13405 let result = encode_value_envelope(&json!({"value": 42}), DEFAULT_CODEC).expect("result");
13406 let ctx = workflow_context(vec![history_event(
13407 "SideEffectRecorded",
13408 json!({"sequence": 1, "result": result}),
13409 )]);
13410 let error = ctx
13411 .side_effect::<Vec<String>, _>(|| panic!("callback must not run"))
13412 .expect_err("changed type must fail replay");
13413 assert!(matches!(
13414 error,
13415 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
13416 if reason == "side_effect_type_mismatch"
13417 ));
13418 }
13419
13420 fn completed_retry_activity_history() -> Vec<HistoryEvent> {
13421 vec![
13422 history_event(
13423 "ActivityScheduled",
13424 json!({
13425 "sequence": 1,
13426 "activity_type": "flaky",
13427 "activity_execution_id": "act-1",
13428 "activity": {
13429 "id": "act-1",
13430 "sequence": 1,
13431 "type": "flaky",
13432 "queue": "critical-activities",
13433 "execution_mode": null,
13434 "retry_policy": {
13435 "snapshot_version": 1,
13436 "max_attempts": 3,
13437 "backoff_seconds": [2, 4],
13438 "start_to_close_timeout": 30,
13439 "schedule_to_start_timeout": 5,
13440 "schedule_to_close_timeout": 90,
13441 "heartbeat_timeout": 10,
13442 "non_retryable_error_types": ["PermanentError"]
13443 }
13444 }
13445 }),
13446 ),
13447 history_event(
13448 "ActivityStarted",
13449 json!({
13450 "sequence": 1,
13451 "activity_type": "flaky",
13452 "activity_execution_id": "act-1",
13453 "activity_attempt_id": "attempt-1",
13454 "attempt_number": 1
13455 }),
13456 ),
13457 history_event(
13458 "ActivityRetryScheduled",
13459 json!({
13460 "sequence": 1,
13461 "activity_type": "flaky",
13462 "activity_execution_id": "act-1",
13463 "activity_attempt_id": "attempt-1",
13464 "attempt_number": 1,
13465 "retry_after_attempt": 1,
13466 "retry_backoff_seconds": 2,
13467 "failure_category": "activity",
13468 "exception_type": "TransientError"
13469 }),
13470 ),
13471 history_event(
13472 "ActivityStarted",
13473 json!({
13474 "sequence": 1,
13475 "activity_type": "flaky",
13476 "activity_execution_id": "act-1",
13477 "activity_attempt_id": "attempt-2",
13478 "attempt_number": 2
13479 }),
13480 ),
13481 history_event(
13482 "ActivityCompleted",
13483 json!({
13484 "sequence": 1,
13485 "activity_type": "flaky",
13486 "activity_execution_id": "act-1",
13487 "activity_attempt_id": "attempt-2",
13488 "attempt_number": 2,
13489 "payload_codec": DEFAULT_CODEC,
13490 "result": fixture_envelope(json!({"status":"recovered"}))
13491 }),
13492 ),
13493 ]
13494 }
13495
13496 fn retry_activity_options() -> ActivityOptions {
13497 ActivityOptions::new()
13498 .task_queue("critical-activities")
13499 .retry_policy(
13500 ActivityRetryPolicy::new(3)
13501 .backoff_intervals([Duration::from_secs(2), Duration::from_secs(4)])
13502 .non_retryable_error_type("PermanentError"),
13503 )
13504 .start_to_close_timeout(Duration::from_secs(30))
13505 .schedule_to_start_timeout(Duration::from_secs(5))
13506 .schedule_to_close_timeout(Duration::from_secs(90))
13507 .heartbeat_timeout(Duration::from_secs(10))
13508 }
13509
13510 #[test]
13511 fn fixed_avro_value_round_trips_json_values() {
13512 let value = json!({"greeting": "hello", "count": 3, "ok": true});
13513 let envelope = PayloadEnvelope::avro(&value).expect("encode");
13514 assert_eq!(envelope.codec, DEFAULT_CODEC);
13515 assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
13516 }
13517
13518 #[tokio::test]
13519 async fn typed_handler_adapters_round_trip_serde_contracts_on_the_fixed_wire() {
13520 let client = Client::new("http://127.0.0.1:8080").expect("client");
13521 let mut worker = Worker::new(client, "rust-workers");
13522 worker.register_typed_workflow(
13523 "typed.contract.workflow",
13524 |_ctx, input: TypedContract| async move { Ok(input) },
13525 );
13526 worker.register_typed_activity(
13527 "typed.contract.activity",
13528 |_ctx, input: TypedContract| async move { Ok(input) },
13529 );
13530
13531 let expected = typed_contract();
13532 let arguments = AvroValue::Array(vec![
13533 AvroValue::from_serialize(&expected).expect("typed request")
13534 ]);
13535 let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("arguments");
13536 let mut workflow = workflow_task("typed.contract.workflow", Vec::new(), DEFAULT_CODEC);
13537 workflow.arguments = Some(envelope.clone());
13538 let commands = worker
13539 .execute_workflow_task(workflow)
13540 .expect("typed workflow task");
13541 let workflow_result: TypedContract =
13542 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
13543 .expect("workflow result envelope")
13544 .deserialize()
13545 .expect("workflow result type");
13546 assert_eq!(workflow_result, expected);
13547
13548 let activity = ActivityTask {
13549 task_id: "typed-contract-activity".to_string(),
13550 activity_attempt_id: Some("typed-contract-attempt".to_string()),
13551 attempt_id: None,
13552 activity_type: "typed.contract.activity".to_string(),
13553 payload_codec: DEFAULT_CODEC.to_string(),
13554 arguments: Some(envelope),
13555 attempt_number: 1,
13556 lease_owner: Some("rust-worker".to_string()),
13557 };
13558 let activity_result: TypedContract = worker
13559 .execute_activity_task(activity)
13560 .await
13561 .expect("typed activity task")
13562 .deserialize()
13563 .expect("activity result type");
13564 assert_eq!(activity_result, expected);
13565 }
13566
13567 #[tokio::test]
13568 async fn typed_handler_errors_include_handler_name_direction_and_rust_type() {
13569 let client = Client::new("http://127.0.0.1:8080").expect("client");
13570 let mut worker = Worker::new(client, "rust-workers");
13571 worker.register_typed_workflow(
13572 "typed.shape.workflow",
13573 |_ctx, input: TypedContract| async move { Ok(input) },
13574 );
13575 worker.register_typed_activity("typed.unsupported.activity", |_ctx, (): ()| async move {
13576 Ok(f64::NAN)
13577 });
13578
13579 let mut workflow = workflow_task("typed.shape.workflow", Vec::new(), DEFAULT_CODEC);
13580 workflow.arguments = Some(
13581 encode_typed_envelope(
13582 &AvroValue::Array(vec![
13583 AvroValue::String("first".to_string()),
13584 AvroValue::String("second".to_string()),
13585 ]),
13586 DEFAULT_CODEC,
13587 )
13588 .expect("malformed typed arguments"),
13589 );
13590 let commands = worker
13591 .execute_workflow_task(workflow)
13592 .expect("shape mismatch becomes a workflow failure");
13593 let message = commands[0]["message"].as_str().expect("failure message");
13594 assert!(message.contains("workflow handler \"typed.shape.workflow\" input type"));
13595 assert!(message.contains(type_name::<TypedContract>()));
13596 assert!(message.contains("task carried 2 arguments"));
13597
13598 let activity = ActivityTask {
13599 task_id: "typed-unsupported-activity".to_string(),
13600 activity_attempt_id: Some("typed-unsupported-attempt".to_string()),
13601 attempt_id: None,
13602 activity_type: "typed.unsupported.activity".to_string(),
13603 payload_codec: DEFAULT_CODEC.to_string(),
13604 arguments: Some(
13605 encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
13606 .expect("unit arguments"),
13607 ),
13608 attempt_number: 1,
13609 lease_owner: Some("rust-worker".to_string()),
13610 };
13611 let Error::HandlerType {
13612 handler_kind,
13613 handler_name,
13614 value_kind,
13615 rust_type,
13616 message,
13617 } = worker
13618 .execute_activity_task(activity)
13619 .await
13620 .expect_err("non-finite handler output must fail")
13621 else {
13622 panic!("expected contextual handler type failure");
13623 };
13624 assert_eq!(handler_kind, HandlerKind::Activity);
13625 assert_eq!(handler_name, "typed.unsupported.activity");
13626 assert_eq!(value_kind, HandlerValueKind::Result);
13627 assert_eq!(rust_type, type_name::<f64>());
13628 assert!(message.contains("non_finite_float"));
13629 }
13630
13631 #[tokio::test]
13632 async fn typed_replayed_workflow_decodes_input_and_activity_result_losslessly() {
13633 #[derive(Clone, Default)]
13634 struct State {
13635 observed: Option<TypedContract>,
13636 }
13637
13638 let client = Client::new("http://127.0.0.1:8080").expect("client");
13639 let mut worker = Worker::new(client, "rust-workers");
13640 worker.register_typed_replayed_workflow(
13641 "typed.contract.replayed",
13642 State::default,
13643 |ctx, input: TypedContract, state| async move {
13644 let result: TypedContract =
13645 ctx.activity_typed("typed.contract.activity", input).await?;
13646 state.update(|current| current.observed = Some(result.clone()))?;
13647 Ok(result)
13648 },
13649 );
13650 worker.register_replayed_query::<State, _, _>(
13651 "typed.contract.replayed",
13652 "observed",
13653 |_ctx, state, _args| async move {
13654 Ok(json!(state.observed.as_ref().map(|value| value.signed)))
13655 },
13656 );
13657
13658 let expected = typed_contract();
13659 let typed_value = AvroValue::from_serialize(&expected).expect("typed value");
13660 let workflow_arguments =
13661 encode_typed_envelope(&AvroValue::Array(vec![typed_value.clone()]), DEFAULT_CODEC)
13662 .expect("workflow arguments");
13663 let result = encode_typed_envelope(&typed_value, DEFAULT_CODEC).expect("activity result");
13664 let task = QueryTask {
13665 query_task_id: "typed-replay-query".to_string(),
13666 query_task_attempt: 1,
13667 lease_owner: Some("rust-worker".to_string()),
13668 workflow_id: Some("typed-replay".to_string()),
13669 run_id: Some("typed-replay-run".to_string()),
13670 workflow_type: "typed.contract.replayed".to_string(),
13671 query_name: "observed".to_string(),
13672 payload_codec: DEFAULT_CODEC.to_string(),
13673 workflow_arguments: Some(workflow_arguments),
13674 query_arguments: Some(
13675 encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
13676 .expect("query arguments"),
13677 ),
13678 history_events: vec![
13679 history_event(
13680 "ActivityScheduled",
13681 json!({
13682 "sequence": 1,
13683 "activity_type": "typed.contract.activity"
13684 }),
13685 ),
13686 history_event(
13687 "ActivityCompleted",
13688 json!({
13689 "sequence": 1,
13690 "activity_type": "typed.contract.activity",
13691 "payload_codec": DEFAULT_CODEC,
13692 "result": result
13693 }),
13694 ),
13695 ],
13696 history_export: None,
13697 run_status: Some("completed".to_string()),
13698 };
13699
13700 assert_eq!(
13701 worker
13702 .execute_query_task(task)
13703 .await
13704 .expect("typed replay query")
13705 .deserialize::<i64>()
13706 .expect("query result"),
13707 expected.signed
13708 );
13709 }
13710
13711 #[tokio::test]
13712 async fn typed_worker_surfaces_preserve_bytes_and_map_list_identity() {
13713 let client = Client::new("http://127.0.0.1:8080").expect("client");
13714 let mut worker = Worker::new(client, "rust-workers");
13715 worker.register_workflow_avro_value("typed.echo", |_ctx, input| async move { Ok(input) });
13716 worker
13717 .register_activity_avro_value("typed.activity", |_ctx, input| async move { Ok(input) });
13718 worker.register_query_avro_value("typed.echo", "inspect", |_ctx, input| async move {
13719 Ok(input)
13720 });
13721 worker.register_update_avro_value("typed.echo", "replace", |_ctx, input| async move {
13722 Ok(input)
13723 });
13724 worker.register_workflow_avro_value("typed.signal", |ctx, _input| async move {
13725 Ok(AvroValue::Array(
13726 ctx.wait_signal_avro_value("changed").await?,
13727 ))
13728 });
13729
13730 let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
13731 let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed envelope");
13732
13733 let mut workflow = workflow_task("typed.echo", Vec::new(), DEFAULT_CODEC);
13734 workflow.arguments = Some(envelope.clone());
13735 let commands = worker
13736 .execute_workflow_task(workflow)
13737 .expect("typed workflow task");
13738 assert_eq!(commands[0]["type"], "complete_workflow");
13739 assert_eq!(
13740 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
13741 .expect("typed workflow result"),
13742 arguments
13743 );
13744
13745 let activity = ActivityTask {
13746 task_id: "activity-typed".to_string(),
13747 activity_attempt_id: Some("attempt-typed".to_string()),
13748 attempt_id: None,
13749 activity_type: "typed.activity".to_string(),
13750 payload_codec: DEFAULT_CODEC.to_string(),
13751 arguments: Some(envelope.clone()),
13752 attempt_number: 1,
13753 lease_owner: Some("rust-worker".to_string()),
13754 };
13755 assert_eq!(
13756 worker
13757 .execute_activity_task(activity)
13758 .await
13759 .expect("typed activity result"),
13760 arguments
13761 );
13762
13763 let query = QueryTask {
13764 query_task_id: "query-typed".to_string(),
13765 query_task_attempt: 1,
13766 lease_owner: Some("rust-worker".to_string()),
13767 workflow_id: Some("typed-1".to_string()),
13768 run_id: Some("run-typed".to_string()),
13769 workflow_type: "typed.echo".to_string(),
13770 query_name: "inspect".to_string(),
13771 payload_codec: DEFAULT_CODEC.to_string(),
13772 workflow_arguments: Some(
13773 encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
13774 .expect("workflow input"),
13775 ),
13776 query_arguments: Some(envelope.clone()),
13777 history_events: Vec::new(),
13778 history_export: None,
13779 run_status: Some("running".to_string()),
13780 };
13781 assert_eq!(
13782 worker
13783 .execute_query_task(query)
13784 .await
13785 .expect("typed query result"),
13786 arguments
13787 );
13788
13789 let mut update = workflow_task(
13790 "typed.echo",
13791 vec![history_event(
13792 "UpdateAccepted",
13793 json!({
13794 "update_id": "update-typed",
13795 "update_name": "replace",
13796 "arguments": envelope.clone(),
13797 }),
13798 )],
13799 DEFAULT_CODEC,
13800 );
13801 update.workflow_update_id = Some("update-typed".to_string());
13802 update.update_name = Some("replace".to_string());
13803 let commands = worker
13804 .execute_workflow_task(update)
13805 .expect("typed update task");
13806 assert_eq!(commands[0]["type"], "complete_update");
13807 assert_eq!(
13808 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
13809 .expect("typed update result"),
13810 arguments
13811 );
13812
13813 let mut signal = workflow_task(
13814 "typed.signal",
13815 vec![history_event(
13816 "SignalReceived",
13817 json!({
13818 "signal_id": "signal-typed",
13819 "signal_name": "changed",
13820 "arguments": envelope.clone(),
13821 }),
13822 )],
13823 DEFAULT_CODEC,
13824 );
13825 signal.workflow_signal_id = Some("signal-typed".to_string());
13826 signal.signal_name = Some("changed".to_string());
13827 signal.signal_arguments = Some(envelope);
13828 let commands = worker
13829 .execute_workflow_task(signal)
13830 .expect("typed signal resume");
13831 assert_eq!(
13832 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
13833 .expect("typed signal result"),
13834 arguments
13835 );
13836 }
13837
13838 #[tokio::test]
13839 async fn typed_helpers_never_parse_json_inspection_projection() {
13840 let collision_values = projection_collision_probe();
13841 let expected = AvroValue::Array(collision_values.clone());
13842 let envelope = encode_typed_envelope(&expected, DEFAULT_CODEC).expect("collision envelope");
13843
13844 let activity_context = workflow_context_with_codec(
13845 vec![history_event(
13846 "ActivityCompleted",
13847 json!({
13848 "sequence": 1,
13849 "activity_type": "collision.activity",
13850 "payload_codec": DEFAULT_CODEC,
13851 "result": envelope.clone(),
13852 }),
13853 )],
13854 DEFAULT_CODEC,
13855 );
13856 assert_eq!(
13857 activity_context
13858 .activity_avro_value("collision.activity", AvroValue::Array(Vec::new()))
13859 .await
13860 .expect("typed activity collision result"),
13861 expected
13862 );
13863
13864 let signal_context = workflow_context_with_codec(
13865 vec![
13866 history_event(
13867 "SignalWaitOpened",
13868 json!({"sequence": 1, "signal_name": "collision"}),
13869 ),
13870 history_event(
13871 "SignalApplied",
13872 json!({
13873 "sequence": 1,
13874 "signal_name": "collision",
13875 "payload_codec": DEFAULT_CODEC,
13876 "value": envelope.clone(),
13877 }),
13878 ),
13879 ],
13880 DEFAULT_CODEC,
13881 );
13882 assert_eq!(
13883 signal_context
13884 .wait_signal_avro_value("collision")
13885 .await
13886 .expect("typed signal collision arguments"),
13887 collision_values
13888 );
13889
13890 let child_context = workflow_context_with_codec(
13891 vec![
13892 history_event(
13893 "ChildWorkflowScheduled",
13894 json!({
13895 "sequence": 1,
13896 "child_workflow_instance_id": "collision-child",
13897 "child_workflow_run_id": "collision-run",
13898 "child_workflow_type": "collision.child",
13899 }),
13900 ),
13901 history_event(
13902 "ChildRunCompleted",
13903 json!({
13904 "sequence": 1,
13905 "child_workflow_instance_id": "collision-child",
13906 "child_workflow_run_id": "collision-run",
13907 "child_workflow_type": "collision.child",
13908 "payload_codec": DEFAULT_CODEC,
13909 "result": envelope,
13910 }),
13911 ),
13912 ],
13913 DEFAULT_CODEC,
13914 );
13915 let child = child_context
13916 .start_child_workflow_avro_value(
13917 "collision.child",
13918 ChildWorkflowOptions::new("collision-workers"),
13919 AvroValue::Array(Vec::new()),
13920 )
13921 .await
13922 .expect("typed child collision result");
13923 assert_eq!(child.result, expected);
13924 }
13925
13926 #[tokio::test]
13927 async fn replayed_typed_query_keeps_lossless_workflow_and_query_inputs() {
13928 let client = Client::new("http://127.0.0.1:8080").expect("client");
13929 let mut worker = Worker::new(client, "rust-workers");
13930 worker.register_replayed_workflow_avro_value(
13931 "typed.replayed",
13932 || (),
13933 |_ctx, input, _state| async move { Ok(input) },
13934 );
13935 worker.register_replayed_query_avro_value::<(), _, _>(
13936 "typed.replayed",
13937 "inspect",
13938 |ctx, _state, args| async move {
13939 let mut signals = ctx.signals_avro_value("collision");
13940 let signal = signals
13941 .pop()
13942 .map(AvroValue::Array)
13943 .unwrap_or_else(|| AvroValue::Array(Vec::new()));
13944 Ok(AvroValue::Array(vec![
13945 ctx.workflow_input_avro_value().clone(),
13946 signal,
13947 args,
13948 ]))
13949 },
13950 );
13951 let arguments = AvroValue::Array(projection_collision_probe());
13952 let signal_arguments =
13953 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed query signal arguments");
13954 let task = QueryTask {
13955 query_task_id: "query-typed-replay".to_string(),
13956 query_task_attempt: 1,
13957 lease_owner: Some("rust-worker".to_string()),
13958 workflow_id: Some("typed-replay".to_string()),
13959 run_id: Some("run-typed-replay".to_string()),
13960 workflow_type: "typed.replayed".to_string(),
13961 query_name: "inspect".to_string(),
13962 payload_codec: DEFAULT_CODEC.to_string(),
13963 workflow_arguments: Some(
13964 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("workflow arguments"),
13965 ),
13966 query_arguments: Some(
13967 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("query arguments"),
13968 ),
13969 history_events: vec![history_event(
13970 "SignalReceived",
13971 json!({
13972 "signal_id": "collision-signal",
13973 "signal_name": "collision",
13974 "workflow_sequence": 1,
13975 "payload_codec": DEFAULT_CODEC,
13976 "arguments": signal_arguments,
13977 }),
13978 )],
13979 history_export: None,
13980 run_status: Some("completed".to_string()),
13981 };
13982
13983 assert_eq!(
13984 worker
13985 .execute_query_task(task)
13986 .await
13987 .expect("typed replay query"),
13988 AvroValue::Array(vec![arguments.clone(), arguments.clone(), arguments])
13989 );
13990 }
13991
13992 #[test]
13993 fn public_avro_adapter_rejects_non_string_map_keys_before_json_conversion() {
13994 let value = BTreeMap::from([(1_i32, "integer key")]);
13995 let error = PayloadEnvelope::avro(&value)
13996 .expect_err("integer map keys must fail")
13997 .to_string();
13998
13999 assert!(error.contains("invalid_map_key"));
14000 }
14001
14002 #[test]
14003 fn json_tagged_payload_fails_closed_with_actionable_diagnostic() {
14004 let envelope = PayloadEnvelope {
14005 codec: "json".to_string(),
14006 blob: r#"{"greeting":"hello"}"#.to_string(),
14007 };
14008
14009 let error = decode_payload::<Value>(&envelope).expect_err("JSON payload must fail");
14010 let diagnostic = error.to_string();
14011 assert!(diagnostic.contains("unsupported_payload_codec"));
14012 assert!(diagnostic.contains("codec=\"avro\""));
14013 assert!(diagnostic.contains("HTTP document transport"));
14014 }
14015
14016 #[test]
14017 fn untagged_json_payload_value_fails_closed() {
14018 let error = decode_wire_value(&json!({"stale": true}), DEFAULT_CODEC)
14019 .expect_err("untagged JSON payload values must fail");
14020 let diagnostic = error.to_string();
14021 assert!(diagnostic.contains("unsupported_payload_codec"));
14022 assert!(diagnostic.contains("untagged durable payload"));
14023 assert!(diagnostic.contains("HTTP document transport"));
14024 }
14025
14026 #[test]
14027 fn prerelease_avro_payload_without_single_object_frame_is_rejected() {
14028 let envelope = PayloadEnvelope {
14029 codec: DEFAULT_CODEC.to_string(),
14030 blob: BASE64.encode([0x01]),
14031 };
14032
14033 let error = decode_payload::<Value>(&envelope).expect_err("prerelease payload must fail");
14034 assert!(error.to_string().contains("invalid_payload_framing"));
14035 }
14036
14037 #[tokio::test]
14038 async fn workflow_completion_rejects_invalid_payload_slots_without_transport() {
14039 let server = MockWorkerServer::start();
14040 let client = Client::builder(server.base_url())
14041 .timeout(Duration::from_secs(2))
14042 .build()
14043 .expect("client");
14044 let invalid_commands = [
14045 json!({
14046 "type": "complete_workflow",
14047 "result": {"codec": "json", "blob": null}
14048 }),
14049 json!({
14050 "type": "schedule_activity",
14051 "arguments": {"codec": "yaml", "blob": "ignored"}
14052 }),
14053 json!({
14054 "type": "start_child_workflow",
14055 "arguments": {"codec": DEFAULT_CODEC, "blob": null}
14056 }),
14057 json!({"type": "continue_as_new", "arguments": []}),
14058 json!({"type": "complete_update"}),
14059 json!({"type": "record_side_effect", "result": null}),
14060 json!({
14061 "type": "start_service_operation",
14062 "payload_codec": DEFAULT_CODEC,
14063 "request_payload": "raw-avro-bytes"
14064 }),
14065 ];
14066
14067 for command in invalid_commands {
14068 let error = client
14069 .complete_workflow_task("invalid-codec", "rust-worker", 1, vec![command])
14070 .await
14071 .expect_err("invalid durable payload must fail locally");
14072 let diagnostic = error.to_string();
14073 assert!(
14074 diagnostic.contains("unsupported_payload_codec")
14075 || diagnostic.contains("invalid_payload_envelope")
14076 || diagnostic.contains("untagged durable payload"),
14077 "unexpected validation diagnostic: {diagnostic}"
14078 );
14079 }
14080
14081 assert_eq!(
14082 server.request_count("/api/worker/workflow-tasks/invalid-codec/complete"),
14083 0,
14084 "invalid command payloads must not reach HTTP transport"
14085 );
14086 }
14087
14088 #[test]
14089 fn workflow_completion_validates_only_protocol_owned_payload_slots() {
14090 let envelope = fixture_envelope(json!({"codec": "customer-value"}));
14091 let commands = [
14092 json!({"type": "complete_workflow", "result": envelope.clone()}),
14093 json!({"type": "schedule_activity", "arguments": envelope.clone()}),
14094 json!({"type": "start_child_workflow", "arguments": envelope.clone()}),
14095 json!({"type": "continue_as_new", "arguments": envelope.clone()}),
14096 json!({"type": "complete_update", "result": envelope.clone()}),
14097 json!({"type": "record_side_effect", "result": envelope.clone()}),
14098 json!({
14099 "type": "start_service_operation",
14100 "payload_codec": DEFAULT_CODEC,
14101 "request_payload": envelope.clone()
14102 }),
14103 json!({
14104 "type": "complete_workflow",
14105 "result": envelope,
14106 "metadata": {
14107 "codec": "json",
14108 "payload_codec": "customer-codec",
14109 "result": {"codec": "yaml", "blob": null}
14110 }
14111 }),
14112 ];
14113
14114 validate_workflow_task_commands(&commands)
14115 .expect("customer metadata must not become a protocol codec declaration");
14116 }
14117
14118 #[test]
14119 fn valid_avro_tasks_normalize_absent_and_null_arguments_to_empty_lists() {
14120 assert_eq!(
14121 decode_task_avro_arguments(None, DEFAULT_CODEC).expect("absent arguments"),
14122 AvroValue::Array(Vec::new())
14123 );
14124 assert_eq!(
14125 decode_task_avro_arguments(Some(&Value::Null), DEFAULT_CODEC).expect("null arguments"),
14126 AvroValue::Array(Vec::new())
14127 );
14128
14129 let mut signal = workflow_task("missing", Vec::new(), DEFAULT_CODEC);
14130 signal.signal_name = Some("empty-signal".to_string());
14131 signal.signal_arguments = None;
14132 let decoded = decode_resume_signal(&signal)
14133 .expect("valid Avro signal")
14134 .expect("named signal resumes the workflow");
14135 assert!(decoded.arguments.is_empty());
14136 }
14137
14138 #[tokio::test]
14139 async fn malformed_task_level_codecs_become_pre_handler_failures() {
14140 let client = Client::new("http://127.0.0.1:8080").expect("client");
14141 let mut worker = Worker::new(client, "rust-workers");
14142 let handler_calls = Arc::new(AtomicUsize::new(0));
14143
14144 let calls = Arc::clone(&handler_calls);
14145 worker.register_workflow("codec.workflow", move |_ctx, _args| {
14146 calls.fetch_add(1, Ordering::SeqCst);
14147 async move { Ok(Value::Null) }
14148 });
14149 let calls = Arc::clone(&handler_calls);
14150 worker.register_activity("codec.activity", move |_ctx, _args| {
14151 calls.fetch_add(1, Ordering::SeqCst);
14152 async move { Ok(Value::Null) }
14153 });
14154 let calls = Arc::clone(&handler_calls);
14155 worker.register_query("codec.workflow", "known", move |_ctx, _args| {
14156 calls.fetch_add(1, Ordering::SeqCst);
14157 async move { Ok(Value::Null) }
14158 });
14159
14160 let mut failures = Vec::new();
14161 for codec_case in [
14162 InvalidTaskPayloadCodec::Missing,
14163 InvalidTaskPayloadCodec::Null,
14164 InvalidTaskPayloadCodec::NonString,
14165 ] {
14166 let mut workflow = json!({
14167 "task_id": format!("workflow-{}", codec_case.label()),
14168 "workflow_type": "codec.workflow"
14169 });
14170 codec_case.apply(&mut workflow);
14171 match serde_json::from_value::<WorkflowTask>(workflow) {
14172 Ok(task) => match worker.execute_workflow_task(task) {
14173 Err(error) if error.to_string().contains("unsupported_payload_codec") => {}
14174 outcome => failures.push(format!(
14175 "workflow {} codec returned {outcome:?}",
14176 codec_case.label()
14177 )),
14178 },
14179 Err(error) => failures.push(format!(
14180 "workflow {} codec failed transport deserialization: {error}",
14181 codec_case.label()
14182 )),
14183 }
14184
14185 let mut activity = json!({
14186 "task_id": format!("activity-{}", codec_case.label()),
14187 "activity_attempt_id": format!("attempt-{}", codec_case.label()),
14188 "activity_type": "codec.activity",
14189 "attempt_number": 1
14190 });
14191 codec_case.apply(&mut activity);
14192 match serde_json::from_value::<ActivityTask>(activity) {
14193 Ok(task) => match worker.execute_activity_task(task).await {
14194 Err(error) if error.to_string().contains("unsupported_payload_codec") => {}
14195 outcome => failures.push(format!(
14196 "activity {} codec returned {outcome:?}",
14197 codec_case.label()
14198 )),
14199 },
14200 Err(error) => failures.push(format!(
14201 "activity {} codec failed transport deserialization: {error}",
14202 codec_case.label()
14203 )),
14204 }
14205
14206 let mut query = json!({
14207 "query_task_id": format!("query-{}", codec_case.label()),
14208 "workflow_type": "codec.workflow",
14209 "query_name": "known"
14210 });
14211 codec_case.apply(&mut query);
14212 match serde_json::from_value::<QueryTask>(query) {
14213 Ok(task) => match worker.execute_query_task(task).await {
14214 Err(failure) if failure.message.contains("unsupported_payload_codec") => {}
14215 outcome => failures.push(format!(
14216 "query {} codec returned {outcome:?}",
14217 codec_case.label()
14218 )),
14219 },
14220 Err(error) => failures.push(format!(
14221 "query {} codec failed transport deserialization: {error}",
14222 codec_case.label()
14223 )),
14224 }
14225 }
14226
14227 assert!(failures.is_empty(), "{}", failures.join("\n"));
14228 assert_eq!(
14229 handler_calls.load(Ordering::SeqCst),
14230 0,
14231 "invalid task codecs must not invoke a handler"
14232 );
14233 }
14234
14235 #[tokio::test]
14236 async fn polled_malformed_task_codecs_are_settled_without_handler_execution() {
14237 for codec_case in [
14238 InvalidTaskPayloadCodec::Missing,
14239 InvalidTaskPayloadCodec::Null,
14240 InvalidTaskPayloadCodec::NonString,
14241 ] {
14242 let server = MockWorkerServer::invalid_task_payload_codec(codec_case);
14243 let client = Client::builder(server.base_url())
14244 .timeout(Duration::from_secs(2))
14245 .build()
14246 .expect("client");
14247 let mut worker = Worker::new(client, "rust-workers")
14248 .worker_id("codec-worker")
14249 .poll_timeout(Duration::from_millis(10));
14250 let handler_calls = Arc::new(AtomicUsize::new(0));
14251
14252 let calls = Arc::clone(&handler_calls);
14253 worker.register_workflow("codec.workflow", move |_ctx, _args| {
14254 calls.fetch_add(1, Ordering::SeqCst);
14255 async move { Ok(Value::Null) }
14256 });
14257 let calls = Arc::clone(&handler_calls);
14258 worker.register_activity("codec.activity", move |_ctx, _args| {
14259 calls.fetch_add(1, Ordering::SeqCst);
14260 async move { Ok(Value::Null) }
14261 });
14262 let calls = Arc::clone(&handler_calls);
14263 worker.register_query("codec.workflow", "known", move |_ctx, _args| {
14264 calls.fetch_add(1, Ordering::SeqCst);
14265 async move { Ok(Value::Null) }
14266 });
14267
14268 assert_eq!(
14269 worker.run_once().await.expect("invalid tasks are settled"),
14270 3,
14271 "all {} codec tasks must be handled",
14272 codec_case.label()
14273 );
14274 assert_eq!(
14275 handler_calls.load(Ordering::SeqCst),
14276 0,
14277 "{} task codecs must fail before every handler",
14278 codec_case.label()
14279 );
14280
14281 for path in [
14282 "/api/worker/workflow-tasks/codec-workflow/fail",
14283 "/api/worker/activity-tasks/codec-activity/fail",
14284 "/api/worker/query-tasks/codec-query/fail",
14285 ] {
14286 let body = server.request_body(path);
14287 assert!(
14288 body["failure"]["message"]
14289 .as_str()
14290 .is_some_and(|message| message.contains("unsupported_payload_codec")),
14291 "{path} must receive the stable codec diagnostic for the {} case: {body}",
14292 codec_case.label()
14293 );
14294 }
14295 assert_eq!(
14296 server.request_body("/api/worker/query-tasks/codec-query/fail")["failure"]
14297 ["reason"],
14298 "query_payload_decode_failed"
14299 );
14300 for path in [
14301 "/api/worker/workflow-tasks/codec-workflow/complete",
14302 "/api/worker/activity-tasks/codec-activity/complete",
14303 "/api/worker/query-tasks/codec-query/complete",
14304 ] {
14305 assert_eq!(
14306 server.request_count(path),
14307 0,
14308 "invalid {} codec task reached {path}",
14309 codec_case.label()
14310 );
14311 }
14312 }
14313 }
14314
14315 #[tokio::test]
14316 async fn invalid_inbound_codecs_precede_handlers_and_unrelated_outcomes() {
14317 let client = Client::new("http://127.0.0.1:8080").expect("client");
14318 let mut worker = Worker::new(client, "rust-workers");
14319 let handler_calls = Arc::new(AtomicUsize::new(0));
14320
14321 let calls = Arc::clone(&handler_calls);
14322 worker.register_workflow("codec.workflow", move |_ctx, _args| {
14323 calls.fetch_add(1, Ordering::SeqCst);
14324 async move { Ok(Value::Null) }
14325 });
14326 let calls = Arc::clone(&handler_calls);
14327 worker.register_activity("codec.activity", move |_ctx, _args| {
14328 calls.fetch_add(1, Ordering::SeqCst);
14329 async move { Ok(Value::Null) }
14330 });
14331 let calls = Arc::clone(&handler_calls);
14332 worker.register_update("codec.workflow", "known", move |_ctx, _args| {
14333 calls.fetch_add(1, Ordering::SeqCst);
14334 async move { Ok(Value::Null) }
14335 });
14336 let calls = Arc::clone(&handler_calls);
14337 worker.register_query("codec.workflow", "known", move |_ctx, _args| {
14338 calls.fetch_add(1, Ordering::SeqCst);
14339 async move { Ok(Value::Null) }
14340 });
14341
14342 let mut workflow = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
14343 workflow.payload_codec = "json".to_string();
14344 workflow.arguments = None;
14345 let error = worker
14346 .execute_workflow_task(workflow)
14347 .expect_err("task codec must be checked before workflow invocation");
14348 assert!(error.to_string().contains("unsupported_payload_codec"));
14349
14350 let activity = ActivityTask {
14351 task_id: "activity-invalid-codec".to_string(),
14352 activity_attempt_id: None,
14353 attempt_id: None,
14354 activity_type: "codec.activity".to_string(),
14355 payload_codec: "unknown".to_string(),
14356 arguments: None,
14357 attempt_number: 1,
14358 lease_owner: None,
14359 };
14360 let error = worker
14361 .execute_activity_task(activity)
14362 .await
14363 .expect_err("task codec must be checked before activity invocation");
14364 assert!(error.to_string().contains("unsupported_payload_codec"));
14365
14366 let mut update = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
14367 update.workflow_update_id = Some("update-invalid-codec".to_string());
14368 update.update_name = Some("known".to_string());
14369 update.history_events.push(history_event(
14370 "UpdateAccepted",
14371 json!({
14372 "update_id": "update-invalid-codec",
14373 "update_name": "known",
14374 "arguments": {"codec": "json", "blob": null}
14375 }),
14376 ));
14377 let error = worker
14378 .execute_workflow_task(update)
14379 .expect_err("nested update codec must be checked before handler lookup");
14380 assert!(error.to_string().contains("unsupported_payload_codec"));
14381
14382 let query: QueryTask = serde_json::from_value(json!({
14383 "query_task_id": "query-invalid-codec",
14384 "workflow_type": "codec.workflow",
14385 "query_name": "known",
14386 "payload_codec": DEFAULT_CODEC,
14387 "workflow_arguments": null,
14388 "query_arguments": null,
14389 "history_export": {
14390 "payloads": {"codec": DEFAULT_CODEC},
14391 "signals": [{
14392 "name": "empty",
14393 "payload_codec": "json",
14394 "arguments": null
14395 }]
14396 }
14397 }))
14398 .expect("query task");
14399 let failure = worker
14400 .execute_query_task(query)
14401 .await
14402 .expect_err("exported signal codec must be checked before query invocation");
14403 assert_eq!(failure.reason, "query_payload_decode_failed");
14404 assert!(failure.message.contains("unsupported_payload_codec"));
14405
14406 let exported_history: QueryTask = serde_json::from_value(json!({
14407 "query_task_id": "query-invalid-history-codec",
14408 "workflow_type": "codec.workflow",
14409 "query_name": "known",
14410 "payload_codec": DEFAULT_CODEC,
14411 "history_export": {
14412 "payloads": {"codec": DEFAULT_CODEC},
14413 "history_events": [{
14414 "type": "ActivityCompleted",
14415 "payload": {"payload_codec": "unknown", "result": null}
14416 }]
14417 }
14418 }))
14419 .expect("query task");
14420 let failure = worker
14421 .execute_query_task(exported_history)
14422 .await
14423 .expect_err("exported history codec must be checked before query invocation");
14424 assert_eq!(failure.reason, "query_payload_decode_failed");
14425 assert!(failure.message.contains("unsupported_payload_codec"));
14426 assert_eq!(handler_calls.load(Ordering::SeqCst), 0);
14427
14428 let mut unknown_workflow = workflow_task("missing", Vec::new(), DEFAULT_CODEC);
14429 unknown_workflow.arguments = None;
14430 unknown_workflow.history_events.push(history_event(
14431 "SignalReceived",
14432 json!({
14433 "signal_name": "empty",
14434 "payload_codec": "json",
14435 "arguments": null
14436 }),
14437 ));
14438 let error = worker
14439 .execute_workflow_task(unknown_workflow)
14440 .expect_err("history codec must precede unknown workflow outcome");
14441 assert!(error.to_string().contains("unsupported_payload_codec"));
14442
14443 let unknown_activity = ActivityTask {
14444 task_id: "activity-unknown".to_string(),
14445 activity_attempt_id: None,
14446 attempt_id: None,
14447 activity_type: "missing".to_string(),
14448 payload_codec: "json".to_string(),
14449 arguments: None,
14450 attempt_number: 1,
14451 lease_owner: None,
14452 };
14453 let error = worker
14454 .execute_activity_task(unknown_activity)
14455 .await
14456 .expect_err("codec must precede unknown activity outcome");
14457 assert!(error.to_string().contains("unsupported_payload_codec"));
14458
14459 let mut unknown_update = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
14460 unknown_update.payload_codec = "json".to_string();
14461 unknown_update.arguments = None;
14462 unknown_update.workflow_update_id = Some("update-unknown".to_string());
14463 unknown_update.update_name = Some("missing".to_string());
14464 let error = worker
14465 .execute_workflow_task(unknown_update)
14466 .expect_err("codec must precede fail_update shortcut");
14467 assert!(error.to_string().contains("unsupported_payload_codec"));
14468
14469 let unknown_query: QueryTask = serde_json::from_value(json!({
14470 "query_task_id": "query-unknown",
14471 "workflow_type": "missing",
14472 "query_name": "missing",
14473 "payload_codec": "json",
14474 "workflow_arguments": null,
14475 "query_arguments": null
14476 }))
14477 .expect("query task");
14478 let failure = worker
14479 .execute_query_task(unknown_query)
14480 .await
14481 .expect_err("codec must precede unknown query outcome");
14482 assert_eq!(failure.reason, "query_payload_decode_failed");
14483 assert!(failure.message.contains("unsupported_payload_codec"));
14484 }
14485
14486 #[tokio::test]
14487 async fn invalid_signal_history_payload_aliases_precede_shortcuts() {
14488 let client = Client::new("http://127.0.0.1:8080").expect("client");
14489 let worker = Worker::new(client, "rust-workers");
14490
14491 for event_type in ["SignalReceived", "SignalApplied"] {
14492 for (payload_field, codec) in [
14493 ("value", "json"),
14494 ("input", "unknown"),
14495 ("arguments", "json"),
14496 ] {
14497 let payload = json!({
14498 "signal_name": "empty",
14499 payload_field: {"codec": codec, "blob": null}
14500 });
14501 let workflow = workflow_task(
14502 "missing",
14503 vec![history_event(event_type, payload.clone())],
14504 DEFAULT_CODEC,
14505 );
14506 let error = worker
14507 .execute_workflow_task(workflow)
14508 .expect_err("signal payload codec must precede unknown workflow outcome");
14509 assert!(
14510 error.to_string().contains("unsupported_payload_codec"),
14511 "{event_type}.{payload_field} returned an unrelated workflow error: {error}"
14512 );
14513
14514 let query: QueryTask = serde_json::from_value(json!({
14515 "query_task_id": format!("query-{event_type}-{payload_field}"),
14516 "workflow_type": "missing",
14517 "query_name": "missing",
14518 "payload_codec": DEFAULT_CODEC,
14519 "workflow_arguments": null,
14520 "query_arguments": null,
14521 "history_events": [{
14522 "event_type": event_type,
14523 "payload": payload
14524 }]
14525 }))
14526 .expect("query task");
14527 let failure = worker
14528 .execute_query_task(query)
14529 .await
14530 .expect_err("signal payload codec must precede unknown query outcome");
14531 assert_eq!(
14532 failure.reason, "query_payload_decode_failed",
14533 "{event_type}.{payload_field} returned an unrelated query outcome"
14534 );
14535 assert!(
14536 failure.message.contains("unsupported_payload_codec"),
14537 "{event_type}.{payload_field} returned an unrelated query error: {}",
14538 failure.message
14539 );
14540 }
14541 }
14542 }
14543
14544 #[test]
14545 fn workflow_context_schedules_activity_until_completion_is_in_history() {
14546 let ctx = WorkflowContext {
14547 state: Arc::new(Mutex::new(
14548 WorkflowState::new_with_identity(
14549 Vec::new(),
14550 Some("wf-parent".to_string()),
14551 Some("run-parent".to_string()),
14552 "rust-workers".to_string(),
14553 DEFAULT_CODEC.to_string(),
14554 None,
14555 )
14556 .expect("workflow state"),
14557 )),
14558 };
14559
14560 let mut call = Box::pin(ctx.activity("hello.activity", json!(["Ada"])));
14561 let mut task_context = TaskContext::from_waker(noop_waker_ref());
14562 assert!(matches!(
14563 call.as_mut().poll(&mut task_context),
14564 Poll::Pending
14565 ));
14566
14567 let commands = ctx.take_commands().expect("commands");
14568 assert_eq!(commands[0]["type"], "schedule_activity");
14569 assert_eq!(commands[0]["activity_type"], "hello.activity");
14570 }
14571
14572 #[test]
14573 fn activity_options_encode_retry_policy_queue_and_every_timeout() {
14574 let ctx = workflow_context(Vec::new());
14575 let options = ActivityOptions::new()
14576 .task_queue("payments")
14577 .retry_policy(
14578 ActivityRetryPolicy::new(4)
14579 .exponential_backoff(Duration::from_secs(1), 3, Some(Duration::from_secs(10)))
14580 .non_retryable_error_type("ValidationError"),
14581 )
14582 .start_to_close_timeout(Duration::from_secs(120))
14583 .schedule_to_start_timeout(Duration::from_secs(10))
14584 .schedule_to_close_timeout(Duration::from_secs(300))
14585 .heartbeat_timeout(Duration::from_secs(15));
14586 let mut call = Box::pin(ctx.activity_with_options(
14587 "charge-card",
14588 options,
14589 json!([{"order_id": "o-1"}]),
14590 ));
14591 let mut task_context = TaskContext::from_waker(noop_waker_ref());
14592
14593 assert!(matches!(
14594 call.as_mut().poll(&mut task_context),
14595 Poll::Pending
14596 ));
14597 assert!(matches!(
14598 call.as_mut().poll(&mut task_context),
14599 Poll::Pending
14600 ));
14601
14602 let commands = ctx.take_commands().expect("activity command");
14603 assert_eq!(commands.len(), 1, "one future emits one logical schedule");
14604 assert_eq!(commands[0]["queue"], "payments");
14605 assert_eq!(
14606 commands[0]["retry_policy"],
14607 json!({
14608 "max_attempts": 4,
14609 "backoff_seconds": [1, 3, 9],
14610 "non_retryable_error_types": ["ValidationError"],
14611 })
14612 );
14613 assert_eq!(commands[0]["start_to_close_timeout"], 120);
14614 assert_eq!(commands[0]["schedule_to_start_timeout"], 10);
14615 assert_eq!(commands[0]["schedule_to_close_timeout"], 300);
14616 assert_eq!(commands[0]["heartbeat_timeout"], 15);
14617 }
14618
14619 #[test]
14620 fn activity_options_encode_explicit_and_rounded_backoff_intervals() {
14621 let ctx = workflow_context(Vec::new());
14622 let options = ActivityOptions::new().retry_policy(
14623 ActivityRetryPolicy::new(3)
14624 .backoff_intervals([Duration::from_millis(1), Duration::from_millis(1_001)]),
14625 );
14626 let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
14627 let mut task_context = TaskContext::from_waker(noop_waker_ref());
14628
14629 assert!(matches!(
14630 call.as_mut().poll(&mut task_context),
14631 Poll::Pending
14632 ));
14633 assert_eq!(
14634 ctx.take_commands().expect("command")[0]["retry_policy"]["backoff_seconds"],
14635 json!([1, 2])
14636 );
14637 }
14638
14639 #[test]
14640 fn invalid_activity_options_return_typed_errors_before_emitting_commands() {
14641 let cases = [
14642 (
14643 ActivityOptions::new().task_queue(" "),
14644 ActivityOptionsErrorKind::EmptyTaskQueue,
14645 ),
14646 (
14647 ActivityOptions::new().retry_policy(ActivityRetryPolicy::default()),
14648 ActivityOptionsErrorKind::EmptyRetryPolicy,
14649 ),
14650 (
14651 ActivityOptions::new().retry_policy(ActivityRetryPolicy::new(0)),
14652 ActivityOptionsErrorKind::InvalidMaxAttempts,
14653 ),
14654 (
14655 ActivityOptions::new().retry_policy(ActivityRetryPolicy {
14656 max_attempts: None,
14657 backoff: Some(ActivityBackoff::Explicit(vec![Duration::from_secs(1)])),
14658 non_retryable_error_types: Vec::new(),
14659 }),
14660 ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
14661 ),
14662 (
14663 ActivityOptions::new().retry_policy(
14664 ActivityRetryPolicy::new(2)
14665 .backoff_intervals([Duration::from_secs(1), Duration::from_secs(2)]),
14666 ),
14667 ActivityOptionsErrorKind::TooManyBackoffIntervals,
14668 ),
14669 (
14670 ActivityOptions::new().retry_policy(
14671 ActivityRetryPolicy::new(2).exponential_backoff(
14672 Duration::from_secs(1),
14673 0,
14674 None,
14675 ),
14676 ),
14677 ActivityOptionsErrorKind::InvalidBackoffCoefficient,
14678 ),
14679 (
14680 ActivityOptions::new()
14681 .retry_policy(ActivityRetryPolicy::new(2).non_retryable_error_type(" ")),
14682 ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
14683 ),
14684 (
14685 ActivityOptions::new().retry_policy(
14686 ActivityRetryPolicy::new(10_002).exponential_backoff(
14687 Duration::from_secs(1),
14688 1,
14689 None,
14690 ),
14691 ),
14692 ActivityOptionsErrorKind::BackoffGenerationTooLarge,
14693 ),
14694 (
14695 ActivityOptions::new().retry_policy(
14696 ActivityRetryPolicy::new(2)
14697 .backoff_intervals([Duration::from_secs(i64::MAX as u64 + 1)]),
14698 ),
14699 ActivityOptionsErrorKind::BackoffOverflow,
14700 ),
14701 ];
14702
14703 for (options, expected_kind) in cases {
14704 let ctx = workflow_context(Vec::new());
14705 let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
14706 let mut task_context = TaskContext::from_waker(noop_waker_ref());
14707 let Poll::Ready(Err(Error::InvalidActivityOptions(error))) =
14708 call.as_mut().poll(&mut task_context)
14709 else {
14710 panic!("expected typed activity validation error");
14711 };
14712 assert_eq!(error.kind, expected_kind);
14713 assert!(ctx.take_commands().expect("commands").is_empty());
14714 }
14715 }
14716
14717 #[test]
14718 fn activity_options_validate_positive_and_ordered_timeouts() {
14719 let zero_timeout_cases = [
14720 ActivityOptions::new().start_to_close_timeout(Duration::ZERO),
14721 ActivityOptions::new().schedule_to_start_timeout(Duration::ZERO),
14722 ActivityOptions::new().schedule_to_close_timeout(Duration::ZERO),
14723 ActivityOptions::new().heartbeat_timeout(Duration::ZERO),
14724 ];
14725 for options in zero_timeout_cases {
14726 assert_eq!(
14727 options.validate().expect_err("zero timeout").kind,
14728 ActivityOptionsErrorKind::TimeoutNotPositive
14729 );
14730 }
14731
14732 let ordering_cases = [
14733 ActivityOptions::new()
14734 .heartbeat_timeout(Duration::from_secs(11))
14735 .start_to_close_timeout(Duration::from_secs(10)),
14736 ActivityOptions::new()
14737 .start_to_close_timeout(Duration::from_secs(31))
14738 .schedule_to_close_timeout(Duration::from_secs(30)),
14739 ActivityOptions::new()
14740 .schedule_to_start_timeout(Duration::from_secs(31))
14741 .schedule_to_close_timeout(Duration::from_secs(30)),
14742 ];
14743 for options in ordering_cases {
14744 assert_eq!(
14745 options.validate().expect_err("timeout order").kind,
14746 ActivityOptionsErrorKind::TimeoutOrder
14747 );
14748 }
14749
14750 assert_eq!(
14751 ActivityOptions::new()
14752 .start_to_close_timeout(Duration::from_secs(i64::MAX as u64 + 1))
14753 .validate()
14754 .expect_err("protocol integer overflow")
14755 .kind,
14756 ActivityOptionsErrorKind::TimeoutOverflow
14757 );
14758 }
14759
14760 #[test]
14761 fn replayed_activity_retry_history_completes_without_duplicate_schedule() {
14762 let ctx = workflow_context(completed_retry_activity_history());
14763 let mut call =
14764 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
14765 let mut task_context = TaskContext::from_waker(noop_waker_ref());
14766
14767 assert!(matches!(
14768 call.as_mut().poll(&mut task_context),
14769 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
14770 ));
14771 assert!(ctx.take_commands().expect("commands").is_empty());
14772 ctx.ensure_history_consumed().expect("history consumed");
14773 }
14774
14775 #[test]
14776 fn duplicate_non_retryable_types_use_one_command_and_replay_representation() {
14777 let mut options = retry_activity_options();
14778 options
14779 .retry_policy
14780 .as_mut()
14781 .expect("retry policy")
14782 .non_retryable_error_types
14783 .extend([" PermanentError ".to_string(), "PermanentError".to_string()]);
14784
14785 let new_ctx = workflow_context(Vec::new());
14786 let mut new_call =
14787 Box::pin(new_ctx.activity_with_options("flaky", options.clone(), json!([])));
14788 let mut task_context = TaskContext::from_waker(noop_waker_ref());
14789 assert!(matches!(
14790 new_call.as_mut().poll(&mut task_context),
14791 Poll::Pending
14792 ));
14793 let commands = new_ctx.take_commands().expect("commands");
14794 assert_eq!(commands.len(), 1);
14795 assert_eq!(
14796 commands[0]["retry_policy"]["non_retryable_error_types"],
14797 json!(["PermanentError"])
14798 );
14799
14800 let replay_ctx = workflow_context(completed_retry_activity_history());
14801 let mut replay_call =
14802 Box::pin(replay_ctx.activity_with_options("flaky", options, json!([])));
14803 assert!(matches!(
14804 replay_call.as_mut().poll(&mut task_context),
14805 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
14806 ));
14807 assert!(replay_ctx.take_commands().expect("commands").is_empty());
14808 replay_ctx
14809 .ensure_history_consumed()
14810 .expect("history consumed");
14811 }
14812
14813 #[test]
14814 fn replayed_intermediate_retry_remains_pending_across_restarts() {
14815 let history = completed_retry_activity_history()
14816 .into_iter()
14817 .take(3)
14818 .collect::<Vec<_>>();
14819
14820 for _restart in 0..2 {
14821 let ctx = workflow_context(history.clone());
14822 let mut call =
14823 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
14824 let mut task_context = TaskContext::from_waker(noop_waker_ref());
14825 assert!(matches!(
14826 call.as_mut().poll(&mut task_context),
14827 Poll::Pending
14828 ));
14829 assert!(ctx.take_commands().expect("commands").is_empty());
14830 }
14831 }
14832
14833 #[test]
14834 fn replayed_activity_rejects_changed_queue_retry_and_every_timeout_field() {
14835 let mut changed_queue = retry_activity_options();
14836 changed_queue.task_queue = Some("different-queue".to_string());
14837
14838 let mut changed_max_attempts = retry_activity_options();
14839 let retry_policy = changed_max_attempts
14840 .retry_policy
14841 .as_mut()
14842 .expect("retry policy");
14843 retry_policy.max_attempts = Some(4);
14844
14845 let mut changed_backoff = retry_activity_options();
14846 let retry_policy = changed_backoff.retry_policy.as_mut().expect("retry policy");
14847 retry_policy.backoff = Some(ActivityBackoff::Explicit(vec![
14848 Duration::from_secs(3),
14849 Duration::from_secs(4),
14850 ]));
14851
14852 let mut changed_non_retryable_types = retry_activity_options();
14853 let retry_policy = changed_non_retryable_types
14854 .retry_policy
14855 .as_mut()
14856 .expect("retry policy");
14857 retry_policy.non_retryable_error_types = vec!["AnotherPermanentError".to_string()];
14858
14859 let mut changed_start_to_close = retry_activity_options();
14860 changed_start_to_close.start_to_close_timeout = Some(Duration::from_secs(31));
14861 let mut changed_schedule_to_start = retry_activity_options();
14862 changed_schedule_to_start.schedule_to_start_timeout = Some(Duration::from_secs(6));
14863 let mut changed_schedule_to_close = retry_activity_options();
14864 changed_schedule_to_close.schedule_to_close_timeout = Some(Duration::from_secs(91));
14865 let mut changed_heartbeat = retry_activity_options();
14866 changed_heartbeat.heartbeat_timeout = Some(Duration::from_secs(11));
14867
14868 let cases = [
14869 (changed_queue, "activity_task_queue_mismatch"),
14870 (changed_max_attempts, "activity_retry_policy_mismatch"),
14871 (changed_backoff, "activity_retry_policy_mismatch"),
14872 (
14873 changed_non_retryable_types,
14874 "activity_retry_policy_mismatch",
14875 ),
14876 (changed_start_to_close, "activity_retry_policy_mismatch"),
14877 (changed_schedule_to_start, "activity_retry_policy_mismatch"),
14878 (changed_schedule_to_close, "activity_retry_policy_mismatch"),
14879 (changed_heartbeat, "activity_retry_policy_mismatch"),
14880 ];
14881
14882 for (options, expected_reason) in cases {
14883 let ctx = workflow_context(completed_retry_activity_history());
14884 let mut call = Box::pin(ctx.activity_with_options("flaky", options, json!([])));
14885 let mut task_context = TaskContext::from_waker(noop_waker_ref());
14886 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
14887 call.as_mut().poll(&mut task_context)
14888 else {
14889 panic!("changed activity options must fail replay");
14890 };
14891 assert_eq!(failure.reason, expected_reason);
14892 assert_eq!(failure.sequence, Some(1));
14893 assert!(ctx.take_commands().expect("commands").is_empty());
14894 }
14895 }
14896
14897 #[test]
14898 fn replayed_activity_rejects_changed_execution_mode_and_snapshot_version() {
14899 let cases = [
14900 (
14901 "execution_mode",
14902 json!("local"),
14903 "activity_execution_mode_mismatch",
14904 ),
14905 (
14906 "snapshot_version",
14907 json!(2),
14908 "activity_retry_policy_mismatch",
14909 ),
14910 ];
14911
14912 for (field, value, expected_reason) in cases {
14913 let mut history = completed_retry_activity_history();
14914 let activity = history[0].payload["activity"]
14915 .as_object_mut()
14916 .expect("activity snapshot");
14917 if field == "execution_mode" {
14918 activity.insert(field.to_string(), value);
14919 } else {
14920 activity["retry_policy"]
14921 .as_object_mut()
14922 .expect("retry snapshot")
14923 .insert(field.to_string(), value);
14924 }
14925
14926 let ctx = workflow_context(history);
14927 let mut call =
14928 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
14929 let mut task_context = TaskContext::from_waker(noop_waker_ref());
14930 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
14931 call.as_mut().poll(&mut task_context)
14932 else {
14933 panic!("changed {field} must fail replay");
14934 };
14935 assert_eq!(failure.reason, expected_reason);
14936 assert_eq!(failure.sequence, Some(1));
14937 assert!(ctx.take_commands().expect("commands").is_empty());
14938 }
14939 }
14940
14941 #[test]
14942 fn replayed_legacy_activity_treats_missing_option_snapshot_as_unknown() {
14943 let mut history = completed_retry_activity_history();
14944 let activity = history[0].payload["activity"]
14945 .as_object_mut()
14946 .expect("activity snapshot");
14947 activity.remove("execution_mode");
14948 activity.remove("retry_policy");
14949
14950 let mut current = retry_activity_options();
14951 current.start_to_close_timeout = Some(Duration::from_secs(45));
14952 current.schedule_to_start_timeout = Some(Duration::from_secs(8));
14953 current.schedule_to_close_timeout = Some(Duration::from_secs(120));
14954 current.heartbeat_timeout = Some(Duration::from_secs(12));
14955
14956 let ctx = workflow_context(history);
14957 let mut call = Box::pin(ctx.activity_with_options("flaky", current, json!([])));
14958 let mut task_context = TaskContext::from_waker(noop_waker_ref());
14959 assert!(matches!(
14960 call.as_mut().poll(&mut task_context),
14961 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
14962 ));
14963 assert!(ctx.take_commands().expect("commands").is_empty());
14964 ctx.ensure_history_consumed().expect("history consumed");
14965 }
14966
14967 #[test]
14968 fn terminal_activity_failed_after_start_returns_typed_failure() {
14969 let history = vec![
14970 history_event(
14971 "ActivityScheduled",
14972 json!({
14973 "sequence": 1,
14974 "activity_type": "flaky",
14975 "activity_execution_id": "act-terminal",
14976 "activity": {
14977 "id": "act-terminal",
14978 "sequence": 1,
14979 "type": "flaky",
14980 "queue": "critical-activities",
14981 "retry_policy": {
14982 "snapshot_version": 1,
14983 "max_attempts": 3,
14984 "backoff_seconds": [2, 4],
14985 "non_retryable_error_types": ["PermanentError"]
14986 }
14987 }
14988 }),
14989 ),
14990 history_event(
14991 "ActivityStarted",
14992 json!({
14993 "sequence": 1,
14994 "activity_type": "flaky",
14995 "activity_execution_id": "act-terminal",
14996 "activity_attempt_id": "attempt-1",
14997 "attempt_number": 1
14998 }),
14999 ),
15000 history_event(
15001 "ActivityFailed",
15002 json!({
15003 "sequence": 1,
15004 "activity_type": "flaky",
15005 "activity_execution_id": "act-terminal",
15006 "activity_attempt_id": "attempt-1",
15007 "attempt_number": 1,
15008 "failure_id": "failure-terminal",
15009 "failure_category": "activity",
15010 "exception_type": "PermanentError",
15011 "message": "cannot retry",
15012 "non_retryable": true
15013 }),
15014 ),
15015 ];
15016 let ctx = workflow_context(history);
15017 let mut call =
15018 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
15019 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15020
15021 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
15022 call.as_mut().poll(&mut task_context)
15023 else {
15024 panic!("terminal ActivityFailed must settle the activity future");
15025 };
15026 assert_eq!(failure.kind, ActivityFailureKind::Failed);
15027 assert_eq!(
15028 failure.activity_execution_id.as_deref(),
15029 Some("act-terminal")
15030 );
15031 assert_eq!(failure.exception_type.as_deref(), Some("PermanentError"));
15032 assert!(failure.non_retryable);
15033 assert!(ctx.take_commands().expect("commands").is_empty());
15034 ctx.ensure_history_consumed().expect("history consumed");
15035 }
15036
15037 #[test]
15038 fn activity_terminal_events_return_machine_readable_failures() {
15039 let cases = [
15040 (
15041 "ActivityFailed",
15042 json!({
15043 "sequence": 1,
15044 "activity_type": "charge-card",
15045 "activity_execution_id": "act-1",
15046 "activity_attempt_id": "attempt-2",
15047 "attempt_number": 2,
15048 "failure_id": "failure-1",
15049 "failure_category": "activity",
15050 "exception_type": "PaymentDeclined",
15051 "exception_class": "payments.PaymentDeclined",
15052 "message": "card declined",
15053 "non_retryable": true
15054 }),
15055 ActivityFailureKind::Failed,
15056 "activity",
15057 ),
15058 (
15059 "ActivityCancelled",
15060 json!({
15061 "sequence": 1,
15062 "activity_type": "charge-card",
15063 "activity_execution_id": "act-1",
15064 "activity_attempt_id": "attempt-1"
15065 }),
15066 ActivityFailureKind::Cancelled,
15067 "cancelled",
15068 ),
15069 ];
15070
15071 for (event_type, payload, expected_kind, expected_reason) in cases {
15072 let ctx = workflow_context(vec![history_event(event_type, payload)]);
15073 let mut call = Box::pin(ctx.activity("charge-card", json!([])));
15074 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15075 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
15076 call.as_mut().poll(&mut task_context)
15077 else {
15078 panic!("expected terminal activity failure");
15079 };
15080 assert_eq!(failure.kind, expected_kind);
15081 assert_eq!(failure.reason, expected_reason);
15082 assert_eq!(failure.activity_execution_id.as_deref(), Some("act-1"));
15083 assert_eq!(failure.activity_type.as_deref(), Some("charge-card"));
15084 }
15085 }
15086
15087 #[test]
15088 fn every_activity_timeout_class_is_typed() {
15089 for timeout_kind in [
15090 "start_to_close",
15091 "schedule_to_start",
15092 "schedule_to_close",
15093 "heartbeat",
15094 ] {
15095 let ctx = workflow_context(vec![history_event(
15096 "ActivityTimedOut",
15097 json!({
15098 "sequence": 1,
15099 "activity_type": "slow",
15100 "activity_execution_id": "act-timeout",
15101 "activity_attempt_id": "attempt-timeout",
15102 "failure_category": "timeout",
15103 "timeout_kind": timeout_kind,
15104 "message": "deadline expired"
15105 }),
15106 )]);
15107 let mut call = Box::pin(ctx.activity("slow", json!([])));
15108 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15109 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
15110 call.as_mut().poll(&mut task_context)
15111 else {
15112 panic!("expected timeout failure");
15113 };
15114 assert_eq!(failure.kind, ActivityFailureKind::TimedOut);
15115 assert_eq!(failure.reason, timeout_kind);
15116 assert_eq!(failure.timeout_kind.as_deref(), Some(timeout_kind));
15117 assert_eq!(failure.failure_category.as_deref(), Some("timeout"));
15118 }
15119 }
15120
15121 #[test]
15122 fn workflow_sleep_emits_one_durable_timer_and_rounds_up() {
15123 let ctx = workflow_context(Vec::new());
15124 let mut sleep = Box::pin(ctx.sleep(Duration::from_millis(1_001)));
15125 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15126
15127 assert!(matches!(
15128 sleep.as_mut().poll(&mut task_context),
15129 Poll::Pending
15130 ));
15131 assert!(matches!(
15132 sleep.as_mut().poll(&mut task_context),
15133 Poll::Pending
15134 ));
15135
15136 let commands = ctx.take_commands().expect("timer command");
15137 assert_eq!(
15138 commands,
15139 vec![json!({
15140 "type": "start_timer",
15141 "delay_seconds": 2,
15142 })]
15143 );
15144 }
15145
15146 #[test]
15147 fn workflow_sleep_replays_matching_schedule_and_fire_without_a_command() {
15148 let history = vec![
15149 history_event(
15150 "TimerScheduled",
15151 json!({
15152 "sequence": 1,
15153 "timer_id": "timer-1",
15154 "delay_seconds": 5,
15155 "fire_at": "2026-07-11T12:00:05Z",
15156 }),
15157 ),
15158 history_event(
15159 "TimerFired",
15160 json!({
15161 "sequence": 1,
15162 "timer_id": "timer-1",
15163 "delay_seconds": 5,
15164 "fire_at": "2026-07-11T12:00:05Z",
15165 "fired_at": "2026-07-11T12:00:05Z",
15166 }),
15167 ),
15168 ];
15169
15170 for _restart in 0..2 {
15171 let ctx = workflow_context(history.clone());
15172 let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(5)));
15173 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15174 assert!(matches!(
15175 sleep.as_mut().poll(&mut task_context),
15176 Poll::Ready(Ok(()))
15177 ));
15178 assert!(ctx.take_commands().expect("commands").is_empty());
15179 ctx.ensure_history_consumed().expect("history consumed");
15180 }
15181 }
15182
15183 #[test]
15184 fn workflow_sleep_rejects_changed_delay_during_replay() {
15185 let ctx = workflow_context(vec![
15186 history_event(
15187 "TimerScheduled",
15188 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
15189 ),
15190 history_event(
15191 "TimerFired",
15192 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
15193 ),
15194 ]);
15195 let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(500)));
15196 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15197
15198 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
15199 sleep.as_mut().poll(&mut task_context)
15200 else {
15201 panic!("changed timer delay must be rejected");
15202 };
15203 assert_eq!(failure.reason, "timer_delay_mismatch");
15204 assert_eq!(failure.sequence, Some(1));
15205 }
15206
15207 #[test]
15208 fn workflow_condition_wait_emits_published_identity_and_timeout_contract() {
15209 let ctx = workflow_context(Vec::new());
15210 let mut wait = Box::pin(
15211 ctx.wait_condition(
15212 ConditionWaitOptions::new("approval.ready", "sha256:approval-v1")
15213 .timeout(Duration::from_millis(60_001)),
15214 || Ok(false),
15215 ),
15216 );
15217 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15218
15219 assert!(matches!(
15220 wait.as_mut().poll(&mut task_context),
15221 Poll::Pending
15222 ));
15223 assert!(matches!(
15224 wait.as_mut().poll(&mut task_context),
15225 Poll::Pending
15226 ));
15227 assert_eq!(
15228 ctx.take_commands().expect("condition command"),
15229 vec![json!({
15230 "type": "open_condition_wait",
15231 "condition_key": "approval.ready",
15232 "condition_definition_fingerprint": "sha256:approval-v1",
15233 "timeout_seconds": 61,
15234 })]
15235 );
15236 }
15237
15238 #[test]
15239 fn workflow_condition_wait_returns_explicit_immediate_results_without_commands() {
15240 let ctx = workflow_context(Vec::new());
15241 let mut satisfied = Box::pin(wait_condition!(ctx, "already-ready", || Ok(true)));
15242 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15243 assert!(matches!(
15244 satisfied.as_mut().poll(&mut task_context),
15245 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
15246 ));
15247
15248 let mut timed_out = Box::pin(wait_condition!(
15249 ctx,
15250 "no-wait",
15251 timeout: Duration::ZERO,
15252 || Ok(false),
15253 ));
15254 assert!(matches!(
15255 timed_out.as_mut().poll(&mut task_context),
15256 Poll::Ready(Ok(ConditionWaitResult::TimedOut))
15257 ));
15258 assert!(ctx.take_commands().expect("commands").is_empty());
15259 }
15260
15261 #[test]
15262 fn signal_and_update_history_reevaluate_open_conditions_after_restart() {
15263 let signal_history = vec![
15264 history_event(
15265 "ConditionWaitOpened",
15266 json!({
15267 "sequence": 4,
15268 "condition_wait_id": "condition:4",
15269 "condition_key": "approval",
15270 "condition_definition_fingerprint": "sha256:approval-v1",
15271 "timeout_seconds": 30,
15272 }),
15273 ),
15274 history_event(
15275 "SignalReceived",
15276 json!({
15277 "workflow_sequence": 4,
15278 "signal_name": "approve",
15279 "arguments": fixture_envelope(json!(["Ada"])),
15280 }),
15281 ),
15282 ];
15283 for _worker_before_or_after_restart in 0..2 {
15284 let ctx = workflow_context(signal_history.clone());
15285 let predicate_ctx = ctx.clone();
15286 let mut wait = Box::pin(
15287 ctx.wait_condition(
15288 ConditionWaitOptions::new("approval", "sha256:approval-v1")
15289 .timeout(Duration::from_secs(30)),
15290 move || Ok(!predicate_ctx.signals("approve")?.is_empty()),
15291 ),
15292 );
15293 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15294 assert!(matches!(
15295 wait.as_mut().poll(&mut task_context),
15296 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
15297 ));
15298 assert!(ctx.take_commands().expect("commands").is_empty());
15299 ctx.ensure_history_consumed().expect("condition consumed");
15300 }
15301
15302 let update_history = vec![
15303 history_event(
15304 "ConditionWaitOpened",
15305 json!({
15306 "sequence": 7,
15307 "condition_wait_id": "condition:7",
15308 "condition_key": "update-approval",
15309 "condition_definition_fingerprint": "sha256:update-approval-v1",
15310 }),
15311 ),
15312 history_event(
15313 "UpdateApplied",
15314 json!({
15315 "sequence": 7,
15316 "update_id": "update-1",
15317 "update_name": "approve",
15318 "arguments": fixture_envelope(json!([true])),
15319 }),
15320 ),
15321 ];
15322 let ctx = workflow_context(update_history);
15323 let predicate_ctx = ctx.clone();
15324 let mut wait = Box::pin(ctx.wait_condition(
15325 ConditionWaitOptions::new("update-approval", "sha256:update-approval-v1"),
15326 move || {
15327 Ok(predicate_ctx
15328 .updates("approve")?
15329 .first()
15330 .and_then(|arguments| arguments.first())
15331 .and_then(Value::as_bool)
15332 == Some(true))
15333 },
15334 ));
15335 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15336 assert!(matches!(
15337 wait.as_mut().poll(&mut task_context),
15338 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
15339 ));
15340 assert!(ctx.take_commands().expect("commands").is_empty());
15341 ctx.ensure_history_consumed().expect("condition consumed");
15342 }
15343
15344 #[test]
15345 fn condition_wait_preserves_open_satisfied_and_timed_out_replay_states() {
15346 let open_history = vec![
15347 history_event(
15348 "ConditionWaitOpened",
15349 json!({
15350 "sequence": 3,
15351 "condition_wait_id": "condition:3",
15352 "condition_key": "two-votes",
15353 "condition_definition_fingerprint": "sha256:two-votes-v1",
15354 "timeout_seconds": 120,
15355 }),
15356 ),
15357 history_event(
15358 "SignalReceived",
15359 json!({
15360 "workflow_sequence": 3,
15361 "signal_name": "vote",
15362 "arguments": fixture_envelope(json!(["first"])),
15363 }),
15364 ),
15365 ];
15366 for _worker_before_or_after_restart in 0..2 {
15367 let ctx = workflow_context(open_history.clone());
15368 let predicate_ctx = ctx.clone();
15369 let mut wait = Box::pin(
15370 ctx.wait_condition(
15371 ConditionWaitOptions::new("two-votes", "sha256:two-votes-v1")
15372 .timeout(Duration::from_secs(120)),
15373 move || Ok(predicate_ctx.signals("vote")?.len() >= 2),
15374 ),
15375 );
15376 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15377 assert!(matches!(
15378 wait.as_mut().poll(&mut task_context),
15379 Poll::Pending
15380 ));
15381 assert_eq!(
15382 ctx.take_commands().expect("reopened condition"),
15383 vec![json!({
15384 "type": "open_condition_wait",
15385 "condition_key": "two-votes",
15386 "condition_definition_fingerprint": "sha256:two-votes-v1",
15387 "timeout_seconds": 120,
15388 })]
15389 );
15390 }
15391
15392 let satisfied_ctx = workflow_context(vec![
15393 history_event(
15394 "ConditionWaitOpened",
15395 json!({
15396 "sequence": 5,
15397 "condition_wait_id": "condition:5",
15398 "condition_key": "approval",
15399 "condition_definition_fingerprint": "sha256:approval-v1",
15400 }),
15401 ),
15402 history_event(
15403 "ConditionWaitSatisfied",
15404 json!({
15405 "sequence": 5,
15406 "condition_wait_id": "condition:5",
15407 "condition_key": "approval",
15408 "condition_definition_fingerprint": "sha256:approval-v1",
15409 }),
15410 ),
15411 ]);
15412 let mut satisfied = Box::pin(satisfied_ctx.wait_condition(
15413 ConditionWaitOptions::new("approval", "sha256:approval-v1"),
15414 || Ok(false),
15415 ));
15416 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15417 assert!(matches!(
15418 satisfied.as_mut().poll(&mut task_context),
15419 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
15420 ));
15421
15422 let timed_out_ctx = workflow_context(vec![
15423 history_event(
15424 "ConditionWaitOpened",
15425 json!({
15426 "sequence": 8,
15427 "condition_wait_id": "condition:8",
15428 "condition_key": "approval-timeout",
15429 "condition_definition_fingerprint": "sha256:approval-timeout-v1",
15430 "timeout_seconds": 5,
15431 }),
15432 ),
15433 history_event(
15434 "TimerScheduled",
15435 json!({
15436 "sequence": 9,
15437 "timer_id": "condition-timer:9",
15438 "timer_kind": "condition_timeout",
15439 "condition_wait_id": "condition:8",
15440 "delay_seconds": 5,
15441 }),
15442 ),
15443 history_event(
15444 "TimerFired",
15445 json!({
15446 "sequence": 9,
15447 "timer_id": "condition-timer:9",
15448 "timer_kind": "condition_timeout",
15449 "condition_wait_id": "condition:8",
15450 "delay_seconds": 5,
15451 }),
15452 ),
15453 ]);
15454 let mut timed_out = Box::pin(
15455 timed_out_ctx.wait_condition(
15456 ConditionWaitOptions::new("approval-timeout", "sha256:approval-timeout-v1")
15457 .timeout(Duration::from_secs(5)),
15458 || Ok(true),
15459 ),
15460 );
15461 assert!(matches!(
15462 timed_out.as_mut().poll(&mut task_context),
15463 Poll::Ready(Ok(ConditionWaitResult::TimedOut))
15464 ));
15465 }
15466
15467 #[test]
15468 fn condition_wait_replays_repeated_physical_opens_as_one_logical_wait() {
15469 let ctx = workflow_context(vec![
15470 history_event(
15471 "ConditionWaitOpened",
15472 json!({
15473 "sequence": 3,
15474 "condition_wait_id": "condition:3",
15475 "condition_key": "two-votes",
15476 "condition_definition_fingerprint": "sha256:two-votes-v1",
15477 }),
15478 ),
15479 history_event(
15480 "SignalReceived",
15481 json!({
15482 "workflow_sequence": 3,
15483 "signal_name": "vote",
15484 "arguments": fixture_envelope(json!(["first"])),
15485 }),
15486 ),
15487 history_event(
15488 "ConditionWaitSatisfied",
15489 json!({
15490 "sequence": 3,
15491 "condition_wait_id": "condition:3",
15492 "condition_key": "two-votes",
15493 "condition_definition_fingerprint": "sha256:two-votes-v1",
15494 }),
15495 ),
15496 history_event(
15497 "ConditionWaitOpened",
15498 json!({
15499 "sequence": 5,
15500 "condition_wait_id": "condition:5",
15501 "condition_key": "two-votes",
15502 "condition_definition_fingerprint": "sha256:two-votes-v1",
15503 }),
15504 ),
15505 history_event(
15506 "SignalReceived",
15507 json!({
15508 "workflow_sequence": 5,
15509 "signal_name": "vote",
15510 "arguments": fixture_envelope(json!(["second"])),
15511 }),
15512 ),
15513 history_event(
15514 "ConditionWaitSatisfied",
15515 json!({
15516 "sequence": 5,
15517 "condition_wait_id": "condition:5",
15518 "condition_key": "two-votes",
15519 "condition_definition_fingerprint": "sha256:two-votes-v1",
15520 }),
15521 ),
15522 ]);
15523 let predicate_ctx = ctx.clone();
15524 let mut wait = Box::pin(ctx.wait_condition(
15525 ConditionWaitOptions::new("two-votes", "sha256:two-votes-v1"),
15526 move || Ok(predicate_ctx.signals("vote")?.len() >= 2),
15527 ));
15528 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15529
15530 assert!(matches!(
15531 wait.as_mut().poll(&mut task_context),
15532 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
15533 ));
15534 ctx.ensure_history_consumed()
15535 .expect("every physical wait-open is consumed");
15536 }
15537
15538 #[test]
15539 fn condition_wait_replay_rejects_identity_predicate_and_timeout_changes() {
15540 let history = vec![history_event(
15541 "ConditionWaitOpened",
15542 json!({
15543 "sequence": 12,
15544 "condition_wait_id": "condition:12",
15545 "condition_key": "approval",
15546 "condition_definition_fingerprint": "sha256:approval-v1",
15547 "timeout_seconds": 30,
15548 }),
15549 )];
15550 for (options, expected_reason) in [
15551 (
15552 ConditionWaitOptions::new("changed", "sha256:approval-v1")
15553 .timeout(Duration::from_secs(30)),
15554 "condition_wait_key_mismatch",
15555 ),
15556 (
15557 ConditionWaitOptions::new("approval", "sha256:approval-v2")
15558 .timeout(Duration::from_secs(30)),
15559 "condition_wait_predicate_mismatch",
15560 ),
15561 (
15562 ConditionWaitOptions::new("approval", "sha256:approval-v1")
15563 .timeout(Duration::from_secs(29)),
15564 "condition_wait_timeout_mismatch",
15565 ),
15566 ] {
15567 let ctx = workflow_context(history.clone());
15568 let mut wait = Box::pin(ctx.wait_condition(options, || Ok(false)));
15569 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15570 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
15571 wait.as_mut().poll(&mut task_context)
15572 else {
15573 panic!("changed condition definition must fail replay");
15574 };
15575 assert_eq!(failure.reason, expected_reason);
15576 assert_eq!(failure.sequence, Some(12));
15577 }
15578 }
15579
15580 #[test]
15581 fn condition_wait_history_requires_the_canonical_predicate_fingerprint() {
15582 let error = WorkflowState::new(
15583 vec![history_event(
15584 "ConditionWaitOpened",
15585 json!({
15586 "sequence": 12,
15587 "condition_wait_id": "condition:12",
15588 "condition_key": "approval",
15589 }),
15590 )],
15591 "rust-workers".to_string(),
15592 DEFAULT_CODEC.to_string(),
15593 None,
15594 )
15595 .expect_err("condition history without a predicate fingerprint must fail");
15596
15597 assert!(matches!(
15598 error,
15599 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
15600 if reason == "condition_wait_predicate_fingerprint_missing"
15601 ));
15602 }
15603
15604 #[test]
15605 fn typed_search_attribute_updates_validate_emit_and_replay() {
15606 let update = SearchAttributeUpdate::new()
15607 .keyword("OrderStatus", " waiting ")
15608 .expect("keyword")
15609 .int("Attempt", 3)
15610 .expect("int")
15611 .bool("Escalated", false)
15612 .expect("bool")
15613 .keyword_list("Regions", ["us-east", "eu-west"])
15614 .expect("list")
15615 .datetime("UpdatedAt", "2026-08-22T04:00:00Z")
15616 .expect("datetime")
15617 .delete("LegacyStatus")
15618 .expect("delete");
15619 let ctx = workflow_context(Vec::new());
15620 ctx.upsert_search_attributes(update.clone())
15621 .expect("typed update");
15622 assert_eq!(
15623 ctx.take_commands().expect("search-attribute command"),
15624 vec![json!({
15625 "type": "upsert_search_attributes",
15626 "attributes": {
15627 "Attempt": 3,
15628 "Escalated": false,
15629 "LegacyStatus": null,
15630 "OrderStatus": "waiting",
15631 "Regions": ["us-east", "eu-west"],
15632 "UpdatedAt": "2026-08-22T04:00:00Z",
15633 },
15634 "attribute_types": {
15635 "Attempt": "int",
15636 "Escalated": "bool",
15637 "OrderStatus": "keyword",
15638 "Regions": "keyword_list",
15639 "UpdatedAt": "datetime",
15640 },
15641 })]
15642 );
15643
15644 let replay = workflow_context(vec![history_event(
15645 "SearchAttributesUpserted",
15646 json!({
15647 "sequence": 6,
15648 "attributes": {
15649 "Attempt": 3,
15650 "Escalated": false,
15651 "LegacyStatus": null,
15652 "OrderStatus": "waiting",
15653 "Regions": ["us-east", "eu-west"],
15654 "UpdatedAt": "2026-08-22T04:00:00Z",
15655 },
15656 "attribute_types": {
15657 "Attempt": "int",
15658 "Escalated": "bool",
15659 "OrderStatus": "keyword",
15660 "Regions": "keyword_list",
15661 "UpdatedAt": "datetime",
15662 },
15663 "merged": {},
15664 }),
15665 )]);
15666 replay
15667 .upsert_search_attributes(update)
15668 .expect("matching update replays");
15669 assert!(replay.take_commands().expect("commands").is_empty());
15670 replay.ensure_history_consumed().expect("history consumed");
15671
15672 let type_drift = workflow_context(vec![history_event(
15673 "SearchAttributesUpserted",
15674 json!({
15675 "sequence": 7,
15676 "attributes": {"OrderStatus": "waiting"},
15677 "attribute_types": {"OrderStatus": "keyword"},
15678 "merged": {"OrderStatus": "waiting"},
15679 }),
15680 )]);
15681 let error = type_drift
15682 .upsert_search_attributes(
15683 SearchAttributeUpdate::new()
15684 .string("OrderStatus", "waiting")
15685 .expect("string update"),
15686 )
15687 .expect_err("same JSON value with a changed type must fail replay");
15688 assert!(matches!(
15689 error,
15690 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
15691 if reason == "search_attribute_type_mismatch"
15692 ));
15693
15694 let malformed_types = WorkflowState::new(
15695 vec![history_event(
15696 "SearchAttributesUpserted",
15697 json!({
15698 "sequence": 8,
15699 "attributes": {"OrderStatus": "waiting"},
15700 "attribute_types": {"OrderStatus": "unsupported"},
15701 "merged": {"OrderStatus": "waiting"},
15702 }),
15703 )],
15704 "rust-workers".to_string(),
15705 DEFAULT_CODEC.to_string(),
15706 None,
15707 )
15708 .expect_err("unsupported search-attribute type metadata must fail");
15709 assert!(matches!(
15710 malformed_types,
15711 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
15712 if reason == "search_attribute_types_malformed"
15713 ));
15714
15715 assert!(matches!(
15716 SearchAttributeUpdate::new().keyword("bad key", "value"),
15717 Err(SearchAttributeUpdateError::InvalidKey(_))
15718 ));
15719 assert!(matches!(
15720 SearchAttributeUpdate::new().float("Ratio", f64::NAN),
15721 Err(SearchAttributeUpdateError::NonFiniteFloat(_))
15722 ));
15723 assert!(matches!(
15724 SearchAttributeUpdate::new().keyword("UnicodeKeyword", "é".repeat(128)),
15725 Err(SearchAttributeUpdateError::ValueTooLong { .. })
15726 ));
15727 assert!(matches!(
15728 SearchAttributeUpdate::new().datetime("UpdatedAt", "2026-02-30T04:00:00Z"),
15729 Err(SearchAttributeUpdateError::InvalidDateTime(_))
15730 ));
15731 assert!(matches!(
15732 workflow_context(Vec::new()).upsert_search_attributes(SearchAttributeUpdate::new()),
15733 Err(Error::InvalidSearchAttributeUpdate(
15734 SearchAttributeUpdateError::Empty
15735 ))
15736 ));
15737 }
15738
15739 #[test]
15740 fn workflow_history_rejects_unpaired_or_mismatched_timer_events() {
15741 let lone_fire = WorkflowState::new(
15742 vec![history_event(
15743 "TimerFired",
15744 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
15745 )],
15746 "rust-workers".to_string(),
15747 DEFAULT_CODEC.to_string(),
15748 None,
15749 )
15750 .expect_err("TimerFired requires TimerScheduled");
15751 assert!(matches!(
15752 lone_fire,
15753 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
15754 if reason == "timer_schedule_missing_or_duplicate"
15755 ));
15756
15757 let wrong_identity = WorkflowState::new(
15758 vec![
15759 history_event(
15760 "TimerScheduled",
15761 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
15762 ),
15763 history_event(
15764 "TimerFired",
15765 json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
15766 ),
15767 ],
15768 "rust-workers".to_string(),
15769 DEFAULT_CODEC.to_string(),
15770 None,
15771 )
15772 .expect_err("fire must match scheduled timer identity");
15773 assert!(matches!(
15774 wrong_identity,
15775 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
15776 if reason == "timer_identity_mismatch"
15777 ));
15778
15779 let duplicate_fire = WorkflowState::new(
15780 vec![
15781 history_event(
15782 "TimerScheduled",
15783 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
15784 ),
15785 history_event(
15786 "TimerFired",
15787 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
15788 ),
15789 history_event(
15790 "TimerFired",
15791 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
15792 ),
15793 ],
15794 "rust-workers".to_string(),
15795 DEFAULT_CODEC.to_string(),
15796 None,
15797 )
15798 .expect_err("a durable timer cannot fire twice");
15799 assert!(matches!(
15800 duplicate_fire,
15801 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
15802 if reason == "duplicate_timer_fire"
15803 ));
15804
15805 let wrong_fired_delay = WorkflowState::new(
15806 vec![
15807 history_event(
15808 "TimerScheduled",
15809 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
15810 ),
15811 history_event(
15812 "TimerFired",
15813 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 6}),
15814 ),
15815 ],
15816 "rust-workers".to_string(),
15817 DEFAULT_CODEC.to_string(),
15818 None,
15819 )
15820 .expect_err("timer schedule and fire delays must agree");
15821 assert!(matches!(
15822 wrong_fired_delay,
15823 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
15824 if reason == "timer_history_delay_mismatch"
15825 ));
15826 }
15827
15828 #[test]
15829 fn replay_rejects_activity_moved_before_recorded_timer() {
15830 let ctx = workflow_context(vec![
15831 history_event(
15832 "TimerScheduled",
15833 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
15834 ),
15835 history_event(
15836 "TimerFired",
15837 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
15838 ),
15839 history_event(
15840 "ActivityCompleted",
15841 json!({
15842 "sequence": 2,
15843 "activity_type": "after-timer",
15844 "payload_codec": DEFAULT_CODEC,
15845 "result": fixture_envelope(json!("done")),
15846 }),
15847 ),
15848 ]);
15849 let mut activity = Box::pin(ctx.activity("after-timer", json!([])));
15850 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15851
15852 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
15853 activity.as_mut().poll(&mut task_context)
15854 else {
15855 panic!("reordered durable command must be rejected");
15856 };
15857 assert_eq!(failure.reason, "recorded_command_mismatch");
15858 assert_eq!(failure.sequence, Some(1));
15859 assert_eq!(failure.expected.as_deref(), Some("timer"));
15860 assert_eq!(failure.actual.as_deref(), Some("activity:after-timer"));
15861 }
15862
15863 #[test]
15864 fn workflow_context_emits_a_typed_named_signal_wait() {
15865 let ctx = workflow_context(Vec::new());
15866 let mut signal = Box::pin(ctx.wait_signal("finish"));
15867 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15868
15869 assert!(matches!(
15870 signal.as_mut().poll(&mut task_context),
15871 Poll::Pending
15872 ));
15873 assert_eq!(
15874 ctx.take_commands().expect("signal-wait command"),
15875 vec![json!({
15876 "type": "open_signal_wait",
15877 "signal_name": "finish",
15878 })]
15879 );
15880 }
15881
15882 #[test]
15883 fn condition_wait_history_cannot_be_consumed_as_a_typed_signal_wait() {
15884 let ctx = workflow_context(vec![
15885 history_event(
15886 "ConditionWaitOpened",
15887 json!({
15888 "sequence": 1,
15889 "condition_wait_id": "condition:1",
15890 "condition_key": "signal:finish",
15891 "condition_definition_fingerprint": "sha256:signal-finish-v1",
15892 }),
15893 ),
15894 history_event(
15895 "ConditionWaitSatisfied",
15896 json!({
15897 "sequence": 1,
15898 "condition_wait_id": "condition:1",
15899 "condition_key": "signal:finish",
15900 "condition_definition_fingerprint": "sha256:signal-finish-v1",
15901 }),
15902 ),
15903 history_event(
15904 "SignalReceived",
15905 json!({"signal_name": "finish", "arguments": []}),
15906 ),
15907 ]);
15908 let mut signal = Box::pin(ctx.wait_signal("finish"));
15909 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15910
15911 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
15912 signal.as_mut().poll(&mut task_context)
15913 else {
15914 panic!("condition history must not resolve as a typed signal wait");
15915 };
15916 assert_eq!(failure.reason, "recorded_command_mismatch");
15917 assert_eq!(failure.expected.as_deref(), Some("condition wait"));
15918 }
15919
15920 #[test]
15921 fn replay_orders_signal_waits_and_timers_in_one_command_stream() {
15922 let signal_then_timer = vec![
15923 history_event(
15924 "SignalWaitOpened",
15925 json!({"sequence": 1, "signal_name": "go"}),
15926 ),
15927 history_event(
15928 "SignalApplied",
15929 json!({
15930 "sequence": 1,
15931 "signal_name": "go",
15932 "value": fixture_envelope(json!(["now"])),
15933 }),
15934 ),
15935 history_event(
15936 "TimerScheduled",
15937 json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
15938 ),
15939 history_event(
15940 "TimerFired",
15941 json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
15942 ),
15943 ];
15944
15945 let ctx = workflow_context(signal_then_timer.clone());
15946 let mut signal = Box::pin(ctx.wait_signal("go"));
15947 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15948 assert!(matches!(
15949 signal.as_mut().poll(&mut task_context),
15950 Poll::Ready(Ok(arguments)) if arguments == vec![json!("now")]
15951 ));
15952 let mut timer = Box::pin(ctx.sleep(Duration::from_secs(5)));
15953 assert!(matches!(
15954 timer.as_mut().poll(&mut task_context),
15955 Poll::Ready(Ok(()))
15956 ));
15957 ctx.ensure_history_consumed()
15958 .expect("signal and timer history consumed in order");
15959
15960 let reordered = workflow_context(signal_then_timer);
15961 let mut timer_first = Box::pin(reordered.sleep(Duration::from_secs(5)));
15962 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
15963 timer_first.as_mut().poll(&mut task_context)
15964 else {
15965 panic!("timer cannot consume signal-wait-first history");
15966 };
15967 assert_eq!(failure.reason, "recorded_command_mismatch");
15968 assert_eq!(failure.sequence, Some(1));
15969 assert_eq!(failure.expected.as_deref(), Some("signal wait"));
15970
15971 let timer_then_signal = vec![
15972 history_event(
15973 "TimerScheduled",
15974 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
15975 ),
15976 history_event(
15977 "TimerFired",
15978 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
15979 ),
15980 history_event(
15981 "SignalWaitOpened",
15982 json!({"sequence": 2, "signal_name": "go"}),
15983 ),
15984 history_event(
15985 "SignalApplied",
15986 json!({
15987 "sequence": 2,
15988 "signal_name": "go",
15989 "value": fixture_envelope(json!([])),
15990 }),
15991 ),
15992 ];
15993 let reordered = workflow_context(timer_then_signal);
15994 let mut signal_first = Box::pin(reordered.wait_signal("go"));
15995 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
15996 signal_first.as_mut().poll(&mut task_context)
15997 else {
15998 panic!("signal wait cannot consume timer-first history");
15999 };
16000 assert_eq!(failure.reason, "recorded_command_mismatch");
16001 assert_eq!(failure.sequence, Some(1));
16002 assert_eq!(failure.expected.as_deref(), Some("timer"));
16003 }
16004
16005 #[test]
16006 fn workflow_history_rejects_duplicate_or_colliding_command_sequences() {
16007 let duplicate_timer = WorkflowState::new(
16008 vec![
16009 history_event(
16010 "TimerScheduled",
16011 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
16012 ),
16013 history_event(
16014 "TimerScheduled",
16015 json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
16016 ),
16017 ],
16018 "rust-workers".to_string(),
16019 DEFAULT_CODEC.to_string(),
16020 None,
16021 )
16022 .expect_err("one workflow sequence cannot schedule two timers");
16023 assert!(matches!(
16024 duplicate_timer,
16025 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16026 if reason == "timer_schedule_missing_or_duplicate"
16027 ));
16028
16029 let colliding_kinds = WorkflowState::new(
16030 vec![
16031 history_event(
16032 "TimerScheduled",
16033 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
16034 ),
16035 history_event(
16036 "ActivityCompleted",
16037 json!({"sequence": 1, "activity_type": "same-sequence"}),
16038 ),
16039 ],
16040 "rust-workers".to_string(),
16041 DEFAULT_CODEC.to_string(),
16042 None,
16043 )
16044 .expect_err("one workflow sequence cannot identify two command kinds");
16045 assert!(matches!(
16046 colliding_kinds,
16047 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16048 if reason == "durable_command_sequence_collision"
16049 ));
16050
16051 let duplicate_signal_wait = WorkflowState::new(
16052 vec![
16053 history_event(
16054 "SignalWaitOpened",
16055 json!({"sequence": 1, "signal_name": "go"}),
16056 ),
16057 history_event(
16058 "SignalWaitOpened",
16059 json!({"sequence": 1, "signal_name": "go"}),
16060 ),
16061 ],
16062 "rust-workers".to_string(),
16063 DEFAULT_CODEC.to_string(),
16064 None,
16065 )
16066 .expect_err("one workflow sequence cannot open two signal waits");
16067 assert!(matches!(
16068 duplicate_signal_wait,
16069 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16070 if reason == "signal_wait_open_missing_or_duplicate"
16071 ));
16072 }
16073
16074 #[test]
16075 fn workflow_history_accepts_a_first_command_after_global_sequence_gaps() {
16076 let result = encode_value_envelope(&json!({"captured": true}), DEFAULT_CODEC)
16077 .expect("side-effect result");
16078 let ctx = workflow_context(vec![history_event(
16079 "SideEffectRecorded",
16080 json!({"sequence": 99, "result": result}),
16081 )]);
16082
16083 let replayed: Value = ctx
16084 .side_effect(|| panic!("recorded side effect must not run"))
16085 .expect("positive global workflow sequence is valid");
16086 assert_eq!(replayed, json!({"captured": true}));
16087 ctx.ensure_history_consumed().expect("history consumed");
16088 }
16089
16090 #[test]
16091 fn workflow_history_rejects_zero_and_descending_command_sequences() {
16092 let result =
16093 encode_value_envelope(&json!("captured"), DEFAULT_CODEC).expect("side-effect result");
16094 let zero = WorkflowState::new(
16095 vec![history_event(
16096 "SideEffectRecorded",
16097 json!({"sequence": 0, "result": result.clone()}),
16098 )],
16099 "rust-workers".to_string(),
16100 DEFAULT_CODEC.to_string(),
16101 None,
16102 )
16103 .expect_err("durable command sequences must be positive");
16104 assert!(matches!(
16105 zero,
16106 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16107 if reason == "durable_command_sequence_invalid"
16108 ));
16109
16110 let descending = WorkflowState::new(
16111 vec![
16112 history_event(
16113 "SideEffectRecorded",
16114 json!({"sequence": 3, "result": result}),
16115 ),
16116 history_event(
16117 "VersionMarkerRecorded",
16118 json!({
16119 "sequence": 2,
16120 "change_id": "descending-marker",
16121 "version": 1,
16122 "min_supported": 1,
16123 "max_supported": 1,
16124 }),
16125 ),
16126 ],
16127 "rust-workers".to_string(),
16128 DEFAULT_CODEC.to_string(),
16129 None,
16130 )
16131 .expect_err("new durable commands must remain strictly ordered");
16132 let Error::NonDeterministicReplay(failure) = descending else {
16133 panic!("expected typed replay failure");
16134 };
16135 assert_eq!(failure.reason, "durable_command_sequence_mismatch");
16136 assert_eq!(failure.sequence, Some(2));
16137 assert_eq!(
16138 failure.expected.as_deref(),
16139 Some("workflow sequence greater than 3")
16140 );
16141 assert_eq!(failure.actual.as_deref(), Some("2"));
16142 }
16143
16144 #[test]
16145 fn workflow_task_replay_completes_after_signals_create_sequence_gaps() {
16146 fn worker() -> Worker {
16147 let client = Client::new("http://127.0.0.1:8080").expect("client");
16148 let mut worker = Worker::new(client, "rust-workers");
16149 worker.register_workflow("rust.finish-after-gaps", |ctx, _input| async move {
16150 ctx.wait_signal("finish").await?;
16151 let marker: String =
16152 ctx.side_effect(|| panic!("recorded side effect must not run"))?;
16153 assert_eq!(marker, "after-finish");
16154 Ok(json!("finished"))
16155 });
16156 worker
16157 }
16158
16159 let marker = encode_value_envelope(&json!("after-finish"), DEFAULT_CODEC)
16160 .expect("side-effect result");
16161 let task = workflow_task(
16162 "rust.finish-after-gaps",
16163 vec![
16164 history_event(
16165 "SignalWaitOpened",
16166 json!({"sequence": 1, "signal_name": "finish"}),
16167 ),
16168 history_event(
16169 "SignalReceived",
16170 json!({
16171 "signal_id": "increment-3",
16172 "signal_name": "increment",
16173 "workflow_sequence": 2,
16174 "payload_codec": DEFAULT_CODEC,
16175 "arguments": fixture_envelope(json!([3])),
16176 }),
16177 ),
16178 history_event(
16179 "SignalReceived",
16180 json!({
16181 "signal_id": "increment-5",
16182 "signal_name": "increment",
16183 "workflow_sequence": 3,
16184 "payload_codec": DEFAULT_CODEC,
16185 "arguments": fixture_envelope(json!([5])),
16186 }),
16187 ),
16188 history_event(
16189 "SignalReceived",
16190 json!({
16191 "signal_id": "finish",
16192 "signal_name": "finish",
16193 "workflow_sequence": 4,
16194 "payload_codec": DEFAULT_CODEC,
16195 "arguments": fixture_envelope(json!([])),
16196 }),
16197 ),
16198 history_event(
16199 "SignalApplied",
16200 json!({
16201 "sequence": 1,
16202 "signal_id": "finish",
16203 "signal_name": "finish",
16204 "payload_codec": DEFAULT_CODEC,
16205 "value": fixture_envelope(json!([])),
16206 }),
16207 ),
16208 history_event(
16209 "SideEffectRecorded",
16210 json!({"sequence": 5, "result": marker}),
16211 ),
16212 ],
16213 DEFAULT_CODEC,
16214 );
16215
16216 for _original_or_cold_worker in 0..2 {
16217 let commands = worker()
16218 .execute_workflow_task(task.clone())
16219 .expect("signal gaps preserve deterministic replay");
16220 assert_eq!(commands.len(), 1, "replay emits only terminal completion");
16221 assert_eq!(commands[0]["type"], "complete_workflow");
16222 assert_eq!(
16223 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("workflow output"),
16224 json!("finished")
16225 );
16226 }
16227 }
16228
16229 #[test]
16230 fn workflow_sleep_rejects_unrepresentable_rounded_duration() {
16231 let ctx = workflow_context(Vec::new());
16232 let mut sleep = Box::pin(ctx.start_timer(Duration::new(u64::MAX, 1)));
16233 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16234 assert!(matches!(
16235 sleep.as_mut().poll(&mut task_context),
16236 Poll::Ready(Err(Error::TimerDurationOverflow))
16237 ));
16238 assert!(ctx.take_commands().expect("commands").is_empty());
16239 }
16240
16241 #[test]
16242 fn workflow_memo_update_emits_canonical_command_and_replays_once() {
16243 let entries = AvroValue::Map(BTreeMap::from([
16244 ("text".to_string(), AvroValue::String("same".to_string())),
16245 (
16246 "nested".to_string(),
16247 AvroValue::Map(BTreeMap::from([
16248 ("beta".to_string(), AvroValue::Long(2)),
16249 ("alpha".to_string(), AvroValue::Long(1)),
16250 ])),
16251 ),
16252 ("long".to_string(), AvroValue::Long(7)),
16253 ("double".to_string(), AvroValue::Double(7.0)),
16254 ("binary".to_string(), AvroValue::Bytes(b"same".to_vec())),
16255 ]));
16256 let ctx = workflow_context(Vec::new());
16257 ctx.upsert_memo(entries.clone()).expect("valid memo update");
16258 let commands = ctx.take_commands().expect("commands");
16259
16260 assert_eq!(commands.len(), 1);
16261 assert_eq!(commands[0]["type"], "upsert_memo");
16262 let server_entries = json!({
16263 "codec": "avro",
16264 "blob": "wwHioz3/VYAiNw4KDGJpbmFyeQgIc2FtZQxkb3VibGUGAAAAAAAAHEAIbG9uZwQODG5lc3RlZA4ECmFscGhhBAIIYmV0YQQEAAh0ZXh0CghzYW1lAA==",
16265 });
16266 assert_eq!(
16267 commands[0]["entries"]
16268 .as_object()
16269 .expect("entries envelope")
16270 .keys()
16271 .collect::<Vec<_>>(),
16272 vec!["blob", "codec"]
16273 );
16274 assert_eq!(commands[0]["entries"], server_entries);
16275 let wire_entries =
16276 decode_wire_avro_value(&commands[0]["entries"], DEFAULT_CODEC).expect("memo entries");
16277 assert_eq!(wire_entries, entries);
16278
16279 let history = vec![history_event(
16280 "MemoUpserted",
16281 json!({
16282 "sequence": 1,
16283 "entries": server_entries.clone(),
16284 "merged": server_entries,
16285 }),
16286 )];
16287 let replay = workflow_context(history.clone());
16288 replay
16289 .upsert_memo(entries.clone())
16290 .expect("matching replay identity");
16291 assert!(replay.take_commands().expect("replay commands").is_empty());
16292
16293 let changed_types = AvroValue::Map(BTreeMap::from([
16294 ("text".to_string(), AvroValue::Bytes(b"same".to_vec())),
16295 (
16296 "nested".to_string(),
16297 AvroValue::Map(BTreeMap::from([
16298 ("alpha".to_string(), AvroValue::Long(1)),
16299 ("beta".to_string(), AvroValue::Long(2)),
16300 ])),
16301 ),
16302 ("long".to_string(), AvroValue::Double(7.0)),
16303 ("double".to_string(), AvroValue::Long(7)),
16304 ("binary".to_string(), AvroValue::String("same".to_string())),
16305 ]));
16306 let error = workflow_context(history)
16307 .upsert_memo(changed_types)
16308 .expect_err("memo replay identity must preserve Avro value types");
16309 assert!(matches!(
16310 error,
16311 Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
16312 ));
16313 }
16314
16315 #[test]
16316 fn workflow_memo_update_rejects_changed_replay_identity_and_invalid_keys() {
16317 let original = encode_value_envelope(&json!({"stage": "original"}), DEFAULT_CODEC)
16318 .expect("memo envelope");
16319 let replay = workflow_context(vec![history_event(
16320 "MemoUpserted",
16321 json!({
16322 "sequence": 1,
16323 "entries": original.clone(),
16324 "merged": original
16325 }),
16326 )]);
16327 let error = replay
16328 .upsert_memo(json!({"stage": "changed"}))
16329 .expect_err("changed memo update must fail replay");
16330 assert!(matches!(
16331 error,
16332 Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
16333 ));
16334
16335 let invalid = workflow_context(Vec::new())
16336 .upsert_memo(
16337 json!({"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx": true}),
16338 )
16339 .expect_err("oversized key");
16340 assert!(matches!(invalid, Error::InvalidMemoUpdate(_)));
16341 }
16342
16343 #[test]
16344 fn workflow_memo_replay_distinguishes_signed_zero_identity() {
16345 let negative_zero = AvroValue::Map(BTreeMap::from([(
16346 "reading".to_string(),
16347 AvroValue::Double(-0.0),
16348 )]));
16349 let negative_zero_envelope =
16350 encode_typed_envelope(&negative_zero, DEFAULT_CODEC).expect("negative zero envelope");
16351 let history = vec![history_event(
16352 "MemoUpserted",
16353 json!({
16354 "sequence": 1,
16355 "entries": negative_zero_envelope.clone(),
16356 "merged": negative_zero_envelope,
16357 }),
16358 )];
16359
16360 workflow_context(history.clone())
16361 .upsert_memo(negative_zero)
16362 .expect("matching negative-zero history identity");
16363
16364 let error = workflow_context(history)
16365 .upsert_memo(AvroValue::Map(BTreeMap::from([(
16366 "reading".to_string(),
16367 AvroValue::Double(0.0),
16368 )])))
16369 .expect_err("positive zero must not consume negative-zero memo history");
16370 assert!(matches!(
16371 error,
16372 Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
16373 ));
16374 }
16375
16376 #[test]
16377 fn workflow_memo_capability_requires_flag_and_command_advertisement() {
16378 let supported = json!({
16379 "workflow_memo_updates": {"supported": true, "minimum_protocol_version": "1.14"},
16380 "supported_workflow_task_commands": ["complete_workflow", "upsert_memo"]
16381 });
16382 assert!(runtime_supports_workflow_memo_updates(Some(&supported)));
16383 assert!(!runtime_supports_workflow_memo_updates(Some(&json!({
16384 "workflow_memo_updates": {"supported": false},
16385 "supported_workflow_task_commands": ["upsert_memo"]
16386 }))));
16387 assert!(commands_use_workflow_memo_updates(&[json!({
16388 "type": "upsert_memo",
16389 "entries": {"stage": "processing"}
16390 })]));
16391 }
16392
16393 #[test]
16394 fn workflow_task_replay_completes_without_rescheduling_recorded_commands() {
16395 let client = Client::new("http://127.0.0.1:8080").expect("client");
16396 let mut worker = Worker::new(client, "rust-workers");
16397 worker.register_workflow("rust.timer", |ctx, _input| async move {
16398 ctx.sleep(Duration::from_secs(5)).await?;
16399 ctx.activity("after-timer", json!([])).await
16400 });
16401
16402 let task = |history_events| WorkflowTask {
16403 task_id: "wft-rust-timer-1".to_string(),
16404 workflow_command_id: None,
16405 workflow_id: Some("wf-rust-timer".to_string()),
16406 run_id: Some("run-rust-timer".to_string()),
16407 workflow_type: "rust.timer".to_string(),
16408 cancel_requested: false,
16409 payload_codec: DEFAULT_CODEC.to_string(),
16410 arguments: Some(
16411 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
16412 ),
16413 history_events,
16414 total_history_events: None,
16415 history_size_bytes: None,
16416 continue_as_new_recommended: None,
16417 history_budget_pressure: None,
16418 next_history_page_token: None,
16419 workflow_task_attempt: 1,
16420 workflow_signal_id: None,
16421 signal_name: None,
16422 signal_arguments: None,
16423 workflow_update_id: None,
16424 update_name: None,
16425 lease_owner: Some("rust-worker".to_string()),
16426 };
16427
16428 let initial = worker
16429 .execute_workflow_task(task(Vec::new()))
16430 .expect("initial timer task");
16431 assert_eq!(
16432 initial,
16433 vec![json!({"type": "start_timer", "delay_seconds": 5})]
16434 );
16435
16436 let activity_result =
16437 encode_value_envelope(&json!("done"), DEFAULT_CODEC).expect("activity result");
16438 let replayed = worker
16439 .execute_workflow_task(task(vec![
16440 history_event(
16441 "TimerScheduled",
16442 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
16443 ),
16444 history_event(
16445 "TimerFired",
16446 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
16447 ),
16448 history_event(
16449 "ActivityCompleted",
16450 json!({
16451 "sequence": 2,
16452 "activity_type": "after-timer",
16453 "payload_codec": DEFAULT_CODEC,
16454 "result": activity_result,
16455 }),
16456 ),
16457 ]))
16458 .expect("replayed workflow task");
16459 assert_eq!(replayed.len(), 1);
16460 assert_eq!(replayed[0]["type"], "complete_workflow");
16461 assert_eq!(
16462 decode_wire_value(&replayed[0]["result"], DEFAULT_CODEC).expect("result"),
16463 json!("done")
16464 );
16465 }
16466
16467 #[test]
16468 fn workflow_continue_as_new_emits_arguments_type_and_queue_once() {
16469 let client = Client::new("http://127.0.0.1:8080").expect("client");
16470 let mut worker = Worker::new(client, "rust-workers");
16471 worker.register_workflow("rust.continue", |ctx, _input| async move {
16472 ctx.continue_as_new_with_options(
16473 ContinueAsNewOptions::new()
16474 .workflow_type("rust.next")
16475 .task_queue("next-workers"),
16476 json!([2, {"cursor": "next"}]),
16477 )
16478 });
16479
16480 let commands = worker
16481 .execute_workflow_task(workflow_task("rust.continue", Vec::new(), DEFAULT_CODEC))
16482 .expect("continue-as-new command");
16483
16484 assert_eq!(commands.len(), 1);
16485 assert_eq!(commands[0]["type"], "continue_as_new");
16486 assert_eq!(commands[0]["workflow_type"], "rust.next");
16487 assert_eq!(commands[0]["queue"], "next-workers");
16488 assert_eq!(
16489 decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC)
16490 .expect("continue-as-new arguments"),
16491 json!([2, {"cursor": "next"}])
16492 );
16493 }
16494
16495 #[test]
16496 fn continue_as_new_preserves_typed_arguments() {
16497 let client = Client::new("http://127.0.0.1:8080").expect("client");
16498 let mut worker = Worker::new(client, "rust-workers");
16499 worker.register_workflow_avro_value("rust.typed-continue", |ctx, _input| async move {
16500 ctx.continue_as_new(AvroValue::Array(vec![typed_fidelity_probe()]))?;
16501 unreachable!("continue-as-new returns a control-flow error")
16502 });
16503
16504 let commands = worker
16505 .execute_workflow_task(workflow_task(
16506 "rust.typed-continue",
16507 Vec::new(),
16508 DEFAULT_CODEC,
16509 ))
16510 .expect("typed continue-as-new command");
16511
16512 assert_eq!(commands[0]["type"], "continue_as_new");
16513 assert_eq!(
16514 decode_wire_avro_value(&commands[0]["arguments"], DEFAULT_CODEC)
16515 .expect("typed continue arguments"),
16516 AvroValue::Array(vec![typed_fidelity_probe()])
16517 );
16518 }
16519
16520 #[test]
16521 fn recorded_continue_as_new_is_consumed_without_duplicate_successor_command() {
16522 let client = Client::new("http://127.0.0.1:8080").expect("client");
16523 let mut worker = Worker::new(client, "rust-workers");
16524 worker.register_workflow("rust.continue", |ctx, _input| async move {
16525 ctx.continue_as_new(json!([2]))
16526 });
16527 let task = workflow_task(
16528 "rust.continue",
16529 vec![history_event(
16530 "WorkflowContinuedAsNew",
16531 json!({"sequence": 1, "continued_to_run_id": "run-next"}),
16532 )],
16533 DEFAULT_CODEC,
16534 );
16535
16536 for _worker_restart_or_redelivery in 0..2 {
16537 let commands = worker
16538 .execute_workflow_task(task.clone())
16539 .expect("recorded transition replays");
16540 assert!(
16541 commands.is_empty(),
16542 "replay must not emit another successor"
16543 );
16544 }
16545 }
16546
16547 #[test]
16548 fn continue_as_new_rejects_invalid_overrides_before_emitting_a_command() {
16549 let ctx = workflow_context(Vec::new());
16550 let error = ctx
16551 .continue_as_new_with_options(ContinueAsNewOptions::new().task_queue(" "), json!([1]))
16552 .expect_err("blank queue must be rejected");
16553
16554 let Error::InvalidContinueAsNewOptions(error) = error else {
16555 panic!("expected typed continue-as-new validation error");
16556 };
16557 assert_eq!(error.field, "task_queue");
16558 assert!(ctx.take_commands().expect("commands").is_empty());
16559 }
16560
16561 #[test]
16562 fn workflow_context_exposes_server_history_budget() {
16563 let client = Client::new("http://127.0.0.1:8080").expect("client");
16564 let mut worker = Worker::new(client, "rust-workers");
16565 worker.register_workflow("rust.history-budget", |ctx, _input| async move {
16566 let budget = ctx.history_budget()?;
16567 Ok(json!({
16568 "events": budget.event_count,
16569 "bytes": budget.size_bytes,
16570 "recommended": budget.continue_as_new_recommended,
16571 "pressure": budget.pressure,
16572 }))
16573 });
16574 let task: WorkflowTask = serde_json::from_value(json!({
16575 "task_id": "task-history-budget",
16576 "workflow_type": "rust.history-budget",
16577 "payload_codec": DEFAULT_CODEC,
16578 "history_events": [],
16579 "total_history_events": 480,
16580 "history_size_bytes": 1_048_576,
16581 "continue_as_new_recommended": true,
16582 "history_budget_pressure": "continue_as_new_recommended",
16583 }))
16584 .expect("published workflow task");
16585
16586 let commands = worker
16587 .execute_workflow_task(task)
16588 .expect("history-budget workflow");
16589 let result = decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("result");
16590 assert_eq!(result["events"], 480);
16591 assert_eq!(result["bytes"], 1_048_576);
16592 assert_eq!(result["recommended"], true);
16593 assert_eq!(result["pressure"], "continue_as_new_recommended");
16594 }
16595
16596 #[test]
16597 fn uncaught_workflow_handler_error_emits_terminal_failure_command() {
16598 let client = Client::new("http://127.0.0.1:8080").expect("client");
16599 let mut worker = Worker::new(client, "rust-workers");
16600 worker.register_workflow("rust.failing", |_ctx, _input| async move {
16601 Err(Error::Codec("rust_conformance_failure".to_string()))
16602 });
16603 let task = WorkflowTask {
16604 task_id: "wft-rust-failing-1".to_string(),
16605 workflow_command_id: None,
16606 workflow_id: Some("wf-rust-failing".to_string()),
16607 run_id: Some("run-rust-failing".to_string()),
16608 workflow_type: "rust.failing".to_string(),
16609 cancel_requested: false,
16610 payload_codec: DEFAULT_CODEC.to_string(),
16611 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
16612 history_events: Vec::new(),
16613 total_history_events: Some(0),
16614 history_size_bytes: None,
16615 continue_as_new_recommended: None,
16616 history_budget_pressure: None,
16617 next_history_page_token: None,
16618 workflow_task_attempt: 1,
16619 workflow_signal_id: None,
16620 signal_name: None,
16621 signal_arguments: None,
16622 workflow_update_id: None,
16623 update_name: None,
16624 lease_owner: Some("rust-worker".to_string()),
16625 };
16626
16627 let commands = worker
16628 .execute_workflow_task(task)
16629 .expect("handler failure becomes a workflow command");
16630
16631 assert_eq!(commands.len(), 1);
16632 assert_eq!(commands[0]["type"], "fail_workflow");
16633 assert_eq!(commands[0]["exception_type"], "RustWorkflowError");
16634 assert_eq!(commands[0]["exception_class"], "durable_workflow::Error");
16635 assert_eq!(commands[0]["non_retryable"], false);
16636 assert_eq!(
16637 commands[0]["message"],
16638 "codec error: rust_conformance_failure"
16639 );
16640 assert_eq!(
16641 commands[0]["exception"]["message"],
16642 "codec error: rust_conformance_failure"
16643 );
16644 }
16645
16646 #[test]
16647 fn ordinary_handler_error_preserves_commands_queued_in_the_same_decision() {
16648 let client = Client::new("http://127.0.0.1:8080").expect("client");
16649 let mut worker = Worker::new(client, "rust-workers");
16650 worker.register_workflow("rust.failing-after-side-effect", |ctx, _input| async move {
16651 let _: String = ctx.side_effect(|| "captured".to_string())?;
16652 Err(Error::WorkerLoop("application failure".to_string()))
16653 });
16654
16655 let commands = worker
16656 .execute_workflow_task(workflow_task(
16657 "rust.failing-after-side-effect",
16658 Vec::new(),
16659 DEFAULT_CODEC,
16660 ))
16661 .expect("ordinary failure remains a workflow decision");
16662
16663 assert_eq!(commands.len(), 2);
16664 assert_eq!(commands[0]["type"], "record_side_effect");
16665 assert_eq!(commands[1]["type"], "fail_workflow");
16666 }
16667
16668 #[test]
16669 fn handler_error_cannot_hide_an_unconsumed_committed_side_effect() {
16670 let client = Client::new("http://127.0.0.1:8080").expect("client");
16671 let mut worker = Worker::new(client, "rust-workers");
16672 worker.register_workflow("rust.removed-side-effect", |_ctx, _input| async move {
16673 Err(Error::WorkerLoop("application failure".to_string()))
16674 });
16675 let result =
16676 encode_value_envelope(&json!("committed"), DEFAULT_CODEC).expect("side-effect result");
16677
16678 let error = worker
16679 .execute_workflow_task(workflow_task(
16680 "rust.removed-side-effect",
16681 vec![history_event(
16682 "SideEffectRecorded",
16683 json!({"sequence": 1, "result": result}),
16684 )],
16685 DEFAULT_CODEC,
16686 ))
16687 .expect_err("removed committed history must not become fail_workflow");
16688
16689 let Error::NonDeterministicReplay(failure) = error else {
16690 panic!("expected typed replay failure");
16691 };
16692 assert_eq!(failure.reason, "recorded_commands_unconsumed");
16693 assert_eq!(failure.sequence, Some(1));
16694 assert_eq!(failure.expected.as_deref(), Some("side effect"));
16695 }
16696
16697 #[test]
16698 fn replay_error_discards_side_effect_queued_before_incompatible_marker_check() {
16699 let client = Client::new("http://127.0.0.1:8080").expect("client");
16700 let mut worker = Worker::new(client, "rust-workers");
16701 worker.register_workflow(
16702 "rust.side-effect-before-marker-error",
16703 |ctx, _input| async move {
16704 assert_eq!(ctx.get_version("restart-safe", 1, 1)?, 1);
16705 let _: String = ctx.side_effect(|| "must-not-commit".to_string())?;
16706 ctx.get_version("restart-safe", 2, 2)?;
16707 Ok(Value::Null)
16708 },
16709 );
16710
16711 let error = worker
16712 .execute_workflow_task(workflow_task(
16713 "rust.side-effect-before-marker-error",
16714 vec![history_event(
16715 "VersionMarkerRecorded",
16716 json!({
16717 "sequence": 1,
16718 "change_id": "restart-safe",
16719 "version": 1,
16720 "min_supported": 1,
16721 "max_supported": 1,
16722 }),
16723 )],
16724 DEFAULT_CODEC,
16725 ))
16726 .expect_err("replay error must return no queued workflow commands");
16727
16728 let Error::NonDeterministicReplay(failure) = error else {
16729 panic!("expected typed replay failure");
16730 };
16731 assert_eq!(failure.reason, "version_marker_incompatible_range");
16732 assert_eq!(failure.sequence, Some(1));
16733 }
16734
16735 #[test]
16736 fn workflow_task_replay_keeps_recorded_unfired_timer_pending_without_rescheduling() {
16737 let client = Client::new("http://127.0.0.1:8080").expect("client");
16738 let mut worker = Worker::new(client, "rust-workers");
16739 worker.register_workflow("rust.timer.pending", |ctx, _input| async move {
16740 ctx.sleep(Duration::from_secs(5)).await?;
16741 Ok(json!({"status": "timer fired"}))
16742 });
16743
16744 let task = WorkflowTask {
16745 task_id: "wft-rust-timer-pending".to_string(),
16746 workflow_command_id: None,
16747 workflow_id: Some("wf-rust-timer".to_string()),
16748 run_id: Some("run-rust-timer".to_string()),
16749 workflow_type: "rust.timer.pending".to_string(),
16750 cancel_requested: false,
16751 payload_codec: DEFAULT_CODEC.to_string(),
16752 arguments: Some(
16753 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
16754 ),
16755 history_events: vec![history_event(
16756 "TimerScheduled",
16757 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
16758 )],
16759 total_history_events: Some(1),
16760 history_size_bytes: None,
16761 continue_as_new_recommended: None,
16762 history_budget_pressure: None,
16763 next_history_page_token: None,
16764 workflow_task_attempt: 1,
16765 workflow_signal_id: None,
16766 signal_name: None,
16767 signal_arguments: None,
16768 workflow_update_id: None,
16769 update_name: None,
16770 lease_owner: Some("rust-worker".to_string()),
16771 };
16772
16773 for _redelivery_or_restart in 0..2 {
16774 let commands = worker
16775 .execute_workflow_task(task.clone())
16776 .expect("recorded timer remains pending");
16777 assert!(
16778 commands.is_empty(),
16779 "recorded timer must not be rescheduled"
16780 );
16781 }
16782 }
16783
16784 #[test]
16785 fn workflow_task_rejects_recorded_command_removed_from_workflow_code() {
16786 let client = Client::new("http://127.0.0.1:8080").expect("client");
16787 let mut worker = Worker::new(client, "rust-workers");
16788 worker.register_workflow("rust.timer.removed", |_ctx, _input| async move {
16789 Ok(json!({"status": "completed"}))
16790 });
16791 let task = WorkflowTask {
16792 task_id: "wft-rust-timer-removed".to_string(),
16793 workflow_command_id: None,
16794 workflow_id: Some("wf-rust-timer".to_string()),
16795 run_id: Some("run-rust-timer".to_string()),
16796 workflow_type: "rust.timer.removed".to_string(),
16797 cancel_requested: false,
16798 payload_codec: DEFAULT_CODEC.to_string(),
16799 arguments: Some(
16800 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
16801 ),
16802 history_events: vec![
16803 history_event(
16804 "TimerScheduled",
16805 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
16806 ),
16807 history_event(
16808 "TimerFired",
16809 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
16810 ),
16811 ],
16812 total_history_events: Some(2),
16813 history_size_bytes: None,
16814 continue_as_new_recommended: None,
16815 history_budget_pressure: None,
16816 next_history_page_token: None,
16817 workflow_task_attempt: 1,
16818 workflow_signal_id: None,
16819 signal_name: None,
16820 signal_arguments: None,
16821 workflow_update_id: None,
16822 update_name: None,
16823 lease_owner: Some("rust-worker".to_string()),
16824 };
16825
16826 let Error::NonDeterministicReplay(failure) = worker
16827 .execute_workflow_task(task)
16828 .expect_err("removed timer must fail replay")
16829 else {
16830 panic!("expected typed replay failure");
16831 };
16832 assert_eq!(failure.reason, "recorded_commands_unconsumed");
16833 assert_eq!(failure.sequence, Some(1));
16834 }
16835
16836 #[test]
16837 fn workflow_context_emits_explicit_child_workflow_contract() {
16838 let ctx = WorkflowContext {
16839 state: Arc::new(Mutex::new(
16840 WorkflowState::new_with_identity(
16841 Vec::new(),
16842 Some("wf-parent".to_string()),
16843 Some("run-parent".to_string()),
16844 "parent-workers".to_string(),
16845 DEFAULT_CODEC.to_string(),
16846 None,
16847 )
16848 .expect("workflow state"),
16849 )),
16850 };
16851 let options = ChildWorkflowOptions::new("python-workers")
16852 .parent_close_policy(ParentClosePolicy::RequestCancel)
16853 .retry_policy(ChildWorkflowRetryPolicy {
16854 max_attempts: Some(3),
16855 backoff_seconds: vec![1, 5],
16856 non_retryable_error_types: vec!["ValidationError".to_string()],
16857 })
16858 .execution_timeout_seconds(600)
16859 .run_timeout_seconds(120);
16860 let mut call = Box::pin(ctx.start_child_workflow(
16861 "python.fulfil-order",
16862 options,
16863 json!([{"order_id": "order-42"}]),
16864 ));
16865 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16866
16867 assert!(matches!(
16868 call.as_mut().poll(&mut task_context),
16869 Poll::Pending
16870 ));
16871 let commands = ctx.take_commands().expect("commands");
16872 assert_eq!(commands.len(), 1);
16873 let command = &commands[0];
16874 assert_eq!(command["type"], "start_child_workflow");
16875 assert_eq!(command["workflow_type"], "python.fulfil-order");
16876 assert_eq!(command["queue"], "python-workers");
16877 assert_eq!(command["parent_close_policy"], "request_cancel");
16878 assert_eq!(command["retry_policy"]["max_attempts"], 3);
16879 assert_eq!(command["execution_timeout_seconds"], 600);
16880 assert_eq!(command["run_timeout_seconds"], 120);
16881 assert_eq!(
16882 decode_wire_value(&command["arguments"], DEFAULT_CODEC).expect("child args"),
16883 json!([{"order_id": "order-42"}])
16884 );
16885 }
16886
16887 fn child_parent_worker() -> Worker {
16888 let client = Client::new("http://127.0.0.1:8080").expect("client");
16889 let mut worker = Worker::new(client, "rust-parent-workers");
16890 worker.register_workflow("rust.parent", |ctx, _input| async move {
16891 let child = ctx
16892 .start_child_workflow(
16893 "python.child",
16894 ChildWorkflowOptions::new("python-child-workers")
16895 .parent_close_policy(ParentClosePolicy::Terminate),
16896 json!([{"codec_probe": [1, true, "rust"]}]),
16897 )
16898 .await?;
16899 Ok(json!({
16900 "parent_workflow_id": child.parent.workflow_id,
16901 "parent_run_id": child.parent.run_id,
16902 "child_workflow_id": child.child.workflow_id,
16903 "child_run_id": child.child.run_id,
16904 "child_workflow_type": child.child_workflow_type,
16905 "result": child.result,
16906 }))
16907 });
16908 worker
16909 }
16910
16911 fn child_parent_task(event_type: &str, payload: Value) -> WorkflowTask {
16912 WorkflowTask {
16913 task_id: "wft-child-parent".to_string(),
16914 workflow_command_id: None,
16915 workflow_id: Some("wf-parent".to_string()),
16916 run_id: Some("run-parent".to_string()),
16917 workflow_type: "rust.parent".to_string(),
16918 cancel_requested: false,
16919 payload_codec: DEFAULT_CODEC.to_string(),
16920 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
16921 history_events: vec![
16922 HistoryEvent {
16923 event_type: "ChildWorkflowScheduled".to_string(),
16924 payload: json!({
16925 "sequence": 1,
16926 "child_call_id": "call-child",
16927 "child_workflow_instance_id": "wf-child",
16928 "child_workflow_run_id": "run-child",
16929 "child_workflow_type": "python.child",
16930 }),
16931 raw: HashMap::new(),
16932 },
16933 HistoryEvent {
16934 event_type: event_type.to_string(),
16935 payload,
16936 raw: HashMap::new(),
16937 },
16938 ],
16939 total_history_events: Some(2),
16940 history_size_bytes: None,
16941 continue_as_new_recommended: None,
16942 history_budget_pressure: None,
16943 next_history_page_token: None,
16944 workflow_task_attempt: 1,
16945 workflow_signal_id: None,
16946 signal_name: None,
16947 signal_arguments: None,
16948 workflow_update_id: None,
16949 update_name: None,
16950 lease_owner: Some("rust-worker".to_string()),
16951 }
16952 }
16953
16954 #[test]
16955 fn committed_child_result_replays_without_starting_a_duplicate() {
16956 let worker = child_parent_worker();
16957 let task = child_parent_task(
16958 "ChildRunCompleted",
16959 json!({
16960 "sequence": 1,
16961 "child_call_id": "call-child",
16962 "child_workflow_instance_id": "wf-child",
16963 "child_workflow_run_id": "run-child",
16964 "child_workflow_type": "python.child",
16965 "payload_codec": DEFAULT_CODEC,
16966 "result": fixture_envelope(json!({"from":"python","ok":true})),
16967 }),
16968 );
16969
16970 for _restart in 0..2 {
16971 let commands = worker
16972 .execute_workflow_task(task.clone())
16973 .expect("replayed parent task");
16974 assert_eq!(commands.len(), 1);
16975 assert_eq!(commands[0]["type"], "complete_workflow");
16976 assert!(!commands
16977 .iter()
16978 .any(|command| command["type"] == "start_child_workflow"));
16979 let output =
16980 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("parent output");
16981 assert_eq!(output["parent_workflow_id"], "wf-parent");
16982 assert_eq!(output["parent_run_id"], "run-parent");
16983 assert_eq!(output["child_workflow_id"], "wf-child");
16984 assert_eq!(output["child_run_id"], "run-child");
16985 assert_eq!(output["result"], json!({"from": "python", "ok": true}));
16986 }
16987 }
16988
16989 #[test]
16990 fn typed_child_arguments_and_results_survive_replay() {
16991 let client = Client::new("http://127.0.0.1:8080").expect("client");
16992 let mut worker = Worker::new(client, "rust-parent-workers");
16993 worker.register_workflow_avro_value("rust.typed-parent", |ctx, _input| async move {
16994 let child = ctx
16995 .start_child_workflow_avro_value(
16996 "python.typed-child",
16997 ChildWorkflowOptions::new("python-workers"),
16998 AvroValue::Array(vec![typed_fidelity_probe()]),
16999 )
17000 .await?;
17001 Ok(child.result)
17002 });
17003
17004 let initial = worker
17005 .execute_workflow_task(workflow_task(
17006 "rust.typed-parent",
17007 Vec::new(),
17008 DEFAULT_CODEC,
17009 ))
17010 .expect("typed child start");
17011 assert_eq!(initial[0]["type"], "start_child_workflow");
17012 assert_eq!(
17013 decode_wire_avro_value(&initial[0]["arguments"], DEFAULT_CODEC)
17014 .expect("typed child arguments"),
17015 AvroValue::Array(vec![typed_fidelity_probe()])
17016 );
17017
17018 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
17019 .expect("typed child result");
17020 let task = workflow_task(
17021 "rust.typed-parent",
17022 vec![
17023 history_event(
17024 "ChildWorkflowScheduled",
17025 json!({
17026 "sequence": 1,
17027 "child_call_id": "call-typed",
17028 "child_workflow_instance_id": "wf-child",
17029 "child_workflow_run_id": "run-child",
17030 "child_workflow_type": "python.typed-child",
17031 }),
17032 ),
17033 history_event(
17034 "ChildRunCompleted",
17035 json!({
17036 "sequence": 1,
17037 "child_call_id": "call-typed",
17038 "child_workflow_instance_id": "wf-child",
17039 "child_workflow_run_id": "run-child",
17040 "child_workflow_type": "python.typed-child",
17041 "payload_codec": DEFAULT_CODEC,
17042 "result": result,
17043 }),
17044 ),
17045 ],
17046 DEFAULT_CODEC,
17047 );
17048
17049 let commands = worker
17050 .execute_workflow_task(task)
17051 .expect("typed child replay");
17052 assert_eq!(commands[0]["type"], "complete_workflow");
17053 assert_eq!(
17054 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17055 .expect("typed parent result"),
17056 typed_fidelity_probe()
17057 );
17058 }
17059
17060 #[test]
17061 fn pending_child_replays_after_restart_without_starting_a_duplicate() {
17062 let worker = child_parent_worker();
17063 let mut task = child_parent_task("unused", Value::Null);
17064 task.history_events.truncate(1);
17065 task.total_history_events = Some(1);
17066
17067 for _redelivery_or_restart in 0..2 {
17068 let commands = worker
17069 .execute_workflow_task(task.clone())
17070 .expect("recorded child remains pending");
17071 assert!(
17072 commands.is_empty(),
17073 "recorded pending child must not be started again"
17074 );
17075 }
17076 }
17077
17078 #[test]
17079 fn child_cancellation_becomes_stable_parent_failure_command() {
17080 let worker = child_parent_worker();
17081 let task = child_parent_task(
17082 "ChildRunCancelled",
17083 json!({
17084 "sequence": 1,
17085 "child_workflow_instance_id": "wf-child",
17086 "child_workflow_run_id": "run-child",
17087 "child_workflow_type": "python.child",
17088 "failure_id": "failure-child",
17089 "failure_category": "cancelled",
17090 "message": "cancelled by parent-close policy",
17091 }),
17092 );
17093
17094 let commands = worker
17095 .execute_workflow_task(task)
17096 .expect("parent settlement");
17097 assert_eq!(commands.len(), 1);
17098 assert_eq!(commands[0]["type"], "fail_workflow");
17099 assert_eq!(commands[0]["exception_type"], "ChildWorkflowCancelled");
17100 assert_eq!(
17101 commands[0]["exception"]["properties"]["reason"],
17102 "cancelled"
17103 );
17104 assert_eq!(
17105 commands[0]["exception"]["properties"]["child_workflow_run_id"],
17106 "run-child"
17107 );
17108 }
17109
17110 #[test]
17111 fn workflow_can_handle_typed_child_failure() {
17112 let client = Client::new("http://127.0.0.1:8080").expect("client");
17113 let mut worker = Worker::new(client, "rust-parent-workers");
17114 worker.register_workflow("rust.handled-parent", |ctx, _input| async move {
17115 match ctx
17116 .start_child_workflow(
17117 "python.child",
17118 ChildWorkflowOptions::new("python-child-workers"),
17119 json!([]),
17120 )
17121 .await
17122 {
17123 Err(Error::ChildWorkflowFailed(failure)) => Ok(json!({
17124 "reason": failure.reason,
17125 "failure_id": failure.failure_id,
17126 "exception_class": failure.exception_class,
17127 "child_run_id": failure.child_workflow_run_id,
17128 })),
17129 Err(error) => Err(error),
17130 Ok(_) => Err(Error::WorkerLoop(
17131 "child unexpectedly succeeded".to_string(),
17132 )),
17133 }
17134 });
17135 let mut task = child_parent_task(
17136 "ChildRunFailed",
17137 json!({
17138 "sequence": 1,
17139 "child_workflow_instance_id": "wf-child",
17140 "child_workflow_run_id": "run-child",
17141 "child_workflow_type": "python.child",
17142 "failure_id": "failure-child",
17143 "failure_category": "child_workflow",
17144 "message": "payment rejected",
17145 "exception": {
17146 "type": "PaymentRejected",
17147 "class": "payments.PaymentRejected",
17148 "message": "payment rejected"
17149 }
17150 }),
17151 );
17152 task.workflow_type = "rust.handled-parent".to_string();
17153
17154 let commands = worker.execute_workflow_task(task).expect("handled failure");
17155 assert_eq!(commands[0]["type"], "complete_workflow");
17156 let output =
17157 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("parent output");
17158 assert_eq!(output["reason"], "child_workflow");
17159 assert_eq!(output["failure_id"], "failure-child");
17160 assert_eq!(output["exception_class"], "payments.PaymentRejected");
17161 assert_eq!(output["child_run_id"], "run-child");
17162 }
17163
17164 #[test]
17165 fn rust_hello_world_uses_signal_arguments_from_resume_payload() {
17166 let client = Client::new("http://127.0.0.1:8080").expect("client");
17167 let mut worker = Worker::new(client, "rust-workers");
17168
17169 worker.register_workflow("rust.hello_workflow", |ctx, _input| async move {
17170 let signal = ctx.wait_signal("start").await?;
17171 let name = signal
17172 .first()
17173 .and_then(|value| value.as_str())
17174 .unwrap_or("world");
17175 let greeting = ctx.activity("rust.hello_activity", json!([name])).await?;
17176 Ok(json!({
17177 "greeting": greeting,
17178 "language": "rust"
17179 }))
17180 });
17181
17182 let signal_arguments =
17183 encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC).expect("signal arguments");
17184 let task = WorkflowTask {
17185 task_id: "wft-rust-signal-1".to_string(),
17186 workflow_command_id: None,
17187 workflow_id: Some("wf-rust-hello".to_string()),
17188 run_id: Some("run-rust-hello".to_string()),
17189 workflow_type: "rust.hello_workflow".to_string(),
17190 cancel_requested: false,
17191 payload_codec: DEFAULT_CODEC.to_string(),
17192 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
17193 history_events: vec![HistoryEvent {
17194 event_type: "SignalReceived".to_string(),
17195 payload: json!({
17196 "signal_id": "sig-rust-1",
17197 "signal_name": "start"
17198 }),
17199 raw: HashMap::new(),
17200 }],
17201 total_history_events: Some(1),
17202 history_size_bytes: None,
17203 continue_as_new_recommended: None,
17204 history_budget_pressure: None,
17205 next_history_page_token: None,
17206 workflow_task_attempt: 1,
17207 workflow_signal_id: Some("sig-rust-1".to_string()),
17208 signal_name: Some("start".to_string()),
17209 signal_arguments: Some(signal_arguments),
17210 workflow_update_id: None,
17211 update_name: None,
17212 lease_owner: Some("rust-worker".to_string()),
17213 };
17214
17215 let commands = worker.execute_workflow_task(task).expect("workflow task");
17216
17217 assert_eq!(commands.len(), 1);
17218 assert_eq!(commands[0]["type"], "schedule_activity");
17219 assert_eq!(commands[0]["activity_type"], "rust.hello_activity");
17220 assert_eq!(
17221 decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC).expect("activity args"),
17222 json!(["Rust"])
17223 );
17224 }
17225
17226 #[test]
17227 fn workflow_task_appends_paginated_history_events() {
17228 let mut task = WorkflowTask {
17229 task_id: "wft-rust-pages-1".to_string(),
17230 workflow_command_id: None,
17231 workflow_id: Some("wf-rust-pages".to_string()),
17232 run_id: Some("run-rust-pages".to_string()),
17233 workflow_type: "rust.hello_workflow".to_string(),
17234 cancel_requested: false,
17235 payload_codec: DEFAULT_CODEC.to_string(),
17236 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
17237 history_events: vec![HistoryEvent {
17238 event_type: "WorkflowStarted".to_string(),
17239 payload: json!({}),
17240 raw: HashMap::new(),
17241 }],
17242 total_history_events: Some(3),
17243 history_size_bytes: None,
17244 continue_as_new_recommended: None,
17245 history_budget_pressure: None,
17246 next_history_page_token: Some("MQ==".to_string()),
17247 workflow_task_attempt: 1,
17248 workflow_signal_id: None,
17249 signal_name: None,
17250 signal_arguments: None,
17251 workflow_update_id: None,
17252 update_name: None,
17253 lease_owner: Some("rust-worker".to_string()),
17254 };
17255
17256 task.append_history_page(WorkflowTaskHistoryPage {
17257 history_events: vec![
17258 HistoryEvent {
17259 event_type: "SignalReceived".to_string(),
17260 payload: json!({
17261 "signal_id": "sig-rust-1",
17262 "signal_name": "start",
17263 "arguments": encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC)
17264 .expect("signal arguments")
17265 }),
17266 raw: HashMap::new(),
17267 },
17268 HistoryEvent {
17269 event_type: "MarkerRecorded".to_string(),
17270 payload: json!({"sequence": 3}),
17271 raw: HashMap::new(),
17272 },
17273 ],
17274 total_history_events: Some(3),
17275 next_history_page_token: None,
17276 });
17277
17278 assert_eq!(task.history_events.len(), 3);
17279 assert_eq!(task.total_history_events, Some(3));
17280 assert_eq!(task.next_history_page_token, None);
17281
17282 let signal = task
17283 .history_events
17284 .iter()
17285 .find(|event| event.event_type == "SignalReceived")
17286 .expect("signal event");
17287 assert_eq!(
17288 decode_signal_event_arguments(signal, DEFAULT_CODEC).expect("signal arguments"),
17289 vec![AvroValue::String("Rust".to_string())]
17290 );
17291 }
17292
17293 #[tokio::test]
17294 async fn query_handler_reads_ordered_cross_codec_signals_without_commands() {
17295 let client = Client::new("http://127.0.0.1:8080").expect("client");
17296 let mut worker = Worker::new(client, "rust-workers");
17297 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
17298 worker.register_query("counter", "current", |ctx, _args| async move {
17299 let mut count = 0_i64;
17300 for signal in ctx.signal_events() {
17301 let value = signal
17302 .arguments
17303 .first()
17304 .and_then(Value::as_i64)
17305 .unwrap_or_default();
17306 match signal.name.as_str() {
17307 "increment" => count += value,
17308 "set" => count = value,
17309 _ => {}
17310 }
17311 }
17312 Ok(json!(count))
17313 });
17314
17315 let task = QueryTask {
17316 query_task_id: "query-rust-counter".to_string(),
17317 query_task_attempt: 1,
17318 lease_owner: Some("rust-worker".to_string()),
17319 workflow_id: Some("counter-1".to_string()),
17320 run_id: Some("run-counter-1".to_string()),
17321 workflow_type: "counter".to_string(),
17322 query_name: "current".to_string(),
17323 payload_codec: DEFAULT_CODEC.to_string(),
17324 workflow_arguments: Some(
17325 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
17326 ),
17327 query_arguments: Some(
17328 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("query arguments"),
17329 ),
17330 history_events: vec![
17331 HistoryEvent {
17332 event_type: "SignalReceived".to_string(),
17333 payload: json!({
17334 "signal_id": "php-signal-1",
17335 "signal_name": "increment",
17336 "workflow_sequence": 1,
17337 "payload_codec": DEFAULT_CODEC,
17338 "arguments": encode_value_envelope(&json!([3]), DEFAULT_CODEC).expect("php avro signal")
17339 }),
17340 raw: HashMap::new(),
17341 },
17342 HistoryEvent {
17343 event_type: "SignalReceived".to_string(),
17344 payload: json!({
17345 "signal_id": "python-signal-2",
17346 "signal_name": "increment",
17347 "workflow_sequence": 2,
17348 "payload_codec": DEFAULT_CODEC,
17349 "arguments": encode_value_envelope(&json!([5]), DEFAULT_CODEC).expect("python avro signal")
17350 }),
17351 raw: HashMap::new(),
17352 },
17353 HistoryEvent {
17354 event_type: "SignalReceived".to_string(),
17355 payload: json!({
17356 "signal_id": "rust-signal-3",
17357 "signal_name": "set",
17358 "workflow_sequence": 3,
17359 "payload_codec": DEFAULT_CODEC,
17360 "arguments": encode_value_envelope(&json!([0]), DEFAULT_CODEC).expect("rust avro signal")
17361 }),
17362 raw: HashMap::new(),
17363 },
17364 ],
17365 history_export: None,
17366 run_status: Some("completed".to_string()),
17367 };
17368
17369 let result = worker.execute_query_task(task).await.expect("query result");
17370 assert_eq!(result.into_json().expect("query projection"), json!(0));
17371 }
17372
17373 #[tokio::test]
17374 async fn replayed_queries_read_running_completed_and_cold_restarted_instance_state() {
17375 let worker = replay_counter_worker();
17376 let running_history = json!([
17377 {
17378 "type": "ActivityCompleted",
17379 "payload": {
17380 "sequence": 1,
17381 "activity_type": "load-counter",
17382 "payload_codec": DEFAULT_CODEC,
17383 "result": fixture_envelope(json!("loaded"))
17384 }
17385 },
17386 {
17387 "type": "SignalWaitOpened",
17388 "payload": {
17389 "sequence": 3,
17390 "signal_name": "increment"
17391 }
17392 },
17393 {
17394 "type": "SignalReceived",
17395 "payload": {
17396 "signal_id": "signal-3",
17397 "signal_name": "increment",
17398 "workflow_sequence": 2,
17399 "payload_codec": DEFAULT_CODEC,
17400 "arguments": fixture_envelope(json!([3]))
17401 }
17402 },
17403 {
17404 "type": "SignalApplied",
17405 "payload": {
17406 "sequence": 3,
17407 "signal_id": "signal-3",
17408 "signal_name": "increment",
17409 "payload_codec": DEFAULT_CODEC,
17410 "value": fixture_envelope(json!([3]))
17411 }
17412 }
17413 ]);
17414
17415 let running = worker
17416 .execute_query_task(replay_counter_query(
17417 "current",
17418 running_history.clone(),
17419 "running",
17420 ))
17421 .await
17422 .expect("running replay query");
17423 assert_eq!(
17424 running.clone().into_json().expect("query projection"),
17425 json!({"loaded": "loaded", "count": 3, "finished": false})
17426 );
17427
17428 let detached = worker
17429 .execute_query_task(replay_counter_query(
17430 "detached-mutation",
17431 running_history.clone(),
17432 "running",
17433 ))
17434 .await
17435 .expect("query mutates only its detached state clone");
17436 assert_eq!(detached.into_json().expect("query projection"), json!(999));
17437 let failed = worker
17438 .execute_query_task(replay_counter_query(
17439 "failed-mutation",
17440 running_history.clone(),
17441 "running",
17442 ))
17443 .await
17444 .expect_err("failed query");
17445 assert_eq!(failed.reason, "query_rejected");
17446 let unchanged = worker
17447 .execute_query_task(replay_counter_query("current", running_history, "running"))
17448 .await
17449 .expect("later query reconstructs unchanged state");
17450 assert_eq!(unchanged, running);
17451
17452 let restarted_worker = replay_counter_worker();
17453 let empty_arguments = fixture_envelope(json!([]));
17454 let loaded_result = fixture_envelope(json!("loaded"));
17455 let signal_three = fixture_blob(json!([3]));
17456 let signal_five = fixture_blob(json!([5]));
17457 let restarted_task: QueryTask = serde_json::from_value(json!({
17458 "query_task_id": "query-after-restart",
17459 "workflow_id": "counter-1",
17460 "run_id": "run-counter-1",
17461 "workflow_type": "replay-counter",
17462 "query_name": "current",
17463 "payload_codec": DEFAULT_CODEC,
17464 "workflow_arguments": empty_arguments.clone(),
17465 "query_arguments": empty_arguments,
17466 "history_events": [],
17467 "history_export": {
17468 "payloads": {"codec": DEFAULT_CODEC},
17469 "history_events": [
17470 {
17471 "type": "ActivityCompleted",
17472 "payload": {
17473 "sequence": 1,
17474 "activity_type": "load-counter",
17475 "payload_codec": DEFAULT_CODEC,
17476 "result": null
17477 }
17478 },
17479 {
17480 "type": "SignalWaitOpened",
17481 "payload": {
17482 "sequence": 3,
17483 "signal_name": "increment"
17484 }
17485 },
17486 {
17487 "type": "SignalReceived",
17488 "payload": {
17489 "signal_id": "signal-3",
17490 "signal_name": "increment",
17491 "workflow_sequence": 2
17492 }
17493 },
17494 {
17495 "type": "SignalApplied",
17496 "payload": {
17497 "sequence": 3,
17498 "signal_id": "signal-3",
17499 "signal_name": "increment"
17500 }
17501 },
17502 {
17503 "type": "SignalWaitOpened",
17504 "payload": {
17505 "sequence": 5,
17506 "signal_name": "increment"
17507 }
17508 },
17509 {
17510 "type": "SignalReceived",
17511 "payload": {
17512 "signal_id": "signal-5",
17513 "signal_name": "increment",
17514 "workflow_sequence": 4
17515 }
17516 },
17517 {
17518 "type": "SignalApplied",
17519 "payload": {
17520 "sequence": 5,
17521 "signal_id": "signal-5",
17522 "signal_name": "increment"
17523 }
17524 }
17525 ],
17526 "activities": [{
17527 "sequence": 1,
17528 "activity_type": "load-counter",
17529 "payload_codec": DEFAULT_CODEC,
17530 "result": loaded_result
17531 }],
17532 "signals": [
17533 {
17534 "id": "signal-3",
17535 "name": "increment",
17536 "workflow_sequence": 2,
17537 "payload_codec": DEFAULT_CODEC,
17538 "arguments": signal_three
17539 },
17540 {
17541 "id": "signal-5",
17542 "name": "increment",
17543 "workflow_sequence": 4,
17544 "payload_codec": DEFAULT_CODEC,
17545 "arguments": signal_five
17546 }
17547 ]
17548 },
17549 "run_status": "completed"
17550 }))
17551 .expect("cold replay query task");
17552 let completed = restarted_worker
17553 .execute_query_task(restarted_task)
17554 .await
17555 .expect("completed cold replay query");
17556 assert_eq!(
17557 completed.into_json().expect("query projection"),
17558 json!({"loaded": "loaded", "count": 8, "finished": true})
17559 );
17560 }
17561
17562 #[tokio::test]
17563 async fn replayed_query_replay_failures_are_machine_readable() {
17564 let worker = replay_counter_worker();
17565 let task = replay_counter_query(
17566 "current",
17567 json!([{
17568 "type": "ActivityCompleted",
17569 "payload": {
17570 "sequence": 1,
17571 "payload_codec": DEFAULT_CODEC,
17572 "result": {"codec": DEFAULT_CODEC, "blob": "{"}
17573 }
17574 }]),
17575 "running",
17576 );
17577 let failure = worker
17578 .execute_query_task(task)
17579 .await
17580 .expect_err("invalid replay history payload");
17581 assert_eq!(failure.reason, "query_payload_decode_failed");
17582 assert_eq!(failure.failure_type, "QueryPayloadDecodeFailed");
17583 assert!(failure.message.contains("invalid_payload_framing"));
17584 }
17585
17586 #[tokio::test]
17587 async fn query_task_restores_compact_history_from_export() {
17588 let client = Client::new("http://127.0.0.1:8080").expect("client");
17589 let mut worker = Worker::new(client, "rust-workers");
17590 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
17591 worker.register_query("counter", "current", |ctx, _args| async move {
17592 Ok(json!(ctx.signals("increment")[0][0]))
17593 });
17594 let empty_arguments = fixture_envelope(json!([]));
17595 let exported_signal = fixture_blob(json!([9]));
17596 let task: QueryTask = serde_json::from_value(json!({
17597 "query_task_id": "query-export",
17598 "workflow_type": "counter",
17599 "query_name": "current",
17600 "payload_codec": DEFAULT_CODEC,
17601 "workflow_arguments": empty_arguments.clone(),
17602 "query_arguments": empty_arguments,
17603 "history_events": [],
17604 "history_export": {
17605 "payloads": {"codec": DEFAULT_CODEC},
17606 "history_events": [{
17607 "type": "SignalReceived",
17608 "payload": {"signal_id": "signal-export", "signal_name": "increment"}
17609 }],
17610 "signals": [{
17611 "id": "signal-export",
17612 "name": "increment",
17613 "status": "applied",
17614 "workflow_sequence": 1,
17615 "payload_codec": DEFAULT_CODEC,
17616 "arguments": exported_signal
17617 }]
17618 }
17619 }))
17620 .expect("query task");
17621
17622 let result = worker.execute_query_task(task).await.expect("query result");
17623 assert_eq!(result.into_json().expect("query projection"), json!(9));
17624 }
17625
17626 #[tokio::test]
17627 async fn query_task_failures_have_stable_reasons() {
17628 let client = Client::new("http://127.0.0.1:8080").expect("client");
17629 let mut worker = Worker::new(client, "rust-workers");
17630 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
17631 worker.register_query(
17632 "counter",
17633 "current",
17634 |_ctx, _args| async move { Ok(json!(0)) },
17635 );
17636
17637 let base_task = QueryTask {
17638 query_task_id: "query-errors".to_string(),
17639 query_task_attempt: 1,
17640 lease_owner: None,
17641 workflow_id: Some("counter-errors".to_string()),
17642 run_id: Some("run-errors".to_string()),
17643 workflow_type: "counter".to_string(),
17644 query_name: "missing".to_string(),
17645 payload_codec: DEFAULT_CODEC.to_string(),
17646 workflow_arguments: Some(fixture_envelope(json!([]))),
17647 query_arguments: Some(fixture_envelope(json!([]))),
17648 history_events: Vec::new(),
17649 history_export: None,
17650 run_status: Some("running".to_string()),
17651 };
17652
17653 let unknown = worker
17654 .execute_query_task(base_task.clone())
17655 .await
17656 .expect_err("unknown query");
17657 assert_eq!(unknown.reason, "rejected_unknown_query");
17658
17659 let mut malformed = base_task;
17660 malformed.query_name = "current".to_string();
17661 malformed.query_arguments = Some(json!({"codec": DEFAULT_CODEC, "blob": "{"}));
17662 let malformed = worker
17663 .execute_query_task(malformed)
17664 .await
17665 .expect_err("malformed payload");
17666 assert_eq!(malformed.reason, "query_payload_decode_failed");
17667
17668 let client = Client::new("http://127.0.0.1:8080").expect("client");
17669 let mut unavailable_worker = Worker::new(client, "rust-workers");
17670 unavailable_worker
17671 .register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
17672 let empty_arguments = fixture_envelope(json!([]));
17673 let unavailable_task: QueryTask = serde_json::from_value(json!({
17674 "query_task_id": "query-unavailable",
17675 "workflow_type": "counter",
17676 "query_name": "current",
17677 "payload_codec": DEFAULT_CODEC,
17678 "workflow_arguments": empty_arguments.clone(),
17679 "query_arguments": empty_arguments
17680 }))
17681 .expect("query task");
17682 let unavailable = unavailable_worker
17683 .execute_query_task(unavailable_task)
17684 .await
17685 .expect_err("query handler unavailable");
17686 assert_eq!(unavailable.reason, "query_handler_unavailable");
17687 }
17688
17689 #[tokio::test]
17690 async fn client_query_decodes_result_and_typed_failure() {
17691 let server = MockWorkerServer::start();
17692 let client = Client::builder(server.base_url())
17693 .timeout(Duration::from_secs(2))
17694 .build()
17695 .expect("client");
17696
17697 let result = client
17698 .query_workflow("counter-1", "current", json!([]))
17699 .await
17700 .expect("query result");
17701 assert_eq!(result, json!({"count": 8}));
17702
17703 let error = client
17704 .query_workflow("counter-1", "missing", json!([]))
17705 .await
17706 .expect_err("unknown query");
17707 let Error::QueryFailed(failure) = error else {
17708 panic!("expected typed query failure");
17709 };
17710 assert_eq!(failure.status, 404);
17711 assert_eq!(failure.reason, "rejected_unknown_query");
17712 }
17713
17714 #[tokio::test]
17715 async fn public_client_surfaces_send_and_receive_lossless_avro_values() {
17716 let server = MockWorkerServer::start();
17717 let client = Client::builder(server.base_url())
17718 .timeout(Duration::from_secs(2))
17719 .build()
17720 .expect("client");
17721 let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
17722
17723 client
17724 .start_workflow(
17725 "typed.echo",
17726 "rust-workers",
17727 "typed-start",
17728 arguments.clone(),
17729 )
17730 .await
17731 .expect("typed workflow start");
17732 assert_eq!(
17733 decode_wire_avro_value(
17734 &server.request_body("/api/workflows")["input"],
17735 DEFAULT_CODEC,
17736 )
17737 .expect("typed start input"),
17738 arguments
17739 );
17740
17741 client
17742 .signal_workflow("typed-1", "changed", arguments.clone())
17743 .await
17744 .expect("typed signal");
17745 assert_eq!(
17746 decode_wire_avro_value(
17747 &server.request_body("/api/workflows/typed-1/signal/changed")["input"],
17748 DEFAULT_CODEC,
17749 )
17750 .expect("typed signal input"),
17751 arguments
17752 );
17753
17754 assert_eq!(
17755 client
17756 .query_workflow_avro_value("typed-1", "inspect", arguments.clone())
17757 .await
17758 .expect("typed query"),
17759 typed_fidelity_probe()
17760 );
17761 assert_eq!(
17762 decode_wire_avro_value(
17763 &server.request_body("/api/workflows/typed-1/query/inspect")["input"],
17764 DEFAULT_CODEC,
17765 )
17766 .expect("typed query input"),
17767 arguments
17768 );
17769
17770 assert_eq!(
17771 client
17772 .update_workflow_avro_value(
17773 "typed-1",
17774 "replace",
17775 arguments.clone(),
17776 Some("typed-request"),
17777 )
17778 .await
17779 .expect("typed update"),
17780 typed_fidelity_probe()
17781 );
17782 let update = server.request_body("/api/workflows/typed-1/update/replace");
17783 assert_eq!(update["request_id"], "typed-request");
17784 assert_eq!(
17785 decode_wire_avro_value(&update["input"], DEFAULT_CODEC).expect("typed update input"),
17786 arguments
17787 );
17788
17789 let handle = WorkflowHandle {
17790 client: client.clone(),
17791 workflow_id: "typed-1".to_string(),
17792 run_id: Some("run-typed-1".to_string()),
17793 workflow_type: "typed.echo".to_string(),
17794 };
17795 assert_eq!(
17796 handle
17797 .result_avro_value(WorkflowResultOptions::default())
17798 .await
17799 .expect("typed workflow result"),
17800 typed_fidelity_probe()
17801 );
17802
17803 client
17804 .complete_activity_task(
17805 "activity-typed",
17806 "attempt-typed",
17807 "rust-worker",
17808 typed_fidelity_probe(),
17809 DEFAULT_CODEC,
17810 )
17811 .await
17812 .expect("typed activity completion");
17813 assert_eq!(
17814 decode_wire_avro_value(
17815 &server.request_body("/api/worker/activity-tasks/activity-typed/complete")
17816 ["result"],
17817 DEFAULT_CODEC,
17818 )
17819 .expect("typed activity result"),
17820 typed_fidelity_probe()
17821 );
17822 client
17823 .fail_activity_task(
17824 "activity-typed",
17825 "attempt-typed",
17826 "rust-worker",
17827 "typed failure",
17828 true,
17829 )
17830 .await
17831 .expect("activity failure");
17832 }
17833
17834 #[tokio::test]
17835 async fn lifecycle_commands_support_instance_and_selected_run_targets() {
17836 let server = MockWorkerServer::start();
17837 let client = Client::builder(server.base_url())
17838 .timeout(Duration::from_secs(2))
17839 .build()
17840 .expect("client");
17841
17842 let options = WorkflowCommandOptions::new()
17843 .reason("cleanup requested")
17844 .request_id("cancel-17");
17845 let cancelled = client
17846 .cancel_workflow("wf-lifecycle", options)
17847 .await
17848 .expect("instance cancellation");
17849 assert_eq!(cancelled.command, WorkflowCommandKind::Cancel);
17850 assert_eq!(cancelled.run_id.as_deref(), Some("run-current"));
17851 assert_eq!(cancelled.outcome.as_deref(), Some("cancelled"));
17852 assert_eq!(
17853 server.request_body("/api/workflows/wf-lifecycle/cancel"),
17854 json!({"reason":"cleanup requested","request_id":"cancel-17"})
17855 );
17856
17857 let terminated = client
17858 .terminate_workflow(
17859 "wf-lifecycle",
17860 WorkflowCommandOptions::new().reason("forced stop"),
17861 )
17862 .await
17863 .expect("instance termination");
17864 assert_eq!(terminated.command, WorkflowCommandKind::Terminate);
17865 assert_eq!(terminated.outcome.as_deref(), Some("terminated"));
17866
17867 client
17868 .cancel_workflow_run(
17869 "wf-lifecycle",
17870 "run-current",
17871 WorkflowCommandOptions::default(),
17872 )
17873 .await
17874 .expect("selected run cancellation");
17875 client
17876 .terminate_workflow_run(
17877 "wf-lifecycle",
17878 "run-current",
17879 WorkflowCommandOptions::default(),
17880 )
17881 .await
17882 .expect("selected run termination");
17883
17884 for (command, error) in [
17885 (
17886 WorkflowCommandKind::Cancel,
17887 client
17888 .cancel_workflow_run(
17889 "wf-lifecycle",
17890 "run-stale",
17891 WorkflowCommandOptions::default(),
17892 )
17893 .await
17894 .expect_err("stale cancellation must be rejected"),
17895 ),
17896 (
17897 WorkflowCommandKind::Terminate,
17898 client
17899 .terminate_workflow_run(
17900 "wf-lifecycle",
17901 "run-stale",
17902 WorkflowCommandOptions::default(),
17903 )
17904 .await
17905 .expect_err("stale termination must be rejected"),
17906 ),
17907 ] {
17908 let Error::WorkflowCommandRejected(rejection) = error else {
17909 panic!("expected typed command rejection");
17910 };
17911 assert_eq!(rejection.command, command);
17912 assert_eq!(rejection.status, 409);
17913 assert_eq!(rejection.reason, "historical_run_command_rejected");
17914 assert_eq!(rejection.run_id.as_deref(), Some("run-stale"));
17915 assert_eq!(rejection.target_scope.as_deref(), Some("run"));
17916 }
17917 }
17918
17919 #[tokio::test]
17920 async fn workflow_start_options_send_server_enforced_deadlines() {
17921 let server = MockWorkerServer::start();
17922 let client = Client::builder(server.base_url())
17923 .timeout(Duration::from_secs(2))
17924 .build()
17925 .expect("client");
17926
17927 let handle = client
17928 .start_workflow_with_options(
17929 "rust.timeout",
17930 "rust-timeouts",
17931 "wf-start-options",
17932 WorkflowStartOptions::new()
17933 .execution_timeout_seconds(30)
17934 .run_timeout_seconds(1),
17935 json!([]),
17936 )
17937 .await
17938 .expect("workflow start");
17939
17940 assert_eq!(handle.run_id.as_deref(), Some("run-start-options"));
17941 let body = server.request_body("/api/workflows");
17942 assert_eq!(body["execution_timeout_seconds"], 30);
17943 assert_eq!(body["run_timeout_seconds"], 1);
17944
17945 let invalid = client
17946 .start_workflow_with_options(
17947 "rust.timeout",
17948 "rust-timeouts",
17949 "wf-invalid-options",
17950 WorkflowStartOptions::new()
17951 .execution_timeout_seconds(1)
17952 .run_timeout_seconds(2),
17953 json!([]),
17954 )
17955 .await
17956 .expect_err("invalid deadline ordering");
17957 assert!(invalid
17958 .to_string()
17959 .contains("run_timeout_seconds cannot exceed execution_timeout_seconds"));
17960 }
17961
17962 #[tokio::test]
17963 async fn workflow_result_returns_each_typed_terminal_outcome() {
17964 let server = MockWorkerServer::start();
17965 let client = Client::builder(server.base_url())
17966 .timeout(Duration::from_secs(2))
17967 .build()
17968 .expect("client");
17969 let options = WorkflowResultOptions {
17970 poll_interval: Duration::ZERO,
17971 timeout: Duration::from_secs(1),
17972 };
17973
17974 let failed = WorkflowHandle {
17975 client: client.clone(),
17976 workflow_id: "wf-failed".to_string(),
17977 run_id: Some("run-failed".to_string()),
17978 workflow_type: "failure".to_string(),
17979 }
17980 .result(options)
17981 .await
17982 .expect_err("failed outcome");
17983 let Error::WorkflowFailed(failure) = failed else {
17984 panic!("expected WorkflowFailed");
17985 };
17986 assert_eq!(failure.workflow_id, "wf-failed");
17987 assert_eq!(failure.run_id.as_deref(), Some("run-failed"));
17988 assert_eq!(failure.failure_id.as_deref(), Some("failure-17"));
17989 assert_eq!(failure.failure_category.as_deref(), Some("application"));
17990 assert_eq!(failure.exception_type.as_deref(), Some("PaymentError"));
17991 assert_eq!(
17992 failure.exception_class.as_deref(),
17993 Some("billing::PaymentError")
17994 );
17995 assert_eq!(failure.non_retryable, Some(true));
17996
17997 for (workflow_id, expected_kind, expected_reason) in [
17998 (
17999 "wf-cancelled",
18000 WorkflowTerminalKind::Cancelled,
18001 "cleanup requested",
18002 ),
18003 (
18004 "wf-terminated",
18005 WorkflowTerminalKind::Terminated,
18006 "forced stop",
18007 ),
18008 (
18009 "wf-timed-out",
18010 WorkflowTerminalKind::TimedOut,
18011 "run_timeout",
18012 ),
18013 ] {
18014 let error = WorkflowHandle {
18015 client: client.clone(),
18016 workflow_id: workflow_id.to_string(),
18017 run_id: None,
18018 workflow_type: "terminal".to_string(),
18019 }
18020 .result(options)
18021 .await
18022 .expect_err("typed terminal outcome");
18023 let outcome = match error {
18024 Error::WorkflowCancelled(outcome) => outcome,
18025 Error::WorkflowTerminated(outcome) => outcome,
18026 Error::WorkflowTimedOut(outcome) => outcome,
18027 other => panic!("unexpected terminal error: {other}"),
18028 };
18029 assert_eq!(outcome.kind, expected_kind);
18030 assert_eq!(outcome.workflow_id, workflow_id);
18031 assert_eq!(outcome.reason, expected_reason);
18032 }
18033
18034 let wait_timeout = WorkflowHandle {
18035 client,
18036 workflow_id: "wf-waiting".to_string(),
18037 run_id: Some("run-waiting".to_string()),
18038 workflow_type: "waiting".to_string(),
18039 }
18040 .result(WorkflowResultOptions {
18041 poll_interval: Duration::ZERO,
18042 timeout: Duration::ZERO,
18043 })
18044 .await
18045 .expect_err("client wait timeout");
18046 let Error::WorkflowTimedOut(timeout) = wait_timeout else {
18047 panic!("expected typed client timeout");
18048 };
18049 assert_eq!(timeout.reason, "result_wait_timeout");
18050 assert_eq!(timeout.failure_category.as_deref(), Some("client_timeout"));
18051 assert_eq!(timeout.run_id.as_deref(), Some("run-waiting"));
18052 }
18053
18054 #[tokio::test]
18055 async fn workflow_result_follows_chain_and_selected_result_preserves_history() {
18056 let server = MockWorkerServer::start();
18057 let client = Client::builder(server.base_url())
18058 .timeout(Duration::from_secs(2))
18059 .build()
18060 .expect("client");
18061
18062 let handle = WorkflowHandle {
18063 client,
18064 workflow_id: "wf-selected".to_string(),
18065 run_id: Some("run-selected".to_string()),
18066 workflow_type: "selected".to_string(),
18067 };
18068 let options = WorkflowResultOptions {
18069 poll_interval: Duration::ZERO,
18070 timeout: Duration::from_secs(1),
18071 };
18072
18073 let current = handle
18074 .result(options)
18075 .await
18076 .expect("instance result follows the current run");
18077 assert_eq!(current, json!("current run output"));
18078
18079 let error = handle
18080 .result_selected_run(options)
18081 .await
18082 .expect_err("the selected run is cancelled even though the current run completed");
18083
18084 let Error::WorkflowCancelled(outcome) = error else {
18085 panic!("expected selected run cancellation");
18086 };
18087 assert_eq!(outcome.run_id.as_deref(), Some("run-selected"));
18088 assert_eq!(outcome.reason, "selected run cancelled");
18089 assert_eq!(
18090 server.request_count("/api/workflows/wf-selected/runs/run-selected"),
18091 1
18092 );
18093 assert_eq!(server.request_count("/api/workflows/wf-selected"), 1);
18094 }
18095
18096 #[tokio::test]
18097 async fn poll_responses_decode_http_conflict_drain_as_a_stable_stop() {
18098 let server = MockWorkerServer::draining_polls();
18099 let client = Client::builder(server.base_url())
18100 .timeout(Duration::from_secs(2))
18101 .build()
18102 .expect("client");
18103
18104 let workflow = client
18105 .poll_workflow_task_response("draining-worker", "rust-workers", Duration::ZERO)
18106 .await
18107 .expect("workflow drain response");
18108 let activity = client
18109 .poll_activity_task_response("draining-worker", "rust-workers", Duration::ZERO)
18110 .await
18111 .expect("activity drain response");
18112 let query = client
18113 .poll_query_task_response("draining-worker", "rust-workers", Duration::ZERO)
18114 .await
18115 .expect("query drain response");
18116
18117 for outcome in [workflow.outcome(), activity.outcome(), query.outcome()] {
18118 assert_eq!(
18119 outcome,
18120 WorkerPollOutcome::Stop {
18121 poll_status: Some("draining".to_string()),
18122 reason: Some("worker_draining".to_string()),
18123 }
18124 );
18125 }
18126
18127 assert!(client
18128 .poll_workflow_task("draining-worker", "rust-workers", Duration::ZERO)
18129 .await
18130 .expect("compatibility poll")
18131 .is_none());
18132 }
18133
18134 #[tokio::test]
18135 async fn managed_worker_honors_drain_stop_for_every_task_family() {
18136 let server = MockWorkerServer::draining_polls();
18137 let client = Client::builder(server.base_url())
18138 .timeout(Duration::from_secs(2))
18139 .build()
18140 .expect("client");
18141
18142 let mut workflow_worker = Worker::new(client.clone(), "rust-workers")
18143 .worker_id("draining-workflow-worker")
18144 .poll_timeout(Duration::ZERO);
18145 workflow_worker.register_workflow("counter", |_ctx, _args| async { Ok(Value::Null) });
18146 workflow_worker
18147 .run()
18148 .await
18149 .expect("workflow drain is a clean stop");
18150
18151 let mut activity_worker = Worker::new(client.clone(), "rust-workers")
18152 .worker_id("draining-activity-worker")
18153 .poll_timeout(Duration::ZERO);
18154 activity_worker.register_activity("write", |_ctx, _args| async { Ok(Value::Null) });
18155 activity_worker
18156 .run()
18157 .await
18158 .expect("activity drain is a clean stop");
18159
18160 let mut query_worker = Worker::new(client, "rust-workers")
18161 .worker_id("draining-query-worker")
18162 .poll_timeout(Duration::ZERO);
18163 query_worker.register_query("counter", "current", |_ctx, _args| async {
18164 Ok(Value::Null)
18165 });
18166 query_worker
18167 .run()
18168 .await
18169 .expect("query drain is a clean stop");
18170 }
18171
18172 #[tokio::test]
18173 async fn activity_cancellation_and_late_completion_remain_machine_readable() {
18174 let server = MockWorkerServer::start();
18175 let client = Client::builder(server.base_url())
18176 .timeout(Duration::from_secs(2))
18177 .build()
18178 .expect("client");
18179
18180 let heartbeat = client
18181 .heartbeat_activity_task(
18182 "activity-cancel",
18183 "attempt-cancel",
18184 "rust-worker",
18185 typed_fidelity_probe(),
18186 )
18187 .await
18188 .expect("cancellation heartbeat");
18189 assert!(heartbeat.cancel_requested);
18190 assert!(heartbeat.should_stop());
18191 assert_eq!(heartbeat.reason.as_deref(), Some("run_cancelled"));
18192 assert_eq!(heartbeat.run_closed_reason.as_deref(), Some("cancelled"));
18193 let heartbeat_body =
18194 server.request_body("/api/worker/activity-tasks/activity-cancel/heartbeat");
18195 assert_eq!(heartbeat_body["details"]["codec"], DEFAULT_CODEC);
18196 assert_eq!(
18197 decode_wire_avro_value(&heartbeat_body["details"], DEFAULT_CODEC)
18198 .expect("typed heartbeat details"),
18199 typed_fidelity_probe()
18200 );
18201
18202 let error = client
18203 .complete_activity_task(
18204 "activity-cancel",
18205 "attempt-cancel",
18206 "rust-worker",
18207 json!({"late":true}),
18208 DEFAULT_CODEC,
18209 )
18210 .await
18211 .expect_err("late completion must be refused");
18212 assert!(activity_task_rejection_is_final(&error));
18213 let Error::ActivityTaskRejected(rejection) = error else {
18214 panic!("expected typed activity rejection");
18215 };
18216 assert_eq!(rejection.status, 409);
18217 assert_eq!(rejection.reason, "run_cancelled");
18218 assert!(rejection.cancel_requested);
18219 assert_eq!(rejection.can_continue, Some(false));
18220 }
18221
18222 #[tokio::test]
18223 async fn managed_worker_survives_late_completion_and_restart_during_cancellation() {
18224 let server = MockWorkerServer::cancelled_activity();
18225 let client = Client::builder(server.base_url())
18226 .timeout(Duration::from_secs(2))
18227 .build()
18228 .expect("client");
18229 let cancellation_observed = Arc::new(AtomicBool::new(false));
18230 let observed = Arc::clone(&cancellation_observed);
18231 let mut worker = Worker::new(client.clone(), "rust-workers")
18232 .worker_id("rust-cancel-worker")
18233 .poll_timeout(Duration::from_millis(10));
18234 worker.register_activity("cancel-aware", move |ctx, _args| {
18235 let observed = Arc::clone(&observed);
18236 async move {
18237 let heartbeat = ctx.heartbeat(json!({"stage":"running"})).await?;
18238 observed.store(heartbeat.should_stop(), Ordering::SeqCst);
18239 Ok(json!({"late":"completion"}))
18240 }
18241 });
18242
18243 assert_eq!(
18244 worker.run_once().await.expect("cancelled attempt handled"),
18245 1
18246 );
18247 assert!(cancellation_observed.load(Ordering::SeqCst));
18248 assert_eq!(
18249 server.request_count("/api/worker/activity-tasks/activity-cancel/complete"),
18250 1
18251 );
18252
18253 let mut restarted = Worker::new(client, "rust-workers")
18254 .worker_id("rust-cancel-worker-restarted")
18255 .poll_timeout(Duration::from_millis(10));
18256 restarted.register_activity("cancel-aware", |_ctx, _args| async move { Ok(Value::Null) });
18257 assert_eq!(
18258 restarted
18259 .run_once()
18260 .await
18261 .expect("replacement worker continues polling"),
18262 0
18263 );
18264 }
18265
18266 #[tokio::test]
18267 async fn managed_worker_absorbs_selected_run_terminal_timeout_completion_race() {
18268 let response = r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"outcome":"completed","recorded":false,"run_id":"run-selected-timeout","run_status":"failed","created_task_ids":[],"reason":"run_timed_out"}"#;
18269 let server = MockWorkerServer::workflow_completion("409 Conflict", response);
18270 let client = Client::builder(server.base_url())
18271 .timeout(Duration::from_secs(2))
18272 .build()
18273 .expect("client");
18274
18275 let direct_error = client
18276 .complete_workflow_task(
18277 "workflow-timeout-task",
18278 "timeout-worker",
18279 3,
18280 vec![json!({
18281 "type": "complete_workflow",
18282 "result": fixture_envelope(Value::Null)
18283 })],
18284 )
18285 .await
18286 .expect_err("the low-level client preserves the completion rejection");
18287 let Error::Http { status, body } = direct_error else {
18288 panic!("expected the original HTTP completion rejection");
18289 };
18290 assert_eq!(status, reqwest::StatusCode::CONFLICT);
18291 assert_eq!(
18292 serde_json::from_str::<Value>(&body).expect("response body")["reason"],
18293 "run_timed_out"
18294 );
18295
18296 let mut worker = Worker::new(client, "rust-workers")
18297 .worker_id("timeout-worker")
18298 .poll_timeout(Duration::from_millis(10));
18299 worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
18300 Ok(json!({"late": "result"}))
18301 });
18302
18303 assert_eq!(
18304 worker
18305 .run_once()
18306 .await
18307 .expect("authoritative selected-run timeout settles the tick"),
18308 1
18309 );
18310 assert_eq!(
18311 server.request_count("/api/worker/workflow-tasks/workflow-timeout-task/complete"),
18312 2,
18313 "both the direct client proof and managed worker must see the rejection"
18314 );
18315 }
18316
18317 #[tokio::test]
18318 async fn managed_worker_does_not_swallow_nearby_completion_errors() {
18319 for (name, status, response) in [
18320 ("bare conflict", "409 Conflict", r#"{"message":"conflict"}"#),
18321 (
18322 "command was recorded",
18323 "409 Conflict",
18324 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":true,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
18325 ),
18326 (
18327 "lease conflict",
18328 "409 Conflict",
18329 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"lease_expired"}"#,
18330 ),
18331 (
18332 "nonterminal run",
18333 "409 Conflict",
18334 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"waiting","reason":"run_timed_out"}"#,
18335 ),
18336 (
18337 "different selected run",
18338 "409 Conflict",
18339 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-reused-workflow-current","run_status":"failed","reason":"run_timed_out"}"#,
18340 ),
18341 (
18342 "different task attempt",
18343 "409 Conflict",
18344 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":4,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
18345 ),
18346 (
18347 "authentication failure",
18348 "401 Unauthorized",
18349 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
18350 ),
18351 (
18352 "authorization failure",
18353 "403 Forbidden",
18354 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
18355 ),
18356 (
18357 "protocol failure",
18358 "400 Bad Request",
18359 r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.2","requested_version":"1.3"}"#,
18360 ),
18361 (
18362 "malformed command",
18363 "422 Unprocessable Entity",
18364 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
18365 ),
18366 (
18367 "transient server failure",
18368 "503 Service Unavailable",
18369 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
18370 ),
18371 ] {
18372 let server = MockWorkerServer::workflow_completion(status, response);
18373 let client = Client::builder(server.base_url())
18374 .timeout(Duration::from_secs(2))
18375 .build()
18376 .expect("client");
18377 let mut worker = Worker::new(client, "rust-workers")
18378 .worker_id("timeout-worker")
18379 .poll_timeout(Duration::from_millis(10));
18380 worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
18381 Ok(json!({"late": "result"}))
18382 });
18383
18384 let error = worker
18385 .run_once()
18386 .await
18387 .expect_err(&format!("{name} must remain an error"));
18388 assert!(
18389 matches!(error, Error::Http { .. } | Error::Protocol(_)),
18390 "{name} returned an unexpected error variant: {error}"
18391 );
18392 }
18393 }
18394
18395 #[tokio::test]
18396 async fn worker_deregistration_uses_worker_plane_method_path_headers_and_result() {
18397 let server = MockWorkerServer::start();
18398 let client = Client::builder(server.base_url())
18399 .worker_token(Some("worker-secret".to_string()))
18400 .namespace("orders")
18401 .timeout(Duration::from_secs(2))
18402 .build()
18403 .expect("client");
18404 let path = "/api/worker/registrations/worker%2F%CE%B1%20space";
18405
18406 let result = client
18407 .deregister_worker_registration("worker/α space")
18408 .await
18409 .expect("deregister worker registration");
18410
18411 assert_eq!(server.method_for(path).as_deref(), Some("DELETE"));
18412 assert_eq!(
18413 server.worker_protocol_for(path).as_deref(),
18414 Some(WORKER_PROTOCOL_VERSION)
18415 );
18416 assert_eq!(server.control_protocol_for(path), None);
18417 assert_eq!(server.namespace_for(path).as_deref(), Some("orders"));
18418 assert_eq!(
18419 server.authorization_for(path).as_deref(),
18420 Some("Bearer worker-secret")
18421 );
18422 assert_eq!(
18423 result,
18424 WorkerDeregistrationEnvelope {
18425 worker_id: "deregistered-worker".to_string(),
18426 outcome: "deregistered".to_string(),
18427 recovered_workflow_task_count: 2,
18428 }
18429 );
18430 }
18431
18432 #[tokio::test]
18433 async fn low_level_registration_rejects_update_validators_before_transport() {
18434 let server = MockWorkerServer::start();
18435 let client = Client::builder(server.base_url())
18436 .timeout(Duration::from_secs(2))
18437 .build()
18438 .expect("client");
18439
18440 for update_validators in [json!(["approve"]), json!("approve")] {
18441 let error = client
18442 .register_worker_with_command_contracts(
18443 "validator-claiming-worker",
18444 "rust-workers",
18445 vec!["orders".to_string()],
18446 vec![],
18447 1,
18448 1,
18449 vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
18450 json!({
18451 "orders": {
18452 "queries": ["current"],
18453 "updates": ["approve"],
18454 "update_validators": update_validators,
18455 },
18456 }),
18457 )
18458 .await
18459 .expect_err("unsupported validator claims must fail before registration");
18460
18461 let Error::UnsupportedUpdateValidators { workflow_type } = error else {
18462 panic!("expected typed unsupported-validator failure");
18463 };
18464 assert_eq!(workflow_type, "orders");
18465 }
18466 assert_eq!(server.request_count("/api/worker/register"), 0);
18467 }
18468
18469 #[tokio::test]
18470 async fn low_level_registration_preserves_query_and_update_contracts() {
18471 let server = MockWorkerServer::start();
18472 let client = Client::builder(server.base_url())
18473 .timeout(Duration::from_secs(2))
18474 .build()
18475 .expect("client");
18476 let contracts = json!({
18477 "orders": {
18478 "queries": ["current"],
18479 "updates": ["approve"],
18480 "update_validators": [],
18481 },
18482 "payments": {
18483 "queries": ["status"],
18484 "updates": ["capture"],
18485 },
18486 });
18487
18488 client
18489 .register_worker_with_command_contracts(
18490 "command-worker",
18491 "rust-workers",
18492 vec!["orders".to_string(), "payments".to_string()],
18493 vec![],
18494 1,
18495 1,
18496 vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
18497 contracts.clone(),
18498 )
18499 .await
18500 .expect("query and update contracts must remain supported");
18501
18502 assert_eq!(
18503 server.request_body("/api/worker/register")["workflow_command_contracts"],
18504 contracts
18505 );
18506 }
18507
18508 #[tokio::test]
18509 async fn role_scoped_tokens_are_never_used_for_the_opposite_plane() {
18510 let server = MockWorkerServer::start();
18511 let control_only = Client::builder(server.base_url())
18512 .control_token(Some("control-secret".to_string()))
18513 .build()
18514 .expect("control client");
18515
18516 let error = control_only
18517 .register_worker("worker", "queue", vec![], vec![], 1, 1)
18518 .await
18519 .expect_err("control token must not authorize a worker request");
18520 assert!(matches!(
18521 error,
18522 Error::MissingRoleCredentials { role: "worker", .. }
18523 ));
18524 assert_eq!(server.request_count("/api/worker/register"), 0);
18525
18526 let worker_only = Client::builder(server.base_url())
18527 .worker_token(Some("worker-secret".to_string()))
18528 .build()
18529 .expect("worker client");
18530 let error = worker_only
18531 .health()
18532 .await
18533 .expect_err("worker token must not authorize a control request");
18534 assert!(matches!(
18535 error,
18536 Error::MissingRoleCredentials {
18537 role: "control",
18538 ..
18539 }
18540 ));
18541 assert_eq!(server.request_count("/api/health"), 0);
18542 }
18543
18544 #[tokio::test]
18545 async fn shared_token_supports_worker_and_control_planes() {
18546 let server = MockWorkerServer::start();
18547 let client = Client::builder(server.base_url())
18548 .token(Some("shared-secret".to_string()))
18549 .build()
18550 .expect("client");
18551
18552 client.health().await.expect("control request");
18553 client
18554 .register_worker("worker", "queue", vec![], vec![], 1, 1)
18555 .await
18556 .expect("worker request");
18557
18558 assert_eq!(
18559 server.authorization_for("/api/health").as_deref(),
18560 Some("Bearer shared-secret")
18561 );
18562 assert_eq!(
18563 server.control_protocol_for("/api/health").as_deref(),
18564 Some(CONTROL_PLANE_VERSION)
18565 );
18566 assert_eq!(
18567 server.authorization_for("/api/worker/register").as_deref(),
18568 Some("Bearer shared-secret")
18569 );
18570 assert_eq!(
18571 server
18572 .worker_protocol_for("/api/worker/register")
18573 .as_deref(),
18574 Some(WORKER_PROTOCOL_VERSION)
18575 );
18576 }
18577
18578 #[tokio::test]
18579 async fn baseline_worker_endpoints_send_the_baseline_protocol() {
18580 let server = MockWorkerServer::start();
18581 let client = Client::builder(server.base_url())
18582 .timeout(Duration::from_secs(2))
18583 .build()
18584 .expect("client");
18585
18586 client
18587 .register_worker("capture-worker", "capture", vec![], vec![], 1, 1)
18588 .await
18589 .expect("register");
18590 client
18591 .heartbeat_worker("capture-worker", 1, 1)
18592 .await
18593 .expect("heartbeat");
18594 client
18595 .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
18596 .await
18597 .expect("workflow poll");
18598 client
18599 .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
18600 .await
18601 .expect("activity poll");
18602
18603 for path in [
18604 "/api/worker/register",
18605 "/api/worker/heartbeat",
18606 "/api/worker/workflow-tasks/poll",
18607 "/api/worker/activity-tasks/poll",
18608 ] {
18609 assert_eq!(
18610 server.worker_protocol_for(path).as_deref(),
18611 Some(WORKER_PROTOCOL_VERSION),
18612 "unexpected protocol for {path}"
18613 );
18614 }
18615
18616 assert_eq!(
18617 server.request_body("/api/worker/workflow-tasks/poll")["timeout_seconds"],
18618 1
18619 );
18620 assert_eq!(
18621 server.request_body("/api/worker/activity-tasks/poll")["timeout_seconds"],
18622 1
18623 );
18624 assert!(
18625 server.request_body("/api/worker/workflow-tasks/poll")["poll_request_id"]
18626 .as_str()
18627 .is_some_and(|id| id.starts_with("rust-workflow-poll-"))
18628 );
18629 assert!(
18630 server.request_body("/api/worker/activity-tasks/poll")["poll_request_id"]
18631 .as_str()
18632 .is_some_and(|id| id.starts_with("rust-activity-poll-"))
18633 );
18634 }
18635
18636 #[tokio::test]
18637 async fn query_task_endpoints_send_the_query_feature_protocol() {
18638 let server = MockWorkerServer::start();
18639 let client = Client::builder(server.base_url())
18640 .timeout(Duration::from_secs(2))
18641 .build()
18642 .expect("client");
18643
18644 client
18645 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
18646 .await
18647 .expect("query poll");
18648 client
18649 .complete_query_task(
18650 "query-capture",
18651 "capture-worker",
18652 1,
18653 json!(8),
18654 DEFAULT_CODEC,
18655 )
18656 .await
18657 .expect("query complete");
18658 client
18659 .fail_query_task(
18660 "query-capture",
18661 "capture-worker",
18662 1,
18663 "failed",
18664 "query_rejected",
18665 "QueryFailed",
18666 )
18667 .await
18668 .expect("query fail");
18669
18670 for path in [
18671 "/api/worker/query-tasks/poll",
18672 "/api/worker/query-tasks/query-capture/complete",
18673 "/api/worker/query-tasks/query-capture/fail",
18674 ] {
18675 assert_eq!(
18676 server.worker_protocol_for(path).as_deref(),
18677 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
18678 "unexpected protocol for {path}"
18679 );
18680 }
18681
18682 assert_eq!(
18683 server.request_body("/api/worker/query-tasks/poll")["timeout_seconds"],
18684 1
18685 );
18686 assert!(
18687 server.request_body("/api/worker/query-tasks/poll")["poll_request_id"]
18688 .as_str()
18689 .is_some_and(|id| id.starts_with("rust-query-poll-"))
18690 );
18691 }
18692
18693 #[tokio::test]
18694 async fn disconnected_client_polls_retry_once_with_the_same_request_id() {
18695 let server = MockWorkerServer::transient_worker_failures();
18696 let client = Client::builder(server.base_url())
18697 .timeout(Duration::from_secs(2))
18698 .build()
18699 .expect("client");
18700
18701 client
18702 .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
18703 .await
18704 .expect("workflow poll retry");
18705 client
18706 .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
18707 .await
18708 .expect("activity poll retry");
18709 client
18710 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
18711 .await
18712 .expect("query poll retry");
18713
18714 for path in [
18715 "/api/worker/workflow-tasks/poll",
18716 "/api/worker/activity-tasks/poll",
18717 "/api/worker/query-tasks/poll",
18718 ] {
18719 let bodies = server.request_bodies(path);
18720 assert_eq!(bodies.len(), 2, "{path} must be retried once");
18721 assert_eq!(
18722 bodies[0]["poll_request_id"], bodies[1]["poll_request_id"],
18723 "{path} must preserve the request binding across retry"
18724 );
18725 }
18726 }
18727
18728 #[tokio::test]
18729 async fn worker_poll_retries_preserve_request_id_across_consecutive_failures() {
18730 let server = MockWorkerServer::consecutive_poll_failures(2);
18731 let client = Client::builder(server.base_url())
18732 .timeout(Duration::from_secs(2))
18733 .build()
18734 .expect("client");
18735 let mut worker = Worker::new(client, "capture")
18736 .worker_id("capture-worker")
18737 .poll_timeout(Duration::from_millis(10))
18738 .retry_policy(WorkerRetryPolicy {
18739 max_retries: 2,
18740 initial_backoff: Duration::from_millis(1),
18741 max_backoff: Duration::from_millis(1),
18742 });
18743 worker.register_workflow(
18744 "capture.workflow",
18745 |_ctx, _input| async move { Ok(Value::Null) },
18746 );
18747 worker.register_activity(
18748 "capture.activity",
18749 |_ctx, _input| async move { Ok(Value::Null) },
18750 );
18751 worker.register_query("capture.workflow", "current", |_ctx, _args| async move {
18752 Ok(Value::Null)
18753 });
18754
18755 assert_eq!(worker.run_once().await.expect("poll retries"), 0);
18756
18757 for path in [
18758 "/api/worker/workflow-tasks/poll",
18759 "/api/worker/activity-tasks/poll",
18760 "/api/worker/query-tasks/poll",
18761 ] {
18762 let bodies = server.request_bodies(path);
18763 assert_eq!(bodies.len(), 3, "{path} must use exactly two retries");
18764 assert!(
18765 bodies
18766 .iter()
18767 .all(|body| body["poll_request_id"] == bodies[0]["poll_request_id"]),
18768 "{path} must preserve one request binding across every retry"
18769 );
18770 }
18771 }
18772
18773 #[tokio::test]
18774 async fn query_protocol_rejection_from_older_server_is_typed() {
18775 let server = MockWorkerServer::reject_query_protocol();
18776 let client = Client::builder(server.base_url())
18777 .timeout(Duration::from_secs(2))
18778 .build()
18779 .expect("client");
18780
18781 let error = client
18782 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
18783 .await
18784 .expect_err("server below query protocol floor must reject");
18785 let Error::Protocol(failure) = error else {
18786 panic!("expected typed protocol failure");
18787 };
18788
18789 assert_eq!(failure.status, 400);
18790 assert_eq!(failure.reason, "unsupported_protocol_version");
18791 assert_eq!(failure.supported_version.as_deref(), Some("1.7"));
18792 assert_eq!(
18793 failure.requested_version.as_deref(),
18794 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
18795 );
18796 assert_eq!(
18797 server
18798 .worker_protocol_for("/api/worker/query-tasks/poll")
18799 .as_deref(),
18800 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
18801 );
18802 }
18803
18804 #[tokio::test]
18805 async fn run_once_without_query_handlers_keeps_pre_query_server_compatibility() {
18806 let server = MockWorkerServer::reject_query_protocol();
18807 let client = Client::builder(server.base_url())
18808 .timeout(Duration::from_secs(2))
18809 .build()
18810 .expect("client");
18811 let mut worker = Worker::new(client, "rust-workers")
18812 .worker_id("baseline-worker")
18813 .poll_timeout(Duration::from_millis(10));
18814
18815 worker.register_workflow("baseline.workflow", |_ctx, _input| async move {
18816 Ok(Value::Null)
18817 });
18818
18819 assert_eq!(worker.run_once().await.expect("baseline run once"), 0);
18820 assert_eq!(
18821 server
18822 .worker_protocol_for("/api/worker/workflow-tasks/poll")
18823 .as_deref(),
18824 Some(WORKER_PROTOCOL_VERSION)
18825 );
18826 assert_eq!(
18827 server.worker_protocol_for("/api/worker/query-tasks/poll"),
18828 None,
18829 "a worker without query handlers must not use the query-task endpoint"
18830 );
18831 }
18832
18833 #[tokio::test]
18834 async fn completion_time_query_rejection_is_typed_without_stopping_worker() {
18835 let server = MockWorkerServer::reject_query_completion();
18836 let client = Client::builder(server.base_url())
18837 .timeout(Duration::from_secs(2))
18838 .build()
18839 .expect("client");
18840
18841 let error = client
18842 .complete_query_task("query-late", "late-worker", 1, json!(8), DEFAULT_CODEC)
18843 .await
18844 .expect_err("expired completion must be rejected");
18845 let Error::QueryFailed(failure) = error else {
18846 panic!("expected typed query failure");
18847 };
18848 assert_eq!(failure.status, 409);
18849 assert_eq!(failure.reason, "query_task_timed_out");
18850
18851 let mut worker = Worker::new(client, "rust-workers")
18852 .worker_id("late-worker")
18853 .poll_timeout(Duration::from_millis(10));
18854 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
18855 worker.register_query(
18856 "counter",
18857 "current",
18858 |_ctx, _args| async move { Ok(json!(8)) },
18859 );
18860
18861 assert_eq!(worker.run_once().await.expect("late task is handled"), 1);
18862 assert_eq!(
18863 worker
18864 .run_once()
18865 .await
18866 .expect("worker continues after late completion"),
18867 0
18868 );
18869 assert_eq!(
18870 server.request_count("/api/worker/query-tasks/query-late/complete"),
18871 2
18872 );
18873 assert_eq!(
18874 server.request_count("/api/worker/query-tasks/query-late/fail"),
18875 0,
18876 "a server completion rejection must not be reported as an encoding failure"
18877 );
18878 }
18879
18880 #[tokio::test]
18881 async fn normal_shutdown_joins_pollers_and_deregisters_once() {
18882 let server = MockWorkerServer::start();
18883 let client = Client::builder(server.base_url())
18884 .timeout(Duration::from_secs(2))
18885 .build()
18886 .expect("client");
18887 let mut worker = Worker::new(client, "rust-workers")
18888 .worker_id("joined-worker")
18889 .poll_timeout(Duration::from_millis(10));
18890 worker.register_workflow(
18891 "joined.workflow",
18892 |_ctx, _input| async move { Ok(Value::Null) },
18893 );
18894 worker.register_activity(
18895 "joined.activity",
18896 |_ctx, _input| async move { Ok(Value::Null) },
18897 );
18898 worker.register_query("joined.workflow", "state", |_ctx, _input| async move {
18899 Ok(Value::Null)
18900 });
18901
18902 worker
18903 .run_until(tokio::time::sleep(Duration::from_millis(20)))
18904 .await
18905 .expect("normal shutdown");
18906
18907 let deregistration_path = "/api/worker/registrations/mock-worker";
18908 assert_eq!(server.request_count(deregistration_path), 1);
18909 for poll_path in [
18910 "/api/worker/workflow-tasks/poll",
18911 "/api/worker/activity-tasks/poll",
18912 "/api/worker/query-tasks/poll",
18913 ] {
18914 assert!(server.request_count(poll_path) > 0, "missing {poll_path}");
18915 }
18916 assert_eq!(
18917 server.captured_paths().last().map(String::as_str),
18918 Some(deregistration_path),
18919 "deregistration must start only after every poller has joined"
18920 );
18921 }
18922
18923 #[tokio::test]
18924 async fn registration_failure_does_not_deregister() {
18925 let server = MockWorkerServer::rejected_registration();
18926 let client = Client::builder(server.base_url())
18927 .timeout(Duration::from_secs(2))
18928 .build()
18929 .expect("client");
18930 let worker = Worker::new(client, "rust-workers").worker_id("never-registered");
18931
18932 let error = worker
18933 .run_until(async {})
18934 .await
18935 .expect_err("registration must fail");
18936 assert!(matches!(
18937 error,
18938 Error::Http {
18939 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
18940 ..
18941 }
18942 ));
18943 assert!(server
18944 .captured_paths()
18945 .iter()
18946 .all(|path| !path.starts_with("/api/worker/registrations/")));
18947 }
18948
18949 #[tokio::test]
18950 async fn declined_registration_does_not_deregister() {
18951 let server = MockWorkerServer::declined_registration();
18952 let client = Client::builder(server.base_url())
18953 .timeout(Duration::from_secs(2))
18954 .build()
18955 .expect("client");
18956 let worker = Worker::new(client, "rust-workers").worker_id("declined-worker");
18957
18958 let error = worker
18959 .run_until(async {})
18960 .await
18961 .expect_err("declined registration must fail");
18962 assert!(matches!(error, Error::WorkerLoop(_)));
18963 assert!(error.to_string().contains("was not accepted"));
18964 assert!(server
18965 .captured_paths()
18966 .iter()
18967 .all(|path| !path.starts_with("/api/worker/registrations/")));
18968 }
18969
18970 #[tokio::test]
18971 async fn deregistration_http_failure_is_returned_after_normal_shutdown() {
18972 let server = MockWorkerServer::rejected_deregistration();
18973 let client = Client::builder(server.base_url())
18974 .timeout(Duration::from_secs(2))
18975 .build()
18976 .expect("client");
18977 let worker = Worker::new(client, "rust-workers").worker_id("forbidden-cleanup");
18978
18979 let error = worker
18980 .run_until(async {})
18981 .await
18982 .expect_err("deregistration must fail");
18983 assert!(matches!(
18984 error,
18985 Error::Http {
18986 status: reqwest::StatusCode::FORBIDDEN,
18987 ..
18988 }
18989 ));
18990 assert_eq!(
18991 server.request_count("/api/worker/registrations/mock-worker"),
18992 1
18993 );
18994 }
18995
18996 #[tokio::test]
18997 async fn deregistration_protocol_failure_is_returned_after_normal_shutdown() {
18998 let server = MockWorkerServer::rejected_deregistration_protocol();
18999 let client = Client::builder(server.base_url())
19000 .timeout(Duration::from_secs(2))
19001 .build()
19002 .expect("client");
19003 let worker = Worker::new(client, "rust-workers").worker_id("protocol-cleanup");
19004
19005 let error = worker
19006 .run_until(async {})
19007 .await
19008 .expect_err("protocol rejection must fail shutdown");
19009 let Error::Protocol(failure) = error else {
19010 panic!("expected typed protocol failure");
19011 };
19012 assert_eq!(failure.reason, "unsupported_protocol_version");
19013 assert_eq!(
19014 failure.requested_version.as_deref(),
19015 Some(WORKER_PROTOCOL_VERSION)
19016 );
19017 assert_eq!(
19018 server.request_count("/api/worker/registrations/mock-worker"),
19019 1
19020 );
19021 }
19022
19023 #[tokio::test]
19024 async fn primary_poller_error_retains_deregistration_failure_context() {
19025 let server = MockWorkerServer::unauthorized_polls_and_rejected_deregistration();
19026 let client = Client::builder(server.base_url())
19027 .timeout(Duration::from_secs(2))
19028 .build()
19029 .expect("client");
19030 let mut worker = Worker::new(client, "rust-workers")
19031 .worker_id("combined-failure")
19032 .poll_timeout(Duration::from_millis(10));
19033 worker.register_workflow("combined.workflow", |_ctx, _input| async move {
19034 Ok(Value::Null)
19035 });
19036
19037 let error = worker
19038 .run()
19039 .await
19040 .expect_err("worker and cleanup must fail");
19041 let summary = error.to_string();
19042 assert!(summary.contains("authentication_failed"));
19043 assert!(summary.contains("worker cannot deregister"));
19044 let Error::WorkerShutdown {
19045 primary,
19046 deregistration,
19047 } = error
19048 else {
19049 panic!("expected combined worker shutdown error");
19050 };
19051 assert!(matches!(
19052 *primary,
19053 Error::Http {
19054 status: reqwest::StatusCode::UNAUTHORIZED,
19055 ..
19056 }
19057 ));
19058 assert!(matches!(
19059 *deregistration,
19060 Error::Http {
19061 status: reqwest::StatusCode::FORBIDDEN,
19062 ..
19063 }
19064 ));
19065 assert_eq!(
19066 server.request_count("/api/worker/registrations/mock-worker"),
19067 1
19068 );
19069 }
19070
19071 #[tokio::test]
19072 async fn activity_only_worker_can_shutdown_without_workflow_poller() {
19073 let server = MockWorkerServer::start();
19074 let client = Client::builder(server.base_url())
19075 .timeout(Duration::from_secs(2))
19076 .build()
19077 .expect("client");
19078 let mut worker = Worker::new(client, "rust-workers")
19079 .worker_id("activity-only-worker")
19080 .poll_timeout(Duration::from_millis(10));
19081
19082 worker.register_activity(
19083 "activity.only",
19084 |_ctx, _args| async move { Ok(Value::Null) },
19085 );
19086
19087 worker.run_until(async {}).await.expect("run worker");
19088 }
19089
19090 #[tokio::test]
19091 async fn workflow_only_worker_can_shutdown_without_activity_poller() {
19092 let server = MockWorkerServer::start();
19093 let client = Client::builder(server.base_url())
19094 .timeout(Duration::from_secs(2))
19095 .build()
19096 .expect("client");
19097 let mut worker = Worker::new(client, "rust-workers")
19098 .worker_id("workflow-only-worker")
19099 .poll_timeout(Duration::from_millis(10));
19100
19101 worker.register_workflow(
19102 "workflow.only",
19103 |_ctx, _input| async move { Ok(Value::Null) },
19104 );
19105
19106 worker.run_until(async {}).await.expect("run worker");
19107 }
19108
19109 #[tokio::test]
19110 async fn worker_heartbeat_observer_receives_server_acknowledgements() {
19111 let server = MockWorkerServer::start();
19112 let client = Client::builder(server.base_url())
19113 .timeout(Duration::from_secs(2))
19114 .build()
19115 .expect("client");
19116 let observations = Arc::new(Mutex::new(Vec::new()));
19117 let observed = Arc::clone(&observations);
19118 let mut worker = Worker::new(client, "rust-workers")
19119 .worker_id("observed-heartbeat-worker")
19120 .poll_timeout(Duration::from_millis(10))
19121 .on_worker_heartbeat(move |observation| {
19122 observed
19123 .lock()
19124 .expect("heartbeat observations")
19125 .push(observation.clone());
19126 });
19127
19128 worker.register_workflow("workflow.observed", |_ctx, _input| async move {
19129 Ok(Value::Null)
19130 });
19131 let acknowledged = Arc::clone(&observations);
19132 worker
19133 .run_until(async move {
19134 tokio::time::timeout(Duration::from_secs(2), async move {
19135 loop {
19136 if !acknowledged
19137 .lock()
19138 .expect("heartbeat observations")
19139 .is_empty()
19140 {
19141 break;
19142 }
19143 tokio::time::sleep(Duration::from_millis(1)).await;
19144 }
19145 })
19146 .await
19147 .expect("heartbeat acknowledgement within timeout");
19148 })
19149 .await
19150 .expect("run worker");
19151
19152 let observations = observations.lock().expect("heartbeat observations");
19153 let first = observations.first().expect("heartbeat acknowledgement");
19154 assert_eq!(first.worker_id, "observed-heartbeat-worker");
19155 assert_eq!(first.task_queue, "rust-workers");
19156 assert!(first.acknowledged_at_unix_millis > 0);
19157 assert_eq!(first.acknowledgement, json!({}));
19158 }
19159
19160 #[tokio::test]
19161 async fn delayed_worker_heartbeat_keeps_cadence_and_pollers_live() {
19162 let server = MockWorkerServer::delayed_heartbeat_worker();
19163 let client = Client::builder(server.base_url())
19164 .timeout(Duration::from_secs(3))
19165 .build()
19166 .expect("client");
19167 let observations = Arc::new(Mutex::new(Vec::new()));
19168 let observed = Arc::clone(&observations);
19169 let mut worker = Worker::new(client, "rust-snapshot-workers")
19170 .worker_id("rust-snapshot-worker")
19171 .poll_timeout(Duration::from_millis(10))
19172 .on_worker_heartbeat(move |observation| {
19173 observed
19174 .lock()
19175 .expect("heartbeat observations")
19176 .push(observation.clone());
19177 });
19178
19179 worker.register_workflow("snapshot", |ctx, _input| async move {
19180 ctx.wait_signal("finish").await?;
19181 Ok(json!({"status": "finished"}))
19182 });
19183 worker.register_query("snapshot", "current", |ctx, _args| async move {
19184 Ok(json!(ctx
19185 .signals("increment")
19186 .iter()
19187 .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
19188 .sum::<i64>()))
19189 });
19190 worker.register_activity("cancel-aware", |_ctx, _args| async move {
19191 Ok(json!({"late": "completion"}))
19192 });
19193
19194 worker
19195 .run_until(tokio::time::sleep(Duration::from_millis(3_800)))
19196 .await
19197 .expect("delayed heartbeat must allow a clean worker shutdown");
19198
19199 let observations = observations.lock().expect("heartbeat observations");
19200 assert!(
19201 observations.len() >= 3,
19202 "the immediate heartbeat, delayed acknowledgement, and next cadence heartbeat must complete"
19203 );
19204 assert!(
19205 observations.windows(2).all(|pair| {
19206 pair[1].acknowledged_at_unix_millis
19207 .saturating_sub(pair[0].acknowledged_at_unix_millis)
19208 >= 850
19209 }),
19210 "successful acknowledgements must not catch up faster than the advertised one-second cadence: {observations:?}"
19211 );
19212 drop(observations);
19213
19214 let heartbeat_times = server.request_times("/api/worker/heartbeat");
19215 let delayed_request_at = *heartbeat_times
19216 .get(1)
19217 .expect("intentionally delayed heartbeat request");
19218 let delay_window_start = delayed_request_at + Duration::from_millis(100);
19219 let delay_window_end = delayed_request_at + Duration::from_millis(1_400);
19220 for path in [
19221 "/api/worker/workflow-tasks/poll",
19222 "/api/worker/activity-tasks/poll",
19223 "/api/worker/query-tasks/poll",
19224 ] {
19225 assert!(
19226 server
19227 .request_times(path)
19228 .iter()
19229 .any(|received_at| *received_at >= delay_window_start
19230 && *received_at <= delay_window_end),
19231 "{path} must keep polling while a heartbeat acknowledgement is delayed"
19232 );
19233 }
19234 assert!(
19235 server.request_count("/api/worker/workflow-tasks/snapshot-wait-3/fail") >= 1,
19236 "workflow work must be settled"
19237 );
19238 assert!(
19239 server.request_count("/api/worker/activity-tasks/activity-cancel/complete") >= 1,
19240 "activity work must be settled"
19241 );
19242 assert!(
19243 server.request_count("/api/worker/query-tasks/snapshot-current/complete") >= 1,
19244 "query work must be settled"
19245 );
19246 }
19247
19248 #[tokio::test]
19249 async fn retried_worker_heartbeat_restarts_the_advertised_cadence() {
19250 let server = MockWorkerServer::heartbeat_retry_worker();
19251 let client = Client::builder(server.base_url())
19252 .timeout(Duration::from_secs(2))
19253 .build()
19254 .expect("client");
19255 let observations = Arc::new(Mutex::new(Vec::new()));
19256 let observed = Arc::clone(&observations);
19257 let worker = Worker::new(client, "rust-workers")
19258 .worker_id("heartbeat-retry-worker")
19259 .retry_policy(WorkerRetryPolicy {
19260 max_retries: 1,
19261 initial_backoff: Duration::from_millis(300),
19262 max_backoff: Duration::from_millis(300),
19263 })
19264 .on_worker_heartbeat(move |observation| {
19265 observed
19266 .lock()
19267 .expect("heartbeat observations")
19268 .push(observation.clone());
19269 });
19270
19271 worker
19272 .run_until(tokio::time::sleep(Duration::from_millis(2_700)))
19273 .await
19274 .expect("retryable heartbeat failure must remain bounded and recover");
19275
19276 let observations = observations.lock().expect("heartbeat observations");
19277 assert!(observations.len() >= 3, "heartbeat retry must recover");
19278 assert!(
19279 observations.windows(2).all(|pair| {
19280 pair[1]
19281 .acknowledged_at_unix_millis
19282 .saturating_sub(pair[0].acknowledged_at_unix_millis)
19283 >= 850
19284 }),
19285 "a successful retry must start a fresh advertised cadence: {observations:?}"
19286 );
19287 assert_eq!(
19288 server.request_count("/api/worker/heartbeat"),
19289 observations.len() + 1,
19290 "one retryable failure must add exactly one bounded request"
19291 );
19292 }
19293
19294 #[tokio::test]
19295 async fn query_enabled_worker_ignores_unmatched_signals_then_completes_once() {
19296 let server = MockWorkerServer::waiting_query_worker();
19297 let client = Client::builder(server.base_url())
19298 .timeout(Duration::from_secs(2))
19299 .build()
19300 .expect("client");
19301 let observations = Arc::new(Mutex::new(Vec::new()));
19302 let observed = Arc::clone(&observations);
19303 let mut worker = Worker::new(client, "rust-snapshot-workers")
19304 .worker_id("rust-snapshot-worker")
19305 .poll_timeout(Duration::from_millis(10))
19306 .on_worker_heartbeat(move |observation| {
19307 observed
19308 .lock()
19309 .expect("heartbeat observations")
19310 .push(observation.clone());
19311 });
19312
19313 worker.register_workflow("snapshot", |ctx, _input| async move {
19314 ctx.wait_signal("finish").await?;
19315 Ok(json!({"status": "finished"}))
19316 });
19317 worker.register_query("snapshot", "current", |ctx, _args| async move {
19318 let current = ctx
19319 .signals("increment")
19320 .iter()
19321 .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
19322 .sum::<i64>();
19323 Ok(json!(current))
19324 });
19325 worker.register_update("snapshot", "replace", |_ctx, args| async move { Ok(args) });
19326
19327 worker
19328 .run_until(tokio::time::sleep(Duration::from_millis(3_200)))
19329 .await
19330 .expect("pending workflow and query poller must remain live until shutdown");
19331
19332 assert!(
19333 observations.lock().expect("heartbeat observations").len() >= 4,
19334 "the immediate heartbeat and at least three advertised one-second intervals must be acknowledged"
19335 );
19336 assert!(
19337 server.request_count("/api/worker/workflow-tasks/poll") >= 3,
19338 "workflow polling must continue after empty replay acknowledgements"
19339 );
19340 assert!(
19341 server.request_count("/api/worker/query-tasks/poll") >= 2,
19342 "query polling must continue after serving the current query"
19343 );
19344 assert_eq!(
19345 server.request_body("/api/worker/register")["capabilities"],
19346 json!([QUERY_TASKS_CAPABILITY, WORKFLOW_UPDATES_CAPABILITY])
19347 );
19348 assert_eq!(
19349 server.request_body("/api/worker/register")["workflow_command_contracts"]["snapshot"],
19350 json!({
19351 "queries": ["current"],
19352 "updates": ["replace"],
19353 "update_validators": [],
19354 })
19355 );
19356
19357 let opened = server.request_body("/api/worker/workflow-tasks/snapshot-open/complete");
19358 assert_eq!(
19359 opened["commands"],
19360 json!([{
19361 "type": "open_signal_wait",
19362 "signal_name": "finish",
19363 }])
19364 );
19365
19366 for task_id in ["snapshot-wait-3", "snapshot-wait-5"] {
19367 let fail_path = format!("/api/worker/workflow-tasks/{task_id}/fail");
19368 let completion_path = format!("/api/worker/workflow-tasks/{task_id}/complete");
19369 let failure = server.request_body(&fail_path);
19370 assert_eq!(
19371 failure["failure"]["type"],
19372 WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE
19373 );
19374 assert_eq!(server.request_count(&completion_path), 0);
19375 }
19376
19377 let query_completion =
19378 server.request_body("/api/worker/query-tasks/snapshot-current/complete");
19379 assert_eq!(query_completion["result"], json!(8));
19380
19381 let terminal_path = "/api/worker/workflow-tasks/snapshot-finish/complete";
19382 assert_eq!(
19383 server.request_count(terminal_path),
19384 1,
19385 "the matching signal must settle the workflow exactly once"
19386 );
19387 let terminal = server.request_body(terminal_path);
19388 assert_eq!(terminal["commands"].as_array().map(Vec::len), Some(1));
19389 assert_eq!(terminal["commands"][0]["type"], "complete_workflow");
19390 assert_eq!(
19391 decode_wire_value(&terminal["commands"][0]["result"], DEFAULT_CODEC)
19392 .expect("terminal workflow result"),
19393 json!({"status": "finished"})
19394 );
19395 }
19396
19397 #[tokio::test]
19398 async fn worker_retries_poll_and_heartbeat_transport_failures_independently() {
19399 let server = MockWorkerServer::transient_worker_failures();
19400 let client = Client::builder(server.base_url())
19401 .timeout(Duration::from_secs(2))
19402 .build()
19403 .expect("client");
19404 let mut worker = Worker::new(client, "rust-workers")
19405 .worker_id("retry-worker")
19406 .poll_timeout(Duration::from_millis(10))
19407 .retry_policy(WorkerRetryPolicy {
19408 max_retries: 2,
19409 initial_backoff: Duration::from_millis(1),
19410 max_backoff: Duration::from_millis(1),
19411 });
19412 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
19413 worker.register_activity(
19414 "counter.activity",
19415 |_ctx, _input| async move { Ok(Value::Null) },
19416 );
19417 worker.register_query(
19418 "counter",
19419 "current",
19420 |_ctx, _args| async move { Ok(json!(8)) },
19421 );
19422
19423 worker
19424 .run_until(tokio::time::sleep(Duration::from_millis(75)))
19425 .await
19426 .expect("transient failures must not stop the worker");
19427
19428 for path in [
19429 "/api/worker/heartbeat",
19430 "/api/worker/workflow-tasks/poll",
19431 "/api/worker/activity-tasks/poll",
19432 "/api/worker/query-tasks/poll",
19433 ] {
19434 assert!(
19435 server.request_count(path) >= 2,
19436 "{path} must continue after its transient failure"
19437 );
19438 }
19439 }
19440
19441 #[tokio::test]
19442 async fn worker_bounds_transport_retries() {
19443 let server = MockWorkerServer::unavailable_polls();
19444 let client = Client::builder(server.base_url())
19445 .timeout(Duration::from_secs(2))
19446 .build()
19447 .expect("client");
19448 let mut worker = Worker::new(client, "rust-workers")
19449 .worker_id("bounded-retry-worker")
19450 .poll_timeout(Duration::from_millis(10))
19451 .retry_policy(WorkerRetryPolicy {
19452 max_retries: 2,
19453 initial_backoff: Duration::from_millis(1),
19454 max_backoff: Duration::from_millis(1),
19455 });
19456 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
19457
19458 let error = worker.run().await.expect_err("retry bound must terminate");
19459 assert!(matches!(error, Error::Transport(_)));
19460 assert_eq!(
19461 server.request_count("/api/worker/workflow-tasks/poll"),
19462 3,
19463 "one initial request plus exactly two retries"
19464 );
19465 }
19466
19467 #[tokio::test]
19468 async fn worker_retry_policy_can_disable_poll_retries() {
19469 let server = MockWorkerServer::unavailable_polls();
19470 let client = Client::builder(server.base_url())
19471 .timeout(Duration::from_secs(2))
19472 .build()
19473 .expect("client");
19474 let mut worker = Worker::new(client, "rust-workers")
19475 .worker_id("no-retry-worker")
19476 .poll_timeout(Duration::from_millis(10))
19477 .retry_policy(WorkerRetryPolicy {
19478 max_retries: 0,
19479 initial_backoff: Duration::from_millis(1),
19480 max_backoff: Duration::from_millis(1),
19481 });
19482 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
19483
19484 let error = worker
19485 .run_once()
19486 .await
19487 .expect_err("disabled retries must return the first transport failure");
19488 assert!(matches!(error, Error::Transport(_)));
19489 assert_eq!(
19490 server.request_count("/api/worker/workflow-tasks/poll"),
19491 1,
19492 "max_retries=0 must send only the initial request"
19493 );
19494 }
19495
19496 #[tokio::test]
19497 async fn worker_does_not_retry_authentication_failures() {
19498 let server = MockWorkerServer::unauthorized_polls();
19499 let client = Client::builder(server.base_url())
19500 .timeout(Duration::from_secs(2))
19501 .build()
19502 .expect("client");
19503 let mut worker = Worker::new(client, "rust-workers")
19504 .worker_id("unauthorized-worker")
19505 .poll_timeout(Duration::from_millis(10));
19506 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
19507
19508 let error = worker
19509 .run()
19510 .await
19511 .expect_err("authentication must terminate");
19512 let Error::Http { status, body } = error else {
19513 panic!("expected stable HTTP authentication error");
19514 };
19515 assert_eq!(status, reqwest::StatusCode::UNAUTHORIZED);
19516 assert!(body.contains("authentication_failed"));
19517 assert_eq!(
19518 server.request_count("/api/worker/workflow-tasks/poll"),
19519 1,
19520 "authentication failures must not be retried"
19521 );
19522 }
19523
19524 #[derive(Clone, Debug)]
19525 struct CapturedRequest {
19526 method: String,
19527 path: String,
19528 authorization: Option<String>,
19529 namespace: Option<String>,
19530 worker_protocol: Option<String>,
19531 control_protocol: Option<String>,
19532 body: String,
19533 received_at: Instant,
19534 }
19535
19536 struct MockWorkerServer {
19537 addr: SocketAddr,
19538 stop: Arc<AtomicBool>,
19539 requests: Arc<Mutex<Vec<CapturedRequest>>>,
19540 thread: Option<thread::JoinHandle<()>>,
19541 }
19542
19543 #[derive(Clone, Copy, Default)]
19544 struct MockWorkerBehavior {
19545 reject_query_protocol: bool,
19546 reject_query_completion: bool,
19547 waiting_query_worker: bool,
19548 decline_registration: bool,
19549 complete_named_signal: bool,
19550 poll_failures_per_path: usize,
19551 heartbeat_failures: usize,
19552 heartbeat_failure_request: Option<usize>,
19553 delayed_heartbeat_request: Option<usize>,
19554 heartbeat_response_delay: Duration,
19555 concurrent_requests: bool,
19556 unauthorized_polls: bool,
19557 reject_registration: bool,
19558 reject_deregistration: bool,
19559 reject_deregistration_protocol: bool,
19560 cancelled_activity: bool,
19561 draining_polls: bool,
19562 invalid_task_payload_codec: Option<InvalidTaskPayloadCodec>,
19563 workflow_completion_status: Option<&'static str>,
19564 workflow_completion_body: Option<&'static str>,
19565 }
19566
19567 impl MockWorkerServer {
19568 fn start() -> Self {
19569 Self::start_with_behavior(MockWorkerBehavior::default())
19570 }
19571
19572 fn reject_query_protocol() -> Self {
19573 Self::start_with_behavior(MockWorkerBehavior {
19574 reject_query_protocol: true,
19575 ..MockWorkerBehavior::default()
19576 })
19577 }
19578
19579 fn reject_query_completion() -> Self {
19580 Self::start_with_behavior(MockWorkerBehavior {
19581 reject_query_completion: true,
19582 ..MockWorkerBehavior::default()
19583 })
19584 }
19585
19586 fn waiting_query_worker() -> Self {
19587 Self::start_with_behavior(MockWorkerBehavior {
19588 waiting_query_worker: true,
19589 complete_named_signal: true,
19590 ..MockWorkerBehavior::default()
19591 })
19592 }
19593
19594 fn transient_worker_failures() -> Self {
19595 Self::start_with_behavior(MockWorkerBehavior {
19596 poll_failures_per_path: 1,
19597 heartbeat_failures: 1,
19598 ..MockWorkerBehavior::default()
19599 })
19600 }
19601
19602 fn consecutive_poll_failures(count: usize) -> Self {
19603 Self::start_with_behavior(MockWorkerBehavior {
19604 poll_failures_per_path: count,
19605 ..MockWorkerBehavior::default()
19606 })
19607 }
19608
19609 fn delayed_heartbeat_worker() -> Self {
19610 Self::start_with_behavior(MockWorkerBehavior {
19611 waiting_query_worker: true,
19612 delayed_heartbeat_request: Some(2),
19613 heartbeat_response_delay: Duration::from_millis(1_500),
19614 concurrent_requests: true,
19615 cancelled_activity: true,
19616 ..MockWorkerBehavior::default()
19617 })
19618 }
19619
19620 fn heartbeat_retry_worker() -> Self {
19621 Self::start_with_behavior(MockWorkerBehavior {
19622 waiting_query_worker: true,
19623 heartbeat_failure_request: Some(2),
19624 concurrent_requests: true,
19625 ..MockWorkerBehavior::default()
19626 })
19627 }
19628
19629 fn unavailable_polls() -> Self {
19630 Self::start_with_behavior(MockWorkerBehavior {
19631 poll_failures_per_path: usize::MAX,
19632 ..MockWorkerBehavior::default()
19633 })
19634 }
19635
19636 fn unauthorized_polls() -> Self {
19637 Self::start_with_behavior(MockWorkerBehavior {
19638 unauthorized_polls: true,
19639 ..MockWorkerBehavior::default()
19640 })
19641 }
19642
19643 fn rejected_registration() -> Self {
19644 Self::start_with_behavior(MockWorkerBehavior {
19645 reject_registration: true,
19646 ..MockWorkerBehavior::default()
19647 })
19648 }
19649
19650 fn declined_registration() -> Self {
19651 Self::start_with_behavior(MockWorkerBehavior {
19652 decline_registration: true,
19653 ..MockWorkerBehavior::default()
19654 })
19655 }
19656
19657 fn rejected_deregistration() -> Self {
19658 Self::start_with_behavior(MockWorkerBehavior {
19659 reject_deregistration: true,
19660 ..MockWorkerBehavior::default()
19661 })
19662 }
19663
19664 fn rejected_deregistration_protocol() -> Self {
19665 Self::start_with_behavior(MockWorkerBehavior {
19666 reject_deregistration_protocol: true,
19667 ..MockWorkerBehavior::default()
19668 })
19669 }
19670
19671 fn unauthorized_polls_and_rejected_deregistration() -> Self {
19672 Self::start_with_behavior(MockWorkerBehavior {
19673 unauthorized_polls: true,
19674 reject_deregistration: true,
19675 ..MockWorkerBehavior::default()
19676 })
19677 }
19678
19679 fn cancelled_activity() -> Self {
19680 Self::start_with_behavior(MockWorkerBehavior {
19681 cancelled_activity: true,
19682 ..MockWorkerBehavior::default()
19683 })
19684 }
19685
19686 fn draining_polls() -> Self {
19687 Self::start_with_behavior(MockWorkerBehavior {
19688 draining_polls: true,
19689 ..MockWorkerBehavior::default()
19690 })
19691 }
19692
19693 fn invalid_task_payload_codec(codec: InvalidTaskPayloadCodec) -> Self {
19694 Self::start_with_behavior(MockWorkerBehavior {
19695 invalid_task_payload_codec: Some(codec),
19696 ..MockWorkerBehavior::default()
19697 })
19698 }
19699
19700 fn workflow_completion(status: &'static str, body: &'static str) -> Self {
19701 Self::start_with_behavior(MockWorkerBehavior {
19702 workflow_completion_status: Some(status),
19703 workflow_completion_body: Some(body),
19704 ..MockWorkerBehavior::default()
19705 })
19706 }
19707
19708 fn start_with_behavior(behavior: MockWorkerBehavior) -> Self {
19709 let listener = TcpListener::bind("127.0.0.1:0").expect("bind mock server");
19710 listener
19711 .set_nonblocking(true)
19712 .expect("configure mock listener");
19713 let addr = listener.local_addr().expect("mock server address");
19714 let stop = Arc::new(AtomicBool::new(false));
19715 let server_stop = Arc::clone(&stop);
19716 let requests = Arc::new(Mutex::new(Vec::new()));
19717 let server_requests = Arc::clone(&requests);
19718 let thread = thread::spawn(move || {
19719 let mut request_threads = Vec::new();
19720 while !server_stop.load(Ordering::SeqCst) {
19721 match listener.accept() {
19722 Ok((mut stream, _)) => {
19723 if behavior.concurrent_requests {
19724 let requests = Arc::clone(&server_requests);
19725 request_threads.push(thread::spawn(move || {
19726 handle_mock_worker_request(&mut stream, &requests, behavior)
19727 }));
19728 } else {
19729 handle_mock_worker_request(&mut stream, &server_requests, behavior);
19730 }
19731 }
19732 Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
19733 let mut index = 0;
19734 while index < request_threads.len() {
19735 if request_threads[index].is_finished() {
19736 request_threads
19737 .swap_remove(index)
19738 .join()
19739 .expect("join mock request");
19740 } else {
19741 index += 1;
19742 }
19743 }
19744 thread::sleep(Duration::from_millis(5));
19745 }
19746 Err(_) => break,
19747 }
19748 }
19749 for request_thread in request_threads {
19750 request_thread.join().expect("join mock request");
19751 }
19752 });
19753
19754 Self {
19755 addr,
19756 stop,
19757 requests,
19758 thread: Some(thread),
19759 }
19760 }
19761
19762 fn base_url(&self) -> String {
19763 format!("http://{}", self.addr)
19764 }
19765
19766 fn worker_protocol_for(&self, path: &str) -> Option<String> {
19767 self.requests
19768 .lock()
19769 .expect("captured requests")
19770 .iter()
19771 .find(|request| request.path == path)
19772 .and_then(|request| request.worker_protocol.clone())
19773 }
19774
19775 fn control_protocol_for(&self, path: &str) -> Option<String> {
19776 self.requests
19777 .lock()
19778 .expect("captured requests")
19779 .iter()
19780 .find(|request| request.path == path)
19781 .and_then(|request| request.control_protocol.clone())
19782 }
19783
19784 fn method_for(&self, path: &str) -> Option<String> {
19785 self.requests
19786 .lock()
19787 .expect("captured requests")
19788 .iter()
19789 .find(|request| request.path == path)
19790 .map(|request| request.method.clone())
19791 }
19792
19793 fn authorization_for(&self, path: &str) -> Option<String> {
19794 self.requests
19795 .lock()
19796 .expect("captured requests")
19797 .iter()
19798 .find(|request| request.path == path)
19799 .and_then(|request| request.authorization.clone())
19800 }
19801
19802 fn namespace_for(&self, path: &str) -> Option<String> {
19803 self.requests
19804 .lock()
19805 .expect("captured requests")
19806 .iter()
19807 .find(|request| request.path == path)
19808 .and_then(|request| request.namespace.clone())
19809 }
19810
19811 fn request_count(&self, path: &str) -> usize {
19812 self.requests
19813 .lock()
19814 .expect("captured requests")
19815 .iter()
19816 .filter(|request| request.path == path)
19817 .count()
19818 }
19819
19820 fn captured_paths(&self) -> Vec<String> {
19821 self.requests
19822 .lock()
19823 .expect("captured requests")
19824 .iter()
19825 .map(|request| request.path.clone())
19826 .collect()
19827 }
19828
19829 fn request_times(&self, path: &str) -> Vec<Instant> {
19830 self.requests
19831 .lock()
19832 .expect("captured requests")
19833 .iter()
19834 .filter(|request| request.path == path)
19835 .map(|request| request.received_at)
19836 .collect()
19837 }
19838
19839 fn request_body(&self, path: &str) -> Value {
19840 let requests = self.requests.lock().expect("captured requests");
19841 let body = &requests
19842 .iter()
19843 .find(|request| request.path == path)
19844 .unwrap_or_else(|| panic!("missing request for {path}"))
19845 .body;
19846 serde_json::from_str(body).unwrap_or_else(|error| {
19847 panic!("invalid JSON request body for {path}: {error}: {body:?}")
19848 })
19849 }
19850
19851 fn request_bodies(&self, path: &str) -> Vec<Value> {
19852 self.requests
19853 .lock()
19854 .expect("captured requests")
19855 .iter()
19856 .filter(|request| request.path == path)
19857 .map(|request| {
19858 serde_json::from_str(&request.body).unwrap_or_else(|error| {
19859 panic!(
19860 "invalid JSON request body for {path}: {error}: {:?}",
19861 request.body
19862 )
19863 })
19864 })
19865 .collect()
19866 }
19867 }
19868
19869 impl Drop for MockWorkerServer {
19870 fn drop(&mut self) {
19871 self.stop.store(true, Ordering::SeqCst);
19872 let _ = TcpStream::connect(self.addr);
19873
19874 if let Some(thread) = self.thread.take() {
19875 thread.join().expect("join mock server");
19876 }
19877 }
19878 }
19879
19880 fn handle_mock_worker_request(
19881 stream: &mut TcpStream,
19882 requests: &Arc<Mutex<Vec<CapturedRequest>>>,
19883 behavior: MockWorkerBehavior,
19884 ) {
19885 let _ = stream.set_read_timeout(Some(Duration::from_millis(200)));
19886 let mut buffer = [0_u8; 8192];
19887 let mut request = Vec::new();
19888
19889 loop {
19890 match stream.read(&mut buffer) {
19891 Ok(0) => break,
19892 Ok(read) => {
19893 request.extend_from_slice(&buffer[..read]);
19894 if mock_request_is_complete(&request) {
19895 break;
19896 }
19897 }
19898 Err(error)
19899 if matches!(
19900 error.kind(),
19901 std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut
19902 ) =>
19903 {
19904 break;
19905 }
19906 Err(_) => return,
19907 }
19908 }
19909
19910 let request = String::from_utf8_lossy(&request);
19911 let body = request
19912 .split_once("\r\n\r\n")
19913 .map(|(_, body)| body)
19914 .unwrap_or_default();
19915 let path = request
19916 .lines()
19917 .next()
19918 .and_then(|line| line.split_whitespace().nth(1))
19919 .unwrap_or_default();
19920 let method = request
19921 .lines()
19922 .next()
19923 .and_then(|line| line.split_whitespace().next())
19924 .unwrap_or_default();
19925 let authorization = request.lines().find_map(|line| {
19926 let (name, value) = line.split_once(':')?;
19927 name.eq_ignore_ascii_case("Authorization")
19928 .then(|| value.trim().to_string())
19929 });
19930 let namespace = request.lines().find_map(|line| {
19931 let (name, value) = line.split_once(':')?;
19932 name.eq_ignore_ascii_case("X-Namespace")
19933 .then(|| value.trim().to_string())
19934 });
19935 let worker_protocol = request.lines().find_map(|line| {
19936 let (name, value) = line.split_once(':')?;
19937 name.eq_ignore_ascii_case("X-Durable-Workflow-Protocol-Version")
19938 .then(|| value.trim().to_string())
19939 });
19940 let control_protocol = request.lines().find_map(|line| {
19941 let (name, value) = line.split_once(':')?;
19942 name.eq_ignore_ascii_case("X-Durable-Workflow-Control-Plane-Version")
19943 .then(|| value.trim().to_string())
19944 });
19945 let request_number = {
19946 let mut requests = requests.lock().expect("captured requests");
19947 requests.push(CapturedRequest {
19948 method: method.to_string(),
19949 path: path.to_string(),
19950 authorization,
19951 namespace,
19952 worker_protocol: worker_protocol.clone(),
19953 control_protocol,
19954 body: body.to_string(),
19955 received_at: Instant::now(),
19956 });
19957 requests
19958 .iter()
19959 .filter(|request| request.path == path)
19960 .count()
19961 };
19962
19963 if behavior.reject_registration && path == "/api/worker/register" {
19964 write_mock_response(
19965 stream,
19966 "503 Service Unavailable",
19967 r#"{"reason":"registration_unavailable","message":"registration failed"}"#,
19968 );
19969 return;
19970 }
19971
19972 if path.starts_with("/api/worker/registrations/") {
19973 if behavior.reject_deregistration_protocol {
19974 write_mock_response(
19975 stream,
19976 "400 Bad Request",
19977 r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.15","requested_version":"1.16"}"#,
19978 );
19979 } else if behavior.reject_deregistration {
19980 write_mock_response(
19981 stream,
19982 "403 Forbidden",
19983 r#"{"reason":"authorization_failed","message":"worker cannot deregister"}"#,
19984 );
19985 } else {
19986 write_mock_response(
19987 stream,
19988 "200 OK",
19989 r#"{"worker_id":"deregistered-worker","outcome":"deregistered","recovered_workflow_task_count":2}"#,
19990 );
19991 }
19992 return;
19993 }
19994
19995 let is_poll = matches!(
19996 path,
19997 "/api/worker/workflow-tasks/poll"
19998 | "/api/worker/activity-tasks/poll"
19999 | "/api/worker/query-tasks/poll"
20000 );
20001 if is_poll && request_number <= behavior.poll_failures_per_path {
20002 return;
20003 }
20004 if path == "/api/worker/heartbeat" && request_number <= behavior.heartbeat_failures {
20005 return;
20006 }
20007 if path == "/api/worker/heartbeat"
20008 && behavior.heartbeat_failure_request == Some(request_number)
20009 {
20010 return;
20011 }
20012 if path == "/api/worker/heartbeat"
20013 && behavior.delayed_heartbeat_request == Some(request_number)
20014 {
20015 thread::sleep(behavior.heartbeat_response_delay);
20016 }
20017 if behavior.unauthorized_polls && is_poll {
20018 write_mock_response(
20019 stream,
20020 "401 Unauthorized",
20021 r#"{"reason":"authentication_failed","message":"invalid worker token"}"#,
20022 );
20023 return;
20024 }
20025 if behavior.draining_polls && is_poll {
20026 write_mock_response(
20027 stream,
20028 "409 Conflict",
20029 r#"{"task":null,"poll_status":"draining","reason":"worker_draining","worker_status":"draining","drain_intent":"draining"}"#,
20030 );
20031 return;
20032 }
20033
20034 if let Some(codec_case) = behavior.invalid_task_payload_codec {
20035 if is_poll && request_number == 1 {
20036 let mut task = match path {
20037 "/api/worker/workflow-tasks/poll" => json!({
20038 "task_id": "codec-workflow",
20039 "workflow_type": "codec.workflow",
20040 "payload_codec": DEFAULT_CODEC,
20041 "workflow_task_attempt": 1,
20042 "lease_owner": "codec-worker"
20043 }),
20044 "/api/worker/activity-tasks/poll" => json!({
20045 "task_id": "codec-activity",
20046 "activity_attempt_id": "codec-activity-attempt",
20047 "activity_type": "codec.activity",
20048 "payload_codec": DEFAULT_CODEC,
20049 "attempt_number": 1,
20050 "lease_owner": "codec-worker"
20051 }),
20052 "/api/worker/query-tasks/poll" => json!({
20053 "query_task_id": "codec-query",
20054 "query_task_attempt": 1,
20055 "workflow_type": "codec.workflow",
20056 "query_name": "known",
20057 "payload_codec": DEFAULT_CODEC,
20058 "lease_owner": "codec-worker"
20059 }),
20060 _ => unreachable!("is_poll limits task codec probe paths"),
20061 };
20062 codec_case.apply(&mut task);
20063 write_mock_response(stream, "200 OK", &json!({"task": task}).to_string());
20064 return;
20065 }
20066
20067 if matches!(
20068 path,
20069 "/api/worker/workflow-tasks/codec-workflow/fail"
20070 | "/api/worker/activity-tasks/codec-activity/fail"
20071 | "/api/worker/query-tasks/codec-query/fail"
20072 ) {
20073 write_mock_response(stream, "200 OK", r#"{"outcome":"failed"}"#);
20074 return;
20075 }
20076 }
20077
20078 if behavior.reject_query_protocol && path.starts_with("/api/worker/query-tasks/") {
20079 let requested_version = worker_protocol.as_deref().unwrap_or("missing");
20080 let body = format!(
20081 r#"{{"reason":"unsupported_protocol_version","message":"Query tasks require worker protocol 1.8 or newer.","supported_version":"1.7","requested_version":"{requested_version}"}}"#
20082 );
20083 write_mock_response(stream, "400 Bad Request", &body);
20084 return;
20085 }
20086
20087 if behavior.reject_query_completion && path == "/api/worker/query-tasks/query-late/complete"
20088 {
20089 write_mock_response(
20090 stream,
20091 "409 Conflict",
20092 r#"{"reason":"query_task_timed_out","message":"query task timed out before completion"}"#,
20093 );
20094 return;
20095 }
20096
20097 if behavior.workflow_completion_status.is_some()
20098 && path == "/api/worker/workflow-tasks/poll"
20099 && request_number == 1
20100 {
20101 write_mock_response(
20102 stream,
20103 "200 OK",
20104 r#"{"task":{"task_id":"workflow-timeout-task","workflow_id":"reused-workflow-id","run_id":"run-selected-timeout","workflow_type":"timeout.workflow","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"history_events":[],"workflow_task_attempt":3,"lease_owner":"timeout-worker"}}"#,
20105 );
20106 return;
20107 }
20108
20109 if path == "/api/worker/workflow-tasks/workflow-timeout-task/complete" {
20110 if let (Some(status), Some(body)) = (
20111 behavior.workflow_completion_status,
20112 behavior.workflow_completion_body,
20113 ) {
20114 write_mock_response(stream, status, body);
20115 return;
20116 }
20117 }
20118
20119 if behavior.waiting_query_worker {
20120 if behavior.complete_named_signal
20121 && path == "/api/worker/workflow-tasks/poll"
20122 && request_number == 1
20123 {
20124 let body = json!({
20125 "task": {
20126 "task_id": "snapshot-open",
20127 "workflow_id": "snapshot-1",
20128 "run_id": "snapshot-run-1",
20129 "workflow_type": "snapshot",
20130 "payload_codec": DEFAULT_CODEC,
20131 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
20132 .expect("Avro workflow arguments"),
20133 "history_events": [],
20134 "workflow_task_attempt": 1,
20135 "lease_owner": "rust-snapshot-worker"
20136 }
20137 })
20138 .to_string();
20139 write_mock_response(stream, "200 OK", &body);
20140 return;
20141 }
20142
20143 let signal_request = request_number - usize::from(behavior.complete_named_signal);
20144 let signal_request_limit = 2 + usize::from(behavior.complete_named_signal);
20145 if path == "/api/worker/workflow-tasks/poll"
20146 && signal_request >= 1
20147 && signal_request <= signal_request_limit
20148 {
20149 let finish = behavior.complete_named_signal && signal_request == 3;
20150 let amounts = if signal_request == 1 {
20151 vec![3]
20152 } else {
20153 vec![3, 5]
20154 };
20155 let task_id = if signal_request == 1 {
20156 "snapshot-wait-3"
20157 } else if finish {
20158 "snapshot-finish"
20159 } else {
20160 "snapshot-wait-5"
20161 };
20162 let mut history_events = std::iter::once(json!({
20163 "event_type": "SignalWaitOpened",
20164 "payload": {"sequence": 1, "signal_name": "finish"}
20165 }))
20166 .chain(amounts.iter().enumerate().map(|(index, amount)| {
20167 json!({
20168 "event_type": "SignalReceived",
20169 "payload": {
20170 "signal_id": format!("increment-{amount}"),
20171 "signal_name": "increment",
20172 "workflow_sequence": index + 2,
20173 "payload_codec": DEFAULT_CODEC,
20174 "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
20175 .expect("Avro signal envelope")
20176 }
20177 })
20178 }))
20179 .collect::<Vec<_>>();
20180 let (resume_id, resume_name, resume_arguments) = if finish {
20181 history_events.push(json!({
20182 "event_type": "SignalReceived",
20183 "payload": {
20184 "signal_id": "finish",
20185 "signal_name": "finish",
20186 "workflow_sequence": 4,
20187 "payload_codec": DEFAULT_CODEC,
20188 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
20189 .expect("Avro finish signal envelope")
20190 }
20191 }));
20192 (
20193 "finish".to_string(),
20194 "finish".to_string(),
20195 encode_value_envelope(&json!([]), DEFAULT_CODEC)
20196 .expect("Avro finish resume signal"),
20197 )
20198 } else {
20199 let amount = amounts.last().expect("amount");
20200 (
20201 format!("increment-{amount}"),
20202 "increment".to_string(),
20203 encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
20204 .expect("Avro increment resume signal"),
20205 )
20206 };
20207 let body = json!({
20208 "task": {
20209 "task_id": task_id,
20210 "workflow_id": "snapshot-1",
20211 "run_id": "snapshot-run-1",
20212 "workflow_type": "snapshot",
20213 "payload_codec": DEFAULT_CODEC,
20214 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
20215 .expect("Avro workflow arguments"),
20216 "history_events": history_events,
20217 "workflow_task_attempt": 1,
20218 "workflow_signal_id": resume_id,
20219 "signal_name": resume_name,
20220 "signal_arguments": resume_arguments,
20221 "lease_owner": "rust-snapshot-worker"
20222 }
20223 })
20224 .to_string();
20225 write_mock_response(stream, "200 OK", &body);
20226 return;
20227 }
20228
20229 if path == "/api/worker/query-tasks/poll" && request_number == 1 {
20230 let history_events = [3, 5]
20231 .into_iter()
20232 .enumerate()
20233 .map(|(index, amount)| {
20234 json!({
20235 "event_type": "SignalReceived",
20236 "payload": {
20237 "signal_id": format!("increment-{amount}"),
20238 "signal_name": "increment",
20239 "workflow_sequence": index + 2,
20240 "payload_codec": DEFAULT_CODEC,
20241 "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
20242 .expect("Avro query signal envelope")
20243 }
20244 })
20245 })
20246 .collect::<Vec<_>>();
20247 let body = json!({
20248 "task": {
20249 "query_task_id": "snapshot-current",
20250 "query_task_attempt": 1,
20251 "lease_owner": "rust-snapshot-worker",
20252 "workflow_id": "snapshot-1",
20253 "run_id": "snapshot-run-1",
20254 "workflow_type": "snapshot",
20255 "query_name": "current",
20256 "payload_codec": DEFAULT_CODEC,
20257 "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
20258 .expect("Avro workflow arguments"),
20259 "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
20260 .expect("Avro query arguments"),
20261 "history_events": history_events,
20262 "run_status": "waiting"
20263 }
20264 })
20265 .to_string();
20266 write_mock_response(stream, "200 OK", &body);
20267 return;
20268 }
20269
20270 if path == "/api/worker/workflow-tasks/snapshot-wait-3/fail"
20271 || path == "/api/worker/workflow-tasks/snapshot-wait-5/fail"
20272 {
20273 write_mock_response(
20274 stream,
20275 "200 OK",
20276 r#"{"outcome":"waiting_for_history","recorded":true}"#,
20277 );
20278 return;
20279 }
20280
20281 if path == "/api/worker/workflow-tasks/snapshot-open/complete" {
20282 write_mock_response(stream, "200 OK", r#"{"outcome":"waiting","recorded":true}"#);
20283 return;
20284 }
20285
20286 if path == "/api/worker/workflow-tasks/snapshot-finish/complete" {
20287 write_mock_response(
20288 stream,
20289 "200 OK",
20290 r#"{"outcome":"completed","run_status":"completed","recorded":true}"#,
20291 );
20292 return;
20293 }
20294
20295 if path == "/api/worker/query-tasks/snapshot-current/complete" {
20296 write_mock_response(stream, "200 OK", r#"{"outcome":"completed"}"#);
20297 return;
20298 }
20299 }
20300
20301 if matches!(
20302 path,
20303 "/api/workflows/typed-1/query/inspect" | "/api/workflows/typed-1/update/replace"
20304 ) {
20305 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
20306 .expect("typed mock result");
20307 let body = json!({
20308 "result": typed_fidelity_probe().into_json().expect("result projection"),
20309 "result_envelope": result,
20310 })
20311 .to_string();
20312 write_mock_response(stream, "200 OK", &body);
20313 return;
20314 }
20315
20316 if path == "/api/workflows/typed-1" {
20317 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
20318 .expect("typed mock result");
20319 let body = json!({
20320 "workflow_id": "typed-1",
20321 "run_id": "run-typed-1",
20322 "workflow_type": "typed.echo",
20323 "status": "completed",
20324 "output": typed_fidelity_probe().into_json().expect("output projection"),
20325 "output_envelope": result,
20326 })
20327 .to_string();
20328 write_mock_response(stream, "200 OK", &body);
20329 return;
20330 }
20331
20332 let (status, body) = match path {
20333 "/api/health" => ("200 OK", r#"{"status":"ok"}"#),
20334 "/api/workflows" => (
20335 "201 Created",
20336 r#"{"workflow_id":"wf-start-options","run_id":"run-start-options","workflow_type":"rust.timeout"}"#,
20337 ),
20338 "/api/worker/register" if behavior.decline_registration => (
20339 "200 OK",
20340 r#"{"worker_id":"declined-worker","registered":false}"#,
20341 ),
20342 "/api/worker/register" if behavior.waiting_query_worker => (
20343 "200 OK",
20344 r#"{"worker_id":"rust-snapshot-worker","registered":true,"heartbeat_interval_seconds":1}"#,
20345 ),
20346 "/api/worker/register" => (
20347 "200 OK",
20348 r#"{"worker_id":"mock-worker","registered":true,"heartbeat_interval_seconds":3600}"#,
20349 ),
20350 "/api/worker/heartbeat" => ("200 OK", "{}"),
20351 "/api/worker/activity-tasks/poll"
20352 if behavior.cancelled_activity && request_number == 1 =>
20353 {
20354 (
20355 "200 OK",
20356 r#"{"task":{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","activity_type":"cancel-aware","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"attempt_number":1,"lease_owner":"rust-cancel-worker"}}"#,
20357 )
20358 }
20359 "/api/worker/activity-tasks/poll" | "/api/worker/workflow-tasks/poll" => {
20360 ("200 OK", r#"{"task":null}"#)
20361 }
20362 "/api/worker/query-tasks/poll"
20363 if behavior.reject_query_completion && request_number == 1 =>
20364 {
20365 (
20366 "200 OK",
20367 r#"{"task":{"query_task_id":"query-late","query_task_attempt":1,"lease_owner":"late-worker","workflow_id":"counter-late","run_id":"run-late","workflow_type":"counter","query_name":"current","payload_codec":"avro","workflow_arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"query_arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"history_events":[],"run_status":"running"}}"#,
20368 )
20369 }
20370 "/api/worker/query-tasks/poll" => ("200 OK", r#"{"task":null}"#),
20371 "/api/worker/query-tasks/query-capture/complete"
20372 | "/api/worker/query-tasks/query-capture/fail" => ("200 OK", "{}"),
20373 "/api/worker/activity-tasks/activity-cancel/heartbeat" => (
20374 "200 OK",
20375 r#"{"activity_attempt_id":"attempt-cancel","cancel_requested":true,"can_continue":false,"reason":"run_cancelled","run_closed_reason":"cancelled","heartbeat_recorded":false}"#,
20376 ),
20377 "/api/worker/activity-tasks/activity-cancel/complete" => (
20378 "409 Conflict",
20379 r#"{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","reason":"run_cancelled","cancel_requested":true,"can_continue":false,"run_closed_reason":"cancelled"}"#,
20380 ),
20381 "/api/worker/activity-tasks/activity-typed/complete"
20382 | "/api/worker/activity-tasks/activity-typed/fail"
20383 | "/api/workflows/typed-1/signal/changed" => ("200 OK", "{}"),
20384 "/api/workflows/counter-1/query/current" => (
20385 "200 OK",
20386 r#"{"workflow_id":"counter-1","query_name":"current","result":{"count":8},"result_envelope":{"codec":"avro","blob":"wwHioz3/VYAiNw4CCmNvdW50BBAA"}}"#,
20387 ),
20388 "/api/workflows/counter-1/query/missing" => (
20389 "404 Not Found",
20390 r#"{"workflow_id":"counter-1","query_name":"missing","reason":"rejected_unknown_query","message":"unknown query"}"#,
20391 ),
20392 "/api/workflows/wf-lifecycle/cancel" => (
20393 "200 OK",
20394 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","reason":"cleanup requested","command_status":"accepted"}"#,
20395 ),
20396 "/api/workflows/wf-lifecycle/terminate" => (
20397 "200 OK",
20398 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","reason":"forced stop","command_status":"accepted"}"#,
20399 ),
20400 "/api/workflows/wf-lifecycle/runs/run-current/cancel" => (
20401 "200 OK",
20402 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","command_status":"accepted"}"#,
20403 ),
20404 "/api/workflows/wf-lifecycle/runs/run-current/terminate" => (
20405 "200 OK",
20406 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","command_status":"accepted"}"#,
20407 ),
20408 "/api/workflows/wf-lifecycle/runs/run-stale/cancel"
20409 | "/api/workflows/wf-lifecycle/runs/run-stale/terminate" => (
20410 "409 Conflict",
20411 r#"{"workflow_id":"wf-lifecycle","run_id":"run-stale","reason":"historical_run_command_rejected","target_scope":"run","message":"Commands cannot target historical runs."}"#,
20412 ),
20413 "/api/workflows/wf-failed" | "/api/workflows/wf-failed/runs/run-failed" => (
20414 "200 OK",
20415 r#"{"workflow_id":"wf-failed","run_id":"run-failed","status":"failed","closed_reason":"failed","error":"payment failed","failure":{"message":"payment failed","failure_category":"application","exception_type":"PaymentError","exception_class":"billing::PaymentError","non_retryable":true,"exception":{"type":"PaymentError","class":"billing::PaymentError","message":"payment failed"},"failures":[{"id":"failure-17","failure_category":"application"}]}}"#,
20416 ),
20417 "/api/workflows/wf-cancelled" => (
20418 "200 OK",
20419 r#"{"workflow_id":"wf-cancelled","run_id":"run-cancelled","status":"cancelled","closed_reason":"cancelled","reason":"cleanup requested"}"#,
20420 ),
20421 "/api/workflows/wf-terminated" => (
20422 "200 OK",
20423 r#"{"workflow_id":"wf-terminated","run_id":"run-terminated","status":"terminated","closed_reason":"terminated","reason":"forced stop"}"#,
20424 ),
20425 "/api/workflows/wf-timed-out" => (
20426 "200 OK",
20427 r#"{"workflow_id":"wf-timed-out","run_id":"run-timed-out","status":"failed","closed_reason":"timed_out","reason":"run_timeout"}"#,
20428 ),
20429 "/api/workflows/wf-waiting" | "/api/workflows/wf-waiting/runs/run-waiting" => (
20430 "200 OK",
20431 r#"{"workflow_id":"wf-waiting","run_id":"run-waiting","status":"waiting"}"#,
20432 ),
20433 "/api/workflows/wf-selected" => (
20434 "200 OK",
20435 r#"{"workflow_id":"wf-selected","run_id":"run-current","status":"completed","output":"current run output"}"#,
20436 ),
20437 "/api/workflows/wf-selected/runs/run-selected" => (
20438 "200 OK",
20439 r#"{"workflow_id":"wf-selected","run_id":"run-selected","status":"cancelled","closed_reason":"cancelled","reason":"selected run cancelled"}"#,
20440 ),
20441 _ => ("404 Not Found", r#"{"message":"not found"}"#),
20442 };
20443 write_mock_response(stream, status, body);
20444 }
20445
20446 fn mock_request_is_complete(request: &[u8]) -> bool {
20447 let Some(header_end) = request
20448 .windows(4)
20449 .position(|window| window == b"\r\n\r\n")
20450 .map(|position| position + 4)
20451 else {
20452 return false;
20453 };
20454 let headers = String::from_utf8_lossy(&request[..header_end]);
20455 let content_length = headers.lines().find_map(|line| {
20456 let (name, value) = line.split_once(':')?;
20457 name.eq_ignore_ascii_case("content-length")
20458 .then(|| value.trim().parse::<usize>().ok())
20459 .flatten()
20460 });
20461
20462 request.len() >= header_end + content_length.unwrap_or(0)
20463 }
20464
20465 fn write_mock_response(stream: &mut TcpStream, status: &str, body: &str) {
20466 let response = format!(
20467 "HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
20468 body.len()
20469 );
20470
20471 let _ = stream.write_all(response.as_bytes());
20472 let _ = stream.flush();
20473 }
20474}