1#![doc = include_str!("../README.md")]
2
3use std::{
4 any::{Any, TypeId},
5 collections::{BTreeMap, HashMap},
6 future::Future,
7 io::{self, Read},
8 pin::Pin,
9 sync::{
10 atomic::{AtomicBool, Ordering},
11 Arc, Mutex, OnceLock,
12 },
13 task::{Context as TaskContext, Poll},
14 time::{Duration, Instant, SystemTime, UNIX_EPOCH},
15};
16
17use apache_avro::{from_avro_datum, to_avro_datum, types::Value as AvroDatum, Schema};
18use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
19use futures_util::{future::OptionFuture, task::noop_waker_ref};
20use serde::{
21 de::DeserializeOwned,
22 ser::{SerializeMap, SerializeSeq},
23 Deserialize, Deserializer, Serialize, Serializer,
24};
25pub use serde_json::{json, Value};
26use thiserror::Error;
27pub use uuid::Uuid;
28
29pub const WORKER_PROTOCOL_VERSION: &str = "1.2";
30pub const CONTROL_PLANE_VERSION: &str = "2";
31pub const DEFAULT_CODEC: &str = "avro";
32pub const SDK_VERSION: &str = concat!("durable-workflow-rust/", env!("CARGO_PKG_VERSION"));
33pub const QUERY_TASKS_CAPABILITY: &str = "query_tasks";
35pub const WORKFLOW_UPDATES_CAPABILITY: &str = "workflow_updates";
37pub const QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
39
40const MAX_LONG_POLL_TIMEOUT_SECONDS: u64 = 60;
41const WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE: &str =
42 "Workflow task waiting for scheduled history.";
43const WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE: &str = "WorkflowTaskWaitingForHistory";
44const MISSING_TASK_PAYLOAD_CODEC: &str = "\0missing-task-payload-codec";
45const NULL_TASK_PAYLOAD_CODEC: &str = "\0null-task-payload-codec";
46const NON_STRING_TASK_PAYLOAD_CODEC: &str = "\0non-string-task-payload-codec";
47
48const QUERY_TASK_FINAL_REJECTION_REASONS: &[&str] = &[
49 "lease_expired",
50 "query_task_not_found",
51 "query_task_not_leased",
52 "query_task_timed_out",
53];
54
55pub const AVRO_VALUE_SCHEMA_JSON: &str =
57 include_str!("../schema/durable_workflow.protocol.Value.v1.avsc");
58pub const AVRO_VALUE_SCHEMA_FINGERPRINT_HEX: &str = "e2a33dff55802237";
59pub const AVRO_VALUE_SCHEMA_FINGERPRINT: [u8; 8] = [0xe2, 0xa3, 0x3d, 0xff, 0x55, 0x80, 0x22, 0x37];
60const AVRO_SINGLE_OBJECT_MAGIC: [u8; 2] = [0xc3, 0x01];
61
62static AVRO_VALUE_SCHEMA: OnceLock<std::result::Result<Schema, String>> = OnceLock::new();
63
64#[derive(Clone, Copy)]
65enum RequestProtocol {
66 ControlPlane,
67 Worker(&'static str),
68}
69
70pub type Result<T> = std::result::Result<T, Error>;
71
72#[derive(Debug, Error)]
73pub enum Error {
74 #[error("transport error: {0}")]
75 Transport(#[from] reqwest::Error),
76 #[error(
77 "invalid Durable Workflow base URL: omit the SDK-owned /api suffix and pass the Server or Cloud runtime base URL; the SDK appends /api automatically"
78 )]
79 InvalidBaseUrl,
80 #[error("json error: {0}")]
81 Json(#[from] serde_json::Error),
82 #[error("http {status}: {body}")]
83 Http {
84 status: reqwest::StatusCode,
85 body: String,
86 },
87 #[error("codec error: {0}")]
88 Codec(String),
89 #[error(transparent)]
90 QueryFailed(QueryFailure),
91 #[error(transparent)]
92 Protocol(ProtocolFailure),
93 #[error(transparent)]
94 NonDeterministicReplay(ReplayFailure),
95 #[error(transparent)]
96 ChildWorkflowFailed(ChildWorkflowFailure),
97 #[error(transparent)]
98 ActivityFailed(ActivityFailure),
99 #[error(transparent)]
100 WorkflowCommandRejected(WorkflowCommandRejection),
101 #[error(transparent)]
102 WorkflowFailed(WorkflowTerminalOutcome),
103 #[error(transparent)]
104 WorkflowCancelled(WorkflowTerminalOutcome),
105 #[error(transparent)]
106 WorkflowTerminated(WorkflowTerminalOutcome),
107 #[error(transparent)]
108 WorkflowTimedOut(WorkflowTerminalOutcome),
109 #[error(transparent)]
110 ActivityTaskRejected(ActivityTaskRejection),
111 #[error("workflow handler {0:?} is not registered")]
112 WorkflowNotRegistered(String),
113 #[error("activity handler {0:?} is not registered")]
114 ActivityNotRegistered(String),
115 #[error("workflow future yielded without emitting a durable command")]
116 WorkflowYieldedWithoutCommand,
117 #[error("workflow state lock is poisoned")]
118 WorkflowStatePoisoned,
119 #[error("timer duration is too large for the worker protocol")]
120 TimerDurationOverflow,
121 #[error("operation timed out")]
122 Timeout,
123 #[error(
124 "missing {role}-plane credentials: configure ClientBuilder::{role}_token or ClientBuilder::token; a {opposite_role}-plane token cannot authorize this request"
125 )]
126 MissingRoleCredentials {
127 role: &'static str,
128 opposite_role: &'static str,
129 },
130 #[error("worker loop error: {0}")]
131 WorkerLoop(String),
132 #[error(
133 "workflow command contract for {workflow_type:?} declares update validators, but this Rust SDK cannot execute synchronous pre-accept update validation"
134 )]
135 UnsupportedUpdateValidators { workflow_type: String },
136 #[error("{primary}; worker deregistration also failed: {deregistration}")]
137 WorkerShutdown {
138 primary: Box<Error>,
139 deregistration: Box<Error>,
140 },
141 #[error("invalid child workflow options: {0}")]
142 InvalidChildWorkflowOptions(String),
143 #[error(transparent)]
144 InvalidActivityOptions(ActivityOptionsError),
145 #[error(transparent)]
146 InvalidContinueAsNewOptions(#[from] ContinueAsNewOptionsError),
147 #[doc(hidden)]
148 #[error("workflow requested continue as new")]
149 ContinueAsNew(ContinueAsNewRequest),
150}
151
152#[derive(Clone, Copy, Debug, PartialEq, Eq)]
154pub enum WorkflowCommandKind {
155 Cancel,
156 Terminate,
157}
158
159impl WorkflowCommandKind {
160 fn as_str(self) -> &'static str {
161 match self {
162 Self::Cancel => "cancel",
163 Self::Terminate => "terminate",
164 }
165 }
166}
167
168#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
170pub struct WorkflowCommandOptions {
171 #[serde(skip_serializing_if = "Option::is_none")]
172 pub reason: Option<String>,
173 #[serde(skip_serializing_if = "Option::is_none")]
174 pub request_id: Option<String>,
175}
176
177#[derive(Clone, Debug, PartialEq, Eq)]
184pub struct WorkflowStartOptions {
185 pub execution_timeout_seconds: u64,
186 pub run_timeout_seconds: u64,
187}
188
189impl Default for WorkflowStartOptions {
190 fn default() -> Self {
191 Self {
192 execution_timeout_seconds: 3600,
193 run_timeout_seconds: 600,
194 }
195 }
196}
197
198impl WorkflowStartOptions {
199 pub fn new() -> Self {
200 Self::default()
201 }
202
203 pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
204 self.execution_timeout_seconds = seconds;
205 self
206 }
207
208 pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
209 self.run_timeout_seconds = seconds;
210 self
211 }
212
213 fn validate(&self) -> Result<()> {
214 if self.execution_timeout_seconds == 0 {
215 return Err(Error::Codec(
216 "execution_timeout_seconds must be at least 1".to_string(),
217 ));
218 }
219 if self.run_timeout_seconds == 0 {
220 return Err(Error::Codec(
221 "run_timeout_seconds must be at least 1".to_string(),
222 ));
223 }
224 if self.run_timeout_seconds > self.execution_timeout_seconds {
225 return Err(Error::Codec(
226 "run_timeout_seconds cannot exceed execution_timeout_seconds".to_string(),
227 ));
228 }
229
230 Ok(())
231 }
232}
233
234#[derive(Clone, Debug, Default, PartialEq, Eq)]
239pub struct ContinueAsNewOptions {
240 pub workflow_type: Option<String>,
241 pub task_queue: Option<String>,
242}
243
244impl ContinueAsNewOptions {
245 pub fn new() -> Self {
246 Self::default()
247 }
248
249 pub fn workflow_type(mut self, workflow_type: impl Into<String>) -> Self {
250 self.workflow_type = Some(workflow_type.into());
251 self
252 }
253
254 pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
255 self.task_queue = Some(task_queue.into());
256 self
257 }
258
259 fn validate(&self) -> std::result::Result<(), ContinueAsNewOptionsError> {
260 for (field, value) in [
261 ("workflow_type", self.workflow_type.as_deref()),
262 ("task_queue", self.task_queue.as_deref()),
263 ] {
264 if value.is_some_and(|value| value.trim().is_empty()) {
265 return Err(ContinueAsNewOptionsError {
266 field,
267 message: format!("{field} must not be empty"),
268 });
269 }
270 }
271 Ok(())
272 }
273}
274
275#[derive(Clone, Debug, Error, PartialEq, Eq)]
277#[error("invalid continue-as-new option {field}: {message}")]
278pub struct ContinueAsNewOptionsError {
279 pub field: &'static str,
280 pub message: String,
281}
282
283#[derive(Clone, Debug, Default, PartialEq, Eq)]
285pub struct WorkflowHistoryBudget {
286 pub event_count: u64,
287 pub size_bytes: Option<u64>,
288 pub continue_as_new_recommended: bool,
289 pub pressure: Option<String>,
290}
291
292#[doc(hidden)]
293#[derive(Clone, Debug)]
294pub struct ContinueAsNewRequest {
295 arguments: AvroValue,
296 options: ContinueAsNewOptions,
297}
298
299impl WorkflowCommandOptions {
300 pub fn new() -> Self {
301 Self::default()
302 }
303
304 pub fn reason(mut self, reason: impl Into<String>) -> Self {
305 self.reason = Some(reason.into());
306 self
307 }
308
309 pub fn request_id(mut self, request_id: impl Into<String>) -> Self {
310 self.request_id = Some(request_id.into());
311 self
312 }
313}
314
315#[derive(Clone, Debug, PartialEq)]
317pub struct WorkflowCommandResult {
318 pub command: WorkflowCommandKind,
319 pub workflow_id: String,
320 pub run_id: Option<String>,
321 pub outcome: Option<String>,
322 pub reason: Option<String>,
323 pub command_status: Option<String>,
324 pub raw: Value,
325}
326
327#[derive(Clone, Debug, Error)]
329#[error("workflow {command:?} rejected ({reason}, HTTP {status}): {message}")]
330pub struct WorkflowCommandRejection {
331 pub command: WorkflowCommandKind,
332 pub status: u16,
333 pub reason: String,
334 pub message: String,
335 pub workflow_id: String,
336 pub run_id: Option<String>,
337 pub target_scope: Option<String>,
338 pub body: Value,
339}
340
341#[derive(Clone, Copy, Debug, PartialEq, Eq)]
343pub enum WorkflowTerminalKind {
344 Failed,
345 Cancelled,
346 Terminated,
347 TimedOut,
348}
349
350#[derive(Clone, Debug, Error)]
356#[error("workflow {workflow_id} run {run_id:?} ended as {kind:?} ({reason})")]
357pub struct WorkflowTerminalOutcome {
358 pub kind: WorkflowTerminalKind,
359 pub workflow_id: String,
360 pub run_id: Option<String>,
361 pub reason: String,
362 pub failure_category: Option<String>,
363 pub failure_id: Option<String>,
364 pub exception_type: Option<String>,
365 pub exception_class: Option<String>,
366 pub non_retryable: Option<bool>,
367 pub message: Option<String>,
368 pub exception: Option<Value>,
369 pub raw: Value,
370}
371
372#[derive(Clone, Debug, Error)]
374#[error("activity task {operation} rejected ({reason}, HTTP {status})")]
375pub struct ActivityTaskRejection {
376 pub operation: String,
377 pub status: u16,
378 pub reason: String,
379 pub task_id: String,
380 pub activity_attempt_id: String,
381 pub cancel_requested: bool,
382 pub can_continue: Option<bool>,
383 pub run_closed_reason: Option<String>,
384 pub body: Value,
385}
386
387#[derive(Clone, Copy, Debug, PartialEq, Eq)]
389pub enum ActivityOptionsErrorKind {
390 EmptyTaskQueue,
391 EmptyRetryPolicy,
392 InvalidMaxAttempts,
393 BackoffWithoutRetryBudget,
394 TooManyBackoffIntervals,
395 InvalidBackoffCoefficient,
396 BackoffGenerationTooLarge,
397 BackoffOverflow,
398 EmptyNonRetryableErrorType,
399 TimeoutNotPositive,
400 TimeoutOverflow,
401 TimeoutOrder,
402}
403
404#[derive(Clone, Debug, Error, PartialEq, Eq)]
406#[error("invalid activity options ({kind:?}, {field:?}): {message}")]
407pub struct ActivityOptionsError {
408 pub kind: ActivityOptionsErrorKind,
409 pub field: Option<&'static str>,
410 pub message: String,
411}
412
413impl ActivityOptionsError {
414 fn new(
415 kind: ActivityOptionsErrorKind,
416 field: Option<&'static str>,
417 message: impl Into<String>,
418 ) -> Self {
419 Self {
420 kind,
421 field,
422 message: message.into(),
423 }
424 }
425}
426
427#[derive(Clone, Copy, Debug, PartialEq, Eq)]
429pub enum ActivityFailureKind {
430 Failed,
431 Cancelled,
432 TimedOut,
433}
434
435#[derive(Clone, Debug, Error)]
440#[error("activity failed ({reason}): {message}")]
441pub struct ActivityFailure {
442 pub kind: ActivityFailureKind,
443 pub reason: String,
444 pub message: String,
445 pub activity_execution_id: Option<String>,
446 pub activity_attempt_id: Option<String>,
447 pub activity_type: Option<String>,
448 pub activity_class: Option<String>,
449 pub attempt_number: Option<u64>,
450 pub failure_id: Option<String>,
451 pub failure_category: Option<String>,
452 pub timeout_kind: Option<String>,
453 pub non_retryable: bool,
454 pub exception_type: Option<String>,
455 pub exception_class: Option<String>,
456 pub code: Option<Value>,
457 pub exception: Option<Value>,
458}
459
460#[derive(Clone, Copy, Debug, PartialEq, Eq)]
462pub enum ChildWorkflowFailureKind {
463 Failed,
464 Cancelled,
465 Terminated,
466}
467
468#[derive(Clone, Debug, Error)]
474#[error("child workflow failed ({reason}): {message}")]
475pub struct ChildWorkflowFailure {
476 pub kind: ChildWorkflowFailureKind,
477 pub reason: String,
478 pub message: String,
479 pub parent_workflow_id: Option<String>,
480 pub parent_workflow_run_id: Option<String>,
481 pub child_workflow_id: Option<String>,
482 pub child_workflow_run_id: Option<String>,
483 pub child_workflow_type: Option<String>,
484 pub failure_id: Option<String>,
485 pub failure_category: Option<String>,
486 pub exception_type: Option<String>,
487 pub exception_class: Option<String>,
488 pub non_retryable: bool,
489 pub code: Option<Value>,
490 pub exception: Option<Value>,
491}
492
493#[derive(Clone, Debug, PartialEq, Eq)]
495pub struct WorkflowIdentity {
496 pub workflow_id: Option<String>,
497 pub run_id: Option<String>,
498}
499
500#[derive(Clone, Debug, PartialEq)]
502pub struct ChildWorkflowResult {
503 pub parent: WorkflowIdentity,
504 pub child: WorkflowIdentity,
505 pub child_workflow_type: Option<String>,
506 pub result: Value,
507}
508
509#[derive(Clone, Debug, PartialEq)]
511pub struct ChildWorkflowAvroResult {
512 pub parent: WorkflowIdentity,
513 pub child: WorkflowIdentity,
514 pub child_workflow_type: Option<String>,
515 pub result: AvroValue,
516}
517
518#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
520pub enum ParentClosePolicy {
521 #[default]
522 Abandon,
523 RequestCancel,
524 Terminate,
525}
526
527impl ParentClosePolicy {
528 fn as_str(self) -> &'static str {
529 match self {
530 Self::Abandon => "abandon",
531 Self::RequestCancel => "request_cancel",
532 Self::Terminate => "terminate",
533 }
534 }
535}
536
537#[derive(Clone, Debug, Default, PartialEq, Eq)]
539pub struct ChildWorkflowRetryPolicy {
540 pub max_attempts: Option<u32>,
541 pub backoff_seconds: Vec<u64>,
542 pub non_retryable_error_types: Vec<String>,
543}
544
545#[derive(Clone, Debug, PartialEq, Eq)]
549pub struct ChildWorkflowOptions {
550 pub task_queue: String,
551 pub parent_close_policy: ParentClosePolicy,
552 pub retry_policy: Option<ChildWorkflowRetryPolicy>,
553 pub execution_timeout_seconds: Option<u64>,
554 pub run_timeout_seconds: Option<u64>,
555}
556
557impl ChildWorkflowOptions {
558 pub fn new(task_queue: impl Into<String>) -> Self {
559 Self {
560 task_queue: task_queue.into(),
561 parent_close_policy: ParentClosePolicy::Abandon,
562 retry_policy: None,
563 execution_timeout_seconds: None,
564 run_timeout_seconds: None,
565 }
566 }
567
568 pub fn parent_close_policy(mut self, policy: ParentClosePolicy) -> Self {
569 self.parent_close_policy = policy;
570 self
571 }
572
573 pub fn retry_policy(mut self, policy: ChildWorkflowRetryPolicy) -> Self {
574 self.retry_policy = Some(policy);
575 self
576 }
577
578 pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
579 self.execution_timeout_seconds = Some(seconds);
580 self
581 }
582
583 pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
584 self.run_timeout_seconds = Some(seconds);
585 self
586 }
587}
588
589#[derive(Clone, Debug, PartialEq, Eq)]
591pub enum ActivityBackoff {
592 Explicit(Vec<Duration>),
595 Exponential {
597 initial_interval: Duration,
598 coefficient: u32,
599 maximum_interval: Option<Duration>,
600 },
601}
602
603#[derive(Clone, Debug, Default, PartialEq, Eq)]
605pub struct ActivityRetryPolicy {
606 pub max_attempts: Option<u32>,
607 pub backoff: Option<ActivityBackoff>,
608 pub non_retryable_error_types: Vec<String>,
609}
610
611impl ActivityRetryPolicy {
612 pub fn new(max_attempts: u32) -> Self {
614 Self {
615 max_attempts: Some(max_attempts),
616 ..Self::default()
617 }
618 }
619
620 pub fn backoff_intervals(mut self, intervals: impl IntoIterator<Item = Duration>) -> Self {
621 self.backoff = Some(ActivityBackoff::Explicit(intervals.into_iter().collect()));
622 self
623 }
624
625 pub fn exponential_backoff(
626 mut self,
627 initial_interval: Duration,
628 coefficient: u32,
629 maximum_interval: Option<Duration>,
630 ) -> Self {
631 self.backoff = Some(ActivityBackoff::Exponential {
632 initial_interval,
633 coefficient,
634 maximum_interval,
635 });
636 self
637 }
638
639 pub fn non_retryable_error_type(mut self, error_type: impl Into<String>) -> Self {
640 self.non_retryable_error_types.push(error_type.into());
641 self
642 }
643
644 pub fn non_retryable_error_types(
645 mut self,
646 error_types: impl IntoIterator<Item = impl Into<String>>,
647 ) -> Self {
648 self.non_retryable_error_types
649 .extend(error_types.into_iter().map(Into::into));
650 self
651 }
652}
653
654#[derive(Clone, Debug, Default, PartialEq, Eq)]
659pub struct ActivityOptions {
660 pub task_queue: Option<String>,
661 pub retry_policy: Option<ActivityRetryPolicy>,
662 pub start_to_close_timeout: Option<Duration>,
663 pub schedule_to_start_timeout: Option<Duration>,
664 pub schedule_to_close_timeout: Option<Duration>,
665 pub heartbeat_timeout: Option<Duration>,
666}
667
668impl ActivityOptions {
669 pub fn new() -> Self {
670 Self::default()
671 }
672
673 pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
674 self.task_queue = Some(task_queue.into());
675 self
676 }
677
678 pub fn retry_policy(mut self, policy: ActivityRetryPolicy) -> Self {
679 self.retry_policy = Some(policy);
680 self
681 }
682
683 pub fn start_to_close_timeout(mut self, timeout: Duration) -> Self {
684 self.start_to_close_timeout = Some(timeout);
685 self
686 }
687
688 pub fn schedule_to_start_timeout(mut self, timeout: Duration) -> Self {
689 self.schedule_to_start_timeout = Some(timeout);
690 self
691 }
692
693 pub fn schedule_to_close_timeout(mut self, timeout: Duration) -> Self {
694 self.schedule_to_close_timeout = Some(timeout);
695 self
696 }
697
698 pub fn heartbeat_timeout(mut self, timeout: Duration) -> Self {
699 self.heartbeat_timeout = Some(timeout);
700 self
701 }
702
703 fn validate(&self) -> std::result::Result<ValidatedActivityOptions, ActivityOptionsError> {
704 if self
705 .task_queue
706 .as_deref()
707 .is_some_and(|queue| queue.trim().is_empty())
708 {
709 return Err(ActivityOptionsError::new(
710 ActivityOptionsErrorKind::EmptyTaskQueue,
711 Some("task_queue"),
712 "task_queue must not be empty",
713 ));
714 }
715
716 for (field, value) in [
717 ("start_to_close_timeout", self.start_to_close_timeout),
718 ("schedule_to_start_timeout", self.schedule_to_start_timeout),
719 ("schedule_to_close_timeout", self.schedule_to_close_timeout),
720 ("heartbeat_timeout", self.heartbeat_timeout),
721 ] {
722 if value.is_some_and(|value| value.is_zero()) {
723 return Err(ActivityOptionsError::new(
724 ActivityOptionsErrorKind::TimeoutNotPositive,
725 Some(field),
726 format!("{field} must be positive"),
727 ));
728 }
729 }
730
731 validate_timeout_order(
732 "heartbeat_timeout",
733 self.heartbeat_timeout,
734 "start_to_close_timeout",
735 self.start_to_close_timeout,
736 )?;
737 validate_timeout_order(
738 "start_to_close_timeout",
739 self.start_to_close_timeout,
740 "schedule_to_close_timeout",
741 self.schedule_to_close_timeout,
742 )?;
743 validate_timeout_order(
744 "schedule_to_start_timeout",
745 self.schedule_to_start_timeout,
746 "schedule_to_close_timeout",
747 self.schedule_to_close_timeout,
748 )?;
749
750 Ok(ValidatedActivityOptions {
751 task_queue: self.task_queue.clone(),
752 retry_policy: self
753 .retry_policy
754 .as_ref()
755 .map(validate_activity_retry_policy)
756 .transpose()?,
757 start_to_close_timeout: timeout_seconds(
758 "start_to_close_timeout",
759 self.start_to_close_timeout,
760 )?,
761 schedule_to_start_timeout: timeout_seconds(
762 "schedule_to_start_timeout",
763 self.schedule_to_start_timeout,
764 )?,
765 schedule_to_close_timeout: timeout_seconds(
766 "schedule_to_close_timeout",
767 self.schedule_to_close_timeout,
768 )?,
769 heartbeat_timeout: timeout_seconds("heartbeat_timeout", self.heartbeat_timeout)?,
770 })
771 }
772}
773
774#[derive(Clone, Debug)]
775struct ValidatedActivityOptions {
776 task_queue: Option<String>,
777 retry_policy: Option<Value>,
778 start_to_close_timeout: Option<u64>,
779 schedule_to_start_timeout: Option<u64>,
780 schedule_to_close_timeout: Option<u64>,
781 heartbeat_timeout: Option<u64>,
782}
783
784fn validate_timeout_order(
785 smaller_name: &'static str,
786 smaller: Option<Duration>,
787 larger_name: &'static str,
788 larger: Option<Duration>,
789) -> std::result::Result<(), ActivityOptionsError> {
790 if matches!((smaller, larger), (Some(smaller), Some(larger)) if smaller > larger) {
791 return Err(ActivityOptionsError::new(
792 ActivityOptionsErrorKind::TimeoutOrder,
793 Some(smaller_name),
794 format!("{smaller_name} must be <= {larger_name}"),
795 ));
796 }
797 Ok(())
798}
799
800fn timeout_seconds(
801 field: &'static str,
802 value: Option<Duration>,
803) -> std::result::Result<Option<u64>, ActivityOptionsError> {
804 value
805 .map(|value| {
806 activity_protocol_seconds(value).ok_or_else(|| {
807 ActivityOptionsError::new(
808 ActivityOptionsErrorKind::TimeoutOverflow,
809 Some(field),
810 format!("{field} is too large for the worker protocol"),
811 )
812 })
813 })
814 .transpose()
815}
816
817fn duration_seconds_ceil(value: Duration) -> Option<u64> {
818 value
819 .as_secs()
820 .checked_add(u64::from(value.subsec_nanos() > 0))
821}
822
823fn activity_protocol_seconds(value: Duration) -> Option<u64> {
824 duration_seconds_ceil(value).filter(|seconds| *seconds <= i64::MAX as u64)
825}
826
827fn validate_activity_retry_policy(
828 policy: &ActivityRetryPolicy,
829) -> std::result::Result<Value, ActivityOptionsError> {
830 if policy.max_attempts.is_none()
831 && policy.backoff.is_none()
832 && policy.non_retryable_error_types.is_empty()
833 {
834 return Err(ActivityOptionsError::new(
835 ActivityOptionsErrorKind::EmptyRetryPolicy,
836 Some("retry_policy"),
837 "retry_policy must configure at least one field",
838 ));
839 }
840 if policy.max_attempts == Some(0) {
841 return Err(ActivityOptionsError::new(
842 ActivityOptionsErrorKind::InvalidMaxAttempts,
843 Some("retry_policy.max_attempts"),
844 "max_attempts must be >= 1",
845 ));
846 }
847 if policy
848 .non_retryable_error_types
849 .iter()
850 .any(|error_type| error_type.trim().is_empty())
851 {
852 return Err(ActivityOptionsError::new(
853 ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
854 Some("retry_policy.non_retryable_error_types"),
855 "non_retryable_error_types must not contain empty values",
856 ));
857 }
858
859 let backoff_seconds = match &policy.backoff {
860 None => None,
861 Some(backoff) => {
862 let max_attempts = policy.max_attempts.ok_or_else(|| {
863 ActivityOptionsError::new(
864 ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
865 Some("retry_policy.backoff"),
866 "backoff requires max_attempts",
867 )
868 })?;
869 let retry_count = max_attempts.saturating_sub(1) as usize;
870 let intervals = match backoff {
871 ActivityBackoff::Explicit(intervals) => {
872 if intervals.len() > retry_count {
873 return Err(ActivityOptionsError::new(
874 ActivityOptionsErrorKind::TooManyBackoffIntervals,
875 Some("retry_policy.backoff"),
876 "backoff interval count must not exceed max_attempts - 1",
877 ));
878 }
879 intervals.clone()
880 }
881 ActivityBackoff::Exponential {
882 initial_interval,
883 coefficient,
884 maximum_interval,
885 } => {
886 if *coefficient < 1 {
887 return Err(ActivityOptionsError::new(
888 ActivityOptionsErrorKind::InvalidBackoffCoefficient,
889 Some("retry_policy.backoff.coefficient"),
890 "backoff coefficient must be >= 1",
891 ));
892 }
893 if retry_count > 10_000 {
894 return Err(ActivityOptionsError::new(
895 ActivityOptionsErrorKind::BackoffGenerationTooLarge,
896 Some("retry_policy.max_attempts"),
897 "generated backoff supports at most 10000 retry intervals",
898 ));
899 }
900 let mut current = *initial_interval;
901 let mut intervals = Vec::with_capacity(retry_count);
902 for _ in 0..retry_count {
903 let interval = maximum_interval
904 .map(|maximum| current.min(maximum))
905 .unwrap_or(current);
906 intervals.push(interval);
907 if maximum_interval.is_some_and(|maximum| interval == maximum) {
908 break;
909 }
910 current = current.checked_mul(*coefficient).ok_or_else(|| {
911 ActivityOptionsError::new(
912 ActivityOptionsErrorKind::BackoffOverflow,
913 Some("retry_policy.backoff"),
914 "generated backoff interval overflowed",
915 )
916 })?;
917 }
918 intervals
919 }
920 };
921 Some(
922 intervals
923 .into_iter()
924 .map(|interval| {
925 activity_protocol_seconds(interval).ok_or_else(|| {
926 ActivityOptionsError::new(
927 ActivityOptionsErrorKind::BackoffOverflow,
928 Some("retry_policy.backoff"),
929 "backoff interval is too large for the worker protocol",
930 )
931 })
932 })
933 .collect::<std::result::Result<Vec<_>, _>>()?,
934 )
935 }
936 };
937
938 let mut encoded = serde_json::Map::new();
939 if let Some(max_attempts) = policy.max_attempts {
940 encoded.insert("max_attempts".to_string(), json!(max_attempts));
941 }
942 if let Some(backoff_seconds) = backoff_seconds {
943 encoded.insert("backoff_seconds".to_string(), json!(backoff_seconds));
944 }
945 if !policy.non_retryable_error_types.is_empty() {
946 let mut canonical_error_types = Vec::new();
947 for error_type in policy
948 .non_retryable_error_types
949 .iter()
950 .map(|error_type| error_type.trim())
951 {
952 if !canonical_error_types.contains(&error_type) {
953 canonical_error_types.push(error_type);
954 }
955 }
956 encoded.insert(
957 "non_retryable_error_types".to_string(),
958 json!(canonical_error_types),
959 );
960 }
961 Ok(Value::Object(encoded))
962}
963
964#[derive(Clone, Debug, Error)]
967#[error("non-deterministic workflow replay ({reason}) at sequence {sequence:?}: {message}")]
968pub struct ReplayFailure {
969 pub reason: String,
970 pub sequence: Option<u64>,
971 pub expected: Option<String>,
972 pub actual: Option<String>,
973 pub message: String,
974}
975
976impl ReplayFailure {
977 fn new(
978 reason: impl Into<String>,
979 sequence: Option<u64>,
980 expected: Option<String>,
981 actual: Option<String>,
982 message: impl Into<String>,
983 ) -> Self {
984 Self {
985 reason: reason.into(),
986 sequence,
987 expected,
988 actual,
989 message: message.into(),
990 }
991 }
992}
993
994#[derive(Clone, Debug, Error)]
996#[error("query failed ({reason}, HTTP {status}): {message}")]
997pub struct QueryFailure {
998 pub status: u16,
999 pub reason: String,
1000 pub message: String,
1001 pub body: Value,
1002}
1003
1004#[derive(Clone, Debug, Error)]
1006#[error("protocol rejected ({reason}, HTTP {status}): {message}")]
1007pub struct ProtocolFailure {
1008 pub status: u16,
1009 pub reason: String,
1010 pub message: String,
1011 pub supported_version: Option<String>,
1012 pub requested_version: Option<String>,
1013 pub body: Value,
1014}
1015
1016#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
1017pub struct PayloadEnvelope {
1018 pub codec: String,
1019 pub blob: String,
1020}
1021
1022impl PayloadEnvelope {
1023 pub fn avro<T: Serialize>(value: &T) -> Result<Self> {
1024 encode_payload(value, DEFAULT_CODEC)
1025 }
1026
1027 pub fn avro_value(value: &AvroValue) -> Result<Self> {
1030 encode_avro_value(value)
1031 }
1032}
1033
1034#[derive(Clone, Debug, PartialEq)]
1036pub enum AvroValue {
1037 Null,
1038 Boolean(bool),
1039 Long(i64),
1040 Double(f64),
1041 Bytes(Vec<u8>),
1042 String(String),
1043 Array(Vec<AvroValue>),
1044 Map(BTreeMap<String, AvroValue>),
1045}
1046
1047impl AvroValue {
1048 fn from_serialize<T: Serialize>(value: &T) -> Result<Self> {
1049 Self::from_serde_value(
1050 serde_value::to_value(value).map_err(|error| {
1051 Error::Codec(format!("could not adapt value for Avro: {error}"))
1052 })?,
1053 )
1054 }
1055
1056 fn from_serde_value(value: serde_value::Value) -> Result<Self> {
1057 use serde_value::Value as SerdeValue;
1058
1059 match value {
1060 SerdeValue::Unit => Ok(Self::Null),
1061 SerdeValue::Bool(value) => Ok(Self::Boolean(value)),
1062 SerdeValue::I8(value) => Ok(Self::Long(i64::from(value))),
1063 SerdeValue::I16(value) => Ok(Self::Long(i64::from(value))),
1064 SerdeValue::I32(value) => Ok(Self::Long(i64::from(value))),
1065 SerdeValue::I64(value) => Ok(Self::Long(value)),
1066 SerdeValue::U8(value) => Ok(Self::Long(i64::from(value))),
1067 SerdeValue::U16(value) => Ok(Self::Long(i64::from(value))),
1068 SerdeValue::U32(value) => Ok(Self::Long(i64::from(value))),
1069 SerdeValue::U64(value) => i64::try_from(value).map(Self::Long).map_err(|_| {
1070 Error::Codec(
1071 "integer_overflow: Avro Value long must be within signed 64-bit range"
1072 .to_string(),
1073 )
1074 }),
1075 SerdeValue::F32(value) => Self::finite_double(f64::from(value)),
1076 SerdeValue::F64(value) => Self::finite_double(value),
1077 SerdeValue::Char(value) => Ok(Self::String(value.to_string())),
1078 SerdeValue::String(value) => Ok(Self::String(value)),
1079 SerdeValue::Bytes(value) => Ok(Self::Bytes(value)),
1080 SerdeValue::Option(None) => Ok(Self::Null),
1081 SerdeValue::Option(Some(value)) | SerdeValue::Newtype(value) => {
1082 Self::from_serde_value(*value)
1083 }
1084 SerdeValue::Seq(values) => values
1085 .into_iter()
1086 .map(Self::from_serde_value)
1087 .collect::<Result<Vec<_>>>()
1088 .map(Self::Array),
1089 SerdeValue::Map(values) => values
1090 .into_iter()
1091 .map(|(key, value)| {
1092 let SerdeValue::String(key) = key else {
1093 return Err(Error::Codec(
1094 "invalid_map_key: Avro Value map keys must be strings".to_string(),
1095 ));
1096 };
1097
1098 Ok((key, Self::from_serde_value(value)?))
1099 })
1100 .collect::<Result<BTreeMap<_, _>>>()
1101 .map(Self::Map),
1102 }
1103 }
1104
1105 fn finite_double(value: f64) -> Result<Self> {
1106 if !value.is_finite() {
1107 return Err(Error::Codec(
1108 "non_finite_float: Avro Value doubles must be finite".to_string(),
1109 ));
1110 }
1111
1112 Ok(Self::Double(value))
1113 }
1114
1115 fn into_json(self) -> Result<Value> {
1116 match self {
1117 Self::Null => Ok(Value::Null),
1118 Self::Boolean(value) => Ok(Value::Bool(value)),
1119 Self::Long(value) => Ok(Value::Number(value.into())),
1120 Self::Double(value) => serde_json::Number::from_f64(value)
1121 .map(Value::Number)
1122 .ok_or_else(|| {
1123 Error::Codec(
1124 "non_finite_float: decoded Avro Value double is not finite".to_string(),
1125 )
1126 }),
1127 Self::Bytes(value) => Ok(json!({
1128 "$type": "bytes",
1129 "base64": BASE64.encode(value),
1130 })),
1131 Self::String(value) => Ok(Value::String(value)),
1132 Self::Array(values) => values
1133 .into_iter()
1134 .map(Self::into_json)
1135 .collect::<Result<Vec<_>>>()
1136 .map(Value::Array),
1137 Self::Map(values) => values
1138 .into_iter()
1139 .map(|(key, value)| Ok((key, value.into_json()?)))
1140 .collect::<Result<serde_json::Map<_, _>>>()
1141 .map(Value::Object),
1142 }
1143 }
1144
1145 fn into_serde_value(self) -> serde_value::Value {
1146 use serde_value::Value as SerdeValue;
1147
1148 match self {
1149 Self::Null => SerdeValue::Unit,
1150 Self::Boolean(value) => SerdeValue::Bool(value),
1151 Self::Long(value) => SerdeValue::I64(value),
1152 Self::Double(value) => SerdeValue::F64(value),
1153 Self::Bytes(value) => SerdeValue::Bytes(value),
1154 Self::String(value) => SerdeValue::String(value),
1155 Self::Array(values) => {
1156 SerdeValue::Seq(values.into_iter().map(Self::into_serde_value).collect())
1157 }
1158 Self::Map(values) => SerdeValue::Map(
1159 values
1160 .into_iter()
1161 .map(|(key, value)| (SerdeValue::String(key), value.into_serde_value()))
1162 .collect(),
1163 ),
1164 }
1165 }
1166
1167 pub fn deserialize<T: DeserializeOwned>(self) -> Result<T> {
1168 self.into_serde_value().deserialize_into().map_err(|error| {
1169 Error::Codec(format!(
1170 "avro_value_type_mismatch: could not adapt decoded value: {error}"
1171 ))
1172 })
1173 }
1174}
1175
1176impl Serialize for AvroValue {
1177 fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
1178 where
1179 S: Serializer,
1180 {
1181 match self {
1182 Self::Null => serializer.serialize_unit(),
1183 Self::Boolean(value) => serializer.serialize_bool(*value),
1184 Self::Long(value) => serializer.serialize_i64(*value),
1185 Self::Double(value) => serializer.serialize_f64(*value),
1186 Self::Bytes(value) => serializer.serialize_bytes(value),
1187 Self::String(value) => serializer.serialize_str(value),
1188 Self::Array(values) => {
1189 let mut sequence = serializer.serialize_seq(Some(values.len()))?;
1190 for value in values {
1191 sequence.serialize_element(value)?;
1192 }
1193 sequence.end()
1194 }
1195 Self::Map(values) => {
1196 let mut map = serializer.serialize_map(Some(values.len()))?;
1197 for (key, value) in values {
1198 map.serialize_entry(key, value)?;
1199 }
1200 map.end()
1201 }
1202 }
1203 }
1204}
1205
1206pub fn encode_avro_value(value: &AvroValue) -> Result<PayloadEnvelope> {
1207 let datum = avro_value_to_datum(value)?;
1208 let datum = to_avro_datum(avro_value_schema()?, datum)
1209 .map_err(|err| Error::Codec(format!("avro_value_encode_failed: {err}")))?;
1210 let mut bytes = Vec::with_capacity(datum.len() + 10);
1211 bytes.extend_from_slice(&AVRO_SINGLE_OBJECT_MAGIC);
1212 bytes.extend_from_slice(&AVRO_VALUE_SCHEMA_FINGERPRINT);
1213 bytes.extend_from_slice(&datum);
1214 Ok(PayloadEnvelope {
1215 codec: DEFAULT_CODEC.to_string(),
1216 blob: BASE64.encode(bytes),
1217 })
1218}
1219
1220pub fn decode_avro_value(envelope: &PayloadEnvelope) -> Result<AvroValue> {
1221 if envelope.codec != DEFAULT_CODEC {
1222 return Err(unsupported_payload_codec(&envelope.codec));
1223 }
1224 decode_avro_value_blob(&envelope.blob)
1225}
1226
1227pub fn encode_payload<T: Serialize>(value: &T, codec: &str) -> Result<PayloadEnvelope> {
1228 let blob = match codec {
1229 DEFAULT_CODEC => encode_avro_value(&AvroValue::from_serialize(value)?)?.blob,
1230 other => return Err(unsupported_payload_codec(other)),
1231 };
1232
1233 Ok(PayloadEnvelope {
1234 codec: codec.to_string(),
1235 blob,
1236 })
1237}
1238
1239pub fn decode_payload<T: DeserializeOwned>(envelope: &PayloadEnvelope) -> Result<T> {
1240 match envelope.codec.as_str() {
1241 DEFAULT_CODEC => decode_avro_value(envelope)?.deserialize(),
1242 other => Err(unsupported_payload_codec(other)),
1243 }
1244}
1245
1246#[cfg(test)]
1247fn encode_value_envelope(value: &Value, codec: &str) -> Result<Value> {
1248 Ok(serde_json::to_value(encode_payload(value, codec)?)?)
1249}
1250
1251fn decode_wire_value(value: &Value, fallback_codec: &str) -> Result<Value> {
1252 validate_payload_codec(fallback_codec)?;
1253
1254 if value.is_null() {
1255 return Ok(Value::Null);
1256 }
1257
1258 if let Some((codec, blob)) = payload_envelope_parts(value)? {
1259 return decode_blob(blob, codec);
1260 }
1261
1262 if let Some(blob) = value.as_str() {
1263 return decode_blob(blob, fallback_codec);
1264 }
1265
1266 Err(untagged_payload_value())
1267}
1268
1269fn encode_typed_envelope(value: &AvroValue, codec: &str) -> Result<Value> {
1270 let envelope = match codec {
1271 DEFAULT_CODEC => encode_avro_value(value)?,
1272 other => return Err(unsupported_payload_codec(other)),
1273 };
1274 Ok(serde_json::to_value(envelope)?)
1275}
1276
1277fn decode_wire_avro_value(value: &Value, fallback_codec: &str) -> Result<AvroValue> {
1278 validate_payload_codec(fallback_codec)?;
1279
1280 if value.is_null() {
1281 return Ok(AvroValue::Null);
1282 }
1283
1284 if let Some((codec, blob)) = payload_envelope_parts(value)? {
1285 validate_payload_codec(codec)?;
1286 return decode_avro_value_blob(blob);
1287 }
1288
1289 if let Some(blob) = value.as_str() {
1290 return match fallback_codec {
1291 DEFAULT_CODEC => decode_avro_value_blob(blob),
1292 other => Err(unsupported_payload_codec(other)),
1293 };
1294 }
1295
1296 Err(untagged_payload_value())
1297}
1298
1299fn normalize_avro_arguments(value: AvroValue) -> AvroValue {
1300 match value {
1301 AvroValue::Null => AvroValue::Array(Vec::new()),
1302 AvroValue::Array(_) => value,
1303 other => AvroValue::Array(vec![other]),
1304 }
1305}
1306
1307fn decode_blob(blob: &str, codec: &str) -> Result<Value> {
1308 match codec {
1309 DEFAULT_CODEC => decode_avro_value_blob(blob)?.into_json(),
1310 other => Err(unsupported_payload_codec(other)),
1311 }
1312}
1313
1314fn validate_payload_codec(codec: &str) -> Result<()> {
1315 match codec {
1316 DEFAULT_CODEC => Ok(()),
1317 MISSING_TASK_PAYLOAD_CODEC => {
1318 Err(invalid_task_payload_codec("task payload_codec is missing"))
1319 }
1320 NULL_TASK_PAYLOAD_CODEC => Err(invalid_task_payload_codec("task payload_codec is null")),
1321 NON_STRING_TASK_PAYLOAD_CODEC => Err(invalid_task_payload_codec(
1322 "task payload_codec must be a string",
1323 )),
1324 other => Err(unsupported_payload_codec(other)),
1325 }
1326}
1327
1328fn invalid_task_payload_codec(reason: &str) -> Error {
1329 Error::Codec(format!(
1330 "unsupported_payload_codec: {reason}; Durable Workflow 2.0 requires an explicit string payload_codec=\"avro\" before worker task execution"
1331 ))
1332}
1333
1334fn payload_envelope_parts(value: &Value) -> Result<Option<(&str, &str)>> {
1335 let Some(object) = value.as_object() else {
1336 return Ok(None);
1337 };
1338 if !object.contains_key("codec") && !object.contains_key("blob") {
1339 return Ok(None);
1340 }
1341
1342 let codec = object
1343 .get("codec")
1344 .and_then(Value::as_str)
1345 .ok_or_else(invalid_payload_envelope)?;
1346 validate_payload_codec(codec)?;
1347 let blob = object
1348 .get("blob")
1349 .and_then(Value::as_str)
1350 .ok_or_else(invalid_payload_envelope)?;
1351 Ok(Some((codec, blob)))
1352}
1353
1354fn invalid_payload_envelope() -> Error {
1355 Error::Codec(
1356 "invalid_payload_envelope: durable payloads must use an object with string codec=\"avro\" and blob fields"
1357 .to_string(),
1358 )
1359}
1360
1361fn validate_workflow_task_commands(commands: &[Value]) -> Result<()> {
1362 for command in commands {
1363 let Some(command) = command.as_object() else {
1364 continue;
1365 };
1366 let Some(command_type) = command.get("type").and_then(Value::as_str) else {
1367 continue;
1368 };
1369 let Some(payload_field) = workflow_command_payload_field(command_type) else {
1370 continue;
1371 };
1372
1373 if let Some(codec) = command.get("payload_codec") {
1374 let codec = codec.as_str().ok_or_else(invalid_payload_envelope)?;
1375 validate_payload_codec(codec)?;
1376 }
1377
1378 let payload = command
1379 .get(payload_field)
1380 .ok_or_else(invalid_payload_envelope)?;
1381 validate_outbound_payload_envelope(payload)?;
1382 }
1383 Ok(())
1384}
1385
1386fn workflow_command_payload_field(command_type: &str) -> Option<&'static str> {
1387 match command_type {
1388 "complete_workflow" | "complete_update" | "record_side_effect" => Some("result"),
1389 "schedule_activity" | "start_child_workflow" | "continue_as_new" => Some("arguments"),
1390 "start_service_operation" => Some("request_payload"),
1391 _ => None,
1392 }
1393}
1394
1395fn validate_outbound_payload_envelope(value: &Value) -> Result<()> {
1396 let Some((codec, blob)) = payload_envelope_parts(value)? else {
1397 return Err(untagged_payload_value());
1398 };
1399 validate_payload_codec(codec)?;
1400 decode_avro_value_blob(blob)?;
1401 Ok(())
1402}
1403
1404fn unsupported_payload_codec(codec: &str) -> Error {
1405 Error::Codec(format!(
1406 "unsupported_payload_codec: workflow payload codec {codec:?} is not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec"
1407 ))
1408}
1409
1410fn untagged_payload_value() -> Error {
1411 Error::Codec(
1412 "unsupported_payload_codec: untagged durable payload values are not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec"
1413 .to_string(),
1414 )
1415}
1416
1417fn decode_avro_value_blob(blob: &str) -> Result<AvroValue> {
1418 let bytes = BASE64.decode(blob).map_err(|err| {
1419 Error::Codec(format!(
1420 "invalid_payload_framing: expected strict base64 Avro single-object bytes: {err}"
1421 ))
1422 })?;
1423
1424 if serde_json::from_slice::<Value>(&bytes).is_ok() {
1425 return Err(unsupported_payload_codec("json"));
1426 }
1427
1428 if bytes.len() < 10 || bytes[..2] != AVRO_SINGLE_OBJECT_MAGIC {
1429 return Err(Error::Codec(
1430 "invalid_payload_framing: expected Avro single-object magic c301".to_string(),
1431 ));
1432 }
1433
1434 let fingerprint: [u8; 8] = bytes[2..10]
1435 .try_into()
1436 .map_err(|_| Error::Codec("invalid Avro fingerprint length".to_string()))?;
1437 if fingerprint != AVRO_VALUE_SCHEMA_FINGERPRINT {
1438 return Err(Error::Codec(format!(
1439 "unsupported_payload_schema: unknown CRC-64-AVRO fingerprint {}",
1440 fingerprint
1441 .iter()
1442 .map(|byte| format!("{byte:02x}"))
1443 .collect::<String>()
1444 )));
1445 }
1446
1447 let mut datum_reader = StrictAvroDatumReader::new(&bytes[10..]);
1448 let datum = from_avro_datum(avro_value_schema()?, &mut datum_reader, None);
1452 if datum_reader.truncated {
1453 return Err(Error::Codec(
1454 "invalid_payload_framing: truncated Avro Value datum".to_string(),
1455 ));
1456 }
1457 let datum = datum.map_err(|err| {
1458 Error::Codec(format!(
1459 "invalid_payload_framing: malformed Avro Value datum: {err}"
1460 ))
1461 })?;
1462 if datum_reader.remaining() != 0 {
1463 return Err(Error::Codec(format!(
1464 "invalid_payload_framing: {} trailing bytes after Avro Value datum",
1465 datum_reader.remaining()
1466 )));
1467 }
1468 avro_value_from_datum(datum)
1469}
1470
1471struct StrictAvroDatumReader<'a> {
1472 bytes: &'a [u8],
1473 offset: usize,
1474 truncated: bool,
1475}
1476
1477impl<'a> StrictAvroDatumReader<'a> {
1478 fn new(bytes: &'a [u8]) -> Self {
1479 Self {
1480 bytes,
1481 offset: 0,
1482 truncated: false,
1483 }
1484 }
1485
1486 fn remaining(&self) -> usize {
1487 self.bytes.len() - self.offset
1488 }
1489}
1490
1491impl Read for StrictAvroDatumReader<'_> {
1492 fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
1493 let count = buffer.len().min(self.remaining());
1494 buffer[..count].copy_from_slice(&self.bytes[self.offset..self.offset + count]);
1495 self.offset += count;
1496 if count < buffer.len() {
1497 self.truncated = true;
1498 }
1499
1500 Ok(count)
1501 }
1502}
1503
1504fn avro_value_to_datum(value: &AvroValue) -> Result<AvroDatum> {
1505 let branch = match value {
1506 AvroValue::Null => AvroDatum::Union(0, Box::new(AvroDatum::Null)),
1507 AvroValue::Boolean(value) => AvroDatum::Union(
1508 1,
1509 Box::new(AvroDatum::Record(vec![(
1510 "boolean".to_string(),
1511 AvroDatum::Boolean(*value),
1512 )])),
1513 ),
1514 AvroValue::Long(value) => AvroDatum::Union(
1515 2,
1516 Box::new(AvroDatum::Record(vec![(
1517 "long".to_string(),
1518 AvroDatum::Long(*value),
1519 )])),
1520 ),
1521 AvroValue::Double(value) => {
1522 if !value.is_finite() {
1523 return Err(Error::Codec(
1524 "non_finite_float: Avro Value doubles must be finite".to_string(),
1525 ));
1526 }
1527 AvroDatum::Union(
1528 3,
1529 Box::new(AvroDatum::Record(vec![(
1530 "double".to_string(),
1531 AvroDatum::Double(*value),
1532 )])),
1533 )
1534 }
1535 AvroValue::Bytes(value) => AvroDatum::Union(
1536 4,
1537 Box::new(AvroDatum::Record(vec![(
1538 "bytes".to_string(),
1539 AvroDatum::Bytes(value.clone()),
1540 )])),
1541 ),
1542 AvroValue::String(value) => AvroDatum::Union(
1543 5,
1544 Box::new(AvroDatum::Record(vec![(
1545 "string".to_string(),
1546 AvroDatum::String(value.clone()),
1547 )])),
1548 ),
1549 AvroValue::Array(values) => AvroDatum::Union(
1550 6,
1551 Box::new(AvroDatum::Record(vec![(
1552 "items".to_string(),
1553 AvroDatum::Array(
1554 values
1555 .iter()
1556 .map(avro_value_to_datum)
1557 .collect::<Result<Vec<_>>>()?,
1558 ),
1559 )])),
1560 ),
1561 AvroValue::Map(values) => AvroDatum::Union(
1562 7,
1563 Box::new(AvroDatum::Record(vec![(
1564 "entries".to_string(),
1565 AvroDatum::Map(
1566 values
1567 .iter()
1568 .map(|(key, value)| Ok((key.clone(), avro_value_to_datum(value)?)))
1569 .collect::<Result<HashMap<_, _>>>()?,
1570 ),
1571 )])),
1572 ),
1573 };
1574 Ok(AvroDatum::Record(vec![("value".to_string(), branch)]))
1575}
1576
1577fn avro_value_from_datum(datum: AvroDatum) -> Result<AvroValue> {
1578 let AvroDatum::Record(mut outer) = datum else {
1579 return Err(Error::Codec(
1580 "invalid_payload_framing: datum is not a Value record".to_string(),
1581 ));
1582 };
1583 let (_, branch) = outer
1584 .pop()
1585 .filter(|(name, _)| name == "value")
1586 .ok_or_else(|| Error::Codec("invalid_payload_framing: Value field missing".to_string()))?;
1587 let AvroDatum::Union(_, branch) = branch else {
1588 return Err(Error::Codec(
1589 "invalid_payload_framing: invalid Value union".to_string(),
1590 ));
1591 };
1592 match *branch {
1593 AvroDatum::Null => Ok(AvroValue::Null),
1594 AvroDatum::Record(mut fields) => {
1595 let (name, value) = fields.pop().ok_or_else(|| {
1596 Error::Codec("invalid_payload_framing: empty Value branch".to_string())
1597 })?;
1598 match (name.as_str(), value) {
1599 ("boolean", AvroDatum::Boolean(value)) => Ok(AvroValue::Boolean(value)),
1600 ("long", AvroDatum::Long(value)) => Ok(AvroValue::Long(value)),
1601 ("double", AvroDatum::Double(value)) if value.is_finite() => {
1602 Ok(AvroValue::Double(value))
1603 }
1604 ("bytes", AvroDatum::Bytes(value)) => Ok(AvroValue::Bytes(value)),
1605 ("string", AvroDatum::String(value)) => Ok(AvroValue::String(value)),
1606 ("items", AvroDatum::Array(values)) => values
1607 .into_iter()
1608 .map(avro_value_from_datum)
1609 .collect::<Result<Vec<_>>>()
1610 .map(AvroValue::Array),
1611 ("entries", AvroDatum::Map(values)) => values
1612 .into_iter()
1613 .map(|(key, value)| Ok((key, avro_value_from_datum(value)?)))
1614 .collect::<Result<BTreeMap<_, _>>>()
1615 .map(AvroValue::Map),
1616 _ => Err(Error::Codec(
1617 "invalid_payload_framing: unknown Value branch".to_string(),
1618 )),
1619 }
1620 }
1621 _ => Err(Error::Codec(
1622 "invalid_payload_framing: invalid Value branch".to_string(),
1623 )),
1624 }
1625}
1626
1627fn avro_value_schema() -> Result<&'static Schema> {
1628 match AVRO_VALUE_SCHEMA.get_or_init(|| {
1629 Schema::parse_str(AVRO_VALUE_SCHEMA_JSON)
1630 .map_err(|err| format!("could not parse Avro Value schema: {err}"))
1631 }) {
1632 Ok(schema) => Ok(schema),
1633 Err(message) => Err(Error::Codec(message.clone())),
1634 }
1635}
1636
1637#[derive(Clone, Debug)]
1638pub struct Client {
1639 http: reqwest::Client,
1640 base_url: String,
1641 token: Option<String>,
1642 control_token: Option<String>,
1643 worker_token: Option<String>,
1644 namespace: String,
1645}
1646
1647impl Client {
1648 pub fn new(base_url: impl Into<String>) -> Result<Self> {
1649 Self::builder(base_url).build()
1650 }
1651
1652 pub fn builder(base_url: impl Into<String>) -> ClientBuilder {
1653 ClientBuilder {
1654 base_url: base_url.into(),
1655 token: None,
1656 control_token: None,
1657 worker_token: None,
1658 namespace: "default".to_string(),
1659 timeout: Duration::from_secs(60),
1660 }
1661 }
1662
1663 pub async fn health(&self) -> Result<Value> {
1664 self.request_json(
1665 reqwest::Method::GET,
1666 "/health",
1667 RequestProtocol::ControlPlane,
1668 Option::<&Value>::None,
1669 )
1670 .await
1671 }
1672
1673 pub async fn cluster_info(&self) -> Result<Value> {
1674 self.request_json(
1675 reqwest::Method::GET,
1676 "/cluster/info",
1677 RequestProtocol::ControlPlane,
1678 Option::<&Value>::None,
1679 )
1680 .await
1681 }
1682
1683 pub async fn start_workflow<T: Serialize>(
1684 &self,
1685 workflow_type: &str,
1686 task_queue: &str,
1687 workflow_id: &str,
1688 input: T,
1689 ) -> Result<WorkflowHandle> {
1690 self.start_workflow_with_options(
1691 workflow_type,
1692 task_queue,
1693 workflow_id,
1694 WorkflowStartOptions::default(),
1695 input,
1696 )
1697 .await
1698 }
1699
1700 pub async fn start_workflow_with_options<T: Serialize>(
1703 &self,
1704 workflow_type: &str,
1705 task_queue: &str,
1706 workflow_id: &str,
1707 options: WorkflowStartOptions,
1708 input: T,
1709 ) -> Result<WorkflowHandle> {
1710 options.validate()?;
1711 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1712 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
1713 let body = json!({
1714 "workflow_id": workflow_id,
1715 "workflow_type": workflow_type,
1716 "task_queue": task_queue,
1717 "input": input_envelope,
1718 "execution_timeout_seconds": options.execution_timeout_seconds,
1719 "run_timeout_seconds": options.run_timeout_seconds
1720 });
1721
1722 let data: Value = self
1723 .request_json(
1724 reqwest::Method::POST,
1725 "/workflows",
1726 RequestProtocol::ControlPlane,
1727 Some(&body),
1728 )
1729 .await?;
1730
1731 Ok(WorkflowHandle {
1732 client: self.clone(),
1733 workflow_id: data
1734 .get("workflow_id")
1735 .and_then(Value::as_str)
1736 .unwrap_or(workflow_id)
1737 .to_string(),
1738 run_id: data
1739 .get("run_id")
1740 .and_then(Value::as_str)
1741 .map(str::to_string),
1742 workflow_type: data
1743 .get("workflow_type")
1744 .and_then(Value::as_str)
1745 .unwrap_or(workflow_type)
1746 .to_string(),
1747 })
1748 }
1749
1750 pub async fn signal_workflow<T: Serialize>(
1751 &self,
1752 workflow_id: &str,
1753 signal_name: &str,
1754 input: T,
1755 ) -> Result<Value> {
1756 self.signal_workflow_target(workflow_id, None, signal_name, input)
1757 .await
1758 }
1759
1760 pub async fn signal_workflow_run<T: Serialize>(
1762 &self,
1763 workflow_id: &str,
1764 run_id: &str,
1765 signal_name: &str,
1766 input: T,
1767 ) -> Result<Value> {
1768 self.signal_workflow_target(workflow_id, Some(run_id), signal_name, input)
1769 .await
1770 }
1771
1772 async fn signal_workflow_target<T: Serialize>(
1773 &self,
1774 workflow_id: &str,
1775 run_id: Option<&str>,
1776 signal_name: &str,
1777 input: T,
1778 ) -> Result<Value> {
1779 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1780 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
1781 let body = json!({
1782 "input": input_envelope
1783 });
1784 let path = match run_id {
1785 Some(run_id) => {
1786 format!("/workflows/{workflow_id}/runs/{run_id}/signal/{signal_name}")
1787 }
1788 None => format!("/workflows/{workflow_id}/signal/{signal_name}"),
1789 };
1790 self.request_json(
1791 reqwest::Method::POST,
1792 &path,
1793 RequestProtocol::ControlPlane,
1794 Some(&body),
1795 )
1796 .await
1797 }
1798
1799 pub async fn cancel_workflow(
1801 &self,
1802 workflow_id: &str,
1803 options: WorkflowCommandOptions,
1804 ) -> Result<WorkflowCommandResult> {
1805 self.workflow_command(workflow_id, None, WorkflowCommandKind::Cancel, options)
1806 .await
1807 }
1808
1809 pub async fn cancel_workflow_run(
1811 &self,
1812 workflow_id: &str,
1813 run_id: &str,
1814 options: WorkflowCommandOptions,
1815 ) -> Result<WorkflowCommandResult> {
1816 self.workflow_command(
1817 workflow_id,
1818 Some(run_id),
1819 WorkflowCommandKind::Cancel,
1820 options,
1821 )
1822 .await
1823 }
1824
1825 pub async fn terminate_workflow(
1827 &self,
1828 workflow_id: &str,
1829 options: WorkflowCommandOptions,
1830 ) -> Result<WorkflowCommandResult> {
1831 self.workflow_command(workflow_id, None, WorkflowCommandKind::Terminate, options)
1832 .await
1833 }
1834
1835 pub async fn terminate_workflow_run(
1837 &self,
1838 workflow_id: &str,
1839 run_id: &str,
1840 options: WorkflowCommandOptions,
1841 ) -> Result<WorkflowCommandResult> {
1842 self.workflow_command(
1843 workflow_id,
1844 Some(run_id),
1845 WorkflowCommandKind::Terminate,
1846 options,
1847 )
1848 .await
1849 }
1850
1851 async fn workflow_command(
1852 &self,
1853 workflow_id: &str,
1854 run_id: Option<&str>,
1855 command: WorkflowCommandKind,
1856 options: WorkflowCommandOptions,
1857 ) -> Result<WorkflowCommandResult> {
1858 let path = match run_id {
1859 Some(run_id) => format!(
1860 "/workflows/{workflow_id}/runs/{run_id}/{}",
1861 command.as_str()
1862 ),
1863 None => format!("/workflows/{workflow_id}/{}", command.as_str()),
1864 };
1865 let data = match self
1866 .request_json(
1867 reqwest::Method::POST,
1868 &path,
1869 RequestProtocol::ControlPlane,
1870 Some(&options),
1871 )
1872 .await
1873 {
1874 Ok(data) => data,
1875 Err(Error::Http { status, body }) => {
1876 return Err(Error::WorkflowCommandRejected(workflow_command_rejection(
1877 command,
1878 status,
1879 body,
1880 workflow_id,
1881 run_id,
1882 )));
1883 }
1884 Err(error) => return Err(error),
1885 };
1886
1887 Ok(workflow_command_result(command, data, workflow_id, run_id))
1888 }
1889
1890 pub async fn query_workflow<T: Serialize>(
1896 &self,
1897 workflow_id: &str,
1898 query_name: &str,
1899 input: T,
1900 ) -> Result<Value> {
1901 self.query_workflow_target(workflow_id, None, query_name, input)
1902 .await
1903 }
1904
1905 pub async fn query_workflow_run<T: Serialize>(
1907 &self,
1908 workflow_id: &str,
1909 run_id: &str,
1910 query_name: &str,
1911 input: T,
1912 ) -> Result<Value> {
1913 self.query_workflow_target(workflow_id, Some(run_id), query_name, input)
1914 .await
1915 }
1916
1917 pub async fn query_workflow_avro_value<T: Serialize>(
1919 &self,
1920 workflow_id: &str,
1921 query_name: &str,
1922 input: T,
1923 ) -> Result<AvroValue> {
1924 self.query_workflow_avro_value_target(workflow_id, None, query_name, input)
1925 .await
1926 }
1927
1928 pub async fn query_workflow_run_avro_value<T: Serialize>(
1930 &self,
1931 workflow_id: &str,
1932 run_id: &str,
1933 query_name: &str,
1934 input: T,
1935 ) -> Result<AvroValue> {
1936 self.query_workflow_avro_value_target(workflow_id, Some(run_id), query_name, input)
1937 .await
1938 }
1939
1940 async fn query_workflow_avro_value_target<T: Serialize>(
1941 &self,
1942 workflow_id: &str,
1943 run_id: Option<&str>,
1944 query_name: &str,
1945 input: T,
1946 ) -> Result<AvroValue> {
1947 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1948 let body = json!({"input": encode_typed_envelope(&input, DEFAULT_CODEC)?});
1949 let path = match run_id {
1950 Some(run_id) => {
1951 format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
1952 }
1953 None => format!("/workflows/{workflow_id}/query/{query_name}"),
1954 };
1955 let response: Value = match self
1956 .request_json(
1957 reqwest::Method::POST,
1958 &path,
1959 RequestProtocol::ControlPlane,
1960 Some(&body),
1961 )
1962 .await
1963 {
1964 Ok(response) => response,
1965 Err(Error::Http { status, body }) => {
1966 return Err(Error::QueryFailed(query_failure(status, body)));
1967 }
1968 Err(error) => return Err(error),
1969 };
1970
1971 let envelope = response
1972 .get("result_envelope")
1973 .filter(|envelope| !envelope.is_null())
1974 .ok_or_else(|| {
1975 Error::Codec(
1976 "missing_payload_envelope: typed query result requires result_envelope"
1977 .to_string(),
1978 )
1979 })?;
1980 decode_wire_avro_value(envelope, DEFAULT_CODEC)
1981 }
1982
1983 async fn query_workflow_target<T: Serialize>(
1984 &self,
1985 workflow_id: &str,
1986 run_id: Option<&str>,
1987 query_name: &str,
1988 input: T,
1989 ) -> Result<Value> {
1990 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1991 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
1992 let body = json!({
1993 "input": input_envelope
1994 });
1995 let path = match run_id {
1996 Some(run_id) => {
1997 format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
1998 }
1999 None => format!("/workflows/{workflow_id}/query/{query_name}"),
2000 };
2001 let response: Value = match self
2002 .request_json(
2003 reqwest::Method::POST,
2004 &path,
2005 RequestProtocol::ControlPlane,
2006 Some(&body),
2007 )
2008 .await
2009 {
2010 Ok(response) => response,
2011 Err(Error::Http { status, body }) => {
2012 return Err(Error::QueryFailed(query_failure(status, body)));
2013 }
2014 Err(error) => return Err(error),
2015 };
2016
2017 if let Some(envelope) = response
2018 .get("result_envelope")
2019 .filter(|envelope| !envelope.is_null())
2020 {
2021 return decode_wire_value(envelope, DEFAULT_CODEC);
2022 }
2023
2024 Ok(response.get("result").cloned().unwrap_or(Value::Null))
2025 }
2026
2027 pub async fn update_workflow<T: Serialize>(
2029 &self,
2030 workflow_id: &str,
2031 update_name: &str,
2032 input: T,
2033 request_id: Option<&str>,
2034 ) -> Result<Value> {
2035 let response = self
2036 .update_workflow_response(workflow_id, update_name, input, request_id)
2037 .await?;
2038 if let Some(envelope) = response
2039 .get("result_envelope")
2040 .filter(|envelope| !envelope.is_null())
2041 {
2042 return decode_wire_value(envelope, DEFAULT_CODEC);
2043 }
2044 Ok(response.get("result").cloned().unwrap_or(response))
2045 }
2046
2047 pub async fn update_workflow_avro_value<T: Serialize>(
2049 &self,
2050 workflow_id: &str,
2051 update_name: &str,
2052 input: T,
2053 request_id: Option<&str>,
2054 ) -> Result<AvroValue> {
2055 let response = self
2056 .update_workflow_response(workflow_id, update_name, input, request_id)
2057 .await?;
2058 let envelope = response
2059 .get("result_envelope")
2060 .filter(|envelope| !envelope.is_null())
2061 .ok_or_else(|| {
2062 Error::Codec(
2063 "missing_payload_envelope: typed update result requires result_envelope"
2064 .to_string(),
2065 )
2066 })?;
2067 decode_wire_avro_value(envelope, DEFAULT_CODEC)
2068 }
2069
2070 async fn update_workflow_response<T: Serialize>(
2071 &self,
2072 workflow_id: &str,
2073 update_name: &str,
2074 input: T,
2075 request_id: Option<&str>,
2076 ) -> Result<Value> {
2077 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2078 let mut body = json!({
2079 "input": encode_typed_envelope(&input, DEFAULT_CODEC)?,
2080 "wait_for": "completed",
2081 });
2082 if let Some(request_id) = request_id {
2083 body["request_id"] = json!(request_id);
2084 }
2085 self.request_json(
2086 reqwest::Method::POST,
2087 &format!("/workflows/{workflow_id}/update/{update_name}"),
2088 RequestProtocol::ControlPlane,
2089 Some(&body),
2090 )
2091 .await
2092 }
2093
2094 pub async fn describe_workflow(&self, workflow_id: &str) -> Result<WorkflowDescription> {
2095 let path = format!("/workflows/{workflow_id}");
2096 let mut data: WorkflowDescription = self
2097 .request_json(
2098 reqwest::Method::GET,
2099 &path,
2100 RequestProtocol::ControlPlane,
2101 Option::<&Value>::None,
2102 )
2103 .await?;
2104 data.decode_payloads()?;
2105 Ok(data)
2106 }
2107
2108 pub async fn describe_workflow_run(
2110 &self,
2111 workflow_id: &str,
2112 run_id: &str,
2113 ) -> Result<WorkflowDescription> {
2114 let path = format!("/workflows/{workflow_id}/runs/{run_id}");
2115 let mut data: WorkflowDescription = self
2116 .request_json(
2117 reqwest::Method::GET,
2118 &path,
2119 RequestProtocol::ControlPlane,
2120 Option::<&Value>::None,
2121 )
2122 .await?;
2123 data.decode_payloads()?;
2124 Ok(data)
2125 }
2126
2127 pub async fn register_worker(
2128 &self,
2129 worker_id: &str,
2130 task_queue: &str,
2131 supported_workflow_types: Vec<String>,
2132 supported_activity_types: Vec<String>,
2133 max_concurrent_workflow_tasks: usize,
2134 max_concurrent_activity_tasks: usize,
2135 ) -> Result<RegisterWorkerResponse> {
2136 self.register_worker_with_capabilities(
2137 worker_id,
2138 task_queue,
2139 supported_workflow_types,
2140 supported_activity_types,
2141 max_concurrent_workflow_tasks,
2142 max_concurrent_activity_tasks,
2143 Vec::new(),
2144 )
2145 .await
2146 }
2147
2148 pub async fn register_worker_with_capabilities(
2150 &self,
2151 worker_id: &str,
2152 task_queue: &str,
2153 supported_workflow_types: Vec<String>,
2154 supported_activity_types: Vec<String>,
2155 max_concurrent_workflow_tasks: usize,
2156 max_concurrent_activity_tasks: usize,
2157 capabilities: Vec<String>,
2158 ) -> Result<RegisterWorkerResponse> {
2159 self.register_worker_with_command_contracts(
2160 worker_id,
2161 task_queue,
2162 supported_workflow_types,
2163 supported_activity_types,
2164 max_concurrent_workflow_tasks,
2165 max_concurrent_activity_tasks,
2166 capabilities,
2167 Value::Object(serde_json::Map::new()),
2168 )
2169 .await
2170 }
2171
2172 #[allow(clippy::too_many_arguments)]
2179 pub async fn register_worker_with_command_contracts(
2180 &self,
2181 worker_id: &str,
2182 task_queue: &str,
2183 supported_workflow_types: Vec<String>,
2184 supported_activity_types: Vec<String>,
2185 max_concurrent_workflow_tasks: usize,
2186 max_concurrent_activity_tasks: usize,
2187 capabilities: Vec<String>,
2188 workflow_command_contracts: Value,
2189 ) -> Result<RegisterWorkerResponse> {
2190 if let Some(contracts) = workflow_command_contracts.as_object() {
2191 for (workflow_type, contract) in contracts {
2192 let Some(update_validators) = contract.get("update_validators") else {
2193 continue;
2194 };
2195 if !update_validators
2196 .as_array()
2197 .is_some_and(|validators| validators.is_empty())
2198 {
2199 return Err(Error::UnsupportedUpdateValidators {
2200 workflow_type: workflow_type.clone(),
2201 });
2202 }
2203 }
2204 }
2205
2206 let mut body = json!({
2207 "worker_id": worker_id,
2208 "task_queue": task_queue,
2209 "runtime": "rust",
2210 "sdk_version": SDK_VERSION,
2211 "supported_workflow_types": supported_workflow_types,
2212 "supported_activity_types": supported_activity_types,
2213 "capabilities": capabilities,
2214 "max_concurrent_workflow_tasks": max_concurrent_workflow_tasks,
2215 "max_concurrent_activity_tasks": max_concurrent_activity_tasks
2216 });
2217 if workflow_command_contracts
2218 .as_object()
2219 .is_some_and(|contracts| !contracts.is_empty())
2220 {
2221 body["workflow_command_contracts"] = workflow_command_contracts;
2222 }
2223
2224 self.request_json(
2225 reqwest::Method::POST,
2226 "/worker/register",
2227 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2228 Some(&body),
2229 )
2230 .await
2231 }
2232
2233 pub async fn deregister_worker_registration(
2239 &self,
2240 worker_id: &str,
2241 ) -> Result<WorkerDeregistrationEnvelope> {
2242 let path = format!(
2243 "/worker/registrations/{}",
2244 percent_encode_path_segment(worker_id)
2245 );
2246 self.request_json(
2247 reqwest::Method::DELETE,
2248 &path,
2249 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2250 Option::<&Value>::None,
2251 )
2252 .await
2253 }
2254
2255 pub async fn poll_query_task(
2257 &self,
2258 worker_id: &str,
2259 task_queue: &str,
2260 timeout: Duration,
2261 ) -> Result<Option<QueryTask>> {
2262 Ok(self
2263 .poll_query_task_response(worker_id, task_queue, timeout)
2264 .await?
2265 .task)
2266 }
2267
2268 pub async fn poll_query_task_response(
2270 &self,
2271 worker_id: &str,
2272 task_queue: &str,
2273 timeout: Duration,
2274 ) -> Result<PollQueryTaskResponse> {
2275 let poll_request_id = unique_request_id("rust-query-poll");
2276 self.poll_query_task_response_with_request_id(
2277 worker_id,
2278 task_queue,
2279 timeout,
2280 &poll_request_id,
2281 1,
2282 )
2283 .await
2284 }
2285
2286 async fn poll_query_task_response_with_request_id(
2287 &self,
2288 worker_id: &str,
2289 task_queue: &str,
2290 timeout: Duration,
2291 poll_request_id: &str,
2292 transport_retries: usize,
2293 ) -> Result<PollQueryTaskResponse> {
2294 let timeout_seconds = long_poll_timeout_seconds(timeout);
2295 let body = json!({
2296 "worker_id": worker_id,
2297 "task_queue": task_queue,
2298 "poll_request_id": poll_request_id,
2299 "timeout_seconds": timeout_seconds,
2300 });
2301 self.poll_request_json(
2302 "/worker/query-tasks/poll",
2303 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
2304 &body,
2305 timeout + Duration::from_secs(5),
2306 transport_retries,
2307 )
2308 .await
2309 }
2310
2311 pub async fn complete_query_task<T: Serialize>(
2313 &self,
2314 query_task_id: &str,
2315 lease_owner: &str,
2316 query_task_attempt: u64,
2317 result: T,
2318 codec: &str,
2319 ) -> Result<Value> {
2320 let typed_result = AvroValue::from_serialize(&result)?;
2321 let result_envelope = encode_typed_envelope(&typed_result, codec)?;
2322 self.complete_query_task_with_envelope(
2323 query_task_id,
2324 lease_owner,
2325 query_task_attempt,
2326 typed_result.into_json()?,
2327 result_envelope,
2328 )
2329 .await
2330 }
2331
2332 async fn complete_query_task_with_envelope(
2333 &self,
2334 query_task_id: &str,
2335 lease_owner: &str,
2336 query_task_attempt: u64,
2337 result: Value,
2338 result_envelope: Value,
2339 ) -> Result<Value> {
2340 let body = json!({
2341 "lease_owner": lease_owner,
2342 "query_task_attempt": query_task_attempt,
2343 "result": result,
2344 "result_envelope": result_envelope,
2345 });
2346 let path = format!("/worker/query-tasks/{query_task_id}/complete");
2347 let response = self
2348 .request_json(
2349 reqwest::Method::POST,
2350 &path,
2351 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
2352 Some(&body),
2353 )
2354 .await;
2355 query_task_response(response)
2356 }
2357
2358 pub async fn fail_query_task(
2360 &self,
2361 query_task_id: &str,
2362 lease_owner: &str,
2363 query_task_attempt: u64,
2364 message: impl Into<String>,
2365 reason: impl Into<String>,
2366 failure_type: impl Into<String>,
2367 ) -> Result<Value> {
2368 let body = json!({
2369 "lease_owner": lease_owner,
2370 "query_task_attempt": query_task_attempt,
2371 "failure": {
2372 "message": message.into(),
2373 "reason": reason.into(),
2374 "type": failure_type.into(),
2375 }
2376 });
2377 let path = format!("/worker/query-tasks/{query_task_id}/fail");
2378 let response = self
2379 .request_json(
2380 reqwest::Method::POST,
2381 &path,
2382 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
2383 Some(&body),
2384 )
2385 .await;
2386 query_task_response(response)
2387 }
2388
2389 pub async fn heartbeat_worker(
2390 &self,
2391 worker_id: &str,
2392 workflow_available: usize,
2393 activity_available: usize,
2394 ) -> Result<Value> {
2395 let body = json!({
2396 "worker_id": worker_id,
2397 "task_slots": {
2398 "workflow_available": workflow_available,
2399 "activity_available": activity_available
2400 },
2401 "process_metrics": {
2402 "process_id": std::process::id(),
2403 "process_uptime_seconds": 0
2404 }
2405 });
2406
2407 self.request_json(
2408 reqwest::Method::POST,
2409 "/worker/heartbeat",
2410 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2411 Some(&body),
2412 )
2413 .await
2414 }
2415
2416 pub async fn poll_workflow_task(
2417 &self,
2418 worker_id: &str,
2419 task_queue: &str,
2420 timeout: Duration,
2421 ) -> Result<Option<WorkflowTask>> {
2422 Ok(self
2423 .poll_workflow_task_response(worker_id, task_queue, timeout)
2424 .await?
2425 .task)
2426 }
2427
2428 pub async fn poll_workflow_task_response(
2429 &self,
2430 worker_id: &str,
2431 task_queue: &str,
2432 timeout: Duration,
2433 ) -> Result<PollWorkflowTaskResponse> {
2434 let poll_request_id = unique_request_id("rust-workflow-poll");
2435 self.poll_workflow_task_response_with_request_id(
2436 worker_id,
2437 task_queue,
2438 timeout,
2439 &poll_request_id,
2440 1,
2441 )
2442 .await
2443 }
2444
2445 async fn poll_workflow_task_response_with_request_id(
2446 &self,
2447 worker_id: &str,
2448 task_queue: &str,
2449 timeout: Duration,
2450 poll_request_id: &str,
2451 transport_retries: usize,
2452 ) -> Result<PollWorkflowTaskResponse> {
2453 let body = json!({
2454 "worker_id": worker_id,
2455 "task_queue": task_queue,
2456 "poll_request_id": poll_request_id,
2457 "timeout_seconds": long_poll_timeout_seconds(timeout),
2458 });
2459 let mut data: PollWorkflowTaskResponse = self
2460 .poll_request_json(
2461 "/worker/workflow-tasks/poll",
2462 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2463 &body,
2464 timeout + Duration::from_secs(5),
2465 transport_retries,
2466 )
2467 .await?;
2468
2469 if let Some(task) = data.task.as_mut() {
2470 self.fetch_remaining_workflow_history(worker_id, task)
2471 .await?;
2472 }
2473
2474 Ok(data)
2475 }
2476
2477 async fn fetch_remaining_workflow_history(
2478 &self,
2479 worker_id: &str,
2480 task: &mut WorkflowTask,
2481 ) -> Result<()> {
2482 let mut next_token = task.next_history_page_token.clone();
2483
2484 while let Some(token) = next_token.take().filter(|token| !token.is_empty()) {
2485 let lease_owner = task
2486 .lease_owner
2487 .clone()
2488 .unwrap_or_else(|| worker_id.to_string());
2489 let page = self
2490 .workflow_task_history_page(
2491 &task.task_id,
2492 &lease_owner,
2493 task.workflow_task_attempt,
2494 &token,
2495 )
2496 .await?;
2497
2498 task.append_history_page(page);
2499
2500 if task.next_history_page_token.as_deref() == Some(token.as_str()) {
2501 return Err(Error::Codec(
2502 "workflow history pagination returned the same page token".to_string(),
2503 ));
2504 }
2505
2506 next_token = task.next_history_page_token.clone();
2507 }
2508
2509 Ok(())
2510 }
2511
2512 async fn workflow_task_history_page(
2513 &self,
2514 task_id: &str,
2515 lease_owner: &str,
2516 workflow_task_attempt: u64,
2517 next_history_page_token: &str,
2518 ) -> Result<WorkflowTaskHistoryPage> {
2519 let body = json!({
2520 "lease_owner": lease_owner,
2521 "workflow_task_attempt": workflow_task_attempt,
2522 "next_history_page_token": next_history_page_token
2523 });
2524 let path = format!("/worker/workflow-tasks/{task_id}/history");
2525
2526 self.request_json(
2527 reqwest::Method::POST,
2528 &path,
2529 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2530 Some(&body),
2531 )
2532 .await
2533 }
2534
2535 pub async fn complete_workflow_task(
2536 &self,
2537 task_id: &str,
2538 lease_owner: &str,
2539 workflow_task_attempt: u64,
2540 commands: Vec<Value>,
2541 ) -> Result<Value> {
2542 validate_workflow_task_commands(&commands)?;
2543 let body = json!({
2544 "lease_owner": lease_owner,
2545 "workflow_task_attempt": workflow_task_attempt,
2546 "commands": commands
2547 });
2548 let path = format!("/worker/workflow-tasks/{task_id}/complete");
2549 self.request_json(
2550 reqwest::Method::POST,
2551 &path,
2552 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2553 Some(&body),
2554 )
2555 .await
2556 }
2557
2558 pub async fn fail_workflow_task(
2559 &self,
2560 task_id: &str,
2561 lease_owner: &str,
2562 workflow_task_attempt: u64,
2563 message: impl Into<String>,
2564 ) -> Result<Value> {
2565 self.fail_workflow_task_with_type(
2566 task_id,
2567 lease_owner,
2568 workflow_task_attempt,
2569 message,
2570 "RustWorkflowTaskFailure",
2571 )
2572 .await
2573 }
2574
2575 async fn fail_workflow_task_with_type(
2576 &self,
2577 task_id: &str,
2578 lease_owner: &str,
2579 workflow_task_attempt: u64,
2580 message: impl Into<String>,
2581 failure_type: &str,
2582 ) -> Result<Value> {
2583 let body = json!({
2584 "lease_owner": lease_owner,
2585 "workflow_task_attempt": workflow_task_attempt,
2586 "failure": {
2587 "message": message.into(),
2588 "type": failure_type
2589 }
2590 });
2591 let path = format!("/worker/workflow-tasks/{task_id}/fail");
2592 self.request_json(
2593 reqwest::Method::POST,
2594 &path,
2595 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2596 Some(&body),
2597 )
2598 .await
2599 }
2600
2601 pub async fn poll_activity_task(
2602 &self,
2603 worker_id: &str,
2604 task_queue: &str,
2605 timeout: Duration,
2606 ) -> Result<Option<ActivityTask>> {
2607 Ok(self
2608 .poll_activity_task_response(worker_id, task_queue, timeout)
2609 .await?
2610 .task)
2611 }
2612
2613 pub async fn poll_activity_task_response(
2615 &self,
2616 worker_id: &str,
2617 task_queue: &str,
2618 timeout: Duration,
2619 ) -> Result<PollActivityTaskResponse> {
2620 let poll_request_id = unique_request_id("rust-activity-poll");
2621 self.poll_activity_task_response_with_request_id(
2622 worker_id,
2623 task_queue,
2624 timeout,
2625 &poll_request_id,
2626 1,
2627 )
2628 .await
2629 }
2630
2631 async fn poll_activity_task_response_with_request_id(
2632 &self,
2633 worker_id: &str,
2634 task_queue: &str,
2635 timeout: Duration,
2636 poll_request_id: &str,
2637 transport_retries: usize,
2638 ) -> Result<PollActivityTaskResponse> {
2639 let body = json!({
2640 "worker_id": worker_id,
2641 "task_queue": task_queue,
2642 "poll_request_id": poll_request_id,
2643 "timeout_seconds": long_poll_timeout_seconds(timeout),
2644 });
2645 let data: PollActivityTaskResponse = self
2646 .poll_request_json(
2647 "/worker/activity-tasks/poll",
2648 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2649 &body,
2650 timeout + Duration::from_secs(5),
2651 transport_retries,
2652 )
2653 .await?;
2654 Ok(data)
2655 }
2656
2657 pub async fn complete_activity_task<T: Serialize>(
2658 &self,
2659 task_id: &str,
2660 activity_attempt_id: &str,
2661 lease_owner: &str,
2662 result: T,
2663 codec: &str,
2664 ) -> Result<Value> {
2665 let result = encode_typed_envelope(&AvroValue::from_serialize(&result)?, codec)?;
2666 let body = json!({
2667 "activity_attempt_id": activity_attempt_id,
2668 "lease_owner": lease_owner,
2669 "result": result
2670 });
2671 let path = format!("/worker/activity-tasks/{task_id}/complete");
2672 activity_task_response(
2673 self.request_json(
2674 reqwest::Method::POST,
2675 &path,
2676 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2677 Some(&body),
2678 )
2679 .await,
2680 "complete",
2681 task_id,
2682 activity_attempt_id,
2683 )
2684 }
2685
2686 pub async fn fail_activity_task(
2687 &self,
2688 task_id: &str,
2689 activity_attempt_id: &str,
2690 lease_owner: &str,
2691 message: impl Into<String>,
2692 non_retryable: bool,
2693 ) -> Result<Value> {
2694 let body = json!({
2695 "activity_attempt_id": activity_attempt_id,
2696 "lease_owner": lease_owner,
2697 "failure": {
2698 "message": message.into(),
2699 "type": "RustActivityFailure",
2700 "non_retryable": non_retryable
2701 }
2702 });
2703 let path = format!("/worker/activity-tasks/{task_id}/fail");
2704 activity_task_response(
2705 self.request_json(
2706 reqwest::Method::POST,
2707 &path,
2708 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2709 Some(&body),
2710 )
2711 .await,
2712 "fail",
2713 task_id,
2714 activity_attempt_id,
2715 )
2716 }
2717
2718 pub async fn heartbeat_activity_task<T: Serialize>(
2719 &self,
2720 task_id: &str,
2721 activity_attempt_id: &str,
2722 lease_owner: &str,
2723 details: T,
2724 ) -> Result<ActivityHeartbeatResponse> {
2725 let details = encode_typed_envelope(&AvroValue::from_serialize(&details)?, DEFAULT_CODEC)?;
2726 let body = json!({
2727 "activity_attempt_id": activity_attempt_id,
2728 "lease_owner": lease_owner,
2729 "details": details
2730 });
2731 let path = format!("/worker/activity-tasks/{task_id}/heartbeat");
2732 activity_task_response(
2733 self.request_json(
2734 reqwest::Method::POST,
2735 &path,
2736 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2737 Some(&body),
2738 )
2739 .await,
2740 "heartbeat",
2741 task_id,
2742 activity_attempt_id,
2743 )
2744 }
2745
2746 async fn request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
2747 &self,
2748 method: reqwest::Method,
2749 path: &str,
2750 protocol: RequestProtocol,
2751 body: Option<&B>,
2752 ) -> Result<T> {
2753 self.request_json_with_timeout(method, path, protocol, body, Duration::from_secs(60))
2754 .await
2755 }
2756
2757 async fn request_json_with_timeout<T: DeserializeOwned, B: Serialize + ?Sized>(
2758 &self,
2759 method: reqwest::Method,
2760 path: &str,
2761 protocol: RequestProtocol,
2762 body: Option<&B>,
2763 timeout: Duration,
2764 ) -> Result<T> {
2765 let auth_token = self.auth_token(protocol)?;
2766 let mut request = self
2767 .http
2768 .request(method, format!("{}/api{}", self.base_url, path))
2769 .timeout(timeout)
2770 .header(reqwest::header::ACCEPT, "application/json")
2771 .header(reqwest::header::CONTENT_TYPE, "application/json")
2772 .header("X-Namespace", &self.namespace);
2773
2774 match protocol {
2775 RequestProtocol::Worker(version) => {
2776 request = request.header("X-Durable-Workflow-Protocol-Version", version);
2777 }
2778 RequestProtocol::ControlPlane => {
2779 request = request.header(
2780 "X-Durable-Workflow-Control-Plane-Version",
2781 CONTROL_PLANE_VERSION,
2782 );
2783 }
2784 }
2785
2786 if let Some(token) = auth_token {
2787 request = request.bearer_auth(token);
2788 }
2789
2790 if let Some(body) = body {
2791 request = request.json(body);
2792 }
2793
2794 let response = request.send().await?;
2795 let status = response.status();
2796 let bytes = response.bytes().await?;
2797
2798 if !status.is_success() {
2799 let body = String::from_utf8_lossy(&bytes).to_string();
2800 if let Some(protocol) = protocol_failure(status, &body) {
2801 return Err(Error::Protocol(protocol));
2802 }
2803 return Err(Error::Http { status, body });
2804 }
2805
2806 if bytes.is_empty() {
2807 return Ok(serde_json::from_value(Value::Null)?);
2808 }
2809
2810 Ok(serde_json::from_slice(&bytes)?)
2811 }
2812
2813 async fn poll_request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
2814 &self,
2815 path: &str,
2816 protocol: RequestProtocol,
2817 body: &B,
2818 timeout: Duration,
2819 max_retries: usize,
2820 ) -> Result<T> {
2821 let mut retries = 0;
2822
2823 loop {
2824 let response = self
2825 .request_json_with_timeout(
2826 reqwest::Method::POST,
2827 path,
2828 protocol,
2829 Some(body),
2830 timeout,
2831 )
2832 .await;
2833
2834 match response {
2835 Err(Error::Transport(_)) if retries < max_retries => retries += 1,
2836 response => return worker_poll_response(response),
2837 }
2838 }
2839 }
2840
2841 fn auth_token(&self, protocol: RequestProtocol) -> Result<Option<&str>> {
2842 match protocol {
2843 RequestProtocol::Worker(_) => {
2844 if let Some(token) = self.worker_token.as_deref().or(self.token.as_deref()) {
2845 return Ok(Some(token));
2846 }
2847 if self.control_token.is_some() {
2848 return Err(Error::MissingRoleCredentials {
2849 role: "worker",
2850 opposite_role: "control",
2851 });
2852 }
2853 Ok(None)
2854 }
2855 RequestProtocol::ControlPlane => {
2856 if let Some(token) = self.control_token.as_deref().or(self.token.as_deref()) {
2857 return Ok(Some(token));
2858 }
2859 if self.worker_token.is_some() {
2860 return Err(Error::MissingRoleCredentials {
2861 role: "control",
2862 opposite_role: "worker",
2863 });
2864 }
2865 Ok(None)
2866 }
2867 }
2868 }
2869}
2870
2871fn query_failure(status: reqwest::StatusCode, raw_body: String) -> QueryFailure {
2872 let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
2873 let reason = body
2874 .get("reason")
2875 .and_then(Value::as_str)
2876 .unwrap_or("query_rejected")
2877 .to_string();
2878 let message = body
2879 .get("message")
2880 .or_else(|| body.get("error"))
2881 .and_then(Value::as_str)
2882 .unwrap_or("workflow query was rejected")
2883 .to_string();
2884
2885 QueryFailure {
2886 status: status.as_u16(),
2887 reason,
2888 message,
2889 body,
2890 }
2891}
2892
2893fn workflow_command_result(
2894 command: WorkflowCommandKind,
2895 data: Value,
2896 workflow_id: &str,
2897 run_id: Option<&str>,
2898) -> WorkflowCommandResult {
2899 WorkflowCommandResult {
2900 command,
2901 workflow_id: data
2902 .get("workflow_id")
2903 .and_then(Value::as_str)
2904 .unwrap_or(workflow_id)
2905 .to_string(),
2906 run_id: data
2907 .get("run_id")
2908 .and_then(Value::as_str)
2909 .or(run_id)
2910 .map(str::to_string),
2911 outcome: data
2912 .get("outcome")
2913 .and_then(Value::as_str)
2914 .map(str::to_string),
2915 reason: data
2916 .get("reason")
2917 .and_then(Value::as_str)
2918 .map(str::to_string),
2919 command_status: data
2920 .get("command_status")
2921 .and_then(Value::as_str)
2922 .map(str::to_string),
2923 raw: data,
2924 }
2925}
2926
2927fn workflow_command_rejection(
2928 command: WorkflowCommandKind,
2929 status: reqwest::StatusCode,
2930 raw_body: String,
2931 workflow_id: &str,
2932 run_id: Option<&str>,
2933) -> WorkflowCommandRejection {
2934 let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
2935 WorkflowCommandRejection {
2936 command,
2937 status: status.as_u16(),
2938 reason: body
2939 .get("reason")
2940 .and_then(Value::as_str)
2941 .unwrap_or("workflow_command_rejected")
2942 .to_string(),
2943 message: body
2944 .get("message")
2945 .or_else(|| body.get("error"))
2946 .and_then(Value::as_str)
2947 .unwrap_or("workflow lifecycle command was rejected")
2948 .to_string(),
2949 workflow_id: body
2950 .get("workflow_id")
2951 .and_then(Value::as_str)
2952 .unwrap_or(workflow_id)
2953 .to_string(),
2954 run_id: body
2955 .get("run_id")
2956 .and_then(Value::as_str)
2957 .or(run_id)
2958 .map(str::to_string),
2959 target_scope: body
2960 .get("target_scope")
2961 .and_then(Value::as_str)
2962 .map(str::to_string),
2963 body,
2964 }
2965}
2966
2967fn query_task_response(response: Result<Value>) -> Result<Value> {
2968 match response {
2969 Err(Error::Http { status, body }) => Err(Error::QueryFailed(query_failure(status, body))),
2970 response => response,
2971 }
2972}
2973
2974fn worker_poll_response<T: DeserializeOwned>(response: Result<T>) -> Result<T> {
2975 match response {
2976 Err(Error::Http { status, body })
2977 if status == reqwest::StatusCode::CONFLICT && worker_poll_body_is_stop(&body) =>
2978 {
2979 Ok(serde_json::from_str(&body)?)
2980 }
2981 response => response,
2982 }
2983}
2984
2985fn worker_poll_body_is_stop(body: &str) -> bool {
2986 serde_json::from_str::<Value>(body)
2987 .ok()
2988 .is_some_and(|body| {
2989 worker_poll_is_stop(
2990 body.get("poll_status").and_then(Value::as_str),
2991 body.get("reason").and_then(Value::as_str),
2992 )
2993 })
2994}
2995
2996fn worker_poll_is_stop(poll_status: Option<&str>, reason: Option<&str>) -> bool {
2997 matches!(poll_status, Some("draining" | "stopped"))
2998 || matches!(reason, Some("worker_draining" | "worker_stopped"))
2999}
3000
3001fn query_task_rejection_is_final(error: &Error) -> bool {
3002 matches!(
3003 error,
3004 Error::QueryFailed(failure)
3005 if QUERY_TASK_FINAL_REJECTION_REASONS.contains(&failure.reason.as_str())
3006 )
3007}
3008
3009fn activity_task_response<T>(
3010 response: Result<T>,
3011 operation: &str,
3012 task_id: &str,
3013 activity_attempt_id: &str,
3014) -> Result<T> {
3015 match response {
3016 Err(Error::Http { status, body }) => {
3017 let body = serde_json::from_str(&body).unwrap_or_else(|_| json!({"message": body}));
3018 Err(Error::ActivityTaskRejected(ActivityTaskRejection {
3019 operation: operation.to_string(),
3020 status: status.as_u16(),
3021 reason: body
3022 .get("reason")
3023 .and_then(Value::as_str)
3024 .unwrap_or("activity_task_rejected")
3025 .to_string(),
3026 task_id: body
3027 .get("task_id")
3028 .and_then(Value::as_str)
3029 .unwrap_or(task_id)
3030 .to_string(),
3031 activity_attempt_id: body
3032 .get("activity_attempt_id")
3033 .and_then(Value::as_str)
3034 .unwrap_or(activity_attempt_id)
3035 .to_string(),
3036 cancel_requested: body
3037 .get("cancel_requested")
3038 .and_then(Value::as_bool)
3039 .unwrap_or(false),
3040 can_continue: body.get("can_continue").and_then(Value::as_bool),
3041 run_closed_reason: body
3042 .get("run_closed_reason")
3043 .and_then(Value::as_str)
3044 .map(str::to_string),
3045 body,
3046 }))
3047 }
3048 response => response,
3049 }
3050}
3051
3052fn activity_task_rejection_is_final(error: &Error) -> bool {
3053 matches!(
3054 error,
3055 Error::ActivityTaskRejected(rejection)
3056 if matches!(
3057 rejection.reason.as_str(),
3058 "run_cancelled"
3059 | "run_terminated"
3060 | "attempt_closed"
3061 | "stale_attempt"
3062 | "activity_cancelled"
3063 | "task_cancelled"
3064 | "run_closed"
3065 | "activity_not_running"
3066 | "attempt_not_found"
3067 )
3068 )
3069}
3070
3071fn workflow_task_completion_is_terminal_timeout(
3072 error: &Error,
3073 task_id: &str,
3074 workflow_task_attempt: u64,
3075 run_id: Option<&str>,
3076) -> bool {
3077 let Error::Http { status, body } = error else {
3078 return false;
3079 };
3080 if *status != reqwest::StatusCode::CONFLICT {
3081 return false;
3082 }
3083
3084 let Some(run_id) = run_id else {
3085 return false;
3086 };
3087 let Ok(body) = serde_json::from_str::<Value>(body) else {
3088 return false;
3089 };
3090
3091 body.get("recorded").and_then(Value::as_bool) == Some(false)
3092 && body.get("reason").and_then(Value::as_str) == Some("run_timed_out")
3093 && body.get("run_status").and_then(Value::as_str) == Some("failed")
3094 && body.get("run_id").and_then(Value::as_str) == Some(run_id)
3095 && body.get("task_id").and_then(Value::as_str) == Some(task_id)
3096 && body.get("workflow_task_attempt").and_then(Value::as_u64) == Some(workflow_task_attempt)
3097}
3098
3099fn protocol_failure(status: reqwest::StatusCode, raw_body: &str) -> Option<ProtocolFailure> {
3100 let body: Value = serde_json::from_str(raw_body).ok()?;
3101 let reason = body.get("reason")?.as_str()?;
3102 if !matches!(
3103 reason,
3104 "missing_protocol_version"
3105 | "unsupported_protocol_version"
3106 | "missing_control_plane_version"
3107 | "unsupported_control_plane_version"
3108 ) {
3109 return None;
3110 }
3111
3112 Some(ProtocolFailure {
3113 status: status.as_u16(),
3114 reason: reason.to_string(),
3115 message: body
3116 .get("message")
3117 .or_else(|| body.get("error"))
3118 .and_then(Value::as_str)
3119 .unwrap_or("protocol version rejected")
3120 .to_string(),
3121 supported_version: body
3122 .get("supported_version")
3123 .and_then(Value::as_str)
3124 .map(str::to_string),
3125 requested_version: body
3126 .get("requested_version")
3127 .and_then(Value::as_str)
3128 .map(str::to_string),
3129 body,
3130 })
3131}
3132
3133fn long_poll_timeout_seconds(timeout: Duration) -> u64 {
3134 timeout
3135 .as_secs()
3136 .saturating_add(u64::from(timeout.subsec_nanos() > 0))
3137 .min(MAX_LONG_POLL_TIMEOUT_SECONDS)
3138}
3139
3140fn worker_operation_is_retryable(error: &Error) -> bool {
3141 match error {
3142 Error::Transport(error) => {
3143 error.is_timeout() || error.is_connect() || error.is_request() || error.is_body()
3144 }
3145 Error::Http { status, .. } => {
3146 matches!(
3147 *status,
3148 reqwest::StatusCode::REQUEST_TIMEOUT | reqwest::StatusCode::TOO_MANY_REQUESTS
3149 ) || status.is_server_error()
3150 }
3151 _ => false,
3152 }
3153}
3154
3155fn worker_retry_delay(policy: WorkerRetryPolicy, retry: usize) -> Duration {
3156 let exponent = retry.saturating_sub(1).min(31) as u32;
3157 policy
3158 .initial_backoff
3159 .saturating_mul(1_u32 << exponent)
3160 .min(policy.max_backoff)
3161}
3162
3163#[derive(Debug)]
3164pub struct ClientBuilder {
3165 base_url: String,
3166 token: Option<String>,
3167 control_token: Option<String>,
3168 worker_token: Option<String>,
3169 namespace: String,
3170 timeout: Duration,
3171}
3172
3173impl ClientBuilder {
3174 pub fn token(mut self, token: Option<String>) -> Self {
3175 self.token = token;
3176 self
3177 }
3178
3179 pub fn control_token(mut self, token: Option<String>) -> Self {
3180 self.control_token = token;
3181 self
3182 }
3183
3184 pub fn worker_token(mut self, token: Option<String>) -> Self {
3185 self.worker_token = token;
3186 self
3187 }
3188
3189 pub fn namespace(mut self, namespace: impl Into<String>) -> Self {
3190 self.namespace = namespace.into();
3191 self
3192 }
3193
3194 pub fn timeout(mut self, timeout: Duration) -> Self {
3195 self.timeout = timeout;
3196 self
3197 }
3198
3199 pub fn build(self) -> Result<Client> {
3200 let base_url = self.base_url.trim_end_matches('/').to_string();
3201 let has_sdk_api_suffix = reqwest::Url::parse(&base_url)
3202 .map(|url| url.path().trim_end_matches('/').ends_with("/api"))
3203 .unwrap_or_else(|_| base_url.ends_with("/api"));
3204
3205 if has_sdk_api_suffix {
3206 return Err(Error::InvalidBaseUrl);
3207 }
3208
3209 Ok(Client {
3210 http: reqwest::Client::builder().timeout(self.timeout).build()?,
3211 base_url,
3212 token: self.token,
3213 control_token: self.control_token,
3214 worker_token: self.worker_token,
3215 namespace: self.namespace,
3216 })
3217 }
3218}
3219
3220#[derive(Clone, Debug)]
3221pub struct WorkflowHandle {
3222 client: Client,
3223 pub workflow_id: String,
3224 pub run_id: Option<String>,
3225 pub workflow_type: String,
3226}
3227
3228impl WorkflowHandle {
3229 pub async fn describe(&self) -> Result<WorkflowDescription> {
3231 self.client.describe_workflow(&self.workflow_id).await
3232 }
3233
3234 pub async fn describe_selected_run(&self) -> Result<WorkflowDescription> {
3236 let run_id = self.run_id.as_deref().ok_or_else(|| {
3237 Error::Codec("run_id is required for selected-run description".to_string())
3238 })?;
3239 self.client
3240 .describe_workflow_run(&self.workflow_id, run_id)
3241 .await
3242 }
3243
3244 pub async fn signal<T: Serialize>(&self, signal_name: &str, input: T) -> Result<Value> {
3245 self.client
3246 .signal_workflow(&self.workflow_id, signal_name, input)
3247 .await
3248 }
3249
3250 pub async fn signal_selected_run<T: Serialize>(
3252 &self,
3253 signal_name: &str,
3254 input: T,
3255 ) -> Result<Value> {
3256 let run_id = self.run_id.as_deref().ok_or_else(|| {
3257 Error::Codec("run_id is required for selected-run signaling".to_string())
3258 })?;
3259 self.client
3260 .signal_workflow_run(&self.workflow_id, run_id, signal_name, input)
3261 .await
3262 }
3263
3264 pub async fn cancel(&self, options: WorkflowCommandOptions) -> Result<WorkflowCommandResult> {
3266 self.client
3267 .cancel_workflow(&self.workflow_id, options)
3268 .await
3269 }
3270
3271 pub async fn cancel_selected_run(
3273 &self,
3274 options: WorkflowCommandOptions,
3275 ) -> Result<WorkflowCommandResult> {
3276 let run_id = self.run_id.as_deref().ok_or_else(|| {
3277 Error::Codec("run_id is required for selected-run cancellation".to_string())
3278 })?;
3279 self.client
3280 .cancel_workflow_run(&self.workflow_id, run_id, options)
3281 .await
3282 }
3283
3284 pub async fn terminate(
3286 &self,
3287 options: WorkflowCommandOptions,
3288 ) -> Result<WorkflowCommandResult> {
3289 self.client
3290 .terminate_workflow(&self.workflow_id, options)
3291 .await
3292 }
3293
3294 pub async fn terminate_selected_run(
3296 &self,
3297 options: WorkflowCommandOptions,
3298 ) -> Result<WorkflowCommandResult> {
3299 let run_id = self.run_id.as_deref().ok_or_else(|| {
3300 Error::Codec("run_id is required for selected-run termination".to_string())
3301 })?;
3302 self.client
3303 .terminate_workflow_run(&self.workflow_id, run_id, options)
3304 .await
3305 }
3306
3307 pub async fn query<T: Serialize>(&self, query_name: &str, input: T) -> Result<Value> {
3309 self.client
3310 .query_workflow(&self.workflow_id, query_name, input)
3311 .await
3312 }
3313
3314 pub async fn query_avro_value<T: Serialize>(
3315 &self,
3316 query_name: &str,
3317 input: T,
3318 ) -> Result<AvroValue> {
3319 self.client
3320 .query_workflow_avro_value(&self.workflow_id, query_name, input)
3321 .await
3322 }
3323
3324 pub async fn update<T: Serialize>(
3325 &self,
3326 update_name: &str,
3327 input: T,
3328 request_id: Option<&str>,
3329 ) -> Result<Value> {
3330 self.client
3331 .update_workflow(&self.workflow_id, update_name, input, request_id)
3332 .await
3333 }
3334
3335 pub async fn update_avro_value<T: Serialize>(
3336 &self,
3337 update_name: &str,
3338 input: T,
3339 request_id: Option<&str>,
3340 ) -> Result<AvroValue> {
3341 self.client
3342 .update_workflow_avro_value(&self.workflow_id, update_name, input, request_id)
3343 .await
3344 }
3345
3346 pub async fn query_selected_run<T: Serialize>(
3348 &self,
3349 query_name: &str,
3350 input: T,
3351 ) -> Result<Value> {
3352 let run_id = self
3353 .run_id
3354 .as_deref()
3355 .ok_or_else(|| Error::Codec("run_id is required for selected-run query".to_string()))?;
3356 self.client
3357 .query_workflow_run(&self.workflow_id, run_id, query_name, input)
3358 .await
3359 }
3360
3361 pub async fn result(&self, options: WorkflowResultOptions) -> Result<Value> {
3363 self.result_target(options, None).await
3364 }
3365
3366 pub async fn result_avro_value(&self, options: WorkflowResultOptions) -> Result<AvroValue> {
3368 self.result_avro_value_target(options, None).await
3369 }
3370
3371 pub async fn result_selected_run(&self, options: WorkflowResultOptions) -> Result<Value> {
3373 let run_id = self.run_id.as_deref().ok_or_else(|| {
3374 Error::Codec("run_id is required for selected-run result".to_string())
3375 })?;
3376 self.result_target(options, Some(run_id)).await
3377 }
3378
3379 pub async fn result_selected_run_avro_value(
3381 &self,
3382 options: WorkflowResultOptions,
3383 ) -> Result<AvroValue> {
3384 let run_id = self.run_id.as_deref().ok_or_else(|| {
3385 Error::Codec("run_id is required for selected-run result".to_string())
3386 })?;
3387 self.result_avro_value_target(options, Some(run_id)).await
3388 }
3389
3390 async fn result_avro_value_target(
3391 &self,
3392 options: WorkflowResultOptions,
3393 selected_run_id: Option<&str>,
3394 ) -> Result<AvroValue> {
3395 let started = Instant::now();
3396
3397 loop {
3398 let description = match selected_run_id {
3399 Some(run_id) => {
3400 self.client
3401 .describe_workflow_run(&self.workflow_id, run_id)
3402 .await?
3403 }
3404 None => self.describe().await?,
3405 };
3406 if description.is_completed() {
3407 return description.output_avro_value.ok_or_else(|| {
3408 Error::Codec(
3409 "missing_payload_envelope: typed workflow result requires output_envelope"
3410 .to_string(),
3411 )
3412 });
3413 }
3414 if description.is_terminal() {
3415 let outcome =
3416 workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
3417 return Err(match outcome.kind {
3418 WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
3419 WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
3420 WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
3421 WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
3422 });
3423 }
3424 if started.elapsed() >= options.timeout {
3425 return Err(Error::Timeout);
3426 }
3427 tokio::time::sleep(options.poll_interval).await;
3428 }
3429 }
3430
3431 async fn result_target(
3432 &self,
3433 options: WorkflowResultOptions,
3434 selected_run_id: Option<&str>,
3435 ) -> Result<Value> {
3436 let started = Instant::now();
3437
3438 loop {
3439 let description = match selected_run_id {
3440 Some(run_id) => {
3441 self.client
3442 .describe_workflow_run(&self.workflow_id, run_id)
3443 .await?
3444 }
3445 None => self.describe().await?,
3446 };
3447 if description.is_completed() {
3448 return Ok(description.output.unwrap_or(Value::Null));
3449 }
3450
3451 if description.is_terminal() {
3452 let outcome =
3453 workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
3454 return Err(match outcome.kind {
3455 WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
3456 WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
3457 WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
3458 WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
3459 });
3460 }
3461
3462 if started.elapsed() >= options.timeout {
3463 return Err(Error::WorkflowTimedOut(WorkflowTerminalOutcome {
3464 kind: WorkflowTerminalKind::TimedOut,
3465 workflow_id: description
3466 .workflow_id
3467 .clone()
3468 .unwrap_or_else(|| self.workflow_id.clone()),
3469 run_id: description
3470 .run_id
3471 .clone()
3472 .or_else(|| selected_run_id.map(str::to_string)),
3473 reason: "result_wait_timeout".to_string(),
3474 failure_category: Some("client_timeout".to_string()),
3475 failure_id: None,
3476 exception_type: None,
3477 exception_class: None,
3478 non_retryable: None,
3479 message: Some(format!(
3480 "workflow result was not terminal within {:?}",
3481 options.timeout
3482 )),
3483 exception: None,
3484 raw: description.raw_value(),
3485 }));
3486 }
3487
3488 tokio::time::sleep(options.poll_interval).await;
3489 }
3490 }
3491}
3492
3493#[derive(Clone, Copy, Debug)]
3494pub struct WorkflowResultOptions {
3495 pub poll_interval: Duration,
3496 pub timeout: Duration,
3497}
3498
3499impl Default for WorkflowResultOptions {
3500 fn default() -> Self {
3501 Self {
3502 poll_interval: Duration::from_millis(500),
3503 timeout: Duration::from_secs(30),
3504 }
3505 }
3506}
3507
3508#[derive(Clone, Debug, Deserialize)]
3509pub struct WorkflowDescription {
3510 pub workflow_id: Option<String>,
3511 pub run_id: Option<String>,
3512 pub workflow_type: Option<String>,
3513 pub status: Option<String>,
3514 #[serde(default)]
3515 pub closed_reason: Option<String>,
3516 #[serde(default)]
3517 pub error: Option<String>,
3518 #[serde(default)]
3519 pub failure: Option<Value>,
3520 #[serde(default)]
3521 pub exception: Option<Value>,
3522 #[serde(default)]
3523 pub failures: Vec<Value>,
3524 #[serde(default)]
3525 pub output: Option<Value>,
3526 #[serde(default)]
3527 pub output_envelope: Option<Value>,
3528 #[serde(skip)]
3529 pub output_avro_value: Option<AvroValue>,
3530 #[serde(flatten)]
3531 pub raw: HashMap<String, Value>,
3532}
3533
3534impl WorkflowDescription {
3535 pub fn is_completed(&self) -> bool {
3536 matches!(self.status.as_deref(), Some("completed" | "Completed"))
3537 }
3538
3539 pub fn is_terminal(&self) -> bool {
3540 matches!(
3541 self.status.as_deref(),
3542 Some(
3543 "completed"
3544 | "Completed"
3545 | "failed"
3546 | "Failed"
3547 | "cancelled"
3548 | "Cancelled"
3549 | "terminated"
3550 | "Terminated"
3551 | "timed_out"
3552 | "TimedOut",
3553 )
3554 )
3555 }
3556
3557 fn decode_payloads(&mut self) -> Result<()> {
3558 if let Some(envelope) = &self.output_envelope {
3559 let value = decode_wire_avro_value(envelope, DEFAULT_CODEC)?;
3560 self.output = Some(value.clone().into_json()?);
3561 self.output_avro_value = Some(value);
3562 }
3563
3564 Ok(())
3565 }
3566
3567 fn raw_value(&self) -> Value {
3568 let mut data = self.raw.clone();
3569 data.insert(
3570 "workflow_id".to_string(),
3571 self.workflow_id
3572 .clone()
3573 .map(Value::String)
3574 .unwrap_or(Value::Null),
3575 );
3576 data.insert(
3577 "run_id".to_string(),
3578 self.run_id
3579 .clone()
3580 .map(Value::String)
3581 .unwrap_or(Value::Null),
3582 );
3583 data.insert(
3584 "workflow_type".to_string(),
3585 self.workflow_type
3586 .clone()
3587 .map(Value::String)
3588 .unwrap_or(Value::Null),
3589 );
3590 data.insert(
3591 "status".to_string(),
3592 self.status
3593 .clone()
3594 .map(Value::String)
3595 .unwrap_or(Value::Null),
3596 );
3597 data.insert(
3598 "closed_reason".to_string(),
3599 self.closed_reason
3600 .clone()
3601 .map(Value::String)
3602 .unwrap_or(Value::Null),
3603 );
3604 if let Some(failure) = &self.failure {
3605 data.insert("failure".to_string(), failure.clone());
3606 }
3607 if let Some(exception) = &self.exception {
3608 data.insert("exception".to_string(), exception.clone());
3609 }
3610 Value::Object(data.into_iter().collect())
3611 }
3612}
3613
3614fn workflow_terminal_outcome(
3615 description: &WorkflowDescription,
3616 workflow_id: &str,
3617 run_id: Option<&str>,
3618) -> WorkflowTerminalOutcome {
3619 let terminal_kind = description
3620 .closed_reason
3621 .as_deref()
3622 .or(description.status.as_deref())
3623 .unwrap_or("failed")
3624 .to_ascii_lowercase();
3625 let kind = match terminal_kind.as_str() {
3626 "cancelled" | "canceled" => WorkflowTerminalKind::Cancelled,
3627 "terminated" => WorkflowTerminalKind::Terminated,
3628 "timed_out" | "timedout" => WorkflowTerminalKind::TimedOut,
3629 _ => WorkflowTerminalKind::Failed,
3630 };
3631 let default_reason = match kind {
3632 WorkflowTerminalKind::Failed => "workflow_failed",
3633 WorkflowTerminalKind::Cancelled => "cancelled",
3634 WorkflowTerminalKind::Terminated => "terminated",
3635 WorkflowTerminalKind::TimedOut => "timed_out",
3636 };
3637 let failure = description
3638 .failure
3639 .as_ref()
3640 .filter(|value| value.is_object());
3641 let nested_failure = failure
3642 .and_then(|value| value.get("failures"))
3643 .and_then(Value::as_array)
3644 .and_then(|failures| failures.last())
3645 .or_else(|| description.failures.last());
3646 let exception = description
3647 .exception
3648 .clone()
3649 .or_else(|| failure.and_then(|value| value.get("exception")).cloned())
3650 .or_else(|| {
3651 nested_failure
3652 .and_then(|value| value.get("exception_payload"))
3653 .cloned()
3654 });
3655 let string_field = |name: &str| {
3656 failure
3657 .and_then(|value| value.get(name))
3658 .and_then(Value::as_str)
3659 .or_else(|| {
3660 nested_failure
3661 .and_then(|value| value.get(name))
3662 .and_then(Value::as_str)
3663 })
3664 .map(str::to_string)
3665 };
3666 let exception_field = |name: &str| {
3667 exception
3668 .as_ref()
3669 .and_then(|value| value.get(name))
3670 .and_then(Value::as_str)
3671 .map(str::to_string)
3672 };
3673 let message = description
3674 .error
3675 .clone()
3676 .or_else(|| string_field("message"))
3677 .or_else(|| exception_field("message"));
3678 let reason = description
3679 .raw
3680 .get("reason")
3681 .and_then(Value::as_str)
3682 .map(str::to_string)
3683 .or_else(|| {
3684 failure
3685 .and_then(|value| value.get("reason"))
3686 .and_then(Value::as_str)
3687 .map(str::to_string)
3688 })
3689 .or_else(|| description.closed_reason.clone())
3690 .unwrap_or_else(|| default_reason.to_string());
3691 let failure_id = string_field("failure_id").or_else(|| {
3692 nested_failure
3693 .and_then(|value| value.get("id"))
3694 .and_then(Value::as_str)
3695 .map(str::to_string)
3696 });
3697
3698 WorkflowTerminalOutcome {
3699 kind,
3700 workflow_id: description
3701 .workflow_id
3702 .clone()
3703 .unwrap_or_else(|| workflow_id.to_string()),
3704 run_id: description
3705 .run_id
3706 .clone()
3707 .or_else(|| run_id.map(str::to_string)),
3708 reason,
3709 failure_category: string_field("failure_category")
3710 .or_else(|| Some(default_reason.to_string())),
3711 failure_id,
3712 exception_type: string_field("exception_type").or_else(|| exception_field("type")),
3713 exception_class: string_field("exception_class").or_else(|| exception_field("class")),
3714 non_retryable: failure
3715 .and_then(|value| value.get("non_retryable"))
3716 .and_then(Value::as_bool)
3717 .or_else(|| {
3718 nested_failure
3719 .and_then(|value| value.get("non_retryable"))
3720 .and_then(Value::as_bool)
3721 }),
3722 message,
3723 exception,
3724 raw: description.raw_value(),
3725 }
3726}
3727
3728#[derive(Clone, Debug, Deserialize)]
3729pub struct RegisterWorkerResponse {
3730 pub worker_id: String,
3731 pub registered: bool,
3732 #[serde(default)]
3733 pub heartbeat_interval_seconds: Option<u64>,
3734 #[serde(default)]
3735 pub protocol_version: Option<String>,
3736 #[serde(default)]
3737 pub server_capabilities: Option<Value>,
3738}
3739
3740#[derive(Clone, Debug, Deserialize, PartialEq, Eq)]
3742pub struct WorkerDeregistrationEnvelope {
3743 pub worker_id: String,
3744 pub outcome: String,
3745 pub recovered_workflow_task_count: u64,
3746}
3747
3748#[derive(Clone, Debug, Deserialize)]
3749pub struct PollWorkflowTaskResponse {
3750 #[serde(default)]
3751 pub task: Option<WorkflowTask>,
3752 #[serde(default)]
3753 pub poll_status: Option<String>,
3754 #[serde(default)]
3755 pub reason: Option<String>,
3756 #[serde(default)]
3757 pub protocol_version: Option<String>,
3758 #[serde(default)]
3759 pub server_capabilities: Option<Value>,
3760}
3761
3762impl PollWorkflowTaskResponse {
3763 pub fn outcome(&self) -> WorkerPollOutcome {
3765 worker_poll_outcome(
3766 self.task.is_some(),
3767 self.poll_status.as_deref(),
3768 self.reason.as_deref(),
3769 )
3770 }
3771}
3772
3773#[derive(Clone, Debug, Deserialize)]
3774pub struct PollActivityTaskResponse {
3775 #[serde(default)]
3776 pub task: Option<ActivityTask>,
3777 #[serde(default)]
3778 pub poll_status: Option<String>,
3779 #[serde(default)]
3780 pub reason: Option<String>,
3781}
3782
3783impl PollActivityTaskResponse {
3784 pub fn outcome(&self) -> WorkerPollOutcome {
3786 worker_poll_outcome(
3787 self.task.is_some(),
3788 self.poll_status.as_deref(),
3789 self.reason.as_deref(),
3790 )
3791 }
3792}
3793
3794#[derive(Clone, Debug, Deserialize)]
3795pub struct PollQueryTaskResponse {
3796 #[serde(default)]
3797 pub task: Option<QueryTask>,
3798 #[serde(default)]
3799 pub poll_status: Option<String>,
3800 #[serde(default)]
3801 pub reason: Option<String>,
3802}
3803
3804impl PollQueryTaskResponse {
3805 pub fn outcome(&self) -> WorkerPollOutcome {
3807 worker_poll_outcome(
3808 self.task.is_some(),
3809 self.poll_status.as_deref(),
3810 self.reason.as_deref(),
3811 )
3812 }
3813}
3814
3815#[derive(Clone, Debug, PartialEq, Eq)]
3817pub enum WorkerPollOutcome {
3818 Task,
3820 Idle {
3822 poll_status: Option<String>,
3823 reason: Option<String>,
3824 },
3825 Stop {
3827 poll_status: Option<String>,
3828 reason: Option<String>,
3829 },
3830}
3831
3832impl WorkerPollOutcome {
3833 pub fn should_stop(&self) -> bool {
3834 matches!(self, Self::Stop { .. })
3835 }
3836}
3837
3838fn worker_poll_outcome(
3839 has_task: bool,
3840 poll_status: Option<&str>,
3841 reason: Option<&str>,
3842) -> WorkerPollOutcome {
3843 if worker_poll_is_stop(poll_status, reason) {
3844 return WorkerPollOutcome::Stop {
3845 poll_status: poll_status.map(str::to_string),
3846 reason: reason.map(str::to_string),
3847 };
3848 }
3849
3850 if has_task {
3851 WorkerPollOutcome::Task
3852 } else {
3853 WorkerPollOutcome::Idle {
3854 poll_status: poll_status.map(str::to_string),
3855 reason: reason.map(str::to_string),
3856 }
3857 }
3858}
3859
3860#[derive(Clone, Debug, Deserialize)]
3862pub struct QueryTask {
3863 pub query_task_id: String,
3864 #[serde(default = "default_workflow_task_attempt")]
3865 pub query_task_attempt: u64,
3866 #[serde(default)]
3867 pub lease_owner: Option<String>,
3868 #[serde(default)]
3869 pub workflow_id: Option<String>,
3870 #[serde(default)]
3871 pub run_id: Option<String>,
3872 pub workflow_type: String,
3873 pub query_name: String,
3874 #[serde(
3875 default = "missing_task_payload_codec",
3876 deserialize_with = "deserialize_task_payload_codec"
3877 )]
3878 pub payload_codec: String,
3879 #[serde(default)]
3880 pub workflow_arguments: Option<Value>,
3881 #[serde(default)]
3882 pub query_arguments: Option<Value>,
3883 #[serde(default)]
3884 pub history_events: Vec<HistoryEvent>,
3885 #[serde(default)]
3886 pub history_export: Option<Value>,
3887 #[serde(default)]
3888 pub run_status: Option<String>,
3889}
3890
3891#[derive(Clone, Debug, Deserialize)]
3892pub struct WorkflowTask {
3893 pub task_id: String,
3894 #[serde(default)]
3895 pub workflow_id: Option<String>,
3896 #[serde(default)]
3897 pub run_id: Option<String>,
3898 pub workflow_type: String,
3899 #[serde(
3900 default = "missing_task_payload_codec",
3901 deserialize_with = "deserialize_task_payload_codec"
3902 )]
3903 pub payload_codec: String,
3904 #[serde(default)]
3905 pub arguments: Option<Value>,
3906 #[serde(default)]
3907 pub history_events: Vec<HistoryEvent>,
3908 #[serde(default)]
3909 pub total_history_events: Option<u64>,
3910 #[serde(default)]
3911 pub history_size_bytes: Option<u64>,
3912 #[serde(default)]
3913 pub continue_as_new_recommended: Option<bool>,
3914 #[serde(default)]
3915 pub history_budget_pressure: Option<String>,
3916 #[serde(default)]
3917 pub next_history_page_token: Option<String>,
3918 #[serde(default = "default_workflow_task_attempt")]
3919 pub workflow_task_attempt: u64,
3920 #[serde(default)]
3921 pub workflow_signal_id: Option<String>,
3922 #[serde(default)]
3923 pub signal_name: Option<String>,
3924 #[serde(default)]
3925 pub signal_arguments: Option<Value>,
3926 #[serde(default)]
3927 pub workflow_update_id: Option<String>,
3928 #[serde(default)]
3929 pub update_name: Option<String>,
3930 #[serde(default)]
3931 pub lease_owner: Option<String>,
3932}
3933
3934impl WorkflowTask {
3935 fn append_history_page(&mut self, page: WorkflowTaskHistoryPage) {
3936 self.history_events.extend(page.history_events);
3937
3938 if page.total_history_events.is_some() {
3939 self.total_history_events = page.total_history_events;
3940 }
3941
3942 self.next_history_page_token = page
3943 .next_history_page_token
3944 .filter(|token| !token.is_empty());
3945 }
3946}
3947
3948#[derive(Clone, Debug, Deserialize)]
3949struct WorkflowTaskHistoryPage {
3950 #[serde(default)]
3951 history_events: Vec<HistoryEvent>,
3952 #[serde(default)]
3953 total_history_events: Option<u64>,
3954 #[serde(default)]
3955 next_history_page_token: Option<String>,
3956}
3957
3958#[derive(Clone, Debug, Deserialize)]
3959pub struct ActivityTask {
3960 pub task_id: String,
3961 #[serde(default)]
3962 pub activity_attempt_id: Option<String>,
3963 #[serde(default)]
3964 pub attempt_id: Option<String>,
3965 pub activity_type: String,
3966 #[serde(
3967 default = "missing_task_payload_codec",
3968 deserialize_with = "deserialize_task_payload_codec"
3969 )]
3970 pub payload_codec: String,
3971 #[serde(default)]
3972 pub arguments: Option<Value>,
3973 #[serde(default = "default_attempt_number")]
3974 pub attempt_number: u64,
3975 #[serde(default)]
3976 pub lease_owner: Option<String>,
3977}
3978
3979#[derive(Clone, Debug, Deserialize)]
3980pub struct HistoryEvent {
3981 #[serde(alias = "type")]
3982 pub event_type: String,
3983 #[serde(default)]
3984 pub payload: Value,
3985 #[serde(flatten)]
3986 pub raw: HashMap<String, Value>,
3987}
3988
3989#[derive(Clone, Debug, PartialEq)]
3991pub struct QuerySignal {
3992 pub id: Option<String>,
3993 pub name: String,
3994 pub arguments: Vec<Value>,
3995 avro_arguments: Vec<AvroValue>,
3996 pub workflow_sequence: Option<u64>,
3997}
3998
3999impl QuerySignal {
4000 pub fn arguments_avro_value(&self) -> &[AvroValue] {
4002 &self.avro_arguments
4003 }
4004}
4005
4006#[derive(Clone, Debug)]
4012pub struct QueryContext {
4013 pub workflow_id: Option<String>,
4014 pub run_id: Option<String>,
4015 pub workflow_type: String,
4016 pub run_status: Option<String>,
4017 workflow_input: Value,
4018 workflow_input_avro_value: AvroValue,
4019 history_events: Arc<Vec<HistoryEvent>>,
4020 signal_events: Arc<Vec<QuerySignal>>,
4021}
4022
4023impl QueryContext {
4024 pub fn workflow_input(&self) -> &Value {
4026 &self.workflow_input
4027 }
4028
4029 pub fn workflow_input_avro_value(&self) -> &AvroValue {
4031 &self.workflow_input_avro_value
4032 }
4033
4034 pub fn history_events(&self) -> &[HistoryEvent] {
4036 self.history_events.as_slice()
4037 }
4038
4039 pub fn signal_events(&self) -> &[QuerySignal] {
4041 self.signal_events.as_slice()
4042 }
4043
4044 pub fn signals(&self, signal_name: &str) -> Vec<Vec<Value>> {
4046 self.signal_events
4047 .iter()
4048 .filter(|signal| signal.name == signal_name)
4049 .map(|signal| signal.arguments.clone())
4050 .collect()
4051 }
4052
4053 pub fn signals_avro_value(&self, signal_name: &str) -> Vec<Vec<AvroValue>> {
4055 self.signal_events
4056 .iter()
4057 .filter(|signal| signal.name == signal_name)
4058 .map(|signal| signal.avro_arguments.clone())
4059 .collect()
4060 }
4061}
4062
4063#[derive(Clone, Debug, Deserialize)]
4064pub struct ActivityHeartbeatResponse {
4065 #[serde(default)]
4066 pub cancel_requested: bool,
4067 #[serde(default)]
4068 pub heartbeat_recorded: bool,
4069 #[serde(default)]
4070 pub can_continue: Option<bool>,
4071 #[serde(default)]
4072 pub reason: Option<String>,
4073 #[serde(default)]
4074 pub run_closed_reason: Option<String>,
4075 #[serde(default)]
4076 pub run_closed_at: Option<String>,
4077 #[serde(default)]
4078 pub lease_expires_at: Option<String>,
4079 #[serde(default)]
4080 pub last_heartbeat_at: Option<String>,
4081}
4082
4083impl ActivityHeartbeatResponse {
4084 pub fn should_stop(&self) -> bool {
4086 self.cancel_requested || self.can_continue == Some(false)
4087 }
4088}
4089
4090fn missing_task_payload_codec() -> String {
4091 MISSING_TASK_PAYLOAD_CODEC.to_string()
4092}
4093
4094fn deserialize_task_payload_codec<'de, D>(deserializer: D) -> std::result::Result<String, D::Error>
4095where
4096 D: Deserializer<'de>,
4097{
4098 Ok(match Value::deserialize(deserializer)? {
4099 Value::String(codec) => codec,
4100 Value::Null => NULL_TASK_PAYLOAD_CODEC.to_string(),
4101 _ => NON_STRING_TASK_PAYLOAD_CODEC.to_string(),
4102 })
4103}
4104
4105fn default_workflow_task_attempt() -> u64 {
4106 1
4107}
4108
4109fn default_attempt_number() -> u64 {
4110 1
4111}
4112
4113type WorkflowFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
4114type WorkflowHandler = Arc<dyn Fn(WorkflowContext, AvroValue) -> WorkflowFuture + Send + Sync>;
4115type ErasedWorkflowState = Arc<dyn Any + Send + Sync>;
4116type WorkflowStateSnapshot = Arc<dyn Fn() -> Result<ErasedWorkflowState> + Send + Sync>;
4117type ReplayedWorkflowHandler =
4118 Arc<dyn Fn(WorkflowContext, AvroValue) -> ReplayedWorkflowInvocation + Send + Sync>;
4119type ActivityFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
4120type ActivityHandler = Arc<dyn Fn(ActivityContext, AvroValue) -> ActivityFuture + Send + Sync>;
4121type QueryFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
4122type QueryHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
4123type UpdateHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
4124type ReplayedQueryHandler = Arc<
4125 dyn Fn(QueryContext, ErasedWorkflowState, AvroValue) -> std::result::Result<QueryFuture, String>
4126 + Send
4127 + Sync,
4128>;
4129type WorkerHeartbeatObserver = Arc<dyn Fn(&WorkerHeartbeatObservation) + Send + Sync>;
4130
4131struct ReplayedWorkflowInvocation {
4132 future: WorkflowFuture,
4133 snapshot: WorkflowStateSnapshot,
4134}
4135
4136#[derive(Clone)]
4137struct RegisteredWorkflow {
4138 execute: WorkflowHandler,
4139 replay: Option<ReplayedWorkflowHandler>,
4140 state_type: Option<TypeId>,
4141}
4142
4143#[derive(Clone)]
4144enum RegisteredQuery {
4145 Snapshot(QueryHandler),
4146 Replayed {
4147 state_type: TypeId,
4148 handler: ReplayedQueryHandler,
4149 },
4150}
4151
4152#[derive(Clone, Debug)]
4153pub struct WorkerHeartbeatObservation {
4154 pub worker_id: String,
4155 pub task_queue: String,
4156 pub acknowledged_at_unix_millis: u64,
4157 pub acknowledgement: Value,
4158}
4159
4160#[derive(Clone, Copy, Debug)]
4167pub struct WorkerRetryPolicy {
4168 pub max_retries: usize,
4170 pub initial_backoff: Duration,
4172 pub max_backoff: Duration,
4174}
4175
4176impl Default for WorkerRetryPolicy {
4177 fn default() -> Self {
4178 Self {
4179 max_retries: 5,
4180 initial_backoff: Duration::from_millis(100),
4181 max_backoff: Duration::from_secs(5),
4182 }
4183 }
4184}
4185
4186#[derive(Clone, Copy, Debug, PartialEq, Eq)]
4187enum ManagedPollOutcome {
4188 Idle,
4189 Handled,
4190 Stop,
4191}
4192
4193#[derive(Clone)]
4194pub struct Worker {
4195 client: Client,
4196 worker_id: String,
4197 task_queue: String,
4198 workflows: HashMap<String, RegisteredWorkflow>,
4199 activities: HashMap<String, ActivityHandler>,
4200 queries: HashMap<String, HashMap<String, RegisteredQuery>>,
4201 updates: HashMap<String, HashMap<String, UpdateHandler>>,
4202 max_concurrent_workflow_tasks: usize,
4203 max_concurrent_activity_tasks: usize,
4204 poll_timeout: Duration,
4205 heartbeat_interval: Duration,
4206 retry_policy: WorkerRetryPolicy,
4207 heartbeat_observer: Option<WorkerHeartbeatObserver>,
4208}
4209
4210impl Worker {
4211 pub fn new(client: Client, task_queue: impl Into<String>) -> Self {
4212 Self {
4213 client,
4214 worker_id: default_worker_id(),
4215 task_queue: task_queue.into(),
4216 workflows: HashMap::new(),
4217 activities: HashMap::new(),
4218 queries: HashMap::new(),
4219 updates: HashMap::new(),
4220 max_concurrent_workflow_tasks: 10,
4221 max_concurrent_activity_tasks: 10,
4222 poll_timeout: Duration::from_secs(30),
4223 heartbeat_interval: Duration::from_secs(60),
4224 retry_policy: WorkerRetryPolicy::default(),
4225 heartbeat_observer: None,
4226 }
4227 }
4228
4229 pub fn worker_id(mut self, worker_id: impl Into<String>) -> Self {
4230 self.worker_id = worker_id.into();
4231 self
4232 }
4233
4234 pub fn poll_timeout(mut self, timeout: Duration) -> Self {
4235 self.poll_timeout = timeout;
4236 self
4237 }
4238
4239 pub fn heartbeat_interval(mut self, interval: Duration) -> Self {
4240 self.heartbeat_interval = interval;
4241 self
4242 }
4243
4244 pub fn retry_policy(mut self, policy: WorkerRetryPolicy) -> Self {
4246 self.retry_policy = policy;
4247 self
4248 }
4249
4250 pub fn on_worker_heartbeat<F>(mut self, observer: F) -> Self
4251 where
4252 F: Fn(&WorkerHeartbeatObservation) + Send + Sync + 'static,
4253 {
4254 self.heartbeat_observer = Some(Arc::new(observer));
4255 self
4256 }
4257
4258 pub fn max_concurrent_workflow_tasks(mut self, count: usize) -> Self {
4259 self.max_concurrent_workflow_tasks = count.max(1);
4260 self
4261 }
4262
4263 pub fn max_concurrent_activity_tasks(mut self, count: usize) -> Self {
4264 self.max_concurrent_activity_tasks = count.max(1);
4265 self
4266 }
4267
4268 pub fn register_workflow<F, Fut>(&mut self, workflow_type: impl Into<String>, handler: F)
4275 where
4276 F: Fn(WorkflowContext, Value) -> Fut + Send + Sync + 'static,
4277 Fut: Future<Output = Result<Value>> + Send + 'static,
4278 {
4279 let handler = Arc::new(handler);
4280 self.workflows.insert(
4281 workflow_type.into(),
4282 RegisteredWorkflow {
4283 execute: Arc::new(move |ctx, input| {
4284 let handler = Arc::clone(&handler);
4285 Box::pin(async move {
4286 let result = handler(ctx, input.into_json()?).await?;
4287 AvroValue::from_serialize(&result)
4288 })
4289 }),
4290 replay: None,
4291 state_type: None,
4292 },
4293 );
4294 }
4295
4296 pub fn register_workflow_avro_value<F, Fut>(
4298 &mut self,
4299 workflow_type: impl Into<String>,
4300 handler: F,
4301 ) where
4302 F: Fn(WorkflowContext, AvroValue) -> Fut + Send + Sync + 'static,
4303 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4304 {
4305 self.workflows.insert(
4306 workflow_type.into(),
4307 RegisteredWorkflow {
4308 execute: Arc::new(move |ctx, input| Box::pin(handler(ctx, input))),
4309 replay: None,
4310 state_type: None,
4311 },
4312 );
4313 }
4314
4315 pub fn register_replayed_workflow<S, Factory, F, Fut>(
4323 &mut self,
4324 workflow_type: impl Into<String>,
4325 state_factory: Factory,
4326 handler: F,
4327 ) where
4328 S: Clone + Send + Sync + 'static,
4329 Factory: Fn() -> S + Send + Sync + 'static,
4330 F: Fn(WorkflowContext, Value, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
4331 Fut: Future<Output = Result<Value>> + Send + 'static,
4332 {
4333 let state_factory = Arc::new(state_factory);
4334 let handler = Arc::new(handler);
4335
4336 let execute_factory = Arc::clone(&state_factory);
4337 let execute_handler = Arc::clone(&handler);
4338 let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4339 let state = WorkflowInstance::new(execute_factory());
4340 let handler = Arc::clone(&execute_handler);
4341 Box::pin(async move {
4342 let result = handler(ctx, input.into_json()?, state).await?;
4343 AvroValue::from_serialize(&result)
4344 }) as WorkflowFuture
4345 });
4346
4347 let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4348 let state = WorkflowInstance::new(state_factory());
4349 let snapshot_state = state.clone();
4350 let snapshot: WorkflowStateSnapshot =
4351 Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
4352 let replay_handler = Arc::clone(&handler);
4353 let future = async move {
4354 let result = replay_handler(ctx, input.into_json()?, state).await?;
4355 AvroValue::from_serialize(&result)
4356 };
4357 ReplayedWorkflowInvocation {
4358 future: Box::pin(future),
4359 snapshot,
4360 }
4361 });
4362
4363 self.workflows.insert(
4364 workflow_type.into(),
4365 RegisteredWorkflow {
4366 execute,
4367 replay: Some(replay),
4368 state_type: Some(TypeId::of::<S>()),
4369 },
4370 );
4371 }
4372
4373 pub fn register_replayed_workflow_avro_value<S, Factory, F, Fut>(
4375 &mut self,
4376 workflow_type: impl Into<String>,
4377 state_factory: Factory,
4378 handler: F,
4379 ) where
4380 S: Clone + Send + Sync + 'static,
4381 Factory: Fn() -> S + Send + Sync + 'static,
4382 F: Fn(WorkflowContext, AvroValue, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
4383 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4384 {
4385 let state_factory = Arc::new(state_factory);
4386 let handler = Arc::new(handler);
4387
4388 let execute_factory = Arc::clone(&state_factory);
4389 let execute_handler = Arc::clone(&handler);
4390 let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4391 let state = WorkflowInstance::new(execute_factory());
4392 Box::pin(execute_handler(ctx, input, state)) as WorkflowFuture
4393 });
4394
4395 let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4396 let state = WorkflowInstance::new(state_factory());
4397 let snapshot_state = state.clone();
4398 let snapshot: WorkflowStateSnapshot =
4399 Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
4400 ReplayedWorkflowInvocation {
4401 future: Box::pin(handler(ctx, input, state)),
4402 snapshot,
4403 }
4404 });
4405
4406 self.workflows.insert(
4407 workflow_type.into(),
4408 RegisteredWorkflow {
4409 execute,
4410 replay: Some(replay),
4411 state_type: Some(TypeId::of::<S>()),
4412 },
4413 );
4414 }
4415
4416 pub fn register_activity<F, Fut>(&mut self, activity_type: impl Into<String>, handler: F)
4417 where
4418 F: Fn(ActivityContext, Value) -> Fut + Send + Sync + 'static,
4419 Fut: Future<Output = Result<Value>> + Send + 'static,
4420 {
4421 let handler = Arc::new(handler);
4422 self.activities.insert(
4423 activity_type.into(),
4424 Arc::new(move |ctx, args| {
4425 let handler = Arc::clone(&handler);
4426 Box::pin(async move {
4427 let result = handler(ctx, args.into_json()?).await?;
4428 AvroValue::from_serialize(&result)
4429 })
4430 }),
4431 );
4432 }
4433
4434 pub fn register_activity_avro_value<F, Fut>(
4436 &mut self,
4437 activity_type: impl Into<String>,
4438 handler: F,
4439 ) where
4440 F: Fn(ActivityContext, AvroValue) -> Fut + Send + Sync + 'static,
4441 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4442 {
4443 self.activities.insert(
4444 activity_type.into(),
4445 Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
4446 );
4447 }
4448
4449 pub fn register_query<F, Fut>(
4455 &mut self,
4456 workflow_type: impl Into<String>,
4457 query_name: impl Into<String>,
4458 handler: F,
4459 ) where
4460 F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
4461 Fut: Future<Output = Result<Value>> + Send + 'static,
4462 {
4463 let handler = Arc::new(handler);
4464 self.queries
4465 .entry(workflow_type.into())
4466 .or_default()
4467 .insert(
4468 query_name.into(),
4469 RegisteredQuery::Snapshot(Arc::new(move |ctx, args| {
4470 let handler = Arc::clone(&handler);
4471 Box::pin(async move {
4472 let result = handler(ctx, args.into_json()?).await?;
4473 AvroValue::from_serialize(&result)
4474 })
4475 })),
4476 );
4477 }
4478
4479 pub fn register_query_avro_value<F, Fut>(
4481 &mut self,
4482 workflow_type: impl Into<String>,
4483 query_name: impl Into<String>,
4484 handler: F,
4485 ) where
4486 F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
4487 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4488 {
4489 self.queries
4490 .entry(workflow_type.into())
4491 .or_default()
4492 .insert(
4493 query_name.into(),
4494 RegisteredQuery::Snapshot(Arc::new(move |ctx, args| Box::pin(handler(ctx, args)))),
4495 );
4496 }
4497
4498 pub fn register_replayed_query<S, F, Fut>(
4505 &mut self,
4506 workflow_type: impl Into<String>,
4507 query_name: impl Into<String>,
4508 handler: F,
4509 ) where
4510 S: Clone + Send + Sync + 'static,
4511 F: Fn(QueryContext, Arc<S>, Value) -> Fut + Send + Sync + 'static,
4512 Fut: Future<Output = Result<Value>> + Send + 'static,
4513 {
4514 let handler = Arc::new(handler);
4515 let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
4516 let state = state.downcast::<S>().map_err(|_| {
4517 "registered query state type does not match the replayed workflow state".to_string()
4518 })?;
4519 let handler = Arc::clone(&handler);
4520 Ok(Box::pin(async move {
4521 let result = handler(ctx, state, args.into_json()?).await?;
4522 AvroValue::from_serialize(&result)
4523 }))
4524 });
4525
4526 self.queries
4527 .entry(workflow_type.into())
4528 .or_default()
4529 .insert(
4530 query_name.into(),
4531 RegisteredQuery::Replayed {
4532 state_type: TypeId::of::<S>(),
4533 handler: erased_handler,
4534 },
4535 );
4536 }
4537
4538 pub fn register_replayed_query_avro_value<S, F, Fut>(
4540 &mut self,
4541 workflow_type: impl Into<String>,
4542 query_name: impl Into<String>,
4543 handler: F,
4544 ) where
4545 S: Clone + Send + Sync + 'static,
4546 F: Fn(QueryContext, Arc<S>, AvroValue) -> Fut + Send + Sync + 'static,
4547 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4548 {
4549 let handler = Arc::new(handler);
4550 let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
4551 let state = state.downcast::<S>().map_err(|_| {
4552 "registered query state type does not match the replayed workflow state".to_string()
4553 })?;
4554 Ok(Box::pin(handler(ctx, state, args)))
4555 });
4556
4557 self.queries
4558 .entry(workflow_type.into())
4559 .or_default()
4560 .insert(
4561 query_name.into(),
4562 RegisteredQuery::Replayed {
4563 state_type: TypeId::of::<S>(),
4564 handler: erased_handler,
4565 },
4566 );
4567 }
4568
4569 pub fn register_update<F, Fut>(
4571 &mut self,
4572 workflow_type: impl Into<String>,
4573 update_name: impl Into<String>,
4574 handler: F,
4575 ) where
4576 F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
4577 Fut: Future<Output = Result<Value>> + Send + 'static,
4578 {
4579 let handler = Arc::new(handler);
4580 self.updates
4581 .entry(workflow_type.into())
4582 .or_default()
4583 .insert(
4584 update_name.into(),
4585 Arc::new(move |ctx, args| {
4586 let handler = Arc::clone(&handler);
4587 Box::pin(async move {
4588 let result = handler(ctx, args.into_json()?).await?;
4589 AvroValue::from_serialize(&result)
4590 })
4591 }),
4592 );
4593 }
4594
4595 pub fn register_update_avro_value<F, Fut>(
4597 &mut self,
4598 workflow_type: impl Into<String>,
4599 update_name: impl Into<String>,
4600 handler: F,
4601 ) where
4602 F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
4603 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4604 {
4605 self.updates
4606 .entry(workflow_type.into())
4607 .or_default()
4608 .insert(
4609 update_name.into(),
4610 Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
4611 );
4612 }
4613
4614 pub async fn register(&self) -> Result<RegisterWorkerResponse> {
4615 let mut command_contracts = serde_json::Map::new();
4616 for workflow_type in self.workflows.keys() {
4617 let mut queries = self
4618 .queries
4619 .get(workflow_type)
4620 .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
4621 .unwrap_or_default();
4622 queries.sort();
4623 let mut updates = self
4624 .updates
4625 .get(workflow_type)
4626 .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
4627 .unwrap_or_default();
4628 updates.sort();
4629 if !queries.is_empty() || !updates.is_empty() {
4630 command_contracts.insert(
4631 workflow_type.clone(),
4632 json!({
4633 "queries": queries,
4634 "updates": updates,
4635 "update_validators": [],
4636 }),
4637 );
4638 }
4639 }
4640
4641 self.client
4642 .register_worker_with_command_contracts(
4643 &self.worker_id,
4644 &self.task_queue,
4645 self.workflows.keys().cloned().collect(),
4646 self.activities.keys().cloned().collect(),
4647 self.max_concurrent_workflow_tasks,
4648 self.max_concurrent_activity_tasks,
4649 [
4650 (!self.queries.is_empty()).then(|| QUERY_TASKS_CAPABILITY.to_string()),
4651 (!self.updates.is_empty()).then(|| WORKFLOW_UPDATES_CAPABILITY.to_string()),
4652 ]
4653 .into_iter()
4654 .flatten()
4655 .collect(),
4656 Value::Object(command_contracts),
4657 )
4658 .await
4659 }
4660
4661 pub async fn run(&self) -> Result<()> {
4667 self.run_until(std::future::pending::<()>()).await
4668 }
4669
4670 pub async fn run_until<F>(&self, shutdown: F) -> Result<()>
4674 where
4675 F: Future<Output = ()>,
4676 {
4677 let registration = self.register().await?;
4678 if !registration.registered {
4679 return Err(Error::WorkerLoop(format!(
4680 "worker registration for {:?} was not accepted",
4681 self.worker_id
4682 )));
4683 }
4684 let registered_worker_id = registration.worker_id.clone();
4685 let primary = self.run_registered_until(shutdown, registration).await;
4686 let deregistration = self
4687 .client
4688 .deregister_worker_registration(®istered_worker_id)
4689 .await;
4690
4691 match (primary, deregistration) {
4692 (Ok(()), Ok(_)) => Ok(()),
4693 (Ok(()), Err(deregistration)) => Err(deregistration),
4694 (Err(primary), Ok(_)) => Err(primary),
4695 (Err(primary), Err(deregistration)) => Err(Error::WorkerShutdown {
4696 primary: Box::new(primary),
4697 deregistration: Box::new(deregistration),
4698 }),
4699 }
4700 }
4701
4702 async fn run_registered_until<F>(
4703 &self,
4704 shutdown: F,
4705 registration: RegisterWorkerResponse,
4706 ) -> Result<()>
4707 where
4708 F: Future<Output = ()>,
4709 {
4710 let heartbeat_interval = Duration::from_secs(
4711 registration
4712 .heartbeat_interval_seconds
4713 .unwrap_or(self.heartbeat_interval.as_secs().max(1)),
4714 );
4715 let heartbeat = tokio::time::sleep(Duration::ZERO);
4721 tokio::pin!(heartbeat);
4722 tokio::pin!(shutdown);
4723 let stop = Arc::new(AtomicBool::new(false));
4724 let mut workflow_poller = (!self.workflows.is_empty()).then(|| {
4728 let worker = self.clone();
4729 let stop = Arc::clone(&stop);
4730 tokio::spawn(async move { worker.poll_workflows_until_stopped(stop).await })
4731 });
4732 let mut activity_poller = (!self.activities.is_empty()).then(|| {
4733 let worker = self.clone();
4734 let stop = Arc::clone(&stop);
4735 tokio::spawn(async move { worker.poll_activities_until_stopped(stop).await })
4736 });
4737 let mut query_poller = (!self.queries.is_empty()).then(|| {
4738 let worker = self.clone();
4739 let stop = Arc::clone(&stop);
4740 tokio::spawn(async move { worker.poll_queries_until_stopped(stop).await })
4741 });
4742
4743 loop {
4744 tokio::select! {
4745 _ = &mut shutdown => {
4746 stop.store(true, Ordering::SeqCst);
4747 break;
4748 }
4749 _ = &mut heartbeat => {
4750 let result = self.retry_worker_operation(|| {
4751 self.client.heartbeat_worker(
4752 &self.worker_id,
4753 self.max_concurrent_workflow_tasks,
4754 self.max_concurrent_activity_tasks,
4755 )
4756 }).await;
4757 heartbeat
4758 .as_mut()
4759 .reset(tokio::time::Instant::now() + heartbeat_interval);
4760 match result {
4761 Ok(acknowledgement) => {
4762 if let Some(observer) = &self.heartbeat_observer {
4763 observer(&WorkerHeartbeatObservation {
4764 worker_id: self.worker_id.clone(),
4765 task_queue: self.task_queue.clone(),
4766 acknowledged_at_unix_millis: SystemTime::now()
4767 .duration_since(UNIX_EPOCH)
4768 .unwrap_or_default()
4769 .as_millis()
4770 .min(u64::MAX as u128)
4771 as u64,
4772 acknowledgement,
4773 });
4774 }
4775 }
4776 Err(error) => {
4777 stop.store(true, Ordering::SeqCst);
4778 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await?;
4779 return Err(error);
4780 }
4781 }
4782 }
4783 result = OptionFuture::from(workflow_poller.as_mut()), if workflow_poller.is_some() => {
4784 workflow_poller = None;
4785 let stopped_by_server = stop.load(Ordering::SeqCst);
4786 stop.store(true, Ordering::SeqCst);
4787 let poller_result = optional_poller_result("workflow", result);
4788 let join_result =
4789 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
4790 poller_result?;
4791 join_result?;
4792 if stopped_by_server {
4793 return Ok(());
4794 }
4795 return Err(Error::WorkerLoop(
4796 "workflow poller stopped unexpectedly".to_string(),
4797 ));
4798 }
4799 result = OptionFuture::from(activity_poller.as_mut()), if activity_poller.is_some() => {
4800 activity_poller = None;
4801 let stopped_by_server = stop.load(Ordering::SeqCst);
4802 stop.store(true, Ordering::SeqCst);
4803 let poller_result = optional_poller_result("activity", result);
4804 let join_result =
4805 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
4806 poller_result?;
4807 join_result?;
4808 if stopped_by_server {
4809 return Ok(());
4810 }
4811 return Err(Error::WorkerLoop(
4812 "activity poller stopped unexpectedly".to_string(),
4813 ));
4814 }
4815 result = OptionFuture::from(query_poller.as_mut()), if query_poller.is_some() => {
4816 query_poller = None;
4817 let stopped_by_server = stop.load(Ordering::SeqCst);
4818 stop.store(true, Ordering::SeqCst);
4819 let poller_result = optional_poller_result("query", result);
4820 let join_result =
4821 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
4822 poller_result?;
4823 join_result?;
4824 if stopped_by_server {
4825 return Ok(());
4826 }
4827 return Err(Error::WorkerLoop(
4828 "query poller stopped unexpectedly".to_string(),
4829 ));
4830 }
4831 }
4832 }
4833
4834 join_pollers(
4835 workflow_poller.take(),
4836 activity_poller.take(),
4837 query_poller.take(),
4838 )
4839 .await
4840 }
4841
4842 pub async fn run_once(&self) -> Result<usize> {
4856 let mut handled = 0;
4857 match self.poll_workflow_once().await? {
4858 ManagedPollOutcome::Handled => handled += 1,
4859 ManagedPollOutcome::Stop => return Ok(handled),
4860 ManagedPollOutcome::Idle => {}
4861 }
4862 match self.poll_activity_once().await? {
4863 ManagedPollOutcome::Handled => handled += 1,
4864 ManagedPollOutcome::Stop => return Ok(handled),
4865 ManagedPollOutcome::Idle => {}
4866 }
4867 if !self.queries.is_empty() {
4868 match self.poll_query_once().await? {
4869 ManagedPollOutcome::Handled => handled += 1,
4870 ManagedPollOutcome::Stop => return Ok(handled),
4871 ManagedPollOutcome::Idle => {}
4872 }
4873 }
4874 Ok(handled)
4875 }
4876
4877 async fn poll_workflow_once(&self) -> Result<ManagedPollOutcome> {
4878 let poll_request_id = unique_request_id("rust-workflow-poll");
4879 let response = self
4880 .retry_worker_operation(|| {
4881 self.client.poll_workflow_task_response_with_request_id(
4882 &self.worker_id,
4883 &self.task_queue,
4884 self.poll_timeout,
4885 &poll_request_id,
4886 0,
4887 )
4888 })
4889 .await?;
4890 if response.outcome().should_stop() {
4891 return Ok(ManagedPollOutcome::Stop);
4892 }
4893 let Some(task) = response.task else {
4894 return Ok(ManagedPollOutcome::Idle);
4895 };
4896
4897 let task_id = task.task_id.clone();
4898 let attempt = task.workflow_task_attempt;
4899 let run_id = task.run_id.clone();
4900 let lease_owner = task
4901 .lease_owner
4902 .clone()
4903 .unwrap_or_else(|| self.worker_id.clone());
4904
4905 match self.execute_workflow_task(task) {
4906 Ok(commands) if commands.is_empty() => {
4907 self.client
4913 .fail_workflow_task_with_type(
4914 &task_id,
4915 &lease_owner,
4916 attempt,
4917 WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE,
4918 WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE,
4919 )
4920 .await?;
4921 }
4922 Ok(commands) => {
4923 let completion = self
4924 .client
4925 .complete_workflow_task(&task_id, &lease_owner, attempt, commands)
4926 .await;
4927 if let Err(error) = completion {
4928 if !workflow_task_completion_is_terminal_timeout(
4929 &error,
4930 &task_id,
4931 attempt,
4932 run_id.as_deref(),
4933 ) {
4934 return Err(error);
4935 }
4936 }
4937 }
4938 Err(error) => {
4939 self.client
4940 .fail_workflow_task(&task_id, &lease_owner, attempt, error.to_string())
4941 .await?;
4942 }
4943 }
4944
4945 Ok(ManagedPollOutcome::Handled)
4946 }
4947
4948 async fn poll_workflows_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
4949 while !stop.load(Ordering::SeqCst) {
4950 if self.poll_workflow_once().await? == ManagedPollOutcome::Stop {
4951 stop.store(true, Ordering::SeqCst);
4952 break;
4953 }
4954 }
4955
4956 Ok(())
4957 }
4958
4959 async fn poll_activity_once(&self) -> Result<ManagedPollOutcome> {
4960 let poll_request_id = unique_request_id("rust-activity-poll");
4961 let response = self
4962 .retry_worker_operation(|| {
4963 self.client.poll_activity_task_response_with_request_id(
4964 &self.worker_id,
4965 &self.task_queue,
4966 self.poll_timeout,
4967 &poll_request_id,
4968 0,
4969 )
4970 })
4971 .await?;
4972 if response.outcome().should_stop() {
4973 return Ok(ManagedPollOutcome::Stop);
4974 }
4975 let Some(task) = response.task else {
4976 return Ok(ManagedPollOutcome::Idle);
4977 };
4978
4979 let task_id = task.task_id.clone();
4980 let attempt_id = task
4981 .activity_attempt_id
4982 .clone()
4983 .or(task.attempt_id.clone())
4984 .unwrap_or_default();
4985 let lease_owner = task
4986 .lease_owner
4987 .clone()
4988 .unwrap_or_else(|| self.worker_id.clone());
4989 let codec = task.payload_codec.clone();
4990 let result = self.execute_activity_task(task).await;
4991 match result {
4992 Ok(value) => {
4993 let completion = self
4994 .client
4995 .complete_activity_task(&task_id, &attempt_id, &lease_owner, value, &codec)
4996 .await;
4997 if let Err(error) = completion {
4998 if !activity_task_rejection_is_final(&error) {
4999 return Err(error);
5000 }
5001 }
5002 }
5003 Err(error) => {
5004 let failure = self
5005 .client
5006 .fail_activity_task(
5007 &task_id,
5008 &attempt_id,
5009 &lease_owner,
5010 error.to_string(),
5011 false,
5012 )
5013 .await;
5014 if let Err(error) = failure {
5015 if !activity_task_rejection_is_final(&error) {
5016 return Err(error);
5017 }
5018 }
5019 }
5020 }
5021
5022 Ok(ManagedPollOutcome::Handled)
5023 }
5024
5025 async fn poll_activities_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
5026 while !stop.load(Ordering::SeqCst) {
5027 if self.poll_activity_once().await? == ManagedPollOutcome::Stop {
5028 stop.store(true, Ordering::SeqCst);
5029 break;
5030 }
5031 }
5032
5033 Ok(())
5034 }
5035
5036 async fn poll_query_once(&self) -> Result<ManagedPollOutcome> {
5037 let poll_request_id = unique_request_id("rust-query-poll");
5038 let response = self
5039 .retry_worker_operation(|| {
5040 self.client.poll_query_task_response_with_request_id(
5041 &self.worker_id,
5042 &self.task_queue,
5043 self.poll_timeout,
5044 &poll_request_id,
5045 0,
5046 )
5047 })
5048 .await?;
5049 if response.outcome().should_stop() {
5050 return Ok(ManagedPollOutcome::Stop);
5051 }
5052 let Some(task) = response.task else {
5053 return Ok(ManagedPollOutcome::Idle);
5054 };
5055
5056 let query_task_id = task.query_task_id.clone();
5057 let attempt = task.query_task_attempt;
5058 let lease_owner = task
5059 .lease_owner
5060 .clone()
5061 .unwrap_or_else(|| self.worker_id.clone());
5062 let codec = task.payload_codec.clone();
5063
5064 match self.execute_query_task(task).await {
5065 Ok(value) => {
5066 let result_envelope = match encode_typed_envelope(&value, &codec) {
5067 Ok(result_envelope) => result_envelope,
5068 Err(error) => {
5069 let failure = self
5070 .client
5071 .fail_query_task(
5072 &query_task_id,
5073 &lease_owner,
5074 attempt,
5075 error.to_string(),
5076 "query_result_encode_failed",
5077 "QueryResultEncodeFailed",
5078 )
5079 .await;
5080 if let Err(error) = failure {
5081 if !query_task_rejection_is_final(&error) {
5082 return Err(error);
5083 }
5084 }
5085 return Ok(ManagedPollOutcome::Handled);
5086 }
5087 };
5088
5089 if let Err(error) = self
5090 .client
5091 .complete_query_task_with_envelope(
5092 &query_task_id,
5093 &lease_owner,
5094 attempt,
5095 value.clone().into_json()?,
5096 result_envelope,
5097 )
5098 .await
5099 {
5100 if !query_task_rejection_is_final(&error) {
5101 return Err(error);
5102 }
5103 }
5104 }
5105 Err(failure) => {
5106 let result = self
5107 .client
5108 .fail_query_task(
5109 &query_task_id,
5110 &lease_owner,
5111 attempt,
5112 failure.message,
5113 failure.reason,
5114 failure.failure_type,
5115 )
5116 .await;
5117 if let Err(error) = result {
5118 if !query_task_rejection_is_final(&error) {
5119 return Err(error);
5120 }
5121 }
5122 }
5123 }
5124
5125 Ok(ManagedPollOutcome::Handled)
5126 }
5127
5128 async fn poll_queries_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
5129 while !stop.load(Ordering::SeqCst) {
5130 if self.poll_query_once().await? == ManagedPollOutcome::Stop {
5131 stop.store(true, Ordering::SeqCst);
5132 break;
5133 }
5134 }
5135
5136 Ok(())
5137 }
5138
5139 async fn retry_worker_operation<T, F, Fut>(&self, mut operation: F) -> Result<T>
5140 where
5141 F: FnMut() -> Fut,
5142 Fut: Future<Output = Result<T>>,
5143 {
5144 let mut retries = 0;
5145
5146 loop {
5147 match operation().await {
5148 Err(error)
5149 if worker_operation_is_retryable(&error)
5150 && retries < self.retry_policy.max_retries =>
5151 {
5152 retries += 1;
5153 tokio::time::sleep(worker_retry_delay(self.retry_policy, retries)).await;
5154 }
5155 result => return result,
5156 }
5157 }
5158 }
5159
5160 async fn execute_query_task(
5161 &self,
5162 mut task: QueryTask,
5163 ) -> std::result::Result<AvroValue, QueryTaskExecutionFailure> {
5164 validate_query_task_payloads(&task).map_err(|error| {
5165 QueryTaskExecutionFailure::new(
5166 "query_payload_decode_failed",
5167 error.to_string(),
5168 "QueryPayloadDecodeFailed",
5169 )
5170 })?;
5171
5172 if !self.workflows.contains_key(&task.workflow_type) {
5173 return Err(QueryTaskExecutionFailure::new(
5174 "query_workflow_type_not_registered",
5175 format!("no workflow registered for type {:?}", task.workflow_type),
5176 "WorkflowTypeNotRegistered",
5177 ));
5178 }
5179
5180 let Some(handlers) = self.queries.get(&task.workflow_type) else {
5181 return Err(QueryTaskExecutionFailure::new(
5182 "query_handler_unavailable",
5183 format!(
5184 "query handlers are unavailable for workflow type {:?}",
5185 task.workflow_type
5186 ),
5187 "QueryHandlerUnavailable",
5188 ));
5189 };
5190 let Some(query) = handlers.get(&task.query_name) else {
5191 return Err(QueryTaskExecutionFailure::new(
5192 "rejected_unknown_query",
5193 format!("unknown query {:?}", task.query_name),
5194 "QueryFailed",
5195 ));
5196 };
5197
5198 let args = decode_task_avro_arguments(task.query_arguments.as_ref(), &task.payload_codec)
5199 .map_err(|error| {
5200 QueryTaskExecutionFailure::new(
5201 "query_payload_decode_failed",
5202 format!("cannot decode query arguments: {error}"),
5203 "QueryPayloadDecodeFailed",
5204 )
5205 })?;
5206 let workflow_input_typed =
5207 decode_task_avro_arguments(task.workflow_arguments.as_ref(), &task.payload_codec)
5208 .map_err(|error| {
5209 QueryTaskExecutionFailure::new(
5210 "query_workflow_state_unavailable",
5211 format!("cannot decode workflow start input: {error}"),
5212 "QueryWorkflowStateUnavailable",
5213 )
5214 })?;
5215 let workflow_input = workflow_input_typed.clone().into_json().map_err(|error| {
5216 QueryTaskExecutionFailure::new(
5217 "query_workflow_state_unavailable",
5218 format!("cannot project workflow start input: {error}"),
5219 "QueryWorkflowStateUnavailable",
5220 )
5221 })?;
5222 hydrate_query_history_from_export(&mut task).map_err(|error| {
5223 QueryTaskExecutionFailure::new(
5224 "query_workflow_state_unavailable",
5225 format!("cannot restore query history snapshot: {error}"),
5226 "QueryWorkflowStateUnavailable",
5227 )
5228 })?;
5229 enrich_query_history_from_export(&mut task).map_err(|error| {
5230 QueryTaskExecutionFailure::new(
5231 "query_workflow_state_unavailable",
5232 format!("cannot restore compact query history payloads: {error}"),
5233 "QueryWorkflowStateUnavailable",
5234 )
5235 })?;
5236 let signal_events = query_signal_events(&task).map_err(|error| {
5237 QueryTaskExecutionFailure::new(
5238 "query_workflow_state_unavailable",
5239 format!("cannot decode committed workflow signals: {error}"),
5240 "QueryWorkflowStateUnavailable",
5241 )
5242 })?;
5243 let history_events = Arc::new(std::mem::take(&mut task.history_events));
5244 let context = QueryContext {
5245 workflow_id: task.workflow_id,
5246 run_id: task.run_id,
5247 workflow_type: task.workflow_type.clone(),
5248 run_status: task.run_status,
5249 workflow_input,
5250 workflow_input_avro_value: workflow_input_typed.clone(),
5251 history_events: Arc::clone(&history_events),
5252 signal_events: Arc::new(signal_events),
5253 };
5254
5255 let future = match query {
5256 RegisteredQuery::Snapshot(handler) => handler(context, args),
5257 RegisteredQuery::Replayed {
5258 state_type,
5259 handler,
5260 } => {
5261 let workflow = self
5262 .workflows
5263 .get(&task.workflow_type)
5264 .expect("workflow registration was checked above");
5265 if workflow.state_type != Some(*state_type) {
5266 return Err(QueryTaskExecutionFailure::new(
5267 "query_workflow_state_unavailable",
5268 "replayed query state type does not match its workflow registration",
5269 "QueryWorkflowStateUnavailable",
5270 ));
5271 }
5272 let replay = workflow.replay.as_ref().ok_or_else(|| {
5273 QueryTaskExecutionFailure::new(
5274 "query_workflow_state_unavailable",
5275 format!(
5276 "workflow type {:?} is not registered for instance-state replay",
5277 task.workflow_type
5278 ),
5279 "QueryWorkflowStateUnavailable",
5280 )
5281 })?;
5282 let workflow_state = Arc::new(Mutex::new(
5283 WorkflowState::new_with_identity(
5284 history_events.as_ref().clone(),
5285 context.workflow_id.clone(),
5286 context.run_id.clone(),
5287 self.task_queue.clone(),
5288 task.payload_codec,
5289 None,
5290 )
5291 .map_err(|error| {
5292 QueryTaskExecutionFailure::new(
5293 "query_workflow_state_unavailable",
5294 format!("workflow replay failed before query: {error}"),
5295 "QueryWorkflowStateUnavailable",
5296 )
5297 })?,
5298 ));
5299 let workflow_context = WorkflowContext {
5300 state: workflow_state,
5301 };
5302 let mut invocation = replay(workflow_context.clone(), workflow_input_typed.clone());
5303 let mut cx = TaskContext::from_waker(noop_waker_ref());
5304 match invocation.future.as_mut().poll(&mut cx) {
5305 Poll::Ready(Ok(_)) => {
5306 workflow_context
5307 .ensure_history_consumed()
5308 .map_err(|error| {
5309 QueryTaskExecutionFailure::new(
5310 "query_workflow_state_unavailable",
5311 format!("workflow replay failed before query: {error}"),
5312 "QueryWorkflowStateUnavailable",
5313 )
5314 })?;
5315 }
5316 Poll::Ready(Err(error)) => {
5317 return Err(QueryTaskExecutionFailure::new(
5318 "query_workflow_state_unavailable",
5319 format!("workflow replay failed before query: {error}"),
5320 "QueryWorkflowStateUnavailable",
5321 ));
5322 }
5323 Poll::Pending => {
5324 let commands = workflow_context.take_commands().map_err(|error| {
5325 QueryTaskExecutionFailure::new(
5326 "query_workflow_state_unavailable",
5327 format!("workflow replay failed before query: {error}"),
5328 "QueryWorkflowStateUnavailable",
5329 )
5330 })?;
5331 if commands.is_empty()
5332 && !workflow_context
5333 .matched_recorded_pending()
5334 .map_err(|error| {
5335 QueryTaskExecutionFailure::new(
5336 "query_workflow_state_unavailable",
5337 format!("workflow replay failed before query: {error}"),
5338 "QueryWorkflowStateUnavailable",
5339 )
5340 })?
5341 {
5342 return Err(QueryTaskExecutionFailure::new(
5343 "query_workflow_state_unavailable",
5344 "workflow replay yielded without a durable command",
5345 "QueryWorkflowStateUnavailable",
5346 ));
5347 }
5348 }
5349 }
5350 let state = (invocation.snapshot)().map_err(|error| {
5351 QueryTaskExecutionFailure::new(
5352 "query_workflow_state_unavailable",
5353 format!("cannot snapshot replayed workflow state: {error}"),
5354 "QueryWorkflowStateUnavailable",
5355 )
5356 })?;
5357 handler(context, state, args).map_err(|message| {
5358 QueryTaskExecutionFailure::new(
5359 "query_workflow_state_unavailable",
5360 message,
5361 "QueryWorkflowStateUnavailable",
5362 )
5363 })?
5364 }
5365 };
5366
5367 future.await.map_err(|error| {
5368 QueryTaskExecutionFailure::new("query_rejected", error.to_string(), "QueryFailed")
5369 })
5370 }
5371
5372 fn execute_workflow_task(&self, task: WorkflowTask) -> Result<Vec<Value>> {
5373 validate_workflow_task_payloads(&task)?;
5374
5375 if let Some(update_id) = task
5376 .workflow_update_id
5377 .as_deref()
5378 .filter(|update_id| !update_id.is_empty())
5379 {
5380 return self.execute_update_task(&task, update_id);
5381 }
5382
5383 let workflow = self
5384 .workflows
5385 .get(&task.workflow_type)
5386 .ok_or_else(|| Error::WorkflowNotRegistered(task.workflow_type.clone()))?;
5387 let input = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
5388 let resume_signal = decode_resume_signal(&task)?;
5389 let history_budget = WorkflowHistoryBudget {
5390 event_count: task
5391 .total_history_events
5392 .unwrap_or_else(|| u64::try_from(task.history_events.len()).unwrap_or(u64::MAX)),
5393 size_bytes: task.history_size_bytes,
5394 continue_as_new_recommended: task.continue_as_new_recommended.unwrap_or(false),
5395 pressure: task.history_budget_pressure.clone(),
5396 };
5397 let mut workflow_state = WorkflowState::new_with_identity(
5398 task.history_events,
5399 task.workflow_id,
5400 task.run_id,
5401 self.task_queue.clone(),
5402 task.payload_codec.clone(),
5403 resume_signal,
5404 )?;
5405 workflow_state.history_budget = history_budget;
5406 let state = Arc::new(Mutex::new(workflow_state));
5407 let ctx = WorkflowContext { state };
5408 let mut future = (workflow.execute)(ctx.clone(), input);
5409 let mut cx = TaskContext::from_waker(noop_waker_ref());
5410
5411 match future.as_mut().poll(&mut cx) {
5412 Poll::Ready(Ok(result)) => {
5413 ctx.ensure_history_consumed()?;
5414 let result = encode_typed_envelope(&result, &task.payload_codec)?;
5415 let mut commands = ctx.take_commands()?;
5416 commands.push(json!({
5417 "type": "complete_workflow",
5418 "result": result
5419 }));
5420 Ok(commands)
5421 }
5422 Poll::Ready(Err(error)) => {
5423 if let Error::ContinueAsNew(request) = error {
5424 let mut commands = ctx.take_commands()?;
5425 if let Some(command) = ctx.continue_as_new_command(request)? {
5426 commands.push(command);
5427 }
5428 ctx.ensure_history_consumed()?;
5429 return Ok(commands);
5430 }
5431 ctx.ensure_history_consumed()?;
5434 if workflow_task_integrity_error(&error) {
5435 return Err(error);
5440 }
5441 let mut commands = ctx.take_commands()?;
5442 commands.push(workflow_failure_command(&error));
5443 Ok(commands)
5444 }
5445 Poll::Pending => {
5446 let commands = ctx.take_commands()?;
5447 if commands.is_empty() && !ctx.matched_recorded_pending()? {
5448 Err(Error::WorkflowYieldedWithoutCommand)
5449 } else {
5450 Ok(commands)
5451 }
5452 }
5453 }
5454 }
5455
5456 fn execute_update_task(&self, task: &WorkflowTask, update_id: &str) -> Result<Vec<Value>> {
5457 if !self.workflows.contains_key(&task.workflow_type) {
5458 return Err(Error::WorkflowNotRegistered(task.workflow_type.clone()));
5459 }
5460
5461 let accepted = task.history_events.iter().rev().find_map(|event| {
5462 (event.event_type == "UpdateAccepted"
5463 && event.payload.get("update_id").and_then(Value::as_str) == Some(update_id))
5464 .then_some(&event.payload)
5465 });
5466 let update_name = accepted
5467 .and_then(|payload| payload.get("update_name"))
5468 .and_then(Value::as_str)
5469 .or(task.update_name.as_deref())
5470 .unwrap_or_default();
5471 let Some(handler) = self
5472 .updates
5473 .get(&task.workflow_type)
5474 .and_then(|handlers| handlers.get(update_name))
5475 else {
5476 return Ok(vec![json!({
5477 "type": "fail_update",
5478 "update_id": update_id,
5479 "message": format!(
5480 "no update handler is registered for {}.{update_name}",
5481 task.workflow_type
5482 ),
5483 "exception_type": "UnknownUpdate",
5484 "non_retryable": true,
5485 })]);
5486 };
5487 let arguments = accepted
5488 .and_then(|payload| payload.get("arguments"))
5489 .or(task.arguments.as_ref());
5490 let arguments = decode_task_avro_arguments(arguments, &task.payload_codec)?;
5491 let context = QueryContext {
5492 workflow_id: task.workflow_id.clone(),
5493 run_id: task.run_id.clone(),
5494 workflow_type: task.workflow_type.clone(),
5495 run_status: Some("running".to_string()),
5496 workflow_input: Value::Null,
5497 workflow_input_avro_value: AvroValue::Null,
5498 history_events: Arc::new(task.history_events.clone()),
5499 signal_events: Arc::new(Vec::new()),
5500 };
5501 let mut future = handler(context, arguments);
5502 let mut cx = TaskContext::from_waker(noop_waker_ref());
5503
5504 match future.as_mut().poll(&mut cx) {
5505 Poll::Ready(Ok(result)) => Ok(vec![json!({
5506 "type": "complete_update",
5507 "update_id": update_id,
5508 "result": encode_typed_envelope(&result, &task.payload_codec)?,
5509 })]),
5510 Poll::Ready(Err(error)) => Ok(vec![json!({
5511 "type": "fail_update",
5512 "update_id": update_id,
5513 "message": error.to_string(),
5514 "exception_type": "UpdateFailed",
5515 "non_retryable": true,
5516 })]),
5517 Poll::Pending => Err(Error::WorkflowYieldedWithoutCommand),
5518 }
5519 }
5520
5521 async fn execute_activity_task(&self, task: ActivityTask) -> Result<AvroValue> {
5522 validate_activity_task_payloads(&task)?;
5523
5524 let handler = self
5525 .activities
5526 .get(&task.activity_type)
5527 .ok_or_else(|| Error::ActivityNotRegistered(task.activity_type.clone()))?;
5528 let args = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
5529 let attempt_id = task
5530 .activity_attempt_id
5531 .clone()
5532 .or(task.attempt_id.clone())
5533 .unwrap_or_default();
5534 let lease_owner = task
5535 .lease_owner
5536 .clone()
5537 .unwrap_or_else(|| self.worker_id.clone());
5538 let ctx = ActivityContext {
5539 client: self.client.clone(),
5540 task_id: task.task_id,
5541 activity_attempt_id: attempt_id,
5542 lease_owner,
5543 activity_type: task.activity_type,
5544 attempt_number: task.attempt_number,
5545 task_queue: self.task_queue.clone(),
5546 worker_id: self.worker_id.clone(),
5547 };
5548
5549 handler(ctx, args).await
5550 }
5551}
5552
5553fn poller_result(
5554 kind: &str,
5555 result: std::result::Result<Result<()>, tokio::task::JoinError>,
5556) -> Result<()> {
5557 match result {
5558 Ok(result) => result,
5559 Err(error) => Err(Error::WorkerLoop(format!(
5560 "{kind} poller join error: {error}"
5561 ))),
5562 }
5563}
5564
5565fn optional_poller_result(
5566 kind: &str,
5567 result: Option<std::result::Result<Result<()>, tokio::task::JoinError>>,
5568) -> Result<()> {
5569 match result {
5570 Some(result) => poller_result(kind, result),
5571 None => Ok(()),
5572 }
5573}
5574
5575async fn join_pollers(
5576 workflow_poller: Option<tokio::task::JoinHandle<Result<()>>>,
5577 activity_poller: Option<tokio::task::JoinHandle<Result<()>>>,
5578 query_poller: Option<tokio::task::JoinHandle<Result<()>>>,
5579) -> Result<()> {
5580 let mut first_error = None;
5581
5582 if let Some(handle) = workflow_poller {
5583 if let Err(error) = poller_result("workflow", handle.await) {
5584 first_error.get_or_insert(error);
5585 }
5586 }
5587
5588 if let Some(handle) = activity_poller {
5589 if let Err(error) = poller_result("activity", handle.await) {
5590 first_error.get_or_insert(error);
5591 }
5592 }
5593
5594 if let Some(handle) = query_poller {
5595 if let Err(error) = poller_result("query", handle.await) {
5596 first_error.get_or_insert(error);
5597 }
5598 }
5599
5600 if let Some(error) = first_error {
5601 Err(error)
5602 } else {
5603 Ok(())
5604 }
5605}
5606
5607fn default_worker_id() -> String {
5608 let millis = SystemTime::now()
5609 .duration_since(UNIX_EPOCH)
5610 .unwrap_or_default()
5611 .as_millis();
5612 format!("rust-worker-{}-{millis}", std::process::id())
5613}
5614
5615fn percent_encode_path_segment(segment: &str) -> String {
5616 const HEX: &[u8; 16] = b"0123456789ABCDEF";
5617 let mut encoded = String::with_capacity(segment.len());
5618
5619 for byte in segment.bytes() {
5620 if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') {
5621 encoded.push(char::from(byte));
5622 } else {
5623 encoded.push('%');
5624 encoded.push(char::from(HEX[(byte >> 4) as usize]));
5625 encoded.push(char::from(HEX[(byte & 0x0f) as usize]));
5626 }
5627 }
5628
5629 encoded
5630}
5631
5632fn unique_request_id(prefix: &str) -> String {
5633 let nanos = SystemTime::now()
5634 .duration_since(UNIX_EPOCH)
5635 .unwrap_or_default()
5636 .as_nanos();
5637 format!("{prefix}-{}-{nanos}", std::process::id())
5638}
5639
5640#[derive(Debug)]
5641struct QueryTaskExecutionFailure {
5642 reason: String,
5643 message: String,
5644 failure_type: String,
5645}
5646
5647impl QueryTaskExecutionFailure {
5648 fn new(
5649 reason: impl Into<String>,
5650 message: impl Into<String>,
5651 failure_type: impl Into<String>,
5652 ) -> Self {
5653 Self {
5654 reason: reason.into(),
5655 message: message.into(),
5656 failure_type: failure_type.into(),
5657 }
5658 }
5659}
5660
5661#[derive(Clone, Debug)]
5667pub struct WorkflowInstance<S> {
5668 state: Arc<Mutex<S>>,
5669}
5670
5671impl<S> WorkflowInstance<S> {
5672 fn new(state: S) -> Self {
5673 Self {
5674 state: Arc::new(Mutex::new(state)),
5675 }
5676 }
5677
5678 pub fn read<R>(&self, reader: impl FnOnce(&S) -> R) -> Result<R> {
5680 let state = self
5681 .state
5682 .lock()
5683 .map_err(|_| Error::WorkflowStatePoisoned)?;
5684 Ok(reader(&state))
5685 }
5686
5687 pub fn update<R>(&self, transition: impl FnOnce(&mut S) -> R) -> Result<R> {
5689 let mut state = self
5690 .state
5691 .lock()
5692 .map_err(|_| Error::WorkflowStatePoisoned)?;
5693 Ok(transition(&mut state))
5694 }
5695}
5696
5697impl<S: Clone> WorkflowInstance<S> {
5698 fn snapshot(&self) -> Result<S> {
5699 self.read(Clone::clone)
5700 }
5701}
5702
5703#[derive(Clone, Debug)]
5704pub struct WorkflowContext {
5705 state: Arc<Mutex<WorkflowState>>,
5706}
5707
5708impl WorkflowContext {
5709 pub fn workflow_identity(&self) -> Result<WorkflowIdentity> {
5711 let state = self
5712 .state
5713 .lock()
5714 .map_err(|_| Error::WorkflowStatePoisoned)?;
5715 Ok(WorkflowIdentity {
5716 workflow_id: state.workflow_id.clone(),
5717 run_id: state.run_id.clone(),
5718 })
5719 }
5720
5721 pub fn history_budget(&self) -> Result<WorkflowHistoryBudget> {
5723 let state = self
5724 .state
5725 .lock()
5726 .map_err(|_| Error::WorkflowStatePoisoned)?;
5727 Ok(state.history_budget.clone())
5728 }
5729
5730 pub fn continue_as_new<T: Serialize>(&self, args: T) -> Result<Value> {
5736 self.continue_as_new_with_options(ContinueAsNewOptions::new(), args)
5737 }
5738
5739 pub fn continue_as_new_with_options<T: Serialize>(
5741 &self,
5742 options: ContinueAsNewOptions,
5743 args: T,
5744 ) -> Result<Value> {
5745 options.validate()?;
5746 Err(Error::ContinueAsNew(ContinueAsNewRequest {
5747 arguments: normalize_avro_arguments(AvroValue::from_serialize(&args)?),
5748 options,
5749 }))
5750 }
5751
5752 pub fn activity<T: Serialize>(
5753 &self,
5754 activity_type: impl Into<String>,
5755 args: T,
5756 ) -> ActivityCall {
5757 self.activity_with_options(activity_type, ActivityOptions::new(), args)
5758 }
5759
5760 pub fn activity_on_queue<T, Q>(
5761 &self,
5762 activity_type: impl Into<String>,
5763 task_queue: Option<Q>,
5764 args: T,
5765 ) -> ActivityCall
5766 where
5767 T: Serialize,
5768 Q: Into<String>,
5769 {
5770 let mut options = ActivityOptions::new();
5771 options.task_queue = task_queue.map(Into::into);
5772 self.activity_with_options(activity_type, options, args)
5773 }
5774
5775 pub fn activity_with_options<T: Serialize>(
5813 &self,
5814 activity_type: impl Into<String>,
5815 options: ActivityOptions,
5816 args: T,
5817 ) -> ActivityCall {
5818 ActivityCall {
5819 ctx: self.clone(),
5820 activity_type: activity_type.into(),
5821 options,
5822 args: Some(AvroValue::from_serialize(&args)),
5823 scheduled: false,
5824 }
5825 }
5826
5827 pub async fn activity_avro_value<T: Serialize>(
5828 &self,
5829 activity_type: impl Into<String>,
5830 args: T,
5831 ) -> Result<AvroValue> {
5832 let mut call = self.activity(activity_type, args);
5833 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
5834 }
5835
5836 pub async fn activity_avro_value_with_options<T: Serialize>(
5837 &self,
5838 activity_type: impl Into<String>,
5839 options: ActivityOptions,
5840 args: T,
5841 ) -> Result<AvroValue> {
5842 let mut call = self.activity_with_options(activity_type, options, args);
5843 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
5844 }
5845
5846 pub fn wait_signal(&self, signal_name: impl Into<String>) -> SignalCall {
5847 SignalCall {
5848 ctx: self.clone(),
5849 signal_name: signal_name.into(),
5850 opened_wait: false,
5851 matched_pending: false,
5852 }
5853 }
5854
5855 pub async fn wait_signal_avro_value(
5856 &self,
5857 signal_name: impl Into<String>,
5858 ) -> Result<Vec<AvroValue>> {
5859 let mut call = self.wait_signal(signal_name);
5860 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
5861 }
5862
5863 pub fn sleep(&self, duration: Duration) -> TimerCall {
5884 let delay_seconds = duration
5885 .as_secs()
5886 .checked_add(u64::from(duration.subsec_nanos() > 0));
5887 TimerCall {
5888 ctx: self.clone(),
5889 delay_seconds,
5890 scheduled: false,
5891 matched_pending: false,
5892 }
5893 }
5894
5895 pub fn start_timer(&self, duration: Duration) -> TimerCall {
5897 self.sleep(duration)
5898 }
5899
5900 pub fn side_effect<T, F>(&self, callback: F) -> Result<T>
5907 where
5908 T: Serialize + DeserializeOwned,
5909 F: FnOnce() -> T,
5910 {
5911 {
5912 let mut state = self
5913 .state
5914 .lock()
5915 .map_err(|_| Error::WorkflowStatePoisoned)?;
5916 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
5917 return match recorded {
5918 RecordedCommand::SideEffect { sequence, value } => {
5919 state.command_cursor += 1;
5920 value.deserialize().map_err(|error| {
5921 Error::NonDeterministicReplay(ReplayFailure::new(
5922 "side_effect_type_mismatch",
5923 Some(sequence),
5924 Some(std::any::type_name::<T>().to_string()),
5925 Some(error.to_string()),
5926 "recorded side-effect value is incompatible with the requested Rust type",
5927 ))
5928 })
5929 }
5930 other => Err(command_mismatch(&other, "side effect")),
5931 };
5932 }
5933 }
5934
5935 let value = callback();
5936 let avro_value = AvroValue::from_serialize(&value)?;
5937 let mut state = self
5938 .state
5939 .lock()
5940 .map_err(|_| Error::WorkflowStatePoisoned)?;
5941 let result = encode_typed_envelope(&avro_value, &state.payload_codec)?;
5942 state.commands.push(json!({
5943 "type": "record_side_effect",
5944 "result": result,
5945 }));
5946 Ok(value)
5947 }
5948
5949 pub fn side_effect_avro_value<F>(&self, callback: F) -> Result<AvroValue>
5951 where
5952 F: FnOnce() -> AvroValue,
5953 {
5954 {
5955 let mut state = self
5956 .state
5957 .lock()
5958 .map_err(|_| Error::WorkflowStatePoisoned)?;
5959 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
5960 return match recorded {
5961 RecordedCommand::SideEffect { value, .. } => {
5962 state.command_cursor += 1;
5963 Ok(value)
5964 }
5965 other => Err(command_mismatch(&other, "side effect")),
5966 };
5967 }
5968 }
5969
5970 let value = callback();
5971 let mut state = self
5972 .state
5973 .lock()
5974 .map_err(|_| Error::WorkflowStatePoisoned)?;
5975 let result = encode_typed_envelope(&value, &state.payload_codec)?;
5976 state.commands.push(json!({
5977 "type": "record_side_effect",
5978 "result": result,
5979 }));
5980 Ok(value)
5981 }
5982
5983 pub fn uuid_v4(&self) -> Result<Uuid> {
5985 self.side_effect(Uuid::new_v4)
5986 }
5987
5988 pub fn get_version(
5991 &self,
5992 change_id: impl Into<String>,
5993 min_supported: i32,
5994 max_supported: i32,
5995 ) -> Result<i32> {
5996 let change_id = change_id.into();
5997 if change_id.trim().is_empty() {
5998 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5999 "version_change_id_invalid",
6000 None,
6001 Some("non-empty change ID".to_string()),
6002 Some(change_id),
6003 "version markers require a stable non-empty change ID",
6004 )));
6005 }
6006 if min_supported > max_supported {
6007 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
6008 "version_range_invalid",
6009 None,
6010 Some("min_supported <= max_supported".to_string()),
6011 Some(format!("{min_supported}..={max_supported}")),
6012 "version marker supported range is invalid",
6013 )));
6014 }
6015
6016 let mut state = self
6017 .state
6018 .lock()
6019 .map_err(|_| Error::WorkflowStatePoisoned)?;
6020 if let Some((version, sequence)) = state.version_markers.get(&change_id).copied() {
6021 ensure_version_supported(&change_id, version, min_supported, max_supported, sequence)?;
6022 return Ok(version);
6023 }
6024
6025 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6026 return match recorded {
6027 RecordedCommand::VersionMarker {
6028 sequence,
6029 change_id: recorded_change_id,
6030 version,
6031 ..
6032 } => {
6033 if recorded_change_id != change_id {
6034 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
6035 "version_change_id_mismatch",
6036 Some(sequence),
6037 Some(recorded_change_id),
6038 Some(change_id),
6039 "recorded version marker change ID differs from current workflow code",
6040 )));
6041 }
6042 ensure_version_supported(
6043 &change_id,
6044 version,
6045 min_supported,
6046 max_supported,
6047 sequence,
6048 )?;
6049 state.command_cursor += 1;
6050 state.version_markers.insert(change_id, (version, sequence));
6051 Ok(version)
6052 }
6053 other => Err(command_mismatch(
6054 &other,
6055 format!("version marker:{change_id}"),
6056 )),
6057 };
6058 }
6059
6060 let version = max_supported;
6061 state.commands.push(json!({
6062 "type": "record_version_marker",
6063 "change_id": change_id,
6064 "version": version,
6065 "min_supported": min_supported,
6066 "max_supported": max_supported,
6067 }));
6068 state.version_markers.insert(change_id, (version, 0));
6071 Ok(version)
6072 }
6073
6074 pub fn patched(&self, change_id: impl Into<String>) -> Result<bool> {
6076 Ok(self.get_version(change_id, -1, 1)? == 1)
6077 }
6078
6079 pub fn deprecate_patch(&self, change_id: impl Into<String>) -> Result<()> {
6081 self.get_version(change_id, -1, 1).map(|_| ())
6082 }
6083
6084 pub fn start_child_workflow<T: Serialize>(
6111 &self,
6112 workflow_type: impl Into<String>,
6113 options: ChildWorkflowOptions,
6114 args: T,
6115 ) -> ChildWorkflowCall {
6116 ChildWorkflowCall {
6117 ctx: self.clone(),
6118 workflow_type: workflow_type.into(),
6119 options,
6120 args: Some(AvroValue::from_serialize(&args)),
6121 scheduled: false,
6122 matched_pending: false,
6123 }
6124 }
6125
6126 pub async fn start_child_workflow_avro_value<T: Serialize>(
6127 &self,
6128 workflow_type: impl Into<String>,
6129 options: ChildWorkflowOptions,
6130 args: T,
6131 ) -> Result<ChildWorkflowAvroResult> {
6132 let mut call = self.start_child_workflow(workflow_type, options, args);
6133 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
6134 }
6135
6136 fn take_commands(&self) -> Result<Vec<Value>> {
6137 let mut state = self
6138 .state
6139 .lock()
6140 .map_err(|_| Error::WorkflowStatePoisoned)?;
6141 Ok(std::mem::take(&mut state.commands))
6142 }
6143
6144 fn continue_as_new_command(&self, request: ContinueAsNewRequest) -> Result<Option<Value>> {
6145 let mut state = self
6146 .state
6147 .lock()
6148 .map_err(|_| Error::WorkflowStatePoisoned)?;
6149
6150 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6151 return Err(command_mismatch(&recorded, "continue as new"));
6152 }
6153 if state.recorded_continue_as_new_sequence.is_some() {
6154 state.continue_as_new_consumed = true;
6155 return Ok(None);
6156 }
6157
6158 let arguments = encode_typed_envelope(&request.arguments, &state.payload_codec)?;
6159 let mut command = serde_json::Map::from_iter([
6160 ("type".to_string(), json!("continue_as_new")),
6161 ("arguments".to_string(), arguments),
6162 ("queue".to_string(), json!(state.task_queue.clone())),
6163 ]);
6164 if let Some(workflow_type) = request.options.workflow_type {
6165 command.insert("workflow_type".to_string(), json!(workflow_type));
6166 }
6167 if let Some(task_queue) = request.options.task_queue {
6168 command.insert("queue".to_string(), json!(task_queue));
6169 }
6170 Ok(Some(Value::Object(command)))
6171 }
6172
6173 fn matched_recorded_pending(&self) -> Result<bool> {
6174 let state = self
6175 .state
6176 .lock()
6177 .map_err(|_| Error::WorkflowStatePoisoned)?;
6178 Ok(state.matched_recorded_pending)
6179 }
6180
6181 fn ensure_history_consumed(&self) -> Result<()> {
6182 let state = self
6183 .state
6184 .lock()
6185 .map_err(|_| Error::WorkflowStatePoisoned)?;
6186 if let Some(command) = state.recorded_commands.get(state.command_cursor) {
6187 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
6188 "recorded_commands_unconsumed",
6189 Some(command.sequence()),
6190 Some(command.shape().to_string()),
6191 Some("workflow completion".to_string()),
6192 "workflow completed before consuming all recorded durable commands",
6193 )));
6194 }
6195 if let Some(sequence) = state
6196 .recorded_continue_as_new_sequence
6197 .filter(|_| !state.continue_as_new_consumed)
6198 {
6199 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
6200 "recorded_continue_as_new_unconsumed",
6201 Some(sequence),
6202 Some("continue as new".to_string()),
6203 Some("workflow completion".to_string()),
6204 "workflow completed without consuming its recorded continue-as-new transition",
6205 )));
6206 }
6207 Ok(())
6208 }
6209}
6210
6211#[derive(Debug)]
6212struct WorkflowState {
6213 workflow_id: Option<String>,
6214 run_id: Option<String>,
6215 task_queue: String,
6216 payload_codec: String,
6217 history_budget: WorkflowHistoryBudget,
6218 resume_signal: Option<ResumeSignal>,
6219 recorded_commands: Vec<RecordedCommand>,
6220 recorded_continue_as_new_sequence: Option<u64>,
6221 continue_as_new_consumed: bool,
6222 command_cursor: usize,
6223 matched_recorded_pending: bool,
6224 version_markers: HashMap<String, (i32, u64)>,
6225 commands: Vec<Value>,
6226}
6227
6228impl WorkflowState {
6229 #[cfg(test)]
6230 fn new(
6231 history: Vec<HistoryEvent>,
6232 task_queue: String,
6233 payload_codec: String,
6234 resume_signal: Option<ResumeSignal>,
6235 ) -> Result<Self> {
6236 Self::new_with_identity(
6237 history,
6238 None,
6239 None,
6240 task_queue,
6241 payload_codec,
6242 resume_signal,
6243 )
6244 }
6245
6246 fn new_with_identity(
6247 history: Vec<HistoryEvent>,
6248 workflow_id: Option<String>,
6249 run_id: Option<String>,
6250 task_queue: String,
6251 payload_codec: String,
6252 resume_signal: Option<ResumeSignal>,
6253 ) -> Result<Self> {
6254 let recorded_commands = recorded_commands(
6255 &history,
6256 &payload_codec,
6257 WorkflowIdentity {
6258 workflow_id: workflow_id.clone(),
6259 run_id: run_id.clone(),
6260 },
6261 )?;
6262 let recorded_continue_as_new = history
6263 .iter()
6264 .filter(|event| event.event_type == "WorkflowContinuedAsNew")
6265 .collect::<Vec<_>>();
6266 if recorded_continue_as_new.len() > 1 {
6267 return Err(invalid_recorded_history(
6268 "duplicate_continue_as_new_transition",
6269 recorded_continue_as_new
6270 .last()
6271 .and_then(|event| durable_event_sequence(event))
6272 .unwrap_or(0),
6273 "one WorkflowContinuedAsNew event",
6274 &format!(
6275 "{} WorkflowContinuedAsNew events",
6276 recorded_continue_as_new.len()
6277 ),
6278 "workflow history records one continue-as-new transition more than once",
6279 ));
6280 }
6281 let recorded_continue_as_new_sequence = recorded_continue_as_new
6282 .first()
6283 .map(|event| {
6284 durable_event_sequence(event).ok_or_else(|| {
6285 Error::NonDeterministicReplay(ReplayFailure::new(
6286 "continue_as_new_sequence_missing",
6287 None,
6288 Some("recorded transition sequence".to_string()),
6289 Some("missing sequence".to_string()),
6290 "WorkflowContinuedAsNew history is missing its recorded sequence",
6291 ))
6292 })
6293 })
6294 .transpose()?;
6295 let event_count = u64::try_from(history.len()).unwrap_or(u64::MAX);
6296 Ok(Self {
6297 workflow_id,
6298 run_id,
6299 task_queue,
6300 payload_codec,
6301 history_budget: WorkflowHistoryBudget {
6302 event_count,
6303 ..WorkflowHistoryBudget::default()
6304 },
6305 resume_signal,
6306 recorded_commands,
6307 recorded_continue_as_new_sequence,
6308 continue_as_new_consumed: false,
6309 command_cursor: 0,
6310 matched_recorded_pending: false,
6311 version_markers: HashMap::new(),
6312 commands: Vec::new(),
6313 })
6314 }
6315}
6316
6317#[derive(Clone, Debug)]
6318enum RecordedCommand {
6319 Activity {
6320 sequence: u64,
6321 activity_type: Option<String>,
6322 options: Option<RecordedActivityOptions>,
6323 outcome: Option<ActivityOutcome>,
6324 },
6325 Timer {
6326 sequence: u64,
6327 delay_seconds: u64,
6328 fired: bool,
6329 },
6330 ChildWorkflow {
6331 sequence: u64,
6332 workflow_type: Option<String>,
6333 outcome: Option<ChildWorkflowOutcome>,
6334 },
6335 SignalWait {
6336 sequence: u64,
6337 signal_name: String,
6338 value: Option<Vec<AvroValue>>,
6339 },
6340 SideEffect {
6341 sequence: u64,
6342 value: AvroValue,
6343 },
6344 VersionMarker {
6345 sequence: u64,
6346 change_id: String,
6347 version: i32,
6348 },
6349}
6350
6351#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
6352struct RecordedActivityOptions {
6353 task_queue: RecordedSnapshotValue<Option<String>>,
6354 execution_mode: RecordedSnapshotValue<Option<String>>,
6355 retry_policy: ActivityRetrySnapshot,
6356}
6357
6358#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
6359enum RecordedSnapshotValue<T> {
6360 Unknown,
6362 Known(T),
6363}
6364
6365impl<T: PartialEq> RecordedSnapshotValue<T> {
6366 fn matches_current(&self, current: &Self) -> bool {
6367 match self {
6368 Self::Unknown => true,
6369 Self::Known(recorded) => matches!(current, Self::Known(value) if value == recorded),
6370 }
6371 }
6372}
6373
6374#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
6375struct ActivityRetrySnapshot {
6376 snapshot_version: RecordedSnapshotValue<Option<u64>>,
6377 max_attempts: RecordedSnapshotValue<Option<u64>>,
6378 backoff_seconds: RecordedSnapshotValue<Vec<u64>>,
6379 start_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
6380 schedule_to_start_timeout: RecordedSnapshotValue<Option<u64>>,
6381 schedule_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
6382 heartbeat_timeout: RecordedSnapshotValue<Option<u64>>,
6383 non_retryable_error_types: RecordedSnapshotValue<Vec<String>>,
6384}
6385
6386impl ActivityRetrySnapshot {
6387 fn matches_current(&self, current: &Self) -> bool {
6388 self.snapshot_version
6389 .matches_current(¤t.snapshot_version)
6390 && self.max_attempts.matches_current(¤t.max_attempts)
6391 && self
6392 .backoff_seconds
6393 .matches_current(¤t.backoff_seconds)
6394 && self
6395 .start_to_close_timeout
6396 .matches_current(¤t.start_to_close_timeout)
6397 && self
6398 .schedule_to_start_timeout
6399 .matches_current(¤t.schedule_to_start_timeout)
6400 && self
6401 .schedule_to_close_timeout
6402 .matches_current(¤t.schedule_to_close_timeout)
6403 && self
6404 .heartbeat_timeout
6405 .matches_current(¤t.heartbeat_timeout)
6406 && self
6407 .non_retryable_error_types
6408 .matches_current(¤t.non_retryable_error_types)
6409 }
6410}
6411
6412fn recorded_optional_u64(
6413 object: Option<&serde_json::Map<String, Value>>,
6414 field: &str,
6415) -> RecordedSnapshotValue<Option<u64>> {
6416 match object.and_then(|object| object.get(field)) {
6417 None => RecordedSnapshotValue::Unknown,
6418 Some(Value::Null) => RecordedSnapshotValue::Known(None),
6419 Some(value) => RecordedSnapshotValue::Known(value_as_u64(value)),
6420 }
6421}
6422
6423fn recorded_optional_string(
6424 object: &serde_json::Map<String, Value>,
6425 field: &str,
6426) -> RecordedSnapshotValue<Option<String>> {
6427 match object.get(field) {
6428 None => RecordedSnapshotValue::Unknown,
6429 Some(Value::Null) => RecordedSnapshotValue::Known(None),
6430 Some(value) => RecordedSnapshotValue::Known(value.as_str().map(str::to_string)),
6431 }
6432}
6433
6434fn recorded_activity_retry_snapshot(policy: Option<&Value>) -> ActivityRetrySnapshot {
6435 let policy = policy.and_then(Value::as_object);
6436 let backoff_seconds = policy
6437 .and_then(|policy| policy.get("backoff_seconds"))
6438 .and_then(Value::as_array)
6439 .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
6440 .map_or(RecordedSnapshotValue::Unknown, RecordedSnapshotValue::Known);
6441 let mut non_retryable_error_types = Vec::new();
6442 for error_type in policy
6443 .and_then(|policy| policy.get("non_retryable_error_types"))
6444 .and_then(Value::as_array)
6445 .into_iter()
6446 .flatten()
6447 .filter_map(Value::as_str)
6448 .map(str::trim)
6449 .filter(|error_type| !error_type.is_empty())
6450 {
6451 if !non_retryable_error_types
6452 .iter()
6453 .any(|recorded| recorded == error_type)
6454 {
6455 non_retryable_error_types.push(error_type.to_string());
6456 }
6457 }
6458
6459 ActivityRetrySnapshot {
6460 snapshot_version: recorded_optional_u64(policy, "snapshot_version"),
6461 max_attempts: recorded_optional_u64(policy, "max_attempts"),
6462 backoff_seconds,
6463 start_to_close_timeout: recorded_optional_u64(policy, "start_to_close_timeout"),
6464 schedule_to_start_timeout: recorded_optional_u64(policy, "schedule_to_start_timeout"),
6465 schedule_to_close_timeout: recorded_optional_u64(policy, "schedule_to_close_timeout"),
6466 heartbeat_timeout: recorded_optional_u64(policy, "heartbeat_timeout"),
6467 non_retryable_error_types: if policy
6468 .is_some_and(|policy| policy.contains_key("non_retryable_error_types"))
6469 {
6470 RecordedSnapshotValue::Known(non_retryable_error_types)
6471 } else {
6472 RecordedSnapshotValue::Unknown
6473 },
6474 }
6475}
6476
6477fn current_activity_retry_snapshot(options: &ValidatedActivityOptions) -> ActivityRetrySnapshot {
6478 let policy = options.retry_policy.as_ref();
6479 let max_attempts = match policy.and_then(|policy| policy.get("max_attempts")) {
6480 Some(Value::Null) => None,
6481 Some(value) => value_as_u64(value),
6482 None => Some(1),
6483 };
6484 let backoff_seconds = policy
6485 .and_then(|policy| policy.get("backoff_seconds"))
6486 .and_then(Value::as_array)
6487 .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
6488 .unwrap_or_default();
6489 let non_retryable_error_types = policy
6490 .and_then(|policy| policy.get("non_retryable_error_types"))
6491 .and_then(Value::as_array)
6492 .into_iter()
6493 .flatten()
6494 .filter_map(Value::as_str)
6495 .map(str::to_string)
6496 .collect();
6497
6498 ActivityRetrySnapshot {
6499 snapshot_version: RecordedSnapshotValue::Known(Some(1)),
6500 max_attempts: RecordedSnapshotValue::Known(max_attempts),
6501 backoff_seconds: RecordedSnapshotValue::Known(backoff_seconds),
6502 start_to_close_timeout: RecordedSnapshotValue::Known(options.start_to_close_timeout),
6503 schedule_to_start_timeout: RecordedSnapshotValue::Known(options.schedule_to_start_timeout),
6504 schedule_to_close_timeout: RecordedSnapshotValue::Known(options.schedule_to_close_timeout),
6505 heartbeat_timeout: RecordedSnapshotValue::Known(options.heartbeat_timeout),
6506 non_retryable_error_types: RecordedSnapshotValue::Known(non_retryable_error_types),
6507 }
6508}
6509
6510fn activity_options_description(options: &RecordedActivityOptions) -> String {
6511 serde_json::to_string(options).unwrap_or_else(|_| format!("{options:?}"))
6512}
6513
6514impl RecordedCommand {
6515 fn sequence(&self) -> u64 {
6516 match self {
6517 Self::Activity { sequence, .. }
6518 | Self::Timer { sequence, .. }
6519 | Self::ChildWorkflow { sequence, .. }
6520 | Self::SignalWait { sequence, .. }
6521 | Self::SideEffect { sequence, .. }
6522 | Self::VersionMarker { sequence, .. } => *sequence,
6523 }
6524 }
6525
6526 fn shape(&self) -> &'static str {
6527 match self {
6528 Self::Activity { .. } => "activity",
6529 Self::Timer { .. } => "timer",
6530 Self::ChildWorkflow { .. } => "child workflow",
6531 Self::SignalWait { .. } => "signal wait",
6532 Self::SideEffect { .. } => "side effect",
6533 Self::VersionMarker { .. } => "version marker",
6534 }
6535 }
6536}
6537
6538fn ensure_version_supported(
6539 change_id: &str,
6540 version: i32,
6541 min_supported: i32,
6542 max_supported: i32,
6543 sequence: u64,
6544) -> Result<()> {
6545 if (min_supported..=max_supported).contains(&version) {
6546 return Ok(());
6547 }
6548 Err(Error::NonDeterministicReplay(ReplayFailure::new(
6549 "version_marker_incompatible_range",
6550 (sequence != 0).then_some(sequence),
6551 Some(format!("{min_supported}..={max_supported}")),
6552 Some(format!("{change_id}:{version}")),
6553 "recorded workflow version is outside the range supported by current code",
6554 )))
6555}
6556
6557#[derive(Clone, Debug)]
6558struct ResumeSignal {
6559 signal_name: String,
6560 arguments: Vec<AvroValue>,
6561}
6562
6563pub struct ActivityCall {
6564 ctx: WorkflowContext,
6565 activity_type: String,
6566 options: ActivityOptions,
6567 args: Option<Result<AvroValue>>,
6568 scheduled: bool,
6569}
6570
6571impl ActivityCall {
6572 fn poll_avro_value(
6573 mut self: Pin<&mut Self>,
6574 _cx: &mut TaskContext<'_>,
6575 ) -> Poll<Result<AvroValue>> {
6576 let ctx = self.ctx.clone();
6577 let mut state = match ctx.state.lock() {
6578 Ok(state) => state,
6579 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6580 };
6581
6582 if self.scheduled {
6583 return Poll::Pending;
6584 }
6585
6586 let options = match self.options.validate() {
6587 Ok(options) => options,
6588 Err(error) => {
6589 return Poll::Ready(Err(Error::InvalidActivityOptions(error)));
6590 }
6591 };
6592 let task_queue = options
6593 .task_queue
6594 .clone()
6595 .unwrap_or_else(|| state.task_queue.clone());
6596 let current_recorded_options = RecordedActivityOptions {
6597 task_queue: RecordedSnapshotValue::Known(Some(task_queue.clone())),
6598 execution_mode: RecordedSnapshotValue::Known(None),
6601 retry_policy: current_activity_retry_snapshot(&options),
6602 };
6603
6604 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6605 let sequence = recorded.sequence();
6606 match recorded {
6607 RecordedCommand::Activity {
6608 activity_type,
6609 options: recorded_options,
6610 outcome,
6611 ..
6612 } => {
6613 if let Some(recorded_type) = activity_type {
6614 if recorded_type != self.activity_type {
6615 return Poll::Ready(Err(Error::NonDeterministicReplay(
6616 ReplayFailure::new(
6617 "recorded_command_detail_mismatch",
6618 Some(sequence),
6619 Some(format!("activity:{recorded_type}")),
6620 Some(format!("activity:{}", self.activity_type)),
6621 "recorded activity type differs from the current workflow command",
6622 ),
6623 )));
6624 }
6625 }
6626 if let Some(recorded_options) = recorded_options {
6627 if !recorded_options
6628 .task_queue
6629 .matches_current(¤t_recorded_options.task_queue)
6630 {
6631 return Poll::Ready(Err(Error::NonDeterministicReplay(
6632 ReplayFailure::new(
6633 "activity_task_queue_mismatch",
6634 Some(sequence),
6635 Some(activity_options_description(&recorded_options)),
6636 Some(activity_options_description(¤t_recorded_options)),
6637 "recorded activity task queue differs from the current workflow command",
6638 ),
6639 )));
6640 }
6641 if !recorded_options
6642 .execution_mode
6643 .matches_current(¤t_recorded_options.execution_mode)
6644 {
6645 return Poll::Ready(Err(Error::NonDeterministicReplay(
6646 ReplayFailure::new(
6647 "activity_execution_mode_mismatch",
6648 Some(sequence),
6649 Some(activity_options_description(&recorded_options)),
6650 Some(activity_options_description(¤t_recorded_options)),
6651 "recorded activity execution mode differs from the current workflow command",
6652 ),
6653 )));
6654 }
6655 if !recorded_options
6656 .retry_policy
6657 .matches_current(¤t_recorded_options.retry_policy)
6658 {
6659 return Poll::Ready(Err(Error::NonDeterministicReplay(
6660 ReplayFailure::new(
6661 "activity_retry_policy_mismatch",
6662 Some(sequence),
6663 Some(activity_options_description(&recorded_options)),
6664 Some(activity_options_description(¤t_recorded_options)),
6665 "recorded activity retry policy differs from the current workflow command",
6666 ),
6667 )));
6668 }
6669 }
6670 state.command_cursor += 1;
6671 if let Some(outcome) = outcome {
6672 return Poll::Ready(outcome.map_err(Error::ActivityFailed));
6673 }
6674 state.matched_recorded_pending = true;
6675 self.scheduled = true;
6676 return Poll::Pending;
6677 }
6678 other => {
6679 return Poll::Ready(Err(command_mismatch(
6680 &other,
6681 format!("activity:{}", self.activity_type),
6682 )));
6683 }
6684 }
6685 }
6686
6687 if !self.scheduled {
6688 let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
6689 Ok(args) => args,
6690 Err(error) => return Poll::Ready(Err(error)),
6691 };
6692 let arguments = normalize_avro_arguments(args);
6693 let envelope = match encode_typed_envelope(&arguments, &state.payload_codec) {
6694 Ok(envelope) => envelope,
6695 Err(error) => return Poll::Ready(Err(error)),
6696 };
6697
6698 let mut command = serde_json::Map::from_iter([
6699 ("type".to_string(), json!("schedule_activity")),
6700 (
6701 "activity_type".to_string(),
6702 json!(self.activity_type.clone()),
6703 ),
6704 ("queue".to_string(), json!(task_queue)),
6705 ("arguments".to_string(), envelope),
6706 ]);
6707 for (field, value) in [
6708 ("start_to_close_timeout", options.start_to_close_timeout),
6709 (
6710 "schedule_to_start_timeout",
6711 options.schedule_to_start_timeout,
6712 ),
6713 (
6714 "schedule_to_close_timeout",
6715 options.schedule_to_close_timeout,
6716 ),
6717 ("heartbeat_timeout", options.heartbeat_timeout),
6718 ] {
6719 if let Some(value) = value {
6720 command.insert(field.to_string(), json!(value));
6721 }
6722 }
6723 if let Some(retry_policy) = options.retry_policy {
6724 command.insert("retry_policy".to_string(), retry_policy);
6725 }
6726 state.commands.push(Value::Object(command));
6727 self.scheduled = true;
6728 }
6729
6730 Poll::Pending
6731 }
6732}
6733
6734impl Future for ActivityCall {
6735 type Output = Result<Value>;
6736
6737 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6738 match self.poll_avro_value(cx) {
6739 Poll::Ready(Ok(value)) => Poll::Ready(value.into_json()),
6740 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
6741 Poll::Pending => Poll::Pending,
6742 }
6743 }
6744}
6745
6746pub struct TimerCall {
6748 ctx: WorkflowContext,
6749 delay_seconds: Option<u64>,
6750 scheduled: bool,
6751 matched_pending: bool,
6752}
6753
6754impl Future for TimerCall {
6755 type Output = Result<()>;
6756
6757 fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6758 if self.matched_pending {
6759 return Poll::Pending;
6760 }
6761
6762 let ctx = self.ctx.clone();
6763 let Some(requested_delay) = self.delay_seconds else {
6764 return Poll::Ready(Err(Error::TimerDurationOverflow));
6765 };
6766 let mut state = match ctx.state.lock() {
6767 Ok(state) => state,
6768 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6769 };
6770
6771 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6772 match recorded {
6773 RecordedCommand::Timer {
6774 sequence,
6775 delay_seconds,
6776 fired,
6777 ..
6778 } => {
6779 if delay_seconds != requested_delay {
6780 return Poll::Ready(Err(Error::NonDeterministicReplay(
6781 ReplayFailure::new(
6782 "timer_delay_mismatch",
6783 Some(sequence),
6784 Some(format!("timer:{delay_seconds}s")),
6785 Some(format!("timer:{requested_delay}s")),
6786 "recorded timer delay differs from the current workflow command",
6787 ),
6788 )));
6789 }
6790 state.command_cursor += 1;
6791 if fired {
6792 return Poll::Ready(Ok(()));
6793 }
6794 state.matched_recorded_pending = true;
6795 self.scheduled = true;
6796 self.matched_pending = true;
6797 return Poll::Pending;
6798 }
6799 other => return Poll::Ready(Err(command_mismatch(&other, "timer"))),
6800 }
6801 }
6802
6803 if !self.scheduled {
6804 state.commands.push(json!({
6805 "type": "start_timer",
6806 "delay_seconds": requested_delay,
6807 }));
6808 self.scheduled = true;
6809 }
6810
6811 Poll::Pending
6812 }
6813}
6814
6815pub struct ChildWorkflowCall {
6817 ctx: WorkflowContext,
6818 workflow_type: String,
6819 options: ChildWorkflowOptions,
6820 args: Option<Result<AvroValue>>,
6821 scheduled: bool,
6822 matched_pending: bool,
6823}
6824
6825impl ChildWorkflowCall {
6826 fn poll_avro_value(
6827 mut self: Pin<&mut Self>,
6828 _cx: &mut TaskContext<'_>,
6829 ) -> Poll<Result<ChildWorkflowAvroResult>> {
6830 if self.matched_pending {
6831 return Poll::Pending;
6832 }
6833
6834 let ctx = self.ctx.clone();
6835 let mut state = match ctx.state.lock() {
6836 Ok(state) => state,
6837 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6838 };
6839
6840 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6841 let sequence = recorded.sequence();
6842 match recorded {
6843 RecordedCommand::ChildWorkflow {
6844 workflow_type,
6845 outcome,
6846 ..
6847 } => {
6848 if let Some(recorded_type) = workflow_type {
6849 if recorded_type != self.workflow_type {
6850 return Poll::Ready(Err(Error::NonDeterministicReplay(
6851 ReplayFailure::new(
6852 "recorded_command_detail_mismatch",
6853 Some(sequence),
6854 Some(format!("child workflow:{recorded_type}")),
6855 Some(format!("child workflow:{}", self.workflow_type)),
6856 "recorded child workflow type differs from the current workflow command",
6857 ),
6858 )));
6859 }
6860 }
6861 state.command_cursor += 1;
6862 if let Some(outcome) = outcome {
6863 return Poll::Ready(outcome.map_err(Error::ChildWorkflowFailed));
6864 }
6865 state.matched_recorded_pending = true;
6866 self.scheduled = true;
6867 self.matched_pending = true;
6868 return Poll::Pending;
6869 }
6870 other => {
6871 return Poll::Ready(Err(command_mismatch(
6872 &other,
6873 format!("child workflow:{}", self.workflow_type),
6874 )));
6875 }
6876 }
6877 }
6878
6879 if !self.scheduled {
6880 if self.options.task_queue.trim().is_empty() {
6881 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
6882 "task_queue must not be empty".to_string(),
6883 )));
6884 }
6885 for (name, value) in [
6886 (
6887 "execution_timeout_seconds",
6888 self.options.execution_timeout_seconds,
6889 ),
6890 ("run_timeout_seconds", self.options.run_timeout_seconds),
6891 ] {
6892 if value == Some(0) {
6893 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(format!(
6894 "{name} must be at least 1"
6895 ))));
6896 }
6897 }
6898
6899 let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
6900 Ok(args) => args,
6901 Err(error) => return Poll::Ready(Err(error)),
6902 };
6903 let arguments = match encode_typed_envelope(
6904 &normalize_avro_arguments(args),
6905 &state.payload_codec,
6906 ) {
6907 Ok(arguments) => arguments,
6908 Err(error) => return Poll::Ready(Err(error)),
6909 };
6910 let mut command = json!({
6911 "type": "start_child_workflow",
6912 "workflow_type": self.workflow_type,
6913 "queue": self.options.task_queue,
6914 "parent_close_policy": self.options.parent_close_policy.as_str(),
6915 "arguments": arguments,
6916 });
6917 let object = command
6918 .as_object_mut()
6919 .expect("child workflow command is always an object");
6920 if let Some(policy) = &self.options.retry_policy {
6921 let mut retry_policy = serde_json::Map::new();
6922 if let Some(max_attempts) = policy.max_attempts {
6923 if max_attempts == 0 {
6924 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
6925 "retry_policy.max_attempts must be at least 1".to_string(),
6926 )));
6927 }
6928 retry_policy.insert("max_attempts".to_string(), json!(max_attempts));
6929 }
6930 if !policy.backoff_seconds.is_empty() {
6931 retry_policy
6932 .insert("backoff_seconds".to_string(), json!(policy.backoff_seconds));
6933 }
6934 if !policy.non_retryable_error_types.is_empty() {
6935 retry_policy.insert(
6936 "non_retryable_error_types".to_string(),
6937 json!(policy.non_retryable_error_types),
6938 );
6939 }
6940 if retry_policy.is_empty() {
6941 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
6942 "retry_policy must configure at least one field".to_string(),
6943 )));
6944 }
6945 object.insert("retry_policy".to_string(), Value::Object(retry_policy));
6946 }
6947 if let Some(seconds) = self.options.execution_timeout_seconds {
6948 object.insert("execution_timeout_seconds".to_string(), json!(seconds));
6949 }
6950 if let Some(seconds) = self.options.run_timeout_seconds {
6951 object.insert("run_timeout_seconds".to_string(), json!(seconds));
6952 }
6953 state.commands.push(command);
6954 self.scheduled = true;
6955 }
6956
6957 Poll::Pending
6958 }
6959}
6960
6961impl Future for ChildWorkflowCall {
6962 type Output = Result<ChildWorkflowResult>;
6963
6964 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6965 match self.poll_avro_value(cx) {
6966 Poll::Ready(Ok(result)) => match result.result.into_json() {
6967 Ok(projected) => Poll::Ready(Ok(ChildWorkflowResult {
6968 parent: result.parent,
6969 child: result.child,
6970 child_workflow_type: result.child_workflow_type,
6971 result: projected,
6972 })),
6973 Err(error) => Poll::Ready(Err(error)),
6974 },
6975 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
6976 Poll::Pending => Poll::Pending,
6977 }
6978 }
6979}
6980
6981fn command_mismatch(recorded: &RecordedCommand, actual: impl Into<String>) -> Error {
6982 Error::NonDeterministicReplay(ReplayFailure::new(
6983 "recorded_command_mismatch",
6984 Some(recorded.sequence()),
6985 Some(recorded.shape().to_string()),
6986 Some(actual.into()),
6987 "current workflow command does not match the recorded durable command sequence",
6988 ))
6989}
6990
6991pub struct SignalCall {
6992 ctx: WorkflowContext,
6993 signal_name: String,
6994 opened_wait: bool,
6995 matched_pending: bool,
6996}
6997
6998impl SignalCall {
6999 fn poll_avro_value(
7000 mut self: Pin<&mut Self>,
7001 _cx: &mut TaskContext<'_>,
7002 ) -> Poll<Result<Vec<AvroValue>>> {
7003 if self.matched_pending {
7004 return Poll::Pending;
7005 }
7006
7007 let ctx = self.ctx.clone();
7008 let mut state = match ctx.state.lock() {
7009 Ok(state) => state,
7010 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
7011 };
7012
7013 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
7014 match recorded {
7015 RecordedCommand::SignalWait {
7016 sequence,
7017 signal_name,
7018 value,
7019 } => {
7020 if signal_name != self.signal_name {
7021 return Poll::Ready(Err(Error::NonDeterministicReplay(
7022 ReplayFailure::new(
7023 "recorded_command_detail_mismatch",
7024 Some(sequence),
7025 Some(format!("signal wait:{signal_name}")),
7026 Some(format!("signal wait:{}", self.signal_name)),
7027 "recorded signal name differs from the current workflow command",
7028 ),
7029 )));
7030 }
7031
7032 state.command_cursor += 1;
7033 if let Some(value) = value {
7034 return Poll::Ready(Ok(value));
7035 }
7036 if state
7037 .resume_signal
7038 .as_ref()
7039 .is_some_and(|signal| signal.signal_name == self.signal_name)
7040 {
7041 let signal = state
7042 .resume_signal
7043 .take()
7044 .expect("matching resume signal is present");
7045 return Poll::Ready(Ok(signal.arguments));
7046 }
7047
7048 state.matched_recorded_pending = true;
7049 self.opened_wait = true;
7050 self.matched_pending = true;
7051 return Poll::Pending;
7052 }
7053 other => {
7054 return Poll::Ready(Err(command_mismatch(
7055 &other,
7056 format!("signal wait:{}", self.signal_name),
7057 )));
7058 }
7059 }
7060 }
7061
7062 if state
7063 .resume_signal
7064 .as_ref()
7065 .is_some_and(|signal| signal.signal_name == self.signal_name)
7066 {
7067 let signal = state
7068 .resume_signal
7069 .take()
7070 .expect("matching resume signal is present");
7071 return Poll::Ready(Ok(signal.arguments));
7072 }
7073
7074 if !self.opened_wait {
7075 state.commands.push(json!({
7076 "type": "open_signal_wait",
7077 "signal_name": self.signal_name
7078 }));
7079 self.opened_wait = true;
7080 }
7081
7082 Poll::Pending
7083 }
7084}
7085
7086impl Future for SignalCall {
7087 type Output = Result<Vec<Value>>;
7088
7089 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
7090 match self.poll_avro_value(cx) {
7091 Poll::Ready(Ok(values)) => Poll::Ready(
7092 values
7093 .into_iter()
7094 .map(AvroValue::into_json)
7095 .collect::<Result<Vec<_>>>(),
7096 ),
7097 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
7098 Poll::Pending => Poll::Pending,
7099 }
7100 }
7101}
7102
7103#[derive(Clone, Debug)]
7104pub struct ActivityContext {
7105 client: Client,
7106 pub task_id: String,
7107 pub activity_attempt_id: String,
7108 pub lease_owner: String,
7109 pub activity_type: String,
7110 pub attempt_number: u64,
7111 pub task_queue: String,
7112 pub worker_id: String,
7113}
7114
7115impl ActivityContext {
7116 pub async fn heartbeat<T: Serialize>(&self, details: T) -> Result<ActivityHeartbeatResponse> {
7117 self.client
7118 .heartbeat_activity_task(
7119 &self.task_id,
7120 &self.activity_attempt_id,
7121 &self.lease_owner,
7122 details,
7123 )
7124 .await
7125 }
7126}
7127
7128fn decode_task_avro_arguments(value: Option<&Value>, codec: &str) -> Result<AvroValue> {
7129 validate_payload_codec(codec)?;
7130 match value {
7131 Some(value) => Ok(normalize_avro_arguments(decode_wire_avro_value(
7132 value, codec,
7133 )?)),
7134 None => Ok(AvroValue::Array(Vec::new())),
7135 }
7136}
7137
7138fn decode_resume_signal(task: &WorkflowTask) -> Result<Option<ResumeSignal>> {
7139 let Some(signal_name) = task
7140 .signal_name
7141 .as_deref()
7142 .filter(|value| !value.is_empty())
7143 else {
7144 return Ok(None);
7145 };
7146 let decoded = decode_task_avro_arguments(task.signal_arguments.as_ref(), &task.payload_codec)?;
7147 let AvroValue::Array(arguments) = decoded else {
7148 unreachable!("normalize_avro_arguments always returns an array");
7149 };
7150
7151 Ok(Some(ResumeSignal {
7152 signal_name: signal_name.to_string(),
7153 arguments,
7154 }))
7155}
7156
7157fn validate_workflow_task_payloads(task: &WorkflowTask) -> Result<()> {
7158 validate_payload_codec(&task.payload_codec)?;
7159 validate_optional_inbound_payload(task.arguments.as_ref(), &task.payload_codec)?;
7160 validate_optional_inbound_payload(task.signal_arguments.as_ref(), &task.payload_codec)?;
7161 for event in &task.history_events {
7162 validate_history_event_payloads(event, &task.payload_codec)?;
7163 }
7164 Ok(())
7165}
7166
7167fn validate_activity_task_payloads(task: &ActivityTask) -> Result<()> {
7168 validate_payload_codec(&task.payload_codec)?;
7169 validate_optional_inbound_payload(task.arguments.as_ref(), &task.payload_codec)
7170}
7171
7172fn validate_query_task_payloads(task: &QueryTask) -> Result<()> {
7173 validate_payload_codec(&task.payload_codec)?;
7174 validate_optional_inbound_payload(task.workflow_arguments.as_ref(), &task.payload_codec)?;
7175 validate_optional_inbound_payload(task.query_arguments.as_ref(), &task.payload_codec)?;
7176 for event in &task.history_events {
7177 validate_history_event_payloads(event, &task.payload_codec)?;
7178 }
7179
7180 let Some(export) = task.history_export.as_ref() else {
7181 return Ok(());
7182 };
7183 let export_codec = match export.get("payloads") {
7184 Some(payloads) => declared_payload_codec(payloads, "codec")?,
7185 None => None,
7186 }
7187 .unwrap_or(&task.payload_codec);
7188 validate_payload_codec(export_codec)?;
7189
7190 if let Some(events) = export.get("history_events").and_then(Value::as_array) {
7191 for event in events {
7192 let event_type = event
7193 .get("event_type")
7194 .or_else(|| event.get("type"))
7195 .and_then(Value::as_str)
7196 .unwrap_or_default();
7197 if let Some(payload) = event.get("payload") {
7198 validate_history_payloads(event_type, payload, export_codec)?;
7199 }
7200 }
7201 }
7202 for signal in export
7203 .get("signals")
7204 .and_then(Value::as_array)
7205 .into_iter()
7206 .flatten()
7207 {
7208 let codec = declared_payload_codec(signal, "payload_codec")?.unwrap_or(export_codec);
7209 validate_payload_codec(codec)?;
7210 validate_optional_inbound_payload(signal.get("arguments"), codec)?;
7211 }
7212 for activity in export
7213 .get("activities")
7214 .and_then(Value::as_array)
7215 .into_iter()
7216 .flatten()
7217 {
7218 let codec = declared_payload_codec(activity, "payload_codec")?.unwrap_or(export_codec);
7219 validate_payload_codec(codec)?;
7220 validate_optional_inbound_payload(activity.get("arguments"), codec)?;
7221 validate_optional_inbound_payload(activity.get("result"), codec)?;
7222 }
7223 Ok(())
7224}
7225
7226fn validate_history_event_payloads(event: &HistoryEvent, fallback_codec: &str) -> Result<()> {
7227 validate_history_payloads(&event.event_type, &event.payload, fallback_codec)
7228}
7229
7230fn validate_history_payloads(
7231 event_type: &str,
7232 payload: &Value,
7233 fallback_codec: &str,
7234) -> Result<()> {
7235 let codec = declared_payload_codec(payload, "payload_codec")?.unwrap_or(fallback_codec);
7236 validate_payload_codec(codec)?;
7237 for field in history_payload_fields(event_type) {
7238 validate_optional_inbound_payload(payload.get(*field), codec)?;
7239 }
7240 Ok(())
7241}
7242
7243const SIGNAL_HISTORY_PAYLOAD_FIELDS: &[&str] = &["value", "input", "arguments"];
7244
7245fn history_payload_fields(event_type: &str) -> &'static [&'static str] {
7246 match event_type {
7247 "ActivityCompleted" => &["result"],
7248 "SignalReceived" | "SignalApplied" => SIGNAL_HISTORY_PAYLOAD_FIELDS,
7249 "UpdateAccepted" | "UpdateRejected" | "UpdateApplied" => &["arguments"],
7250 "UpdateCompleted" | "SideEffectRecorded" => &["result"],
7251 "ChildRunCompleted" => &["result", "output"],
7252 "WorkflowCompleted" => &["output"],
7253 "ServiceCallStarted"
7254 | "ServiceCallCompleted"
7255 | "ServiceCallFailed"
7256 | "ServiceCallCancelled" => &["request_payload", "response_payload"],
7257 _ => &[],
7258 }
7259}
7260
7261fn signal_history_payload(payload: &Value) -> Option<&Value> {
7262 SIGNAL_HISTORY_PAYLOAD_FIELDS
7263 .iter()
7264 .find_map(|field| payload.get(*field))
7265}
7266
7267fn declared_payload_codec<'a>(value: &'a Value, field: &str) -> Result<Option<&'a str>> {
7268 match value.get(field) {
7269 None => Ok(None),
7270 Some(Value::String(codec)) => Ok(Some(codec)),
7271 Some(_) => Err(invalid_payload_envelope()),
7272 }
7273}
7274
7275fn validate_optional_inbound_payload(value: Option<&Value>, codec: &str) -> Result<()> {
7276 validate_payload_codec(codec)?;
7277 if let Some(value) = value.filter(|value| !value.is_null()) {
7278 decode_wire_avro_value(value, codec)?;
7279 }
7280 Ok(())
7281}
7282
7283fn recorded_commands(
7284 events: &[HistoryEvent],
7285 fallback_codec: &str,
7286 parent: WorkflowIdentity,
7287) -> Result<Vec<RecordedCommand>> {
7288 let mut events_by_sequence: BTreeMap<u64, Vec<&HistoryEvent>> = BTreeMap::new();
7289 let mut last_new_sequence = None;
7290
7291 for event in events {
7292 let is_activity = matches!(
7293 event.event_type.as_str(),
7294 "ActivityScheduled"
7295 | "ActivityStarted"
7296 | "ActivityHeartbeatRecorded"
7297 | "ActivityRetryScheduled"
7298 | "ActivityCompleted"
7299 | "ActivityFailed"
7300 | "ActivityCancelled"
7301 | "ActivityTimedOut"
7302 );
7303 let is_workflow_timer = matches!(
7304 event.event_type.as_str(),
7305 "TimerScheduled" | "TimerCancelled" | "TimerFired"
7306 ) && !is_internal_timer_event(event);
7307 let is_child_workflow = matches!(
7308 event.event_type.as_str(),
7309 "ChildWorkflowScheduled"
7310 | "ChildRunCompleted"
7311 | "ChildRunFailed"
7312 | "ChildRunCancelled"
7313 | "ChildRunTerminated"
7314 );
7315 let is_signal_wait = is_recorded_signal_wait_event(event);
7316 let is_side_effect = event.event_type == "SideEffectRecorded";
7317 let is_version_marker = event.event_type == "VersionMarkerRecorded";
7318 if !is_activity
7319 && !is_workflow_timer
7320 && !is_child_workflow
7321 && !is_signal_wait
7322 && !is_side_effect
7323 && !is_version_marker
7324 {
7325 continue;
7326 }
7327
7328 let sequence = durable_event_sequence(event).ok_or_else(|| {
7329 Error::NonDeterministicReplay(ReplayFailure::new(
7330 "durable_command_sequence_missing",
7331 None,
7332 Some("positive workflow sequence".to_string()),
7333 Some(event.event_type.clone()),
7334 "durable command history event has no workflow sequence",
7335 ))
7336 })?;
7337 if sequence == 0 {
7338 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
7339 "durable_command_sequence_invalid",
7340 Some(sequence),
7341 Some("positive workflow sequence".to_string()),
7342 Some(sequence.to_string()),
7343 "durable command history uses an invalid workflow sequence",
7344 )));
7345 }
7346 if !events_by_sequence.contains_key(&sequence) {
7347 if let Some(previous) = last_new_sequence {
7348 if sequence < previous {
7349 return Err(invalid_recorded_history(
7350 "durable_command_sequence_mismatch",
7351 sequence,
7352 &format!("workflow sequence greater than {previous}"),
7353 &sequence.to_string(),
7354 "durable commands are not strictly ordered by their recorded workflow sequence",
7355 ));
7356 }
7357 }
7358 last_new_sequence = Some(sequence);
7359 }
7360 events_by_sequence.entry(sequence).or_default().push(event);
7361 }
7362
7363 let commands: Vec<RecordedCommand> = events_by_sequence
7364 .into_iter()
7365 .map(|(sequence, sequence_events)| {
7366 let activity_events: Vec<_> = sequence_events
7367 .iter()
7368 .copied()
7369 .filter(|event| event.event_type.starts_with("Activity"))
7370 .collect();
7371 let timer_events: Vec<_> = sequence_events
7372 .iter()
7373 .copied()
7374 .filter(|event| event.event_type.starts_with("Timer"))
7375 .collect();
7376 let child_events: Vec<_> = sequence_events
7377 .iter()
7378 .copied()
7379 .filter(|event| {
7380 event.event_type == "ChildWorkflowScheduled"
7381 || event.event_type.starts_with("ChildRun")
7382 })
7383 .collect();
7384 let signal_wait_events: Vec<_> = sequence_events
7385 .iter()
7386 .copied()
7387 .filter(|event| is_recorded_signal_wait_event(event))
7388 .collect();
7389 let side_effect_events: Vec<_> = sequence_events
7390 .iter()
7391 .copied()
7392 .filter(|event| event.event_type == "SideEffectRecorded")
7393 .collect();
7394 let version_marker_events: Vec<_> = sequence_events
7395 .iter()
7396 .copied()
7397 .filter(|event| event.event_type == "VersionMarkerRecorded")
7398 .collect();
7399
7400 let command_kind_count = usize::from(!activity_events.is_empty())
7401 + usize::from(!timer_events.is_empty())
7402 + usize::from(!child_events.is_empty())
7403 + usize::from(!signal_wait_events.is_empty())
7404 + usize::from(!side_effect_events.is_empty())
7405 + usize::from(!version_marker_events.is_empty());
7406 if command_kind_count > 1 {
7407 let actual = [
7408 (!activity_events.is_empty()).then_some("activity"),
7409 (!timer_events.is_empty()).then_some("timer"),
7410 (!child_events.is_empty()).then_some("child workflow"),
7411 (!signal_wait_events.is_empty()).then_some("signal wait"),
7412 (!side_effect_events.is_empty()).then_some("side effect"),
7413 (!version_marker_events.is_empty()).then_some("version marker"),
7414 ]
7415 .into_iter()
7416 .flatten()
7417 .collect::<Vec<_>>()
7418 .join(" and ");
7419 return Err(invalid_recorded_history(
7420 "durable_command_sequence_collision",
7421 sequence,
7422 "one durable command kind",
7423 &actual,
7424 "one workflow sequence records more than one durable command kind",
7425 ));
7426 }
7427
7428 if !activity_events.is_empty() {
7429 let scheduled_count = activity_events
7430 .iter()
7431 .filter(|event| event.event_type == "ActivityScheduled")
7432 .count();
7433 if scheduled_count > 1 {
7434 return Err(invalid_recorded_history(
7435 "duplicate_activity_schedule",
7436 sequence,
7437 "at most one ActivityScheduled event",
7438 "multiple ActivityScheduled events",
7439 "activity history schedules more than one command at one workflow sequence",
7440 ));
7441 }
7442 let activity_type = activity_events.iter().find_map(|event| {
7443 event
7444 .payload
7445 .get("activity_type")
7446 .or_else(|| event.payload.get("activity_name"))
7447 .and_then(Value::as_str)
7448 .map(str::to_string)
7449 });
7450 if activity_events.iter().filter_map(|event| {
7451 event
7452 .payload
7453 .get("activity_type")
7454 .or_else(|| event.payload.get("activity_name"))
7455 .and_then(Value::as_str)
7456 }).any(|candidate| Some(candidate) != activity_type.as_deref()) {
7457 return Err(invalid_recorded_history(
7458 "activity_identity_mismatch",
7459 sequence,
7460 activity_type.as_deref().unwrap_or("one activity identity"),
7461 "conflicting activity identities",
7462 "activity lifecycle events at one workflow sequence disagree on identity",
7463 ));
7464 }
7465 let terminal: Vec<_> = activity_events
7466 .iter()
7467 .copied()
7468 .filter(|event| {
7469 matches!(
7470 event.event_type.as_str(),
7471 "ActivityCompleted"
7472 | "ActivityFailed"
7473 | "ActivityCancelled"
7474 | "ActivityTimedOut"
7475 )
7476 })
7477 .collect();
7478 if terminal.len() > 1 {
7479 return Err(invalid_recorded_history(
7480 "duplicate_activity_terminal_event",
7481 sequence,
7482 "at most one terminal activity event",
7483 "multiple terminal activity events",
7484 "activity history settles one command more than once",
7485 ));
7486 }
7487 let outcome = terminal
7488 .first()
7489 .map(|event| activity_outcome(event, fallback_codec, activity_type.clone()))
7490 .transpose()?;
7491 let options = activity_events
7492 .iter()
7493 .find(|event| event.event_type == "ActivityScheduled")
7494 .and_then(|event| event.payload.get("activity"))
7495 .and_then(Value::as_object)
7496 .map(|activity| RecordedActivityOptions {
7497 task_queue: recorded_optional_string(activity, "queue"),
7498 execution_mode: recorded_optional_string(activity, "execution_mode"),
7499 retry_policy: recorded_activity_retry_snapshot(
7500 activity.get("retry_policy"),
7501 ),
7502 });
7503 return Ok(RecordedCommand::Activity {
7504 sequence,
7505 activity_type,
7506 options,
7507 outcome,
7508 });
7509 }
7510
7511 if !child_events.is_empty() {
7512 let scheduled: Vec<_> = child_events
7513 .iter()
7514 .copied()
7515 .filter(|event| event.event_type == "ChildWorkflowScheduled")
7516 .collect();
7517 if scheduled.len() != 1 {
7518 return Err(invalid_recorded_history(
7519 "child_workflow_schedule_missing_or_duplicate",
7520 sequence,
7521 "one ChildWorkflowScheduled event",
7522 &format!("{} ChildWorkflowScheduled events", scheduled.len()),
7523 "child workflow replay requires exactly one recorded schedule event",
7524 ));
7525 }
7526 let workflow_type = child_events.iter().find_map(|event| {
7527 event
7528 .payload
7529 .get("child_workflow_type")
7530 .or_else(|| event.payload.get("workflow_type"))
7531 .and_then(Value::as_str)
7532 .filter(|value| !value.is_empty())
7533 .map(str::to_string)
7534 });
7535 if child_events
7536 .iter()
7537 .filter_map(|event| {
7538 event
7539 .payload
7540 .get("child_workflow_type")
7541 .or_else(|| event.payload.get("workflow_type"))
7542 .and_then(Value::as_str)
7543 })
7544 .any(|candidate| Some(candidate) != workflow_type.as_deref())
7545 {
7546 return Err(invalid_recorded_history(
7547 "child_workflow_identity_mismatch",
7548 sequence,
7549 workflow_type
7550 .as_deref()
7551 .unwrap_or("one child workflow type"),
7552 "conflicting child workflow types",
7553 "child workflow lifecycle events at one sequence disagree on type",
7554 ));
7555 }
7556 let mut outcomes = child_workflow_outcomes(
7557 &child_events.iter().map(|event| (*event).clone()).collect::<Vec<_>>(),
7558 fallback_codec,
7559 parent.clone(),
7560 )?;
7561 if outcomes.len() > 1 {
7562 return Err(invalid_recorded_history(
7563 "duplicate_child_workflow_terminal_event",
7564 sequence,
7565 "at most one terminal child event",
7566 "multiple terminal child events",
7567 "child workflow history settles one command more than once",
7568 ));
7569 }
7570 return Ok(RecordedCommand::ChildWorkflow {
7571 sequence,
7572 workflow_type,
7573 outcome: outcomes.pop(),
7574 });
7575 }
7576
7577 if !signal_wait_events.is_empty() {
7578 let opened: Vec<_> = signal_wait_events
7579 .iter()
7580 .copied()
7581 .filter(|event| event.event_type == "SignalWaitOpened")
7582 .collect();
7583 if opened.len() != 1 {
7584 return Err(invalid_recorded_history(
7585 "signal_wait_open_missing_or_duplicate",
7586 sequence,
7587 "one SignalWaitOpened event",
7588 &format!("{} SignalWaitOpened events", opened.len()),
7589 "signal replay requires exactly one canonical wait-open event",
7590 ));
7591 }
7592
7593 let applied: Vec<_> = signal_wait_events
7594 .iter()
7595 .copied()
7596 .filter(|event| event.event_type == "SignalApplied")
7597 .collect();
7598 if applied.len() > 1 {
7599 return Err(invalid_recorded_history(
7600 "duplicate_signal_wait_apply",
7601 sequence,
7602 "at most one SignalApplied event",
7603 "multiple SignalApplied events",
7604 "signal history applies one durable wait more than once",
7605 ));
7606 }
7607
7608 let signal_names = signal_wait_events
7609 .iter()
7610 .map(|event| required_signal_wait_name(event, sequence))
7611 .collect::<Result<Vec<_>>>()?;
7612 let signal_name = signal_names
7613 .first()
7614 .expect("signal wait events are not empty")
7615 .clone();
7616 if signal_names.iter().any(|candidate| candidate != &signal_name) {
7617 return Err(invalid_recorded_history(
7618 "signal_wait_identity_mismatch",
7619 sequence,
7620 &signal_name,
7621 "conflicting signal names",
7622 "signal wait lifecycle events at one workflow sequence disagree on identity",
7623 ));
7624 }
7625 let value = applied
7626 .first()
7627 .map(|event| decode_signal_event_arguments(event, fallback_codec))
7628 .transpose()?;
7629 return Ok(RecordedCommand::SignalWait {
7630 sequence,
7631 signal_name,
7632 value,
7633 });
7634 }
7635
7636 if !side_effect_events.is_empty() {
7637 if side_effect_events.len() != 1 {
7638 return Err(invalid_recorded_history(
7639 "duplicate_side_effect_record",
7640 sequence,
7641 "one SideEffectRecorded event",
7642 &format!("{} SideEffectRecorded events", side_effect_events.len()),
7643 "side-effect history records one workflow command more than once",
7644 ));
7645 }
7646 let event = side_effect_events[0];
7647 let result = event.payload.get("result").ok_or_else(|| {
7648 invalid_recorded_history(
7649 "side_effect_result_missing",
7650 sequence,
7651 "recorded result payload",
7652 "missing result",
7653 "side-effect history is missing its recorded value",
7654 )
7655 })?;
7656 let has_published_envelope = result.as_str().is_some()
7657 || result.as_object().is_some_and(|envelope| {
7658 envelope.get("codec").and_then(Value::as_str).is_some()
7659 && envelope.get("blob").and_then(Value::as_str).is_some()
7660 });
7661 if !has_published_envelope {
7662 return Err(invalid_recorded_history(
7663 "side_effect_payload_malformed",
7664 sequence,
7665 "payload blob or {codec, blob} envelope",
7666 &result.to_string(),
7667 "side-effect history result does not use a published payload envelope",
7668 ));
7669 }
7670 let codec = event
7671 .payload
7672 .get("payload_codec")
7673 .and_then(Value::as_str)
7674 .unwrap_or(fallback_codec);
7675 let value = decode_wire_avro_value(result, codec).map_err(|error| {
7676 if error.to_string().contains("unsupported_payload_codec") {
7677 return error;
7678 }
7679
7680 invalid_recorded_history(
7681 "side_effect_payload_incompatible",
7682 sequence,
7683 &format!("valid {codec} payload envelope"),
7684 &error.to_string(),
7685 "side-effect history payload cannot be decoded with its recorded codec",
7686 )
7687 })?;
7688 return Ok(RecordedCommand::SideEffect { sequence, value });
7689 }
7690
7691 if !version_marker_events.is_empty() {
7692 if version_marker_events.len() != 1 {
7693 return Err(invalid_recorded_history(
7694 "duplicate_version_marker_record",
7695 sequence,
7696 "one VersionMarkerRecorded event",
7697 &format!("{} VersionMarkerRecorded events", version_marker_events.len()),
7698 "version-marker history records one workflow command more than once",
7699 ));
7700 }
7701 let payload = &version_marker_events[0].payload;
7702 let change_id = payload
7703 .get("change_id")
7704 .and_then(Value::as_str)
7705 .filter(|value| !value.is_empty())
7706 .map(str::to_string)
7707 .ok_or_else(|| {
7708 invalid_recorded_history(
7709 "version_marker_field_missing",
7710 sequence,
7711 "non-empty change_id",
7712 "missing or invalid change_id",
7713 "version-marker history is missing its stable change ID",
7714 )
7715 })?;
7716 let version = required_version_i32(payload, "version", sequence)?;
7717 let min_supported = required_version_i32(payload, "min_supported", sequence)?;
7718 let max_supported = required_version_i32(payload, "max_supported", sequence)?;
7719 if min_supported > max_supported || version < min_supported || version > max_supported {
7720 return Err(invalid_recorded_history(
7721 "version_marker_history_range_invalid",
7722 sequence,
7723 "min_supported <= version <= max_supported",
7724 &format!("{min_supported} <= {version} <= {max_supported}"),
7725 "recorded version marker contains an internally incompatible range",
7726 ));
7727 }
7728 return Ok(RecordedCommand::VersionMarker {
7729 sequence,
7730 change_id,
7731 version,
7732 });
7733 }
7734
7735 let scheduled: Vec<_> = timer_events
7736 .iter()
7737 .copied()
7738 .filter(|event| event.event_type == "TimerScheduled")
7739 .collect();
7740 let fired: Vec<_> = timer_events
7741 .iter()
7742 .copied()
7743 .filter(|event| event.event_type == "TimerFired")
7744 .collect();
7745 if scheduled.len() != 1 {
7746 return Err(invalid_recorded_history(
7747 "timer_schedule_missing_or_duplicate",
7748 sequence,
7749 "one TimerScheduled event",
7750 &format!("{} TimerScheduled events", scheduled.len()),
7751 "timer replay requires exactly one recorded schedule event",
7752 ));
7753 }
7754 if fired.len() > 1 {
7755 return Err(invalid_recorded_history(
7756 "duplicate_timer_fire",
7757 sequence,
7758 "at most one TimerFired event",
7759 "multiple TimerFired events",
7760 "timer history contains more than one fire event for a workflow sequence",
7761 ));
7762 }
7763
7764 let scheduled = scheduled[0];
7765 let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
7766 let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
7767 if let Some(fired) = fired.first() {
7768 let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
7769 if fired_timer_id != timer_id {
7770 return Err(invalid_recorded_history(
7771 "timer_identity_mismatch",
7772 sequence,
7773 &timer_id,
7774 &fired_timer_id,
7775 "TimerFired does not correspond to the recorded TimerScheduled event",
7776 ));
7777 }
7778 let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
7779 if fired_delay != delay_seconds {
7780 return Err(invalid_recorded_history(
7781 "timer_history_delay_mismatch",
7782 sequence,
7783 &delay_seconds.to_string(),
7784 &fired_delay.to_string(),
7785 "TimerScheduled and TimerFired record different delays",
7786 ));
7787 }
7788 }
7789
7790 Ok(RecordedCommand::Timer {
7791 sequence,
7792 delay_seconds,
7793 fired: !fired.is_empty(),
7794 })
7795 })
7796 .collect::<Result<_>>()?;
7797
7798 let mut marker_sequences = HashMap::new();
7799 for command in &commands {
7800 if let RecordedCommand::VersionMarker {
7801 sequence,
7802 change_id,
7803 ..
7804 } = command
7805 {
7806 if let Some(first_sequence) = marker_sequences.insert(change_id.clone(), *sequence) {
7807 return Err(invalid_recorded_history(
7808 "duplicate_version_marker",
7809 *sequence,
7810 &format!("one marker for change ID {change_id:?}"),
7811 &format!("markers at sequences {first_sequence} and {sequence}"),
7812 "workflow history contains duplicate markers for one stable change ID",
7813 ));
7814 }
7815 }
7816 }
7817
7818 Ok(commands)
7819}
7820
7821fn required_version_i32(payload: &Value, field: &str, sequence: u64) -> Result<i32> {
7822 payload
7823 .get(field)
7824 .and_then(Value::as_i64)
7825 .and_then(|value| i32::try_from(value).ok())
7826 .ok_or_else(|| {
7827 invalid_recorded_history(
7828 "version_marker_field_missing",
7829 sequence,
7830 &format!("integer {field}"),
7831 "missing or out-of-range integer",
7832 "version-marker history is missing a required integer field",
7833 )
7834 })
7835}
7836
7837fn durable_event_sequence(event: &HistoryEvent) -> Option<u64> {
7838 event
7839 .payload
7840 .get("sequence")
7841 .or_else(|| event.payload.get("workflow_sequence"))
7842 .or_else(|| event.raw.get("sequence"))
7843 .or_else(|| event.raw.get("workflow_sequence"))
7844 .and_then(value_as_u64)
7845}
7846
7847fn is_internal_timer_event(event: &HistoryEvent) -> bool {
7848 matches!(
7849 event
7850 .payload
7851 .get("timer_kind")
7852 .or_else(|| event.raw.get("timer_kind"))
7853 .and_then(Value::as_str),
7854 Some("condition_timeout" | "signal_timeout")
7855 )
7856}
7857
7858fn required_signal_wait_name(event: &HistoryEvent, sequence: u64) -> Result<String> {
7859 event
7860 .payload
7861 .get("signal_name")
7862 .or_else(|| event.raw.get("signal_name"))
7863 .and_then(Value::as_str)
7864 .filter(|value| !value.is_empty())
7865 .map(str::to_string)
7866 .ok_or_else(|| {
7867 invalid_recorded_history(
7868 "signal_wait_name_missing",
7869 sequence,
7870 "non-empty signal_name",
7871 &event.event_type,
7872 "canonical signal-wait history is missing its signal identity",
7873 )
7874 })
7875}
7876
7877fn is_recorded_signal_wait_event(event: &HistoryEvent) -> bool {
7878 matches!(
7879 event.event_type.as_str(),
7880 "SignalWaitOpened" | "SignalApplied"
7881 )
7882}
7883
7884fn required_history_string(event: &HistoryEvent, field: &str, sequence: u64) -> Result<String> {
7885 event
7886 .payload
7887 .get(field)
7888 .and_then(Value::as_str)
7889 .filter(|value| !value.is_empty())
7890 .map(str::to_string)
7891 .ok_or_else(|| {
7892 invalid_recorded_history(
7893 "timer_history_field_missing",
7894 sequence,
7895 field,
7896 &event.event_type,
7897 "timer history is missing a required identity field",
7898 )
7899 })
7900}
7901
7902fn required_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<u64> {
7903 event
7904 .payload
7905 .get(field)
7906 .and_then(value_as_u64)
7907 .ok_or_else(|| {
7908 invalid_recorded_history(
7909 "timer_history_field_missing",
7910 sequence,
7911 field,
7912 &event.event_type,
7913 "timer history is missing a required numeric field",
7914 )
7915 })
7916}
7917
7918fn invalid_recorded_history(
7919 reason: &str,
7920 sequence: u64,
7921 expected: &str,
7922 actual: &str,
7923 message: &str,
7924) -> Error {
7925 Error::NonDeterministicReplay(ReplayFailure::new(
7926 reason,
7927 Some(sequence),
7928 Some(expected.to_string()),
7929 Some(actual.to_string()),
7930 message,
7931 ))
7932}
7933
7934type ActivityOutcome = std::result::Result<AvroValue, ActivityFailure>;
7935
7936fn activity_outcome(
7937 event: &HistoryEvent,
7938 fallback_codec: &str,
7939 recorded_activity_type: Option<String>,
7940) -> Result<ActivityOutcome> {
7941 if event.event_type == "ActivityCompleted" {
7942 let codec = event
7943 .payload
7944 .get("payload_codec")
7945 .and_then(Value::as_str)
7946 .unwrap_or(fallback_codec);
7947 return Ok(Ok(decode_wire_avro_value(
7948 event.payload.get("result").unwrap_or(&Value::Null),
7949 codec,
7950 )?));
7951 }
7952
7953 let payload = &event.payload;
7954 let (kind, fallback_reason, fallback_message) = match event.event_type.as_str() {
7955 "ActivityFailed" => (ActivityFailureKind::Failed, "activity", "activity failed"),
7956 "ActivityCancelled" => (
7957 ActivityFailureKind::Cancelled,
7958 "cancelled",
7959 "activity was cancelled",
7960 ),
7961 "ActivityTimedOut" => (
7962 ActivityFailureKind::TimedOut,
7963 "timeout",
7964 "activity timed out",
7965 ),
7966 _ => unreachable!("activity_outcome is called only for terminal activity events"),
7967 };
7968 let exception = payload
7969 .get("exception")
7970 .filter(|value| !value.is_null())
7971 .cloned();
7972 let failure_category = payload_string(payload, "failure_category");
7973 let timeout_kind = payload_string(payload, "timeout_kind");
7974 let reason = payload_string(payload, "reason").unwrap_or_else(|| match kind {
7975 ActivityFailureKind::Failed => failure_category
7976 .clone()
7977 .unwrap_or_else(|| fallback_reason.to_string()),
7978 ActivityFailureKind::Cancelled => fallback_reason.to_string(),
7979 ActivityFailureKind::TimedOut => timeout_kind
7980 .clone()
7981 .unwrap_or_else(|| fallback_reason.to_string()),
7982 });
7983 let message = payload_string(payload, "message")
7984 .or_else(|| {
7985 exception
7986 .as_ref()
7987 .and_then(|value| payload_string(value, "message"))
7988 })
7989 .unwrap_or_else(|| fallback_message.to_string());
7990
7991 Ok(Err(ActivityFailure {
7992 kind,
7993 reason,
7994 message,
7995 activity_execution_id: payload_string(payload, "activity_execution_id"),
7996 activity_attempt_id: payload_string(payload, "activity_attempt_id"),
7997 activity_type: payload_string(payload, "activity_type")
7998 .or_else(|| payload_string(payload, "activity_name"))
7999 .or(recorded_activity_type),
8000 activity_class: payload_string(payload, "activity_class"),
8001 attempt_number: payload.get("attempt_number").and_then(value_as_u64),
8002 failure_id: payload_string(payload, "failure_id"),
8003 failure_category,
8004 timeout_kind,
8005 non_retryable: payload
8006 .get("non_retryable")
8007 .and_then(Value::as_bool)
8008 .unwrap_or(false),
8009 exception_type: payload_string(payload, "exception_type").or_else(|| {
8010 exception
8011 .as_ref()
8012 .and_then(|value| payload_string(value, "type"))
8013 }),
8014 exception_class: payload_string(payload, "exception_class").or_else(|| {
8015 exception
8016 .as_ref()
8017 .and_then(|value| payload_string(value, "class"))
8018 }),
8019 code: payload
8020 .get("code")
8021 .filter(|value| !value.is_null())
8022 .cloned(),
8023 exception,
8024 }))
8025}
8026
8027type ChildWorkflowOutcome = std::result::Result<ChildWorkflowAvroResult, ChildWorkflowFailure>;
8028
8029fn child_workflow_outcomes(
8030 events: &[HistoryEvent],
8031 fallback_codec: &str,
8032 parent: WorkflowIdentity,
8033) -> Result<Vec<ChildWorkflowOutcome>> {
8034 let mut outcomes = Vec::new();
8035
8036 for event in events {
8037 let kind = match event.event_type.as_str() {
8038 "ChildRunCompleted" => None,
8039 "ChildRunFailed" => Some((
8040 ChildWorkflowFailureKind::Failed,
8041 "child_workflow",
8042 "child workflow failed",
8043 )),
8044 "ChildRunCancelled" => Some((
8045 ChildWorkflowFailureKind::Cancelled,
8046 "cancelled",
8047 "child workflow was cancelled",
8048 )),
8049 "ChildRunTerminated" => Some((
8050 ChildWorkflowFailureKind::Terminated,
8051 "terminated",
8052 "child workflow was terminated",
8053 )),
8054 _ => continue,
8055 };
8056 let payload = &event.payload;
8057 let child_workflow_id = payload_string(payload, "child_workflow_instance_id");
8058 let child_workflow_run_id = payload_string(payload, "child_workflow_run_id");
8059 let child_workflow_type = payload_string(payload, "child_workflow_type");
8060
8061 if let Some((kind, reason, fallback_message)) = kind {
8062 let exception = payload
8063 .get("exception")
8064 .filter(|value| !value.is_null())
8065 .cloned();
8066 let message = payload_string(payload, "message")
8067 .or_else(|| {
8068 exception
8069 .as_ref()
8070 .and_then(|value| payload_string(value, "message"))
8071 })
8072 .unwrap_or_else(|| fallback_message.to_string());
8073 let exception_type = payload_string(payload, "exception_type").or_else(|| {
8074 exception
8075 .as_ref()
8076 .and_then(|value| payload_string(value, "type"))
8077 });
8078 let exception_class = payload_string(payload, "exception_class").or_else(|| {
8079 exception
8080 .as_ref()
8081 .and_then(|value| payload_string(value, "class"))
8082 });
8083 outcomes.push(Err(ChildWorkflowFailure {
8084 kind,
8085 reason: reason.to_string(),
8086 message,
8087 parent_workflow_id: parent.workflow_id.clone(),
8088 parent_workflow_run_id: parent.run_id.clone(),
8089 child_workflow_id,
8090 child_workflow_run_id,
8091 child_workflow_type,
8092 failure_id: payload_string(payload, "failure_id"),
8093 failure_category: payload_string(payload, "failure_category"),
8094 exception_type,
8095 exception_class,
8096 non_retryable: payload
8097 .get("non_retryable")
8098 .and_then(Value::as_bool)
8099 .unwrap_or(false),
8100 code: payload
8101 .get("code")
8102 .filter(|value| !value.is_null())
8103 .cloned(),
8104 exception,
8105 }));
8106 continue;
8107 }
8108
8109 let codec = payload
8110 .get("payload_codec")
8111 .and_then(Value::as_str)
8112 .unwrap_or(fallback_codec);
8113 let result = payload
8114 .get("result")
8115 .or_else(|| payload.get("output"))
8116 .unwrap_or(&Value::Null);
8117 outcomes.push(Ok(ChildWorkflowAvroResult {
8118 parent: parent.clone(),
8119 child: WorkflowIdentity {
8120 workflow_id: child_workflow_id,
8121 run_id: child_workflow_run_id,
8122 },
8123 child_workflow_type,
8124 result: decode_wire_avro_value(result, codec)?,
8125 }));
8126 }
8127
8128 Ok(outcomes)
8129}
8130
8131fn payload_string(payload: &Value, key: &str) -> Option<String> {
8132 payload
8133 .get(key)
8134 .and_then(Value::as_str)
8135 .filter(|value| !value.is_empty())
8136 .map(str::to_string)
8137}
8138
8139fn workflow_failure_command(error: &Error) -> Value {
8140 let (exception_type, exception_class, properties) = match error {
8141 Error::ActivityFailed(failure) => (
8142 match failure.kind {
8143 ActivityFailureKind::Failed => "ActivityFailed",
8144 ActivityFailureKind::Cancelled => "ActivityCancelled",
8145 ActivityFailureKind::TimedOut => "ActivityTimedOut",
8146 },
8147 "durable_workflow::ActivityFailure",
8148 json!({
8149 "reason": failure.reason,
8150 "activity_execution_id": failure.activity_execution_id,
8151 "activity_attempt_id": failure.activity_attempt_id,
8152 "activity_type": failure.activity_type,
8153 "activity_class": failure.activity_class,
8154 "attempt_number": failure.attempt_number,
8155 "failure_id": failure.failure_id,
8156 "failure_category": failure.failure_category,
8157 "timeout_kind": failure.timeout_kind,
8158 "activity_non_retryable": failure.non_retryable,
8159 "activity_exception_type": failure.exception_type,
8160 "activity_exception_class": failure.exception_class,
8161 "activity_code": failure.code,
8162 "activity_exception": failure.exception,
8163 }),
8164 ),
8165 Error::ChildWorkflowFailed(failure) => (
8166 match failure.kind {
8167 ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
8168 ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
8169 ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
8170 },
8171 "durable_workflow::ChildWorkflowFailure",
8172 json!({
8173 "reason": failure.reason,
8174 "parent_workflow_id": failure.parent_workflow_id,
8175 "parent_workflow_run_id": failure.parent_workflow_run_id,
8176 "child_workflow_id": failure.child_workflow_id,
8177 "child_workflow_run_id": failure.child_workflow_run_id,
8178 "child_workflow_type": failure.child_workflow_type,
8179 "failure_id": failure.failure_id,
8180 "failure_category": failure.failure_category,
8181 "child_exception_type": failure.exception_type,
8182 "child_exception_class": failure.exception_class,
8183 "child_non_retryable": failure.non_retryable,
8184 "child_code": failure.code,
8185 "child_exception": failure.exception,
8186 }),
8187 ),
8188 Error::NonDeterministicReplay(_) => (
8189 "NonDeterministicReplay",
8190 "durable_workflow::Error",
8191 Value::Null,
8192 ),
8193 _ => ("RustWorkflowError", "durable_workflow::Error", Value::Null),
8194 };
8195 let non_retryable = match error {
8196 Error::ActivityFailed(failure) => failure.non_retryable,
8197 Error::ChildWorkflowFailed(failure) => failure.non_retryable,
8198 Error::NonDeterministicReplay(_) => true,
8199 _ => false,
8200 };
8201
8202 json!({
8203 "type": "fail_workflow",
8204 "message": error.to_string(),
8205 "exception_type": exception_type,
8206 "exception_class": exception_class,
8207 "non_retryable": non_retryable,
8208 "exception": {
8209 "type": exception_type,
8210 "class": exception_class,
8211 "message": error.to_string(),
8212 "properties": properties,
8213 }
8214 })
8215}
8216
8217fn workflow_task_integrity_error(error: &Error) -> bool {
8218 matches!(
8219 error,
8220 Error::NonDeterministicReplay(_) | Error::Protocol(_) | Error::WorkflowStatePoisoned
8221 )
8222}
8223
8224fn decode_signal_event_arguments(
8225 event: &HistoryEvent,
8226 fallback_codec: &str,
8227) -> Result<Vec<AvroValue>> {
8228 let codec = declared_payload_codec(&event.payload, "payload_codec")?.unwrap_or(fallback_codec);
8229 validate_payload_codec(codec)?;
8230 let raw = signal_history_payload(&event.payload);
8231 let decoded = match raw.filter(|value| !value.is_null()) {
8232 Some(value) => decode_wire_avro_value(value, codec)?,
8233 None => AvroValue::Array(Vec::new()),
8234 };
8235 let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
8236 unreachable!("normalize_avro_arguments always returns an array");
8237 };
8238 Ok(arguments)
8239}
8240
8241fn hydrate_query_history_from_export(task: &mut QueryTask) -> Result<()> {
8242 let Some(export_events) = task
8243 .history_export
8244 .as_ref()
8245 .and_then(|export| export.get("history_events"))
8246 .and_then(Value::as_array)
8247 else {
8248 return Ok(());
8249 };
8250
8251 if export_events.len() > task.history_events.len() {
8252 task.history_events = serde_json::from_value(Value::Array(export_events.clone()))?;
8253 }
8254
8255 Ok(())
8256}
8257
8258fn enrich_query_history_from_export(task: &mut QueryTask) -> Result<()> {
8259 let Some(export) = task.history_export.as_ref() else {
8260 return Ok(());
8261 };
8262 let signals = export
8263 .get("signals")
8264 .and_then(Value::as_array)
8265 .cloned()
8266 .unwrap_or_default();
8267 let activities = export
8268 .get("activities")
8269 .and_then(Value::as_array)
8270 .cloned()
8271 .unwrap_or_default();
8272 let export_codec = export
8273 .get("payloads")
8274 .and_then(|payloads| payloads.get("codec"))
8275 .and_then(Value::as_str)
8276 .unwrap_or(&task.payload_codec)
8277 .to_string();
8278 let mut signal_name_offsets: HashMap<String, usize> = HashMap::new();
8279
8280 for event in &mut task.history_events {
8281 if event.event_type == "ActivityCompleted" {
8282 let sequence = event
8283 .payload
8284 .get("sequence")
8285 .or_else(|| event.payload.get("workflow_sequence"))
8286 .and_then(value_as_u64);
8287 let Some(activity) = sequence.and_then(|sequence| {
8288 activities.iter().find(|activity| {
8289 activity.get("sequence").and_then(value_as_u64) == Some(sequence)
8290 })
8291 }) else {
8292 continue;
8293 };
8294 let Some(payload) = event.payload.as_object_mut() else {
8295 continue;
8296 };
8297 if missing_payload(payload.get("result")) {
8298 if let Some(result) = activity
8299 .get("result")
8300 .filter(|value| !missing_payload(Some(value)))
8301 {
8302 payload.insert("result".to_string(), result.clone());
8303 }
8304 }
8305 for field in ["payload_codec", "activity_type"] {
8306 if payload
8307 .get(field)
8308 .and_then(Value::as_str)
8309 .unwrap_or_default()
8310 .is_empty()
8311 {
8312 if let Some(value) = activity.get(field) {
8313 payload.insert(field.to_string(), value.clone());
8314 }
8315 }
8316 }
8317 continue;
8318 }
8319
8320 if event.event_type != "SignalReceived" && event.event_type != "SignalApplied" {
8321 continue;
8322 }
8323 let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
8324 let command_id = event
8325 .payload
8326 .get("workflow_command_id")
8327 .or_else(|| event.raw.get("workflow_command_id"))
8328 .and_then(Value::as_str);
8329 let signal_name = event
8330 .payload
8331 .get("signal_name")
8332 .and_then(Value::as_str)
8333 .unwrap_or_default()
8334 .to_string();
8335 let matched = signals
8336 .iter()
8337 .find(|signal| {
8338 signal_id.is_some() && signal.get("id").and_then(Value::as_str) == signal_id
8339 })
8340 .or_else(|| {
8341 signals.iter().find(|signal| {
8342 command_id.is_some()
8343 && signal.get("command_id").and_then(Value::as_str) == command_id
8344 })
8345 })
8346 .or_else(|| {
8347 let offset = signal_name_offsets.entry(signal_name.clone()).or_default();
8348 let signal = signals
8349 .iter()
8350 .filter(|signal| {
8351 signal.get("name").and_then(Value::as_str) == Some(signal_name.as_str())
8352 })
8353 .nth(*offset);
8354 if signal.is_some() {
8355 *offset += 1;
8356 }
8357 signal
8358 });
8359 let Some(signal) = matched else {
8360 continue;
8361 };
8362 let signal_codec = signal
8363 .get("payload_codec")
8364 .and_then(Value::as_str)
8365 .unwrap_or(&export_codec);
8366 let Some(payload) = event.payload.as_object_mut() else {
8367 continue;
8368 };
8369 if missing_payload(payload.get("arguments")) {
8370 if let Some(arguments) = signal
8371 .get("arguments")
8372 .filter(|value| !missing_payload(Some(value)))
8373 {
8374 let envelope = match arguments {
8375 Value::String(blob) => json!({"codec": signal_codec, "blob": blob}),
8376 other => other.clone(),
8377 };
8378 payload.insert("arguments".to_string(), envelope);
8379 }
8380 }
8381 if payload
8382 .get("payload_codec")
8383 .and_then(Value::as_str)
8384 .unwrap_or_default()
8385 .is_empty()
8386 {
8387 payload.insert("payload_codec".to_string(), json!(signal_codec));
8388 }
8389 }
8390
8391 Ok(())
8392}
8393
8394fn missing_payload(value: Option<&Value>) -> bool {
8395 match value {
8396 None | Some(Value::Null) => true,
8397 Some(Value::String(value)) => value.is_empty(),
8398 Some(_) => false,
8399 }
8400}
8401
8402fn query_signal_events(task: &QueryTask) -> Result<Vec<QuerySignal>> {
8403 let export_signals = task
8404 .history_export
8405 .as_ref()
8406 .and_then(|export| export.get("signals"))
8407 .and_then(Value::as_array)
8408 .cloned()
8409 .unwrap_or_default();
8410 let export_codec = task
8411 .history_export
8412 .as_ref()
8413 .and_then(|export| export.get("payloads"))
8414 .and_then(|payloads| payloads.get("codec"))
8415 .and_then(Value::as_str)
8416 .unwrap_or(&task.payload_codec);
8417 let mut name_offsets: HashMap<String, usize> = HashMap::new();
8418 let mut signals = Vec::new();
8419
8420 for event in &task.history_events {
8421 if event.event_type != "SignalApplied" && event.event_type != "SignalReceived" {
8422 continue;
8423 }
8424
8425 let name = event
8426 .payload
8427 .get("signal_name")
8428 .and_then(Value::as_str)
8429 .unwrap_or_default();
8430 if name.is_empty() {
8431 continue;
8432 }
8433 let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
8434 let command_id = event
8435 .payload
8436 .get("workflow_command_id")
8437 .or_else(|| event.raw.get("workflow_command_id"))
8438 .and_then(Value::as_str);
8439 let matched_export = export_signals
8440 .iter()
8441 .find(|candidate| {
8442 signal_id.is_some() && candidate.get("id").and_then(Value::as_str) == signal_id
8443 })
8444 .or_else(|| {
8445 export_signals.iter().find(|candidate| {
8446 command_id.is_some()
8447 && candidate.get("command_id").and_then(Value::as_str) == command_id
8448 })
8449 })
8450 .or_else(|| {
8451 let offset = name_offsets.entry(name.to_string()).or_default();
8452 let candidate = export_signals
8453 .iter()
8454 .filter(|candidate| candidate.get("name").and_then(Value::as_str) == Some(name))
8455 .nth(*offset);
8456 if candidate.is_some() {
8457 *offset += 1;
8458 }
8459 candidate
8460 });
8461 let codec = event
8462 .payload
8463 .get("payload_codec")
8464 .and_then(Value::as_str)
8465 .or_else(|| {
8466 matched_export
8467 .and_then(|signal| signal.get("payload_codec"))
8468 .and_then(Value::as_str)
8469 })
8470 .unwrap_or(export_codec);
8471 let raw_arguments = signal_history_payload(&event.payload)
8472 .filter(|value| !value.is_null())
8473 .or_else(|| matched_export.and_then(|signal| signal.get("arguments")));
8474 let (arguments, avro_arguments) = decode_query_signal_arguments(raw_arguments, codec)?;
8475 let workflow_sequence = event
8476 .payload
8477 .get("workflow_sequence")
8478 .and_then(value_as_u64)
8479 .or_else(|| {
8480 matched_export
8481 .and_then(|signal| signal.get("workflow_sequence"))
8482 .and_then(value_as_u64)
8483 });
8484
8485 signals.push(QuerySignal {
8486 id: signal_id.map(str::to_string).or_else(|| {
8487 matched_export
8488 .and_then(|signal| signal.get("id"))
8489 .and_then(Value::as_str)
8490 .map(str::to_string)
8491 }),
8492 name: name.to_string(),
8493 arguments,
8494 avro_arguments,
8495 workflow_sequence,
8496 });
8497 }
8498
8499 if signals.is_empty() {
8500 for signal in export_signals {
8501 if signal.get("status").and_then(Value::as_str) == Some("rejected") {
8502 continue;
8503 }
8504 let Some(name) = signal.get("name").and_then(Value::as_str) else {
8505 continue;
8506 };
8507 let codec = signal
8508 .get("payload_codec")
8509 .and_then(Value::as_str)
8510 .unwrap_or(export_codec);
8511 let (arguments, avro_arguments) =
8512 decode_query_signal_arguments(signal.get("arguments"), codec)?;
8513 signals.push(QuerySignal {
8514 id: signal.get("id").and_then(Value::as_str).map(str::to_string),
8515 name: name.to_string(),
8516 arguments,
8517 avro_arguments,
8518 workflow_sequence: signal.get("workflow_sequence").and_then(value_as_u64),
8519 });
8520 }
8521 signals.sort_by_key(|signal| signal.workflow_sequence.unwrap_or(u64::MAX));
8522 }
8523
8524 Ok(signals)
8525}
8526
8527fn decode_query_signal_arguments(
8528 raw: Option<&Value>,
8529 codec: &str,
8530) -> Result<(Vec<Value>, Vec<AvroValue>)> {
8531 validate_payload_codec(codec)?;
8532 let decoded = match raw.filter(|value| !value.is_null()) {
8533 Some(value) => decode_wire_avro_value(value, codec)?,
8534 None => AvroValue::Array(Vec::new()),
8535 };
8536 let AvroValue::Array(avro_arguments) = normalize_avro_arguments(decoded) else {
8537 unreachable!("normalize_avro_arguments always returns an array");
8538 };
8539 let arguments = avro_arguments
8540 .iter()
8541 .cloned()
8542 .map(AvroValue::into_json)
8543 .collect::<Result<Vec<_>>>()?;
8544 Ok((arguments, avro_arguments))
8545}
8546
8547fn value_as_u64(value: &Value) -> Option<u64> {
8548 value
8549 .as_u64()
8550 .or_else(|| value.as_str().and_then(|value| value.parse().ok()))
8551}
8552
8553#[cfg(test)]
8554mod tests {
8555 use super::*;
8556 use std::{
8557 io::{Read, Write},
8558 net::{SocketAddr, TcpListener, TcpStream},
8559 sync::atomic::AtomicUsize,
8560 thread,
8561 };
8562
8563 #[derive(Clone, Copy, Debug)]
8564 enum InvalidTaskPayloadCodec {
8565 Missing,
8566 Null,
8567 NonString,
8568 }
8569
8570 impl InvalidTaskPayloadCodec {
8571 fn label(self) -> &'static str {
8572 match self {
8573 Self::Missing => "missing",
8574 Self::Null => "null",
8575 Self::NonString => "non-string",
8576 }
8577 }
8578
8579 fn apply(self, task: &mut Value) {
8580 let task = task.as_object_mut().expect("task fixture object");
8581 match self {
8582 Self::Missing => {
8583 task.remove("payload_codec");
8584 }
8585 Self::Null => {
8586 task.insert("payload_codec".to_string(), Value::Null);
8587 }
8588 Self::NonString => {
8589 task.insert("payload_codec".to_string(), json!(42));
8590 }
8591 }
8592 }
8593 }
8594
8595 fn fixture_envelope(value: Value) -> Value {
8596 encode_value_envelope(&value, DEFAULT_CODEC).expect("encode Avro test fixture")
8597 }
8598
8599 fn fixture_blob(value: Value) -> String {
8600 encode_payload(&value, DEFAULT_CODEC)
8601 .expect("encode Avro test fixture")
8602 .blob
8603 }
8604
8605 #[test]
8606 fn client_builder_rejects_the_sdk_owned_api_suffix() {
8607 for base_url in [
8608 "http://127.0.0.1:8080/api",
8609 "http://localhost:8080/api/",
8610 "https://runtime.example.test/namespaces/orders/api",
8611 ] {
8612 let error = Client::builder(base_url)
8613 .build()
8614 .expect_err("SDK-owned /api suffix must be rejected during build");
8615
8616 assert!(matches!(error, Error::InvalidBaseUrl), "{base_url}");
8617 assert!(
8618 error.to_string().contains("SDK appends /api automatically"),
8619 "the validation error must explain how to fix the endpoint"
8620 );
8621 }
8622 }
8623
8624 #[test]
8625 fn client_builder_preserves_self_hosted_and_managed_runtime_prefixes() {
8626 for (base_url, expected) in [
8627 ("http://127.0.0.1:8080", "http://127.0.0.1:8080"),
8628 (
8629 "http://localhost:8080/durable-workflow/",
8630 "http://localhost:8080/durable-workflow",
8631 ),
8632 (
8633 "https://runtime.example.test/namespaces/orders",
8634 "https://runtime.example.test/namespaces/orders",
8635 ),
8636 (
8637 "https://runtime.example.test/gateway/api/namespaces/orders",
8638 "https://runtime.example.test/gateway/api/namespaces/orders",
8639 ),
8640 (
8641 "https://api.example.test/runtime/orders/",
8642 "https://api.example.test/runtime/orders",
8643 ),
8644 ] {
8645 let client = Client::builder(base_url)
8646 .build()
8647 .expect("Server and Cloud runtime base URL must remain valid");
8648
8649 assert_eq!(client.base_url, expected);
8650 }
8651 }
8652
8653 fn typed_fidelity_probe() -> AvroValue {
8654 AvroValue::Map(BTreeMap::from([
8655 ("bytes".to_string(), AvroValue::Bytes(vec![0, 0xff])),
8656 ("empty".to_string(), AvroValue::Map(BTreeMap::new())),
8657 (
8658 "numeric".to_string(),
8659 AvroValue::Map(BTreeMap::from([
8660 ("0".to_string(), AvroValue::String("zero".to_string())),
8661 ("1".to_string(), AvroValue::String("one".to_string())),
8662 ])),
8663 ),
8664 (
8665 "nested".to_string(),
8666 AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([(
8667 "enabled".to_string(),
8668 AvroValue::Boolean(true),
8669 )]))]),
8670 ),
8671 (
8672 "projection_collisions".to_string(),
8673 AvroValue::Array(projection_collision_probe()),
8674 ),
8675 ]))
8676 }
8677
8678 fn projection_collision_probe() -> Vec<AvroValue> {
8679 vec![
8680 AvroValue::Map(BTreeMap::from([
8681 ("$type".to_string(), AvroValue::String("bytes".to_string())),
8682 (
8683 "base64".to_string(),
8684 AvroValue::String("ordinary user text".to_string()),
8685 ),
8686 ])),
8687 AvroValue::Map(BTreeMap::from([
8688 ("$type".to_string(), AvroValue::String("map".to_string())),
8689 (
8690 "entries".to_string(),
8691 AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([
8692 ("key".to_string(), AvroValue::String("ordinary".to_string())),
8693 (
8694 "value".to_string(),
8695 AvroValue::String("user map".to_string()),
8696 ),
8697 ]))]),
8698 ),
8699 ])),
8700 ]
8701 }
8702
8703 #[derive(Clone, Debug, Default, PartialEq)]
8704 struct ReplayCounterState {
8705 loaded: Option<String>,
8706 count: i64,
8707 finished: bool,
8708 }
8709
8710 fn replay_counter_worker() -> Worker {
8711 let client = Client::new("http://127.0.0.1:8080").expect("client");
8712 let mut worker = Worker::new(client, "rust-workers");
8713 worker.register_replayed_workflow(
8714 "replay-counter",
8715 ReplayCounterState::default,
8716 |ctx, _input, state| async move {
8717 let loaded = ctx.activity("load-counter", json!([])).await?;
8718 state.update(|current| {
8719 current.loaded = loaded.as_str().map(str::to_string);
8720 })?;
8721 for _ in 0..2 {
8722 let signal = ctx.wait_signal("increment").await?;
8723 let amount = signal.first().and_then(Value::as_i64).unwrap_or_default();
8724 state.update(|current| current.count += amount)?;
8725 }
8726 state.update(|current| current.finished = true)?;
8727 state.read(|current| Ok(json!(current.count)))?
8728 },
8729 );
8730 worker.register_replayed_query::<ReplayCounterState, _, _>(
8731 "replay-counter",
8732 "current",
8733 |_ctx, state, _args| async move {
8734 Ok(json!({
8735 "loaded": state.loaded,
8736 "count": state.count,
8737 "finished": state.finished,
8738 }))
8739 },
8740 );
8741 worker.register_replayed_query::<ReplayCounterState, _, _>(
8742 "replay-counter",
8743 "detached-mutation",
8744 |_ctx, state, _args| async move {
8745 let mut detached = (*state).clone();
8746 detached.count = 999;
8747 Ok(json!(detached.count))
8748 },
8749 );
8750 worker.register_replayed_query::<ReplayCounterState, _, _>(
8751 "replay-counter",
8752 "failed-mutation",
8753 |_ctx, state, _args| async move {
8754 let mut detached = (*state).clone();
8755 detached.count = 999;
8756 Err(Error::WorkerLoop("query refused".to_string()))
8757 },
8758 );
8759 worker
8760 }
8761
8762 fn replay_counter_query(
8763 query_name: &str,
8764 history_events: Value,
8765 run_status: &str,
8766 ) -> QueryTask {
8767 let arguments = fixture_envelope(json!([]));
8768 serde_json::from_value(json!({
8769 "query_task_id": format!("query-{query_name}"),
8770 "workflow_type": "replay-counter",
8771 "query_name": query_name,
8772 "payload_codec": DEFAULT_CODEC,
8773 "workflow_arguments": arguments.clone(),
8774 "query_arguments": arguments,
8775 "history_events": history_events,
8776 "run_status": run_status,
8777 }))
8778 .expect("query task")
8779 }
8780
8781 fn workflow_context(history: Vec<HistoryEvent>) -> WorkflowContext {
8782 workflow_context_with_codec(history, DEFAULT_CODEC)
8783 }
8784
8785 fn workflow_context_with_codec(
8786 history: Vec<HistoryEvent>,
8787 payload_codec: &str,
8788 ) -> WorkflowContext {
8789 WorkflowContext {
8790 state: Arc::new(Mutex::new(
8791 WorkflowState::new_with_identity(
8792 history,
8793 None,
8794 None,
8795 "rust-workers".to_string(),
8796 payload_codec.to_string(),
8797 None,
8798 )
8799 .expect("valid workflow history"),
8800 )),
8801 }
8802 }
8803
8804 fn history_event(event_type: &str, payload: Value) -> HistoryEvent {
8805 HistoryEvent {
8806 event_type: event_type.to_string(),
8807 payload,
8808 raw: HashMap::new(),
8809 }
8810 }
8811
8812 fn workflow_task(
8813 workflow_type: &str,
8814 history_events: Vec<HistoryEvent>,
8815 payload_codec: &str,
8816 ) -> WorkflowTask {
8817 WorkflowTask {
8818 task_id: format!("wft-{workflow_type}"),
8819 workflow_id: Some(format!("wf-{workflow_type}")),
8820 run_id: Some(format!("run-{workflow_type}")),
8821 workflow_type: workflow_type.to_string(),
8822 payload_codec: payload_codec.to_string(),
8823 arguments: Some(
8824 encode_value_envelope(&json!([]), payload_codec).expect("workflow arguments"),
8825 ),
8826 total_history_events: Some(history_events.len() as u64),
8827 history_size_bytes: None,
8828 continue_as_new_recommended: None,
8829 history_budget_pressure: None,
8830 history_events,
8831 next_history_page_token: None,
8832 workflow_task_attempt: 1,
8833 workflow_signal_id: None,
8834 signal_name: None,
8835 signal_arguments: None,
8836 workflow_update_id: None,
8837 update_name: None,
8838 lease_owner: Some("rust-worker".to_string()),
8839 }
8840 }
8841
8842 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
8843 struct SideEffectProbe {
8844 request_id: String,
8845 attempt: u32,
8846 }
8847
8848 #[test]
8849 fn typed_side_effect_runs_callback_once_and_replay_skips_it() {
8850 let calls = AtomicUsize::new(0);
8851 let ctx = workflow_context(Vec::new());
8852 let value = ctx
8853 .side_effect(|| {
8854 calls.fetch_add(1, Ordering::SeqCst);
8855 SideEffectProbe {
8856 request_id: "request-42".to_string(),
8857 attempt: 3,
8858 }
8859 })
8860 .expect("first side effect");
8861 assert_eq!(value.attempt, 3);
8862 assert_eq!(calls.load(Ordering::SeqCst), 1);
8863 let commands = ctx.take_commands().expect("commands");
8864 assert_eq!(commands.len(), 1);
8865 assert_eq!(commands[0]["type"], "record_side_effect");
8866 assert_eq!(
8867 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
8868 serde_json::to_value(&value).expect("value")
8869 );
8870
8871 let replay = workflow_context(vec![history_event(
8872 "SideEffectRecorded",
8873 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
8874 )]);
8875 let replayed: SideEffectProbe = replay
8876 .side_effect(|| {
8877 calls.fetch_add(1, Ordering::SeqCst);
8878 panic!("committed side-effect callbacks must not run during replay")
8879 })
8880 .expect("replayed side effect");
8881 assert_eq!(replayed, value);
8882 assert_eq!(calls.load(Ordering::SeqCst), 1);
8883 assert!(replay.take_commands().expect("commands").is_empty());
8884 replay.ensure_history_consumed().expect("history consumed");
8885 }
8886
8887 #[test]
8888 fn side_effect_uses_avro_envelope_and_uuid_is_replay_stable() {
8889 let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
8890 let value = ctx
8891 .side_effect(|| SideEffectProbe {
8892 request_id: "avro-request".to_string(),
8893 attempt: 1,
8894 })
8895 .expect("Avro side effect");
8896 let uuid = ctx.uuid_v4().expect("deterministic UUID");
8897 let commands = ctx.take_commands().expect("commands");
8898 assert_eq!(commands.len(), 2);
8899 assert_eq!(commands[0]["result"]["codec"], DEFAULT_CODEC);
8900 assert_eq!(commands[1]["result"]["codec"], DEFAULT_CODEC);
8901 assert_eq!(
8902 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
8903 serde_json::to_value(&value).expect("value")
8904 );
8905
8906 let replay = workflow_context_with_codec(
8907 vec![
8908 history_event(
8909 "SideEffectRecorded",
8910 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
8911 ),
8912 history_event(
8913 "SideEffectRecorded",
8914 json!({"sequence": 2, "result": commands[1]["result"].clone()}),
8915 ),
8916 ],
8917 DEFAULT_CODEC,
8918 );
8919 let replayed: SideEffectProbe = replay
8920 .side_effect(|| panic!("Avro callback must not run"))
8921 .expect("replayed Avro value");
8922 let replayed_uuid = replay.uuid_v4().expect("replayed UUID");
8923 assert_eq!(replayed, value);
8924 assert_eq!(replayed_uuid, uuid);
8925 assert!(replay.take_commands().expect("commands").is_empty());
8926 }
8927
8928 #[test]
8929 fn typed_side_effect_replay_preserves_bytes_and_maps() {
8930 let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
8931 let value = ctx
8932 .side_effect_avro_value(typed_fidelity_probe)
8933 .expect("typed side effect");
8934 let commands = ctx.take_commands().expect("side-effect command");
8935 assert_eq!(
8936 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
8937 .expect("recorded side effect"),
8938 value
8939 );
8940
8941 let replay = workflow_context_with_codec(
8942 vec![history_event(
8943 "SideEffectRecorded",
8944 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
8945 )],
8946 DEFAULT_CODEC,
8947 );
8948 assert_eq!(
8949 replay
8950 .side_effect_avro_value(|| panic!("replay must not invoke callback"))
8951 .expect("replayed typed side effect"),
8952 value
8953 );
8954 }
8955
8956 #[test]
8957 fn ordered_side_effects_share_the_durable_command_stream() {
8958 let first = encode_value_envelope(&json!("first"), DEFAULT_CODEC).expect("first");
8959 let second = encode_value_envelope(&json!(29), DEFAULT_CODEC).expect("second");
8960 let ctx = workflow_context(vec![
8961 history_event(
8962 "SideEffectRecorded",
8963 json!({"sequence": 1, "result": first}),
8964 ),
8965 history_event(
8966 "SideEffectRecorded",
8967 json!({"sequence": 2, "result": second}),
8968 ),
8969 ]);
8970 let first: String = ctx
8971 .side_effect(|| panic!("first callback must not run"))
8972 .expect("first replay");
8973 let second: i32 = ctx
8974 .side_effect(|| panic!("second callback must not run"))
8975 .expect("second replay");
8976 assert_eq!(first, "first");
8977 assert_eq!(second, 29);
8978 ctx.ensure_history_consumed().expect("ordered history");
8979
8980 let reordered = workflow_context(vec![history_event(
8981 "VersionMarkerRecorded",
8982 json!({
8983 "sequence": 1,
8984 "change_id": "before-side-effect",
8985 "version": 1,
8986 "min_supported": 1,
8987 "max_supported": 1,
8988 }),
8989 )]);
8990 let error = reordered
8991 .side_effect(|| "new".to_string())
8992 .expect_err("command reordering must fail");
8993 assert!(matches!(
8994 error,
8995 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8996 if reason == "recorded_command_mismatch"
8997 ));
8998 }
8999
9000 #[test]
9001 fn version_markers_replay_across_upgrades_and_do_not_duplicate() {
9002 let ctx = workflow_context(Vec::new());
9003 assert_eq!(ctx.get_version("checkout-v2", 1, 2).expect("version"), 2);
9004 assert_eq!(ctx.get_version("checkout-v2", 1, 3).expect("cached"), 2);
9005 assert!(ctx.patched("new-search").expect("patch"));
9006 ctx.deprecate_patch("new-search").expect("deprecate patch");
9007 let commands = ctx.take_commands().expect("commands");
9008 assert_eq!(commands.len(), 2);
9009 assert_eq!(commands[0]["type"], "record_version_marker");
9010 assert_eq!(commands[0]["version"], 2);
9011 assert_eq!(commands[1]["change_id"], "new-search");
9012
9013 let replay = workflow_context(vec![history_event(
9014 "VersionMarkerRecorded",
9015 json!({
9016 "sequence": 1,
9017 "change_id": "checkout-v2",
9018 "version": 2,
9019 "min_supported": 1,
9020 "max_supported": 2,
9021 }),
9022 )]);
9023 assert_eq!(replay.get_version("checkout-v2", 1, 4).expect("upgrade"), 2);
9024 assert_eq!(replay.get_version("checkout-v2", 2, 5).expect("repeat"), 2);
9025 assert!(replay.take_commands().expect("commands").is_empty());
9026 replay.ensure_history_consumed().expect("history consumed");
9027 }
9028
9029 #[test]
9030 fn version_markers_reject_incompatible_or_malformed_history() {
9031 let incompatible = workflow_context(vec![history_event(
9032 "VersionMarkerRecorded",
9033 json!({
9034 "sequence": 1,
9035 "change_id": "checkout-v2",
9036 "version": 1,
9037 "min_supported": 1,
9038 "max_supported": 2,
9039 }),
9040 )]);
9041 let error = incompatible
9042 .get_version("checkout-v2", 2, 3)
9043 .expect_err("old version is unsupported");
9044 assert!(matches!(
9045 error,
9046 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
9047 if reason == "version_marker_incompatible_range"
9048 ));
9049
9050 for (history, reason) in [
9051 (
9052 vec![history_event("SideEffectRecorded", json!({"sequence": 1}))],
9053 "side_effect_result_missing",
9054 ),
9055 (
9056 vec![history_event(
9057 "SideEffectRecorded",
9058 json!({
9059 "sequence": 1,
9060 "result": {"codec": "avro", "blob": "not-base64"},
9061 }),
9062 )],
9063 "side_effect_payload_incompatible",
9064 ),
9065 (
9066 vec![history_event(
9067 "SideEffectRecorded",
9068 json!({"sequence": 1, "result": {"unwrapped": true}}),
9069 )],
9070 "side_effect_payload_malformed",
9071 ),
9072 (
9073 vec![history_event(
9074 "VersionMarkerRecorded",
9075 json!({
9076 "sequence": 1,
9077 "change_id": "change",
9078 "version": 1,
9079 "min_supported": 2,
9080 "max_supported": 1,
9081 }),
9082 )],
9083 "version_marker_history_range_invalid",
9084 ),
9085 ] {
9086 let error = WorkflowState::new(
9087 history,
9088 "rust-workers".to_string(),
9089 DEFAULT_CODEC.to_string(),
9090 None,
9091 )
9092 .expect_err("malformed history must fail");
9093 assert!(matches!(
9094 error,
9095 Error::NonDeterministicReplay(ReplayFailure { reason: actual, .. })
9096 if actual == reason
9097 ));
9098 }
9099 }
9100
9101 #[test]
9102 fn duplicate_side_effects_and_version_markers_are_rejected() {
9103 let duplicate_side_effect = WorkflowState::new(
9104 vec![
9105 history_event(
9106 "SideEffectRecorded",
9107 json!({"sequence": 1, "result": fixture_envelope(json!(1))}),
9108 ),
9109 history_event(
9110 "SideEffectRecorded",
9111 json!({"sequence": 1, "result": fixture_envelope(json!(2))}),
9112 ),
9113 ],
9114 "rust-workers".to_string(),
9115 DEFAULT_CODEC.to_string(),
9116 None,
9117 )
9118 .expect_err("duplicate side effect");
9119 assert!(matches!(
9120 duplicate_side_effect,
9121 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
9122 if reason == "duplicate_side_effect_record"
9123 ));
9124
9125 let marker = |sequence| {
9126 history_event(
9127 "VersionMarkerRecorded",
9128 json!({
9129 "sequence": sequence,
9130 "change_id": "same-change",
9131 "version": 1,
9132 "min_supported": 1,
9133 "max_supported": 1,
9134 }),
9135 )
9136 };
9137 let duplicate_marker = WorkflowState::new(
9138 vec![marker(1), marker(3)],
9139 "rust-workers".to_string(),
9140 DEFAULT_CODEC.to_string(),
9141 None,
9142 )
9143 .expect_err("duplicate marker");
9144 assert!(matches!(
9145 duplicate_marker,
9146 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
9147 if reason == "duplicate_version_marker"
9148 ));
9149 }
9150
9151 #[test]
9152 fn cold_worker_replay_does_not_repeat_committed_side_effects_or_markers() {
9153 fn worker(calls: Arc<AtomicUsize>) -> Worker {
9154 let client = Client::new("http://127.0.0.1:8080").expect("client");
9155 let mut worker = Worker::new(client, "rust-workers");
9156 worker.register_workflow("rust.side-effect-version", move |ctx, _input| {
9157 let calls = Arc::clone(&calls);
9158 async move {
9159 let captured = ctx.side_effect(|| {
9160 calls.fetch_add(1, Ordering::SeqCst);
9161 "captured-once".to_string()
9162 })?;
9163 let version = ctx.get_version("cold-restart", 1, 2)?;
9164 Ok(json!({"captured": captured, "version": version}))
9165 }
9166 });
9167 worker
9168 }
9169
9170 fn task(history_events: Vec<HistoryEvent>) -> WorkflowTask {
9171 WorkflowTask {
9172 task_id: "wft-side-effect-version".to_string(),
9173 workflow_id: Some("wf-side-effect-version".to_string()),
9174 run_id: Some("run-side-effect-version".to_string()),
9175 workflow_type: "rust.side-effect-version".to_string(),
9176 payload_codec: DEFAULT_CODEC.to_string(),
9177 arguments: Some(
9178 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("arguments"),
9179 ),
9180 history_events,
9181 total_history_events: None,
9182 history_size_bytes: None,
9183 continue_as_new_recommended: None,
9184 history_budget_pressure: None,
9185 next_history_page_token: None,
9186 workflow_task_attempt: 1,
9187 workflow_signal_id: None,
9188 signal_name: None,
9189 signal_arguments: None,
9190 workflow_update_id: None,
9191 update_name: None,
9192 lease_owner: Some("rust-worker".to_string()),
9193 }
9194 }
9195
9196 let calls = Arc::new(AtomicUsize::new(0));
9197 let initial = worker(Arc::clone(&calls))
9198 .execute_workflow_task(task(Vec::new()))
9199 .expect("initial execution");
9200 assert_eq!(
9201 initial
9202 .iter()
9203 .map(|command| &command["type"])
9204 .collect::<Vec<_>>(),
9205 vec![
9206 "record_side_effect",
9207 "record_version_marker",
9208 "complete_workflow"
9209 ]
9210 );
9211 assert_eq!(calls.load(Ordering::SeqCst), 1);
9212
9213 let restarted = worker(Arc::clone(&calls));
9214 let replayed = restarted
9215 .execute_workflow_task(task(vec![
9216 history_event(
9217 "SideEffectRecorded",
9218 json!({"sequence": 1, "result": initial[0]["result"].clone()}),
9219 ),
9220 history_event(
9221 "VersionMarkerRecorded",
9222 json!({
9223 "sequence": 2,
9224 "change_id": "cold-restart",
9225 "version": 2,
9226 "min_supported": 1,
9227 "max_supported": 2,
9228 }),
9229 ),
9230 ]))
9231 .expect("cold replay");
9232 assert_eq!(replayed.len(), 1);
9233 assert_eq!(replayed[0]["type"], "complete_workflow");
9234 assert_eq!(calls.load(Ordering::SeqCst), 1);
9235 }
9236
9237 #[test]
9238 fn side_effect_replay_rejects_changed_rust_value_type() {
9239 let result = encode_value_envelope(&json!({"value": 42}), DEFAULT_CODEC).expect("result");
9240 let ctx = workflow_context(vec![history_event(
9241 "SideEffectRecorded",
9242 json!({"sequence": 1, "result": result}),
9243 )]);
9244 let error = ctx
9245 .side_effect::<Vec<String>, _>(|| panic!("callback must not run"))
9246 .expect_err("changed type must fail replay");
9247 assert!(matches!(
9248 error,
9249 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
9250 if reason == "side_effect_type_mismatch"
9251 ));
9252 }
9253
9254 fn completed_retry_activity_history() -> Vec<HistoryEvent> {
9255 vec![
9256 history_event(
9257 "ActivityScheduled",
9258 json!({
9259 "sequence": 1,
9260 "activity_type": "flaky",
9261 "activity_execution_id": "act-1",
9262 "activity": {
9263 "id": "act-1",
9264 "sequence": 1,
9265 "type": "flaky",
9266 "queue": "critical-activities",
9267 "execution_mode": null,
9268 "retry_policy": {
9269 "snapshot_version": 1,
9270 "max_attempts": 3,
9271 "backoff_seconds": [2, 4],
9272 "start_to_close_timeout": 30,
9273 "schedule_to_start_timeout": 5,
9274 "schedule_to_close_timeout": 90,
9275 "heartbeat_timeout": 10,
9276 "non_retryable_error_types": ["PermanentError"]
9277 }
9278 }
9279 }),
9280 ),
9281 history_event(
9282 "ActivityStarted",
9283 json!({
9284 "sequence": 1,
9285 "activity_type": "flaky",
9286 "activity_execution_id": "act-1",
9287 "activity_attempt_id": "attempt-1",
9288 "attempt_number": 1
9289 }),
9290 ),
9291 history_event(
9292 "ActivityRetryScheduled",
9293 json!({
9294 "sequence": 1,
9295 "activity_type": "flaky",
9296 "activity_execution_id": "act-1",
9297 "activity_attempt_id": "attempt-1",
9298 "attempt_number": 1,
9299 "retry_after_attempt": 1,
9300 "retry_backoff_seconds": 2,
9301 "failure_category": "activity",
9302 "exception_type": "TransientError"
9303 }),
9304 ),
9305 history_event(
9306 "ActivityStarted",
9307 json!({
9308 "sequence": 1,
9309 "activity_type": "flaky",
9310 "activity_execution_id": "act-1",
9311 "activity_attempt_id": "attempt-2",
9312 "attempt_number": 2
9313 }),
9314 ),
9315 history_event(
9316 "ActivityCompleted",
9317 json!({
9318 "sequence": 1,
9319 "activity_type": "flaky",
9320 "activity_execution_id": "act-1",
9321 "activity_attempt_id": "attempt-2",
9322 "attempt_number": 2,
9323 "payload_codec": DEFAULT_CODEC,
9324 "result": fixture_envelope(json!({"status":"recovered"}))
9325 }),
9326 ),
9327 ]
9328 }
9329
9330 fn retry_activity_options() -> ActivityOptions {
9331 ActivityOptions::new()
9332 .task_queue("critical-activities")
9333 .retry_policy(
9334 ActivityRetryPolicy::new(3)
9335 .backoff_intervals([Duration::from_secs(2), Duration::from_secs(4)])
9336 .non_retryable_error_type("PermanentError"),
9337 )
9338 .start_to_close_timeout(Duration::from_secs(30))
9339 .schedule_to_start_timeout(Duration::from_secs(5))
9340 .schedule_to_close_timeout(Duration::from_secs(90))
9341 .heartbeat_timeout(Duration::from_secs(10))
9342 }
9343
9344 #[test]
9345 fn fixed_avro_value_round_trips_json_values() {
9346 let value = json!({"greeting": "hello", "count": 3, "ok": true});
9347 let envelope = PayloadEnvelope::avro(&value).expect("encode");
9348 assert_eq!(envelope.codec, DEFAULT_CODEC);
9349 assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
9350 }
9351
9352 #[tokio::test]
9353 async fn typed_worker_surfaces_preserve_bytes_and_map_list_identity() {
9354 let client = Client::new("http://127.0.0.1:8080").expect("client");
9355 let mut worker = Worker::new(client, "rust-workers");
9356 worker.register_workflow_avro_value("typed.echo", |_ctx, input| async move { Ok(input) });
9357 worker
9358 .register_activity_avro_value("typed.activity", |_ctx, input| async move { Ok(input) });
9359 worker.register_query_avro_value("typed.echo", "inspect", |_ctx, input| async move {
9360 Ok(input)
9361 });
9362 worker.register_update_avro_value("typed.echo", "replace", |_ctx, input| async move {
9363 Ok(input)
9364 });
9365 worker.register_workflow_avro_value("typed.signal", |ctx, _input| async move {
9366 Ok(AvroValue::Array(
9367 ctx.wait_signal_avro_value("changed").await?,
9368 ))
9369 });
9370
9371 let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
9372 let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed envelope");
9373
9374 let mut workflow = workflow_task("typed.echo", Vec::new(), DEFAULT_CODEC);
9375 workflow.arguments = Some(envelope.clone());
9376 let commands = worker
9377 .execute_workflow_task(workflow)
9378 .expect("typed workflow task");
9379 assert_eq!(commands[0]["type"], "complete_workflow");
9380 assert_eq!(
9381 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
9382 .expect("typed workflow result"),
9383 arguments
9384 );
9385
9386 let activity = ActivityTask {
9387 task_id: "activity-typed".to_string(),
9388 activity_attempt_id: Some("attempt-typed".to_string()),
9389 attempt_id: None,
9390 activity_type: "typed.activity".to_string(),
9391 payload_codec: DEFAULT_CODEC.to_string(),
9392 arguments: Some(envelope.clone()),
9393 attempt_number: 1,
9394 lease_owner: Some("rust-worker".to_string()),
9395 };
9396 assert_eq!(
9397 worker
9398 .execute_activity_task(activity)
9399 .await
9400 .expect("typed activity result"),
9401 arguments
9402 );
9403
9404 let query = QueryTask {
9405 query_task_id: "query-typed".to_string(),
9406 query_task_attempt: 1,
9407 lease_owner: Some("rust-worker".to_string()),
9408 workflow_id: Some("typed-1".to_string()),
9409 run_id: Some("run-typed".to_string()),
9410 workflow_type: "typed.echo".to_string(),
9411 query_name: "inspect".to_string(),
9412 payload_codec: DEFAULT_CODEC.to_string(),
9413 workflow_arguments: Some(
9414 encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
9415 .expect("workflow input"),
9416 ),
9417 query_arguments: Some(envelope.clone()),
9418 history_events: Vec::new(),
9419 history_export: None,
9420 run_status: Some("running".to_string()),
9421 };
9422 assert_eq!(
9423 worker
9424 .execute_query_task(query)
9425 .await
9426 .expect("typed query result"),
9427 arguments
9428 );
9429
9430 let mut update = workflow_task(
9431 "typed.echo",
9432 vec![history_event(
9433 "UpdateAccepted",
9434 json!({
9435 "update_id": "update-typed",
9436 "update_name": "replace",
9437 "arguments": envelope.clone(),
9438 }),
9439 )],
9440 DEFAULT_CODEC,
9441 );
9442 update.workflow_update_id = Some("update-typed".to_string());
9443 update.update_name = Some("replace".to_string());
9444 let commands = worker
9445 .execute_workflow_task(update)
9446 .expect("typed update task");
9447 assert_eq!(commands[0]["type"], "complete_update");
9448 assert_eq!(
9449 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
9450 .expect("typed update result"),
9451 arguments
9452 );
9453
9454 let mut signal = workflow_task(
9455 "typed.signal",
9456 vec![history_event(
9457 "SignalReceived",
9458 json!({
9459 "signal_id": "signal-typed",
9460 "signal_name": "changed",
9461 "arguments": envelope.clone(),
9462 }),
9463 )],
9464 DEFAULT_CODEC,
9465 );
9466 signal.workflow_signal_id = Some("signal-typed".to_string());
9467 signal.signal_name = Some("changed".to_string());
9468 signal.signal_arguments = Some(envelope);
9469 let commands = worker
9470 .execute_workflow_task(signal)
9471 .expect("typed signal resume");
9472 assert_eq!(
9473 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
9474 .expect("typed signal result"),
9475 arguments
9476 );
9477 }
9478
9479 #[tokio::test]
9480 async fn typed_helpers_never_parse_json_inspection_projection() {
9481 let collision_values = projection_collision_probe();
9482 let expected = AvroValue::Array(collision_values.clone());
9483 let envelope = encode_typed_envelope(&expected, DEFAULT_CODEC).expect("collision envelope");
9484
9485 let activity_context = workflow_context_with_codec(
9486 vec![history_event(
9487 "ActivityCompleted",
9488 json!({
9489 "sequence": 1,
9490 "activity_type": "collision.activity",
9491 "payload_codec": DEFAULT_CODEC,
9492 "result": envelope.clone(),
9493 }),
9494 )],
9495 DEFAULT_CODEC,
9496 );
9497 assert_eq!(
9498 activity_context
9499 .activity_avro_value("collision.activity", AvroValue::Array(Vec::new()))
9500 .await
9501 .expect("typed activity collision result"),
9502 expected
9503 );
9504
9505 let signal_context = workflow_context_with_codec(
9506 vec![
9507 history_event(
9508 "SignalWaitOpened",
9509 json!({"sequence": 1, "signal_name": "collision"}),
9510 ),
9511 history_event(
9512 "SignalApplied",
9513 json!({
9514 "sequence": 1,
9515 "signal_name": "collision",
9516 "payload_codec": DEFAULT_CODEC,
9517 "value": envelope.clone(),
9518 }),
9519 ),
9520 ],
9521 DEFAULT_CODEC,
9522 );
9523 assert_eq!(
9524 signal_context
9525 .wait_signal_avro_value("collision")
9526 .await
9527 .expect("typed signal collision arguments"),
9528 collision_values
9529 );
9530
9531 let child_context = workflow_context_with_codec(
9532 vec![
9533 history_event(
9534 "ChildWorkflowScheduled",
9535 json!({
9536 "sequence": 1,
9537 "child_workflow_instance_id": "collision-child",
9538 "child_workflow_run_id": "collision-run",
9539 "child_workflow_type": "collision.child",
9540 }),
9541 ),
9542 history_event(
9543 "ChildRunCompleted",
9544 json!({
9545 "sequence": 1,
9546 "child_workflow_instance_id": "collision-child",
9547 "child_workflow_run_id": "collision-run",
9548 "child_workflow_type": "collision.child",
9549 "payload_codec": DEFAULT_CODEC,
9550 "result": envelope,
9551 }),
9552 ),
9553 ],
9554 DEFAULT_CODEC,
9555 );
9556 let child = child_context
9557 .start_child_workflow_avro_value(
9558 "collision.child",
9559 ChildWorkflowOptions::new("collision-workers"),
9560 AvroValue::Array(Vec::new()),
9561 )
9562 .await
9563 .expect("typed child collision result");
9564 assert_eq!(child.result, expected);
9565 }
9566
9567 #[tokio::test]
9568 async fn replayed_typed_query_keeps_lossless_workflow_and_query_inputs() {
9569 let client = Client::new("http://127.0.0.1:8080").expect("client");
9570 let mut worker = Worker::new(client, "rust-workers");
9571 worker.register_replayed_workflow_avro_value(
9572 "typed.replayed",
9573 || (),
9574 |_ctx, input, _state| async move { Ok(input) },
9575 );
9576 worker.register_replayed_query_avro_value::<(), _, _>(
9577 "typed.replayed",
9578 "inspect",
9579 |ctx, _state, args| async move {
9580 let mut signals = ctx.signals_avro_value("collision");
9581 let signal = signals
9582 .pop()
9583 .map(AvroValue::Array)
9584 .unwrap_or_else(|| AvroValue::Array(Vec::new()));
9585 Ok(AvroValue::Array(vec![
9586 ctx.workflow_input_avro_value().clone(),
9587 signal,
9588 args,
9589 ]))
9590 },
9591 );
9592 let arguments = AvroValue::Array(projection_collision_probe());
9593 let signal_arguments =
9594 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed query signal arguments");
9595 let task = QueryTask {
9596 query_task_id: "query-typed-replay".to_string(),
9597 query_task_attempt: 1,
9598 lease_owner: Some("rust-worker".to_string()),
9599 workflow_id: Some("typed-replay".to_string()),
9600 run_id: Some("run-typed-replay".to_string()),
9601 workflow_type: "typed.replayed".to_string(),
9602 query_name: "inspect".to_string(),
9603 payload_codec: DEFAULT_CODEC.to_string(),
9604 workflow_arguments: Some(
9605 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("workflow arguments"),
9606 ),
9607 query_arguments: Some(
9608 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("query arguments"),
9609 ),
9610 history_events: vec![history_event(
9611 "SignalReceived",
9612 json!({
9613 "signal_id": "collision-signal",
9614 "signal_name": "collision",
9615 "workflow_sequence": 1,
9616 "payload_codec": DEFAULT_CODEC,
9617 "arguments": signal_arguments,
9618 }),
9619 )],
9620 history_export: None,
9621 run_status: Some("completed".to_string()),
9622 };
9623
9624 assert_eq!(
9625 worker
9626 .execute_query_task(task)
9627 .await
9628 .expect("typed replay query"),
9629 AvroValue::Array(vec![arguments.clone(), arguments.clone(), arguments])
9630 );
9631 }
9632
9633 #[test]
9634 fn public_avro_adapter_rejects_non_string_map_keys_before_json_conversion() {
9635 let value = BTreeMap::from([(1_i32, "integer key")]);
9636 let error = PayloadEnvelope::avro(&value)
9637 .expect_err("integer map keys must fail")
9638 .to_string();
9639
9640 assert!(error.contains("invalid_map_key"));
9641 }
9642
9643 #[test]
9644 fn json_tagged_payload_fails_closed_with_actionable_diagnostic() {
9645 let envelope = PayloadEnvelope {
9646 codec: "json".to_string(),
9647 blob: r#"{"greeting":"hello"}"#.to_string(),
9648 };
9649
9650 let error = decode_payload::<Value>(&envelope).expect_err("JSON payload must fail");
9651 let diagnostic = error.to_string();
9652 assert!(diagnostic.contains("unsupported_payload_codec"));
9653 assert!(diagnostic.contains("codec=\"avro\""));
9654 assert!(diagnostic.contains("HTTP document transport"));
9655 }
9656
9657 #[test]
9658 fn untagged_json_payload_value_fails_closed() {
9659 let error = decode_wire_value(&json!({"stale": true}), DEFAULT_CODEC)
9660 .expect_err("untagged JSON payload values must fail");
9661 let diagnostic = error.to_string();
9662 assert!(diagnostic.contains("unsupported_payload_codec"));
9663 assert!(diagnostic.contains("untagged durable payload"));
9664 assert!(diagnostic.contains("HTTP document transport"));
9665 }
9666
9667 #[test]
9668 fn prerelease_avro_payload_without_single_object_frame_is_rejected() {
9669 let envelope = PayloadEnvelope {
9670 codec: DEFAULT_CODEC.to_string(),
9671 blob: BASE64.encode([0x01]),
9672 };
9673
9674 let error = decode_payload::<Value>(&envelope).expect_err("prerelease payload must fail");
9675 assert!(error.to_string().contains("invalid_payload_framing"));
9676 }
9677
9678 #[tokio::test]
9679 async fn workflow_completion_rejects_invalid_payload_slots_without_transport() {
9680 let server = MockWorkerServer::start();
9681 let client = Client::builder(server.base_url())
9682 .timeout(Duration::from_secs(2))
9683 .build()
9684 .expect("client");
9685 let invalid_commands = [
9686 json!({
9687 "type": "complete_workflow",
9688 "result": {"codec": "json", "blob": null}
9689 }),
9690 json!({
9691 "type": "schedule_activity",
9692 "arguments": {"codec": "yaml", "blob": "ignored"}
9693 }),
9694 json!({
9695 "type": "start_child_workflow",
9696 "arguments": {"codec": DEFAULT_CODEC, "blob": null}
9697 }),
9698 json!({"type": "continue_as_new", "arguments": []}),
9699 json!({"type": "complete_update"}),
9700 json!({"type": "record_side_effect", "result": null}),
9701 json!({
9702 "type": "start_service_operation",
9703 "payload_codec": DEFAULT_CODEC,
9704 "request_payload": "raw-avro-bytes"
9705 }),
9706 ];
9707
9708 for command in invalid_commands {
9709 let error = client
9710 .complete_workflow_task("invalid-codec", "rust-worker", 1, vec![command])
9711 .await
9712 .expect_err("invalid durable payload must fail locally");
9713 let diagnostic = error.to_string();
9714 assert!(
9715 diagnostic.contains("unsupported_payload_codec")
9716 || diagnostic.contains("invalid_payload_envelope")
9717 || diagnostic.contains("untagged durable payload"),
9718 "unexpected validation diagnostic: {diagnostic}"
9719 );
9720 }
9721
9722 assert_eq!(
9723 server.request_count("/api/worker/workflow-tasks/invalid-codec/complete"),
9724 0,
9725 "invalid command payloads must not reach HTTP transport"
9726 );
9727 }
9728
9729 #[test]
9730 fn workflow_completion_validates_only_protocol_owned_payload_slots() {
9731 let envelope = fixture_envelope(json!({"codec": "customer-value"}));
9732 let commands = [
9733 json!({"type": "complete_workflow", "result": envelope.clone()}),
9734 json!({"type": "schedule_activity", "arguments": envelope.clone()}),
9735 json!({"type": "start_child_workflow", "arguments": envelope.clone()}),
9736 json!({"type": "continue_as_new", "arguments": envelope.clone()}),
9737 json!({"type": "complete_update", "result": envelope.clone()}),
9738 json!({"type": "record_side_effect", "result": envelope.clone()}),
9739 json!({
9740 "type": "start_service_operation",
9741 "payload_codec": DEFAULT_CODEC,
9742 "request_payload": envelope.clone()
9743 }),
9744 json!({
9745 "type": "complete_workflow",
9746 "result": envelope,
9747 "metadata": {
9748 "codec": "json",
9749 "payload_codec": "customer-codec",
9750 "result": {"codec": "yaml", "blob": null}
9751 }
9752 }),
9753 ];
9754
9755 validate_workflow_task_commands(&commands)
9756 .expect("customer metadata must not become a protocol codec declaration");
9757 }
9758
9759 #[test]
9760 fn valid_avro_tasks_normalize_absent_and_null_arguments_to_empty_lists() {
9761 assert_eq!(
9762 decode_task_avro_arguments(None, DEFAULT_CODEC).expect("absent arguments"),
9763 AvroValue::Array(Vec::new())
9764 );
9765 assert_eq!(
9766 decode_task_avro_arguments(Some(&Value::Null), DEFAULT_CODEC).expect("null arguments"),
9767 AvroValue::Array(Vec::new())
9768 );
9769
9770 let mut signal = workflow_task("missing", Vec::new(), DEFAULT_CODEC);
9771 signal.signal_name = Some("empty-signal".to_string());
9772 signal.signal_arguments = None;
9773 let decoded = decode_resume_signal(&signal)
9774 .expect("valid Avro signal")
9775 .expect("named signal resumes the workflow");
9776 assert!(decoded.arguments.is_empty());
9777 }
9778
9779 #[tokio::test]
9780 async fn malformed_task_level_codecs_become_pre_handler_failures() {
9781 let client = Client::new("http://127.0.0.1:8080").expect("client");
9782 let mut worker = Worker::new(client, "rust-workers");
9783 let handler_calls = Arc::new(AtomicUsize::new(0));
9784
9785 let calls = Arc::clone(&handler_calls);
9786 worker.register_workflow("codec.workflow", move |_ctx, _args| {
9787 calls.fetch_add(1, Ordering::SeqCst);
9788 async move { Ok(Value::Null) }
9789 });
9790 let calls = Arc::clone(&handler_calls);
9791 worker.register_activity("codec.activity", move |_ctx, _args| {
9792 calls.fetch_add(1, Ordering::SeqCst);
9793 async move { Ok(Value::Null) }
9794 });
9795 let calls = Arc::clone(&handler_calls);
9796 worker.register_query("codec.workflow", "known", move |_ctx, _args| {
9797 calls.fetch_add(1, Ordering::SeqCst);
9798 async move { Ok(Value::Null) }
9799 });
9800
9801 let mut failures = Vec::new();
9802 for codec_case in [
9803 InvalidTaskPayloadCodec::Missing,
9804 InvalidTaskPayloadCodec::Null,
9805 InvalidTaskPayloadCodec::NonString,
9806 ] {
9807 let mut workflow = json!({
9808 "task_id": format!("workflow-{}", codec_case.label()),
9809 "workflow_type": "codec.workflow"
9810 });
9811 codec_case.apply(&mut workflow);
9812 match serde_json::from_value::<WorkflowTask>(workflow) {
9813 Ok(task) => match worker.execute_workflow_task(task) {
9814 Err(error) if error.to_string().contains("unsupported_payload_codec") => {}
9815 outcome => failures.push(format!(
9816 "workflow {} codec returned {outcome:?}",
9817 codec_case.label()
9818 )),
9819 },
9820 Err(error) => failures.push(format!(
9821 "workflow {} codec failed transport deserialization: {error}",
9822 codec_case.label()
9823 )),
9824 }
9825
9826 let mut activity = json!({
9827 "task_id": format!("activity-{}", codec_case.label()),
9828 "activity_attempt_id": format!("attempt-{}", codec_case.label()),
9829 "activity_type": "codec.activity",
9830 "attempt_number": 1
9831 });
9832 codec_case.apply(&mut activity);
9833 match serde_json::from_value::<ActivityTask>(activity) {
9834 Ok(task) => match worker.execute_activity_task(task).await {
9835 Err(error) if error.to_string().contains("unsupported_payload_codec") => {}
9836 outcome => failures.push(format!(
9837 "activity {} codec returned {outcome:?}",
9838 codec_case.label()
9839 )),
9840 },
9841 Err(error) => failures.push(format!(
9842 "activity {} codec failed transport deserialization: {error}",
9843 codec_case.label()
9844 )),
9845 }
9846
9847 let mut query = json!({
9848 "query_task_id": format!("query-{}", codec_case.label()),
9849 "workflow_type": "codec.workflow",
9850 "query_name": "known"
9851 });
9852 codec_case.apply(&mut query);
9853 match serde_json::from_value::<QueryTask>(query) {
9854 Ok(task) => match worker.execute_query_task(task).await {
9855 Err(failure) if failure.message.contains("unsupported_payload_codec") => {}
9856 outcome => failures.push(format!(
9857 "query {} codec returned {outcome:?}",
9858 codec_case.label()
9859 )),
9860 },
9861 Err(error) => failures.push(format!(
9862 "query {} codec failed transport deserialization: {error}",
9863 codec_case.label()
9864 )),
9865 }
9866 }
9867
9868 assert!(failures.is_empty(), "{}", failures.join("\n"));
9869 assert_eq!(
9870 handler_calls.load(Ordering::SeqCst),
9871 0,
9872 "invalid task codecs must not invoke a handler"
9873 );
9874 }
9875
9876 #[tokio::test]
9877 async fn polled_malformed_task_codecs_are_settled_without_handler_execution() {
9878 for codec_case in [
9879 InvalidTaskPayloadCodec::Missing,
9880 InvalidTaskPayloadCodec::Null,
9881 InvalidTaskPayloadCodec::NonString,
9882 ] {
9883 let server = MockWorkerServer::invalid_task_payload_codec(codec_case);
9884 let client = Client::builder(server.base_url())
9885 .timeout(Duration::from_secs(2))
9886 .build()
9887 .expect("client");
9888 let mut worker = Worker::new(client, "rust-workers")
9889 .worker_id("codec-worker")
9890 .poll_timeout(Duration::from_millis(10));
9891 let handler_calls = Arc::new(AtomicUsize::new(0));
9892
9893 let calls = Arc::clone(&handler_calls);
9894 worker.register_workflow("codec.workflow", move |_ctx, _args| {
9895 calls.fetch_add(1, Ordering::SeqCst);
9896 async move { Ok(Value::Null) }
9897 });
9898 let calls = Arc::clone(&handler_calls);
9899 worker.register_activity("codec.activity", move |_ctx, _args| {
9900 calls.fetch_add(1, Ordering::SeqCst);
9901 async move { Ok(Value::Null) }
9902 });
9903 let calls = Arc::clone(&handler_calls);
9904 worker.register_query("codec.workflow", "known", move |_ctx, _args| {
9905 calls.fetch_add(1, Ordering::SeqCst);
9906 async move { Ok(Value::Null) }
9907 });
9908
9909 assert_eq!(
9910 worker.run_once().await.expect("invalid tasks are settled"),
9911 3,
9912 "all {} codec tasks must be handled",
9913 codec_case.label()
9914 );
9915 assert_eq!(
9916 handler_calls.load(Ordering::SeqCst),
9917 0,
9918 "{} task codecs must fail before every handler",
9919 codec_case.label()
9920 );
9921
9922 for path in [
9923 "/api/worker/workflow-tasks/codec-workflow/fail",
9924 "/api/worker/activity-tasks/codec-activity/fail",
9925 "/api/worker/query-tasks/codec-query/fail",
9926 ] {
9927 let body = server.request_body(path);
9928 assert!(
9929 body["failure"]["message"]
9930 .as_str()
9931 .is_some_and(|message| message.contains("unsupported_payload_codec")),
9932 "{path} must receive the stable codec diagnostic for the {} case: {body}",
9933 codec_case.label()
9934 );
9935 }
9936 assert_eq!(
9937 server.request_body("/api/worker/query-tasks/codec-query/fail")["failure"]
9938 ["reason"],
9939 "query_payload_decode_failed"
9940 );
9941 for path in [
9942 "/api/worker/workflow-tasks/codec-workflow/complete",
9943 "/api/worker/activity-tasks/codec-activity/complete",
9944 "/api/worker/query-tasks/codec-query/complete",
9945 ] {
9946 assert_eq!(
9947 server.request_count(path),
9948 0,
9949 "invalid {} codec task reached {path}",
9950 codec_case.label()
9951 );
9952 }
9953 }
9954 }
9955
9956 #[tokio::test]
9957 async fn invalid_inbound_codecs_precede_handlers_and_unrelated_outcomes() {
9958 let client = Client::new("http://127.0.0.1:8080").expect("client");
9959 let mut worker = Worker::new(client, "rust-workers");
9960 let handler_calls = Arc::new(AtomicUsize::new(0));
9961
9962 let calls = Arc::clone(&handler_calls);
9963 worker.register_workflow("codec.workflow", move |_ctx, _args| {
9964 calls.fetch_add(1, Ordering::SeqCst);
9965 async move { Ok(Value::Null) }
9966 });
9967 let calls = Arc::clone(&handler_calls);
9968 worker.register_activity("codec.activity", move |_ctx, _args| {
9969 calls.fetch_add(1, Ordering::SeqCst);
9970 async move { Ok(Value::Null) }
9971 });
9972 let calls = Arc::clone(&handler_calls);
9973 worker.register_update("codec.workflow", "known", move |_ctx, _args| {
9974 calls.fetch_add(1, Ordering::SeqCst);
9975 async move { Ok(Value::Null) }
9976 });
9977 let calls = Arc::clone(&handler_calls);
9978 worker.register_query("codec.workflow", "known", move |_ctx, _args| {
9979 calls.fetch_add(1, Ordering::SeqCst);
9980 async move { Ok(Value::Null) }
9981 });
9982
9983 let mut workflow = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
9984 workflow.payload_codec = "json".to_string();
9985 workflow.arguments = None;
9986 let error = worker
9987 .execute_workflow_task(workflow)
9988 .expect_err("task codec must be checked before workflow invocation");
9989 assert!(error.to_string().contains("unsupported_payload_codec"));
9990
9991 let activity = ActivityTask {
9992 task_id: "activity-invalid-codec".to_string(),
9993 activity_attempt_id: None,
9994 attempt_id: None,
9995 activity_type: "codec.activity".to_string(),
9996 payload_codec: "unknown".to_string(),
9997 arguments: None,
9998 attempt_number: 1,
9999 lease_owner: None,
10000 };
10001 let error = worker
10002 .execute_activity_task(activity)
10003 .await
10004 .expect_err("task codec must be checked before activity invocation");
10005 assert!(error.to_string().contains("unsupported_payload_codec"));
10006
10007 let mut update = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
10008 update.workflow_update_id = Some("update-invalid-codec".to_string());
10009 update.update_name = Some("known".to_string());
10010 update.history_events.push(history_event(
10011 "UpdateAccepted",
10012 json!({
10013 "update_id": "update-invalid-codec",
10014 "update_name": "known",
10015 "arguments": {"codec": "json", "blob": null}
10016 }),
10017 ));
10018 let error = worker
10019 .execute_workflow_task(update)
10020 .expect_err("nested update codec must be checked before handler lookup");
10021 assert!(error.to_string().contains("unsupported_payload_codec"));
10022
10023 let query: QueryTask = serde_json::from_value(json!({
10024 "query_task_id": "query-invalid-codec",
10025 "workflow_type": "codec.workflow",
10026 "query_name": "known",
10027 "payload_codec": DEFAULT_CODEC,
10028 "workflow_arguments": null,
10029 "query_arguments": null,
10030 "history_export": {
10031 "payloads": {"codec": DEFAULT_CODEC},
10032 "signals": [{
10033 "name": "empty",
10034 "payload_codec": "json",
10035 "arguments": null
10036 }]
10037 }
10038 }))
10039 .expect("query task");
10040 let failure = worker
10041 .execute_query_task(query)
10042 .await
10043 .expect_err("exported signal codec must be checked before query invocation");
10044 assert_eq!(failure.reason, "query_payload_decode_failed");
10045 assert!(failure.message.contains("unsupported_payload_codec"));
10046
10047 let exported_history: QueryTask = serde_json::from_value(json!({
10048 "query_task_id": "query-invalid-history-codec",
10049 "workflow_type": "codec.workflow",
10050 "query_name": "known",
10051 "payload_codec": DEFAULT_CODEC,
10052 "history_export": {
10053 "payloads": {"codec": DEFAULT_CODEC},
10054 "history_events": [{
10055 "type": "ActivityCompleted",
10056 "payload": {"payload_codec": "unknown", "result": null}
10057 }]
10058 }
10059 }))
10060 .expect("query task");
10061 let failure = worker
10062 .execute_query_task(exported_history)
10063 .await
10064 .expect_err("exported history codec must be checked before query invocation");
10065 assert_eq!(failure.reason, "query_payload_decode_failed");
10066 assert!(failure.message.contains("unsupported_payload_codec"));
10067 assert_eq!(handler_calls.load(Ordering::SeqCst), 0);
10068
10069 let mut unknown_workflow = workflow_task("missing", Vec::new(), DEFAULT_CODEC);
10070 unknown_workflow.arguments = None;
10071 unknown_workflow.history_events.push(history_event(
10072 "SignalReceived",
10073 json!({
10074 "signal_name": "empty",
10075 "payload_codec": "json",
10076 "arguments": null
10077 }),
10078 ));
10079 let error = worker
10080 .execute_workflow_task(unknown_workflow)
10081 .expect_err("history codec must precede unknown workflow outcome");
10082 assert!(error.to_string().contains("unsupported_payload_codec"));
10083
10084 let unknown_activity = ActivityTask {
10085 task_id: "activity-unknown".to_string(),
10086 activity_attempt_id: None,
10087 attempt_id: None,
10088 activity_type: "missing".to_string(),
10089 payload_codec: "json".to_string(),
10090 arguments: None,
10091 attempt_number: 1,
10092 lease_owner: None,
10093 };
10094 let error = worker
10095 .execute_activity_task(unknown_activity)
10096 .await
10097 .expect_err("codec must precede unknown activity outcome");
10098 assert!(error.to_string().contains("unsupported_payload_codec"));
10099
10100 let mut unknown_update = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
10101 unknown_update.payload_codec = "json".to_string();
10102 unknown_update.arguments = None;
10103 unknown_update.workflow_update_id = Some("update-unknown".to_string());
10104 unknown_update.update_name = Some("missing".to_string());
10105 let error = worker
10106 .execute_workflow_task(unknown_update)
10107 .expect_err("codec must precede fail_update shortcut");
10108 assert!(error.to_string().contains("unsupported_payload_codec"));
10109
10110 let unknown_query: QueryTask = serde_json::from_value(json!({
10111 "query_task_id": "query-unknown",
10112 "workflow_type": "missing",
10113 "query_name": "missing",
10114 "payload_codec": "json",
10115 "workflow_arguments": null,
10116 "query_arguments": null
10117 }))
10118 .expect("query task");
10119 let failure = worker
10120 .execute_query_task(unknown_query)
10121 .await
10122 .expect_err("codec must precede unknown query outcome");
10123 assert_eq!(failure.reason, "query_payload_decode_failed");
10124 assert!(failure.message.contains("unsupported_payload_codec"));
10125 }
10126
10127 #[tokio::test]
10128 async fn invalid_signal_history_payload_aliases_precede_shortcuts() {
10129 let client = Client::new("http://127.0.0.1:8080").expect("client");
10130 let worker = Worker::new(client, "rust-workers");
10131
10132 for event_type in ["SignalReceived", "SignalApplied"] {
10133 for (payload_field, codec) in [
10134 ("value", "json"),
10135 ("input", "unknown"),
10136 ("arguments", "json"),
10137 ] {
10138 let payload = json!({
10139 "signal_name": "empty",
10140 payload_field: {"codec": codec, "blob": null}
10141 });
10142 let workflow = workflow_task(
10143 "missing",
10144 vec![history_event(event_type, payload.clone())],
10145 DEFAULT_CODEC,
10146 );
10147 let error = worker
10148 .execute_workflow_task(workflow)
10149 .expect_err("signal payload codec must precede unknown workflow outcome");
10150 assert!(
10151 error.to_string().contains("unsupported_payload_codec"),
10152 "{event_type}.{payload_field} returned an unrelated workflow error: {error}"
10153 );
10154
10155 let query: QueryTask = serde_json::from_value(json!({
10156 "query_task_id": format!("query-{event_type}-{payload_field}"),
10157 "workflow_type": "missing",
10158 "query_name": "missing",
10159 "payload_codec": DEFAULT_CODEC,
10160 "workflow_arguments": null,
10161 "query_arguments": null,
10162 "history_events": [{
10163 "event_type": event_type,
10164 "payload": payload
10165 }]
10166 }))
10167 .expect("query task");
10168 let failure = worker
10169 .execute_query_task(query)
10170 .await
10171 .expect_err("signal payload codec must precede unknown query outcome");
10172 assert_eq!(
10173 failure.reason, "query_payload_decode_failed",
10174 "{event_type}.{payload_field} returned an unrelated query outcome"
10175 );
10176 assert!(
10177 failure.message.contains("unsupported_payload_codec"),
10178 "{event_type}.{payload_field} returned an unrelated query error: {}",
10179 failure.message
10180 );
10181 }
10182 }
10183 }
10184
10185 #[test]
10186 fn workflow_context_schedules_activity_until_completion_is_in_history() {
10187 let ctx = WorkflowContext {
10188 state: Arc::new(Mutex::new(
10189 WorkflowState::new_with_identity(
10190 Vec::new(),
10191 Some("wf-parent".to_string()),
10192 Some("run-parent".to_string()),
10193 "rust-workers".to_string(),
10194 DEFAULT_CODEC.to_string(),
10195 None,
10196 )
10197 .expect("workflow state"),
10198 )),
10199 };
10200
10201 let mut call = Box::pin(ctx.activity("hello.activity", json!(["Ada"])));
10202 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10203 assert!(matches!(
10204 call.as_mut().poll(&mut task_context),
10205 Poll::Pending
10206 ));
10207
10208 let commands = ctx.take_commands().expect("commands");
10209 assert_eq!(commands[0]["type"], "schedule_activity");
10210 assert_eq!(commands[0]["activity_type"], "hello.activity");
10211 }
10212
10213 #[test]
10214 fn activity_options_encode_retry_policy_queue_and_every_timeout() {
10215 let ctx = workflow_context(Vec::new());
10216 let options = ActivityOptions::new()
10217 .task_queue("payments")
10218 .retry_policy(
10219 ActivityRetryPolicy::new(4)
10220 .exponential_backoff(Duration::from_secs(1), 3, Some(Duration::from_secs(10)))
10221 .non_retryable_error_type("ValidationError"),
10222 )
10223 .start_to_close_timeout(Duration::from_secs(120))
10224 .schedule_to_start_timeout(Duration::from_secs(10))
10225 .schedule_to_close_timeout(Duration::from_secs(300))
10226 .heartbeat_timeout(Duration::from_secs(15));
10227 let mut call = Box::pin(ctx.activity_with_options(
10228 "charge-card",
10229 options,
10230 json!([{"order_id": "o-1"}]),
10231 ));
10232 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10233
10234 assert!(matches!(
10235 call.as_mut().poll(&mut task_context),
10236 Poll::Pending
10237 ));
10238 assert!(matches!(
10239 call.as_mut().poll(&mut task_context),
10240 Poll::Pending
10241 ));
10242
10243 let commands = ctx.take_commands().expect("activity command");
10244 assert_eq!(commands.len(), 1, "one future emits one logical schedule");
10245 assert_eq!(commands[0]["queue"], "payments");
10246 assert_eq!(
10247 commands[0]["retry_policy"],
10248 json!({
10249 "max_attempts": 4,
10250 "backoff_seconds": [1, 3, 9],
10251 "non_retryable_error_types": ["ValidationError"],
10252 })
10253 );
10254 assert_eq!(commands[0]["start_to_close_timeout"], 120);
10255 assert_eq!(commands[0]["schedule_to_start_timeout"], 10);
10256 assert_eq!(commands[0]["schedule_to_close_timeout"], 300);
10257 assert_eq!(commands[0]["heartbeat_timeout"], 15);
10258 }
10259
10260 #[test]
10261 fn activity_options_encode_explicit_and_rounded_backoff_intervals() {
10262 let ctx = workflow_context(Vec::new());
10263 let options = ActivityOptions::new().retry_policy(
10264 ActivityRetryPolicy::new(3)
10265 .backoff_intervals([Duration::from_millis(1), Duration::from_millis(1_001)]),
10266 );
10267 let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
10268 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10269
10270 assert!(matches!(
10271 call.as_mut().poll(&mut task_context),
10272 Poll::Pending
10273 ));
10274 assert_eq!(
10275 ctx.take_commands().expect("command")[0]["retry_policy"]["backoff_seconds"],
10276 json!([1, 2])
10277 );
10278 }
10279
10280 #[test]
10281 fn invalid_activity_options_return_typed_errors_before_emitting_commands() {
10282 let cases = [
10283 (
10284 ActivityOptions::new().task_queue(" "),
10285 ActivityOptionsErrorKind::EmptyTaskQueue,
10286 ),
10287 (
10288 ActivityOptions::new().retry_policy(ActivityRetryPolicy::default()),
10289 ActivityOptionsErrorKind::EmptyRetryPolicy,
10290 ),
10291 (
10292 ActivityOptions::new().retry_policy(ActivityRetryPolicy::new(0)),
10293 ActivityOptionsErrorKind::InvalidMaxAttempts,
10294 ),
10295 (
10296 ActivityOptions::new().retry_policy(ActivityRetryPolicy {
10297 max_attempts: None,
10298 backoff: Some(ActivityBackoff::Explicit(vec![Duration::from_secs(1)])),
10299 non_retryable_error_types: Vec::new(),
10300 }),
10301 ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
10302 ),
10303 (
10304 ActivityOptions::new().retry_policy(
10305 ActivityRetryPolicy::new(2)
10306 .backoff_intervals([Duration::from_secs(1), Duration::from_secs(2)]),
10307 ),
10308 ActivityOptionsErrorKind::TooManyBackoffIntervals,
10309 ),
10310 (
10311 ActivityOptions::new().retry_policy(
10312 ActivityRetryPolicy::new(2).exponential_backoff(
10313 Duration::from_secs(1),
10314 0,
10315 None,
10316 ),
10317 ),
10318 ActivityOptionsErrorKind::InvalidBackoffCoefficient,
10319 ),
10320 (
10321 ActivityOptions::new()
10322 .retry_policy(ActivityRetryPolicy::new(2).non_retryable_error_type(" ")),
10323 ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
10324 ),
10325 (
10326 ActivityOptions::new().retry_policy(
10327 ActivityRetryPolicy::new(10_002).exponential_backoff(
10328 Duration::from_secs(1),
10329 1,
10330 None,
10331 ),
10332 ),
10333 ActivityOptionsErrorKind::BackoffGenerationTooLarge,
10334 ),
10335 (
10336 ActivityOptions::new().retry_policy(
10337 ActivityRetryPolicy::new(2)
10338 .backoff_intervals([Duration::from_secs(i64::MAX as u64 + 1)]),
10339 ),
10340 ActivityOptionsErrorKind::BackoffOverflow,
10341 ),
10342 ];
10343
10344 for (options, expected_kind) in cases {
10345 let ctx = workflow_context(Vec::new());
10346 let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
10347 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10348 let Poll::Ready(Err(Error::InvalidActivityOptions(error))) =
10349 call.as_mut().poll(&mut task_context)
10350 else {
10351 panic!("expected typed activity validation error");
10352 };
10353 assert_eq!(error.kind, expected_kind);
10354 assert!(ctx.take_commands().expect("commands").is_empty());
10355 }
10356 }
10357
10358 #[test]
10359 fn activity_options_validate_positive_and_ordered_timeouts() {
10360 let zero_timeout_cases = [
10361 ActivityOptions::new().start_to_close_timeout(Duration::ZERO),
10362 ActivityOptions::new().schedule_to_start_timeout(Duration::ZERO),
10363 ActivityOptions::new().schedule_to_close_timeout(Duration::ZERO),
10364 ActivityOptions::new().heartbeat_timeout(Duration::ZERO),
10365 ];
10366 for options in zero_timeout_cases {
10367 assert_eq!(
10368 options.validate().expect_err("zero timeout").kind,
10369 ActivityOptionsErrorKind::TimeoutNotPositive
10370 );
10371 }
10372
10373 let ordering_cases = [
10374 ActivityOptions::new()
10375 .heartbeat_timeout(Duration::from_secs(11))
10376 .start_to_close_timeout(Duration::from_secs(10)),
10377 ActivityOptions::new()
10378 .start_to_close_timeout(Duration::from_secs(31))
10379 .schedule_to_close_timeout(Duration::from_secs(30)),
10380 ActivityOptions::new()
10381 .schedule_to_start_timeout(Duration::from_secs(31))
10382 .schedule_to_close_timeout(Duration::from_secs(30)),
10383 ];
10384 for options in ordering_cases {
10385 assert_eq!(
10386 options.validate().expect_err("timeout order").kind,
10387 ActivityOptionsErrorKind::TimeoutOrder
10388 );
10389 }
10390
10391 assert_eq!(
10392 ActivityOptions::new()
10393 .start_to_close_timeout(Duration::from_secs(i64::MAX as u64 + 1))
10394 .validate()
10395 .expect_err("protocol integer overflow")
10396 .kind,
10397 ActivityOptionsErrorKind::TimeoutOverflow
10398 );
10399 }
10400
10401 #[test]
10402 fn replayed_activity_retry_history_completes_without_duplicate_schedule() {
10403 let ctx = workflow_context(completed_retry_activity_history());
10404 let mut call =
10405 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
10406 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10407
10408 assert!(matches!(
10409 call.as_mut().poll(&mut task_context),
10410 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
10411 ));
10412 assert!(ctx.take_commands().expect("commands").is_empty());
10413 ctx.ensure_history_consumed().expect("history consumed");
10414 }
10415
10416 #[test]
10417 fn duplicate_non_retryable_types_use_one_command_and_replay_representation() {
10418 let mut options = retry_activity_options();
10419 options
10420 .retry_policy
10421 .as_mut()
10422 .expect("retry policy")
10423 .non_retryable_error_types
10424 .extend([" PermanentError ".to_string(), "PermanentError".to_string()]);
10425
10426 let new_ctx = workflow_context(Vec::new());
10427 let mut new_call =
10428 Box::pin(new_ctx.activity_with_options("flaky", options.clone(), json!([])));
10429 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10430 assert!(matches!(
10431 new_call.as_mut().poll(&mut task_context),
10432 Poll::Pending
10433 ));
10434 let commands = new_ctx.take_commands().expect("commands");
10435 assert_eq!(commands.len(), 1);
10436 assert_eq!(
10437 commands[0]["retry_policy"]["non_retryable_error_types"],
10438 json!(["PermanentError"])
10439 );
10440
10441 let replay_ctx = workflow_context(completed_retry_activity_history());
10442 let mut replay_call =
10443 Box::pin(replay_ctx.activity_with_options("flaky", options, json!([])));
10444 assert!(matches!(
10445 replay_call.as_mut().poll(&mut task_context),
10446 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
10447 ));
10448 assert!(replay_ctx.take_commands().expect("commands").is_empty());
10449 replay_ctx
10450 .ensure_history_consumed()
10451 .expect("history consumed");
10452 }
10453
10454 #[test]
10455 fn replayed_intermediate_retry_remains_pending_across_restarts() {
10456 let history = completed_retry_activity_history()
10457 .into_iter()
10458 .take(3)
10459 .collect::<Vec<_>>();
10460
10461 for _restart in 0..2 {
10462 let ctx = workflow_context(history.clone());
10463 let mut call =
10464 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
10465 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10466 assert!(matches!(
10467 call.as_mut().poll(&mut task_context),
10468 Poll::Pending
10469 ));
10470 assert!(ctx.take_commands().expect("commands").is_empty());
10471 }
10472 }
10473
10474 #[test]
10475 fn replayed_activity_rejects_changed_queue_retry_and_every_timeout_field() {
10476 let mut changed_queue = retry_activity_options();
10477 changed_queue.task_queue = Some("different-queue".to_string());
10478
10479 let mut changed_max_attempts = retry_activity_options();
10480 let retry_policy = changed_max_attempts
10481 .retry_policy
10482 .as_mut()
10483 .expect("retry policy");
10484 retry_policy.max_attempts = Some(4);
10485
10486 let mut changed_backoff = retry_activity_options();
10487 let retry_policy = changed_backoff.retry_policy.as_mut().expect("retry policy");
10488 retry_policy.backoff = Some(ActivityBackoff::Explicit(vec![
10489 Duration::from_secs(3),
10490 Duration::from_secs(4),
10491 ]));
10492
10493 let mut changed_non_retryable_types = retry_activity_options();
10494 let retry_policy = changed_non_retryable_types
10495 .retry_policy
10496 .as_mut()
10497 .expect("retry policy");
10498 retry_policy.non_retryable_error_types = vec!["AnotherPermanentError".to_string()];
10499
10500 let mut changed_start_to_close = retry_activity_options();
10501 changed_start_to_close.start_to_close_timeout = Some(Duration::from_secs(31));
10502 let mut changed_schedule_to_start = retry_activity_options();
10503 changed_schedule_to_start.schedule_to_start_timeout = Some(Duration::from_secs(6));
10504 let mut changed_schedule_to_close = retry_activity_options();
10505 changed_schedule_to_close.schedule_to_close_timeout = Some(Duration::from_secs(91));
10506 let mut changed_heartbeat = retry_activity_options();
10507 changed_heartbeat.heartbeat_timeout = Some(Duration::from_secs(11));
10508
10509 let cases = [
10510 (changed_queue, "activity_task_queue_mismatch"),
10511 (changed_max_attempts, "activity_retry_policy_mismatch"),
10512 (changed_backoff, "activity_retry_policy_mismatch"),
10513 (
10514 changed_non_retryable_types,
10515 "activity_retry_policy_mismatch",
10516 ),
10517 (changed_start_to_close, "activity_retry_policy_mismatch"),
10518 (changed_schedule_to_start, "activity_retry_policy_mismatch"),
10519 (changed_schedule_to_close, "activity_retry_policy_mismatch"),
10520 (changed_heartbeat, "activity_retry_policy_mismatch"),
10521 ];
10522
10523 for (options, expected_reason) in cases {
10524 let ctx = workflow_context(completed_retry_activity_history());
10525 let mut call = Box::pin(ctx.activity_with_options("flaky", options, json!([])));
10526 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10527 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10528 call.as_mut().poll(&mut task_context)
10529 else {
10530 panic!("changed activity options must fail replay");
10531 };
10532 assert_eq!(failure.reason, expected_reason);
10533 assert_eq!(failure.sequence, Some(1));
10534 assert!(ctx.take_commands().expect("commands").is_empty());
10535 }
10536 }
10537
10538 #[test]
10539 fn replayed_activity_rejects_changed_execution_mode_and_snapshot_version() {
10540 let cases = [
10541 (
10542 "execution_mode",
10543 json!("local"),
10544 "activity_execution_mode_mismatch",
10545 ),
10546 (
10547 "snapshot_version",
10548 json!(2),
10549 "activity_retry_policy_mismatch",
10550 ),
10551 ];
10552
10553 for (field, value, expected_reason) in cases {
10554 let mut history = completed_retry_activity_history();
10555 let activity = history[0].payload["activity"]
10556 .as_object_mut()
10557 .expect("activity snapshot");
10558 if field == "execution_mode" {
10559 activity.insert(field.to_string(), value);
10560 } else {
10561 activity["retry_policy"]
10562 .as_object_mut()
10563 .expect("retry snapshot")
10564 .insert(field.to_string(), value);
10565 }
10566
10567 let ctx = workflow_context(history);
10568 let mut call =
10569 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
10570 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10571 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10572 call.as_mut().poll(&mut task_context)
10573 else {
10574 panic!("changed {field} must fail replay");
10575 };
10576 assert_eq!(failure.reason, expected_reason);
10577 assert_eq!(failure.sequence, Some(1));
10578 assert!(ctx.take_commands().expect("commands").is_empty());
10579 }
10580 }
10581
10582 #[test]
10583 fn replayed_legacy_activity_treats_missing_option_snapshot_as_unknown() {
10584 let mut history = completed_retry_activity_history();
10585 let activity = history[0].payload["activity"]
10586 .as_object_mut()
10587 .expect("activity snapshot");
10588 activity.remove("execution_mode");
10589 activity.remove("retry_policy");
10590
10591 let mut current = retry_activity_options();
10592 current.start_to_close_timeout = Some(Duration::from_secs(45));
10593 current.schedule_to_start_timeout = Some(Duration::from_secs(8));
10594 current.schedule_to_close_timeout = Some(Duration::from_secs(120));
10595 current.heartbeat_timeout = Some(Duration::from_secs(12));
10596
10597 let ctx = workflow_context(history);
10598 let mut call = Box::pin(ctx.activity_with_options("flaky", current, json!([])));
10599 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10600 assert!(matches!(
10601 call.as_mut().poll(&mut task_context),
10602 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
10603 ));
10604 assert!(ctx.take_commands().expect("commands").is_empty());
10605 ctx.ensure_history_consumed().expect("history consumed");
10606 }
10607
10608 #[test]
10609 fn terminal_activity_failed_after_start_returns_typed_failure() {
10610 let history = vec![
10611 history_event(
10612 "ActivityScheduled",
10613 json!({
10614 "sequence": 1,
10615 "activity_type": "flaky",
10616 "activity_execution_id": "act-terminal",
10617 "activity": {
10618 "id": "act-terminal",
10619 "sequence": 1,
10620 "type": "flaky",
10621 "queue": "critical-activities",
10622 "retry_policy": {
10623 "snapshot_version": 1,
10624 "max_attempts": 3,
10625 "backoff_seconds": [2, 4],
10626 "non_retryable_error_types": ["PermanentError"]
10627 }
10628 }
10629 }),
10630 ),
10631 history_event(
10632 "ActivityStarted",
10633 json!({
10634 "sequence": 1,
10635 "activity_type": "flaky",
10636 "activity_execution_id": "act-terminal",
10637 "activity_attempt_id": "attempt-1",
10638 "attempt_number": 1
10639 }),
10640 ),
10641 history_event(
10642 "ActivityFailed",
10643 json!({
10644 "sequence": 1,
10645 "activity_type": "flaky",
10646 "activity_execution_id": "act-terminal",
10647 "activity_attempt_id": "attempt-1",
10648 "attempt_number": 1,
10649 "failure_id": "failure-terminal",
10650 "failure_category": "activity",
10651 "exception_type": "PermanentError",
10652 "message": "cannot retry",
10653 "non_retryable": true
10654 }),
10655 ),
10656 ];
10657 let ctx = workflow_context(history);
10658 let mut call =
10659 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
10660 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10661
10662 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
10663 call.as_mut().poll(&mut task_context)
10664 else {
10665 panic!("terminal ActivityFailed must settle the activity future");
10666 };
10667 assert_eq!(failure.kind, ActivityFailureKind::Failed);
10668 assert_eq!(
10669 failure.activity_execution_id.as_deref(),
10670 Some("act-terminal")
10671 );
10672 assert_eq!(failure.exception_type.as_deref(), Some("PermanentError"));
10673 assert!(failure.non_retryable);
10674 assert!(ctx.take_commands().expect("commands").is_empty());
10675 ctx.ensure_history_consumed().expect("history consumed");
10676 }
10677
10678 #[test]
10679 fn activity_terminal_events_return_machine_readable_failures() {
10680 let cases = [
10681 (
10682 "ActivityFailed",
10683 json!({
10684 "sequence": 1,
10685 "activity_type": "charge-card",
10686 "activity_execution_id": "act-1",
10687 "activity_attempt_id": "attempt-2",
10688 "attempt_number": 2,
10689 "failure_id": "failure-1",
10690 "failure_category": "activity",
10691 "exception_type": "PaymentDeclined",
10692 "exception_class": "payments.PaymentDeclined",
10693 "message": "card declined",
10694 "non_retryable": true
10695 }),
10696 ActivityFailureKind::Failed,
10697 "activity",
10698 ),
10699 (
10700 "ActivityCancelled",
10701 json!({
10702 "sequence": 1,
10703 "activity_type": "charge-card",
10704 "activity_execution_id": "act-1",
10705 "activity_attempt_id": "attempt-1"
10706 }),
10707 ActivityFailureKind::Cancelled,
10708 "cancelled",
10709 ),
10710 ];
10711
10712 for (event_type, payload, expected_kind, expected_reason) in cases {
10713 let ctx = workflow_context(vec![history_event(event_type, payload)]);
10714 let mut call = Box::pin(ctx.activity("charge-card", json!([])));
10715 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10716 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
10717 call.as_mut().poll(&mut task_context)
10718 else {
10719 panic!("expected terminal activity failure");
10720 };
10721 assert_eq!(failure.kind, expected_kind);
10722 assert_eq!(failure.reason, expected_reason);
10723 assert_eq!(failure.activity_execution_id.as_deref(), Some("act-1"));
10724 assert_eq!(failure.activity_type.as_deref(), Some("charge-card"));
10725 }
10726 }
10727
10728 #[test]
10729 fn every_activity_timeout_class_is_typed() {
10730 for timeout_kind in [
10731 "start_to_close",
10732 "schedule_to_start",
10733 "schedule_to_close",
10734 "heartbeat",
10735 ] {
10736 let ctx = workflow_context(vec![history_event(
10737 "ActivityTimedOut",
10738 json!({
10739 "sequence": 1,
10740 "activity_type": "slow",
10741 "activity_execution_id": "act-timeout",
10742 "activity_attempt_id": "attempt-timeout",
10743 "failure_category": "timeout",
10744 "timeout_kind": timeout_kind,
10745 "message": "deadline expired"
10746 }),
10747 )]);
10748 let mut call = Box::pin(ctx.activity("slow", json!([])));
10749 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10750 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
10751 call.as_mut().poll(&mut task_context)
10752 else {
10753 panic!("expected timeout failure");
10754 };
10755 assert_eq!(failure.kind, ActivityFailureKind::TimedOut);
10756 assert_eq!(failure.reason, timeout_kind);
10757 assert_eq!(failure.timeout_kind.as_deref(), Some(timeout_kind));
10758 assert_eq!(failure.failure_category.as_deref(), Some("timeout"));
10759 }
10760 }
10761
10762 #[test]
10763 fn workflow_sleep_emits_one_durable_timer_and_rounds_up() {
10764 let ctx = workflow_context(Vec::new());
10765 let mut sleep = Box::pin(ctx.sleep(Duration::from_millis(1_001)));
10766 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10767
10768 assert!(matches!(
10769 sleep.as_mut().poll(&mut task_context),
10770 Poll::Pending
10771 ));
10772 assert!(matches!(
10773 sleep.as_mut().poll(&mut task_context),
10774 Poll::Pending
10775 ));
10776
10777 let commands = ctx.take_commands().expect("timer command");
10778 assert_eq!(
10779 commands,
10780 vec![json!({
10781 "type": "start_timer",
10782 "delay_seconds": 2,
10783 })]
10784 );
10785 }
10786
10787 #[test]
10788 fn workflow_sleep_replays_matching_schedule_and_fire_without_a_command() {
10789 let history = vec![
10790 history_event(
10791 "TimerScheduled",
10792 json!({
10793 "sequence": 1,
10794 "timer_id": "timer-1",
10795 "delay_seconds": 5,
10796 "fire_at": "2026-07-11T12:00:05Z",
10797 }),
10798 ),
10799 history_event(
10800 "TimerFired",
10801 json!({
10802 "sequence": 1,
10803 "timer_id": "timer-1",
10804 "delay_seconds": 5,
10805 "fire_at": "2026-07-11T12:00:05Z",
10806 "fired_at": "2026-07-11T12:00:05Z",
10807 }),
10808 ),
10809 ];
10810
10811 for _restart in 0..2 {
10812 let ctx = workflow_context(history.clone());
10813 let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(5)));
10814 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10815 assert!(matches!(
10816 sleep.as_mut().poll(&mut task_context),
10817 Poll::Ready(Ok(()))
10818 ));
10819 assert!(ctx.take_commands().expect("commands").is_empty());
10820 ctx.ensure_history_consumed().expect("history consumed");
10821 }
10822 }
10823
10824 #[test]
10825 fn workflow_sleep_rejects_changed_delay_during_replay() {
10826 let ctx = workflow_context(vec![
10827 history_event(
10828 "TimerScheduled",
10829 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10830 ),
10831 history_event(
10832 "TimerFired",
10833 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10834 ),
10835 ]);
10836 let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(500)));
10837 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10838
10839 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10840 sleep.as_mut().poll(&mut task_context)
10841 else {
10842 panic!("changed timer delay must be rejected");
10843 };
10844 assert_eq!(failure.reason, "timer_delay_mismatch");
10845 assert_eq!(failure.sequence, Some(1));
10846 }
10847
10848 #[test]
10849 fn workflow_history_rejects_unpaired_or_mismatched_timer_events() {
10850 let lone_fire = WorkflowState::new(
10851 vec![history_event(
10852 "TimerFired",
10853 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10854 )],
10855 "rust-workers".to_string(),
10856 DEFAULT_CODEC.to_string(),
10857 None,
10858 )
10859 .expect_err("TimerFired requires TimerScheduled");
10860 assert!(matches!(
10861 lone_fire,
10862 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10863 if reason == "timer_schedule_missing_or_duplicate"
10864 ));
10865
10866 let wrong_identity = WorkflowState::new(
10867 vec![
10868 history_event(
10869 "TimerScheduled",
10870 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10871 ),
10872 history_event(
10873 "TimerFired",
10874 json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
10875 ),
10876 ],
10877 "rust-workers".to_string(),
10878 DEFAULT_CODEC.to_string(),
10879 None,
10880 )
10881 .expect_err("fire must match scheduled timer identity");
10882 assert!(matches!(
10883 wrong_identity,
10884 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10885 if reason == "timer_identity_mismatch"
10886 ));
10887
10888 let duplicate_fire = WorkflowState::new(
10889 vec![
10890 history_event(
10891 "TimerScheduled",
10892 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10893 ),
10894 history_event(
10895 "TimerFired",
10896 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10897 ),
10898 history_event(
10899 "TimerFired",
10900 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10901 ),
10902 ],
10903 "rust-workers".to_string(),
10904 DEFAULT_CODEC.to_string(),
10905 None,
10906 )
10907 .expect_err("a durable timer cannot fire twice");
10908 assert!(matches!(
10909 duplicate_fire,
10910 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10911 if reason == "duplicate_timer_fire"
10912 ));
10913
10914 let wrong_fired_delay = WorkflowState::new(
10915 vec![
10916 history_event(
10917 "TimerScheduled",
10918 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10919 ),
10920 history_event(
10921 "TimerFired",
10922 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 6}),
10923 ),
10924 ],
10925 "rust-workers".to_string(),
10926 DEFAULT_CODEC.to_string(),
10927 None,
10928 )
10929 .expect_err("timer schedule and fire delays must agree");
10930 assert!(matches!(
10931 wrong_fired_delay,
10932 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10933 if reason == "timer_history_delay_mismatch"
10934 ));
10935 }
10936
10937 #[test]
10938 fn replay_rejects_activity_moved_before_recorded_timer() {
10939 let ctx = workflow_context(vec![
10940 history_event(
10941 "TimerScheduled",
10942 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10943 ),
10944 history_event(
10945 "TimerFired",
10946 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10947 ),
10948 history_event(
10949 "ActivityCompleted",
10950 json!({
10951 "sequence": 2,
10952 "activity_type": "after-timer",
10953 "payload_codec": DEFAULT_CODEC,
10954 "result": fixture_envelope(json!("done")),
10955 }),
10956 ),
10957 ]);
10958 let mut activity = Box::pin(ctx.activity("after-timer", json!([])));
10959 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10960
10961 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10962 activity.as_mut().poll(&mut task_context)
10963 else {
10964 panic!("reordered durable command must be rejected");
10965 };
10966 assert_eq!(failure.reason, "recorded_command_mismatch");
10967 assert_eq!(failure.sequence, Some(1));
10968 assert_eq!(failure.expected.as_deref(), Some("timer"));
10969 assert_eq!(failure.actual.as_deref(), Some("activity:after-timer"));
10970 }
10971
10972 #[test]
10973 fn workflow_context_emits_a_typed_named_signal_wait() {
10974 let ctx = workflow_context(Vec::new());
10975 let mut signal = Box::pin(ctx.wait_signal("finish"));
10976 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10977
10978 assert!(matches!(
10979 signal.as_mut().poll(&mut task_context),
10980 Poll::Pending
10981 ));
10982 assert_eq!(
10983 ctx.take_commands().expect("signal-wait command"),
10984 vec![json!({
10985 "type": "open_signal_wait",
10986 "signal_name": "finish",
10987 })]
10988 );
10989 }
10990
10991 #[test]
10992 fn condition_wait_history_does_not_resolve_a_typed_signal_wait() {
10993 let ctx = workflow_context(vec![
10994 history_event(
10995 "ConditionWaitOpened",
10996 json!({"sequence": 1, "condition_key": "signal:finish"}),
10997 ),
10998 history_event(
10999 "ConditionWaitSatisfied",
11000 json!({"sequence": 1, "condition_key": "signal:finish"}),
11001 ),
11002 history_event(
11003 "SignalReceived",
11004 json!({"signal_name": "finish", "arguments": []}),
11005 ),
11006 ]);
11007 let mut signal = Box::pin(ctx.wait_signal("finish"));
11008 let mut task_context = TaskContext::from_waker(noop_waker_ref());
11009
11010 assert!(matches!(
11011 signal.as_mut().poll(&mut task_context),
11012 Poll::Pending
11013 ));
11014 assert_eq!(
11015 ctx.take_commands().expect("typed signal-wait command"),
11016 vec![json!({
11017 "type": "open_signal_wait",
11018 "signal_name": "finish",
11019 })]
11020 );
11021 }
11022
11023 #[test]
11024 fn replay_orders_signal_waits_and_timers_in_one_command_stream() {
11025 let signal_then_timer = vec![
11026 history_event(
11027 "SignalWaitOpened",
11028 json!({"sequence": 1, "signal_name": "go"}),
11029 ),
11030 history_event(
11031 "SignalApplied",
11032 json!({
11033 "sequence": 1,
11034 "signal_name": "go",
11035 "value": fixture_envelope(json!(["now"])),
11036 }),
11037 ),
11038 history_event(
11039 "TimerScheduled",
11040 json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
11041 ),
11042 history_event(
11043 "TimerFired",
11044 json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
11045 ),
11046 ];
11047
11048 let ctx = workflow_context(signal_then_timer.clone());
11049 let mut signal = Box::pin(ctx.wait_signal("go"));
11050 let mut task_context = TaskContext::from_waker(noop_waker_ref());
11051 assert!(matches!(
11052 signal.as_mut().poll(&mut task_context),
11053 Poll::Ready(Ok(arguments)) if arguments == vec![json!("now")]
11054 ));
11055 let mut timer = Box::pin(ctx.sleep(Duration::from_secs(5)));
11056 assert!(matches!(
11057 timer.as_mut().poll(&mut task_context),
11058 Poll::Ready(Ok(()))
11059 ));
11060 ctx.ensure_history_consumed()
11061 .expect("signal and timer history consumed in order");
11062
11063 let reordered = workflow_context(signal_then_timer);
11064 let mut timer_first = Box::pin(reordered.sleep(Duration::from_secs(5)));
11065 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
11066 timer_first.as_mut().poll(&mut task_context)
11067 else {
11068 panic!("timer cannot consume signal-wait-first history");
11069 };
11070 assert_eq!(failure.reason, "recorded_command_mismatch");
11071 assert_eq!(failure.sequence, Some(1));
11072 assert_eq!(failure.expected.as_deref(), Some("signal wait"));
11073
11074 let timer_then_signal = vec![
11075 history_event(
11076 "TimerScheduled",
11077 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
11078 ),
11079 history_event(
11080 "TimerFired",
11081 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
11082 ),
11083 history_event(
11084 "SignalWaitOpened",
11085 json!({"sequence": 2, "signal_name": "go"}),
11086 ),
11087 history_event(
11088 "SignalApplied",
11089 json!({
11090 "sequence": 2,
11091 "signal_name": "go",
11092 "value": fixture_envelope(json!([])),
11093 }),
11094 ),
11095 ];
11096 let reordered = workflow_context(timer_then_signal);
11097 let mut signal_first = Box::pin(reordered.wait_signal("go"));
11098 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
11099 signal_first.as_mut().poll(&mut task_context)
11100 else {
11101 panic!("signal wait cannot consume timer-first history");
11102 };
11103 assert_eq!(failure.reason, "recorded_command_mismatch");
11104 assert_eq!(failure.sequence, Some(1));
11105 assert_eq!(failure.expected.as_deref(), Some("timer"));
11106 }
11107
11108 #[test]
11109 fn workflow_history_rejects_duplicate_or_colliding_command_sequences() {
11110 let duplicate_timer = WorkflowState::new(
11111 vec![
11112 history_event(
11113 "TimerScheduled",
11114 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
11115 ),
11116 history_event(
11117 "TimerScheduled",
11118 json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
11119 ),
11120 ],
11121 "rust-workers".to_string(),
11122 DEFAULT_CODEC.to_string(),
11123 None,
11124 )
11125 .expect_err("one workflow sequence cannot schedule two timers");
11126 assert!(matches!(
11127 duplicate_timer,
11128 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
11129 if reason == "timer_schedule_missing_or_duplicate"
11130 ));
11131
11132 let colliding_kinds = WorkflowState::new(
11133 vec![
11134 history_event(
11135 "TimerScheduled",
11136 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
11137 ),
11138 history_event(
11139 "ActivityCompleted",
11140 json!({"sequence": 1, "activity_type": "same-sequence"}),
11141 ),
11142 ],
11143 "rust-workers".to_string(),
11144 DEFAULT_CODEC.to_string(),
11145 None,
11146 )
11147 .expect_err("one workflow sequence cannot identify two command kinds");
11148 assert!(matches!(
11149 colliding_kinds,
11150 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
11151 if reason == "durable_command_sequence_collision"
11152 ));
11153
11154 let duplicate_signal_wait = WorkflowState::new(
11155 vec![
11156 history_event(
11157 "SignalWaitOpened",
11158 json!({"sequence": 1, "signal_name": "go"}),
11159 ),
11160 history_event(
11161 "SignalWaitOpened",
11162 json!({"sequence": 1, "signal_name": "go"}),
11163 ),
11164 ],
11165 "rust-workers".to_string(),
11166 DEFAULT_CODEC.to_string(),
11167 None,
11168 )
11169 .expect_err("one workflow sequence cannot open two signal waits");
11170 assert!(matches!(
11171 duplicate_signal_wait,
11172 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
11173 if reason == "signal_wait_open_missing_or_duplicate"
11174 ));
11175 }
11176
11177 #[test]
11178 fn workflow_history_accepts_a_first_command_after_global_sequence_gaps() {
11179 let result = encode_value_envelope(&json!({"captured": true}), DEFAULT_CODEC)
11180 .expect("side-effect result");
11181 let ctx = workflow_context(vec![history_event(
11182 "SideEffectRecorded",
11183 json!({"sequence": 99, "result": result}),
11184 )]);
11185
11186 let replayed: Value = ctx
11187 .side_effect(|| panic!("recorded side effect must not run"))
11188 .expect("positive global workflow sequence is valid");
11189 assert_eq!(replayed, json!({"captured": true}));
11190 ctx.ensure_history_consumed().expect("history consumed");
11191 }
11192
11193 #[test]
11194 fn workflow_history_rejects_zero_and_descending_command_sequences() {
11195 let result =
11196 encode_value_envelope(&json!("captured"), DEFAULT_CODEC).expect("side-effect result");
11197 let zero = WorkflowState::new(
11198 vec![history_event(
11199 "SideEffectRecorded",
11200 json!({"sequence": 0, "result": result.clone()}),
11201 )],
11202 "rust-workers".to_string(),
11203 DEFAULT_CODEC.to_string(),
11204 None,
11205 )
11206 .expect_err("durable command sequences must be positive");
11207 assert!(matches!(
11208 zero,
11209 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
11210 if reason == "durable_command_sequence_invalid"
11211 ));
11212
11213 let descending = WorkflowState::new(
11214 vec![
11215 history_event(
11216 "SideEffectRecorded",
11217 json!({"sequence": 3, "result": result}),
11218 ),
11219 history_event(
11220 "VersionMarkerRecorded",
11221 json!({
11222 "sequence": 2,
11223 "change_id": "descending-marker",
11224 "version": 1,
11225 "min_supported": 1,
11226 "max_supported": 1,
11227 }),
11228 ),
11229 ],
11230 "rust-workers".to_string(),
11231 DEFAULT_CODEC.to_string(),
11232 None,
11233 )
11234 .expect_err("new durable commands must remain strictly ordered");
11235 let Error::NonDeterministicReplay(failure) = descending else {
11236 panic!("expected typed replay failure");
11237 };
11238 assert_eq!(failure.reason, "durable_command_sequence_mismatch");
11239 assert_eq!(failure.sequence, Some(2));
11240 assert_eq!(
11241 failure.expected.as_deref(),
11242 Some("workflow sequence greater than 3")
11243 );
11244 assert_eq!(failure.actual.as_deref(), Some("2"));
11245 }
11246
11247 #[test]
11248 fn workflow_task_replay_completes_after_signals_create_sequence_gaps() {
11249 fn worker() -> Worker {
11250 let client = Client::new("http://127.0.0.1:8080").expect("client");
11251 let mut worker = Worker::new(client, "rust-workers");
11252 worker.register_workflow("rust.finish-after-gaps", |ctx, _input| async move {
11253 ctx.wait_signal("finish").await?;
11254 let marker: String =
11255 ctx.side_effect(|| panic!("recorded side effect must not run"))?;
11256 assert_eq!(marker, "after-finish");
11257 Ok(json!("finished"))
11258 });
11259 worker
11260 }
11261
11262 let marker = encode_value_envelope(&json!("after-finish"), DEFAULT_CODEC)
11263 .expect("side-effect result");
11264 let task = workflow_task(
11265 "rust.finish-after-gaps",
11266 vec![
11267 history_event(
11268 "SignalWaitOpened",
11269 json!({"sequence": 1, "signal_name": "finish"}),
11270 ),
11271 history_event(
11272 "SignalReceived",
11273 json!({
11274 "signal_id": "increment-3",
11275 "signal_name": "increment",
11276 "workflow_sequence": 2,
11277 "payload_codec": DEFAULT_CODEC,
11278 "arguments": fixture_envelope(json!([3])),
11279 }),
11280 ),
11281 history_event(
11282 "SignalReceived",
11283 json!({
11284 "signal_id": "increment-5",
11285 "signal_name": "increment",
11286 "workflow_sequence": 3,
11287 "payload_codec": DEFAULT_CODEC,
11288 "arguments": fixture_envelope(json!([5])),
11289 }),
11290 ),
11291 history_event(
11292 "SignalReceived",
11293 json!({
11294 "signal_id": "finish",
11295 "signal_name": "finish",
11296 "workflow_sequence": 4,
11297 "payload_codec": DEFAULT_CODEC,
11298 "arguments": fixture_envelope(json!([])),
11299 }),
11300 ),
11301 history_event(
11302 "SignalApplied",
11303 json!({
11304 "sequence": 1,
11305 "signal_id": "finish",
11306 "signal_name": "finish",
11307 "payload_codec": DEFAULT_CODEC,
11308 "value": fixture_envelope(json!([])),
11309 }),
11310 ),
11311 history_event(
11312 "SideEffectRecorded",
11313 json!({"sequence": 5, "result": marker}),
11314 ),
11315 ],
11316 DEFAULT_CODEC,
11317 );
11318
11319 for _original_or_cold_worker in 0..2 {
11320 let commands = worker()
11321 .execute_workflow_task(task.clone())
11322 .expect("signal gaps preserve deterministic replay");
11323 assert_eq!(commands.len(), 1, "replay emits only terminal completion");
11324 assert_eq!(commands[0]["type"], "complete_workflow");
11325 assert_eq!(
11326 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("workflow output"),
11327 json!("finished")
11328 );
11329 }
11330 }
11331
11332 #[test]
11333 fn workflow_sleep_rejects_unrepresentable_rounded_duration() {
11334 let ctx = workflow_context(Vec::new());
11335 let mut sleep = Box::pin(ctx.start_timer(Duration::new(u64::MAX, 1)));
11336 let mut task_context = TaskContext::from_waker(noop_waker_ref());
11337 assert!(matches!(
11338 sleep.as_mut().poll(&mut task_context),
11339 Poll::Ready(Err(Error::TimerDurationOverflow))
11340 ));
11341 assert!(ctx.take_commands().expect("commands").is_empty());
11342 }
11343
11344 #[test]
11345 fn workflow_task_replay_completes_without_rescheduling_recorded_commands() {
11346 let client = Client::new("http://127.0.0.1:8080").expect("client");
11347 let mut worker = Worker::new(client, "rust-workers");
11348 worker.register_workflow("rust.timer", |ctx, _input| async move {
11349 ctx.sleep(Duration::from_secs(5)).await?;
11350 ctx.activity("after-timer", json!([])).await
11351 });
11352
11353 let task = |history_events| WorkflowTask {
11354 task_id: "wft-rust-timer-1".to_string(),
11355 workflow_id: Some("wf-rust-timer".to_string()),
11356 run_id: Some("run-rust-timer".to_string()),
11357 workflow_type: "rust.timer".to_string(),
11358 payload_codec: DEFAULT_CODEC.to_string(),
11359 arguments: Some(
11360 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
11361 ),
11362 history_events,
11363 total_history_events: None,
11364 history_size_bytes: None,
11365 continue_as_new_recommended: None,
11366 history_budget_pressure: None,
11367 next_history_page_token: None,
11368 workflow_task_attempt: 1,
11369 workflow_signal_id: None,
11370 signal_name: None,
11371 signal_arguments: None,
11372 workflow_update_id: None,
11373 update_name: None,
11374 lease_owner: Some("rust-worker".to_string()),
11375 };
11376
11377 let initial = worker
11378 .execute_workflow_task(task(Vec::new()))
11379 .expect("initial timer task");
11380 assert_eq!(
11381 initial,
11382 vec![json!({"type": "start_timer", "delay_seconds": 5})]
11383 );
11384
11385 let activity_result =
11386 encode_value_envelope(&json!("done"), DEFAULT_CODEC).expect("activity result");
11387 let replayed = worker
11388 .execute_workflow_task(task(vec![
11389 history_event(
11390 "TimerScheduled",
11391 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
11392 ),
11393 history_event(
11394 "TimerFired",
11395 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
11396 ),
11397 history_event(
11398 "ActivityCompleted",
11399 json!({
11400 "sequence": 2,
11401 "activity_type": "after-timer",
11402 "payload_codec": DEFAULT_CODEC,
11403 "result": activity_result,
11404 }),
11405 ),
11406 ]))
11407 .expect("replayed workflow task");
11408 assert_eq!(replayed.len(), 1);
11409 assert_eq!(replayed[0]["type"], "complete_workflow");
11410 assert_eq!(
11411 decode_wire_value(&replayed[0]["result"], DEFAULT_CODEC).expect("result"),
11412 json!("done")
11413 );
11414 }
11415
11416 #[test]
11417 fn workflow_continue_as_new_emits_arguments_type_and_queue_once() {
11418 let client = Client::new("http://127.0.0.1:8080").expect("client");
11419 let mut worker = Worker::new(client, "rust-workers");
11420 worker.register_workflow("rust.continue", |ctx, _input| async move {
11421 ctx.continue_as_new_with_options(
11422 ContinueAsNewOptions::new()
11423 .workflow_type("rust.next")
11424 .task_queue("next-workers"),
11425 json!([2, {"cursor": "next"}]),
11426 )
11427 });
11428
11429 let commands = worker
11430 .execute_workflow_task(workflow_task("rust.continue", Vec::new(), DEFAULT_CODEC))
11431 .expect("continue-as-new command");
11432
11433 assert_eq!(commands.len(), 1);
11434 assert_eq!(commands[0]["type"], "continue_as_new");
11435 assert_eq!(commands[0]["workflow_type"], "rust.next");
11436 assert_eq!(commands[0]["queue"], "next-workers");
11437 assert_eq!(
11438 decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC)
11439 .expect("continue-as-new arguments"),
11440 json!([2, {"cursor": "next"}])
11441 );
11442 }
11443
11444 #[test]
11445 fn continue_as_new_preserves_typed_arguments() {
11446 let client = Client::new("http://127.0.0.1:8080").expect("client");
11447 let mut worker = Worker::new(client, "rust-workers");
11448 worker.register_workflow_avro_value("rust.typed-continue", |ctx, _input| async move {
11449 ctx.continue_as_new(AvroValue::Array(vec![typed_fidelity_probe()]))?;
11450 unreachable!("continue-as-new returns a control-flow error")
11451 });
11452
11453 let commands = worker
11454 .execute_workflow_task(workflow_task(
11455 "rust.typed-continue",
11456 Vec::new(),
11457 DEFAULT_CODEC,
11458 ))
11459 .expect("typed continue-as-new command");
11460
11461 assert_eq!(commands[0]["type"], "continue_as_new");
11462 assert_eq!(
11463 decode_wire_avro_value(&commands[0]["arguments"], DEFAULT_CODEC)
11464 .expect("typed continue arguments"),
11465 AvroValue::Array(vec![typed_fidelity_probe()])
11466 );
11467 }
11468
11469 #[test]
11470 fn recorded_continue_as_new_is_consumed_without_duplicate_successor_command() {
11471 let client = Client::new("http://127.0.0.1:8080").expect("client");
11472 let mut worker = Worker::new(client, "rust-workers");
11473 worker.register_workflow("rust.continue", |ctx, _input| async move {
11474 ctx.continue_as_new(json!([2]))
11475 });
11476 let task = workflow_task(
11477 "rust.continue",
11478 vec![history_event(
11479 "WorkflowContinuedAsNew",
11480 json!({"sequence": 1, "continued_to_run_id": "run-next"}),
11481 )],
11482 DEFAULT_CODEC,
11483 );
11484
11485 for _worker_restart_or_redelivery in 0..2 {
11486 let commands = worker
11487 .execute_workflow_task(task.clone())
11488 .expect("recorded transition replays");
11489 assert!(
11490 commands.is_empty(),
11491 "replay must not emit another successor"
11492 );
11493 }
11494 }
11495
11496 #[test]
11497 fn continue_as_new_rejects_invalid_overrides_before_emitting_a_command() {
11498 let ctx = workflow_context(Vec::new());
11499 let error = ctx
11500 .continue_as_new_with_options(ContinueAsNewOptions::new().task_queue(" "), json!([1]))
11501 .expect_err("blank queue must be rejected");
11502
11503 let Error::InvalidContinueAsNewOptions(error) = error else {
11504 panic!("expected typed continue-as-new validation error");
11505 };
11506 assert_eq!(error.field, "task_queue");
11507 assert!(ctx.take_commands().expect("commands").is_empty());
11508 }
11509
11510 #[test]
11511 fn workflow_context_exposes_server_history_budget() {
11512 let client = Client::new("http://127.0.0.1:8080").expect("client");
11513 let mut worker = Worker::new(client, "rust-workers");
11514 worker.register_workflow("rust.history-budget", |ctx, _input| async move {
11515 let budget = ctx.history_budget()?;
11516 Ok(json!({
11517 "events": budget.event_count,
11518 "bytes": budget.size_bytes,
11519 "recommended": budget.continue_as_new_recommended,
11520 "pressure": budget.pressure,
11521 }))
11522 });
11523 let task: WorkflowTask = serde_json::from_value(json!({
11524 "task_id": "task-history-budget",
11525 "workflow_type": "rust.history-budget",
11526 "payload_codec": DEFAULT_CODEC,
11527 "history_events": [],
11528 "total_history_events": 480,
11529 "history_size_bytes": 1_048_576,
11530 "continue_as_new_recommended": true,
11531 "history_budget_pressure": "continue_as_new_recommended",
11532 }))
11533 .expect("published workflow task");
11534
11535 let commands = worker
11536 .execute_workflow_task(task)
11537 .expect("history-budget workflow");
11538 let result = decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("result");
11539 assert_eq!(result["events"], 480);
11540 assert_eq!(result["bytes"], 1_048_576);
11541 assert_eq!(result["recommended"], true);
11542 assert_eq!(result["pressure"], "continue_as_new_recommended");
11543 }
11544
11545 #[test]
11546 fn uncaught_workflow_handler_error_emits_terminal_failure_command() {
11547 let client = Client::new("http://127.0.0.1:8080").expect("client");
11548 let mut worker = Worker::new(client, "rust-workers");
11549 worker.register_workflow("rust.failing", |_ctx, _input| async move {
11550 Err(Error::Codec("rust_conformance_failure".to_string()))
11551 });
11552 let task = WorkflowTask {
11553 task_id: "wft-rust-failing-1".to_string(),
11554 workflow_id: Some("wf-rust-failing".to_string()),
11555 run_id: Some("run-rust-failing".to_string()),
11556 workflow_type: "rust.failing".to_string(),
11557 payload_codec: DEFAULT_CODEC.to_string(),
11558 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
11559 history_events: Vec::new(),
11560 total_history_events: Some(0),
11561 history_size_bytes: None,
11562 continue_as_new_recommended: None,
11563 history_budget_pressure: None,
11564 next_history_page_token: None,
11565 workflow_task_attempt: 1,
11566 workflow_signal_id: None,
11567 signal_name: None,
11568 signal_arguments: None,
11569 workflow_update_id: None,
11570 update_name: None,
11571 lease_owner: Some("rust-worker".to_string()),
11572 };
11573
11574 let commands = worker
11575 .execute_workflow_task(task)
11576 .expect("handler failure becomes a workflow command");
11577
11578 assert_eq!(commands.len(), 1);
11579 assert_eq!(commands[0]["type"], "fail_workflow");
11580 assert_eq!(commands[0]["exception_type"], "RustWorkflowError");
11581 assert_eq!(commands[0]["exception_class"], "durable_workflow::Error");
11582 assert_eq!(commands[0]["non_retryable"], false);
11583 assert_eq!(
11584 commands[0]["message"],
11585 "codec error: rust_conformance_failure"
11586 );
11587 assert_eq!(
11588 commands[0]["exception"]["message"],
11589 "codec error: rust_conformance_failure"
11590 );
11591 }
11592
11593 #[test]
11594 fn ordinary_handler_error_preserves_commands_queued_in_the_same_decision() {
11595 let client = Client::new("http://127.0.0.1:8080").expect("client");
11596 let mut worker = Worker::new(client, "rust-workers");
11597 worker.register_workflow("rust.failing-after-side-effect", |ctx, _input| async move {
11598 let _: String = ctx.side_effect(|| "captured".to_string())?;
11599 Err(Error::WorkerLoop("application failure".to_string()))
11600 });
11601
11602 let commands = worker
11603 .execute_workflow_task(workflow_task(
11604 "rust.failing-after-side-effect",
11605 Vec::new(),
11606 DEFAULT_CODEC,
11607 ))
11608 .expect("ordinary failure remains a workflow decision");
11609
11610 assert_eq!(commands.len(), 2);
11611 assert_eq!(commands[0]["type"], "record_side_effect");
11612 assert_eq!(commands[1]["type"], "fail_workflow");
11613 }
11614
11615 #[test]
11616 fn handler_error_cannot_hide_an_unconsumed_committed_side_effect() {
11617 let client = Client::new("http://127.0.0.1:8080").expect("client");
11618 let mut worker = Worker::new(client, "rust-workers");
11619 worker.register_workflow("rust.removed-side-effect", |_ctx, _input| async move {
11620 Err(Error::WorkerLoop("application failure".to_string()))
11621 });
11622 let result =
11623 encode_value_envelope(&json!("committed"), DEFAULT_CODEC).expect("side-effect result");
11624
11625 let error = worker
11626 .execute_workflow_task(workflow_task(
11627 "rust.removed-side-effect",
11628 vec![history_event(
11629 "SideEffectRecorded",
11630 json!({"sequence": 1, "result": result}),
11631 )],
11632 DEFAULT_CODEC,
11633 ))
11634 .expect_err("removed committed history must not become fail_workflow");
11635
11636 let Error::NonDeterministicReplay(failure) = error else {
11637 panic!("expected typed replay failure");
11638 };
11639 assert_eq!(failure.reason, "recorded_commands_unconsumed");
11640 assert_eq!(failure.sequence, Some(1));
11641 assert_eq!(failure.expected.as_deref(), Some("side effect"));
11642 }
11643
11644 #[test]
11645 fn replay_error_discards_side_effect_queued_before_incompatible_marker_check() {
11646 let client = Client::new("http://127.0.0.1:8080").expect("client");
11647 let mut worker = Worker::new(client, "rust-workers");
11648 worker.register_workflow(
11649 "rust.side-effect-before-marker-error",
11650 |ctx, _input| async move {
11651 assert_eq!(ctx.get_version("restart-safe", 1, 1)?, 1);
11652 let _: String = ctx.side_effect(|| "must-not-commit".to_string())?;
11653 ctx.get_version("restart-safe", 2, 2)?;
11654 Ok(Value::Null)
11655 },
11656 );
11657
11658 let error = worker
11659 .execute_workflow_task(workflow_task(
11660 "rust.side-effect-before-marker-error",
11661 vec![history_event(
11662 "VersionMarkerRecorded",
11663 json!({
11664 "sequence": 1,
11665 "change_id": "restart-safe",
11666 "version": 1,
11667 "min_supported": 1,
11668 "max_supported": 1,
11669 }),
11670 )],
11671 DEFAULT_CODEC,
11672 ))
11673 .expect_err("replay error must return no queued workflow commands");
11674
11675 let Error::NonDeterministicReplay(failure) = error else {
11676 panic!("expected typed replay failure");
11677 };
11678 assert_eq!(failure.reason, "version_marker_incompatible_range");
11679 assert_eq!(failure.sequence, Some(1));
11680 }
11681
11682 #[test]
11683 fn workflow_task_replay_keeps_recorded_unfired_timer_pending_without_rescheduling() {
11684 let client = Client::new("http://127.0.0.1:8080").expect("client");
11685 let mut worker = Worker::new(client, "rust-workers");
11686 worker.register_workflow("rust.timer.pending", |ctx, _input| async move {
11687 ctx.sleep(Duration::from_secs(5)).await?;
11688 Ok(json!({"status": "timer fired"}))
11689 });
11690
11691 let task = WorkflowTask {
11692 task_id: "wft-rust-timer-pending".to_string(),
11693 workflow_id: Some("wf-rust-timer".to_string()),
11694 run_id: Some("run-rust-timer".to_string()),
11695 workflow_type: "rust.timer.pending".to_string(),
11696 payload_codec: DEFAULT_CODEC.to_string(),
11697 arguments: Some(
11698 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
11699 ),
11700 history_events: vec![history_event(
11701 "TimerScheduled",
11702 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
11703 )],
11704 total_history_events: Some(1),
11705 history_size_bytes: None,
11706 continue_as_new_recommended: None,
11707 history_budget_pressure: None,
11708 next_history_page_token: None,
11709 workflow_task_attempt: 1,
11710 workflow_signal_id: None,
11711 signal_name: None,
11712 signal_arguments: None,
11713 workflow_update_id: None,
11714 update_name: None,
11715 lease_owner: Some("rust-worker".to_string()),
11716 };
11717
11718 for _redelivery_or_restart in 0..2 {
11719 let commands = worker
11720 .execute_workflow_task(task.clone())
11721 .expect("recorded timer remains pending");
11722 assert!(
11723 commands.is_empty(),
11724 "recorded timer must not be rescheduled"
11725 );
11726 }
11727 }
11728
11729 #[test]
11730 fn workflow_task_rejects_recorded_command_removed_from_workflow_code() {
11731 let client = Client::new("http://127.0.0.1:8080").expect("client");
11732 let mut worker = Worker::new(client, "rust-workers");
11733 worker.register_workflow("rust.timer.removed", |_ctx, _input| async move {
11734 Ok(json!({"status": "completed"}))
11735 });
11736 let task = WorkflowTask {
11737 task_id: "wft-rust-timer-removed".to_string(),
11738 workflow_id: Some("wf-rust-timer".to_string()),
11739 run_id: Some("run-rust-timer".to_string()),
11740 workflow_type: "rust.timer.removed".to_string(),
11741 payload_codec: DEFAULT_CODEC.to_string(),
11742 arguments: Some(
11743 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
11744 ),
11745 history_events: vec![
11746 history_event(
11747 "TimerScheduled",
11748 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
11749 ),
11750 history_event(
11751 "TimerFired",
11752 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
11753 ),
11754 ],
11755 total_history_events: Some(2),
11756 history_size_bytes: None,
11757 continue_as_new_recommended: None,
11758 history_budget_pressure: None,
11759 next_history_page_token: None,
11760 workflow_task_attempt: 1,
11761 workflow_signal_id: None,
11762 signal_name: None,
11763 signal_arguments: None,
11764 workflow_update_id: None,
11765 update_name: None,
11766 lease_owner: Some("rust-worker".to_string()),
11767 };
11768
11769 let Error::NonDeterministicReplay(failure) = worker
11770 .execute_workflow_task(task)
11771 .expect_err("removed timer must fail replay")
11772 else {
11773 panic!("expected typed replay failure");
11774 };
11775 assert_eq!(failure.reason, "recorded_commands_unconsumed");
11776 assert_eq!(failure.sequence, Some(1));
11777 }
11778
11779 #[test]
11780 fn workflow_context_emits_explicit_child_workflow_contract() {
11781 let ctx = WorkflowContext {
11782 state: Arc::new(Mutex::new(
11783 WorkflowState::new_with_identity(
11784 Vec::new(),
11785 Some("wf-parent".to_string()),
11786 Some("run-parent".to_string()),
11787 "parent-workers".to_string(),
11788 DEFAULT_CODEC.to_string(),
11789 None,
11790 )
11791 .expect("workflow state"),
11792 )),
11793 };
11794 let options = ChildWorkflowOptions::new("python-workers")
11795 .parent_close_policy(ParentClosePolicy::RequestCancel)
11796 .retry_policy(ChildWorkflowRetryPolicy {
11797 max_attempts: Some(3),
11798 backoff_seconds: vec![1, 5],
11799 non_retryable_error_types: vec!["ValidationError".to_string()],
11800 })
11801 .execution_timeout_seconds(600)
11802 .run_timeout_seconds(120);
11803 let mut call = Box::pin(ctx.start_child_workflow(
11804 "python.fulfil-order",
11805 options,
11806 json!([{"order_id": "order-42"}]),
11807 ));
11808 let mut task_context = TaskContext::from_waker(noop_waker_ref());
11809
11810 assert!(matches!(
11811 call.as_mut().poll(&mut task_context),
11812 Poll::Pending
11813 ));
11814 let commands = ctx.take_commands().expect("commands");
11815 assert_eq!(commands.len(), 1);
11816 let command = &commands[0];
11817 assert_eq!(command["type"], "start_child_workflow");
11818 assert_eq!(command["workflow_type"], "python.fulfil-order");
11819 assert_eq!(command["queue"], "python-workers");
11820 assert_eq!(command["parent_close_policy"], "request_cancel");
11821 assert_eq!(command["retry_policy"]["max_attempts"], 3);
11822 assert_eq!(command["execution_timeout_seconds"], 600);
11823 assert_eq!(command["run_timeout_seconds"], 120);
11824 assert_eq!(
11825 decode_wire_value(&command["arguments"], DEFAULT_CODEC).expect("child args"),
11826 json!([{"order_id": "order-42"}])
11827 );
11828 }
11829
11830 fn child_parent_worker() -> Worker {
11831 let client = Client::new("http://127.0.0.1:8080").expect("client");
11832 let mut worker = Worker::new(client, "rust-parent-workers");
11833 worker.register_workflow("rust.parent", |ctx, _input| async move {
11834 let child = ctx
11835 .start_child_workflow(
11836 "python.child",
11837 ChildWorkflowOptions::new("python-child-workers")
11838 .parent_close_policy(ParentClosePolicy::Terminate),
11839 json!([{"codec_probe": [1, true, "rust"]}]),
11840 )
11841 .await?;
11842 Ok(json!({
11843 "parent_workflow_id": child.parent.workflow_id,
11844 "parent_run_id": child.parent.run_id,
11845 "child_workflow_id": child.child.workflow_id,
11846 "child_run_id": child.child.run_id,
11847 "child_workflow_type": child.child_workflow_type,
11848 "result": child.result,
11849 }))
11850 });
11851 worker
11852 }
11853
11854 fn child_parent_task(event_type: &str, payload: Value) -> WorkflowTask {
11855 WorkflowTask {
11856 task_id: "wft-child-parent".to_string(),
11857 workflow_id: Some("wf-parent".to_string()),
11858 run_id: Some("run-parent".to_string()),
11859 workflow_type: "rust.parent".to_string(),
11860 payload_codec: DEFAULT_CODEC.to_string(),
11861 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
11862 history_events: vec![
11863 HistoryEvent {
11864 event_type: "ChildWorkflowScheduled".to_string(),
11865 payload: json!({
11866 "sequence": 1,
11867 "child_call_id": "call-child",
11868 "child_workflow_instance_id": "wf-child",
11869 "child_workflow_run_id": "run-child",
11870 "child_workflow_type": "python.child",
11871 }),
11872 raw: HashMap::new(),
11873 },
11874 HistoryEvent {
11875 event_type: event_type.to_string(),
11876 payload,
11877 raw: HashMap::new(),
11878 },
11879 ],
11880 total_history_events: Some(2),
11881 history_size_bytes: None,
11882 continue_as_new_recommended: None,
11883 history_budget_pressure: None,
11884 next_history_page_token: None,
11885 workflow_task_attempt: 1,
11886 workflow_signal_id: None,
11887 signal_name: None,
11888 signal_arguments: None,
11889 workflow_update_id: None,
11890 update_name: None,
11891 lease_owner: Some("rust-worker".to_string()),
11892 }
11893 }
11894
11895 #[test]
11896 fn committed_child_result_replays_without_starting_a_duplicate() {
11897 let worker = child_parent_worker();
11898 let task = child_parent_task(
11899 "ChildRunCompleted",
11900 json!({
11901 "sequence": 1,
11902 "child_call_id": "call-child",
11903 "child_workflow_instance_id": "wf-child",
11904 "child_workflow_run_id": "run-child",
11905 "child_workflow_type": "python.child",
11906 "payload_codec": DEFAULT_CODEC,
11907 "result": fixture_envelope(json!({"from":"python","ok":true})),
11908 }),
11909 );
11910
11911 for _restart in 0..2 {
11912 let commands = worker
11913 .execute_workflow_task(task.clone())
11914 .expect("replayed parent task");
11915 assert_eq!(commands.len(), 1);
11916 assert_eq!(commands[0]["type"], "complete_workflow");
11917 assert!(!commands
11918 .iter()
11919 .any(|command| command["type"] == "start_child_workflow"));
11920 let output =
11921 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("parent output");
11922 assert_eq!(output["parent_workflow_id"], "wf-parent");
11923 assert_eq!(output["parent_run_id"], "run-parent");
11924 assert_eq!(output["child_workflow_id"], "wf-child");
11925 assert_eq!(output["child_run_id"], "run-child");
11926 assert_eq!(output["result"], json!({"from": "python", "ok": true}));
11927 }
11928 }
11929
11930 #[test]
11931 fn typed_child_arguments_and_results_survive_replay() {
11932 let client = Client::new("http://127.0.0.1:8080").expect("client");
11933 let mut worker = Worker::new(client, "rust-parent-workers");
11934 worker.register_workflow_avro_value("rust.typed-parent", |ctx, _input| async move {
11935 let child = ctx
11936 .start_child_workflow_avro_value(
11937 "python.typed-child",
11938 ChildWorkflowOptions::new("python-workers"),
11939 AvroValue::Array(vec![typed_fidelity_probe()]),
11940 )
11941 .await?;
11942 Ok(child.result)
11943 });
11944
11945 let initial = worker
11946 .execute_workflow_task(workflow_task(
11947 "rust.typed-parent",
11948 Vec::new(),
11949 DEFAULT_CODEC,
11950 ))
11951 .expect("typed child start");
11952 assert_eq!(initial[0]["type"], "start_child_workflow");
11953 assert_eq!(
11954 decode_wire_avro_value(&initial[0]["arguments"], DEFAULT_CODEC)
11955 .expect("typed child arguments"),
11956 AvroValue::Array(vec![typed_fidelity_probe()])
11957 );
11958
11959 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
11960 .expect("typed child result");
11961 let task = workflow_task(
11962 "rust.typed-parent",
11963 vec![
11964 history_event(
11965 "ChildWorkflowScheduled",
11966 json!({
11967 "sequence": 1,
11968 "child_call_id": "call-typed",
11969 "child_workflow_instance_id": "wf-child",
11970 "child_workflow_run_id": "run-child",
11971 "child_workflow_type": "python.typed-child",
11972 }),
11973 ),
11974 history_event(
11975 "ChildRunCompleted",
11976 json!({
11977 "sequence": 1,
11978 "child_call_id": "call-typed",
11979 "child_workflow_instance_id": "wf-child",
11980 "child_workflow_run_id": "run-child",
11981 "child_workflow_type": "python.typed-child",
11982 "payload_codec": DEFAULT_CODEC,
11983 "result": result,
11984 }),
11985 ),
11986 ],
11987 DEFAULT_CODEC,
11988 );
11989
11990 let commands = worker
11991 .execute_workflow_task(task)
11992 .expect("typed child replay");
11993 assert_eq!(commands[0]["type"], "complete_workflow");
11994 assert_eq!(
11995 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
11996 .expect("typed parent result"),
11997 typed_fidelity_probe()
11998 );
11999 }
12000
12001 #[test]
12002 fn pending_child_replays_after_restart_without_starting_a_duplicate() {
12003 let worker = child_parent_worker();
12004 let mut task = child_parent_task("unused", Value::Null);
12005 task.history_events.truncate(1);
12006 task.total_history_events = Some(1);
12007
12008 for _redelivery_or_restart in 0..2 {
12009 let commands = worker
12010 .execute_workflow_task(task.clone())
12011 .expect("recorded child remains pending");
12012 assert!(
12013 commands.is_empty(),
12014 "recorded pending child must not be started again"
12015 );
12016 }
12017 }
12018
12019 #[test]
12020 fn child_cancellation_becomes_stable_parent_failure_command() {
12021 let worker = child_parent_worker();
12022 let task = child_parent_task(
12023 "ChildRunCancelled",
12024 json!({
12025 "sequence": 1,
12026 "child_workflow_instance_id": "wf-child",
12027 "child_workflow_run_id": "run-child",
12028 "child_workflow_type": "python.child",
12029 "failure_id": "failure-child",
12030 "failure_category": "cancelled",
12031 "message": "cancelled by parent-close policy",
12032 }),
12033 );
12034
12035 let commands = worker
12036 .execute_workflow_task(task)
12037 .expect("parent settlement");
12038 assert_eq!(commands.len(), 1);
12039 assert_eq!(commands[0]["type"], "fail_workflow");
12040 assert_eq!(commands[0]["exception_type"], "ChildWorkflowCancelled");
12041 assert_eq!(
12042 commands[0]["exception"]["properties"]["reason"],
12043 "cancelled"
12044 );
12045 assert_eq!(
12046 commands[0]["exception"]["properties"]["child_workflow_run_id"],
12047 "run-child"
12048 );
12049 }
12050
12051 #[test]
12052 fn workflow_can_handle_typed_child_failure() {
12053 let client = Client::new("http://127.0.0.1:8080").expect("client");
12054 let mut worker = Worker::new(client, "rust-parent-workers");
12055 worker.register_workflow("rust.handled-parent", |ctx, _input| async move {
12056 match ctx
12057 .start_child_workflow(
12058 "python.child",
12059 ChildWorkflowOptions::new("python-child-workers"),
12060 json!([]),
12061 )
12062 .await
12063 {
12064 Err(Error::ChildWorkflowFailed(failure)) => Ok(json!({
12065 "reason": failure.reason,
12066 "failure_id": failure.failure_id,
12067 "exception_class": failure.exception_class,
12068 "child_run_id": failure.child_workflow_run_id,
12069 })),
12070 Err(error) => Err(error),
12071 Ok(_) => Err(Error::WorkerLoop(
12072 "child unexpectedly succeeded".to_string(),
12073 )),
12074 }
12075 });
12076 let mut task = child_parent_task(
12077 "ChildRunFailed",
12078 json!({
12079 "sequence": 1,
12080 "child_workflow_instance_id": "wf-child",
12081 "child_workflow_run_id": "run-child",
12082 "child_workflow_type": "python.child",
12083 "failure_id": "failure-child",
12084 "failure_category": "child_workflow",
12085 "message": "payment rejected",
12086 "exception": {
12087 "type": "PaymentRejected",
12088 "class": "payments.PaymentRejected",
12089 "message": "payment rejected"
12090 }
12091 }),
12092 );
12093 task.workflow_type = "rust.handled-parent".to_string();
12094
12095 let commands = worker.execute_workflow_task(task).expect("handled failure");
12096 assert_eq!(commands[0]["type"], "complete_workflow");
12097 let output =
12098 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("parent output");
12099 assert_eq!(output["reason"], "child_workflow");
12100 assert_eq!(output["failure_id"], "failure-child");
12101 assert_eq!(output["exception_class"], "payments.PaymentRejected");
12102 assert_eq!(output["child_run_id"], "run-child");
12103 }
12104
12105 #[test]
12106 fn rust_hello_world_uses_signal_arguments_from_resume_payload() {
12107 let client = Client::new("http://127.0.0.1:8080").expect("client");
12108 let mut worker = Worker::new(client, "rust-workers");
12109
12110 worker.register_workflow("rust.hello_workflow", |ctx, _input| async move {
12111 let signal = ctx.wait_signal("start").await?;
12112 let name = signal
12113 .first()
12114 .and_then(|value| value.as_str())
12115 .unwrap_or("world");
12116 let greeting = ctx.activity("rust.hello_activity", json!([name])).await?;
12117 Ok(json!({
12118 "greeting": greeting,
12119 "language": "rust"
12120 }))
12121 });
12122
12123 let signal_arguments =
12124 encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC).expect("signal arguments");
12125 let task = WorkflowTask {
12126 task_id: "wft-rust-signal-1".to_string(),
12127 workflow_id: Some("wf-rust-hello".to_string()),
12128 run_id: Some("run-rust-hello".to_string()),
12129 workflow_type: "rust.hello_workflow".to_string(),
12130 payload_codec: DEFAULT_CODEC.to_string(),
12131 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
12132 history_events: vec![HistoryEvent {
12133 event_type: "SignalReceived".to_string(),
12134 payload: json!({
12135 "signal_id": "sig-rust-1",
12136 "signal_name": "start"
12137 }),
12138 raw: HashMap::new(),
12139 }],
12140 total_history_events: Some(1),
12141 history_size_bytes: None,
12142 continue_as_new_recommended: None,
12143 history_budget_pressure: None,
12144 next_history_page_token: None,
12145 workflow_task_attempt: 1,
12146 workflow_signal_id: Some("sig-rust-1".to_string()),
12147 signal_name: Some("start".to_string()),
12148 signal_arguments: Some(signal_arguments),
12149 workflow_update_id: None,
12150 update_name: None,
12151 lease_owner: Some("rust-worker".to_string()),
12152 };
12153
12154 let commands = worker.execute_workflow_task(task).expect("workflow task");
12155
12156 assert_eq!(commands.len(), 1);
12157 assert_eq!(commands[0]["type"], "schedule_activity");
12158 assert_eq!(commands[0]["activity_type"], "rust.hello_activity");
12159 assert_eq!(
12160 decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC).expect("activity args"),
12161 json!(["Rust"])
12162 );
12163 }
12164
12165 #[test]
12166 fn workflow_task_appends_paginated_history_events() {
12167 let mut task = WorkflowTask {
12168 task_id: "wft-rust-pages-1".to_string(),
12169 workflow_id: Some("wf-rust-pages".to_string()),
12170 run_id: Some("run-rust-pages".to_string()),
12171 workflow_type: "rust.hello_workflow".to_string(),
12172 payload_codec: DEFAULT_CODEC.to_string(),
12173 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
12174 history_events: vec![HistoryEvent {
12175 event_type: "WorkflowStarted".to_string(),
12176 payload: json!({}),
12177 raw: HashMap::new(),
12178 }],
12179 total_history_events: Some(3),
12180 history_size_bytes: None,
12181 continue_as_new_recommended: None,
12182 history_budget_pressure: None,
12183 next_history_page_token: Some("MQ==".to_string()),
12184 workflow_task_attempt: 1,
12185 workflow_signal_id: None,
12186 signal_name: None,
12187 signal_arguments: None,
12188 workflow_update_id: None,
12189 update_name: None,
12190 lease_owner: Some("rust-worker".to_string()),
12191 };
12192
12193 task.append_history_page(WorkflowTaskHistoryPage {
12194 history_events: vec![
12195 HistoryEvent {
12196 event_type: "SignalReceived".to_string(),
12197 payload: json!({
12198 "signal_id": "sig-rust-1",
12199 "signal_name": "start",
12200 "arguments": encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC)
12201 .expect("signal arguments")
12202 }),
12203 raw: HashMap::new(),
12204 },
12205 HistoryEvent {
12206 event_type: "MarkerRecorded".to_string(),
12207 payload: json!({"sequence": 3}),
12208 raw: HashMap::new(),
12209 },
12210 ],
12211 total_history_events: Some(3),
12212 next_history_page_token: None,
12213 });
12214
12215 assert_eq!(task.history_events.len(), 3);
12216 assert_eq!(task.total_history_events, Some(3));
12217 assert_eq!(task.next_history_page_token, None);
12218
12219 let signal = task
12220 .history_events
12221 .iter()
12222 .find(|event| event.event_type == "SignalReceived")
12223 .expect("signal event");
12224 assert_eq!(
12225 decode_signal_event_arguments(signal, DEFAULT_CODEC).expect("signal arguments"),
12226 vec![AvroValue::String("Rust".to_string())]
12227 );
12228 }
12229
12230 #[tokio::test]
12231 async fn query_handler_reads_ordered_cross_codec_signals_without_commands() {
12232 let client = Client::new("http://127.0.0.1:8080").expect("client");
12233 let mut worker = Worker::new(client, "rust-workers");
12234 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
12235 worker.register_query("counter", "current", |ctx, _args| async move {
12236 let mut count = 0_i64;
12237 for signal in ctx.signal_events() {
12238 let value = signal
12239 .arguments
12240 .first()
12241 .and_then(Value::as_i64)
12242 .unwrap_or_default();
12243 match signal.name.as_str() {
12244 "increment" => count += value,
12245 "set" => count = value,
12246 _ => {}
12247 }
12248 }
12249 Ok(json!(count))
12250 });
12251
12252 let task = QueryTask {
12253 query_task_id: "query-rust-counter".to_string(),
12254 query_task_attempt: 1,
12255 lease_owner: Some("rust-worker".to_string()),
12256 workflow_id: Some("counter-1".to_string()),
12257 run_id: Some("run-counter-1".to_string()),
12258 workflow_type: "counter".to_string(),
12259 query_name: "current".to_string(),
12260 payload_codec: DEFAULT_CODEC.to_string(),
12261 workflow_arguments: Some(
12262 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
12263 ),
12264 query_arguments: Some(
12265 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("query arguments"),
12266 ),
12267 history_events: vec![
12268 HistoryEvent {
12269 event_type: "SignalReceived".to_string(),
12270 payload: json!({
12271 "signal_id": "php-signal-1",
12272 "signal_name": "increment",
12273 "workflow_sequence": 1,
12274 "payload_codec": DEFAULT_CODEC,
12275 "arguments": encode_value_envelope(&json!([3]), DEFAULT_CODEC).expect("php avro signal")
12276 }),
12277 raw: HashMap::new(),
12278 },
12279 HistoryEvent {
12280 event_type: "SignalReceived".to_string(),
12281 payload: json!({
12282 "signal_id": "python-signal-2",
12283 "signal_name": "increment",
12284 "workflow_sequence": 2,
12285 "payload_codec": DEFAULT_CODEC,
12286 "arguments": encode_value_envelope(&json!([5]), DEFAULT_CODEC).expect("python avro signal")
12287 }),
12288 raw: HashMap::new(),
12289 },
12290 HistoryEvent {
12291 event_type: "SignalReceived".to_string(),
12292 payload: json!({
12293 "signal_id": "rust-signal-3",
12294 "signal_name": "set",
12295 "workflow_sequence": 3,
12296 "payload_codec": DEFAULT_CODEC,
12297 "arguments": encode_value_envelope(&json!([0]), DEFAULT_CODEC).expect("rust avro signal")
12298 }),
12299 raw: HashMap::new(),
12300 },
12301 ],
12302 history_export: None,
12303 run_status: Some("completed".to_string()),
12304 };
12305
12306 let result = worker.execute_query_task(task).await.expect("query result");
12307 assert_eq!(result.into_json().expect("query projection"), json!(0));
12308 }
12309
12310 #[tokio::test]
12311 async fn replayed_queries_read_running_completed_and_cold_restarted_instance_state() {
12312 let worker = replay_counter_worker();
12313 let running_history = json!([
12314 {
12315 "type": "ActivityCompleted",
12316 "payload": {
12317 "sequence": 1,
12318 "activity_type": "load-counter",
12319 "payload_codec": DEFAULT_CODEC,
12320 "result": fixture_envelope(json!("loaded"))
12321 }
12322 },
12323 {
12324 "type": "SignalWaitOpened",
12325 "payload": {
12326 "sequence": 3,
12327 "signal_name": "increment"
12328 }
12329 },
12330 {
12331 "type": "SignalReceived",
12332 "payload": {
12333 "signal_id": "signal-3",
12334 "signal_name": "increment",
12335 "workflow_sequence": 2,
12336 "payload_codec": DEFAULT_CODEC,
12337 "arguments": fixture_envelope(json!([3]))
12338 }
12339 },
12340 {
12341 "type": "SignalApplied",
12342 "payload": {
12343 "sequence": 3,
12344 "signal_id": "signal-3",
12345 "signal_name": "increment",
12346 "payload_codec": DEFAULT_CODEC,
12347 "value": fixture_envelope(json!([3]))
12348 }
12349 }
12350 ]);
12351
12352 let running = worker
12353 .execute_query_task(replay_counter_query(
12354 "current",
12355 running_history.clone(),
12356 "running",
12357 ))
12358 .await
12359 .expect("running replay query");
12360 assert_eq!(
12361 running.clone().into_json().expect("query projection"),
12362 json!({"loaded": "loaded", "count": 3, "finished": false})
12363 );
12364
12365 let detached = worker
12366 .execute_query_task(replay_counter_query(
12367 "detached-mutation",
12368 running_history.clone(),
12369 "running",
12370 ))
12371 .await
12372 .expect("query mutates only its detached state clone");
12373 assert_eq!(detached.into_json().expect("query projection"), json!(999));
12374 let failed = worker
12375 .execute_query_task(replay_counter_query(
12376 "failed-mutation",
12377 running_history.clone(),
12378 "running",
12379 ))
12380 .await
12381 .expect_err("failed query");
12382 assert_eq!(failed.reason, "query_rejected");
12383 let unchanged = worker
12384 .execute_query_task(replay_counter_query("current", running_history, "running"))
12385 .await
12386 .expect("later query reconstructs unchanged state");
12387 assert_eq!(unchanged, running);
12388
12389 let restarted_worker = replay_counter_worker();
12390 let empty_arguments = fixture_envelope(json!([]));
12391 let loaded_result = fixture_envelope(json!("loaded"));
12392 let signal_three = fixture_blob(json!([3]));
12393 let signal_five = fixture_blob(json!([5]));
12394 let restarted_task: QueryTask = serde_json::from_value(json!({
12395 "query_task_id": "query-after-restart",
12396 "workflow_id": "counter-1",
12397 "run_id": "run-counter-1",
12398 "workflow_type": "replay-counter",
12399 "query_name": "current",
12400 "payload_codec": DEFAULT_CODEC,
12401 "workflow_arguments": empty_arguments.clone(),
12402 "query_arguments": empty_arguments,
12403 "history_events": [],
12404 "history_export": {
12405 "payloads": {"codec": DEFAULT_CODEC},
12406 "history_events": [
12407 {
12408 "type": "ActivityCompleted",
12409 "payload": {
12410 "sequence": 1,
12411 "activity_type": "load-counter",
12412 "payload_codec": DEFAULT_CODEC,
12413 "result": null
12414 }
12415 },
12416 {
12417 "type": "SignalWaitOpened",
12418 "payload": {
12419 "sequence": 3,
12420 "signal_name": "increment"
12421 }
12422 },
12423 {
12424 "type": "SignalReceived",
12425 "payload": {
12426 "signal_id": "signal-3",
12427 "signal_name": "increment",
12428 "workflow_sequence": 2
12429 }
12430 },
12431 {
12432 "type": "SignalApplied",
12433 "payload": {
12434 "sequence": 3,
12435 "signal_id": "signal-3",
12436 "signal_name": "increment"
12437 }
12438 },
12439 {
12440 "type": "SignalWaitOpened",
12441 "payload": {
12442 "sequence": 5,
12443 "signal_name": "increment"
12444 }
12445 },
12446 {
12447 "type": "SignalReceived",
12448 "payload": {
12449 "signal_id": "signal-5",
12450 "signal_name": "increment",
12451 "workflow_sequence": 4
12452 }
12453 },
12454 {
12455 "type": "SignalApplied",
12456 "payload": {
12457 "sequence": 5,
12458 "signal_id": "signal-5",
12459 "signal_name": "increment"
12460 }
12461 }
12462 ],
12463 "activities": [{
12464 "sequence": 1,
12465 "activity_type": "load-counter",
12466 "payload_codec": DEFAULT_CODEC,
12467 "result": loaded_result
12468 }],
12469 "signals": [
12470 {
12471 "id": "signal-3",
12472 "name": "increment",
12473 "workflow_sequence": 2,
12474 "payload_codec": DEFAULT_CODEC,
12475 "arguments": signal_three
12476 },
12477 {
12478 "id": "signal-5",
12479 "name": "increment",
12480 "workflow_sequence": 4,
12481 "payload_codec": DEFAULT_CODEC,
12482 "arguments": signal_five
12483 }
12484 ]
12485 },
12486 "run_status": "completed"
12487 }))
12488 .expect("cold replay query task");
12489 let completed = restarted_worker
12490 .execute_query_task(restarted_task)
12491 .await
12492 .expect("completed cold replay query");
12493 assert_eq!(
12494 completed.into_json().expect("query projection"),
12495 json!({"loaded": "loaded", "count": 8, "finished": true})
12496 );
12497 }
12498
12499 #[tokio::test]
12500 async fn replayed_query_replay_failures_are_machine_readable() {
12501 let worker = replay_counter_worker();
12502 let task = replay_counter_query(
12503 "current",
12504 json!([{
12505 "type": "ActivityCompleted",
12506 "payload": {
12507 "sequence": 1,
12508 "payload_codec": DEFAULT_CODEC,
12509 "result": {"codec": DEFAULT_CODEC, "blob": "{"}
12510 }
12511 }]),
12512 "running",
12513 );
12514 let failure = worker
12515 .execute_query_task(task)
12516 .await
12517 .expect_err("invalid replay history payload");
12518 assert_eq!(failure.reason, "query_payload_decode_failed");
12519 assert_eq!(failure.failure_type, "QueryPayloadDecodeFailed");
12520 assert!(failure.message.contains("invalid_payload_framing"));
12521 }
12522
12523 #[tokio::test]
12524 async fn query_task_restores_compact_history_from_export() {
12525 let client = Client::new("http://127.0.0.1:8080").expect("client");
12526 let mut worker = Worker::new(client, "rust-workers");
12527 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
12528 worker.register_query("counter", "current", |ctx, _args| async move {
12529 Ok(json!(ctx.signals("increment")[0][0]))
12530 });
12531 let empty_arguments = fixture_envelope(json!([]));
12532 let exported_signal = fixture_blob(json!([9]));
12533 let task: QueryTask = serde_json::from_value(json!({
12534 "query_task_id": "query-export",
12535 "workflow_type": "counter",
12536 "query_name": "current",
12537 "payload_codec": DEFAULT_CODEC,
12538 "workflow_arguments": empty_arguments.clone(),
12539 "query_arguments": empty_arguments,
12540 "history_events": [],
12541 "history_export": {
12542 "payloads": {"codec": DEFAULT_CODEC},
12543 "history_events": [{
12544 "type": "SignalReceived",
12545 "payload": {"signal_id": "signal-export", "signal_name": "increment"}
12546 }],
12547 "signals": [{
12548 "id": "signal-export",
12549 "name": "increment",
12550 "status": "applied",
12551 "workflow_sequence": 1,
12552 "payload_codec": DEFAULT_CODEC,
12553 "arguments": exported_signal
12554 }]
12555 }
12556 }))
12557 .expect("query task");
12558
12559 let result = worker.execute_query_task(task).await.expect("query result");
12560 assert_eq!(result.into_json().expect("query projection"), json!(9));
12561 }
12562
12563 #[tokio::test]
12564 async fn query_task_failures_have_stable_reasons() {
12565 let client = Client::new("http://127.0.0.1:8080").expect("client");
12566 let mut worker = Worker::new(client, "rust-workers");
12567 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
12568 worker.register_query(
12569 "counter",
12570 "current",
12571 |_ctx, _args| async move { Ok(json!(0)) },
12572 );
12573
12574 let base_task = QueryTask {
12575 query_task_id: "query-errors".to_string(),
12576 query_task_attempt: 1,
12577 lease_owner: None,
12578 workflow_id: Some("counter-errors".to_string()),
12579 run_id: Some("run-errors".to_string()),
12580 workflow_type: "counter".to_string(),
12581 query_name: "missing".to_string(),
12582 payload_codec: DEFAULT_CODEC.to_string(),
12583 workflow_arguments: Some(fixture_envelope(json!([]))),
12584 query_arguments: Some(fixture_envelope(json!([]))),
12585 history_events: Vec::new(),
12586 history_export: None,
12587 run_status: Some("running".to_string()),
12588 };
12589
12590 let unknown = worker
12591 .execute_query_task(base_task.clone())
12592 .await
12593 .expect_err("unknown query");
12594 assert_eq!(unknown.reason, "rejected_unknown_query");
12595
12596 let mut malformed = base_task;
12597 malformed.query_name = "current".to_string();
12598 malformed.query_arguments = Some(json!({"codec": DEFAULT_CODEC, "blob": "{"}));
12599 let malformed = worker
12600 .execute_query_task(malformed)
12601 .await
12602 .expect_err("malformed payload");
12603 assert_eq!(malformed.reason, "query_payload_decode_failed");
12604
12605 let client = Client::new("http://127.0.0.1:8080").expect("client");
12606 let mut unavailable_worker = Worker::new(client, "rust-workers");
12607 unavailable_worker
12608 .register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
12609 let empty_arguments = fixture_envelope(json!([]));
12610 let unavailable_task: QueryTask = serde_json::from_value(json!({
12611 "query_task_id": "query-unavailable",
12612 "workflow_type": "counter",
12613 "query_name": "current",
12614 "payload_codec": DEFAULT_CODEC,
12615 "workflow_arguments": empty_arguments.clone(),
12616 "query_arguments": empty_arguments
12617 }))
12618 .expect("query task");
12619 let unavailable = unavailable_worker
12620 .execute_query_task(unavailable_task)
12621 .await
12622 .expect_err("query handler unavailable");
12623 assert_eq!(unavailable.reason, "query_handler_unavailable");
12624 }
12625
12626 #[tokio::test]
12627 async fn client_query_decodes_result_and_typed_failure() {
12628 let server = MockWorkerServer::start();
12629 let client = Client::builder(server.base_url())
12630 .timeout(Duration::from_secs(2))
12631 .build()
12632 .expect("client");
12633
12634 let result = client
12635 .query_workflow("counter-1", "current", json!([]))
12636 .await
12637 .expect("query result");
12638 assert_eq!(result, json!({"count": 8}));
12639
12640 let error = client
12641 .query_workflow("counter-1", "missing", json!([]))
12642 .await
12643 .expect_err("unknown query");
12644 let Error::QueryFailed(failure) = error else {
12645 panic!("expected typed query failure");
12646 };
12647 assert_eq!(failure.status, 404);
12648 assert_eq!(failure.reason, "rejected_unknown_query");
12649 }
12650
12651 #[tokio::test]
12652 async fn public_client_surfaces_send_and_receive_lossless_avro_values() {
12653 let server = MockWorkerServer::start();
12654 let client = Client::builder(server.base_url())
12655 .timeout(Duration::from_secs(2))
12656 .build()
12657 .expect("client");
12658 let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
12659
12660 client
12661 .start_workflow(
12662 "typed.echo",
12663 "rust-workers",
12664 "typed-start",
12665 arguments.clone(),
12666 )
12667 .await
12668 .expect("typed workflow start");
12669 assert_eq!(
12670 decode_wire_avro_value(
12671 &server.request_body("/api/workflows")["input"],
12672 DEFAULT_CODEC,
12673 )
12674 .expect("typed start input"),
12675 arguments
12676 );
12677
12678 client
12679 .signal_workflow("typed-1", "changed", arguments.clone())
12680 .await
12681 .expect("typed signal");
12682 assert_eq!(
12683 decode_wire_avro_value(
12684 &server.request_body("/api/workflows/typed-1/signal/changed")["input"],
12685 DEFAULT_CODEC,
12686 )
12687 .expect("typed signal input"),
12688 arguments
12689 );
12690
12691 assert_eq!(
12692 client
12693 .query_workflow_avro_value("typed-1", "inspect", arguments.clone())
12694 .await
12695 .expect("typed query"),
12696 typed_fidelity_probe()
12697 );
12698 assert_eq!(
12699 decode_wire_avro_value(
12700 &server.request_body("/api/workflows/typed-1/query/inspect")["input"],
12701 DEFAULT_CODEC,
12702 )
12703 .expect("typed query input"),
12704 arguments
12705 );
12706
12707 assert_eq!(
12708 client
12709 .update_workflow_avro_value(
12710 "typed-1",
12711 "replace",
12712 arguments.clone(),
12713 Some("typed-request"),
12714 )
12715 .await
12716 .expect("typed update"),
12717 typed_fidelity_probe()
12718 );
12719 let update = server.request_body("/api/workflows/typed-1/update/replace");
12720 assert_eq!(update["request_id"], "typed-request");
12721 assert_eq!(
12722 decode_wire_avro_value(&update["input"], DEFAULT_CODEC).expect("typed update input"),
12723 arguments
12724 );
12725
12726 let handle = WorkflowHandle {
12727 client: client.clone(),
12728 workflow_id: "typed-1".to_string(),
12729 run_id: Some("run-typed-1".to_string()),
12730 workflow_type: "typed.echo".to_string(),
12731 };
12732 assert_eq!(
12733 handle
12734 .result_avro_value(WorkflowResultOptions::default())
12735 .await
12736 .expect("typed workflow result"),
12737 typed_fidelity_probe()
12738 );
12739
12740 client
12741 .complete_activity_task(
12742 "activity-typed",
12743 "attempt-typed",
12744 "rust-worker",
12745 typed_fidelity_probe(),
12746 DEFAULT_CODEC,
12747 )
12748 .await
12749 .expect("typed activity completion");
12750 assert_eq!(
12751 decode_wire_avro_value(
12752 &server.request_body("/api/worker/activity-tasks/activity-typed/complete")
12753 ["result"],
12754 DEFAULT_CODEC,
12755 )
12756 .expect("typed activity result"),
12757 typed_fidelity_probe()
12758 );
12759 client
12760 .fail_activity_task(
12761 "activity-typed",
12762 "attempt-typed",
12763 "rust-worker",
12764 "typed failure",
12765 true,
12766 )
12767 .await
12768 .expect("activity failure");
12769 }
12770
12771 #[tokio::test]
12772 async fn lifecycle_commands_support_instance_and_selected_run_targets() {
12773 let server = MockWorkerServer::start();
12774 let client = Client::builder(server.base_url())
12775 .timeout(Duration::from_secs(2))
12776 .build()
12777 .expect("client");
12778
12779 let options = WorkflowCommandOptions::new()
12780 .reason("cleanup requested")
12781 .request_id("cancel-17");
12782 let cancelled = client
12783 .cancel_workflow("wf-lifecycle", options)
12784 .await
12785 .expect("instance cancellation");
12786 assert_eq!(cancelled.command, WorkflowCommandKind::Cancel);
12787 assert_eq!(cancelled.run_id.as_deref(), Some("run-current"));
12788 assert_eq!(cancelled.outcome.as_deref(), Some("cancelled"));
12789 assert_eq!(
12790 server.request_body("/api/workflows/wf-lifecycle/cancel"),
12791 json!({"reason":"cleanup requested","request_id":"cancel-17"})
12792 );
12793
12794 let terminated = client
12795 .terminate_workflow(
12796 "wf-lifecycle",
12797 WorkflowCommandOptions::new().reason("forced stop"),
12798 )
12799 .await
12800 .expect("instance termination");
12801 assert_eq!(terminated.command, WorkflowCommandKind::Terminate);
12802 assert_eq!(terminated.outcome.as_deref(), Some("terminated"));
12803
12804 client
12805 .cancel_workflow_run(
12806 "wf-lifecycle",
12807 "run-current",
12808 WorkflowCommandOptions::default(),
12809 )
12810 .await
12811 .expect("selected run cancellation");
12812 client
12813 .terminate_workflow_run(
12814 "wf-lifecycle",
12815 "run-current",
12816 WorkflowCommandOptions::default(),
12817 )
12818 .await
12819 .expect("selected run termination");
12820
12821 for (command, error) in [
12822 (
12823 WorkflowCommandKind::Cancel,
12824 client
12825 .cancel_workflow_run(
12826 "wf-lifecycle",
12827 "run-stale",
12828 WorkflowCommandOptions::default(),
12829 )
12830 .await
12831 .expect_err("stale cancellation must be rejected"),
12832 ),
12833 (
12834 WorkflowCommandKind::Terminate,
12835 client
12836 .terminate_workflow_run(
12837 "wf-lifecycle",
12838 "run-stale",
12839 WorkflowCommandOptions::default(),
12840 )
12841 .await
12842 .expect_err("stale termination must be rejected"),
12843 ),
12844 ] {
12845 let Error::WorkflowCommandRejected(rejection) = error else {
12846 panic!("expected typed command rejection");
12847 };
12848 assert_eq!(rejection.command, command);
12849 assert_eq!(rejection.status, 409);
12850 assert_eq!(rejection.reason, "historical_run_command_rejected");
12851 assert_eq!(rejection.run_id.as_deref(), Some("run-stale"));
12852 assert_eq!(rejection.target_scope.as_deref(), Some("run"));
12853 }
12854 }
12855
12856 #[tokio::test]
12857 async fn workflow_start_options_send_server_enforced_deadlines() {
12858 let server = MockWorkerServer::start();
12859 let client = Client::builder(server.base_url())
12860 .timeout(Duration::from_secs(2))
12861 .build()
12862 .expect("client");
12863
12864 let handle = client
12865 .start_workflow_with_options(
12866 "rust.timeout",
12867 "rust-timeouts",
12868 "wf-start-options",
12869 WorkflowStartOptions::new()
12870 .execution_timeout_seconds(30)
12871 .run_timeout_seconds(1),
12872 json!([]),
12873 )
12874 .await
12875 .expect("workflow start");
12876
12877 assert_eq!(handle.run_id.as_deref(), Some("run-start-options"));
12878 let body = server.request_body("/api/workflows");
12879 assert_eq!(body["execution_timeout_seconds"], 30);
12880 assert_eq!(body["run_timeout_seconds"], 1);
12881
12882 let invalid = client
12883 .start_workflow_with_options(
12884 "rust.timeout",
12885 "rust-timeouts",
12886 "wf-invalid-options",
12887 WorkflowStartOptions::new()
12888 .execution_timeout_seconds(1)
12889 .run_timeout_seconds(2),
12890 json!([]),
12891 )
12892 .await
12893 .expect_err("invalid deadline ordering");
12894 assert!(invalid
12895 .to_string()
12896 .contains("run_timeout_seconds cannot exceed execution_timeout_seconds"));
12897 }
12898
12899 #[tokio::test]
12900 async fn workflow_result_returns_each_typed_terminal_outcome() {
12901 let server = MockWorkerServer::start();
12902 let client = Client::builder(server.base_url())
12903 .timeout(Duration::from_secs(2))
12904 .build()
12905 .expect("client");
12906 let options = WorkflowResultOptions {
12907 poll_interval: Duration::ZERO,
12908 timeout: Duration::from_secs(1),
12909 };
12910
12911 let failed = WorkflowHandle {
12912 client: client.clone(),
12913 workflow_id: "wf-failed".to_string(),
12914 run_id: Some("run-failed".to_string()),
12915 workflow_type: "failure".to_string(),
12916 }
12917 .result(options)
12918 .await
12919 .expect_err("failed outcome");
12920 let Error::WorkflowFailed(failure) = failed else {
12921 panic!("expected WorkflowFailed");
12922 };
12923 assert_eq!(failure.workflow_id, "wf-failed");
12924 assert_eq!(failure.run_id.as_deref(), Some("run-failed"));
12925 assert_eq!(failure.failure_id.as_deref(), Some("failure-17"));
12926 assert_eq!(failure.failure_category.as_deref(), Some("application"));
12927 assert_eq!(failure.exception_type.as_deref(), Some("PaymentError"));
12928 assert_eq!(
12929 failure.exception_class.as_deref(),
12930 Some("billing::PaymentError")
12931 );
12932 assert_eq!(failure.non_retryable, Some(true));
12933
12934 for (workflow_id, expected_kind, expected_reason) in [
12935 (
12936 "wf-cancelled",
12937 WorkflowTerminalKind::Cancelled,
12938 "cleanup requested",
12939 ),
12940 (
12941 "wf-terminated",
12942 WorkflowTerminalKind::Terminated,
12943 "forced stop",
12944 ),
12945 (
12946 "wf-timed-out",
12947 WorkflowTerminalKind::TimedOut,
12948 "run_timeout",
12949 ),
12950 ] {
12951 let error = WorkflowHandle {
12952 client: client.clone(),
12953 workflow_id: workflow_id.to_string(),
12954 run_id: None,
12955 workflow_type: "terminal".to_string(),
12956 }
12957 .result(options)
12958 .await
12959 .expect_err("typed terminal outcome");
12960 let outcome = match error {
12961 Error::WorkflowCancelled(outcome) => outcome,
12962 Error::WorkflowTerminated(outcome) => outcome,
12963 Error::WorkflowTimedOut(outcome) => outcome,
12964 other => panic!("unexpected terminal error: {other}"),
12965 };
12966 assert_eq!(outcome.kind, expected_kind);
12967 assert_eq!(outcome.workflow_id, workflow_id);
12968 assert_eq!(outcome.reason, expected_reason);
12969 }
12970
12971 let wait_timeout = WorkflowHandle {
12972 client,
12973 workflow_id: "wf-waiting".to_string(),
12974 run_id: Some("run-waiting".to_string()),
12975 workflow_type: "waiting".to_string(),
12976 }
12977 .result(WorkflowResultOptions {
12978 poll_interval: Duration::ZERO,
12979 timeout: Duration::ZERO,
12980 })
12981 .await
12982 .expect_err("client wait timeout");
12983 let Error::WorkflowTimedOut(timeout) = wait_timeout else {
12984 panic!("expected typed client timeout");
12985 };
12986 assert_eq!(timeout.reason, "result_wait_timeout");
12987 assert_eq!(timeout.failure_category.as_deref(), Some("client_timeout"));
12988 assert_eq!(timeout.run_id.as_deref(), Some("run-waiting"));
12989 }
12990
12991 #[tokio::test]
12992 async fn workflow_result_follows_chain_and_selected_result_preserves_history() {
12993 let server = MockWorkerServer::start();
12994 let client = Client::builder(server.base_url())
12995 .timeout(Duration::from_secs(2))
12996 .build()
12997 .expect("client");
12998
12999 let handle = WorkflowHandle {
13000 client,
13001 workflow_id: "wf-selected".to_string(),
13002 run_id: Some("run-selected".to_string()),
13003 workflow_type: "selected".to_string(),
13004 };
13005 let options = WorkflowResultOptions {
13006 poll_interval: Duration::ZERO,
13007 timeout: Duration::from_secs(1),
13008 };
13009
13010 let current = handle
13011 .result(options)
13012 .await
13013 .expect("instance result follows the current run");
13014 assert_eq!(current, json!("current run output"));
13015
13016 let error = handle
13017 .result_selected_run(options)
13018 .await
13019 .expect_err("the selected run is cancelled even though the current run completed");
13020
13021 let Error::WorkflowCancelled(outcome) = error else {
13022 panic!("expected selected run cancellation");
13023 };
13024 assert_eq!(outcome.run_id.as_deref(), Some("run-selected"));
13025 assert_eq!(outcome.reason, "selected run cancelled");
13026 assert_eq!(
13027 server.request_count("/api/workflows/wf-selected/runs/run-selected"),
13028 1
13029 );
13030 assert_eq!(server.request_count("/api/workflows/wf-selected"), 1);
13031 }
13032
13033 #[tokio::test]
13034 async fn poll_responses_decode_http_conflict_drain_as_a_stable_stop() {
13035 let server = MockWorkerServer::draining_polls();
13036 let client = Client::builder(server.base_url())
13037 .timeout(Duration::from_secs(2))
13038 .build()
13039 .expect("client");
13040
13041 let workflow = client
13042 .poll_workflow_task_response("draining-worker", "rust-workers", Duration::ZERO)
13043 .await
13044 .expect("workflow drain response");
13045 let activity = client
13046 .poll_activity_task_response("draining-worker", "rust-workers", Duration::ZERO)
13047 .await
13048 .expect("activity drain response");
13049 let query = client
13050 .poll_query_task_response("draining-worker", "rust-workers", Duration::ZERO)
13051 .await
13052 .expect("query drain response");
13053
13054 for outcome in [workflow.outcome(), activity.outcome(), query.outcome()] {
13055 assert_eq!(
13056 outcome,
13057 WorkerPollOutcome::Stop {
13058 poll_status: Some("draining".to_string()),
13059 reason: Some("worker_draining".to_string()),
13060 }
13061 );
13062 }
13063
13064 assert!(client
13065 .poll_workflow_task("draining-worker", "rust-workers", Duration::ZERO)
13066 .await
13067 .expect("compatibility poll")
13068 .is_none());
13069 }
13070
13071 #[tokio::test]
13072 async fn managed_worker_honors_drain_stop_for_every_task_family() {
13073 let server = MockWorkerServer::draining_polls();
13074 let client = Client::builder(server.base_url())
13075 .timeout(Duration::from_secs(2))
13076 .build()
13077 .expect("client");
13078
13079 let mut workflow_worker = Worker::new(client.clone(), "rust-workers")
13080 .worker_id("draining-workflow-worker")
13081 .poll_timeout(Duration::ZERO);
13082 workflow_worker.register_workflow("counter", |_ctx, _args| async { Ok(Value::Null) });
13083 workflow_worker
13084 .run()
13085 .await
13086 .expect("workflow drain is a clean stop");
13087
13088 let mut activity_worker = Worker::new(client.clone(), "rust-workers")
13089 .worker_id("draining-activity-worker")
13090 .poll_timeout(Duration::ZERO);
13091 activity_worker.register_activity("write", |_ctx, _args| async { Ok(Value::Null) });
13092 activity_worker
13093 .run()
13094 .await
13095 .expect("activity drain is a clean stop");
13096
13097 let mut query_worker = Worker::new(client, "rust-workers")
13098 .worker_id("draining-query-worker")
13099 .poll_timeout(Duration::ZERO);
13100 query_worker.register_query("counter", "current", |_ctx, _args| async {
13101 Ok(Value::Null)
13102 });
13103 query_worker
13104 .run()
13105 .await
13106 .expect("query drain is a clean stop");
13107 }
13108
13109 #[tokio::test]
13110 async fn activity_cancellation_and_late_completion_remain_machine_readable() {
13111 let server = MockWorkerServer::start();
13112 let client = Client::builder(server.base_url())
13113 .timeout(Duration::from_secs(2))
13114 .build()
13115 .expect("client");
13116
13117 let heartbeat = client
13118 .heartbeat_activity_task(
13119 "activity-cancel",
13120 "attempt-cancel",
13121 "rust-worker",
13122 typed_fidelity_probe(),
13123 )
13124 .await
13125 .expect("cancellation heartbeat");
13126 assert!(heartbeat.cancel_requested);
13127 assert!(heartbeat.should_stop());
13128 assert_eq!(heartbeat.reason.as_deref(), Some("run_cancelled"));
13129 assert_eq!(heartbeat.run_closed_reason.as_deref(), Some("cancelled"));
13130 let heartbeat_body =
13131 server.request_body("/api/worker/activity-tasks/activity-cancel/heartbeat");
13132 assert_eq!(heartbeat_body["details"]["codec"], DEFAULT_CODEC);
13133 assert_eq!(
13134 decode_wire_avro_value(&heartbeat_body["details"], DEFAULT_CODEC)
13135 .expect("typed heartbeat details"),
13136 typed_fidelity_probe()
13137 );
13138
13139 let error = client
13140 .complete_activity_task(
13141 "activity-cancel",
13142 "attempt-cancel",
13143 "rust-worker",
13144 json!({"late":true}),
13145 DEFAULT_CODEC,
13146 )
13147 .await
13148 .expect_err("late completion must be refused");
13149 assert!(activity_task_rejection_is_final(&error));
13150 let Error::ActivityTaskRejected(rejection) = error else {
13151 panic!("expected typed activity rejection");
13152 };
13153 assert_eq!(rejection.status, 409);
13154 assert_eq!(rejection.reason, "run_cancelled");
13155 assert!(rejection.cancel_requested);
13156 assert_eq!(rejection.can_continue, Some(false));
13157 }
13158
13159 #[tokio::test]
13160 async fn managed_worker_survives_late_completion_and_restart_during_cancellation() {
13161 let server = MockWorkerServer::cancelled_activity();
13162 let client = Client::builder(server.base_url())
13163 .timeout(Duration::from_secs(2))
13164 .build()
13165 .expect("client");
13166 let cancellation_observed = Arc::new(AtomicBool::new(false));
13167 let observed = Arc::clone(&cancellation_observed);
13168 let mut worker = Worker::new(client.clone(), "rust-workers")
13169 .worker_id("rust-cancel-worker")
13170 .poll_timeout(Duration::from_millis(10));
13171 worker.register_activity("cancel-aware", move |ctx, _args| {
13172 let observed = Arc::clone(&observed);
13173 async move {
13174 let heartbeat = ctx.heartbeat(json!({"stage":"running"})).await?;
13175 observed.store(heartbeat.should_stop(), Ordering::SeqCst);
13176 Ok(json!({"late":"completion"}))
13177 }
13178 });
13179
13180 assert_eq!(
13181 worker.run_once().await.expect("cancelled attempt handled"),
13182 1
13183 );
13184 assert!(cancellation_observed.load(Ordering::SeqCst));
13185 assert_eq!(
13186 server.request_count("/api/worker/activity-tasks/activity-cancel/complete"),
13187 1
13188 );
13189
13190 let mut restarted = Worker::new(client, "rust-workers")
13191 .worker_id("rust-cancel-worker-restarted")
13192 .poll_timeout(Duration::from_millis(10));
13193 restarted.register_activity("cancel-aware", |_ctx, _args| async move { Ok(Value::Null) });
13194 assert_eq!(
13195 restarted
13196 .run_once()
13197 .await
13198 .expect("replacement worker continues polling"),
13199 0
13200 );
13201 }
13202
13203 #[tokio::test]
13204 async fn managed_worker_absorbs_selected_run_terminal_timeout_completion_race() {
13205 let response = r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"outcome":"completed","recorded":false,"run_id":"run-selected-timeout","run_status":"failed","created_task_ids":[],"reason":"run_timed_out"}"#;
13206 let server = MockWorkerServer::workflow_completion("409 Conflict", response);
13207 let client = Client::builder(server.base_url())
13208 .timeout(Duration::from_secs(2))
13209 .build()
13210 .expect("client");
13211
13212 let direct_error = client
13213 .complete_workflow_task(
13214 "workflow-timeout-task",
13215 "timeout-worker",
13216 3,
13217 vec![json!({
13218 "type": "complete_workflow",
13219 "result": fixture_envelope(Value::Null)
13220 })],
13221 )
13222 .await
13223 .expect_err("the low-level client preserves the completion rejection");
13224 let Error::Http { status, body } = direct_error else {
13225 panic!("expected the original HTTP completion rejection");
13226 };
13227 assert_eq!(status, reqwest::StatusCode::CONFLICT);
13228 assert_eq!(
13229 serde_json::from_str::<Value>(&body).expect("response body")["reason"],
13230 "run_timed_out"
13231 );
13232
13233 let mut worker = Worker::new(client, "rust-workers")
13234 .worker_id("timeout-worker")
13235 .poll_timeout(Duration::from_millis(10));
13236 worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
13237 Ok(json!({"late": "result"}))
13238 });
13239
13240 assert_eq!(
13241 worker
13242 .run_once()
13243 .await
13244 .expect("authoritative selected-run timeout settles the tick"),
13245 1
13246 );
13247 assert_eq!(
13248 server.request_count("/api/worker/workflow-tasks/workflow-timeout-task/complete"),
13249 2,
13250 "both the direct client proof and managed worker must see the rejection"
13251 );
13252 }
13253
13254 #[tokio::test]
13255 async fn managed_worker_does_not_swallow_nearby_completion_errors() {
13256 for (name, status, response) in [
13257 ("bare conflict", "409 Conflict", r#"{"message":"conflict"}"#),
13258 (
13259 "command was recorded",
13260 "409 Conflict",
13261 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":true,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
13262 ),
13263 (
13264 "lease conflict",
13265 "409 Conflict",
13266 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"lease_expired"}"#,
13267 ),
13268 (
13269 "nonterminal run",
13270 "409 Conflict",
13271 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"waiting","reason":"run_timed_out"}"#,
13272 ),
13273 (
13274 "different selected run",
13275 "409 Conflict",
13276 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-reused-workflow-current","run_status":"failed","reason":"run_timed_out"}"#,
13277 ),
13278 (
13279 "different task attempt",
13280 "409 Conflict",
13281 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":4,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
13282 ),
13283 (
13284 "authentication failure",
13285 "401 Unauthorized",
13286 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
13287 ),
13288 (
13289 "authorization failure",
13290 "403 Forbidden",
13291 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
13292 ),
13293 (
13294 "protocol failure",
13295 "400 Bad Request",
13296 r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.2","requested_version":"1.3"}"#,
13297 ),
13298 (
13299 "malformed command",
13300 "422 Unprocessable Entity",
13301 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
13302 ),
13303 (
13304 "transient server failure",
13305 "503 Service Unavailable",
13306 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
13307 ),
13308 ] {
13309 let server = MockWorkerServer::workflow_completion(status, response);
13310 let client = Client::builder(server.base_url())
13311 .timeout(Duration::from_secs(2))
13312 .build()
13313 .expect("client");
13314 let mut worker = Worker::new(client, "rust-workers")
13315 .worker_id("timeout-worker")
13316 .poll_timeout(Duration::from_millis(10));
13317 worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
13318 Ok(json!({"late": "result"}))
13319 });
13320
13321 let error = worker
13322 .run_once()
13323 .await
13324 .expect_err(&format!("{name} must remain an error"));
13325 assert!(
13326 matches!(error, Error::Http { .. } | Error::Protocol(_)),
13327 "{name} returned an unexpected error variant: {error}"
13328 );
13329 }
13330 }
13331
13332 #[tokio::test]
13333 async fn worker_deregistration_uses_worker_plane_method_path_headers_and_result() {
13334 let server = MockWorkerServer::start();
13335 let client = Client::builder(server.base_url())
13336 .worker_token(Some("worker-secret".to_string()))
13337 .namespace("orders")
13338 .timeout(Duration::from_secs(2))
13339 .build()
13340 .expect("client");
13341 let path = "/api/worker/registrations/worker%2F%CE%B1%20space";
13342
13343 let result = client
13344 .deregister_worker_registration("worker/α space")
13345 .await
13346 .expect("deregister worker registration");
13347
13348 assert_eq!(server.method_for(path).as_deref(), Some("DELETE"));
13349 assert_eq!(
13350 server.worker_protocol_for(path).as_deref(),
13351 Some(WORKER_PROTOCOL_VERSION)
13352 );
13353 assert_eq!(server.control_protocol_for(path), None);
13354 assert_eq!(server.namespace_for(path).as_deref(), Some("orders"));
13355 assert_eq!(
13356 server.authorization_for(path).as_deref(),
13357 Some("Bearer worker-secret")
13358 );
13359 assert_eq!(
13360 result,
13361 WorkerDeregistrationEnvelope {
13362 worker_id: "deregistered-worker".to_string(),
13363 outcome: "deregistered".to_string(),
13364 recovered_workflow_task_count: 2,
13365 }
13366 );
13367 }
13368
13369 #[tokio::test]
13370 async fn low_level_registration_rejects_update_validators_before_transport() {
13371 let server = MockWorkerServer::start();
13372 let client = Client::builder(server.base_url())
13373 .timeout(Duration::from_secs(2))
13374 .build()
13375 .expect("client");
13376
13377 for update_validators in [json!(["approve"]), json!("approve")] {
13378 let error = client
13379 .register_worker_with_command_contracts(
13380 "validator-claiming-worker",
13381 "rust-workers",
13382 vec!["orders".to_string()],
13383 vec![],
13384 1,
13385 1,
13386 vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
13387 json!({
13388 "orders": {
13389 "queries": ["current"],
13390 "updates": ["approve"],
13391 "update_validators": update_validators,
13392 },
13393 }),
13394 )
13395 .await
13396 .expect_err("unsupported validator claims must fail before registration");
13397
13398 let Error::UnsupportedUpdateValidators { workflow_type } = error else {
13399 panic!("expected typed unsupported-validator failure");
13400 };
13401 assert_eq!(workflow_type, "orders");
13402 }
13403 assert_eq!(server.request_count("/api/worker/register"), 0);
13404 }
13405
13406 #[tokio::test]
13407 async fn low_level_registration_preserves_query_and_update_contracts() {
13408 let server = MockWorkerServer::start();
13409 let client = Client::builder(server.base_url())
13410 .timeout(Duration::from_secs(2))
13411 .build()
13412 .expect("client");
13413 let contracts = json!({
13414 "orders": {
13415 "queries": ["current"],
13416 "updates": ["approve"],
13417 "update_validators": [],
13418 },
13419 "payments": {
13420 "queries": ["status"],
13421 "updates": ["capture"],
13422 },
13423 });
13424
13425 client
13426 .register_worker_with_command_contracts(
13427 "command-worker",
13428 "rust-workers",
13429 vec!["orders".to_string(), "payments".to_string()],
13430 vec![],
13431 1,
13432 1,
13433 vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
13434 contracts.clone(),
13435 )
13436 .await
13437 .expect("query and update contracts must remain supported");
13438
13439 assert_eq!(
13440 server.request_body("/api/worker/register")["workflow_command_contracts"],
13441 contracts
13442 );
13443 }
13444
13445 #[tokio::test]
13446 async fn role_scoped_tokens_are_never_used_for_the_opposite_plane() {
13447 let server = MockWorkerServer::start();
13448 let control_only = Client::builder(server.base_url())
13449 .control_token(Some("control-secret".to_string()))
13450 .build()
13451 .expect("control client");
13452
13453 let error = control_only
13454 .register_worker("worker", "queue", vec![], vec![], 1, 1)
13455 .await
13456 .expect_err("control token must not authorize a worker request");
13457 assert!(matches!(
13458 error,
13459 Error::MissingRoleCredentials { role: "worker", .. }
13460 ));
13461 assert_eq!(server.request_count("/api/worker/register"), 0);
13462
13463 let worker_only = Client::builder(server.base_url())
13464 .worker_token(Some("worker-secret".to_string()))
13465 .build()
13466 .expect("worker client");
13467 let error = worker_only
13468 .health()
13469 .await
13470 .expect_err("worker token must not authorize a control request");
13471 assert!(matches!(
13472 error,
13473 Error::MissingRoleCredentials {
13474 role: "control",
13475 ..
13476 }
13477 ));
13478 assert_eq!(server.request_count("/api/health"), 0);
13479 }
13480
13481 #[tokio::test]
13482 async fn shared_token_supports_worker_and_control_planes() {
13483 let server = MockWorkerServer::start();
13484 let client = Client::builder(server.base_url())
13485 .token(Some("shared-secret".to_string()))
13486 .build()
13487 .expect("client");
13488
13489 client.health().await.expect("control request");
13490 client
13491 .register_worker("worker", "queue", vec![], vec![], 1, 1)
13492 .await
13493 .expect("worker request");
13494
13495 assert_eq!(
13496 server.authorization_for("/api/health").as_deref(),
13497 Some("Bearer shared-secret")
13498 );
13499 assert_eq!(
13500 server.control_protocol_for("/api/health").as_deref(),
13501 Some(CONTROL_PLANE_VERSION)
13502 );
13503 assert_eq!(
13504 server.authorization_for("/api/worker/register").as_deref(),
13505 Some("Bearer shared-secret")
13506 );
13507 assert_eq!(
13508 server
13509 .worker_protocol_for("/api/worker/register")
13510 .as_deref(),
13511 Some(WORKER_PROTOCOL_VERSION)
13512 );
13513 }
13514
13515 #[tokio::test]
13516 async fn baseline_worker_endpoints_send_the_baseline_protocol() {
13517 let server = MockWorkerServer::start();
13518 let client = Client::builder(server.base_url())
13519 .timeout(Duration::from_secs(2))
13520 .build()
13521 .expect("client");
13522
13523 client
13524 .register_worker("capture-worker", "capture", vec![], vec![], 1, 1)
13525 .await
13526 .expect("register");
13527 client
13528 .heartbeat_worker("capture-worker", 1, 1)
13529 .await
13530 .expect("heartbeat");
13531 client
13532 .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
13533 .await
13534 .expect("workflow poll");
13535 client
13536 .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
13537 .await
13538 .expect("activity poll");
13539
13540 for path in [
13541 "/api/worker/register",
13542 "/api/worker/heartbeat",
13543 "/api/worker/workflow-tasks/poll",
13544 "/api/worker/activity-tasks/poll",
13545 ] {
13546 assert_eq!(
13547 server.worker_protocol_for(path).as_deref(),
13548 Some(WORKER_PROTOCOL_VERSION),
13549 "unexpected protocol for {path}"
13550 );
13551 }
13552
13553 assert_eq!(
13554 server.request_body("/api/worker/workflow-tasks/poll")["timeout_seconds"],
13555 1
13556 );
13557 assert_eq!(
13558 server.request_body("/api/worker/activity-tasks/poll")["timeout_seconds"],
13559 1
13560 );
13561 assert!(
13562 server.request_body("/api/worker/workflow-tasks/poll")["poll_request_id"]
13563 .as_str()
13564 .is_some_and(|id| id.starts_with("rust-workflow-poll-"))
13565 );
13566 assert!(
13567 server.request_body("/api/worker/activity-tasks/poll")["poll_request_id"]
13568 .as_str()
13569 .is_some_and(|id| id.starts_with("rust-activity-poll-"))
13570 );
13571 }
13572
13573 #[tokio::test]
13574 async fn query_task_endpoints_send_the_query_feature_protocol() {
13575 let server = MockWorkerServer::start();
13576 let client = Client::builder(server.base_url())
13577 .timeout(Duration::from_secs(2))
13578 .build()
13579 .expect("client");
13580
13581 client
13582 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
13583 .await
13584 .expect("query poll");
13585 client
13586 .complete_query_task(
13587 "query-capture",
13588 "capture-worker",
13589 1,
13590 json!(8),
13591 DEFAULT_CODEC,
13592 )
13593 .await
13594 .expect("query complete");
13595 client
13596 .fail_query_task(
13597 "query-capture",
13598 "capture-worker",
13599 1,
13600 "failed",
13601 "query_rejected",
13602 "QueryFailed",
13603 )
13604 .await
13605 .expect("query fail");
13606
13607 for path in [
13608 "/api/worker/query-tasks/poll",
13609 "/api/worker/query-tasks/query-capture/complete",
13610 "/api/worker/query-tasks/query-capture/fail",
13611 ] {
13612 assert_eq!(
13613 server.worker_protocol_for(path).as_deref(),
13614 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
13615 "unexpected protocol for {path}"
13616 );
13617 }
13618
13619 assert_eq!(
13620 server.request_body("/api/worker/query-tasks/poll")["timeout_seconds"],
13621 1
13622 );
13623 assert!(
13624 server.request_body("/api/worker/query-tasks/poll")["poll_request_id"]
13625 .as_str()
13626 .is_some_and(|id| id.starts_with("rust-query-poll-"))
13627 );
13628 }
13629
13630 #[tokio::test]
13631 async fn disconnected_client_polls_retry_once_with_the_same_request_id() {
13632 let server = MockWorkerServer::transient_worker_failures();
13633 let client = Client::builder(server.base_url())
13634 .timeout(Duration::from_secs(2))
13635 .build()
13636 .expect("client");
13637
13638 client
13639 .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
13640 .await
13641 .expect("workflow poll retry");
13642 client
13643 .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
13644 .await
13645 .expect("activity poll retry");
13646 client
13647 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
13648 .await
13649 .expect("query poll retry");
13650
13651 for path in [
13652 "/api/worker/workflow-tasks/poll",
13653 "/api/worker/activity-tasks/poll",
13654 "/api/worker/query-tasks/poll",
13655 ] {
13656 let bodies = server.request_bodies(path);
13657 assert_eq!(bodies.len(), 2, "{path} must be retried once");
13658 assert_eq!(
13659 bodies[0]["poll_request_id"], bodies[1]["poll_request_id"],
13660 "{path} must preserve the request binding across retry"
13661 );
13662 }
13663 }
13664
13665 #[tokio::test]
13666 async fn worker_poll_retries_preserve_request_id_across_consecutive_failures() {
13667 let server = MockWorkerServer::consecutive_poll_failures(2);
13668 let client = Client::builder(server.base_url())
13669 .timeout(Duration::from_secs(2))
13670 .build()
13671 .expect("client");
13672 let mut worker = Worker::new(client, "capture")
13673 .worker_id("capture-worker")
13674 .poll_timeout(Duration::from_millis(10))
13675 .retry_policy(WorkerRetryPolicy {
13676 max_retries: 2,
13677 initial_backoff: Duration::from_millis(1),
13678 max_backoff: Duration::from_millis(1),
13679 });
13680 worker.register_workflow(
13681 "capture.workflow",
13682 |_ctx, _input| async move { Ok(Value::Null) },
13683 );
13684 worker.register_activity(
13685 "capture.activity",
13686 |_ctx, _input| async move { Ok(Value::Null) },
13687 );
13688 worker.register_query("capture.workflow", "current", |_ctx, _args| async move {
13689 Ok(Value::Null)
13690 });
13691
13692 assert_eq!(worker.run_once().await.expect("poll retries"), 0);
13693
13694 for path in [
13695 "/api/worker/workflow-tasks/poll",
13696 "/api/worker/activity-tasks/poll",
13697 "/api/worker/query-tasks/poll",
13698 ] {
13699 let bodies = server.request_bodies(path);
13700 assert_eq!(bodies.len(), 3, "{path} must use exactly two retries");
13701 assert!(
13702 bodies
13703 .iter()
13704 .all(|body| body["poll_request_id"] == bodies[0]["poll_request_id"]),
13705 "{path} must preserve one request binding across every retry"
13706 );
13707 }
13708 }
13709
13710 #[tokio::test]
13711 async fn query_protocol_rejection_from_older_server_is_typed() {
13712 let server = MockWorkerServer::reject_query_protocol();
13713 let client = Client::builder(server.base_url())
13714 .timeout(Duration::from_secs(2))
13715 .build()
13716 .expect("client");
13717
13718 let error = client
13719 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
13720 .await
13721 .expect_err("server below query protocol floor must reject");
13722 let Error::Protocol(failure) = error else {
13723 panic!("expected typed protocol failure");
13724 };
13725
13726 assert_eq!(failure.status, 400);
13727 assert_eq!(failure.reason, "unsupported_protocol_version");
13728 assert_eq!(failure.supported_version.as_deref(), Some("1.7"));
13729 assert_eq!(
13730 failure.requested_version.as_deref(),
13731 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
13732 );
13733 assert_eq!(
13734 server
13735 .worker_protocol_for("/api/worker/query-tasks/poll")
13736 .as_deref(),
13737 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
13738 );
13739 }
13740
13741 #[tokio::test]
13742 async fn run_once_without_query_handlers_keeps_pre_query_server_compatibility() {
13743 let server = MockWorkerServer::reject_query_protocol();
13744 let client = Client::builder(server.base_url())
13745 .timeout(Duration::from_secs(2))
13746 .build()
13747 .expect("client");
13748 let mut worker = Worker::new(client, "rust-workers")
13749 .worker_id("baseline-worker")
13750 .poll_timeout(Duration::from_millis(10));
13751
13752 worker.register_workflow("baseline.workflow", |_ctx, _input| async move {
13753 Ok(Value::Null)
13754 });
13755
13756 assert_eq!(worker.run_once().await.expect("baseline run once"), 0);
13757 assert_eq!(
13758 server
13759 .worker_protocol_for("/api/worker/workflow-tasks/poll")
13760 .as_deref(),
13761 Some(WORKER_PROTOCOL_VERSION)
13762 );
13763 assert_eq!(
13764 server.worker_protocol_for("/api/worker/query-tasks/poll"),
13765 None,
13766 "a worker without query handlers must not use the query-task endpoint"
13767 );
13768 }
13769
13770 #[tokio::test]
13771 async fn completion_time_query_rejection_is_typed_without_stopping_worker() {
13772 let server = MockWorkerServer::reject_query_completion();
13773 let client = Client::builder(server.base_url())
13774 .timeout(Duration::from_secs(2))
13775 .build()
13776 .expect("client");
13777
13778 let error = client
13779 .complete_query_task("query-late", "late-worker", 1, json!(8), DEFAULT_CODEC)
13780 .await
13781 .expect_err("expired completion must be rejected");
13782 let Error::QueryFailed(failure) = error else {
13783 panic!("expected typed query failure");
13784 };
13785 assert_eq!(failure.status, 409);
13786 assert_eq!(failure.reason, "query_task_timed_out");
13787
13788 let mut worker = Worker::new(client, "rust-workers")
13789 .worker_id("late-worker")
13790 .poll_timeout(Duration::from_millis(10));
13791 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
13792 worker.register_query(
13793 "counter",
13794 "current",
13795 |_ctx, _args| async move { Ok(json!(8)) },
13796 );
13797
13798 assert_eq!(worker.run_once().await.expect("late task is handled"), 1);
13799 assert_eq!(
13800 worker
13801 .run_once()
13802 .await
13803 .expect("worker continues after late completion"),
13804 0
13805 );
13806 assert_eq!(
13807 server.request_count("/api/worker/query-tasks/query-late/complete"),
13808 2
13809 );
13810 assert_eq!(
13811 server.request_count("/api/worker/query-tasks/query-late/fail"),
13812 0,
13813 "a server completion rejection must not be reported as an encoding failure"
13814 );
13815 }
13816
13817 #[tokio::test]
13818 async fn normal_shutdown_joins_pollers_and_deregisters_once() {
13819 let server = MockWorkerServer::start();
13820 let client = Client::builder(server.base_url())
13821 .timeout(Duration::from_secs(2))
13822 .build()
13823 .expect("client");
13824 let mut worker = Worker::new(client, "rust-workers")
13825 .worker_id("joined-worker")
13826 .poll_timeout(Duration::from_millis(10));
13827 worker.register_workflow(
13828 "joined.workflow",
13829 |_ctx, _input| async move { Ok(Value::Null) },
13830 );
13831 worker.register_activity(
13832 "joined.activity",
13833 |_ctx, _input| async move { Ok(Value::Null) },
13834 );
13835 worker.register_query("joined.workflow", "state", |_ctx, _input| async move {
13836 Ok(Value::Null)
13837 });
13838
13839 worker
13840 .run_until(tokio::time::sleep(Duration::from_millis(20)))
13841 .await
13842 .expect("normal shutdown");
13843
13844 let deregistration_path = "/api/worker/registrations/mock-worker";
13845 assert_eq!(server.request_count(deregistration_path), 1);
13846 for poll_path in [
13847 "/api/worker/workflow-tasks/poll",
13848 "/api/worker/activity-tasks/poll",
13849 "/api/worker/query-tasks/poll",
13850 ] {
13851 assert!(server.request_count(poll_path) > 0, "missing {poll_path}");
13852 }
13853 assert_eq!(
13854 server.captured_paths().last().map(String::as_str),
13855 Some(deregistration_path),
13856 "deregistration must start only after every poller has joined"
13857 );
13858 }
13859
13860 #[tokio::test]
13861 async fn registration_failure_does_not_deregister() {
13862 let server = MockWorkerServer::rejected_registration();
13863 let client = Client::builder(server.base_url())
13864 .timeout(Duration::from_secs(2))
13865 .build()
13866 .expect("client");
13867 let worker = Worker::new(client, "rust-workers").worker_id("never-registered");
13868
13869 let error = worker
13870 .run_until(async {})
13871 .await
13872 .expect_err("registration must fail");
13873 assert!(matches!(
13874 error,
13875 Error::Http {
13876 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
13877 ..
13878 }
13879 ));
13880 assert!(server
13881 .captured_paths()
13882 .iter()
13883 .all(|path| !path.starts_with("/api/worker/registrations/")));
13884 }
13885
13886 #[tokio::test]
13887 async fn declined_registration_does_not_deregister() {
13888 let server = MockWorkerServer::declined_registration();
13889 let client = Client::builder(server.base_url())
13890 .timeout(Duration::from_secs(2))
13891 .build()
13892 .expect("client");
13893 let worker = Worker::new(client, "rust-workers").worker_id("declined-worker");
13894
13895 let error = worker
13896 .run_until(async {})
13897 .await
13898 .expect_err("declined registration must fail");
13899 assert!(matches!(error, Error::WorkerLoop(_)));
13900 assert!(error.to_string().contains("was not accepted"));
13901 assert!(server
13902 .captured_paths()
13903 .iter()
13904 .all(|path| !path.starts_with("/api/worker/registrations/")));
13905 }
13906
13907 #[tokio::test]
13908 async fn deregistration_http_failure_is_returned_after_normal_shutdown() {
13909 let server = MockWorkerServer::rejected_deregistration();
13910 let client = Client::builder(server.base_url())
13911 .timeout(Duration::from_secs(2))
13912 .build()
13913 .expect("client");
13914 let worker = Worker::new(client, "rust-workers").worker_id("forbidden-cleanup");
13915
13916 let error = worker
13917 .run_until(async {})
13918 .await
13919 .expect_err("deregistration must fail");
13920 assert!(matches!(
13921 error,
13922 Error::Http {
13923 status: reqwest::StatusCode::FORBIDDEN,
13924 ..
13925 }
13926 ));
13927 assert_eq!(
13928 server.request_count("/api/worker/registrations/mock-worker"),
13929 1
13930 );
13931 }
13932
13933 #[tokio::test]
13934 async fn deregistration_protocol_failure_is_returned_after_normal_shutdown() {
13935 let server = MockWorkerServer::rejected_deregistration_protocol();
13936 let client = Client::builder(server.base_url())
13937 .timeout(Duration::from_secs(2))
13938 .build()
13939 .expect("client");
13940 let worker = Worker::new(client, "rust-workers").worker_id("protocol-cleanup");
13941
13942 let error = worker
13943 .run_until(async {})
13944 .await
13945 .expect_err("protocol rejection must fail shutdown");
13946 let Error::Protocol(failure) = error else {
13947 panic!("expected typed protocol failure");
13948 };
13949 assert_eq!(failure.reason, "unsupported_protocol_version");
13950 assert_eq!(failure.requested_version.as_deref(), Some("1.2"));
13951 assert_eq!(
13952 server.request_count("/api/worker/registrations/mock-worker"),
13953 1
13954 );
13955 }
13956
13957 #[tokio::test]
13958 async fn primary_poller_error_retains_deregistration_failure_context() {
13959 let server = MockWorkerServer::unauthorized_polls_and_rejected_deregistration();
13960 let client = Client::builder(server.base_url())
13961 .timeout(Duration::from_secs(2))
13962 .build()
13963 .expect("client");
13964 let mut worker = Worker::new(client, "rust-workers")
13965 .worker_id("combined-failure")
13966 .poll_timeout(Duration::from_millis(10));
13967 worker.register_workflow("combined.workflow", |_ctx, _input| async move {
13968 Ok(Value::Null)
13969 });
13970
13971 let error = worker
13972 .run()
13973 .await
13974 .expect_err("worker and cleanup must fail");
13975 let summary = error.to_string();
13976 assert!(summary.contains("authentication_failed"));
13977 assert!(summary.contains("worker cannot deregister"));
13978 let Error::WorkerShutdown {
13979 primary,
13980 deregistration,
13981 } = error
13982 else {
13983 panic!("expected combined worker shutdown error");
13984 };
13985 assert!(matches!(
13986 *primary,
13987 Error::Http {
13988 status: reqwest::StatusCode::UNAUTHORIZED,
13989 ..
13990 }
13991 ));
13992 assert!(matches!(
13993 *deregistration,
13994 Error::Http {
13995 status: reqwest::StatusCode::FORBIDDEN,
13996 ..
13997 }
13998 ));
13999 assert_eq!(
14000 server.request_count("/api/worker/registrations/mock-worker"),
14001 1
14002 );
14003 }
14004
14005 #[tokio::test]
14006 async fn activity_only_worker_can_shutdown_without_workflow_poller() {
14007 let server = MockWorkerServer::start();
14008 let client = Client::builder(server.base_url())
14009 .timeout(Duration::from_secs(2))
14010 .build()
14011 .expect("client");
14012 let mut worker = Worker::new(client, "rust-workers")
14013 .worker_id("activity-only-worker")
14014 .poll_timeout(Duration::from_millis(10));
14015
14016 worker.register_activity(
14017 "activity.only",
14018 |_ctx, _args| async move { Ok(Value::Null) },
14019 );
14020
14021 worker.run_until(async {}).await.expect("run worker");
14022 }
14023
14024 #[tokio::test]
14025 async fn workflow_only_worker_can_shutdown_without_activity_poller() {
14026 let server = MockWorkerServer::start();
14027 let client = Client::builder(server.base_url())
14028 .timeout(Duration::from_secs(2))
14029 .build()
14030 .expect("client");
14031 let mut worker = Worker::new(client, "rust-workers")
14032 .worker_id("workflow-only-worker")
14033 .poll_timeout(Duration::from_millis(10));
14034
14035 worker.register_workflow(
14036 "workflow.only",
14037 |_ctx, _input| async move { Ok(Value::Null) },
14038 );
14039
14040 worker.run_until(async {}).await.expect("run worker");
14041 }
14042
14043 #[tokio::test]
14044 async fn worker_heartbeat_observer_receives_server_acknowledgements() {
14045 let server = MockWorkerServer::start();
14046 let client = Client::builder(server.base_url())
14047 .timeout(Duration::from_secs(2))
14048 .build()
14049 .expect("client");
14050 let observations = Arc::new(Mutex::new(Vec::new()));
14051 let observed = Arc::clone(&observations);
14052 let mut worker = Worker::new(client, "rust-workers")
14053 .worker_id("observed-heartbeat-worker")
14054 .poll_timeout(Duration::from_millis(10))
14055 .on_worker_heartbeat(move |observation| {
14056 observed
14057 .lock()
14058 .expect("heartbeat observations")
14059 .push(observation.clone());
14060 });
14061
14062 worker.register_workflow("workflow.observed", |_ctx, _input| async move {
14063 Ok(Value::Null)
14064 });
14065 let acknowledged = Arc::clone(&observations);
14066 worker
14067 .run_until(async move {
14068 tokio::time::timeout(Duration::from_secs(2), async move {
14069 loop {
14070 if !acknowledged
14071 .lock()
14072 .expect("heartbeat observations")
14073 .is_empty()
14074 {
14075 break;
14076 }
14077 tokio::time::sleep(Duration::from_millis(1)).await;
14078 }
14079 })
14080 .await
14081 .expect("heartbeat acknowledgement within timeout");
14082 })
14083 .await
14084 .expect("run worker");
14085
14086 let observations = observations.lock().expect("heartbeat observations");
14087 let first = observations.first().expect("heartbeat acknowledgement");
14088 assert_eq!(first.worker_id, "observed-heartbeat-worker");
14089 assert_eq!(first.task_queue, "rust-workers");
14090 assert!(first.acknowledged_at_unix_millis > 0);
14091 assert_eq!(first.acknowledgement, json!({}));
14092 }
14093
14094 #[tokio::test]
14095 async fn delayed_worker_heartbeat_keeps_cadence_and_pollers_live() {
14096 let server = MockWorkerServer::delayed_heartbeat_worker();
14097 let client = Client::builder(server.base_url())
14098 .timeout(Duration::from_secs(3))
14099 .build()
14100 .expect("client");
14101 let observations = Arc::new(Mutex::new(Vec::new()));
14102 let observed = Arc::clone(&observations);
14103 let mut worker = Worker::new(client, "rust-snapshot-workers")
14104 .worker_id("rust-snapshot-worker")
14105 .poll_timeout(Duration::from_millis(10))
14106 .on_worker_heartbeat(move |observation| {
14107 observed
14108 .lock()
14109 .expect("heartbeat observations")
14110 .push(observation.clone());
14111 });
14112
14113 worker.register_workflow("snapshot", |ctx, _input| async move {
14114 ctx.wait_signal("finish").await?;
14115 Ok(json!({"status": "finished"}))
14116 });
14117 worker.register_query("snapshot", "current", |ctx, _args| async move {
14118 Ok(json!(ctx
14119 .signals("increment")
14120 .iter()
14121 .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
14122 .sum::<i64>()))
14123 });
14124 worker.register_activity("cancel-aware", |_ctx, _args| async move {
14125 Ok(json!({"late": "completion"}))
14126 });
14127
14128 worker
14129 .run_until(tokio::time::sleep(Duration::from_millis(3_800)))
14130 .await
14131 .expect("delayed heartbeat must allow a clean worker shutdown");
14132
14133 let observations = observations.lock().expect("heartbeat observations");
14134 assert!(
14135 observations.len() >= 3,
14136 "the immediate heartbeat, delayed acknowledgement, and next cadence heartbeat must complete"
14137 );
14138 assert!(
14139 observations.windows(2).all(|pair| {
14140 pair[1].acknowledged_at_unix_millis
14141 .saturating_sub(pair[0].acknowledged_at_unix_millis)
14142 >= 850
14143 }),
14144 "successful acknowledgements must not catch up faster than the advertised one-second cadence: {observations:?}"
14145 );
14146 drop(observations);
14147
14148 let heartbeat_times = server.request_times("/api/worker/heartbeat");
14149 let delayed_request_at = *heartbeat_times
14150 .get(1)
14151 .expect("intentionally delayed heartbeat request");
14152 let delay_window_start = delayed_request_at + Duration::from_millis(100);
14153 let delay_window_end = delayed_request_at + Duration::from_millis(1_400);
14154 for path in [
14155 "/api/worker/workflow-tasks/poll",
14156 "/api/worker/activity-tasks/poll",
14157 "/api/worker/query-tasks/poll",
14158 ] {
14159 assert!(
14160 server
14161 .request_times(path)
14162 .iter()
14163 .any(|received_at| *received_at >= delay_window_start
14164 && *received_at <= delay_window_end),
14165 "{path} must keep polling while a heartbeat acknowledgement is delayed"
14166 );
14167 }
14168 assert!(
14169 server.request_count("/api/worker/workflow-tasks/snapshot-wait-3/fail") >= 1,
14170 "workflow work must be settled"
14171 );
14172 assert!(
14173 server.request_count("/api/worker/activity-tasks/activity-cancel/complete") >= 1,
14174 "activity work must be settled"
14175 );
14176 assert!(
14177 server.request_count("/api/worker/query-tasks/snapshot-current/complete") >= 1,
14178 "query work must be settled"
14179 );
14180 }
14181
14182 #[tokio::test]
14183 async fn retried_worker_heartbeat_restarts_the_advertised_cadence() {
14184 let server = MockWorkerServer::heartbeat_retry_worker();
14185 let client = Client::builder(server.base_url())
14186 .timeout(Duration::from_secs(2))
14187 .build()
14188 .expect("client");
14189 let observations = Arc::new(Mutex::new(Vec::new()));
14190 let observed = Arc::clone(&observations);
14191 let worker = Worker::new(client, "rust-workers")
14192 .worker_id("heartbeat-retry-worker")
14193 .retry_policy(WorkerRetryPolicy {
14194 max_retries: 1,
14195 initial_backoff: Duration::from_millis(300),
14196 max_backoff: Duration::from_millis(300),
14197 })
14198 .on_worker_heartbeat(move |observation| {
14199 observed
14200 .lock()
14201 .expect("heartbeat observations")
14202 .push(observation.clone());
14203 });
14204
14205 worker
14206 .run_until(tokio::time::sleep(Duration::from_millis(2_700)))
14207 .await
14208 .expect("retryable heartbeat failure must remain bounded and recover");
14209
14210 let observations = observations.lock().expect("heartbeat observations");
14211 assert!(observations.len() >= 3, "heartbeat retry must recover");
14212 assert!(
14213 observations.windows(2).all(|pair| {
14214 pair[1]
14215 .acknowledged_at_unix_millis
14216 .saturating_sub(pair[0].acknowledged_at_unix_millis)
14217 >= 850
14218 }),
14219 "a successful retry must start a fresh advertised cadence: {observations:?}"
14220 );
14221 assert_eq!(
14222 server.request_count("/api/worker/heartbeat"),
14223 observations.len() + 1,
14224 "one retryable failure must add exactly one bounded request"
14225 );
14226 }
14227
14228 #[tokio::test]
14229 async fn query_enabled_worker_ignores_unmatched_signals_then_completes_once() {
14230 let server = MockWorkerServer::waiting_query_worker();
14231 let client = Client::builder(server.base_url())
14232 .timeout(Duration::from_secs(2))
14233 .build()
14234 .expect("client");
14235 let observations = Arc::new(Mutex::new(Vec::new()));
14236 let observed = Arc::clone(&observations);
14237 let mut worker = Worker::new(client, "rust-snapshot-workers")
14238 .worker_id("rust-snapshot-worker")
14239 .poll_timeout(Duration::from_millis(10))
14240 .on_worker_heartbeat(move |observation| {
14241 observed
14242 .lock()
14243 .expect("heartbeat observations")
14244 .push(observation.clone());
14245 });
14246
14247 worker.register_workflow("snapshot", |ctx, _input| async move {
14248 ctx.wait_signal("finish").await?;
14249 Ok(json!({"status": "finished"}))
14250 });
14251 worker.register_query("snapshot", "current", |ctx, _args| async move {
14252 let current = ctx
14253 .signals("increment")
14254 .iter()
14255 .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
14256 .sum::<i64>();
14257 Ok(json!(current))
14258 });
14259 worker.register_update("snapshot", "replace", |_ctx, args| async move { Ok(args) });
14260
14261 worker
14262 .run_until(tokio::time::sleep(Duration::from_millis(3_200)))
14263 .await
14264 .expect("pending workflow and query poller must remain live until shutdown");
14265
14266 assert!(
14267 observations.lock().expect("heartbeat observations").len() >= 4,
14268 "the immediate heartbeat and at least three advertised one-second intervals must be acknowledged"
14269 );
14270 assert!(
14271 server.request_count("/api/worker/workflow-tasks/poll") >= 3,
14272 "workflow polling must continue after empty replay acknowledgements"
14273 );
14274 assert!(
14275 server.request_count("/api/worker/query-tasks/poll") >= 2,
14276 "query polling must continue after serving the current query"
14277 );
14278 assert_eq!(
14279 server.request_body("/api/worker/register")["capabilities"],
14280 json!([QUERY_TASKS_CAPABILITY, WORKFLOW_UPDATES_CAPABILITY])
14281 );
14282 assert_eq!(
14283 server.request_body("/api/worker/register")["workflow_command_contracts"]["snapshot"],
14284 json!({
14285 "queries": ["current"],
14286 "updates": ["replace"],
14287 "update_validators": [],
14288 })
14289 );
14290
14291 let opened = server.request_body("/api/worker/workflow-tasks/snapshot-open/complete");
14292 assert_eq!(
14293 opened["commands"],
14294 json!([{
14295 "type": "open_signal_wait",
14296 "signal_name": "finish",
14297 }])
14298 );
14299
14300 for task_id in ["snapshot-wait-3", "snapshot-wait-5"] {
14301 let fail_path = format!("/api/worker/workflow-tasks/{task_id}/fail");
14302 let completion_path = format!("/api/worker/workflow-tasks/{task_id}/complete");
14303 let failure = server.request_body(&fail_path);
14304 assert_eq!(
14305 failure["failure"]["type"],
14306 WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE
14307 );
14308 assert_eq!(server.request_count(&completion_path), 0);
14309 }
14310
14311 let query_completion =
14312 server.request_body("/api/worker/query-tasks/snapshot-current/complete");
14313 assert_eq!(query_completion["result"], json!(8));
14314
14315 let terminal_path = "/api/worker/workflow-tasks/snapshot-finish/complete";
14316 assert_eq!(
14317 server.request_count(terminal_path),
14318 1,
14319 "the matching signal must settle the workflow exactly once"
14320 );
14321 let terminal = server.request_body(terminal_path);
14322 assert_eq!(terminal["commands"].as_array().map(Vec::len), Some(1));
14323 assert_eq!(terminal["commands"][0]["type"], "complete_workflow");
14324 assert_eq!(
14325 decode_wire_value(&terminal["commands"][0]["result"], DEFAULT_CODEC)
14326 .expect("terminal workflow result"),
14327 json!({"status": "finished"})
14328 );
14329 }
14330
14331 #[tokio::test]
14332 async fn worker_retries_poll_and_heartbeat_transport_failures_independently() {
14333 let server = MockWorkerServer::transient_worker_failures();
14334 let client = Client::builder(server.base_url())
14335 .timeout(Duration::from_secs(2))
14336 .build()
14337 .expect("client");
14338 let mut worker = Worker::new(client, "rust-workers")
14339 .worker_id("retry-worker")
14340 .poll_timeout(Duration::from_millis(10))
14341 .retry_policy(WorkerRetryPolicy {
14342 max_retries: 2,
14343 initial_backoff: Duration::from_millis(1),
14344 max_backoff: Duration::from_millis(1),
14345 });
14346 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
14347 worker.register_activity(
14348 "counter.activity",
14349 |_ctx, _input| async move { Ok(Value::Null) },
14350 );
14351 worker.register_query(
14352 "counter",
14353 "current",
14354 |_ctx, _args| async move { Ok(json!(8)) },
14355 );
14356
14357 worker
14358 .run_until(tokio::time::sleep(Duration::from_millis(75)))
14359 .await
14360 .expect("transient failures must not stop the worker");
14361
14362 for path in [
14363 "/api/worker/heartbeat",
14364 "/api/worker/workflow-tasks/poll",
14365 "/api/worker/activity-tasks/poll",
14366 "/api/worker/query-tasks/poll",
14367 ] {
14368 assert!(
14369 server.request_count(path) >= 2,
14370 "{path} must continue after its transient failure"
14371 );
14372 }
14373 }
14374
14375 #[tokio::test]
14376 async fn worker_bounds_transport_retries() {
14377 let server = MockWorkerServer::unavailable_polls();
14378 let client = Client::builder(server.base_url())
14379 .timeout(Duration::from_secs(2))
14380 .build()
14381 .expect("client");
14382 let mut worker = Worker::new(client, "rust-workers")
14383 .worker_id("bounded-retry-worker")
14384 .poll_timeout(Duration::from_millis(10))
14385 .retry_policy(WorkerRetryPolicy {
14386 max_retries: 2,
14387 initial_backoff: Duration::from_millis(1),
14388 max_backoff: Duration::from_millis(1),
14389 });
14390 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
14391
14392 let error = worker.run().await.expect_err("retry bound must terminate");
14393 assert!(matches!(error, Error::Transport(_)));
14394 assert_eq!(
14395 server.request_count("/api/worker/workflow-tasks/poll"),
14396 3,
14397 "one initial request plus exactly two retries"
14398 );
14399 }
14400
14401 #[tokio::test]
14402 async fn worker_retry_policy_can_disable_poll_retries() {
14403 let server = MockWorkerServer::unavailable_polls();
14404 let client = Client::builder(server.base_url())
14405 .timeout(Duration::from_secs(2))
14406 .build()
14407 .expect("client");
14408 let mut worker = Worker::new(client, "rust-workers")
14409 .worker_id("no-retry-worker")
14410 .poll_timeout(Duration::from_millis(10))
14411 .retry_policy(WorkerRetryPolicy {
14412 max_retries: 0,
14413 initial_backoff: Duration::from_millis(1),
14414 max_backoff: Duration::from_millis(1),
14415 });
14416 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
14417
14418 let error = worker
14419 .run_once()
14420 .await
14421 .expect_err("disabled retries must return the first transport failure");
14422 assert!(matches!(error, Error::Transport(_)));
14423 assert_eq!(
14424 server.request_count("/api/worker/workflow-tasks/poll"),
14425 1,
14426 "max_retries=0 must send only the initial request"
14427 );
14428 }
14429
14430 #[tokio::test]
14431 async fn worker_does_not_retry_authentication_failures() {
14432 let server = MockWorkerServer::unauthorized_polls();
14433 let client = Client::builder(server.base_url())
14434 .timeout(Duration::from_secs(2))
14435 .build()
14436 .expect("client");
14437 let mut worker = Worker::new(client, "rust-workers")
14438 .worker_id("unauthorized-worker")
14439 .poll_timeout(Duration::from_millis(10));
14440 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
14441
14442 let error = worker
14443 .run()
14444 .await
14445 .expect_err("authentication must terminate");
14446 let Error::Http { status, body } = error else {
14447 panic!("expected stable HTTP authentication error");
14448 };
14449 assert_eq!(status, reqwest::StatusCode::UNAUTHORIZED);
14450 assert!(body.contains("authentication_failed"));
14451 assert_eq!(
14452 server.request_count("/api/worker/workflow-tasks/poll"),
14453 1,
14454 "authentication failures must not be retried"
14455 );
14456 }
14457
14458 #[derive(Clone, Debug)]
14459 struct CapturedRequest {
14460 method: String,
14461 path: String,
14462 authorization: Option<String>,
14463 namespace: Option<String>,
14464 worker_protocol: Option<String>,
14465 control_protocol: Option<String>,
14466 body: String,
14467 received_at: Instant,
14468 }
14469
14470 struct MockWorkerServer {
14471 addr: SocketAddr,
14472 stop: Arc<AtomicBool>,
14473 requests: Arc<Mutex<Vec<CapturedRequest>>>,
14474 thread: Option<thread::JoinHandle<()>>,
14475 }
14476
14477 #[derive(Clone, Copy, Default)]
14478 struct MockWorkerBehavior {
14479 reject_query_protocol: bool,
14480 reject_query_completion: bool,
14481 waiting_query_worker: bool,
14482 decline_registration: bool,
14483 complete_named_signal: bool,
14484 poll_failures_per_path: usize,
14485 heartbeat_failures: usize,
14486 heartbeat_failure_request: Option<usize>,
14487 delayed_heartbeat_request: Option<usize>,
14488 heartbeat_response_delay: Duration,
14489 concurrent_requests: bool,
14490 unauthorized_polls: bool,
14491 reject_registration: bool,
14492 reject_deregistration: bool,
14493 reject_deregistration_protocol: bool,
14494 cancelled_activity: bool,
14495 draining_polls: bool,
14496 invalid_task_payload_codec: Option<InvalidTaskPayloadCodec>,
14497 workflow_completion_status: Option<&'static str>,
14498 workflow_completion_body: Option<&'static str>,
14499 }
14500
14501 impl MockWorkerServer {
14502 fn start() -> Self {
14503 Self::start_with_behavior(MockWorkerBehavior::default())
14504 }
14505
14506 fn reject_query_protocol() -> Self {
14507 Self::start_with_behavior(MockWorkerBehavior {
14508 reject_query_protocol: true,
14509 ..MockWorkerBehavior::default()
14510 })
14511 }
14512
14513 fn reject_query_completion() -> Self {
14514 Self::start_with_behavior(MockWorkerBehavior {
14515 reject_query_completion: true,
14516 ..MockWorkerBehavior::default()
14517 })
14518 }
14519
14520 fn waiting_query_worker() -> Self {
14521 Self::start_with_behavior(MockWorkerBehavior {
14522 waiting_query_worker: true,
14523 complete_named_signal: true,
14524 ..MockWorkerBehavior::default()
14525 })
14526 }
14527
14528 fn transient_worker_failures() -> Self {
14529 Self::start_with_behavior(MockWorkerBehavior {
14530 poll_failures_per_path: 1,
14531 heartbeat_failures: 1,
14532 ..MockWorkerBehavior::default()
14533 })
14534 }
14535
14536 fn consecutive_poll_failures(count: usize) -> Self {
14537 Self::start_with_behavior(MockWorkerBehavior {
14538 poll_failures_per_path: count,
14539 ..MockWorkerBehavior::default()
14540 })
14541 }
14542
14543 fn delayed_heartbeat_worker() -> Self {
14544 Self::start_with_behavior(MockWorkerBehavior {
14545 waiting_query_worker: true,
14546 delayed_heartbeat_request: Some(2),
14547 heartbeat_response_delay: Duration::from_millis(1_500),
14548 concurrent_requests: true,
14549 cancelled_activity: true,
14550 ..MockWorkerBehavior::default()
14551 })
14552 }
14553
14554 fn heartbeat_retry_worker() -> Self {
14555 Self::start_with_behavior(MockWorkerBehavior {
14556 waiting_query_worker: true,
14557 heartbeat_failure_request: Some(2),
14558 concurrent_requests: true,
14559 ..MockWorkerBehavior::default()
14560 })
14561 }
14562
14563 fn unavailable_polls() -> Self {
14564 Self::start_with_behavior(MockWorkerBehavior {
14565 poll_failures_per_path: usize::MAX,
14566 ..MockWorkerBehavior::default()
14567 })
14568 }
14569
14570 fn unauthorized_polls() -> Self {
14571 Self::start_with_behavior(MockWorkerBehavior {
14572 unauthorized_polls: true,
14573 ..MockWorkerBehavior::default()
14574 })
14575 }
14576
14577 fn rejected_registration() -> Self {
14578 Self::start_with_behavior(MockWorkerBehavior {
14579 reject_registration: true,
14580 ..MockWorkerBehavior::default()
14581 })
14582 }
14583
14584 fn declined_registration() -> Self {
14585 Self::start_with_behavior(MockWorkerBehavior {
14586 decline_registration: true,
14587 ..MockWorkerBehavior::default()
14588 })
14589 }
14590
14591 fn rejected_deregistration() -> Self {
14592 Self::start_with_behavior(MockWorkerBehavior {
14593 reject_deregistration: true,
14594 ..MockWorkerBehavior::default()
14595 })
14596 }
14597
14598 fn rejected_deregistration_protocol() -> Self {
14599 Self::start_with_behavior(MockWorkerBehavior {
14600 reject_deregistration_protocol: true,
14601 ..MockWorkerBehavior::default()
14602 })
14603 }
14604
14605 fn unauthorized_polls_and_rejected_deregistration() -> Self {
14606 Self::start_with_behavior(MockWorkerBehavior {
14607 unauthorized_polls: true,
14608 reject_deregistration: true,
14609 ..MockWorkerBehavior::default()
14610 })
14611 }
14612
14613 fn cancelled_activity() -> Self {
14614 Self::start_with_behavior(MockWorkerBehavior {
14615 cancelled_activity: true,
14616 ..MockWorkerBehavior::default()
14617 })
14618 }
14619
14620 fn draining_polls() -> Self {
14621 Self::start_with_behavior(MockWorkerBehavior {
14622 draining_polls: true,
14623 ..MockWorkerBehavior::default()
14624 })
14625 }
14626
14627 fn invalid_task_payload_codec(codec: InvalidTaskPayloadCodec) -> Self {
14628 Self::start_with_behavior(MockWorkerBehavior {
14629 invalid_task_payload_codec: Some(codec),
14630 ..MockWorkerBehavior::default()
14631 })
14632 }
14633
14634 fn workflow_completion(status: &'static str, body: &'static str) -> Self {
14635 Self::start_with_behavior(MockWorkerBehavior {
14636 workflow_completion_status: Some(status),
14637 workflow_completion_body: Some(body),
14638 ..MockWorkerBehavior::default()
14639 })
14640 }
14641
14642 fn start_with_behavior(behavior: MockWorkerBehavior) -> Self {
14643 let listener = TcpListener::bind("127.0.0.1:0").expect("bind mock server");
14644 listener
14645 .set_nonblocking(true)
14646 .expect("configure mock listener");
14647 let addr = listener.local_addr().expect("mock server address");
14648 let stop = Arc::new(AtomicBool::new(false));
14649 let server_stop = Arc::clone(&stop);
14650 let requests = Arc::new(Mutex::new(Vec::new()));
14651 let server_requests = Arc::clone(&requests);
14652 let thread = thread::spawn(move || {
14653 let mut request_threads = Vec::new();
14654 while !server_stop.load(Ordering::SeqCst) {
14655 match listener.accept() {
14656 Ok((mut stream, _)) => {
14657 if behavior.concurrent_requests {
14658 let requests = Arc::clone(&server_requests);
14659 request_threads.push(thread::spawn(move || {
14660 handle_mock_worker_request(&mut stream, &requests, behavior)
14661 }));
14662 } else {
14663 handle_mock_worker_request(&mut stream, &server_requests, behavior);
14664 }
14665 }
14666 Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
14667 let mut index = 0;
14668 while index < request_threads.len() {
14669 if request_threads[index].is_finished() {
14670 request_threads
14671 .swap_remove(index)
14672 .join()
14673 .expect("join mock request");
14674 } else {
14675 index += 1;
14676 }
14677 }
14678 thread::sleep(Duration::from_millis(5));
14679 }
14680 Err(_) => break,
14681 }
14682 }
14683 for request_thread in request_threads {
14684 request_thread.join().expect("join mock request");
14685 }
14686 });
14687
14688 Self {
14689 addr,
14690 stop,
14691 requests,
14692 thread: Some(thread),
14693 }
14694 }
14695
14696 fn base_url(&self) -> String {
14697 format!("http://{}", self.addr)
14698 }
14699
14700 fn worker_protocol_for(&self, path: &str) -> Option<String> {
14701 self.requests
14702 .lock()
14703 .expect("captured requests")
14704 .iter()
14705 .find(|request| request.path == path)
14706 .and_then(|request| request.worker_protocol.clone())
14707 }
14708
14709 fn control_protocol_for(&self, path: &str) -> Option<String> {
14710 self.requests
14711 .lock()
14712 .expect("captured requests")
14713 .iter()
14714 .find(|request| request.path == path)
14715 .and_then(|request| request.control_protocol.clone())
14716 }
14717
14718 fn method_for(&self, path: &str) -> Option<String> {
14719 self.requests
14720 .lock()
14721 .expect("captured requests")
14722 .iter()
14723 .find(|request| request.path == path)
14724 .map(|request| request.method.clone())
14725 }
14726
14727 fn authorization_for(&self, path: &str) -> Option<String> {
14728 self.requests
14729 .lock()
14730 .expect("captured requests")
14731 .iter()
14732 .find(|request| request.path == path)
14733 .and_then(|request| request.authorization.clone())
14734 }
14735
14736 fn namespace_for(&self, path: &str) -> Option<String> {
14737 self.requests
14738 .lock()
14739 .expect("captured requests")
14740 .iter()
14741 .find(|request| request.path == path)
14742 .and_then(|request| request.namespace.clone())
14743 }
14744
14745 fn request_count(&self, path: &str) -> usize {
14746 self.requests
14747 .lock()
14748 .expect("captured requests")
14749 .iter()
14750 .filter(|request| request.path == path)
14751 .count()
14752 }
14753
14754 fn captured_paths(&self) -> Vec<String> {
14755 self.requests
14756 .lock()
14757 .expect("captured requests")
14758 .iter()
14759 .map(|request| request.path.clone())
14760 .collect()
14761 }
14762
14763 fn request_times(&self, path: &str) -> Vec<Instant> {
14764 self.requests
14765 .lock()
14766 .expect("captured requests")
14767 .iter()
14768 .filter(|request| request.path == path)
14769 .map(|request| request.received_at)
14770 .collect()
14771 }
14772
14773 fn request_body(&self, path: &str) -> Value {
14774 let requests = self.requests.lock().expect("captured requests");
14775 let body = &requests
14776 .iter()
14777 .find(|request| request.path == path)
14778 .unwrap_or_else(|| panic!("missing request for {path}"))
14779 .body;
14780 serde_json::from_str(body).unwrap_or_else(|error| {
14781 panic!("invalid JSON request body for {path}: {error}: {body:?}")
14782 })
14783 }
14784
14785 fn request_bodies(&self, path: &str) -> Vec<Value> {
14786 self.requests
14787 .lock()
14788 .expect("captured requests")
14789 .iter()
14790 .filter(|request| request.path == path)
14791 .map(|request| {
14792 serde_json::from_str(&request.body).unwrap_or_else(|error| {
14793 panic!(
14794 "invalid JSON request body for {path}: {error}: {:?}",
14795 request.body
14796 )
14797 })
14798 })
14799 .collect()
14800 }
14801 }
14802
14803 impl Drop for MockWorkerServer {
14804 fn drop(&mut self) {
14805 self.stop.store(true, Ordering::SeqCst);
14806 let _ = TcpStream::connect(self.addr);
14807
14808 if let Some(thread) = self.thread.take() {
14809 thread.join().expect("join mock server");
14810 }
14811 }
14812 }
14813
14814 fn handle_mock_worker_request(
14815 stream: &mut TcpStream,
14816 requests: &Arc<Mutex<Vec<CapturedRequest>>>,
14817 behavior: MockWorkerBehavior,
14818 ) {
14819 let _ = stream.set_read_timeout(Some(Duration::from_millis(200)));
14820 let mut buffer = [0_u8; 8192];
14821 let mut request = Vec::new();
14822
14823 loop {
14824 match stream.read(&mut buffer) {
14825 Ok(0) => break,
14826 Ok(read) => {
14827 request.extend_from_slice(&buffer[..read]);
14828 if mock_request_is_complete(&request) {
14829 break;
14830 }
14831 }
14832 Err(error)
14833 if matches!(
14834 error.kind(),
14835 std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut
14836 ) =>
14837 {
14838 break;
14839 }
14840 Err(_) => return,
14841 }
14842 }
14843
14844 let request = String::from_utf8_lossy(&request);
14845 let body = request
14846 .split_once("\r\n\r\n")
14847 .map(|(_, body)| body)
14848 .unwrap_or_default();
14849 let path = request
14850 .lines()
14851 .next()
14852 .and_then(|line| line.split_whitespace().nth(1))
14853 .unwrap_or_default();
14854 let method = request
14855 .lines()
14856 .next()
14857 .and_then(|line| line.split_whitespace().next())
14858 .unwrap_or_default();
14859 let authorization = request.lines().find_map(|line| {
14860 let (name, value) = line.split_once(':')?;
14861 name.eq_ignore_ascii_case("Authorization")
14862 .then(|| value.trim().to_string())
14863 });
14864 let namespace = request.lines().find_map(|line| {
14865 let (name, value) = line.split_once(':')?;
14866 name.eq_ignore_ascii_case("X-Namespace")
14867 .then(|| value.trim().to_string())
14868 });
14869 let worker_protocol = request.lines().find_map(|line| {
14870 let (name, value) = line.split_once(':')?;
14871 name.eq_ignore_ascii_case("X-Durable-Workflow-Protocol-Version")
14872 .then(|| value.trim().to_string())
14873 });
14874 let control_protocol = request.lines().find_map(|line| {
14875 let (name, value) = line.split_once(':')?;
14876 name.eq_ignore_ascii_case("X-Durable-Workflow-Control-Plane-Version")
14877 .then(|| value.trim().to_string())
14878 });
14879 let request_number = {
14880 let mut requests = requests.lock().expect("captured requests");
14881 requests.push(CapturedRequest {
14882 method: method.to_string(),
14883 path: path.to_string(),
14884 authorization,
14885 namespace,
14886 worker_protocol: worker_protocol.clone(),
14887 control_protocol,
14888 body: body.to_string(),
14889 received_at: Instant::now(),
14890 });
14891 requests
14892 .iter()
14893 .filter(|request| request.path == path)
14894 .count()
14895 };
14896
14897 if behavior.reject_registration && path == "/api/worker/register" {
14898 write_mock_response(
14899 stream,
14900 "503 Service Unavailable",
14901 r#"{"reason":"registration_unavailable","message":"registration failed"}"#,
14902 );
14903 return;
14904 }
14905
14906 if path.starts_with("/api/worker/registrations/") {
14907 if behavior.reject_deregistration_protocol {
14908 write_mock_response(
14909 stream,
14910 "400 Bad Request",
14911 r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.1","requested_version":"1.2"}"#,
14912 );
14913 } else if behavior.reject_deregistration {
14914 write_mock_response(
14915 stream,
14916 "403 Forbidden",
14917 r#"{"reason":"authorization_failed","message":"worker cannot deregister"}"#,
14918 );
14919 } else {
14920 write_mock_response(
14921 stream,
14922 "200 OK",
14923 r#"{"worker_id":"deregistered-worker","outcome":"deregistered","recovered_workflow_task_count":2}"#,
14924 );
14925 }
14926 return;
14927 }
14928
14929 let is_poll = matches!(
14930 path,
14931 "/api/worker/workflow-tasks/poll"
14932 | "/api/worker/activity-tasks/poll"
14933 | "/api/worker/query-tasks/poll"
14934 );
14935 if is_poll && request_number <= behavior.poll_failures_per_path {
14936 return;
14937 }
14938 if path == "/api/worker/heartbeat" && request_number <= behavior.heartbeat_failures {
14939 return;
14940 }
14941 if path == "/api/worker/heartbeat"
14942 && behavior.heartbeat_failure_request == Some(request_number)
14943 {
14944 return;
14945 }
14946 if path == "/api/worker/heartbeat"
14947 && behavior.delayed_heartbeat_request == Some(request_number)
14948 {
14949 thread::sleep(behavior.heartbeat_response_delay);
14950 }
14951 if behavior.unauthorized_polls && is_poll {
14952 write_mock_response(
14953 stream,
14954 "401 Unauthorized",
14955 r#"{"reason":"authentication_failed","message":"invalid worker token"}"#,
14956 );
14957 return;
14958 }
14959 if behavior.draining_polls && is_poll {
14960 write_mock_response(
14961 stream,
14962 "409 Conflict",
14963 r#"{"task":null,"poll_status":"draining","reason":"worker_draining","worker_status":"draining","drain_intent":"draining"}"#,
14964 );
14965 return;
14966 }
14967
14968 if let Some(codec_case) = behavior.invalid_task_payload_codec {
14969 if is_poll && request_number == 1 {
14970 let mut task = match path {
14971 "/api/worker/workflow-tasks/poll" => json!({
14972 "task_id": "codec-workflow",
14973 "workflow_type": "codec.workflow",
14974 "payload_codec": DEFAULT_CODEC,
14975 "workflow_task_attempt": 1,
14976 "lease_owner": "codec-worker"
14977 }),
14978 "/api/worker/activity-tasks/poll" => json!({
14979 "task_id": "codec-activity",
14980 "activity_attempt_id": "codec-activity-attempt",
14981 "activity_type": "codec.activity",
14982 "payload_codec": DEFAULT_CODEC,
14983 "attempt_number": 1,
14984 "lease_owner": "codec-worker"
14985 }),
14986 "/api/worker/query-tasks/poll" => json!({
14987 "query_task_id": "codec-query",
14988 "query_task_attempt": 1,
14989 "workflow_type": "codec.workflow",
14990 "query_name": "known",
14991 "payload_codec": DEFAULT_CODEC,
14992 "lease_owner": "codec-worker"
14993 }),
14994 _ => unreachable!("is_poll limits task codec probe paths"),
14995 };
14996 codec_case.apply(&mut task);
14997 write_mock_response(stream, "200 OK", &json!({"task": task}).to_string());
14998 return;
14999 }
15000
15001 if matches!(
15002 path,
15003 "/api/worker/workflow-tasks/codec-workflow/fail"
15004 | "/api/worker/activity-tasks/codec-activity/fail"
15005 | "/api/worker/query-tasks/codec-query/fail"
15006 ) {
15007 write_mock_response(stream, "200 OK", r#"{"outcome":"failed"}"#);
15008 return;
15009 }
15010 }
15011
15012 if behavior.reject_query_protocol && path.starts_with("/api/worker/query-tasks/") {
15013 let requested_version = worker_protocol.as_deref().unwrap_or("missing");
15014 let body = format!(
15015 r#"{{"reason":"unsupported_protocol_version","message":"Query tasks require worker protocol 1.8 or newer.","supported_version":"1.7","requested_version":"{requested_version}"}}"#
15016 );
15017 write_mock_response(stream, "400 Bad Request", &body);
15018 return;
15019 }
15020
15021 if behavior.reject_query_completion && path == "/api/worker/query-tasks/query-late/complete"
15022 {
15023 write_mock_response(
15024 stream,
15025 "409 Conflict",
15026 r#"{"reason":"query_task_timed_out","message":"query task timed out before completion"}"#,
15027 );
15028 return;
15029 }
15030
15031 if behavior.workflow_completion_status.is_some()
15032 && path == "/api/worker/workflow-tasks/poll"
15033 && request_number == 1
15034 {
15035 write_mock_response(
15036 stream,
15037 "200 OK",
15038 r#"{"task":{"task_id":"workflow-timeout-task","workflow_id":"reused-workflow-id","run_id":"run-selected-timeout","workflow_type":"timeout.workflow","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"history_events":[],"workflow_task_attempt":3,"lease_owner":"timeout-worker"}}"#,
15039 );
15040 return;
15041 }
15042
15043 if path == "/api/worker/workflow-tasks/workflow-timeout-task/complete" {
15044 if let (Some(status), Some(body)) = (
15045 behavior.workflow_completion_status,
15046 behavior.workflow_completion_body,
15047 ) {
15048 write_mock_response(stream, status, body);
15049 return;
15050 }
15051 }
15052
15053 if behavior.waiting_query_worker {
15054 if behavior.complete_named_signal
15055 && path == "/api/worker/workflow-tasks/poll"
15056 && request_number == 1
15057 {
15058 let body = json!({
15059 "task": {
15060 "task_id": "snapshot-open",
15061 "workflow_id": "snapshot-1",
15062 "run_id": "snapshot-run-1",
15063 "workflow_type": "snapshot",
15064 "payload_codec": DEFAULT_CODEC,
15065 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
15066 .expect("Avro workflow arguments"),
15067 "history_events": [],
15068 "workflow_task_attempt": 1,
15069 "lease_owner": "rust-snapshot-worker"
15070 }
15071 })
15072 .to_string();
15073 write_mock_response(stream, "200 OK", &body);
15074 return;
15075 }
15076
15077 let signal_request = request_number - usize::from(behavior.complete_named_signal);
15078 let signal_request_limit = 2 + usize::from(behavior.complete_named_signal);
15079 if path == "/api/worker/workflow-tasks/poll"
15080 && signal_request >= 1
15081 && signal_request <= signal_request_limit
15082 {
15083 let finish = behavior.complete_named_signal && signal_request == 3;
15084 let amounts = if signal_request == 1 {
15085 vec![3]
15086 } else {
15087 vec![3, 5]
15088 };
15089 let task_id = if signal_request == 1 {
15090 "snapshot-wait-3"
15091 } else if finish {
15092 "snapshot-finish"
15093 } else {
15094 "snapshot-wait-5"
15095 };
15096 let mut history_events = std::iter::once(json!({
15097 "event_type": "SignalWaitOpened",
15098 "payload": {"sequence": 1, "signal_name": "finish"}
15099 }))
15100 .chain(amounts.iter().enumerate().map(|(index, amount)| {
15101 json!({
15102 "event_type": "SignalReceived",
15103 "payload": {
15104 "signal_id": format!("increment-{amount}"),
15105 "signal_name": "increment",
15106 "workflow_sequence": index + 2,
15107 "payload_codec": DEFAULT_CODEC,
15108 "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
15109 .expect("Avro signal envelope")
15110 }
15111 })
15112 }))
15113 .collect::<Vec<_>>();
15114 let (resume_id, resume_name, resume_arguments) = if finish {
15115 history_events.push(json!({
15116 "event_type": "SignalReceived",
15117 "payload": {
15118 "signal_id": "finish",
15119 "signal_name": "finish",
15120 "workflow_sequence": 4,
15121 "payload_codec": DEFAULT_CODEC,
15122 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
15123 .expect("Avro finish signal envelope")
15124 }
15125 }));
15126 (
15127 "finish".to_string(),
15128 "finish".to_string(),
15129 encode_value_envelope(&json!([]), DEFAULT_CODEC)
15130 .expect("Avro finish resume signal"),
15131 )
15132 } else {
15133 let amount = amounts.last().expect("amount");
15134 (
15135 format!("increment-{amount}"),
15136 "increment".to_string(),
15137 encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
15138 .expect("Avro increment resume signal"),
15139 )
15140 };
15141 let body = json!({
15142 "task": {
15143 "task_id": task_id,
15144 "workflow_id": "snapshot-1",
15145 "run_id": "snapshot-run-1",
15146 "workflow_type": "snapshot",
15147 "payload_codec": DEFAULT_CODEC,
15148 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
15149 .expect("Avro workflow arguments"),
15150 "history_events": history_events,
15151 "workflow_task_attempt": 1,
15152 "workflow_signal_id": resume_id,
15153 "signal_name": resume_name,
15154 "signal_arguments": resume_arguments,
15155 "lease_owner": "rust-snapshot-worker"
15156 }
15157 })
15158 .to_string();
15159 write_mock_response(stream, "200 OK", &body);
15160 return;
15161 }
15162
15163 if path == "/api/worker/query-tasks/poll" && request_number == 1 {
15164 let history_events = [3, 5]
15165 .into_iter()
15166 .enumerate()
15167 .map(|(index, amount)| {
15168 json!({
15169 "event_type": "SignalReceived",
15170 "payload": {
15171 "signal_id": format!("increment-{amount}"),
15172 "signal_name": "increment",
15173 "workflow_sequence": index + 2,
15174 "payload_codec": DEFAULT_CODEC,
15175 "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
15176 .expect("Avro query signal envelope")
15177 }
15178 })
15179 })
15180 .collect::<Vec<_>>();
15181 let body = json!({
15182 "task": {
15183 "query_task_id": "snapshot-current",
15184 "query_task_attempt": 1,
15185 "lease_owner": "rust-snapshot-worker",
15186 "workflow_id": "snapshot-1",
15187 "run_id": "snapshot-run-1",
15188 "workflow_type": "snapshot",
15189 "query_name": "current",
15190 "payload_codec": DEFAULT_CODEC,
15191 "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
15192 .expect("Avro workflow arguments"),
15193 "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
15194 .expect("Avro query arguments"),
15195 "history_events": history_events,
15196 "run_status": "waiting"
15197 }
15198 })
15199 .to_string();
15200 write_mock_response(stream, "200 OK", &body);
15201 return;
15202 }
15203
15204 if path == "/api/worker/workflow-tasks/snapshot-wait-3/fail"
15205 || path == "/api/worker/workflow-tasks/snapshot-wait-5/fail"
15206 {
15207 write_mock_response(
15208 stream,
15209 "200 OK",
15210 r#"{"outcome":"waiting_for_history","recorded":true}"#,
15211 );
15212 return;
15213 }
15214
15215 if path == "/api/worker/workflow-tasks/snapshot-open/complete" {
15216 write_mock_response(stream, "200 OK", r#"{"outcome":"waiting","recorded":true}"#);
15217 return;
15218 }
15219
15220 if path == "/api/worker/workflow-tasks/snapshot-finish/complete" {
15221 write_mock_response(
15222 stream,
15223 "200 OK",
15224 r#"{"outcome":"completed","run_status":"completed","recorded":true}"#,
15225 );
15226 return;
15227 }
15228
15229 if path == "/api/worker/query-tasks/snapshot-current/complete" {
15230 write_mock_response(stream, "200 OK", r#"{"outcome":"completed"}"#);
15231 return;
15232 }
15233 }
15234
15235 if matches!(
15236 path,
15237 "/api/workflows/typed-1/query/inspect" | "/api/workflows/typed-1/update/replace"
15238 ) {
15239 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
15240 .expect("typed mock result");
15241 let body = json!({
15242 "result": typed_fidelity_probe().into_json().expect("result projection"),
15243 "result_envelope": result,
15244 })
15245 .to_string();
15246 write_mock_response(stream, "200 OK", &body);
15247 return;
15248 }
15249
15250 if path == "/api/workflows/typed-1" {
15251 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
15252 .expect("typed mock result");
15253 let body = json!({
15254 "workflow_id": "typed-1",
15255 "run_id": "run-typed-1",
15256 "workflow_type": "typed.echo",
15257 "status": "completed",
15258 "output": typed_fidelity_probe().into_json().expect("output projection"),
15259 "output_envelope": result,
15260 })
15261 .to_string();
15262 write_mock_response(stream, "200 OK", &body);
15263 return;
15264 }
15265
15266 let (status, body) = match path {
15267 "/api/health" => ("200 OK", r#"{"status":"ok"}"#),
15268 "/api/workflows" => (
15269 "201 Created",
15270 r#"{"workflow_id":"wf-start-options","run_id":"run-start-options","workflow_type":"rust.timeout"}"#,
15271 ),
15272 "/api/worker/register" if behavior.decline_registration => (
15273 "200 OK",
15274 r#"{"worker_id":"declined-worker","registered":false}"#,
15275 ),
15276 "/api/worker/register" if behavior.waiting_query_worker => (
15277 "200 OK",
15278 r#"{"worker_id":"rust-snapshot-worker","registered":true,"heartbeat_interval_seconds":1}"#,
15279 ),
15280 "/api/worker/register" => (
15281 "200 OK",
15282 r#"{"worker_id":"mock-worker","registered":true,"heartbeat_interval_seconds":3600}"#,
15283 ),
15284 "/api/worker/heartbeat" => ("200 OK", "{}"),
15285 "/api/worker/activity-tasks/poll"
15286 if behavior.cancelled_activity && request_number == 1 =>
15287 {
15288 (
15289 "200 OK",
15290 r#"{"task":{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","activity_type":"cancel-aware","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"attempt_number":1,"lease_owner":"rust-cancel-worker"}}"#,
15291 )
15292 }
15293 "/api/worker/activity-tasks/poll" | "/api/worker/workflow-tasks/poll" => {
15294 ("200 OK", r#"{"task":null}"#)
15295 }
15296 "/api/worker/query-tasks/poll"
15297 if behavior.reject_query_completion && request_number == 1 =>
15298 {
15299 (
15300 "200 OK",
15301 r#"{"task":{"query_task_id":"query-late","query_task_attempt":1,"lease_owner":"late-worker","workflow_id":"counter-late","run_id":"run-late","workflow_type":"counter","query_name":"current","payload_codec":"avro","workflow_arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"query_arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"history_events":[],"run_status":"running"}}"#,
15302 )
15303 }
15304 "/api/worker/query-tasks/poll" => ("200 OK", r#"{"task":null}"#),
15305 "/api/worker/query-tasks/query-capture/complete"
15306 | "/api/worker/query-tasks/query-capture/fail" => ("200 OK", "{}"),
15307 "/api/worker/activity-tasks/activity-cancel/heartbeat" => (
15308 "200 OK",
15309 r#"{"activity_attempt_id":"attempt-cancel","cancel_requested":true,"can_continue":false,"reason":"run_cancelled","run_closed_reason":"cancelled","heartbeat_recorded":false}"#,
15310 ),
15311 "/api/worker/activity-tasks/activity-cancel/complete" => (
15312 "409 Conflict",
15313 r#"{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","reason":"run_cancelled","cancel_requested":true,"can_continue":false,"run_closed_reason":"cancelled"}"#,
15314 ),
15315 "/api/worker/activity-tasks/activity-typed/complete"
15316 | "/api/worker/activity-tasks/activity-typed/fail"
15317 | "/api/workflows/typed-1/signal/changed" => ("200 OK", "{}"),
15318 "/api/workflows/counter-1/query/current" => (
15319 "200 OK",
15320 r#"{"workflow_id":"counter-1","query_name":"current","result":{"count":8},"result_envelope":{"codec":"avro","blob":"wwHioz3/VYAiNw4CCmNvdW50BBAA"}}"#,
15321 ),
15322 "/api/workflows/counter-1/query/missing" => (
15323 "404 Not Found",
15324 r#"{"workflow_id":"counter-1","query_name":"missing","reason":"rejected_unknown_query","message":"unknown query"}"#,
15325 ),
15326 "/api/workflows/wf-lifecycle/cancel" => (
15327 "200 OK",
15328 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","reason":"cleanup requested","command_status":"accepted"}"#,
15329 ),
15330 "/api/workflows/wf-lifecycle/terminate" => (
15331 "200 OK",
15332 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","reason":"forced stop","command_status":"accepted"}"#,
15333 ),
15334 "/api/workflows/wf-lifecycle/runs/run-current/cancel" => (
15335 "200 OK",
15336 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","command_status":"accepted"}"#,
15337 ),
15338 "/api/workflows/wf-lifecycle/runs/run-current/terminate" => (
15339 "200 OK",
15340 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","command_status":"accepted"}"#,
15341 ),
15342 "/api/workflows/wf-lifecycle/runs/run-stale/cancel"
15343 | "/api/workflows/wf-lifecycle/runs/run-stale/terminate" => (
15344 "409 Conflict",
15345 r#"{"workflow_id":"wf-lifecycle","run_id":"run-stale","reason":"historical_run_command_rejected","target_scope":"run","message":"Commands cannot target historical runs."}"#,
15346 ),
15347 "/api/workflows/wf-failed" | "/api/workflows/wf-failed/runs/run-failed" => (
15348 "200 OK",
15349 r#"{"workflow_id":"wf-failed","run_id":"run-failed","status":"failed","closed_reason":"failed","error":"payment failed","failure":{"message":"payment failed","failure_category":"application","exception_type":"PaymentError","exception_class":"billing::PaymentError","non_retryable":true,"exception":{"type":"PaymentError","class":"billing::PaymentError","message":"payment failed"},"failures":[{"id":"failure-17","failure_category":"application"}]}}"#,
15350 ),
15351 "/api/workflows/wf-cancelled" => (
15352 "200 OK",
15353 r#"{"workflow_id":"wf-cancelled","run_id":"run-cancelled","status":"cancelled","closed_reason":"cancelled","reason":"cleanup requested"}"#,
15354 ),
15355 "/api/workflows/wf-terminated" => (
15356 "200 OK",
15357 r#"{"workflow_id":"wf-terminated","run_id":"run-terminated","status":"terminated","closed_reason":"terminated","reason":"forced stop"}"#,
15358 ),
15359 "/api/workflows/wf-timed-out" => (
15360 "200 OK",
15361 r#"{"workflow_id":"wf-timed-out","run_id":"run-timed-out","status":"failed","closed_reason":"timed_out","reason":"run_timeout"}"#,
15362 ),
15363 "/api/workflows/wf-waiting" | "/api/workflows/wf-waiting/runs/run-waiting" => (
15364 "200 OK",
15365 r#"{"workflow_id":"wf-waiting","run_id":"run-waiting","status":"waiting"}"#,
15366 ),
15367 "/api/workflows/wf-selected" => (
15368 "200 OK",
15369 r#"{"workflow_id":"wf-selected","run_id":"run-current","status":"completed","output":"current run output"}"#,
15370 ),
15371 "/api/workflows/wf-selected/runs/run-selected" => (
15372 "200 OK",
15373 r#"{"workflow_id":"wf-selected","run_id":"run-selected","status":"cancelled","closed_reason":"cancelled","reason":"selected run cancelled"}"#,
15374 ),
15375 _ => ("404 Not Found", r#"{"message":"not found"}"#),
15376 };
15377 write_mock_response(stream, status, body);
15378 }
15379
15380 fn mock_request_is_complete(request: &[u8]) -> bool {
15381 let Some(header_end) = request
15382 .windows(4)
15383 .position(|window| window == b"\r\n\r\n")
15384 .map(|position| position + 4)
15385 else {
15386 return false;
15387 };
15388 let headers = String::from_utf8_lossy(&request[..header_end]);
15389 let content_length = headers.lines().find_map(|line| {
15390 let (name, value) = line.split_once(':')?;
15391 name.eq_ignore_ascii_case("content-length")
15392 .then(|| value.trim().parse::<usize>().ok())
15393 .flatten()
15394 });
15395
15396 request.len() >= header_end + content_length.unwrap_or(0)
15397 }
15398
15399 fn write_mock_response(stream: &mut TcpStream, status: &str, body: &str) {
15400 let response = format!(
15401 "HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
15402 body.len()
15403 );
15404
15405 let _ = stream.write_all(response.as_bytes());
15406 let _ = stream.flush();
15407 }
15408}