1#![doc = include_str!("../README.md")]
2
3use std::{
4 any::{Any, TypeId},
5 collections::{BTreeMap, HashMap},
6 future::Future,
7 io::{self, Read},
8 pin::Pin,
9 sync::{
10 atomic::{AtomicBool, Ordering},
11 Arc, Mutex, OnceLock,
12 },
13 task::{Context as TaskContext, Poll},
14 time::{Duration, Instant, SystemTime, UNIX_EPOCH},
15};
16
17use apache_avro::{from_avro_datum, to_avro_datum, types::Value as AvroDatum, Schema};
18use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
19use futures_util::{future::OptionFuture, task::noop_waker_ref};
20use serde::{
21 de::DeserializeOwned,
22 ser::{SerializeMap, SerializeSeq},
23 Deserialize, Serialize, Serializer,
24};
25pub use serde_json::{json, Value};
26use thiserror::Error;
27pub use uuid::Uuid;
28
29pub const WORKER_PROTOCOL_VERSION: &str = "1.2";
30pub const CONTROL_PLANE_VERSION: &str = "2";
31pub const DEFAULT_CODEC: &str = "avro";
32pub const JSON_CODEC: &str = "json";
33pub const SDK_VERSION: &str = concat!("durable-workflow-rust/", env!("CARGO_PKG_VERSION"));
34pub const QUERY_TASKS_CAPABILITY: &str = "query_tasks";
36pub const WORKFLOW_UPDATES_CAPABILITY: &str = "workflow_updates";
38pub const QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
40
41const MAX_LONG_POLL_TIMEOUT_SECONDS: u64 = 60;
42const WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE: &str =
43 "Workflow task waiting for scheduled history.";
44const WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE: &str = "WorkflowTaskWaitingForHistory";
45
46const QUERY_TASK_FINAL_REJECTION_REASONS: &[&str] = &[
47 "lease_expired",
48 "query_task_not_found",
49 "query_task_not_leased",
50 "query_task_timed_out",
51];
52
53pub const AVRO_VALUE_SCHEMA_JSON: &str =
55 include_str!("../schema/durable_workflow.protocol.Value.v1.avsc");
56pub const AVRO_VALUE_SCHEMA_FINGERPRINT_HEX: &str = "e2a33dff55802237";
57pub const AVRO_VALUE_SCHEMA_FINGERPRINT: [u8; 8] = [0xe2, 0xa3, 0x3d, 0xff, 0x55, 0x80, 0x22, 0x37];
58const AVRO_SINGLE_OBJECT_MAGIC: [u8; 2] = [0xc3, 0x01];
59
60static AVRO_VALUE_SCHEMA: OnceLock<std::result::Result<Schema, String>> = OnceLock::new();
61
62#[derive(Clone, Copy)]
63enum RequestProtocol {
64 ControlPlane,
65 Worker(&'static str),
66}
67
68impl RequestProtocol {
69 fn is_worker(self) -> bool {
70 matches!(self, Self::Worker(_))
71 }
72}
73
74pub type Result<T> = std::result::Result<T, Error>;
75
76#[derive(Debug, Error)]
77pub enum Error {
78 #[error("transport error: {0}")]
79 Transport(#[from] reqwest::Error),
80 #[error(
81 "invalid Durable Workflow base URL: omit the SDK-owned /api suffix and pass the Server or Cloud runtime base URL; the SDK appends /api automatically"
82 )]
83 InvalidBaseUrl,
84 #[error("json error: {0}")]
85 Json(#[from] serde_json::Error),
86 #[error("http {status}: {body}")]
87 Http {
88 status: reqwest::StatusCode,
89 body: String,
90 },
91 #[error("codec error: {0}")]
92 Codec(String),
93 #[error(transparent)]
94 QueryFailed(QueryFailure),
95 #[error(transparent)]
96 Protocol(ProtocolFailure),
97 #[error(transparent)]
98 NonDeterministicReplay(ReplayFailure),
99 #[error(transparent)]
100 ChildWorkflowFailed(ChildWorkflowFailure),
101 #[error(transparent)]
102 ActivityFailed(ActivityFailure),
103 #[error(transparent)]
104 WorkflowCommandRejected(WorkflowCommandRejection),
105 #[error(transparent)]
106 WorkflowFailed(WorkflowTerminalOutcome),
107 #[error(transparent)]
108 WorkflowCancelled(WorkflowTerminalOutcome),
109 #[error(transparent)]
110 WorkflowTerminated(WorkflowTerminalOutcome),
111 #[error(transparent)]
112 WorkflowTimedOut(WorkflowTerminalOutcome),
113 #[error(transparent)]
114 ActivityTaskRejected(ActivityTaskRejection),
115 #[error("workflow handler {0:?} is not registered")]
116 WorkflowNotRegistered(String),
117 #[error("activity handler {0:?} is not registered")]
118 ActivityNotRegistered(String),
119 #[error("workflow future yielded without emitting a durable command")]
120 WorkflowYieldedWithoutCommand,
121 #[error("workflow state lock is poisoned")]
122 WorkflowStatePoisoned,
123 #[error("timer duration is too large for the worker protocol")]
124 TimerDurationOverflow,
125 #[error("operation timed out")]
126 Timeout,
127 #[error("worker loop error: {0}")]
128 WorkerLoop(String),
129 #[error("invalid child workflow options: {0}")]
130 InvalidChildWorkflowOptions(String),
131 #[error(transparent)]
132 InvalidActivityOptions(ActivityOptionsError),
133 #[error(transparent)]
134 InvalidContinueAsNewOptions(#[from] ContinueAsNewOptionsError),
135 #[doc(hidden)]
136 #[error("workflow requested continue as new")]
137 ContinueAsNew(ContinueAsNewRequest),
138}
139
140#[derive(Clone, Copy, Debug, PartialEq, Eq)]
142pub enum WorkflowCommandKind {
143 Cancel,
144 Terminate,
145}
146
147impl WorkflowCommandKind {
148 fn as_str(self) -> &'static str {
149 match self {
150 Self::Cancel => "cancel",
151 Self::Terminate => "terminate",
152 }
153 }
154}
155
156#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
158pub struct WorkflowCommandOptions {
159 #[serde(skip_serializing_if = "Option::is_none")]
160 pub reason: Option<String>,
161 #[serde(skip_serializing_if = "Option::is_none")]
162 pub request_id: Option<String>,
163}
164
165#[derive(Clone, Debug, PartialEq, Eq)]
172pub struct WorkflowStartOptions {
173 pub execution_timeout_seconds: u64,
174 pub run_timeout_seconds: u64,
175}
176
177impl Default for WorkflowStartOptions {
178 fn default() -> Self {
179 Self {
180 execution_timeout_seconds: 3600,
181 run_timeout_seconds: 600,
182 }
183 }
184}
185
186impl WorkflowStartOptions {
187 pub fn new() -> Self {
188 Self::default()
189 }
190
191 pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
192 self.execution_timeout_seconds = seconds;
193 self
194 }
195
196 pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
197 self.run_timeout_seconds = seconds;
198 self
199 }
200
201 fn validate(&self) -> Result<()> {
202 if self.execution_timeout_seconds == 0 {
203 return Err(Error::Codec(
204 "execution_timeout_seconds must be at least 1".to_string(),
205 ));
206 }
207 if self.run_timeout_seconds == 0 {
208 return Err(Error::Codec(
209 "run_timeout_seconds must be at least 1".to_string(),
210 ));
211 }
212 if self.run_timeout_seconds > self.execution_timeout_seconds {
213 return Err(Error::Codec(
214 "run_timeout_seconds cannot exceed execution_timeout_seconds".to_string(),
215 ));
216 }
217
218 Ok(())
219 }
220}
221
222#[derive(Clone, Debug, Default, PartialEq, Eq)]
227pub struct ContinueAsNewOptions {
228 pub workflow_type: Option<String>,
229 pub task_queue: Option<String>,
230}
231
232impl ContinueAsNewOptions {
233 pub fn new() -> Self {
234 Self::default()
235 }
236
237 pub fn workflow_type(mut self, workflow_type: impl Into<String>) -> Self {
238 self.workflow_type = Some(workflow_type.into());
239 self
240 }
241
242 pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
243 self.task_queue = Some(task_queue.into());
244 self
245 }
246
247 fn validate(&self) -> std::result::Result<(), ContinueAsNewOptionsError> {
248 for (field, value) in [
249 ("workflow_type", self.workflow_type.as_deref()),
250 ("task_queue", self.task_queue.as_deref()),
251 ] {
252 if value.is_some_and(|value| value.trim().is_empty()) {
253 return Err(ContinueAsNewOptionsError {
254 field,
255 message: format!("{field} must not be empty"),
256 });
257 }
258 }
259 Ok(())
260 }
261}
262
263#[derive(Clone, Debug, Error, PartialEq, Eq)]
265#[error("invalid continue-as-new option {field}: {message}")]
266pub struct ContinueAsNewOptionsError {
267 pub field: &'static str,
268 pub message: String,
269}
270
271#[derive(Clone, Debug, Default, PartialEq, Eq)]
273pub struct WorkflowHistoryBudget {
274 pub event_count: u64,
275 pub size_bytes: Option<u64>,
276 pub continue_as_new_recommended: bool,
277 pub pressure: Option<String>,
278}
279
280#[doc(hidden)]
281#[derive(Clone, Debug)]
282pub struct ContinueAsNewRequest {
283 arguments: AvroValue,
284 options: ContinueAsNewOptions,
285}
286
287impl WorkflowCommandOptions {
288 pub fn new() -> Self {
289 Self::default()
290 }
291
292 pub fn reason(mut self, reason: impl Into<String>) -> Self {
293 self.reason = Some(reason.into());
294 self
295 }
296
297 pub fn request_id(mut self, request_id: impl Into<String>) -> Self {
298 self.request_id = Some(request_id.into());
299 self
300 }
301}
302
303#[derive(Clone, Debug, PartialEq)]
305pub struct WorkflowCommandResult {
306 pub command: WorkflowCommandKind,
307 pub workflow_id: String,
308 pub run_id: Option<String>,
309 pub outcome: Option<String>,
310 pub reason: Option<String>,
311 pub command_status: Option<String>,
312 pub raw: Value,
313}
314
315#[derive(Clone, Debug, Error)]
317#[error("workflow {command:?} rejected ({reason}, HTTP {status}): {message}")]
318pub struct WorkflowCommandRejection {
319 pub command: WorkflowCommandKind,
320 pub status: u16,
321 pub reason: String,
322 pub message: String,
323 pub workflow_id: String,
324 pub run_id: Option<String>,
325 pub target_scope: Option<String>,
326 pub body: Value,
327}
328
329#[derive(Clone, Copy, Debug, PartialEq, Eq)]
331pub enum WorkflowTerminalKind {
332 Failed,
333 Cancelled,
334 Terminated,
335 TimedOut,
336}
337
338#[derive(Clone, Debug, Error)]
344#[error("workflow {workflow_id} run {run_id:?} ended as {kind:?} ({reason})")]
345pub struct WorkflowTerminalOutcome {
346 pub kind: WorkflowTerminalKind,
347 pub workflow_id: String,
348 pub run_id: Option<String>,
349 pub reason: String,
350 pub failure_category: Option<String>,
351 pub failure_id: Option<String>,
352 pub exception_type: Option<String>,
353 pub exception_class: Option<String>,
354 pub non_retryable: Option<bool>,
355 pub message: Option<String>,
356 pub exception: Option<Value>,
357 pub raw: Value,
358}
359
360#[derive(Clone, Debug, Error)]
362#[error("activity task {operation} rejected ({reason}, HTTP {status})")]
363pub struct ActivityTaskRejection {
364 pub operation: String,
365 pub status: u16,
366 pub reason: String,
367 pub task_id: String,
368 pub activity_attempt_id: String,
369 pub cancel_requested: bool,
370 pub can_continue: Option<bool>,
371 pub run_closed_reason: Option<String>,
372 pub body: Value,
373}
374
375#[derive(Clone, Copy, Debug, PartialEq, Eq)]
377pub enum ActivityOptionsErrorKind {
378 EmptyTaskQueue,
379 EmptyRetryPolicy,
380 InvalidMaxAttempts,
381 BackoffWithoutRetryBudget,
382 TooManyBackoffIntervals,
383 InvalidBackoffCoefficient,
384 BackoffGenerationTooLarge,
385 BackoffOverflow,
386 EmptyNonRetryableErrorType,
387 TimeoutNotPositive,
388 TimeoutOverflow,
389 TimeoutOrder,
390}
391
392#[derive(Clone, Debug, Error, PartialEq, Eq)]
394#[error("invalid activity options ({kind:?}, {field:?}): {message}")]
395pub struct ActivityOptionsError {
396 pub kind: ActivityOptionsErrorKind,
397 pub field: Option<&'static str>,
398 pub message: String,
399}
400
401impl ActivityOptionsError {
402 fn new(
403 kind: ActivityOptionsErrorKind,
404 field: Option<&'static str>,
405 message: impl Into<String>,
406 ) -> Self {
407 Self {
408 kind,
409 field,
410 message: message.into(),
411 }
412 }
413}
414
415#[derive(Clone, Copy, Debug, PartialEq, Eq)]
417pub enum ActivityFailureKind {
418 Failed,
419 Cancelled,
420 TimedOut,
421}
422
423#[derive(Clone, Debug, Error)]
428#[error("activity failed ({reason}): {message}")]
429pub struct ActivityFailure {
430 pub kind: ActivityFailureKind,
431 pub reason: String,
432 pub message: String,
433 pub activity_execution_id: Option<String>,
434 pub activity_attempt_id: Option<String>,
435 pub activity_type: Option<String>,
436 pub activity_class: Option<String>,
437 pub attempt_number: Option<u64>,
438 pub failure_id: Option<String>,
439 pub failure_category: Option<String>,
440 pub timeout_kind: Option<String>,
441 pub non_retryable: bool,
442 pub exception_type: Option<String>,
443 pub exception_class: Option<String>,
444 pub code: Option<Value>,
445 pub exception: Option<Value>,
446}
447
448#[derive(Clone, Copy, Debug, PartialEq, Eq)]
450pub enum ChildWorkflowFailureKind {
451 Failed,
452 Cancelled,
453 Terminated,
454}
455
456#[derive(Clone, Debug, Error)]
462#[error("child workflow failed ({reason}): {message}")]
463pub struct ChildWorkflowFailure {
464 pub kind: ChildWorkflowFailureKind,
465 pub reason: String,
466 pub message: String,
467 pub parent_workflow_id: Option<String>,
468 pub parent_workflow_run_id: Option<String>,
469 pub child_workflow_id: Option<String>,
470 pub child_workflow_run_id: Option<String>,
471 pub child_workflow_type: Option<String>,
472 pub failure_id: Option<String>,
473 pub failure_category: Option<String>,
474 pub exception_type: Option<String>,
475 pub exception_class: Option<String>,
476 pub non_retryable: bool,
477 pub code: Option<Value>,
478 pub exception: Option<Value>,
479}
480
481#[derive(Clone, Debug, PartialEq, Eq)]
483pub struct WorkflowIdentity {
484 pub workflow_id: Option<String>,
485 pub run_id: Option<String>,
486}
487
488#[derive(Clone, Debug, PartialEq)]
490pub struct ChildWorkflowResult {
491 pub parent: WorkflowIdentity,
492 pub child: WorkflowIdentity,
493 pub child_workflow_type: Option<String>,
494 pub result: Value,
495}
496
497#[derive(Clone, Debug, PartialEq)]
499pub struct ChildWorkflowAvroResult {
500 pub parent: WorkflowIdentity,
501 pub child: WorkflowIdentity,
502 pub child_workflow_type: Option<String>,
503 pub result: AvroValue,
504}
505
506#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
508pub enum ParentClosePolicy {
509 #[default]
510 Abandon,
511 RequestCancel,
512 Terminate,
513}
514
515impl ParentClosePolicy {
516 fn as_str(self) -> &'static str {
517 match self {
518 Self::Abandon => "abandon",
519 Self::RequestCancel => "request_cancel",
520 Self::Terminate => "terminate",
521 }
522 }
523}
524
525#[derive(Clone, Debug, Default, PartialEq, Eq)]
527pub struct ChildWorkflowRetryPolicy {
528 pub max_attempts: Option<u32>,
529 pub backoff_seconds: Vec<u64>,
530 pub non_retryable_error_types: Vec<String>,
531}
532
533#[derive(Clone, Debug, PartialEq, Eq)]
537pub struct ChildWorkflowOptions {
538 pub task_queue: String,
539 pub parent_close_policy: ParentClosePolicy,
540 pub retry_policy: Option<ChildWorkflowRetryPolicy>,
541 pub execution_timeout_seconds: Option<u64>,
542 pub run_timeout_seconds: Option<u64>,
543}
544
545impl ChildWorkflowOptions {
546 pub fn new(task_queue: impl Into<String>) -> Self {
547 Self {
548 task_queue: task_queue.into(),
549 parent_close_policy: ParentClosePolicy::Abandon,
550 retry_policy: None,
551 execution_timeout_seconds: None,
552 run_timeout_seconds: None,
553 }
554 }
555
556 pub fn parent_close_policy(mut self, policy: ParentClosePolicy) -> Self {
557 self.parent_close_policy = policy;
558 self
559 }
560
561 pub fn retry_policy(mut self, policy: ChildWorkflowRetryPolicy) -> Self {
562 self.retry_policy = Some(policy);
563 self
564 }
565
566 pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
567 self.execution_timeout_seconds = Some(seconds);
568 self
569 }
570
571 pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
572 self.run_timeout_seconds = Some(seconds);
573 self
574 }
575}
576
577#[derive(Clone, Debug, PartialEq, Eq)]
579pub enum ActivityBackoff {
580 Explicit(Vec<Duration>),
583 Exponential {
585 initial_interval: Duration,
586 coefficient: u32,
587 maximum_interval: Option<Duration>,
588 },
589}
590
591#[derive(Clone, Debug, Default, PartialEq, Eq)]
593pub struct ActivityRetryPolicy {
594 pub max_attempts: Option<u32>,
595 pub backoff: Option<ActivityBackoff>,
596 pub non_retryable_error_types: Vec<String>,
597}
598
599impl ActivityRetryPolicy {
600 pub fn new(max_attempts: u32) -> Self {
602 Self {
603 max_attempts: Some(max_attempts),
604 ..Self::default()
605 }
606 }
607
608 pub fn backoff_intervals(mut self, intervals: impl IntoIterator<Item = Duration>) -> Self {
609 self.backoff = Some(ActivityBackoff::Explicit(intervals.into_iter().collect()));
610 self
611 }
612
613 pub fn exponential_backoff(
614 mut self,
615 initial_interval: Duration,
616 coefficient: u32,
617 maximum_interval: Option<Duration>,
618 ) -> Self {
619 self.backoff = Some(ActivityBackoff::Exponential {
620 initial_interval,
621 coefficient,
622 maximum_interval,
623 });
624 self
625 }
626
627 pub fn non_retryable_error_type(mut self, error_type: impl Into<String>) -> Self {
628 self.non_retryable_error_types.push(error_type.into());
629 self
630 }
631
632 pub fn non_retryable_error_types(
633 mut self,
634 error_types: impl IntoIterator<Item = impl Into<String>>,
635 ) -> Self {
636 self.non_retryable_error_types
637 .extend(error_types.into_iter().map(Into::into));
638 self
639 }
640}
641
642#[derive(Clone, Debug, Default, PartialEq, Eq)]
647pub struct ActivityOptions {
648 pub task_queue: Option<String>,
649 pub retry_policy: Option<ActivityRetryPolicy>,
650 pub start_to_close_timeout: Option<Duration>,
651 pub schedule_to_start_timeout: Option<Duration>,
652 pub schedule_to_close_timeout: Option<Duration>,
653 pub heartbeat_timeout: Option<Duration>,
654}
655
656impl ActivityOptions {
657 pub fn new() -> Self {
658 Self::default()
659 }
660
661 pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
662 self.task_queue = Some(task_queue.into());
663 self
664 }
665
666 pub fn retry_policy(mut self, policy: ActivityRetryPolicy) -> Self {
667 self.retry_policy = Some(policy);
668 self
669 }
670
671 pub fn start_to_close_timeout(mut self, timeout: Duration) -> Self {
672 self.start_to_close_timeout = Some(timeout);
673 self
674 }
675
676 pub fn schedule_to_start_timeout(mut self, timeout: Duration) -> Self {
677 self.schedule_to_start_timeout = Some(timeout);
678 self
679 }
680
681 pub fn schedule_to_close_timeout(mut self, timeout: Duration) -> Self {
682 self.schedule_to_close_timeout = Some(timeout);
683 self
684 }
685
686 pub fn heartbeat_timeout(mut self, timeout: Duration) -> Self {
687 self.heartbeat_timeout = Some(timeout);
688 self
689 }
690
691 fn validate(&self) -> std::result::Result<ValidatedActivityOptions, ActivityOptionsError> {
692 if self
693 .task_queue
694 .as_deref()
695 .is_some_and(|queue| queue.trim().is_empty())
696 {
697 return Err(ActivityOptionsError::new(
698 ActivityOptionsErrorKind::EmptyTaskQueue,
699 Some("task_queue"),
700 "task_queue must not be empty",
701 ));
702 }
703
704 for (field, value) in [
705 ("start_to_close_timeout", self.start_to_close_timeout),
706 ("schedule_to_start_timeout", self.schedule_to_start_timeout),
707 ("schedule_to_close_timeout", self.schedule_to_close_timeout),
708 ("heartbeat_timeout", self.heartbeat_timeout),
709 ] {
710 if value.is_some_and(|value| value.is_zero()) {
711 return Err(ActivityOptionsError::new(
712 ActivityOptionsErrorKind::TimeoutNotPositive,
713 Some(field),
714 format!("{field} must be positive"),
715 ));
716 }
717 }
718
719 validate_timeout_order(
720 "heartbeat_timeout",
721 self.heartbeat_timeout,
722 "start_to_close_timeout",
723 self.start_to_close_timeout,
724 )?;
725 validate_timeout_order(
726 "start_to_close_timeout",
727 self.start_to_close_timeout,
728 "schedule_to_close_timeout",
729 self.schedule_to_close_timeout,
730 )?;
731 validate_timeout_order(
732 "schedule_to_start_timeout",
733 self.schedule_to_start_timeout,
734 "schedule_to_close_timeout",
735 self.schedule_to_close_timeout,
736 )?;
737
738 Ok(ValidatedActivityOptions {
739 task_queue: self.task_queue.clone(),
740 retry_policy: self
741 .retry_policy
742 .as_ref()
743 .map(validate_activity_retry_policy)
744 .transpose()?,
745 start_to_close_timeout: timeout_seconds(
746 "start_to_close_timeout",
747 self.start_to_close_timeout,
748 )?,
749 schedule_to_start_timeout: timeout_seconds(
750 "schedule_to_start_timeout",
751 self.schedule_to_start_timeout,
752 )?,
753 schedule_to_close_timeout: timeout_seconds(
754 "schedule_to_close_timeout",
755 self.schedule_to_close_timeout,
756 )?,
757 heartbeat_timeout: timeout_seconds("heartbeat_timeout", self.heartbeat_timeout)?,
758 })
759 }
760}
761
762#[derive(Clone, Debug)]
763struct ValidatedActivityOptions {
764 task_queue: Option<String>,
765 retry_policy: Option<Value>,
766 start_to_close_timeout: Option<u64>,
767 schedule_to_start_timeout: Option<u64>,
768 schedule_to_close_timeout: Option<u64>,
769 heartbeat_timeout: Option<u64>,
770}
771
772fn validate_timeout_order(
773 smaller_name: &'static str,
774 smaller: Option<Duration>,
775 larger_name: &'static str,
776 larger: Option<Duration>,
777) -> std::result::Result<(), ActivityOptionsError> {
778 if matches!((smaller, larger), (Some(smaller), Some(larger)) if smaller > larger) {
779 return Err(ActivityOptionsError::new(
780 ActivityOptionsErrorKind::TimeoutOrder,
781 Some(smaller_name),
782 format!("{smaller_name} must be <= {larger_name}"),
783 ));
784 }
785 Ok(())
786}
787
788fn timeout_seconds(
789 field: &'static str,
790 value: Option<Duration>,
791) -> std::result::Result<Option<u64>, ActivityOptionsError> {
792 value
793 .map(|value| {
794 activity_protocol_seconds(value).ok_or_else(|| {
795 ActivityOptionsError::new(
796 ActivityOptionsErrorKind::TimeoutOverflow,
797 Some(field),
798 format!("{field} is too large for the worker protocol"),
799 )
800 })
801 })
802 .transpose()
803}
804
805fn duration_seconds_ceil(value: Duration) -> Option<u64> {
806 value
807 .as_secs()
808 .checked_add(u64::from(value.subsec_nanos() > 0))
809}
810
811fn activity_protocol_seconds(value: Duration) -> Option<u64> {
812 duration_seconds_ceil(value).filter(|seconds| *seconds <= i64::MAX as u64)
813}
814
815fn validate_activity_retry_policy(
816 policy: &ActivityRetryPolicy,
817) -> std::result::Result<Value, ActivityOptionsError> {
818 if policy.max_attempts.is_none()
819 && policy.backoff.is_none()
820 && policy.non_retryable_error_types.is_empty()
821 {
822 return Err(ActivityOptionsError::new(
823 ActivityOptionsErrorKind::EmptyRetryPolicy,
824 Some("retry_policy"),
825 "retry_policy must configure at least one field",
826 ));
827 }
828 if policy.max_attempts == Some(0) {
829 return Err(ActivityOptionsError::new(
830 ActivityOptionsErrorKind::InvalidMaxAttempts,
831 Some("retry_policy.max_attempts"),
832 "max_attempts must be >= 1",
833 ));
834 }
835 if policy
836 .non_retryable_error_types
837 .iter()
838 .any(|error_type| error_type.trim().is_empty())
839 {
840 return Err(ActivityOptionsError::new(
841 ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
842 Some("retry_policy.non_retryable_error_types"),
843 "non_retryable_error_types must not contain empty values",
844 ));
845 }
846
847 let backoff_seconds = match &policy.backoff {
848 None => None,
849 Some(backoff) => {
850 let max_attempts = policy.max_attempts.ok_or_else(|| {
851 ActivityOptionsError::new(
852 ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
853 Some("retry_policy.backoff"),
854 "backoff requires max_attempts",
855 )
856 })?;
857 let retry_count = max_attempts.saturating_sub(1) as usize;
858 let intervals = match backoff {
859 ActivityBackoff::Explicit(intervals) => {
860 if intervals.len() > retry_count {
861 return Err(ActivityOptionsError::new(
862 ActivityOptionsErrorKind::TooManyBackoffIntervals,
863 Some("retry_policy.backoff"),
864 "backoff interval count must not exceed max_attempts - 1",
865 ));
866 }
867 intervals.clone()
868 }
869 ActivityBackoff::Exponential {
870 initial_interval,
871 coefficient,
872 maximum_interval,
873 } => {
874 if *coefficient < 1 {
875 return Err(ActivityOptionsError::new(
876 ActivityOptionsErrorKind::InvalidBackoffCoefficient,
877 Some("retry_policy.backoff.coefficient"),
878 "backoff coefficient must be >= 1",
879 ));
880 }
881 if retry_count > 10_000 {
882 return Err(ActivityOptionsError::new(
883 ActivityOptionsErrorKind::BackoffGenerationTooLarge,
884 Some("retry_policy.max_attempts"),
885 "generated backoff supports at most 10000 retry intervals",
886 ));
887 }
888 let mut current = *initial_interval;
889 let mut intervals = Vec::with_capacity(retry_count);
890 for _ in 0..retry_count {
891 let interval = maximum_interval
892 .map(|maximum| current.min(maximum))
893 .unwrap_or(current);
894 intervals.push(interval);
895 if maximum_interval.is_some_and(|maximum| interval == maximum) {
896 break;
897 }
898 current = current.checked_mul(*coefficient).ok_or_else(|| {
899 ActivityOptionsError::new(
900 ActivityOptionsErrorKind::BackoffOverflow,
901 Some("retry_policy.backoff"),
902 "generated backoff interval overflowed",
903 )
904 })?;
905 }
906 intervals
907 }
908 };
909 Some(
910 intervals
911 .into_iter()
912 .map(|interval| {
913 activity_protocol_seconds(interval).ok_or_else(|| {
914 ActivityOptionsError::new(
915 ActivityOptionsErrorKind::BackoffOverflow,
916 Some("retry_policy.backoff"),
917 "backoff interval is too large for the worker protocol",
918 )
919 })
920 })
921 .collect::<std::result::Result<Vec<_>, _>>()?,
922 )
923 }
924 };
925
926 let mut encoded = serde_json::Map::new();
927 if let Some(max_attempts) = policy.max_attempts {
928 encoded.insert("max_attempts".to_string(), json!(max_attempts));
929 }
930 if let Some(backoff_seconds) = backoff_seconds {
931 encoded.insert("backoff_seconds".to_string(), json!(backoff_seconds));
932 }
933 if !policy.non_retryable_error_types.is_empty() {
934 let mut canonical_error_types = Vec::new();
935 for error_type in policy
936 .non_retryable_error_types
937 .iter()
938 .map(|error_type| error_type.trim())
939 {
940 if !canonical_error_types.contains(&error_type) {
941 canonical_error_types.push(error_type);
942 }
943 }
944 encoded.insert(
945 "non_retryable_error_types".to_string(),
946 json!(canonical_error_types),
947 );
948 }
949 Ok(Value::Object(encoded))
950}
951
952#[derive(Clone, Debug, Error)]
955#[error("non-deterministic workflow replay ({reason}) at sequence {sequence:?}: {message}")]
956pub struct ReplayFailure {
957 pub reason: String,
958 pub sequence: Option<u64>,
959 pub expected: Option<String>,
960 pub actual: Option<String>,
961 pub message: String,
962}
963
964impl ReplayFailure {
965 fn new(
966 reason: impl Into<String>,
967 sequence: Option<u64>,
968 expected: Option<String>,
969 actual: Option<String>,
970 message: impl Into<String>,
971 ) -> Self {
972 Self {
973 reason: reason.into(),
974 sequence,
975 expected,
976 actual,
977 message: message.into(),
978 }
979 }
980}
981
982#[derive(Clone, Debug, Error)]
984#[error("query failed ({reason}, HTTP {status}): {message}")]
985pub struct QueryFailure {
986 pub status: u16,
987 pub reason: String,
988 pub message: String,
989 pub body: Value,
990}
991
992#[derive(Clone, Debug, Error)]
994#[error("protocol rejected ({reason}, HTTP {status}): {message}")]
995pub struct ProtocolFailure {
996 pub status: u16,
997 pub reason: String,
998 pub message: String,
999 pub supported_version: Option<String>,
1000 pub requested_version: Option<String>,
1001 pub body: Value,
1002}
1003
1004#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
1005pub struct PayloadEnvelope {
1006 pub codec: String,
1007 pub blob: String,
1008}
1009
1010impl PayloadEnvelope {
1011 pub fn avro<T: Serialize>(value: &T) -> Result<Self> {
1012 encode_payload(value, DEFAULT_CODEC)
1013 }
1014
1015 pub fn json<T: Serialize>(value: &T) -> Result<Self> {
1016 encode_payload(value, JSON_CODEC)
1017 }
1018
1019 pub fn avro_value(value: &AvroValue) -> Result<Self> {
1022 encode_avro_value(value)
1023 }
1024}
1025
1026#[derive(Clone, Debug, PartialEq)]
1028pub enum AvroValue {
1029 Null,
1030 Boolean(bool),
1031 Long(i64),
1032 Double(f64),
1033 Bytes(Vec<u8>),
1034 String(String),
1035 Array(Vec<AvroValue>),
1036 Map(BTreeMap<String, AvroValue>),
1037}
1038
1039impl AvroValue {
1040 fn from_serialize<T: Serialize>(value: &T) -> Result<Self> {
1041 Self::from_serde_value(
1042 serde_value::to_value(value).map_err(|error| {
1043 Error::Codec(format!("could not adapt value for Avro: {error}"))
1044 })?,
1045 )
1046 }
1047
1048 fn from_serde_value(value: serde_value::Value) -> Result<Self> {
1049 use serde_value::Value as SerdeValue;
1050
1051 match value {
1052 SerdeValue::Unit => Ok(Self::Null),
1053 SerdeValue::Bool(value) => Ok(Self::Boolean(value)),
1054 SerdeValue::I8(value) => Ok(Self::Long(i64::from(value))),
1055 SerdeValue::I16(value) => Ok(Self::Long(i64::from(value))),
1056 SerdeValue::I32(value) => Ok(Self::Long(i64::from(value))),
1057 SerdeValue::I64(value) => Ok(Self::Long(value)),
1058 SerdeValue::U8(value) => Ok(Self::Long(i64::from(value))),
1059 SerdeValue::U16(value) => Ok(Self::Long(i64::from(value))),
1060 SerdeValue::U32(value) => Ok(Self::Long(i64::from(value))),
1061 SerdeValue::U64(value) => i64::try_from(value).map(Self::Long).map_err(|_| {
1062 Error::Codec(
1063 "integer_overflow: Avro Value long must be within signed 64-bit range"
1064 .to_string(),
1065 )
1066 }),
1067 SerdeValue::F32(value) => Self::finite_double(f64::from(value)),
1068 SerdeValue::F64(value) => Self::finite_double(value),
1069 SerdeValue::Char(value) => Ok(Self::String(value.to_string())),
1070 SerdeValue::String(value) => Ok(Self::String(value)),
1071 SerdeValue::Bytes(value) => Ok(Self::Bytes(value)),
1072 SerdeValue::Option(None) => Ok(Self::Null),
1073 SerdeValue::Option(Some(value)) | SerdeValue::Newtype(value) => {
1074 Self::from_serde_value(*value)
1075 }
1076 SerdeValue::Seq(values) => values
1077 .into_iter()
1078 .map(Self::from_serde_value)
1079 .collect::<Result<Vec<_>>>()
1080 .map(Self::Array),
1081 SerdeValue::Map(values) => values
1082 .into_iter()
1083 .map(|(key, value)| {
1084 let SerdeValue::String(key) = key else {
1085 return Err(Error::Codec(
1086 "invalid_map_key: Avro Value map keys must be strings".to_string(),
1087 ));
1088 };
1089
1090 Ok((key, Self::from_serde_value(value)?))
1091 })
1092 .collect::<Result<BTreeMap<_, _>>>()
1093 .map(Self::Map),
1094 }
1095 }
1096
1097 fn finite_double(value: f64) -> Result<Self> {
1098 if !value.is_finite() {
1099 return Err(Error::Codec(
1100 "non_finite_float: Avro Value doubles must be finite".to_string(),
1101 ));
1102 }
1103
1104 Ok(Self::Double(value))
1105 }
1106
1107 fn into_json(self) -> Result<Value> {
1108 match self {
1109 Self::Null => Ok(Value::Null),
1110 Self::Boolean(value) => Ok(Value::Bool(value)),
1111 Self::Long(value) => Ok(Value::Number(value.into())),
1112 Self::Double(value) => serde_json::Number::from_f64(value)
1113 .map(Value::Number)
1114 .ok_or_else(|| {
1115 Error::Codec(
1116 "non_finite_float: decoded Avro Value double is not finite".to_string(),
1117 )
1118 }),
1119 Self::Bytes(value) => Ok(json!({
1120 "$type": "bytes",
1121 "base64": BASE64.encode(value),
1122 })),
1123 Self::String(value) => Ok(Value::String(value)),
1124 Self::Array(values) => values
1125 .into_iter()
1126 .map(Self::into_json)
1127 .collect::<Result<Vec<_>>>()
1128 .map(Value::Array),
1129 Self::Map(values) => values
1130 .into_iter()
1131 .map(|(key, value)| Ok((key, value.into_json()?)))
1132 .collect::<Result<serde_json::Map<_, _>>>()
1133 .map(Value::Object),
1134 }
1135 }
1136
1137 fn into_serde_value(self) -> serde_value::Value {
1138 use serde_value::Value as SerdeValue;
1139
1140 match self {
1141 Self::Null => SerdeValue::Unit,
1142 Self::Boolean(value) => SerdeValue::Bool(value),
1143 Self::Long(value) => SerdeValue::I64(value),
1144 Self::Double(value) => SerdeValue::F64(value),
1145 Self::Bytes(value) => SerdeValue::Bytes(value),
1146 Self::String(value) => SerdeValue::String(value),
1147 Self::Array(values) => {
1148 SerdeValue::Seq(values.into_iter().map(Self::into_serde_value).collect())
1149 }
1150 Self::Map(values) => SerdeValue::Map(
1151 values
1152 .into_iter()
1153 .map(|(key, value)| (SerdeValue::String(key), value.into_serde_value()))
1154 .collect(),
1155 ),
1156 }
1157 }
1158
1159 pub fn deserialize<T: DeserializeOwned>(self) -> Result<T> {
1160 self.into_serde_value().deserialize_into().map_err(|error| {
1161 Error::Codec(format!(
1162 "avro_value_type_mismatch: could not adapt decoded value: {error}"
1163 ))
1164 })
1165 }
1166}
1167
1168impl Serialize for AvroValue {
1169 fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
1170 where
1171 S: Serializer,
1172 {
1173 match self {
1174 Self::Null => serializer.serialize_unit(),
1175 Self::Boolean(value) => serializer.serialize_bool(*value),
1176 Self::Long(value) => serializer.serialize_i64(*value),
1177 Self::Double(value) => serializer.serialize_f64(*value),
1178 Self::Bytes(value) => serializer.serialize_bytes(value),
1179 Self::String(value) => serializer.serialize_str(value),
1180 Self::Array(values) => {
1181 let mut sequence = serializer.serialize_seq(Some(values.len()))?;
1182 for value in values {
1183 sequence.serialize_element(value)?;
1184 }
1185 sequence.end()
1186 }
1187 Self::Map(values) => {
1188 let mut map = serializer.serialize_map(Some(values.len()))?;
1189 for (key, value) in values {
1190 map.serialize_entry(key, value)?;
1191 }
1192 map.end()
1193 }
1194 }
1195 }
1196}
1197
1198pub fn encode_avro_value(value: &AvroValue) -> Result<PayloadEnvelope> {
1199 let datum = avro_value_to_datum(value)?;
1200 let datum = to_avro_datum(avro_value_schema()?, datum)
1201 .map_err(|err| Error::Codec(format!("avro_value_encode_failed: {err}")))?;
1202 let mut bytes = Vec::with_capacity(datum.len() + 10);
1203 bytes.extend_from_slice(&AVRO_SINGLE_OBJECT_MAGIC);
1204 bytes.extend_from_slice(&AVRO_VALUE_SCHEMA_FINGERPRINT);
1205 bytes.extend_from_slice(&datum);
1206 Ok(PayloadEnvelope {
1207 codec: DEFAULT_CODEC.to_string(),
1208 blob: BASE64.encode(bytes),
1209 })
1210}
1211
1212pub fn decode_avro_value(envelope: &PayloadEnvelope) -> Result<AvroValue> {
1213 if envelope.codec != DEFAULT_CODEC {
1214 return Err(Error::Codec(format!(
1215 "unsupported payload codec {:?}",
1216 envelope.codec
1217 )));
1218 }
1219 decode_avro_value_blob(&envelope.blob)
1220}
1221
1222pub fn encode_payload<T: Serialize>(value: &T, codec: &str) -> Result<PayloadEnvelope> {
1223 let blob = match codec {
1224 JSON_CODEC => serde_json::to_string(value)?,
1225 DEFAULT_CODEC => encode_avro_value(&AvroValue::from_serialize(value)?)?.blob,
1226 other => return Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1227 };
1228
1229 Ok(PayloadEnvelope {
1230 codec: codec.to_string(),
1231 blob,
1232 })
1233}
1234
1235pub fn decode_payload<T: DeserializeOwned>(envelope: &PayloadEnvelope) -> Result<T> {
1236 match envelope.codec.as_str() {
1237 JSON_CODEC => Ok(serde_json::from_str(&envelope.blob)?),
1238 DEFAULT_CODEC => decode_avro_value(envelope)?.deserialize(),
1239 other => Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1240 }
1241}
1242
1243#[cfg(test)]
1244fn encode_value_envelope(value: &Value, codec: &str) -> Result<Value> {
1245 Ok(serde_json::to_value(encode_payload(value, codec)?)?)
1246}
1247
1248fn decode_wire_value(value: &Value, fallback_codec: &str) -> Result<Value> {
1249 if value.is_null() {
1250 return Ok(Value::Null);
1251 }
1252
1253 if let Some(object) = value.as_object() {
1254 if let (Some(codec), Some(blob)) = (
1255 object.get("codec").and_then(Value::as_str),
1256 object.get("blob").and_then(Value::as_str),
1257 ) {
1258 return decode_blob(blob, codec);
1259 }
1260 }
1261
1262 if let Some(blob) = value.as_str() {
1263 return decode_blob(blob, fallback_codec);
1264 }
1265
1266 Ok(value.clone())
1267}
1268
1269fn encode_typed_envelope(value: &AvroValue, codec: &str) -> Result<Value> {
1270 let envelope = match codec {
1271 DEFAULT_CODEC => encode_avro_value(value)?,
1272 JSON_CODEC => PayloadEnvelope {
1273 codec: JSON_CODEC.to_string(),
1274 blob: serde_json::to_string(&value.clone().into_json()?)?,
1275 },
1276 other => return Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1277 };
1278 Ok(serde_json::to_value(envelope)?)
1279}
1280
1281fn decode_wire_avro_value(value: &Value, fallback_codec: &str) -> Result<AvroValue> {
1282 if value.is_null() {
1283 return Ok(AvroValue::Null);
1284 }
1285
1286 if let Some(object) = value.as_object() {
1287 if let (Some(codec), Some(blob)) = (
1288 object.get("codec").and_then(Value::as_str),
1289 object.get("blob").and_then(Value::as_str),
1290 ) {
1291 return match codec {
1292 DEFAULT_CODEC => decode_avro_value_blob(blob),
1293 JSON_CODEC => AvroValue::from_serialize(&serde_json::from_str::<Value>(blob)?),
1294 other => Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1295 };
1296 }
1297 }
1298
1299 if let Some(blob) = value.as_str() {
1300 return match fallback_codec {
1301 DEFAULT_CODEC => decode_avro_value_blob(blob),
1302 JSON_CODEC => AvroValue::from_serialize(&serde_json::from_str::<Value>(blob)?),
1303 other => Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1304 };
1305 }
1306
1307 AvroValue::from_serialize(value)
1308}
1309
1310fn normalize_avro_arguments(value: AvroValue) -> AvroValue {
1311 match value {
1312 AvroValue::Null => AvroValue::Array(Vec::new()),
1313 AvroValue::Array(_) => value,
1314 other => AvroValue::Array(vec![other]),
1315 }
1316}
1317
1318fn decode_blob(blob: &str, codec: &str) -> Result<Value> {
1319 match codec {
1320 JSON_CODEC => Ok(serde_json::from_str(blob)?),
1321 DEFAULT_CODEC => decode_avro_value_blob(blob)?.into_json(),
1322 other => Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1323 }
1324}
1325
1326fn decode_avro_value_blob(blob: &str) -> Result<AvroValue> {
1327 let bytes = BASE64.decode(blob).map_err(|err| {
1328 Error::Codec(format!(
1329 "invalid_payload_framing: expected strict base64 Avro single-object bytes: {err}"
1330 ))
1331 })?;
1332
1333 if bytes.len() < 10 || bytes[..2] != AVRO_SINGLE_OBJECT_MAGIC {
1334 return Err(Error::Codec(
1335 "invalid_payload_framing: expected Avro single-object magic c301".to_string(),
1336 ));
1337 }
1338
1339 let fingerprint: [u8; 8] = bytes[2..10]
1340 .try_into()
1341 .map_err(|_| Error::Codec("invalid Avro fingerprint length".to_string()))?;
1342 if fingerprint != AVRO_VALUE_SCHEMA_FINGERPRINT {
1343 return Err(Error::Codec(format!(
1344 "unsupported_payload_schema: unknown CRC-64-AVRO fingerprint {}",
1345 fingerprint
1346 .iter()
1347 .map(|byte| format!("{byte:02x}"))
1348 .collect::<String>()
1349 )));
1350 }
1351
1352 let mut datum_reader = StrictAvroDatumReader::new(&bytes[10..]);
1353 let datum = from_avro_datum(avro_value_schema()?, &mut datum_reader, None);
1357 if datum_reader.truncated {
1358 return Err(Error::Codec(
1359 "invalid_payload_framing: truncated Avro Value datum".to_string(),
1360 ));
1361 }
1362 let datum = datum.map_err(|err| {
1363 Error::Codec(format!(
1364 "invalid_payload_framing: malformed Avro Value datum: {err}"
1365 ))
1366 })?;
1367 if datum_reader.remaining() != 0 {
1368 return Err(Error::Codec(format!(
1369 "invalid_payload_framing: {} trailing bytes after Avro Value datum",
1370 datum_reader.remaining()
1371 )));
1372 }
1373 avro_value_from_datum(datum)
1374}
1375
1376struct StrictAvroDatumReader<'a> {
1377 bytes: &'a [u8],
1378 offset: usize,
1379 truncated: bool,
1380}
1381
1382impl<'a> StrictAvroDatumReader<'a> {
1383 fn new(bytes: &'a [u8]) -> Self {
1384 Self {
1385 bytes,
1386 offset: 0,
1387 truncated: false,
1388 }
1389 }
1390
1391 fn remaining(&self) -> usize {
1392 self.bytes.len() - self.offset
1393 }
1394}
1395
1396impl Read for StrictAvroDatumReader<'_> {
1397 fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
1398 let count = buffer.len().min(self.remaining());
1399 buffer[..count].copy_from_slice(&self.bytes[self.offset..self.offset + count]);
1400 self.offset += count;
1401 if count < buffer.len() {
1402 self.truncated = true;
1403 }
1404
1405 Ok(count)
1406 }
1407}
1408
1409fn avro_value_to_datum(value: &AvroValue) -> Result<AvroDatum> {
1410 let branch = match value {
1411 AvroValue::Null => AvroDatum::Union(0, Box::new(AvroDatum::Null)),
1412 AvroValue::Boolean(value) => AvroDatum::Union(
1413 1,
1414 Box::new(AvroDatum::Record(vec![(
1415 "boolean".to_string(),
1416 AvroDatum::Boolean(*value),
1417 )])),
1418 ),
1419 AvroValue::Long(value) => AvroDatum::Union(
1420 2,
1421 Box::new(AvroDatum::Record(vec![(
1422 "long".to_string(),
1423 AvroDatum::Long(*value),
1424 )])),
1425 ),
1426 AvroValue::Double(value) => {
1427 if !value.is_finite() {
1428 return Err(Error::Codec(
1429 "non_finite_float: Avro Value doubles must be finite".to_string(),
1430 ));
1431 }
1432 AvroDatum::Union(
1433 3,
1434 Box::new(AvroDatum::Record(vec![(
1435 "double".to_string(),
1436 AvroDatum::Double(*value),
1437 )])),
1438 )
1439 }
1440 AvroValue::Bytes(value) => AvroDatum::Union(
1441 4,
1442 Box::new(AvroDatum::Record(vec![(
1443 "bytes".to_string(),
1444 AvroDatum::Bytes(value.clone()),
1445 )])),
1446 ),
1447 AvroValue::String(value) => AvroDatum::Union(
1448 5,
1449 Box::new(AvroDatum::Record(vec![(
1450 "string".to_string(),
1451 AvroDatum::String(value.clone()),
1452 )])),
1453 ),
1454 AvroValue::Array(values) => AvroDatum::Union(
1455 6,
1456 Box::new(AvroDatum::Record(vec![(
1457 "items".to_string(),
1458 AvroDatum::Array(
1459 values
1460 .iter()
1461 .map(avro_value_to_datum)
1462 .collect::<Result<Vec<_>>>()?,
1463 ),
1464 )])),
1465 ),
1466 AvroValue::Map(values) => AvroDatum::Union(
1467 7,
1468 Box::new(AvroDatum::Record(vec![(
1469 "entries".to_string(),
1470 AvroDatum::Map(
1471 values
1472 .iter()
1473 .map(|(key, value)| Ok((key.clone(), avro_value_to_datum(value)?)))
1474 .collect::<Result<HashMap<_, _>>>()?,
1475 ),
1476 )])),
1477 ),
1478 };
1479 Ok(AvroDatum::Record(vec![("value".to_string(), branch)]))
1480}
1481
1482fn avro_value_from_datum(datum: AvroDatum) -> Result<AvroValue> {
1483 let AvroDatum::Record(mut outer) = datum else {
1484 return Err(Error::Codec(
1485 "invalid_payload_framing: datum is not a Value record".to_string(),
1486 ));
1487 };
1488 let (_, branch) = outer
1489 .pop()
1490 .filter(|(name, _)| name == "value")
1491 .ok_or_else(|| Error::Codec("invalid_payload_framing: Value field missing".to_string()))?;
1492 let AvroDatum::Union(_, branch) = branch else {
1493 return Err(Error::Codec(
1494 "invalid_payload_framing: invalid Value union".to_string(),
1495 ));
1496 };
1497 match *branch {
1498 AvroDatum::Null => Ok(AvroValue::Null),
1499 AvroDatum::Record(mut fields) => {
1500 let (name, value) = fields.pop().ok_or_else(|| {
1501 Error::Codec("invalid_payload_framing: empty Value branch".to_string())
1502 })?;
1503 match (name.as_str(), value) {
1504 ("boolean", AvroDatum::Boolean(value)) => Ok(AvroValue::Boolean(value)),
1505 ("long", AvroDatum::Long(value)) => Ok(AvroValue::Long(value)),
1506 ("double", AvroDatum::Double(value)) if value.is_finite() => {
1507 Ok(AvroValue::Double(value))
1508 }
1509 ("bytes", AvroDatum::Bytes(value)) => Ok(AvroValue::Bytes(value)),
1510 ("string", AvroDatum::String(value)) => Ok(AvroValue::String(value)),
1511 ("items", AvroDatum::Array(values)) => values
1512 .into_iter()
1513 .map(avro_value_from_datum)
1514 .collect::<Result<Vec<_>>>()
1515 .map(AvroValue::Array),
1516 ("entries", AvroDatum::Map(values)) => values
1517 .into_iter()
1518 .map(|(key, value)| Ok((key, avro_value_from_datum(value)?)))
1519 .collect::<Result<BTreeMap<_, _>>>()
1520 .map(AvroValue::Map),
1521 _ => Err(Error::Codec(
1522 "invalid_payload_framing: unknown Value branch".to_string(),
1523 )),
1524 }
1525 }
1526 _ => Err(Error::Codec(
1527 "invalid_payload_framing: invalid Value branch".to_string(),
1528 )),
1529 }
1530}
1531
1532fn avro_value_schema() -> Result<&'static Schema> {
1533 match AVRO_VALUE_SCHEMA.get_or_init(|| {
1534 Schema::parse_str(AVRO_VALUE_SCHEMA_JSON)
1535 .map_err(|err| format!("could not parse Avro Value schema: {err}"))
1536 }) {
1537 Ok(schema) => Ok(schema),
1538 Err(message) => Err(Error::Codec(message.clone())),
1539 }
1540}
1541
1542#[derive(Clone, Debug)]
1543pub struct Client {
1544 http: reqwest::Client,
1545 base_url: String,
1546 token: Option<String>,
1547 control_token: Option<String>,
1548 worker_token: Option<String>,
1549 namespace: String,
1550}
1551
1552impl Client {
1553 pub fn new(base_url: impl Into<String>) -> Result<Self> {
1554 Self::builder(base_url).build()
1555 }
1556
1557 pub fn builder(base_url: impl Into<String>) -> ClientBuilder {
1558 ClientBuilder {
1559 base_url: base_url.into(),
1560 token: None,
1561 control_token: None,
1562 worker_token: None,
1563 namespace: "default".to_string(),
1564 timeout: Duration::from_secs(60),
1565 }
1566 }
1567
1568 pub async fn health(&self) -> Result<Value> {
1569 self.request_json(
1570 reqwest::Method::GET,
1571 "/health",
1572 RequestProtocol::ControlPlane,
1573 Option::<&Value>::None,
1574 )
1575 .await
1576 }
1577
1578 pub async fn cluster_info(&self) -> Result<Value> {
1579 self.request_json(
1580 reqwest::Method::GET,
1581 "/cluster/info",
1582 RequestProtocol::ControlPlane,
1583 Option::<&Value>::None,
1584 )
1585 .await
1586 }
1587
1588 pub async fn start_workflow<T: Serialize>(
1589 &self,
1590 workflow_type: &str,
1591 task_queue: &str,
1592 workflow_id: &str,
1593 input: T,
1594 ) -> Result<WorkflowHandle> {
1595 self.start_workflow_with_options(
1596 workflow_type,
1597 task_queue,
1598 workflow_id,
1599 WorkflowStartOptions::default(),
1600 input,
1601 )
1602 .await
1603 }
1604
1605 pub async fn start_workflow_with_options<T: Serialize>(
1608 &self,
1609 workflow_type: &str,
1610 task_queue: &str,
1611 workflow_id: &str,
1612 options: WorkflowStartOptions,
1613 input: T,
1614 ) -> Result<WorkflowHandle> {
1615 options.validate()?;
1616 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1617 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
1618 let body = json!({
1619 "workflow_id": workflow_id,
1620 "workflow_type": workflow_type,
1621 "task_queue": task_queue,
1622 "input": input_envelope,
1623 "execution_timeout_seconds": options.execution_timeout_seconds,
1624 "run_timeout_seconds": options.run_timeout_seconds
1625 });
1626
1627 let data: Value = self
1628 .request_json(
1629 reqwest::Method::POST,
1630 "/workflows",
1631 RequestProtocol::ControlPlane,
1632 Some(&body),
1633 )
1634 .await?;
1635
1636 Ok(WorkflowHandle {
1637 client: self.clone(),
1638 workflow_id: data
1639 .get("workflow_id")
1640 .and_then(Value::as_str)
1641 .unwrap_or(workflow_id)
1642 .to_string(),
1643 run_id: data
1644 .get("run_id")
1645 .and_then(Value::as_str)
1646 .map(str::to_string),
1647 workflow_type: data
1648 .get("workflow_type")
1649 .and_then(Value::as_str)
1650 .unwrap_or(workflow_type)
1651 .to_string(),
1652 })
1653 }
1654
1655 pub async fn signal_workflow<T: Serialize>(
1656 &self,
1657 workflow_id: &str,
1658 signal_name: &str,
1659 input: T,
1660 ) -> Result<Value> {
1661 self.signal_workflow_target(workflow_id, None, signal_name, input)
1662 .await
1663 }
1664
1665 pub async fn signal_workflow_run<T: Serialize>(
1667 &self,
1668 workflow_id: &str,
1669 run_id: &str,
1670 signal_name: &str,
1671 input: T,
1672 ) -> Result<Value> {
1673 self.signal_workflow_target(workflow_id, Some(run_id), signal_name, input)
1674 .await
1675 }
1676
1677 async fn signal_workflow_target<T: Serialize>(
1678 &self,
1679 workflow_id: &str,
1680 run_id: Option<&str>,
1681 signal_name: &str,
1682 input: T,
1683 ) -> Result<Value> {
1684 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1685 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
1686 let body = json!({
1687 "input": input_envelope
1688 });
1689 let path = match run_id {
1690 Some(run_id) => {
1691 format!("/workflows/{workflow_id}/runs/{run_id}/signal/{signal_name}")
1692 }
1693 None => format!("/workflows/{workflow_id}/signal/{signal_name}"),
1694 };
1695 self.request_json(
1696 reqwest::Method::POST,
1697 &path,
1698 RequestProtocol::ControlPlane,
1699 Some(&body),
1700 )
1701 .await
1702 }
1703
1704 pub async fn cancel_workflow(
1706 &self,
1707 workflow_id: &str,
1708 options: WorkflowCommandOptions,
1709 ) -> Result<WorkflowCommandResult> {
1710 self.workflow_command(workflow_id, None, WorkflowCommandKind::Cancel, options)
1711 .await
1712 }
1713
1714 pub async fn cancel_workflow_run(
1716 &self,
1717 workflow_id: &str,
1718 run_id: &str,
1719 options: WorkflowCommandOptions,
1720 ) -> Result<WorkflowCommandResult> {
1721 self.workflow_command(
1722 workflow_id,
1723 Some(run_id),
1724 WorkflowCommandKind::Cancel,
1725 options,
1726 )
1727 .await
1728 }
1729
1730 pub async fn terminate_workflow(
1732 &self,
1733 workflow_id: &str,
1734 options: WorkflowCommandOptions,
1735 ) -> Result<WorkflowCommandResult> {
1736 self.workflow_command(workflow_id, None, WorkflowCommandKind::Terminate, options)
1737 .await
1738 }
1739
1740 pub async fn terminate_workflow_run(
1742 &self,
1743 workflow_id: &str,
1744 run_id: &str,
1745 options: WorkflowCommandOptions,
1746 ) -> Result<WorkflowCommandResult> {
1747 self.workflow_command(
1748 workflow_id,
1749 Some(run_id),
1750 WorkflowCommandKind::Terminate,
1751 options,
1752 )
1753 .await
1754 }
1755
1756 async fn workflow_command(
1757 &self,
1758 workflow_id: &str,
1759 run_id: Option<&str>,
1760 command: WorkflowCommandKind,
1761 options: WorkflowCommandOptions,
1762 ) -> Result<WorkflowCommandResult> {
1763 let path = match run_id {
1764 Some(run_id) => format!(
1765 "/workflows/{workflow_id}/runs/{run_id}/{}",
1766 command.as_str()
1767 ),
1768 None => format!("/workflows/{workflow_id}/{}", command.as_str()),
1769 };
1770 let data = match self
1771 .request_json(
1772 reqwest::Method::POST,
1773 &path,
1774 RequestProtocol::ControlPlane,
1775 Some(&options),
1776 )
1777 .await
1778 {
1779 Ok(data) => data,
1780 Err(Error::Http { status, body }) => {
1781 return Err(Error::WorkflowCommandRejected(workflow_command_rejection(
1782 command,
1783 status,
1784 body,
1785 workflow_id,
1786 run_id,
1787 )));
1788 }
1789 Err(error) => return Err(error),
1790 };
1791
1792 Ok(workflow_command_result(command, data, workflow_id, run_id))
1793 }
1794
1795 pub async fn query_workflow<T: Serialize>(
1801 &self,
1802 workflow_id: &str,
1803 query_name: &str,
1804 input: T,
1805 ) -> Result<Value> {
1806 self.query_workflow_target(workflow_id, None, query_name, input)
1807 .await
1808 }
1809
1810 pub async fn query_workflow_run<T: Serialize>(
1812 &self,
1813 workflow_id: &str,
1814 run_id: &str,
1815 query_name: &str,
1816 input: T,
1817 ) -> Result<Value> {
1818 self.query_workflow_target(workflow_id, Some(run_id), query_name, input)
1819 .await
1820 }
1821
1822 pub async fn query_workflow_avro_value<T: Serialize>(
1824 &self,
1825 workflow_id: &str,
1826 query_name: &str,
1827 input: T,
1828 ) -> Result<AvroValue> {
1829 self.query_workflow_avro_value_target(workflow_id, None, query_name, input)
1830 .await
1831 }
1832
1833 pub async fn query_workflow_run_avro_value<T: Serialize>(
1835 &self,
1836 workflow_id: &str,
1837 run_id: &str,
1838 query_name: &str,
1839 input: T,
1840 ) -> Result<AvroValue> {
1841 self.query_workflow_avro_value_target(workflow_id, Some(run_id), query_name, input)
1842 .await
1843 }
1844
1845 async fn query_workflow_avro_value_target<T: Serialize>(
1846 &self,
1847 workflow_id: &str,
1848 run_id: Option<&str>,
1849 query_name: &str,
1850 input: T,
1851 ) -> Result<AvroValue> {
1852 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1853 let body = json!({"input": encode_typed_envelope(&input, DEFAULT_CODEC)?});
1854 let path = match run_id {
1855 Some(run_id) => {
1856 format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
1857 }
1858 None => format!("/workflows/{workflow_id}/query/{query_name}"),
1859 };
1860 let response: Value = match self
1861 .request_json(
1862 reqwest::Method::POST,
1863 &path,
1864 RequestProtocol::ControlPlane,
1865 Some(&body),
1866 )
1867 .await
1868 {
1869 Ok(response) => response,
1870 Err(Error::Http { status, body }) => {
1871 return Err(Error::QueryFailed(query_failure(status, body)));
1872 }
1873 Err(error) => return Err(error),
1874 };
1875
1876 let envelope = response
1877 .get("result_envelope")
1878 .filter(|envelope| !envelope.is_null())
1879 .ok_or_else(|| {
1880 Error::Codec(
1881 "missing_payload_envelope: typed query result requires result_envelope"
1882 .to_string(),
1883 )
1884 })?;
1885 decode_wire_avro_value(envelope, DEFAULT_CODEC)
1886 }
1887
1888 async fn query_workflow_target<T: Serialize>(
1889 &self,
1890 workflow_id: &str,
1891 run_id: Option<&str>,
1892 query_name: &str,
1893 input: T,
1894 ) -> Result<Value> {
1895 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1896 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
1897 let body = json!({
1898 "input": input_envelope
1899 });
1900 let path = match run_id {
1901 Some(run_id) => {
1902 format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
1903 }
1904 None => format!("/workflows/{workflow_id}/query/{query_name}"),
1905 };
1906 let response: Value = match self
1907 .request_json(
1908 reqwest::Method::POST,
1909 &path,
1910 RequestProtocol::ControlPlane,
1911 Some(&body),
1912 )
1913 .await
1914 {
1915 Ok(response) => response,
1916 Err(Error::Http { status, body }) => {
1917 return Err(Error::QueryFailed(query_failure(status, body)));
1918 }
1919 Err(error) => return Err(error),
1920 };
1921
1922 if let Some(envelope) = response
1923 .get("result_envelope")
1924 .filter(|envelope| !envelope.is_null())
1925 {
1926 return decode_wire_value(envelope, DEFAULT_CODEC);
1927 }
1928
1929 Ok(response.get("result").cloned().unwrap_or(Value::Null))
1930 }
1931
1932 pub async fn update_workflow<T: Serialize>(
1934 &self,
1935 workflow_id: &str,
1936 update_name: &str,
1937 input: T,
1938 request_id: Option<&str>,
1939 ) -> Result<Value> {
1940 let response = self
1941 .update_workflow_response(workflow_id, update_name, input, request_id)
1942 .await?;
1943 if let Some(envelope) = response
1944 .get("result_envelope")
1945 .filter(|envelope| !envelope.is_null())
1946 {
1947 return decode_wire_value(envelope, DEFAULT_CODEC);
1948 }
1949 Ok(response.get("result").cloned().unwrap_or(response))
1950 }
1951
1952 pub async fn update_workflow_avro_value<T: Serialize>(
1954 &self,
1955 workflow_id: &str,
1956 update_name: &str,
1957 input: T,
1958 request_id: Option<&str>,
1959 ) -> Result<AvroValue> {
1960 let response = self
1961 .update_workflow_response(workflow_id, update_name, input, request_id)
1962 .await?;
1963 let envelope = response
1964 .get("result_envelope")
1965 .filter(|envelope| !envelope.is_null())
1966 .ok_or_else(|| {
1967 Error::Codec(
1968 "missing_payload_envelope: typed update result requires result_envelope"
1969 .to_string(),
1970 )
1971 })?;
1972 decode_wire_avro_value(envelope, DEFAULT_CODEC)
1973 }
1974
1975 async fn update_workflow_response<T: Serialize>(
1976 &self,
1977 workflow_id: &str,
1978 update_name: &str,
1979 input: T,
1980 request_id: Option<&str>,
1981 ) -> Result<Value> {
1982 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1983 let mut body = json!({
1984 "input": encode_typed_envelope(&input, DEFAULT_CODEC)?,
1985 "wait_for": "completed",
1986 });
1987 if let Some(request_id) = request_id {
1988 body["request_id"] = json!(request_id);
1989 }
1990 self.request_json(
1991 reqwest::Method::POST,
1992 &format!("/workflows/{workflow_id}/update/{update_name}"),
1993 RequestProtocol::ControlPlane,
1994 Some(&body),
1995 )
1996 .await
1997 }
1998
1999 pub async fn describe_workflow(&self, workflow_id: &str) -> Result<WorkflowDescription> {
2000 let path = format!("/workflows/{workflow_id}");
2001 let mut data: WorkflowDescription = self
2002 .request_json(
2003 reqwest::Method::GET,
2004 &path,
2005 RequestProtocol::ControlPlane,
2006 Option::<&Value>::None,
2007 )
2008 .await?;
2009 data.decode_payloads()?;
2010 Ok(data)
2011 }
2012
2013 pub async fn describe_workflow_run(
2015 &self,
2016 workflow_id: &str,
2017 run_id: &str,
2018 ) -> Result<WorkflowDescription> {
2019 let path = format!("/workflows/{workflow_id}/runs/{run_id}");
2020 let mut data: WorkflowDescription = self
2021 .request_json(
2022 reqwest::Method::GET,
2023 &path,
2024 RequestProtocol::ControlPlane,
2025 Option::<&Value>::None,
2026 )
2027 .await?;
2028 data.decode_payloads()?;
2029 Ok(data)
2030 }
2031
2032 pub async fn register_worker(
2033 &self,
2034 worker_id: &str,
2035 task_queue: &str,
2036 supported_workflow_types: Vec<String>,
2037 supported_activity_types: Vec<String>,
2038 max_concurrent_workflow_tasks: usize,
2039 max_concurrent_activity_tasks: usize,
2040 ) -> Result<RegisterWorkerResponse> {
2041 self.register_worker_with_capabilities(
2042 worker_id,
2043 task_queue,
2044 supported_workflow_types,
2045 supported_activity_types,
2046 max_concurrent_workflow_tasks,
2047 max_concurrent_activity_tasks,
2048 Vec::new(),
2049 )
2050 .await
2051 }
2052
2053 pub async fn register_worker_with_capabilities(
2055 &self,
2056 worker_id: &str,
2057 task_queue: &str,
2058 supported_workflow_types: Vec<String>,
2059 supported_activity_types: Vec<String>,
2060 max_concurrent_workflow_tasks: usize,
2061 max_concurrent_activity_tasks: usize,
2062 capabilities: Vec<String>,
2063 ) -> Result<RegisterWorkerResponse> {
2064 self.register_worker_with_command_contracts(
2065 worker_id,
2066 task_queue,
2067 supported_workflow_types,
2068 supported_activity_types,
2069 max_concurrent_workflow_tasks,
2070 max_concurrent_activity_tasks,
2071 capabilities,
2072 Value::Object(serde_json::Map::new()),
2073 )
2074 .await
2075 }
2076
2077 #[allow(clippy::too_many_arguments)]
2079 pub async fn register_worker_with_command_contracts(
2080 &self,
2081 worker_id: &str,
2082 task_queue: &str,
2083 supported_workflow_types: Vec<String>,
2084 supported_activity_types: Vec<String>,
2085 max_concurrent_workflow_tasks: usize,
2086 max_concurrent_activity_tasks: usize,
2087 capabilities: Vec<String>,
2088 workflow_command_contracts: Value,
2089 ) -> Result<RegisterWorkerResponse> {
2090 let mut body = json!({
2091 "worker_id": worker_id,
2092 "task_queue": task_queue,
2093 "runtime": "rust",
2094 "sdk_version": SDK_VERSION,
2095 "supported_workflow_types": supported_workflow_types,
2096 "supported_activity_types": supported_activity_types,
2097 "capabilities": capabilities,
2098 "max_concurrent_workflow_tasks": max_concurrent_workflow_tasks,
2099 "max_concurrent_activity_tasks": max_concurrent_activity_tasks
2100 });
2101 if workflow_command_contracts
2102 .as_object()
2103 .is_some_and(|contracts| !contracts.is_empty())
2104 {
2105 body["workflow_command_contracts"] = workflow_command_contracts;
2106 }
2107
2108 self.request_json(
2109 reqwest::Method::POST,
2110 "/worker/register",
2111 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2112 Some(&body),
2113 )
2114 .await
2115 }
2116
2117 pub async fn poll_query_task(
2119 &self,
2120 worker_id: &str,
2121 task_queue: &str,
2122 timeout: Duration,
2123 ) -> Result<Option<QueryTask>> {
2124 Ok(self
2125 .poll_query_task_response(worker_id, task_queue, timeout)
2126 .await?
2127 .task)
2128 }
2129
2130 pub async fn poll_query_task_response(
2132 &self,
2133 worker_id: &str,
2134 task_queue: &str,
2135 timeout: Duration,
2136 ) -> Result<PollQueryTaskResponse> {
2137 let poll_request_id = unique_request_id("rust-query-poll");
2138 self.poll_query_task_response_with_request_id(
2139 worker_id,
2140 task_queue,
2141 timeout,
2142 &poll_request_id,
2143 1,
2144 )
2145 .await
2146 }
2147
2148 async fn poll_query_task_response_with_request_id(
2149 &self,
2150 worker_id: &str,
2151 task_queue: &str,
2152 timeout: Duration,
2153 poll_request_id: &str,
2154 transport_retries: usize,
2155 ) -> Result<PollQueryTaskResponse> {
2156 let timeout_seconds = long_poll_timeout_seconds(timeout);
2157 let body = json!({
2158 "worker_id": worker_id,
2159 "task_queue": task_queue,
2160 "poll_request_id": poll_request_id,
2161 "timeout_seconds": timeout_seconds,
2162 });
2163 self.poll_request_json(
2164 "/worker/query-tasks/poll",
2165 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
2166 &body,
2167 timeout + Duration::from_secs(5),
2168 transport_retries,
2169 )
2170 .await
2171 }
2172
2173 pub async fn complete_query_task<T: Serialize>(
2175 &self,
2176 query_task_id: &str,
2177 lease_owner: &str,
2178 query_task_attempt: u64,
2179 result: T,
2180 codec: &str,
2181 ) -> Result<Value> {
2182 let typed_result = AvroValue::from_serialize(&result)?;
2183 let result_envelope = encode_typed_envelope(&typed_result, codec)?;
2184 self.complete_query_task_with_envelope(
2185 query_task_id,
2186 lease_owner,
2187 query_task_attempt,
2188 typed_result.into_json()?,
2189 result_envelope,
2190 )
2191 .await
2192 }
2193
2194 async fn complete_query_task_with_envelope(
2195 &self,
2196 query_task_id: &str,
2197 lease_owner: &str,
2198 query_task_attempt: u64,
2199 result: Value,
2200 result_envelope: Value,
2201 ) -> Result<Value> {
2202 let body = json!({
2203 "lease_owner": lease_owner,
2204 "query_task_attempt": query_task_attempt,
2205 "result": result,
2206 "result_envelope": result_envelope,
2207 });
2208 let path = format!("/worker/query-tasks/{query_task_id}/complete");
2209 let response = self
2210 .request_json(
2211 reqwest::Method::POST,
2212 &path,
2213 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
2214 Some(&body),
2215 )
2216 .await;
2217 query_task_response(response)
2218 }
2219
2220 pub async fn fail_query_task(
2222 &self,
2223 query_task_id: &str,
2224 lease_owner: &str,
2225 query_task_attempt: u64,
2226 message: impl Into<String>,
2227 reason: impl Into<String>,
2228 failure_type: impl Into<String>,
2229 ) -> Result<Value> {
2230 let body = json!({
2231 "lease_owner": lease_owner,
2232 "query_task_attempt": query_task_attempt,
2233 "failure": {
2234 "message": message.into(),
2235 "reason": reason.into(),
2236 "type": failure_type.into(),
2237 }
2238 });
2239 let path = format!("/worker/query-tasks/{query_task_id}/fail");
2240 let response = self
2241 .request_json(
2242 reqwest::Method::POST,
2243 &path,
2244 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
2245 Some(&body),
2246 )
2247 .await;
2248 query_task_response(response)
2249 }
2250
2251 pub async fn heartbeat_worker(
2252 &self,
2253 worker_id: &str,
2254 workflow_available: usize,
2255 activity_available: usize,
2256 ) -> Result<Value> {
2257 let body = json!({
2258 "worker_id": worker_id,
2259 "task_slots": {
2260 "workflow_available": workflow_available,
2261 "activity_available": activity_available
2262 },
2263 "process_metrics": {
2264 "process_id": std::process::id(),
2265 "process_uptime_seconds": 0
2266 }
2267 });
2268
2269 self.request_json(
2270 reqwest::Method::POST,
2271 "/worker/heartbeat",
2272 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2273 Some(&body),
2274 )
2275 .await
2276 }
2277
2278 pub async fn poll_workflow_task(
2279 &self,
2280 worker_id: &str,
2281 task_queue: &str,
2282 timeout: Duration,
2283 ) -> Result<Option<WorkflowTask>> {
2284 Ok(self
2285 .poll_workflow_task_response(worker_id, task_queue, timeout)
2286 .await?
2287 .task)
2288 }
2289
2290 pub async fn poll_workflow_task_response(
2291 &self,
2292 worker_id: &str,
2293 task_queue: &str,
2294 timeout: Duration,
2295 ) -> Result<PollWorkflowTaskResponse> {
2296 let poll_request_id = unique_request_id("rust-workflow-poll");
2297 self.poll_workflow_task_response_with_request_id(
2298 worker_id,
2299 task_queue,
2300 timeout,
2301 &poll_request_id,
2302 1,
2303 )
2304 .await
2305 }
2306
2307 async fn poll_workflow_task_response_with_request_id(
2308 &self,
2309 worker_id: &str,
2310 task_queue: &str,
2311 timeout: Duration,
2312 poll_request_id: &str,
2313 transport_retries: usize,
2314 ) -> Result<PollWorkflowTaskResponse> {
2315 let body = json!({
2316 "worker_id": worker_id,
2317 "task_queue": task_queue,
2318 "poll_request_id": poll_request_id,
2319 "timeout_seconds": long_poll_timeout_seconds(timeout),
2320 });
2321 let mut data: PollWorkflowTaskResponse = self
2322 .poll_request_json(
2323 "/worker/workflow-tasks/poll",
2324 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2325 &body,
2326 timeout + Duration::from_secs(5),
2327 transport_retries,
2328 )
2329 .await?;
2330
2331 if let Some(task) = data.task.as_mut() {
2332 self.fetch_remaining_workflow_history(worker_id, task)
2333 .await?;
2334 }
2335
2336 Ok(data)
2337 }
2338
2339 async fn fetch_remaining_workflow_history(
2340 &self,
2341 worker_id: &str,
2342 task: &mut WorkflowTask,
2343 ) -> Result<()> {
2344 let mut next_token = task.next_history_page_token.clone();
2345
2346 while let Some(token) = next_token.take().filter(|token| !token.is_empty()) {
2347 let lease_owner = task
2348 .lease_owner
2349 .clone()
2350 .unwrap_or_else(|| worker_id.to_string());
2351 let page = self
2352 .workflow_task_history_page(
2353 &task.task_id,
2354 &lease_owner,
2355 task.workflow_task_attempt,
2356 &token,
2357 )
2358 .await?;
2359
2360 task.append_history_page(page);
2361
2362 if task.next_history_page_token.as_deref() == Some(token.as_str()) {
2363 return Err(Error::Codec(
2364 "workflow history pagination returned the same page token".to_string(),
2365 ));
2366 }
2367
2368 next_token = task.next_history_page_token.clone();
2369 }
2370
2371 Ok(())
2372 }
2373
2374 async fn workflow_task_history_page(
2375 &self,
2376 task_id: &str,
2377 lease_owner: &str,
2378 workflow_task_attempt: u64,
2379 next_history_page_token: &str,
2380 ) -> Result<WorkflowTaskHistoryPage> {
2381 let body = json!({
2382 "lease_owner": lease_owner,
2383 "workflow_task_attempt": workflow_task_attempt,
2384 "next_history_page_token": next_history_page_token
2385 });
2386 let path = format!("/worker/workflow-tasks/{task_id}/history");
2387
2388 self.request_json(
2389 reqwest::Method::POST,
2390 &path,
2391 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2392 Some(&body),
2393 )
2394 .await
2395 }
2396
2397 pub async fn complete_workflow_task(
2398 &self,
2399 task_id: &str,
2400 lease_owner: &str,
2401 workflow_task_attempt: u64,
2402 commands: Vec<Value>,
2403 ) -> Result<Value> {
2404 let body = json!({
2405 "lease_owner": lease_owner,
2406 "workflow_task_attempt": workflow_task_attempt,
2407 "commands": commands
2408 });
2409 let path = format!("/worker/workflow-tasks/{task_id}/complete");
2410 self.request_json(
2411 reqwest::Method::POST,
2412 &path,
2413 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2414 Some(&body),
2415 )
2416 .await
2417 }
2418
2419 pub async fn fail_workflow_task(
2420 &self,
2421 task_id: &str,
2422 lease_owner: &str,
2423 workflow_task_attempt: u64,
2424 message: impl Into<String>,
2425 ) -> Result<Value> {
2426 self.fail_workflow_task_with_type(
2427 task_id,
2428 lease_owner,
2429 workflow_task_attempt,
2430 message,
2431 "RustWorkflowTaskFailure",
2432 )
2433 .await
2434 }
2435
2436 async fn fail_workflow_task_with_type(
2437 &self,
2438 task_id: &str,
2439 lease_owner: &str,
2440 workflow_task_attempt: u64,
2441 message: impl Into<String>,
2442 failure_type: &str,
2443 ) -> Result<Value> {
2444 let body = json!({
2445 "lease_owner": lease_owner,
2446 "workflow_task_attempt": workflow_task_attempt,
2447 "failure": {
2448 "message": message.into(),
2449 "type": failure_type
2450 }
2451 });
2452 let path = format!("/worker/workflow-tasks/{task_id}/fail");
2453 self.request_json(
2454 reqwest::Method::POST,
2455 &path,
2456 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2457 Some(&body),
2458 )
2459 .await
2460 }
2461
2462 pub async fn poll_activity_task(
2463 &self,
2464 worker_id: &str,
2465 task_queue: &str,
2466 timeout: Duration,
2467 ) -> Result<Option<ActivityTask>> {
2468 Ok(self
2469 .poll_activity_task_response(worker_id, task_queue, timeout)
2470 .await?
2471 .task)
2472 }
2473
2474 pub async fn poll_activity_task_response(
2476 &self,
2477 worker_id: &str,
2478 task_queue: &str,
2479 timeout: Duration,
2480 ) -> Result<PollActivityTaskResponse> {
2481 let poll_request_id = unique_request_id("rust-activity-poll");
2482 self.poll_activity_task_response_with_request_id(
2483 worker_id,
2484 task_queue,
2485 timeout,
2486 &poll_request_id,
2487 1,
2488 )
2489 .await
2490 }
2491
2492 async fn poll_activity_task_response_with_request_id(
2493 &self,
2494 worker_id: &str,
2495 task_queue: &str,
2496 timeout: Duration,
2497 poll_request_id: &str,
2498 transport_retries: usize,
2499 ) -> Result<PollActivityTaskResponse> {
2500 let body = json!({
2501 "worker_id": worker_id,
2502 "task_queue": task_queue,
2503 "poll_request_id": poll_request_id,
2504 "timeout_seconds": long_poll_timeout_seconds(timeout),
2505 });
2506 let data: PollActivityTaskResponse = self
2507 .poll_request_json(
2508 "/worker/activity-tasks/poll",
2509 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2510 &body,
2511 timeout + Duration::from_secs(5),
2512 transport_retries,
2513 )
2514 .await?;
2515 Ok(data)
2516 }
2517
2518 pub async fn complete_activity_task<T: Serialize>(
2519 &self,
2520 task_id: &str,
2521 activity_attempt_id: &str,
2522 lease_owner: &str,
2523 result: T,
2524 codec: &str,
2525 ) -> Result<Value> {
2526 let result = encode_typed_envelope(&AvroValue::from_serialize(&result)?, codec)?;
2527 let body = json!({
2528 "activity_attempt_id": activity_attempt_id,
2529 "lease_owner": lease_owner,
2530 "result": result
2531 });
2532 let path = format!("/worker/activity-tasks/{task_id}/complete");
2533 activity_task_response(
2534 self.request_json(
2535 reqwest::Method::POST,
2536 &path,
2537 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2538 Some(&body),
2539 )
2540 .await,
2541 "complete",
2542 task_id,
2543 activity_attempt_id,
2544 )
2545 }
2546
2547 pub async fn fail_activity_task(
2548 &self,
2549 task_id: &str,
2550 activity_attempt_id: &str,
2551 lease_owner: &str,
2552 message: impl Into<String>,
2553 non_retryable: bool,
2554 ) -> Result<Value> {
2555 let body = json!({
2556 "activity_attempt_id": activity_attempt_id,
2557 "lease_owner": lease_owner,
2558 "failure": {
2559 "message": message.into(),
2560 "type": "RustActivityFailure",
2561 "non_retryable": non_retryable
2562 }
2563 });
2564 let path = format!("/worker/activity-tasks/{task_id}/fail");
2565 activity_task_response(
2566 self.request_json(
2567 reqwest::Method::POST,
2568 &path,
2569 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2570 Some(&body),
2571 )
2572 .await,
2573 "fail",
2574 task_id,
2575 activity_attempt_id,
2576 )
2577 }
2578
2579 pub async fn heartbeat_activity_task<T: Serialize>(
2580 &self,
2581 task_id: &str,
2582 activity_attempt_id: &str,
2583 lease_owner: &str,
2584 details: T,
2585 ) -> Result<ActivityHeartbeatResponse> {
2586 let details = encode_typed_envelope(&AvroValue::from_serialize(&details)?, DEFAULT_CODEC)?;
2587 let body = json!({
2588 "activity_attempt_id": activity_attempt_id,
2589 "lease_owner": lease_owner,
2590 "details": details
2591 });
2592 let path = format!("/worker/activity-tasks/{task_id}/heartbeat");
2593 activity_task_response(
2594 self.request_json(
2595 reqwest::Method::POST,
2596 &path,
2597 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2598 Some(&body),
2599 )
2600 .await,
2601 "heartbeat",
2602 task_id,
2603 activity_attempt_id,
2604 )
2605 }
2606
2607 async fn request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
2608 &self,
2609 method: reqwest::Method,
2610 path: &str,
2611 protocol: RequestProtocol,
2612 body: Option<&B>,
2613 ) -> Result<T> {
2614 self.request_json_with_timeout(method, path, protocol, body, Duration::from_secs(60))
2615 .await
2616 }
2617
2618 async fn request_json_with_timeout<T: DeserializeOwned, B: Serialize + ?Sized>(
2619 &self,
2620 method: reqwest::Method,
2621 path: &str,
2622 protocol: RequestProtocol,
2623 body: Option<&B>,
2624 timeout: Duration,
2625 ) -> Result<T> {
2626 let mut request = self
2627 .http
2628 .request(method, format!("{}/api{}", self.base_url, path))
2629 .timeout(timeout)
2630 .header(reqwest::header::ACCEPT, "application/json")
2631 .header(reqwest::header::CONTENT_TYPE, "application/json")
2632 .header("X-Namespace", &self.namespace);
2633
2634 match protocol {
2635 RequestProtocol::Worker(version) => {
2636 request = request.header("X-Durable-Workflow-Protocol-Version", version);
2637 }
2638 RequestProtocol::ControlPlane => {
2639 request = request.header(
2640 "X-Durable-Workflow-Control-Plane-Version",
2641 CONTROL_PLANE_VERSION,
2642 );
2643 }
2644 }
2645
2646 if let Some(token) = self.auth_token(protocol.is_worker()) {
2647 request = request.bearer_auth(token);
2648 }
2649
2650 if let Some(body) = body {
2651 request = request.json(body);
2652 }
2653
2654 let response = request.send().await?;
2655 let status = response.status();
2656 let bytes = response.bytes().await?;
2657
2658 if !status.is_success() {
2659 let body = String::from_utf8_lossy(&bytes).to_string();
2660 if let Some(protocol) = protocol_failure(status, &body) {
2661 return Err(Error::Protocol(protocol));
2662 }
2663 return Err(Error::Http { status, body });
2664 }
2665
2666 if bytes.is_empty() {
2667 return Ok(serde_json::from_value(Value::Null)?);
2668 }
2669
2670 Ok(serde_json::from_slice(&bytes)?)
2671 }
2672
2673 async fn poll_request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
2674 &self,
2675 path: &str,
2676 protocol: RequestProtocol,
2677 body: &B,
2678 timeout: Duration,
2679 max_retries: usize,
2680 ) -> Result<T> {
2681 let mut retries = 0;
2682
2683 loop {
2684 let response = self
2685 .request_json_with_timeout(
2686 reqwest::Method::POST,
2687 path,
2688 protocol,
2689 Some(body),
2690 timeout,
2691 )
2692 .await;
2693
2694 match response {
2695 Err(Error::Transport(_)) if retries < max_retries => retries += 1,
2696 response => return worker_poll_response(response),
2697 }
2698 }
2699 }
2700
2701 fn auth_token(&self, worker: bool) -> Option<&str> {
2702 if worker {
2703 self.worker_token
2704 .as_deref()
2705 .or(self.token.as_deref())
2706 .or(self.control_token.as_deref())
2707 } else {
2708 self.control_token
2709 .as_deref()
2710 .or(self.token.as_deref())
2711 .or(self.worker_token.as_deref())
2712 }
2713 }
2714}
2715
2716fn query_failure(status: reqwest::StatusCode, raw_body: String) -> QueryFailure {
2717 let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
2718 let reason = body
2719 .get("reason")
2720 .and_then(Value::as_str)
2721 .unwrap_or("query_rejected")
2722 .to_string();
2723 let message = body
2724 .get("message")
2725 .or_else(|| body.get("error"))
2726 .and_then(Value::as_str)
2727 .unwrap_or("workflow query was rejected")
2728 .to_string();
2729
2730 QueryFailure {
2731 status: status.as_u16(),
2732 reason,
2733 message,
2734 body,
2735 }
2736}
2737
2738fn workflow_command_result(
2739 command: WorkflowCommandKind,
2740 data: Value,
2741 workflow_id: &str,
2742 run_id: Option<&str>,
2743) -> WorkflowCommandResult {
2744 WorkflowCommandResult {
2745 command,
2746 workflow_id: data
2747 .get("workflow_id")
2748 .and_then(Value::as_str)
2749 .unwrap_or(workflow_id)
2750 .to_string(),
2751 run_id: data
2752 .get("run_id")
2753 .and_then(Value::as_str)
2754 .or(run_id)
2755 .map(str::to_string),
2756 outcome: data
2757 .get("outcome")
2758 .and_then(Value::as_str)
2759 .map(str::to_string),
2760 reason: data
2761 .get("reason")
2762 .and_then(Value::as_str)
2763 .map(str::to_string),
2764 command_status: data
2765 .get("command_status")
2766 .and_then(Value::as_str)
2767 .map(str::to_string),
2768 raw: data,
2769 }
2770}
2771
2772fn workflow_command_rejection(
2773 command: WorkflowCommandKind,
2774 status: reqwest::StatusCode,
2775 raw_body: String,
2776 workflow_id: &str,
2777 run_id: Option<&str>,
2778) -> WorkflowCommandRejection {
2779 let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
2780 WorkflowCommandRejection {
2781 command,
2782 status: status.as_u16(),
2783 reason: body
2784 .get("reason")
2785 .and_then(Value::as_str)
2786 .unwrap_or("workflow_command_rejected")
2787 .to_string(),
2788 message: body
2789 .get("message")
2790 .or_else(|| body.get("error"))
2791 .and_then(Value::as_str)
2792 .unwrap_or("workflow lifecycle command was rejected")
2793 .to_string(),
2794 workflow_id: body
2795 .get("workflow_id")
2796 .and_then(Value::as_str)
2797 .unwrap_or(workflow_id)
2798 .to_string(),
2799 run_id: body
2800 .get("run_id")
2801 .and_then(Value::as_str)
2802 .or(run_id)
2803 .map(str::to_string),
2804 target_scope: body
2805 .get("target_scope")
2806 .and_then(Value::as_str)
2807 .map(str::to_string),
2808 body,
2809 }
2810}
2811
2812fn query_task_response(response: Result<Value>) -> Result<Value> {
2813 match response {
2814 Err(Error::Http { status, body }) => Err(Error::QueryFailed(query_failure(status, body))),
2815 response => response,
2816 }
2817}
2818
2819fn worker_poll_response<T: DeserializeOwned>(response: Result<T>) -> Result<T> {
2820 match response {
2821 Err(Error::Http { status, body })
2822 if status == reqwest::StatusCode::CONFLICT && worker_poll_body_is_stop(&body) =>
2823 {
2824 Ok(serde_json::from_str(&body)?)
2825 }
2826 response => response,
2827 }
2828}
2829
2830fn worker_poll_body_is_stop(body: &str) -> bool {
2831 serde_json::from_str::<Value>(body)
2832 .ok()
2833 .is_some_and(|body| {
2834 worker_poll_is_stop(
2835 body.get("poll_status").and_then(Value::as_str),
2836 body.get("reason").and_then(Value::as_str),
2837 )
2838 })
2839}
2840
2841fn worker_poll_is_stop(poll_status: Option<&str>, reason: Option<&str>) -> bool {
2842 matches!(poll_status, Some("draining" | "stopped"))
2843 || matches!(reason, Some("worker_draining" | "worker_stopped"))
2844}
2845
2846fn query_task_rejection_is_final(error: &Error) -> bool {
2847 matches!(
2848 error,
2849 Error::QueryFailed(failure)
2850 if QUERY_TASK_FINAL_REJECTION_REASONS.contains(&failure.reason.as_str())
2851 )
2852}
2853
2854fn activity_task_response<T>(
2855 response: Result<T>,
2856 operation: &str,
2857 task_id: &str,
2858 activity_attempt_id: &str,
2859) -> Result<T> {
2860 match response {
2861 Err(Error::Http { status, body }) => {
2862 let body = serde_json::from_str(&body).unwrap_or_else(|_| json!({"message": body}));
2863 Err(Error::ActivityTaskRejected(ActivityTaskRejection {
2864 operation: operation.to_string(),
2865 status: status.as_u16(),
2866 reason: body
2867 .get("reason")
2868 .and_then(Value::as_str)
2869 .unwrap_or("activity_task_rejected")
2870 .to_string(),
2871 task_id: body
2872 .get("task_id")
2873 .and_then(Value::as_str)
2874 .unwrap_or(task_id)
2875 .to_string(),
2876 activity_attempt_id: body
2877 .get("activity_attempt_id")
2878 .and_then(Value::as_str)
2879 .unwrap_or(activity_attempt_id)
2880 .to_string(),
2881 cancel_requested: body
2882 .get("cancel_requested")
2883 .and_then(Value::as_bool)
2884 .unwrap_or(false),
2885 can_continue: body.get("can_continue").and_then(Value::as_bool),
2886 run_closed_reason: body
2887 .get("run_closed_reason")
2888 .and_then(Value::as_str)
2889 .map(str::to_string),
2890 body,
2891 }))
2892 }
2893 response => response,
2894 }
2895}
2896
2897fn activity_task_rejection_is_final(error: &Error) -> bool {
2898 matches!(
2899 error,
2900 Error::ActivityTaskRejected(rejection)
2901 if matches!(
2902 rejection.reason.as_str(),
2903 "run_cancelled"
2904 | "run_terminated"
2905 | "attempt_closed"
2906 | "stale_attempt"
2907 | "activity_cancelled"
2908 | "task_cancelled"
2909 | "run_closed"
2910 | "activity_not_running"
2911 | "attempt_not_found"
2912 )
2913 )
2914}
2915
2916fn workflow_task_completion_is_terminal_timeout(
2917 error: &Error,
2918 task_id: &str,
2919 workflow_task_attempt: u64,
2920 run_id: Option<&str>,
2921) -> bool {
2922 let Error::Http { status, body } = error else {
2923 return false;
2924 };
2925 if *status != reqwest::StatusCode::CONFLICT {
2926 return false;
2927 }
2928
2929 let Some(run_id) = run_id else {
2930 return false;
2931 };
2932 let Ok(body) = serde_json::from_str::<Value>(body) else {
2933 return false;
2934 };
2935
2936 body.get("recorded").and_then(Value::as_bool) == Some(false)
2937 && body.get("reason").and_then(Value::as_str) == Some("run_timed_out")
2938 && body.get("run_status").and_then(Value::as_str) == Some("failed")
2939 && body.get("run_id").and_then(Value::as_str) == Some(run_id)
2940 && body.get("task_id").and_then(Value::as_str) == Some(task_id)
2941 && body.get("workflow_task_attempt").and_then(Value::as_u64) == Some(workflow_task_attempt)
2942}
2943
2944fn protocol_failure(status: reqwest::StatusCode, raw_body: &str) -> Option<ProtocolFailure> {
2945 let body: Value = serde_json::from_str(raw_body).ok()?;
2946 let reason = body.get("reason")?.as_str()?;
2947 if !matches!(
2948 reason,
2949 "missing_protocol_version"
2950 | "unsupported_protocol_version"
2951 | "missing_control_plane_version"
2952 | "unsupported_control_plane_version"
2953 ) {
2954 return None;
2955 }
2956
2957 Some(ProtocolFailure {
2958 status: status.as_u16(),
2959 reason: reason.to_string(),
2960 message: body
2961 .get("message")
2962 .or_else(|| body.get("error"))
2963 .and_then(Value::as_str)
2964 .unwrap_or("protocol version rejected")
2965 .to_string(),
2966 supported_version: body
2967 .get("supported_version")
2968 .and_then(Value::as_str)
2969 .map(str::to_string),
2970 requested_version: body
2971 .get("requested_version")
2972 .and_then(Value::as_str)
2973 .map(str::to_string),
2974 body,
2975 })
2976}
2977
2978fn long_poll_timeout_seconds(timeout: Duration) -> u64 {
2979 timeout
2980 .as_secs()
2981 .saturating_add(u64::from(timeout.subsec_nanos() > 0))
2982 .min(MAX_LONG_POLL_TIMEOUT_SECONDS)
2983}
2984
2985fn worker_operation_is_retryable(error: &Error) -> bool {
2986 match error {
2987 Error::Transport(error) => {
2988 error.is_timeout() || error.is_connect() || error.is_request() || error.is_body()
2989 }
2990 Error::Http { status, .. } => {
2991 matches!(
2992 *status,
2993 reqwest::StatusCode::REQUEST_TIMEOUT | reqwest::StatusCode::TOO_MANY_REQUESTS
2994 ) || status.is_server_error()
2995 }
2996 _ => false,
2997 }
2998}
2999
3000fn worker_retry_delay(policy: WorkerRetryPolicy, retry: usize) -> Duration {
3001 let exponent = retry.saturating_sub(1).min(31) as u32;
3002 policy
3003 .initial_backoff
3004 .saturating_mul(1_u32 << exponent)
3005 .min(policy.max_backoff)
3006}
3007
3008#[derive(Debug)]
3009pub struct ClientBuilder {
3010 base_url: String,
3011 token: Option<String>,
3012 control_token: Option<String>,
3013 worker_token: Option<String>,
3014 namespace: String,
3015 timeout: Duration,
3016}
3017
3018impl ClientBuilder {
3019 pub fn token(mut self, token: Option<String>) -> Self {
3020 self.token = token;
3021 self
3022 }
3023
3024 pub fn control_token(mut self, token: Option<String>) -> Self {
3025 self.control_token = token;
3026 self
3027 }
3028
3029 pub fn worker_token(mut self, token: Option<String>) -> Self {
3030 self.worker_token = token;
3031 self
3032 }
3033
3034 pub fn namespace(mut self, namespace: impl Into<String>) -> Self {
3035 self.namespace = namespace.into();
3036 self
3037 }
3038
3039 pub fn timeout(mut self, timeout: Duration) -> Self {
3040 self.timeout = timeout;
3041 self
3042 }
3043
3044 pub fn build(self) -> Result<Client> {
3045 let base_url = self.base_url.trim_end_matches('/').to_string();
3046 let has_sdk_api_suffix = reqwest::Url::parse(&base_url)
3047 .map(|url| url.path().trim_end_matches('/').ends_with("/api"))
3048 .unwrap_or_else(|_| base_url.ends_with("/api"));
3049
3050 if has_sdk_api_suffix {
3051 return Err(Error::InvalidBaseUrl);
3052 }
3053
3054 Ok(Client {
3055 http: reqwest::Client::builder().timeout(self.timeout).build()?,
3056 base_url,
3057 token: self.token,
3058 control_token: self.control_token,
3059 worker_token: self.worker_token,
3060 namespace: self.namespace,
3061 })
3062 }
3063}
3064
3065#[derive(Clone, Debug)]
3066pub struct WorkflowHandle {
3067 client: Client,
3068 pub workflow_id: String,
3069 pub run_id: Option<String>,
3070 pub workflow_type: String,
3071}
3072
3073impl WorkflowHandle {
3074 pub async fn describe(&self) -> Result<WorkflowDescription> {
3076 self.client.describe_workflow(&self.workflow_id).await
3077 }
3078
3079 pub async fn describe_selected_run(&self) -> Result<WorkflowDescription> {
3081 let run_id = self.run_id.as_deref().ok_or_else(|| {
3082 Error::Codec("run_id is required for selected-run description".to_string())
3083 })?;
3084 self.client
3085 .describe_workflow_run(&self.workflow_id, run_id)
3086 .await
3087 }
3088
3089 pub async fn signal<T: Serialize>(&self, signal_name: &str, input: T) -> Result<Value> {
3090 self.client
3091 .signal_workflow(&self.workflow_id, signal_name, input)
3092 .await
3093 }
3094
3095 pub async fn signal_selected_run<T: Serialize>(
3097 &self,
3098 signal_name: &str,
3099 input: T,
3100 ) -> Result<Value> {
3101 let run_id = self.run_id.as_deref().ok_or_else(|| {
3102 Error::Codec("run_id is required for selected-run signaling".to_string())
3103 })?;
3104 self.client
3105 .signal_workflow_run(&self.workflow_id, run_id, signal_name, input)
3106 .await
3107 }
3108
3109 pub async fn cancel(&self, options: WorkflowCommandOptions) -> Result<WorkflowCommandResult> {
3111 self.client
3112 .cancel_workflow(&self.workflow_id, options)
3113 .await
3114 }
3115
3116 pub async fn cancel_selected_run(
3118 &self,
3119 options: WorkflowCommandOptions,
3120 ) -> Result<WorkflowCommandResult> {
3121 let run_id = self.run_id.as_deref().ok_or_else(|| {
3122 Error::Codec("run_id is required for selected-run cancellation".to_string())
3123 })?;
3124 self.client
3125 .cancel_workflow_run(&self.workflow_id, run_id, options)
3126 .await
3127 }
3128
3129 pub async fn terminate(
3131 &self,
3132 options: WorkflowCommandOptions,
3133 ) -> Result<WorkflowCommandResult> {
3134 self.client
3135 .terminate_workflow(&self.workflow_id, options)
3136 .await
3137 }
3138
3139 pub async fn terminate_selected_run(
3141 &self,
3142 options: WorkflowCommandOptions,
3143 ) -> Result<WorkflowCommandResult> {
3144 let run_id = self.run_id.as_deref().ok_or_else(|| {
3145 Error::Codec("run_id is required for selected-run termination".to_string())
3146 })?;
3147 self.client
3148 .terminate_workflow_run(&self.workflow_id, run_id, options)
3149 .await
3150 }
3151
3152 pub async fn query<T: Serialize>(&self, query_name: &str, input: T) -> Result<Value> {
3154 self.client
3155 .query_workflow(&self.workflow_id, query_name, input)
3156 .await
3157 }
3158
3159 pub async fn query_avro_value<T: Serialize>(
3160 &self,
3161 query_name: &str,
3162 input: T,
3163 ) -> Result<AvroValue> {
3164 self.client
3165 .query_workflow_avro_value(&self.workflow_id, query_name, input)
3166 .await
3167 }
3168
3169 pub async fn update<T: Serialize>(
3170 &self,
3171 update_name: &str,
3172 input: T,
3173 request_id: Option<&str>,
3174 ) -> Result<Value> {
3175 self.client
3176 .update_workflow(&self.workflow_id, update_name, input, request_id)
3177 .await
3178 }
3179
3180 pub async fn update_avro_value<T: Serialize>(
3181 &self,
3182 update_name: &str,
3183 input: T,
3184 request_id: Option<&str>,
3185 ) -> Result<AvroValue> {
3186 self.client
3187 .update_workflow_avro_value(&self.workflow_id, update_name, input, request_id)
3188 .await
3189 }
3190
3191 pub async fn query_selected_run<T: Serialize>(
3193 &self,
3194 query_name: &str,
3195 input: T,
3196 ) -> Result<Value> {
3197 let run_id = self
3198 .run_id
3199 .as_deref()
3200 .ok_or_else(|| Error::Codec("run_id is required for selected-run query".to_string()))?;
3201 self.client
3202 .query_workflow_run(&self.workflow_id, run_id, query_name, input)
3203 .await
3204 }
3205
3206 pub async fn result(&self, options: WorkflowResultOptions) -> Result<Value> {
3208 self.result_target(options, None).await
3209 }
3210
3211 pub async fn result_avro_value(&self, options: WorkflowResultOptions) -> Result<AvroValue> {
3213 self.result_avro_value_target(options, None).await
3214 }
3215
3216 pub async fn result_selected_run(&self, options: WorkflowResultOptions) -> Result<Value> {
3218 let run_id = self.run_id.as_deref().ok_or_else(|| {
3219 Error::Codec("run_id is required for selected-run result".to_string())
3220 })?;
3221 self.result_target(options, Some(run_id)).await
3222 }
3223
3224 pub async fn result_selected_run_avro_value(
3226 &self,
3227 options: WorkflowResultOptions,
3228 ) -> Result<AvroValue> {
3229 let run_id = self.run_id.as_deref().ok_or_else(|| {
3230 Error::Codec("run_id is required for selected-run result".to_string())
3231 })?;
3232 self.result_avro_value_target(options, Some(run_id)).await
3233 }
3234
3235 async fn result_avro_value_target(
3236 &self,
3237 options: WorkflowResultOptions,
3238 selected_run_id: Option<&str>,
3239 ) -> Result<AvroValue> {
3240 let started = Instant::now();
3241
3242 loop {
3243 let description = match selected_run_id {
3244 Some(run_id) => {
3245 self.client
3246 .describe_workflow_run(&self.workflow_id, run_id)
3247 .await?
3248 }
3249 None => self.describe().await?,
3250 };
3251 if description.is_completed() {
3252 return description.output_avro_value.ok_or_else(|| {
3253 Error::Codec(
3254 "missing_payload_envelope: typed workflow result requires output_envelope"
3255 .to_string(),
3256 )
3257 });
3258 }
3259 if description.is_terminal() {
3260 let outcome =
3261 workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
3262 return Err(match outcome.kind {
3263 WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
3264 WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
3265 WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
3266 WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
3267 });
3268 }
3269 if started.elapsed() >= options.timeout {
3270 return Err(Error::Timeout);
3271 }
3272 tokio::time::sleep(options.poll_interval).await;
3273 }
3274 }
3275
3276 async fn result_target(
3277 &self,
3278 options: WorkflowResultOptions,
3279 selected_run_id: Option<&str>,
3280 ) -> Result<Value> {
3281 let started = Instant::now();
3282
3283 loop {
3284 let description = match selected_run_id {
3285 Some(run_id) => {
3286 self.client
3287 .describe_workflow_run(&self.workflow_id, run_id)
3288 .await?
3289 }
3290 None => self.describe().await?,
3291 };
3292 if description.is_completed() {
3293 return Ok(description.output.unwrap_or(Value::Null));
3294 }
3295
3296 if description.is_terminal() {
3297 let outcome =
3298 workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
3299 return Err(match outcome.kind {
3300 WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
3301 WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
3302 WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
3303 WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
3304 });
3305 }
3306
3307 if started.elapsed() >= options.timeout {
3308 return Err(Error::WorkflowTimedOut(WorkflowTerminalOutcome {
3309 kind: WorkflowTerminalKind::TimedOut,
3310 workflow_id: description
3311 .workflow_id
3312 .clone()
3313 .unwrap_or_else(|| self.workflow_id.clone()),
3314 run_id: description
3315 .run_id
3316 .clone()
3317 .or_else(|| selected_run_id.map(str::to_string)),
3318 reason: "result_wait_timeout".to_string(),
3319 failure_category: Some("client_timeout".to_string()),
3320 failure_id: None,
3321 exception_type: None,
3322 exception_class: None,
3323 non_retryable: None,
3324 message: Some(format!(
3325 "workflow result was not terminal within {:?}",
3326 options.timeout
3327 )),
3328 exception: None,
3329 raw: description.raw_value(),
3330 }));
3331 }
3332
3333 tokio::time::sleep(options.poll_interval).await;
3334 }
3335 }
3336}
3337
3338#[derive(Clone, Copy, Debug)]
3339pub struct WorkflowResultOptions {
3340 pub poll_interval: Duration,
3341 pub timeout: Duration,
3342}
3343
3344impl Default for WorkflowResultOptions {
3345 fn default() -> Self {
3346 Self {
3347 poll_interval: Duration::from_millis(500),
3348 timeout: Duration::from_secs(30),
3349 }
3350 }
3351}
3352
3353#[derive(Clone, Debug, Deserialize)]
3354pub struct WorkflowDescription {
3355 pub workflow_id: Option<String>,
3356 pub run_id: Option<String>,
3357 pub workflow_type: Option<String>,
3358 pub status: Option<String>,
3359 #[serde(default)]
3360 pub closed_reason: Option<String>,
3361 #[serde(default)]
3362 pub error: Option<String>,
3363 #[serde(default)]
3364 pub failure: Option<Value>,
3365 #[serde(default)]
3366 pub exception: Option<Value>,
3367 #[serde(default)]
3368 pub failures: Vec<Value>,
3369 #[serde(default)]
3370 pub output: Option<Value>,
3371 #[serde(default)]
3372 pub output_envelope: Option<Value>,
3373 #[serde(skip)]
3374 pub output_avro_value: Option<AvroValue>,
3375 #[serde(flatten)]
3376 pub raw: HashMap<String, Value>,
3377}
3378
3379impl WorkflowDescription {
3380 pub fn is_completed(&self) -> bool {
3381 matches!(self.status.as_deref(), Some("completed" | "Completed"))
3382 }
3383
3384 pub fn is_terminal(&self) -> bool {
3385 matches!(
3386 self.status.as_deref(),
3387 Some(
3388 "completed"
3389 | "Completed"
3390 | "failed"
3391 | "Failed"
3392 | "cancelled"
3393 | "Cancelled"
3394 | "terminated"
3395 | "Terminated"
3396 | "timed_out"
3397 | "TimedOut",
3398 )
3399 )
3400 }
3401
3402 fn decode_payloads(&mut self) -> Result<()> {
3403 if let Some(envelope) = &self.output_envelope {
3404 let value = decode_wire_avro_value(envelope, DEFAULT_CODEC)?;
3405 self.output = Some(value.clone().into_json()?);
3406 self.output_avro_value = Some(value);
3407 }
3408
3409 Ok(())
3410 }
3411
3412 fn raw_value(&self) -> Value {
3413 let mut data = self.raw.clone();
3414 data.insert(
3415 "workflow_id".to_string(),
3416 self.workflow_id
3417 .clone()
3418 .map(Value::String)
3419 .unwrap_or(Value::Null),
3420 );
3421 data.insert(
3422 "run_id".to_string(),
3423 self.run_id
3424 .clone()
3425 .map(Value::String)
3426 .unwrap_or(Value::Null),
3427 );
3428 data.insert(
3429 "workflow_type".to_string(),
3430 self.workflow_type
3431 .clone()
3432 .map(Value::String)
3433 .unwrap_or(Value::Null),
3434 );
3435 data.insert(
3436 "status".to_string(),
3437 self.status
3438 .clone()
3439 .map(Value::String)
3440 .unwrap_or(Value::Null),
3441 );
3442 data.insert(
3443 "closed_reason".to_string(),
3444 self.closed_reason
3445 .clone()
3446 .map(Value::String)
3447 .unwrap_or(Value::Null),
3448 );
3449 if let Some(failure) = &self.failure {
3450 data.insert("failure".to_string(), failure.clone());
3451 }
3452 if let Some(exception) = &self.exception {
3453 data.insert("exception".to_string(), exception.clone());
3454 }
3455 Value::Object(data.into_iter().collect())
3456 }
3457}
3458
3459fn workflow_terminal_outcome(
3460 description: &WorkflowDescription,
3461 workflow_id: &str,
3462 run_id: Option<&str>,
3463) -> WorkflowTerminalOutcome {
3464 let terminal_kind = description
3465 .closed_reason
3466 .as_deref()
3467 .or(description.status.as_deref())
3468 .unwrap_or("failed")
3469 .to_ascii_lowercase();
3470 let kind = match terminal_kind.as_str() {
3471 "cancelled" | "canceled" => WorkflowTerminalKind::Cancelled,
3472 "terminated" => WorkflowTerminalKind::Terminated,
3473 "timed_out" | "timedout" => WorkflowTerminalKind::TimedOut,
3474 _ => WorkflowTerminalKind::Failed,
3475 };
3476 let default_reason = match kind {
3477 WorkflowTerminalKind::Failed => "workflow_failed",
3478 WorkflowTerminalKind::Cancelled => "cancelled",
3479 WorkflowTerminalKind::Terminated => "terminated",
3480 WorkflowTerminalKind::TimedOut => "timed_out",
3481 };
3482 let failure = description
3483 .failure
3484 .as_ref()
3485 .filter(|value| value.is_object());
3486 let nested_failure = failure
3487 .and_then(|value| value.get("failures"))
3488 .and_then(Value::as_array)
3489 .and_then(|failures| failures.last())
3490 .or_else(|| description.failures.last());
3491 let exception = description
3492 .exception
3493 .clone()
3494 .or_else(|| failure.and_then(|value| value.get("exception")).cloned())
3495 .or_else(|| {
3496 nested_failure
3497 .and_then(|value| value.get("exception_payload"))
3498 .cloned()
3499 });
3500 let string_field = |name: &str| {
3501 failure
3502 .and_then(|value| value.get(name))
3503 .and_then(Value::as_str)
3504 .or_else(|| {
3505 nested_failure
3506 .and_then(|value| value.get(name))
3507 .and_then(Value::as_str)
3508 })
3509 .map(str::to_string)
3510 };
3511 let exception_field = |name: &str| {
3512 exception
3513 .as_ref()
3514 .and_then(|value| value.get(name))
3515 .and_then(Value::as_str)
3516 .map(str::to_string)
3517 };
3518 let message = description
3519 .error
3520 .clone()
3521 .or_else(|| string_field("message"))
3522 .or_else(|| exception_field("message"));
3523 let reason = description
3524 .raw
3525 .get("reason")
3526 .and_then(Value::as_str)
3527 .map(str::to_string)
3528 .or_else(|| {
3529 failure
3530 .and_then(|value| value.get("reason"))
3531 .and_then(Value::as_str)
3532 .map(str::to_string)
3533 })
3534 .or_else(|| description.closed_reason.clone())
3535 .unwrap_or_else(|| default_reason.to_string());
3536 let failure_id = string_field("failure_id").or_else(|| {
3537 nested_failure
3538 .and_then(|value| value.get("id"))
3539 .and_then(Value::as_str)
3540 .map(str::to_string)
3541 });
3542
3543 WorkflowTerminalOutcome {
3544 kind,
3545 workflow_id: description
3546 .workflow_id
3547 .clone()
3548 .unwrap_or_else(|| workflow_id.to_string()),
3549 run_id: description
3550 .run_id
3551 .clone()
3552 .or_else(|| run_id.map(str::to_string)),
3553 reason,
3554 failure_category: string_field("failure_category")
3555 .or_else(|| Some(default_reason.to_string())),
3556 failure_id,
3557 exception_type: string_field("exception_type").or_else(|| exception_field("type")),
3558 exception_class: string_field("exception_class").or_else(|| exception_field("class")),
3559 non_retryable: failure
3560 .and_then(|value| value.get("non_retryable"))
3561 .and_then(Value::as_bool)
3562 .or_else(|| {
3563 nested_failure
3564 .and_then(|value| value.get("non_retryable"))
3565 .and_then(Value::as_bool)
3566 }),
3567 message,
3568 exception,
3569 raw: description.raw_value(),
3570 }
3571}
3572
3573#[derive(Clone, Debug, Deserialize)]
3574pub struct RegisterWorkerResponse {
3575 pub worker_id: String,
3576 pub registered: bool,
3577 #[serde(default)]
3578 pub heartbeat_interval_seconds: Option<u64>,
3579 #[serde(default)]
3580 pub protocol_version: Option<String>,
3581 #[serde(default)]
3582 pub server_capabilities: Option<Value>,
3583}
3584
3585#[derive(Clone, Debug, Deserialize)]
3586pub struct PollWorkflowTaskResponse {
3587 #[serde(default)]
3588 pub task: Option<WorkflowTask>,
3589 #[serde(default)]
3590 pub poll_status: Option<String>,
3591 #[serde(default)]
3592 pub reason: Option<String>,
3593 #[serde(default)]
3594 pub protocol_version: Option<String>,
3595 #[serde(default)]
3596 pub server_capabilities: Option<Value>,
3597}
3598
3599impl PollWorkflowTaskResponse {
3600 pub fn outcome(&self) -> WorkerPollOutcome {
3602 worker_poll_outcome(
3603 self.task.is_some(),
3604 self.poll_status.as_deref(),
3605 self.reason.as_deref(),
3606 )
3607 }
3608}
3609
3610#[derive(Clone, Debug, Deserialize)]
3611pub struct PollActivityTaskResponse {
3612 #[serde(default)]
3613 pub task: Option<ActivityTask>,
3614 #[serde(default)]
3615 pub poll_status: Option<String>,
3616 #[serde(default)]
3617 pub reason: Option<String>,
3618}
3619
3620impl PollActivityTaskResponse {
3621 pub fn outcome(&self) -> WorkerPollOutcome {
3623 worker_poll_outcome(
3624 self.task.is_some(),
3625 self.poll_status.as_deref(),
3626 self.reason.as_deref(),
3627 )
3628 }
3629}
3630
3631#[derive(Clone, Debug, Deserialize)]
3632pub struct PollQueryTaskResponse {
3633 #[serde(default)]
3634 pub task: Option<QueryTask>,
3635 #[serde(default)]
3636 pub poll_status: Option<String>,
3637 #[serde(default)]
3638 pub reason: Option<String>,
3639}
3640
3641impl PollQueryTaskResponse {
3642 pub fn outcome(&self) -> WorkerPollOutcome {
3644 worker_poll_outcome(
3645 self.task.is_some(),
3646 self.poll_status.as_deref(),
3647 self.reason.as_deref(),
3648 )
3649 }
3650}
3651
3652#[derive(Clone, Debug, PartialEq, Eq)]
3654pub enum WorkerPollOutcome {
3655 Task,
3657 Idle {
3659 poll_status: Option<String>,
3660 reason: Option<String>,
3661 },
3662 Stop {
3664 poll_status: Option<String>,
3665 reason: Option<String>,
3666 },
3667}
3668
3669impl WorkerPollOutcome {
3670 pub fn should_stop(&self) -> bool {
3671 matches!(self, Self::Stop { .. })
3672 }
3673}
3674
3675fn worker_poll_outcome(
3676 has_task: bool,
3677 poll_status: Option<&str>,
3678 reason: Option<&str>,
3679) -> WorkerPollOutcome {
3680 if worker_poll_is_stop(poll_status, reason) {
3681 return WorkerPollOutcome::Stop {
3682 poll_status: poll_status.map(str::to_string),
3683 reason: reason.map(str::to_string),
3684 };
3685 }
3686
3687 if has_task {
3688 WorkerPollOutcome::Task
3689 } else {
3690 WorkerPollOutcome::Idle {
3691 poll_status: poll_status.map(str::to_string),
3692 reason: reason.map(str::to_string),
3693 }
3694 }
3695}
3696
3697#[derive(Clone, Debug, Deserialize)]
3699pub struct QueryTask {
3700 pub query_task_id: String,
3701 #[serde(default = "default_workflow_task_attempt")]
3702 pub query_task_attempt: u64,
3703 #[serde(default)]
3704 pub lease_owner: Option<String>,
3705 #[serde(default)]
3706 pub workflow_id: Option<String>,
3707 #[serde(default)]
3708 pub run_id: Option<String>,
3709 pub workflow_type: String,
3710 pub query_name: String,
3711 #[serde(default = "default_payload_codec")]
3712 pub payload_codec: String,
3713 #[serde(default)]
3714 pub workflow_arguments: Option<Value>,
3715 #[serde(default)]
3716 pub query_arguments: Option<Value>,
3717 #[serde(default)]
3718 pub history_events: Vec<HistoryEvent>,
3719 #[serde(default)]
3720 pub history_export: Option<Value>,
3721 #[serde(default)]
3722 pub run_status: Option<String>,
3723}
3724
3725#[derive(Clone, Debug, Deserialize)]
3726pub struct WorkflowTask {
3727 pub task_id: String,
3728 #[serde(default)]
3729 pub workflow_id: Option<String>,
3730 #[serde(default)]
3731 pub run_id: Option<String>,
3732 pub workflow_type: String,
3733 #[serde(default = "default_payload_codec")]
3734 pub payload_codec: String,
3735 #[serde(default)]
3736 pub arguments: Option<Value>,
3737 #[serde(default)]
3738 pub history_events: Vec<HistoryEvent>,
3739 #[serde(default)]
3740 pub total_history_events: Option<u64>,
3741 #[serde(default)]
3742 pub history_size_bytes: Option<u64>,
3743 #[serde(default)]
3744 pub continue_as_new_recommended: Option<bool>,
3745 #[serde(default)]
3746 pub history_budget_pressure: Option<String>,
3747 #[serde(default)]
3748 pub next_history_page_token: Option<String>,
3749 #[serde(default = "default_workflow_task_attempt")]
3750 pub workflow_task_attempt: u64,
3751 #[serde(default)]
3752 pub workflow_signal_id: Option<String>,
3753 #[serde(default)]
3754 pub signal_name: Option<String>,
3755 #[serde(default)]
3756 pub signal_arguments: Option<Value>,
3757 #[serde(default)]
3758 pub workflow_update_id: Option<String>,
3759 #[serde(default)]
3760 pub update_name: Option<String>,
3761 #[serde(default)]
3762 pub lease_owner: Option<String>,
3763}
3764
3765impl WorkflowTask {
3766 fn append_history_page(&mut self, page: WorkflowTaskHistoryPage) {
3767 self.history_events.extend(page.history_events);
3768
3769 if page.total_history_events.is_some() {
3770 self.total_history_events = page.total_history_events;
3771 }
3772
3773 self.next_history_page_token = page
3774 .next_history_page_token
3775 .filter(|token| !token.is_empty());
3776 }
3777}
3778
3779#[derive(Clone, Debug, Deserialize)]
3780struct WorkflowTaskHistoryPage {
3781 #[serde(default)]
3782 history_events: Vec<HistoryEvent>,
3783 #[serde(default)]
3784 total_history_events: Option<u64>,
3785 #[serde(default)]
3786 next_history_page_token: Option<String>,
3787}
3788
3789#[derive(Clone, Debug, Deserialize)]
3790pub struct ActivityTask {
3791 pub task_id: String,
3792 #[serde(default)]
3793 pub activity_attempt_id: Option<String>,
3794 #[serde(default)]
3795 pub attempt_id: Option<String>,
3796 pub activity_type: String,
3797 #[serde(default = "default_payload_codec")]
3798 pub payload_codec: String,
3799 #[serde(default)]
3800 pub arguments: Option<Value>,
3801 #[serde(default = "default_attempt_number")]
3802 pub attempt_number: u64,
3803 #[serde(default)]
3804 pub lease_owner: Option<String>,
3805}
3806
3807#[derive(Clone, Debug, Deserialize)]
3808pub struct HistoryEvent {
3809 #[serde(alias = "type")]
3810 pub event_type: String,
3811 #[serde(default)]
3812 pub payload: Value,
3813 #[serde(flatten)]
3814 pub raw: HashMap<String, Value>,
3815}
3816
3817#[derive(Clone, Debug, PartialEq)]
3819pub struct QuerySignal {
3820 pub id: Option<String>,
3821 pub name: String,
3822 pub arguments: Vec<Value>,
3823 avro_arguments: Vec<AvroValue>,
3824 pub workflow_sequence: Option<u64>,
3825}
3826
3827impl QuerySignal {
3828 pub fn arguments_avro_value(&self) -> &[AvroValue] {
3830 &self.avro_arguments
3831 }
3832}
3833
3834#[derive(Clone, Debug)]
3840pub struct QueryContext {
3841 pub workflow_id: Option<String>,
3842 pub run_id: Option<String>,
3843 pub workflow_type: String,
3844 pub run_status: Option<String>,
3845 workflow_input: Value,
3846 workflow_input_avro_value: AvroValue,
3847 history_events: Arc<Vec<HistoryEvent>>,
3848 signal_events: Arc<Vec<QuerySignal>>,
3849}
3850
3851impl QueryContext {
3852 pub fn workflow_input(&self) -> &Value {
3854 &self.workflow_input
3855 }
3856
3857 pub fn workflow_input_avro_value(&self) -> &AvroValue {
3859 &self.workflow_input_avro_value
3860 }
3861
3862 pub fn history_events(&self) -> &[HistoryEvent] {
3864 self.history_events.as_slice()
3865 }
3866
3867 pub fn signal_events(&self) -> &[QuerySignal] {
3869 self.signal_events.as_slice()
3870 }
3871
3872 pub fn signals(&self, signal_name: &str) -> Vec<Vec<Value>> {
3874 self.signal_events
3875 .iter()
3876 .filter(|signal| signal.name == signal_name)
3877 .map(|signal| signal.arguments.clone())
3878 .collect()
3879 }
3880
3881 pub fn signals_avro_value(&self, signal_name: &str) -> Vec<Vec<AvroValue>> {
3883 self.signal_events
3884 .iter()
3885 .filter(|signal| signal.name == signal_name)
3886 .map(|signal| signal.avro_arguments.clone())
3887 .collect()
3888 }
3889}
3890
3891#[derive(Clone, Debug, Deserialize)]
3892pub struct ActivityHeartbeatResponse {
3893 #[serde(default)]
3894 pub cancel_requested: bool,
3895 #[serde(default)]
3896 pub heartbeat_recorded: bool,
3897 #[serde(default)]
3898 pub can_continue: Option<bool>,
3899 #[serde(default)]
3900 pub reason: Option<String>,
3901 #[serde(default)]
3902 pub run_closed_reason: Option<String>,
3903 #[serde(default)]
3904 pub run_closed_at: Option<String>,
3905 #[serde(default)]
3906 pub lease_expires_at: Option<String>,
3907 #[serde(default)]
3908 pub last_heartbeat_at: Option<String>,
3909}
3910
3911impl ActivityHeartbeatResponse {
3912 pub fn should_stop(&self) -> bool {
3914 self.cancel_requested || self.can_continue == Some(false)
3915 }
3916}
3917
3918fn default_payload_codec() -> String {
3919 DEFAULT_CODEC.to_string()
3920}
3921
3922fn default_workflow_task_attempt() -> u64 {
3923 1
3924}
3925
3926fn default_attempt_number() -> u64 {
3927 1
3928}
3929
3930type WorkflowFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
3931type WorkflowHandler = Arc<dyn Fn(WorkflowContext, AvroValue) -> WorkflowFuture + Send + Sync>;
3932type ErasedWorkflowState = Arc<dyn Any + Send + Sync>;
3933type WorkflowStateSnapshot = Arc<dyn Fn() -> Result<ErasedWorkflowState> + Send + Sync>;
3934type ReplayedWorkflowHandler =
3935 Arc<dyn Fn(WorkflowContext, AvroValue) -> ReplayedWorkflowInvocation + Send + Sync>;
3936type ActivityFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
3937type ActivityHandler = Arc<dyn Fn(ActivityContext, AvroValue) -> ActivityFuture + Send + Sync>;
3938type QueryFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
3939type QueryHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
3940type UpdateHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
3941type ReplayedQueryHandler = Arc<
3942 dyn Fn(QueryContext, ErasedWorkflowState, AvroValue) -> std::result::Result<QueryFuture, String>
3943 + Send
3944 + Sync,
3945>;
3946type WorkerHeartbeatObserver = Arc<dyn Fn(&WorkerHeartbeatObservation) + Send + Sync>;
3947
3948struct ReplayedWorkflowInvocation {
3949 future: WorkflowFuture,
3950 snapshot: WorkflowStateSnapshot,
3951}
3952
3953#[derive(Clone)]
3954struct RegisteredWorkflow {
3955 execute: WorkflowHandler,
3956 replay: Option<ReplayedWorkflowHandler>,
3957 state_type: Option<TypeId>,
3958}
3959
3960#[derive(Clone)]
3961enum RegisteredQuery {
3962 Snapshot(QueryHandler),
3963 Replayed {
3964 state_type: TypeId,
3965 handler: ReplayedQueryHandler,
3966 },
3967}
3968
3969#[derive(Clone, Debug)]
3970pub struct WorkerHeartbeatObservation {
3971 pub worker_id: String,
3972 pub task_queue: String,
3973 pub acknowledged_at_unix_millis: u64,
3974 pub acknowledgement: Value,
3975}
3976
3977#[derive(Clone, Copy, Debug)]
3984pub struct WorkerRetryPolicy {
3985 pub max_retries: usize,
3987 pub initial_backoff: Duration,
3989 pub max_backoff: Duration,
3991}
3992
3993impl Default for WorkerRetryPolicy {
3994 fn default() -> Self {
3995 Self {
3996 max_retries: 5,
3997 initial_backoff: Duration::from_millis(100),
3998 max_backoff: Duration::from_secs(5),
3999 }
4000 }
4001}
4002
4003#[derive(Clone, Copy, Debug, PartialEq, Eq)]
4004enum ManagedPollOutcome {
4005 Idle,
4006 Handled,
4007 Stop,
4008}
4009
4010#[derive(Clone)]
4011pub struct Worker {
4012 client: Client,
4013 worker_id: String,
4014 task_queue: String,
4015 workflows: HashMap<String, RegisteredWorkflow>,
4016 activities: HashMap<String, ActivityHandler>,
4017 queries: HashMap<String, HashMap<String, RegisteredQuery>>,
4018 updates: HashMap<String, HashMap<String, UpdateHandler>>,
4019 max_concurrent_workflow_tasks: usize,
4020 max_concurrent_activity_tasks: usize,
4021 poll_timeout: Duration,
4022 heartbeat_interval: Duration,
4023 retry_policy: WorkerRetryPolicy,
4024 heartbeat_observer: Option<WorkerHeartbeatObserver>,
4025}
4026
4027impl Worker {
4028 pub fn new(client: Client, task_queue: impl Into<String>) -> Self {
4029 Self {
4030 client,
4031 worker_id: default_worker_id(),
4032 task_queue: task_queue.into(),
4033 workflows: HashMap::new(),
4034 activities: HashMap::new(),
4035 queries: HashMap::new(),
4036 updates: HashMap::new(),
4037 max_concurrent_workflow_tasks: 10,
4038 max_concurrent_activity_tasks: 10,
4039 poll_timeout: Duration::from_secs(30),
4040 heartbeat_interval: Duration::from_secs(60),
4041 retry_policy: WorkerRetryPolicy::default(),
4042 heartbeat_observer: None,
4043 }
4044 }
4045
4046 pub fn worker_id(mut self, worker_id: impl Into<String>) -> Self {
4047 self.worker_id = worker_id.into();
4048 self
4049 }
4050
4051 pub fn poll_timeout(mut self, timeout: Duration) -> Self {
4052 self.poll_timeout = timeout;
4053 self
4054 }
4055
4056 pub fn heartbeat_interval(mut self, interval: Duration) -> Self {
4057 self.heartbeat_interval = interval;
4058 self
4059 }
4060
4061 pub fn retry_policy(mut self, policy: WorkerRetryPolicy) -> Self {
4063 self.retry_policy = policy;
4064 self
4065 }
4066
4067 pub fn on_worker_heartbeat<F>(mut self, observer: F) -> Self
4068 where
4069 F: Fn(&WorkerHeartbeatObservation) + Send + Sync + 'static,
4070 {
4071 self.heartbeat_observer = Some(Arc::new(observer));
4072 self
4073 }
4074
4075 pub fn max_concurrent_workflow_tasks(mut self, count: usize) -> Self {
4076 self.max_concurrent_workflow_tasks = count.max(1);
4077 self
4078 }
4079
4080 pub fn max_concurrent_activity_tasks(mut self, count: usize) -> Self {
4081 self.max_concurrent_activity_tasks = count.max(1);
4082 self
4083 }
4084
4085 pub fn register_workflow<F, Fut>(&mut self, workflow_type: impl Into<String>, handler: F)
4092 where
4093 F: Fn(WorkflowContext, Value) -> Fut + Send + Sync + 'static,
4094 Fut: Future<Output = Result<Value>> + Send + 'static,
4095 {
4096 let handler = Arc::new(handler);
4097 self.workflows.insert(
4098 workflow_type.into(),
4099 RegisteredWorkflow {
4100 execute: Arc::new(move |ctx, input| {
4101 let handler = Arc::clone(&handler);
4102 Box::pin(async move {
4103 let result = handler(ctx, input.into_json()?).await?;
4104 AvroValue::from_serialize(&result)
4105 })
4106 }),
4107 replay: None,
4108 state_type: None,
4109 },
4110 );
4111 }
4112
4113 pub fn register_workflow_avro_value<F, Fut>(
4115 &mut self,
4116 workflow_type: impl Into<String>,
4117 handler: F,
4118 ) where
4119 F: Fn(WorkflowContext, AvroValue) -> Fut + Send + Sync + 'static,
4120 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4121 {
4122 self.workflows.insert(
4123 workflow_type.into(),
4124 RegisteredWorkflow {
4125 execute: Arc::new(move |ctx, input| Box::pin(handler(ctx, input))),
4126 replay: None,
4127 state_type: None,
4128 },
4129 );
4130 }
4131
4132 pub fn register_replayed_workflow<S, Factory, F, Fut>(
4140 &mut self,
4141 workflow_type: impl Into<String>,
4142 state_factory: Factory,
4143 handler: F,
4144 ) where
4145 S: Clone + Send + Sync + 'static,
4146 Factory: Fn() -> S + Send + Sync + 'static,
4147 F: Fn(WorkflowContext, Value, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
4148 Fut: Future<Output = Result<Value>> + Send + 'static,
4149 {
4150 let state_factory = Arc::new(state_factory);
4151 let handler = Arc::new(handler);
4152
4153 let execute_factory = Arc::clone(&state_factory);
4154 let execute_handler = Arc::clone(&handler);
4155 let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4156 let state = WorkflowInstance::new(execute_factory());
4157 let handler = Arc::clone(&execute_handler);
4158 Box::pin(async move {
4159 let result = handler(ctx, input.into_json()?, state).await?;
4160 AvroValue::from_serialize(&result)
4161 }) as WorkflowFuture
4162 });
4163
4164 let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4165 let state = WorkflowInstance::new(state_factory());
4166 let snapshot_state = state.clone();
4167 let snapshot: WorkflowStateSnapshot =
4168 Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
4169 let replay_handler = Arc::clone(&handler);
4170 let future = async move {
4171 let result = replay_handler(ctx, input.into_json()?, state).await?;
4172 AvroValue::from_serialize(&result)
4173 };
4174 ReplayedWorkflowInvocation {
4175 future: Box::pin(future),
4176 snapshot,
4177 }
4178 });
4179
4180 self.workflows.insert(
4181 workflow_type.into(),
4182 RegisteredWorkflow {
4183 execute,
4184 replay: Some(replay),
4185 state_type: Some(TypeId::of::<S>()),
4186 },
4187 );
4188 }
4189
4190 pub fn register_replayed_workflow_avro_value<S, Factory, F, Fut>(
4192 &mut self,
4193 workflow_type: impl Into<String>,
4194 state_factory: Factory,
4195 handler: F,
4196 ) where
4197 S: Clone + Send + Sync + 'static,
4198 Factory: Fn() -> S + Send + Sync + 'static,
4199 F: Fn(WorkflowContext, AvroValue, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
4200 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4201 {
4202 let state_factory = Arc::new(state_factory);
4203 let handler = Arc::new(handler);
4204
4205 let execute_factory = Arc::clone(&state_factory);
4206 let execute_handler = Arc::clone(&handler);
4207 let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4208 let state = WorkflowInstance::new(execute_factory());
4209 Box::pin(execute_handler(ctx, input, state)) as WorkflowFuture
4210 });
4211
4212 let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4213 let state = WorkflowInstance::new(state_factory());
4214 let snapshot_state = state.clone();
4215 let snapshot: WorkflowStateSnapshot =
4216 Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
4217 ReplayedWorkflowInvocation {
4218 future: Box::pin(handler(ctx, input, state)),
4219 snapshot,
4220 }
4221 });
4222
4223 self.workflows.insert(
4224 workflow_type.into(),
4225 RegisteredWorkflow {
4226 execute,
4227 replay: Some(replay),
4228 state_type: Some(TypeId::of::<S>()),
4229 },
4230 );
4231 }
4232
4233 pub fn register_activity<F, Fut>(&mut self, activity_type: impl Into<String>, handler: F)
4234 where
4235 F: Fn(ActivityContext, Value) -> Fut + Send + Sync + 'static,
4236 Fut: Future<Output = Result<Value>> + Send + 'static,
4237 {
4238 let handler = Arc::new(handler);
4239 self.activities.insert(
4240 activity_type.into(),
4241 Arc::new(move |ctx, args| {
4242 let handler = Arc::clone(&handler);
4243 Box::pin(async move {
4244 let result = handler(ctx, args.into_json()?).await?;
4245 AvroValue::from_serialize(&result)
4246 })
4247 }),
4248 );
4249 }
4250
4251 pub fn register_activity_avro_value<F, Fut>(
4253 &mut self,
4254 activity_type: impl Into<String>,
4255 handler: F,
4256 ) where
4257 F: Fn(ActivityContext, AvroValue) -> Fut + Send + Sync + 'static,
4258 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4259 {
4260 self.activities.insert(
4261 activity_type.into(),
4262 Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
4263 );
4264 }
4265
4266 pub fn register_query<F, Fut>(
4272 &mut self,
4273 workflow_type: impl Into<String>,
4274 query_name: impl Into<String>,
4275 handler: F,
4276 ) where
4277 F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
4278 Fut: Future<Output = Result<Value>> + Send + 'static,
4279 {
4280 let handler = Arc::new(handler);
4281 self.queries
4282 .entry(workflow_type.into())
4283 .or_default()
4284 .insert(
4285 query_name.into(),
4286 RegisteredQuery::Snapshot(Arc::new(move |ctx, args| {
4287 let handler = Arc::clone(&handler);
4288 Box::pin(async move {
4289 let result = handler(ctx, args.into_json()?).await?;
4290 AvroValue::from_serialize(&result)
4291 })
4292 })),
4293 );
4294 }
4295
4296 pub fn register_query_avro_value<F, Fut>(
4298 &mut self,
4299 workflow_type: impl Into<String>,
4300 query_name: impl Into<String>,
4301 handler: F,
4302 ) where
4303 F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
4304 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4305 {
4306 self.queries
4307 .entry(workflow_type.into())
4308 .or_default()
4309 .insert(
4310 query_name.into(),
4311 RegisteredQuery::Snapshot(Arc::new(move |ctx, args| Box::pin(handler(ctx, args)))),
4312 );
4313 }
4314
4315 pub fn register_replayed_query<S, F, Fut>(
4322 &mut self,
4323 workflow_type: impl Into<String>,
4324 query_name: impl Into<String>,
4325 handler: F,
4326 ) where
4327 S: Clone + Send + Sync + 'static,
4328 F: Fn(QueryContext, Arc<S>, Value) -> Fut + Send + Sync + 'static,
4329 Fut: Future<Output = Result<Value>> + Send + 'static,
4330 {
4331 let handler = Arc::new(handler);
4332 let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
4333 let state = state.downcast::<S>().map_err(|_| {
4334 "registered query state type does not match the replayed workflow state".to_string()
4335 })?;
4336 let handler = Arc::clone(&handler);
4337 Ok(Box::pin(async move {
4338 let result = handler(ctx, state, args.into_json()?).await?;
4339 AvroValue::from_serialize(&result)
4340 }))
4341 });
4342
4343 self.queries
4344 .entry(workflow_type.into())
4345 .or_default()
4346 .insert(
4347 query_name.into(),
4348 RegisteredQuery::Replayed {
4349 state_type: TypeId::of::<S>(),
4350 handler: erased_handler,
4351 },
4352 );
4353 }
4354
4355 pub fn register_replayed_query_avro_value<S, F, Fut>(
4357 &mut self,
4358 workflow_type: impl Into<String>,
4359 query_name: impl Into<String>,
4360 handler: F,
4361 ) where
4362 S: Clone + Send + Sync + 'static,
4363 F: Fn(QueryContext, Arc<S>, AvroValue) -> Fut + Send + Sync + 'static,
4364 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4365 {
4366 let handler = Arc::new(handler);
4367 let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
4368 let state = state.downcast::<S>().map_err(|_| {
4369 "registered query state type does not match the replayed workflow state".to_string()
4370 })?;
4371 Ok(Box::pin(handler(ctx, state, args)))
4372 });
4373
4374 self.queries
4375 .entry(workflow_type.into())
4376 .or_default()
4377 .insert(
4378 query_name.into(),
4379 RegisteredQuery::Replayed {
4380 state_type: TypeId::of::<S>(),
4381 handler: erased_handler,
4382 },
4383 );
4384 }
4385
4386 pub fn register_update<F, Fut>(
4388 &mut self,
4389 workflow_type: impl Into<String>,
4390 update_name: impl Into<String>,
4391 handler: F,
4392 ) where
4393 F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
4394 Fut: Future<Output = Result<Value>> + Send + 'static,
4395 {
4396 let handler = Arc::new(handler);
4397 self.updates
4398 .entry(workflow_type.into())
4399 .or_default()
4400 .insert(
4401 update_name.into(),
4402 Arc::new(move |ctx, args| {
4403 let handler = Arc::clone(&handler);
4404 Box::pin(async move {
4405 let result = handler(ctx, args.into_json()?).await?;
4406 AvroValue::from_serialize(&result)
4407 })
4408 }),
4409 );
4410 }
4411
4412 pub fn register_update_avro_value<F, Fut>(
4414 &mut self,
4415 workflow_type: impl Into<String>,
4416 update_name: impl Into<String>,
4417 handler: F,
4418 ) where
4419 F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
4420 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4421 {
4422 self.updates
4423 .entry(workflow_type.into())
4424 .or_default()
4425 .insert(
4426 update_name.into(),
4427 Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
4428 );
4429 }
4430
4431 pub async fn register(&self) -> Result<RegisterWorkerResponse> {
4432 let mut command_contracts = serde_json::Map::new();
4433 for workflow_type in self.workflows.keys() {
4434 let mut queries = self
4435 .queries
4436 .get(workflow_type)
4437 .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
4438 .unwrap_or_default();
4439 queries.sort();
4440 let mut updates = self
4441 .updates
4442 .get(workflow_type)
4443 .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
4444 .unwrap_or_default();
4445 updates.sort();
4446 if !queries.is_empty() || !updates.is_empty() {
4447 command_contracts.insert(
4448 workflow_type.clone(),
4449 json!({
4450 "queries": queries,
4451 "updates": updates,
4452 }),
4453 );
4454 }
4455 }
4456
4457 self.client
4458 .register_worker_with_command_contracts(
4459 &self.worker_id,
4460 &self.task_queue,
4461 self.workflows.keys().cloned().collect(),
4462 self.activities.keys().cloned().collect(),
4463 self.max_concurrent_workflow_tasks,
4464 self.max_concurrent_activity_tasks,
4465 [
4466 (!self.queries.is_empty()).then(|| QUERY_TASKS_CAPABILITY.to_string()),
4467 (!self.updates.is_empty()).then(|| WORKFLOW_UPDATES_CAPABILITY.to_string()),
4468 ]
4469 .into_iter()
4470 .flatten()
4471 .collect(),
4472 Value::Object(command_contracts),
4473 )
4474 .await
4475 }
4476
4477 pub async fn run(&self) -> Result<()> {
4483 self.run_until(std::future::pending::<()>()).await
4484 }
4485
4486 pub async fn run_until<F>(&self, shutdown: F) -> Result<()>
4490 where
4491 F: Future<Output = ()>,
4492 {
4493 let registration = self.register().await?;
4494 let heartbeat_interval = Duration::from_secs(
4495 registration
4496 .heartbeat_interval_seconds
4497 .unwrap_or(self.heartbeat_interval.as_secs().max(1)),
4498 );
4499 let heartbeat = tokio::time::sleep(Duration::ZERO);
4505 tokio::pin!(heartbeat);
4506 tokio::pin!(shutdown);
4507 let stop = Arc::new(AtomicBool::new(false));
4508 let mut workflow_poller = (!self.workflows.is_empty()).then(|| {
4512 let worker = self.clone();
4513 let stop = Arc::clone(&stop);
4514 tokio::spawn(async move { worker.poll_workflows_until_stopped(stop).await })
4515 });
4516 let mut activity_poller = (!self.activities.is_empty()).then(|| {
4517 let worker = self.clone();
4518 let stop = Arc::clone(&stop);
4519 tokio::spawn(async move { worker.poll_activities_until_stopped(stop).await })
4520 });
4521 let mut query_poller = (!self.queries.is_empty()).then(|| {
4522 let worker = self.clone();
4523 let stop = Arc::clone(&stop);
4524 tokio::spawn(async move { worker.poll_queries_until_stopped(stop).await })
4525 });
4526
4527 loop {
4528 tokio::select! {
4529 _ = &mut shutdown => {
4530 stop.store(true, Ordering::SeqCst);
4531 break;
4532 }
4533 _ = &mut heartbeat => {
4534 let result = self.retry_worker_operation(|| {
4535 self.client.heartbeat_worker(
4536 &self.worker_id,
4537 self.max_concurrent_workflow_tasks,
4538 self.max_concurrent_activity_tasks,
4539 )
4540 }).await;
4541 heartbeat
4542 .as_mut()
4543 .reset(tokio::time::Instant::now() + heartbeat_interval);
4544 match result {
4545 Ok(acknowledgement) => {
4546 if let Some(observer) = &self.heartbeat_observer {
4547 observer(&WorkerHeartbeatObservation {
4548 worker_id: self.worker_id.clone(),
4549 task_queue: self.task_queue.clone(),
4550 acknowledged_at_unix_millis: SystemTime::now()
4551 .duration_since(UNIX_EPOCH)
4552 .unwrap_or_default()
4553 .as_millis()
4554 .min(u64::MAX as u128)
4555 as u64,
4556 acknowledgement,
4557 });
4558 }
4559 }
4560 Err(error) => {
4561 stop.store(true, Ordering::SeqCst);
4562 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await?;
4563 return Err(error);
4564 }
4565 }
4566 }
4567 result = OptionFuture::from(workflow_poller.as_mut()), if workflow_poller.is_some() => {
4568 workflow_poller = None;
4569 let stopped_by_server = stop.load(Ordering::SeqCst);
4570 stop.store(true, Ordering::SeqCst);
4571 let poller_result = optional_poller_result("workflow", result);
4572 let join_result =
4573 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
4574 poller_result?;
4575 join_result?;
4576 if stopped_by_server {
4577 return Ok(());
4578 }
4579 return Err(Error::WorkerLoop(
4580 "workflow poller stopped unexpectedly".to_string(),
4581 ));
4582 }
4583 result = OptionFuture::from(activity_poller.as_mut()), if activity_poller.is_some() => {
4584 activity_poller = None;
4585 let stopped_by_server = stop.load(Ordering::SeqCst);
4586 stop.store(true, Ordering::SeqCst);
4587 let poller_result = optional_poller_result("activity", result);
4588 let join_result =
4589 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
4590 poller_result?;
4591 join_result?;
4592 if stopped_by_server {
4593 return Ok(());
4594 }
4595 return Err(Error::WorkerLoop(
4596 "activity poller stopped unexpectedly".to_string(),
4597 ));
4598 }
4599 result = OptionFuture::from(query_poller.as_mut()), if query_poller.is_some() => {
4600 query_poller = None;
4601 let stopped_by_server = stop.load(Ordering::SeqCst);
4602 stop.store(true, Ordering::SeqCst);
4603 let poller_result = optional_poller_result("query", result);
4604 let join_result =
4605 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
4606 poller_result?;
4607 join_result?;
4608 if stopped_by_server {
4609 return Ok(());
4610 }
4611 return Err(Error::WorkerLoop(
4612 "query poller stopped unexpectedly".to_string(),
4613 ));
4614 }
4615 }
4616 }
4617
4618 join_pollers(
4619 workflow_poller.take(),
4620 activity_poller.take(),
4621 query_poller.take(),
4622 )
4623 .await
4624 }
4625
4626 pub async fn run_once(&self) -> Result<usize> {
4640 let mut handled = 0;
4641 match self.poll_workflow_once().await? {
4642 ManagedPollOutcome::Handled => handled += 1,
4643 ManagedPollOutcome::Stop => return Ok(handled),
4644 ManagedPollOutcome::Idle => {}
4645 }
4646 match self.poll_activity_once().await? {
4647 ManagedPollOutcome::Handled => handled += 1,
4648 ManagedPollOutcome::Stop => return Ok(handled),
4649 ManagedPollOutcome::Idle => {}
4650 }
4651 if !self.queries.is_empty() {
4652 match self.poll_query_once().await? {
4653 ManagedPollOutcome::Handled => handled += 1,
4654 ManagedPollOutcome::Stop => return Ok(handled),
4655 ManagedPollOutcome::Idle => {}
4656 }
4657 }
4658 Ok(handled)
4659 }
4660
4661 async fn poll_workflow_once(&self) -> Result<ManagedPollOutcome> {
4662 let poll_request_id = unique_request_id("rust-workflow-poll");
4663 let response = self
4664 .retry_worker_operation(|| {
4665 self.client.poll_workflow_task_response_with_request_id(
4666 &self.worker_id,
4667 &self.task_queue,
4668 self.poll_timeout,
4669 &poll_request_id,
4670 0,
4671 )
4672 })
4673 .await?;
4674 if response.outcome().should_stop() {
4675 return Ok(ManagedPollOutcome::Stop);
4676 }
4677 let Some(task) = response.task else {
4678 return Ok(ManagedPollOutcome::Idle);
4679 };
4680
4681 let task_id = task.task_id.clone();
4682 let attempt = task.workflow_task_attempt;
4683 let run_id = task.run_id.clone();
4684 let lease_owner = task
4685 .lease_owner
4686 .clone()
4687 .unwrap_or_else(|| self.worker_id.clone());
4688
4689 match self.execute_workflow_task(task) {
4690 Ok(commands) if commands.is_empty() => {
4691 self.client
4697 .fail_workflow_task_with_type(
4698 &task_id,
4699 &lease_owner,
4700 attempt,
4701 WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE,
4702 WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE,
4703 )
4704 .await?;
4705 }
4706 Ok(commands) => {
4707 let completion = self
4708 .client
4709 .complete_workflow_task(&task_id, &lease_owner, attempt, commands)
4710 .await;
4711 if let Err(error) = completion {
4712 if !workflow_task_completion_is_terminal_timeout(
4713 &error,
4714 &task_id,
4715 attempt,
4716 run_id.as_deref(),
4717 ) {
4718 return Err(error);
4719 }
4720 }
4721 }
4722 Err(error) => {
4723 self.client
4724 .fail_workflow_task(&task_id, &lease_owner, attempt, error.to_string())
4725 .await?;
4726 }
4727 }
4728
4729 Ok(ManagedPollOutcome::Handled)
4730 }
4731
4732 async fn poll_workflows_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
4733 while !stop.load(Ordering::SeqCst) {
4734 if self.poll_workflow_once().await? == ManagedPollOutcome::Stop {
4735 stop.store(true, Ordering::SeqCst);
4736 break;
4737 }
4738 }
4739
4740 Ok(())
4741 }
4742
4743 async fn poll_activity_once(&self) -> Result<ManagedPollOutcome> {
4744 let poll_request_id = unique_request_id("rust-activity-poll");
4745 let response = self
4746 .retry_worker_operation(|| {
4747 self.client.poll_activity_task_response_with_request_id(
4748 &self.worker_id,
4749 &self.task_queue,
4750 self.poll_timeout,
4751 &poll_request_id,
4752 0,
4753 )
4754 })
4755 .await?;
4756 if response.outcome().should_stop() {
4757 return Ok(ManagedPollOutcome::Stop);
4758 }
4759 let Some(task) = response.task else {
4760 return Ok(ManagedPollOutcome::Idle);
4761 };
4762
4763 let task_id = task.task_id.clone();
4764 let attempt_id = task
4765 .activity_attempt_id
4766 .clone()
4767 .or(task.attempt_id.clone())
4768 .unwrap_or_default();
4769 let lease_owner = task
4770 .lease_owner
4771 .clone()
4772 .unwrap_or_else(|| self.worker_id.clone());
4773 let codec = task.payload_codec.clone();
4774 let result = self.execute_activity_task(task).await;
4775 match result {
4776 Ok(value) => {
4777 let completion = self
4778 .client
4779 .complete_activity_task(&task_id, &attempt_id, &lease_owner, value, &codec)
4780 .await;
4781 if let Err(error) = completion {
4782 if !activity_task_rejection_is_final(&error) {
4783 return Err(error);
4784 }
4785 }
4786 }
4787 Err(error) => {
4788 let failure = self
4789 .client
4790 .fail_activity_task(
4791 &task_id,
4792 &attempt_id,
4793 &lease_owner,
4794 error.to_string(),
4795 false,
4796 )
4797 .await;
4798 if let Err(error) = failure {
4799 if !activity_task_rejection_is_final(&error) {
4800 return Err(error);
4801 }
4802 }
4803 }
4804 }
4805
4806 Ok(ManagedPollOutcome::Handled)
4807 }
4808
4809 async fn poll_activities_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
4810 while !stop.load(Ordering::SeqCst) {
4811 if self.poll_activity_once().await? == ManagedPollOutcome::Stop {
4812 stop.store(true, Ordering::SeqCst);
4813 break;
4814 }
4815 }
4816
4817 Ok(())
4818 }
4819
4820 async fn poll_query_once(&self) -> Result<ManagedPollOutcome> {
4821 let poll_request_id = unique_request_id("rust-query-poll");
4822 let response = self
4823 .retry_worker_operation(|| {
4824 self.client.poll_query_task_response_with_request_id(
4825 &self.worker_id,
4826 &self.task_queue,
4827 self.poll_timeout,
4828 &poll_request_id,
4829 0,
4830 )
4831 })
4832 .await?;
4833 if response.outcome().should_stop() {
4834 return Ok(ManagedPollOutcome::Stop);
4835 }
4836 let Some(task) = response.task else {
4837 return Ok(ManagedPollOutcome::Idle);
4838 };
4839
4840 let query_task_id = task.query_task_id.clone();
4841 let attempt = task.query_task_attempt;
4842 let lease_owner = task
4843 .lease_owner
4844 .clone()
4845 .unwrap_or_else(|| self.worker_id.clone());
4846 let codec = task.payload_codec.clone();
4847
4848 match self.execute_query_task(task).await {
4849 Ok(value) => {
4850 let result_envelope = match encode_typed_envelope(&value, &codec) {
4851 Ok(result_envelope) => result_envelope,
4852 Err(error) => {
4853 let failure = self
4854 .client
4855 .fail_query_task(
4856 &query_task_id,
4857 &lease_owner,
4858 attempt,
4859 error.to_string(),
4860 "query_result_encode_failed",
4861 "QueryResultEncodeFailed",
4862 )
4863 .await;
4864 if let Err(error) = failure {
4865 if !query_task_rejection_is_final(&error) {
4866 return Err(error);
4867 }
4868 }
4869 return Ok(ManagedPollOutcome::Handled);
4870 }
4871 };
4872
4873 if let Err(error) = self
4874 .client
4875 .complete_query_task_with_envelope(
4876 &query_task_id,
4877 &lease_owner,
4878 attempt,
4879 value.clone().into_json()?,
4880 result_envelope,
4881 )
4882 .await
4883 {
4884 if !query_task_rejection_is_final(&error) {
4885 return Err(error);
4886 }
4887 }
4888 }
4889 Err(failure) => {
4890 let result = self
4891 .client
4892 .fail_query_task(
4893 &query_task_id,
4894 &lease_owner,
4895 attempt,
4896 failure.message,
4897 failure.reason,
4898 failure.failure_type,
4899 )
4900 .await;
4901 if let Err(error) = result {
4902 if !query_task_rejection_is_final(&error) {
4903 return Err(error);
4904 }
4905 }
4906 }
4907 }
4908
4909 Ok(ManagedPollOutcome::Handled)
4910 }
4911
4912 async fn poll_queries_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
4913 while !stop.load(Ordering::SeqCst) {
4914 if self.poll_query_once().await? == ManagedPollOutcome::Stop {
4915 stop.store(true, Ordering::SeqCst);
4916 break;
4917 }
4918 }
4919
4920 Ok(())
4921 }
4922
4923 async fn retry_worker_operation<T, F, Fut>(&self, mut operation: F) -> Result<T>
4924 where
4925 F: FnMut() -> Fut,
4926 Fut: Future<Output = Result<T>>,
4927 {
4928 let mut retries = 0;
4929
4930 loop {
4931 match operation().await {
4932 Err(error)
4933 if worker_operation_is_retryable(&error)
4934 && retries < self.retry_policy.max_retries =>
4935 {
4936 retries += 1;
4937 tokio::time::sleep(worker_retry_delay(self.retry_policy, retries)).await;
4938 }
4939 result => return result,
4940 }
4941 }
4942 }
4943
4944 async fn execute_query_task(
4945 &self,
4946 mut task: QueryTask,
4947 ) -> std::result::Result<AvroValue, QueryTaskExecutionFailure> {
4948 if !matches!(task.payload_codec.as_str(), DEFAULT_CODEC | JSON_CODEC) {
4949 return Err(QueryTaskExecutionFailure::new(
4950 "query_payload_decode_failed",
4951 format!(
4952 "cannot decode query payload with unsupported codec {:?}",
4953 task.payload_codec
4954 ),
4955 "QueryPayloadDecodeFailed",
4956 ));
4957 }
4958
4959 if !self.workflows.contains_key(&task.workflow_type) {
4960 return Err(QueryTaskExecutionFailure::new(
4961 "query_workflow_type_not_registered",
4962 format!("no workflow registered for type {:?}", task.workflow_type),
4963 "WorkflowTypeNotRegistered",
4964 ));
4965 }
4966
4967 let Some(handlers) = self.queries.get(&task.workflow_type) else {
4968 return Err(QueryTaskExecutionFailure::new(
4969 "query_handler_unavailable",
4970 format!(
4971 "query handlers are unavailable for workflow type {:?}",
4972 task.workflow_type
4973 ),
4974 "QueryHandlerUnavailable",
4975 ));
4976 };
4977 let Some(query) = handlers.get(&task.query_name) else {
4978 return Err(QueryTaskExecutionFailure::new(
4979 "rejected_unknown_query",
4980 format!("unknown query {:?}", task.query_name),
4981 "QueryFailed",
4982 ));
4983 };
4984
4985 let args = decode_task_avro_arguments(task.query_arguments.as_ref(), &task.payload_codec)
4986 .map_err(|error| {
4987 QueryTaskExecutionFailure::new(
4988 "query_payload_decode_failed",
4989 format!("cannot decode query arguments: {error}"),
4990 "QueryPayloadDecodeFailed",
4991 )
4992 })?;
4993 let workflow_input_typed =
4994 decode_task_avro_arguments(task.workflow_arguments.as_ref(), &task.payload_codec)
4995 .map_err(|error| {
4996 QueryTaskExecutionFailure::new(
4997 "query_workflow_state_unavailable",
4998 format!("cannot decode workflow start input: {error}"),
4999 "QueryWorkflowStateUnavailable",
5000 )
5001 })?;
5002 let workflow_input = workflow_input_typed.clone().into_json().map_err(|error| {
5003 QueryTaskExecutionFailure::new(
5004 "query_workflow_state_unavailable",
5005 format!("cannot project workflow start input: {error}"),
5006 "QueryWorkflowStateUnavailable",
5007 )
5008 })?;
5009 hydrate_query_history_from_export(&mut task).map_err(|error| {
5010 QueryTaskExecutionFailure::new(
5011 "query_workflow_state_unavailable",
5012 format!("cannot restore query history snapshot: {error}"),
5013 "QueryWorkflowStateUnavailable",
5014 )
5015 })?;
5016 enrich_query_history_from_export(&mut task).map_err(|error| {
5017 QueryTaskExecutionFailure::new(
5018 "query_workflow_state_unavailable",
5019 format!("cannot restore compact query history payloads: {error}"),
5020 "QueryWorkflowStateUnavailable",
5021 )
5022 })?;
5023 let signal_events = query_signal_events(&task).map_err(|error| {
5024 QueryTaskExecutionFailure::new(
5025 "query_workflow_state_unavailable",
5026 format!("cannot decode committed workflow signals: {error}"),
5027 "QueryWorkflowStateUnavailable",
5028 )
5029 })?;
5030 let history_events = Arc::new(std::mem::take(&mut task.history_events));
5031 let context = QueryContext {
5032 workflow_id: task.workflow_id,
5033 run_id: task.run_id,
5034 workflow_type: task.workflow_type.clone(),
5035 run_status: task.run_status,
5036 workflow_input,
5037 workflow_input_avro_value: workflow_input_typed.clone(),
5038 history_events: Arc::clone(&history_events),
5039 signal_events: Arc::new(signal_events),
5040 };
5041
5042 let future = match query {
5043 RegisteredQuery::Snapshot(handler) => handler(context, args),
5044 RegisteredQuery::Replayed {
5045 state_type,
5046 handler,
5047 } => {
5048 let workflow = self
5049 .workflows
5050 .get(&task.workflow_type)
5051 .expect("workflow registration was checked above");
5052 if workflow.state_type != Some(*state_type) {
5053 return Err(QueryTaskExecutionFailure::new(
5054 "query_workflow_state_unavailable",
5055 "replayed query state type does not match its workflow registration",
5056 "QueryWorkflowStateUnavailable",
5057 ));
5058 }
5059 let replay = workflow.replay.as_ref().ok_or_else(|| {
5060 QueryTaskExecutionFailure::new(
5061 "query_workflow_state_unavailable",
5062 format!(
5063 "workflow type {:?} is not registered for instance-state replay",
5064 task.workflow_type
5065 ),
5066 "QueryWorkflowStateUnavailable",
5067 )
5068 })?;
5069 let workflow_state = Arc::new(Mutex::new(
5070 WorkflowState::new_with_identity(
5071 history_events.as_ref().clone(),
5072 context.workflow_id.clone(),
5073 context.run_id.clone(),
5074 self.task_queue.clone(),
5075 task.payload_codec,
5076 None,
5077 )
5078 .map_err(|error| {
5079 QueryTaskExecutionFailure::new(
5080 "query_workflow_state_unavailable",
5081 format!("workflow replay failed before query: {error}"),
5082 "QueryWorkflowStateUnavailable",
5083 )
5084 })?,
5085 ));
5086 let workflow_context = WorkflowContext {
5087 state: workflow_state,
5088 };
5089 let mut invocation = replay(workflow_context.clone(), workflow_input_typed.clone());
5090 let mut cx = TaskContext::from_waker(noop_waker_ref());
5091 match invocation.future.as_mut().poll(&mut cx) {
5092 Poll::Ready(Ok(_)) => {
5093 workflow_context
5094 .ensure_history_consumed()
5095 .map_err(|error| {
5096 QueryTaskExecutionFailure::new(
5097 "query_workflow_state_unavailable",
5098 format!("workflow replay failed before query: {error}"),
5099 "QueryWorkflowStateUnavailable",
5100 )
5101 })?;
5102 }
5103 Poll::Ready(Err(error)) => {
5104 return Err(QueryTaskExecutionFailure::new(
5105 "query_workflow_state_unavailable",
5106 format!("workflow replay failed before query: {error}"),
5107 "QueryWorkflowStateUnavailable",
5108 ));
5109 }
5110 Poll::Pending => {
5111 let commands = workflow_context.take_commands().map_err(|error| {
5112 QueryTaskExecutionFailure::new(
5113 "query_workflow_state_unavailable",
5114 format!("workflow replay failed before query: {error}"),
5115 "QueryWorkflowStateUnavailable",
5116 )
5117 })?;
5118 if commands.is_empty()
5119 && !workflow_context
5120 .matched_recorded_pending()
5121 .map_err(|error| {
5122 QueryTaskExecutionFailure::new(
5123 "query_workflow_state_unavailable",
5124 format!("workflow replay failed before query: {error}"),
5125 "QueryWorkflowStateUnavailable",
5126 )
5127 })?
5128 {
5129 return Err(QueryTaskExecutionFailure::new(
5130 "query_workflow_state_unavailable",
5131 "workflow replay yielded without a durable command",
5132 "QueryWorkflowStateUnavailable",
5133 ));
5134 }
5135 }
5136 }
5137 let state = (invocation.snapshot)().map_err(|error| {
5138 QueryTaskExecutionFailure::new(
5139 "query_workflow_state_unavailable",
5140 format!("cannot snapshot replayed workflow state: {error}"),
5141 "QueryWorkflowStateUnavailable",
5142 )
5143 })?;
5144 handler(context, state, args).map_err(|message| {
5145 QueryTaskExecutionFailure::new(
5146 "query_workflow_state_unavailable",
5147 message,
5148 "QueryWorkflowStateUnavailable",
5149 )
5150 })?
5151 }
5152 };
5153
5154 future.await.map_err(|error| {
5155 QueryTaskExecutionFailure::new("query_rejected", error.to_string(), "QueryFailed")
5156 })
5157 }
5158
5159 fn execute_workflow_task(&self, task: WorkflowTask) -> Result<Vec<Value>> {
5160 if let Some(update_id) = task
5161 .workflow_update_id
5162 .as_deref()
5163 .filter(|update_id| !update_id.is_empty())
5164 {
5165 return self.execute_update_task(&task, update_id);
5166 }
5167
5168 let workflow = self
5169 .workflows
5170 .get(&task.workflow_type)
5171 .ok_or_else(|| Error::WorkflowNotRegistered(task.workflow_type.clone()))?;
5172 let input = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
5173 let resume_signal = decode_resume_signal(&task)?;
5174 let history_budget = WorkflowHistoryBudget {
5175 event_count: task
5176 .total_history_events
5177 .unwrap_or_else(|| u64::try_from(task.history_events.len()).unwrap_or(u64::MAX)),
5178 size_bytes: task.history_size_bytes,
5179 continue_as_new_recommended: task.continue_as_new_recommended.unwrap_or(false),
5180 pressure: task.history_budget_pressure.clone(),
5181 };
5182 let mut workflow_state = WorkflowState::new_with_identity(
5183 task.history_events,
5184 task.workflow_id,
5185 task.run_id,
5186 self.task_queue.clone(),
5187 task.payload_codec.clone(),
5188 resume_signal,
5189 )?;
5190 workflow_state.history_budget = history_budget;
5191 let state = Arc::new(Mutex::new(workflow_state));
5192 let ctx = WorkflowContext { state };
5193 let mut future = (workflow.execute)(ctx.clone(), input);
5194 let mut cx = TaskContext::from_waker(noop_waker_ref());
5195
5196 match future.as_mut().poll(&mut cx) {
5197 Poll::Ready(Ok(result)) => {
5198 ctx.ensure_history_consumed()?;
5199 let result = encode_typed_envelope(&result, &task.payload_codec)?;
5200 let mut commands = ctx.take_commands()?;
5201 commands.push(json!({
5202 "type": "complete_workflow",
5203 "result": result
5204 }));
5205 Ok(commands)
5206 }
5207 Poll::Ready(Err(error)) => {
5208 if let Error::ContinueAsNew(request) = error {
5209 let mut commands = ctx.take_commands()?;
5210 if let Some(command) = ctx.continue_as_new_command(request)? {
5211 commands.push(command);
5212 }
5213 ctx.ensure_history_consumed()?;
5214 return Ok(commands);
5215 }
5216 ctx.ensure_history_consumed()?;
5219 if workflow_task_integrity_error(&error) {
5220 return Err(error);
5225 }
5226 let mut commands = ctx.take_commands()?;
5227 commands.push(workflow_failure_command(&error));
5228 Ok(commands)
5229 }
5230 Poll::Pending => {
5231 let commands = ctx.take_commands()?;
5232 if commands.is_empty() && !ctx.matched_recorded_pending()? {
5233 Err(Error::WorkflowYieldedWithoutCommand)
5234 } else {
5235 Ok(commands)
5236 }
5237 }
5238 }
5239 }
5240
5241 fn execute_update_task(&self, task: &WorkflowTask, update_id: &str) -> Result<Vec<Value>> {
5242 if !self.workflows.contains_key(&task.workflow_type) {
5243 return Err(Error::WorkflowNotRegistered(task.workflow_type.clone()));
5244 }
5245
5246 let accepted = task.history_events.iter().rev().find_map(|event| {
5247 (event.event_type == "UpdateAccepted"
5248 && event.payload.get("update_id").and_then(Value::as_str) == Some(update_id))
5249 .then_some(&event.payload)
5250 });
5251 let update_name = accepted
5252 .and_then(|payload| payload.get("update_name"))
5253 .and_then(Value::as_str)
5254 .or(task.update_name.as_deref())
5255 .unwrap_or_default();
5256 let Some(handler) = self
5257 .updates
5258 .get(&task.workflow_type)
5259 .and_then(|handlers| handlers.get(update_name))
5260 else {
5261 return Ok(vec![json!({
5262 "type": "fail_update",
5263 "update_id": update_id,
5264 "message": format!(
5265 "no update handler is registered for {}.{update_name}",
5266 task.workflow_type
5267 ),
5268 "exception_type": "UnknownUpdate",
5269 "non_retryable": true,
5270 })]);
5271 };
5272 let arguments = accepted
5273 .and_then(|payload| payload.get("arguments"))
5274 .or(task.arguments.as_ref());
5275 let arguments = decode_task_avro_arguments(arguments, &task.payload_codec)?;
5276 let context = QueryContext {
5277 workflow_id: task.workflow_id.clone(),
5278 run_id: task.run_id.clone(),
5279 workflow_type: task.workflow_type.clone(),
5280 run_status: Some("running".to_string()),
5281 workflow_input: Value::Null,
5282 workflow_input_avro_value: AvroValue::Null,
5283 history_events: Arc::new(task.history_events.clone()),
5284 signal_events: Arc::new(Vec::new()),
5285 };
5286 let mut future = handler(context, arguments);
5287 let mut cx = TaskContext::from_waker(noop_waker_ref());
5288
5289 match future.as_mut().poll(&mut cx) {
5290 Poll::Ready(Ok(result)) => Ok(vec![json!({
5291 "type": "complete_update",
5292 "update_id": update_id,
5293 "result": encode_typed_envelope(&result, &task.payload_codec)?,
5294 })]),
5295 Poll::Ready(Err(error)) => Ok(vec![json!({
5296 "type": "fail_update",
5297 "update_id": update_id,
5298 "message": error.to_string(),
5299 "exception_type": "UpdateFailed",
5300 "non_retryable": true,
5301 })]),
5302 Poll::Pending => Err(Error::WorkflowYieldedWithoutCommand),
5303 }
5304 }
5305
5306 async fn execute_activity_task(&self, task: ActivityTask) -> Result<AvroValue> {
5307 let handler = self
5308 .activities
5309 .get(&task.activity_type)
5310 .ok_or_else(|| Error::ActivityNotRegistered(task.activity_type.clone()))?;
5311 let args = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
5312 let attempt_id = task
5313 .activity_attempt_id
5314 .clone()
5315 .or(task.attempt_id.clone())
5316 .unwrap_or_default();
5317 let lease_owner = task
5318 .lease_owner
5319 .clone()
5320 .unwrap_or_else(|| self.worker_id.clone());
5321 let ctx = ActivityContext {
5322 client: self.client.clone(),
5323 task_id: task.task_id,
5324 activity_attempt_id: attempt_id,
5325 lease_owner,
5326 activity_type: task.activity_type,
5327 attempt_number: task.attempt_number,
5328 task_queue: self.task_queue.clone(),
5329 worker_id: self.worker_id.clone(),
5330 };
5331
5332 handler(ctx, args).await
5333 }
5334}
5335
5336fn poller_result(
5337 kind: &str,
5338 result: std::result::Result<Result<()>, tokio::task::JoinError>,
5339) -> Result<()> {
5340 match result {
5341 Ok(result) => result,
5342 Err(error) => Err(Error::WorkerLoop(format!(
5343 "{kind} poller join error: {error}"
5344 ))),
5345 }
5346}
5347
5348fn optional_poller_result(
5349 kind: &str,
5350 result: Option<std::result::Result<Result<()>, tokio::task::JoinError>>,
5351) -> Result<()> {
5352 match result {
5353 Some(result) => poller_result(kind, result),
5354 None => Ok(()),
5355 }
5356}
5357
5358async fn join_pollers(
5359 workflow_poller: Option<tokio::task::JoinHandle<Result<()>>>,
5360 activity_poller: Option<tokio::task::JoinHandle<Result<()>>>,
5361 query_poller: Option<tokio::task::JoinHandle<Result<()>>>,
5362) -> Result<()> {
5363 let mut first_error = None;
5364
5365 if let Some(handle) = workflow_poller {
5366 if let Err(error) = poller_result("workflow", handle.await) {
5367 first_error.get_or_insert(error);
5368 }
5369 }
5370
5371 if let Some(handle) = activity_poller {
5372 if let Err(error) = poller_result("activity", handle.await) {
5373 first_error.get_or_insert(error);
5374 }
5375 }
5376
5377 if let Some(handle) = query_poller {
5378 if let Err(error) = poller_result("query", handle.await) {
5379 first_error.get_or_insert(error);
5380 }
5381 }
5382
5383 if let Some(error) = first_error {
5384 Err(error)
5385 } else {
5386 Ok(())
5387 }
5388}
5389
5390fn default_worker_id() -> String {
5391 let millis = SystemTime::now()
5392 .duration_since(UNIX_EPOCH)
5393 .unwrap_or_default()
5394 .as_millis();
5395 format!("rust-worker-{}-{millis}", std::process::id())
5396}
5397
5398fn unique_request_id(prefix: &str) -> String {
5399 let nanos = SystemTime::now()
5400 .duration_since(UNIX_EPOCH)
5401 .unwrap_or_default()
5402 .as_nanos();
5403 format!("{prefix}-{}-{nanos}", std::process::id())
5404}
5405
5406#[derive(Debug)]
5407struct QueryTaskExecutionFailure {
5408 reason: String,
5409 message: String,
5410 failure_type: String,
5411}
5412
5413impl QueryTaskExecutionFailure {
5414 fn new(
5415 reason: impl Into<String>,
5416 message: impl Into<String>,
5417 failure_type: impl Into<String>,
5418 ) -> Self {
5419 Self {
5420 reason: reason.into(),
5421 message: message.into(),
5422 failure_type: failure_type.into(),
5423 }
5424 }
5425}
5426
5427#[derive(Clone, Debug)]
5433pub struct WorkflowInstance<S> {
5434 state: Arc<Mutex<S>>,
5435}
5436
5437impl<S> WorkflowInstance<S> {
5438 fn new(state: S) -> Self {
5439 Self {
5440 state: Arc::new(Mutex::new(state)),
5441 }
5442 }
5443
5444 pub fn read<R>(&self, reader: impl FnOnce(&S) -> R) -> Result<R> {
5446 let state = self
5447 .state
5448 .lock()
5449 .map_err(|_| Error::WorkflowStatePoisoned)?;
5450 Ok(reader(&state))
5451 }
5452
5453 pub fn update<R>(&self, transition: impl FnOnce(&mut S) -> R) -> Result<R> {
5455 let mut state = self
5456 .state
5457 .lock()
5458 .map_err(|_| Error::WorkflowStatePoisoned)?;
5459 Ok(transition(&mut state))
5460 }
5461}
5462
5463impl<S: Clone> WorkflowInstance<S> {
5464 fn snapshot(&self) -> Result<S> {
5465 self.read(Clone::clone)
5466 }
5467}
5468
5469#[derive(Clone, Debug)]
5470pub struct WorkflowContext {
5471 state: Arc<Mutex<WorkflowState>>,
5472}
5473
5474impl WorkflowContext {
5475 pub fn workflow_identity(&self) -> Result<WorkflowIdentity> {
5477 let state = self
5478 .state
5479 .lock()
5480 .map_err(|_| Error::WorkflowStatePoisoned)?;
5481 Ok(WorkflowIdentity {
5482 workflow_id: state.workflow_id.clone(),
5483 run_id: state.run_id.clone(),
5484 })
5485 }
5486
5487 pub fn history_budget(&self) -> Result<WorkflowHistoryBudget> {
5489 let state = self
5490 .state
5491 .lock()
5492 .map_err(|_| Error::WorkflowStatePoisoned)?;
5493 Ok(state.history_budget.clone())
5494 }
5495
5496 pub fn continue_as_new<T: Serialize>(&self, args: T) -> Result<Value> {
5502 self.continue_as_new_with_options(ContinueAsNewOptions::new(), args)
5503 }
5504
5505 pub fn continue_as_new_with_options<T: Serialize>(
5507 &self,
5508 options: ContinueAsNewOptions,
5509 args: T,
5510 ) -> Result<Value> {
5511 options.validate()?;
5512 Err(Error::ContinueAsNew(ContinueAsNewRequest {
5513 arguments: normalize_avro_arguments(AvroValue::from_serialize(&args)?),
5514 options,
5515 }))
5516 }
5517
5518 pub fn activity<T: Serialize>(
5519 &self,
5520 activity_type: impl Into<String>,
5521 args: T,
5522 ) -> ActivityCall {
5523 self.activity_with_options(activity_type, ActivityOptions::new(), args)
5524 }
5525
5526 pub fn activity_on_queue<T, Q>(
5527 &self,
5528 activity_type: impl Into<String>,
5529 task_queue: Option<Q>,
5530 args: T,
5531 ) -> ActivityCall
5532 where
5533 T: Serialize,
5534 Q: Into<String>,
5535 {
5536 let mut options = ActivityOptions::new();
5537 options.task_queue = task_queue.map(Into::into);
5538 self.activity_with_options(activity_type, options, args)
5539 }
5540
5541 pub fn activity_with_options<T: Serialize>(
5579 &self,
5580 activity_type: impl Into<String>,
5581 options: ActivityOptions,
5582 args: T,
5583 ) -> ActivityCall {
5584 ActivityCall {
5585 ctx: self.clone(),
5586 activity_type: activity_type.into(),
5587 options,
5588 args: Some(AvroValue::from_serialize(&args)),
5589 scheduled: false,
5590 }
5591 }
5592
5593 pub async fn activity_avro_value<T: Serialize>(
5594 &self,
5595 activity_type: impl Into<String>,
5596 args: T,
5597 ) -> Result<AvroValue> {
5598 let mut call = self.activity(activity_type, args);
5599 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
5600 }
5601
5602 pub async fn activity_avro_value_with_options<T: Serialize>(
5603 &self,
5604 activity_type: impl Into<String>,
5605 options: ActivityOptions,
5606 args: T,
5607 ) -> Result<AvroValue> {
5608 let mut call = self.activity_with_options(activity_type, options, args);
5609 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
5610 }
5611
5612 pub fn wait_signal(&self, signal_name: impl Into<String>) -> SignalCall {
5613 SignalCall {
5614 ctx: self.clone(),
5615 signal_name: signal_name.into(),
5616 opened_wait: false,
5617 matched_pending: false,
5618 }
5619 }
5620
5621 pub async fn wait_signal_avro_value(
5622 &self,
5623 signal_name: impl Into<String>,
5624 ) -> Result<Vec<AvroValue>> {
5625 let mut call = self.wait_signal(signal_name);
5626 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
5627 }
5628
5629 pub fn sleep(&self, duration: Duration) -> TimerCall {
5650 let delay_seconds = duration
5651 .as_secs()
5652 .checked_add(u64::from(duration.subsec_nanos() > 0));
5653 TimerCall {
5654 ctx: self.clone(),
5655 delay_seconds,
5656 scheduled: false,
5657 matched_pending: false,
5658 }
5659 }
5660
5661 pub fn start_timer(&self, duration: Duration) -> TimerCall {
5663 self.sleep(duration)
5664 }
5665
5666 pub fn side_effect<T, F>(&self, callback: F) -> Result<T>
5673 where
5674 T: Serialize + DeserializeOwned,
5675 F: FnOnce() -> T,
5676 {
5677 {
5678 let mut state = self
5679 .state
5680 .lock()
5681 .map_err(|_| Error::WorkflowStatePoisoned)?;
5682 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
5683 return match recorded {
5684 RecordedCommand::SideEffect { sequence, value } => {
5685 state.command_cursor += 1;
5686 value.deserialize().map_err(|error| {
5687 Error::NonDeterministicReplay(ReplayFailure::new(
5688 "side_effect_type_mismatch",
5689 Some(sequence),
5690 Some(std::any::type_name::<T>().to_string()),
5691 Some(error.to_string()),
5692 "recorded side-effect value is incompatible with the requested Rust type",
5693 ))
5694 })
5695 }
5696 other => Err(command_mismatch(&other, "side effect")),
5697 };
5698 }
5699 }
5700
5701 let value = callback();
5702 let avro_value = AvroValue::from_serialize(&value)?;
5703 let mut state = self
5704 .state
5705 .lock()
5706 .map_err(|_| Error::WorkflowStatePoisoned)?;
5707 let result = encode_typed_envelope(&avro_value, &state.payload_codec)?;
5708 state.commands.push(json!({
5709 "type": "record_side_effect",
5710 "result": result,
5711 }));
5712 Ok(value)
5713 }
5714
5715 pub fn side_effect_avro_value<F>(&self, callback: F) -> Result<AvroValue>
5717 where
5718 F: FnOnce() -> AvroValue,
5719 {
5720 {
5721 let mut state = self
5722 .state
5723 .lock()
5724 .map_err(|_| Error::WorkflowStatePoisoned)?;
5725 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
5726 return match recorded {
5727 RecordedCommand::SideEffect { value, .. } => {
5728 state.command_cursor += 1;
5729 Ok(value)
5730 }
5731 other => Err(command_mismatch(&other, "side effect")),
5732 };
5733 }
5734 }
5735
5736 let value = callback();
5737 let mut state = self
5738 .state
5739 .lock()
5740 .map_err(|_| Error::WorkflowStatePoisoned)?;
5741 let result = encode_typed_envelope(&value, &state.payload_codec)?;
5742 state.commands.push(json!({
5743 "type": "record_side_effect",
5744 "result": result,
5745 }));
5746 Ok(value)
5747 }
5748
5749 pub fn uuid_v4(&self) -> Result<Uuid> {
5751 self.side_effect(Uuid::new_v4)
5752 }
5753
5754 pub fn get_version(
5757 &self,
5758 change_id: impl Into<String>,
5759 min_supported: i32,
5760 max_supported: i32,
5761 ) -> Result<i32> {
5762 let change_id = change_id.into();
5763 if change_id.trim().is_empty() {
5764 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5765 "version_change_id_invalid",
5766 None,
5767 Some("non-empty change ID".to_string()),
5768 Some(change_id),
5769 "version markers require a stable non-empty change ID",
5770 )));
5771 }
5772 if min_supported > max_supported {
5773 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5774 "version_range_invalid",
5775 None,
5776 Some("min_supported <= max_supported".to_string()),
5777 Some(format!("{min_supported}..={max_supported}")),
5778 "version marker supported range is invalid",
5779 )));
5780 }
5781
5782 let mut state = self
5783 .state
5784 .lock()
5785 .map_err(|_| Error::WorkflowStatePoisoned)?;
5786 if let Some((version, sequence)) = state.version_markers.get(&change_id).copied() {
5787 ensure_version_supported(&change_id, version, min_supported, max_supported, sequence)?;
5788 return Ok(version);
5789 }
5790
5791 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
5792 return match recorded {
5793 RecordedCommand::VersionMarker {
5794 sequence,
5795 change_id: recorded_change_id,
5796 version,
5797 ..
5798 } => {
5799 if recorded_change_id != change_id {
5800 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5801 "version_change_id_mismatch",
5802 Some(sequence),
5803 Some(recorded_change_id),
5804 Some(change_id),
5805 "recorded version marker change ID differs from current workflow code",
5806 )));
5807 }
5808 ensure_version_supported(
5809 &change_id,
5810 version,
5811 min_supported,
5812 max_supported,
5813 sequence,
5814 )?;
5815 state.command_cursor += 1;
5816 state.version_markers.insert(change_id, (version, sequence));
5817 Ok(version)
5818 }
5819 other => Err(command_mismatch(
5820 &other,
5821 format!("version marker:{change_id}"),
5822 )),
5823 };
5824 }
5825
5826 let version = max_supported;
5827 state.commands.push(json!({
5828 "type": "record_version_marker",
5829 "change_id": change_id,
5830 "version": version,
5831 "min_supported": min_supported,
5832 "max_supported": max_supported,
5833 }));
5834 state.version_markers.insert(change_id, (version, 0));
5837 Ok(version)
5838 }
5839
5840 pub fn patched(&self, change_id: impl Into<String>) -> Result<bool> {
5842 Ok(self.get_version(change_id, -1, 1)? == 1)
5843 }
5844
5845 pub fn deprecate_patch(&self, change_id: impl Into<String>) -> Result<()> {
5847 self.get_version(change_id, -1, 1).map(|_| ())
5848 }
5849
5850 pub fn start_child_workflow<T: Serialize>(
5877 &self,
5878 workflow_type: impl Into<String>,
5879 options: ChildWorkflowOptions,
5880 args: T,
5881 ) -> ChildWorkflowCall {
5882 ChildWorkflowCall {
5883 ctx: self.clone(),
5884 workflow_type: workflow_type.into(),
5885 options,
5886 args: Some(AvroValue::from_serialize(&args)),
5887 scheduled: false,
5888 matched_pending: false,
5889 }
5890 }
5891
5892 pub async fn start_child_workflow_avro_value<T: Serialize>(
5893 &self,
5894 workflow_type: impl Into<String>,
5895 options: ChildWorkflowOptions,
5896 args: T,
5897 ) -> Result<ChildWorkflowAvroResult> {
5898 let mut call = self.start_child_workflow(workflow_type, options, args);
5899 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
5900 }
5901
5902 fn take_commands(&self) -> Result<Vec<Value>> {
5903 let mut state = self
5904 .state
5905 .lock()
5906 .map_err(|_| Error::WorkflowStatePoisoned)?;
5907 Ok(std::mem::take(&mut state.commands))
5908 }
5909
5910 fn continue_as_new_command(&self, request: ContinueAsNewRequest) -> Result<Option<Value>> {
5911 let mut state = self
5912 .state
5913 .lock()
5914 .map_err(|_| Error::WorkflowStatePoisoned)?;
5915
5916 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
5917 return Err(command_mismatch(&recorded, "continue as new"));
5918 }
5919 if state.recorded_continue_as_new_sequence.is_some() {
5920 state.continue_as_new_consumed = true;
5921 return Ok(None);
5922 }
5923
5924 let arguments = encode_typed_envelope(&request.arguments, &state.payload_codec)?;
5925 let mut command = serde_json::Map::from_iter([
5926 ("type".to_string(), json!("continue_as_new")),
5927 ("arguments".to_string(), arguments),
5928 ("queue".to_string(), json!(state.task_queue.clone())),
5929 ]);
5930 if let Some(workflow_type) = request.options.workflow_type {
5931 command.insert("workflow_type".to_string(), json!(workflow_type));
5932 }
5933 if let Some(task_queue) = request.options.task_queue {
5934 command.insert("queue".to_string(), json!(task_queue));
5935 }
5936 Ok(Some(Value::Object(command)))
5937 }
5938
5939 fn matched_recorded_pending(&self) -> Result<bool> {
5940 let state = self
5941 .state
5942 .lock()
5943 .map_err(|_| Error::WorkflowStatePoisoned)?;
5944 Ok(state.matched_recorded_pending)
5945 }
5946
5947 fn ensure_history_consumed(&self) -> Result<()> {
5948 let state = self
5949 .state
5950 .lock()
5951 .map_err(|_| Error::WorkflowStatePoisoned)?;
5952 if let Some(command) = state.recorded_commands.get(state.command_cursor) {
5953 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5954 "recorded_commands_unconsumed",
5955 Some(command.sequence()),
5956 Some(command.shape().to_string()),
5957 Some("workflow completion".to_string()),
5958 "workflow completed before consuming all recorded durable commands",
5959 )));
5960 }
5961 if let Some(sequence) = state
5962 .recorded_continue_as_new_sequence
5963 .filter(|_| !state.continue_as_new_consumed)
5964 {
5965 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5966 "recorded_continue_as_new_unconsumed",
5967 Some(sequence),
5968 Some("continue as new".to_string()),
5969 Some("workflow completion".to_string()),
5970 "workflow completed without consuming its recorded continue-as-new transition",
5971 )));
5972 }
5973 Ok(())
5974 }
5975}
5976
5977#[derive(Debug)]
5978struct WorkflowState {
5979 workflow_id: Option<String>,
5980 run_id: Option<String>,
5981 task_queue: String,
5982 payload_codec: String,
5983 history_budget: WorkflowHistoryBudget,
5984 resume_signal: Option<ResumeSignal>,
5985 recorded_commands: Vec<RecordedCommand>,
5986 recorded_continue_as_new_sequence: Option<u64>,
5987 continue_as_new_consumed: bool,
5988 command_cursor: usize,
5989 matched_recorded_pending: bool,
5990 version_markers: HashMap<String, (i32, u64)>,
5991 commands: Vec<Value>,
5992}
5993
5994impl WorkflowState {
5995 #[cfg(test)]
5996 fn new(
5997 history: Vec<HistoryEvent>,
5998 task_queue: String,
5999 payload_codec: String,
6000 resume_signal: Option<ResumeSignal>,
6001 ) -> Result<Self> {
6002 Self::new_with_identity(
6003 history,
6004 None,
6005 None,
6006 task_queue,
6007 payload_codec,
6008 resume_signal,
6009 )
6010 }
6011
6012 fn new_with_identity(
6013 history: Vec<HistoryEvent>,
6014 workflow_id: Option<String>,
6015 run_id: Option<String>,
6016 task_queue: String,
6017 payload_codec: String,
6018 resume_signal: Option<ResumeSignal>,
6019 ) -> Result<Self> {
6020 let recorded_commands = recorded_commands(
6021 &history,
6022 &payload_codec,
6023 WorkflowIdentity {
6024 workflow_id: workflow_id.clone(),
6025 run_id: run_id.clone(),
6026 },
6027 )?;
6028 let recorded_continue_as_new = history
6029 .iter()
6030 .filter(|event| event.event_type == "WorkflowContinuedAsNew")
6031 .collect::<Vec<_>>();
6032 if recorded_continue_as_new.len() > 1 {
6033 return Err(invalid_recorded_history(
6034 "duplicate_continue_as_new_transition",
6035 recorded_continue_as_new
6036 .last()
6037 .and_then(|event| durable_event_sequence(event))
6038 .unwrap_or(0),
6039 "one WorkflowContinuedAsNew event",
6040 &format!(
6041 "{} WorkflowContinuedAsNew events",
6042 recorded_continue_as_new.len()
6043 ),
6044 "workflow history records one continue-as-new transition more than once",
6045 ));
6046 }
6047 let recorded_continue_as_new_sequence = recorded_continue_as_new
6048 .first()
6049 .map(|event| {
6050 durable_event_sequence(event).ok_or_else(|| {
6051 Error::NonDeterministicReplay(ReplayFailure::new(
6052 "continue_as_new_sequence_missing",
6053 None,
6054 Some("recorded transition sequence".to_string()),
6055 Some("missing sequence".to_string()),
6056 "WorkflowContinuedAsNew history is missing its recorded sequence",
6057 ))
6058 })
6059 })
6060 .transpose()?;
6061 let event_count = u64::try_from(history.len()).unwrap_or(u64::MAX);
6062 Ok(Self {
6063 workflow_id,
6064 run_id,
6065 task_queue,
6066 payload_codec,
6067 history_budget: WorkflowHistoryBudget {
6068 event_count,
6069 ..WorkflowHistoryBudget::default()
6070 },
6071 resume_signal,
6072 recorded_commands,
6073 recorded_continue_as_new_sequence,
6074 continue_as_new_consumed: false,
6075 command_cursor: 0,
6076 matched_recorded_pending: false,
6077 version_markers: HashMap::new(),
6078 commands: Vec::new(),
6079 })
6080 }
6081}
6082
6083#[derive(Clone, Debug)]
6084enum RecordedCommand {
6085 Activity {
6086 sequence: u64,
6087 activity_type: Option<String>,
6088 options: Option<RecordedActivityOptions>,
6089 outcome: Option<ActivityOutcome>,
6090 },
6091 Timer {
6092 sequence: u64,
6093 delay_seconds: u64,
6094 fired: bool,
6095 },
6096 ChildWorkflow {
6097 sequence: u64,
6098 workflow_type: Option<String>,
6099 outcome: Option<ChildWorkflowOutcome>,
6100 },
6101 SignalWait {
6102 sequence: u64,
6103 signal_name: String,
6104 value: Option<Vec<AvroValue>>,
6105 },
6106 SideEffect {
6107 sequence: u64,
6108 value: AvroValue,
6109 },
6110 VersionMarker {
6111 sequence: u64,
6112 change_id: String,
6113 version: i32,
6114 },
6115}
6116
6117#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
6118struct RecordedActivityOptions {
6119 task_queue: RecordedSnapshotValue<Option<String>>,
6120 execution_mode: RecordedSnapshotValue<Option<String>>,
6121 retry_policy: ActivityRetrySnapshot,
6122}
6123
6124#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
6125enum RecordedSnapshotValue<T> {
6126 Unknown,
6128 Known(T),
6129}
6130
6131impl<T: PartialEq> RecordedSnapshotValue<T> {
6132 fn matches_current(&self, current: &Self) -> bool {
6133 match self {
6134 Self::Unknown => true,
6135 Self::Known(recorded) => matches!(current, Self::Known(value) if value == recorded),
6136 }
6137 }
6138}
6139
6140#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
6141struct ActivityRetrySnapshot {
6142 snapshot_version: RecordedSnapshotValue<Option<u64>>,
6143 max_attempts: RecordedSnapshotValue<Option<u64>>,
6144 backoff_seconds: RecordedSnapshotValue<Vec<u64>>,
6145 start_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
6146 schedule_to_start_timeout: RecordedSnapshotValue<Option<u64>>,
6147 schedule_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
6148 heartbeat_timeout: RecordedSnapshotValue<Option<u64>>,
6149 non_retryable_error_types: RecordedSnapshotValue<Vec<String>>,
6150}
6151
6152impl ActivityRetrySnapshot {
6153 fn matches_current(&self, current: &Self) -> bool {
6154 self.snapshot_version
6155 .matches_current(¤t.snapshot_version)
6156 && self.max_attempts.matches_current(¤t.max_attempts)
6157 && self
6158 .backoff_seconds
6159 .matches_current(¤t.backoff_seconds)
6160 && self
6161 .start_to_close_timeout
6162 .matches_current(¤t.start_to_close_timeout)
6163 && self
6164 .schedule_to_start_timeout
6165 .matches_current(¤t.schedule_to_start_timeout)
6166 && self
6167 .schedule_to_close_timeout
6168 .matches_current(¤t.schedule_to_close_timeout)
6169 && self
6170 .heartbeat_timeout
6171 .matches_current(¤t.heartbeat_timeout)
6172 && self
6173 .non_retryable_error_types
6174 .matches_current(¤t.non_retryable_error_types)
6175 }
6176}
6177
6178fn recorded_optional_u64(
6179 object: Option<&serde_json::Map<String, Value>>,
6180 field: &str,
6181) -> RecordedSnapshotValue<Option<u64>> {
6182 match object.and_then(|object| object.get(field)) {
6183 None => RecordedSnapshotValue::Unknown,
6184 Some(Value::Null) => RecordedSnapshotValue::Known(None),
6185 Some(value) => RecordedSnapshotValue::Known(value_as_u64(value)),
6186 }
6187}
6188
6189fn recorded_optional_string(
6190 object: &serde_json::Map<String, Value>,
6191 field: &str,
6192) -> RecordedSnapshotValue<Option<String>> {
6193 match object.get(field) {
6194 None => RecordedSnapshotValue::Unknown,
6195 Some(Value::Null) => RecordedSnapshotValue::Known(None),
6196 Some(value) => RecordedSnapshotValue::Known(value.as_str().map(str::to_string)),
6197 }
6198}
6199
6200fn recorded_activity_retry_snapshot(policy: Option<&Value>) -> ActivityRetrySnapshot {
6201 let policy = policy.and_then(Value::as_object);
6202 let backoff_seconds = policy
6203 .and_then(|policy| policy.get("backoff_seconds"))
6204 .and_then(Value::as_array)
6205 .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
6206 .map_or(RecordedSnapshotValue::Unknown, RecordedSnapshotValue::Known);
6207 let mut non_retryable_error_types = Vec::new();
6208 for error_type in policy
6209 .and_then(|policy| policy.get("non_retryable_error_types"))
6210 .and_then(Value::as_array)
6211 .into_iter()
6212 .flatten()
6213 .filter_map(Value::as_str)
6214 .map(str::trim)
6215 .filter(|error_type| !error_type.is_empty())
6216 {
6217 if !non_retryable_error_types
6218 .iter()
6219 .any(|recorded| recorded == error_type)
6220 {
6221 non_retryable_error_types.push(error_type.to_string());
6222 }
6223 }
6224
6225 ActivityRetrySnapshot {
6226 snapshot_version: recorded_optional_u64(policy, "snapshot_version"),
6227 max_attempts: recorded_optional_u64(policy, "max_attempts"),
6228 backoff_seconds,
6229 start_to_close_timeout: recorded_optional_u64(policy, "start_to_close_timeout"),
6230 schedule_to_start_timeout: recorded_optional_u64(policy, "schedule_to_start_timeout"),
6231 schedule_to_close_timeout: recorded_optional_u64(policy, "schedule_to_close_timeout"),
6232 heartbeat_timeout: recorded_optional_u64(policy, "heartbeat_timeout"),
6233 non_retryable_error_types: if policy
6234 .is_some_and(|policy| policy.contains_key("non_retryable_error_types"))
6235 {
6236 RecordedSnapshotValue::Known(non_retryable_error_types)
6237 } else {
6238 RecordedSnapshotValue::Unknown
6239 },
6240 }
6241}
6242
6243fn current_activity_retry_snapshot(options: &ValidatedActivityOptions) -> ActivityRetrySnapshot {
6244 let policy = options.retry_policy.as_ref();
6245 let max_attempts = match policy.and_then(|policy| policy.get("max_attempts")) {
6246 Some(Value::Null) => None,
6247 Some(value) => value_as_u64(value),
6248 None => Some(1),
6249 };
6250 let backoff_seconds = policy
6251 .and_then(|policy| policy.get("backoff_seconds"))
6252 .and_then(Value::as_array)
6253 .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
6254 .unwrap_or_default();
6255 let non_retryable_error_types = policy
6256 .and_then(|policy| policy.get("non_retryable_error_types"))
6257 .and_then(Value::as_array)
6258 .into_iter()
6259 .flatten()
6260 .filter_map(Value::as_str)
6261 .map(str::to_string)
6262 .collect();
6263
6264 ActivityRetrySnapshot {
6265 snapshot_version: RecordedSnapshotValue::Known(Some(1)),
6266 max_attempts: RecordedSnapshotValue::Known(max_attempts),
6267 backoff_seconds: RecordedSnapshotValue::Known(backoff_seconds),
6268 start_to_close_timeout: RecordedSnapshotValue::Known(options.start_to_close_timeout),
6269 schedule_to_start_timeout: RecordedSnapshotValue::Known(options.schedule_to_start_timeout),
6270 schedule_to_close_timeout: RecordedSnapshotValue::Known(options.schedule_to_close_timeout),
6271 heartbeat_timeout: RecordedSnapshotValue::Known(options.heartbeat_timeout),
6272 non_retryable_error_types: RecordedSnapshotValue::Known(non_retryable_error_types),
6273 }
6274}
6275
6276fn activity_options_description(options: &RecordedActivityOptions) -> String {
6277 serde_json::to_string(options).unwrap_or_else(|_| format!("{options:?}"))
6278}
6279
6280impl RecordedCommand {
6281 fn sequence(&self) -> u64 {
6282 match self {
6283 Self::Activity { sequence, .. }
6284 | Self::Timer { sequence, .. }
6285 | Self::ChildWorkflow { sequence, .. }
6286 | Self::SignalWait { sequence, .. }
6287 | Self::SideEffect { sequence, .. }
6288 | Self::VersionMarker { sequence, .. } => *sequence,
6289 }
6290 }
6291
6292 fn shape(&self) -> &'static str {
6293 match self {
6294 Self::Activity { .. } => "activity",
6295 Self::Timer { .. } => "timer",
6296 Self::ChildWorkflow { .. } => "child workflow",
6297 Self::SignalWait { .. } => "signal wait",
6298 Self::SideEffect { .. } => "side effect",
6299 Self::VersionMarker { .. } => "version marker",
6300 }
6301 }
6302}
6303
6304fn ensure_version_supported(
6305 change_id: &str,
6306 version: i32,
6307 min_supported: i32,
6308 max_supported: i32,
6309 sequence: u64,
6310) -> Result<()> {
6311 if (min_supported..=max_supported).contains(&version) {
6312 return Ok(());
6313 }
6314 Err(Error::NonDeterministicReplay(ReplayFailure::new(
6315 "version_marker_incompatible_range",
6316 (sequence != 0).then_some(sequence),
6317 Some(format!("{min_supported}..={max_supported}")),
6318 Some(format!("{change_id}:{version}")),
6319 "recorded workflow version is outside the range supported by current code",
6320 )))
6321}
6322
6323#[derive(Clone, Debug)]
6324struct ResumeSignal {
6325 signal_name: String,
6326 arguments: Vec<AvroValue>,
6327}
6328
6329pub struct ActivityCall {
6330 ctx: WorkflowContext,
6331 activity_type: String,
6332 options: ActivityOptions,
6333 args: Option<Result<AvroValue>>,
6334 scheduled: bool,
6335}
6336
6337impl ActivityCall {
6338 fn poll_avro_value(
6339 mut self: Pin<&mut Self>,
6340 _cx: &mut TaskContext<'_>,
6341 ) -> Poll<Result<AvroValue>> {
6342 let ctx = self.ctx.clone();
6343 let mut state = match ctx.state.lock() {
6344 Ok(state) => state,
6345 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6346 };
6347
6348 if self.scheduled {
6349 return Poll::Pending;
6350 }
6351
6352 let options = match self.options.validate() {
6353 Ok(options) => options,
6354 Err(error) => {
6355 return Poll::Ready(Err(Error::InvalidActivityOptions(error)));
6356 }
6357 };
6358 let task_queue = options
6359 .task_queue
6360 .clone()
6361 .unwrap_or_else(|| state.task_queue.clone());
6362 let current_recorded_options = RecordedActivityOptions {
6363 task_queue: RecordedSnapshotValue::Known(Some(task_queue.clone())),
6364 execution_mode: RecordedSnapshotValue::Known(None),
6367 retry_policy: current_activity_retry_snapshot(&options),
6368 };
6369
6370 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6371 let sequence = recorded.sequence();
6372 match recorded {
6373 RecordedCommand::Activity {
6374 activity_type,
6375 options: recorded_options,
6376 outcome,
6377 ..
6378 } => {
6379 if let Some(recorded_type) = activity_type {
6380 if recorded_type != self.activity_type {
6381 return Poll::Ready(Err(Error::NonDeterministicReplay(
6382 ReplayFailure::new(
6383 "recorded_command_detail_mismatch",
6384 Some(sequence),
6385 Some(format!("activity:{recorded_type}")),
6386 Some(format!("activity:{}", self.activity_type)),
6387 "recorded activity type differs from the current workflow command",
6388 ),
6389 )));
6390 }
6391 }
6392 if let Some(recorded_options) = recorded_options {
6393 if !recorded_options
6394 .task_queue
6395 .matches_current(¤t_recorded_options.task_queue)
6396 {
6397 return Poll::Ready(Err(Error::NonDeterministicReplay(
6398 ReplayFailure::new(
6399 "activity_task_queue_mismatch",
6400 Some(sequence),
6401 Some(activity_options_description(&recorded_options)),
6402 Some(activity_options_description(¤t_recorded_options)),
6403 "recorded activity task queue differs from the current workflow command",
6404 ),
6405 )));
6406 }
6407 if !recorded_options
6408 .execution_mode
6409 .matches_current(¤t_recorded_options.execution_mode)
6410 {
6411 return Poll::Ready(Err(Error::NonDeterministicReplay(
6412 ReplayFailure::new(
6413 "activity_execution_mode_mismatch",
6414 Some(sequence),
6415 Some(activity_options_description(&recorded_options)),
6416 Some(activity_options_description(¤t_recorded_options)),
6417 "recorded activity execution mode differs from the current workflow command",
6418 ),
6419 )));
6420 }
6421 if !recorded_options
6422 .retry_policy
6423 .matches_current(¤t_recorded_options.retry_policy)
6424 {
6425 return Poll::Ready(Err(Error::NonDeterministicReplay(
6426 ReplayFailure::new(
6427 "activity_retry_policy_mismatch",
6428 Some(sequence),
6429 Some(activity_options_description(&recorded_options)),
6430 Some(activity_options_description(¤t_recorded_options)),
6431 "recorded activity retry policy differs from the current workflow command",
6432 ),
6433 )));
6434 }
6435 }
6436 state.command_cursor += 1;
6437 if let Some(outcome) = outcome {
6438 return Poll::Ready(outcome.map_err(Error::ActivityFailed));
6439 }
6440 state.matched_recorded_pending = true;
6441 self.scheduled = true;
6442 return Poll::Pending;
6443 }
6444 other => {
6445 return Poll::Ready(Err(command_mismatch(
6446 &other,
6447 format!("activity:{}", self.activity_type),
6448 )));
6449 }
6450 }
6451 }
6452
6453 if !self.scheduled {
6454 let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
6455 Ok(args) => args,
6456 Err(error) => return Poll::Ready(Err(error)),
6457 };
6458 let arguments = normalize_avro_arguments(args);
6459 let envelope = match encode_typed_envelope(&arguments, &state.payload_codec) {
6460 Ok(envelope) => envelope,
6461 Err(error) => return Poll::Ready(Err(error)),
6462 };
6463
6464 let mut command = serde_json::Map::from_iter([
6465 ("type".to_string(), json!("schedule_activity")),
6466 (
6467 "activity_type".to_string(),
6468 json!(self.activity_type.clone()),
6469 ),
6470 ("queue".to_string(), json!(task_queue)),
6471 ("arguments".to_string(), envelope),
6472 ]);
6473 for (field, value) in [
6474 ("start_to_close_timeout", options.start_to_close_timeout),
6475 (
6476 "schedule_to_start_timeout",
6477 options.schedule_to_start_timeout,
6478 ),
6479 (
6480 "schedule_to_close_timeout",
6481 options.schedule_to_close_timeout,
6482 ),
6483 ("heartbeat_timeout", options.heartbeat_timeout),
6484 ] {
6485 if let Some(value) = value {
6486 command.insert(field.to_string(), json!(value));
6487 }
6488 }
6489 if let Some(retry_policy) = options.retry_policy {
6490 command.insert("retry_policy".to_string(), retry_policy);
6491 }
6492 state.commands.push(Value::Object(command));
6493 self.scheduled = true;
6494 }
6495
6496 Poll::Pending
6497 }
6498}
6499
6500impl Future for ActivityCall {
6501 type Output = Result<Value>;
6502
6503 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6504 match self.poll_avro_value(cx) {
6505 Poll::Ready(Ok(value)) => Poll::Ready(value.into_json()),
6506 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
6507 Poll::Pending => Poll::Pending,
6508 }
6509 }
6510}
6511
6512pub struct TimerCall {
6514 ctx: WorkflowContext,
6515 delay_seconds: Option<u64>,
6516 scheduled: bool,
6517 matched_pending: bool,
6518}
6519
6520impl Future for TimerCall {
6521 type Output = Result<()>;
6522
6523 fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6524 if self.matched_pending {
6525 return Poll::Pending;
6526 }
6527
6528 let ctx = self.ctx.clone();
6529 let Some(requested_delay) = self.delay_seconds else {
6530 return Poll::Ready(Err(Error::TimerDurationOverflow));
6531 };
6532 let mut state = match ctx.state.lock() {
6533 Ok(state) => state,
6534 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6535 };
6536
6537 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6538 match recorded {
6539 RecordedCommand::Timer {
6540 sequence,
6541 delay_seconds,
6542 fired,
6543 ..
6544 } => {
6545 if delay_seconds != requested_delay {
6546 return Poll::Ready(Err(Error::NonDeterministicReplay(
6547 ReplayFailure::new(
6548 "timer_delay_mismatch",
6549 Some(sequence),
6550 Some(format!("timer:{delay_seconds}s")),
6551 Some(format!("timer:{requested_delay}s")),
6552 "recorded timer delay differs from the current workflow command",
6553 ),
6554 )));
6555 }
6556 state.command_cursor += 1;
6557 if fired {
6558 return Poll::Ready(Ok(()));
6559 }
6560 state.matched_recorded_pending = true;
6561 self.scheduled = true;
6562 self.matched_pending = true;
6563 return Poll::Pending;
6564 }
6565 other => return Poll::Ready(Err(command_mismatch(&other, "timer"))),
6566 }
6567 }
6568
6569 if !self.scheduled {
6570 state.commands.push(json!({
6571 "type": "start_timer",
6572 "delay_seconds": requested_delay,
6573 }));
6574 self.scheduled = true;
6575 }
6576
6577 Poll::Pending
6578 }
6579}
6580
6581pub struct ChildWorkflowCall {
6583 ctx: WorkflowContext,
6584 workflow_type: String,
6585 options: ChildWorkflowOptions,
6586 args: Option<Result<AvroValue>>,
6587 scheduled: bool,
6588 matched_pending: bool,
6589}
6590
6591impl ChildWorkflowCall {
6592 fn poll_avro_value(
6593 mut self: Pin<&mut Self>,
6594 _cx: &mut TaskContext<'_>,
6595 ) -> Poll<Result<ChildWorkflowAvroResult>> {
6596 if self.matched_pending {
6597 return Poll::Pending;
6598 }
6599
6600 let ctx = self.ctx.clone();
6601 let mut state = match ctx.state.lock() {
6602 Ok(state) => state,
6603 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6604 };
6605
6606 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6607 let sequence = recorded.sequence();
6608 match recorded {
6609 RecordedCommand::ChildWorkflow {
6610 workflow_type,
6611 outcome,
6612 ..
6613 } => {
6614 if let Some(recorded_type) = workflow_type {
6615 if recorded_type != self.workflow_type {
6616 return Poll::Ready(Err(Error::NonDeterministicReplay(
6617 ReplayFailure::new(
6618 "recorded_command_detail_mismatch",
6619 Some(sequence),
6620 Some(format!("child workflow:{recorded_type}")),
6621 Some(format!("child workflow:{}", self.workflow_type)),
6622 "recorded child workflow type differs from the current workflow command",
6623 ),
6624 )));
6625 }
6626 }
6627 state.command_cursor += 1;
6628 if let Some(outcome) = outcome {
6629 return Poll::Ready(outcome.map_err(Error::ChildWorkflowFailed));
6630 }
6631 state.matched_recorded_pending = true;
6632 self.scheduled = true;
6633 self.matched_pending = true;
6634 return Poll::Pending;
6635 }
6636 other => {
6637 return Poll::Ready(Err(command_mismatch(
6638 &other,
6639 format!("child workflow:{}", self.workflow_type),
6640 )));
6641 }
6642 }
6643 }
6644
6645 if !self.scheduled {
6646 if self.options.task_queue.trim().is_empty() {
6647 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
6648 "task_queue must not be empty".to_string(),
6649 )));
6650 }
6651 for (name, value) in [
6652 (
6653 "execution_timeout_seconds",
6654 self.options.execution_timeout_seconds,
6655 ),
6656 ("run_timeout_seconds", self.options.run_timeout_seconds),
6657 ] {
6658 if value == Some(0) {
6659 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(format!(
6660 "{name} must be at least 1"
6661 ))));
6662 }
6663 }
6664
6665 let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
6666 Ok(args) => args,
6667 Err(error) => return Poll::Ready(Err(error)),
6668 };
6669 let arguments = match encode_typed_envelope(
6670 &normalize_avro_arguments(args),
6671 &state.payload_codec,
6672 ) {
6673 Ok(arguments) => arguments,
6674 Err(error) => return Poll::Ready(Err(error)),
6675 };
6676 let mut command = json!({
6677 "type": "start_child_workflow",
6678 "workflow_type": self.workflow_type,
6679 "queue": self.options.task_queue,
6680 "parent_close_policy": self.options.parent_close_policy.as_str(),
6681 "arguments": arguments,
6682 });
6683 let object = command
6684 .as_object_mut()
6685 .expect("child workflow command is always an object");
6686 if let Some(policy) = &self.options.retry_policy {
6687 let mut retry_policy = serde_json::Map::new();
6688 if let Some(max_attempts) = policy.max_attempts {
6689 if max_attempts == 0 {
6690 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
6691 "retry_policy.max_attempts must be at least 1".to_string(),
6692 )));
6693 }
6694 retry_policy.insert("max_attempts".to_string(), json!(max_attempts));
6695 }
6696 if !policy.backoff_seconds.is_empty() {
6697 retry_policy
6698 .insert("backoff_seconds".to_string(), json!(policy.backoff_seconds));
6699 }
6700 if !policy.non_retryable_error_types.is_empty() {
6701 retry_policy.insert(
6702 "non_retryable_error_types".to_string(),
6703 json!(policy.non_retryable_error_types),
6704 );
6705 }
6706 if retry_policy.is_empty() {
6707 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
6708 "retry_policy must configure at least one field".to_string(),
6709 )));
6710 }
6711 object.insert("retry_policy".to_string(), Value::Object(retry_policy));
6712 }
6713 if let Some(seconds) = self.options.execution_timeout_seconds {
6714 object.insert("execution_timeout_seconds".to_string(), json!(seconds));
6715 }
6716 if let Some(seconds) = self.options.run_timeout_seconds {
6717 object.insert("run_timeout_seconds".to_string(), json!(seconds));
6718 }
6719 state.commands.push(command);
6720 self.scheduled = true;
6721 }
6722
6723 Poll::Pending
6724 }
6725}
6726
6727impl Future for ChildWorkflowCall {
6728 type Output = Result<ChildWorkflowResult>;
6729
6730 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6731 match self.poll_avro_value(cx) {
6732 Poll::Ready(Ok(result)) => match result.result.into_json() {
6733 Ok(projected) => Poll::Ready(Ok(ChildWorkflowResult {
6734 parent: result.parent,
6735 child: result.child,
6736 child_workflow_type: result.child_workflow_type,
6737 result: projected,
6738 })),
6739 Err(error) => Poll::Ready(Err(error)),
6740 },
6741 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
6742 Poll::Pending => Poll::Pending,
6743 }
6744 }
6745}
6746
6747fn command_mismatch(recorded: &RecordedCommand, actual: impl Into<String>) -> Error {
6748 Error::NonDeterministicReplay(ReplayFailure::new(
6749 "recorded_command_mismatch",
6750 Some(recorded.sequence()),
6751 Some(recorded.shape().to_string()),
6752 Some(actual.into()),
6753 "current workflow command does not match the recorded durable command sequence",
6754 ))
6755}
6756
6757pub struct SignalCall {
6758 ctx: WorkflowContext,
6759 signal_name: String,
6760 opened_wait: bool,
6761 matched_pending: bool,
6762}
6763
6764impl SignalCall {
6765 fn poll_avro_value(
6766 mut self: Pin<&mut Self>,
6767 _cx: &mut TaskContext<'_>,
6768 ) -> Poll<Result<Vec<AvroValue>>> {
6769 if self.matched_pending {
6770 return Poll::Pending;
6771 }
6772
6773 let ctx = self.ctx.clone();
6774 let mut state = match ctx.state.lock() {
6775 Ok(state) => state,
6776 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6777 };
6778
6779 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6780 match recorded {
6781 RecordedCommand::SignalWait {
6782 sequence,
6783 signal_name,
6784 value,
6785 } => {
6786 if signal_name != self.signal_name {
6787 return Poll::Ready(Err(Error::NonDeterministicReplay(
6788 ReplayFailure::new(
6789 "recorded_command_detail_mismatch",
6790 Some(sequence),
6791 Some(format!("signal wait:{signal_name}")),
6792 Some(format!("signal wait:{}", self.signal_name)),
6793 "recorded signal name differs from the current workflow command",
6794 ),
6795 )));
6796 }
6797
6798 state.command_cursor += 1;
6799 if let Some(value) = value {
6800 return Poll::Ready(Ok(value));
6801 }
6802 if state
6803 .resume_signal
6804 .as_ref()
6805 .is_some_and(|signal| signal.signal_name == self.signal_name)
6806 {
6807 let signal = state
6808 .resume_signal
6809 .take()
6810 .expect("matching resume signal is present");
6811 return Poll::Ready(Ok(signal.arguments));
6812 }
6813
6814 state.matched_recorded_pending = true;
6815 self.opened_wait = true;
6816 self.matched_pending = true;
6817 return Poll::Pending;
6818 }
6819 other => {
6820 return Poll::Ready(Err(command_mismatch(
6821 &other,
6822 format!("signal wait:{}", self.signal_name),
6823 )));
6824 }
6825 }
6826 }
6827
6828 if state
6829 .resume_signal
6830 .as_ref()
6831 .is_some_and(|signal| signal.signal_name == self.signal_name)
6832 {
6833 let signal = state
6834 .resume_signal
6835 .take()
6836 .expect("matching resume signal is present");
6837 return Poll::Ready(Ok(signal.arguments));
6838 }
6839
6840 if !self.opened_wait {
6841 state.commands.push(json!({
6842 "type": "open_signal_wait",
6843 "signal_name": self.signal_name
6844 }));
6845 self.opened_wait = true;
6846 }
6847
6848 Poll::Pending
6849 }
6850}
6851
6852impl Future for SignalCall {
6853 type Output = Result<Vec<Value>>;
6854
6855 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6856 match self.poll_avro_value(cx) {
6857 Poll::Ready(Ok(values)) => Poll::Ready(
6858 values
6859 .into_iter()
6860 .map(AvroValue::into_json)
6861 .collect::<Result<Vec<_>>>(),
6862 ),
6863 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
6864 Poll::Pending => Poll::Pending,
6865 }
6866 }
6867}
6868
6869#[derive(Clone, Debug)]
6870pub struct ActivityContext {
6871 client: Client,
6872 pub task_id: String,
6873 pub activity_attempt_id: String,
6874 pub lease_owner: String,
6875 pub activity_type: String,
6876 pub attempt_number: u64,
6877 pub task_queue: String,
6878 pub worker_id: String,
6879}
6880
6881impl ActivityContext {
6882 pub async fn heartbeat<T: Serialize>(&self, details: T) -> Result<ActivityHeartbeatResponse> {
6883 self.client
6884 .heartbeat_activity_task(
6885 &self.task_id,
6886 &self.activity_attempt_id,
6887 &self.lease_owner,
6888 details,
6889 )
6890 .await
6891 }
6892}
6893
6894fn decode_task_avro_arguments(value: Option<&Value>, codec: &str) -> Result<AvroValue> {
6895 match value {
6896 Some(value) => Ok(normalize_avro_arguments(decode_wire_avro_value(
6897 value, codec,
6898 )?)),
6899 None => Ok(AvroValue::Array(Vec::new())),
6900 }
6901}
6902
6903fn decode_resume_signal(task: &WorkflowTask) -> Result<Option<ResumeSignal>> {
6904 let Some(signal_name) = task
6905 .signal_name
6906 .as_deref()
6907 .filter(|value| !value.is_empty())
6908 else {
6909 return Ok(None);
6910 };
6911 let Some(arguments) = task.signal_arguments.as_ref() else {
6912 return Ok(None);
6913 };
6914
6915 let decoded = normalize_avro_arguments(decode_wire_avro_value(arguments, &task.payload_codec)?);
6916 let AvroValue::Array(arguments) = decoded else {
6917 unreachable!("normalize_avro_arguments always returns an array");
6918 };
6919
6920 Ok(Some(ResumeSignal {
6921 signal_name: signal_name.to_string(),
6922 arguments,
6923 }))
6924}
6925
6926fn recorded_commands(
6927 events: &[HistoryEvent],
6928 fallback_codec: &str,
6929 parent: WorkflowIdentity,
6930) -> Result<Vec<RecordedCommand>> {
6931 let mut events_by_sequence: BTreeMap<u64, Vec<&HistoryEvent>> = BTreeMap::new();
6932 let mut last_new_sequence = None;
6933
6934 for event in events {
6935 let is_activity = matches!(
6936 event.event_type.as_str(),
6937 "ActivityScheduled"
6938 | "ActivityStarted"
6939 | "ActivityHeartbeatRecorded"
6940 | "ActivityRetryScheduled"
6941 | "ActivityCompleted"
6942 | "ActivityFailed"
6943 | "ActivityCancelled"
6944 | "ActivityTimedOut"
6945 );
6946 let is_workflow_timer = matches!(
6947 event.event_type.as_str(),
6948 "TimerScheduled" | "TimerCancelled" | "TimerFired"
6949 ) && !is_internal_timer_event(event);
6950 let is_child_workflow = matches!(
6951 event.event_type.as_str(),
6952 "ChildWorkflowScheduled"
6953 | "ChildRunCompleted"
6954 | "ChildRunFailed"
6955 | "ChildRunCancelled"
6956 | "ChildRunTerminated"
6957 );
6958 let is_signal_wait = is_recorded_signal_wait_event(event);
6959 let is_side_effect = event.event_type == "SideEffectRecorded";
6960 let is_version_marker = event.event_type == "VersionMarkerRecorded";
6961 if !is_activity
6962 && !is_workflow_timer
6963 && !is_child_workflow
6964 && !is_signal_wait
6965 && !is_side_effect
6966 && !is_version_marker
6967 {
6968 continue;
6969 }
6970
6971 let sequence = durable_event_sequence(event).ok_or_else(|| {
6972 Error::NonDeterministicReplay(ReplayFailure::new(
6973 "durable_command_sequence_missing",
6974 None,
6975 Some("positive workflow sequence".to_string()),
6976 Some(event.event_type.clone()),
6977 "durable command history event has no workflow sequence",
6978 ))
6979 })?;
6980 if sequence == 0 {
6981 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
6982 "durable_command_sequence_invalid",
6983 Some(sequence),
6984 Some("positive workflow sequence".to_string()),
6985 Some(sequence.to_string()),
6986 "durable command history uses an invalid workflow sequence",
6987 )));
6988 }
6989 if !events_by_sequence.contains_key(&sequence) {
6990 if let Some(previous) = last_new_sequence {
6991 if sequence < previous {
6992 return Err(invalid_recorded_history(
6993 "durable_command_sequence_mismatch",
6994 sequence,
6995 &format!("workflow sequence greater than {previous}"),
6996 &sequence.to_string(),
6997 "durable commands are not strictly ordered by their recorded workflow sequence",
6998 ));
6999 }
7000 }
7001 last_new_sequence = Some(sequence);
7002 }
7003 events_by_sequence.entry(sequence).or_default().push(event);
7004 }
7005
7006 let commands: Vec<RecordedCommand> = events_by_sequence
7007 .into_iter()
7008 .map(|(sequence, sequence_events)| {
7009 let activity_events: Vec<_> = sequence_events
7010 .iter()
7011 .copied()
7012 .filter(|event| event.event_type.starts_with("Activity"))
7013 .collect();
7014 let timer_events: Vec<_> = sequence_events
7015 .iter()
7016 .copied()
7017 .filter(|event| event.event_type.starts_with("Timer"))
7018 .collect();
7019 let child_events: Vec<_> = sequence_events
7020 .iter()
7021 .copied()
7022 .filter(|event| {
7023 event.event_type == "ChildWorkflowScheduled"
7024 || event.event_type.starts_with("ChildRun")
7025 })
7026 .collect();
7027 let signal_wait_events: Vec<_> = sequence_events
7028 .iter()
7029 .copied()
7030 .filter(|event| is_recorded_signal_wait_event(event))
7031 .collect();
7032 let side_effect_events: Vec<_> = sequence_events
7033 .iter()
7034 .copied()
7035 .filter(|event| event.event_type == "SideEffectRecorded")
7036 .collect();
7037 let version_marker_events: Vec<_> = sequence_events
7038 .iter()
7039 .copied()
7040 .filter(|event| event.event_type == "VersionMarkerRecorded")
7041 .collect();
7042
7043 let command_kind_count = usize::from(!activity_events.is_empty())
7044 + usize::from(!timer_events.is_empty())
7045 + usize::from(!child_events.is_empty())
7046 + usize::from(!signal_wait_events.is_empty())
7047 + usize::from(!side_effect_events.is_empty())
7048 + usize::from(!version_marker_events.is_empty());
7049 if command_kind_count > 1 {
7050 let actual = [
7051 (!activity_events.is_empty()).then_some("activity"),
7052 (!timer_events.is_empty()).then_some("timer"),
7053 (!child_events.is_empty()).then_some("child workflow"),
7054 (!signal_wait_events.is_empty()).then_some("signal wait"),
7055 (!side_effect_events.is_empty()).then_some("side effect"),
7056 (!version_marker_events.is_empty()).then_some("version marker"),
7057 ]
7058 .into_iter()
7059 .flatten()
7060 .collect::<Vec<_>>()
7061 .join(" and ");
7062 return Err(invalid_recorded_history(
7063 "durable_command_sequence_collision",
7064 sequence,
7065 "one durable command kind",
7066 &actual,
7067 "one workflow sequence records more than one durable command kind",
7068 ));
7069 }
7070
7071 if !activity_events.is_empty() {
7072 let scheduled_count = activity_events
7073 .iter()
7074 .filter(|event| event.event_type == "ActivityScheduled")
7075 .count();
7076 if scheduled_count > 1 {
7077 return Err(invalid_recorded_history(
7078 "duplicate_activity_schedule",
7079 sequence,
7080 "at most one ActivityScheduled event",
7081 "multiple ActivityScheduled events",
7082 "activity history schedules more than one command at one workflow sequence",
7083 ));
7084 }
7085 let activity_type = activity_events.iter().find_map(|event| {
7086 event
7087 .payload
7088 .get("activity_type")
7089 .or_else(|| event.payload.get("activity_name"))
7090 .and_then(Value::as_str)
7091 .map(str::to_string)
7092 });
7093 if activity_events.iter().filter_map(|event| {
7094 event
7095 .payload
7096 .get("activity_type")
7097 .or_else(|| event.payload.get("activity_name"))
7098 .and_then(Value::as_str)
7099 }).any(|candidate| Some(candidate) != activity_type.as_deref()) {
7100 return Err(invalid_recorded_history(
7101 "activity_identity_mismatch",
7102 sequence,
7103 activity_type.as_deref().unwrap_or("one activity identity"),
7104 "conflicting activity identities",
7105 "activity lifecycle events at one workflow sequence disagree on identity",
7106 ));
7107 }
7108 let terminal: Vec<_> = activity_events
7109 .iter()
7110 .copied()
7111 .filter(|event| {
7112 matches!(
7113 event.event_type.as_str(),
7114 "ActivityCompleted"
7115 | "ActivityFailed"
7116 | "ActivityCancelled"
7117 | "ActivityTimedOut"
7118 )
7119 })
7120 .collect();
7121 if terminal.len() > 1 {
7122 return Err(invalid_recorded_history(
7123 "duplicate_activity_terminal_event",
7124 sequence,
7125 "at most one terminal activity event",
7126 "multiple terminal activity events",
7127 "activity history settles one command more than once",
7128 ));
7129 }
7130 let outcome = terminal
7131 .first()
7132 .map(|event| activity_outcome(event, fallback_codec, activity_type.clone()))
7133 .transpose()?;
7134 let options = activity_events
7135 .iter()
7136 .find(|event| event.event_type == "ActivityScheduled")
7137 .and_then(|event| event.payload.get("activity"))
7138 .and_then(Value::as_object)
7139 .map(|activity| RecordedActivityOptions {
7140 task_queue: recorded_optional_string(activity, "queue"),
7141 execution_mode: recorded_optional_string(activity, "execution_mode"),
7142 retry_policy: recorded_activity_retry_snapshot(
7143 activity.get("retry_policy"),
7144 ),
7145 });
7146 return Ok(RecordedCommand::Activity {
7147 sequence,
7148 activity_type,
7149 options,
7150 outcome,
7151 });
7152 }
7153
7154 if !child_events.is_empty() {
7155 let scheduled: Vec<_> = child_events
7156 .iter()
7157 .copied()
7158 .filter(|event| event.event_type == "ChildWorkflowScheduled")
7159 .collect();
7160 if scheduled.len() != 1 {
7161 return Err(invalid_recorded_history(
7162 "child_workflow_schedule_missing_or_duplicate",
7163 sequence,
7164 "one ChildWorkflowScheduled event",
7165 &format!("{} ChildWorkflowScheduled events", scheduled.len()),
7166 "child workflow replay requires exactly one recorded schedule event",
7167 ));
7168 }
7169 let workflow_type = child_events.iter().find_map(|event| {
7170 event
7171 .payload
7172 .get("child_workflow_type")
7173 .or_else(|| event.payload.get("workflow_type"))
7174 .and_then(Value::as_str)
7175 .filter(|value| !value.is_empty())
7176 .map(str::to_string)
7177 });
7178 if child_events
7179 .iter()
7180 .filter_map(|event| {
7181 event
7182 .payload
7183 .get("child_workflow_type")
7184 .or_else(|| event.payload.get("workflow_type"))
7185 .and_then(Value::as_str)
7186 })
7187 .any(|candidate| Some(candidate) != workflow_type.as_deref())
7188 {
7189 return Err(invalid_recorded_history(
7190 "child_workflow_identity_mismatch",
7191 sequence,
7192 workflow_type
7193 .as_deref()
7194 .unwrap_or("one child workflow type"),
7195 "conflicting child workflow types",
7196 "child workflow lifecycle events at one sequence disagree on type",
7197 ));
7198 }
7199 let mut outcomes = child_workflow_outcomes(
7200 &child_events.iter().map(|event| (*event).clone()).collect::<Vec<_>>(),
7201 fallback_codec,
7202 parent.clone(),
7203 )?;
7204 if outcomes.len() > 1 {
7205 return Err(invalid_recorded_history(
7206 "duplicate_child_workflow_terminal_event",
7207 sequence,
7208 "at most one terminal child event",
7209 "multiple terminal child events",
7210 "child workflow history settles one command more than once",
7211 ));
7212 }
7213 return Ok(RecordedCommand::ChildWorkflow {
7214 sequence,
7215 workflow_type,
7216 outcome: outcomes.pop(),
7217 });
7218 }
7219
7220 if !signal_wait_events.is_empty() {
7221 let opened: Vec<_> = signal_wait_events
7222 .iter()
7223 .copied()
7224 .filter(|event| event.event_type == "SignalWaitOpened")
7225 .collect();
7226 if opened.len() != 1 {
7227 return Err(invalid_recorded_history(
7228 "signal_wait_open_missing_or_duplicate",
7229 sequence,
7230 "one SignalWaitOpened event",
7231 &format!("{} SignalWaitOpened events", opened.len()),
7232 "signal replay requires exactly one canonical wait-open event",
7233 ));
7234 }
7235
7236 let applied: Vec<_> = signal_wait_events
7237 .iter()
7238 .copied()
7239 .filter(|event| event.event_type == "SignalApplied")
7240 .collect();
7241 if applied.len() > 1 {
7242 return Err(invalid_recorded_history(
7243 "duplicate_signal_wait_apply",
7244 sequence,
7245 "at most one SignalApplied event",
7246 "multiple SignalApplied events",
7247 "signal history applies one durable wait more than once",
7248 ));
7249 }
7250
7251 let signal_names = signal_wait_events
7252 .iter()
7253 .map(|event| required_signal_wait_name(event, sequence))
7254 .collect::<Result<Vec<_>>>()?;
7255 let signal_name = signal_names
7256 .first()
7257 .expect("signal wait events are not empty")
7258 .clone();
7259 if signal_names.iter().any(|candidate| candidate != &signal_name) {
7260 return Err(invalid_recorded_history(
7261 "signal_wait_identity_mismatch",
7262 sequence,
7263 &signal_name,
7264 "conflicting signal names",
7265 "signal wait lifecycle events at one workflow sequence disagree on identity",
7266 ));
7267 }
7268 let value = applied
7269 .first()
7270 .map(|event| decode_signal_event_arguments(event, fallback_codec))
7271 .transpose()?;
7272 return Ok(RecordedCommand::SignalWait {
7273 sequence,
7274 signal_name,
7275 value,
7276 });
7277 }
7278
7279 if !side_effect_events.is_empty() {
7280 if side_effect_events.len() != 1 {
7281 return Err(invalid_recorded_history(
7282 "duplicate_side_effect_record",
7283 sequence,
7284 "one SideEffectRecorded event",
7285 &format!("{} SideEffectRecorded events", side_effect_events.len()),
7286 "side-effect history records one workflow command more than once",
7287 ));
7288 }
7289 let event = side_effect_events[0];
7290 let result = event.payload.get("result").ok_or_else(|| {
7291 invalid_recorded_history(
7292 "side_effect_result_missing",
7293 sequence,
7294 "recorded result payload",
7295 "missing result",
7296 "side-effect history is missing its recorded value",
7297 )
7298 })?;
7299 let has_published_envelope = result.as_str().is_some()
7300 || result.as_object().is_some_and(|envelope| {
7301 envelope.get("codec").and_then(Value::as_str).is_some()
7302 && envelope.get("blob").and_then(Value::as_str).is_some()
7303 });
7304 if !has_published_envelope {
7305 return Err(invalid_recorded_history(
7306 "side_effect_payload_malformed",
7307 sequence,
7308 "payload blob or {codec, blob} envelope",
7309 &result.to_string(),
7310 "side-effect history result does not use a published payload envelope",
7311 ));
7312 }
7313 let codec = event
7314 .payload
7315 .get("payload_codec")
7316 .and_then(Value::as_str)
7317 .unwrap_or(fallback_codec);
7318 let value = decode_wire_avro_value(result, codec).map_err(|error| {
7319 invalid_recorded_history(
7320 "side_effect_payload_incompatible",
7321 sequence,
7322 &format!("valid {codec} payload envelope"),
7323 &error.to_string(),
7324 "side-effect history payload cannot be decoded with its recorded codec",
7325 )
7326 })?;
7327 return Ok(RecordedCommand::SideEffect { sequence, value });
7328 }
7329
7330 if !version_marker_events.is_empty() {
7331 if version_marker_events.len() != 1 {
7332 return Err(invalid_recorded_history(
7333 "duplicate_version_marker_record",
7334 sequence,
7335 "one VersionMarkerRecorded event",
7336 &format!("{} VersionMarkerRecorded events", version_marker_events.len()),
7337 "version-marker history records one workflow command more than once",
7338 ));
7339 }
7340 let payload = &version_marker_events[0].payload;
7341 let change_id = payload
7342 .get("change_id")
7343 .and_then(Value::as_str)
7344 .filter(|value| !value.is_empty())
7345 .map(str::to_string)
7346 .ok_or_else(|| {
7347 invalid_recorded_history(
7348 "version_marker_field_missing",
7349 sequence,
7350 "non-empty change_id",
7351 "missing or invalid change_id",
7352 "version-marker history is missing its stable change ID",
7353 )
7354 })?;
7355 let version = required_version_i32(payload, "version", sequence)?;
7356 let min_supported = required_version_i32(payload, "min_supported", sequence)?;
7357 let max_supported = required_version_i32(payload, "max_supported", sequence)?;
7358 if min_supported > max_supported || version < min_supported || version > max_supported {
7359 return Err(invalid_recorded_history(
7360 "version_marker_history_range_invalid",
7361 sequence,
7362 "min_supported <= version <= max_supported",
7363 &format!("{min_supported} <= {version} <= {max_supported}"),
7364 "recorded version marker contains an internally incompatible range",
7365 ));
7366 }
7367 return Ok(RecordedCommand::VersionMarker {
7368 sequence,
7369 change_id,
7370 version,
7371 });
7372 }
7373
7374 let scheduled: Vec<_> = timer_events
7375 .iter()
7376 .copied()
7377 .filter(|event| event.event_type == "TimerScheduled")
7378 .collect();
7379 let fired: Vec<_> = timer_events
7380 .iter()
7381 .copied()
7382 .filter(|event| event.event_type == "TimerFired")
7383 .collect();
7384 if scheduled.len() != 1 {
7385 return Err(invalid_recorded_history(
7386 "timer_schedule_missing_or_duplicate",
7387 sequence,
7388 "one TimerScheduled event",
7389 &format!("{} TimerScheduled events", scheduled.len()),
7390 "timer replay requires exactly one recorded schedule event",
7391 ));
7392 }
7393 if fired.len() > 1 {
7394 return Err(invalid_recorded_history(
7395 "duplicate_timer_fire",
7396 sequence,
7397 "at most one TimerFired event",
7398 "multiple TimerFired events",
7399 "timer history contains more than one fire event for a workflow sequence",
7400 ));
7401 }
7402
7403 let scheduled = scheduled[0];
7404 let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
7405 let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
7406 if let Some(fired) = fired.first() {
7407 let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
7408 if fired_timer_id != timer_id {
7409 return Err(invalid_recorded_history(
7410 "timer_identity_mismatch",
7411 sequence,
7412 &timer_id,
7413 &fired_timer_id,
7414 "TimerFired does not correspond to the recorded TimerScheduled event",
7415 ));
7416 }
7417 let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
7418 if fired_delay != delay_seconds {
7419 return Err(invalid_recorded_history(
7420 "timer_history_delay_mismatch",
7421 sequence,
7422 &delay_seconds.to_string(),
7423 &fired_delay.to_string(),
7424 "TimerScheduled and TimerFired record different delays",
7425 ));
7426 }
7427 }
7428
7429 Ok(RecordedCommand::Timer {
7430 sequence,
7431 delay_seconds,
7432 fired: !fired.is_empty(),
7433 })
7434 })
7435 .collect::<Result<_>>()?;
7436
7437 let mut marker_sequences = HashMap::new();
7438 for command in &commands {
7439 if let RecordedCommand::VersionMarker {
7440 sequence,
7441 change_id,
7442 ..
7443 } = command
7444 {
7445 if let Some(first_sequence) = marker_sequences.insert(change_id.clone(), *sequence) {
7446 return Err(invalid_recorded_history(
7447 "duplicate_version_marker",
7448 *sequence,
7449 &format!("one marker for change ID {change_id:?}"),
7450 &format!("markers at sequences {first_sequence} and {sequence}"),
7451 "workflow history contains duplicate markers for one stable change ID",
7452 ));
7453 }
7454 }
7455 }
7456
7457 Ok(commands)
7458}
7459
7460fn required_version_i32(payload: &Value, field: &str, sequence: u64) -> Result<i32> {
7461 payload
7462 .get(field)
7463 .and_then(Value::as_i64)
7464 .and_then(|value| i32::try_from(value).ok())
7465 .ok_or_else(|| {
7466 invalid_recorded_history(
7467 "version_marker_field_missing",
7468 sequence,
7469 &format!("integer {field}"),
7470 "missing or out-of-range integer",
7471 "version-marker history is missing a required integer field",
7472 )
7473 })
7474}
7475
7476fn durable_event_sequence(event: &HistoryEvent) -> Option<u64> {
7477 event
7478 .payload
7479 .get("sequence")
7480 .or_else(|| event.payload.get("workflow_sequence"))
7481 .or_else(|| event.raw.get("sequence"))
7482 .or_else(|| event.raw.get("workflow_sequence"))
7483 .and_then(value_as_u64)
7484}
7485
7486fn is_internal_timer_event(event: &HistoryEvent) -> bool {
7487 matches!(
7488 event
7489 .payload
7490 .get("timer_kind")
7491 .or_else(|| event.raw.get("timer_kind"))
7492 .and_then(Value::as_str),
7493 Some("condition_timeout" | "signal_timeout")
7494 )
7495}
7496
7497fn required_signal_wait_name(event: &HistoryEvent, sequence: u64) -> Result<String> {
7498 event
7499 .payload
7500 .get("signal_name")
7501 .or_else(|| event.raw.get("signal_name"))
7502 .and_then(Value::as_str)
7503 .filter(|value| !value.is_empty())
7504 .map(str::to_string)
7505 .ok_or_else(|| {
7506 invalid_recorded_history(
7507 "signal_wait_name_missing",
7508 sequence,
7509 "non-empty signal_name",
7510 &event.event_type,
7511 "canonical signal-wait history is missing its signal identity",
7512 )
7513 })
7514}
7515
7516fn is_recorded_signal_wait_event(event: &HistoryEvent) -> bool {
7517 matches!(
7518 event.event_type.as_str(),
7519 "SignalWaitOpened" | "SignalApplied"
7520 )
7521}
7522
7523fn required_history_string(event: &HistoryEvent, field: &str, sequence: u64) -> Result<String> {
7524 event
7525 .payload
7526 .get(field)
7527 .and_then(Value::as_str)
7528 .filter(|value| !value.is_empty())
7529 .map(str::to_string)
7530 .ok_or_else(|| {
7531 invalid_recorded_history(
7532 "timer_history_field_missing",
7533 sequence,
7534 field,
7535 &event.event_type,
7536 "timer history is missing a required identity field",
7537 )
7538 })
7539}
7540
7541fn required_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<u64> {
7542 event
7543 .payload
7544 .get(field)
7545 .and_then(value_as_u64)
7546 .ok_or_else(|| {
7547 invalid_recorded_history(
7548 "timer_history_field_missing",
7549 sequence,
7550 field,
7551 &event.event_type,
7552 "timer history is missing a required numeric field",
7553 )
7554 })
7555}
7556
7557fn invalid_recorded_history(
7558 reason: &str,
7559 sequence: u64,
7560 expected: &str,
7561 actual: &str,
7562 message: &str,
7563) -> Error {
7564 Error::NonDeterministicReplay(ReplayFailure::new(
7565 reason,
7566 Some(sequence),
7567 Some(expected.to_string()),
7568 Some(actual.to_string()),
7569 message,
7570 ))
7571}
7572
7573type ActivityOutcome = std::result::Result<AvroValue, ActivityFailure>;
7574
7575fn activity_outcome(
7576 event: &HistoryEvent,
7577 fallback_codec: &str,
7578 recorded_activity_type: Option<String>,
7579) -> Result<ActivityOutcome> {
7580 if event.event_type == "ActivityCompleted" {
7581 let codec = event
7582 .payload
7583 .get("payload_codec")
7584 .and_then(Value::as_str)
7585 .unwrap_or(fallback_codec);
7586 return Ok(Ok(decode_wire_avro_value(
7587 event.payload.get("result").unwrap_or(&Value::Null),
7588 codec,
7589 )?));
7590 }
7591
7592 let payload = &event.payload;
7593 let (kind, fallback_reason, fallback_message) = match event.event_type.as_str() {
7594 "ActivityFailed" => (ActivityFailureKind::Failed, "activity", "activity failed"),
7595 "ActivityCancelled" => (
7596 ActivityFailureKind::Cancelled,
7597 "cancelled",
7598 "activity was cancelled",
7599 ),
7600 "ActivityTimedOut" => (
7601 ActivityFailureKind::TimedOut,
7602 "timeout",
7603 "activity timed out",
7604 ),
7605 _ => unreachable!("activity_outcome is called only for terminal activity events"),
7606 };
7607 let exception = payload
7608 .get("exception")
7609 .filter(|value| !value.is_null())
7610 .cloned();
7611 let failure_category = payload_string(payload, "failure_category");
7612 let timeout_kind = payload_string(payload, "timeout_kind");
7613 let reason = payload_string(payload, "reason").unwrap_or_else(|| match kind {
7614 ActivityFailureKind::Failed => failure_category
7615 .clone()
7616 .unwrap_or_else(|| fallback_reason.to_string()),
7617 ActivityFailureKind::Cancelled => fallback_reason.to_string(),
7618 ActivityFailureKind::TimedOut => timeout_kind
7619 .clone()
7620 .unwrap_or_else(|| fallback_reason.to_string()),
7621 });
7622 let message = payload_string(payload, "message")
7623 .or_else(|| {
7624 exception
7625 .as_ref()
7626 .and_then(|value| payload_string(value, "message"))
7627 })
7628 .unwrap_or_else(|| fallback_message.to_string());
7629
7630 Ok(Err(ActivityFailure {
7631 kind,
7632 reason,
7633 message,
7634 activity_execution_id: payload_string(payload, "activity_execution_id"),
7635 activity_attempt_id: payload_string(payload, "activity_attempt_id"),
7636 activity_type: payload_string(payload, "activity_type")
7637 .or_else(|| payload_string(payload, "activity_name"))
7638 .or(recorded_activity_type),
7639 activity_class: payload_string(payload, "activity_class"),
7640 attempt_number: payload.get("attempt_number").and_then(value_as_u64),
7641 failure_id: payload_string(payload, "failure_id"),
7642 failure_category,
7643 timeout_kind,
7644 non_retryable: payload
7645 .get("non_retryable")
7646 .and_then(Value::as_bool)
7647 .unwrap_or(false),
7648 exception_type: payload_string(payload, "exception_type").or_else(|| {
7649 exception
7650 .as_ref()
7651 .and_then(|value| payload_string(value, "type"))
7652 }),
7653 exception_class: payload_string(payload, "exception_class").or_else(|| {
7654 exception
7655 .as_ref()
7656 .and_then(|value| payload_string(value, "class"))
7657 }),
7658 code: payload
7659 .get("code")
7660 .filter(|value| !value.is_null())
7661 .cloned(),
7662 exception,
7663 }))
7664}
7665
7666type ChildWorkflowOutcome = std::result::Result<ChildWorkflowAvroResult, ChildWorkflowFailure>;
7667
7668fn child_workflow_outcomes(
7669 events: &[HistoryEvent],
7670 fallback_codec: &str,
7671 parent: WorkflowIdentity,
7672) -> Result<Vec<ChildWorkflowOutcome>> {
7673 let mut outcomes = Vec::new();
7674
7675 for event in events {
7676 let kind = match event.event_type.as_str() {
7677 "ChildRunCompleted" => None,
7678 "ChildRunFailed" => Some((
7679 ChildWorkflowFailureKind::Failed,
7680 "child_workflow",
7681 "child workflow failed",
7682 )),
7683 "ChildRunCancelled" => Some((
7684 ChildWorkflowFailureKind::Cancelled,
7685 "cancelled",
7686 "child workflow was cancelled",
7687 )),
7688 "ChildRunTerminated" => Some((
7689 ChildWorkflowFailureKind::Terminated,
7690 "terminated",
7691 "child workflow was terminated",
7692 )),
7693 _ => continue,
7694 };
7695 let payload = &event.payload;
7696 let child_workflow_id = payload_string(payload, "child_workflow_instance_id");
7697 let child_workflow_run_id = payload_string(payload, "child_workflow_run_id");
7698 let child_workflow_type = payload_string(payload, "child_workflow_type");
7699
7700 if let Some((kind, reason, fallback_message)) = kind {
7701 let exception = payload
7702 .get("exception")
7703 .filter(|value| !value.is_null())
7704 .cloned();
7705 let message = payload_string(payload, "message")
7706 .or_else(|| {
7707 exception
7708 .as_ref()
7709 .and_then(|value| payload_string(value, "message"))
7710 })
7711 .unwrap_or_else(|| fallback_message.to_string());
7712 let exception_type = payload_string(payload, "exception_type").or_else(|| {
7713 exception
7714 .as_ref()
7715 .and_then(|value| payload_string(value, "type"))
7716 });
7717 let exception_class = payload_string(payload, "exception_class").or_else(|| {
7718 exception
7719 .as_ref()
7720 .and_then(|value| payload_string(value, "class"))
7721 });
7722 outcomes.push(Err(ChildWorkflowFailure {
7723 kind,
7724 reason: reason.to_string(),
7725 message,
7726 parent_workflow_id: parent.workflow_id.clone(),
7727 parent_workflow_run_id: parent.run_id.clone(),
7728 child_workflow_id,
7729 child_workflow_run_id,
7730 child_workflow_type,
7731 failure_id: payload_string(payload, "failure_id"),
7732 failure_category: payload_string(payload, "failure_category"),
7733 exception_type,
7734 exception_class,
7735 non_retryable: payload
7736 .get("non_retryable")
7737 .and_then(Value::as_bool)
7738 .unwrap_or(false),
7739 code: payload
7740 .get("code")
7741 .filter(|value| !value.is_null())
7742 .cloned(),
7743 exception,
7744 }));
7745 continue;
7746 }
7747
7748 let codec = payload
7749 .get("payload_codec")
7750 .and_then(Value::as_str)
7751 .unwrap_or(fallback_codec);
7752 let result = payload
7753 .get("result")
7754 .or_else(|| payload.get("output"))
7755 .unwrap_or(&Value::Null);
7756 outcomes.push(Ok(ChildWorkflowAvroResult {
7757 parent: parent.clone(),
7758 child: WorkflowIdentity {
7759 workflow_id: child_workflow_id,
7760 run_id: child_workflow_run_id,
7761 },
7762 child_workflow_type,
7763 result: decode_wire_avro_value(result, codec)?,
7764 }));
7765 }
7766
7767 Ok(outcomes)
7768}
7769
7770fn payload_string(payload: &Value, key: &str) -> Option<String> {
7771 payload
7772 .get(key)
7773 .and_then(Value::as_str)
7774 .filter(|value| !value.is_empty())
7775 .map(str::to_string)
7776}
7777
7778fn workflow_failure_command(error: &Error) -> Value {
7779 let (exception_type, exception_class, properties) = match error {
7780 Error::ActivityFailed(failure) => (
7781 match failure.kind {
7782 ActivityFailureKind::Failed => "ActivityFailed",
7783 ActivityFailureKind::Cancelled => "ActivityCancelled",
7784 ActivityFailureKind::TimedOut => "ActivityTimedOut",
7785 },
7786 "durable_workflow::ActivityFailure",
7787 json!({
7788 "reason": failure.reason,
7789 "activity_execution_id": failure.activity_execution_id,
7790 "activity_attempt_id": failure.activity_attempt_id,
7791 "activity_type": failure.activity_type,
7792 "activity_class": failure.activity_class,
7793 "attempt_number": failure.attempt_number,
7794 "failure_id": failure.failure_id,
7795 "failure_category": failure.failure_category,
7796 "timeout_kind": failure.timeout_kind,
7797 "activity_non_retryable": failure.non_retryable,
7798 "activity_exception_type": failure.exception_type,
7799 "activity_exception_class": failure.exception_class,
7800 "activity_code": failure.code,
7801 "activity_exception": failure.exception,
7802 }),
7803 ),
7804 Error::ChildWorkflowFailed(failure) => (
7805 match failure.kind {
7806 ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
7807 ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
7808 ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
7809 },
7810 "durable_workflow::ChildWorkflowFailure",
7811 json!({
7812 "reason": failure.reason,
7813 "parent_workflow_id": failure.parent_workflow_id,
7814 "parent_workflow_run_id": failure.parent_workflow_run_id,
7815 "child_workflow_id": failure.child_workflow_id,
7816 "child_workflow_run_id": failure.child_workflow_run_id,
7817 "child_workflow_type": failure.child_workflow_type,
7818 "failure_id": failure.failure_id,
7819 "failure_category": failure.failure_category,
7820 "child_exception_type": failure.exception_type,
7821 "child_exception_class": failure.exception_class,
7822 "child_non_retryable": failure.non_retryable,
7823 "child_code": failure.code,
7824 "child_exception": failure.exception,
7825 }),
7826 ),
7827 Error::NonDeterministicReplay(_) => (
7828 "NonDeterministicReplay",
7829 "durable_workflow::Error",
7830 Value::Null,
7831 ),
7832 _ => ("RustWorkflowError", "durable_workflow::Error", Value::Null),
7833 };
7834 let non_retryable = match error {
7835 Error::ActivityFailed(failure) => failure.non_retryable,
7836 Error::ChildWorkflowFailed(failure) => failure.non_retryable,
7837 Error::NonDeterministicReplay(_) => true,
7838 _ => false,
7839 };
7840
7841 json!({
7842 "type": "fail_workflow",
7843 "message": error.to_string(),
7844 "exception_type": exception_type,
7845 "exception_class": exception_class,
7846 "non_retryable": non_retryable,
7847 "exception": {
7848 "type": exception_type,
7849 "class": exception_class,
7850 "message": error.to_string(),
7851 "properties": properties,
7852 }
7853 })
7854}
7855
7856fn workflow_task_integrity_error(error: &Error) -> bool {
7857 matches!(
7858 error,
7859 Error::NonDeterministicReplay(_) | Error::Protocol(_) | Error::WorkflowStatePoisoned
7860 )
7861}
7862
7863fn decode_signal_event_arguments(
7864 event: &HistoryEvent,
7865 fallback_codec: &str,
7866) -> Result<Vec<AvroValue>> {
7867 let codec = event
7868 .payload
7869 .get("payload_codec")
7870 .and_then(Value::as_str)
7871 .unwrap_or(fallback_codec);
7872 let raw = event
7873 .payload
7874 .get("value")
7875 .or_else(|| event.payload.get("input"))
7876 .or_else(|| event.payload.get("arguments"));
7877 let decoded = match raw.filter(|value| !value.is_null()) {
7878 Some(value) => decode_wire_avro_value(value, codec)?,
7879 None => AvroValue::Array(Vec::new()),
7880 };
7881 let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
7882 unreachable!("normalize_avro_arguments always returns an array");
7883 };
7884 Ok(arguments)
7885}
7886
7887fn hydrate_query_history_from_export(task: &mut QueryTask) -> Result<()> {
7888 let Some(export_events) = task
7889 .history_export
7890 .as_ref()
7891 .and_then(|export| export.get("history_events"))
7892 .and_then(Value::as_array)
7893 else {
7894 return Ok(());
7895 };
7896
7897 if export_events.len() > task.history_events.len() {
7898 task.history_events = serde_json::from_value(Value::Array(export_events.clone()))?;
7899 }
7900
7901 Ok(())
7902}
7903
7904fn enrich_query_history_from_export(task: &mut QueryTask) -> Result<()> {
7905 let Some(export) = task.history_export.as_ref() else {
7906 return Ok(());
7907 };
7908 let signals = export
7909 .get("signals")
7910 .and_then(Value::as_array)
7911 .cloned()
7912 .unwrap_or_default();
7913 let activities = export
7914 .get("activities")
7915 .and_then(Value::as_array)
7916 .cloned()
7917 .unwrap_or_default();
7918 let export_codec = export
7919 .get("payloads")
7920 .and_then(|payloads| payloads.get("codec"))
7921 .and_then(Value::as_str)
7922 .unwrap_or(&task.payload_codec)
7923 .to_string();
7924 let mut signal_name_offsets: HashMap<String, usize> = HashMap::new();
7925
7926 for event in &mut task.history_events {
7927 if event.event_type == "ActivityCompleted" {
7928 let sequence = event
7929 .payload
7930 .get("sequence")
7931 .or_else(|| event.payload.get("workflow_sequence"))
7932 .and_then(value_as_u64);
7933 let Some(activity) = sequence.and_then(|sequence| {
7934 activities.iter().find(|activity| {
7935 activity.get("sequence").and_then(value_as_u64) == Some(sequence)
7936 })
7937 }) else {
7938 continue;
7939 };
7940 let Some(payload) = event.payload.as_object_mut() else {
7941 continue;
7942 };
7943 if missing_payload(payload.get("result")) {
7944 if let Some(result) = activity
7945 .get("result")
7946 .filter(|value| !missing_payload(Some(value)))
7947 {
7948 payload.insert("result".to_string(), result.clone());
7949 }
7950 }
7951 for field in ["payload_codec", "activity_type"] {
7952 if payload
7953 .get(field)
7954 .and_then(Value::as_str)
7955 .unwrap_or_default()
7956 .is_empty()
7957 {
7958 if let Some(value) = activity.get(field) {
7959 payload.insert(field.to_string(), value.clone());
7960 }
7961 }
7962 }
7963 continue;
7964 }
7965
7966 if event.event_type != "SignalReceived" && event.event_type != "SignalApplied" {
7967 continue;
7968 }
7969 let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
7970 let command_id = event
7971 .payload
7972 .get("workflow_command_id")
7973 .or_else(|| event.raw.get("workflow_command_id"))
7974 .and_then(Value::as_str);
7975 let signal_name = event
7976 .payload
7977 .get("signal_name")
7978 .and_then(Value::as_str)
7979 .unwrap_or_default()
7980 .to_string();
7981 let matched = signals
7982 .iter()
7983 .find(|signal| {
7984 signal_id.is_some() && signal.get("id").and_then(Value::as_str) == signal_id
7985 })
7986 .or_else(|| {
7987 signals.iter().find(|signal| {
7988 command_id.is_some()
7989 && signal.get("command_id").and_then(Value::as_str) == command_id
7990 })
7991 })
7992 .or_else(|| {
7993 let offset = signal_name_offsets.entry(signal_name.clone()).or_default();
7994 let signal = signals
7995 .iter()
7996 .filter(|signal| {
7997 signal.get("name").and_then(Value::as_str) == Some(signal_name.as_str())
7998 })
7999 .nth(*offset);
8000 if signal.is_some() {
8001 *offset += 1;
8002 }
8003 signal
8004 });
8005 let Some(signal) = matched else {
8006 continue;
8007 };
8008 let signal_codec = signal
8009 .get("payload_codec")
8010 .and_then(Value::as_str)
8011 .unwrap_or(&export_codec);
8012 let Some(payload) = event.payload.as_object_mut() else {
8013 continue;
8014 };
8015 if missing_payload(payload.get("arguments")) {
8016 if let Some(arguments) = signal
8017 .get("arguments")
8018 .filter(|value| !missing_payload(Some(value)))
8019 {
8020 let envelope = match arguments {
8021 Value::String(blob) => json!({"codec": signal_codec, "blob": blob}),
8022 other => other.clone(),
8023 };
8024 payload.insert("arguments".to_string(), envelope);
8025 }
8026 }
8027 if payload
8028 .get("payload_codec")
8029 .and_then(Value::as_str)
8030 .unwrap_or_default()
8031 .is_empty()
8032 {
8033 payload.insert("payload_codec".to_string(), json!(signal_codec));
8034 }
8035 }
8036
8037 Ok(())
8038}
8039
8040fn missing_payload(value: Option<&Value>) -> bool {
8041 match value {
8042 None | Some(Value::Null) => true,
8043 Some(Value::String(value)) => value.is_empty(),
8044 Some(_) => false,
8045 }
8046}
8047
8048fn query_signal_events(task: &QueryTask) -> Result<Vec<QuerySignal>> {
8049 let export_signals = task
8050 .history_export
8051 .as_ref()
8052 .and_then(|export| export.get("signals"))
8053 .and_then(Value::as_array)
8054 .cloned()
8055 .unwrap_or_default();
8056 let export_codec = task
8057 .history_export
8058 .as_ref()
8059 .and_then(|export| export.get("payloads"))
8060 .and_then(|payloads| payloads.get("codec"))
8061 .and_then(Value::as_str)
8062 .unwrap_or(&task.payload_codec);
8063 let mut name_offsets: HashMap<String, usize> = HashMap::new();
8064 let mut signals = Vec::new();
8065
8066 for event in &task.history_events {
8067 if event.event_type != "SignalApplied" && event.event_type != "SignalReceived" {
8068 continue;
8069 }
8070
8071 let name = event
8072 .payload
8073 .get("signal_name")
8074 .and_then(Value::as_str)
8075 .unwrap_or_default();
8076 if name.is_empty() {
8077 continue;
8078 }
8079 let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
8080 let command_id = event
8081 .payload
8082 .get("workflow_command_id")
8083 .or_else(|| event.raw.get("workflow_command_id"))
8084 .and_then(Value::as_str);
8085 let matched_export = export_signals
8086 .iter()
8087 .find(|candidate| {
8088 signal_id.is_some() && candidate.get("id").and_then(Value::as_str) == signal_id
8089 })
8090 .or_else(|| {
8091 export_signals.iter().find(|candidate| {
8092 command_id.is_some()
8093 && candidate.get("command_id").and_then(Value::as_str) == command_id
8094 })
8095 })
8096 .or_else(|| {
8097 let offset = name_offsets.entry(name.to_string()).or_default();
8098 let candidate = export_signals
8099 .iter()
8100 .filter(|candidate| candidate.get("name").and_then(Value::as_str) == Some(name))
8101 .nth(*offset);
8102 if candidate.is_some() {
8103 *offset += 1;
8104 }
8105 candidate
8106 });
8107 let codec = event
8108 .payload
8109 .get("payload_codec")
8110 .and_then(Value::as_str)
8111 .or_else(|| {
8112 matched_export
8113 .and_then(|signal| signal.get("payload_codec"))
8114 .and_then(Value::as_str)
8115 })
8116 .unwrap_or(export_codec);
8117 let raw_arguments = event
8118 .payload
8119 .get("value")
8120 .or_else(|| event.payload.get("input"))
8121 .or_else(|| event.payload.get("arguments"))
8122 .filter(|value| !value.is_null())
8123 .or_else(|| matched_export.and_then(|signal| signal.get("arguments")));
8124 let (arguments, avro_arguments) = decode_query_signal_arguments(raw_arguments, codec)?;
8125 let workflow_sequence = event
8126 .payload
8127 .get("workflow_sequence")
8128 .and_then(value_as_u64)
8129 .or_else(|| {
8130 matched_export
8131 .and_then(|signal| signal.get("workflow_sequence"))
8132 .and_then(value_as_u64)
8133 });
8134
8135 signals.push(QuerySignal {
8136 id: signal_id.map(str::to_string).or_else(|| {
8137 matched_export
8138 .and_then(|signal| signal.get("id"))
8139 .and_then(Value::as_str)
8140 .map(str::to_string)
8141 }),
8142 name: name.to_string(),
8143 arguments,
8144 avro_arguments,
8145 workflow_sequence,
8146 });
8147 }
8148
8149 if signals.is_empty() {
8150 for signal in export_signals {
8151 if signal.get("status").and_then(Value::as_str) == Some("rejected") {
8152 continue;
8153 }
8154 let Some(name) = signal.get("name").and_then(Value::as_str) else {
8155 continue;
8156 };
8157 let codec = signal
8158 .get("payload_codec")
8159 .and_then(Value::as_str)
8160 .unwrap_or(export_codec);
8161 let (arguments, avro_arguments) =
8162 decode_query_signal_arguments(signal.get("arguments"), codec)?;
8163 signals.push(QuerySignal {
8164 id: signal.get("id").and_then(Value::as_str).map(str::to_string),
8165 name: name.to_string(),
8166 arguments,
8167 avro_arguments,
8168 workflow_sequence: signal.get("workflow_sequence").and_then(value_as_u64),
8169 });
8170 }
8171 signals.sort_by_key(|signal| signal.workflow_sequence.unwrap_or(u64::MAX));
8172 }
8173
8174 Ok(signals)
8175}
8176
8177fn decode_query_signal_arguments(
8178 raw: Option<&Value>,
8179 codec: &str,
8180) -> Result<(Vec<Value>, Vec<AvroValue>)> {
8181 let decoded = match raw.filter(|value| !value.is_null()) {
8182 Some(value) => decode_wire_avro_value(value, codec)?,
8183 None => AvroValue::Array(Vec::new()),
8184 };
8185 let AvroValue::Array(avro_arguments) = normalize_avro_arguments(decoded) else {
8186 unreachable!("normalize_avro_arguments always returns an array");
8187 };
8188 let arguments = avro_arguments
8189 .iter()
8190 .cloned()
8191 .map(AvroValue::into_json)
8192 .collect::<Result<Vec<_>>>()?;
8193 Ok((arguments, avro_arguments))
8194}
8195
8196fn value_as_u64(value: &Value) -> Option<u64> {
8197 value
8198 .as_u64()
8199 .or_else(|| value.as_str().and_then(|value| value.parse().ok()))
8200}
8201
8202#[cfg(test)]
8203mod tests {
8204 use super::*;
8205 use std::{
8206 io::{Read, Write},
8207 net::{SocketAddr, TcpListener, TcpStream},
8208 sync::atomic::AtomicUsize,
8209 thread,
8210 };
8211
8212 #[test]
8213 fn client_builder_rejects_the_sdk_owned_api_suffix() {
8214 for base_url in [
8215 "http://127.0.0.1:8080/api",
8216 "http://localhost:8080/api/",
8217 "https://runtime.example.test/namespaces/orders/api",
8218 ] {
8219 let error = Client::builder(base_url)
8220 .build()
8221 .expect_err("SDK-owned /api suffix must be rejected during build");
8222
8223 assert!(matches!(error, Error::InvalidBaseUrl), "{base_url}");
8224 assert!(
8225 error.to_string().contains("SDK appends /api automatically"),
8226 "the validation error must explain how to fix the endpoint"
8227 );
8228 }
8229 }
8230
8231 #[test]
8232 fn client_builder_preserves_self_hosted_and_managed_runtime_prefixes() {
8233 for (base_url, expected) in [
8234 ("http://127.0.0.1:8080", "http://127.0.0.1:8080"),
8235 (
8236 "http://localhost:8080/durable-workflow/",
8237 "http://localhost:8080/durable-workflow",
8238 ),
8239 (
8240 "https://runtime.example.test/namespaces/orders",
8241 "https://runtime.example.test/namespaces/orders",
8242 ),
8243 (
8244 "https://runtime.example.test/gateway/api/namespaces/orders",
8245 "https://runtime.example.test/gateway/api/namespaces/orders",
8246 ),
8247 (
8248 "https://api.example.test/runtime/orders/",
8249 "https://api.example.test/runtime/orders",
8250 ),
8251 ] {
8252 let client = Client::builder(base_url)
8253 .build()
8254 .expect("Server and Cloud runtime base URL must remain valid");
8255
8256 assert_eq!(client.base_url, expected);
8257 }
8258 }
8259
8260 fn typed_fidelity_probe() -> AvroValue {
8261 AvroValue::Map(BTreeMap::from([
8262 ("bytes".to_string(), AvroValue::Bytes(vec![0, 0xff])),
8263 ("empty".to_string(), AvroValue::Map(BTreeMap::new())),
8264 (
8265 "numeric".to_string(),
8266 AvroValue::Map(BTreeMap::from([
8267 ("0".to_string(), AvroValue::String("zero".to_string())),
8268 ("1".to_string(), AvroValue::String("one".to_string())),
8269 ])),
8270 ),
8271 (
8272 "nested".to_string(),
8273 AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([(
8274 "enabled".to_string(),
8275 AvroValue::Boolean(true),
8276 )]))]),
8277 ),
8278 (
8279 "projection_collisions".to_string(),
8280 AvroValue::Array(projection_collision_probe()),
8281 ),
8282 ]))
8283 }
8284
8285 fn projection_collision_probe() -> Vec<AvroValue> {
8286 vec![
8287 AvroValue::Map(BTreeMap::from([
8288 ("$type".to_string(), AvroValue::String("bytes".to_string())),
8289 (
8290 "base64".to_string(),
8291 AvroValue::String("ordinary user text".to_string()),
8292 ),
8293 ])),
8294 AvroValue::Map(BTreeMap::from([
8295 ("$type".to_string(), AvroValue::String("map".to_string())),
8296 (
8297 "entries".to_string(),
8298 AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([
8299 ("key".to_string(), AvroValue::String("ordinary".to_string())),
8300 (
8301 "value".to_string(),
8302 AvroValue::String("user map".to_string()),
8303 ),
8304 ]))]),
8305 ),
8306 ])),
8307 ]
8308 }
8309
8310 #[derive(Clone, Debug, Default, PartialEq)]
8311 struct ReplayCounterState {
8312 loaded: Option<String>,
8313 count: i64,
8314 finished: bool,
8315 }
8316
8317 fn replay_counter_worker() -> Worker {
8318 let client = Client::new("http://127.0.0.1:8080").expect("client");
8319 let mut worker = Worker::new(client, "rust-workers");
8320 worker.register_replayed_workflow(
8321 "replay-counter",
8322 ReplayCounterState::default,
8323 |ctx, _input, state| async move {
8324 let loaded = ctx.activity("load-counter", json!([])).await?;
8325 state.update(|current| {
8326 current.loaded = loaded.as_str().map(str::to_string);
8327 })?;
8328 for _ in 0..2 {
8329 let signal = ctx.wait_signal("increment").await?;
8330 let amount = signal.first().and_then(Value::as_i64).unwrap_or_default();
8331 state.update(|current| current.count += amount)?;
8332 }
8333 state.update(|current| current.finished = true)?;
8334 state.read(|current| Ok(json!(current.count)))?
8335 },
8336 );
8337 worker.register_replayed_query::<ReplayCounterState, _, _>(
8338 "replay-counter",
8339 "current",
8340 |_ctx, state, _args| async move {
8341 Ok(json!({
8342 "loaded": state.loaded,
8343 "count": state.count,
8344 "finished": state.finished,
8345 }))
8346 },
8347 );
8348 worker.register_replayed_query::<ReplayCounterState, _, _>(
8349 "replay-counter",
8350 "detached-mutation",
8351 |_ctx, state, _args| async move {
8352 let mut detached = (*state).clone();
8353 detached.count = 999;
8354 Ok(json!(detached.count))
8355 },
8356 );
8357 worker.register_replayed_query::<ReplayCounterState, _, _>(
8358 "replay-counter",
8359 "failed-mutation",
8360 |_ctx, state, _args| async move {
8361 let mut detached = (*state).clone();
8362 detached.count = 999;
8363 Err(Error::WorkerLoop("query refused".to_string()))
8364 },
8365 );
8366 worker
8367 }
8368
8369 fn replay_counter_query(
8370 query_name: &str,
8371 history_events: Value,
8372 run_status: &str,
8373 ) -> QueryTask {
8374 serde_json::from_value(json!({
8375 "query_task_id": format!("query-{query_name}"),
8376 "workflow_type": "replay-counter",
8377 "query_name": query_name,
8378 "payload_codec": "json",
8379 "workflow_arguments": {"codec": "json", "blob": "[]"},
8380 "query_arguments": {"codec": "json", "blob": "[]"},
8381 "history_events": history_events,
8382 "run_status": run_status,
8383 }))
8384 .expect("query task")
8385 }
8386
8387 fn workflow_context(history: Vec<HistoryEvent>) -> WorkflowContext {
8388 workflow_context_with_codec(history, JSON_CODEC)
8389 }
8390
8391 fn workflow_context_with_codec(
8392 history: Vec<HistoryEvent>,
8393 payload_codec: &str,
8394 ) -> WorkflowContext {
8395 WorkflowContext {
8396 state: Arc::new(Mutex::new(
8397 WorkflowState::new_with_identity(
8398 history,
8399 None,
8400 None,
8401 "rust-workers".to_string(),
8402 payload_codec.to_string(),
8403 None,
8404 )
8405 .expect("valid workflow history"),
8406 )),
8407 }
8408 }
8409
8410 fn history_event(event_type: &str, payload: Value) -> HistoryEvent {
8411 HistoryEvent {
8412 event_type: event_type.to_string(),
8413 payload,
8414 raw: HashMap::new(),
8415 }
8416 }
8417
8418 fn workflow_task(
8419 workflow_type: &str,
8420 history_events: Vec<HistoryEvent>,
8421 payload_codec: &str,
8422 ) -> WorkflowTask {
8423 WorkflowTask {
8424 task_id: format!("wft-{workflow_type}"),
8425 workflow_id: Some(format!("wf-{workflow_type}")),
8426 run_id: Some(format!("run-{workflow_type}")),
8427 workflow_type: workflow_type.to_string(),
8428 payload_codec: payload_codec.to_string(),
8429 arguments: Some(
8430 encode_value_envelope(&json!([]), payload_codec).expect("workflow arguments"),
8431 ),
8432 total_history_events: Some(history_events.len() as u64),
8433 history_size_bytes: None,
8434 continue_as_new_recommended: None,
8435 history_budget_pressure: None,
8436 history_events,
8437 next_history_page_token: None,
8438 workflow_task_attempt: 1,
8439 workflow_signal_id: None,
8440 signal_name: None,
8441 signal_arguments: None,
8442 workflow_update_id: None,
8443 update_name: None,
8444 lease_owner: Some("rust-worker".to_string()),
8445 }
8446 }
8447
8448 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
8449 struct SideEffectProbe {
8450 request_id: String,
8451 attempt: u32,
8452 }
8453
8454 #[test]
8455 fn typed_side_effect_runs_callback_once_and_replay_skips_it() {
8456 let calls = AtomicUsize::new(0);
8457 let ctx = workflow_context(Vec::new());
8458 let value = ctx
8459 .side_effect(|| {
8460 calls.fetch_add(1, Ordering::SeqCst);
8461 SideEffectProbe {
8462 request_id: "request-42".to_string(),
8463 attempt: 3,
8464 }
8465 })
8466 .expect("first side effect");
8467 assert_eq!(value.attempt, 3);
8468 assert_eq!(calls.load(Ordering::SeqCst), 1);
8469 let commands = ctx.take_commands().expect("commands");
8470 assert_eq!(commands.len(), 1);
8471 assert_eq!(commands[0]["type"], "record_side_effect");
8472 assert_eq!(
8473 decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("JSON result"),
8474 serde_json::to_value(&value).expect("value")
8475 );
8476
8477 let replay = workflow_context(vec![history_event(
8478 "SideEffectRecorded",
8479 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
8480 )]);
8481 let replayed: SideEffectProbe = replay
8482 .side_effect(|| {
8483 calls.fetch_add(1, Ordering::SeqCst);
8484 panic!("committed side-effect callbacks must not run during replay")
8485 })
8486 .expect("replayed side effect");
8487 assert_eq!(replayed, value);
8488 assert_eq!(calls.load(Ordering::SeqCst), 1);
8489 assert!(replay.take_commands().expect("commands").is_empty());
8490 replay.ensure_history_consumed().expect("history consumed");
8491 }
8492
8493 #[test]
8494 fn side_effect_uses_avro_envelope_and_uuid_is_replay_stable() {
8495 let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
8496 let value = ctx
8497 .side_effect(|| SideEffectProbe {
8498 request_id: "avro-request".to_string(),
8499 attempt: 1,
8500 })
8501 .expect("Avro side effect");
8502 let uuid = ctx.uuid_v4().expect("deterministic UUID");
8503 let commands = ctx.take_commands().expect("commands");
8504 assert_eq!(commands.len(), 2);
8505 assert_eq!(commands[0]["result"]["codec"], DEFAULT_CODEC);
8506 assert_eq!(commands[1]["result"]["codec"], DEFAULT_CODEC);
8507 assert_eq!(
8508 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
8509 serde_json::to_value(&value).expect("value")
8510 );
8511
8512 let replay = workflow_context_with_codec(
8513 vec![
8514 history_event(
8515 "SideEffectRecorded",
8516 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
8517 ),
8518 history_event(
8519 "SideEffectRecorded",
8520 json!({"sequence": 2, "result": commands[1]["result"].clone()}),
8521 ),
8522 ],
8523 DEFAULT_CODEC,
8524 );
8525 let replayed: SideEffectProbe = replay
8526 .side_effect(|| panic!("Avro callback must not run"))
8527 .expect("replayed Avro value");
8528 let replayed_uuid = replay.uuid_v4().expect("replayed UUID");
8529 assert_eq!(replayed, value);
8530 assert_eq!(replayed_uuid, uuid);
8531 assert!(replay.take_commands().expect("commands").is_empty());
8532 }
8533
8534 #[test]
8535 fn typed_side_effect_replay_preserves_bytes_and_maps() {
8536 let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
8537 let value = ctx
8538 .side_effect_avro_value(typed_fidelity_probe)
8539 .expect("typed side effect");
8540 let commands = ctx.take_commands().expect("side-effect command");
8541 assert_eq!(
8542 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
8543 .expect("recorded side effect"),
8544 value
8545 );
8546
8547 let replay = workflow_context_with_codec(
8548 vec![history_event(
8549 "SideEffectRecorded",
8550 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
8551 )],
8552 DEFAULT_CODEC,
8553 );
8554 assert_eq!(
8555 replay
8556 .side_effect_avro_value(|| panic!("replay must not invoke callback"))
8557 .expect("replayed typed side effect"),
8558 value
8559 );
8560 }
8561
8562 #[test]
8563 fn ordered_side_effects_share_the_durable_command_stream() {
8564 let first = encode_value_envelope(&json!("first"), JSON_CODEC).expect("first");
8565 let second = encode_value_envelope(&json!(29), JSON_CODEC).expect("second");
8566 let ctx = workflow_context(vec![
8567 history_event(
8568 "SideEffectRecorded",
8569 json!({"sequence": 1, "result": first}),
8570 ),
8571 history_event(
8572 "SideEffectRecorded",
8573 json!({"sequence": 2, "result": second}),
8574 ),
8575 ]);
8576 let first: String = ctx
8577 .side_effect(|| panic!("first callback must not run"))
8578 .expect("first replay");
8579 let second: i32 = ctx
8580 .side_effect(|| panic!("second callback must not run"))
8581 .expect("second replay");
8582 assert_eq!(first, "first");
8583 assert_eq!(second, 29);
8584 ctx.ensure_history_consumed().expect("ordered history");
8585
8586 let reordered = workflow_context(vec![history_event(
8587 "VersionMarkerRecorded",
8588 json!({
8589 "sequence": 1,
8590 "change_id": "before-side-effect",
8591 "version": 1,
8592 "min_supported": 1,
8593 "max_supported": 1,
8594 }),
8595 )]);
8596 let error = reordered
8597 .side_effect(|| "new".to_string())
8598 .expect_err("command reordering must fail");
8599 assert!(matches!(
8600 error,
8601 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8602 if reason == "recorded_command_mismatch"
8603 ));
8604 }
8605
8606 #[test]
8607 fn version_markers_replay_across_upgrades_and_do_not_duplicate() {
8608 let ctx = workflow_context(Vec::new());
8609 assert_eq!(ctx.get_version("checkout-v2", 1, 2).expect("version"), 2);
8610 assert_eq!(ctx.get_version("checkout-v2", 1, 3).expect("cached"), 2);
8611 assert!(ctx.patched("new-search").expect("patch"));
8612 ctx.deprecate_patch("new-search").expect("deprecate patch");
8613 let commands = ctx.take_commands().expect("commands");
8614 assert_eq!(commands.len(), 2);
8615 assert_eq!(commands[0]["type"], "record_version_marker");
8616 assert_eq!(commands[0]["version"], 2);
8617 assert_eq!(commands[1]["change_id"], "new-search");
8618
8619 let replay = workflow_context(vec![history_event(
8620 "VersionMarkerRecorded",
8621 json!({
8622 "sequence": 1,
8623 "change_id": "checkout-v2",
8624 "version": 2,
8625 "min_supported": 1,
8626 "max_supported": 2,
8627 }),
8628 )]);
8629 assert_eq!(replay.get_version("checkout-v2", 1, 4).expect("upgrade"), 2);
8630 assert_eq!(replay.get_version("checkout-v2", 2, 5).expect("repeat"), 2);
8631 assert!(replay.take_commands().expect("commands").is_empty());
8632 replay.ensure_history_consumed().expect("history consumed");
8633 }
8634
8635 #[test]
8636 fn version_markers_reject_incompatible_or_malformed_history() {
8637 let incompatible = workflow_context(vec![history_event(
8638 "VersionMarkerRecorded",
8639 json!({
8640 "sequence": 1,
8641 "change_id": "checkout-v2",
8642 "version": 1,
8643 "min_supported": 1,
8644 "max_supported": 2,
8645 }),
8646 )]);
8647 let error = incompatible
8648 .get_version("checkout-v2", 2, 3)
8649 .expect_err("old version is unsupported");
8650 assert!(matches!(
8651 error,
8652 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8653 if reason == "version_marker_incompatible_range"
8654 ));
8655
8656 for (history, reason) in [
8657 (
8658 vec![history_event("SideEffectRecorded", json!({"sequence": 1}))],
8659 "side_effect_result_missing",
8660 ),
8661 (
8662 vec![history_event(
8663 "SideEffectRecorded",
8664 json!({
8665 "sequence": 1,
8666 "result": {"codec": "avro", "blob": "not-base64"},
8667 }),
8668 )],
8669 "side_effect_payload_incompatible",
8670 ),
8671 (
8672 vec![history_event(
8673 "SideEffectRecorded",
8674 json!({"sequence": 1, "result": {"unwrapped": true}}),
8675 )],
8676 "side_effect_payload_malformed",
8677 ),
8678 (
8679 vec![history_event(
8680 "VersionMarkerRecorded",
8681 json!({
8682 "sequence": 1,
8683 "change_id": "change",
8684 "version": 1,
8685 "min_supported": 2,
8686 "max_supported": 1,
8687 }),
8688 )],
8689 "version_marker_history_range_invalid",
8690 ),
8691 ] {
8692 let error = WorkflowState::new(
8693 history,
8694 "rust-workers".to_string(),
8695 JSON_CODEC.to_string(),
8696 None,
8697 )
8698 .expect_err("malformed history must fail");
8699 assert!(matches!(
8700 error,
8701 Error::NonDeterministicReplay(ReplayFailure { reason: actual, .. })
8702 if actual == reason
8703 ));
8704 }
8705 }
8706
8707 #[test]
8708 fn duplicate_side_effects_and_version_markers_are_rejected() {
8709 let duplicate_side_effect = WorkflowState::new(
8710 vec![
8711 history_event(
8712 "SideEffectRecorded",
8713 json!({"sequence": 1, "result": {"codec": "json", "blob": "1"}}),
8714 ),
8715 history_event(
8716 "SideEffectRecorded",
8717 json!({"sequence": 1, "result": {"codec": "json", "blob": "2"}}),
8718 ),
8719 ],
8720 "rust-workers".to_string(),
8721 JSON_CODEC.to_string(),
8722 None,
8723 )
8724 .expect_err("duplicate side effect");
8725 assert!(matches!(
8726 duplicate_side_effect,
8727 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8728 if reason == "duplicate_side_effect_record"
8729 ));
8730
8731 let marker = |sequence| {
8732 history_event(
8733 "VersionMarkerRecorded",
8734 json!({
8735 "sequence": sequence,
8736 "change_id": "same-change",
8737 "version": 1,
8738 "min_supported": 1,
8739 "max_supported": 1,
8740 }),
8741 )
8742 };
8743 let duplicate_marker = WorkflowState::new(
8744 vec![marker(1), marker(3)],
8745 "rust-workers".to_string(),
8746 JSON_CODEC.to_string(),
8747 None,
8748 )
8749 .expect_err("duplicate marker");
8750 assert!(matches!(
8751 duplicate_marker,
8752 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8753 if reason == "duplicate_version_marker"
8754 ));
8755 }
8756
8757 #[test]
8758 fn cold_worker_replay_does_not_repeat_committed_side_effects_or_markers() {
8759 fn worker(calls: Arc<AtomicUsize>) -> Worker {
8760 let client = Client::new("http://127.0.0.1:8080").expect("client");
8761 let mut worker = Worker::new(client, "rust-workers");
8762 worker.register_workflow("rust.side-effect-version", move |ctx, _input| {
8763 let calls = Arc::clone(&calls);
8764 async move {
8765 let captured = ctx.side_effect(|| {
8766 calls.fetch_add(1, Ordering::SeqCst);
8767 "captured-once".to_string()
8768 })?;
8769 let version = ctx.get_version("cold-restart", 1, 2)?;
8770 Ok(json!({"captured": captured, "version": version}))
8771 }
8772 });
8773 worker
8774 }
8775
8776 fn task(history_events: Vec<HistoryEvent>) -> WorkflowTask {
8777 WorkflowTask {
8778 task_id: "wft-side-effect-version".to_string(),
8779 workflow_id: Some("wf-side-effect-version".to_string()),
8780 run_id: Some("run-side-effect-version".to_string()),
8781 workflow_type: "rust.side-effect-version".to_string(),
8782 payload_codec: JSON_CODEC.to_string(),
8783 arguments: Some(encode_value_envelope(&json!([]), JSON_CODEC).expect("arguments")),
8784 history_events,
8785 total_history_events: None,
8786 history_size_bytes: None,
8787 continue_as_new_recommended: None,
8788 history_budget_pressure: None,
8789 next_history_page_token: None,
8790 workflow_task_attempt: 1,
8791 workflow_signal_id: None,
8792 signal_name: None,
8793 signal_arguments: None,
8794 workflow_update_id: None,
8795 update_name: None,
8796 lease_owner: Some("rust-worker".to_string()),
8797 }
8798 }
8799
8800 let calls = Arc::new(AtomicUsize::new(0));
8801 let initial = worker(Arc::clone(&calls))
8802 .execute_workflow_task(task(Vec::new()))
8803 .expect("initial execution");
8804 assert_eq!(
8805 initial
8806 .iter()
8807 .map(|command| &command["type"])
8808 .collect::<Vec<_>>(),
8809 vec![
8810 "record_side_effect",
8811 "record_version_marker",
8812 "complete_workflow"
8813 ]
8814 );
8815 assert_eq!(calls.load(Ordering::SeqCst), 1);
8816
8817 let restarted = worker(Arc::clone(&calls));
8818 let replayed = restarted
8819 .execute_workflow_task(task(vec![
8820 history_event(
8821 "SideEffectRecorded",
8822 json!({"sequence": 1, "result": initial[0]["result"].clone()}),
8823 ),
8824 history_event(
8825 "VersionMarkerRecorded",
8826 json!({
8827 "sequence": 2,
8828 "change_id": "cold-restart",
8829 "version": 2,
8830 "min_supported": 1,
8831 "max_supported": 2,
8832 }),
8833 ),
8834 ]))
8835 .expect("cold replay");
8836 assert_eq!(replayed.len(), 1);
8837 assert_eq!(replayed[0]["type"], "complete_workflow");
8838 assert_eq!(calls.load(Ordering::SeqCst), 1);
8839 }
8840
8841 #[test]
8842 fn side_effect_replay_rejects_changed_rust_value_type() {
8843 let result = encode_value_envelope(&json!({"value": 42}), JSON_CODEC).expect("result");
8844 let ctx = workflow_context(vec![history_event(
8845 "SideEffectRecorded",
8846 json!({"sequence": 1, "result": result}),
8847 )]);
8848 let error = ctx
8849 .side_effect::<Vec<String>, _>(|| panic!("callback must not run"))
8850 .expect_err("changed type must fail replay");
8851 assert!(matches!(
8852 error,
8853 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8854 if reason == "side_effect_type_mismatch"
8855 ));
8856 }
8857
8858 fn completed_retry_activity_history() -> Vec<HistoryEvent> {
8859 vec![
8860 history_event(
8861 "ActivityScheduled",
8862 json!({
8863 "sequence": 1,
8864 "activity_type": "flaky",
8865 "activity_execution_id": "act-1",
8866 "activity": {
8867 "id": "act-1",
8868 "sequence": 1,
8869 "type": "flaky",
8870 "queue": "critical-activities",
8871 "execution_mode": null,
8872 "retry_policy": {
8873 "snapshot_version": 1,
8874 "max_attempts": 3,
8875 "backoff_seconds": [2, 4],
8876 "start_to_close_timeout": 30,
8877 "schedule_to_start_timeout": 5,
8878 "schedule_to_close_timeout": 90,
8879 "heartbeat_timeout": 10,
8880 "non_retryable_error_types": ["PermanentError"]
8881 }
8882 }
8883 }),
8884 ),
8885 history_event(
8886 "ActivityStarted",
8887 json!({
8888 "sequence": 1,
8889 "activity_type": "flaky",
8890 "activity_execution_id": "act-1",
8891 "activity_attempt_id": "attempt-1",
8892 "attempt_number": 1
8893 }),
8894 ),
8895 history_event(
8896 "ActivityRetryScheduled",
8897 json!({
8898 "sequence": 1,
8899 "activity_type": "flaky",
8900 "activity_execution_id": "act-1",
8901 "activity_attempt_id": "attempt-1",
8902 "attempt_number": 1,
8903 "retry_after_attempt": 1,
8904 "retry_backoff_seconds": 2,
8905 "failure_category": "activity",
8906 "exception_type": "TransientError"
8907 }),
8908 ),
8909 history_event(
8910 "ActivityStarted",
8911 json!({
8912 "sequence": 1,
8913 "activity_type": "flaky",
8914 "activity_execution_id": "act-1",
8915 "activity_attempt_id": "attempt-2",
8916 "attempt_number": 2
8917 }),
8918 ),
8919 history_event(
8920 "ActivityCompleted",
8921 json!({
8922 "sequence": 1,
8923 "activity_type": "flaky",
8924 "activity_execution_id": "act-1",
8925 "activity_attempt_id": "attempt-2",
8926 "attempt_number": 2,
8927 "payload_codec": "json",
8928 "result": {"codec": "json", "blob": "{\"status\":\"recovered\"}"}
8929 }),
8930 ),
8931 ]
8932 }
8933
8934 fn retry_activity_options() -> ActivityOptions {
8935 ActivityOptions::new()
8936 .task_queue("critical-activities")
8937 .retry_policy(
8938 ActivityRetryPolicy::new(3)
8939 .backoff_intervals([Duration::from_secs(2), Duration::from_secs(4)])
8940 .non_retryable_error_type("PermanentError"),
8941 )
8942 .start_to_close_timeout(Duration::from_secs(30))
8943 .schedule_to_start_timeout(Duration::from_secs(5))
8944 .schedule_to_close_timeout(Duration::from_secs(90))
8945 .heartbeat_timeout(Duration::from_secs(10))
8946 }
8947
8948 #[test]
8949 fn fixed_avro_value_round_trips_json_values() {
8950 let value = json!({"greeting": "hello", "count": 3, "ok": true});
8951 let envelope = PayloadEnvelope::avro(&value).expect("encode");
8952 assert_eq!(envelope.codec, DEFAULT_CODEC);
8953 assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
8954 }
8955
8956 #[tokio::test]
8957 async fn typed_worker_surfaces_preserve_bytes_and_map_list_identity() {
8958 let client = Client::new("http://127.0.0.1:8080").expect("client");
8959 let mut worker = Worker::new(client, "rust-workers");
8960 worker.register_workflow_avro_value("typed.echo", |_ctx, input| async move { Ok(input) });
8961 worker
8962 .register_activity_avro_value("typed.activity", |_ctx, input| async move { Ok(input) });
8963 worker.register_query_avro_value("typed.echo", "inspect", |_ctx, input| async move {
8964 Ok(input)
8965 });
8966 worker.register_update_avro_value("typed.echo", "replace", |_ctx, input| async move {
8967 Ok(input)
8968 });
8969 worker.register_workflow_avro_value("typed.signal", |ctx, _input| async move {
8970 Ok(AvroValue::Array(
8971 ctx.wait_signal_avro_value("changed").await?,
8972 ))
8973 });
8974
8975 let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
8976 let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed envelope");
8977
8978 let mut workflow = workflow_task("typed.echo", Vec::new(), DEFAULT_CODEC);
8979 workflow.arguments = Some(envelope.clone());
8980 let commands = worker
8981 .execute_workflow_task(workflow)
8982 .expect("typed workflow task");
8983 assert_eq!(commands[0]["type"], "complete_workflow");
8984 assert_eq!(
8985 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
8986 .expect("typed workflow result"),
8987 arguments
8988 );
8989
8990 let activity = ActivityTask {
8991 task_id: "activity-typed".to_string(),
8992 activity_attempt_id: Some("attempt-typed".to_string()),
8993 attempt_id: None,
8994 activity_type: "typed.activity".to_string(),
8995 payload_codec: DEFAULT_CODEC.to_string(),
8996 arguments: Some(envelope.clone()),
8997 attempt_number: 1,
8998 lease_owner: Some("rust-worker".to_string()),
8999 };
9000 assert_eq!(
9001 worker
9002 .execute_activity_task(activity)
9003 .await
9004 .expect("typed activity result"),
9005 arguments
9006 );
9007
9008 let query = QueryTask {
9009 query_task_id: "query-typed".to_string(),
9010 query_task_attempt: 1,
9011 lease_owner: Some("rust-worker".to_string()),
9012 workflow_id: Some("typed-1".to_string()),
9013 run_id: Some("run-typed".to_string()),
9014 workflow_type: "typed.echo".to_string(),
9015 query_name: "inspect".to_string(),
9016 payload_codec: DEFAULT_CODEC.to_string(),
9017 workflow_arguments: Some(
9018 encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
9019 .expect("workflow input"),
9020 ),
9021 query_arguments: Some(envelope.clone()),
9022 history_events: Vec::new(),
9023 history_export: None,
9024 run_status: Some("running".to_string()),
9025 };
9026 assert_eq!(
9027 worker
9028 .execute_query_task(query)
9029 .await
9030 .expect("typed query result"),
9031 arguments
9032 );
9033
9034 let mut update = workflow_task(
9035 "typed.echo",
9036 vec![history_event(
9037 "UpdateAccepted",
9038 json!({
9039 "update_id": "update-typed",
9040 "update_name": "replace",
9041 "arguments": envelope.clone(),
9042 }),
9043 )],
9044 DEFAULT_CODEC,
9045 );
9046 update.workflow_update_id = Some("update-typed".to_string());
9047 update.update_name = Some("replace".to_string());
9048 let commands = worker
9049 .execute_workflow_task(update)
9050 .expect("typed update task");
9051 assert_eq!(commands[0]["type"], "complete_update");
9052 assert_eq!(
9053 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
9054 .expect("typed update result"),
9055 arguments
9056 );
9057
9058 let mut signal = workflow_task(
9059 "typed.signal",
9060 vec![history_event(
9061 "SignalReceived",
9062 json!({
9063 "signal_id": "signal-typed",
9064 "signal_name": "changed",
9065 "arguments": envelope.clone(),
9066 }),
9067 )],
9068 DEFAULT_CODEC,
9069 );
9070 signal.workflow_signal_id = Some("signal-typed".to_string());
9071 signal.signal_name = Some("changed".to_string());
9072 signal.signal_arguments = Some(envelope);
9073 let commands = worker
9074 .execute_workflow_task(signal)
9075 .expect("typed signal resume");
9076 assert_eq!(
9077 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
9078 .expect("typed signal result"),
9079 arguments
9080 );
9081 }
9082
9083 #[tokio::test]
9084 async fn typed_helpers_never_parse_json_inspection_projection() {
9085 let collision_values = projection_collision_probe();
9086 let expected = AvroValue::Array(collision_values.clone());
9087 let envelope = encode_typed_envelope(&expected, DEFAULT_CODEC).expect("collision envelope");
9088
9089 let activity_context = workflow_context_with_codec(
9090 vec![history_event(
9091 "ActivityCompleted",
9092 json!({
9093 "sequence": 1,
9094 "activity_type": "collision.activity",
9095 "payload_codec": DEFAULT_CODEC,
9096 "result": envelope.clone(),
9097 }),
9098 )],
9099 DEFAULT_CODEC,
9100 );
9101 assert_eq!(
9102 activity_context
9103 .activity_avro_value("collision.activity", AvroValue::Array(Vec::new()))
9104 .await
9105 .expect("typed activity collision result"),
9106 expected
9107 );
9108
9109 let signal_context = workflow_context_with_codec(
9110 vec![
9111 history_event(
9112 "SignalWaitOpened",
9113 json!({"sequence": 1, "signal_name": "collision"}),
9114 ),
9115 history_event(
9116 "SignalApplied",
9117 json!({
9118 "sequence": 1,
9119 "signal_name": "collision",
9120 "payload_codec": DEFAULT_CODEC,
9121 "value": envelope.clone(),
9122 }),
9123 ),
9124 ],
9125 DEFAULT_CODEC,
9126 );
9127 assert_eq!(
9128 signal_context
9129 .wait_signal_avro_value("collision")
9130 .await
9131 .expect("typed signal collision arguments"),
9132 collision_values
9133 );
9134
9135 let child_context = workflow_context_with_codec(
9136 vec![
9137 history_event(
9138 "ChildWorkflowScheduled",
9139 json!({
9140 "sequence": 1,
9141 "child_workflow_instance_id": "collision-child",
9142 "child_workflow_run_id": "collision-run",
9143 "child_workflow_type": "collision.child",
9144 }),
9145 ),
9146 history_event(
9147 "ChildRunCompleted",
9148 json!({
9149 "sequence": 1,
9150 "child_workflow_instance_id": "collision-child",
9151 "child_workflow_run_id": "collision-run",
9152 "child_workflow_type": "collision.child",
9153 "payload_codec": DEFAULT_CODEC,
9154 "result": envelope,
9155 }),
9156 ),
9157 ],
9158 DEFAULT_CODEC,
9159 );
9160 let child = child_context
9161 .start_child_workflow_avro_value(
9162 "collision.child",
9163 ChildWorkflowOptions::new("collision-workers"),
9164 AvroValue::Array(Vec::new()),
9165 )
9166 .await
9167 .expect("typed child collision result");
9168 assert_eq!(child.result, expected);
9169 }
9170
9171 #[tokio::test]
9172 async fn replayed_typed_query_keeps_lossless_workflow_and_query_inputs() {
9173 let client = Client::new("http://127.0.0.1:8080").expect("client");
9174 let mut worker = Worker::new(client, "rust-workers");
9175 worker.register_replayed_workflow_avro_value(
9176 "typed.replayed",
9177 || (),
9178 |_ctx, input, _state| async move { Ok(input) },
9179 );
9180 worker.register_replayed_query_avro_value::<(), _, _>(
9181 "typed.replayed",
9182 "inspect",
9183 |ctx, _state, args| async move {
9184 let mut signals = ctx.signals_avro_value("collision");
9185 let signal = signals
9186 .pop()
9187 .map(AvroValue::Array)
9188 .unwrap_or_else(|| AvroValue::Array(Vec::new()));
9189 Ok(AvroValue::Array(vec![
9190 ctx.workflow_input_avro_value().clone(),
9191 signal,
9192 args,
9193 ]))
9194 },
9195 );
9196 let arguments = AvroValue::Array(projection_collision_probe());
9197 let signal_arguments =
9198 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed query signal arguments");
9199 let task = QueryTask {
9200 query_task_id: "query-typed-replay".to_string(),
9201 query_task_attempt: 1,
9202 lease_owner: Some("rust-worker".to_string()),
9203 workflow_id: Some("typed-replay".to_string()),
9204 run_id: Some("run-typed-replay".to_string()),
9205 workflow_type: "typed.replayed".to_string(),
9206 query_name: "inspect".to_string(),
9207 payload_codec: DEFAULT_CODEC.to_string(),
9208 workflow_arguments: Some(
9209 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("workflow arguments"),
9210 ),
9211 query_arguments: Some(
9212 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("query arguments"),
9213 ),
9214 history_events: vec![history_event(
9215 "SignalReceived",
9216 json!({
9217 "signal_id": "collision-signal",
9218 "signal_name": "collision",
9219 "workflow_sequence": 1,
9220 "payload_codec": DEFAULT_CODEC,
9221 "arguments": signal_arguments,
9222 }),
9223 )],
9224 history_export: None,
9225 run_status: Some("completed".to_string()),
9226 };
9227
9228 assert_eq!(
9229 worker
9230 .execute_query_task(task)
9231 .await
9232 .expect("typed replay query"),
9233 AvroValue::Array(vec![arguments.clone(), arguments.clone(), arguments])
9234 );
9235 }
9236
9237 #[test]
9238 fn public_avro_adapter_rejects_non_string_map_keys_before_json_conversion() {
9239 let value = BTreeMap::from([(1_i32, "integer key")]);
9240 let error = PayloadEnvelope::avro(&value)
9241 .expect_err("integer map keys must fail")
9242 .to_string();
9243
9244 assert!(error.contains("invalid_map_key"));
9245 }
9246
9247 #[test]
9248 fn json_codec_remains_plain_json() {
9249 let value = json!({"greeting": "hello", "count": 3, "ok": true});
9250 let envelope = PayloadEnvelope::json(&value).expect("encode");
9251
9252 assert_eq!(envelope.codec, JSON_CODEC);
9253 assert_eq!(envelope.blob, serde_json::to_string(&value).expect("json"));
9254 assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
9255 }
9256
9257 #[test]
9258 fn prerelease_avro_payload_without_single_object_frame_is_rejected() {
9259 let envelope = PayloadEnvelope {
9260 codec: DEFAULT_CODEC.to_string(),
9261 blob: BASE64.encode([0x01]),
9262 };
9263
9264 let error = decode_payload::<Value>(&envelope).expect_err("prerelease payload must fail");
9265 assert!(error.to_string().contains("invalid_payload_framing"));
9266 }
9267
9268 #[test]
9269 fn workflow_context_schedules_activity_until_completion_is_in_history() {
9270 let ctx = WorkflowContext {
9271 state: Arc::new(Mutex::new(
9272 WorkflowState::new_with_identity(
9273 Vec::new(),
9274 Some("wf-parent".to_string()),
9275 Some("run-parent".to_string()),
9276 "rust-workers".to_string(),
9277 DEFAULT_CODEC.to_string(),
9278 None,
9279 )
9280 .expect("workflow state"),
9281 )),
9282 };
9283
9284 let mut call = Box::pin(ctx.activity("hello.activity", json!(["Ada"])));
9285 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9286 assert!(matches!(
9287 call.as_mut().poll(&mut task_context),
9288 Poll::Pending
9289 ));
9290
9291 let commands = ctx.take_commands().expect("commands");
9292 assert_eq!(commands[0]["type"], "schedule_activity");
9293 assert_eq!(commands[0]["activity_type"], "hello.activity");
9294 }
9295
9296 #[test]
9297 fn activity_options_encode_retry_policy_queue_and_every_timeout() {
9298 let ctx = workflow_context(Vec::new());
9299 let options = ActivityOptions::new()
9300 .task_queue("payments")
9301 .retry_policy(
9302 ActivityRetryPolicy::new(4)
9303 .exponential_backoff(Duration::from_secs(1), 3, Some(Duration::from_secs(10)))
9304 .non_retryable_error_type("ValidationError"),
9305 )
9306 .start_to_close_timeout(Duration::from_secs(120))
9307 .schedule_to_start_timeout(Duration::from_secs(10))
9308 .schedule_to_close_timeout(Duration::from_secs(300))
9309 .heartbeat_timeout(Duration::from_secs(15));
9310 let mut call = Box::pin(ctx.activity_with_options(
9311 "charge-card",
9312 options,
9313 json!([{"order_id": "o-1"}]),
9314 ));
9315 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9316
9317 assert!(matches!(
9318 call.as_mut().poll(&mut task_context),
9319 Poll::Pending
9320 ));
9321 assert!(matches!(
9322 call.as_mut().poll(&mut task_context),
9323 Poll::Pending
9324 ));
9325
9326 let commands = ctx.take_commands().expect("activity command");
9327 assert_eq!(commands.len(), 1, "one future emits one logical schedule");
9328 assert_eq!(commands[0]["queue"], "payments");
9329 assert_eq!(
9330 commands[0]["retry_policy"],
9331 json!({
9332 "max_attempts": 4,
9333 "backoff_seconds": [1, 3, 9],
9334 "non_retryable_error_types": ["ValidationError"],
9335 })
9336 );
9337 assert_eq!(commands[0]["start_to_close_timeout"], 120);
9338 assert_eq!(commands[0]["schedule_to_start_timeout"], 10);
9339 assert_eq!(commands[0]["schedule_to_close_timeout"], 300);
9340 assert_eq!(commands[0]["heartbeat_timeout"], 15);
9341 }
9342
9343 #[test]
9344 fn activity_options_encode_explicit_and_rounded_backoff_intervals() {
9345 let ctx = workflow_context(Vec::new());
9346 let options = ActivityOptions::new().retry_policy(
9347 ActivityRetryPolicy::new(3)
9348 .backoff_intervals([Duration::from_millis(1), Duration::from_millis(1_001)]),
9349 );
9350 let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
9351 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9352
9353 assert!(matches!(
9354 call.as_mut().poll(&mut task_context),
9355 Poll::Pending
9356 ));
9357 assert_eq!(
9358 ctx.take_commands().expect("command")[0]["retry_policy"]["backoff_seconds"],
9359 json!([1, 2])
9360 );
9361 }
9362
9363 #[test]
9364 fn invalid_activity_options_return_typed_errors_before_emitting_commands() {
9365 let cases = [
9366 (
9367 ActivityOptions::new().task_queue(" "),
9368 ActivityOptionsErrorKind::EmptyTaskQueue,
9369 ),
9370 (
9371 ActivityOptions::new().retry_policy(ActivityRetryPolicy::default()),
9372 ActivityOptionsErrorKind::EmptyRetryPolicy,
9373 ),
9374 (
9375 ActivityOptions::new().retry_policy(ActivityRetryPolicy::new(0)),
9376 ActivityOptionsErrorKind::InvalidMaxAttempts,
9377 ),
9378 (
9379 ActivityOptions::new().retry_policy(ActivityRetryPolicy {
9380 max_attempts: None,
9381 backoff: Some(ActivityBackoff::Explicit(vec![Duration::from_secs(1)])),
9382 non_retryable_error_types: Vec::new(),
9383 }),
9384 ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
9385 ),
9386 (
9387 ActivityOptions::new().retry_policy(
9388 ActivityRetryPolicy::new(2)
9389 .backoff_intervals([Duration::from_secs(1), Duration::from_secs(2)]),
9390 ),
9391 ActivityOptionsErrorKind::TooManyBackoffIntervals,
9392 ),
9393 (
9394 ActivityOptions::new().retry_policy(
9395 ActivityRetryPolicy::new(2).exponential_backoff(
9396 Duration::from_secs(1),
9397 0,
9398 None,
9399 ),
9400 ),
9401 ActivityOptionsErrorKind::InvalidBackoffCoefficient,
9402 ),
9403 (
9404 ActivityOptions::new()
9405 .retry_policy(ActivityRetryPolicy::new(2).non_retryable_error_type(" ")),
9406 ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
9407 ),
9408 (
9409 ActivityOptions::new().retry_policy(
9410 ActivityRetryPolicy::new(10_002).exponential_backoff(
9411 Duration::from_secs(1),
9412 1,
9413 None,
9414 ),
9415 ),
9416 ActivityOptionsErrorKind::BackoffGenerationTooLarge,
9417 ),
9418 (
9419 ActivityOptions::new().retry_policy(
9420 ActivityRetryPolicy::new(2)
9421 .backoff_intervals([Duration::from_secs(i64::MAX as u64 + 1)]),
9422 ),
9423 ActivityOptionsErrorKind::BackoffOverflow,
9424 ),
9425 ];
9426
9427 for (options, expected_kind) in cases {
9428 let ctx = workflow_context(Vec::new());
9429 let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
9430 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9431 let Poll::Ready(Err(Error::InvalidActivityOptions(error))) =
9432 call.as_mut().poll(&mut task_context)
9433 else {
9434 panic!("expected typed activity validation error");
9435 };
9436 assert_eq!(error.kind, expected_kind);
9437 assert!(ctx.take_commands().expect("commands").is_empty());
9438 }
9439 }
9440
9441 #[test]
9442 fn activity_options_validate_positive_and_ordered_timeouts() {
9443 let zero_timeout_cases = [
9444 ActivityOptions::new().start_to_close_timeout(Duration::ZERO),
9445 ActivityOptions::new().schedule_to_start_timeout(Duration::ZERO),
9446 ActivityOptions::new().schedule_to_close_timeout(Duration::ZERO),
9447 ActivityOptions::new().heartbeat_timeout(Duration::ZERO),
9448 ];
9449 for options in zero_timeout_cases {
9450 assert_eq!(
9451 options.validate().expect_err("zero timeout").kind,
9452 ActivityOptionsErrorKind::TimeoutNotPositive
9453 );
9454 }
9455
9456 let ordering_cases = [
9457 ActivityOptions::new()
9458 .heartbeat_timeout(Duration::from_secs(11))
9459 .start_to_close_timeout(Duration::from_secs(10)),
9460 ActivityOptions::new()
9461 .start_to_close_timeout(Duration::from_secs(31))
9462 .schedule_to_close_timeout(Duration::from_secs(30)),
9463 ActivityOptions::new()
9464 .schedule_to_start_timeout(Duration::from_secs(31))
9465 .schedule_to_close_timeout(Duration::from_secs(30)),
9466 ];
9467 for options in ordering_cases {
9468 assert_eq!(
9469 options.validate().expect_err("timeout order").kind,
9470 ActivityOptionsErrorKind::TimeoutOrder
9471 );
9472 }
9473
9474 assert_eq!(
9475 ActivityOptions::new()
9476 .start_to_close_timeout(Duration::from_secs(i64::MAX as u64 + 1))
9477 .validate()
9478 .expect_err("protocol integer overflow")
9479 .kind,
9480 ActivityOptionsErrorKind::TimeoutOverflow
9481 );
9482 }
9483
9484 #[test]
9485 fn replayed_activity_retry_history_completes_without_duplicate_schedule() {
9486 let ctx = workflow_context(completed_retry_activity_history());
9487 let mut call =
9488 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
9489 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9490
9491 assert!(matches!(
9492 call.as_mut().poll(&mut task_context),
9493 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
9494 ));
9495 assert!(ctx.take_commands().expect("commands").is_empty());
9496 ctx.ensure_history_consumed().expect("history consumed");
9497 }
9498
9499 #[test]
9500 fn duplicate_non_retryable_types_use_one_command_and_replay_representation() {
9501 let mut options = retry_activity_options();
9502 options
9503 .retry_policy
9504 .as_mut()
9505 .expect("retry policy")
9506 .non_retryable_error_types
9507 .extend([" PermanentError ".to_string(), "PermanentError".to_string()]);
9508
9509 let new_ctx = workflow_context(Vec::new());
9510 let mut new_call =
9511 Box::pin(new_ctx.activity_with_options("flaky", options.clone(), json!([])));
9512 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9513 assert!(matches!(
9514 new_call.as_mut().poll(&mut task_context),
9515 Poll::Pending
9516 ));
9517 let commands = new_ctx.take_commands().expect("commands");
9518 assert_eq!(commands.len(), 1);
9519 assert_eq!(
9520 commands[0]["retry_policy"]["non_retryable_error_types"],
9521 json!(["PermanentError"])
9522 );
9523
9524 let replay_ctx = workflow_context(completed_retry_activity_history());
9525 let mut replay_call =
9526 Box::pin(replay_ctx.activity_with_options("flaky", options, json!([])));
9527 assert!(matches!(
9528 replay_call.as_mut().poll(&mut task_context),
9529 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
9530 ));
9531 assert!(replay_ctx.take_commands().expect("commands").is_empty());
9532 replay_ctx
9533 .ensure_history_consumed()
9534 .expect("history consumed");
9535 }
9536
9537 #[test]
9538 fn replayed_intermediate_retry_remains_pending_across_restarts() {
9539 let history = completed_retry_activity_history()
9540 .into_iter()
9541 .take(3)
9542 .collect::<Vec<_>>();
9543
9544 for _restart in 0..2 {
9545 let ctx = workflow_context(history.clone());
9546 let mut call =
9547 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
9548 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9549 assert!(matches!(
9550 call.as_mut().poll(&mut task_context),
9551 Poll::Pending
9552 ));
9553 assert!(ctx.take_commands().expect("commands").is_empty());
9554 }
9555 }
9556
9557 #[test]
9558 fn replayed_activity_rejects_changed_queue_retry_and_every_timeout_field() {
9559 let mut changed_queue = retry_activity_options();
9560 changed_queue.task_queue = Some("different-queue".to_string());
9561
9562 let mut changed_max_attempts = retry_activity_options();
9563 let retry_policy = changed_max_attempts
9564 .retry_policy
9565 .as_mut()
9566 .expect("retry policy");
9567 retry_policy.max_attempts = Some(4);
9568
9569 let mut changed_backoff = retry_activity_options();
9570 let retry_policy = changed_backoff.retry_policy.as_mut().expect("retry policy");
9571 retry_policy.backoff = Some(ActivityBackoff::Explicit(vec![
9572 Duration::from_secs(3),
9573 Duration::from_secs(4),
9574 ]));
9575
9576 let mut changed_non_retryable_types = retry_activity_options();
9577 let retry_policy = changed_non_retryable_types
9578 .retry_policy
9579 .as_mut()
9580 .expect("retry policy");
9581 retry_policy.non_retryable_error_types = vec!["AnotherPermanentError".to_string()];
9582
9583 let mut changed_start_to_close = retry_activity_options();
9584 changed_start_to_close.start_to_close_timeout = Some(Duration::from_secs(31));
9585 let mut changed_schedule_to_start = retry_activity_options();
9586 changed_schedule_to_start.schedule_to_start_timeout = Some(Duration::from_secs(6));
9587 let mut changed_schedule_to_close = retry_activity_options();
9588 changed_schedule_to_close.schedule_to_close_timeout = Some(Duration::from_secs(91));
9589 let mut changed_heartbeat = retry_activity_options();
9590 changed_heartbeat.heartbeat_timeout = Some(Duration::from_secs(11));
9591
9592 let cases = [
9593 (changed_queue, "activity_task_queue_mismatch"),
9594 (changed_max_attempts, "activity_retry_policy_mismatch"),
9595 (changed_backoff, "activity_retry_policy_mismatch"),
9596 (
9597 changed_non_retryable_types,
9598 "activity_retry_policy_mismatch",
9599 ),
9600 (changed_start_to_close, "activity_retry_policy_mismatch"),
9601 (changed_schedule_to_start, "activity_retry_policy_mismatch"),
9602 (changed_schedule_to_close, "activity_retry_policy_mismatch"),
9603 (changed_heartbeat, "activity_retry_policy_mismatch"),
9604 ];
9605
9606 for (options, expected_reason) in cases {
9607 let ctx = workflow_context(completed_retry_activity_history());
9608 let mut call = Box::pin(ctx.activity_with_options("flaky", options, json!([])));
9609 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9610 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
9611 call.as_mut().poll(&mut task_context)
9612 else {
9613 panic!("changed activity options must fail replay");
9614 };
9615 assert_eq!(failure.reason, expected_reason);
9616 assert_eq!(failure.sequence, Some(1));
9617 assert!(ctx.take_commands().expect("commands").is_empty());
9618 }
9619 }
9620
9621 #[test]
9622 fn replayed_activity_rejects_changed_execution_mode_and_snapshot_version() {
9623 let cases = [
9624 (
9625 "execution_mode",
9626 json!("local"),
9627 "activity_execution_mode_mismatch",
9628 ),
9629 (
9630 "snapshot_version",
9631 json!(2),
9632 "activity_retry_policy_mismatch",
9633 ),
9634 ];
9635
9636 for (field, value, expected_reason) in cases {
9637 let mut history = completed_retry_activity_history();
9638 let activity = history[0].payload["activity"]
9639 .as_object_mut()
9640 .expect("activity snapshot");
9641 if field == "execution_mode" {
9642 activity.insert(field.to_string(), value);
9643 } else {
9644 activity["retry_policy"]
9645 .as_object_mut()
9646 .expect("retry snapshot")
9647 .insert(field.to_string(), value);
9648 }
9649
9650 let ctx = workflow_context(history);
9651 let mut call =
9652 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
9653 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9654 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
9655 call.as_mut().poll(&mut task_context)
9656 else {
9657 panic!("changed {field} must fail replay");
9658 };
9659 assert_eq!(failure.reason, expected_reason);
9660 assert_eq!(failure.sequence, Some(1));
9661 assert!(ctx.take_commands().expect("commands").is_empty());
9662 }
9663 }
9664
9665 #[test]
9666 fn replayed_legacy_activity_treats_missing_option_snapshot_as_unknown() {
9667 let mut history = completed_retry_activity_history();
9668 let activity = history[0].payload["activity"]
9669 .as_object_mut()
9670 .expect("activity snapshot");
9671 activity.remove("execution_mode");
9672 activity.remove("retry_policy");
9673
9674 let mut current = retry_activity_options();
9675 current.start_to_close_timeout = Some(Duration::from_secs(45));
9676 current.schedule_to_start_timeout = Some(Duration::from_secs(8));
9677 current.schedule_to_close_timeout = Some(Duration::from_secs(120));
9678 current.heartbeat_timeout = Some(Duration::from_secs(12));
9679
9680 let ctx = workflow_context(history);
9681 let mut call = Box::pin(ctx.activity_with_options("flaky", current, json!([])));
9682 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9683 assert!(matches!(
9684 call.as_mut().poll(&mut task_context),
9685 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
9686 ));
9687 assert!(ctx.take_commands().expect("commands").is_empty());
9688 ctx.ensure_history_consumed().expect("history consumed");
9689 }
9690
9691 #[test]
9692 fn terminal_activity_failed_after_start_returns_typed_failure() {
9693 let history = vec![
9694 history_event(
9695 "ActivityScheduled",
9696 json!({
9697 "sequence": 1,
9698 "activity_type": "flaky",
9699 "activity_execution_id": "act-terminal",
9700 "activity": {
9701 "id": "act-terminal",
9702 "sequence": 1,
9703 "type": "flaky",
9704 "queue": "critical-activities",
9705 "retry_policy": {
9706 "snapshot_version": 1,
9707 "max_attempts": 3,
9708 "backoff_seconds": [2, 4],
9709 "non_retryable_error_types": ["PermanentError"]
9710 }
9711 }
9712 }),
9713 ),
9714 history_event(
9715 "ActivityStarted",
9716 json!({
9717 "sequence": 1,
9718 "activity_type": "flaky",
9719 "activity_execution_id": "act-terminal",
9720 "activity_attempt_id": "attempt-1",
9721 "attempt_number": 1
9722 }),
9723 ),
9724 history_event(
9725 "ActivityFailed",
9726 json!({
9727 "sequence": 1,
9728 "activity_type": "flaky",
9729 "activity_execution_id": "act-terminal",
9730 "activity_attempt_id": "attempt-1",
9731 "attempt_number": 1,
9732 "failure_id": "failure-terminal",
9733 "failure_category": "activity",
9734 "exception_type": "PermanentError",
9735 "message": "cannot retry",
9736 "non_retryable": true
9737 }),
9738 ),
9739 ];
9740 let ctx = workflow_context(history);
9741 let mut call =
9742 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
9743 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9744
9745 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
9746 call.as_mut().poll(&mut task_context)
9747 else {
9748 panic!("terminal ActivityFailed must settle the activity future");
9749 };
9750 assert_eq!(failure.kind, ActivityFailureKind::Failed);
9751 assert_eq!(
9752 failure.activity_execution_id.as_deref(),
9753 Some("act-terminal")
9754 );
9755 assert_eq!(failure.exception_type.as_deref(), Some("PermanentError"));
9756 assert!(failure.non_retryable);
9757 assert!(ctx.take_commands().expect("commands").is_empty());
9758 ctx.ensure_history_consumed().expect("history consumed");
9759 }
9760
9761 #[test]
9762 fn activity_terminal_events_return_machine_readable_failures() {
9763 let cases = [
9764 (
9765 "ActivityFailed",
9766 json!({
9767 "sequence": 1,
9768 "activity_type": "charge-card",
9769 "activity_execution_id": "act-1",
9770 "activity_attempt_id": "attempt-2",
9771 "attempt_number": 2,
9772 "failure_id": "failure-1",
9773 "failure_category": "activity",
9774 "exception_type": "PaymentDeclined",
9775 "exception_class": "payments.PaymentDeclined",
9776 "message": "card declined",
9777 "non_retryable": true
9778 }),
9779 ActivityFailureKind::Failed,
9780 "activity",
9781 ),
9782 (
9783 "ActivityCancelled",
9784 json!({
9785 "sequence": 1,
9786 "activity_type": "charge-card",
9787 "activity_execution_id": "act-1",
9788 "activity_attempt_id": "attempt-1"
9789 }),
9790 ActivityFailureKind::Cancelled,
9791 "cancelled",
9792 ),
9793 ];
9794
9795 for (event_type, payload, expected_kind, expected_reason) in cases {
9796 let ctx = workflow_context(vec![history_event(event_type, payload)]);
9797 let mut call = Box::pin(ctx.activity("charge-card", json!([])));
9798 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9799 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
9800 call.as_mut().poll(&mut task_context)
9801 else {
9802 panic!("expected terminal activity failure");
9803 };
9804 assert_eq!(failure.kind, expected_kind);
9805 assert_eq!(failure.reason, expected_reason);
9806 assert_eq!(failure.activity_execution_id.as_deref(), Some("act-1"));
9807 assert_eq!(failure.activity_type.as_deref(), Some("charge-card"));
9808 }
9809 }
9810
9811 #[test]
9812 fn every_activity_timeout_class_is_typed() {
9813 for timeout_kind in [
9814 "start_to_close",
9815 "schedule_to_start",
9816 "schedule_to_close",
9817 "heartbeat",
9818 ] {
9819 let ctx = workflow_context(vec![history_event(
9820 "ActivityTimedOut",
9821 json!({
9822 "sequence": 1,
9823 "activity_type": "slow",
9824 "activity_execution_id": "act-timeout",
9825 "activity_attempt_id": "attempt-timeout",
9826 "failure_category": "timeout",
9827 "timeout_kind": timeout_kind,
9828 "message": "deadline expired"
9829 }),
9830 )]);
9831 let mut call = Box::pin(ctx.activity("slow", json!([])));
9832 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9833 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
9834 call.as_mut().poll(&mut task_context)
9835 else {
9836 panic!("expected timeout failure");
9837 };
9838 assert_eq!(failure.kind, ActivityFailureKind::TimedOut);
9839 assert_eq!(failure.reason, timeout_kind);
9840 assert_eq!(failure.timeout_kind.as_deref(), Some(timeout_kind));
9841 assert_eq!(failure.failure_category.as_deref(), Some("timeout"));
9842 }
9843 }
9844
9845 #[test]
9846 fn workflow_sleep_emits_one_durable_timer_and_rounds_up() {
9847 let ctx = workflow_context(Vec::new());
9848 let mut sleep = Box::pin(ctx.sleep(Duration::from_millis(1_001)));
9849 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9850
9851 assert!(matches!(
9852 sleep.as_mut().poll(&mut task_context),
9853 Poll::Pending
9854 ));
9855 assert!(matches!(
9856 sleep.as_mut().poll(&mut task_context),
9857 Poll::Pending
9858 ));
9859
9860 let commands = ctx.take_commands().expect("timer command");
9861 assert_eq!(
9862 commands,
9863 vec![json!({
9864 "type": "start_timer",
9865 "delay_seconds": 2,
9866 })]
9867 );
9868 }
9869
9870 #[test]
9871 fn workflow_sleep_replays_matching_schedule_and_fire_without_a_command() {
9872 let history = vec![
9873 history_event(
9874 "TimerScheduled",
9875 json!({
9876 "sequence": 1,
9877 "timer_id": "timer-1",
9878 "delay_seconds": 5,
9879 "fire_at": "2026-07-11T12:00:05Z",
9880 }),
9881 ),
9882 history_event(
9883 "TimerFired",
9884 json!({
9885 "sequence": 1,
9886 "timer_id": "timer-1",
9887 "delay_seconds": 5,
9888 "fire_at": "2026-07-11T12:00:05Z",
9889 "fired_at": "2026-07-11T12:00:05Z",
9890 }),
9891 ),
9892 ];
9893
9894 for _restart in 0..2 {
9895 let ctx = workflow_context(history.clone());
9896 let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(5)));
9897 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9898 assert!(matches!(
9899 sleep.as_mut().poll(&mut task_context),
9900 Poll::Ready(Ok(()))
9901 ));
9902 assert!(ctx.take_commands().expect("commands").is_empty());
9903 ctx.ensure_history_consumed().expect("history consumed");
9904 }
9905 }
9906
9907 #[test]
9908 fn workflow_sleep_rejects_changed_delay_during_replay() {
9909 let ctx = workflow_context(vec![
9910 history_event(
9911 "TimerScheduled",
9912 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
9913 ),
9914 history_event(
9915 "TimerFired",
9916 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
9917 ),
9918 ]);
9919 let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(500)));
9920 let mut task_context = TaskContext::from_waker(noop_waker_ref());
9921
9922 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
9923 sleep.as_mut().poll(&mut task_context)
9924 else {
9925 panic!("changed timer delay must be rejected");
9926 };
9927 assert_eq!(failure.reason, "timer_delay_mismatch");
9928 assert_eq!(failure.sequence, Some(1));
9929 }
9930
9931 #[test]
9932 fn workflow_history_rejects_unpaired_or_mismatched_timer_events() {
9933 let lone_fire = WorkflowState::new(
9934 vec![history_event(
9935 "TimerFired",
9936 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
9937 )],
9938 "rust-workers".to_string(),
9939 JSON_CODEC.to_string(),
9940 None,
9941 )
9942 .expect_err("TimerFired requires TimerScheduled");
9943 assert!(matches!(
9944 lone_fire,
9945 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
9946 if reason == "timer_schedule_missing_or_duplicate"
9947 ));
9948
9949 let wrong_identity = WorkflowState::new(
9950 vec![
9951 history_event(
9952 "TimerScheduled",
9953 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
9954 ),
9955 history_event(
9956 "TimerFired",
9957 json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
9958 ),
9959 ],
9960 "rust-workers".to_string(),
9961 JSON_CODEC.to_string(),
9962 None,
9963 )
9964 .expect_err("fire must match scheduled timer identity");
9965 assert!(matches!(
9966 wrong_identity,
9967 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
9968 if reason == "timer_identity_mismatch"
9969 ));
9970
9971 let duplicate_fire = WorkflowState::new(
9972 vec![
9973 history_event(
9974 "TimerScheduled",
9975 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
9976 ),
9977 history_event(
9978 "TimerFired",
9979 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
9980 ),
9981 history_event(
9982 "TimerFired",
9983 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
9984 ),
9985 ],
9986 "rust-workers".to_string(),
9987 JSON_CODEC.to_string(),
9988 None,
9989 )
9990 .expect_err("a durable timer cannot fire twice");
9991 assert!(matches!(
9992 duplicate_fire,
9993 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
9994 if reason == "duplicate_timer_fire"
9995 ));
9996
9997 let wrong_fired_delay = WorkflowState::new(
9998 vec![
9999 history_event(
10000 "TimerScheduled",
10001 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10002 ),
10003 history_event(
10004 "TimerFired",
10005 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 6}),
10006 ),
10007 ],
10008 "rust-workers".to_string(),
10009 JSON_CODEC.to_string(),
10010 None,
10011 )
10012 .expect_err("timer schedule and fire delays must agree");
10013 assert!(matches!(
10014 wrong_fired_delay,
10015 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10016 if reason == "timer_history_delay_mismatch"
10017 ));
10018 }
10019
10020 #[test]
10021 fn replay_rejects_activity_moved_before_recorded_timer() {
10022 let ctx = workflow_context(vec![
10023 history_event(
10024 "TimerScheduled",
10025 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10026 ),
10027 history_event(
10028 "TimerFired",
10029 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10030 ),
10031 history_event(
10032 "ActivityCompleted",
10033 json!({
10034 "sequence": 2,
10035 "activity_type": "after-timer",
10036 "payload_codec": "json",
10037 "result": {"codec": "json", "blob": "\"done\""},
10038 }),
10039 ),
10040 ]);
10041 let mut activity = Box::pin(ctx.activity("after-timer", json!([])));
10042 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10043
10044 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10045 activity.as_mut().poll(&mut task_context)
10046 else {
10047 panic!("reordered durable command must be rejected");
10048 };
10049 assert_eq!(failure.reason, "recorded_command_mismatch");
10050 assert_eq!(failure.sequence, Some(1));
10051 assert_eq!(failure.expected.as_deref(), Some("timer"));
10052 assert_eq!(failure.actual.as_deref(), Some("activity:after-timer"));
10053 }
10054
10055 #[test]
10056 fn workflow_context_emits_a_typed_named_signal_wait() {
10057 let ctx = workflow_context(Vec::new());
10058 let mut signal = Box::pin(ctx.wait_signal("finish"));
10059 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10060
10061 assert!(matches!(
10062 signal.as_mut().poll(&mut task_context),
10063 Poll::Pending
10064 ));
10065 assert_eq!(
10066 ctx.take_commands().expect("signal-wait command"),
10067 vec![json!({
10068 "type": "open_signal_wait",
10069 "signal_name": "finish",
10070 })]
10071 );
10072 }
10073
10074 #[test]
10075 fn condition_wait_history_does_not_resolve_a_typed_signal_wait() {
10076 let ctx = workflow_context(vec![
10077 history_event(
10078 "ConditionWaitOpened",
10079 json!({"sequence": 1, "condition_key": "signal:finish"}),
10080 ),
10081 history_event(
10082 "ConditionWaitSatisfied",
10083 json!({"sequence": 1, "condition_key": "signal:finish"}),
10084 ),
10085 history_event(
10086 "SignalReceived",
10087 json!({"signal_name": "finish", "arguments": []}),
10088 ),
10089 ]);
10090 let mut signal = Box::pin(ctx.wait_signal("finish"));
10091 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10092
10093 assert!(matches!(
10094 signal.as_mut().poll(&mut task_context),
10095 Poll::Pending
10096 ));
10097 assert_eq!(
10098 ctx.take_commands().expect("typed signal-wait command"),
10099 vec![json!({
10100 "type": "open_signal_wait",
10101 "signal_name": "finish",
10102 })]
10103 );
10104 }
10105
10106 #[test]
10107 fn replay_orders_signal_waits_and_timers_in_one_command_stream() {
10108 let signal_then_timer = vec![
10109 history_event(
10110 "SignalWaitOpened",
10111 json!({"sequence": 1, "signal_name": "go"}),
10112 ),
10113 history_event(
10114 "SignalApplied",
10115 json!({
10116 "sequence": 1,
10117 "signal_name": "go",
10118 "value": {"codec": "json", "blob": "[\"now\"]"},
10119 }),
10120 ),
10121 history_event(
10122 "TimerScheduled",
10123 json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
10124 ),
10125 history_event(
10126 "TimerFired",
10127 json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
10128 ),
10129 ];
10130
10131 let ctx = workflow_context(signal_then_timer.clone());
10132 let mut signal = Box::pin(ctx.wait_signal("go"));
10133 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10134 assert!(matches!(
10135 signal.as_mut().poll(&mut task_context),
10136 Poll::Ready(Ok(arguments)) if arguments == vec![json!("now")]
10137 ));
10138 let mut timer = Box::pin(ctx.sleep(Duration::from_secs(5)));
10139 assert!(matches!(
10140 timer.as_mut().poll(&mut task_context),
10141 Poll::Ready(Ok(()))
10142 ));
10143 ctx.ensure_history_consumed()
10144 .expect("signal and timer history consumed in order");
10145
10146 let reordered = workflow_context(signal_then_timer);
10147 let mut timer_first = Box::pin(reordered.sleep(Duration::from_secs(5)));
10148 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10149 timer_first.as_mut().poll(&mut task_context)
10150 else {
10151 panic!("timer cannot consume signal-wait-first history");
10152 };
10153 assert_eq!(failure.reason, "recorded_command_mismatch");
10154 assert_eq!(failure.sequence, Some(1));
10155 assert_eq!(failure.expected.as_deref(), Some("signal wait"));
10156
10157 let timer_then_signal = vec![
10158 history_event(
10159 "TimerScheduled",
10160 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10161 ),
10162 history_event(
10163 "TimerFired",
10164 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10165 ),
10166 history_event(
10167 "SignalWaitOpened",
10168 json!({"sequence": 2, "signal_name": "go"}),
10169 ),
10170 history_event(
10171 "SignalApplied",
10172 json!({
10173 "sequence": 2,
10174 "signal_name": "go",
10175 "value": {"codec": "json", "blob": "[]"},
10176 }),
10177 ),
10178 ];
10179 let reordered = workflow_context(timer_then_signal);
10180 let mut signal_first = Box::pin(reordered.wait_signal("go"));
10181 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10182 signal_first.as_mut().poll(&mut task_context)
10183 else {
10184 panic!("signal wait cannot consume timer-first history");
10185 };
10186 assert_eq!(failure.reason, "recorded_command_mismatch");
10187 assert_eq!(failure.sequence, Some(1));
10188 assert_eq!(failure.expected.as_deref(), Some("timer"));
10189 }
10190
10191 #[test]
10192 fn workflow_history_rejects_duplicate_or_colliding_command_sequences() {
10193 let duplicate_timer = WorkflowState::new(
10194 vec![
10195 history_event(
10196 "TimerScheduled",
10197 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10198 ),
10199 history_event(
10200 "TimerScheduled",
10201 json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
10202 ),
10203 ],
10204 "rust-workers".to_string(),
10205 JSON_CODEC.to_string(),
10206 None,
10207 )
10208 .expect_err("one workflow sequence cannot schedule two timers");
10209 assert!(matches!(
10210 duplicate_timer,
10211 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10212 if reason == "timer_schedule_missing_or_duplicate"
10213 ));
10214
10215 let colliding_kinds = WorkflowState::new(
10216 vec![
10217 history_event(
10218 "TimerScheduled",
10219 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10220 ),
10221 history_event(
10222 "ActivityCompleted",
10223 json!({"sequence": 1, "activity_type": "same-sequence"}),
10224 ),
10225 ],
10226 "rust-workers".to_string(),
10227 JSON_CODEC.to_string(),
10228 None,
10229 )
10230 .expect_err("one workflow sequence cannot identify two command kinds");
10231 assert!(matches!(
10232 colliding_kinds,
10233 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10234 if reason == "durable_command_sequence_collision"
10235 ));
10236
10237 let duplicate_signal_wait = WorkflowState::new(
10238 vec![
10239 history_event(
10240 "SignalWaitOpened",
10241 json!({"sequence": 1, "signal_name": "go"}),
10242 ),
10243 history_event(
10244 "SignalWaitOpened",
10245 json!({"sequence": 1, "signal_name": "go"}),
10246 ),
10247 ],
10248 "rust-workers".to_string(),
10249 JSON_CODEC.to_string(),
10250 None,
10251 )
10252 .expect_err("one workflow sequence cannot open two signal waits");
10253 assert!(matches!(
10254 duplicate_signal_wait,
10255 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10256 if reason == "signal_wait_open_missing_or_duplicate"
10257 ));
10258 }
10259
10260 #[test]
10261 fn workflow_history_accepts_a_first_command_after_global_sequence_gaps() {
10262 let result = encode_value_envelope(&json!({"captured": true}), JSON_CODEC)
10263 .expect("side-effect result");
10264 let ctx = workflow_context(vec![history_event(
10265 "SideEffectRecorded",
10266 json!({"sequence": 99, "result": result}),
10267 )]);
10268
10269 let replayed: Value = ctx
10270 .side_effect(|| panic!("recorded side effect must not run"))
10271 .expect("positive global workflow sequence is valid");
10272 assert_eq!(replayed, json!({"captured": true}));
10273 ctx.ensure_history_consumed().expect("history consumed");
10274 }
10275
10276 #[test]
10277 fn workflow_history_rejects_zero_and_descending_command_sequences() {
10278 let result =
10279 encode_value_envelope(&json!("captured"), JSON_CODEC).expect("side-effect result");
10280 let zero = WorkflowState::new(
10281 vec![history_event(
10282 "SideEffectRecorded",
10283 json!({"sequence": 0, "result": result.clone()}),
10284 )],
10285 "rust-workers".to_string(),
10286 JSON_CODEC.to_string(),
10287 None,
10288 )
10289 .expect_err("durable command sequences must be positive");
10290 assert!(matches!(
10291 zero,
10292 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10293 if reason == "durable_command_sequence_invalid"
10294 ));
10295
10296 let descending = WorkflowState::new(
10297 vec![
10298 history_event(
10299 "SideEffectRecorded",
10300 json!({"sequence": 3, "result": result}),
10301 ),
10302 history_event(
10303 "VersionMarkerRecorded",
10304 json!({
10305 "sequence": 2,
10306 "change_id": "descending-marker",
10307 "version": 1,
10308 "min_supported": 1,
10309 "max_supported": 1,
10310 }),
10311 ),
10312 ],
10313 "rust-workers".to_string(),
10314 JSON_CODEC.to_string(),
10315 None,
10316 )
10317 .expect_err("new durable commands must remain strictly ordered");
10318 let Error::NonDeterministicReplay(failure) = descending else {
10319 panic!("expected typed replay failure");
10320 };
10321 assert_eq!(failure.reason, "durable_command_sequence_mismatch");
10322 assert_eq!(failure.sequence, Some(2));
10323 assert_eq!(
10324 failure.expected.as_deref(),
10325 Some("workflow sequence greater than 3")
10326 );
10327 assert_eq!(failure.actual.as_deref(), Some("2"));
10328 }
10329
10330 #[test]
10331 fn workflow_task_replay_completes_after_signals_create_sequence_gaps() {
10332 fn worker() -> Worker {
10333 let client = Client::new("http://127.0.0.1:8080").expect("client");
10334 let mut worker = Worker::new(client, "rust-workers");
10335 worker.register_workflow("rust.finish-after-gaps", |ctx, _input| async move {
10336 ctx.wait_signal("finish").await?;
10337 let marker: String =
10338 ctx.side_effect(|| panic!("recorded side effect must not run"))?;
10339 assert_eq!(marker, "after-finish");
10340 Ok(json!("finished"))
10341 });
10342 worker
10343 }
10344
10345 let marker =
10346 encode_value_envelope(&json!("after-finish"), JSON_CODEC).expect("side-effect result");
10347 let task = workflow_task(
10348 "rust.finish-after-gaps",
10349 vec![
10350 history_event(
10351 "SignalWaitOpened",
10352 json!({"sequence": 1, "signal_name": "finish"}),
10353 ),
10354 history_event(
10355 "SignalReceived",
10356 json!({
10357 "signal_id": "increment-3",
10358 "signal_name": "increment",
10359 "workflow_sequence": 2,
10360 "payload_codec": "json",
10361 "arguments": {"codec": "json", "blob": "[3]"},
10362 }),
10363 ),
10364 history_event(
10365 "SignalReceived",
10366 json!({
10367 "signal_id": "increment-5",
10368 "signal_name": "increment",
10369 "workflow_sequence": 3,
10370 "payload_codec": "json",
10371 "arguments": {"codec": "json", "blob": "[5]"},
10372 }),
10373 ),
10374 history_event(
10375 "SignalReceived",
10376 json!({
10377 "signal_id": "finish",
10378 "signal_name": "finish",
10379 "workflow_sequence": 4,
10380 "payload_codec": "json",
10381 "arguments": {"codec": "json", "blob": "[]"},
10382 }),
10383 ),
10384 history_event(
10385 "SignalApplied",
10386 json!({
10387 "sequence": 1,
10388 "signal_id": "finish",
10389 "signal_name": "finish",
10390 "payload_codec": "json",
10391 "value": {"codec": "json", "blob": "[]"},
10392 }),
10393 ),
10394 history_event(
10395 "SideEffectRecorded",
10396 json!({"sequence": 5, "result": marker}),
10397 ),
10398 ],
10399 JSON_CODEC,
10400 );
10401
10402 for _original_or_cold_worker in 0..2 {
10403 let commands = worker()
10404 .execute_workflow_task(task.clone())
10405 .expect("signal gaps preserve deterministic replay");
10406 assert_eq!(commands.len(), 1, "replay emits only terminal completion");
10407 assert_eq!(commands[0]["type"], "complete_workflow");
10408 assert_eq!(
10409 decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("workflow output"),
10410 json!("finished")
10411 );
10412 }
10413 }
10414
10415 #[test]
10416 fn workflow_sleep_rejects_unrepresentable_rounded_duration() {
10417 let ctx = workflow_context(Vec::new());
10418 let mut sleep = Box::pin(ctx.start_timer(Duration::new(u64::MAX, 1)));
10419 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10420 assert!(matches!(
10421 sleep.as_mut().poll(&mut task_context),
10422 Poll::Ready(Err(Error::TimerDurationOverflow))
10423 ));
10424 assert!(ctx.take_commands().expect("commands").is_empty());
10425 }
10426
10427 #[test]
10428 fn workflow_task_replay_completes_without_rescheduling_recorded_commands() {
10429 let client = Client::new("http://127.0.0.1:8080").expect("client");
10430 let mut worker = Worker::new(client, "rust-workers");
10431 worker.register_workflow("rust.timer", |ctx, _input| async move {
10432 ctx.sleep(Duration::from_secs(5)).await?;
10433 ctx.activity("after-timer", json!([])).await
10434 });
10435
10436 let task = |history_events| WorkflowTask {
10437 task_id: "wft-rust-timer-1".to_string(),
10438 workflow_id: Some("wf-rust-timer".to_string()),
10439 run_id: Some("run-rust-timer".to_string()),
10440 workflow_type: "rust.timer".to_string(),
10441 payload_codec: JSON_CODEC.to_string(),
10442 arguments: Some(json!({"codec": "json", "blob": "[]"})),
10443 history_events,
10444 total_history_events: None,
10445 history_size_bytes: None,
10446 continue_as_new_recommended: None,
10447 history_budget_pressure: None,
10448 next_history_page_token: None,
10449 workflow_task_attempt: 1,
10450 workflow_signal_id: None,
10451 signal_name: None,
10452 signal_arguments: None,
10453 workflow_update_id: None,
10454 update_name: None,
10455 lease_owner: Some("rust-worker".to_string()),
10456 };
10457
10458 let initial = worker
10459 .execute_workflow_task(task(Vec::new()))
10460 .expect("initial timer task");
10461 assert_eq!(
10462 initial,
10463 vec![json!({"type": "start_timer", "delay_seconds": 5})]
10464 );
10465
10466 let replayed = worker
10467 .execute_workflow_task(task(vec![
10468 history_event(
10469 "TimerScheduled",
10470 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10471 ),
10472 history_event(
10473 "TimerFired",
10474 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10475 ),
10476 history_event(
10477 "ActivityCompleted",
10478 json!({
10479 "sequence": 2,
10480 "activity_type": "after-timer",
10481 "payload_codec": "json",
10482 "result": {"codec": "json", "blob": "\"done\""},
10483 }),
10484 ),
10485 ]))
10486 .expect("replayed workflow task");
10487 assert_eq!(replayed.len(), 1);
10488 assert_eq!(replayed[0]["type"], "complete_workflow");
10489 assert_eq!(
10490 decode_wire_value(&replayed[0]["result"], JSON_CODEC).expect("result"),
10491 json!("done")
10492 );
10493 }
10494
10495 #[test]
10496 fn workflow_continue_as_new_emits_arguments_type_and_queue_once() {
10497 let client = Client::new("http://127.0.0.1:8080").expect("client");
10498 let mut worker = Worker::new(client, "rust-workers");
10499 worker.register_workflow("rust.continue", |ctx, _input| async move {
10500 ctx.continue_as_new_with_options(
10501 ContinueAsNewOptions::new()
10502 .workflow_type("rust.next")
10503 .task_queue("next-workers"),
10504 json!([2, {"cursor": "next"}]),
10505 )
10506 });
10507
10508 let commands = worker
10509 .execute_workflow_task(workflow_task("rust.continue", Vec::new(), DEFAULT_CODEC))
10510 .expect("continue-as-new command");
10511
10512 assert_eq!(commands.len(), 1);
10513 assert_eq!(commands[0]["type"], "continue_as_new");
10514 assert_eq!(commands[0]["workflow_type"], "rust.next");
10515 assert_eq!(commands[0]["queue"], "next-workers");
10516 assert_eq!(
10517 decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC)
10518 .expect("continue-as-new arguments"),
10519 json!([2, {"cursor": "next"}])
10520 );
10521 }
10522
10523 #[test]
10524 fn continue_as_new_preserves_typed_arguments() {
10525 let client = Client::new("http://127.0.0.1:8080").expect("client");
10526 let mut worker = Worker::new(client, "rust-workers");
10527 worker.register_workflow_avro_value("rust.typed-continue", |ctx, _input| async move {
10528 ctx.continue_as_new(AvroValue::Array(vec![typed_fidelity_probe()]))?;
10529 unreachable!("continue-as-new returns a control-flow error")
10530 });
10531
10532 let commands = worker
10533 .execute_workflow_task(workflow_task(
10534 "rust.typed-continue",
10535 Vec::new(),
10536 DEFAULT_CODEC,
10537 ))
10538 .expect("typed continue-as-new command");
10539
10540 assert_eq!(commands[0]["type"], "continue_as_new");
10541 assert_eq!(
10542 decode_wire_avro_value(&commands[0]["arguments"], DEFAULT_CODEC)
10543 .expect("typed continue arguments"),
10544 AvroValue::Array(vec![typed_fidelity_probe()])
10545 );
10546 }
10547
10548 #[test]
10549 fn recorded_continue_as_new_is_consumed_without_duplicate_successor_command() {
10550 let client = Client::new("http://127.0.0.1:8080").expect("client");
10551 let mut worker = Worker::new(client, "rust-workers");
10552 worker.register_workflow("rust.continue", |ctx, _input| async move {
10553 ctx.continue_as_new(json!([2]))
10554 });
10555 let task = workflow_task(
10556 "rust.continue",
10557 vec![history_event(
10558 "WorkflowContinuedAsNew",
10559 json!({"sequence": 1, "continued_to_run_id": "run-next"}),
10560 )],
10561 JSON_CODEC,
10562 );
10563
10564 for _worker_restart_or_redelivery in 0..2 {
10565 let commands = worker
10566 .execute_workflow_task(task.clone())
10567 .expect("recorded transition replays");
10568 assert!(
10569 commands.is_empty(),
10570 "replay must not emit another successor"
10571 );
10572 }
10573 }
10574
10575 #[test]
10576 fn continue_as_new_rejects_invalid_overrides_before_emitting_a_command() {
10577 let ctx = workflow_context(Vec::new());
10578 let error = ctx
10579 .continue_as_new_with_options(ContinueAsNewOptions::new().task_queue(" "), json!([1]))
10580 .expect_err("blank queue must be rejected");
10581
10582 let Error::InvalidContinueAsNewOptions(error) = error else {
10583 panic!("expected typed continue-as-new validation error");
10584 };
10585 assert_eq!(error.field, "task_queue");
10586 assert!(ctx.take_commands().expect("commands").is_empty());
10587 }
10588
10589 #[test]
10590 fn workflow_context_exposes_server_history_budget() {
10591 let client = Client::new("http://127.0.0.1:8080").expect("client");
10592 let mut worker = Worker::new(client, "rust-workers");
10593 worker.register_workflow("rust.history-budget", |ctx, _input| async move {
10594 let budget = ctx.history_budget()?;
10595 Ok(json!({
10596 "events": budget.event_count,
10597 "bytes": budget.size_bytes,
10598 "recommended": budget.continue_as_new_recommended,
10599 "pressure": budget.pressure,
10600 }))
10601 });
10602 let task: WorkflowTask = serde_json::from_value(json!({
10603 "task_id": "task-history-budget",
10604 "workflow_type": "rust.history-budget",
10605 "payload_codec": JSON_CODEC,
10606 "history_events": [],
10607 "total_history_events": 480,
10608 "history_size_bytes": 1_048_576,
10609 "continue_as_new_recommended": true,
10610 "history_budget_pressure": "continue_as_new_recommended",
10611 }))
10612 .expect("published workflow task");
10613
10614 let commands = worker
10615 .execute_workflow_task(task)
10616 .expect("history-budget workflow");
10617 let result = decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("result");
10618 assert_eq!(result["events"], 480);
10619 assert_eq!(result["bytes"], 1_048_576);
10620 assert_eq!(result["recommended"], true);
10621 assert_eq!(result["pressure"], "continue_as_new_recommended");
10622 }
10623
10624 #[test]
10625 fn uncaught_workflow_handler_error_emits_terminal_failure_command() {
10626 let client = Client::new("http://127.0.0.1:8080").expect("client");
10627 let mut worker = Worker::new(client, "rust-workers");
10628 worker.register_workflow("rust.failing", |_ctx, _input| async move {
10629 Err(Error::Codec("rust_conformance_failure".to_string()))
10630 });
10631 let task = WorkflowTask {
10632 task_id: "wft-rust-failing-1".to_string(),
10633 workflow_id: Some("wf-rust-failing".to_string()),
10634 run_id: Some("run-rust-failing".to_string()),
10635 workflow_type: "rust.failing".to_string(),
10636 payload_codec: JSON_CODEC.to_string(),
10637 arguments: Some(encode_value_envelope(&json!([]), JSON_CODEC).expect("input")),
10638 history_events: Vec::new(),
10639 total_history_events: Some(0),
10640 history_size_bytes: None,
10641 continue_as_new_recommended: None,
10642 history_budget_pressure: None,
10643 next_history_page_token: None,
10644 workflow_task_attempt: 1,
10645 workflow_signal_id: None,
10646 signal_name: None,
10647 signal_arguments: None,
10648 workflow_update_id: None,
10649 update_name: None,
10650 lease_owner: Some("rust-worker".to_string()),
10651 };
10652
10653 let commands = worker
10654 .execute_workflow_task(task)
10655 .expect("handler failure becomes a workflow command");
10656
10657 assert_eq!(commands.len(), 1);
10658 assert_eq!(commands[0]["type"], "fail_workflow");
10659 assert_eq!(commands[0]["exception_type"], "RustWorkflowError");
10660 assert_eq!(commands[0]["exception_class"], "durable_workflow::Error");
10661 assert_eq!(commands[0]["non_retryable"], false);
10662 assert_eq!(
10663 commands[0]["message"],
10664 "codec error: rust_conformance_failure"
10665 );
10666 assert_eq!(
10667 commands[0]["exception"]["message"],
10668 "codec error: rust_conformance_failure"
10669 );
10670 }
10671
10672 #[test]
10673 fn ordinary_handler_error_preserves_commands_queued_in_the_same_decision() {
10674 let client = Client::new("http://127.0.0.1:8080").expect("client");
10675 let mut worker = Worker::new(client, "rust-workers");
10676 worker.register_workflow("rust.failing-after-side-effect", |ctx, _input| async move {
10677 let _: String = ctx.side_effect(|| "captured".to_string())?;
10678 Err(Error::WorkerLoop("application failure".to_string()))
10679 });
10680
10681 let commands = worker
10682 .execute_workflow_task(workflow_task(
10683 "rust.failing-after-side-effect",
10684 Vec::new(),
10685 JSON_CODEC,
10686 ))
10687 .expect("ordinary failure remains a workflow decision");
10688
10689 assert_eq!(commands.len(), 2);
10690 assert_eq!(commands[0]["type"], "record_side_effect");
10691 assert_eq!(commands[1]["type"], "fail_workflow");
10692 }
10693
10694 #[test]
10695 fn handler_error_cannot_hide_an_unconsumed_committed_side_effect() {
10696 let client = Client::new("http://127.0.0.1:8080").expect("client");
10697 let mut worker = Worker::new(client, "rust-workers");
10698 worker.register_workflow("rust.removed-side-effect", |_ctx, _input| async move {
10699 Err(Error::WorkerLoop("application failure".to_string()))
10700 });
10701 let result =
10702 encode_value_envelope(&json!("committed"), JSON_CODEC).expect("side-effect result");
10703
10704 let error = worker
10705 .execute_workflow_task(workflow_task(
10706 "rust.removed-side-effect",
10707 vec![history_event(
10708 "SideEffectRecorded",
10709 json!({"sequence": 1, "result": result}),
10710 )],
10711 JSON_CODEC,
10712 ))
10713 .expect_err("removed committed history must not become fail_workflow");
10714
10715 let Error::NonDeterministicReplay(failure) = error else {
10716 panic!("expected typed replay failure");
10717 };
10718 assert_eq!(failure.reason, "recorded_commands_unconsumed");
10719 assert_eq!(failure.sequence, Some(1));
10720 assert_eq!(failure.expected.as_deref(), Some("side effect"));
10721 }
10722
10723 #[test]
10724 fn replay_error_discards_side_effect_queued_before_incompatible_marker_check() {
10725 let client = Client::new("http://127.0.0.1:8080").expect("client");
10726 let mut worker = Worker::new(client, "rust-workers");
10727 worker.register_workflow(
10728 "rust.side-effect-before-marker-error",
10729 |ctx, _input| async move {
10730 assert_eq!(ctx.get_version("restart-safe", 1, 1)?, 1);
10731 let _: String = ctx.side_effect(|| "must-not-commit".to_string())?;
10732 ctx.get_version("restart-safe", 2, 2)?;
10733 Ok(Value::Null)
10734 },
10735 );
10736
10737 let error = worker
10738 .execute_workflow_task(workflow_task(
10739 "rust.side-effect-before-marker-error",
10740 vec![history_event(
10741 "VersionMarkerRecorded",
10742 json!({
10743 "sequence": 1,
10744 "change_id": "restart-safe",
10745 "version": 1,
10746 "min_supported": 1,
10747 "max_supported": 1,
10748 }),
10749 )],
10750 JSON_CODEC,
10751 ))
10752 .expect_err("replay error must return no queued workflow commands");
10753
10754 let Error::NonDeterministicReplay(failure) = error else {
10755 panic!("expected typed replay failure");
10756 };
10757 assert_eq!(failure.reason, "version_marker_incompatible_range");
10758 assert_eq!(failure.sequence, Some(1));
10759 }
10760
10761 #[test]
10762 fn workflow_task_replay_keeps_recorded_unfired_timer_pending_without_rescheduling() {
10763 let client = Client::new("http://127.0.0.1:8080").expect("client");
10764 let mut worker = Worker::new(client, "rust-workers");
10765 worker.register_workflow("rust.timer.pending", |ctx, _input| async move {
10766 ctx.sleep(Duration::from_secs(5)).await?;
10767 Ok(json!({"status": "timer fired"}))
10768 });
10769
10770 let task = WorkflowTask {
10771 task_id: "wft-rust-timer-pending".to_string(),
10772 workflow_id: Some("wf-rust-timer".to_string()),
10773 run_id: Some("run-rust-timer".to_string()),
10774 workflow_type: "rust.timer.pending".to_string(),
10775 payload_codec: JSON_CODEC.to_string(),
10776 arguments: Some(json!({"codec": "json", "blob": "[]"})),
10777 history_events: vec![history_event(
10778 "TimerScheduled",
10779 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10780 )],
10781 total_history_events: Some(1),
10782 history_size_bytes: None,
10783 continue_as_new_recommended: None,
10784 history_budget_pressure: None,
10785 next_history_page_token: None,
10786 workflow_task_attempt: 1,
10787 workflow_signal_id: None,
10788 signal_name: None,
10789 signal_arguments: None,
10790 workflow_update_id: None,
10791 update_name: None,
10792 lease_owner: Some("rust-worker".to_string()),
10793 };
10794
10795 for _redelivery_or_restart in 0..2 {
10796 let commands = worker
10797 .execute_workflow_task(task.clone())
10798 .expect("recorded timer remains pending");
10799 assert!(
10800 commands.is_empty(),
10801 "recorded timer must not be rescheduled"
10802 );
10803 }
10804 }
10805
10806 #[test]
10807 fn workflow_task_rejects_recorded_command_removed_from_workflow_code() {
10808 let client = Client::new("http://127.0.0.1:8080").expect("client");
10809 let mut worker = Worker::new(client, "rust-workers");
10810 worker.register_workflow("rust.timer.removed", |_ctx, _input| async move {
10811 Ok(json!({"status": "completed"}))
10812 });
10813 let task = WorkflowTask {
10814 task_id: "wft-rust-timer-removed".to_string(),
10815 workflow_id: Some("wf-rust-timer".to_string()),
10816 run_id: Some("run-rust-timer".to_string()),
10817 workflow_type: "rust.timer.removed".to_string(),
10818 payload_codec: JSON_CODEC.to_string(),
10819 arguments: Some(json!({"codec": "json", "blob": "[]"})),
10820 history_events: vec![
10821 history_event(
10822 "TimerScheduled",
10823 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10824 ),
10825 history_event(
10826 "TimerFired",
10827 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10828 ),
10829 ],
10830 total_history_events: Some(2),
10831 history_size_bytes: None,
10832 continue_as_new_recommended: None,
10833 history_budget_pressure: None,
10834 next_history_page_token: None,
10835 workflow_task_attempt: 1,
10836 workflow_signal_id: None,
10837 signal_name: None,
10838 signal_arguments: None,
10839 workflow_update_id: None,
10840 update_name: None,
10841 lease_owner: Some("rust-worker".to_string()),
10842 };
10843
10844 let Error::NonDeterministicReplay(failure) = worker
10845 .execute_workflow_task(task)
10846 .expect_err("removed timer must fail replay")
10847 else {
10848 panic!("expected typed replay failure");
10849 };
10850 assert_eq!(failure.reason, "recorded_commands_unconsumed");
10851 assert_eq!(failure.sequence, Some(1));
10852 }
10853
10854 #[test]
10855 fn workflow_context_emits_explicit_child_workflow_contract() {
10856 let ctx = WorkflowContext {
10857 state: Arc::new(Mutex::new(
10858 WorkflowState::new_with_identity(
10859 Vec::new(),
10860 Some("wf-parent".to_string()),
10861 Some("run-parent".to_string()),
10862 "parent-workers".to_string(),
10863 JSON_CODEC.to_string(),
10864 None,
10865 )
10866 .expect("workflow state"),
10867 )),
10868 };
10869 let options = ChildWorkflowOptions::new("python-workers")
10870 .parent_close_policy(ParentClosePolicy::RequestCancel)
10871 .retry_policy(ChildWorkflowRetryPolicy {
10872 max_attempts: Some(3),
10873 backoff_seconds: vec![1, 5],
10874 non_retryable_error_types: vec!["ValidationError".to_string()],
10875 })
10876 .execution_timeout_seconds(600)
10877 .run_timeout_seconds(120);
10878 let mut call = Box::pin(ctx.start_child_workflow(
10879 "python.fulfil-order",
10880 options,
10881 json!([{"order_id": "order-42"}]),
10882 ));
10883 let mut task_context = TaskContext::from_waker(noop_waker_ref());
10884
10885 assert!(matches!(
10886 call.as_mut().poll(&mut task_context),
10887 Poll::Pending
10888 ));
10889 let commands = ctx.take_commands().expect("commands");
10890 assert_eq!(commands.len(), 1);
10891 let command = &commands[0];
10892 assert_eq!(command["type"], "start_child_workflow");
10893 assert_eq!(command["workflow_type"], "python.fulfil-order");
10894 assert_eq!(command["queue"], "python-workers");
10895 assert_eq!(command["parent_close_policy"], "request_cancel");
10896 assert_eq!(command["retry_policy"]["max_attempts"], 3);
10897 assert_eq!(command["execution_timeout_seconds"], 600);
10898 assert_eq!(command["run_timeout_seconds"], 120);
10899 assert_eq!(
10900 decode_wire_value(&command["arguments"], JSON_CODEC).expect("child args"),
10901 json!([{"order_id": "order-42"}])
10902 );
10903 }
10904
10905 fn child_parent_worker() -> Worker {
10906 let client = Client::new("http://127.0.0.1:8080").expect("client");
10907 let mut worker = Worker::new(client, "rust-parent-workers");
10908 worker.register_workflow("rust.parent", |ctx, _input| async move {
10909 let child = ctx
10910 .start_child_workflow(
10911 "python.child",
10912 ChildWorkflowOptions::new("python-child-workers")
10913 .parent_close_policy(ParentClosePolicy::Terminate),
10914 json!([{"codec_probe": [1, true, "rust"]}]),
10915 )
10916 .await?;
10917 Ok(json!({
10918 "parent_workflow_id": child.parent.workflow_id,
10919 "parent_run_id": child.parent.run_id,
10920 "child_workflow_id": child.child.workflow_id,
10921 "child_run_id": child.child.run_id,
10922 "child_workflow_type": child.child_workflow_type,
10923 "result": child.result,
10924 }))
10925 });
10926 worker
10927 }
10928
10929 fn child_parent_task(event_type: &str, payload: Value) -> WorkflowTask {
10930 WorkflowTask {
10931 task_id: "wft-child-parent".to_string(),
10932 workflow_id: Some("wf-parent".to_string()),
10933 run_id: Some("run-parent".to_string()),
10934 workflow_type: "rust.parent".to_string(),
10935 payload_codec: JSON_CODEC.to_string(),
10936 arguments: Some(encode_value_envelope(&json!([]), JSON_CODEC).expect("input")),
10937 history_events: vec![
10938 HistoryEvent {
10939 event_type: "ChildWorkflowScheduled".to_string(),
10940 payload: json!({
10941 "sequence": 1,
10942 "child_call_id": "call-child",
10943 "child_workflow_instance_id": "wf-child",
10944 "child_workflow_run_id": "run-child",
10945 "child_workflow_type": "python.child",
10946 }),
10947 raw: HashMap::new(),
10948 },
10949 HistoryEvent {
10950 event_type: event_type.to_string(),
10951 payload,
10952 raw: HashMap::new(),
10953 },
10954 ],
10955 total_history_events: Some(2),
10956 history_size_bytes: None,
10957 continue_as_new_recommended: None,
10958 history_budget_pressure: None,
10959 next_history_page_token: None,
10960 workflow_task_attempt: 1,
10961 workflow_signal_id: None,
10962 signal_name: None,
10963 signal_arguments: None,
10964 workflow_update_id: None,
10965 update_name: None,
10966 lease_owner: Some("rust-worker".to_string()),
10967 }
10968 }
10969
10970 #[test]
10971 fn committed_child_result_replays_without_starting_a_duplicate() {
10972 let worker = child_parent_worker();
10973 let task = child_parent_task(
10974 "ChildRunCompleted",
10975 json!({
10976 "sequence": 1,
10977 "child_call_id": "call-child",
10978 "child_workflow_instance_id": "wf-child",
10979 "child_workflow_run_id": "run-child",
10980 "child_workflow_type": "python.child",
10981 "payload_codec": "json",
10982 "result": {"codec": "json", "blob": "{\"from\":\"python\",\"ok\":true}"},
10983 }),
10984 );
10985
10986 for _restart in 0..2 {
10987 let commands = worker
10988 .execute_workflow_task(task.clone())
10989 .expect("replayed parent task");
10990 assert_eq!(commands.len(), 1);
10991 assert_eq!(commands[0]["type"], "complete_workflow");
10992 assert!(!commands
10993 .iter()
10994 .any(|command| command["type"] == "start_child_workflow"));
10995 let output =
10996 decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("parent output");
10997 assert_eq!(output["parent_workflow_id"], "wf-parent");
10998 assert_eq!(output["parent_run_id"], "run-parent");
10999 assert_eq!(output["child_workflow_id"], "wf-child");
11000 assert_eq!(output["child_run_id"], "run-child");
11001 assert_eq!(output["result"], json!({"from": "python", "ok": true}));
11002 }
11003 }
11004
11005 #[test]
11006 fn typed_child_arguments_and_results_survive_replay() {
11007 let client = Client::new("http://127.0.0.1:8080").expect("client");
11008 let mut worker = Worker::new(client, "rust-parent-workers");
11009 worker.register_workflow_avro_value("rust.typed-parent", |ctx, _input| async move {
11010 let child = ctx
11011 .start_child_workflow_avro_value(
11012 "python.typed-child",
11013 ChildWorkflowOptions::new("python-workers"),
11014 AvroValue::Array(vec![typed_fidelity_probe()]),
11015 )
11016 .await?;
11017 Ok(child.result)
11018 });
11019
11020 let initial = worker
11021 .execute_workflow_task(workflow_task(
11022 "rust.typed-parent",
11023 Vec::new(),
11024 DEFAULT_CODEC,
11025 ))
11026 .expect("typed child start");
11027 assert_eq!(initial[0]["type"], "start_child_workflow");
11028 assert_eq!(
11029 decode_wire_avro_value(&initial[0]["arguments"], DEFAULT_CODEC)
11030 .expect("typed child arguments"),
11031 AvroValue::Array(vec![typed_fidelity_probe()])
11032 );
11033
11034 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
11035 .expect("typed child result");
11036 let task = workflow_task(
11037 "rust.typed-parent",
11038 vec![
11039 history_event(
11040 "ChildWorkflowScheduled",
11041 json!({
11042 "sequence": 1,
11043 "child_call_id": "call-typed",
11044 "child_workflow_instance_id": "wf-child",
11045 "child_workflow_run_id": "run-child",
11046 "child_workflow_type": "python.typed-child",
11047 }),
11048 ),
11049 history_event(
11050 "ChildRunCompleted",
11051 json!({
11052 "sequence": 1,
11053 "child_call_id": "call-typed",
11054 "child_workflow_instance_id": "wf-child",
11055 "child_workflow_run_id": "run-child",
11056 "child_workflow_type": "python.typed-child",
11057 "payload_codec": DEFAULT_CODEC,
11058 "result": result,
11059 }),
11060 ),
11061 ],
11062 DEFAULT_CODEC,
11063 );
11064
11065 let commands = worker
11066 .execute_workflow_task(task)
11067 .expect("typed child replay");
11068 assert_eq!(commands[0]["type"], "complete_workflow");
11069 assert_eq!(
11070 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
11071 .expect("typed parent result"),
11072 typed_fidelity_probe()
11073 );
11074 }
11075
11076 #[test]
11077 fn pending_child_replays_after_restart_without_starting_a_duplicate() {
11078 let worker = child_parent_worker();
11079 let mut task = child_parent_task("unused", Value::Null);
11080 task.history_events.truncate(1);
11081 task.total_history_events = Some(1);
11082
11083 for _redelivery_or_restart in 0..2 {
11084 let commands = worker
11085 .execute_workflow_task(task.clone())
11086 .expect("recorded child remains pending");
11087 assert!(
11088 commands.is_empty(),
11089 "recorded pending child must not be started again"
11090 );
11091 }
11092 }
11093
11094 #[test]
11095 fn child_cancellation_becomes_stable_parent_failure_command() {
11096 let worker = child_parent_worker();
11097 let task = child_parent_task(
11098 "ChildRunCancelled",
11099 json!({
11100 "sequence": 1,
11101 "child_workflow_instance_id": "wf-child",
11102 "child_workflow_run_id": "run-child",
11103 "child_workflow_type": "python.child",
11104 "failure_id": "failure-child",
11105 "failure_category": "cancelled",
11106 "message": "cancelled by parent-close policy",
11107 }),
11108 );
11109
11110 let commands = worker
11111 .execute_workflow_task(task)
11112 .expect("parent settlement");
11113 assert_eq!(commands.len(), 1);
11114 assert_eq!(commands[0]["type"], "fail_workflow");
11115 assert_eq!(commands[0]["exception_type"], "ChildWorkflowCancelled");
11116 assert_eq!(
11117 commands[0]["exception"]["properties"]["reason"],
11118 "cancelled"
11119 );
11120 assert_eq!(
11121 commands[0]["exception"]["properties"]["child_workflow_run_id"],
11122 "run-child"
11123 );
11124 }
11125
11126 #[test]
11127 fn workflow_can_handle_typed_child_failure() {
11128 let client = Client::new("http://127.0.0.1:8080").expect("client");
11129 let mut worker = Worker::new(client, "rust-parent-workers");
11130 worker.register_workflow("rust.handled-parent", |ctx, _input| async move {
11131 match ctx
11132 .start_child_workflow(
11133 "python.child",
11134 ChildWorkflowOptions::new("python-child-workers"),
11135 json!([]),
11136 )
11137 .await
11138 {
11139 Err(Error::ChildWorkflowFailed(failure)) => Ok(json!({
11140 "reason": failure.reason,
11141 "failure_id": failure.failure_id,
11142 "exception_class": failure.exception_class,
11143 "child_run_id": failure.child_workflow_run_id,
11144 })),
11145 Err(error) => Err(error),
11146 Ok(_) => Err(Error::WorkerLoop(
11147 "child unexpectedly succeeded".to_string(),
11148 )),
11149 }
11150 });
11151 let mut task = child_parent_task(
11152 "ChildRunFailed",
11153 json!({
11154 "sequence": 1,
11155 "child_workflow_instance_id": "wf-child",
11156 "child_workflow_run_id": "run-child",
11157 "child_workflow_type": "python.child",
11158 "failure_id": "failure-child",
11159 "failure_category": "child_workflow",
11160 "message": "payment rejected",
11161 "exception": {
11162 "type": "PaymentRejected",
11163 "class": "payments.PaymentRejected",
11164 "message": "payment rejected"
11165 }
11166 }),
11167 );
11168 task.workflow_type = "rust.handled-parent".to_string();
11169
11170 let commands = worker.execute_workflow_task(task).expect("handled failure");
11171 assert_eq!(commands[0]["type"], "complete_workflow");
11172 let output = decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("parent output");
11173 assert_eq!(output["reason"], "child_workflow");
11174 assert_eq!(output["failure_id"], "failure-child");
11175 assert_eq!(output["exception_class"], "payments.PaymentRejected");
11176 assert_eq!(output["child_run_id"], "run-child");
11177 }
11178
11179 #[test]
11180 fn rust_hello_world_uses_signal_arguments_from_resume_payload() {
11181 let client = Client::new("http://127.0.0.1:8080").expect("client");
11182 let mut worker = Worker::new(client, "rust-workers");
11183
11184 worker.register_workflow("rust.hello_workflow", |ctx, _input| async move {
11185 let signal = ctx.wait_signal("start").await?;
11186 let name = signal
11187 .first()
11188 .and_then(|value| value.as_str())
11189 .unwrap_or("world");
11190 let greeting = ctx.activity("rust.hello_activity", json!([name])).await?;
11191 Ok(json!({
11192 "greeting": greeting,
11193 "language": "rust"
11194 }))
11195 });
11196
11197 let signal_arguments =
11198 encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC).expect("signal arguments");
11199 let task = WorkflowTask {
11200 task_id: "wft-rust-signal-1".to_string(),
11201 workflow_id: Some("wf-rust-hello".to_string()),
11202 run_id: Some("run-rust-hello".to_string()),
11203 workflow_type: "rust.hello_workflow".to_string(),
11204 payload_codec: DEFAULT_CODEC.to_string(),
11205 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
11206 history_events: vec![HistoryEvent {
11207 event_type: "SignalReceived".to_string(),
11208 payload: json!({
11209 "signal_id": "sig-rust-1",
11210 "signal_name": "start"
11211 }),
11212 raw: HashMap::new(),
11213 }],
11214 total_history_events: Some(1),
11215 history_size_bytes: None,
11216 continue_as_new_recommended: None,
11217 history_budget_pressure: None,
11218 next_history_page_token: None,
11219 workflow_task_attempt: 1,
11220 workflow_signal_id: Some("sig-rust-1".to_string()),
11221 signal_name: Some("start".to_string()),
11222 signal_arguments: Some(signal_arguments),
11223 workflow_update_id: None,
11224 update_name: None,
11225 lease_owner: Some("rust-worker".to_string()),
11226 };
11227
11228 let commands = worker.execute_workflow_task(task).expect("workflow task");
11229
11230 assert_eq!(commands.len(), 1);
11231 assert_eq!(commands[0]["type"], "schedule_activity");
11232 assert_eq!(commands[0]["activity_type"], "rust.hello_activity");
11233 assert_eq!(
11234 decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC).expect("activity args"),
11235 json!(["Rust"])
11236 );
11237 }
11238
11239 #[test]
11240 fn workflow_task_appends_paginated_history_events() {
11241 let mut task = WorkflowTask {
11242 task_id: "wft-rust-pages-1".to_string(),
11243 workflow_id: Some("wf-rust-pages".to_string()),
11244 run_id: Some("run-rust-pages".to_string()),
11245 workflow_type: "rust.hello_workflow".to_string(),
11246 payload_codec: DEFAULT_CODEC.to_string(),
11247 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
11248 history_events: vec![HistoryEvent {
11249 event_type: "WorkflowStarted".to_string(),
11250 payload: json!({}),
11251 raw: HashMap::new(),
11252 }],
11253 total_history_events: Some(3),
11254 history_size_bytes: None,
11255 continue_as_new_recommended: None,
11256 history_budget_pressure: None,
11257 next_history_page_token: Some("MQ==".to_string()),
11258 workflow_task_attempt: 1,
11259 workflow_signal_id: None,
11260 signal_name: None,
11261 signal_arguments: None,
11262 workflow_update_id: None,
11263 update_name: None,
11264 lease_owner: Some("rust-worker".to_string()),
11265 };
11266
11267 task.append_history_page(WorkflowTaskHistoryPage {
11268 history_events: vec![
11269 HistoryEvent {
11270 event_type: "SignalReceived".to_string(),
11271 payload: json!({
11272 "signal_id": "sig-rust-1",
11273 "signal_name": "start",
11274 "arguments": encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC)
11275 .expect("signal arguments")
11276 }),
11277 raw: HashMap::new(),
11278 },
11279 HistoryEvent {
11280 event_type: "MarkerRecorded".to_string(),
11281 payload: json!({"sequence": 3}),
11282 raw: HashMap::new(),
11283 },
11284 ],
11285 total_history_events: Some(3),
11286 next_history_page_token: None,
11287 });
11288
11289 assert_eq!(task.history_events.len(), 3);
11290 assert_eq!(task.total_history_events, Some(3));
11291 assert_eq!(task.next_history_page_token, None);
11292
11293 let signal = task
11294 .history_events
11295 .iter()
11296 .find(|event| event.event_type == "SignalReceived")
11297 .expect("signal event");
11298 assert_eq!(
11299 decode_signal_event_arguments(signal, DEFAULT_CODEC).expect("signal arguments"),
11300 vec![AvroValue::String("Rust".to_string())]
11301 );
11302 }
11303
11304 #[tokio::test]
11305 async fn query_handler_reads_ordered_cross_codec_signals_without_commands() {
11306 let client = Client::new("http://127.0.0.1:8080").expect("client");
11307 let mut worker = Worker::new(client, "rust-workers");
11308 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
11309 worker.register_query("counter", "current", |ctx, _args| async move {
11310 let mut count = 0_i64;
11311 for signal in ctx.signal_events() {
11312 let value = signal
11313 .arguments
11314 .first()
11315 .and_then(Value::as_i64)
11316 .unwrap_or_default();
11317 match signal.name.as_str() {
11318 "increment" => count += value,
11319 "set" => count = value,
11320 _ => {}
11321 }
11322 }
11323 Ok(json!(count))
11324 });
11325
11326 let task = QueryTask {
11327 query_task_id: "query-rust-counter".to_string(),
11328 query_task_attempt: 1,
11329 lease_owner: Some("rust-worker".to_string()),
11330 workflow_id: Some("counter-1".to_string()),
11331 run_id: Some("run-counter-1".to_string()),
11332 workflow_type: "counter".to_string(),
11333 query_name: "current".to_string(),
11334 payload_codec: DEFAULT_CODEC.to_string(),
11335 workflow_arguments: Some(
11336 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
11337 ),
11338 query_arguments: Some(
11339 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("query arguments"),
11340 ),
11341 history_events: vec![
11342 HistoryEvent {
11343 event_type: "SignalReceived".to_string(),
11344 payload: json!({
11345 "signal_id": "php-signal-1",
11346 "signal_name": "increment",
11347 "workflow_sequence": 1,
11348 "payload_codec": DEFAULT_CODEC,
11349 "arguments": encode_value_envelope(&json!([3]), DEFAULT_CODEC).expect("php avro signal")
11350 }),
11351 raw: HashMap::new(),
11352 },
11353 HistoryEvent {
11354 event_type: "SignalReceived".to_string(),
11355 payload: json!({
11356 "signal_id": "python-signal-2",
11357 "signal_name": "increment",
11358 "workflow_sequence": 2,
11359 "payload_codec": JSON_CODEC,
11360 "arguments": encode_value_envelope(&json!([5]), JSON_CODEC).expect("python json signal")
11361 }),
11362 raw: HashMap::new(),
11363 },
11364 HistoryEvent {
11365 event_type: "SignalReceived".to_string(),
11366 payload: json!({
11367 "signal_id": "rust-signal-3",
11368 "signal_name": "set",
11369 "workflow_sequence": 3,
11370 "payload_codec": DEFAULT_CODEC,
11371 "arguments": encode_value_envelope(&json!([0]), DEFAULT_CODEC).expect("rust avro signal")
11372 }),
11373 raw: HashMap::new(),
11374 },
11375 ],
11376 history_export: None,
11377 run_status: Some("completed".to_string()),
11378 };
11379
11380 let result = worker.execute_query_task(task).await.expect("query result");
11381 assert_eq!(result.into_json().expect("query projection"), json!(0));
11382 }
11383
11384 #[tokio::test]
11385 async fn replayed_queries_read_running_completed_and_cold_restarted_instance_state() {
11386 let worker = replay_counter_worker();
11387 let running_history = json!([
11388 {
11389 "type": "ActivityCompleted",
11390 "payload": {
11391 "sequence": 1,
11392 "activity_type": "load-counter",
11393 "payload_codec": "json",
11394 "result": {"codec": "json", "blob": "\"loaded\""}
11395 }
11396 },
11397 {
11398 "type": "SignalWaitOpened",
11399 "payload": {
11400 "sequence": 3,
11401 "signal_name": "increment"
11402 }
11403 },
11404 {
11405 "type": "SignalReceived",
11406 "payload": {
11407 "signal_id": "signal-3",
11408 "signal_name": "increment",
11409 "workflow_sequence": 2,
11410 "payload_codec": "json",
11411 "arguments": {"codec": "json", "blob": "[3]"}
11412 }
11413 },
11414 {
11415 "type": "SignalApplied",
11416 "payload": {
11417 "sequence": 3,
11418 "signal_id": "signal-3",
11419 "signal_name": "increment",
11420 "payload_codec": "json",
11421 "value": {"codec": "json", "blob": "[3]"}
11422 }
11423 }
11424 ]);
11425
11426 let running = worker
11427 .execute_query_task(replay_counter_query(
11428 "current",
11429 running_history.clone(),
11430 "running",
11431 ))
11432 .await
11433 .expect("running replay query");
11434 assert_eq!(
11435 running.clone().into_json().expect("query projection"),
11436 json!({"loaded": "loaded", "count": 3, "finished": false})
11437 );
11438
11439 let detached = worker
11440 .execute_query_task(replay_counter_query(
11441 "detached-mutation",
11442 running_history.clone(),
11443 "running",
11444 ))
11445 .await
11446 .expect("query mutates only its detached state clone");
11447 assert_eq!(detached.into_json().expect("query projection"), json!(999));
11448 let failed = worker
11449 .execute_query_task(replay_counter_query(
11450 "failed-mutation",
11451 running_history.clone(),
11452 "running",
11453 ))
11454 .await
11455 .expect_err("failed query");
11456 assert_eq!(failed.reason, "query_rejected");
11457 let unchanged = worker
11458 .execute_query_task(replay_counter_query("current", running_history, "running"))
11459 .await
11460 .expect("later query reconstructs unchanged state");
11461 assert_eq!(unchanged, running);
11462
11463 let restarted_worker = replay_counter_worker();
11464 let restarted_task: QueryTask = serde_json::from_value(json!({
11465 "query_task_id": "query-after-restart",
11466 "workflow_id": "counter-1",
11467 "run_id": "run-counter-1",
11468 "workflow_type": "replay-counter",
11469 "query_name": "current",
11470 "payload_codec": "json",
11471 "workflow_arguments": {"codec": "json", "blob": "[]"},
11472 "query_arguments": {"codec": "json", "blob": "[]"},
11473 "history_events": [],
11474 "history_export": {
11475 "payloads": {"codec": "json"},
11476 "history_events": [
11477 {
11478 "type": "ActivityCompleted",
11479 "payload": {
11480 "sequence": 1,
11481 "activity_type": "load-counter",
11482 "payload_codec": "json",
11483 "result": null
11484 }
11485 },
11486 {
11487 "type": "SignalWaitOpened",
11488 "payload": {
11489 "sequence": 3,
11490 "signal_name": "increment"
11491 }
11492 },
11493 {
11494 "type": "SignalReceived",
11495 "payload": {
11496 "signal_id": "signal-3",
11497 "signal_name": "increment",
11498 "workflow_sequence": 2
11499 }
11500 },
11501 {
11502 "type": "SignalApplied",
11503 "payload": {
11504 "sequence": 3,
11505 "signal_id": "signal-3",
11506 "signal_name": "increment"
11507 }
11508 },
11509 {
11510 "type": "SignalWaitOpened",
11511 "payload": {
11512 "sequence": 5,
11513 "signal_name": "increment"
11514 }
11515 },
11516 {
11517 "type": "SignalReceived",
11518 "payload": {
11519 "signal_id": "signal-5",
11520 "signal_name": "increment",
11521 "workflow_sequence": 4
11522 }
11523 },
11524 {
11525 "type": "SignalApplied",
11526 "payload": {
11527 "sequence": 5,
11528 "signal_id": "signal-5",
11529 "signal_name": "increment"
11530 }
11531 }
11532 ],
11533 "activities": [{
11534 "sequence": 1,
11535 "activity_type": "load-counter",
11536 "payload_codec": "json",
11537 "result": {"codec": "json", "blob": "\"loaded\""}
11538 }],
11539 "signals": [
11540 {
11541 "id": "signal-3",
11542 "name": "increment",
11543 "workflow_sequence": 2,
11544 "payload_codec": "json",
11545 "arguments": "[3]"
11546 },
11547 {
11548 "id": "signal-5",
11549 "name": "increment",
11550 "workflow_sequence": 4,
11551 "payload_codec": "json",
11552 "arguments": "[5]"
11553 }
11554 ]
11555 },
11556 "run_status": "completed"
11557 }))
11558 .expect("cold replay query task");
11559 let completed = restarted_worker
11560 .execute_query_task(restarted_task)
11561 .await
11562 .expect("completed cold replay query");
11563 assert_eq!(
11564 completed.into_json().expect("query projection"),
11565 json!({"loaded": "loaded", "count": 8, "finished": true})
11566 );
11567 }
11568
11569 #[tokio::test]
11570 async fn replayed_query_replay_failures_are_machine_readable() {
11571 let worker = replay_counter_worker();
11572 let task = replay_counter_query(
11573 "current",
11574 json!([{
11575 "type": "ActivityCompleted",
11576 "payload": {
11577 "sequence": 1,
11578 "payload_codec": "json",
11579 "result": {"codec": "json", "blob": "{"}
11580 }
11581 }]),
11582 "running",
11583 );
11584 let failure = worker
11585 .execute_query_task(task)
11586 .await
11587 .expect_err("invalid replay history payload");
11588 assert_eq!(failure.reason, "query_workflow_state_unavailable");
11589 assert_eq!(failure.failure_type, "QueryWorkflowStateUnavailable");
11590 }
11591
11592 #[tokio::test]
11593 async fn query_task_restores_compact_history_from_export() {
11594 let client = Client::new("http://127.0.0.1:8080").expect("client");
11595 let mut worker = Worker::new(client, "rust-workers");
11596 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
11597 worker.register_query("counter", "current", |ctx, _args| async move {
11598 Ok(json!(ctx.signals("increment")[0][0]))
11599 });
11600 let task: QueryTask = serde_json::from_value(json!({
11601 "query_task_id": "query-export",
11602 "workflow_type": "counter",
11603 "query_name": "current",
11604 "payload_codec": "json",
11605 "workflow_arguments": {"codec": "json", "blob": "[]"},
11606 "query_arguments": {"codec": "json", "blob": "[]"},
11607 "history_events": [],
11608 "history_export": {
11609 "payloads": {"codec": "json"},
11610 "history_events": [{
11611 "type": "SignalReceived",
11612 "payload": {"signal_id": "signal-export", "signal_name": "increment"}
11613 }],
11614 "signals": [{
11615 "id": "signal-export",
11616 "name": "increment",
11617 "status": "applied",
11618 "workflow_sequence": 1,
11619 "payload_codec": "json",
11620 "arguments": "[9]"
11621 }]
11622 }
11623 }))
11624 .expect("query task");
11625
11626 let result = worker.execute_query_task(task).await.expect("query result");
11627 assert_eq!(result.into_json().expect("query projection"), json!(9));
11628 }
11629
11630 #[tokio::test]
11631 async fn query_task_failures_have_stable_reasons() {
11632 let client = Client::new("http://127.0.0.1:8080").expect("client");
11633 let mut worker = Worker::new(client, "rust-workers");
11634 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
11635 worker.register_query(
11636 "counter",
11637 "current",
11638 |_ctx, _args| async move { Ok(json!(0)) },
11639 );
11640
11641 let base_task = QueryTask {
11642 query_task_id: "query-errors".to_string(),
11643 query_task_attempt: 1,
11644 lease_owner: None,
11645 workflow_id: Some("counter-errors".to_string()),
11646 run_id: Some("run-errors".to_string()),
11647 workflow_type: "counter".to_string(),
11648 query_name: "missing".to_string(),
11649 payload_codec: JSON_CODEC.to_string(),
11650 workflow_arguments: Some(json!({"codec": "json", "blob": "[]"})),
11651 query_arguments: Some(json!({"codec": "json", "blob": "[]"})),
11652 history_events: Vec::new(),
11653 history_export: None,
11654 run_status: Some("running".to_string()),
11655 };
11656
11657 let unknown = worker
11658 .execute_query_task(base_task.clone())
11659 .await
11660 .expect_err("unknown query");
11661 assert_eq!(unknown.reason, "rejected_unknown_query");
11662
11663 let mut malformed = base_task;
11664 malformed.query_name = "current".to_string();
11665 malformed.query_arguments = Some(json!({"codec": "json", "blob": "{"}));
11666 let malformed = worker
11667 .execute_query_task(malformed)
11668 .await
11669 .expect_err("malformed payload");
11670 assert_eq!(malformed.reason, "query_payload_decode_failed");
11671
11672 let client = Client::new("http://127.0.0.1:8080").expect("client");
11673 let mut unavailable_worker = Worker::new(client, "rust-workers");
11674 unavailable_worker
11675 .register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
11676 let unavailable_task: QueryTask = serde_json::from_value(json!({
11677 "query_task_id": "query-unavailable",
11678 "workflow_type": "counter",
11679 "query_name": "current",
11680 "payload_codec": "json",
11681 "workflow_arguments": {"codec": "json", "blob": "[]"},
11682 "query_arguments": {"codec": "json", "blob": "[]"}
11683 }))
11684 .expect("query task");
11685 let unavailable = unavailable_worker
11686 .execute_query_task(unavailable_task)
11687 .await
11688 .expect_err("query handler unavailable");
11689 assert_eq!(unavailable.reason, "query_handler_unavailable");
11690 }
11691
11692 #[tokio::test]
11693 async fn client_query_decodes_result_and_typed_failure() {
11694 let server = MockWorkerServer::start();
11695 let client = Client::builder(server.base_url())
11696 .timeout(Duration::from_secs(2))
11697 .build()
11698 .expect("client");
11699
11700 let result = client
11701 .query_workflow("counter-1", "current", json!([]))
11702 .await
11703 .expect("query result");
11704 assert_eq!(result, json!({"count": 8}));
11705
11706 let error = client
11707 .query_workflow("counter-1", "missing", json!([]))
11708 .await
11709 .expect_err("unknown query");
11710 let Error::QueryFailed(failure) = error else {
11711 panic!("expected typed query failure");
11712 };
11713 assert_eq!(failure.status, 404);
11714 assert_eq!(failure.reason, "rejected_unknown_query");
11715 }
11716
11717 #[tokio::test]
11718 async fn public_client_surfaces_send_and_receive_lossless_avro_values() {
11719 let server = MockWorkerServer::start();
11720 let client = Client::builder(server.base_url())
11721 .timeout(Duration::from_secs(2))
11722 .build()
11723 .expect("client");
11724 let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
11725
11726 client
11727 .start_workflow(
11728 "typed.echo",
11729 "rust-workers",
11730 "typed-start",
11731 arguments.clone(),
11732 )
11733 .await
11734 .expect("typed workflow start");
11735 assert_eq!(
11736 decode_wire_avro_value(
11737 &server.request_body("/api/workflows")["input"],
11738 DEFAULT_CODEC,
11739 )
11740 .expect("typed start input"),
11741 arguments
11742 );
11743
11744 client
11745 .signal_workflow("typed-1", "changed", arguments.clone())
11746 .await
11747 .expect("typed signal");
11748 assert_eq!(
11749 decode_wire_avro_value(
11750 &server.request_body("/api/workflows/typed-1/signal/changed")["input"],
11751 DEFAULT_CODEC,
11752 )
11753 .expect("typed signal input"),
11754 arguments
11755 );
11756
11757 assert_eq!(
11758 client
11759 .query_workflow_avro_value("typed-1", "inspect", arguments.clone())
11760 .await
11761 .expect("typed query"),
11762 typed_fidelity_probe()
11763 );
11764 assert_eq!(
11765 decode_wire_avro_value(
11766 &server.request_body("/api/workflows/typed-1/query/inspect")["input"],
11767 DEFAULT_CODEC,
11768 )
11769 .expect("typed query input"),
11770 arguments
11771 );
11772
11773 assert_eq!(
11774 client
11775 .update_workflow_avro_value(
11776 "typed-1",
11777 "replace",
11778 arguments.clone(),
11779 Some("typed-request"),
11780 )
11781 .await
11782 .expect("typed update"),
11783 typed_fidelity_probe()
11784 );
11785 let update = server.request_body("/api/workflows/typed-1/update/replace");
11786 assert_eq!(update["request_id"], "typed-request");
11787 assert_eq!(
11788 decode_wire_avro_value(&update["input"], DEFAULT_CODEC).expect("typed update input"),
11789 arguments
11790 );
11791
11792 let handle = WorkflowHandle {
11793 client: client.clone(),
11794 workflow_id: "typed-1".to_string(),
11795 run_id: Some("run-typed-1".to_string()),
11796 workflow_type: "typed.echo".to_string(),
11797 };
11798 assert_eq!(
11799 handle
11800 .result_avro_value(WorkflowResultOptions::default())
11801 .await
11802 .expect("typed workflow result"),
11803 typed_fidelity_probe()
11804 );
11805
11806 client
11807 .complete_activity_task(
11808 "activity-typed",
11809 "attempt-typed",
11810 "rust-worker",
11811 typed_fidelity_probe(),
11812 DEFAULT_CODEC,
11813 )
11814 .await
11815 .expect("typed activity completion");
11816 assert_eq!(
11817 decode_wire_avro_value(
11818 &server.request_body("/api/worker/activity-tasks/activity-typed/complete")
11819 ["result"],
11820 DEFAULT_CODEC,
11821 )
11822 .expect("typed activity result"),
11823 typed_fidelity_probe()
11824 );
11825 client
11826 .fail_activity_task(
11827 "activity-typed",
11828 "attempt-typed",
11829 "rust-worker",
11830 "typed failure",
11831 true,
11832 )
11833 .await
11834 .expect("activity failure");
11835 }
11836
11837 #[tokio::test]
11838 async fn lifecycle_commands_support_instance_and_selected_run_targets() {
11839 let server = MockWorkerServer::start();
11840 let client = Client::builder(server.base_url())
11841 .timeout(Duration::from_secs(2))
11842 .build()
11843 .expect("client");
11844
11845 let options = WorkflowCommandOptions::new()
11846 .reason("cleanup requested")
11847 .request_id("cancel-17");
11848 let cancelled = client
11849 .cancel_workflow("wf-lifecycle", options)
11850 .await
11851 .expect("instance cancellation");
11852 assert_eq!(cancelled.command, WorkflowCommandKind::Cancel);
11853 assert_eq!(cancelled.run_id.as_deref(), Some("run-current"));
11854 assert_eq!(cancelled.outcome.as_deref(), Some("cancelled"));
11855 assert_eq!(
11856 server.request_body("/api/workflows/wf-lifecycle/cancel"),
11857 json!({"reason":"cleanup requested","request_id":"cancel-17"})
11858 );
11859
11860 let terminated = client
11861 .terminate_workflow(
11862 "wf-lifecycle",
11863 WorkflowCommandOptions::new().reason("forced stop"),
11864 )
11865 .await
11866 .expect("instance termination");
11867 assert_eq!(terminated.command, WorkflowCommandKind::Terminate);
11868 assert_eq!(terminated.outcome.as_deref(), Some("terminated"));
11869
11870 client
11871 .cancel_workflow_run(
11872 "wf-lifecycle",
11873 "run-current",
11874 WorkflowCommandOptions::default(),
11875 )
11876 .await
11877 .expect("selected run cancellation");
11878 client
11879 .terminate_workflow_run(
11880 "wf-lifecycle",
11881 "run-current",
11882 WorkflowCommandOptions::default(),
11883 )
11884 .await
11885 .expect("selected run termination");
11886
11887 for (command, error) in [
11888 (
11889 WorkflowCommandKind::Cancel,
11890 client
11891 .cancel_workflow_run(
11892 "wf-lifecycle",
11893 "run-stale",
11894 WorkflowCommandOptions::default(),
11895 )
11896 .await
11897 .expect_err("stale cancellation must be rejected"),
11898 ),
11899 (
11900 WorkflowCommandKind::Terminate,
11901 client
11902 .terminate_workflow_run(
11903 "wf-lifecycle",
11904 "run-stale",
11905 WorkflowCommandOptions::default(),
11906 )
11907 .await
11908 .expect_err("stale termination must be rejected"),
11909 ),
11910 ] {
11911 let Error::WorkflowCommandRejected(rejection) = error else {
11912 panic!("expected typed command rejection");
11913 };
11914 assert_eq!(rejection.command, command);
11915 assert_eq!(rejection.status, 409);
11916 assert_eq!(rejection.reason, "historical_run_command_rejected");
11917 assert_eq!(rejection.run_id.as_deref(), Some("run-stale"));
11918 assert_eq!(rejection.target_scope.as_deref(), Some("run"));
11919 }
11920 }
11921
11922 #[tokio::test]
11923 async fn workflow_start_options_send_server_enforced_deadlines() {
11924 let server = MockWorkerServer::start();
11925 let client = Client::builder(server.base_url())
11926 .timeout(Duration::from_secs(2))
11927 .build()
11928 .expect("client");
11929
11930 let handle = client
11931 .start_workflow_with_options(
11932 "rust.timeout",
11933 "rust-timeouts",
11934 "wf-start-options",
11935 WorkflowStartOptions::new()
11936 .execution_timeout_seconds(30)
11937 .run_timeout_seconds(1),
11938 json!([]),
11939 )
11940 .await
11941 .expect("workflow start");
11942
11943 assert_eq!(handle.run_id.as_deref(), Some("run-start-options"));
11944 let body = server.request_body("/api/workflows");
11945 assert_eq!(body["execution_timeout_seconds"], 30);
11946 assert_eq!(body["run_timeout_seconds"], 1);
11947
11948 let invalid = client
11949 .start_workflow_with_options(
11950 "rust.timeout",
11951 "rust-timeouts",
11952 "wf-invalid-options",
11953 WorkflowStartOptions::new()
11954 .execution_timeout_seconds(1)
11955 .run_timeout_seconds(2),
11956 json!([]),
11957 )
11958 .await
11959 .expect_err("invalid deadline ordering");
11960 assert!(invalid
11961 .to_string()
11962 .contains("run_timeout_seconds cannot exceed execution_timeout_seconds"));
11963 }
11964
11965 #[tokio::test]
11966 async fn workflow_result_returns_each_typed_terminal_outcome() {
11967 let server = MockWorkerServer::start();
11968 let client = Client::builder(server.base_url())
11969 .timeout(Duration::from_secs(2))
11970 .build()
11971 .expect("client");
11972 let options = WorkflowResultOptions {
11973 poll_interval: Duration::ZERO,
11974 timeout: Duration::from_secs(1),
11975 };
11976
11977 let failed = WorkflowHandle {
11978 client: client.clone(),
11979 workflow_id: "wf-failed".to_string(),
11980 run_id: Some("run-failed".to_string()),
11981 workflow_type: "failure".to_string(),
11982 }
11983 .result(options)
11984 .await
11985 .expect_err("failed outcome");
11986 let Error::WorkflowFailed(failure) = failed else {
11987 panic!("expected WorkflowFailed");
11988 };
11989 assert_eq!(failure.workflow_id, "wf-failed");
11990 assert_eq!(failure.run_id.as_deref(), Some("run-failed"));
11991 assert_eq!(failure.failure_id.as_deref(), Some("failure-17"));
11992 assert_eq!(failure.failure_category.as_deref(), Some("application"));
11993 assert_eq!(failure.exception_type.as_deref(), Some("PaymentError"));
11994 assert_eq!(
11995 failure.exception_class.as_deref(),
11996 Some("billing::PaymentError")
11997 );
11998 assert_eq!(failure.non_retryable, Some(true));
11999
12000 for (workflow_id, expected_kind, expected_reason) in [
12001 (
12002 "wf-cancelled",
12003 WorkflowTerminalKind::Cancelled,
12004 "cleanup requested",
12005 ),
12006 (
12007 "wf-terminated",
12008 WorkflowTerminalKind::Terminated,
12009 "forced stop",
12010 ),
12011 (
12012 "wf-timed-out",
12013 WorkflowTerminalKind::TimedOut,
12014 "run_timeout",
12015 ),
12016 ] {
12017 let error = WorkflowHandle {
12018 client: client.clone(),
12019 workflow_id: workflow_id.to_string(),
12020 run_id: None,
12021 workflow_type: "terminal".to_string(),
12022 }
12023 .result(options)
12024 .await
12025 .expect_err("typed terminal outcome");
12026 let outcome = match error {
12027 Error::WorkflowCancelled(outcome) => outcome,
12028 Error::WorkflowTerminated(outcome) => outcome,
12029 Error::WorkflowTimedOut(outcome) => outcome,
12030 other => panic!("unexpected terminal error: {other}"),
12031 };
12032 assert_eq!(outcome.kind, expected_kind);
12033 assert_eq!(outcome.workflow_id, workflow_id);
12034 assert_eq!(outcome.reason, expected_reason);
12035 }
12036
12037 let wait_timeout = WorkflowHandle {
12038 client,
12039 workflow_id: "wf-waiting".to_string(),
12040 run_id: Some("run-waiting".to_string()),
12041 workflow_type: "waiting".to_string(),
12042 }
12043 .result(WorkflowResultOptions {
12044 poll_interval: Duration::ZERO,
12045 timeout: Duration::ZERO,
12046 })
12047 .await
12048 .expect_err("client wait timeout");
12049 let Error::WorkflowTimedOut(timeout) = wait_timeout else {
12050 panic!("expected typed client timeout");
12051 };
12052 assert_eq!(timeout.reason, "result_wait_timeout");
12053 assert_eq!(timeout.failure_category.as_deref(), Some("client_timeout"));
12054 assert_eq!(timeout.run_id.as_deref(), Some("run-waiting"));
12055 }
12056
12057 #[tokio::test]
12058 async fn workflow_result_follows_chain_and_selected_result_preserves_history() {
12059 let server = MockWorkerServer::start();
12060 let client = Client::builder(server.base_url())
12061 .timeout(Duration::from_secs(2))
12062 .build()
12063 .expect("client");
12064
12065 let handle = WorkflowHandle {
12066 client,
12067 workflow_id: "wf-selected".to_string(),
12068 run_id: Some("run-selected".to_string()),
12069 workflow_type: "selected".to_string(),
12070 };
12071 let options = WorkflowResultOptions {
12072 poll_interval: Duration::ZERO,
12073 timeout: Duration::from_secs(1),
12074 };
12075
12076 let current = handle
12077 .result(options)
12078 .await
12079 .expect("instance result follows the current run");
12080 assert_eq!(current, json!("current run output"));
12081
12082 let error = handle
12083 .result_selected_run(options)
12084 .await
12085 .expect_err("the selected run is cancelled even though the current run completed");
12086
12087 let Error::WorkflowCancelled(outcome) = error else {
12088 panic!("expected selected run cancellation");
12089 };
12090 assert_eq!(outcome.run_id.as_deref(), Some("run-selected"));
12091 assert_eq!(outcome.reason, "selected run cancelled");
12092 assert_eq!(
12093 server.request_count("/api/workflows/wf-selected/runs/run-selected"),
12094 1
12095 );
12096 assert_eq!(server.request_count("/api/workflows/wf-selected"), 1);
12097 }
12098
12099 #[tokio::test]
12100 async fn poll_responses_decode_http_conflict_drain_as_a_stable_stop() {
12101 let server = MockWorkerServer::draining_polls();
12102 let client = Client::builder(server.base_url())
12103 .timeout(Duration::from_secs(2))
12104 .build()
12105 .expect("client");
12106
12107 let workflow = client
12108 .poll_workflow_task_response("draining-worker", "rust-workers", Duration::ZERO)
12109 .await
12110 .expect("workflow drain response");
12111 let activity = client
12112 .poll_activity_task_response("draining-worker", "rust-workers", Duration::ZERO)
12113 .await
12114 .expect("activity drain response");
12115 let query = client
12116 .poll_query_task_response("draining-worker", "rust-workers", Duration::ZERO)
12117 .await
12118 .expect("query drain response");
12119
12120 for outcome in [workflow.outcome(), activity.outcome(), query.outcome()] {
12121 assert_eq!(
12122 outcome,
12123 WorkerPollOutcome::Stop {
12124 poll_status: Some("draining".to_string()),
12125 reason: Some("worker_draining".to_string()),
12126 }
12127 );
12128 }
12129
12130 assert!(client
12131 .poll_workflow_task("draining-worker", "rust-workers", Duration::ZERO)
12132 .await
12133 .expect("compatibility poll")
12134 .is_none());
12135 }
12136
12137 #[tokio::test]
12138 async fn managed_worker_honors_drain_stop_for_every_task_family() {
12139 let server = MockWorkerServer::draining_polls();
12140 let client = Client::builder(server.base_url())
12141 .timeout(Duration::from_secs(2))
12142 .build()
12143 .expect("client");
12144
12145 let mut workflow_worker = Worker::new(client.clone(), "rust-workers")
12146 .worker_id("draining-workflow-worker")
12147 .poll_timeout(Duration::ZERO);
12148 workflow_worker.register_workflow("counter", |_ctx, _args| async { Ok(Value::Null) });
12149 workflow_worker
12150 .run()
12151 .await
12152 .expect("workflow drain is a clean stop");
12153
12154 let mut activity_worker = Worker::new(client.clone(), "rust-workers")
12155 .worker_id("draining-activity-worker")
12156 .poll_timeout(Duration::ZERO);
12157 activity_worker.register_activity("write", |_ctx, _args| async { Ok(Value::Null) });
12158 activity_worker
12159 .run()
12160 .await
12161 .expect("activity drain is a clean stop");
12162
12163 let mut query_worker = Worker::new(client, "rust-workers")
12164 .worker_id("draining-query-worker")
12165 .poll_timeout(Duration::ZERO);
12166 query_worker.register_query("counter", "current", |_ctx, _args| async {
12167 Ok(Value::Null)
12168 });
12169 query_worker
12170 .run()
12171 .await
12172 .expect("query drain is a clean stop");
12173 }
12174
12175 #[tokio::test]
12176 async fn activity_cancellation_and_late_completion_remain_machine_readable() {
12177 let server = MockWorkerServer::start();
12178 let client = Client::builder(server.base_url())
12179 .timeout(Duration::from_secs(2))
12180 .build()
12181 .expect("client");
12182
12183 let heartbeat = client
12184 .heartbeat_activity_task(
12185 "activity-cancel",
12186 "attempt-cancel",
12187 "rust-worker",
12188 typed_fidelity_probe(),
12189 )
12190 .await
12191 .expect("cancellation heartbeat");
12192 assert!(heartbeat.cancel_requested);
12193 assert!(heartbeat.should_stop());
12194 assert_eq!(heartbeat.reason.as_deref(), Some("run_cancelled"));
12195 assert_eq!(heartbeat.run_closed_reason.as_deref(), Some("cancelled"));
12196 let heartbeat_body =
12197 server.request_body("/api/worker/activity-tasks/activity-cancel/heartbeat");
12198 assert_eq!(heartbeat_body["details"]["codec"], DEFAULT_CODEC);
12199 assert_eq!(
12200 decode_wire_avro_value(&heartbeat_body["details"], DEFAULT_CODEC)
12201 .expect("typed heartbeat details"),
12202 typed_fidelity_probe()
12203 );
12204
12205 let error = client
12206 .complete_activity_task(
12207 "activity-cancel",
12208 "attempt-cancel",
12209 "rust-worker",
12210 json!({"late":true}),
12211 JSON_CODEC,
12212 )
12213 .await
12214 .expect_err("late completion must be refused");
12215 assert!(activity_task_rejection_is_final(&error));
12216 let Error::ActivityTaskRejected(rejection) = error else {
12217 panic!("expected typed activity rejection");
12218 };
12219 assert_eq!(rejection.status, 409);
12220 assert_eq!(rejection.reason, "run_cancelled");
12221 assert!(rejection.cancel_requested);
12222 assert_eq!(rejection.can_continue, Some(false));
12223 }
12224
12225 #[tokio::test]
12226 async fn managed_worker_survives_late_completion_and_restart_during_cancellation() {
12227 let server = MockWorkerServer::cancelled_activity();
12228 let client = Client::builder(server.base_url())
12229 .timeout(Duration::from_secs(2))
12230 .build()
12231 .expect("client");
12232 let cancellation_observed = Arc::new(AtomicBool::new(false));
12233 let observed = Arc::clone(&cancellation_observed);
12234 let mut worker = Worker::new(client.clone(), "rust-workers")
12235 .worker_id("rust-cancel-worker")
12236 .poll_timeout(Duration::from_millis(10));
12237 worker.register_activity("cancel-aware", move |ctx, _args| {
12238 let observed = Arc::clone(&observed);
12239 async move {
12240 let heartbeat = ctx.heartbeat(json!({"stage":"running"})).await?;
12241 observed.store(heartbeat.should_stop(), Ordering::SeqCst);
12242 Ok(json!({"late":"completion"}))
12243 }
12244 });
12245
12246 assert_eq!(
12247 worker.run_once().await.expect("cancelled attempt handled"),
12248 1
12249 );
12250 assert!(cancellation_observed.load(Ordering::SeqCst));
12251 assert_eq!(
12252 server.request_count("/api/worker/activity-tasks/activity-cancel/complete"),
12253 1
12254 );
12255
12256 let mut restarted = Worker::new(client, "rust-workers")
12257 .worker_id("rust-cancel-worker-restarted")
12258 .poll_timeout(Duration::from_millis(10));
12259 restarted.register_activity("cancel-aware", |_ctx, _args| async move { Ok(Value::Null) });
12260 assert_eq!(
12261 restarted
12262 .run_once()
12263 .await
12264 .expect("replacement worker continues polling"),
12265 0
12266 );
12267 }
12268
12269 #[tokio::test]
12270 async fn managed_worker_absorbs_selected_run_terminal_timeout_completion_race() {
12271 let response = r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"outcome":"completed","recorded":false,"run_id":"run-selected-timeout","run_status":"failed","created_task_ids":[],"reason":"run_timed_out"}"#;
12272 let server = MockWorkerServer::workflow_completion("409 Conflict", response);
12273 let client = Client::builder(server.base_url())
12274 .timeout(Duration::from_secs(2))
12275 .build()
12276 .expect("client");
12277
12278 let direct_error = client
12279 .complete_workflow_task(
12280 "workflow-timeout-task",
12281 "timeout-worker",
12282 3,
12283 vec![json!({"type": "complete_workflow", "result": null})],
12284 )
12285 .await
12286 .expect_err("the low-level client preserves the completion rejection");
12287 let Error::Http { status, body } = direct_error else {
12288 panic!("expected the original HTTP completion rejection");
12289 };
12290 assert_eq!(status, reqwest::StatusCode::CONFLICT);
12291 assert_eq!(
12292 serde_json::from_str::<Value>(&body).expect("response body")["reason"],
12293 "run_timed_out"
12294 );
12295
12296 let mut worker = Worker::new(client, "rust-workers")
12297 .worker_id("timeout-worker")
12298 .poll_timeout(Duration::from_millis(10));
12299 worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
12300 Ok(json!({"late": "result"}))
12301 });
12302
12303 assert_eq!(
12304 worker
12305 .run_once()
12306 .await
12307 .expect("authoritative selected-run timeout settles the tick"),
12308 1
12309 );
12310 assert_eq!(
12311 server.request_count("/api/worker/workflow-tasks/workflow-timeout-task/complete"),
12312 2,
12313 "both the direct client proof and managed worker must see the rejection"
12314 );
12315 }
12316
12317 #[tokio::test]
12318 async fn managed_worker_does_not_swallow_nearby_completion_errors() {
12319 for (name, status, response) in [
12320 ("bare conflict", "409 Conflict", r#"{"message":"conflict"}"#),
12321 (
12322 "command was recorded",
12323 "409 Conflict",
12324 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":true,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12325 ),
12326 (
12327 "lease conflict",
12328 "409 Conflict",
12329 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"lease_expired"}"#,
12330 ),
12331 (
12332 "nonterminal run",
12333 "409 Conflict",
12334 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"waiting","reason":"run_timed_out"}"#,
12335 ),
12336 (
12337 "different selected run",
12338 "409 Conflict",
12339 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-reused-workflow-current","run_status":"failed","reason":"run_timed_out"}"#,
12340 ),
12341 (
12342 "different task attempt",
12343 "409 Conflict",
12344 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":4,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12345 ),
12346 (
12347 "authentication failure",
12348 "401 Unauthorized",
12349 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12350 ),
12351 (
12352 "authorization failure",
12353 "403 Forbidden",
12354 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12355 ),
12356 (
12357 "protocol failure",
12358 "400 Bad Request",
12359 r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.2","requested_version":"1.3"}"#,
12360 ),
12361 (
12362 "malformed command",
12363 "422 Unprocessable Entity",
12364 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12365 ),
12366 (
12367 "transient server failure",
12368 "503 Service Unavailable",
12369 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12370 ),
12371 ] {
12372 let server = MockWorkerServer::workflow_completion(status, response);
12373 let client = Client::builder(server.base_url())
12374 .timeout(Duration::from_secs(2))
12375 .build()
12376 .expect("client");
12377 let mut worker = Worker::new(client, "rust-workers")
12378 .worker_id("timeout-worker")
12379 .poll_timeout(Duration::from_millis(10));
12380 worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
12381 Ok(json!({"late": "result"}))
12382 });
12383
12384 let error = worker
12385 .run_once()
12386 .await
12387 .expect_err(&format!("{name} must remain an error"));
12388 assert!(
12389 matches!(error, Error::Http { .. } | Error::Protocol(_)),
12390 "{name} returned an unexpected error variant: {error}"
12391 );
12392 }
12393 }
12394
12395 #[tokio::test]
12396 async fn baseline_worker_endpoints_send_the_baseline_protocol() {
12397 let server = MockWorkerServer::start();
12398 let client = Client::builder(server.base_url())
12399 .timeout(Duration::from_secs(2))
12400 .build()
12401 .expect("client");
12402
12403 client
12404 .register_worker("capture-worker", "capture", vec![], vec![], 1, 1)
12405 .await
12406 .expect("register");
12407 client
12408 .heartbeat_worker("capture-worker", 1, 1)
12409 .await
12410 .expect("heartbeat");
12411 client
12412 .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
12413 .await
12414 .expect("workflow poll");
12415 client
12416 .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
12417 .await
12418 .expect("activity poll");
12419
12420 for path in [
12421 "/api/worker/register",
12422 "/api/worker/heartbeat",
12423 "/api/worker/workflow-tasks/poll",
12424 "/api/worker/activity-tasks/poll",
12425 ] {
12426 assert_eq!(
12427 server.worker_protocol_for(path).as_deref(),
12428 Some(WORKER_PROTOCOL_VERSION),
12429 "unexpected protocol for {path}"
12430 );
12431 }
12432
12433 assert_eq!(
12434 server.request_body("/api/worker/workflow-tasks/poll")["timeout_seconds"],
12435 1
12436 );
12437 assert_eq!(
12438 server.request_body("/api/worker/activity-tasks/poll")["timeout_seconds"],
12439 1
12440 );
12441 assert!(
12442 server.request_body("/api/worker/workflow-tasks/poll")["poll_request_id"]
12443 .as_str()
12444 .is_some_and(|id| id.starts_with("rust-workflow-poll-"))
12445 );
12446 assert!(
12447 server.request_body("/api/worker/activity-tasks/poll")["poll_request_id"]
12448 .as_str()
12449 .is_some_and(|id| id.starts_with("rust-activity-poll-"))
12450 );
12451 }
12452
12453 #[tokio::test]
12454 async fn query_task_endpoints_send_the_query_feature_protocol() {
12455 let server = MockWorkerServer::start();
12456 let client = Client::builder(server.base_url())
12457 .timeout(Duration::from_secs(2))
12458 .build()
12459 .expect("client");
12460
12461 client
12462 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
12463 .await
12464 .expect("query poll");
12465 client
12466 .complete_query_task("query-capture", "capture-worker", 1, json!(8), JSON_CODEC)
12467 .await
12468 .expect("query complete");
12469 client
12470 .fail_query_task(
12471 "query-capture",
12472 "capture-worker",
12473 1,
12474 "failed",
12475 "query_rejected",
12476 "QueryFailed",
12477 )
12478 .await
12479 .expect("query fail");
12480
12481 for path in [
12482 "/api/worker/query-tasks/poll",
12483 "/api/worker/query-tasks/query-capture/complete",
12484 "/api/worker/query-tasks/query-capture/fail",
12485 ] {
12486 assert_eq!(
12487 server.worker_protocol_for(path).as_deref(),
12488 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
12489 "unexpected protocol for {path}"
12490 );
12491 }
12492
12493 assert_eq!(
12494 server.request_body("/api/worker/query-tasks/poll")["timeout_seconds"],
12495 1
12496 );
12497 assert!(
12498 server.request_body("/api/worker/query-tasks/poll")["poll_request_id"]
12499 .as_str()
12500 .is_some_and(|id| id.starts_with("rust-query-poll-"))
12501 );
12502 }
12503
12504 #[tokio::test]
12505 async fn disconnected_client_polls_retry_once_with_the_same_request_id() {
12506 let server = MockWorkerServer::transient_worker_failures();
12507 let client = Client::builder(server.base_url())
12508 .timeout(Duration::from_secs(2))
12509 .build()
12510 .expect("client");
12511
12512 client
12513 .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
12514 .await
12515 .expect("workflow poll retry");
12516 client
12517 .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
12518 .await
12519 .expect("activity poll retry");
12520 client
12521 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
12522 .await
12523 .expect("query poll retry");
12524
12525 for path in [
12526 "/api/worker/workflow-tasks/poll",
12527 "/api/worker/activity-tasks/poll",
12528 "/api/worker/query-tasks/poll",
12529 ] {
12530 let bodies = server.request_bodies(path);
12531 assert_eq!(bodies.len(), 2, "{path} must be retried once");
12532 assert_eq!(
12533 bodies[0]["poll_request_id"], bodies[1]["poll_request_id"],
12534 "{path} must preserve the request binding across retry"
12535 );
12536 }
12537 }
12538
12539 #[tokio::test]
12540 async fn worker_poll_retries_preserve_request_id_across_consecutive_failures() {
12541 let server = MockWorkerServer::consecutive_poll_failures(2);
12542 let client = Client::builder(server.base_url())
12543 .timeout(Duration::from_secs(2))
12544 .build()
12545 .expect("client");
12546 let mut worker = Worker::new(client, "capture")
12547 .worker_id("capture-worker")
12548 .poll_timeout(Duration::from_millis(10))
12549 .retry_policy(WorkerRetryPolicy {
12550 max_retries: 2,
12551 initial_backoff: Duration::from_millis(1),
12552 max_backoff: Duration::from_millis(1),
12553 });
12554 worker.register_workflow(
12555 "capture.workflow",
12556 |_ctx, _input| async move { Ok(Value::Null) },
12557 );
12558 worker.register_activity(
12559 "capture.activity",
12560 |_ctx, _input| async move { Ok(Value::Null) },
12561 );
12562 worker.register_query("capture.workflow", "current", |_ctx, _args| async move {
12563 Ok(Value::Null)
12564 });
12565
12566 assert_eq!(worker.run_once().await.expect("poll retries"), 0);
12567
12568 for path in [
12569 "/api/worker/workflow-tasks/poll",
12570 "/api/worker/activity-tasks/poll",
12571 "/api/worker/query-tasks/poll",
12572 ] {
12573 let bodies = server.request_bodies(path);
12574 assert_eq!(bodies.len(), 3, "{path} must use exactly two retries");
12575 assert!(
12576 bodies
12577 .iter()
12578 .all(|body| body["poll_request_id"] == bodies[0]["poll_request_id"]),
12579 "{path} must preserve one request binding across every retry"
12580 );
12581 }
12582 }
12583
12584 #[tokio::test]
12585 async fn query_protocol_rejection_from_older_server_is_typed() {
12586 let server = MockWorkerServer::reject_query_protocol();
12587 let client = Client::builder(server.base_url())
12588 .timeout(Duration::from_secs(2))
12589 .build()
12590 .expect("client");
12591
12592 let error = client
12593 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
12594 .await
12595 .expect_err("server below query protocol floor must reject");
12596 let Error::Protocol(failure) = error else {
12597 panic!("expected typed protocol failure");
12598 };
12599
12600 assert_eq!(failure.status, 400);
12601 assert_eq!(failure.reason, "unsupported_protocol_version");
12602 assert_eq!(failure.supported_version.as_deref(), Some("1.7"));
12603 assert_eq!(
12604 failure.requested_version.as_deref(),
12605 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
12606 );
12607 assert_eq!(
12608 server
12609 .worker_protocol_for("/api/worker/query-tasks/poll")
12610 .as_deref(),
12611 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
12612 );
12613 }
12614
12615 #[tokio::test]
12616 async fn run_once_without_query_handlers_keeps_pre_query_server_compatibility() {
12617 let server = MockWorkerServer::reject_query_protocol();
12618 let client = Client::builder(server.base_url())
12619 .timeout(Duration::from_secs(2))
12620 .build()
12621 .expect("client");
12622 let mut worker = Worker::new(client, "rust-workers")
12623 .worker_id("baseline-worker")
12624 .poll_timeout(Duration::from_millis(10));
12625
12626 worker.register_workflow("baseline.workflow", |_ctx, _input| async move {
12627 Ok(Value::Null)
12628 });
12629
12630 assert_eq!(worker.run_once().await.expect("baseline run once"), 0);
12631 assert_eq!(
12632 server
12633 .worker_protocol_for("/api/worker/workflow-tasks/poll")
12634 .as_deref(),
12635 Some(WORKER_PROTOCOL_VERSION)
12636 );
12637 assert_eq!(
12638 server.worker_protocol_for("/api/worker/query-tasks/poll"),
12639 None,
12640 "a worker without query handlers must not use the query-task endpoint"
12641 );
12642 }
12643
12644 #[tokio::test]
12645 async fn completion_time_query_rejection_is_typed_without_stopping_worker() {
12646 let server = MockWorkerServer::reject_query_completion();
12647 let client = Client::builder(server.base_url())
12648 .timeout(Duration::from_secs(2))
12649 .build()
12650 .expect("client");
12651
12652 let error = client
12653 .complete_query_task("query-late", "late-worker", 1, json!(8), JSON_CODEC)
12654 .await
12655 .expect_err("expired completion must be rejected");
12656 let Error::QueryFailed(failure) = error else {
12657 panic!("expected typed query failure");
12658 };
12659 assert_eq!(failure.status, 409);
12660 assert_eq!(failure.reason, "query_task_timed_out");
12661
12662 let mut worker = Worker::new(client, "rust-workers")
12663 .worker_id("late-worker")
12664 .poll_timeout(Duration::from_millis(10));
12665 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
12666 worker.register_query(
12667 "counter",
12668 "current",
12669 |_ctx, _args| async move { Ok(json!(8)) },
12670 );
12671
12672 assert_eq!(worker.run_once().await.expect("late task is handled"), 1);
12673 assert_eq!(
12674 worker
12675 .run_once()
12676 .await
12677 .expect("worker continues after late completion"),
12678 0
12679 );
12680 assert_eq!(
12681 server.request_count("/api/worker/query-tasks/query-late/complete"),
12682 2
12683 );
12684 assert_eq!(
12685 server.request_count("/api/worker/query-tasks/query-late/fail"),
12686 0,
12687 "a server completion rejection must not be reported as an encoding failure"
12688 );
12689 }
12690
12691 #[tokio::test]
12692 async fn activity_only_worker_can_shutdown_without_workflow_poller() {
12693 let server = MockWorkerServer::start();
12694 let client = Client::builder(server.base_url())
12695 .timeout(Duration::from_secs(2))
12696 .build()
12697 .expect("client");
12698 let mut worker = Worker::new(client, "rust-workers")
12699 .worker_id("activity-only-worker")
12700 .poll_timeout(Duration::from_millis(10));
12701
12702 worker.register_activity(
12703 "activity.only",
12704 |_ctx, _args| async move { Ok(Value::Null) },
12705 );
12706
12707 worker.run_until(async {}).await.expect("run worker");
12708 }
12709
12710 #[tokio::test]
12711 async fn workflow_only_worker_can_shutdown_without_activity_poller() {
12712 let server = MockWorkerServer::start();
12713 let client = Client::builder(server.base_url())
12714 .timeout(Duration::from_secs(2))
12715 .build()
12716 .expect("client");
12717 let mut worker = Worker::new(client, "rust-workers")
12718 .worker_id("workflow-only-worker")
12719 .poll_timeout(Duration::from_millis(10));
12720
12721 worker.register_workflow(
12722 "workflow.only",
12723 |_ctx, _input| async move { Ok(Value::Null) },
12724 );
12725
12726 worker.run_until(async {}).await.expect("run worker");
12727 }
12728
12729 #[tokio::test]
12730 async fn worker_heartbeat_observer_receives_server_acknowledgements() {
12731 let server = MockWorkerServer::start();
12732 let client = Client::builder(server.base_url())
12733 .timeout(Duration::from_secs(2))
12734 .build()
12735 .expect("client");
12736 let observations = Arc::new(Mutex::new(Vec::new()));
12737 let observed = Arc::clone(&observations);
12738 let mut worker = Worker::new(client, "rust-workers")
12739 .worker_id("observed-heartbeat-worker")
12740 .poll_timeout(Duration::from_millis(10))
12741 .on_worker_heartbeat(move |observation| {
12742 observed
12743 .lock()
12744 .expect("heartbeat observations")
12745 .push(observation.clone());
12746 });
12747
12748 worker.register_workflow("workflow.observed", |_ctx, _input| async move {
12749 Ok(Value::Null)
12750 });
12751 let acknowledged = Arc::clone(&observations);
12752 worker
12753 .run_until(async move {
12754 tokio::time::timeout(Duration::from_secs(2), async move {
12755 loop {
12756 if !acknowledged
12757 .lock()
12758 .expect("heartbeat observations")
12759 .is_empty()
12760 {
12761 break;
12762 }
12763 tokio::time::sleep(Duration::from_millis(1)).await;
12764 }
12765 })
12766 .await
12767 .expect("heartbeat acknowledgement within timeout");
12768 })
12769 .await
12770 .expect("run worker");
12771
12772 let observations = observations.lock().expect("heartbeat observations");
12773 let first = observations.first().expect("heartbeat acknowledgement");
12774 assert_eq!(first.worker_id, "observed-heartbeat-worker");
12775 assert_eq!(first.task_queue, "rust-workers");
12776 assert!(first.acknowledged_at_unix_millis > 0);
12777 assert_eq!(first.acknowledgement, json!({}));
12778 }
12779
12780 #[tokio::test]
12781 async fn delayed_worker_heartbeat_keeps_cadence_and_pollers_live() {
12782 let server = MockWorkerServer::delayed_heartbeat_worker();
12783 let client = Client::builder(server.base_url())
12784 .timeout(Duration::from_secs(3))
12785 .build()
12786 .expect("client");
12787 let observations = Arc::new(Mutex::new(Vec::new()));
12788 let observed = Arc::clone(&observations);
12789 let mut worker = Worker::new(client, "rust-snapshot-workers")
12790 .worker_id("rust-snapshot-worker")
12791 .poll_timeout(Duration::from_millis(10))
12792 .on_worker_heartbeat(move |observation| {
12793 observed
12794 .lock()
12795 .expect("heartbeat observations")
12796 .push(observation.clone());
12797 });
12798
12799 worker.register_workflow("snapshot", |ctx, _input| async move {
12800 ctx.wait_signal("finish").await?;
12801 Ok(json!({"status": "finished"}))
12802 });
12803 worker.register_query("snapshot", "current", |ctx, _args| async move {
12804 Ok(json!(ctx
12805 .signals("increment")
12806 .iter()
12807 .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
12808 .sum::<i64>()))
12809 });
12810 worker.register_activity("cancel-aware", |_ctx, _args| async move {
12811 Ok(json!({"late": "completion"}))
12812 });
12813
12814 worker
12815 .run_until(tokio::time::sleep(Duration::from_millis(3_800)))
12816 .await
12817 .expect("delayed heartbeat must allow a clean worker shutdown");
12818
12819 let observations = observations.lock().expect("heartbeat observations");
12820 assert!(
12821 observations.len() >= 3,
12822 "the immediate heartbeat, delayed acknowledgement, and next cadence heartbeat must complete"
12823 );
12824 assert!(
12825 observations.windows(2).all(|pair| {
12826 pair[1].acknowledged_at_unix_millis
12827 .saturating_sub(pair[0].acknowledged_at_unix_millis)
12828 >= 850
12829 }),
12830 "successful acknowledgements must not catch up faster than the advertised one-second cadence: {observations:?}"
12831 );
12832 drop(observations);
12833
12834 let heartbeat_times = server.request_times("/api/worker/heartbeat");
12835 let delayed_request_at = *heartbeat_times
12836 .get(1)
12837 .expect("intentionally delayed heartbeat request");
12838 let delay_window_start = delayed_request_at + Duration::from_millis(100);
12839 let delay_window_end = delayed_request_at + Duration::from_millis(1_400);
12840 for path in [
12841 "/api/worker/workflow-tasks/poll",
12842 "/api/worker/activity-tasks/poll",
12843 "/api/worker/query-tasks/poll",
12844 ] {
12845 assert!(
12846 server
12847 .request_times(path)
12848 .iter()
12849 .any(|received_at| *received_at >= delay_window_start
12850 && *received_at <= delay_window_end),
12851 "{path} must keep polling while a heartbeat acknowledgement is delayed"
12852 );
12853 }
12854 assert!(
12855 server.request_count("/api/worker/workflow-tasks/snapshot-wait-3/fail") >= 1,
12856 "workflow work must be settled"
12857 );
12858 assert!(
12859 server.request_count("/api/worker/activity-tasks/activity-cancel/complete") >= 1,
12860 "activity work must be settled"
12861 );
12862 assert!(
12863 server.request_count("/api/worker/query-tasks/snapshot-current/complete") >= 1,
12864 "query work must be settled"
12865 );
12866 }
12867
12868 #[tokio::test]
12869 async fn retried_worker_heartbeat_restarts_the_advertised_cadence() {
12870 let server = MockWorkerServer::heartbeat_retry_worker();
12871 let client = Client::builder(server.base_url())
12872 .timeout(Duration::from_secs(2))
12873 .build()
12874 .expect("client");
12875 let observations = Arc::new(Mutex::new(Vec::new()));
12876 let observed = Arc::clone(&observations);
12877 let worker = Worker::new(client, "rust-workers")
12878 .worker_id("heartbeat-retry-worker")
12879 .retry_policy(WorkerRetryPolicy {
12880 max_retries: 1,
12881 initial_backoff: Duration::from_millis(300),
12882 max_backoff: Duration::from_millis(300),
12883 })
12884 .on_worker_heartbeat(move |observation| {
12885 observed
12886 .lock()
12887 .expect("heartbeat observations")
12888 .push(observation.clone());
12889 });
12890
12891 worker
12892 .run_until(tokio::time::sleep(Duration::from_millis(2_700)))
12893 .await
12894 .expect("retryable heartbeat failure must remain bounded and recover");
12895
12896 let observations = observations.lock().expect("heartbeat observations");
12897 assert!(observations.len() >= 3, "heartbeat retry must recover");
12898 assert!(
12899 observations.windows(2).all(|pair| {
12900 pair[1]
12901 .acknowledged_at_unix_millis
12902 .saturating_sub(pair[0].acknowledged_at_unix_millis)
12903 >= 850
12904 }),
12905 "a successful retry must start a fresh advertised cadence: {observations:?}"
12906 );
12907 assert_eq!(
12908 server.request_count("/api/worker/heartbeat"),
12909 observations.len() + 1,
12910 "one retryable failure must add exactly one bounded request"
12911 );
12912 }
12913
12914 #[tokio::test]
12915 async fn query_enabled_worker_ignores_unmatched_signals_then_completes_once() {
12916 let server = MockWorkerServer::waiting_query_worker();
12917 let client = Client::builder(server.base_url())
12918 .timeout(Duration::from_secs(2))
12919 .build()
12920 .expect("client");
12921 let observations = Arc::new(Mutex::new(Vec::new()));
12922 let observed = Arc::clone(&observations);
12923 let mut worker = Worker::new(client, "rust-snapshot-workers")
12924 .worker_id("rust-snapshot-worker")
12925 .poll_timeout(Duration::from_millis(10))
12926 .on_worker_heartbeat(move |observation| {
12927 observed
12928 .lock()
12929 .expect("heartbeat observations")
12930 .push(observation.clone());
12931 });
12932
12933 worker.register_workflow("snapshot", |ctx, _input| async move {
12934 ctx.wait_signal("finish").await?;
12935 Ok(json!({"status": "finished"}))
12936 });
12937 worker.register_query("snapshot", "current", |ctx, _args| async move {
12938 let current = ctx
12939 .signals("increment")
12940 .iter()
12941 .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
12942 .sum::<i64>();
12943 Ok(json!(current))
12944 });
12945 worker.register_update("snapshot", "replace", |_ctx, args| async move { Ok(args) });
12946
12947 worker
12948 .run_until(tokio::time::sleep(Duration::from_millis(3_200)))
12949 .await
12950 .expect("pending workflow and query poller must remain live until shutdown");
12951
12952 assert!(
12953 observations.lock().expect("heartbeat observations").len() >= 4,
12954 "the immediate heartbeat and at least three advertised one-second intervals must be acknowledged"
12955 );
12956 assert!(
12957 server.request_count("/api/worker/workflow-tasks/poll") >= 3,
12958 "workflow polling must continue after empty replay acknowledgements"
12959 );
12960 assert!(
12961 server.request_count("/api/worker/query-tasks/poll") >= 2,
12962 "query polling must continue after serving the current query"
12963 );
12964 assert_eq!(
12965 server.request_body("/api/worker/register")["capabilities"],
12966 json!([QUERY_TASKS_CAPABILITY, WORKFLOW_UPDATES_CAPABILITY])
12967 );
12968 assert_eq!(
12969 server.request_body("/api/worker/register")["workflow_command_contracts"]["snapshot"],
12970 json!({
12971 "queries": ["current"],
12972 "updates": ["replace"],
12973 })
12974 );
12975
12976 let opened = server.request_body("/api/worker/workflow-tasks/snapshot-open/complete");
12977 assert_eq!(
12978 opened["commands"],
12979 json!([{
12980 "type": "open_signal_wait",
12981 "signal_name": "finish",
12982 }])
12983 );
12984
12985 for task_id in ["snapshot-wait-3", "snapshot-wait-5"] {
12986 let fail_path = format!("/api/worker/workflow-tasks/{task_id}/fail");
12987 let completion_path = format!("/api/worker/workflow-tasks/{task_id}/complete");
12988 let failure = server.request_body(&fail_path);
12989 assert_eq!(
12990 failure["failure"]["type"],
12991 WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE
12992 );
12993 assert_eq!(server.request_count(&completion_path), 0);
12994 }
12995
12996 let query_completion =
12997 server.request_body("/api/worker/query-tasks/snapshot-current/complete");
12998 assert_eq!(query_completion["result"], json!(8));
12999
13000 let terminal_path = "/api/worker/workflow-tasks/snapshot-finish/complete";
13001 assert_eq!(
13002 server.request_count(terminal_path),
13003 1,
13004 "the matching signal must settle the workflow exactly once"
13005 );
13006 let terminal = server.request_body(terminal_path);
13007 assert_eq!(terminal["commands"].as_array().map(Vec::len), Some(1));
13008 assert_eq!(terminal["commands"][0]["type"], "complete_workflow");
13009 assert_eq!(
13010 decode_wire_value(&terminal["commands"][0]["result"], DEFAULT_CODEC)
13011 .expect("terminal workflow result"),
13012 json!({"status": "finished"})
13013 );
13014 }
13015
13016 #[tokio::test]
13017 async fn worker_retries_poll_and_heartbeat_transport_failures_independently() {
13018 let server = MockWorkerServer::transient_worker_failures();
13019 let client = Client::builder(server.base_url())
13020 .timeout(Duration::from_secs(2))
13021 .build()
13022 .expect("client");
13023 let mut worker = Worker::new(client, "rust-workers")
13024 .worker_id("retry-worker")
13025 .poll_timeout(Duration::from_millis(10))
13026 .retry_policy(WorkerRetryPolicy {
13027 max_retries: 2,
13028 initial_backoff: Duration::from_millis(1),
13029 max_backoff: Duration::from_millis(1),
13030 });
13031 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
13032 worker.register_activity(
13033 "counter.activity",
13034 |_ctx, _input| async move { Ok(Value::Null) },
13035 );
13036 worker.register_query(
13037 "counter",
13038 "current",
13039 |_ctx, _args| async move { Ok(json!(8)) },
13040 );
13041
13042 worker
13043 .run_until(tokio::time::sleep(Duration::from_millis(75)))
13044 .await
13045 .expect("transient failures must not stop the worker");
13046
13047 for path in [
13048 "/api/worker/heartbeat",
13049 "/api/worker/workflow-tasks/poll",
13050 "/api/worker/activity-tasks/poll",
13051 "/api/worker/query-tasks/poll",
13052 ] {
13053 assert!(
13054 server.request_count(path) >= 2,
13055 "{path} must continue after its transient failure"
13056 );
13057 }
13058 }
13059
13060 #[tokio::test]
13061 async fn worker_bounds_transport_retries() {
13062 let server = MockWorkerServer::unavailable_polls();
13063 let client = Client::builder(server.base_url())
13064 .timeout(Duration::from_secs(2))
13065 .build()
13066 .expect("client");
13067 let mut worker = Worker::new(client, "rust-workers")
13068 .worker_id("bounded-retry-worker")
13069 .poll_timeout(Duration::from_millis(10))
13070 .retry_policy(WorkerRetryPolicy {
13071 max_retries: 2,
13072 initial_backoff: Duration::from_millis(1),
13073 max_backoff: Duration::from_millis(1),
13074 });
13075 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
13076
13077 let error = worker.run().await.expect_err("retry bound must terminate");
13078 assert!(matches!(error, Error::Transport(_)));
13079 assert_eq!(
13080 server.request_count("/api/worker/workflow-tasks/poll"),
13081 3,
13082 "one initial request plus exactly two retries"
13083 );
13084 }
13085
13086 #[tokio::test]
13087 async fn worker_retry_policy_can_disable_poll_retries() {
13088 let server = MockWorkerServer::unavailable_polls();
13089 let client = Client::builder(server.base_url())
13090 .timeout(Duration::from_secs(2))
13091 .build()
13092 .expect("client");
13093 let mut worker = Worker::new(client, "rust-workers")
13094 .worker_id("no-retry-worker")
13095 .poll_timeout(Duration::from_millis(10))
13096 .retry_policy(WorkerRetryPolicy {
13097 max_retries: 0,
13098 initial_backoff: Duration::from_millis(1),
13099 max_backoff: Duration::from_millis(1),
13100 });
13101 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
13102
13103 let error = worker
13104 .run_once()
13105 .await
13106 .expect_err("disabled retries must return the first transport failure");
13107 assert!(matches!(error, Error::Transport(_)));
13108 assert_eq!(
13109 server.request_count("/api/worker/workflow-tasks/poll"),
13110 1,
13111 "max_retries=0 must send only the initial request"
13112 );
13113 }
13114
13115 #[tokio::test]
13116 async fn worker_does_not_retry_authentication_failures() {
13117 let server = MockWorkerServer::unauthorized_polls();
13118 let client = Client::builder(server.base_url())
13119 .timeout(Duration::from_secs(2))
13120 .build()
13121 .expect("client");
13122 let mut worker = Worker::new(client, "rust-workers")
13123 .worker_id("unauthorized-worker")
13124 .poll_timeout(Duration::from_millis(10));
13125 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
13126
13127 let error = worker
13128 .run()
13129 .await
13130 .expect_err("authentication must terminate");
13131 let Error::Http { status, body } = error else {
13132 panic!("expected stable HTTP authentication error");
13133 };
13134 assert_eq!(status, reqwest::StatusCode::UNAUTHORIZED);
13135 assert!(body.contains("authentication_failed"));
13136 assert_eq!(
13137 server.request_count("/api/worker/workflow-tasks/poll"),
13138 1,
13139 "authentication failures must not be retried"
13140 );
13141 }
13142
13143 #[derive(Clone, Debug)]
13144 struct CapturedRequest {
13145 path: String,
13146 worker_protocol: Option<String>,
13147 body: String,
13148 received_at: Instant,
13149 }
13150
13151 struct MockWorkerServer {
13152 addr: SocketAddr,
13153 stop: Arc<AtomicBool>,
13154 requests: Arc<Mutex<Vec<CapturedRequest>>>,
13155 thread: Option<thread::JoinHandle<()>>,
13156 }
13157
13158 #[derive(Clone, Copy, Default)]
13159 struct MockWorkerBehavior {
13160 reject_query_protocol: bool,
13161 reject_query_completion: bool,
13162 waiting_query_worker: bool,
13163 complete_named_signal: bool,
13164 poll_failures_per_path: usize,
13165 heartbeat_failures: usize,
13166 heartbeat_failure_request: Option<usize>,
13167 delayed_heartbeat_request: Option<usize>,
13168 heartbeat_response_delay: Duration,
13169 concurrent_requests: bool,
13170 unauthorized_polls: bool,
13171 cancelled_activity: bool,
13172 draining_polls: bool,
13173 workflow_completion_status: Option<&'static str>,
13174 workflow_completion_body: Option<&'static str>,
13175 }
13176
13177 impl MockWorkerServer {
13178 fn start() -> Self {
13179 Self::start_with_behavior(MockWorkerBehavior::default())
13180 }
13181
13182 fn reject_query_protocol() -> Self {
13183 Self::start_with_behavior(MockWorkerBehavior {
13184 reject_query_protocol: true,
13185 ..MockWorkerBehavior::default()
13186 })
13187 }
13188
13189 fn reject_query_completion() -> Self {
13190 Self::start_with_behavior(MockWorkerBehavior {
13191 reject_query_completion: true,
13192 ..MockWorkerBehavior::default()
13193 })
13194 }
13195
13196 fn waiting_query_worker() -> Self {
13197 Self::start_with_behavior(MockWorkerBehavior {
13198 waiting_query_worker: true,
13199 complete_named_signal: true,
13200 ..MockWorkerBehavior::default()
13201 })
13202 }
13203
13204 fn transient_worker_failures() -> Self {
13205 Self::start_with_behavior(MockWorkerBehavior {
13206 poll_failures_per_path: 1,
13207 heartbeat_failures: 1,
13208 ..MockWorkerBehavior::default()
13209 })
13210 }
13211
13212 fn consecutive_poll_failures(count: usize) -> Self {
13213 Self::start_with_behavior(MockWorkerBehavior {
13214 poll_failures_per_path: count,
13215 ..MockWorkerBehavior::default()
13216 })
13217 }
13218
13219 fn delayed_heartbeat_worker() -> Self {
13220 Self::start_with_behavior(MockWorkerBehavior {
13221 waiting_query_worker: true,
13222 delayed_heartbeat_request: Some(2),
13223 heartbeat_response_delay: Duration::from_millis(1_500),
13224 concurrent_requests: true,
13225 cancelled_activity: true,
13226 ..MockWorkerBehavior::default()
13227 })
13228 }
13229
13230 fn heartbeat_retry_worker() -> Self {
13231 Self::start_with_behavior(MockWorkerBehavior {
13232 waiting_query_worker: true,
13233 heartbeat_failure_request: Some(2),
13234 concurrent_requests: true,
13235 ..MockWorkerBehavior::default()
13236 })
13237 }
13238
13239 fn unavailable_polls() -> Self {
13240 Self::start_with_behavior(MockWorkerBehavior {
13241 poll_failures_per_path: usize::MAX,
13242 ..MockWorkerBehavior::default()
13243 })
13244 }
13245
13246 fn unauthorized_polls() -> Self {
13247 Self::start_with_behavior(MockWorkerBehavior {
13248 unauthorized_polls: true,
13249 ..MockWorkerBehavior::default()
13250 })
13251 }
13252
13253 fn cancelled_activity() -> Self {
13254 Self::start_with_behavior(MockWorkerBehavior {
13255 cancelled_activity: true,
13256 ..MockWorkerBehavior::default()
13257 })
13258 }
13259
13260 fn draining_polls() -> Self {
13261 Self::start_with_behavior(MockWorkerBehavior {
13262 draining_polls: true,
13263 ..MockWorkerBehavior::default()
13264 })
13265 }
13266
13267 fn workflow_completion(status: &'static str, body: &'static str) -> Self {
13268 Self::start_with_behavior(MockWorkerBehavior {
13269 workflow_completion_status: Some(status),
13270 workflow_completion_body: Some(body),
13271 ..MockWorkerBehavior::default()
13272 })
13273 }
13274
13275 fn start_with_behavior(behavior: MockWorkerBehavior) -> Self {
13276 let listener = TcpListener::bind("127.0.0.1:0").expect("bind mock server");
13277 listener
13278 .set_nonblocking(true)
13279 .expect("configure mock listener");
13280 let addr = listener.local_addr().expect("mock server address");
13281 let stop = Arc::new(AtomicBool::new(false));
13282 let server_stop = Arc::clone(&stop);
13283 let requests = Arc::new(Mutex::new(Vec::new()));
13284 let server_requests = Arc::clone(&requests);
13285 let thread = thread::spawn(move || {
13286 let mut request_threads = Vec::new();
13287 while !server_stop.load(Ordering::SeqCst) {
13288 match listener.accept() {
13289 Ok((mut stream, _)) => {
13290 if behavior.concurrent_requests {
13291 let requests = Arc::clone(&server_requests);
13292 request_threads.push(thread::spawn(move || {
13293 handle_mock_worker_request(&mut stream, &requests, behavior)
13294 }));
13295 } else {
13296 handle_mock_worker_request(&mut stream, &server_requests, behavior);
13297 }
13298 }
13299 Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
13300 let mut index = 0;
13301 while index < request_threads.len() {
13302 if request_threads[index].is_finished() {
13303 request_threads
13304 .swap_remove(index)
13305 .join()
13306 .expect("join mock request");
13307 } else {
13308 index += 1;
13309 }
13310 }
13311 thread::sleep(Duration::from_millis(5));
13312 }
13313 Err(_) => break,
13314 }
13315 }
13316 for request_thread in request_threads {
13317 request_thread.join().expect("join mock request");
13318 }
13319 });
13320
13321 Self {
13322 addr,
13323 stop,
13324 requests,
13325 thread: Some(thread),
13326 }
13327 }
13328
13329 fn base_url(&self) -> String {
13330 format!("http://{}", self.addr)
13331 }
13332
13333 fn worker_protocol_for(&self, path: &str) -> Option<String> {
13334 self.requests
13335 .lock()
13336 .expect("captured requests")
13337 .iter()
13338 .find(|request| request.path == path)
13339 .and_then(|request| request.worker_protocol.clone())
13340 }
13341
13342 fn request_count(&self, path: &str) -> usize {
13343 self.requests
13344 .lock()
13345 .expect("captured requests")
13346 .iter()
13347 .filter(|request| request.path == path)
13348 .count()
13349 }
13350
13351 fn request_times(&self, path: &str) -> Vec<Instant> {
13352 self.requests
13353 .lock()
13354 .expect("captured requests")
13355 .iter()
13356 .filter(|request| request.path == path)
13357 .map(|request| request.received_at)
13358 .collect()
13359 }
13360
13361 fn request_body(&self, path: &str) -> Value {
13362 let requests = self.requests.lock().expect("captured requests");
13363 let body = &requests
13364 .iter()
13365 .find(|request| request.path == path)
13366 .unwrap_or_else(|| panic!("missing request for {path}"))
13367 .body;
13368 serde_json::from_str(body).unwrap_or_else(|error| {
13369 panic!("invalid JSON request body for {path}: {error}: {body:?}")
13370 })
13371 }
13372
13373 fn request_bodies(&self, path: &str) -> Vec<Value> {
13374 self.requests
13375 .lock()
13376 .expect("captured requests")
13377 .iter()
13378 .filter(|request| request.path == path)
13379 .map(|request| {
13380 serde_json::from_str(&request.body).unwrap_or_else(|error| {
13381 panic!(
13382 "invalid JSON request body for {path}: {error}: {:?}",
13383 request.body
13384 )
13385 })
13386 })
13387 .collect()
13388 }
13389 }
13390
13391 impl Drop for MockWorkerServer {
13392 fn drop(&mut self) {
13393 self.stop.store(true, Ordering::SeqCst);
13394 let _ = TcpStream::connect(self.addr);
13395
13396 if let Some(thread) = self.thread.take() {
13397 thread.join().expect("join mock server");
13398 }
13399 }
13400 }
13401
13402 fn handle_mock_worker_request(
13403 stream: &mut TcpStream,
13404 requests: &Arc<Mutex<Vec<CapturedRequest>>>,
13405 behavior: MockWorkerBehavior,
13406 ) {
13407 let _ = stream.set_read_timeout(Some(Duration::from_millis(200)));
13408 let mut buffer = [0_u8; 8192];
13409 let mut request = Vec::new();
13410
13411 loop {
13412 match stream.read(&mut buffer) {
13413 Ok(0) => break,
13414 Ok(read) => {
13415 request.extend_from_slice(&buffer[..read]);
13416 if mock_request_is_complete(&request) {
13417 break;
13418 }
13419 }
13420 Err(error)
13421 if matches!(
13422 error.kind(),
13423 std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut
13424 ) =>
13425 {
13426 break;
13427 }
13428 Err(_) => return,
13429 }
13430 }
13431
13432 let request = String::from_utf8_lossy(&request);
13433 let body = request
13434 .split_once("\r\n\r\n")
13435 .map(|(_, body)| body)
13436 .unwrap_or_default();
13437 let path = request
13438 .lines()
13439 .next()
13440 .and_then(|line| line.split_whitespace().nth(1))
13441 .unwrap_or_default();
13442 let worker_protocol = request.lines().find_map(|line| {
13443 let (name, value) = line.split_once(':')?;
13444 name.eq_ignore_ascii_case("X-Durable-Workflow-Protocol-Version")
13445 .then(|| value.trim().to_string())
13446 });
13447 let request_number = {
13448 let mut requests = requests.lock().expect("captured requests");
13449 requests.push(CapturedRequest {
13450 path: path.to_string(),
13451 worker_protocol: worker_protocol.clone(),
13452 body: body.to_string(),
13453 received_at: Instant::now(),
13454 });
13455 requests
13456 .iter()
13457 .filter(|request| request.path == path)
13458 .count()
13459 };
13460
13461 let is_poll = matches!(
13462 path,
13463 "/api/worker/workflow-tasks/poll"
13464 | "/api/worker/activity-tasks/poll"
13465 | "/api/worker/query-tasks/poll"
13466 );
13467 if is_poll && request_number <= behavior.poll_failures_per_path {
13468 return;
13469 }
13470 if path == "/api/worker/heartbeat" && request_number <= behavior.heartbeat_failures {
13471 return;
13472 }
13473 if path == "/api/worker/heartbeat"
13474 && behavior.heartbeat_failure_request == Some(request_number)
13475 {
13476 return;
13477 }
13478 if path == "/api/worker/heartbeat"
13479 && behavior.delayed_heartbeat_request == Some(request_number)
13480 {
13481 thread::sleep(behavior.heartbeat_response_delay);
13482 }
13483 if behavior.unauthorized_polls && is_poll {
13484 write_mock_response(
13485 stream,
13486 "401 Unauthorized",
13487 r#"{"reason":"authentication_failed","message":"invalid worker token"}"#,
13488 );
13489 return;
13490 }
13491 if behavior.draining_polls && is_poll {
13492 write_mock_response(
13493 stream,
13494 "409 Conflict",
13495 r#"{"task":null,"poll_status":"draining","reason":"worker_draining","worker_status":"draining","drain_intent":"draining"}"#,
13496 );
13497 return;
13498 }
13499
13500 if behavior.reject_query_protocol && path.starts_with("/api/worker/query-tasks/") {
13501 let requested_version = worker_protocol.as_deref().unwrap_or("missing");
13502 let body = format!(
13503 r#"{{"reason":"unsupported_protocol_version","message":"Query tasks require worker protocol 1.8 or newer.","supported_version":"1.7","requested_version":"{requested_version}"}}"#
13504 );
13505 write_mock_response(stream, "400 Bad Request", &body);
13506 return;
13507 }
13508
13509 if behavior.reject_query_completion && path == "/api/worker/query-tasks/query-late/complete"
13510 {
13511 write_mock_response(
13512 stream,
13513 "409 Conflict",
13514 r#"{"reason":"query_task_timed_out","message":"query task timed out before completion"}"#,
13515 );
13516 return;
13517 }
13518
13519 if behavior.workflow_completion_status.is_some()
13520 && path == "/api/worker/workflow-tasks/poll"
13521 && request_number == 1
13522 {
13523 write_mock_response(
13524 stream,
13525 "200 OK",
13526 r#"{"task":{"task_id":"workflow-timeout-task","workflow_id":"reused-workflow-id","run_id":"run-selected-timeout","workflow_type":"timeout.workflow","payload_codec":"json","arguments":{"codec":"json","blob":"[]"},"history_events":[],"workflow_task_attempt":3,"lease_owner":"timeout-worker"}}"#,
13527 );
13528 return;
13529 }
13530
13531 if path == "/api/worker/workflow-tasks/workflow-timeout-task/complete" {
13532 if let (Some(status), Some(body)) = (
13533 behavior.workflow_completion_status,
13534 behavior.workflow_completion_body,
13535 ) {
13536 write_mock_response(stream, status, body);
13537 return;
13538 }
13539 }
13540
13541 if behavior.waiting_query_worker {
13542 if behavior.complete_named_signal
13543 && path == "/api/worker/workflow-tasks/poll"
13544 && request_number == 1
13545 {
13546 let body = json!({
13547 "task": {
13548 "task_id": "snapshot-open",
13549 "workflow_id": "snapshot-1",
13550 "run_id": "snapshot-run-1",
13551 "workflow_type": "snapshot",
13552 "payload_codec": DEFAULT_CODEC,
13553 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
13554 .expect("Avro workflow arguments"),
13555 "history_events": [],
13556 "workflow_task_attempt": 1,
13557 "lease_owner": "rust-snapshot-worker"
13558 }
13559 })
13560 .to_string();
13561 write_mock_response(stream, "200 OK", &body);
13562 return;
13563 }
13564
13565 let signal_request = request_number - usize::from(behavior.complete_named_signal);
13566 let signal_request_limit = 2 + usize::from(behavior.complete_named_signal);
13567 if path == "/api/worker/workflow-tasks/poll"
13568 && signal_request >= 1
13569 && signal_request <= signal_request_limit
13570 {
13571 let finish = behavior.complete_named_signal && signal_request == 3;
13572 let amounts = if signal_request == 1 {
13573 vec![3]
13574 } else {
13575 vec![3, 5]
13576 };
13577 let task_id = if signal_request == 1 {
13578 "snapshot-wait-3"
13579 } else if finish {
13580 "snapshot-finish"
13581 } else {
13582 "snapshot-wait-5"
13583 };
13584 let mut history_events = std::iter::once(json!({
13585 "event_type": "SignalWaitOpened",
13586 "payload": {"sequence": 1, "signal_name": "finish"}
13587 }))
13588 .chain(amounts.iter().enumerate().map(|(index, amount)| {
13589 json!({
13590 "event_type": "SignalReceived",
13591 "payload": {
13592 "signal_id": format!("increment-{amount}"),
13593 "signal_name": "increment",
13594 "workflow_sequence": index + 2,
13595 "payload_codec": DEFAULT_CODEC,
13596 "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
13597 .expect("Avro signal envelope")
13598 }
13599 })
13600 }))
13601 .collect::<Vec<_>>();
13602 let (resume_id, resume_name, resume_arguments) = if finish {
13603 history_events.push(json!({
13604 "event_type": "SignalReceived",
13605 "payload": {
13606 "signal_id": "finish",
13607 "signal_name": "finish",
13608 "workflow_sequence": 4,
13609 "payload_codec": DEFAULT_CODEC,
13610 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
13611 .expect("Avro finish signal envelope")
13612 }
13613 }));
13614 (
13615 "finish".to_string(),
13616 "finish".to_string(),
13617 encode_value_envelope(&json!([]), DEFAULT_CODEC)
13618 .expect("Avro finish resume signal"),
13619 )
13620 } else {
13621 let amount = amounts.last().expect("amount");
13622 (
13623 format!("increment-{amount}"),
13624 "increment".to_string(),
13625 encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
13626 .expect("Avro increment resume signal"),
13627 )
13628 };
13629 let body = json!({
13630 "task": {
13631 "task_id": task_id,
13632 "workflow_id": "snapshot-1",
13633 "run_id": "snapshot-run-1",
13634 "workflow_type": "snapshot",
13635 "payload_codec": DEFAULT_CODEC,
13636 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
13637 .expect("Avro workflow arguments"),
13638 "history_events": history_events,
13639 "workflow_task_attempt": 1,
13640 "workflow_signal_id": resume_id,
13641 "signal_name": resume_name,
13642 "signal_arguments": resume_arguments,
13643 "lease_owner": "rust-snapshot-worker"
13644 }
13645 })
13646 .to_string();
13647 write_mock_response(stream, "200 OK", &body);
13648 return;
13649 }
13650
13651 if path == "/api/worker/query-tasks/poll" && request_number == 1 {
13652 let history_events = [3, 5]
13653 .into_iter()
13654 .enumerate()
13655 .map(|(index, amount)| {
13656 json!({
13657 "event_type": "SignalReceived",
13658 "payload": {
13659 "signal_id": format!("increment-{amount}"),
13660 "signal_name": "increment",
13661 "workflow_sequence": index + 2,
13662 "payload_codec": DEFAULT_CODEC,
13663 "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
13664 .expect("Avro query signal envelope")
13665 }
13666 })
13667 })
13668 .collect::<Vec<_>>();
13669 let body = json!({
13670 "task": {
13671 "query_task_id": "snapshot-current",
13672 "query_task_attempt": 1,
13673 "lease_owner": "rust-snapshot-worker",
13674 "workflow_id": "snapshot-1",
13675 "run_id": "snapshot-run-1",
13676 "workflow_type": "snapshot",
13677 "query_name": "current",
13678 "payload_codec": DEFAULT_CODEC,
13679 "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
13680 .expect("Avro workflow arguments"),
13681 "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
13682 .expect("Avro query arguments"),
13683 "history_events": history_events,
13684 "run_status": "waiting"
13685 }
13686 })
13687 .to_string();
13688 write_mock_response(stream, "200 OK", &body);
13689 return;
13690 }
13691
13692 if path == "/api/worker/workflow-tasks/snapshot-wait-3/fail"
13693 || path == "/api/worker/workflow-tasks/snapshot-wait-5/fail"
13694 {
13695 write_mock_response(
13696 stream,
13697 "200 OK",
13698 r#"{"outcome":"waiting_for_history","recorded":true}"#,
13699 );
13700 return;
13701 }
13702
13703 if path == "/api/worker/workflow-tasks/snapshot-open/complete" {
13704 write_mock_response(stream, "200 OK", r#"{"outcome":"waiting","recorded":true}"#);
13705 return;
13706 }
13707
13708 if path == "/api/worker/workflow-tasks/snapshot-finish/complete" {
13709 write_mock_response(
13710 stream,
13711 "200 OK",
13712 r#"{"outcome":"completed","run_status":"completed","recorded":true}"#,
13713 );
13714 return;
13715 }
13716
13717 if path == "/api/worker/query-tasks/snapshot-current/complete" {
13718 write_mock_response(stream, "200 OK", r#"{"outcome":"completed"}"#);
13719 return;
13720 }
13721 }
13722
13723 if matches!(
13724 path,
13725 "/api/workflows/typed-1/query/inspect" | "/api/workflows/typed-1/update/replace"
13726 ) {
13727 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
13728 .expect("typed mock result");
13729 let body = json!({
13730 "result": typed_fidelity_probe().into_json().expect("result projection"),
13731 "result_envelope": result,
13732 })
13733 .to_string();
13734 write_mock_response(stream, "200 OK", &body);
13735 return;
13736 }
13737
13738 if path == "/api/workflows/typed-1" {
13739 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
13740 .expect("typed mock result");
13741 let body = json!({
13742 "workflow_id": "typed-1",
13743 "run_id": "run-typed-1",
13744 "workflow_type": "typed.echo",
13745 "status": "completed",
13746 "output": typed_fidelity_probe().into_json().expect("output projection"),
13747 "output_envelope": result,
13748 })
13749 .to_string();
13750 write_mock_response(stream, "200 OK", &body);
13751 return;
13752 }
13753
13754 let (status, body) = match path {
13755 "/api/workflows" => (
13756 "201 Created",
13757 r#"{"workflow_id":"wf-start-options","run_id":"run-start-options","workflow_type":"rust.timeout"}"#,
13758 ),
13759 "/api/worker/register" if behavior.waiting_query_worker => (
13760 "200 OK",
13761 r#"{"worker_id":"rust-snapshot-worker","registered":true,"heartbeat_interval_seconds":1}"#,
13762 ),
13763 "/api/worker/register" => (
13764 "200 OK",
13765 r#"{"worker_id":"mock-worker","registered":true,"heartbeat_interval_seconds":3600}"#,
13766 ),
13767 "/api/worker/heartbeat" => ("200 OK", "{}"),
13768 "/api/worker/activity-tasks/poll"
13769 if behavior.cancelled_activity && request_number == 1 =>
13770 {
13771 (
13772 "200 OK",
13773 r#"{"task":{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","activity_type":"cancel-aware","payload_codec":"json","arguments":{"codec":"json","blob":"[]"},"attempt_number":1,"lease_owner":"rust-cancel-worker"}}"#,
13774 )
13775 }
13776 "/api/worker/activity-tasks/poll" | "/api/worker/workflow-tasks/poll" => {
13777 ("200 OK", r#"{"task":null}"#)
13778 }
13779 "/api/worker/query-tasks/poll"
13780 if behavior.reject_query_completion && request_number == 1 =>
13781 {
13782 (
13783 "200 OK",
13784 r#"{"task":{"query_task_id":"query-late","query_task_attempt":1,"lease_owner":"late-worker","workflow_id":"counter-late","run_id":"run-late","workflow_type":"counter","query_name":"current","payload_codec":"json","workflow_arguments":{"codec":"json","blob":"[]"},"query_arguments":{"codec":"json","blob":"[]"},"history_events":[],"run_status":"running"}}"#,
13785 )
13786 }
13787 "/api/worker/query-tasks/poll" => ("200 OK", r#"{"task":null}"#),
13788 "/api/worker/query-tasks/query-capture/complete"
13789 | "/api/worker/query-tasks/query-capture/fail" => ("200 OK", "{}"),
13790 "/api/worker/activity-tasks/activity-cancel/heartbeat" => (
13791 "200 OK",
13792 r#"{"activity_attempt_id":"attempt-cancel","cancel_requested":true,"can_continue":false,"reason":"run_cancelled","run_closed_reason":"cancelled","heartbeat_recorded":false}"#,
13793 ),
13794 "/api/worker/activity-tasks/activity-cancel/complete" => (
13795 "409 Conflict",
13796 r#"{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","reason":"run_cancelled","cancel_requested":true,"can_continue":false,"run_closed_reason":"cancelled"}"#,
13797 ),
13798 "/api/worker/activity-tasks/activity-typed/complete"
13799 | "/api/worker/activity-tasks/activity-typed/fail"
13800 | "/api/workflows/typed-1/signal/changed" => ("200 OK", "{}"),
13801 "/api/workflows/counter-1/query/current" => (
13802 "200 OK",
13803 r#"{"workflow_id":"counter-1","query_name":"current","result":{"count":8},"result_envelope":{"codec":"json","blob":"{\"count\":8}"}}"#,
13804 ),
13805 "/api/workflows/counter-1/query/missing" => (
13806 "404 Not Found",
13807 r#"{"workflow_id":"counter-1","query_name":"missing","reason":"rejected_unknown_query","message":"unknown query"}"#,
13808 ),
13809 "/api/workflows/wf-lifecycle/cancel" => (
13810 "200 OK",
13811 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","reason":"cleanup requested","command_status":"accepted"}"#,
13812 ),
13813 "/api/workflows/wf-lifecycle/terminate" => (
13814 "200 OK",
13815 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","reason":"forced stop","command_status":"accepted"}"#,
13816 ),
13817 "/api/workflows/wf-lifecycle/runs/run-current/cancel" => (
13818 "200 OK",
13819 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","command_status":"accepted"}"#,
13820 ),
13821 "/api/workflows/wf-lifecycle/runs/run-current/terminate" => (
13822 "200 OK",
13823 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","command_status":"accepted"}"#,
13824 ),
13825 "/api/workflows/wf-lifecycle/runs/run-stale/cancel"
13826 | "/api/workflows/wf-lifecycle/runs/run-stale/terminate" => (
13827 "409 Conflict",
13828 r#"{"workflow_id":"wf-lifecycle","run_id":"run-stale","reason":"historical_run_command_rejected","target_scope":"run","message":"Commands cannot target historical runs."}"#,
13829 ),
13830 "/api/workflows/wf-failed" | "/api/workflows/wf-failed/runs/run-failed" => (
13831 "200 OK",
13832 r#"{"workflow_id":"wf-failed","run_id":"run-failed","status":"failed","closed_reason":"failed","error":"payment failed","failure":{"message":"payment failed","failure_category":"application","exception_type":"PaymentError","exception_class":"billing::PaymentError","non_retryable":true,"exception":{"type":"PaymentError","class":"billing::PaymentError","message":"payment failed"},"failures":[{"id":"failure-17","failure_category":"application"}]}}"#,
13833 ),
13834 "/api/workflows/wf-cancelled" => (
13835 "200 OK",
13836 r#"{"workflow_id":"wf-cancelled","run_id":"run-cancelled","status":"cancelled","closed_reason":"cancelled","reason":"cleanup requested"}"#,
13837 ),
13838 "/api/workflows/wf-terminated" => (
13839 "200 OK",
13840 r#"{"workflow_id":"wf-terminated","run_id":"run-terminated","status":"terminated","closed_reason":"terminated","reason":"forced stop"}"#,
13841 ),
13842 "/api/workflows/wf-timed-out" => (
13843 "200 OK",
13844 r#"{"workflow_id":"wf-timed-out","run_id":"run-timed-out","status":"failed","closed_reason":"timed_out","reason":"run_timeout"}"#,
13845 ),
13846 "/api/workflows/wf-waiting" | "/api/workflows/wf-waiting/runs/run-waiting" => (
13847 "200 OK",
13848 r#"{"workflow_id":"wf-waiting","run_id":"run-waiting","status":"waiting"}"#,
13849 ),
13850 "/api/workflows/wf-selected" => (
13851 "200 OK",
13852 r#"{"workflow_id":"wf-selected","run_id":"run-current","status":"completed","output":"current run output"}"#,
13853 ),
13854 "/api/workflows/wf-selected/runs/run-selected" => (
13855 "200 OK",
13856 r#"{"workflow_id":"wf-selected","run_id":"run-selected","status":"cancelled","closed_reason":"cancelled","reason":"selected run cancelled"}"#,
13857 ),
13858 _ => ("404 Not Found", r#"{"message":"not found"}"#),
13859 };
13860 write_mock_response(stream, status, body);
13861 }
13862
13863 fn mock_request_is_complete(request: &[u8]) -> bool {
13864 let Some(header_end) = request
13865 .windows(4)
13866 .position(|window| window == b"\r\n\r\n")
13867 .map(|position| position + 4)
13868 else {
13869 return false;
13870 };
13871 let headers = String::from_utf8_lossy(&request[..header_end]);
13872 let content_length = headers.lines().find_map(|line| {
13873 let (name, value) = line.split_once(':')?;
13874 name.eq_ignore_ascii_case("content-length")
13875 .then(|| value.trim().parse::<usize>().ok())
13876 .flatten()
13877 });
13878
13879 request.len() >= header_end + content_length.unwrap_or(0)
13880 }
13881
13882 fn write_mock_response(stream: &mut TcpStream, status: &str, body: &str) {
13883 let response = format!(
13884 "HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
13885 body.len()
13886 );
13887
13888 let _ = stream.write_all(response.as_bytes());
13889 let _ = stream.flush();
13890 }
13891}