1#![doc = include_str!("../README.md")]
2
3mod runtime_payloads;
4mod runtime_uploads;
5
6use std::{
7 any::{type_name, Any, TypeId},
8 collections::{BTreeMap, HashMap},
9 future::Future,
10 io::{self, Read},
11 pin::Pin,
12 sync::{
13 atomic::{AtomicBool, Ordering},
14 Arc, Mutex, OnceLock,
15 },
16 task::{Context as TaskContext, Poll},
17 time::{Duration, Instant, SystemTime, UNIX_EPOCH},
18};
19
20use apache_avro::{from_avro_datum, to_avro_datum, types::Value as AvroDatum, Schema};
21use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
22use chrono::DateTime;
23use futures_util::{future::OptionFuture, task::noop_waker_ref};
24use serde::{
25 de::DeserializeOwned,
26 ser::{SerializeMap, SerializeSeq},
27 Deserialize, Deserializer, Serialize, Serializer,
28};
29pub use serde_json::{json, Value};
30use sha2::{Digest, Sha256};
31use thiserror::Error;
32pub use uuid::Uuid;
33
34pub const WORKER_PROTOCOL_VERSION: &str = "1.19";
35pub const PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION: &str = "1.18";
37pub const CONTROL_PLANE_VERSION: &str = "2";
38pub const DEFAULT_CODEC: &str = "avro";
39pub const SDK_VERSION: &str = concat!("durable-workflow-rust/", env!("CARGO_PKG_VERSION"));
40pub const CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY: &str =
42 "condition_wait_occurrence_identity";
43pub const MEMO_UPSERTS_CAPABILITY: &str = "memo_upserts";
45pub const QUERY_TASKS_CAPABILITY: &str = "query_tasks";
47pub const TYPED_SEARCH_ATTRIBUTES_CAPABILITY: &str = "typed_search_attributes";
49pub const WORKFLOW_UPDATES_CAPABILITY: &str = "workflow_updates";
51pub const MESSAGE_STREAMS_CAPABILITY: &str = "message_streams";
53pub const DURABLE_SELECTION_CAPABILITY: &str = "durable_selection";
55pub const MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.15";
56pub const MESSAGE_STREAM_SIGNAL: &str = "__durable_workflow_message_stream";
57pub const MESSAGE_STREAM_SCHEMA: &str = "durable-workflow.v2.message-stream.message";
58pub const MESSAGE_STREAM_CURSOR_SCHEMA: &str = "durable-workflow.v2.message-stream.cursor";
59pub const MESSAGE_STREAM_MAX_BATCH: usize = 100;
60pub const QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
62pub const SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
64pub const MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.14";
66pub const TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.16";
68pub const CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.9";
70pub const CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.17";
72pub const DURABLE_SELECTION_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.19";
74
75pub fn worker_protocol_supports_message_streams(version: &str) -> bool {
76 let Some((major, minor)) = version.split_once('.') else {
77 return false;
78 };
79 major == "1" && minor.parse::<u64>().is_ok_and(|minor| minor >= 15)
80}
81
82fn validate_user_signal_name(signal_name: &str) -> Result<()> {
83 if signal_name == MESSAGE_STREAM_SIGNAL {
84 return Err(Error::Codec(format!(
85 "signal name {MESSAGE_STREAM_SIGNAL:?} is reserved by the workflow runtime"
86 )));
87 }
88 Ok(())
89}
90
91const MAX_LONG_POLL_TIMEOUT_SECONDS: u64 = 60;
92const WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE: &str =
93 "Workflow task waiting for scheduled history.";
94const WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE: &str = "WorkflowTaskWaitingForHistory";
95const MISSING_TASK_PAYLOAD_CODEC: &str = "\0missing-task-payload-codec";
96const NULL_TASK_PAYLOAD_CODEC: &str = "\0null-task-payload-codec";
97const NON_STRING_TASK_PAYLOAD_CODEC: &str = "\0non-string-task-payload-codec";
98const MAX_MEMO_ENTRIES: usize = 100;
99const MAX_MEMO_VALUE_SIZE_BYTES: usize = 10_240;
100const MAX_MEMO_TOTAL_SIZE_BYTES: usize = 65_536;
101
102const QUERY_TASK_FINAL_REJECTION_REASONS: &[&str] = &[
103 "lease_expired",
104 "query_task_not_found",
105 "query_task_not_leased",
106 "query_task_timed_out",
107];
108
109pub fn portable_worker_affinity_capability_manifest() -> Value {
111 json!({
112 "local_activities": {
113 "supported": false,
114 "minimum_protocol_version": PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
115 "reason": "rust_worker_does_not_execute_record_local_activity",
116 },
117 "worker_sessions": {
118 "supported": false,
119 "minimum_protocol_version": PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
120 "reason": "rust_worker_has_no_typed_session_lifecycle",
121 },
122 "sticky_execution": {
123 "supported": false,
124 "minimum_protocol_version": PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
125 "reason": "rust_worker_uses_complete_durable_history_replay",
126 },
127 })
128}
129
130pub const AVRO_VALUE_SCHEMA_JSON: &str =
132 include_str!("../schema/durable_workflow.protocol.Value.v1.avsc");
133pub const AVRO_VALUE_SCHEMA_FINGERPRINT_HEX: &str = "e2a33dff55802237";
134pub const AVRO_VALUE_SCHEMA_FINGERPRINT: [u8; 8] = [0xe2, 0xa3, 0x3d, 0xff, 0x55, 0x80, 0x22, 0x37];
135const AVRO_SINGLE_OBJECT_MAGIC: [u8; 2] = [0xc3, 0x01];
136
137static AVRO_VALUE_SCHEMA: OnceLock<std::result::Result<Schema, String>> = OnceLock::new();
138static AVRO_VALUE_ORDERED_MAP_ENCODING_SCHEMA: OnceLock<std::result::Result<Schema, String>> =
139 OnceLock::new();
140
141#[derive(Clone, Copy)]
142enum RequestProtocol {
143 ControlPlane,
144 Worker(&'static str),
145}
146
147pub type Result<T> = std::result::Result<T, Error>;
148
149#[derive(Debug, Error)]
150pub enum Error {
151 #[error("transport error: {0}")]
152 Transport(#[from] reqwest::Error),
153 #[error(
154 "invalid Durable Workflow base URL: omit the SDK-owned /api suffix and pass the Server or Cloud runtime base URL; the SDK appends /api automatically"
155 )]
156 InvalidBaseUrl,
157 #[error("json error: {0}")]
158 Json(#[from] serde_json::Error),
159 #[error("http {status}: {body}")]
160 Http {
161 status: reqwest::StatusCode,
162 body: String,
163 },
164 #[error("codec error: {0}")]
165 Codec(String),
166 #[error(transparent)]
167 QueryFailed(QueryFailure),
168 #[error(transparent)]
169 Protocol(ProtocolFailure),
170 #[error(transparent)]
171 NonDeterministicReplay(ReplayFailure),
172 #[error(transparent)]
173 ChildWorkflowFailed(ChildWorkflowFailure),
174 #[error(transparent)]
175 ActivityFailed(ActivityFailure),
176 #[error(transparent)]
177 ParallelFailed(ParallelFailure),
178 #[error(transparent)]
179 SagaCompensationFailed(SagaCompensationFailure),
180 #[error(transparent)]
181 InvalidParallelGroup(ParallelGroupError),
182 #[error(transparent)]
183 DurableOperationCancelled(DurableOperationCancelled),
184 #[error(transparent)]
185 WorkflowCancellationRequested(WorkflowCancellationRequested),
186 #[error(transparent)]
187 WorkflowCommandRejected(WorkflowCommandRejection),
188 #[error(transparent)]
189 WorkflowFailed(WorkflowTerminalOutcome),
190 #[error(transparent)]
191 WorkflowCancelled(WorkflowTerminalOutcome),
192 #[error(transparent)]
193 WorkflowTerminated(WorkflowTerminalOutcome),
194 #[error(transparent)]
195 WorkflowTimedOut(WorkflowTerminalOutcome),
196 #[error(transparent)]
197 ActivityTaskRejected(ActivityTaskRejection),
198 #[error("workflow handler {0:?} is not registered")]
199 WorkflowNotRegistered(String),
200 #[error("activity handler {0:?} is not registered")]
201 ActivityNotRegistered(String),
202 #[error(
203 "{handler_kind} handler {handler_name:?} {value_kind} type {rust_type} is incompatible with the fixed Avro Value codec: {message}"
204 )]
205 HandlerType {
206 handler_kind: HandlerKind,
207 handler_name: String,
208 value_kind: HandlerValueKind,
209 rust_type: &'static str,
210 message: String,
211 },
212 #[error("workflow future yielded without emitting a durable command")]
213 WorkflowYieldedWithoutCommand,
214 #[error(
215 "workflow_stream_command_identity_missing: workflow stream authoring requires a non-empty server-provided workflow_command_id"
216 )]
217 MissingWorkflowCommandIdentity,
218 #[error("workflow state lock is poisoned")]
219 WorkflowStatePoisoned,
220 #[error("timer duration is too large for the worker protocol")]
221 TimerDurationOverflow,
222 #[error(transparent)]
223 InvalidConditionWaitOptions(#[from] ConditionWaitOptionsError),
224 #[error(transparent)]
225 InvalidSearchAttributeUpdate(#[from] SearchAttributeUpdateError),
226 #[error("operation timed out")]
227 Timeout,
228 #[error(
229 "missing {role}-plane credentials: configure ClientBuilder::{role}_token or ClientBuilder::token; a {opposite_role}-plane token cannot authorize this request"
230 )]
231 MissingRoleCredentials {
232 role: &'static str,
233 opposite_role: &'static str,
234 },
235 #[error("worker loop error: {0}")]
236 WorkerLoop(String),
237 #[error(
238 "workflow command contract for {workflow_type:?} declares update validators, but this Rust SDK cannot execute synchronous pre-accept update validation"
239 )]
240 UnsupportedUpdateValidators { workflow_type: String },
241 #[error("{primary}; worker deregistration also failed: {deregistration}")]
242 WorkerShutdown {
243 primary: Box<Error>,
244 deregistration: Box<Error>,
245 },
246 #[error("invalid child workflow options: {0}")]
247 InvalidChildWorkflowOptions(String),
248 #[error("invalid workflow memo update: {0}")]
249 InvalidMemoUpdate(String),
250 #[error(
251 "workflow_memo_updates_unavailable: the connected runtime did not advertise workflow memo update support"
252 )]
253 WorkflowMemoUpdatesUnavailable,
254 #[error(transparent)]
255 InvalidActivityOptions(ActivityOptionsError),
256 #[error(transparent)]
257 InvalidContinueAsNewOptions(#[from] ContinueAsNewOptionsError),
258 #[doc(hidden)]
259 #[error("workflow requested continue as new")]
260 ContinueAsNew(ContinueAsNewRequest),
261}
262
263#[derive(Clone, Debug, Error, PartialEq, Eq)]
265pub enum ConditionWaitOptionsError {
266 #[error("condition_key must be non-empty")]
267 EmptyKey,
268 #[error("condition_definition_fingerprint must be non-empty")]
269 EmptyPredicateIdentity,
270 #[error("condition timeout is too large for the worker protocol")]
271 TimeoutOverflow,
272}
273
274#[derive(Clone, Debug, PartialEq, Eq)]
281pub struct ConditionWaitOptions {
282 condition_key: String,
283 predicate_identity: String,
284 timeout: Option<Duration>,
285}
286
287impl ConditionWaitOptions {
288 pub fn new(condition_key: impl Into<String>, predicate_identity: impl Into<String>) -> Self {
289 Self {
290 condition_key: condition_key.into(),
291 predicate_identity: predicate_identity.into(),
292 timeout: None,
293 }
294 }
295
296 pub fn timeout(mut self, timeout: Duration) -> Self {
297 self.timeout = Some(timeout);
298 self
299 }
300
301 fn validate(
302 &self,
303 ) -> std::result::Result<ValidatedConditionWaitOptions, ConditionWaitOptionsError> {
304 let condition_key = self.condition_key.trim();
305 if condition_key.is_empty() {
306 return Err(ConditionWaitOptionsError::EmptyKey);
307 }
308 let predicate_identity = self.predicate_identity.trim();
309 if predicate_identity.is_empty() {
310 return Err(ConditionWaitOptionsError::EmptyPredicateIdentity);
311 }
312 let timeout_seconds = self
313 .timeout
314 .map(|timeout| {
315 timeout
316 .as_secs()
317 .checked_add(u64::from(timeout.subsec_nanos() > 0))
318 .ok_or(ConditionWaitOptionsError::TimeoutOverflow)
319 })
320 .transpose()?;
321
322 Ok(ValidatedConditionWaitOptions {
323 condition_key: condition_key.to_string(),
324 predicate_identity: predicate_identity.to_string(),
325 timeout_seconds,
326 })
327 }
328}
329
330#[derive(Clone, Debug, PartialEq, Eq)]
331struct ValidatedConditionWaitOptions {
332 condition_key: String,
333 predicate_identity: String,
334 timeout_seconds: Option<u64>,
335}
336
337const CONDITION_WAIT_OCCURRENCE_PREFIX: &str = "rust:condition-wait:";
338
339#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
341#[serde(rename_all = "snake_case")]
342pub enum ConditionWaitResult {
343 Satisfied,
344 TimedOut,
345}
346
347impl ConditionWaitResult {
348 pub fn is_satisfied(self) -> bool {
349 self == Self::Satisfied
350 }
351
352 pub fn is_timed_out(self) -> bool {
353 self == Self::TimedOut
354 }
355}
356
357#[doc(hidden)]
359pub fn __condition_definition_fingerprint(source: &str) -> String {
360 let mut digest = Sha256::new();
361 digest.update(b"durable-workflow-rust.wait-condition.v1\0");
362 digest.update(source.as_bytes());
363 format!("sha256:{:x}", digest.finalize())
364}
365
366#[macro_export]
372macro_rules! wait_condition {
373 ($ctx:expr, $key:expr, timeout: $timeout:expr, $predicate:expr $(,)?) => {{
374 $ctx.wait_condition(
375 $crate::ConditionWaitOptions::new(
376 $key,
377 $crate::__condition_definition_fingerprint(concat!(
378 module_path!(),
379 "\0",
380 stringify!($predicate)
381 )),
382 )
383 .timeout($timeout),
384 $predicate,
385 )
386 }};
387 ($ctx:expr, $key:expr, $predicate:expr $(,)?) => {{
388 $ctx.wait_condition(
389 $crate::ConditionWaitOptions::new(
390 $key,
391 $crate::__condition_definition_fingerprint(concat!(
392 module_path!(),
393 "\0",
394 stringify!($predicate)
395 )),
396 ),
397 $predicate,
398 )
399 }};
400}
401
402const MAX_SEARCH_ATTRIBUTES_PER_UPDATE: usize = 100;
403const MAX_SEARCH_ATTRIBUTE_KEY_LENGTH: usize = 64;
404const MAX_SEARCH_ATTRIBUTE_STRING_LENGTH: usize = 2_048;
405const MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH: usize = 255;
406const MAX_SEARCH_ATTRIBUTE_UPDATE_BYTES: usize = 65_536;
407
408#[derive(Clone, Debug, Error, PartialEq, Eq)]
410pub enum SearchAttributeUpdateError {
411 #[error("search-attribute update requires at least one attribute")]
412 Empty,
413 #[error("search attribute key {0:?} must be 1-64 URL-safe ASCII characters")]
414 InvalidKey(String),
415 #[error("search-attribute update exceeds the limit of 100 attributes")]
416 TooManyAttributes,
417 #[error("search attribute {key:?} {kind} value exceeds {limit} bytes")]
418 ValueTooLong {
419 key: String,
420 kind: &'static str,
421 limit: usize,
422 },
423 #[error(
424 "search attribute {0:?} must not contain an empty string value; use delete() to remove it"
425 )]
426 EmptyString(String),
427 #[error("search attribute {0:?} has a non-finite float value")]
428 NonFiniteFloat(String),
429 #[error("search attribute {0:?} must use an RFC 3339 datetime with an explicit timezone")]
430 InvalidDateTime(String),
431 #[error("search-attribute update exceeds the 65536-byte protocol limit")]
432 PayloadTooLarge,
433}
434
435#[derive(Clone, Debug, PartialEq)]
437pub enum SearchAttributeValue {
438 String(String),
439 Keyword(String),
440 KeywordList(Vec<String>),
441 Int(i64),
442 Float(f64),
443 Bool(bool),
444 DateTime(String),
445 Delete,
446}
447
448impl SearchAttributeValue {
449 fn type_name(&self) -> Option<&'static str> {
450 match self {
451 Self::String(_) => Some("string"),
452 Self::Keyword(_) => Some("keyword"),
453 Self::KeywordList(_) => Some("keyword_list"),
454 Self::Int(_) => Some("int"),
455 Self::Float(_) => Some("float"),
456 Self::Bool(_) => Some("bool"),
457 Self::DateTime(_) => Some("datetime"),
458 Self::Delete => None,
459 }
460 }
461
462 fn normalized(self, key: &str) -> std::result::Result<Self, SearchAttributeUpdateError> {
463 let normalize_string = |value: String, kind: &'static str, limit: usize| {
464 let value = value.trim().to_string();
465 if value.is_empty() {
466 return Err(SearchAttributeUpdateError::EmptyString(key.to_string()));
467 }
468 if value.len() > limit {
469 return Err(SearchAttributeUpdateError::ValueTooLong {
470 key: key.to_string(),
471 kind,
472 limit,
473 });
474 }
475 Ok(value)
476 };
477
478 match self {
479 Self::String(value) => Ok(Self::String(normalize_string(
480 value,
481 "string",
482 MAX_SEARCH_ATTRIBUTE_STRING_LENGTH,
483 )?)),
484 Self::Keyword(value) => Ok(Self::Keyword(normalize_string(
485 value,
486 "keyword",
487 MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH,
488 )?)),
489 Self::KeywordList(values) => {
490 let values = values
491 .into_iter()
492 .map(|value| {
493 let value = value.trim().to_string();
494 if value.len() > MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH {
495 return Err(SearchAttributeUpdateError::ValueTooLong {
496 key: key.to_string(),
497 kind: "keyword-list entry",
498 limit: MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH,
499 });
500 }
501 Ok(value)
502 })
503 .collect::<std::result::Result<Vec<_>, _>>()?;
504 Ok(Self::KeywordList(values))
505 }
506 Self::Float(value) if !value.is_finite() => {
507 Err(SearchAttributeUpdateError::NonFiniteFloat(key.to_string()))
508 }
509 Self::DateTime(value) => {
510 let value =
511 normalize_string(value, "datetime", MAX_SEARCH_ATTRIBUTE_STRING_LENGTH)?;
512 if DateTime::parse_from_rfc3339(&value).is_err() {
513 return Err(SearchAttributeUpdateError::InvalidDateTime(key.to_string()));
514 }
515 Ok(Self::DateTime(value))
516 }
517 value => Ok(value),
518 }
519 }
520
521 fn into_json(self) -> Value {
522 match self {
523 Self::String(value) | Self::Keyword(value) | Self::DateTime(value) => {
524 Value::String(value)
525 }
526 Self::KeywordList(values) => {
527 Value::Array(values.into_iter().map(Value::String).collect())
528 }
529 Self::Int(value) => json!(value),
530 Self::Float(value) => json!(value),
531 Self::Bool(value) => json!(value),
532 Self::Delete => Value::Null,
533 }
534 }
535}
536
537#[derive(Clone, Debug, Default, PartialEq)]
539pub struct SearchAttributeUpdate {
540 attributes: BTreeMap<String, SearchAttributeValue>,
541}
542
543impl SearchAttributeUpdate {
544 pub fn new() -> Self {
545 Self::default()
546 }
547
548 pub fn set(
549 mut self,
550 key: impl Into<String>,
551 value: SearchAttributeValue,
552 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
553 let key = key.into();
554 validate_search_attribute_key(&key)?;
555 if !self.attributes.contains_key(&key)
556 && self.attributes.len() >= MAX_SEARCH_ATTRIBUTES_PER_UPDATE
557 {
558 return Err(SearchAttributeUpdateError::TooManyAttributes);
559 }
560 self.attributes.insert(key.clone(), value.normalized(&key)?);
561 self.validate_size()?;
562 Ok(self)
563 }
564
565 pub fn string(
566 self,
567 key: impl Into<String>,
568 value: impl Into<String>,
569 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
570 self.set(key, SearchAttributeValue::String(value.into()))
571 }
572
573 pub fn keyword(
574 self,
575 key: impl Into<String>,
576 value: impl Into<String>,
577 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
578 self.set(key, SearchAttributeValue::Keyword(value.into()))
579 }
580
581 pub fn keyword_list<I, V>(
582 self,
583 key: impl Into<String>,
584 values: I,
585 ) -> std::result::Result<Self, SearchAttributeUpdateError>
586 where
587 I: IntoIterator<Item = V>,
588 V: Into<String>,
589 {
590 self.set(
591 key,
592 SearchAttributeValue::KeywordList(values.into_iter().map(Into::into).collect()),
593 )
594 }
595
596 pub fn int(
597 self,
598 key: impl Into<String>,
599 value: i64,
600 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
601 self.set(key, SearchAttributeValue::Int(value))
602 }
603
604 pub fn float(
605 self,
606 key: impl Into<String>,
607 value: f64,
608 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
609 self.set(key, SearchAttributeValue::Float(value))
610 }
611
612 pub fn bool(
613 self,
614 key: impl Into<String>,
615 value: bool,
616 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
617 self.set(key, SearchAttributeValue::Bool(value))
618 }
619
620 pub fn datetime(
621 self,
622 key: impl Into<String>,
623 value: impl Into<String>,
624 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
625 self.set(key, SearchAttributeValue::DateTime(value.into()))
626 }
627
628 pub fn delete(
629 self,
630 key: impl Into<String>,
631 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
632 self.set(key, SearchAttributeValue::Delete)
633 }
634
635 fn validate_size(&self) -> std::result::Result<(), SearchAttributeUpdateError> {
636 let (attributes, _) = self.clone().into_wire_parts();
637 if serde_json::to_vec(&attributes)
638 .map(|payload| payload.len() > MAX_SEARCH_ATTRIBUTE_UPDATE_BYTES)
639 .unwrap_or(true)
640 {
641 return Err(SearchAttributeUpdateError::PayloadTooLarge);
642 }
643 Ok(())
644 }
645
646 fn into_wire_parts(self) -> (Value, BTreeMap<String, String>) {
647 let mut attributes = serde_json::Map::new();
648 let mut attribute_types = BTreeMap::new();
649 for (key, value) in self.attributes {
650 if let Some(type_name) = value.type_name() {
651 attribute_types.insert(key.clone(), type_name.to_string());
652 }
653 attributes.insert(key, value.into_json());
654 }
655 (Value::Object(attributes), attribute_types)
656 }
657
658 fn validate(&self) -> std::result::Result<(), SearchAttributeUpdateError> {
659 if self.attributes.is_empty() {
660 return Err(SearchAttributeUpdateError::Empty);
661 }
662 self.validate_size()
663 }
664}
665
666fn validate_search_attribute_key(key: &str) -> std::result::Result<(), SearchAttributeUpdateError> {
667 let valid = !key.is_empty()
668 && key.len() <= MAX_SEARCH_ATTRIBUTE_KEY_LENGTH
669 && key
670 .bytes()
671 .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-' | b':'));
672 if valid {
673 Ok(())
674 } else {
675 Err(SearchAttributeUpdateError::InvalidKey(key.to_string()))
676 }
677}
678
679#[derive(Clone, Copy, Debug, PartialEq, Eq)]
681pub enum HandlerKind {
682 Workflow,
683 Activity,
684}
685
686impl std::fmt::Display for HandlerKind {
687 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
688 formatter.write_str(match self {
689 Self::Workflow => "workflow",
690 Self::Activity => "activity",
691 })
692 }
693}
694
695#[derive(Clone, Copy, Debug, PartialEq, Eq)]
697pub enum HandlerValueKind {
698 Input,
699 Result,
700}
701
702impl std::fmt::Display for HandlerValueKind {
703 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
704 formatter.write_str(match self {
705 Self::Input => "input",
706 Self::Result => "result",
707 })
708 }
709}
710
711#[derive(Clone, Copy, Debug, PartialEq, Eq)]
713pub enum WorkflowCommandKind {
714 Cancel,
715 Terminate,
716}
717
718impl WorkflowCommandKind {
719 fn as_str(self) -> &'static str {
720 match self {
721 Self::Cancel => "cancel",
722 Self::Terminate => "terminate",
723 }
724 }
725}
726
727#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
729pub struct WorkflowCommandOptions {
730 #[serde(skip_serializing_if = "Option::is_none")]
731 pub reason: Option<String>,
732 #[serde(skip_serializing_if = "Option::is_none")]
733 pub request_id: Option<String>,
734}
735
736#[derive(Clone, Debug, PartialEq, Eq)]
743pub struct WorkflowStartOptions {
744 pub execution_timeout_seconds: u64,
745 pub run_timeout_seconds: u64,
746}
747
748impl Default for WorkflowStartOptions {
749 fn default() -> Self {
750 Self {
751 execution_timeout_seconds: 3600,
752 run_timeout_seconds: 600,
753 }
754 }
755}
756
757impl WorkflowStartOptions {
758 pub fn new() -> Self {
759 Self::default()
760 }
761
762 pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
763 self.execution_timeout_seconds = seconds;
764 self
765 }
766
767 pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
768 self.run_timeout_seconds = seconds;
769 self
770 }
771
772 fn validate(&self) -> Result<()> {
773 if self.execution_timeout_seconds == 0 {
774 return Err(Error::Codec(
775 "execution_timeout_seconds must be at least 1".to_string(),
776 ));
777 }
778 if self.run_timeout_seconds == 0 {
779 return Err(Error::Codec(
780 "run_timeout_seconds must be at least 1".to_string(),
781 ));
782 }
783 if self.run_timeout_seconds > self.execution_timeout_seconds {
784 return Err(Error::Codec(
785 "run_timeout_seconds cannot exceed execution_timeout_seconds".to_string(),
786 ));
787 }
788
789 Ok(())
790 }
791}
792
793#[derive(Clone, Debug, Default, PartialEq, Eq)]
798pub struct ContinueAsNewOptions {
799 pub workflow_type: Option<String>,
800 pub task_queue: Option<String>,
801}
802
803impl ContinueAsNewOptions {
804 pub fn new() -> Self {
805 Self::default()
806 }
807
808 pub fn workflow_type(mut self, workflow_type: impl Into<String>) -> Self {
809 self.workflow_type = Some(workflow_type.into());
810 self
811 }
812
813 pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
814 self.task_queue = Some(task_queue.into());
815 self
816 }
817
818 fn validate(&self) -> std::result::Result<(), ContinueAsNewOptionsError> {
819 for (field, value) in [
820 ("workflow_type", self.workflow_type.as_deref()),
821 ("task_queue", self.task_queue.as_deref()),
822 ] {
823 if value.is_some_and(|value| value.trim().is_empty()) {
824 return Err(ContinueAsNewOptionsError {
825 field,
826 message: format!("{field} must not be empty"),
827 });
828 }
829 }
830 Ok(())
831 }
832}
833
834#[derive(Clone, Debug, Error, PartialEq, Eq)]
836#[error("invalid continue-as-new option {field}: {message}")]
837pub struct ContinueAsNewOptionsError {
838 pub field: &'static str,
839 pub message: String,
840}
841
842#[derive(Clone, Debug, Default, PartialEq, Eq)]
844pub struct WorkflowHistoryBudget {
845 pub event_count: u64,
846 pub size_bytes: Option<u64>,
847 pub continue_as_new_recommended: bool,
848 pub pressure: Option<String>,
849}
850
851#[doc(hidden)]
852#[derive(Clone, Debug)]
853pub struct ContinueAsNewRequest {
854 arguments: AvroValue,
855 options: ContinueAsNewOptions,
856}
857
858impl WorkflowCommandOptions {
859 pub fn new() -> Self {
860 Self::default()
861 }
862
863 pub fn reason(mut self, reason: impl Into<String>) -> Self {
864 self.reason = Some(reason.into());
865 self
866 }
867
868 pub fn request_id(mut self, request_id: impl Into<String>) -> Self {
869 self.request_id = Some(request_id.into());
870 self
871 }
872}
873
874#[derive(Clone, Debug, PartialEq)]
876pub struct WorkflowCommandResult {
877 pub command: WorkflowCommandKind,
878 pub workflow_id: String,
879 pub run_id: Option<String>,
880 pub outcome: Option<String>,
881 pub reason: Option<String>,
882 pub command_status: Option<String>,
883 pub raw: Value,
884}
885
886#[derive(Clone, Debug, Error)]
888#[error("workflow {command:?} rejected ({reason}, HTTP {status}): {message}")]
889pub struct WorkflowCommandRejection {
890 pub command: WorkflowCommandKind,
891 pub status: u16,
892 pub reason: String,
893 pub message: String,
894 pub workflow_id: String,
895 pub run_id: Option<String>,
896 pub target_scope: Option<String>,
897 pub body: Value,
898}
899
900#[derive(Clone, Copy, Debug, PartialEq, Eq)]
902pub enum WorkflowTerminalKind {
903 Failed,
904 Cancelled,
905 Terminated,
906 TimedOut,
907}
908
909#[derive(Clone, Debug, Error)]
915#[error("workflow {workflow_id} run {run_id:?} ended as {kind:?} ({reason})")]
916pub struct WorkflowTerminalOutcome {
917 pub kind: WorkflowTerminalKind,
918 pub workflow_id: String,
919 pub run_id: Option<String>,
920 pub reason: String,
921 pub failure_category: Option<String>,
922 pub failure_id: Option<String>,
923 pub exception_type: Option<String>,
924 pub exception_class: Option<String>,
925 pub non_retryable: Option<bool>,
926 pub message: Option<String>,
927 pub exception: Option<Value>,
928 pub raw: Value,
929}
930
931#[derive(Clone, Debug, Error)]
933#[error("activity task {operation} rejected ({reason}, HTTP {status})")]
934pub struct ActivityTaskRejection {
935 pub operation: String,
936 pub status: u16,
937 pub reason: String,
938 pub task_id: String,
939 pub activity_attempt_id: String,
940 pub cancel_requested: bool,
941 pub can_continue: Option<bool>,
942 pub run_closed_reason: Option<String>,
943 pub body: Value,
944}
945
946#[derive(Clone, Copy, Debug, PartialEq, Eq)]
948pub enum ActivityOptionsErrorKind {
949 EmptyTaskQueue,
950 EmptyRetryPolicy,
951 InvalidMaxAttempts,
952 BackoffWithoutRetryBudget,
953 TooManyBackoffIntervals,
954 InvalidBackoffCoefficient,
955 BackoffGenerationTooLarge,
956 BackoffOverflow,
957 EmptyNonRetryableErrorType,
958 TimeoutNotPositive,
959 TimeoutOverflow,
960 TimeoutOrder,
961}
962
963#[derive(Clone, Debug, Error, PartialEq, Eq)]
965#[error("invalid activity options ({kind:?}, {field:?}): {message}")]
966pub struct ActivityOptionsError {
967 pub kind: ActivityOptionsErrorKind,
968 pub field: Option<&'static str>,
969 pub message: String,
970}
971
972impl ActivityOptionsError {
973 fn new(
974 kind: ActivityOptionsErrorKind,
975 field: Option<&'static str>,
976 message: impl Into<String>,
977 ) -> Self {
978 Self {
979 kind,
980 field,
981 message: message.into(),
982 }
983 }
984}
985
986#[derive(Clone, Copy, Debug, PartialEq, Eq)]
988pub enum ActivityFailureKind {
989 Failed,
990 Cancelled,
991 TimedOut,
992}
993
994#[derive(Clone, Debug, Error)]
999#[error("activity failed ({reason}): {message}")]
1000pub struct ActivityFailure {
1001 pub kind: ActivityFailureKind,
1002 pub reason: String,
1003 pub message: String,
1004 pub activity_execution_id: Option<String>,
1005 pub activity_attempt_id: Option<String>,
1006 pub activity_type: Option<String>,
1007 pub activity_class: Option<String>,
1008 pub attempt_number: Option<u64>,
1009 pub failure_id: Option<String>,
1010 pub failure_category: Option<String>,
1011 pub timeout_kind: Option<String>,
1012 pub non_retryable: bool,
1013 pub exception_type: Option<String>,
1014 pub exception_class: Option<String>,
1015 pub code: Option<Value>,
1016 pub exception: Option<Value>,
1017}
1018
1019#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1021pub enum ChildWorkflowFailureKind {
1022 Failed,
1023 Cancelled,
1024 Terminated,
1025}
1026
1027#[derive(Clone, Debug, Error)]
1033#[error("child workflow failed ({reason}): {message}")]
1034pub struct ChildWorkflowFailure {
1035 pub kind: ChildWorkflowFailureKind,
1036 pub reason: String,
1037 pub message: String,
1038 pub parent_workflow_id: Option<String>,
1039 pub parent_workflow_run_id: Option<String>,
1040 pub child_workflow_id: Option<String>,
1041 pub child_workflow_run_id: Option<String>,
1042 pub child_workflow_type: Option<String>,
1043 pub failure_id: Option<String>,
1044 pub failure_category: Option<String>,
1045 pub exception_type: Option<String>,
1046 pub exception_class: Option<String>,
1047 pub non_retryable: bool,
1048 pub code: Option<Value>,
1049 pub exception: Option<Value>,
1050}
1051
1052#[derive(Clone, Debug, PartialEq, Eq)]
1054pub struct WorkflowIdentity {
1055 pub workflow_id: Option<String>,
1056 pub run_id: Option<String>,
1057}
1058
1059#[derive(Clone, Debug, PartialEq)]
1061pub struct ChildWorkflowResult {
1062 pub parent: WorkflowIdentity,
1063 pub child: WorkflowIdentity,
1064 pub child_workflow_type: Option<String>,
1065 pub result: Value,
1066}
1067
1068#[derive(Clone, Debug, PartialEq)]
1070pub struct ChildWorkflowAvroResult {
1071 pub parent: WorkflowIdentity,
1072 pub child: WorkflowIdentity,
1073 pub child_workflow_type: Option<String>,
1074 pub result: AvroValue,
1075}
1076
1077#[derive(Clone, Debug, Deserialize, Hash, PartialEq, Eq, Serialize)]
1079#[serde(untagged)]
1080pub enum SelectionKey {
1081 Index(usize),
1082 Name(String),
1083}
1084
1085impl From<usize> for SelectionKey {
1086 fn from(value: usize) -> Self {
1087 Self::Index(value)
1088 }
1089}
1090
1091impl From<String> for SelectionKey {
1092 fn from(value: String) -> Self {
1093 Self::Name(value)
1094 }
1095}
1096
1097impl From<&str> for SelectionKey {
1098 fn from(value: &str) -> Self {
1099 Self::Name(value.to_string())
1100 }
1101}
1102
1103#[derive(Clone, Debug, Error, PartialEq, Eq)]
1105#[error("selected {operation_kind} operation {operation_identity} was explicitly cancelled")]
1106pub struct DurableOperationCancelled {
1107 pub selection_group_id: String,
1108 pub member_key: SelectionKey,
1109 pub member_index: usize,
1110 pub operation_kind: String,
1111 pub operation_identity: String,
1112}
1113
1114#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
1120pub struct ParallelGroupMetadata {
1121 pub parallel_group_id: String,
1122 pub parallel_group_kind: String,
1123 pub parallel_group_base_sequence: u64,
1124 pub parallel_group_size: usize,
1125 pub parallel_group_index: usize,
1126 #[serde(default, skip_serializing_if = "Option::is_none")]
1127 pub parallel_group_mode: Option<String>,
1128 #[serde(default, skip_serializing_if = "Option::is_none")]
1129 pub selection_member_key: Option<SelectionKey>,
1130 #[serde(default, skip_serializing_if = "Option::is_none")]
1131 pub selection_member_index: Option<usize>,
1132 #[serde(default, skip_serializing_if = "Option::is_none")]
1133 pub selection_member_base_sequence: Option<u64>,
1134 #[serde(default, skip_serializing_if = "Option::is_none")]
1135 pub selection_member_size: Option<usize>,
1136 #[serde(default, skip_serializing_if = "Option::is_none")]
1137 pub selection_member_kind: Option<String>,
1138}
1139
1140#[derive(Clone, Debug, PartialEq)]
1142pub enum ParallelResult {
1143 Activity(Value),
1144 ChildWorkflow(ChildWorkflowResult),
1145 Timer,
1146 Signal(Vec<Value>),
1147 Condition(ConditionWaitResult),
1148 Group(Vec<ParallelResult>),
1149}
1150
1151#[derive(Clone, Debug, PartialEq)]
1153pub enum ParallelAvroResult {
1154 Activity(AvroValue),
1155 ChildWorkflow(ChildWorkflowAvroResult),
1156 Timer,
1157 Signal(Vec<AvroValue>),
1158 Condition(ConditionWaitResult),
1159 Group(Vec<ParallelAvroResult>),
1160}
1161
1162impl ParallelAvroResult {
1163 fn into_json_result(self) -> Result<ParallelResult> {
1164 match self {
1165 Self::Activity(value) => Ok(ParallelResult::Activity(value.into_json()?)),
1166 Self::ChildWorkflow(result) => Ok(ParallelResult::ChildWorkflow(ChildWorkflowResult {
1167 parent: result.parent,
1168 child: result.child,
1169 child_workflow_type: result.child_workflow_type,
1170 result: result.result.into_json()?,
1171 })),
1172 Self::Timer => Ok(ParallelResult::Timer),
1173 Self::Signal(values) => Ok(ParallelResult::Signal(
1174 values
1175 .into_iter()
1176 .map(AvroValue::into_json)
1177 .collect::<Result<Vec<_>>>()?,
1178 )),
1179 Self::Condition(result) => Ok(ParallelResult::Condition(result)),
1180 Self::Group(results) => Ok(ParallelResult::Group(
1181 results
1182 .into_iter()
1183 .map(Self::into_json_result)
1184 .collect::<Result<Vec<_>>>()?,
1185 )),
1186 }
1187 }
1188}
1189
1190#[derive(Clone, Debug, PartialEq)]
1192pub struct ParallelCompletion {
1193 pub member_path: Vec<usize>,
1194 pub result: ParallelResult,
1195}
1196
1197#[derive(Debug, Error)]
1205#[error("parallel group {group_id} member {member_path:?} failed: {cause}")]
1206pub struct ParallelFailure {
1207 pub group_id: String,
1208 pub member_path: Vec<usize>,
1209 pub group_path: Vec<ParallelGroupMetadata>,
1210 pub completed: Vec<ParallelCompletion>,
1211 #[source]
1212 pub cause: Box<Error>,
1213}
1214
1215#[derive(Clone, Debug, Error, PartialEq, Eq)]
1217#[error("invalid deterministic parallel group ({reason}): {message}")]
1218pub struct ParallelGroupError {
1219 pub reason: &'static str,
1220 pub member_path: Vec<usize>,
1221 pub message: String,
1222}
1223
1224#[derive(Clone, Debug, Error, PartialEq, Eq)]
1226#[error("workflow cancellation was requested")]
1227pub struct WorkflowCancellationRequested;
1228
1229#[derive(Debug, Error)]
1231#[error(
1232 "saga forward execution failed; compensation activity {compensation_activity_type} (registration {compensation_registration_order}) also failed: {compensation_failure}"
1233)]
1234pub struct SagaCompensationFailure {
1235 pub initiating_failure: Box<Error>,
1236 pub compensation_failure: Box<Error>,
1237 pub compensation_activity_type: String,
1238 pub compensation_registration_order: usize,
1239}
1240
1241#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
1243pub enum ParentClosePolicy {
1244 #[default]
1245 Abandon,
1246 RequestCancel,
1247 Terminate,
1248}
1249
1250impl ParentClosePolicy {
1251 fn as_str(self) -> &'static str {
1252 match self {
1253 Self::Abandon => "abandon",
1254 Self::RequestCancel => "request_cancel",
1255 Self::Terminate => "terminate",
1256 }
1257 }
1258}
1259
1260#[derive(Clone, Debug, Default, PartialEq, Eq)]
1262pub struct ChildWorkflowRetryPolicy {
1263 pub max_attempts: Option<u32>,
1264 pub backoff_seconds: Vec<u64>,
1265 pub non_retryable_error_types: Vec<String>,
1266}
1267
1268#[derive(Clone, Debug, PartialEq, Eq)]
1272pub struct ChildWorkflowOptions {
1273 pub task_queue: String,
1274 pub parent_close_policy: ParentClosePolicy,
1275 pub retry_policy: Option<ChildWorkflowRetryPolicy>,
1276 pub execution_timeout_seconds: Option<u64>,
1277 pub run_timeout_seconds: Option<u64>,
1278}
1279
1280impl ChildWorkflowOptions {
1281 pub fn new(task_queue: impl Into<String>) -> Self {
1282 Self {
1283 task_queue: task_queue.into(),
1284 parent_close_policy: ParentClosePolicy::Abandon,
1285 retry_policy: None,
1286 execution_timeout_seconds: None,
1287 run_timeout_seconds: None,
1288 }
1289 }
1290
1291 pub fn parent_close_policy(mut self, policy: ParentClosePolicy) -> Self {
1292 self.parent_close_policy = policy;
1293 self
1294 }
1295
1296 pub fn retry_policy(mut self, policy: ChildWorkflowRetryPolicy) -> Self {
1297 self.retry_policy = Some(policy);
1298 self
1299 }
1300
1301 pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
1302 self.execution_timeout_seconds = Some(seconds);
1303 self
1304 }
1305
1306 pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
1307 self.run_timeout_seconds = Some(seconds);
1308 self
1309 }
1310}
1311
1312#[derive(Clone, Debug, PartialEq, Eq)]
1314pub enum ActivityBackoff {
1315 Explicit(Vec<Duration>),
1318 Exponential {
1320 initial_interval: Duration,
1321 coefficient: u32,
1322 maximum_interval: Option<Duration>,
1323 },
1324}
1325
1326#[derive(Clone, Debug, Default, PartialEq, Eq)]
1328pub struct ActivityRetryPolicy {
1329 pub max_attempts: Option<u32>,
1330 pub backoff: Option<ActivityBackoff>,
1331 pub non_retryable_error_types: Vec<String>,
1332}
1333
1334impl ActivityRetryPolicy {
1335 pub fn new(max_attempts: u32) -> Self {
1337 Self {
1338 max_attempts: Some(max_attempts),
1339 ..Self::default()
1340 }
1341 }
1342
1343 pub fn backoff_intervals(mut self, intervals: impl IntoIterator<Item = Duration>) -> Self {
1344 self.backoff = Some(ActivityBackoff::Explicit(intervals.into_iter().collect()));
1345 self
1346 }
1347
1348 pub fn exponential_backoff(
1349 mut self,
1350 initial_interval: Duration,
1351 coefficient: u32,
1352 maximum_interval: Option<Duration>,
1353 ) -> Self {
1354 self.backoff = Some(ActivityBackoff::Exponential {
1355 initial_interval,
1356 coefficient,
1357 maximum_interval,
1358 });
1359 self
1360 }
1361
1362 pub fn non_retryable_error_type(mut self, error_type: impl Into<String>) -> Self {
1363 self.non_retryable_error_types.push(error_type.into());
1364 self
1365 }
1366
1367 pub fn non_retryable_error_types(
1368 mut self,
1369 error_types: impl IntoIterator<Item = impl Into<String>>,
1370 ) -> Self {
1371 self.non_retryable_error_types
1372 .extend(error_types.into_iter().map(Into::into));
1373 self
1374 }
1375}
1376
1377#[derive(Clone, Debug, Default, PartialEq, Eq)]
1382pub struct ActivityOptions {
1383 pub task_queue: Option<String>,
1384 pub retry_policy: Option<ActivityRetryPolicy>,
1385 pub start_to_close_timeout: Option<Duration>,
1386 pub schedule_to_start_timeout: Option<Duration>,
1387 pub schedule_to_close_timeout: Option<Duration>,
1388 pub heartbeat_timeout: Option<Duration>,
1389}
1390
1391impl ActivityOptions {
1392 pub fn new() -> Self {
1393 Self::default()
1394 }
1395
1396 pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
1397 self.task_queue = Some(task_queue.into());
1398 self
1399 }
1400
1401 pub fn retry_policy(mut self, policy: ActivityRetryPolicy) -> Self {
1402 self.retry_policy = Some(policy);
1403 self
1404 }
1405
1406 pub fn start_to_close_timeout(mut self, timeout: Duration) -> Self {
1407 self.start_to_close_timeout = Some(timeout);
1408 self
1409 }
1410
1411 pub fn schedule_to_start_timeout(mut self, timeout: Duration) -> Self {
1412 self.schedule_to_start_timeout = Some(timeout);
1413 self
1414 }
1415
1416 pub fn schedule_to_close_timeout(mut self, timeout: Duration) -> Self {
1417 self.schedule_to_close_timeout = Some(timeout);
1418 self
1419 }
1420
1421 pub fn heartbeat_timeout(mut self, timeout: Duration) -> Self {
1422 self.heartbeat_timeout = Some(timeout);
1423 self
1424 }
1425
1426 fn validate(&self) -> std::result::Result<ValidatedActivityOptions, ActivityOptionsError> {
1427 if self
1428 .task_queue
1429 .as_deref()
1430 .is_some_and(|queue| queue.trim().is_empty())
1431 {
1432 return Err(ActivityOptionsError::new(
1433 ActivityOptionsErrorKind::EmptyTaskQueue,
1434 Some("task_queue"),
1435 "task_queue must not be empty",
1436 ));
1437 }
1438
1439 for (field, value) in [
1440 ("start_to_close_timeout", self.start_to_close_timeout),
1441 ("schedule_to_start_timeout", self.schedule_to_start_timeout),
1442 ("schedule_to_close_timeout", self.schedule_to_close_timeout),
1443 ("heartbeat_timeout", self.heartbeat_timeout),
1444 ] {
1445 if value.is_some_and(|value| value.is_zero()) {
1446 return Err(ActivityOptionsError::new(
1447 ActivityOptionsErrorKind::TimeoutNotPositive,
1448 Some(field),
1449 format!("{field} must be positive"),
1450 ));
1451 }
1452 }
1453
1454 validate_timeout_order(
1455 "heartbeat_timeout",
1456 self.heartbeat_timeout,
1457 "start_to_close_timeout",
1458 self.start_to_close_timeout,
1459 )?;
1460 validate_timeout_order(
1461 "start_to_close_timeout",
1462 self.start_to_close_timeout,
1463 "schedule_to_close_timeout",
1464 self.schedule_to_close_timeout,
1465 )?;
1466 validate_timeout_order(
1467 "schedule_to_start_timeout",
1468 self.schedule_to_start_timeout,
1469 "schedule_to_close_timeout",
1470 self.schedule_to_close_timeout,
1471 )?;
1472
1473 Ok(ValidatedActivityOptions {
1474 task_queue: self.task_queue.clone(),
1475 retry_policy: self
1476 .retry_policy
1477 .as_ref()
1478 .map(validate_activity_retry_policy)
1479 .transpose()?,
1480 start_to_close_timeout: timeout_seconds(
1481 "start_to_close_timeout",
1482 self.start_to_close_timeout,
1483 )?,
1484 schedule_to_start_timeout: timeout_seconds(
1485 "schedule_to_start_timeout",
1486 self.schedule_to_start_timeout,
1487 )?,
1488 schedule_to_close_timeout: timeout_seconds(
1489 "schedule_to_close_timeout",
1490 self.schedule_to_close_timeout,
1491 )?,
1492 heartbeat_timeout: timeout_seconds("heartbeat_timeout", self.heartbeat_timeout)?,
1493 })
1494 }
1495}
1496
1497pub enum ParallelOperation {
1503 Activity {
1504 activity_type: String,
1505 options: ActivityOptions,
1506 arguments: Result<AvroValue>,
1507 },
1508 ChildWorkflow {
1509 workflow_type: String,
1510 options: ChildWorkflowOptions,
1511 arguments: Result<AvroValue>,
1512 },
1513 Timer(Duration),
1514 Signal(String),
1515 Condition {
1516 options: ConditionWaitOptions,
1517 predicate: Box<dyn Fn() -> Result<bool> + Send + 'static>,
1518 },
1519 Group(Vec<ParallelOperation>),
1520}
1521
1522impl ParallelOperation {
1523 pub fn activity<T: Serialize>(activity_type: impl Into<String>, args: T) -> Self {
1524 Self::activity_with_options(activity_type, ActivityOptions::new(), args)
1525 }
1526
1527 pub fn activity_with_options<T: Serialize>(
1528 activity_type: impl Into<String>,
1529 options: ActivityOptions,
1530 args: T,
1531 ) -> Self {
1532 Self::Activity {
1533 activity_type: activity_type.into(),
1534 options,
1535 arguments: AvroValue::from_serialize(&args),
1536 }
1537 }
1538
1539 pub fn child_workflow<T: Serialize>(
1540 workflow_type: impl Into<String>,
1541 options: ChildWorkflowOptions,
1542 args: T,
1543 ) -> Self {
1544 Self::ChildWorkflow {
1545 workflow_type: workflow_type.into(),
1546 options,
1547 arguments: AvroValue::from_serialize(&args),
1548 }
1549 }
1550
1551 pub fn timer(duration: Duration) -> Self {
1552 Self::Timer(duration)
1553 }
1554
1555 pub fn signal(signal_name: impl Into<String>) -> Self {
1556 Self::Signal(signal_name.into())
1557 }
1558
1559 pub fn condition<F>(options: ConditionWaitOptions, predicate: F) -> Self
1560 where
1561 F: Fn() -> Result<bool> + Send + 'static,
1562 {
1563 Self::Condition {
1564 options,
1565 predicate: Box::new(predicate),
1566 }
1567 }
1568
1569 pub fn group(operations: Vec<ParallelOperation>) -> Self {
1570 Self::Group(operations)
1571 }
1572}
1573
1574#[derive(Clone, Debug)]
1575struct ValidatedActivityOptions {
1576 task_queue: Option<String>,
1577 retry_policy: Option<Value>,
1578 start_to_close_timeout: Option<u64>,
1579 schedule_to_start_timeout: Option<u64>,
1580 schedule_to_close_timeout: Option<u64>,
1581 heartbeat_timeout: Option<u64>,
1582}
1583
1584fn validate_timeout_order(
1585 smaller_name: &'static str,
1586 smaller: Option<Duration>,
1587 larger_name: &'static str,
1588 larger: Option<Duration>,
1589) -> std::result::Result<(), ActivityOptionsError> {
1590 if matches!((smaller, larger), (Some(smaller), Some(larger)) if smaller > larger) {
1591 return Err(ActivityOptionsError::new(
1592 ActivityOptionsErrorKind::TimeoutOrder,
1593 Some(smaller_name),
1594 format!("{smaller_name} must be <= {larger_name}"),
1595 ));
1596 }
1597 Ok(())
1598}
1599
1600fn timeout_seconds(
1601 field: &'static str,
1602 value: Option<Duration>,
1603) -> std::result::Result<Option<u64>, ActivityOptionsError> {
1604 value
1605 .map(|value| {
1606 activity_protocol_seconds(value).ok_or_else(|| {
1607 ActivityOptionsError::new(
1608 ActivityOptionsErrorKind::TimeoutOverflow,
1609 Some(field),
1610 format!("{field} is too large for the worker protocol"),
1611 )
1612 })
1613 })
1614 .transpose()
1615}
1616
1617fn duration_seconds_ceil(value: Duration) -> Option<u64> {
1618 value
1619 .as_secs()
1620 .checked_add(u64::from(value.subsec_nanos() > 0))
1621}
1622
1623fn activity_protocol_seconds(value: Duration) -> Option<u64> {
1624 duration_seconds_ceil(value).filter(|seconds| *seconds <= i64::MAX as u64)
1625}
1626
1627fn validate_activity_retry_policy(
1628 policy: &ActivityRetryPolicy,
1629) -> std::result::Result<Value, ActivityOptionsError> {
1630 if policy.max_attempts.is_none()
1631 && policy.backoff.is_none()
1632 && policy.non_retryable_error_types.is_empty()
1633 {
1634 return Err(ActivityOptionsError::new(
1635 ActivityOptionsErrorKind::EmptyRetryPolicy,
1636 Some("retry_policy"),
1637 "retry_policy must configure at least one field",
1638 ));
1639 }
1640 if policy.max_attempts == Some(0) {
1641 return Err(ActivityOptionsError::new(
1642 ActivityOptionsErrorKind::InvalidMaxAttempts,
1643 Some("retry_policy.max_attempts"),
1644 "max_attempts must be >= 1",
1645 ));
1646 }
1647 if policy
1648 .non_retryable_error_types
1649 .iter()
1650 .any(|error_type| error_type.trim().is_empty())
1651 {
1652 return Err(ActivityOptionsError::new(
1653 ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
1654 Some("retry_policy.non_retryable_error_types"),
1655 "non_retryable_error_types must not contain empty values",
1656 ));
1657 }
1658
1659 let backoff_seconds = match &policy.backoff {
1660 None => None,
1661 Some(backoff) => {
1662 let max_attempts = policy.max_attempts.ok_or_else(|| {
1663 ActivityOptionsError::new(
1664 ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
1665 Some("retry_policy.backoff"),
1666 "backoff requires max_attempts",
1667 )
1668 })?;
1669 let retry_count = max_attempts.saturating_sub(1) as usize;
1670 let intervals = match backoff {
1671 ActivityBackoff::Explicit(intervals) => {
1672 if intervals.len() > retry_count {
1673 return Err(ActivityOptionsError::new(
1674 ActivityOptionsErrorKind::TooManyBackoffIntervals,
1675 Some("retry_policy.backoff"),
1676 "backoff interval count must not exceed max_attempts - 1",
1677 ));
1678 }
1679 intervals.clone()
1680 }
1681 ActivityBackoff::Exponential {
1682 initial_interval,
1683 coefficient,
1684 maximum_interval,
1685 } => {
1686 if *coefficient < 1 {
1687 return Err(ActivityOptionsError::new(
1688 ActivityOptionsErrorKind::InvalidBackoffCoefficient,
1689 Some("retry_policy.backoff.coefficient"),
1690 "backoff coefficient must be >= 1",
1691 ));
1692 }
1693 if retry_count > 10_000 {
1694 return Err(ActivityOptionsError::new(
1695 ActivityOptionsErrorKind::BackoffGenerationTooLarge,
1696 Some("retry_policy.max_attempts"),
1697 "generated backoff supports at most 10000 retry intervals",
1698 ));
1699 }
1700 let mut current = *initial_interval;
1701 let mut intervals = Vec::with_capacity(retry_count);
1702 for _ in 0..retry_count {
1703 let interval = maximum_interval
1704 .map(|maximum| current.min(maximum))
1705 .unwrap_or(current);
1706 intervals.push(interval);
1707 if maximum_interval.is_some_and(|maximum| interval == maximum) {
1708 break;
1709 }
1710 current = current.checked_mul(*coefficient).ok_or_else(|| {
1711 ActivityOptionsError::new(
1712 ActivityOptionsErrorKind::BackoffOverflow,
1713 Some("retry_policy.backoff"),
1714 "generated backoff interval overflowed",
1715 )
1716 })?;
1717 }
1718 intervals
1719 }
1720 };
1721 Some(
1722 intervals
1723 .into_iter()
1724 .map(|interval| {
1725 activity_protocol_seconds(interval).ok_or_else(|| {
1726 ActivityOptionsError::new(
1727 ActivityOptionsErrorKind::BackoffOverflow,
1728 Some("retry_policy.backoff"),
1729 "backoff interval is too large for the worker protocol",
1730 )
1731 })
1732 })
1733 .collect::<std::result::Result<Vec<_>, _>>()?,
1734 )
1735 }
1736 };
1737
1738 let mut encoded = serde_json::Map::new();
1739 if let Some(max_attempts) = policy.max_attempts {
1740 encoded.insert("max_attempts".to_string(), json!(max_attempts));
1741 }
1742 if let Some(backoff_seconds) = backoff_seconds {
1743 encoded.insert("backoff_seconds".to_string(), json!(backoff_seconds));
1744 }
1745 if !policy.non_retryable_error_types.is_empty() {
1746 let mut canonical_error_types = Vec::new();
1747 for error_type in policy
1748 .non_retryable_error_types
1749 .iter()
1750 .map(|error_type| error_type.trim())
1751 {
1752 if !canonical_error_types.contains(&error_type) {
1753 canonical_error_types.push(error_type);
1754 }
1755 }
1756 encoded.insert(
1757 "non_retryable_error_types".to_string(),
1758 json!(canonical_error_types),
1759 );
1760 }
1761 Ok(Value::Object(encoded))
1762}
1763
1764#[derive(Clone, Debug, Error)]
1767#[error("non-deterministic workflow replay ({reason}) at sequence {sequence:?}: {message}")]
1768pub struct ReplayFailure {
1769 pub reason: String,
1770 pub sequence: Option<u64>,
1771 pub expected: Option<String>,
1772 pub actual: Option<String>,
1773 pub message: String,
1774}
1775
1776impl ReplayFailure {
1777 fn new(
1778 reason: impl Into<String>,
1779 sequence: Option<u64>,
1780 expected: Option<String>,
1781 actual: Option<String>,
1782 message: impl Into<String>,
1783 ) -> Self {
1784 Self {
1785 reason: reason.into(),
1786 sequence,
1787 expected,
1788 actual,
1789 message: message.into(),
1790 }
1791 }
1792}
1793
1794#[derive(Clone, Debug, Error)]
1796#[error("query failed ({reason}, HTTP {status}): {message}")]
1797pub struct QueryFailure {
1798 pub status: u16,
1799 pub reason: String,
1800 pub message: String,
1801 pub body: Value,
1802}
1803
1804#[derive(Clone, Debug, Error)]
1806#[error("protocol rejected ({reason}, HTTP {status}): {message}")]
1807pub struct ProtocolFailure {
1808 pub status: u16,
1809 pub reason: String,
1810 pub message: String,
1811 pub supported_version: Option<String>,
1812 pub requested_version: Option<String>,
1813 pub body: Value,
1814}
1815
1816#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
1817pub struct PayloadEnvelope {
1818 pub codec: String,
1819 pub blob: String,
1820}
1821
1822impl PayloadEnvelope {
1823 pub fn avro<T: Serialize>(value: &T) -> Result<Self> {
1824 encode_payload(value, DEFAULT_CODEC)
1825 }
1826
1827 pub fn avro_value(value: &AvroValue) -> Result<Self> {
1830 encode_avro_value(value)
1831 }
1832}
1833
1834#[derive(Clone, Debug)]
1836pub enum AvroValue {
1837 Null,
1838 Boolean(bool),
1839 Long(i64),
1840 Double(f64),
1841 Bytes(Vec<u8>),
1842 String(String),
1843 Array(Vec<AvroValue>),
1844 Map(BTreeMap<String, AvroValue>),
1845}
1846
1847impl PartialEq for AvroValue {
1848 fn eq(&self, other: &Self) -> bool {
1849 match (self, other) {
1850 (Self::Null, Self::Null) => true,
1851 (Self::Boolean(left), Self::Boolean(right)) => left == right,
1852 (Self::Long(left), Self::Long(right)) => left == right,
1853 (Self::Double(left), Self::Double(right)) => left.to_bits() == right.to_bits(),
1854 (Self::Bytes(left), Self::Bytes(right)) => left == right,
1855 (Self::String(left), Self::String(right)) => left == right,
1856 (Self::Array(left), Self::Array(right)) => left == right,
1857 (Self::Map(left), Self::Map(right)) => left == right,
1858 _ => false,
1859 }
1860 }
1861}
1862
1863impl AvroValue {
1864 fn from_serialize<T: Serialize>(value: &T) -> Result<Self> {
1865 Self::from_serde_value(
1866 serde_value::to_value(value).map_err(|error| {
1867 Error::Codec(format!("could not adapt value for Avro: {error}"))
1868 })?,
1869 )
1870 }
1871
1872 fn from_serde_value(value: serde_value::Value) -> Result<Self> {
1873 use serde_value::Value as SerdeValue;
1874
1875 match value {
1876 SerdeValue::Unit => Ok(Self::Null),
1877 SerdeValue::Bool(value) => Ok(Self::Boolean(value)),
1878 SerdeValue::I8(value) => Ok(Self::Long(i64::from(value))),
1879 SerdeValue::I16(value) => Ok(Self::Long(i64::from(value))),
1880 SerdeValue::I32(value) => Ok(Self::Long(i64::from(value))),
1881 SerdeValue::I64(value) => Ok(Self::Long(value)),
1882 SerdeValue::U8(value) => Ok(Self::Long(i64::from(value))),
1883 SerdeValue::U16(value) => Ok(Self::Long(i64::from(value))),
1884 SerdeValue::U32(value) => Ok(Self::Long(i64::from(value))),
1885 SerdeValue::U64(value) => i64::try_from(value).map(Self::Long).map_err(|_| {
1886 Error::Codec(
1887 "integer_overflow: Avro Value long must be within signed 64-bit range"
1888 .to_string(),
1889 )
1890 }),
1891 SerdeValue::F32(value) => Self::finite_double(f64::from(value)),
1892 SerdeValue::F64(value) => Self::finite_double(value),
1893 SerdeValue::Char(value) => Ok(Self::String(value.to_string())),
1894 SerdeValue::String(value) => Ok(Self::String(value)),
1895 SerdeValue::Bytes(value) => Ok(Self::Bytes(value)),
1896 SerdeValue::Option(None) => Ok(Self::Null),
1897 SerdeValue::Option(Some(value)) | SerdeValue::Newtype(value) => {
1898 Self::from_serde_value(*value)
1899 }
1900 SerdeValue::Seq(values) => values
1901 .into_iter()
1902 .map(Self::from_serde_value)
1903 .collect::<Result<Vec<_>>>()
1904 .map(Self::Array),
1905 SerdeValue::Map(values) => values
1906 .into_iter()
1907 .map(|(key, value)| {
1908 let SerdeValue::String(key) = key else {
1909 return Err(Error::Codec(
1910 "invalid_map_key: Avro Value map keys must be strings".to_string(),
1911 ));
1912 };
1913
1914 Ok((key, Self::from_serde_value(value)?))
1915 })
1916 .collect::<Result<BTreeMap<_, _>>>()
1917 .map(Self::Map),
1918 }
1919 }
1920
1921 fn finite_double(value: f64) -> Result<Self> {
1922 if !value.is_finite() {
1923 return Err(Error::Codec(
1924 "non_finite_float: Avro Value doubles must be finite".to_string(),
1925 ));
1926 }
1927
1928 Ok(Self::Double(value))
1929 }
1930
1931 fn into_json(self) -> Result<Value> {
1932 match self {
1933 Self::Null => Ok(Value::Null),
1934 Self::Boolean(value) => Ok(Value::Bool(value)),
1935 Self::Long(value) => Ok(Value::Number(value.into())),
1936 Self::Double(value) => serde_json::Number::from_f64(value)
1937 .map(Value::Number)
1938 .ok_or_else(|| {
1939 Error::Codec(
1940 "non_finite_float: decoded Avro Value double is not finite".to_string(),
1941 )
1942 }),
1943 Self::Bytes(value) => Ok(json!({
1944 "$type": "bytes",
1945 "base64": BASE64.encode(value),
1946 })),
1947 Self::String(value) => Ok(Value::String(value)),
1948 Self::Array(values) => values
1949 .into_iter()
1950 .map(Self::into_json)
1951 .collect::<Result<Vec<_>>>()
1952 .map(Value::Array),
1953 Self::Map(values) => values
1954 .into_iter()
1955 .map(|(key, value)| Ok((key, value.into_json()?)))
1956 .collect::<Result<serde_json::Map<_, _>>>()
1957 .map(Value::Object),
1958 }
1959 }
1960
1961 fn into_serde_value(self) -> serde_value::Value {
1962 use serde_value::Value as SerdeValue;
1963
1964 match self {
1965 Self::Null => SerdeValue::Unit,
1966 Self::Boolean(value) => SerdeValue::Bool(value),
1967 Self::Long(value) => SerdeValue::I64(value),
1968 Self::Double(value) => SerdeValue::F64(value),
1969 Self::Bytes(value) => SerdeValue::Bytes(value),
1970 Self::String(value) => SerdeValue::String(value),
1971 Self::Array(values) => {
1972 SerdeValue::Seq(values.into_iter().map(Self::into_serde_value).collect())
1973 }
1974 Self::Map(values) => SerdeValue::Map(
1975 values
1976 .into_iter()
1977 .map(|(key, value)| (SerdeValue::String(key), value.into_serde_value()))
1978 .collect(),
1979 ),
1980 }
1981 }
1982
1983 pub fn deserialize<T: DeserializeOwned>(self) -> Result<T> {
1984 self.into_serde_value().deserialize_into().map_err(|error| {
1985 Error::Codec(format!(
1986 "avro_value_type_mismatch: could not adapt decoded value: {error}"
1987 ))
1988 })
1989 }
1990}
1991
1992impl Serialize for AvroValue {
1993 fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
1994 where
1995 S: Serializer,
1996 {
1997 match self {
1998 Self::Null => serializer.serialize_unit(),
1999 Self::Boolean(value) => serializer.serialize_bool(*value),
2000 Self::Long(value) => serializer.serialize_i64(*value),
2001 Self::Double(value) => serializer.serialize_f64(*value),
2002 Self::Bytes(value) => serializer.serialize_bytes(value),
2003 Self::String(value) => serializer.serialize_str(value),
2004 Self::Array(values) => {
2005 let mut sequence = serializer.serialize_seq(Some(values.len()))?;
2006 for value in values {
2007 sequence.serialize_element(value)?;
2008 }
2009 sequence.end()
2010 }
2011 Self::Map(values) => {
2012 let mut map = serializer.serialize_map(Some(values.len()))?;
2013 for (key, value) in values {
2014 map.serialize_entry(key, value)?;
2015 }
2016 map.end()
2017 }
2018 }
2019 }
2020}
2021
2022pub fn encode_avro_value(value: &AvroValue) -> Result<PayloadEnvelope> {
2023 let datum = avro_value_to_datum(value)?;
2024 let datum = to_avro_datum(avro_value_ordered_map_encoding_schema()?, datum)
2025 .map_err(|err| Error::Codec(format!("avro_value_encode_failed: {err}")))?;
2026 let mut bytes = Vec::with_capacity(datum.len() + 10);
2027 bytes.extend_from_slice(&AVRO_SINGLE_OBJECT_MAGIC);
2028 bytes.extend_from_slice(&AVRO_VALUE_SCHEMA_FINGERPRINT);
2029 bytes.extend_from_slice(&datum);
2030 Ok(PayloadEnvelope {
2031 codec: DEFAULT_CODEC.to_string(),
2032 blob: BASE64.encode(bytes),
2033 })
2034}
2035
2036pub fn decode_avro_value(envelope: &PayloadEnvelope) -> Result<AvroValue> {
2037 if envelope.codec != DEFAULT_CODEC {
2038 return Err(unsupported_payload_codec(&envelope.codec));
2039 }
2040 decode_avro_value_blob(&envelope.blob)
2041}
2042
2043pub fn encode_payload<T: Serialize>(value: &T, codec: &str) -> Result<PayloadEnvelope> {
2044 let blob = match codec {
2045 DEFAULT_CODEC => encode_avro_value(&AvroValue::from_serialize(value)?)?.blob,
2046 other => return Err(unsupported_payload_codec(other)),
2047 };
2048
2049 Ok(PayloadEnvelope {
2050 codec: codec.to_string(),
2051 blob,
2052 })
2053}
2054
2055pub fn decode_payload<T: DeserializeOwned>(envelope: &PayloadEnvelope) -> Result<T> {
2056 match envelope.codec.as_str() {
2057 DEFAULT_CODEC => decode_avro_value(envelope)?.deserialize(),
2058 other => Err(unsupported_payload_codec(other)),
2059 }
2060}
2061
2062fn handler_type_error<T>(
2063 handler_kind: HandlerKind,
2064 handler_name: &str,
2065 value_kind: HandlerValueKind,
2066 message: impl Into<String>,
2067) -> Error {
2068 Error::HandlerType {
2069 handler_kind,
2070 handler_name: handler_name.to_string(),
2071 value_kind,
2072 rust_type: type_name::<T>(),
2073 message: message.into(),
2074 }
2075}
2076
2077fn decode_handler_input<T: DeserializeOwned>(
2078 arguments: AvroValue,
2079 handler_kind: HandlerKind,
2080 handler_name: &str,
2081) -> Result<T> {
2082 let argument = match arguments {
2083 AvroValue::Array(mut arguments) if arguments.len() == 1 => {
2084 arguments.pop().expect("one typed handler argument")
2085 }
2086 AvroValue::Array(arguments) if arguments.is_empty() => AvroValue::Null,
2087 AvroValue::Array(arguments) => {
2088 return Err(handler_type_error::<T>(
2089 handler_kind,
2090 handler_name,
2091 HandlerValueKind::Input,
2092 format!(
2093 "typed handlers accept one request value, but the task carried {} arguments",
2094 arguments.len()
2095 ),
2096 ));
2097 }
2098 argument => argument,
2099 };
2100
2101 argument.deserialize().map_err(|error| {
2102 handler_type_error::<T>(
2103 handler_kind,
2104 handler_name,
2105 HandlerValueKind::Input,
2106 error.to_string(),
2107 )
2108 })
2109}
2110
2111fn encode_handler_result<T: Serialize>(
2112 result: &T,
2113 handler_kind: HandlerKind,
2114 handler_name: &str,
2115) -> Result<AvroValue> {
2116 AvroValue::from_serialize(result).map_err(|error| {
2117 handler_type_error::<T>(
2118 handler_kind,
2119 handler_name,
2120 HandlerValueKind::Result,
2121 error.to_string(),
2122 )
2123 })
2124}
2125
2126fn decode_handler_result<T: DeserializeOwned>(
2127 result: AvroValue,
2128 handler_kind: HandlerKind,
2129 handler_name: &str,
2130) -> Result<T> {
2131 result.deserialize().map_err(|error| {
2132 handler_type_error::<T>(
2133 handler_kind,
2134 handler_name,
2135 HandlerValueKind::Result,
2136 error.to_string(),
2137 )
2138 })
2139}
2140
2141#[cfg(test)]
2142fn encode_value_envelope(value: &Value, codec: &str) -> Result<Value> {
2143 Ok(serde_json::to_value(encode_payload(value, codec)?)?)
2144}
2145
2146fn decode_wire_value(value: &Value, fallback_codec: &str) -> Result<Value> {
2147 validate_payload_codec(fallback_codec)?;
2148
2149 if value.is_null() {
2150 return Ok(Value::Null);
2151 }
2152
2153 if let Some((codec, blob)) = payload_envelope_parts(value)? {
2154 return decode_blob(blob, codec);
2155 }
2156
2157 if let Some(blob) = value.as_str() {
2158 return decode_blob(blob, fallback_codec);
2159 }
2160
2161 Err(untagged_payload_value())
2162}
2163
2164fn encode_typed_envelope(value: &AvroValue, codec: &str) -> Result<Value> {
2165 let envelope = match codec {
2166 DEFAULT_CODEC => encode_avro_value(value)?,
2167 other => return Err(unsupported_payload_codec(other)),
2168 };
2169 Ok(serde_json::to_value(envelope)?)
2170}
2171
2172fn decode_wire_avro_value(value: &Value, fallback_codec: &str) -> Result<AvroValue> {
2173 validate_payload_codec(fallback_codec)?;
2174
2175 if value.is_null() {
2176 return Ok(AvroValue::Null);
2177 }
2178
2179 if let Some((codec, blob)) = payload_envelope_parts(value)? {
2180 validate_payload_codec(codec)?;
2181 return decode_avro_value_blob(blob);
2182 }
2183
2184 if let Some(blob) = value.as_str() {
2185 return match fallback_codec {
2186 DEFAULT_CODEC => decode_avro_value_blob(blob),
2187 other => Err(unsupported_payload_codec(other)),
2188 };
2189 }
2190
2191 Err(untagged_payload_value())
2192}
2193
2194fn normalize_avro_arguments(value: AvroValue) -> AvroValue {
2195 match value {
2196 AvroValue::Null => AvroValue::Array(Vec::new()),
2197 AvroValue::Array(_) => value,
2198 other => AvroValue::Array(vec![other]),
2199 }
2200}
2201
2202fn decode_blob(blob: &str, codec: &str) -> Result<Value> {
2203 match codec {
2204 DEFAULT_CODEC => decode_avro_value_blob(blob)?.into_json(),
2205 other => Err(unsupported_payload_codec(other)),
2206 }
2207}
2208
2209fn validate_payload_codec(codec: &str) -> Result<()> {
2210 match codec {
2211 DEFAULT_CODEC => Ok(()),
2212 MISSING_TASK_PAYLOAD_CODEC => {
2213 Err(invalid_task_payload_codec("task payload_codec is missing"))
2214 }
2215 NULL_TASK_PAYLOAD_CODEC => Err(invalid_task_payload_codec("task payload_codec is null")),
2216 NON_STRING_TASK_PAYLOAD_CODEC => Err(invalid_task_payload_codec(
2217 "task payload_codec must be a string",
2218 )),
2219 other => Err(unsupported_payload_codec(other)),
2220 }
2221}
2222
2223fn invalid_task_payload_codec(reason: &str) -> Error {
2224 Error::Codec(format!(
2225 "unsupported_payload_codec: {reason}; Durable Workflow 2.0 requires an explicit string payload_codec=\"avro\" before worker task execution"
2226 ))
2227}
2228
2229fn payload_envelope_parts(value: &Value) -> Result<Option<(&str, &str)>> {
2230 let Some(object) = value.as_object() else {
2231 return Ok(None);
2232 };
2233 if !object.contains_key("codec") && !object.contains_key("blob") {
2234 return Ok(None);
2235 }
2236
2237 let codec = object
2238 .get("codec")
2239 .and_then(Value::as_str)
2240 .ok_or_else(invalid_payload_envelope)?;
2241 validate_payload_codec(codec)?;
2242 let blob = object
2243 .get("blob")
2244 .and_then(Value::as_str)
2245 .ok_or_else(invalid_payload_envelope)?;
2246 Ok(Some((codec, blob)))
2247}
2248
2249fn invalid_payload_envelope() -> Error {
2250 Error::Codec(
2251 "invalid_payload_envelope: durable payloads must use an object with string codec=\"avro\" and blob fields"
2252 .to_string(),
2253 )
2254}
2255
2256fn validate_workflow_task_commands(commands: &[Value]) -> Result<()> {
2257 for command in commands {
2258 let Some(command) = command.as_object() else {
2259 continue;
2260 };
2261 let Some(command_type) = command.get("type").and_then(Value::as_str) else {
2262 continue;
2263 };
2264 let Some(payload_field) = workflow_command_payload_field(command_type) else {
2265 continue;
2266 };
2267
2268 if let Some(codec) = command.get("payload_codec") {
2269 let codec = codec.as_str().ok_or_else(invalid_payload_envelope)?;
2270 validate_payload_codec(codec)?;
2271 }
2272
2273 let payload = command
2274 .get(payload_field)
2275 .ok_or_else(invalid_payload_envelope)?;
2276 if runtime_payloads::Reference::parse(payload)?.is_none() {
2277 validate_outbound_payload_envelope(payload)?;
2278 }
2279 }
2280 Ok(())
2281}
2282
2283fn workflow_completion_protocol_version(commands: &[Value]) -> &'static str {
2284 if commands.iter().any(|command| {
2285 command.get("type").and_then(Value::as_str) == Some("open_condition_wait")
2286 && command
2287 .get("condition_wait_occurrence_id")
2288 .and_then(Value::as_str)
2289 .is_some_and(|occurrence_id| !occurrence_id.is_empty())
2290 }) {
2291 CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
2292 } else if commands.iter().any(|command| {
2293 command.get("type").and_then(Value::as_str) == Some("upsert_search_attributes")
2294 && command.get("attribute_types").is_some()
2295 }) {
2296 TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION
2297 } else if commands
2298 .iter()
2299 .any(|command| command.get("type").and_then(Value::as_str) == Some("upsert_memo"))
2300 {
2301 MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION
2302 } else if commands
2303 .iter()
2304 .any(|command| command.get("type").and_then(Value::as_str) == Some("open_condition_wait"))
2305 {
2306 CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION
2307 } else if commands.iter().any(|command| {
2308 command.get("type").and_then(Value::as_str) == Some("upsert_search_attributes")
2309 }) {
2310 SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION
2311 } else {
2312 WORKER_PROTOCOL_VERSION
2313 }
2314}
2315
2316fn workflow_completion_protocol_version_with_message_streams(
2317 commands: &[Value],
2318 has_message_stream_metadata: bool,
2319) -> &'static str {
2320 let command_protocol = workflow_completion_protocol_version(commands);
2321 if has_message_stream_metadata && !worker_protocol_supports_message_streams(command_protocol) {
2322 MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION
2323 } else {
2324 command_protocol
2325 }
2326}
2327
2328fn workflow_command_payload_field(command_type: &str) -> Option<&'static str> {
2329 match command_type {
2330 "complete_workflow" | "complete_update" | "record_side_effect" => Some("result"),
2331 "schedule_activity" | "start_child_workflow" | "continue_as_new" => Some("arguments"),
2332 "start_service_operation" => Some("request_payload"),
2333 "upsert_memo" => Some("entries"),
2334 _ => None,
2335 }
2336}
2337
2338fn validate_outbound_payload_envelope(value: &Value) -> Result<()> {
2339 let Some((codec, blob)) = payload_envelope_parts(value)? else {
2340 return Err(untagged_payload_value());
2341 };
2342 validate_payload_codec(codec)?;
2343 decode_avro_value_blob(blob)?;
2344 Ok(())
2345}
2346
2347fn unsupported_payload_codec(codec: &str) -> Error {
2348 Error::Codec(format!(
2349 "unsupported_payload_codec: workflow payload codec {codec:?} is not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec"
2350 ))
2351}
2352
2353fn untagged_payload_value() -> Error {
2354 Error::Codec(
2355 "unsupported_payload_codec: untagged durable payload values are not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec"
2356 .to_string(),
2357 )
2358}
2359
2360fn decode_avro_value_blob(blob: &str) -> Result<AvroValue> {
2361 let bytes = BASE64.decode(blob).map_err(|err| {
2362 Error::Codec(format!(
2363 "invalid_payload_framing: expected strict base64 Avro single-object bytes: {err}"
2364 ))
2365 })?;
2366
2367 if serde_json::from_slice::<Value>(&bytes).is_ok() {
2368 return Err(unsupported_payload_codec("json"));
2369 }
2370
2371 if bytes.len() < 10 || bytes[..2] != AVRO_SINGLE_OBJECT_MAGIC {
2372 return Err(Error::Codec(
2373 "invalid_payload_framing: expected Avro single-object magic c301".to_string(),
2374 ));
2375 }
2376
2377 let fingerprint: [u8; 8] = bytes[2..10]
2378 .try_into()
2379 .map_err(|_| Error::Codec("invalid Avro fingerprint length".to_string()))?;
2380 if fingerprint != AVRO_VALUE_SCHEMA_FINGERPRINT {
2381 return Err(Error::Codec(format!(
2382 "unsupported_payload_schema: unknown CRC-64-AVRO fingerprint {}",
2383 fingerprint
2384 .iter()
2385 .map(|byte| format!("{byte:02x}"))
2386 .collect::<String>()
2387 )));
2388 }
2389
2390 let mut datum_reader = StrictAvroDatumReader::new(&bytes[10..]);
2391 let datum = from_avro_datum(avro_value_schema()?, &mut datum_reader, None);
2395 if datum_reader.truncated {
2396 return Err(Error::Codec(
2397 "invalid_payload_framing: truncated Avro Value datum".to_string(),
2398 ));
2399 }
2400 let datum = datum.map_err(|err| {
2401 Error::Codec(format!(
2402 "invalid_payload_framing: malformed Avro Value datum: {err}"
2403 ))
2404 })?;
2405 if datum_reader.remaining() != 0 {
2406 return Err(Error::Codec(format!(
2407 "invalid_payload_framing: {} trailing bytes after Avro Value datum",
2408 datum_reader.remaining()
2409 )));
2410 }
2411 avro_value_from_datum(datum)
2412}
2413
2414struct StrictAvroDatumReader<'a> {
2415 bytes: &'a [u8],
2416 offset: usize,
2417 truncated: bool,
2418}
2419
2420impl<'a> StrictAvroDatumReader<'a> {
2421 fn new(bytes: &'a [u8]) -> Self {
2422 Self {
2423 bytes,
2424 offset: 0,
2425 truncated: false,
2426 }
2427 }
2428
2429 fn remaining(&self) -> usize {
2430 self.bytes.len() - self.offset
2431 }
2432}
2433
2434impl Read for StrictAvroDatumReader<'_> {
2435 fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
2436 let count = buffer.len().min(self.remaining());
2437 buffer[..count].copy_from_slice(&self.bytes[self.offset..self.offset + count]);
2438 self.offset += count;
2439 if count < buffer.len() {
2440 self.truncated = true;
2441 }
2442
2443 Ok(count)
2444 }
2445}
2446
2447fn avro_value_to_datum(value: &AvroValue) -> Result<AvroDatum> {
2448 let branch = match value {
2449 AvroValue::Null => AvroDatum::Union(0, Box::new(AvroDatum::Null)),
2450 AvroValue::Boolean(value) => AvroDatum::Union(
2451 1,
2452 Box::new(AvroDatum::Record(vec![(
2453 "boolean".to_string(),
2454 AvroDatum::Boolean(*value),
2455 )])),
2456 ),
2457 AvroValue::Long(value) => AvroDatum::Union(
2458 2,
2459 Box::new(AvroDatum::Record(vec![(
2460 "long".to_string(),
2461 AvroDatum::Long(*value),
2462 )])),
2463 ),
2464 AvroValue::Double(value) => {
2465 if !value.is_finite() {
2466 return Err(Error::Codec(
2467 "non_finite_float: Avro Value doubles must be finite".to_string(),
2468 ));
2469 }
2470 AvroDatum::Union(
2471 3,
2472 Box::new(AvroDatum::Record(vec![(
2473 "double".to_string(),
2474 AvroDatum::Double(*value),
2475 )])),
2476 )
2477 }
2478 AvroValue::Bytes(value) => AvroDatum::Union(
2479 4,
2480 Box::new(AvroDatum::Record(vec![(
2481 "bytes".to_string(),
2482 AvroDatum::Bytes(value.clone()),
2483 )])),
2484 ),
2485 AvroValue::String(value) => AvroDatum::Union(
2486 5,
2487 Box::new(AvroDatum::Record(vec![(
2488 "string".to_string(),
2489 AvroDatum::String(value.clone()),
2490 )])),
2491 ),
2492 AvroValue::Array(values) => AvroDatum::Union(
2493 6,
2494 Box::new(AvroDatum::Record(vec![(
2495 "items".to_string(),
2496 AvroDatum::Array(
2497 values
2498 .iter()
2499 .map(avro_value_to_datum)
2500 .collect::<Result<Vec<_>>>()?,
2501 ),
2502 )])),
2503 ),
2504 AvroValue::Map(values) => AvroDatum::Union(
2505 7,
2506 Box::new(AvroDatum::Record(vec![(
2507 "entries".to_string(),
2508 AvroDatum::Array(
2509 values
2510 .iter()
2511 .map(|(key, value)| {
2512 Ok(AvroDatum::Record(vec![
2513 ("key".to_string(), AvroDatum::String(key.clone())),
2514 ("value".to_string(), avro_value_to_datum(value)?),
2515 ]))
2516 })
2517 .collect::<Result<Vec<_>>>()?,
2518 ),
2519 )])),
2520 ),
2521 };
2522 Ok(AvroDatum::Record(vec![("value".to_string(), branch)]))
2523}
2524
2525fn avro_value_from_datum(datum: AvroDatum) -> Result<AvroValue> {
2526 let AvroDatum::Record(mut outer) = datum else {
2527 return Err(Error::Codec(
2528 "invalid_payload_framing: datum is not a Value record".to_string(),
2529 ));
2530 };
2531 let (_, branch) = outer
2532 .pop()
2533 .filter(|(name, _)| name == "value")
2534 .ok_or_else(|| Error::Codec("invalid_payload_framing: Value field missing".to_string()))?;
2535 let AvroDatum::Union(_, branch) = branch else {
2536 return Err(Error::Codec(
2537 "invalid_payload_framing: invalid Value union".to_string(),
2538 ));
2539 };
2540 match *branch {
2541 AvroDatum::Null => Ok(AvroValue::Null),
2542 AvroDatum::Record(mut fields) => {
2543 let (name, value) = fields.pop().ok_or_else(|| {
2544 Error::Codec("invalid_payload_framing: empty Value branch".to_string())
2545 })?;
2546 match (name.as_str(), value) {
2547 ("boolean", AvroDatum::Boolean(value)) => Ok(AvroValue::Boolean(value)),
2548 ("long", AvroDatum::Long(value)) => Ok(AvroValue::Long(value)),
2549 ("double", AvroDatum::Double(value)) if value.is_finite() => {
2550 Ok(AvroValue::Double(value))
2551 }
2552 ("bytes", AvroDatum::Bytes(value)) => Ok(AvroValue::Bytes(value)),
2553 ("string", AvroDatum::String(value)) => Ok(AvroValue::String(value)),
2554 ("items", AvroDatum::Array(values)) => values
2555 .into_iter()
2556 .map(avro_value_from_datum)
2557 .collect::<Result<Vec<_>>>()
2558 .map(AvroValue::Array),
2559 ("entries", AvroDatum::Map(values)) => values
2560 .into_iter()
2561 .map(|(key, value)| Ok((key, avro_value_from_datum(value)?)))
2562 .collect::<Result<BTreeMap<_, _>>>()
2563 .map(AvroValue::Map),
2564 _ => Err(Error::Codec(
2565 "invalid_payload_framing: unknown Value branch".to_string(),
2566 )),
2567 }
2568 }
2569 _ => Err(Error::Codec(
2570 "invalid_payload_framing: invalid Value branch".to_string(),
2571 )),
2572 }
2573}
2574
2575fn avro_value_schema() -> Result<&'static Schema> {
2576 match AVRO_VALUE_SCHEMA.get_or_init(|| {
2577 Schema::parse_str(AVRO_VALUE_SCHEMA_JSON)
2578 .map_err(|err| format!("could not parse Avro Value schema: {err}"))
2579 }) {
2580 Ok(schema) => Ok(schema),
2581 Err(message) => Err(Error::Codec(message.clone())),
2582 }
2583}
2584
2585fn avro_value_ordered_map_encoding_schema() -> Result<&'static Schema> {
2586 match AVRO_VALUE_ORDERED_MAP_ENCODING_SCHEMA.get_or_init(|| {
2587 let mut schema: Value = serde_json::from_str(AVRO_VALUE_SCHEMA_JSON)
2592 .map_err(|err| format!("could not read packaged Avro Value schema: {err}"))?;
2593 let entries_schema = schema
2594 .pointer_mut("/fields/0/type/7/fields/0/type")
2595 .ok_or_else(|| "packaged Avro Value map schema is missing".to_string())?;
2596 if *entries_schema != json!({"type": "map", "values": "Value"}) {
2597 return Err("packaged Avro Value map schema changed unexpectedly".to_string());
2598 }
2599 *entries_schema = json!({
2600 "type": "array",
2601 "items": {
2602 "type": "record",
2603 "name": "MapEntry",
2604 "fields": [
2605 {"name": "key", "type": "string"},
2606 {"name": "value", "type": "Value"}
2607 ]
2608 }
2609 });
2610 Schema::parse_str(&schema.to_string())
2611 .map_err(|err| format!("could not parse ordered-map Avro Value schema: {err}"))
2612 }) {
2613 Ok(schema) => Ok(schema),
2614 Err(message) => Err(Error::Codec(message.clone())),
2615 }
2616}
2617
2618#[derive(Clone, Debug)]
2619pub struct Client {
2620 http: reqwest::Client,
2621 base_url: String,
2622 token: Option<String>,
2623 control_token: Option<String>,
2624 worker_token: Option<String>,
2625 namespace: String,
2626 max_external_payload_bytes: usize,
2627 worker_storage_admission: Option<WorkerStorageAdmission>,
2628 runtime_upload_policy: Arc<Mutex<runtime_uploads::PolicyCache>>,
2629}
2630
2631impl Client {
2632 pub fn new(base_url: impl Into<String>) -> Result<Self> {
2633 Self::builder(base_url).build()
2634 }
2635
2636 pub fn builder(base_url: impl Into<String>) -> ClientBuilder {
2637 ClientBuilder {
2638 base_url: base_url.into(),
2639 token: None,
2640 control_token: None,
2641 worker_token: None,
2642 namespace: "default".to_string(),
2643 timeout: Duration::from_secs(60),
2644 max_external_payload_bytes: 64 * 1024 * 1024,
2645 }
2646 }
2647
2648 pub async fn health(&self) -> Result<Value> {
2649 self.request_json(
2650 reqwest::Method::GET,
2651 "/health",
2652 RequestProtocol::ControlPlane,
2653 Option::<&Value>::None,
2654 )
2655 .await
2656 }
2657
2658 pub async fn cluster_info(&self) -> Result<Value> {
2659 self.request_json(
2660 reqwest::Method::GET,
2661 "/cluster/info",
2662 RequestProtocol::ControlPlane,
2663 Option::<&Value>::None,
2664 )
2665 .await
2666 }
2667
2668 pub async fn start_workflow<T: Serialize>(
2669 &self,
2670 workflow_type: &str,
2671 task_queue: &str,
2672 workflow_id: &str,
2673 input: T,
2674 ) -> Result<WorkflowHandle> {
2675 self.start_workflow_with_options(
2676 workflow_type,
2677 task_queue,
2678 workflow_id,
2679 WorkflowStartOptions::default(),
2680 input,
2681 )
2682 .await
2683 }
2684
2685 pub async fn start_workflow_with_options<T: Serialize>(
2688 &self,
2689 workflow_type: &str,
2690 task_queue: &str,
2691 workflow_id: &str,
2692 options: WorkflowStartOptions,
2693 input: T,
2694 ) -> Result<WorkflowHandle> {
2695 options.validate()?;
2696 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2697 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2698 let body = json!({
2699 "workflow_id": workflow_id,
2700 "workflow_type": workflow_type,
2701 "task_queue": task_queue,
2702 "input": input_envelope,
2703 "execution_timeout_seconds": options.execution_timeout_seconds,
2704 "run_timeout_seconds": options.run_timeout_seconds
2705 });
2706
2707 let data: Value = self
2708 .request_json(
2709 reqwest::Method::POST,
2710 "/workflows",
2711 RequestProtocol::ControlPlane,
2712 Some(&body),
2713 )
2714 .await?;
2715
2716 Ok(WorkflowHandle {
2717 client: self.clone(),
2718 workflow_id: data
2719 .get("workflow_id")
2720 .and_then(Value::as_str)
2721 .unwrap_or(workflow_id)
2722 .to_string(),
2723 run_id: data
2724 .get("run_id")
2725 .and_then(Value::as_str)
2726 .map(str::to_string),
2727 workflow_type: data
2728 .get("workflow_type")
2729 .and_then(Value::as_str)
2730 .unwrap_or(workflow_type)
2731 .to_string(),
2732 })
2733 }
2734
2735 pub async fn signal_workflow<T: Serialize>(
2736 &self,
2737 workflow_id: &str,
2738 signal_name: &str,
2739 input: T,
2740 ) -> Result<Value> {
2741 self.signal_workflow_target(workflow_id, None, signal_name, input)
2742 .await
2743 }
2744
2745 pub async fn append_message_stream<T: Serialize>(
2747 &self,
2748 workflow_id: &str,
2749 stream_name: &str,
2750 message_id: &str,
2751 input: T,
2752 ) -> Result<Value> {
2753 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2754 let body = json!({
2755 "message_id": message_id,
2756 "input": encode_typed_envelope(&input, DEFAULT_CODEC)?
2757 });
2758 self.request_json(
2759 reqwest::Method::POST,
2760 &format!("/workflows/{workflow_id}/message-streams/{stream_name}/messages"),
2761 RequestProtocol::ControlPlane,
2762 Some(&body),
2763 )
2764 .await
2765 }
2766
2767 pub async fn signal_workflow_run<T: Serialize>(
2769 &self,
2770 workflow_id: &str,
2771 run_id: &str,
2772 signal_name: &str,
2773 input: T,
2774 ) -> Result<Value> {
2775 self.signal_workflow_target(workflow_id, Some(run_id), signal_name, input)
2776 .await
2777 }
2778
2779 async fn signal_workflow_target<T: Serialize>(
2780 &self,
2781 workflow_id: &str,
2782 run_id: Option<&str>,
2783 signal_name: &str,
2784 input: T,
2785 ) -> Result<Value> {
2786 validate_user_signal_name(signal_name)?;
2787 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2788 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2789 let body = json!({
2790 "input": input_envelope
2791 });
2792 let path = match run_id {
2793 Some(run_id) => {
2794 format!("/workflows/{workflow_id}/runs/{run_id}/signal/{signal_name}")
2795 }
2796 None => format!("/workflows/{workflow_id}/signal/{signal_name}"),
2797 };
2798 self.request_json(
2799 reqwest::Method::POST,
2800 &path,
2801 RequestProtocol::ControlPlane,
2802 Some(&body),
2803 )
2804 .await
2805 }
2806
2807 pub async fn cancel_workflow(
2809 &self,
2810 workflow_id: &str,
2811 options: WorkflowCommandOptions,
2812 ) -> Result<WorkflowCommandResult> {
2813 self.workflow_command(workflow_id, None, WorkflowCommandKind::Cancel, options)
2814 .await
2815 }
2816
2817 pub async fn cancel_workflow_run(
2819 &self,
2820 workflow_id: &str,
2821 run_id: &str,
2822 options: WorkflowCommandOptions,
2823 ) -> Result<WorkflowCommandResult> {
2824 self.workflow_command(
2825 workflow_id,
2826 Some(run_id),
2827 WorkflowCommandKind::Cancel,
2828 options,
2829 )
2830 .await
2831 }
2832
2833 pub async fn terminate_workflow(
2835 &self,
2836 workflow_id: &str,
2837 options: WorkflowCommandOptions,
2838 ) -> Result<WorkflowCommandResult> {
2839 self.workflow_command(workflow_id, None, WorkflowCommandKind::Terminate, options)
2840 .await
2841 }
2842
2843 pub async fn terminate_workflow_run(
2845 &self,
2846 workflow_id: &str,
2847 run_id: &str,
2848 options: WorkflowCommandOptions,
2849 ) -> Result<WorkflowCommandResult> {
2850 self.workflow_command(
2851 workflow_id,
2852 Some(run_id),
2853 WorkflowCommandKind::Terminate,
2854 options,
2855 )
2856 .await
2857 }
2858
2859 async fn workflow_command(
2860 &self,
2861 workflow_id: &str,
2862 run_id: Option<&str>,
2863 command: WorkflowCommandKind,
2864 options: WorkflowCommandOptions,
2865 ) -> Result<WorkflowCommandResult> {
2866 let path = match run_id {
2867 Some(run_id) => format!(
2868 "/workflows/{workflow_id}/runs/{run_id}/{}",
2869 command.as_str()
2870 ),
2871 None => format!("/workflows/{workflow_id}/{}", command.as_str()),
2872 };
2873 let data = match self
2874 .request_json(
2875 reqwest::Method::POST,
2876 &path,
2877 RequestProtocol::ControlPlane,
2878 Some(&options),
2879 )
2880 .await
2881 {
2882 Ok(data) => data,
2883 Err(Error::Http { status, body }) => {
2884 return Err(Error::WorkflowCommandRejected(workflow_command_rejection(
2885 command,
2886 status,
2887 body,
2888 workflow_id,
2889 run_id,
2890 )));
2891 }
2892 Err(error) => return Err(error),
2893 };
2894
2895 Ok(workflow_command_result(command, data, workflow_id, run_id))
2896 }
2897
2898 pub async fn query_workflow<T: Serialize>(
2904 &self,
2905 workflow_id: &str,
2906 query_name: &str,
2907 input: T,
2908 ) -> Result<Value> {
2909 self.query_workflow_target(workflow_id, None, query_name, input)
2910 .await
2911 }
2912
2913 pub async fn query_workflow_run<T: Serialize>(
2915 &self,
2916 workflow_id: &str,
2917 run_id: &str,
2918 query_name: &str,
2919 input: T,
2920 ) -> Result<Value> {
2921 self.query_workflow_target(workflow_id, Some(run_id), query_name, input)
2922 .await
2923 }
2924
2925 pub async fn query_workflow_avro_value<T: Serialize>(
2927 &self,
2928 workflow_id: &str,
2929 query_name: &str,
2930 input: T,
2931 ) -> Result<AvroValue> {
2932 self.query_workflow_avro_value_target(workflow_id, None, query_name, input)
2933 .await
2934 }
2935
2936 pub async fn query_workflow_run_avro_value<T: Serialize>(
2938 &self,
2939 workflow_id: &str,
2940 run_id: &str,
2941 query_name: &str,
2942 input: T,
2943 ) -> Result<AvroValue> {
2944 self.query_workflow_avro_value_target(workflow_id, Some(run_id), query_name, input)
2945 .await
2946 }
2947
2948 async fn query_workflow_avro_value_target<T: Serialize>(
2949 &self,
2950 workflow_id: &str,
2951 run_id: Option<&str>,
2952 query_name: &str,
2953 input: T,
2954 ) -> Result<AvroValue> {
2955 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2956 let body = json!({"input": encode_typed_envelope(&input, DEFAULT_CODEC)?});
2957 let path = match run_id {
2958 Some(run_id) => {
2959 format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
2960 }
2961 None => format!("/workflows/{workflow_id}/query/{query_name}"),
2962 };
2963 let response: Value = match self
2964 .request_json(
2965 reqwest::Method::POST,
2966 &path,
2967 RequestProtocol::ControlPlane,
2968 Some(&body),
2969 )
2970 .await
2971 {
2972 Ok(response) => response,
2973 Err(Error::Http { status, body }) => {
2974 return Err(Error::QueryFailed(query_failure(status, body)));
2975 }
2976 Err(error) => return Err(error),
2977 };
2978
2979 let envelope = response
2980 .get("result_envelope")
2981 .filter(|envelope| !envelope.is_null())
2982 .ok_or_else(|| {
2983 Error::Codec(
2984 "missing_payload_envelope: typed query result requires result_envelope"
2985 .to_string(),
2986 )
2987 })?;
2988 decode_wire_avro_value(envelope, DEFAULT_CODEC)
2989 }
2990
2991 async fn query_workflow_target<T: Serialize>(
2992 &self,
2993 workflow_id: &str,
2994 run_id: Option<&str>,
2995 query_name: &str,
2996 input: T,
2997 ) -> Result<Value> {
2998 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2999 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
3000 let body = json!({
3001 "input": input_envelope
3002 });
3003 let path = match run_id {
3004 Some(run_id) => {
3005 format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
3006 }
3007 None => format!("/workflows/{workflow_id}/query/{query_name}"),
3008 };
3009 let response: Value = match self
3010 .request_json(
3011 reqwest::Method::POST,
3012 &path,
3013 RequestProtocol::ControlPlane,
3014 Some(&body),
3015 )
3016 .await
3017 {
3018 Ok(response) => response,
3019 Err(Error::Http { status, body }) => {
3020 return Err(Error::QueryFailed(query_failure(status, body)));
3021 }
3022 Err(error) => return Err(error),
3023 };
3024
3025 if let Some(envelope) = response
3026 .get("result_envelope")
3027 .filter(|envelope| !envelope.is_null())
3028 {
3029 return decode_wire_value(envelope, DEFAULT_CODEC);
3030 }
3031
3032 Ok(response.get("result").cloned().unwrap_or(Value::Null))
3033 }
3034
3035 pub async fn update_workflow<T: Serialize>(
3037 &self,
3038 workflow_id: &str,
3039 update_name: &str,
3040 input: T,
3041 request_id: Option<&str>,
3042 ) -> Result<Value> {
3043 let response = self
3044 .update_workflow_response(workflow_id, update_name, input, request_id)
3045 .await?;
3046 if let Some(envelope) = response
3047 .get("result_envelope")
3048 .filter(|envelope| !envelope.is_null())
3049 {
3050 return decode_wire_value(envelope, DEFAULT_CODEC);
3051 }
3052 Ok(response.get("result").cloned().unwrap_or(response))
3053 }
3054
3055 pub async fn update_workflow_avro_value<T: Serialize>(
3057 &self,
3058 workflow_id: &str,
3059 update_name: &str,
3060 input: T,
3061 request_id: Option<&str>,
3062 ) -> Result<AvroValue> {
3063 let response = self
3064 .update_workflow_response(workflow_id, update_name, input, request_id)
3065 .await?;
3066 let envelope = response
3067 .get("result_envelope")
3068 .filter(|envelope| !envelope.is_null())
3069 .ok_or_else(|| {
3070 Error::Codec(
3071 "missing_payload_envelope: typed update result requires result_envelope"
3072 .to_string(),
3073 )
3074 })?;
3075 decode_wire_avro_value(envelope, DEFAULT_CODEC)
3076 }
3077
3078 async fn update_workflow_response<T: Serialize>(
3079 &self,
3080 workflow_id: &str,
3081 update_name: &str,
3082 input: T,
3083 request_id: Option<&str>,
3084 ) -> Result<Value> {
3085 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
3086 let mut body = json!({
3087 "input": encode_typed_envelope(&input, DEFAULT_CODEC)?,
3088 "wait_for": "completed",
3089 });
3090 if let Some(request_id) = request_id {
3091 body["request_id"] = json!(request_id);
3092 }
3093 self.request_json(
3094 reqwest::Method::POST,
3095 &format!("/workflows/{workflow_id}/update/{update_name}"),
3096 RequestProtocol::ControlPlane,
3097 Some(&body),
3098 )
3099 .await
3100 }
3101
3102 pub async fn describe_workflow(&self, workflow_id: &str) -> Result<WorkflowDescription> {
3103 let path = format!("/workflows/{workflow_id}");
3104 let mut data: WorkflowDescription = self
3105 .request_json(
3106 reqwest::Method::GET,
3107 &path,
3108 RequestProtocol::ControlPlane,
3109 Option::<&Value>::None,
3110 )
3111 .await?;
3112 data.decode_payloads()?;
3113 Ok(data)
3114 }
3115
3116 pub async fn describe_workflow_run(
3118 &self,
3119 workflow_id: &str,
3120 run_id: &str,
3121 ) -> Result<WorkflowDescription> {
3122 let path = format!("/workflows/{workflow_id}/runs/{run_id}");
3123 let mut data: WorkflowDescription = self
3124 .request_json(
3125 reqwest::Method::GET,
3126 &path,
3127 RequestProtocol::ControlPlane,
3128 Option::<&Value>::None,
3129 )
3130 .await?;
3131 data.decode_payloads()?;
3132 Ok(data)
3133 }
3134
3135 fn workflow_stream_path(workflow_id: &str, run_id: &str, stream_name: Option<&str>) -> String {
3136 let mut path = format!(
3137 "/workflows/{}/runs/{}/streams",
3138 percent_encode_path_segment(workflow_id),
3139 percent_encode_path_segment(run_id),
3140 );
3141 if let Some(stream_name) = stream_name {
3142 path.push('/');
3143 path.push_str(&percent_encode_path_segment(stream_name));
3144 }
3145 path
3146 }
3147
3148 pub async fn list_workflow_streams(
3150 &self,
3151 workflow_id: &str,
3152 run_id: &str,
3153 ) -> Result<Vec<WorkflowStreamDescription>> {
3154 let response: WorkflowStreamListResponse = self
3155 .request_json(
3156 reqwest::Method::GET,
3157 &Self::workflow_stream_path(workflow_id, run_id, None),
3158 RequestProtocol::ControlPlane,
3159 Option::<&Value>::None,
3160 )
3161 .await?;
3162 Ok(response.streams)
3163 }
3164
3165 pub async fn describe_workflow_stream(
3167 &self,
3168 workflow_id: &str,
3169 run_id: &str,
3170 stream_name: &str,
3171 ) -> Result<WorkflowStreamDescription> {
3172 let response: WorkflowStreamDescriptionResponse = self
3173 .request_json(
3174 reqwest::Method::GET,
3175 &Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3176 RequestProtocol::ControlPlane,
3177 Option::<&Value>::None,
3178 )
3179 .await?;
3180 Ok(response.stream)
3181 }
3182
3183 pub async fn subscribe_workflow_stream(
3190 &self,
3191 workflow_id: &str,
3192 run_id: &str,
3193 stream_name: &str,
3194 from_offset: u64,
3195 max_items: usize,
3196 wait: Duration,
3197 ) -> Result<WorkflowStreamPage> {
3198 let max_items = max_items.clamp(1, 500);
3199 let wait_seconds = wait.as_secs().min(MAX_LONG_POLL_TIMEOUT_SECONDS);
3200 let path = format!(
3201 "{}/items?from={from_offset}&max_items={max_items}&wait_seconds={wait_seconds}",
3202 Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3203 );
3204 let response: WorkflowStreamPageResponse = self
3205 .request_json_with_timeout(
3206 reqwest::Method::GET,
3207 &path,
3208 RequestProtocol::ControlPlane,
3209 Option::<&Value>::None,
3210 Duration::from_secs(wait_seconds.saturating_add(5).max(5)),
3211 )
3212 .await?;
3213
3214 let items = response
3215 .items
3216 .into_iter()
3217 .map(|raw| {
3218 let offset = raw.get("offset").and_then(Value::as_u64).unwrap_or(0);
3219 let envelope = raw.get("payload").cloned();
3220 let payload = envelope
3221 .as_ref()
3222 .filter(|value| value.get("blob").is_some())
3223 .map(|value| decode_wire_avro_value(value, DEFAULT_CODEC))
3224 .transpose()?
3225 .map(AvroValue::into_json)
3226 .transpose()?;
3227 Ok(WorkflowStreamItem {
3228 offset,
3229 payload,
3230 payload_envelope: envelope,
3231 payload_reference: raw
3232 .get("payload_reference")
3233 .and_then(Value::as_str)
3234 .map(str::to_string),
3235 payload_codec: raw
3236 .get("payload_codec")
3237 .and_then(Value::as_str)
3238 .map(str::to_string),
3239 idempotency_key: raw
3240 .get("idempotency_key")
3241 .and_then(Value::as_str)
3242 .map(str::to_string),
3243 item_type: raw
3244 .get("item_type")
3245 .and_then(Value::as_str)
3246 .map(str::to_string),
3247 content_type: raw
3248 .get("content_type")
3249 .and_then(Value::as_str)
3250 .map(str::to_string),
3251 origin: raw
3252 .get("origin")
3253 .and_then(Value::as_str)
3254 .map(str::to_string),
3255 origin_reference: raw
3256 .get("origin_reference")
3257 .and_then(Value::as_str)
3258 .map(str::to_string),
3259 emitted_at: raw
3260 .get("emitted_at")
3261 .and_then(Value::as_str)
3262 .map(str::to_string),
3263 raw,
3264 })
3265 })
3266 .collect::<Result<Vec<_>>>()?;
3267 Ok(WorkflowStreamPage {
3268 stream: response.stream,
3269 items,
3270 next_offset: response.next_offset,
3271 terminal: response.terminal,
3272 })
3273 }
3274
3275 pub async fn append_workflow_stream(
3277 &self,
3278 workflow_id: &str,
3279 run_id: &str,
3280 stream_name: &str,
3281 items: &[WorkflowStreamAppendItem],
3282 max_pending_items: Option<u64>,
3283 ) -> Result<WorkflowStreamAppendResult> {
3284 if items.is_empty() {
3285 return Err(Error::Codec(
3286 "workflow_stream_items_empty: append requires at least one item".to_string(),
3287 ));
3288 }
3289 let mut body = json!({
3290 "items": items
3291 .iter()
3292 .map(|item| item.wire_value(None))
3293 .collect::<Vec<_>>(),
3294 });
3295 if let Some(max_pending_items) = max_pending_items {
3296 if max_pending_items == 0 {
3297 return Err(Error::Codec(
3298 "workflow_stream_pending_limit_invalid: max_pending_items must be positive"
3299 .to_string(),
3300 ));
3301 }
3302 body["max_pending_items"] = json!(max_pending_items);
3303 }
3304 let response: WorkflowStreamAppendResponse = self
3305 .request_json(
3306 reqwest::Method::POST,
3307 &format!(
3308 "{}/items",
3309 Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3310 ),
3311 RequestProtocol::ControlPlane,
3312 Some(&body),
3313 )
3314 .await?;
3315 Ok(WorkflowStreamAppendResult {
3316 stream: response.stream,
3317 accepted_offsets: response.accepted_offsets,
3318 accepted: response.accepted,
3319 deduped: response.deduped,
3320 })
3321 }
3322
3323 pub async fn close_workflow_stream(
3325 &self,
3326 workflow_id: &str,
3327 run_id: &str,
3328 stream_name: &str,
3329 error_reason: Option<&str>,
3330 retention_seconds: Option<u64>,
3331 ) -> Result<WorkflowStreamDescription> {
3332 let mut body = json!({});
3333 if let Some(error_reason) = error_reason {
3334 body["error_reason"] = json!(error_reason);
3335 }
3336 if let Some(retention_seconds) = retention_seconds {
3337 if retention_seconds == 0 {
3338 return Err(Error::Codec(
3339 "workflow_stream_retention_invalid: retention_seconds must be positive"
3340 .to_string(),
3341 ));
3342 }
3343 body["retention_seconds"] = json!(retention_seconds);
3344 }
3345 let response: WorkflowStreamDescriptionResponse = self
3346 .request_json(
3347 reqwest::Method::POST,
3348 &format!(
3349 "{}/close",
3350 Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3351 ),
3352 RequestProtocol::ControlPlane,
3353 Some(&body),
3354 )
3355 .await?;
3356 Ok(response.stream)
3357 }
3358
3359 pub async fn register_worker(
3360 &self,
3361 worker_id: &str,
3362 task_queue: &str,
3363 supported_workflow_types: Vec<String>,
3364 supported_activity_types: Vec<String>,
3365 max_concurrent_workflow_tasks: usize,
3366 max_concurrent_activity_tasks: usize,
3367 ) -> Result<RegisterWorkerResponse> {
3368 self.register_worker_with_capabilities(
3369 worker_id,
3370 task_queue,
3371 supported_workflow_types,
3372 supported_activity_types,
3373 max_concurrent_workflow_tasks,
3374 max_concurrent_activity_tasks,
3375 Vec::new(),
3376 )
3377 .await
3378 }
3379
3380 pub async fn register_worker_with_capabilities(
3382 &self,
3383 worker_id: &str,
3384 task_queue: &str,
3385 supported_workflow_types: Vec<String>,
3386 supported_activity_types: Vec<String>,
3387 max_concurrent_workflow_tasks: usize,
3388 max_concurrent_activity_tasks: usize,
3389 capabilities: Vec<String>,
3390 ) -> Result<RegisterWorkerResponse> {
3391 self.register_worker_with_command_contracts(
3392 worker_id,
3393 task_queue,
3394 supported_workflow_types,
3395 supported_activity_types,
3396 max_concurrent_workflow_tasks,
3397 max_concurrent_activity_tasks,
3398 capabilities,
3399 Value::Object(serde_json::Map::new()),
3400 )
3401 .await
3402 }
3403
3404 #[allow(clippy::too_many_arguments)]
3411 pub async fn register_worker_with_command_contracts(
3412 &self,
3413 worker_id: &str,
3414 task_queue: &str,
3415 supported_workflow_types: Vec<String>,
3416 supported_activity_types: Vec<String>,
3417 max_concurrent_workflow_tasks: usize,
3418 max_concurrent_activity_tasks: usize,
3419 capabilities: Vec<String>,
3420 workflow_command_contracts: Value,
3421 ) -> Result<RegisterWorkerResponse> {
3422 if let Some(contracts) = workflow_command_contracts.as_object() {
3423 for (workflow_type, contract) in contracts {
3424 let Some(update_validators) = contract.get("update_validators") else {
3425 continue;
3426 };
3427 if !update_validators
3428 .as_array()
3429 .is_some_and(|validators| validators.is_empty())
3430 {
3431 return Err(Error::UnsupportedUpdateValidators {
3432 workflow_type: workflow_type.clone(),
3433 });
3434 }
3435 }
3436 }
3437
3438 let mut body = json!({
3439 "worker_id": worker_id,
3440 "task_queue": task_queue,
3441 "runtime": "rust",
3442 "sdk_version": SDK_VERSION,
3443 "supported_workflow_types": supported_workflow_types,
3444 "supported_activity_types": supported_activity_types,
3445 "capabilities": capabilities,
3446 "capability_manifest": portable_worker_affinity_capability_manifest(),
3447 "max_concurrent_workflow_tasks": max_concurrent_workflow_tasks,
3448 "max_concurrent_activity_tasks": max_concurrent_activity_tasks
3449 });
3450 if workflow_command_contracts
3451 .as_object()
3452 .is_some_and(|contracts| !contracts.is_empty())
3453 {
3454 body["workflow_command_contracts"] = workflow_command_contracts;
3455 }
3456
3457 self.request_json(
3458 reqwest::Method::POST,
3459 "/worker/register",
3460 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3461 Some(&body),
3462 )
3463 .await
3464 }
3465
3466 pub async fn deregister_worker_registration(
3472 &self,
3473 worker_id: &str,
3474 ) -> Result<WorkerDeregistrationEnvelope> {
3475 let path = format!(
3476 "/worker/registrations/{}",
3477 percent_encode_path_segment(worker_id)
3478 );
3479 self.request_json(
3480 reqwest::Method::DELETE,
3481 &path,
3482 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3483 Option::<&Value>::None,
3484 )
3485 .await
3486 }
3487
3488 pub async fn poll_query_task(
3490 &self,
3491 worker_id: &str,
3492 task_queue: &str,
3493 timeout: Duration,
3494 ) -> Result<Option<QueryTask>> {
3495 Ok(self
3496 .poll_query_task_response(worker_id, task_queue, timeout)
3497 .await?
3498 .task)
3499 }
3500
3501 pub async fn poll_query_task_response(
3503 &self,
3504 worker_id: &str,
3505 task_queue: &str,
3506 timeout: Duration,
3507 ) -> Result<PollQueryTaskResponse> {
3508 let poll_request_id = unique_request_id("rust-query-poll");
3509 self.poll_query_task_response_with_request_id(
3510 worker_id,
3511 task_queue,
3512 timeout,
3513 &poll_request_id,
3514 1,
3515 )
3516 .await
3517 }
3518
3519 async fn poll_query_task_response_with_request_id(
3520 &self,
3521 worker_id: &str,
3522 task_queue: &str,
3523 timeout: Duration,
3524 poll_request_id: &str,
3525 transport_retries: usize,
3526 ) -> Result<PollQueryTaskResponse> {
3527 let timeout_seconds = long_poll_timeout_seconds(timeout);
3528 let body = json!({
3529 "worker_id": worker_id,
3530 "task_queue": task_queue,
3531 "poll_request_id": poll_request_id,
3532 "timeout_seconds": timeout_seconds,
3533 });
3534 self.poll_request_json(
3535 "/worker/query-tasks/poll",
3536 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3537 &body,
3538 timeout + Duration::from_secs(5),
3539 transport_retries,
3540 )
3541 .await
3542 }
3543
3544 pub async fn complete_query_task<T: Serialize>(
3546 &self,
3547 query_task_id: &str,
3548 lease_owner: &str,
3549 query_task_attempt: u64,
3550 result: T,
3551 codec: &str,
3552 ) -> Result<Value> {
3553 let typed_result = AvroValue::from_serialize(&result)?;
3554 let result_envelope = encode_typed_envelope(&typed_result, codec)?;
3555 self.complete_query_task_with_envelope(
3556 query_task_id,
3557 lease_owner,
3558 query_task_attempt,
3559 typed_result.into_json()?,
3560 result_envelope,
3561 )
3562 .await
3563 }
3564
3565 async fn complete_query_task_with_envelope(
3566 &self,
3567 query_task_id: &str,
3568 lease_owner: &str,
3569 query_task_attempt: u64,
3570 result: Value,
3571 result_envelope: Value,
3572 ) -> Result<Value> {
3573 let body = json!({
3574 "lease_owner": lease_owner,
3575 "query_task_attempt": query_task_attempt,
3576 "result": result,
3577 "result_envelope": result_envelope,
3578 });
3579 let path = format!("/worker/query-tasks/{query_task_id}/complete");
3580 let response = self
3581 .request_json(
3582 reqwest::Method::POST,
3583 &path,
3584 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3585 Some(&body),
3586 )
3587 .await;
3588 query_task_response(response)
3589 }
3590
3591 pub async fn fail_query_task(
3593 &self,
3594 query_task_id: &str,
3595 lease_owner: &str,
3596 query_task_attempt: u64,
3597 message: impl Into<String>,
3598 reason: impl Into<String>,
3599 failure_type: impl Into<String>,
3600 ) -> Result<Value> {
3601 let body = json!({
3602 "lease_owner": lease_owner,
3603 "query_task_attempt": query_task_attempt,
3604 "failure": {
3605 "message": message.into(),
3606 "reason": reason.into(),
3607 "type": failure_type.into(),
3608 }
3609 });
3610 let path = format!("/worker/query-tasks/{query_task_id}/fail");
3611 let response = self
3612 .request_json(
3613 reqwest::Method::POST,
3614 &path,
3615 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3616 Some(&body),
3617 )
3618 .await;
3619 query_task_response(response)
3620 }
3621
3622 pub async fn heartbeat_worker(
3623 &self,
3624 worker_id: &str,
3625 workflow_available: usize,
3626 activity_available: usize,
3627 ) -> Result<Value> {
3628 let body = json!({
3629 "worker_id": worker_id,
3630 "task_slots": {
3631 "workflow_available": workflow_available,
3632 "activity_available": activity_available
3633 },
3634 "process_metrics": {
3635 "process_id": std::process::id(),
3636 "process_uptime_seconds": 0
3637 }
3638 });
3639
3640 self.request_json(
3641 reqwest::Method::POST,
3642 "/worker/heartbeat",
3643 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3644 Some(&body),
3645 )
3646 .await
3647 }
3648
3649 pub async fn poll_workflow_task(
3650 &self,
3651 worker_id: &str,
3652 task_queue: &str,
3653 timeout: Duration,
3654 ) -> Result<Option<WorkflowTask>> {
3655 Ok(self
3656 .poll_workflow_task_response(worker_id, task_queue, timeout)
3657 .await?
3658 .task)
3659 }
3660
3661 pub async fn poll_workflow_task_response(
3662 &self,
3663 worker_id: &str,
3664 task_queue: &str,
3665 timeout: Duration,
3666 ) -> Result<PollWorkflowTaskResponse> {
3667 let poll_request_id = unique_request_id("rust-workflow-poll");
3668 self.poll_workflow_task_response_with_request_id(
3669 worker_id,
3670 task_queue,
3671 timeout,
3672 &poll_request_id,
3673 1,
3674 )
3675 .await
3676 }
3677
3678 async fn poll_workflow_task_response_with_request_id(
3679 &self,
3680 worker_id: &str,
3681 task_queue: &str,
3682 timeout: Duration,
3683 poll_request_id: &str,
3684 transport_retries: usize,
3685 ) -> Result<PollWorkflowTaskResponse> {
3686 let body = json!({
3687 "worker_id": worker_id,
3688 "task_queue": task_queue,
3689 "poll_request_id": poll_request_id,
3690 "timeout_seconds": long_poll_timeout_seconds(timeout),
3691 });
3692 let mut data: PollWorkflowTaskResponse = self
3693 .poll_request_json(
3694 "/worker/workflow-tasks/poll",
3695 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3696 &body,
3697 timeout + Duration::from_secs(5),
3698 transport_retries,
3699 )
3700 .await?;
3701
3702 if let Some(task) = data.task.as_mut() {
3703 self.fetch_remaining_workflow_history(worker_id, task)
3704 .await?;
3705 }
3706
3707 Ok(data)
3708 }
3709
3710 async fn fetch_remaining_workflow_history(
3711 &self,
3712 worker_id: &str,
3713 task: &mut WorkflowTask,
3714 ) -> Result<()> {
3715 let mut next_token = task.next_history_page_token.clone();
3716
3717 while let Some(token) = next_token.take().filter(|token| !token.is_empty()) {
3718 let lease_owner = task
3719 .lease_owner
3720 .clone()
3721 .unwrap_or_else(|| worker_id.to_string());
3722 let page = self
3723 .workflow_task_history_page(
3724 &task.task_id,
3725 &lease_owner,
3726 task.workflow_task_attempt,
3727 &token,
3728 )
3729 .await?;
3730
3731 task.append_history_page(page);
3732
3733 if task.next_history_page_token.as_deref() == Some(token.as_str()) {
3734 return Err(Error::Codec(
3735 "workflow history pagination returned the same page token".to_string(),
3736 ));
3737 }
3738
3739 next_token = task.next_history_page_token.clone();
3740 }
3741
3742 Ok(())
3743 }
3744
3745 async fn workflow_task_history_page(
3746 &self,
3747 task_id: &str,
3748 lease_owner: &str,
3749 workflow_task_attempt: u64,
3750 next_history_page_token: &str,
3751 ) -> Result<WorkflowTaskHistoryPage> {
3752 let body = json!({
3753 "lease_owner": lease_owner,
3754 "workflow_task_attempt": workflow_task_attempt,
3755 "next_history_page_token": next_history_page_token
3756 });
3757 let path = format!("/worker/workflow-tasks/{task_id}/history");
3758
3759 self.request_json(
3760 reqwest::Method::POST,
3761 &path,
3762 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3763 Some(&body),
3764 )
3765 .await
3766 }
3767
3768 pub async fn complete_workflow_task(
3769 &self,
3770 task_id: &str,
3771 lease_owner: &str,
3772 workflow_task_attempt: u64,
3773 commands: Vec<Value>,
3774 ) -> Result<Value> {
3775 self.complete_workflow_task_with_message_streams(
3776 task_id,
3777 lease_owner,
3778 workflow_task_attempt,
3779 commands,
3780 Vec::new(),
3781 Vec::new(),
3782 )
3783 .await
3784 }
3785
3786 async fn complete_workflow_task_with_message_streams(
3787 &self,
3788 task_id: &str,
3789 lease_owner: &str,
3790 workflow_task_attempt: u64,
3791 commands: Vec<Value>,
3792 message_stream_cursors: Vec<Value>,
3793 message_stream_waits: Vec<Value>,
3794 ) -> Result<Value> {
3795 validate_workflow_task_commands(&commands)?;
3796 let has_message_stream_metadata =
3797 !message_stream_cursors.is_empty() || !message_stream_waits.is_empty();
3798 if has_message_stream_metadata
3799 && !worker_protocol_supports_message_streams(WORKER_PROTOCOL_VERSION)
3800 {
3801 return Err(Error::Codec(
3802 "message_streams_unavailable: message stream completion metadata requires worker protocol 1.15 or newer"
3803 .to_string(),
3804 ));
3805 }
3806 let protocol_version = workflow_completion_protocol_version_with_message_streams(
3807 &commands,
3808 has_message_stream_metadata,
3809 );
3810 let mut body = json!({
3811 "lease_owner": lease_owner,
3812 "workflow_task_attempt": workflow_task_attempt,
3813 "commands": commands
3814 });
3815 if !message_stream_cursors.is_empty() {
3816 body["message_stream_cursors"] = Value::Array(message_stream_cursors);
3817 }
3818 if !message_stream_waits.is_empty() {
3819 body["message_stream_waits"] = Value::Array(message_stream_waits);
3820 }
3821 let path = format!("/worker/workflow-tasks/{task_id}/complete");
3822 self.request_json(
3823 reqwest::Method::POST,
3824 &path,
3825 RequestProtocol::Worker(protocol_version),
3826 Some(&body),
3827 )
3828 .await
3829 }
3830
3831 pub async fn fail_workflow_task(
3832 &self,
3833 task_id: &str,
3834 lease_owner: &str,
3835 workflow_task_attempt: u64,
3836 message: impl Into<String>,
3837 ) -> Result<Value> {
3838 self.fail_workflow_task_with_type(
3839 task_id,
3840 lease_owner,
3841 workflow_task_attempt,
3842 message,
3843 "RustWorkflowTaskFailure",
3844 )
3845 .await
3846 }
3847
3848 async fn fail_workflow_task_with_type(
3849 &self,
3850 task_id: &str,
3851 lease_owner: &str,
3852 workflow_task_attempt: u64,
3853 message: impl Into<String>,
3854 failure_type: &str,
3855 ) -> Result<Value> {
3856 let body = json!({
3857 "lease_owner": lease_owner,
3858 "workflow_task_attempt": workflow_task_attempt,
3859 "failure": {
3860 "message": message.into(),
3861 "type": failure_type
3862 }
3863 });
3864 let path = format!("/worker/workflow-tasks/{task_id}/fail");
3865 self.request_json(
3866 reqwest::Method::POST,
3867 &path,
3868 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3869 Some(&body),
3870 )
3871 .await
3872 }
3873
3874 pub async fn poll_activity_task(
3875 &self,
3876 worker_id: &str,
3877 task_queue: &str,
3878 timeout: Duration,
3879 ) -> Result<Option<ActivityTask>> {
3880 Ok(self
3881 .poll_activity_task_response(worker_id, task_queue, timeout)
3882 .await?
3883 .task)
3884 }
3885
3886 pub async fn poll_activity_task_response(
3888 &self,
3889 worker_id: &str,
3890 task_queue: &str,
3891 timeout: Duration,
3892 ) -> Result<PollActivityTaskResponse> {
3893 let poll_request_id = unique_request_id("rust-activity-poll");
3894 self.poll_activity_task_response_with_request_id(
3895 worker_id,
3896 task_queue,
3897 timeout,
3898 &poll_request_id,
3899 1,
3900 )
3901 .await
3902 }
3903
3904 async fn poll_activity_task_response_with_request_id(
3905 &self,
3906 worker_id: &str,
3907 task_queue: &str,
3908 timeout: Duration,
3909 poll_request_id: &str,
3910 transport_retries: usize,
3911 ) -> Result<PollActivityTaskResponse> {
3912 let body = json!({
3913 "worker_id": worker_id,
3914 "task_queue": task_queue,
3915 "poll_request_id": poll_request_id,
3916 "timeout_seconds": long_poll_timeout_seconds(timeout),
3917 });
3918 let data: PollActivityTaskResponse = self
3919 .poll_request_json(
3920 "/worker/activity-tasks/poll",
3921 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3922 &body,
3923 timeout + Duration::from_secs(5),
3924 transport_retries,
3925 )
3926 .await?;
3927 Ok(data)
3928 }
3929
3930 pub async fn complete_activity_task<T: Serialize>(
3931 &self,
3932 task_id: &str,
3933 activity_attempt_id: &str,
3934 lease_owner: &str,
3935 result: T,
3936 codec: &str,
3937 ) -> Result<Value> {
3938 let result = encode_typed_envelope(&AvroValue::from_serialize(&result)?, codec)?;
3939 let body = json!({
3940 "activity_attempt_id": activity_attempt_id,
3941 "lease_owner": lease_owner,
3942 "result": result
3943 });
3944 let path = format!("/worker/activity-tasks/{task_id}/complete");
3945 activity_task_response(
3946 self.request_json(
3947 reqwest::Method::POST,
3948 &path,
3949 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3950 Some(&body),
3951 )
3952 .await,
3953 "complete",
3954 task_id,
3955 activity_attempt_id,
3956 )
3957 }
3958
3959 pub async fn fail_activity_task(
3960 &self,
3961 task_id: &str,
3962 activity_attempt_id: &str,
3963 lease_owner: &str,
3964 message: impl Into<String>,
3965 non_retryable: bool,
3966 ) -> Result<Value> {
3967 let body = json!({
3968 "activity_attempt_id": activity_attempt_id,
3969 "lease_owner": lease_owner,
3970 "failure": {
3971 "message": message.into(),
3972 "type": "RustActivityFailure",
3973 "non_retryable": non_retryable
3974 }
3975 });
3976 let path = format!("/worker/activity-tasks/{task_id}/fail");
3977 activity_task_response(
3978 self.request_json(
3979 reqwest::Method::POST,
3980 &path,
3981 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3982 Some(&body),
3983 )
3984 .await,
3985 "fail",
3986 task_id,
3987 activity_attempt_id,
3988 )
3989 }
3990
3991 pub async fn heartbeat_activity_task<T: Serialize>(
3992 &self,
3993 task_id: &str,
3994 activity_attempt_id: &str,
3995 lease_owner: &str,
3996 details: T,
3997 ) -> Result<ActivityHeartbeatResponse> {
3998 let details = encode_typed_envelope(&AvroValue::from_serialize(&details)?, DEFAULT_CODEC)?;
3999 let body = json!({
4000 "activity_attempt_id": activity_attempt_id,
4001 "lease_owner": lease_owner,
4002 "details": details
4003 });
4004 let path = format!("/worker/activity-tasks/{task_id}/heartbeat");
4005 activity_task_response(
4006 self.request_json(
4007 reqwest::Method::POST,
4008 &path,
4009 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
4010 Some(&body),
4011 )
4012 .await,
4013 "heartbeat",
4014 task_id,
4015 activity_attempt_id,
4016 )
4017 }
4018
4019 async fn request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
4020 &self,
4021 method: reqwest::Method,
4022 path: &str,
4023 protocol: RequestProtocol,
4024 body: Option<&B>,
4025 ) -> Result<T> {
4026 self.request_json_with_timeout(method, path, protocol, body, Duration::from_secs(60))
4027 .await
4028 }
4029
4030 async fn request_json_with_timeout<T: DeserializeOwned, B: Serialize + ?Sized>(
4031 &self,
4032 method: reqwest::Method,
4033 path: &str,
4034 protocol: RequestProtocol,
4035 body: Option<&B>,
4036 timeout: Duration,
4037 ) -> Result<T> {
4038 let auth_token = self.auth_token(protocol)?;
4039 let mut request = self
4040 .http
4041 .request(method.clone(), format!("{}/api{}", self.base_url, path))
4042 .timeout(timeout)
4043 .header(reqwest::header::ACCEPT, "application/json")
4044 .header(reqwest::header::CONTENT_TYPE, "application/json")
4045 .header("X-Namespace", &self.namespace);
4046
4047 match protocol {
4048 RequestProtocol::Worker(version) => {
4049 request = request.header("X-Durable-Workflow-Protocol-Version", version);
4050 }
4051 RequestProtocol::ControlPlane => {
4052 request = request.header(
4053 "X-Durable-Workflow-Control-Plane-Version",
4054 CONTROL_PLANE_VERSION,
4055 );
4056 }
4057 }
4058
4059 if let Some(token) = auth_token {
4060 request = request.bearer_auth(token);
4061 }
4062
4063 if let Some(body) = body {
4064 let mut body = serde_json::to_value(body)?;
4065 if matches!(
4066 method,
4067 reqwest::Method::POST | reqwest::Method::PUT | reqwest::Method::PATCH
4068 ) {
4069 self.externalize_runtime_payloads(&mut body, path, protocol)
4070 .await?;
4071 }
4072 request = request.json(&body);
4073 }
4074
4075 let request = request.build()?;
4076 let poll_request_id = path.ends_with("/poll").then(|| {
4077 request
4078 .body()
4079 .and_then(reqwest::Body::as_bytes)
4080 .and_then(|body| serde_json::from_slice::<Value>(body).ok())
4081 .and_then(|body| body.get("poll_request_id")?.as_str().map(str::to_owned))
4082 .unwrap_or_default()
4083 });
4084 let mut storage_retries = 0_usize;
4085
4086 loop {
4087 let response = self
4089 .http
4090 .execute(request.try_clone().ok_or_else(|| {
4091 Error::WorkerLoop("worker request body cannot be retried".to_string())
4092 })?)
4093 .await?;
4094 let status = response.status();
4095 let bytes = response.bytes().await?;
4096
4097 if !status.is_success() {
4098 let body = String::from_utf8_lossy(&bytes).to_string();
4099 if let Some(protocol) = protocol_failure(status, &body) {
4100 return Err(Error::Protocol(protocol));
4101 }
4102 let error = Error::Http { status, body };
4103 if self
4104 .wait_for_storage_admission(
4105 &error,
4106 protocol,
4107 Some(path),
4108 poll_request_id.as_deref(),
4109 &mut storage_retries,
4110 )
4111 .await
4112 {
4113 continue;
4114 }
4115 return Err(error);
4116 }
4117
4118 if bytes.is_empty() {
4119 return Ok(serde_json::from_value(Value::Null)?);
4120 }
4121
4122 let mut value: Value = serde_json::from_slice(&bytes)?;
4123 self.resolve_runtime_payloads(&mut value, path, protocol)
4124 .await?;
4125 return Ok(serde_json::from_value(value)?);
4126 }
4127 }
4128
4129 async fn wait_for_storage_admission(
4130 &self,
4131 error: &Error,
4132 protocol: RequestProtocol,
4133 path: Option<&str>,
4134 poll_request_id: Option<&str>,
4135 retries: &mut usize,
4136 ) -> bool {
4137 let Some(admission) = self
4138 .worker_storage_admission
4139 .as_ref()
4140 .filter(|_| matches!(protocol, RequestProtocol::Worker(_)))
4141 else {
4142 return false;
4143 };
4144 let Some(advertised_delay) = worker_storage_admission_retry_after(error, poll_request_id)
4145 .or_else(|| {
4146 path.and_then(|path| {
4147 worker_backend_unavailable_retry_after(error, path, poll_request_id)
4148 })
4149 })
4150 else {
4151 return false;
4152 };
4153 *retries = retries.saturating_add(1);
4154 let delay = worker_retry_delay(admission.policy, *retries)
4155 .max(advertised_delay)
4156 .min(admission.policy.max_backoff.max(Duration::from_millis(1)));
4157 let deadline = tokio::time::Instant::now() + delay;
4158 loop {
4159 if admission.stop.load(Ordering::SeqCst) {
4160 return false;
4161 }
4162 let remaining = deadline.saturating_duration_since(tokio::time::Instant::now());
4163 if remaining.is_zero() {
4164 return true;
4165 }
4166 tokio::time::sleep(remaining.min(Duration::from_millis(100))).await;
4167 }
4168 }
4169
4170 async fn poll_request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
4171 &self,
4172 path: &str,
4173 protocol: RequestProtocol,
4174 body: &B,
4175 timeout: Duration,
4176 max_retries: usize,
4177 ) -> Result<T> {
4178 let mut retries = 0;
4179
4180 loop {
4181 let response = self
4182 .request_json_with_timeout(
4183 reqwest::Method::POST,
4184 path,
4185 protocol,
4186 Some(body),
4187 timeout,
4188 )
4189 .await;
4190
4191 match response {
4192 Err(Error::Transport(_)) if retries < max_retries => retries += 1,
4193 response => return worker_poll_response(response),
4194 }
4195 }
4196 }
4197
4198 fn auth_token(&self, protocol: RequestProtocol) -> Result<Option<&str>> {
4199 match protocol {
4200 RequestProtocol::Worker(_) => {
4201 if let Some(token) = self.worker_token.as_deref().or(self.token.as_deref()) {
4202 return Ok(Some(token));
4203 }
4204 if self.control_token.is_some() {
4205 return Err(Error::MissingRoleCredentials {
4206 role: "worker",
4207 opposite_role: "control",
4208 });
4209 }
4210 Ok(None)
4211 }
4212 RequestProtocol::ControlPlane => {
4213 if let Some(token) = self.control_token.as_deref().or(self.token.as_deref()) {
4214 return Ok(Some(token));
4215 }
4216 if self.worker_token.is_some() {
4217 return Err(Error::MissingRoleCredentials {
4218 role: "control",
4219 opposite_role: "worker",
4220 });
4221 }
4222 Ok(None)
4223 }
4224 }
4225 }
4226}
4227
4228fn query_failure(status: reqwest::StatusCode, raw_body: String) -> QueryFailure {
4229 let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
4230 let reason = body
4231 .get("reason")
4232 .and_then(Value::as_str)
4233 .unwrap_or("query_rejected")
4234 .to_string();
4235 let message = body
4236 .get("message")
4237 .or_else(|| body.get("error"))
4238 .and_then(Value::as_str)
4239 .unwrap_or("workflow query was rejected")
4240 .to_string();
4241
4242 QueryFailure {
4243 status: status.as_u16(),
4244 reason,
4245 message,
4246 body,
4247 }
4248}
4249
4250fn workflow_command_result(
4251 command: WorkflowCommandKind,
4252 data: Value,
4253 workflow_id: &str,
4254 run_id: Option<&str>,
4255) -> WorkflowCommandResult {
4256 WorkflowCommandResult {
4257 command,
4258 workflow_id: data
4259 .get("workflow_id")
4260 .and_then(Value::as_str)
4261 .unwrap_or(workflow_id)
4262 .to_string(),
4263 run_id: data
4264 .get("run_id")
4265 .and_then(Value::as_str)
4266 .or(run_id)
4267 .map(str::to_string),
4268 outcome: data
4269 .get("outcome")
4270 .and_then(Value::as_str)
4271 .map(str::to_string),
4272 reason: data
4273 .get("reason")
4274 .and_then(Value::as_str)
4275 .map(str::to_string),
4276 command_status: data
4277 .get("command_status")
4278 .and_then(Value::as_str)
4279 .map(str::to_string),
4280 raw: data,
4281 }
4282}
4283
4284fn workflow_command_rejection(
4285 command: WorkflowCommandKind,
4286 status: reqwest::StatusCode,
4287 raw_body: String,
4288 workflow_id: &str,
4289 run_id: Option<&str>,
4290) -> WorkflowCommandRejection {
4291 let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
4292 WorkflowCommandRejection {
4293 command,
4294 status: status.as_u16(),
4295 reason: body
4296 .get("reason")
4297 .and_then(Value::as_str)
4298 .unwrap_or("workflow_command_rejected")
4299 .to_string(),
4300 message: body
4301 .get("message")
4302 .or_else(|| body.get("error"))
4303 .and_then(Value::as_str)
4304 .unwrap_or("workflow lifecycle command was rejected")
4305 .to_string(),
4306 workflow_id: body
4307 .get("workflow_id")
4308 .and_then(Value::as_str)
4309 .unwrap_or(workflow_id)
4310 .to_string(),
4311 run_id: body
4312 .get("run_id")
4313 .and_then(Value::as_str)
4314 .or(run_id)
4315 .map(str::to_string),
4316 target_scope: body
4317 .get("target_scope")
4318 .and_then(Value::as_str)
4319 .map(str::to_string),
4320 body,
4321 }
4322}
4323
4324fn query_task_response(response: Result<Value>) -> Result<Value> {
4325 match response {
4326 Err(Error::Http { status, body }) => Err(Error::QueryFailed(query_failure(status, body))),
4327 response => response,
4328 }
4329}
4330
4331fn worker_poll_response<T: DeserializeOwned>(response: Result<T>) -> Result<T> {
4332 match response {
4333 Err(Error::Http { status, body })
4334 if status == reqwest::StatusCode::CONFLICT && worker_poll_body_is_stop(&body) =>
4335 {
4336 Ok(serde_json::from_str(&body)?)
4337 }
4338 response => response,
4339 }
4340}
4341
4342fn worker_poll_body_is_stop(body: &str) -> bool {
4343 serde_json::from_str::<Value>(body)
4344 .ok()
4345 .is_some_and(|body| {
4346 worker_poll_is_stop(
4347 body.get("poll_status").and_then(Value::as_str),
4348 body.get("reason").and_then(Value::as_str),
4349 )
4350 })
4351}
4352
4353fn worker_poll_is_stop(poll_status: Option<&str>, reason: Option<&str>) -> bool {
4354 matches!(poll_status, Some("draining" | "stopped"))
4355 || matches!(reason, Some("worker_draining" | "worker_stopped"))
4356}
4357
4358fn query_task_rejection_is_final(error: &Error) -> bool {
4359 matches!(
4360 error,
4361 Error::QueryFailed(failure)
4362 if QUERY_TASK_FINAL_REJECTION_REASONS.contains(&failure.reason.as_str())
4363 )
4364}
4365
4366fn activity_task_response<T>(
4367 response: Result<T>,
4368 operation: &str,
4369 task_id: &str,
4370 activity_attempt_id: &str,
4371) -> Result<T> {
4372 match response {
4373 Err(Error::Http { status, body }) => {
4374 let body = serde_json::from_str(&body).unwrap_or_else(|_| json!({"message": body}));
4375 Err(Error::ActivityTaskRejected(ActivityTaskRejection {
4376 operation: operation.to_string(),
4377 status: status.as_u16(),
4378 reason: body
4379 .get("reason")
4380 .and_then(Value::as_str)
4381 .unwrap_or("activity_task_rejected")
4382 .to_string(),
4383 task_id: body
4384 .get("task_id")
4385 .and_then(Value::as_str)
4386 .unwrap_or(task_id)
4387 .to_string(),
4388 activity_attempt_id: body
4389 .get("activity_attempt_id")
4390 .and_then(Value::as_str)
4391 .unwrap_or(activity_attempt_id)
4392 .to_string(),
4393 cancel_requested: body
4394 .get("cancel_requested")
4395 .and_then(Value::as_bool)
4396 .unwrap_or(false),
4397 can_continue: body.get("can_continue").and_then(Value::as_bool),
4398 run_closed_reason: body
4399 .get("run_closed_reason")
4400 .and_then(Value::as_str)
4401 .map(str::to_string),
4402 body,
4403 }))
4404 }
4405 response => response,
4406 }
4407}
4408
4409fn activity_task_rejection_is_final(error: &Error) -> bool {
4410 matches!(
4411 error,
4412 Error::ActivityTaskRejected(rejection)
4413 if matches!(
4414 rejection.reason.as_str(),
4415 "run_cancelled"
4416 | "run_terminated"
4417 | "attempt_closed"
4418 | "stale_attempt"
4419 | "activity_cancelled"
4420 | "task_cancelled"
4421 | "run_closed"
4422 | "activity_not_running"
4423 | "attempt_not_found"
4424 )
4425 )
4426}
4427
4428fn workflow_task_completion_is_terminal_timeout(
4429 error: &Error,
4430 task_id: &str,
4431 workflow_task_attempt: u64,
4432 run_id: Option<&str>,
4433) -> bool {
4434 let Error::Http { status, body } = error else {
4435 return false;
4436 };
4437 if *status != reqwest::StatusCode::CONFLICT {
4438 return false;
4439 }
4440
4441 let Some(run_id) = run_id else {
4442 return false;
4443 };
4444 let Ok(body) = serde_json::from_str::<Value>(body) else {
4445 return false;
4446 };
4447
4448 body.get("recorded").and_then(Value::as_bool) == Some(false)
4449 && body.get("reason").and_then(Value::as_str) == Some("run_timed_out")
4450 && body.get("run_status").and_then(Value::as_str) == Some("failed")
4451 && body.get("run_id").and_then(Value::as_str) == Some(run_id)
4452 && body.get("task_id").and_then(Value::as_str) == Some(task_id)
4453 && body.get("workflow_task_attempt").and_then(Value::as_u64) == Some(workflow_task_attempt)
4454}
4455
4456fn protocol_failure(status: reqwest::StatusCode, raw_body: &str) -> Option<ProtocolFailure> {
4457 let body: Value = serde_json::from_str(raw_body).ok()?;
4458 let reason = body.get("reason")?.as_str()?;
4459 if !matches!(
4460 reason,
4461 "missing_protocol_version"
4462 | "unsupported_protocol_version"
4463 | "missing_control_plane_version"
4464 | "unsupported_control_plane_version"
4465 ) {
4466 return None;
4467 }
4468
4469 Some(ProtocolFailure {
4470 status: status.as_u16(),
4471 reason: reason.to_string(),
4472 message: body
4473 .get("message")
4474 .or_else(|| body.get("error"))
4475 .and_then(Value::as_str)
4476 .unwrap_or("protocol version rejected")
4477 .to_string(),
4478 supported_version: body
4479 .get("supported_version")
4480 .and_then(Value::as_str)
4481 .map(str::to_string),
4482 requested_version: body
4483 .get("requested_version")
4484 .and_then(Value::as_str)
4485 .map(str::to_string),
4486 body,
4487 })
4488}
4489
4490fn long_poll_timeout_seconds(timeout: Duration) -> u64 {
4491 timeout
4492 .as_secs()
4493 .saturating_add(u64::from(timeout.subsec_nanos() > 0))
4494 .min(MAX_LONG_POLL_TIMEOUT_SECONDS)
4495}
4496
4497fn worker_operation_is_retryable(error: &Error) -> bool {
4498 if worker_poll_capacity_retry_after(error).is_some()
4499 || worker_storage_admission_body(error).is_some()
4500 || worker_backend_unavailable_body(error).is_some()
4501 || worker_operation_is_explicitly_non_retryable(error)
4502 {
4503 return false;
4504 }
4505
4506 match error {
4507 Error::Transport(error) => {
4508 error.is_timeout() || error.is_connect() || error.is_request() || error.is_body()
4509 }
4510 Error::Http { status, .. } => {
4511 matches!(
4512 *status,
4513 reqwest::StatusCode::REQUEST_TIMEOUT | reqwest::StatusCode::TOO_MANY_REQUESTS
4514 ) || status.is_server_error()
4515 }
4516 _ => false,
4517 }
4518}
4519
4520fn worker_storage_admission_body(error: &Error) -> Option<Value> {
4521 let body: Value = match error {
4522 Error::Http { body, .. } => serde_json::from_str(body).ok()?,
4523 Error::ActivityTaskRejected(rejection) => rejection.body.clone(),
4524 _ => return None,
4525 };
4526 matches!(
4527 body.get("reason").and_then(Value::as_str),
4528 Some("storage_pressure" | "storage_admission_unavailable")
4529 )
4530 .then_some(body)
4531}
4532
4533fn worker_storage_admission_retry_after(
4534 error: &Error,
4535 poll_request_id: Option<&str>,
4536) -> Option<Duration> {
4537 let Error::Http { status, .. } = error else {
4538 return None;
4539 };
4540 let body = worker_storage_admission_body(error)?;
4541 let delay = body.get("retry_after_seconds")?.as_u64()?;
4542 if *status != reqwest::StatusCode::SERVICE_UNAVAILABLE
4543 || delay == 0
4544 || body.get("retryable") != Some(&Value::Bool(true))
4545 || !matches!(body.get("storage_state")?.as_str()?, "draining" | "fenced")
4546 || (body["reason"] == "storage_admission_unavailable" && body["storage_state"] != "fenced")
4547 || body
4548 .get("request_admitted")
4549 .is_some_and(|admitted| admitted != &Value::Bool(false))
4550 {
4551 return None;
4552 }
4553 match poll_request_id {
4554 Some(id) => {
4555 if id.is_empty()
4556 || body.get("task") != Some(&Value::Null)
4557 || body.get("poll_request_id").and_then(Value::as_str) != Some(id)
4558 || body.get("poll_status") != body.get("reason")
4559 || body.get("retry_same_poll_request_id") != Some(&Value::Bool(true))
4560 || body.get("claim_admitted") != Some(&Value::Bool(false))
4561 {
4562 return None;
4563 }
4564 }
4565 None if body.get("request_admitted") != Some(&Value::Bool(false)) => return None,
4566 None => {}
4567 }
4568 Some(Duration::from_secs(delay))
4569}
4570
4571fn worker_backend_unavailable_retry_after(
4572 error: &Error,
4573 path: &str,
4574 poll_request_id: Option<&str>,
4575) -> Option<Duration> {
4576 let operation = match path {
4577 "/worker/register" => "register_worker",
4578 "/worker/heartbeat" => "heartbeat_worker",
4579 "/worker/workflow-tasks/poll" => "poll_workflow_task",
4580 "/worker/activity-tasks/poll" => "poll_activity_task",
4581 "/worker/query-tasks/poll" => "poll_query_task",
4582 "/worker/update-validation-tasks/poll" => "poll_update_validation_task",
4583 _ => return None,
4584 };
4585 let body = worker_backend_unavailable_body(error)?;
4586 let delay = body.get("retry_after_seconds")?.as_u64()?;
4587 if delay == 0
4588 || body.get("operation")?.as_str()? != operation
4589 || body.get("outcome")?.as_str()? != "unknown"
4590 || body.get("retryable") != Some(&Value::Bool(true))
4591 || body.get("worker_id")?.as_str()?.is_empty()
4592 {
4593 return None;
4594 }
4595 if let Some(id) = poll_request_id {
4596 if id.is_empty()
4597 || body.get("task") != Some(&Value::Null)
4598 || body.get("poll_status")?.as_str()? != "backend_unavailable"
4599 || body.get("poll_request_id")?.as_str()? != id
4600 || body.get("retry_same_poll_request_id") != Some(&Value::Bool(true))
4601 {
4602 return None;
4603 }
4604 } else if path.ends_with("/poll") {
4605 return None;
4606 }
4607 Some(Duration::from_secs(delay))
4608}
4609
4610fn worker_backend_unavailable_body(error: &Error) -> Option<Value> {
4611 let Error::Http { status, body } = error else {
4612 return None;
4613 };
4614 if *status != reqwest::StatusCode::SERVICE_UNAVAILABLE {
4615 return None;
4616 }
4617 let body: Value = serde_json::from_str(body).ok()?;
4618 (body.get("reason")?.as_str()? == "backend_unavailable").then_some(body)
4619}
4620
4621fn worker_operation_is_explicitly_non_retryable(error: &Error) -> bool {
4622 let Error::Http { body, .. } = error else {
4623 return false;
4624 };
4625
4626 serde_json::from_str::<Value>(body)
4627 .ok()
4628 .and_then(|body| body.get("retryable").and_then(Value::as_bool))
4629 == Some(false)
4630}
4631
4632fn worker_poll_capacity_retry_after(error: &Error) -> Option<Duration> {
4633 let Error::Http { status, body } = error else {
4634 return None;
4635 };
4636 if *status != reqwest::StatusCode::TOO_MANY_REQUESTS {
4637 return None;
4638 }
4639
4640 let body = serde_json::from_str::<Value>(body).ok()?;
4641 let capacity_exhausted = body.get("poll_status").and_then(Value::as_str)
4642 == Some("long_poll_capacity_exhausted")
4643 || body.get("reason").and_then(Value::as_str) == Some("long_poll_capacity_exhausted");
4644 if !capacity_exhausted || body.get("retryable").and_then(Value::as_bool) != Some(true) {
4645 return None;
4646 }
4647
4648 Some(Duration::from_secs(
4649 body.get("retry_after_seconds")
4650 .and_then(Value::as_u64)
4651 .unwrap_or_default(),
4652 ))
4653}
4654
4655fn worker_retry_delay(policy: WorkerRetryPolicy, retry: usize) -> Duration {
4656 let exponent = retry.saturating_sub(1).min(31) as u32;
4657 policy
4658 .initial_backoff
4659 .saturating_mul(1_u32 << exponent)
4660 .min(policy.max_backoff)
4661}
4662
4663#[derive(Debug)]
4664pub struct ClientBuilder {
4665 base_url: String,
4666 token: Option<String>,
4667 control_token: Option<String>,
4668 worker_token: Option<String>,
4669 namespace: String,
4670 timeout: Duration,
4671 max_external_payload_bytes: usize,
4672}
4673
4674impl ClientBuilder {
4675 pub fn token(mut self, token: Option<String>) -> Self {
4676 self.token = token;
4677 self
4678 }
4679
4680 pub fn control_token(mut self, token: Option<String>) -> Self {
4681 self.control_token = token;
4682 self
4683 }
4684
4685 pub fn worker_token(mut self, token: Option<String>) -> Self {
4686 self.worker_token = token;
4687 self
4688 }
4689
4690 pub fn namespace(mut self, namespace: impl Into<String>) -> Self {
4691 self.namespace = namespace.into();
4692 self
4693 }
4694
4695 pub fn timeout(mut self, timeout: Duration) -> Self {
4696 self.timeout = timeout;
4697 self
4698 }
4699
4700 pub fn max_external_payload_bytes(mut self, bytes: usize) -> Self {
4703 self.max_external_payload_bytes = bytes;
4704 self
4705 }
4706
4707 pub fn build(self) -> Result<Client> {
4708 let base_url = self.base_url.trim_end_matches('/').to_string();
4709 let has_sdk_api_suffix = reqwest::Url::parse(&base_url)
4710 .map(|url| url.path().trim_end_matches('/').ends_with("/api"))
4711 .unwrap_or_else(|_| base_url.ends_with("/api"));
4712
4713 if has_sdk_api_suffix {
4714 return Err(Error::InvalidBaseUrl);
4715 }
4716
4717 Ok(Client {
4718 http: reqwest::Client::builder()
4719 .timeout(self.timeout)
4720 .redirect(reqwest::redirect::Policy::none())
4721 .build()?,
4722 base_url,
4723 token: self.token,
4724 control_token: self.control_token,
4725 worker_token: self.worker_token,
4726 namespace: self.namespace,
4727 max_external_payload_bytes: self.max_external_payload_bytes,
4728 worker_storage_admission: None,
4729 runtime_upload_policy: Arc::new(Mutex::new([None, None])),
4730 })
4731 }
4732}
4733
4734#[derive(Clone, Debug)]
4735pub struct WorkflowHandle {
4736 client: Client,
4737 pub workflow_id: String,
4738 pub run_id: Option<String>,
4739 pub workflow_type: String,
4740}
4741
4742impl WorkflowHandle {
4743 pub async fn describe(&self) -> Result<WorkflowDescription> {
4745 self.client.describe_workflow(&self.workflow_id).await
4746 }
4747
4748 pub async fn describe_selected_run(&self) -> Result<WorkflowDescription> {
4750 let run_id = self.run_id.as_deref().ok_or_else(|| {
4751 Error::Codec("run_id is required for selected-run description".to_string())
4752 })?;
4753 self.client
4754 .describe_workflow_run(&self.workflow_id, run_id)
4755 .await
4756 }
4757
4758 pub async fn signal<T: Serialize>(&self, signal_name: &str, input: T) -> Result<Value> {
4759 self.client
4760 .signal_workflow(&self.workflow_id, signal_name, input)
4761 .await
4762 }
4763
4764 pub async fn append_message<T: Serialize>(
4765 &self,
4766 stream_name: &str,
4767 message_id: &str,
4768 input: T,
4769 ) -> Result<Value> {
4770 self.client
4771 .append_message_stream(&self.workflow_id, stream_name, message_id, input)
4772 .await
4773 }
4774
4775 pub async fn signal_selected_run<T: Serialize>(
4777 &self,
4778 signal_name: &str,
4779 input: T,
4780 ) -> Result<Value> {
4781 let run_id = self.run_id.as_deref().ok_or_else(|| {
4782 Error::Codec("run_id is required for selected-run signaling".to_string())
4783 })?;
4784 self.client
4785 .signal_workflow_run(&self.workflow_id, run_id, signal_name, input)
4786 .await
4787 }
4788
4789 pub async fn cancel(&self, options: WorkflowCommandOptions) -> Result<WorkflowCommandResult> {
4791 self.client
4792 .cancel_workflow(&self.workflow_id, options)
4793 .await
4794 }
4795
4796 pub async fn cancel_selected_run(
4798 &self,
4799 options: WorkflowCommandOptions,
4800 ) -> Result<WorkflowCommandResult> {
4801 let run_id = self.run_id.as_deref().ok_or_else(|| {
4802 Error::Codec("run_id is required for selected-run cancellation".to_string())
4803 })?;
4804 self.client
4805 .cancel_workflow_run(&self.workflow_id, run_id, options)
4806 .await
4807 }
4808
4809 pub async fn terminate(
4811 &self,
4812 options: WorkflowCommandOptions,
4813 ) -> Result<WorkflowCommandResult> {
4814 self.client
4815 .terminate_workflow(&self.workflow_id, options)
4816 .await
4817 }
4818
4819 pub async fn terminate_selected_run(
4821 &self,
4822 options: WorkflowCommandOptions,
4823 ) -> Result<WorkflowCommandResult> {
4824 let run_id = self.run_id.as_deref().ok_or_else(|| {
4825 Error::Codec("run_id is required for selected-run termination".to_string())
4826 })?;
4827 self.client
4828 .terminate_workflow_run(&self.workflow_id, run_id, options)
4829 .await
4830 }
4831
4832 pub async fn query<T: Serialize>(&self, query_name: &str, input: T) -> Result<Value> {
4834 self.client
4835 .query_workflow(&self.workflow_id, query_name, input)
4836 .await
4837 }
4838
4839 pub async fn query_avro_value<T: Serialize>(
4840 &self,
4841 query_name: &str,
4842 input: T,
4843 ) -> Result<AvroValue> {
4844 self.client
4845 .query_workflow_avro_value(&self.workflow_id, query_name, input)
4846 .await
4847 }
4848
4849 pub async fn update<T: Serialize>(
4850 &self,
4851 update_name: &str,
4852 input: T,
4853 request_id: Option<&str>,
4854 ) -> Result<Value> {
4855 self.client
4856 .update_workflow(&self.workflow_id, update_name, input, request_id)
4857 .await
4858 }
4859
4860 pub async fn update_avro_value<T: Serialize>(
4861 &self,
4862 update_name: &str,
4863 input: T,
4864 request_id: Option<&str>,
4865 ) -> Result<AvroValue> {
4866 self.client
4867 .update_workflow_avro_value(&self.workflow_id, update_name, input, request_id)
4868 .await
4869 }
4870
4871 pub async fn query_selected_run<T: Serialize>(
4873 &self,
4874 query_name: &str,
4875 input: T,
4876 ) -> Result<Value> {
4877 let run_id = self
4878 .run_id
4879 .as_deref()
4880 .ok_or_else(|| Error::Codec("run_id is required for selected-run query".to_string()))?;
4881 self.client
4882 .query_workflow_run(&self.workflow_id, run_id, query_name, input)
4883 .await
4884 }
4885
4886 pub async fn result(&self, options: WorkflowResultOptions) -> Result<Value> {
4888 self.result_target(options, None).await
4889 }
4890
4891 pub async fn result_avro_value(&self, options: WorkflowResultOptions) -> Result<AvroValue> {
4893 self.result_avro_value_target(options, None).await
4894 }
4895
4896 pub async fn result_typed<T: DeserializeOwned>(
4898 &self,
4899 options: WorkflowResultOptions,
4900 ) -> Result<T> {
4901 let result = self.result_avro_value(options).await?;
4902 decode_handler_result(result, HandlerKind::Workflow, &self.workflow_type)
4903 }
4904
4905 pub async fn result_selected_run(&self, options: WorkflowResultOptions) -> Result<Value> {
4907 let run_id = self.run_id.as_deref().ok_or_else(|| {
4908 Error::Codec("run_id is required for selected-run result".to_string())
4909 })?;
4910 self.result_target(options, Some(run_id)).await
4911 }
4912
4913 pub async fn result_selected_run_avro_value(
4915 &self,
4916 options: WorkflowResultOptions,
4917 ) -> Result<AvroValue> {
4918 let run_id = self.run_id.as_deref().ok_or_else(|| {
4919 Error::Codec("run_id is required for selected-run result".to_string())
4920 })?;
4921 self.result_avro_value_target(options, Some(run_id)).await
4922 }
4923
4924 pub async fn result_selected_run_typed<T: DeserializeOwned>(
4926 &self,
4927 options: WorkflowResultOptions,
4928 ) -> Result<T> {
4929 let result = self.result_selected_run_avro_value(options).await?;
4930 decode_handler_result(result, HandlerKind::Workflow, &self.workflow_type)
4931 }
4932
4933 async fn result_avro_value_target(
4934 &self,
4935 options: WorkflowResultOptions,
4936 selected_run_id: Option<&str>,
4937 ) -> Result<AvroValue> {
4938 let started = Instant::now();
4939
4940 loop {
4941 let description = match selected_run_id {
4942 Some(run_id) => {
4943 self.client
4944 .describe_workflow_run(&self.workflow_id, run_id)
4945 .await?
4946 }
4947 None => self.describe().await?,
4948 };
4949 if description.is_completed() {
4950 return description.output_avro_value.ok_or_else(|| {
4951 Error::Codec(
4952 "missing_payload_envelope: typed workflow result requires output_envelope"
4953 .to_string(),
4954 )
4955 });
4956 }
4957 if description.is_terminal() {
4958 let outcome =
4959 workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
4960 return Err(match outcome.kind {
4961 WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
4962 WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
4963 WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
4964 WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
4965 });
4966 }
4967 if started.elapsed() >= options.timeout {
4968 return Err(Error::Timeout);
4969 }
4970 tokio::time::sleep(options.poll_interval).await;
4971 }
4972 }
4973
4974 async fn result_target(
4975 &self,
4976 options: WorkflowResultOptions,
4977 selected_run_id: Option<&str>,
4978 ) -> Result<Value> {
4979 let started = Instant::now();
4980
4981 loop {
4982 let description = match selected_run_id {
4983 Some(run_id) => {
4984 self.client
4985 .describe_workflow_run(&self.workflow_id, run_id)
4986 .await?
4987 }
4988 None => self.describe().await?,
4989 };
4990 if description.is_completed() {
4991 return Ok(description.output.unwrap_or(Value::Null));
4992 }
4993
4994 if description.is_terminal() {
4995 let outcome =
4996 workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
4997 return Err(match outcome.kind {
4998 WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
4999 WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
5000 WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
5001 WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
5002 });
5003 }
5004
5005 if started.elapsed() >= options.timeout {
5006 return Err(Error::WorkflowTimedOut(WorkflowTerminalOutcome {
5007 kind: WorkflowTerminalKind::TimedOut,
5008 workflow_id: description
5009 .workflow_id
5010 .clone()
5011 .unwrap_or_else(|| self.workflow_id.clone()),
5012 run_id: description
5013 .run_id
5014 .clone()
5015 .or_else(|| selected_run_id.map(str::to_string)),
5016 reason: "result_wait_timeout".to_string(),
5017 failure_category: Some("client_timeout".to_string()),
5018 failure_id: None,
5019 exception_type: None,
5020 exception_class: None,
5021 non_retryable: None,
5022 message: Some(format!(
5023 "workflow result was not terminal within {:?}",
5024 options.timeout
5025 )),
5026 exception: None,
5027 raw: description.raw_value(),
5028 }));
5029 }
5030
5031 tokio::time::sleep(options.poll_interval).await;
5032 }
5033 }
5034}
5035
5036#[derive(Clone, Copy, Debug)]
5037pub struct WorkflowResultOptions {
5038 pub poll_interval: Duration,
5039 pub timeout: Duration,
5040}
5041
5042impl Default for WorkflowResultOptions {
5043 fn default() -> Self {
5044 Self {
5045 poll_interval: Duration::from_millis(500),
5046 timeout: Duration::from_secs(30),
5047 }
5048 }
5049}
5050
5051#[derive(Clone, Debug, Deserialize)]
5052pub struct WorkflowDescription {
5053 pub workflow_id: Option<String>,
5054 pub run_id: Option<String>,
5055 pub workflow_type: Option<String>,
5056 pub status: Option<String>,
5057 #[serde(default)]
5058 pub closed_reason: Option<String>,
5059 #[serde(default)]
5060 pub error: Option<String>,
5061 #[serde(default)]
5062 pub failure: Option<Value>,
5063 #[serde(default)]
5064 pub exception: Option<Value>,
5065 #[serde(default)]
5066 pub failures: Vec<Value>,
5067 #[serde(default)]
5068 pub output: Option<Value>,
5069 #[serde(default)]
5070 pub output_envelope: Option<Value>,
5071 #[serde(skip)]
5072 pub output_avro_value: Option<AvroValue>,
5073 #[serde(flatten)]
5074 pub raw: HashMap<String, Value>,
5075}
5076
5077#[derive(Clone, Debug, Deserialize)]
5079pub struct WorkflowStreamDescription {
5080 pub stream_name: String,
5081 pub status: String,
5082 pub last_offset: i64,
5083 pub total_items: u64,
5084 pub pending_items: u64,
5085 #[serde(default)]
5086 pub opened_at: Option<String>,
5087 #[serde(default)]
5088 pub last_appended_at: Option<String>,
5089 #[serde(default)]
5090 pub closed_at: Option<String>,
5091 #[serde(default)]
5092 pub error_reason: Option<String>,
5093 #[serde(default)]
5094 pub retention_seconds: Option<u64>,
5095 #[serde(flatten)]
5096 pub raw: HashMap<String, Value>,
5097}
5098
5099impl WorkflowStreamDescription {
5100 pub fn is_terminal(&self) -> bool {
5101 matches!(self.status.as_str(), "closed" | "errored")
5102 }
5103}
5104
5105#[derive(Clone, Debug, Default)]
5107pub struct WorkflowStreamAppendItem {
5108 pub payload_envelope: Option<Value>,
5109 pub payload_reference: Option<String>,
5110 pub item_type: Option<String>,
5111 pub content_type: Option<String>,
5112 pub idempotency_key: Option<String>,
5113}
5114
5115impl WorkflowStreamAppendItem {
5116 pub fn new<T: Serialize>(payload: T) -> Result<Self> {
5118 let value = AvroValue::from_serialize(&payload)?;
5119 Ok(Self {
5120 payload_envelope: Some(encode_typed_envelope(&value, DEFAULT_CODEC)?),
5121 ..Self::default()
5122 })
5123 }
5124
5125 pub fn from_reference(reference: impl Into<String>) -> Self {
5127 Self {
5128 payload_reference: Some(reference.into()),
5129 ..Self::default()
5130 }
5131 }
5132
5133 pub fn item_type(mut self, item_type: impl Into<String>) -> Self {
5134 self.item_type = Some(item_type.into());
5135 self
5136 }
5137
5138 pub fn content_type(mut self, content_type: impl Into<String>) -> Self {
5139 self.content_type = Some(content_type.into());
5140 self
5141 }
5142
5143 pub fn idempotency_key(mut self, idempotency_key: impl Into<String>) -> Self {
5144 self.idempotency_key = Some(idempotency_key.into());
5145 self
5146 }
5147
5148 fn wire_value(&self, derived_idempotency_key: Option<String>) -> Value {
5149 let mut item = serde_json::Map::new();
5150 if let Some(payload) = &self.payload_envelope {
5151 item.insert("payload".to_string(), payload.clone());
5152 item.insert("payload_codec".to_string(), json!(DEFAULT_CODEC));
5153 }
5154 if let Some(reference) = &self.payload_reference {
5155 item.insert("payload_reference".to_string(), json!(reference));
5156 }
5157 if let Some(item_type) = &self.item_type {
5158 item.insert("item_type".to_string(), json!(item_type));
5159 }
5160 if let Some(content_type) = &self.content_type {
5161 item.insert("content_type".to_string(), json!(content_type));
5162 }
5163 if let Some(key) = derived_idempotency_key
5164 .as_ref()
5165 .or(self.idempotency_key.as_ref())
5166 {
5167 item.insert("idempotency_key".to_string(), json!(key));
5168 }
5169 Value::Object(item)
5170 }
5171}
5172
5173#[derive(Clone, Debug)]
5175pub struct WorkflowStreamItem {
5176 pub offset: u64,
5177 pub payload: Option<Value>,
5178 pub payload_envelope: Option<Value>,
5179 pub payload_reference: Option<String>,
5180 pub payload_codec: Option<String>,
5181 pub idempotency_key: Option<String>,
5182 pub item_type: Option<String>,
5183 pub content_type: Option<String>,
5184 pub origin: Option<String>,
5185 pub origin_reference: Option<String>,
5186 pub emitted_at: Option<String>,
5187 pub raw: Value,
5188}
5189
5190#[derive(Clone, Debug)]
5192pub struct WorkflowStreamPage {
5193 pub stream: WorkflowStreamDescription,
5194 pub items: Vec<WorkflowStreamItem>,
5195 pub next_offset: u64,
5196 pub terminal: bool,
5197}
5198
5199#[derive(Clone, Debug)]
5201pub struct WorkflowStreamAppendResult {
5202 pub stream: WorkflowStreamDescription,
5203 pub accepted_offsets: Vec<u64>,
5204 pub accepted: u64,
5205 pub deduped: u64,
5206}
5207
5208#[derive(Deserialize)]
5209struct WorkflowStreamListResponse {
5210 #[serde(default)]
5211 streams: Vec<WorkflowStreamDescription>,
5212}
5213
5214#[derive(Deserialize)]
5215struct WorkflowStreamDescriptionResponse {
5216 stream: WorkflowStreamDescription,
5217}
5218
5219#[derive(Deserialize)]
5220struct WorkflowStreamPageResponse {
5221 stream: WorkflowStreamDescription,
5222 #[serde(default)]
5223 items: Vec<Value>,
5224 next_offset: u64,
5225 terminal: bool,
5226}
5227
5228#[derive(Deserialize)]
5229struct WorkflowStreamAppendResponse {
5230 stream: WorkflowStreamDescription,
5231 #[serde(default)]
5232 accepted_offsets: Vec<u64>,
5233 accepted: u64,
5234 deduped: u64,
5235}
5236
5237impl WorkflowDescription {
5238 pub fn is_completed(&self) -> bool {
5239 matches!(self.status.as_deref(), Some("completed" | "Completed"))
5240 }
5241
5242 pub fn is_terminal(&self) -> bool {
5243 matches!(
5244 self.status.as_deref(),
5245 Some(
5246 "completed"
5247 | "Completed"
5248 | "failed"
5249 | "Failed"
5250 | "cancelled"
5251 | "Cancelled"
5252 | "terminated"
5253 | "Terminated"
5254 | "timed_out"
5255 | "TimedOut",
5256 )
5257 )
5258 }
5259
5260 fn decode_payloads(&mut self) -> Result<()> {
5261 if let Some(envelope) = &self.output_envelope {
5262 let value = decode_wire_avro_value(envelope, DEFAULT_CODEC)?;
5263 self.output = Some(value.clone().into_json()?);
5264 self.output_avro_value = Some(value);
5265 }
5266
5267 Ok(())
5268 }
5269
5270 fn raw_value(&self) -> Value {
5271 let mut data = self.raw.clone();
5272 data.insert(
5273 "workflow_id".to_string(),
5274 self.workflow_id
5275 .clone()
5276 .map(Value::String)
5277 .unwrap_or(Value::Null),
5278 );
5279 data.insert(
5280 "run_id".to_string(),
5281 self.run_id
5282 .clone()
5283 .map(Value::String)
5284 .unwrap_or(Value::Null),
5285 );
5286 data.insert(
5287 "workflow_type".to_string(),
5288 self.workflow_type
5289 .clone()
5290 .map(Value::String)
5291 .unwrap_or(Value::Null),
5292 );
5293 data.insert(
5294 "status".to_string(),
5295 self.status
5296 .clone()
5297 .map(Value::String)
5298 .unwrap_or(Value::Null),
5299 );
5300 data.insert(
5301 "closed_reason".to_string(),
5302 self.closed_reason
5303 .clone()
5304 .map(Value::String)
5305 .unwrap_or(Value::Null),
5306 );
5307 if let Some(failure) = &self.failure {
5308 data.insert("failure".to_string(), failure.clone());
5309 }
5310 if let Some(exception) = &self.exception {
5311 data.insert("exception".to_string(), exception.clone());
5312 }
5313 Value::Object(data.into_iter().collect())
5314 }
5315}
5316
5317fn workflow_terminal_outcome(
5318 description: &WorkflowDescription,
5319 workflow_id: &str,
5320 run_id: Option<&str>,
5321) -> WorkflowTerminalOutcome {
5322 let terminal_kind = description
5323 .closed_reason
5324 .as_deref()
5325 .or(description.status.as_deref())
5326 .unwrap_or("failed")
5327 .to_ascii_lowercase();
5328 let kind = match terminal_kind.as_str() {
5329 "cancelled" | "canceled" => WorkflowTerminalKind::Cancelled,
5330 "terminated" => WorkflowTerminalKind::Terminated,
5331 "timed_out" | "timedout" => WorkflowTerminalKind::TimedOut,
5332 _ => WorkflowTerminalKind::Failed,
5333 };
5334 let default_reason = match kind {
5335 WorkflowTerminalKind::Failed => "workflow_failed",
5336 WorkflowTerminalKind::Cancelled => "cancelled",
5337 WorkflowTerminalKind::Terminated => "terminated",
5338 WorkflowTerminalKind::TimedOut => "timed_out",
5339 };
5340 let failure = description
5341 .failure
5342 .as_ref()
5343 .filter(|value| value.is_object());
5344 let nested_failure = failure
5345 .and_then(|value| value.get("failures"))
5346 .and_then(Value::as_array)
5347 .and_then(|failures| failures.last())
5348 .or_else(|| description.failures.last());
5349 let exception = description
5350 .exception
5351 .clone()
5352 .or_else(|| failure.and_then(|value| value.get("exception")).cloned())
5353 .or_else(|| {
5354 nested_failure
5355 .and_then(|value| value.get("exception_payload"))
5356 .cloned()
5357 });
5358 let string_field = |name: &str| {
5359 failure
5360 .and_then(|value| value.get(name))
5361 .and_then(Value::as_str)
5362 .or_else(|| {
5363 nested_failure
5364 .and_then(|value| value.get(name))
5365 .and_then(Value::as_str)
5366 })
5367 .map(str::to_string)
5368 };
5369 let exception_field = |name: &str| {
5370 exception
5371 .as_ref()
5372 .and_then(|value| value.get(name))
5373 .and_then(Value::as_str)
5374 .map(str::to_string)
5375 };
5376 let message = description
5377 .error
5378 .clone()
5379 .or_else(|| string_field("message"))
5380 .or_else(|| exception_field("message"));
5381 let reason = description
5382 .raw
5383 .get("reason")
5384 .and_then(Value::as_str)
5385 .map(str::to_string)
5386 .or_else(|| {
5387 failure
5388 .and_then(|value| value.get("reason"))
5389 .and_then(Value::as_str)
5390 .map(str::to_string)
5391 })
5392 .or_else(|| description.closed_reason.clone())
5393 .unwrap_or_else(|| default_reason.to_string());
5394 let failure_id = string_field("failure_id").or_else(|| {
5395 nested_failure
5396 .and_then(|value| value.get("id"))
5397 .and_then(Value::as_str)
5398 .map(str::to_string)
5399 });
5400
5401 WorkflowTerminalOutcome {
5402 kind,
5403 workflow_id: description
5404 .workflow_id
5405 .clone()
5406 .unwrap_or_else(|| workflow_id.to_string()),
5407 run_id: description
5408 .run_id
5409 .clone()
5410 .or_else(|| run_id.map(str::to_string)),
5411 reason,
5412 failure_category: string_field("failure_category")
5413 .or_else(|| Some(default_reason.to_string())),
5414 failure_id,
5415 exception_type: string_field("exception_type").or_else(|| exception_field("type")),
5416 exception_class: string_field("exception_class").or_else(|| exception_field("class")),
5417 non_retryable: failure
5418 .and_then(|value| value.get("non_retryable"))
5419 .and_then(Value::as_bool)
5420 .or_else(|| {
5421 nested_failure
5422 .and_then(|value| value.get("non_retryable"))
5423 .and_then(Value::as_bool)
5424 }),
5425 message,
5426 exception,
5427 raw: description.raw_value(),
5428 }
5429}
5430
5431#[derive(Clone, Debug, Deserialize)]
5432pub struct RegisterWorkerResponse {
5433 pub worker_id: String,
5434 pub registered: bool,
5435 #[serde(default)]
5436 pub heartbeat_interval_seconds: Option<u64>,
5437 #[serde(default)]
5438 pub protocol_version: Option<String>,
5439 #[serde(default)]
5440 pub server_capabilities: Option<Value>,
5441}
5442
5443#[derive(Clone, Debug, Deserialize, PartialEq, Eq)]
5445pub struct WorkerDeregistrationEnvelope {
5446 pub worker_id: String,
5447 pub outcome: String,
5448 pub recovered_workflow_task_count: u64,
5449}
5450
5451#[derive(Clone, Debug, Deserialize)]
5452pub struct PollWorkflowTaskResponse {
5453 #[serde(default)]
5454 pub task: Option<WorkflowTask>,
5455 #[serde(default)]
5456 pub poll_status: Option<String>,
5457 #[serde(default)]
5458 pub reason: Option<String>,
5459 #[serde(default)]
5460 pub protocol_version: Option<String>,
5461 #[serde(default)]
5462 pub server_capabilities: Option<Value>,
5463}
5464
5465impl PollWorkflowTaskResponse {
5466 pub fn outcome(&self) -> WorkerPollOutcome {
5468 worker_poll_outcome(
5469 self.task.is_some(),
5470 self.poll_status.as_deref(),
5471 self.reason.as_deref(),
5472 )
5473 }
5474}
5475
5476fn runtime_supports_workflow_memo_updates(capabilities: Option<&Value>) -> bool {
5477 let Some(capabilities) = capabilities.and_then(Value::as_object) else {
5478 return false;
5479 };
5480 let supported = capabilities
5481 .get("workflow_memo_updates")
5482 .and_then(Value::as_object)
5483 .and_then(|memo| memo.get("supported"))
5484 .and_then(Value::as_bool)
5485 == Some(true);
5486 let command_advertised = capabilities
5487 .get("supported_workflow_task_commands")
5488 .and_then(Value::as_array)
5489 .is_some_and(|commands| {
5490 commands
5491 .iter()
5492 .any(|command| command.as_str() == Some("upsert_memo"))
5493 });
5494 supported && command_advertised
5495}
5496
5497fn commands_use_workflow_memo_updates(commands: &[Value]) -> bool {
5498 commands
5499 .iter()
5500 .any(|command| command.get("type").and_then(Value::as_str) == Some("upsert_memo"))
5501}
5502
5503#[derive(Clone, Debug, Deserialize)]
5504pub struct PollActivityTaskResponse {
5505 #[serde(default)]
5506 pub task: Option<ActivityTask>,
5507 #[serde(default)]
5508 pub poll_status: Option<String>,
5509 #[serde(default)]
5510 pub reason: Option<String>,
5511}
5512
5513impl PollActivityTaskResponse {
5514 pub fn outcome(&self) -> WorkerPollOutcome {
5516 worker_poll_outcome(
5517 self.task.is_some(),
5518 self.poll_status.as_deref(),
5519 self.reason.as_deref(),
5520 )
5521 }
5522}
5523
5524#[derive(Clone, Debug, Deserialize)]
5525pub struct PollQueryTaskResponse {
5526 #[serde(default)]
5527 pub task: Option<QueryTask>,
5528 #[serde(default)]
5529 pub poll_status: Option<String>,
5530 #[serde(default)]
5531 pub reason: Option<String>,
5532}
5533
5534impl PollQueryTaskResponse {
5535 pub fn outcome(&self) -> WorkerPollOutcome {
5537 worker_poll_outcome(
5538 self.task.is_some(),
5539 self.poll_status.as_deref(),
5540 self.reason.as_deref(),
5541 )
5542 }
5543}
5544
5545#[derive(Clone, Debug, PartialEq, Eq)]
5547pub enum WorkerPollOutcome {
5548 Task,
5550 Idle {
5552 poll_status: Option<String>,
5553 reason: Option<String>,
5554 },
5555 Stop {
5557 poll_status: Option<String>,
5558 reason: Option<String>,
5559 },
5560}
5561
5562impl WorkerPollOutcome {
5563 pub fn should_stop(&self) -> bool {
5564 matches!(self, Self::Stop { .. })
5565 }
5566}
5567
5568fn worker_poll_outcome(
5569 has_task: bool,
5570 poll_status: Option<&str>,
5571 reason: Option<&str>,
5572) -> WorkerPollOutcome {
5573 if worker_poll_is_stop(poll_status, reason) {
5574 return WorkerPollOutcome::Stop {
5575 poll_status: poll_status.map(str::to_string),
5576 reason: reason.map(str::to_string),
5577 };
5578 }
5579
5580 if has_task {
5581 WorkerPollOutcome::Task
5582 } else {
5583 WorkerPollOutcome::Idle {
5584 poll_status: poll_status.map(str::to_string),
5585 reason: reason.map(str::to_string),
5586 }
5587 }
5588}
5589
5590#[derive(Clone, Debug, Deserialize)]
5592pub struct QueryTask {
5593 pub query_task_id: String,
5594 #[serde(default = "default_workflow_task_attempt")]
5595 pub query_task_attempt: u64,
5596 #[serde(default)]
5597 pub lease_owner: Option<String>,
5598 #[serde(default)]
5599 pub workflow_id: Option<String>,
5600 #[serde(default)]
5601 pub run_id: Option<String>,
5602 pub workflow_type: String,
5603 pub query_name: String,
5604 #[serde(
5605 default = "missing_task_payload_codec",
5606 deserialize_with = "deserialize_task_payload_codec"
5607 )]
5608 pub payload_codec: String,
5609 #[serde(default)]
5610 pub workflow_arguments: Option<Value>,
5611 #[serde(default)]
5612 pub query_arguments: Option<Value>,
5613 #[serde(default)]
5614 pub history_events: Vec<HistoryEvent>,
5615 #[serde(default)]
5616 pub history_export: Option<Value>,
5617 #[serde(default)]
5618 pub run_status: Option<String>,
5619}
5620
5621#[derive(Clone, Debug, Deserialize)]
5622pub struct WorkflowTask {
5623 pub task_id: String,
5624 #[serde(default)]
5625 pub workflow_command_id: Option<String>,
5626 #[serde(default)]
5627 pub workflow_id: Option<String>,
5628 #[serde(default)]
5629 pub run_id: Option<String>,
5630 pub workflow_type: String,
5631 #[serde(default)]
5632 pub cancel_requested: bool,
5633 #[serde(
5634 default = "missing_task_payload_codec",
5635 deserialize_with = "deserialize_task_payload_codec"
5636 )]
5637 pub payload_codec: String,
5638 #[serde(default)]
5639 pub arguments: Option<Value>,
5640 #[serde(default)]
5641 pub history_events: Vec<HistoryEvent>,
5642 #[serde(default)]
5643 pub total_history_events: Option<u64>,
5644 #[serde(default)]
5645 pub history_size_bytes: Option<u64>,
5646 #[serde(default)]
5647 pub continue_as_new_recommended: Option<bool>,
5648 #[serde(default)]
5649 pub history_budget_pressure: Option<String>,
5650 #[serde(default)]
5651 pub next_history_page_token: Option<String>,
5652 #[serde(default = "default_workflow_task_attempt")]
5653 pub workflow_task_attempt: u64,
5654 #[serde(default)]
5655 pub workflow_signal_id: Option<String>,
5656 #[serde(default)]
5657 pub signal_name: Option<String>,
5658 #[serde(default)]
5659 pub signal_arguments: Option<Value>,
5660 #[serde(default)]
5661 pub workflow_update_id: Option<String>,
5662 #[serde(default)]
5663 pub update_name: Option<String>,
5664 #[serde(default)]
5665 pub lease_owner: Option<String>,
5666}
5667
5668impl WorkflowTask {
5669 fn append_history_page(&mut self, page: WorkflowTaskHistoryPage) {
5670 self.history_events.extend(page.history_events);
5671
5672 if page.total_history_events.is_some() {
5673 self.total_history_events = page.total_history_events;
5674 }
5675
5676 self.next_history_page_token = page
5677 .next_history_page_token
5678 .filter(|token| !token.is_empty());
5679 }
5680}
5681
5682#[derive(Clone, Debug, Deserialize)]
5683struct WorkflowTaskHistoryPage {
5684 #[serde(default)]
5685 history_events: Vec<HistoryEvent>,
5686 #[serde(default)]
5687 total_history_events: Option<u64>,
5688 #[serde(default)]
5689 next_history_page_token: Option<String>,
5690}
5691
5692#[derive(Clone, Debug, Deserialize)]
5693pub struct ActivityTask {
5694 pub task_id: String,
5695 #[serde(default)]
5696 pub activity_attempt_id: Option<String>,
5697 #[serde(default)]
5698 pub attempt_id: Option<String>,
5699 pub activity_type: String,
5700 #[serde(
5701 default = "missing_task_payload_codec",
5702 deserialize_with = "deserialize_task_payload_codec"
5703 )]
5704 pub payload_codec: String,
5705 #[serde(default)]
5706 pub arguments: Option<Value>,
5707 #[serde(default = "default_attempt_number")]
5708 pub attempt_number: u64,
5709 #[serde(default)]
5710 pub lease_owner: Option<String>,
5711}
5712
5713#[derive(Clone, Debug, Deserialize)]
5714pub struct HistoryEvent {
5715 #[serde(alias = "type")]
5716 pub event_type: String,
5717 #[serde(default)]
5718 pub payload: Value,
5719 #[serde(flatten)]
5720 pub raw: HashMap<String, Value>,
5721}
5722
5723#[derive(Clone, Debug, PartialEq)]
5725pub struct QuerySignal {
5726 pub id: Option<String>,
5727 pub name: String,
5728 pub arguments: Vec<Value>,
5729 avro_arguments: Vec<AvroValue>,
5730 pub workflow_sequence: Option<u64>,
5731}
5732
5733impl QuerySignal {
5734 pub fn arguments_avro_value(&self) -> &[AvroValue] {
5736 &self.avro_arguments
5737 }
5738}
5739
5740#[derive(Clone, Debug)]
5746pub struct QueryContext {
5747 pub workflow_id: Option<String>,
5748 pub run_id: Option<String>,
5749 pub workflow_type: String,
5750 pub run_status: Option<String>,
5751 workflow_input: Value,
5752 workflow_input_avro_value: AvroValue,
5753 history_events: Arc<Vec<HistoryEvent>>,
5754 signal_events: Arc<Vec<QuerySignal>>,
5755}
5756
5757impl QueryContext {
5758 pub fn workflow_input(&self) -> &Value {
5760 &self.workflow_input
5761 }
5762
5763 pub fn workflow_input_avro_value(&self) -> &AvroValue {
5765 &self.workflow_input_avro_value
5766 }
5767
5768 pub fn history_events(&self) -> &[HistoryEvent] {
5770 self.history_events.as_slice()
5771 }
5772
5773 pub fn signal_events(&self) -> &[QuerySignal] {
5775 self.signal_events.as_slice()
5776 }
5777
5778 pub fn signals(&self, signal_name: &str) -> Vec<Vec<Value>> {
5780 self.signal_events
5781 .iter()
5782 .filter(|signal| signal.name == signal_name)
5783 .map(|signal| signal.arguments.clone())
5784 .collect()
5785 }
5786
5787 pub fn signals_avro_value(&self, signal_name: &str) -> Vec<Vec<AvroValue>> {
5789 self.signal_events
5790 .iter()
5791 .filter(|signal| signal.name == signal_name)
5792 .map(|signal| signal.avro_arguments.clone())
5793 .collect()
5794 }
5795}
5796
5797#[derive(Clone, Debug, Deserialize)]
5798pub struct ActivityHeartbeatResponse {
5799 #[serde(default)]
5800 pub cancel_requested: bool,
5801 #[serde(default)]
5802 pub heartbeat_recorded: bool,
5803 #[serde(default)]
5804 pub can_continue: Option<bool>,
5805 #[serde(default)]
5806 pub reason: Option<String>,
5807 #[serde(default)]
5808 pub run_closed_reason: Option<String>,
5809 #[serde(default)]
5810 pub run_closed_at: Option<String>,
5811 #[serde(default)]
5812 pub lease_expires_at: Option<String>,
5813 #[serde(default)]
5814 pub last_heartbeat_at: Option<String>,
5815}
5816
5817impl ActivityHeartbeatResponse {
5818 pub fn should_stop(&self) -> bool {
5820 self.cancel_requested || self.can_continue == Some(false)
5821 }
5822}
5823
5824fn missing_task_payload_codec() -> String {
5825 MISSING_TASK_PAYLOAD_CODEC.to_string()
5826}
5827
5828fn deserialize_task_payload_codec<'de, D>(deserializer: D) -> std::result::Result<String, D::Error>
5829where
5830 D: Deserializer<'de>,
5831{
5832 Ok(match Value::deserialize(deserializer)? {
5833 Value::String(codec) => codec,
5834 Value::Null => NULL_TASK_PAYLOAD_CODEC.to_string(),
5835 _ => NON_STRING_TASK_PAYLOAD_CODEC.to_string(),
5836 })
5837}
5838
5839fn default_workflow_task_attempt() -> u64 {
5840 1
5841}
5842
5843fn default_attempt_number() -> u64 {
5844 1
5845}
5846
5847type WorkflowFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
5848type WorkflowHandler = Arc<dyn Fn(WorkflowContext, AvroValue) -> WorkflowFuture + Send + Sync>;
5849type ErasedWorkflowState = Arc<dyn Any + Send + Sync>;
5850type WorkflowStateSnapshot = Arc<dyn Fn() -> Result<ErasedWorkflowState> + Send + Sync>;
5851type ReplayedWorkflowHandler =
5852 Arc<dyn Fn(WorkflowContext, AvroValue) -> ReplayedWorkflowInvocation + Send + Sync>;
5853type ActivityFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
5854type ActivityHandler = Arc<dyn Fn(ActivityContext, AvroValue) -> ActivityFuture + Send + Sync>;
5855type QueryFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
5856type QueryHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
5857type UpdateHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
5858type ReplayedQueryHandler = Arc<
5859 dyn Fn(QueryContext, ErasedWorkflowState, AvroValue) -> std::result::Result<QueryFuture, String>
5860 + Send
5861 + Sync,
5862>;
5863type WorkerHeartbeatObserver = Arc<dyn Fn(&WorkerHeartbeatObservation) + Send + Sync>;
5864
5865struct ReplayedWorkflowInvocation {
5866 future: WorkflowFuture,
5867 snapshot: WorkflowStateSnapshot,
5868}
5869
5870#[derive(Clone)]
5871struct RegisteredWorkflow {
5872 execute: WorkflowHandler,
5873 replay: Option<ReplayedWorkflowHandler>,
5874 state_type: Option<TypeId>,
5875}
5876
5877#[derive(Debug)]
5878struct WorkflowTaskDecision {
5879 commands: Vec<Value>,
5880 message_stream_cursors: Vec<Value>,
5881 message_stream_waits: Vec<Value>,
5882}
5883
5884impl WorkflowTaskDecision {
5885 fn without_message_streams(commands: Vec<Value>) -> Self {
5886 Self {
5887 commands,
5888 message_stream_cursors: Vec::new(),
5889 message_stream_waits: Vec::new(),
5890 }
5891 }
5892}
5893
5894#[derive(Clone)]
5895enum RegisteredQuery {
5896 Snapshot(QueryHandler),
5897 Replayed {
5898 state_type: TypeId,
5899 handler: ReplayedQueryHandler,
5900 },
5901}
5902
5903#[derive(Clone, Debug)]
5904pub struct WorkerHeartbeatObservation {
5905 pub worker_id: String,
5906 pub task_queue: String,
5907 pub acknowledged_at_unix_millis: u64,
5908 pub acknowledgement: Value,
5909}
5910
5911#[derive(Clone, Copy, Debug)]
5923pub struct WorkerRetryPolicy {
5924 pub max_retries: usize,
5926 pub initial_backoff: Duration,
5928 pub max_backoff: Duration,
5930}
5931
5932impl Default for WorkerRetryPolicy {
5933 fn default() -> Self {
5934 Self {
5935 max_retries: 5,
5936 initial_backoff: Duration::from_millis(100),
5937 max_backoff: Duration::from_secs(5),
5938 }
5939 }
5940}
5941
5942#[derive(Clone, Debug)]
5943struct WorkerStorageAdmission {
5944 policy: WorkerRetryPolicy,
5945 stop: Arc<AtomicBool>,
5946}
5947
5948struct StopWorkerOnDrop(Arc<AtomicBool>);
5949
5950impl Drop for StopWorkerOnDrop {
5951 fn drop(&mut self) {
5952 self.0.store(true, Ordering::SeqCst);
5953 }
5954}
5955
5956async fn wait_for_worker_stop(stop: &AtomicBool) {
5957 while !stop.load(Ordering::SeqCst) {
5958 tokio::time::sleep(Duration::from_millis(100)).await;
5959 }
5960}
5961
5962#[derive(Clone, Copy, Debug, PartialEq, Eq)]
5963enum ManagedPollOutcome {
5964 Idle,
5965 Handled,
5966 Stop,
5967}
5968
5969#[derive(Clone)]
5970pub struct Worker {
5971 client: Client,
5972 worker_id: String,
5973 task_queue: String,
5974 workflows: HashMap<String, RegisteredWorkflow>,
5975 activities: HashMap<String, ActivityHandler>,
5976 queries: HashMap<String, HashMap<String, RegisteredQuery>>,
5977 updates: HashMap<String, HashMap<String, UpdateHandler>>,
5978 max_concurrent_workflow_tasks: usize,
5979 max_concurrent_activity_tasks: usize,
5980 poll_timeout: Duration,
5981 heartbeat_interval: Duration,
5982 retry_policy: WorkerRetryPolicy,
5983 heartbeat_observer: Option<WorkerHeartbeatObserver>,
5984}
5985
5986impl Worker {
5987 pub fn new(client: Client, task_queue: impl Into<String>) -> Self {
5988 Self {
5989 client,
5990 worker_id: default_worker_id(),
5991 task_queue: task_queue.into(),
5992 workflows: HashMap::new(),
5993 activities: HashMap::new(),
5994 queries: HashMap::new(),
5995 updates: HashMap::new(),
5996 max_concurrent_workflow_tasks: 10,
5997 max_concurrent_activity_tasks: 10,
5998 poll_timeout: Duration::from_secs(30),
5999 heartbeat_interval: Duration::from_secs(60),
6000 retry_policy: WorkerRetryPolicy::default(),
6001 heartbeat_observer: None,
6002 }
6003 }
6004
6005 pub fn worker_id(mut self, worker_id: impl Into<String>) -> Self {
6006 self.worker_id = worker_id.into();
6007 self
6008 }
6009
6010 pub fn poll_timeout(mut self, timeout: Duration) -> Self {
6011 self.poll_timeout = timeout;
6012 self
6013 }
6014
6015 pub fn heartbeat_interval(mut self, interval: Duration) -> Self {
6016 self.heartbeat_interval = interval;
6017 self
6018 }
6019
6020 pub fn retry_policy(mut self, policy: WorkerRetryPolicy) -> Self {
6022 self.retry_policy = policy;
6023 self
6024 }
6025
6026 pub fn on_worker_heartbeat<F>(mut self, observer: F) -> Self
6027 where
6028 F: Fn(&WorkerHeartbeatObservation) + Send + Sync + 'static,
6029 {
6030 self.heartbeat_observer = Some(Arc::new(observer));
6031 self
6032 }
6033
6034 pub fn max_concurrent_workflow_tasks(mut self, count: usize) -> Self {
6035 self.max_concurrent_workflow_tasks = count.max(1);
6036 self
6037 }
6038
6039 pub fn max_concurrent_activity_tasks(mut self, count: usize) -> Self {
6040 self.max_concurrent_activity_tasks = count.max(1);
6041 self
6042 }
6043
6044 pub fn register_workflow<F, Fut>(&mut self, workflow_type: impl Into<String>, handler: F)
6051 where
6052 F: Fn(WorkflowContext, Value) -> Fut + Send + Sync + 'static,
6053 Fut: Future<Output = Result<Value>> + Send + 'static,
6054 {
6055 let handler = Arc::new(handler);
6056 self.workflows.insert(
6057 workflow_type.into(),
6058 RegisteredWorkflow {
6059 execute: Arc::new(move |ctx, input| {
6060 let handler = Arc::clone(&handler);
6061 Box::pin(async move {
6062 let result = handler(ctx, input.into_json()?).await?;
6063 AvroValue::from_serialize(&result)
6064 })
6065 }),
6066 replay: None,
6067 state_type: None,
6068 },
6069 );
6070 }
6071
6072 pub fn register_typed_workflow<I, O, F, Fut>(
6083 &mut self,
6084 workflow_type: impl Into<String>,
6085 handler: F,
6086 ) where
6087 I: DeserializeOwned + Send + 'static,
6088 O: Serialize + Send + 'static,
6089 F: Fn(WorkflowContext, I) -> Fut + Send + Sync + 'static,
6090 Fut: Future<Output = Result<O>> + Send + 'static,
6091 {
6092 let workflow_type = workflow_type.into();
6093 let handler_name = workflow_type.clone();
6094 let handler = Arc::new(handler);
6095 self.workflows.insert(
6096 workflow_type,
6097 RegisteredWorkflow {
6098 execute: Arc::new(move |ctx, input| {
6099 let handler = Arc::clone(&handler);
6100 let handler_name = handler_name.clone();
6101 Box::pin(async move {
6102 let input =
6103 decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
6104 let result = handler(ctx, input).await?;
6105 encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
6106 })
6107 }),
6108 replay: None,
6109 state_type: None,
6110 },
6111 );
6112 }
6113
6114 pub fn register_workflow_avro_value<F, Fut>(
6116 &mut self,
6117 workflow_type: impl Into<String>,
6118 handler: F,
6119 ) where
6120 F: Fn(WorkflowContext, AvroValue) -> Fut + Send + Sync + 'static,
6121 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6122 {
6123 self.workflows.insert(
6124 workflow_type.into(),
6125 RegisteredWorkflow {
6126 execute: Arc::new(move |ctx, input| Box::pin(handler(ctx, input))),
6127 replay: None,
6128 state_type: None,
6129 },
6130 );
6131 }
6132
6133 pub fn register_replayed_workflow<S, Factory, F, Fut>(
6141 &mut self,
6142 workflow_type: impl Into<String>,
6143 state_factory: Factory,
6144 handler: F,
6145 ) where
6146 S: Clone + Send + Sync + 'static,
6147 Factory: Fn() -> S + Send + Sync + 'static,
6148 F: Fn(WorkflowContext, Value, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
6149 Fut: Future<Output = Result<Value>> + Send + 'static,
6150 {
6151 let state_factory = Arc::new(state_factory);
6152 let handler = Arc::new(handler);
6153
6154 let execute_factory = Arc::clone(&state_factory);
6155 let execute_handler = Arc::clone(&handler);
6156 let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6157 let state = WorkflowInstance::new(execute_factory());
6158 let handler = Arc::clone(&execute_handler);
6159 Box::pin(async move {
6160 let result = handler(ctx, input.into_json()?, state).await?;
6161 AvroValue::from_serialize(&result)
6162 }) as WorkflowFuture
6163 });
6164
6165 let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6166 let state = WorkflowInstance::new(state_factory());
6167 let snapshot_state = state.clone();
6168 let snapshot: WorkflowStateSnapshot =
6169 Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
6170 let replay_handler = Arc::clone(&handler);
6171 let future = async move {
6172 let result = replay_handler(ctx, input.into_json()?, state).await?;
6173 AvroValue::from_serialize(&result)
6174 };
6175 ReplayedWorkflowInvocation {
6176 future: Box::pin(future),
6177 snapshot,
6178 }
6179 });
6180
6181 self.workflows.insert(
6182 workflow_type.into(),
6183 RegisteredWorkflow {
6184 execute,
6185 replay: Some(replay),
6186 state_type: Some(TypeId::of::<S>()),
6187 },
6188 );
6189 }
6190
6191 pub fn register_typed_replayed_workflow<I, O, S, Factory, F, Fut>(
6197 &mut self,
6198 workflow_type: impl Into<String>,
6199 state_factory: Factory,
6200 handler: F,
6201 ) where
6202 I: DeserializeOwned + Send + 'static,
6203 O: Serialize + Send + 'static,
6204 S: Clone + Send + Sync + 'static,
6205 Factory: Fn() -> S + Send + Sync + 'static,
6206 F: Fn(WorkflowContext, I, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
6207 Fut: Future<Output = Result<O>> + Send + 'static,
6208 {
6209 let workflow_type = workflow_type.into();
6210 let state_factory = Arc::new(state_factory);
6211 let handler = Arc::new(handler);
6212
6213 let execute_name = workflow_type.clone();
6214 let execute_factory = Arc::clone(&state_factory);
6215 let execute_handler = Arc::clone(&handler);
6216 let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6217 let state = WorkflowInstance::new(execute_factory());
6218 let handler = Arc::clone(&execute_handler);
6219 let handler_name = execute_name.clone();
6220 Box::pin(async move {
6221 let input = decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
6222 let result = handler(ctx, input, state).await?;
6223 encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
6224 }) as WorkflowFuture
6225 });
6226
6227 let replay_name = workflow_type.clone();
6228 let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6229 let state = WorkflowInstance::new(state_factory());
6230 let snapshot_state = state.clone();
6231 let snapshot: WorkflowStateSnapshot =
6232 Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
6233 let handler = Arc::clone(&handler);
6234 let handler_name = replay_name.clone();
6235 let future = async move {
6236 let input = decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
6237 let result = handler(ctx, input, state).await?;
6238 encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
6239 };
6240 ReplayedWorkflowInvocation {
6241 future: Box::pin(future),
6242 snapshot,
6243 }
6244 });
6245
6246 self.workflows.insert(
6247 workflow_type,
6248 RegisteredWorkflow {
6249 execute,
6250 replay: Some(replay),
6251 state_type: Some(TypeId::of::<S>()),
6252 },
6253 );
6254 }
6255
6256 pub fn register_replayed_workflow_avro_value<S, Factory, F, Fut>(
6258 &mut self,
6259 workflow_type: impl Into<String>,
6260 state_factory: Factory,
6261 handler: F,
6262 ) where
6263 S: Clone + Send + Sync + 'static,
6264 Factory: Fn() -> S + Send + Sync + 'static,
6265 F: Fn(WorkflowContext, AvroValue, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
6266 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6267 {
6268 let state_factory = Arc::new(state_factory);
6269 let handler = Arc::new(handler);
6270
6271 let execute_factory = Arc::clone(&state_factory);
6272 let execute_handler = Arc::clone(&handler);
6273 let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6274 let state = WorkflowInstance::new(execute_factory());
6275 Box::pin(execute_handler(ctx, input, state)) as WorkflowFuture
6276 });
6277
6278 let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6279 let state = WorkflowInstance::new(state_factory());
6280 let snapshot_state = state.clone();
6281 let snapshot: WorkflowStateSnapshot =
6282 Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
6283 ReplayedWorkflowInvocation {
6284 future: Box::pin(handler(ctx, input, state)),
6285 snapshot,
6286 }
6287 });
6288
6289 self.workflows.insert(
6290 workflow_type.into(),
6291 RegisteredWorkflow {
6292 execute,
6293 replay: Some(replay),
6294 state_type: Some(TypeId::of::<S>()),
6295 },
6296 );
6297 }
6298
6299 pub fn register_activity<F, Fut>(&mut self, activity_type: impl Into<String>, handler: F)
6300 where
6301 F: Fn(ActivityContext, Value) -> Fut + Send + Sync + 'static,
6302 Fut: Future<Output = Result<Value>> + Send + 'static,
6303 {
6304 let handler = Arc::new(handler);
6305 self.activities.insert(
6306 activity_type.into(),
6307 Arc::new(move |ctx, args| {
6308 let handler = Arc::clone(&handler);
6309 Box::pin(async move {
6310 let result = handler(ctx, args.into_json()?).await?;
6311 AvroValue::from_serialize(&result)
6312 })
6313 }),
6314 );
6315 }
6316
6317 pub fn register_typed_activity<I, O, F, Fut>(
6323 &mut self,
6324 activity_type: impl Into<String>,
6325 handler: F,
6326 ) where
6327 I: DeserializeOwned + Send + 'static,
6328 O: Serialize + Send + 'static,
6329 F: Fn(ActivityContext, I) -> Fut + Send + Sync + 'static,
6330 Fut: Future<Output = Result<O>> + Send + 'static,
6331 {
6332 let activity_type = activity_type.into();
6333 let handler_name = activity_type.clone();
6334 let handler = Arc::new(handler);
6335 self.activities.insert(
6336 activity_type,
6337 Arc::new(move |ctx, input| {
6338 let handler = Arc::clone(&handler);
6339 let handler_name = handler_name.clone();
6340 Box::pin(async move {
6341 let input =
6342 decode_handler_input::<I>(input, HandlerKind::Activity, &handler_name)?;
6343 let result = handler(ctx, input).await?;
6344 encode_handler_result(&result, HandlerKind::Activity, &handler_name)
6345 })
6346 }),
6347 );
6348 }
6349
6350 pub fn register_activity_avro_value<F, Fut>(
6352 &mut self,
6353 activity_type: impl Into<String>,
6354 handler: F,
6355 ) where
6356 F: Fn(ActivityContext, AvroValue) -> Fut + Send + Sync + 'static,
6357 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6358 {
6359 self.activities.insert(
6360 activity_type.into(),
6361 Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
6362 );
6363 }
6364
6365 pub fn register_query<F, Fut>(
6371 &mut self,
6372 workflow_type: impl Into<String>,
6373 query_name: impl Into<String>,
6374 handler: F,
6375 ) where
6376 F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
6377 Fut: Future<Output = Result<Value>> + Send + 'static,
6378 {
6379 let handler = Arc::new(handler);
6380 self.queries
6381 .entry(workflow_type.into())
6382 .or_default()
6383 .insert(
6384 query_name.into(),
6385 RegisteredQuery::Snapshot(Arc::new(move |ctx, args| {
6386 let handler = Arc::clone(&handler);
6387 Box::pin(async move {
6388 let result = handler(ctx, args.into_json()?).await?;
6389 AvroValue::from_serialize(&result)
6390 })
6391 })),
6392 );
6393 }
6394
6395 pub fn register_query_avro_value<F, Fut>(
6397 &mut self,
6398 workflow_type: impl Into<String>,
6399 query_name: impl Into<String>,
6400 handler: F,
6401 ) where
6402 F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
6403 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6404 {
6405 self.queries
6406 .entry(workflow_type.into())
6407 .or_default()
6408 .insert(
6409 query_name.into(),
6410 RegisteredQuery::Snapshot(Arc::new(move |ctx, args| Box::pin(handler(ctx, args)))),
6411 );
6412 }
6413
6414 pub fn register_replayed_query<S, F, Fut>(
6421 &mut self,
6422 workflow_type: impl Into<String>,
6423 query_name: impl Into<String>,
6424 handler: F,
6425 ) where
6426 S: Clone + Send + Sync + 'static,
6427 F: Fn(QueryContext, Arc<S>, Value) -> Fut + Send + Sync + 'static,
6428 Fut: Future<Output = Result<Value>> + Send + 'static,
6429 {
6430 let handler = Arc::new(handler);
6431 let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
6432 let state = state.downcast::<S>().map_err(|_| {
6433 "registered query state type does not match the replayed workflow state".to_string()
6434 })?;
6435 let handler = Arc::clone(&handler);
6436 Ok(Box::pin(async move {
6437 let result = handler(ctx, state, args.into_json()?).await?;
6438 AvroValue::from_serialize(&result)
6439 }))
6440 });
6441
6442 self.queries
6443 .entry(workflow_type.into())
6444 .or_default()
6445 .insert(
6446 query_name.into(),
6447 RegisteredQuery::Replayed {
6448 state_type: TypeId::of::<S>(),
6449 handler: erased_handler,
6450 },
6451 );
6452 }
6453
6454 pub fn register_replayed_query_avro_value<S, F, Fut>(
6456 &mut self,
6457 workflow_type: impl Into<String>,
6458 query_name: impl Into<String>,
6459 handler: F,
6460 ) where
6461 S: Clone + Send + Sync + 'static,
6462 F: Fn(QueryContext, Arc<S>, AvroValue) -> Fut + Send + Sync + 'static,
6463 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6464 {
6465 let handler = Arc::new(handler);
6466 let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
6467 let state = state.downcast::<S>().map_err(|_| {
6468 "registered query state type does not match the replayed workflow state".to_string()
6469 })?;
6470 Ok(Box::pin(handler(ctx, state, args)))
6471 });
6472
6473 self.queries
6474 .entry(workflow_type.into())
6475 .or_default()
6476 .insert(
6477 query_name.into(),
6478 RegisteredQuery::Replayed {
6479 state_type: TypeId::of::<S>(),
6480 handler: erased_handler,
6481 },
6482 );
6483 }
6484
6485 pub fn register_update<F, Fut>(
6487 &mut self,
6488 workflow_type: impl Into<String>,
6489 update_name: impl Into<String>,
6490 handler: F,
6491 ) where
6492 F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
6493 Fut: Future<Output = Result<Value>> + Send + 'static,
6494 {
6495 let handler = Arc::new(handler);
6496 self.updates
6497 .entry(workflow_type.into())
6498 .or_default()
6499 .insert(
6500 update_name.into(),
6501 Arc::new(move |ctx, args| {
6502 let handler = Arc::clone(&handler);
6503 Box::pin(async move {
6504 let result = handler(ctx, args.into_json()?).await?;
6505 AvroValue::from_serialize(&result)
6506 })
6507 }),
6508 );
6509 }
6510
6511 pub fn register_update_avro_value<F, Fut>(
6513 &mut self,
6514 workflow_type: impl Into<String>,
6515 update_name: impl Into<String>,
6516 handler: F,
6517 ) where
6518 F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
6519 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6520 {
6521 self.updates
6522 .entry(workflow_type.into())
6523 .or_default()
6524 .insert(
6525 update_name.into(),
6526 Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
6527 );
6528 }
6529
6530 pub async fn register(&self) -> Result<RegisterWorkerResponse> {
6531 let mut command_contracts = serde_json::Map::new();
6532 for workflow_type in self.workflows.keys() {
6533 let mut queries = self
6534 .queries
6535 .get(workflow_type)
6536 .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
6537 .unwrap_or_default();
6538 queries.sort();
6539 let mut updates = self
6540 .updates
6541 .get(workflow_type)
6542 .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
6543 .unwrap_or_default();
6544 updates.sort();
6545 command_contracts.insert(
6546 workflow_type.clone(),
6547 json!({
6548 "queries": queries,
6549 "query_contracts": [],
6550 "signals": [],
6551 "signal_contracts": [],
6552 "updates": updates,
6553 "update_contracts": [],
6554 "update_validators": [],
6555 }),
6556 );
6557 }
6558
6559 self.client
6560 .register_worker_with_command_contracts(
6561 &self.worker_id,
6562 &self.task_queue,
6563 self.workflows.keys().cloned().collect(),
6564 self.activities.keys().cloned().collect(),
6565 self.max_concurrent_workflow_tasks,
6566 self.max_concurrent_activity_tasks,
6567 [
6568 Some(CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY.to_string()),
6569 Some(DURABLE_SELECTION_CAPABILITY.to_string()),
6570 Some(MEMO_UPSERTS_CAPABILITY.to_string()),
6571 Some(TYPED_SEARCH_ATTRIBUTES_CAPABILITY.to_string()),
6572 (!self.queries.is_empty()).then(|| QUERY_TASKS_CAPABILITY.to_string()),
6573 (!self.updates.is_empty()).then(|| WORKFLOW_UPDATES_CAPABILITY.to_string()),
6574 worker_protocol_supports_message_streams(WORKER_PROTOCOL_VERSION)
6575 .then(|| MESSAGE_STREAMS_CAPABILITY.to_string()),
6576 ]
6577 .into_iter()
6578 .flatten()
6579 .collect(),
6580 Value::Object(command_contracts),
6581 )
6582 .await
6583 }
6584
6585 pub async fn run(&self) -> Result<()> {
6591 self.run_until(std::future::pending::<()>()).await
6592 }
6593
6594 pub async fn run_until<F>(&self, shutdown: F) -> Result<()>
6598 where
6599 F: Future<Output = ()>,
6600 {
6601 let stop = Arc::new(AtomicBool::new(false));
6602 let _stop_on_drop = StopWorkerOnDrop(Arc::clone(&stop));
6603 let worker = self.with_storage_admission(Arc::clone(&stop));
6604 let run = worker.run_with_storage_admission(Arc::clone(&stop));
6605 tokio::pin!(run);
6606 tokio::pin!(shutdown);
6607 tokio::select! {
6608 result = &mut run => result,
6609 _ = &mut shutdown => {
6610 stop.store(true, Ordering::SeqCst);
6611 run.await
6612 }
6613 }
6614 }
6615
6616 fn with_storage_admission(&self, stop: Arc<AtomicBool>) -> Self {
6617 let mut worker = self.clone();
6618 worker.client.worker_storage_admission = Some(WorkerStorageAdmission {
6619 policy: self.retry_policy,
6620 stop,
6621 });
6622 worker
6623 }
6624
6625 async fn run_with_storage_admission(&self, stop: Arc<AtomicBool>) -> Result<()> {
6626 let registration = self.register().await?;
6627 if !registration.registered {
6628 return Err(Error::WorkerLoop(format!(
6629 "worker registration for {:?} was not accepted",
6630 self.worker_id
6631 )));
6632 }
6633 let registered_worker_id = registration.worker_id.clone();
6634 let primary = self.run_registered_until(stop, registration).await;
6635 let deregistration = self
6636 .client
6637 .deregister_worker_registration(®istered_worker_id)
6638 .await;
6639
6640 match (primary, deregistration) {
6641 (Ok(()), Ok(_)) => Ok(()),
6642 (Ok(()), Err(deregistration)) => Err(deregistration),
6643 (Err(primary), Ok(_)) => Err(primary),
6644 (Err(primary), Err(deregistration)) => Err(Error::WorkerShutdown {
6645 primary: Box::new(primary),
6646 deregistration: Box::new(deregistration),
6647 }),
6648 }
6649 }
6650
6651 async fn run_registered_until(
6652 &self,
6653 stop: Arc<AtomicBool>,
6654 registration: RegisterWorkerResponse,
6655 ) -> Result<()> {
6656 let heartbeat_interval = Duration::from_secs(
6657 registration
6658 .heartbeat_interval_seconds
6659 .unwrap_or(self.heartbeat_interval.as_secs().max(1)),
6660 );
6661 let heartbeat = tokio::time::sleep(Duration::ZERO);
6667 tokio::pin!(heartbeat);
6668 let mut workflow_poller = (!self.workflows.is_empty()).then(|| {
6672 let worker = self.clone();
6673 let stop = Arc::clone(&stop);
6674 tokio::spawn(async move { worker.poll_workflows_until_stopped(stop).await })
6675 });
6676 let mut activity_poller = (!self.activities.is_empty()).then(|| {
6677 let worker = self.clone();
6678 let stop = Arc::clone(&stop);
6679 tokio::spawn(async move { worker.poll_activities_until_stopped(stop).await })
6680 });
6681 let mut query_poller = (!self.queries.is_empty()).then(|| {
6682 let worker = self.clone();
6683 let stop = Arc::clone(&stop);
6684 tokio::spawn(async move { worker.poll_queries_until_stopped(stop).await })
6685 });
6686
6687 loop {
6688 tokio::select! {
6689 _ = wait_for_worker_stop(&stop) => {
6690 stop.store(true, Ordering::SeqCst);
6691 break;
6692 }
6693 _ = &mut heartbeat => {
6694 let result = self.retry_worker_operation(|| {
6695 self.client.heartbeat_worker(
6696 &self.worker_id,
6697 self.max_concurrent_workflow_tasks,
6698 self.max_concurrent_activity_tasks,
6699 )
6700 }).await;
6701 heartbeat
6702 .as_mut()
6703 .reset(tokio::time::Instant::now() + heartbeat_interval);
6704 match result {
6705 Ok(acknowledgement) => {
6706 if let Some(observer) = &self.heartbeat_observer {
6707 observer(&WorkerHeartbeatObservation {
6708 worker_id: self.worker_id.clone(),
6709 task_queue: self.task_queue.clone(),
6710 acknowledged_at_unix_millis: SystemTime::now()
6711 .duration_since(UNIX_EPOCH)
6712 .unwrap_or_default()
6713 .as_millis()
6714 .min(u64::MAX as u128)
6715 as u64,
6716 acknowledgement,
6717 });
6718 }
6719 }
6720 Err(error) => {
6721 stop.store(true, Ordering::SeqCst);
6722 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await?;
6723 return Err(error);
6724 }
6725 }
6726 }
6727 result = OptionFuture::from(workflow_poller.as_mut()), if workflow_poller.is_some() => {
6728 workflow_poller = None;
6729 let stopped_by_server = stop.load(Ordering::SeqCst);
6730 stop.store(true, Ordering::SeqCst);
6731 let poller_result = optional_poller_result("workflow", result);
6732 let join_result =
6733 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6734 poller_result?;
6735 join_result?;
6736 if stopped_by_server {
6737 return Ok(());
6738 }
6739 return Err(Error::WorkerLoop(
6740 "workflow poller stopped unexpectedly".to_string(),
6741 ));
6742 }
6743 result = OptionFuture::from(activity_poller.as_mut()), if activity_poller.is_some() => {
6744 activity_poller = None;
6745 let stopped_by_server = stop.load(Ordering::SeqCst);
6746 stop.store(true, Ordering::SeqCst);
6747 let poller_result = optional_poller_result("activity", result);
6748 let join_result =
6749 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6750 poller_result?;
6751 join_result?;
6752 if stopped_by_server {
6753 return Ok(());
6754 }
6755 return Err(Error::WorkerLoop(
6756 "activity poller stopped unexpectedly".to_string(),
6757 ));
6758 }
6759 result = OptionFuture::from(query_poller.as_mut()), if query_poller.is_some() => {
6760 query_poller = None;
6761 let stopped_by_server = stop.load(Ordering::SeqCst);
6762 stop.store(true, Ordering::SeqCst);
6763 let poller_result = optional_poller_result("query", result);
6764 let join_result =
6765 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6766 poller_result?;
6767 join_result?;
6768 if stopped_by_server {
6769 return Ok(());
6770 }
6771 return Err(Error::WorkerLoop(
6772 "query poller stopped unexpectedly".to_string(),
6773 ));
6774 }
6775 }
6776 }
6777
6778 join_pollers(
6779 workflow_poller.take(),
6780 activity_poller.take(),
6781 query_poller.take(),
6782 )
6783 .await
6784 }
6785
6786 pub async fn run_once(&self) -> Result<usize> {
6800 let worker = self.with_storage_admission(Arc::new(AtomicBool::new(false)));
6801 let mut handled = 0;
6802 match worker.poll_workflow_once().await? {
6803 ManagedPollOutcome::Handled => handled += 1,
6804 ManagedPollOutcome::Stop => return Ok(handled),
6805 ManagedPollOutcome::Idle => {}
6806 }
6807 match worker.poll_activity_once().await? {
6808 ManagedPollOutcome::Handled => handled += 1,
6809 ManagedPollOutcome::Stop => return Ok(handled),
6810 ManagedPollOutcome::Idle => {}
6811 }
6812 if !self.queries.is_empty() {
6813 match worker.poll_query_once().await? {
6814 ManagedPollOutcome::Handled => handled += 1,
6815 ManagedPollOutcome::Stop => return Ok(handled),
6816 ManagedPollOutcome::Idle => {}
6817 }
6818 }
6819 Ok(handled)
6820 }
6821
6822 async fn poll_workflow_once(&self) -> Result<ManagedPollOutcome> {
6823 let poll_request_id = unique_request_id("rust-workflow-poll");
6824 let response = self
6825 .retry_worker_operation(|| {
6826 self.client.poll_workflow_task_response_with_request_id(
6827 &self.worker_id,
6828 &self.task_queue,
6829 self.poll_timeout,
6830 &poll_request_id,
6831 0,
6832 )
6833 })
6834 .await;
6835 let Some(response) = self.settle_worker_poll_response(response).await? else {
6836 return Ok(ManagedPollOutcome::Idle);
6837 };
6838 if response.outcome().should_stop() {
6839 return Ok(ManagedPollOutcome::Stop);
6840 }
6841 let memo_updates_supported =
6842 runtime_supports_workflow_memo_updates(response.server_capabilities.as_ref());
6843 let Some(task) = response.task else {
6844 return Ok(ManagedPollOutcome::Idle);
6845 };
6846
6847 let task_id = task.task_id.clone();
6848 let attempt = task.workflow_task_attempt;
6849 let run_id = task.run_id.clone();
6850 let lease_owner = task
6851 .lease_owner
6852 .clone()
6853 .unwrap_or_else(|| self.worker_id.clone());
6854
6855 match self.execute_workflow_task_decision(task) {
6856 Ok(decision)
6857 if commands_use_workflow_memo_updates(&decision.commands)
6858 && !memo_updates_supported =>
6859 {
6860 self.client
6861 .fail_workflow_task(
6862 &task_id,
6863 &lease_owner,
6864 attempt,
6865 Error::WorkflowMemoUpdatesUnavailable.to_string(),
6866 )
6867 .await?;
6868 }
6869 Ok(decision) if decision.commands.is_empty() => {
6870 self.client
6876 .fail_workflow_task_with_type(
6877 &task_id,
6878 &lease_owner,
6879 attempt,
6880 WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE,
6881 WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE,
6882 )
6883 .await?;
6884 }
6885 Ok(decision) => {
6886 let completion = self
6887 .client
6888 .complete_workflow_task_with_message_streams(
6889 &task_id,
6890 &lease_owner,
6891 attempt,
6892 decision.commands,
6893 decision.message_stream_cursors,
6894 decision.message_stream_waits,
6895 )
6896 .await;
6897 if let Err(error) = completion {
6898 if !workflow_task_completion_is_terminal_timeout(
6899 &error,
6900 &task_id,
6901 attempt,
6902 run_id.as_deref(),
6903 ) {
6904 return Err(error);
6905 }
6906 }
6907 }
6908 Err(error) => {
6909 self.client
6910 .fail_workflow_task(&task_id, &lease_owner, attempt, error.to_string())
6911 .await?;
6912 }
6913 }
6914
6915 Ok(ManagedPollOutcome::Handled)
6916 }
6917
6918 async fn poll_workflows_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
6919 while !stop.load(Ordering::SeqCst) {
6920 if self.poll_workflow_once().await? == ManagedPollOutcome::Stop {
6921 stop.store(true, Ordering::SeqCst);
6922 break;
6923 }
6924 }
6925
6926 Ok(())
6927 }
6928
6929 async fn poll_activity_once(&self) -> Result<ManagedPollOutcome> {
6930 let poll_request_id = unique_request_id("rust-activity-poll");
6931 let response = self
6932 .retry_worker_operation(|| {
6933 self.client.poll_activity_task_response_with_request_id(
6934 &self.worker_id,
6935 &self.task_queue,
6936 self.poll_timeout,
6937 &poll_request_id,
6938 0,
6939 )
6940 })
6941 .await;
6942 let Some(response) = self.settle_worker_poll_response(response).await? else {
6943 return Ok(ManagedPollOutcome::Idle);
6944 };
6945 if response.outcome().should_stop() {
6946 return Ok(ManagedPollOutcome::Stop);
6947 }
6948 let Some(task) = response.task else {
6949 return Ok(ManagedPollOutcome::Idle);
6950 };
6951
6952 let task_id = task.task_id.clone();
6953 let attempt_id = task
6954 .activity_attempt_id
6955 .clone()
6956 .or(task.attempt_id.clone())
6957 .unwrap_or_default();
6958 let lease_owner = task
6959 .lease_owner
6960 .clone()
6961 .unwrap_or_else(|| self.worker_id.clone());
6962 let codec = task.payload_codec.clone();
6963 let result = self.execute_activity_task(task).await;
6964 match result {
6965 Err(error) if worker_storage_admission_body(&error).is_some() => return Err(error),
6966 Ok(value) => {
6967 let completion = self
6968 .client
6969 .complete_activity_task(&task_id, &attempt_id, &lease_owner, value, &codec)
6970 .await;
6971 if let Err(error) = completion {
6972 if !activity_task_rejection_is_final(&error) {
6973 return Err(error);
6974 }
6975 }
6976 }
6977 Err(error) => {
6978 let failure = self
6979 .client
6980 .fail_activity_task(
6981 &task_id,
6982 &attempt_id,
6983 &lease_owner,
6984 error.to_string(),
6985 false,
6986 )
6987 .await;
6988 if let Err(error) = failure {
6989 if !activity_task_rejection_is_final(&error) {
6990 return Err(error);
6991 }
6992 }
6993 }
6994 }
6995
6996 Ok(ManagedPollOutcome::Handled)
6997 }
6998
6999 async fn poll_activities_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
7000 while !stop.load(Ordering::SeqCst) {
7001 if self.poll_activity_once().await? == ManagedPollOutcome::Stop {
7002 stop.store(true, Ordering::SeqCst);
7003 break;
7004 }
7005 }
7006
7007 Ok(())
7008 }
7009
7010 async fn poll_query_once(&self) -> Result<ManagedPollOutcome> {
7011 let poll_request_id = unique_request_id("rust-query-poll");
7012 let response = self
7013 .retry_worker_operation(|| {
7014 self.client.poll_query_task_response_with_request_id(
7015 &self.worker_id,
7016 &self.task_queue,
7017 self.poll_timeout,
7018 &poll_request_id,
7019 0,
7020 )
7021 })
7022 .await;
7023 let Some(response) = self.settle_worker_poll_response(response).await? else {
7024 return Ok(ManagedPollOutcome::Idle);
7025 };
7026 if response.outcome().should_stop() {
7027 return Ok(ManagedPollOutcome::Stop);
7028 }
7029 let Some(task) = response.task else {
7030 return Ok(ManagedPollOutcome::Idle);
7031 };
7032
7033 let query_task_id = task.query_task_id.clone();
7034 let attempt = task.query_task_attempt;
7035 let lease_owner = task
7036 .lease_owner
7037 .clone()
7038 .unwrap_or_else(|| self.worker_id.clone());
7039 let codec = task.payload_codec.clone();
7040
7041 match self.execute_query_task(task).await {
7042 Ok(value) => {
7043 let result_envelope = match encode_typed_envelope(&value, &codec) {
7044 Ok(result_envelope) => result_envelope,
7045 Err(error) => {
7046 let failure = self
7047 .client
7048 .fail_query_task(
7049 &query_task_id,
7050 &lease_owner,
7051 attempt,
7052 error.to_string(),
7053 "query_result_encode_failed",
7054 "QueryResultEncodeFailed",
7055 )
7056 .await;
7057 if let Err(error) = failure {
7058 if !query_task_rejection_is_final(&error) {
7059 return Err(error);
7060 }
7061 }
7062 return Ok(ManagedPollOutcome::Handled);
7063 }
7064 };
7065
7066 if let Err(error) = self
7067 .client
7068 .complete_query_task_with_envelope(
7069 &query_task_id,
7070 &lease_owner,
7071 attempt,
7072 value.clone().into_json()?,
7073 result_envelope,
7074 )
7075 .await
7076 {
7077 if !query_task_rejection_is_final(&error) {
7078 return Err(error);
7079 }
7080 }
7081 }
7082 Err(failure) => {
7083 let result = self
7084 .client
7085 .fail_query_task(
7086 &query_task_id,
7087 &lease_owner,
7088 attempt,
7089 failure.message,
7090 failure.reason,
7091 failure.failure_type,
7092 )
7093 .await;
7094 if let Err(error) = result {
7095 if !query_task_rejection_is_final(&error) {
7096 return Err(error);
7097 }
7098 }
7099 }
7100 }
7101
7102 Ok(ManagedPollOutcome::Handled)
7103 }
7104
7105 async fn poll_queries_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
7106 while !stop.load(Ordering::SeqCst) {
7107 if self.poll_query_once().await? == ManagedPollOutcome::Stop {
7108 stop.store(true, Ordering::SeqCst);
7109 break;
7110 }
7111 }
7112
7113 Ok(())
7114 }
7115
7116 async fn retry_worker_operation<T, F, Fut>(&self, mut operation: F) -> Result<T>
7117 where
7118 F: FnMut() -> Fut,
7119 Fut: Future<Output = Result<T>>,
7120 {
7121 let mut retries = 0;
7122
7123 loop {
7124 match operation().await {
7125 Err(error)
7126 if worker_operation_is_retryable(&error)
7127 && retries < self.retry_policy.max_retries =>
7128 {
7129 retries += 1;
7130 tokio::time::sleep(worker_retry_delay(self.retry_policy, retries)).await;
7131 }
7132 result => return result,
7133 }
7134 }
7135 }
7136
7137 async fn settle_worker_poll_response<T>(&self, response: Result<T>) -> Result<Option<T>> {
7138 match response {
7139 Ok(response) => Ok(Some(response)),
7140 Err(error) => {
7141 let Some(advertised_delay) = worker_poll_capacity_retry_after(&error) else {
7142 return Err(error);
7143 };
7144 let minimum_delay = self
7145 .retry_policy
7146 .initial_backoff
7147 .max(Duration::from_millis(1));
7148 let maximum_delay = self.retry_policy.max_backoff.max(minimum_delay);
7149 tokio::time::sleep(advertised_delay.max(minimum_delay).min(maximum_delay)).await;
7150 Ok(None)
7151 }
7152 }
7153 }
7154
7155 async fn execute_query_task(
7156 &self,
7157 mut task: QueryTask,
7158 ) -> std::result::Result<AvroValue, QueryTaskExecutionFailure> {
7159 validate_query_task_payloads(&task).map_err(|error| {
7160 QueryTaskExecutionFailure::new(
7161 "query_payload_decode_failed",
7162 error.to_string(),
7163 "QueryPayloadDecodeFailed",
7164 )
7165 })?;
7166
7167 if !self.workflows.contains_key(&task.workflow_type) {
7168 return Err(QueryTaskExecutionFailure::new(
7169 "query_workflow_type_not_registered",
7170 format!("no workflow registered for type {:?}", task.workflow_type),
7171 "WorkflowTypeNotRegistered",
7172 ));
7173 }
7174
7175 let Some(handlers) = self.queries.get(&task.workflow_type) else {
7176 return Err(QueryTaskExecutionFailure::new(
7177 "query_handler_unavailable",
7178 format!(
7179 "query handlers are unavailable for workflow type {:?}",
7180 task.workflow_type
7181 ),
7182 "QueryHandlerUnavailable",
7183 ));
7184 };
7185 let Some(query) = handlers.get(&task.query_name) else {
7186 return Err(QueryTaskExecutionFailure::new(
7187 "rejected_unknown_query",
7188 format!("unknown query {:?}", task.query_name),
7189 "QueryFailed",
7190 ));
7191 };
7192
7193 let args = decode_task_avro_arguments(task.query_arguments.as_ref(), &task.payload_codec)
7194 .map_err(|error| {
7195 QueryTaskExecutionFailure::new(
7196 "query_payload_decode_failed",
7197 format!("cannot decode query arguments: {error}"),
7198 "QueryPayloadDecodeFailed",
7199 )
7200 })?;
7201 let workflow_input_typed =
7202 decode_task_avro_arguments(task.workflow_arguments.as_ref(), &task.payload_codec)
7203 .map_err(|error| {
7204 QueryTaskExecutionFailure::new(
7205 "query_workflow_state_unavailable",
7206 format!("cannot decode workflow start input: {error}"),
7207 "QueryWorkflowStateUnavailable",
7208 )
7209 })?;
7210 let workflow_input = workflow_input_typed.clone().into_json().map_err(|error| {
7211 QueryTaskExecutionFailure::new(
7212 "query_workflow_state_unavailable",
7213 format!("cannot project workflow start input: {error}"),
7214 "QueryWorkflowStateUnavailable",
7215 )
7216 })?;
7217 hydrate_query_history_from_export(&mut task).map_err(|error| {
7218 QueryTaskExecutionFailure::new(
7219 "query_workflow_state_unavailable",
7220 format!("cannot restore query history snapshot: {error}"),
7221 "QueryWorkflowStateUnavailable",
7222 )
7223 })?;
7224 enrich_query_history_from_export(&mut task).map_err(|error| {
7225 QueryTaskExecutionFailure::new(
7226 "query_workflow_state_unavailable",
7227 format!("cannot restore compact query history payloads: {error}"),
7228 "QueryWorkflowStateUnavailable",
7229 )
7230 })?;
7231 let signal_events = query_signal_events(&task).map_err(|error| {
7232 QueryTaskExecutionFailure::new(
7233 "query_workflow_state_unavailable",
7234 format!("cannot decode committed workflow signals: {error}"),
7235 "QueryWorkflowStateUnavailable",
7236 )
7237 })?;
7238 let history_events = Arc::new(std::mem::take(&mut task.history_events));
7239 let context = QueryContext {
7240 workflow_id: task.workflow_id,
7241 run_id: task.run_id,
7242 workflow_type: task.workflow_type.clone(),
7243 run_status: task.run_status,
7244 workflow_input,
7245 workflow_input_avro_value: workflow_input_typed.clone(),
7246 history_events: Arc::clone(&history_events),
7247 signal_events: Arc::new(signal_events),
7248 };
7249
7250 let future = match query {
7251 RegisteredQuery::Snapshot(handler) => handler(context, args),
7252 RegisteredQuery::Replayed {
7253 state_type,
7254 handler,
7255 } => {
7256 let workflow = self
7257 .workflows
7258 .get(&task.workflow_type)
7259 .expect("workflow registration was checked above");
7260 if workflow.state_type != Some(*state_type) {
7261 return Err(QueryTaskExecutionFailure::new(
7262 "query_workflow_state_unavailable",
7263 "replayed query state type does not match its workflow registration",
7264 "QueryWorkflowStateUnavailable",
7265 ));
7266 }
7267 let replay = workflow.replay.as_ref().ok_or_else(|| {
7268 QueryTaskExecutionFailure::new(
7269 "query_workflow_state_unavailable",
7270 format!(
7271 "workflow type {:?} is not registered for instance-state replay",
7272 task.workflow_type
7273 ),
7274 "QueryWorkflowStateUnavailable",
7275 )
7276 })?;
7277 let workflow_state = Arc::new(Mutex::new(
7278 WorkflowState::new_with_identity(
7279 history_events.as_ref().clone(),
7280 context.workflow_id.clone(),
7281 context.run_id.clone(),
7282 self.task_queue.clone(),
7283 task.payload_codec,
7284 None,
7285 )
7286 .map_err(|error| {
7287 QueryTaskExecutionFailure::new(
7288 "query_workflow_state_unavailable",
7289 format!("workflow replay failed before query: {error}"),
7290 "QueryWorkflowStateUnavailable",
7291 )
7292 })?,
7293 ));
7294 let workflow_context = WorkflowContext {
7295 state: workflow_state,
7296 };
7297 let mut invocation = replay(workflow_context.clone(), workflow_input_typed.clone());
7298 let mut cx = TaskContext::from_waker(noop_waker_ref());
7299 match invocation.future.as_mut().poll(&mut cx) {
7300 Poll::Ready(Ok(_)) => {
7301 workflow_context
7302 .ensure_history_consumed()
7303 .map_err(|error| {
7304 QueryTaskExecutionFailure::new(
7305 "query_workflow_state_unavailable",
7306 format!("workflow replay failed before query: {error}"),
7307 "QueryWorkflowStateUnavailable",
7308 )
7309 })?;
7310 }
7311 Poll::Ready(Err(error)) => {
7312 return Err(QueryTaskExecutionFailure::new(
7313 "query_workflow_state_unavailable",
7314 format!("workflow replay failed before query: {error}"),
7315 "QueryWorkflowStateUnavailable",
7316 ));
7317 }
7318 Poll::Pending => {
7319 let commands = workflow_context.take_commands().map_err(|error| {
7320 QueryTaskExecutionFailure::new(
7321 "query_workflow_state_unavailable",
7322 format!("workflow replay failed before query: {error}"),
7323 "QueryWorkflowStateUnavailable",
7324 )
7325 })?;
7326 if commands.is_empty()
7327 && !workflow_context
7328 .matched_recorded_pending()
7329 .map_err(|error| {
7330 QueryTaskExecutionFailure::new(
7331 "query_workflow_state_unavailable",
7332 format!("workflow replay failed before query: {error}"),
7333 "QueryWorkflowStateUnavailable",
7334 )
7335 })?
7336 {
7337 return Err(QueryTaskExecutionFailure::new(
7338 "query_workflow_state_unavailable",
7339 "workflow replay yielded without a durable command",
7340 "QueryWorkflowStateUnavailable",
7341 ));
7342 }
7343 }
7344 }
7345 let state = (invocation.snapshot)().map_err(|error| {
7346 QueryTaskExecutionFailure::new(
7347 "query_workflow_state_unavailable",
7348 format!("cannot snapshot replayed workflow state: {error}"),
7349 "QueryWorkflowStateUnavailable",
7350 )
7351 })?;
7352 handler(context, state, args).map_err(|message| {
7353 QueryTaskExecutionFailure::new(
7354 "query_workflow_state_unavailable",
7355 message,
7356 "QueryWorkflowStateUnavailable",
7357 )
7358 })?
7359 }
7360 };
7361
7362 future.await.map_err(|error| {
7363 QueryTaskExecutionFailure::new("query_rejected", error.to_string(), "QueryFailed")
7364 })
7365 }
7366
7367 #[cfg(test)]
7368 fn execute_workflow_task(&self, task: WorkflowTask) -> Result<Vec<Value>> {
7369 Ok(self.execute_workflow_task_decision(task)?.commands)
7370 }
7371
7372 fn execute_workflow_task_decision(&self, task: WorkflowTask) -> Result<WorkflowTaskDecision> {
7373 validate_workflow_task_payloads(&task)?;
7374
7375 if let Some(update_id) = task
7376 .workflow_update_id
7377 .as_deref()
7378 .filter(|update_id| !update_id.is_empty())
7379 {
7380 return self
7381 .execute_update_task(&task, update_id)
7382 .map(WorkflowTaskDecision::without_message_streams);
7383 }
7384
7385 let workflow = self
7386 .workflows
7387 .get(&task.workflow_type)
7388 .ok_or_else(|| Error::WorkflowNotRegistered(task.workflow_type.clone()))?;
7389 let input = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
7390 let resume_signal = decode_resume_signal(&task)?;
7391 let history_budget = WorkflowHistoryBudget {
7392 event_count: task
7393 .total_history_events
7394 .unwrap_or_else(|| u64::try_from(task.history_events.len()).unwrap_or(u64::MAX)),
7395 size_bytes: task.history_size_bytes,
7396 continue_as_new_recommended: task.continue_as_new_recommended.unwrap_or(false),
7397 pressure: task.history_budget_pressure.clone(),
7398 };
7399 let workflow_command_identity = task
7400 .workflow_command_id
7401 .clone()
7402 .filter(|identity| !identity.is_empty())
7403 .unwrap_or_default();
7404 let mut workflow_state = WorkflowState::new_with_identity(
7405 task.history_events,
7406 task.workflow_id,
7407 task.run_id,
7408 self.task_queue.clone(),
7409 task.payload_codec.clone(),
7410 resume_signal,
7411 )?;
7412 workflow_state.history_budget = history_budget;
7413 workflow_state.workflow_command_identity = workflow_command_identity;
7414 workflow_state.cancel_requested = task.cancel_requested;
7415 let state = Arc::new(Mutex::new(workflow_state));
7416 let ctx = WorkflowContext { state };
7417 let mut future = (workflow.execute)(ctx.clone(), input);
7418 let mut cx = TaskContext::from_waker(noop_waker_ref());
7419
7420 match future.as_mut().poll(&mut cx) {
7421 Poll::Ready(Ok(result)) => {
7422 ctx.ensure_history_consumed()?;
7423 let result = encode_typed_envelope(&result, &task.payload_codec)?;
7424 let mut commands = ctx.take_commands()?;
7425 commands.push(json!({
7426 "type": "complete_workflow",
7427 "result": result
7428 }));
7429 self.message_stream_decision(&ctx, commands)
7430 }
7431 Poll::Ready(Err(error)) => {
7432 if let Error::ContinueAsNew(request) = error {
7433 let mut commands = ctx.take_commands()?;
7434 if let Some(command) = ctx.continue_as_new_command(request)? {
7435 commands.push(command);
7436 }
7437 ctx.ensure_history_consumed()?;
7438 return self.message_stream_decision(&ctx, commands);
7439 }
7440 if workflow_task_integrity_error(&error) {
7441 return Err(error);
7446 }
7447 ctx.ensure_history_consumed()?;
7450 let mut commands = ctx.take_commands()?;
7451 commands.push(workflow_failure_command(&error));
7452 self.message_stream_decision(&ctx, commands)
7453 }
7454 Poll::Pending => {
7455 let commands = ctx.take_commands()?;
7456 if commands.is_empty() && !ctx.matched_recorded_pending()? {
7457 Err(Error::WorkflowYieldedWithoutCommand)
7458 } else {
7459 self.message_stream_decision(&ctx, commands)
7460 }
7461 }
7462 }
7463 }
7464
7465 fn message_stream_decision(
7466 &self,
7467 ctx: &WorkflowContext,
7468 commands: Vec<Value>,
7469 ) -> Result<WorkflowTaskDecision> {
7470 let (message_stream_cursors, message_stream_waits) = ctx.message_stream_metadata()?;
7471 Ok(WorkflowTaskDecision {
7472 commands,
7473 message_stream_cursors,
7474 message_stream_waits,
7475 })
7476 }
7477
7478 fn execute_update_task(&self, task: &WorkflowTask, update_id: &str) -> Result<Vec<Value>> {
7479 if !self.workflows.contains_key(&task.workflow_type) {
7480 return Err(Error::WorkflowNotRegistered(task.workflow_type.clone()));
7481 }
7482
7483 let accepted = task.history_events.iter().rev().find_map(|event| {
7484 (event.event_type == "UpdateAccepted"
7485 && event.payload.get("update_id").and_then(Value::as_str) == Some(update_id))
7486 .then_some(&event.payload)
7487 });
7488 let update_name = accepted
7489 .and_then(|payload| payload.get("update_name"))
7490 .and_then(Value::as_str)
7491 .or(task.update_name.as_deref())
7492 .unwrap_or_default();
7493 let Some(handler) = self
7494 .updates
7495 .get(&task.workflow_type)
7496 .and_then(|handlers| handlers.get(update_name))
7497 else {
7498 return Ok(vec![json!({
7499 "type": "fail_update",
7500 "update_id": update_id,
7501 "message": format!(
7502 "no update handler is registered for {}.{update_name}",
7503 task.workflow_type
7504 ),
7505 "exception_type": "UnknownUpdate",
7506 "non_retryable": true,
7507 })]);
7508 };
7509 let arguments = accepted
7510 .and_then(|payload| payload.get("arguments"))
7511 .or(task.arguments.as_ref());
7512 let arguments = decode_task_avro_arguments(arguments, &task.payload_codec)?;
7513 let context = QueryContext {
7514 workflow_id: task.workflow_id.clone(),
7515 run_id: task.run_id.clone(),
7516 workflow_type: task.workflow_type.clone(),
7517 run_status: Some("running".to_string()),
7518 workflow_input: Value::Null,
7519 workflow_input_avro_value: AvroValue::Null,
7520 history_events: Arc::new(task.history_events.clone()),
7521 signal_events: Arc::new(Vec::new()),
7522 };
7523 let mut future = handler(context, arguments);
7524 let mut cx = TaskContext::from_waker(noop_waker_ref());
7525
7526 match future.as_mut().poll(&mut cx) {
7527 Poll::Ready(Ok(result)) => Ok(vec![json!({
7528 "type": "complete_update",
7529 "update_id": update_id,
7530 "result": encode_typed_envelope(&result, &task.payload_codec)?,
7531 })]),
7532 Poll::Ready(Err(error)) => Ok(vec![json!({
7533 "type": "fail_update",
7534 "update_id": update_id,
7535 "message": error.to_string(),
7536 "exception_type": "UpdateFailed",
7537 "non_retryable": true,
7538 })]),
7539 Poll::Pending => Err(Error::WorkflowYieldedWithoutCommand),
7540 }
7541 }
7542
7543 async fn execute_activity_task(&self, task: ActivityTask) -> Result<AvroValue> {
7544 validate_activity_task_payloads(&task)?;
7545
7546 let handler = self
7547 .activities
7548 .get(&task.activity_type)
7549 .ok_or_else(|| Error::ActivityNotRegistered(task.activity_type.clone()))?;
7550 let args = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
7551 let attempt_id = task
7552 .activity_attempt_id
7553 .clone()
7554 .or(task.attempt_id.clone())
7555 .unwrap_or_default();
7556 let lease_owner = task
7557 .lease_owner
7558 .clone()
7559 .unwrap_or_else(|| self.worker_id.clone());
7560 let ctx = ActivityContext {
7561 client: self.client.clone(),
7562 task_id: task.task_id,
7563 activity_attempt_id: attempt_id,
7564 lease_owner,
7565 activity_type: task.activity_type,
7566 attempt_number: task.attempt_number,
7567 task_queue: self.task_queue.clone(),
7568 worker_id: self.worker_id.clone(),
7569 };
7570
7571 handler(ctx, args).await
7572 }
7573}
7574
7575fn poller_result(
7576 kind: &str,
7577 result: std::result::Result<Result<()>, tokio::task::JoinError>,
7578) -> Result<()> {
7579 match result {
7580 Ok(result) => result,
7581 Err(error) => Err(Error::WorkerLoop(format!(
7582 "{kind} poller join error: {error}"
7583 ))),
7584 }
7585}
7586
7587fn optional_poller_result(
7588 kind: &str,
7589 result: Option<std::result::Result<Result<()>, tokio::task::JoinError>>,
7590) -> Result<()> {
7591 match result {
7592 Some(result) => poller_result(kind, result),
7593 None => Ok(()),
7594 }
7595}
7596
7597async fn join_pollers(
7598 workflow_poller: Option<tokio::task::JoinHandle<Result<()>>>,
7599 activity_poller: Option<tokio::task::JoinHandle<Result<()>>>,
7600 query_poller: Option<tokio::task::JoinHandle<Result<()>>>,
7601) -> Result<()> {
7602 let mut first_error = None;
7603
7604 if let Some(handle) = workflow_poller {
7605 if let Err(error) = poller_result("workflow", handle.await) {
7606 first_error.get_or_insert(error);
7607 }
7608 }
7609
7610 if let Some(handle) = activity_poller {
7611 if let Err(error) = poller_result("activity", handle.await) {
7612 first_error.get_or_insert(error);
7613 }
7614 }
7615
7616 if let Some(handle) = query_poller {
7617 if let Err(error) = poller_result("query", handle.await) {
7618 first_error.get_or_insert(error);
7619 }
7620 }
7621
7622 if let Some(error) = first_error {
7623 Err(error)
7624 } else {
7625 Ok(())
7626 }
7627}
7628
7629fn default_worker_id() -> String {
7630 let millis = SystemTime::now()
7631 .duration_since(UNIX_EPOCH)
7632 .unwrap_or_default()
7633 .as_millis();
7634 format!("rust-worker-{}-{millis}", std::process::id())
7635}
7636
7637fn percent_encode_path_segment(segment: &str) -> String {
7638 const HEX: &[u8; 16] = b"0123456789ABCDEF";
7639 let mut encoded = String::with_capacity(segment.len());
7640
7641 for byte in segment.bytes() {
7642 if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') {
7643 encoded.push(char::from(byte));
7644 } else {
7645 encoded.push('%');
7646 encoded.push(char::from(HEX[(byte >> 4) as usize]));
7647 encoded.push(char::from(HEX[(byte & 0x0f) as usize]));
7648 }
7649 }
7650
7651 encoded
7652}
7653
7654fn unique_request_id(prefix: &str) -> String {
7655 let nanos = SystemTime::now()
7656 .duration_since(UNIX_EPOCH)
7657 .unwrap_or_default()
7658 .as_nanos();
7659 format!("{prefix}-{}-{nanos}", std::process::id())
7660}
7661
7662#[derive(Debug)]
7663struct QueryTaskExecutionFailure {
7664 reason: String,
7665 message: String,
7666 failure_type: String,
7667}
7668
7669impl QueryTaskExecutionFailure {
7670 fn new(
7671 reason: impl Into<String>,
7672 message: impl Into<String>,
7673 failure_type: impl Into<String>,
7674 ) -> Self {
7675 Self {
7676 reason: reason.into(),
7677 message: message.into(),
7678 failure_type: failure_type.into(),
7679 }
7680 }
7681}
7682
7683#[derive(Clone, Debug)]
7689pub struct WorkflowInstance<S> {
7690 state: Arc<Mutex<S>>,
7691}
7692
7693impl<S> WorkflowInstance<S> {
7694 fn new(state: S) -> Self {
7695 Self {
7696 state: Arc::new(Mutex::new(state)),
7697 }
7698 }
7699
7700 pub fn read<R>(&self, reader: impl FnOnce(&S) -> R) -> Result<R> {
7702 let state = self
7703 .state
7704 .lock()
7705 .map_err(|_| Error::WorkflowStatePoisoned)?;
7706 Ok(reader(&state))
7707 }
7708
7709 pub fn update<R>(&self, transition: impl FnOnce(&mut S) -> R) -> Result<R> {
7711 let mut state = self
7712 .state
7713 .lock()
7714 .map_err(|_| Error::WorkflowStatePoisoned)?;
7715 Ok(transition(&mut state))
7716 }
7717}
7718
7719impl<S: Clone> WorkflowInstance<S> {
7720 fn snapshot(&self) -> Result<S> {
7721 self.read(Clone::clone)
7722 }
7723}
7724
7725#[derive(Clone, Debug, PartialEq)]
7726pub struct MessageStreamMessage {
7727 pub stream_name: String,
7728 pub message_id: String,
7729 pub position: u64,
7730 pub arguments: Vec<AvroValue>,
7731}
7732
7733#[derive(Clone, Debug)]
7734pub struct MessageStream {
7735 ctx: WorkflowContext,
7736 name: String,
7737}
7738
7739impl MessageStream {
7740 pub async fn receive(&self, max_items: usize) -> Result<Vec<MessageStreamMessage>> {
7742 if !(1..=MESSAGE_STREAM_MAX_BATCH).contains(&max_items) {
7743 return Err(Error::Codec(format!(
7744 "message stream max_items must be between 1 and {MESSAGE_STREAM_MAX_BATCH}"
7745 )));
7746 }
7747 loop {
7748 if let Some(batch) = self.ctx.take_message_stream_batch(&self.name, max_items)? {
7749 return Ok(batch);
7750 }
7751
7752 self.ctx.record_message_stream_wait(&self.name)?;
7753 let replay_wait_sequence = self.ctx.next_message_stream_wait_sequence()?;
7754 let arguments = self.ctx.wait_runtime_signal(MESSAGE_STREAM_SIGNAL).await?;
7755 self.ctx.buffer_message_stream_delivery(arguments)?;
7756 if let Some(sequence) = replay_wait_sequence {
7757 self.ctx.buffer_message_stream_history_for_wait(sequence)?;
7758 }
7759 }
7760 }
7761
7762 pub async fn receive_one(&self) -> Result<MessageStreamMessage> {
7763 self.receive(1)
7764 .await?
7765 .into_iter()
7766 .next()
7767 .ok_or_else(|| Error::Codec("message stream resumed without a message".to_string()))
7768 }
7769}
7770
7771#[derive(Clone, Debug)]
7772pub struct WorkflowContext {
7773 state: Arc<Mutex<WorkflowState>>,
7774}
7775
7776fn valid_memo_key(key: &str) -> bool {
7777 let numeric_candidate = key.strip_prefix('-').unwrap_or(key);
7778
7779 !key.is_empty()
7780 && key.len() <= 64
7781 && (numeric_candidate.is_empty()
7782 || !numeric_candidate.bytes().all(|byte| byte.is_ascii_digit()))
7783 && key
7784 .bytes()
7785 .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'.' | b':' | b'-'))
7786}
7787
7788fn avro_encoded_size(value: &AvroValue) -> Result<usize> {
7789 BASE64
7790 .decode(encode_avro_value(value)?.blob)
7791 .map(|bytes| bytes.len())
7792 .map_err(|error| Error::Codec(format!("memo Avro encoding was not strict base64: {error}")))
7793}
7794
7795fn canonical_memo_entries(value: AvroValue, require_entries: bool) -> Result<AvroValue> {
7796 let AvroValue::Map(entries) = value else {
7797 return Err(Error::InvalidMemoUpdate(
7798 "entries must serialize to an Avro string-keyed map".to_string(),
7799 ));
7800 };
7801 if require_entries && entries.is_empty() {
7802 return Err(Error::InvalidMemoUpdate(
7803 "at least one entry is required".to_string(),
7804 ));
7805 }
7806 if entries.len() > MAX_MEMO_ENTRIES {
7807 return Err(Error::InvalidMemoUpdate(format!(
7808 "at most {MAX_MEMO_ENTRIES} entries are allowed"
7809 )));
7810 }
7811
7812 for (key, value) in &entries {
7813 if !valid_memo_key(&key) {
7814 return Err(Error::InvalidMemoUpdate(
7815 "keys must match ^(?!-?[0-9]+$)[A-Za-z0-9_.:-]{1,64}$".to_string(),
7816 ));
7817 }
7818 if avro_encoded_size(value)? > MAX_MEMO_VALUE_SIZE_BYTES {
7819 return Err(Error::InvalidMemoUpdate(format!(
7820 "value {key:?} exceeds the {MAX_MEMO_VALUE_SIZE_BYTES}-byte limit"
7821 )));
7822 }
7823 }
7824
7825 let value = AvroValue::Map(entries);
7826 if avro_encoded_size(&value)? > MAX_MEMO_TOTAL_SIZE_BYTES {
7827 return Err(Error::InvalidMemoUpdate(format!(
7828 "update exceeds the {MAX_MEMO_TOTAL_SIZE_BYTES}-byte total limit"
7829 )));
7830 }
7831 Ok(value)
7832}
7833
7834fn decode_memo_history_map(envelope: &Value, require_entries: bool) -> Result<AvroValue> {
7835 let object = envelope.as_object().ok_or_else(|| {
7836 Error::InvalidMemoUpdate(
7837 "history field must use the public {codec, blob} payload envelope".to_string(),
7838 )
7839 })?;
7840 if object.len() != 2 || !object.contains_key("codec") || !object.contains_key("blob") {
7841 return Err(Error::InvalidMemoUpdate(
7842 "history field must use exactly the public {codec, blob} payload envelope".to_string(),
7843 ));
7844 }
7845
7846 canonical_memo_entries(
7847 decode_wire_avro_value(envelope, DEFAULT_CODEC)?,
7848 require_entries,
7849 )
7850}
7851
7852impl WorkflowContext {
7853 pub fn message_stream(&self, name: impl Into<String>) -> Result<MessageStream> {
7854 let name = name.into();
7855 if name.is_empty()
7856 || name.len() > 128
7857 || !name.bytes().all(|byte| {
7858 byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b':' | b'-')
7859 })
7860 {
7861 return Err(Error::Codec(
7862 "message stream names must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens"
7863 .to_string(),
7864 ));
7865 }
7866 Ok(MessageStream {
7867 ctx: self.clone(),
7868 name,
7869 })
7870 }
7871
7872 fn record_message_stream_wait(&self, name: &str) -> Result<()> {
7873 let mut state = self
7874 .state
7875 .lock()
7876 .map_err(|_| Error::WorkflowStatePoisoned)?;
7877 let position = state.message_stream_cursors.get(name).copied().unwrap_or(0);
7878 state
7879 .message_stream_waits
7880 .insert(name.to_string(), position);
7881 Ok(())
7882 }
7883
7884 fn buffer_message_stream(&self, message: MessageStreamMessage) -> Result<()> {
7885 let mut state = self
7886 .state
7887 .lock()
7888 .map_err(|_| Error::WorkflowStatePoisoned)?;
7889 let cursor = state
7890 .message_stream_cursors
7891 .get(&message.stream_name)
7892 .copied()
7893 .unwrap_or(0);
7894 if message.position <= cursor {
7895 return Ok(());
7896 }
7897 let pending = state
7898 .message_stream_messages
7899 .entry(message.stream_name.clone())
7900 .or_default();
7901 if pending.iter().any(|candidate| {
7902 candidate.position == message.position || candidate.message_id == message.message_id
7903 }) {
7904 return Ok(());
7905 }
7906 pending.push(message);
7907 pending.sort_by_key(|candidate| candidate.position);
7908 Ok(())
7909 }
7910
7911 fn buffer_message_stream_delivery(&self, arguments: Vec<Value>) -> Result<Option<String>> {
7912 if let Some(delivery) = decode_message_stream_delivery(arguments)? {
7913 match delivery {
7914 MessageStreamDelivery::Message(message) => {
7915 let stream_name = message.stream_name.clone();
7916 self.buffer_message_stream(message)?;
7917 return Ok(Some(stream_name));
7918 }
7919 MessageStreamDelivery::Cursor {
7920 stream_name,
7921 through_position,
7922 } => self.apply_message_stream_cursor(&stream_name, through_position)?,
7923 }
7924 }
7925 Ok(None)
7926 }
7927
7928 fn next_message_stream_wait_sequence(&self) -> Result<Option<u64>> {
7929 let state = self
7930 .state
7931 .lock()
7932 .map_err(|_| Error::WorkflowStatePoisoned)?;
7933 Ok(match state.recorded_commands.get(state.command_cursor) {
7934 Some(RecordedCommand::SignalWait {
7935 sequence,
7936 signal_name,
7937 ..
7938 }) if signal_name == MESSAGE_STREAM_SIGNAL => Some(*sequence),
7939 _ => None,
7940 })
7941 }
7942
7943 fn buffer_message_stream_history_for_wait(&self, wait_sequence: u64) -> Result<()> {
7944 let (history, payload_codec) = {
7945 let state = self
7946 .state
7947 .lock()
7948 .map_err(|_| Error::WorkflowStatePoisoned)?;
7949 (
7950 Arc::clone(&state.history_events),
7951 state.payload_codec.clone(),
7952 )
7953 };
7954
7955 let Some(opened_index) = history.iter().position(|event| {
7956 event.event_type == "SignalWaitOpened"
7957 && durable_event_sequence(event) == Some(wait_sequence)
7958 && event.payload.get("signal_name").and_then(Value::as_str)
7959 == Some(MESSAGE_STREAM_SIGNAL)
7960 }) else {
7961 return Ok(());
7962 };
7963 let boundary_index = history
7964 .iter()
7965 .enumerate()
7966 .skip(opened_index + 1)
7967 .find_map(|(index, event)| {
7968 (durable_event_sequence(event).is_some_and(|sequence| sequence > wait_sequence)
7969 && is_authored_command_open_event(event))
7970 .then_some(index)
7971 })
7972 .unwrap_or(history.len());
7973
7974 for event in history[opened_index + 1..boundary_index]
7975 .iter()
7976 .filter(|event| {
7977 event.event_type == "SignalReceived"
7978 && event.payload.get("signal_name").and_then(Value::as_str)
7979 == Some(MESSAGE_STREAM_SIGNAL)
7980 })
7981 {
7982 let arguments = decode_signal_event_arguments(event, &payload_codec)?
7983 .into_iter()
7984 .map(AvroValue::into_json)
7985 .collect::<Result<Vec<_>>>()?;
7986 self.buffer_message_stream_delivery(arguments)?;
7987 }
7988 Ok(())
7989 }
7990
7991 fn apply_message_stream_cursor(&self, name: &str, through_position: u64) -> Result<()> {
7992 let mut state = self
7993 .state
7994 .lock()
7995 .map_err(|_| Error::WorkflowStatePoisoned)?;
7996 let cursor = state
7997 .message_stream_cursors
7998 .entry(name.to_string())
7999 .or_default();
8000 *cursor = (*cursor).max(through_position);
8001 if let Some(pending) = state.message_stream_messages.get_mut(name) {
8002 pending.retain(|message| message.position > through_position);
8003 }
8004 Ok(())
8005 }
8006
8007 fn take_message_stream_batch(
8008 &self,
8009 name: &str,
8010 max_items: usize,
8011 ) -> Result<Option<Vec<MessageStreamMessage>>> {
8012 let mut state = self
8013 .state
8014 .lock()
8015 .map_err(|_| Error::WorkflowStatePoisoned)?;
8016 let cursor = state.message_stream_cursors.get(name).copied().unwrap_or(0);
8017 let pending = state
8018 .message_stream_messages
8019 .entry(name.to_string())
8020 .or_default();
8021 let count = contiguous_message_stream_count(pending, cursor, max_items);
8022 if count == 0 {
8023 return Ok(None);
8024 }
8025 let batch = pending.drain(..count).collect::<Vec<_>>();
8026 let position = batch.last().map(|message| message.position).unwrap_or(0);
8027 state
8028 .message_stream_cursors
8029 .insert(name.to_string(), position);
8030 state.message_stream_waits.remove(name);
8031 Ok(Some(batch))
8032 }
8033
8034 fn message_stream_metadata(&self) -> Result<(Vec<Value>, Vec<Value>)> {
8035 let state = self
8036 .state
8037 .lock()
8038 .map_err(|_| Error::WorkflowStatePoisoned)?;
8039 let mut cursors = state.message_stream_cursors.iter().collect::<Vec<_>>();
8040 cursors.sort_by_key(|(name, _)| *name);
8041 let mut waits = state.message_stream_waits.iter().collect::<Vec<_>>();
8042 waits.sort_by_key(|(name, _)| *name);
8043 Ok((
8044 cursors
8045 .into_iter()
8046 .map(|(name, position)| json!({"stream_name": name, "through_position": position}))
8047 .collect(),
8048 waits
8049 .into_iter()
8050 .map(|(name, position)| json!({"stream_name": name, "after_position": position}))
8051 .collect(),
8052 ))
8053 }
8054 pub fn workflow_identity(&self) -> Result<WorkflowIdentity> {
8056 let state = self
8057 .state
8058 .lock()
8059 .map_err(|_| Error::WorkflowStatePoisoned)?;
8060 Ok(WorkflowIdentity {
8061 workflow_id: state.workflow_id.clone(),
8062 run_id: state.run_id.clone(),
8063 })
8064 }
8065
8066 pub fn history_budget(&self) -> Result<WorkflowHistoryBudget> {
8068 let state = self
8069 .state
8070 .lock()
8071 .map_err(|_| Error::WorkflowStatePoisoned)?;
8072 Ok(state.history_budget.clone())
8073 }
8074
8075 pub fn continue_as_new<T: Serialize>(&self, args: T) -> Result<Value> {
8081 self.continue_as_new_with_options(ContinueAsNewOptions::new(), args)
8082 }
8083
8084 pub fn continue_as_new_with_options<T: Serialize>(
8086 &self,
8087 options: ContinueAsNewOptions,
8088 args: T,
8089 ) -> Result<Value> {
8090 options.validate()?;
8091 Err(Error::ContinueAsNew(ContinueAsNewRequest {
8092 arguments: normalize_avro_arguments(AvroValue::from_serialize(&args)?),
8093 options,
8094 }))
8095 }
8096
8097 pub fn activity<T: Serialize>(
8098 &self,
8099 activity_type: impl Into<String>,
8100 args: T,
8101 ) -> ActivityCall {
8102 self.activity_with_options(activity_type, ActivityOptions::new(), args)
8103 }
8104
8105 pub fn activity_on_queue<T, Q>(
8106 &self,
8107 activity_type: impl Into<String>,
8108 task_queue: Option<Q>,
8109 args: T,
8110 ) -> ActivityCall
8111 where
8112 T: Serialize,
8113 Q: Into<String>,
8114 {
8115 let mut options = ActivityOptions::new();
8116 options.task_queue = task_queue.map(Into::into);
8117 self.activity_with_options(activity_type, options, args)
8118 }
8119
8120 pub fn activity_with_options<T: Serialize>(
8158 &self,
8159 activity_type: impl Into<String>,
8160 options: ActivityOptions,
8161 args: T,
8162 ) -> ActivityCall {
8163 ActivityCall {
8164 ctx: self.clone(),
8165 activity_type: activity_type.into(),
8166 options,
8167 args: Some(AvroValue::from_serialize(&args)),
8168 scheduled: false,
8169 parallel_group_path: Vec::new(),
8170 }
8171 }
8172
8173 pub async fn activity_avro_value<T: Serialize>(
8174 &self,
8175 activity_type: impl Into<String>,
8176 args: T,
8177 ) -> Result<AvroValue> {
8178 let mut call = self.activity(activity_type, args);
8179 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8180 }
8181
8182 pub async fn activity_avro_value_with_options<T: Serialize>(
8183 &self,
8184 activity_type: impl Into<String>,
8185 options: ActivityOptions,
8186 args: T,
8187 ) -> Result<AvroValue> {
8188 let mut call = self.activity_with_options(activity_type, options, args);
8189 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8190 }
8191
8192 pub async fn activity_typed<I, O>(&self, activity_type: impl Into<String>, args: I) -> Result<O>
8194 where
8195 I: Serialize,
8196 O: DeserializeOwned,
8197 {
8198 self.activity_typed_with_options(activity_type, ActivityOptions::new(), args)
8199 .await
8200 }
8201
8202 pub async fn activity_typed_with_options<I, O>(
8208 &self,
8209 activity_type: impl Into<String>,
8210 options: ActivityOptions,
8211 args: I,
8212 ) -> Result<O>
8213 where
8214 I: Serialize,
8215 O: DeserializeOwned,
8216 {
8217 let activity_type = activity_type.into();
8218 let encoded = AvroValue::from_serialize(&args).map_err(|error| {
8219 handler_type_error::<I>(
8220 HandlerKind::Activity,
8221 &activity_type,
8222 HandlerValueKind::Input,
8223 error.to_string(),
8224 )
8225 });
8226 let mut call = ActivityCall {
8227 ctx: self.clone(),
8228 activity_type: activity_type.clone(),
8229 options,
8230 args: Some(encoded),
8231 scheduled: false,
8232 parallel_group_path: Vec::new(),
8233 };
8234 let result = std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await?;
8235 decode_handler_result(result, HandlerKind::Activity, &activity_type)
8236 }
8237
8238 pub fn parallel(&self, operations: Vec<ParallelOperation>) -> ParallelCall {
8245 ParallelCall::new(self.clone(), operations)
8246 }
8247
8248 pub fn join(&self, operations: Vec<ParallelOperation>) -> ParallelCall {
8250 self.parallel(operations)
8251 }
8252
8253 pub async fn parallel_avro_value(
8255 &self,
8256 operations: Vec<ParallelOperation>,
8257 ) -> Result<Vec<ParallelAvroResult>> {
8258 let mut call = self.parallel(operations);
8259 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8260 }
8261
8262 pub fn select(&self, operations: Vec<ParallelOperation>) -> SelectCall {
8265 let operations = operations
8266 .into_iter()
8267 .enumerate()
8268 .map(|(index, operation)| (SelectionKey::Index(index), operation))
8269 .collect();
8270 SelectCall::new(self.clone(), operations)
8271 }
8272
8273 pub fn select_keyed<K>(&self, operations: Vec<(K, ParallelOperation)>) -> SelectCall
8275 where
8276 K: Into<SelectionKey>,
8277 {
8278 SelectCall::new(
8279 self.clone(),
8280 operations
8281 .into_iter()
8282 .map(|(key, operation)| (key.into(), operation))
8283 .collect(),
8284 )
8285 }
8286
8287 pub fn saga(&self) -> Saga {
8289 Saga::new(self.clone())
8290 }
8291
8292 pub fn is_cancellation_requested(&self) -> Result<bool> {
8294 let state = self
8295 .state
8296 .lock()
8297 .map_err(|_| Error::WorkflowStatePoisoned)?;
8298 Ok(state.cancel_requested)
8299 }
8300
8301 pub fn throw_if_cancellation_requested(&self) -> Result<()> {
8306 if self.is_cancellation_requested()? {
8307 return Err(Error::WorkflowCancellationRequested(
8308 WorkflowCancellationRequested,
8309 ));
8310 }
8311 Ok(())
8312 }
8313
8314 pub fn wait_signal(&self, signal_name: impl Into<String>) -> SignalCall {
8315 SignalCall {
8316 ctx: self.clone(),
8317 signal_name: signal_name.into(),
8318 runtime_reserved_allowed: false,
8319 opened_wait: false,
8320 matched_pending: false,
8321 parallel_group_path: Vec::new(),
8322 }
8323 }
8324
8325 fn wait_runtime_signal(&self, signal_name: impl Into<String>) -> SignalCall {
8326 SignalCall {
8327 ctx: self.clone(),
8328 signal_name: signal_name.into(),
8329 runtime_reserved_allowed: true,
8330 opened_wait: false,
8331 matched_pending: false,
8332 parallel_group_path: Vec::new(),
8333 }
8334 }
8335
8336 pub async fn wait_signal_avro_value(
8337 &self,
8338 signal_name: impl Into<String>,
8339 ) -> Result<Vec<AvroValue>> {
8340 let mut call = self.wait_signal(signal_name);
8341 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8342 }
8343
8344 pub fn signals(&self, signal_name: &str) -> Result<Vec<Vec<Value>>> {
8350 self.signals_avro_value(signal_name)?
8351 .into_iter()
8352 .map(|arguments| {
8353 arguments
8354 .into_iter()
8355 .map(AvroValue::into_json)
8356 .collect::<Result<Vec<_>>>()
8357 })
8358 .collect()
8359 }
8360
8361 pub fn signals_avro_value(&self, signal_name: &str) -> Result<Vec<Vec<AvroValue>>> {
8363 let state = self
8364 .state
8365 .lock()
8366 .map_err(|_| Error::WorkflowStatePoisoned)?;
8367 state
8368 .history_events
8369 .iter()
8370 .filter(|event| {
8371 event.event_type == "SignalReceived"
8372 && event.payload.get("signal_name").and_then(Value::as_str) == Some(signal_name)
8373 })
8374 .map(|event| decode_signal_event_arguments(event, &state.payload_codec))
8375 .collect()
8376 }
8377
8378 pub fn updates(&self, update_name: &str) -> Result<Vec<Vec<Value>>> {
8384 self.updates_avro_value(update_name)?
8385 .into_iter()
8386 .map(|arguments| {
8387 arguments
8388 .into_iter()
8389 .map(AvroValue::into_json)
8390 .collect::<Result<Vec<_>>>()
8391 })
8392 .collect()
8393 }
8394
8395 pub fn updates_avro_value(&self, update_name: &str) -> Result<Vec<Vec<AvroValue>>> {
8397 let state = self
8398 .state
8399 .lock()
8400 .map_err(|_| Error::WorkflowStatePoisoned)?;
8401 let mut seen = Vec::new();
8402 let mut updates = Vec::new();
8403 for event in state.history_events.iter() {
8404 if !matches!(
8405 event.event_type.as_str(),
8406 "UpdateAccepted" | "UpdateApplied"
8407 ) || event.payload.get("update_name").and_then(Value::as_str) != Some(update_name)
8408 || event.payload.get("arguments").is_none()
8409 {
8410 continue;
8411 }
8412 if let Some(update_id) = event.payload.get("update_id").and_then(Value::as_str) {
8413 if seen.iter().any(|recorded| recorded == update_id) {
8414 continue;
8415 }
8416 seen.push(update_id.to_string());
8417 }
8418 updates.push(decode_update_event_arguments(event, &state.payload_codec)?);
8419 }
8420 Ok(updates)
8421 }
8422
8423 pub fn wait_condition<F>(
8431 &self,
8432 options: ConditionWaitOptions,
8433 predicate: F,
8434 ) -> ConditionWaitCall
8435 where
8436 F: Fn() -> Result<bool> + Send + 'static,
8437 {
8438 ConditionWaitCall {
8439 ctx: self.clone(),
8440 options,
8441 predicate: Box::new(predicate),
8442 occurrence_id: None,
8443 opened_wait: false,
8444 parallel_group_path: Vec::new(),
8445 }
8446 }
8447
8448 pub fn sleep(&self, duration: Duration) -> TimerCall {
8469 let delay_seconds = duration
8470 .as_secs()
8471 .checked_add(u64::from(duration.subsec_nanos() > 0));
8472 TimerCall {
8473 ctx: self.clone(),
8474 delay_seconds,
8475 scheduled: false,
8476 matched_pending: false,
8477 parallel_group_path: Vec::new(),
8478 }
8479 }
8480
8481 pub fn start_timer(&self, duration: Duration) -> TimerCall {
8483 self.sleep(duration)
8484 }
8485
8486 pub fn side_effect<T, F>(&self, callback: F) -> Result<T>
8493 where
8494 T: Serialize + DeserializeOwned,
8495 F: FnOnce() -> T,
8496 {
8497 {
8498 let mut state = self
8499 .state
8500 .lock()
8501 .map_err(|_| Error::WorkflowStatePoisoned)?;
8502 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8503 return match recorded {
8504 RecordedCommand::SideEffect { sequence, value } => {
8505 state.command_cursor += 1;
8506 value.deserialize().map_err(|error| {
8507 Error::NonDeterministicReplay(ReplayFailure::new(
8508 "side_effect_type_mismatch",
8509 Some(sequence),
8510 Some(std::any::type_name::<T>().to_string()),
8511 Some(error.to_string()),
8512 "recorded side-effect value is incompatible with the requested Rust type",
8513 ))
8514 })
8515 }
8516 other => Err(command_mismatch(&other, "side effect")),
8517 };
8518 }
8519 }
8520
8521 let value = callback();
8522 let avro_value = AvroValue::from_serialize(&value)?;
8523 let mut state = self
8524 .state
8525 .lock()
8526 .map_err(|_| Error::WorkflowStatePoisoned)?;
8527 let result = encode_typed_envelope(&avro_value, &state.payload_codec)?;
8528 state.commands.push(json!({
8529 "type": "record_side_effect",
8530 "result": result,
8531 }));
8532 Ok(value)
8533 }
8534
8535 pub fn side_effect_avro_value<F>(&self, callback: F) -> Result<AvroValue>
8537 where
8538 F: FnOnce() -> AvroValue,
8539 {
8540 {
8541 let mut state = self
8542 .state
8543 .lock()
8544 .map_err(|_| Error::WorkflowStatePoisoned)?;
8545 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8546 return match recorded {
8547 RecordedCommand::SideEffect { value, .. } => {
8548 state.command_cursor += 1;
8549 Ok(value)
8550 }
8551 other => Err(command_mismatch(&other, "side effect")),
8552 };
8553 }
8554 }
8555
8556 let value = callback();
8557 let mut state = self
8558 .state
8559 .lock()
8560 .map_err(|_| Error::WorkflowStatePoisoned)?;
8561 let result = encode_typed_envelope(&value, &state.payload_codec)?;
8562 state.commands.push(json!({
8563 "type": "record_side_effect",
8564 "result": result,
8565 }));
8566 Ok(value)
8567 }
8568
8569 pub fn append_workflow_stream(
8575 &self,
8576 stream_name: impl Into<String>,
8577 items: &[WorkflowStreamAppendItem],
8578 max_pending_items: Option<u64>,
8579 ) -> Result<()> {
8580 if items.is_empty() {
8581 return Err(Error::Codec(
8582 "workflow_stream_items_empty: append requires at least one item".to_string(),
8583 ));
8584 }
8585 if max_pending_items == Some(0) {
8586 return Err(Error::Codec(
8587 "workflow_stream_pending_limit_invalid: max_pending_items must be positive"
8588 .to_string(),
8589 ));
8590 }
8591 let stream_name = stream_name.into();
8592 if stream_name.is_empty() {
8593 return Err(Error::Codec(
8594 "workflow_stream_name_invalid: stream name must not be empty".to_string(),
8595 ));
8596 }
8597
8598 let mut state = self
8599 .state
8600 .lock()
8601 .map_err(|_| Error::WorkflowStatePoisoned)?;
8602 let command_ordinal = state.workflow_stream_command_counter;
8603 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8604 state.workflow_stream_command_counter += 1;
8605 return match recorded {
8606 RecordedCommand::SideEffect { .. } => {
8607 state.command_cursor += 1;
8608 Ok(())
8609 }
8610 other => Err(command_mismatch(&other, "workflow stream append")),
8611 };
8612 }
8613
8614 let identity = Self::workflow_stream_command_identity(&state)?.to_string();
8615 state.workflow_stream_command_counter += 1;
8616 let wire_items = items
8617 .iter()
8618 .enumerate()
8619 .map(|(item_index, item)| {
8620 item.wire_value(Some(format!(
8621 "dw-stream:{identity}:{command_ordinal}:{item_index}"
8622 )))
8623 })
8624 .collect::<Vec<_>>();
8625 let mut directive = json!({
8626 "operation": "append",
8627 "stream_name": stream_name,
8628 "command_identity": identity,
8629 "command_ordinal": command_ordinal,
8630 "items": wire_items,
8631 });
8632 if let Some(max_pending_items) = max_pending_items {
8633 directive["max_pending_items"] = json!(max_pending_items);
8634 }
8635 let result = encode_typed_envelope(&AvroValue::Null, &state.payload_codec)?;
8636 state.commands.push(json!({
8637 "type": "record_side_effect",
8638 "result": result,
8639 "workflow_stream": directive,
8640 }));
8641 Ok(())
8642 }
8643
8644 pub fn close_workflow_stream(
8646 &self,
8647 stream_name: impl Into<String>,
8648 retention_seconds: Option<u64>,
8649 ) -> Result<()> {
8650 self.finish_workflow_stream(stream_name.into(), None, retention_seconds)
8651 }
8652
8653 pub fn error_workflow_stream(
8655 &self,
8656 stream_name: impl Into<String>,
8657 error_reason: impl Into<String>,
8658 retention_seconds: Option<u64>,
8659 ) -> Result<()> {
8660 let error_reason = error_reason.into();
8661 if error_reason.is_empty() {
8662 return Err(Error::Codec(
8663 "workflow_stream_error_invalid: error reason must not be empty".to_string(),
8664 ));
8665 }
8666 self.finish_workflow_stream(stream_name.into(), Some(error_reason), retention_seconds)
8667 }
8668
8669 fn finish_workflow_stream(
8670 &self,
8671 stream_name: String,
8672 error_reason: Option<String>,
8673 retention_seconds: Option<u64>,
8674 ) -> Result<()> {
8675 if stream_name.is_empty() {
8676 return Err(Error::Codec(
8677 "workflow_stream_name_invalid: stream name must not be empty".to_string(),
8678 ));
8679 }
8680 if retention_seconds == Some(0) {
8681 return Err(Error::Codec(
8682 "workflow_stream_retention_invalid: retention_seconds must be positive".to_string(),
8683 ));
8684 }
8685 let mut state = self
8686 .state
8687 .lock()
8688 .map_err(|_| Error::WorkflowStatePoisoned)?;
8689 let command_ordinal = state.workflow_stream_command_counter;
8690 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8691 state.workflow_stream_command_counter += 1;
8692 return match recorded {
8693 RecordedCommand::SideEffect { .. } => {
8694 state.command_cursor += 1;
8695 Ok(())
8696 }
8697 other => Err(command_mismatch(&other, "workflow stream close")),
8698 };
8699 }
8700 let identity = Self::workflow_stream_command_identity(&state)?.to_string();
8701 state.workflow_stream_command_counter += 1;
8702 let mut directive = json!({
8703 "operation": if error_reason.is_some() { "error" } else { "close" },
8704 "stream_name": stream_name,
8705 "command_identity": identity,
8706 "command_ordinal": command_ordinal,
8707 });
8708 if let Some(error_reason) = error_reason {
8709 directive["error_reason"] = json!(error_reason);
8710 }
8711 if let Some(retention_seconds) = retention_seconds {
8712 directive["retention_seconds"] = json!(retention_seconds);
8713 }
8714 let result = encode_typed_envelope(&AvroValue::Null, &state.payload_codec)?;
8715 state.commands.push(json!({
8716 "type": "record_side_effect",
8717 "result": result,
8718 "workflow_stream": directive,
8719 }));
8720 Ok(())
8721 }
8722
8723 fn workflow_stream_command_identity(state: &WorkflowState) -> Result<&str> {
8724 let identity = state.workflow_command_identity.as_str();
8725 if identity.is_empty() {
8726 return Err(Error::MissingWorkflowCommandIdentity);
8727 }
8728 Ok(identity)
8729 }
8730
8731 pub fn upsert_search_attributes(&self, update: SearchAttributeUpdate) -> Result<()> {
8737 update.validate()?;
8738 let (attributes, attribute_types) = update.into_wire_parts();
8739 let mut state = self
8740 .state
8741 .lock()
8742 .map_err(|_| Error::WorkflowStatePoisoned)?;
8743
8744 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8745 return match recorded {
8746 RecordedCommand::SearchAttributes {
8747 sequence,
8748 attributes: recorded_attributes,
8749 attribute_types: recorded_attribute_types,
8750 } => {
8751 if recorded_attributes != attributes {
8752 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8753 "search_attribute_value_mismatch",
8754 Some(sequence),
8755 Some(recorded_attributes.to_string()),
8756 Some(attributes.to_string()),
8757 "search-attribute values differ from the recorded durable command",
8758 )));
8759 }
8760 if let RecordedSnapshotValue::Known(recorded_types) = recorded_attribute_types {
8761 if recorded_types != attribute_types {
8762 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8763 "search_attribute_type_mismatch",
8764 Some(sequence),
8765 Some(json!(recorded_types).to_string()),
8766 Some(json!(attribute_types).to_string()),
8767 "search-attribute declared types differ from the recorded durable command",
8768 )));
8769 }
8770 }
8771 state.command_cursor += 1;
8772 Ok(())
8773 }
8774 other => Err(command_mismatch(&other, "search-attribute update")),
8775 };
8776 }
8777
8778 let mut command = serde_json::Map::from_iter([
8779 ("type".to_string(), json!("upsert_search_attributes")),
8780 ("attributes".to_string(), attributes),
8781 ]);
8782 if !attribute_types.is_empty() {
8783 command.insert("attribute_types".to_string(), json!(attribute_types));
8784 }
8785 state.commands.push(Value::Object(command));
8786 Ok(())
8787 }
8788
8789 pub fn uuid_v4(&self) -> Result<Uuid> {
8791 self.side_effect(Uuid::new_v4)
8792 }
8793
8794 pub fn get_version(
8797 &self,
8798 change_id: impl Into<String>,
8799 min_supported: i32,
8800 max_supported: i32,
8801 ) -> Result<i32> {
8802 let change_id = change_id.into();
8803 if change_id.trim().is_empty() {
8804 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8805 "version_change_id_invalid",
8806 None,
8807 Some("non-empty change ID".to_string()),
8808 Some(change_id),
8809 "version markers require a stable non-empty change ID",
8810 )));
8811 }
8812 if min_supported > max_supported {
8813 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8814 "version_range_invalid",
8815 None,
8816 Some("min_supported <= max_supported".to_string()),
8817 Some(format!("{min_supported}..={max_supported}")),
8818 "version marker supported range is invalid",
8819 )));
8820 }
8821
8822 let mut state = self
8823 .state
8824 .lock()
8825 .map_err(|_| Error::WorkflowStatePoisoned)?;
8826 if let Some((version, sequence)) = state.version_markers.get(&change_id).copied() {
8827 ensure_version_supported(&change_id, version, min_supported, max_supported, sequence)?;
8828 return Ok(version);
8829 }
8830
8831 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8832 return match recorded {
8833 RecordedCommand::VersionMarker {
8834 sequence,
8835 change_id: recorded_change_id,
8836 version,
8837 ..
8838 } => {
8839 if recorded_change_id != change_id {
8840 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8841 "version_change_id_mismatch",
8842 Some(sequence),
8843 Some(recorded_change_id),
8844 Some(change_id),
8845 "recorded version marker change ID differs from current workflow code",
8846 )));
8847 }
8848 ensure_version_supported(
8849 &change_id,
8850 version,
8851 min_supported,
8852 max_supported,
8853 sequence,
8854 )?;
8855 state.command_cursor += 1;
8856 state.version_markers.insert(change_id, (version, sequence));
8857 Ok(version)
8858 }
8859 other => Err(command_mismatch(
8860 &other,
8861 format!("version marker:{change_id}"),
8862 )),
8863 };
8864 }
8865
8866 let version = max_supported;
8867 state.commands.push(json!({
8868 "type": "record_version_marker",
8869 "change_id": change_id,
8870 "version": version,
8871 "min_supported": min_supported,
8872 "max_supported": max_supported,
8873 }));
8874 state.version_markers.insert(change_id, (version, 0));
8877 Ok(version)
8878 }
8879
8880 pub fn patched(&self, change_id: impl Into<String>) -> Result<bool> {
8882 Ok(self.get_version(change_id, -1, 1)? == 1)
8883 }
8884
8885 pub fn deprecate_patch(&self, change_id: impl Into<String>) -> Result<()> {
8887 self.get_version(change_id, -1, 1).map(|_| ())
8888 }
8889
8890 pub fn upsert_memo<T: Serialize>(&self, entries: T) -> Result<()> {
8895 let entries = canonical_memo_entries(AvroValue::from_serialize(&entries)?, true)?;
8896 let mut state = self
8897 .state
8898 .lock()
8899 .map_err(|_| Error::WorkflowStatePoisoned)?;
8900
8901 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8902 return match recorded {
8903 RecordedCommand::Memo {
8904 sequence,
8905 entries: recorded_entries,
8906 } => {
8907 if recorded_entries != entries {
8908 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8909 "memo_update_mismatch",
8910 Some(sequence),
8911 Some(format!("{recorded_entries:?}")),
8912 Some(format!("{entries:?}")),
8913 "recorded memo entries differ from the current workflow update",
8914 )));
8915 }
8916 state.command_cursor += 1;
8917 Ok(())
8918 }
8919 other => Err(command_mismatch(&other, "memo upsert")),
8920 };
8921 }
8922
8923 let entries_envelope = encode_typed_envelope(&entries, DEFAULT_CODEC)?;
8924 state.commands.push(json!({
8925 "type": "upsert_memo",
8926 "entries": entries_envelope,
8927 }));
8928 Ok(())
8929 }
8930
8931 pub fn start_child_workflow<T: Serialize>(
8958 &self,
8959 workflow_type: impl Into<String>,
8960 options: ChildWorkflowOptions,
8961 args: T,
8962 ) -> ChildWorkflowCall {
8963 ChildWorkflowCall {
8964 ctx: self.clone(),
8965 workflow_type: workflow_type.into(),
8966 options,
8967 args: Some(AvroValue::from_serialize(&args)),
8968 scheduled: false,
8969 matched_pending: false,
8970 parallel_group_path: Vec::new(),
8971 }
8972 }
8973
8974 pub async fn start_child_workflow_avro_value<T: Serialize>(
8975 &self,
8976 workflow_type: impl Into<String>,
8977 options: ChildWorkflowOptions,
8978 args: T,
8979 ) -> Result<ChildWorkflowAvroResult> {
8980 let mut call = self.start_child_workflow(workflow_type, options, args);
8981 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8982 }
8983
8984 fn take_commands(&self) -> Result<Vec<Value>> {
8985 let mut state = self
8986 .state
8987 .lock()
8988 .map_err(|_| Error::WorkflowStatePoisoned)?;
8989 Ok(std::mem::take(&mut state.commands))
8990 }
8991
8992 fn continue_as_new_command(&self, request: ContinueAsNewRequest) -> Result<Option<Value>> {
8993 let mut state = self
8994 .state
8995 .lock()
8996 .map_err(|_| Error::WorkflowStatePoisoned)?;
8997
8998 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8999 return Err(command_mismatch(&recorded, "continue as new"));
9000 }
9001 if state.recorded_continue_as_new_sequence.is_some() {
9002 state.continue_as_new_consumed = true;
9003 return Ok(None);
9004 }
9005
9006 let arguments = encode_typed_envelope(&request.arguments, &state.payload_codec)?;
9007 let mut command = serde_json::Map::from_iter([
9008 ("type".to_string(), json!("continue_as_new")),
9009 ("arguments".to_string(), arguments),
9010 ("queue".to_string(), json!(state.task_queue.clone())),
9011 ]);
9012 if let Some(workflow_type) = request.options.workflow_type {
9013 command.insert("workflow_type".to_string(), json!(workflow_type));
9014 }
9015 if let Some(task_queue) = request.options.task_queue {
9016 command.insert("queue".to_string(), json!(task_queue));
9017 }
9018 Ok(Some(Value::Object(command)))
9019 }
9020
9021 fn matched_recorded_pending(&self) -> Result<bool> {
9022 let state = self
9023 .state
9024 .lock()
9025 .map_err(|_| Error::WorkflowStatePoisoned)?;
9026 Ok(state.matched_recorded_pending)
9027 }
9028
9029 fn ensure_history_consumed(&self) -> Result<()> {
9030 let state = self
9031 .state
9032 .lock()
9033 .map_err(|_| Error::WorkflowStatePoisoned)?;
9034 if let Some(command) = state.recorded_commands.get(state.command_cursor) {
9035 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
9036 "recorded_commands_unconsumed",
9037 Some(command.sequence()),
9038 Some(command.shape().to_string()),
9039 Some("workflow completion".to_string()),
9040 "workflow completed before consuming all recorded durable commands",
9041 )));
9042 }
9043 if let Some(sequence) = state
9044 .recorded_continue_as_new_sequence
9045 .filter(|_| !state.continue_as_new_consumed)
9046 {
9047 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
9048 "recorded_continue_as_new_unconsumed",
9049 Some(sequence),
9050 Some("continue as new".to_string()),
9051 Some("workflow completion".to_string()),
9052 "workflow completed without consuming its recorded continue-as-new transition",
9053 )));
9054 }
9055 Ok(())
9056 }
9057}
9058
9059fn contiguous_message_stream_count(
9060 pending: &[MessageStreamMessage],
9061 cursor: u64,
9062 max_items: usize,
9063) -> usize {
9064 pending
9065 .iter()
9066 .take(max_items)
9067 .enumerate()
9068 .take_while(|(offset, message)| {
9069 u64::try_from(*offset)
9070 .ok()
9071 .and_then(|offset| cursor.checked_add(offset + 1))
9072 == Some(message.position)
9073 })
9074 .count()
9075}
9076
9077fn is_authored_command_open_event(event: &HistoryEvent) -> bool {
9078 matches!(
9079 event.event_type.as_str(),
9080 "ActivityScheduled"
9081 | "TimerScheduled"
9082 | "ChildWorkflowScheduled"
9083 | "SignalWaitOpened"
9084 | "ConditionWaitOpened"
9085 | "SearchAttributesUpserted"
9086 | "SideEffectRecorded"
9087 | "VersionMarkerRecorded"
9088 | "MemoUpserted"
9089 | "WorkflowContinuedAsNew"
9090 )
9091}
9092
9093#[derive(Debug)]
9094struct WorkflowState {
9095 workflow_id: Option<String>,
9096 run_id: Option<String>,
9097 task_queue: String,
9098 payload_codec: String,
9099 history_events: Arc<Vec<HistoryEvent>>,
9100 history_budget: WorkflowHistoryBudget,
9101 cancel_requested: bool,
9102 resume_signal: Option<ResumeSignal>,
9103 recorded_commands: Vec<RecordedCommand>,
9104 selection_markers: Vec<SelectionMarker>,
9105 selection_marker_cursor: usize,
9106 cancelled_selection_members: Vec<SelectionCancellation>,
9107 recorded_continue_as_new_sequence: Option<u64>,
9108 continue_as_new_consumed: bool,
9109 command_cursor: usize,
9110 condition_wait_occurrence_counter: u64,
9111 matched_recorded_pending: bool,
9112 version_markers: HashMap<String, (i32, u64)>,
9113 workflow_command_identity: String,
9114 workflow_stream_command_counter: u64,
9115 commands: Vec<Value>,
9116 message_stream_messages: HashMap<String, Vec<MessageStreamMessage>>,
9117 message_stream_cursors: HashMap<String, u64>,
9118 message_stream_waits: HashMap<String, u64>,
9119}
9120
9121impl WorkflowState {
9122 #[cfg(test)]
9123 fn new(
9124 history: Vec<HistoryEvent>,
9125 task_queue: String,
9126 payload_codec: String,
9127 resume_signal: Option<ResumeSignal>,
9128 ) -> Result<Self> {
9129 Self::new_with_identity(
9130 history,
9131 None,
9132 None,
9133 task_queue,
9134 payload_codec,
9135 resume_signal,
9136 )
9137 }
9138
9139 fn new_with_identity(
9140 history: Vec<HistoryEvent>,
9141 workflow_id: Option<String>,
9142 run_id: Option<String>,
9143 task_queue: String,
9144 payload_codec: String,
9145 resume_signal: Option<ResumeSignal>,
9146 ) -> Result<Self> {
9147 let recorded_commands = recorded_commands(
9148 &history,
9149 &payload_codec,
9150 WorkflowIdentity {
9151 workflow_id: workflow_id.clone(),
9152 run_id: run_id.clone(),
9153 },
9154 )?;
9155 let selection_markers = recorded_selection_markers(&history)?;
9156 let cancelled_selection_members = recorded_selection_cancellations(&history)?;
9157 let recorded_continue_as_new = history
9158 .iter()
9159 .filter(|event| event.event_type == "WorkflowContinuedAsNew")
9160 .collect::<Vec<_>>();
9161 if recorded_continue_as_new.len() > 1 {
9162 return Err(invalid_recorded_history(
9163 "duplicate_continue_as_new_transition",
9164 recorded_continue_as_new
9165 .last()
9166 .and_then(|event| durable_event_sequence(event))
9167 .unwrap_or(0),
9168 "one WorkflowContinuedAsNew event",
9169 &format!(
9170 "{} WorkflowContinuedAsNew events",
9171 recorded_continue_as_new.len()
9172 ),
9173 "workflow history records one continue-as-new transition more than once",
9174 ));
9175 }
9176 let recorded_continue_as_new_sequence = recorded_continue_as_new
9177 .first()
9178 .map(|event| {
9179 durable_event_sequence(event).ok_or_else(|| {
9180 Error::NonDeterministicReplay(ReplayFailure::new(
9181 "continue_as_new_sequence_missing",
9182 None,
9183 Some("recorded transition sequence".to_string()),
9184 Some("missing sequence".to_string()),
9185 "WorkflowContinuedAsNew history is missing its recorded sequence",
9186 ))
9187 })
9188 })
9189 .transpose()?;
9190 let mut message_stream_cursors = HashMap::new();
9191 for event in &history {
9192 if !matches!(
9193 event.event_type.as_str(),
9194 "SignalReceived" | "SignalApplied"
9195 ) || event.payload.get("signal_name").and_then(Value::as_str)
9196 != Some(MESSAGE_STREAM_SIGNAL)
9197 {
9198 continue;
9199 }
9200 let arguments = decode_signal_event_arguments(event, &payload_codec)?;
9201 if arguments.len() != 1 {
9202 continue;
9203 }
9204 let envelope = arguments[0].clone().into_json()?;
9205 let Some(envelope) = envelope.as_object() else {
9206 continue;
9207 };
9208 if envelope.get("schema").and_then(Value::as_str) != Some(MESSAGE_STREAM_CURSOR_SCHEMA)
9209 {
9210 continue;
9211 }
9212 let Some(stream_name) = envelope.get("stream_name").and_then(Value::as_str) else {
9213 continue;
9214 };
9215 let Some(through_position) = envelope.get("through_position").and_then(Value::as_u64)
9216 else {
9217 continue;
9218 };
9219 let cursor = message_stream_cursors
9220 .entry(stream_name.to_string())
9221 .or_insert(0);
9222 *cursor = (*cursor).max(through_position);
9223 }
9224 let event_count = u64::try_from(history.len()).unwrap_or(u64::MAX);
9225 let cancel_requested = history.iter().any(|event| {
9226 matches!(
9227 event.event_type.as_str(),
9228 "WorkflowCancellationRequested" | "WorkflowCancelRequested"
9229 )
9230 });
9231 Ok(Self {
9232 workflow_command_identity: String::new(),
9233 workflow_stream_command_counter: 0,
9234 workflow_id,
9235 run_id,
9236 task_queue,
9237 payload_codec,
9238 history_events: Arc::new(history),
9239 history_budget: WorkflowHistoryBudget {
9240 event_count,
9241 ..WorkflowHistoryBudget::default()
9242 },
9243 cancel_requested,
9244 resume_signal,
9245 recorded_commands,
9246 selection_markers,
9247 selection_marker_cursor: 0,
9248 cancelled_selection_members,
9249 recorded_continue_as_new_sequence,
9250 continue_as_new_consumed: false,
9251 command_cursor: 0,
9252 condition_wait_occurrence_counter: 0,
9253 matched_recorded_pending: false,
9254 version_markers: HashMap::new(),
9255 commands: Vec::new(),
9256 message_stream_messages: HashMap::new(),
9257 message_stream_cursors,
9258 message_stream_waits: HashMap::new(),
9259 })
9260 }
9261}
9262
9263enum MessageStreamDelivery {
9264 Message(MessageStreamMessage),
9265 Cursor {
9266 stream_name: String,
9267 through_position: u64,
9268 },
9269}
9270
9271fn decode_message_stream_delivery(arguments: Vec<Value>) -> Result<Option<MessageStreamDelivery>> {
9272 if arguments.len() != 1 {
9273 return Ok(None);
9274 }
9275 let envelope = arguments
9276 .into_iter()
9277 .next()
9278 .expect("one argument was checked");
9279 let Some(envelope) = envelope.as_object() else {
9280 return Ok(None);
9281 };
9282 let Some(stream_name) = envelope.get("stream_name").and_then(Value::as_str) else {
9283 return Ok(None);
9284 };
9285 if envelope.get("schema").and_then(Value::as_str) == Some(MESSAGE_STREAM_CURSOR_SCHEMA) {
9286 let Some(through_position) = envelope.get("through_position").and_then(Value::as_u64)
9287 else {
9288 return Ok(None);
9289 };
9290 return Ok(Some(MessageStreamDelivery::Cursor {
9291 stream_name: stream_name.to_string(),
9292 through_position,
9293 }));
9294 }
9295 if envelope.get("schema").and_then(Value::as_str) != Some(MESSAGE_STREAM_SCHEMA) {
9296 return Ok(None);
9297 }
9298 let Some(message_id) = envelope.get("message_id").and_then(Value::as_str) else {
9299 return Ok(None);
9300 };
9301 let Some(position) = envelope
9302 .get("position")
9303 .and_then(Value::as_u64)
9304 .filter(|value| *value > 0)
9305 else {
9306 return Ok(None);
9307 };
9308 let Some(payload_envelope) = envelope.get("payload_envelope") else {
9309 return Ok(None);
9310 };
9311 let Ok(payload_envelope) = serde_json::from_value::<PayloadEnvelope>(payload_envelope.clone())
9312 else {
9313 return Ok(None);
9314 };
9315 let decoded = decode_avro_value(&payload_envelope)?;
9316 let AvroValue::Array(values) = decoded else {
9317 return Ok(None);
9318 };
9319 Ok(Some(MessageStreamDelivery::Message(MessageStreamMessage {
9320 stream_name: stream_name.to_string(),
9321 message_id: message_id.to_string(),
9322 position,
9323 arguments: values,
9324 })))
9325}
9326
9327#[derive(Clone, Debug)]
9328enum RecordedCommand {
9329 Activity {
9330 sequence: u64,
9331 activity_type: Option<String>,
9332 options: Option<RecordedActivityOptions>,
9333 outcome: Option<ActivityOutcome>,
9334 parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9335 },
9336 Timer {
9337 sequence: u64,
9338 delay_seconds: u64,
9339 fired: bool,
9340 parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9341 },
9342 ChildWorkflow {
9343 sequence: u64,
9344 workflow_type: Option<String>,
9345 outcome: Option<ChildWorkflowOutcome>,
9346 parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9347 },
9348 SignalWait {
9349 sequence: u64,
9350 signal_name: String,
9351 value: Option<Vec<AvroValue>>,
9352 parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9353 },
9354 ConditionWait {
9355 sequence: u64,
9356 occurrence_id: String,
9357 condition_key: Option<String>,
9358 predicate_identity: String,
9359 timeout_seconds: Option<u64>,
9360 result: Option<ConditionWaitResult>,
9361 parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9362 },
9363 SearchAttributes {
9364 sequence: u64,
9365 attributes: Value,
9366 attribute_types: RecordedSnapshotValue<BTreeMap<String, String>>,
9367 },
9368 SideEffect {
9369 sequence: u64,
9370 value: AvroValue,
9371 },
9372 VersionMarker {
9373 sequence: u64,
9374 change_id: String,
9375 version: i32,
9376 },
9377 Memo {
9378 sequence: u64,
9379 entries: AvroValue,
9380 },
9381}
9382
9383#[derive(Clone, Debug, PartialEq, Eq)]
9384struct SelectionMarker {
9385 selection_group_id: String,
9386 selection_group_base_sequence: u64,
9387 selection_group_size: usize,
9388 member_key: SelectionKey,
9389 member_index: usize,
9390 member_base_sequence: u64,
9391 member_size: usize,
9392 operation_kind: String,
9393 operation_identity: String,
9394 outcome: String,
9395 resolution_event_id: String,
9396 resolution_event_type: String,
9397}
9398
9399#[derive(Clone, Debug, PartialEq, Eq)]
9400struct SelectionCancellation {
9401 selection_group_id: String,
9402 member_key: SelectionKey,
9403 member_index: usize,
9404 member_base_sequence: u64,
9405 member_size: usize,
9406 operation_kind: String,
9407 operation_identity: String,
9408}
9409
9410fn recorded_selection_markers(events: &[HistoryEvent]) -> Result<Vec<SelectionMarker>> {
9411 let mut markers: Vec<SelectionMarker> = Vec::new();
9412 for event in events
9413 .iter()
9414 .filter(|event| event.event_type == "SelectionResolved")
9415 {
9416 let payload = &event.payload;
9417 let base_sequence = required_selection_u64(payload, "selection_group_base_sequence")?;
9418 let group_size = required_selection_usize(payload, "selection_group_size")?;
9419 let member_base_sequence = required_selection_u64(payload, "member_base_sequence")?;
9420 let member_size = required_selection_usize(payload, "member_size")?;
9421 let member_index = required_selection_usize_allow_zero(payload, "member_index")?;
9422 let group_id = payload_string(payload, "selection_group_id").ok_or_else(|| {
9423 invalid_recorded_history(
9424 "selection_marker_invalid",
9425 base_sequence,
9426 "non-empty selection_group_id",
9427 &payload.to_string(),
9428 "selection winner history is missing its durable group identity",
9429 )
9430 })?;
9431 let expected_group_id = format!("select-calls:{base_sequence}:{group_size}");
9432 if group_id != expected_group_id {
9433 return Err(invalid_recorded_history(
9434 "selection_marker_invalid",
9435 base_sequence,
9436 &expected_group_id,
9437 &group_id,
9438 "selection winner history contains an incompatible group identity",
9439 ));
9440 }
9441 let group_end = base_sequence
9442 .checked_add(u64::try_from(group_size).unwrap_or(u64::MAX))
9443 .unwrap_or(u64::MAX);
9444 let member_end = member_base_sequence
9445 .checked_add(u64::try_from(member_size).unwrap_or(u64::MAX))
9446 .unwrap_or(u64::MAX);
9447 if member_index >= group_size
9448 || member_base_sequence < base_sequence
9449 || member_end > group_end
9450 {
9451 return Err(invalid_recorded_history(
9452 "selection_marker_invalid",
9453 base_sequence,
9454 "winner member within selection group bounds",
9455 &payload.to_string(),
9456 "selection winner history contains an invalid member range",
9457 ));
9458 }
9459 let operation_kind = payload_string(payload, "operation_kind").ok_or_else(|| {
9460 invalid_recorded_history(
9461 "selection_marker_invalid",
9462 base_sequence,
9463 "selection operation kind",
9464 &payload.to_string(),
9465 "selection winner history is missing its operation kind",
9466 )
9467 })?;
9468 if !matches!(
9469 operation_kind.as_str(),
9470 "activity" | "child" | "timer" | "signal" | "condition" | "group"
9471 ) {
9472 return Err(invalid_recorded_history(
9473 "selection_marker_invalid",
9474 base_sequence,
9475 "activity, child, timer, signal, condition, or group",
9476 &operation_kind,
9477 "selection winner history contains an unsupported operation kind",
9478 ));
9479 }
9480 let operation_identity =
9481 payload_string(payload, "operation_identity").ok_or_else(|| {
9482 invalid_recorded_history(
9483 "selection_marker_invalid",
9484 base_sequence,
9485 "non-empty operation identity",
9486 &payload.to_string(),
9487 "selection winner history is missing its durable operation identity",
9488 )
9489 })?;
9490 let outcome = payload_string(payload, "outcome").ok_or_else(|| {
9491 invalid_recorded_history(
9492 "selection_marker_invalid",
9493 base_sequence,
9494 "completed or failed selection outcome",
9495 &payload.to_string(),
9496 "selection winner history is missing its outcome",
9497 )
9498 })?;
9499 if !matches!(outcome.as_str(), "completed" | "failed") {
9500 return Err(invalid_recorded_history(
9501 "selection_marker_invalid",
9502 base_sequence,
9503 "completed or failed selection outcome",
9504 &outcome,
9505 "selection winner history contains an unsupported outcome",
9506 ));
9507 }
9508 let marker = SelectionMarker {
9509 selection_group_id: group_id,
9510 selection_group_base_sequence: base_sequence,
9511 selection_group_size: group_size,
9512 member_key: selection_key_from_value(payload.get("member_key"), base_sequence)?,
9513 member_index,
9514 member_base_sequence,
9515 member_size,
9516 operation_kind,
9517 operation_identity,
9518 outcome,
9519 resolution_event_id: payload_string(payload, "resolution_event_id").ok_or_else(
9520 || {
9521 invalid_recorded_history(
9522 "selection_marker_invalid",
9523 base_sequence,
9524 "durable resolution_event_id",
9525 &payload.to_string(),
9526 "selection winner history is missing its terminal event identity",
9527 )
9528 },
9529 )?,
9530 resolution_event_type: payload_string(payload, "resolution_event_type").ok_or_else(
9531 || {
9532 invalid_recorded_history(
9533 "selection_marker_invalid",
9534 base_sequence,
9535 "durable resolution_event_type",
9536 &payload.to_string(),
9537 "selection winner history is missing its terminal event type",
9538 )
9539 },
9540 )?,
9541 };
9542 if let Some(existing) = markers
9543 .iter()
9544 .find(|existing| existing.selection_group_id == marker.selection_group_id)
9545 {
9546 if existing != &marker {
9547 return Err(invalid_recorded_history(
9548 "selection_marker_conflict",
9549 base_sequence,
9550 &format!("one winner for {}", marker.selection_group_id),
9551 &payload.to_string(),
9552 "selection history records conflicting winners for one durable group",
9553 ));
9554 }
9555 continue;
9556 }
9557 markers.push(marker);
9558 }
9559 Ok(markers)
9560}
9561
9562fn recorded_selection_cancellations(events: &[HistoryEvent]) -> Result<Vec<SelectionCancellation>> {
9563 let mut cancelled: Vec<SelectionCancellation> = Vec::new();
9564 for event in events
9565 .iter()
9566 .filter(|event| event.event_type == "SelectionOperationCancelled")
9567 {
9568 let group_id = payload_string(&event.payload, "selection_group_id").ok_or_else(|| {
9569 invalid_recorded_history(
9570 "selection_cancellation_invalid",
9571 0,
9572 "non-empty selection_group_id",
9573 &event.payload.to_string(),
9574 "selection cancellation history is missing its group identity",
9575 )
9576 })?;
9577 let member_base_sequence = required_selection_u64(&event.payload, "member_base_sequence")?;
9578 let marker = SelectionCancellation {
9579 selection_group_id: group_id,
9580 member_key: selection_key_from_value(
9581 event.payload.get("member_key"),
9582 member_base_sequence,
9583 )?,
9584 member_index: required_selection_usize_allow_zero(&event.payload, "member_index")?,
9585 member_base_sequence,
9586 member_size: required_selection_usize(&event.payload, "member_size")?,
9587 operation_kind: payload_string(&event.payload, "operation_kind").ok_or_else(|| {
9588 invalid_recorded_history(
9589 "selection_cancellation_invalid",
9590 member_base_sequence,
9591 "selection operation kind",
9592 &event.payload.to_string(),
9593 "selection cancellation is missing its operation kind",
9594 )
9595 })?,
9596 operation_identity: payload_string(&event.payload, "operation_identity").ok_or_else(
9597 || {
9598 invalid_recorded_history(
9599 "selection_cancellation_invalid",
9600 member_base_sequence,
9601 "selection operation identity",
9602 &event.payload.to_string(),
9603 "selection cancellation is missing its operation identity",
9604 )
9605 },
9606 )?,
9607 };
9608 if let Some(existing) = cancelled.iter().find(|recorded| {
9609 recorded.selection_group_id == marker.selection_group_id
9610 && recorded.member_base_sequence == marker.member_base_sequence
9611 }) {
9612 if existing != &marker {
9613 return Err(invalid_recorded_history(
9614 "selection_cancellation_conflict",
9615 member_base_sequence,
9616 "one stable SelectionOperationCancelled marker",
9617 &event.payload.to_string(),
9618 "selection cancellation history contains conflicting member metadata",
9619 ));
9620 }
9621 } else {
9622 cancelled.push(marker);
9623 }
9624 }
9625 Ok(cancelled)
9626}
9627
9628fn required_selection_u64(payload: &Value, field: &str) -> Result<u64> {
9629 payload
9630 .get(field)
9631 .and_then(value_as_u64)
9632 .filter(|value| *value > 0)
9633 .ok_or_else(|| {
9634 invalid_recorded_history(
9635 "selection_marker_invalid",
9636 0,
9637 &format!("positive integer {field}"),
9638 &payload.to_string(),
9639 "selection history contains invalid durable identity metadata",
9640 )
9641 })
9642}
9643
9644fn required_selection_usize(payload: &Value, field: &str) -> Result<usize> {
9645 required_selection_usize_allow_zero(payload, field).and_then(|value| {
9646 if value > 0 {
9647 Ok(value)
9648 } else {
9649 Err(invalid_recorded_history(
9650 "selection_marker_invalid",
9651 0,
9652 &format!("positive integer {field}"),
9653 &payload.to_string(),
9654 "selection history contains invalid durable identity metadata",
9655 ))
9656 }
9657 })
9658}
9659
9660fn required_selection_usize_allow_zero(payload: &Value, field: &str) -> Result<usize> {
9661 payload
9662 .get(field)
9663 .and_then(value_as_u64)
9664 .and_then(|value| usize::try_from(value).ok())
9665 .ok_or_else(|| {
9666 invalid_recorded_history(
9667 "selection_marker_invalid",
9668 0,
9669 &format!("non-negative integer {field}"),
9670 &payload.to_string(),
9671 "selection history contains invalid durable identity metadata",
9672 )
9673 })
9674}
9675
9676#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
9677struct RecordedActivityOptions {
9678 task_queue: RecordedSnapshotValue<Option<String>>,
9679 execution_mode: RecordedSnapshotValue<Option<String>>,
9680 retry_policy: ActivityRetrySnapshot,
9681}
9682
9683#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
9684enum RecordedSnapshotValue<T> {
9685 Unknown,
9687 Known(T),
9688}
9689
9690impl<T: PartialEq> RecordedSnapshotValue<T> {
9691 fn matches_current(&self, current: &Self) -> bool {
9692 match self {
9693 Self::Unknown => true,
9694 Self::Known(recorded) => matches!(current, Self::Known(value) if value == recorded),
9695 }
9696 }
9697}
9698
9699#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
9700struct ActivityRetrySnapshot {
9701 snapshot_version: RecordedSnapshotValue<Option<u64>>,
9702 max_attempts: RecordedSnapshotValue<Option<u64>>,
9703 backoff_seconds: RecordedSnapshotValue<Vec<u64>>,
9704 start_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
9705 schedule_to_start_timeout: RecordedSnapshotValue<Option<u64>>,
9706 schedule_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
9707 heartbeat_timeout: RecordedSnapshotValue<Option<u64>>,
9708 non_retryable_error_types: RecordedSnapshotValue<Vec<String>>,
9709}
9710
9711impl ActivityRetrySnapshot {
9712 fn matches_current(&self, current: &Self) -> bool {
9713 self.snapshot_version
9714 .matches_current(¤t.snapshot_version)
9715 && self.max_attempts.matches_current(¤t.max_attempts)
9716 && self
9717 .backoff_seconds
9718 .matches_current(¤t.backoff_seconds)
9719 && self
9720 .start_to_close_timeout
9721 .matches_current(¤t.start_to_close_timeout)
9722 && self
9723 .schedule_to_start_timeout
9724 .matches_current(¤t.schedule_to_start_timeout)
9725 && self
9726 .schedule_to_close_timeout
9727 .matches_current(¤t.schedule_to_close_timeout)
9728 && self
9729 .heartbeat_timeout
9730 .matches_current(¤t.heartbeat_timeout)
9731 && self
9732 .non_retryable_error_types
9733 .matches_current(¤t.non_retryable_error_types)
9734 }
9735}
9736
9737fn recorded_optional_u64(
9738 object: Option<&serde_json::Map<String, Value>>,
9739 field: &str,
9740) -> RecordedSnapshotValue<Option<u64>> {
9741 match object.and_then(|object| object.get(field)) {
9742 None => RecordedSnapshotValue::Unknown,
9743 Some(Value::Null) => RecordedSnapshotValue::Known(None),
9744 Some(value) => RecordedSnapshotValue::Known(value_as_u64(value)),
9745 }
9746}
9747
9748fn recorded_optional_string(
9749 object: &serde_json::Map<String, Value>,
9750 field: &str,
9751) -> RecordedSnapshotValue<Option<String>> {
9752 match object.get(field) {
9753 None => RecordedSnapshotValue::Unknown,
9754 Some(Value::Null) => RecordedSnapshotValue::Known(None),
9755 Some(value) => RecordedSnapshotValue::Known(value.as_str().map(str::to_string)),
9756 }
9757}
9758
9759fn recorded_activity_retry_snapshot(policy: Option<&Value>) -> ActivityRetrySnapshot {
9760 let policy = policy.and_then(Value::as_object);
9761 let backoff_seconds = policy
9762 .and_then(|policy| policy.get("backoff_seconds"))
9763 .and_then(Value::as_array)
9764 .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
9765 .map_or(RecordedSnapshotValue::Unknown, RecordedSnapshotValue::Known);
9766 let mut non_retryable_error_types = Vec::new();
9767 for error_type in policy
9768 .and_then(|policy| policy.get("non_retryable_error_types"))
9769 .and_then(Value::as_array)
9770 .into_iter()
9771 .flatten()
9772 .filter_map(Value::as_str)
9773 .map(str::trim)
9774 .filter(|error_type| !error_type.is_empty())
9775 {
9776 if !non_retryable_error_types
9777 .iter()
9778 .any(|recorded| recorded == error_type)
9779 {
9780 non_retryable_error_types.push(error_type.to_string());
9781 }
9782 }
9783
9784 ActivityRetrySnapshot {
9785 snapshot_version: recorded_optional_u64(policy, "snapshot_version"),
9786 max_attempts: recorded_optional_u64(policy, "max_attempts"),
9787 backoff_seconds,
9788 start_to_close_timeout: recorded_optional_u64(policy, "start_to_close_timeout"),
9789 schedule_to_start_timeout: recorded_optional_u64(policy, "schedule_to_start_timeout"),
9790 schedule_to_close_timeout: recorded_optional_u64(policy, "schedule_to_close_timeout"),
9791 heartbeat_timeout: recorded_optional_u64(policy, "heartbeat_timeout"),
9792 non_retryable_error_types: if policy
9793 .is_some_and(|policy| policy.contains_key("non_retryable_error_types"))
9794 {
9795 RecordedSnapshotValue::Known(non_retryable_error_types)
9796 } else {
9797 RecordedSnapshotValue::Unknown
9798 },
9799 }
9800}
9801
9802fn current_activity_retry_snapshot(options: &ValidatedActivityOptions) -> ActivityRetrySnapshot {
9803 let policy = options.retry_policy.as_ref();
9804 let max_attempts = match policy.and_then(|policy| policy.get("max_attempts")) {
9805 Some(Value::Null) => None,
9806 Some(value) => value_as_u64(value),
9807 None => Some(1),
9808 };
9809 let backoff_seconds = policy
9810 .and_then(|policy| policy.get("backoff_seconds"))
9811 .and_then(Value::as_array)
9812 .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
9813 .unwrap_or_default();
9814 let non_retryable_error_types = policy
9815 .and_then(|policy| policy.get("non_retryable_error_types"))
9816 .and_then(Value::as_array)
9817 .into_iter()
9818 .flatten()
9819 .filter_map(Value::as_str)
9820 .map(str::to_string)
9821 .collect();
9822
9823 ActivityRetrySnapshot {
9824 snapshot_version: RecordedSnapshotValue::Known(Some(1)),
9825 max_attempts: RecordedSnapshotValue::Known(max_attempts),
9826 backoff_seconds: RecordedSnapshotValue::Known(backoff_seconds),
9827 start_to_close_timeout: RecordedSnapshotValue::Known(options.start_to_close_timeout),
9828 schedule_to_start_timeout: RecordedSnapshotValue::Known(options.schedule_to_start_timeout),
9829 schedule_to_close_timeout: RecordedSnapshotValue::Known(options.schedule_to_close_timeout),
9830 heartbeat_timeout: RecordedSnapshotValue::Known(options.heartbeat_timeout),
9831 non_retryable_error_types: RecordedSnapshotValue::Known(non_retryable_error_types),
9832 }
9833}
9834
9835fn activity_options_description(options: &RecordedActivityOptions) -> String {
9836 serde_json::to_string(options).unwrap_or_else(|_| format!("{options:?}"))
9837}
9838
9839impl RecordedCommand {
9840 fn sequence(&self) -> u64 {
9841 match self {
9842 Self::Activity { sequence, .. }
9843 | Self::Timer { sequence, .. }
9844 | Self::ChildWorkflow { sequence, .. }
9845 | Self::SignalWait { sequence, .. }
9846 | Self::ConditionWait { sequence, .. }
9847 | Self::SearchAttributes { sequence, .. }
9848 | Self::SideEffect { sequence, .. }
9849 | Self::VersionMarker { sequence, .. }
9850 | Self::Memo { sequence, .. } => *sequence,
9851 }
9852 }
9853
9854 fn shape(&self) -> &'static str {
9855 match self {
9856 Self::Activity { .. } => "activity",
9857 Self::Timer { .. } => "timer",
9858 Self::ChildWorkflow { .. } => "child workflow",
9859 Self::SignalWait { .. } => "signal wait",
9860 Self::ConditionWait { .. } => "condition wait",
9861 Self::SearchAttributes { .. } => "search-attribute update",
9862 Self::SideEffect { .. } => "side effect",
9863 Self::VersionMarker { .. } => "version marker",
9864 Self::Memo { .. } => "memo upsert",
9865 }
9866 }
9867}
9868
9869fn ensure_version_supported(
9870 change_id: &str,
9871 version: i32,
9872 min_supported: i32,
9873 max_supported: i32,
9874 sequence: u64,
9875) -> Result<()> {
9876 if (min_supported..=max_supported).contains(&version) {
9877 return Ok(());
9878 }
9879 Err(Error::NonDeterministicReplay(ReplayFailure::new(
9880 "version_marker_incompatible_range",
9881 (sequence != 0).then_some(sequence),
9882 Some(format!("{min_supported}..={max_supported}")),
9883 Some(format!("{change_id}:{version}")),
9884 "recorded workflow version is outside the range supported by current code",
9885 )))
9886}
9887
9888#[derive(Clone, Debug)]
9889struct ResumeSignal {
9890 signal_name: String,
9891 arguments: Vec<AvroValue>,
9892}
9893
9894const MAX_PARALLEL_OPERATIONS: usize = 1000;
9895
9896fn parallel_group_prefix(kind: &str) -> &'static str {
9897 match kind {
9898 "activity" => "parallel-activities",
9899 "child" => "parallel-children",
9900 "timer" => "parallel-timers",
9901 _ => "parallel-calls",
9902 }
9903}
9904
9905fn parallel_group_entry(
9906 base_sequence: u64,
9907 size: usize,
9908 index: usize,
9909 kind: &str,
9910) -> ParallelGroupMetadata {
9911 ParallelGroupMetadata {
9912 parallel_group_id: format!("{}:{base_sequence}:{size}", parallel_group_prefix(kind)),
9913 parallel_group_kind: kind.to_string(),
9914 parallel_group_base_sequence: base_sequence,
9915 parallel_group_size: size,
9916 parallel_group_index: index,
9917 parallel_group_mode: None,
9918 selection_member_key: None,
9919 selection_member_index: None,
9920 selection_member_base_sequence: None,
9921 selection_member_size: None,
9922 selection_member_kind: None,
9923 }
9924}
9925
9926struct SelectionMemberMetadata {
9927 key: SelectionKey,
9928 index: usize,
9929 base_sequence: u64,
9930 size: usize,
9931 kind: String,
9932}
9933
9934fn selection_group_entry(
9935 base_sequence: u64,
9936 size: usize,
9937 index: usize,
9938 kind: &str,
9939 member: &SelectionMemberMetadata,
9940) -> ParallelGroupMetadata {
9941 ParallelGroupMetadata {
9942 parallel_group_id: format!("select-calls:{base_sequence}:{size}"),
9943 parallel_group_kind: kind.to_string(),
9944 parallel_group_base_sequence: base_sequence,
9945 parallel_group_size: size,
9946 parallel_group_index: index,
9947 parallel_group_mode: Some("select".to_string()),
9948 selection_member_key: Some(member.key.clone()),
9949 selection_member_index: Some(member.index),
9950 selection_member_base_sequence: Some(member.base_sequence),
9951 selection_member_size: Some(member.size),
9952 selection_member_kind: Some(member.kind.clone()),
9953 }
9954}
9955
9956fn apply_parallel_group_path(
9957 command: &mut serde_json::Map<String, Value>,
9958 path: &[ParallelGroupMetadata],
9959) {
9960 let Some(inner) = path.last() else {
9961 return;
9962 };
9963 command.insert(
9964 "parallel_group_id".to_string(),
9965 json!(inner.parallel_group_id),
9966 );
9967 command.insert(
9968 "parallel_group_kind".to_string(),
9969 json!(inner.parallel_group_kind),
9970 );
9971 command.insert(
9972 "parallel_group_base_sequence".to_string(),
9973 json!(inner.parallel_group_base_sequence),
9974 );
9975 command.insert(
9976 "parallel_group_size".to_string(),
9977 json!(inner.parallel_group_size),
9978 );
9979 command.insert(
9980 "parallel_group_index".to_string(),
9981 json!(inner.parallel_group_index),
9982 );
9983 if let Some(mode) = &inner.parallel_group_mode {
9984 command.insert("parallel_group_mode".to_string(), json!(mode));
9985 }
9986 if let Some(key) = &inner.selection_member_key {
9987 command.insert("selection_member_key".to_string(), json!(key));
9988 }
9989 if let Some(index) = inner.selection_member_index {
9990 command.insert("selection_member_index".to_string(), json!(index));
9991 }
9992 if let Some(base_sequence) = inner.selection_member_base_sequence {
9993 command.insert(
9994 "selection_member_base_sequence".to_string(),
9995 json!(base_sequence),
9996 );
9997 }
9998 if let Some(size) = inner.selection_member_size {
9999 command.insert("selection_member_size".to_string(), json!(size));
10000 }
10001 if let Some(kind) = &inner.selection_member_kind {
10002 command.insert("selection_member_kind".to_string(), json!(kind));
10003 }
10004 command.insert("parallel_group_path".to_string(), json!(path));
10005}
10006
10007fn ensure_parallel_path_matches(
10008 sequence: u64,
10009 recorded: Option<&[ParallelGroupMetadata]>,
10010 expected: &[ParallelGroupMetadata],
10011) -> Result<()> {
10012 match (recorded, expected.is_empty()) {
10013 (None, true) => Ok(()),
10014 (Some(recorded), false) if recorded == expected => Ok(()),
10015 (None, false) => Err(invalid_recorded_history(
10016 "parallel_group_metadata_missing",
10017 sequence,
10018 &serde_json::to_string(expected).unwrap_or_default(),
10019 "<missing>",
10020 "recorded parallel member is missing its durable group path",
10021 )),
10022 (Some(recorded), true) => Err(invalid_recorded_history(
10023 "parallel_group_shape_mismatch",
10024 sequence,
10025 "sequential command",
10026 &serde_json::to_string(recorded).unwrap_or_default(),
10027 "recorded command belonged to a parallel group but current code schedules it sequentially",
10028 )),
10029 (Some(recorded), false) => Err(invalid_recorded_history(
10030 "parallel_group_shape_mismatch",
10031 sequence,
10032 &serde_json::to_string(recorded).unwrap_or_default(),
10033 &serde_json::to_string(expected).unwrap_or_default(),
10034 "recorded parallel-group identity or path changed during replay",
10035 )),
10036 }
10037}
10038
10039#[derive(Clone, Debug)]
10040enum ParallelShape {
10041 Leaf,
10042 Group(Vec<ParallelShape>),
10043}
10044
10045struct ParallelDescriptor {
10046 operation: ParallelOperation,
10047 offset: usize,
10048 member_path: Vec<usize>,
10049 group_path: Vec<ParallelGroupMetadata>,
10050}
10051
10052fn parallel_leaf_count(operations: &[ParallelOperation]) -> usize {
10053 operations
10054 .iter()
10055 .map(|operation| match operation {
10056 ParallelOperation::Group(children) => parallel_leaf_count(children),
10057 _ => 1,
10058 })
10059 .sum()
10060}
10061
10062fn parallel_operation_kind(operation: &ParallelOperation) -> Option<&'static str> {
10063 match operation {
10064 ParallelOperation::Activity { .. } => Some("activity"),
10065 ParallelOperation::ChildWorkflow { .. } => Some("child"),
10066 ParallelOperation::Timer(_) => Some("timer"),
10067 ParallelOperation::Signal(_) => Some("signal"),
10068 ParallelOperation::Condition { .. } => Some("condition"),
10069 ParallelOperation::Group(children) => parallel_group_kind(children),
10070 }
10071}
10072
10073fn parallel_group_kind(operations: &[ParallelOperation]) -> Option<&'static str> {
10074 let mut kind = None;
10075 for operation in operations {
10076 let Some(operation_kind) = parallel_operation_kind(operation) else {
10077 continue;
10078 };
10079 match kind {
10080 None => kind = Some(operation_kind),
10081 Some(current) if current == operation_kind => {}
10082 Some(_) => return Some("mixed"),
10083 }
10084 }
10085 kind
10086}
10087
10088fn validate_parallel_operations(
10089 operations: &[ParallelOperation],
10090 member_path: &mut Vec<usize>,
10091 root: bool,
10092) -> Result<()> {
10093 let leaves = parallel_leaf_count(operations);
10094 if leaves > MAX_PARALLEL_OPERATIONS {
10095 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10096 reason: "fan_out_limit_exceeded",
10097 member_path: member_path.clone(),
10098 message: format!(
10099 "group contains {leaves} durable leaves; the limit is {MAX_PARALLEL_OPERATIONS}"
10100 ),
10101 }));
10102 }
10103 if !root && operations.is_empty() {
10104 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10105 reason: "nested_group_empty",
10106 member_path: member_path.clone(),
10107 message: "a nested group must contain at least one durable leaf".to_string(),
10108 }));
10109 }
10110
10111 for (index, operation) in operations.iter().enumerate() {
10112 member_path.push(index);
10113 match operation {
10114 ParallelOperation::Activity {
10115 options, arguments, ..
10116 } => {
10117 options
10118 .validate()
10119 .map_err(|error| Error::InvalidActivityOptions(error))?;
10120 if let Err(error) = arguments {
10121 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10122 reason: "arguments_invalid",
10123 member_path: member_path.clone(),
10124 message: error.to_string(),
10125 }));
10126 }
10127 }
10128 ParallelOperation::ChildWorkflow {
10129 options, arguments, ..
10130 } => {
10131 validate_parallel_child_options(options)?;
10132 if let Err(error) = arguments {
10133 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10134 reason: "arguments_invalid",
10135 member_path: member_path.clone(),
10136 message: error.to_string(),
10137 }));
10138 }
10139 }
10140 ParallelOperation::Timer(duration)
10141 if duration.as_secs() == u64::MAX && duration.subsec_nanos() > 0 =>
10142 {
10143 return Err(Error::TimerDurationOverflow);
10144 }
10145 ParallelOperation::Timer(_) => {}
10146 ParallelOperation::Signal(signal_name) => {
10147 validate_user_signal_name(signal_name)?;
10148 if signal_name.trim().is_empty() {
10149 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10150 reason: "signal_name_empty",
10151 member_path: member_path.clone(),
10152 message: "signal wait name must not be empty".to_string(),
10153 }));
10154 }
10155 }
10156 ParallelOperation::Condition { options, .. } => {
10157 options.validate()?;
10158 }
10159 ParallelOperation::Group(children) => {
10160 validate_parallel_operations(children, member_path, false)?;
10161 }
10162 }
10163 member_path.pop();
10164 }
10165 Ok(())
10166}
10167
10168fn validate_parallel_child_options(options: &ChildWorkflowOptions) -> Result<()> {
10169 if options.task_queue.trim().is_empty() {
10170 return Err(Error::InvalidChildWorkflowOptions(
10171 "task_queue must not be empty".to_string(),
10172 ));
10173 }
10174 for (name, value) in [
10175 (
10176 "execution_timeout_seconds",
10177 options.execution_timeout_seconds,
10178 ),
10179 ("run_timeout_seconds", options.run_timeout_seconds),
10180 ] {
10181 if value == Some(0) {
10182 return Err(Error::InvalidChildWorkflowOptions(format!(
10183 "{name} must be at least 1"
10184 )));
10185 }
10186 }
10187 if options
10188 .retry_policy
10189 .as_ref()
10190 .is_some_and(|policy| policy.max_attempts == Some(0))
10191 {
10192 return Err(Error::InvalidChildWorkflowOptions(
10193 "retry_policy.max_attempts must be at least 1".to_string(),
10194 ));
10195 }
10196 Ok(())
10197}
10198
10199fn parallel_shape(operations: &[ParallelOperation]) -> ParallelShape {
10200 ParallelShape::Group(
10201 operations
10202 .iter()
10203 .map(|operation| match operation {
10204 ParallelOperation::Group(children) => parallel_shape(children),
10205 _ => ParallelShape::Leaf,
10206 })
10207 .collect(),
10208 )
10209}
10210
10211fn parallel_descriptors(
10212 operations: Vec<ParallelOperation>,
10213 base_sequence: u64,
10214) -> Result<Vec<ParallelDescriptor>> {
10215 let size = parallel_leaf_count(&operations);
10216 let kind = parallel_group_kind(&operations).unwrap_or("activity");
10217 let mut descriptors = Vec::with_capacity(size);
10218 let mut cursor = 0;
10219
10220 for (index, operation) in operations.into_iter().enumerate() {
10221 match operation {
10222 ParallelOperation::Group(children) => {
10223 let child_base = base_sequence
10224 .checked_add(u64::try_from(cursor).unwrap_or(u64::MAX))
10225 .ok_or(Error::TimerDurationOverflow)?;
10226 for mut descriptor in parallel_descriptors(children, child_base)? {
10227 let outer_index = cursor + descriptor.offset;
10228 descriptor.group_path.insert(
10229 0,
10230 parallel_group_entry(base_sequence, size, outer_index, kind),
10231 );
10232 descriptor.member_path.insert(0, index);
10233 descriptor.offset = outer_index;
10234 descriptors.push(descriptor);
10235 }
10236 cursor = descriptors.len();
10237 }
10238 operation => {
10239 descriptors.push(ParallelDescriptor {
10240 operation,
10241 offset: cursor,
10242 member_path: vec![index],
10243 group_path: vec![parallel_group_entry(base_sequence, size, cursor, kind)],
10244 });
10245 cursor += 1;
10246 }
10247 }
10248 }
10249 Ok(descriptors)
10250}
10251
10252enum ParallelLeafCall {
10253 Activity(ActivityCall),
10254 ChildWorkflow(ChildWorkflowCall),
10255 Timer(TimerCall),
10256 Signal(SignalCall),
10257 Condition(ConditionWaitCall),
10258}
10259
10260fn parallel_leaf_call(
10261 ctx: &WorkflowContext,
10262 operation: ParallelOperation,
10263 parallel_group_path: Vec<ParallelGroupMetadata>,
10264) -> ParallelLeafCall {
10265 match operation {
10266 ParallelOperation::Activity {
10267 activity_type,
10268 options,
10269 arguments,
10270 } => ParallelLeafCall::Activity(ActivityCall {
10271 ctx: ctx.clone(),
10272 activity_type,
10273 options,
10274 args: Some(arguments),
10275 scheduled: false,
10276 parallel_group_path,
10277 }),
10278 ParallelOperation::ChildWorkflow {
10279 workflow_type,
10280 options,
10281 arguments,
10282 } => ParallelLeafCall::ChildWorkflow(ChildWorkflowCall {
10283 ctx: ctx.clone(),
10284 workflow_type,
10285 options,
10286 args: Some(arguments),
10287 scheduled: false,
10288 matched_pending: false,
10289 parallel_group_path,
10290 }),
10291 ParallelOperation::Timer(duration) => {
10292 let delay_seconds = duration
10293 .as_secs()
10294 .checked_add(u64::from(duration.subsec_nanos() > 0));
10295 ParallelLeafCall::Timer(TimerCall {
10296 ctx: ctx.clone(),
10297 delay_seconds,
10298 scheduled: false,
10299 matched_pending: false,
10300 parallel_group_path,
10301 })
10302 }
10303 ParallelOperation::Signal(signal_name) => ParallelLeafCall::Signal(SignalCall {
10304 ctx: ctx.clone(),
10305 signal_name,
10306 runtime_reserved_allowed: false,
10307 opened_wait: false,
10308 matched_pending: false,
10309 parallel_group_path,
10310 }),
10311 ParallelOperation::Condition { options, predicate } => {
10312 ParallelLeafCall::Condition(ConditionWaitCall {
10313 ctx: ctx.clone(),
10314 options,
10315 predicate,
10316 occurrence_id: None,
10317 opened_wait: false,
10318 parallel_group_path,
10319 })
10320 }
10321 ParallelOperation::Group(_) => {
10322 unreachable!("parallel descriptors contain only durable leaves")
10323 }
10324 }
10325}
10326
10327impl ParallelLeafCall {
10328 fn poll_avro_value(&mut self, cx: &mut TaskContext<'_>) -> Poll<Result<ParallelAvroResult>> {
10329 match self {
10330 Self::Activity(call) => Pin::new(call)
10331 .poll_avro_value(cx)
10332 .map_ok(ParallelAvroResult::Activity),
10333 Self::ChildWorkflow(call) => Pin::new(call)
10334 .poll_avro_value(cx)
10335 .map_ok(ParallelAvroResult::ChildWorkflow),
10336 Self::Timer(call) => Pin::new(call)
10337 .poll(cx)
10338 .map_ok(|()| ParallelAvroResult::Timer),
10339 Self::Signal(call) => Pin::new(call)
10340 .poll_avro_value(cx)
10341 .map_ok(ParallelAvroResult::Signal),
10342 Self::Condition(call) => Pin::new(call)
10343 .poll(cx)
10344 .map_ok(ParallelAvroResult::Condition),
10345 }
10346 }
10347}
10348
10349struct ParallelLeaf {
10350 call: ParallelLeafCall,
10351 member_path: Vec<usize>,
10352 group_path: Vec<ParallelGroupMetadata>,
10353 result: Option<ParallelAvroResult>,
10354}
10355
10356pub struct ParallelCall {
10358 ctx: WorkflowContext,
10359 operations: Option<Vec<ParallelOperation>>,
10360 shape: Option<ParallelShape>,
10361 leaves: Vec<ParallelLeaf>,
10362}
10363
10364impl ParallelCall {
10365 fn new(ctx: WorkflowContext, operations: Vec<ParallelOperation>) -> Self {
10366 Self {
10367 ctx,
10368 operations: Some(operations),
10369 shape: None,
10370 leaves: Vec::new(),
10371 }
10372 }
10373
10374 fn initialize(&mut self) -> Result<()> {
10375 let operations = self.operations.take().unwrap_or_default();
10376 validate_parallel_operations(&operations, &mut Vec::new(), true)?;
10377 self.shape = Some(parallel_shape(&operations));
10378 if operations.is_empty() {
10379 return Ok(());
10380 }
10381
10382 let base_sequence = {
10383 let state = self
10384 .ctx
10385 .state
10386 .lock()
10387 .map_err(|_| Error::WorkflowStatePoisoned)?;
10388 if let Some(recorded) = state.recorded_commands.get(state.command_cursor) {
10389 recorded.sequence()
10390 } else {
10391 let last = state
10392 .recorded_commands
10393 .last()
10394 .map(RecordedCommand::sequence)
10395 .unwrap_or(0);
10396 last.checked_add(u64::try_from(state.commands.len()).unwrap_or(u64::MAX))
10397 .and_then(|sequence| sequence.checked_add(1))
10398 .ok_or_else(|| {
10399 Error::InvalidParallelGroup(ParallelGroupError {
10400 reason: "sequence_overflow",
10401 member_path: Vec::new(),
10402 message: "parallel group sequence identity overflowed u64".to_string(),
10403 })
10404 })?
10405 }
10406 };
10407
10408 self.leaves = parallel_descriptors(operations, base_sequence)?
10409 .into_iter()
10410 .map(|descriptor| {
10411 let call = parallel_leaf_call(
10412 &self.ctx,
10413 descriptor.operation,
10414 descriptor.group_path.clone(),
10415 );
10416 ParallelLeaf {
10417 call,
10418 member_path: descriptor.member_path,
10419 group_path: descriptor.group_path,
10420 result: None,
10421 }
10422 })
10423 .collect();
10424 Ok(())
10425 }
10426
10427 fn poll_avro_value(
10428 mut self: Pin<&mut Self>,
10429 cx: &mut TaskContext<'_>,
10430 ) -> Poll<Result<Vec<ParallelAvroResult>>> {
10431 if self.operations.is_some() {
10432 if let Err(error) = self.initialize() {
10433 return Poll::Ready(Err(error));
10434 }
10435 }
10436 if self.leaves.is_empty() {
10437 return Poll::Ready(Ok(Vec::new()));
10438 }
10439
10440 let mut failures = Vec::new();
10441 let mut pending = false;
10442 for (index, leaf) in self.leaves.iter_mut().enumerate() {
10443 if leaf.result.is_some() {
10444 continue;
10445 }
10446 match leaf.call.poll_avro_value(cx) {
10447 Poll::Ready(Ok(result)) => leaf.result = Some(result),
10448 Poll::Ready(Err(error)) => failures.push((index, error)),
10449 Poll::Pending => pending = true,
10450 }
10451 }
10452
10453 if !failures.is_empty() {
10454 if let Some(position) = failures
10455 .iter()
10456 .position(|(_, error)| workflow_task_integrity_error(error))
10457 {
10458 return Poll::Ready(Err(failures.remove(position).1));
10459 }
10460 failures.sort_by_key(|(index, _)| *index);
10461 let (failed_index, cause) = failures.remove(0);
10462 let failed = &self.leaves[failed_index];
10463 let completed = self
10464 .leaves
10465 .iter()
10466 .filter_map(|leaf| {
10467 leaf.result
10468 .clone()
10469 .and_then(|result| result.into_json_result().ok())
10470 .map(|result| ParallelCompletion {
10471 member_path: leaf.member_path.clone(),
10472 result,
10473 })
10474 })
10475 .collect();
10476 let group_id = failed
10477 .group_path
10478 .first()
10479 .map(|entry| entry.parallel_group_id.clone())
10480 .unwrap_or_default();
10481 return Poll::Ready(Err(Error::ParallelFailed(ParallelFailure {
10482 group_id,
10483 member_path: failed.member_path.clone(),
10484 group_path: failed.group_path.clone(),
10485 completed,
10486 cause: Box::new(cause),
10487 })));
10488 }
10489 if pending {
10490 return Poll::Pending;
10491 }
10492
10493 let mut flat_results = self
10494 .leaves
10495 .iter_mut()
10496 .map(|leaf| leaf.result.take().expect("completed parallel leaf"))
10497 .collect::<Vec<_>>()
10498 .into_iter();
10499 let results = parallel_results_for_shape(
10500 self.shape.as_ref().expect("initialized parallel shape"),
10501 &mut flat_results,
10502 );
10503 Poll::Ready(Ok(match results {
10504 ParallelAvroResult::Group(results) => results,
10505 ParallelAvroResult::Activity(_)
10506 | ParallelAvroResult::ChildWorkflow(_)
10507 | ParallelAvroResult::Timer
10508 | ParallelAvroResult::Signal(_)
10509 | ParallelAvroResult::Condition(_) => {
10510 unreachable!("root parallel shape is a group")
10511 }
10512 }))
10513 }
10514}
10515
10516fn parallel_results_for_shape(
10517 shape: &ParallelShape,
10518 flat_results: &mut impl Iterator<Item = ParallelAvroResult>,
10519) -> ParallelAvroResult {
10520 match shape {
10521 ParallelShape::Leaf => flat_results.next().expect("one result per parallel leaf"),
10522 ParallelShape::Group(children) => ParallelAvroResult::Group(
10523 children
10524 .iter()
10525 .map(|child| parallel_results_for_shape(child, flat_results))
10526 .collect(),
10527 ),
10528 }
10529}
10530
10531impl Future for ParallelCall {
10532 type Output = Result<Vec<ParallelResult>>;
10533
10534 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
10535 self.poll_avro_value(cx)
10536 .map_ok(|results| {
10537 results
10538 .into_iter()
10539 .map(ParallelAvroResult::into_json_result)
10540 .collect::<Result<Vec<_>>>()
10541 })
10542 .map_ok(|result| result)
10543 .flatten_result()
10544 }
10545}
10546
10547#[derive(Clone, Debug)]
10548struct SelectionMemberPlan {
10549 key: SelectionKey,
10550 index: usize,
10551 base_sequence: u64,
10552 size: usize,
10553 kind: String,
10554 shape: ParallelShape,
10555 leaf_start: usize,
10556}
10557
10558fn selection_operation_kind(operation: &ParallelOperation) -> &'static str {
10559 match operation {
10560 ParallelOperation::Activity { .. } => "activity",
10561 ParallelOperation::ChildWorkflow { .. } => "child",
10562 ParallelOperation::Timer(_) => "timer",
10563 ParallelOperation::Signal(_) => "signal",
10564 ParallelOperation::Condition { .. } => "condition",
10565 ParallelOperation::Group(_) => "group",
10566 }
10567}
10568
10569fn selection_operation_shape(operation: &ParallelOperation) -> ParallelShape {
10570 match operation {
10571 ParallelOperation::Group(children) => parallel_shape(children),
10572 _ => ParallelShape::Leaf,
10573 }
10574}
10575
10576fn selection_descriptors(
10577 operations: Vec<(SelectionKey, ParallelOperation)>,
10578 base_sequence: u64,
10579) -> Result<(Vec<ParallelDescriptor>, Vec<SelectionMemberPlan>)> {
10580 if operations.is_empty() {
10581 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10582 reason: "selection_empty",
10583 member_path: Vec::new(),
10584 message: "durable selection requires at least one operation".to_string(),
10585 }));
10586 }
10587 let operation_refs = operations
10588 .iter()
10589 .map(|(_, operation)| operation)
10590 .collect::<Vec<_>>();
10591 let total_size = operation_refs
10592 .iter()
10593 .map(|operation| match operation {
10594 ParallelOperation::Group(children) => parallel_leaf_count(children),
10595 _ => 1,
10596 })
10597 .sum::<usize>();
10598 if total_size > MAX_PARALLEL_OPERATIONS {
10599 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10600 reason: "fan_out_limit_exceeded",
10601 member_path: Vec::new(),
10602 message: format!(
10603 "selection contains {total_size} durable leaves; the limit is {MAX_PARALLEL_OPERATIONS}"
10604 ),
10605 }));
10606 }
10607 let group_kind = {
10608 let mut kind = None;
10609 for operation in &operation_refs {
10610 let operation_kind = parallel_operation_kind(operation).unwrap_or("mixed");
10611 match kind {
10612 None => kind = Some(operation_kind),
10613 Some(current) if current == operation_kind => {}
10614 Some(_) => {
10615 kind = Some("mixed");
10616 break;
10617 }
10618 }
10619 }
10620 kind.unwrap_or("mixed")
10621 };
10622
10623 let mut descriptors = Vec::with_capacity(total_size);
10624 let mut members = Vec::with_capacity(operations.len());
10625 let mut cursor = 0usize;
10626 let mut seen_keys: Vec<SelectionKey> = Vec::new();
10627 for (member_index, (key, operation)) in operations.into_iter().enumerate() {
10628 if matches!(&key, SelectionKey::Name(value) if value.is_empty()) {
10629 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10630 reason: "selection_key_invalid",
10631 member_path: vec![member_index],
10632 message: "selection member keys must be non-empty strings or non-negative integers"
10633 .to_string(),
10634 }));
10635 }
10636 if seen_keys.contains(&key) {
10637 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10638 reason: "selection_key_duplicate",
10639 member_path: vec![member_index],
10640 message: format!("selection member key {key:?} is duplicated"),
10641 }));
10642 }
10643 seen_keys.push(key.clone());
10644 let member_size = match &operation {
10645 ParallelOperation::Group(children) => parallel_leaf_count(children),
10646 _ => 1,
10647 };
10648 if member_size == 0 {
10649 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10650 reason: "selection_member_empty",
10651 member_path: vec![member_index],
10652 message: "a selection member must contain at least one durable leaf".to_string(),
10653 }));
10654 }
10655 let member_base = base_sequence
10656 .checked_add(u64::try_from(cursor).unwrap_or(u64::MAX))
10657 .ok_or(Error::TimerDurationOverflow)?;
10658 let member_kind = selection_operation_kind(&operation).to_string();
10659 let member_shape = selection_operation_shape(&operation);
10660 let leaf_start = descriptors.len();
10661 match operation {
10662 ParallelOperation::Group(children) => {
10663 validate_parallel_operations(&children, &mut vec![member_index], false)?;
10664 for mut descriptor in parallel_descriptors(children, member_base)? {
10665 let flat_index = cursor + descriptor.offset;
10666 descriptor.group_path.insert(
10667 0,
10668 selection_group_entry(
10669 base_sequence,
10670 total_size,
10671 flat_index,
10672 group_kind,
10673 &SelectionMemberMetadata {
10674 key: key.clone(),
10675 index: member_index,
10676 base_sequence: member_base,
10677 size: member_size,
10678 kind: member_kind.clone(),
10679 },
10680 ),
10681 );
10682 descriptor.member_path.insert(0, member_index);
10683 descriptor.offset = flat_index;
10684 descriptors.push(descriptor);
10685 }
10686 }
10687 operation => {
10688 validate_parallel_operations(
10689 std::slice::from_ref(&operation),
10690 &mut Vec::new(),
10691 true,
10692 )?;
10693 descriptors.push(ParallelDescriptor {
10694 operation,
10695 offset: cursor,
10696 member_path: vec![member_index],
10697 group_path: vec![selection_group_entry(
10698 base_sequence,
10699 total_size,
10700 cursor,
10701 group_kind,
10702 &SelectionMemberMetadata {
10703 key: key.clone(),
10704 index: member_index,
10705 base_sequence: member_base,
10706 size: member_size,
10707 kind: member_kind.clone(),
10708 },
10709 )],
10710 });
10711 }
10712 }
10713 members.push(SelectionMemberPlan {
10714 key,
10715 index: member_index,
10716 base_sequence: member_base,
10717 size: member_size,
10718 kind: member_kind,
10719 shape: member_shape,
10720 leaf_start,
10721 });
10722 cursor += member_size;
10723 }
10724 Ok((descriptors, members))
10725}
10726
10727struct SelectionLeaf {
10728 call: ParallelLeafCall,
10729 outcome: Option<Result<ParallelAvroResult>>,
10730}
10731
10732#[derive(Clone)]
10734pub struct DurableOperationHandle {
10735 ctx: WorkflowContext,
10736 pub key: SelectionKey,
10737 pub index: usize,
10738 pub kind: String,
10739 pub identity: String,
10740 pub base_sequence: u64,
10741 pub size: usize,
10742 pub selection_group_id: String,
10743 shape: ParallelShape,
10744}
10745
10746impl std::fmt::Debug for DurableOperationHandle {
10747 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10748 formatter
10749 .debug_struct("DurableOperationHandle")
10750 .field("key", &self.key)
10751 .field("index", &self.index)
10752 .field("kind", &self.kind)
10753 .field("identity", &self.identity)
10754 .field("base_sequence", &self.base_sequence)
10755 .field("size", &self.size)
10756 .field("selection_group_id", &self.selection_group_id)
10757 .finish()
10758 }
10759}
10760
10761impl DurableOperationHandle {
10762 pub fn await_result(&self) -> DurableOperationAwaitCall {
10764 DurableOperationAwaitCall {
10765 handle: self.clone(),
10766 }
10767 }
10768
10769 pub fn cancel(&self) -> CancelDurableOperationCall {
10773 CancelDurableOperationCall {
10774 handle: self.clone(),
10775 emitted: false,
10776 }
10777 }
10778}
10779
10780#[derive(Debug)]
10782pub struct SelectionResult {
10783 pub key: SelectionKey,
10784 pub index: usize,
10785 pub kind: String,
10786 pub identity: String,
10787 pub value: Option<ParallelResult>,
10788 pub failure: Option<Error>,
10789 pub winner: DurableOperationHandle,
10790 pub handles: Vec<DurableOperationHandle>,
10791}
10792
10793impl SelectionResult {
10794 pub fn succeeded(&self) -> bool {
10795 self.failure.is_none()
10796 }
10797
10798 pub fn handle(&self, key: &SelectionKey) -> Option<&DurableOperationHandle> {
10799 self.handles.iter().find(|handle| &handle.key == key)
10800 }
10801
10802 pub fn remaining(&self) -> Vec<&DurableOperationHandle> {
10803 self.handles
10804 .iter()
10805 .filter(|handle| handle.index != self.index)
10806 .collect()
10807 }
10808
10809 pub fn into_result(self) -> Result<ParallelResult> {
10810 match (self.value, self.failure) {
10811 (Some(value), None) => Ok(value),
10812 (_, Some(error)) => Err(error),
10813 _ => Err(Error::WorkerLoop(
10814 "selection result contained neither a value nor a failure".to_string(),
10815 )),
10816 }
10817 }
10818}
10819
10820pub struct SelectCall {
10822 ctx: WorkflowContext,
10823 operations: Option<Vec<(SelectionKey, ParallelOperation)>>,
10824 members: Vec<SelectionMemberPlan>,
10825 leaves: Vec<SelectionLeaf>,
10826 group_id: Option<String>,
10827}
10828
10829impl SelectCall {
10830 fn new(ctx: WorkflowContext, operations: Vec<(SelectionKey, ParallelOperation)>) -> Self {
10831 Self {
10832 ctx,
10833 operations: Some(operations),
10834 members: Vec::new(),
10835 leaves: Vec::new(),
10836 group_id: None,
10837 }
10838 }
10839
10840 fn initialize(&mut self) -> Result<()> {
10841 let operations = self.operations.take().unwrap_or_default();
10842 let base_sequence = {
10843 let state = self
10844 .ctx
10845 .state
10846 .lock()
10847 .map_err(|_| Error::WorkflowStatePoisoned)?;
10848 if let Some(marker) = state.selection_markers.get(state.selection_marker_cursor) {
10849 marker.selection_group_base_sequence
10850 } else if let Some(recorded) = state.recorded_commands.get(state.command_cursor) {
10851 recorded.sequence()
10852 } else {
10853 let last = state
10854 .recorded_commands
10855 .last()
10856 .map(RecordedCommand::sequence)
10857 .unwrap_or(0);
10858 last.checked_add(u64::try_from(state.commands.len()).unwrap_or(u64::MAX))
10859 .and_then(|sequence| sequence.checked_add(1))
10860 .ok_or(Error::TimerDurationOverflow)?
10861 }
10862 };
10863 let (descriptors, members) = selection_descriptors(operations, base_sequence)?;
10864 let group_id = format!("select-calls:{base_sequence}:{}", descriptors.len());
10865 self.leaves = descriptors
10866 .into_iter()
10867 .map(|descriptor| SelectionLeaf {
10868 call: parallel_leaf_call(&self.ctx, descriptor.operation, descriptor.group_path),
10869 outcome: None,
10870 })
10871 .collect();
10872 self.members = members;
10873 self.group_id = Some(group_id);
10874 Ok(())
10875 }
10876}
10877
10878impl Future for SelectCall {
10879 type Output = Result<SelectionResult>;
10880
10881 fn poll(mut self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
10882 if self.operations.is_some() {
10883 if let Err(error) = self.initialize() {
10884 return Poll::Ready(Err(error));
10885 }
10886 }
10887
10888 for leaf in &mut self.leaves {
10889 if leaf.outcome.is_some() {
10890 continue;
10891 }
10892 if let Poll::Ready(outcome) = leaf.call.poll_avro_value(cx) {
10893 if outcome
10894 .as_ref()
10895 .err()
10896 .is_some_and(workflow_task_integrity_error)
10897 {
10898 return Poll::Ready(outcome.map(|_| unreachable!()));
10899 }
10900 leaf.outcome = Some(outcome);
10901 }
10902 }
10903
10904 let all_members_terminal = self.leaves.iter().all(|leaf| leaf.outcome.is_some());
10905 let selection_member_range = self
10906 .members
10907 .first()
10908 .map(|member| member.base_sequence)
10909 .zip(self.leaves.len().try_into().ok())
10910 .map(|(base_sequence, size): (u64, u64)| {
10911 base_sequence..base_sequence.saturating_add(size)
10912 });
10913 let marker = {
10914 let mut state = match self.ctx.state.lock() {
10915 Ok(state) => state,
10916 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
10917 };
10918 let marker = state
10919 .selection_markers
10920 .get(state.selection_marker_cursor)
10921 .cloned();
10922 if marker.is_none()
10923 && all_members_terminal
10924 && selection_member_range.as_ref().is_some_and(|member_range| {
10925 state
10926 .recorded_commands
10927 .iter()
10928 .any(|command| member_range.contains(&command.sequence()))
10929 })
10930 {
10931 state.matched_recorded_pending = true;
10937 }
10938 marker
10939 };
10940 let Some(marker) = marker else {
10941 return Poll::Pending;
10942 };
10943 if self.group_id.as_deref() != Some(marker.selection_group_id.as_str())
10944 || marker.selection_group_size != self.leaves.len()
10945 || self.members.first().map(|member| member.base_sequence)
10946 != Some(marker.selection_group_base_sequence)
10947 {
10948 return Poll::Ready(Err(invalid_recorded_history(
10949 "selection_group_shape_mismatch",
10950 marker.selection_group_base_sequence,
10951 self.group_id
10952 .as_deref()
10953 .unwrap_or("initialized selection group"),
10954 &marker.selection_group_id,
10955 "recorded selection group differs from current workflow code",
10956 )));
10957 }
10958 let Some(member_position) = self.members.iter().position(|member| {
10959 member.key == marker.member_key
10960 && member.index == marker.member_index
10961 && member.base_sequence == marker.member_base_sequence
10962 && member.size == marker.member_size
10963 && member.kind == marker.operation_kind
10964 }) else {
10965 return Poll::Ready(Err(invalid_recorded_history(
10966 "selection_member_shape_mismatch",
10967 marker.member_base_sequence,
10968 "winner member matching current workflow code",
10969 &format!("{:?}", marker.member_key),
10970 "recorded selection winner differs from the authored member identity",
10971 )));
10972 };
10973 let member = self.members[member_position].clone();
10974 let (handles, resolution_sequence) = {
10975 let mut state = match self.ctx.state.lock() {
10976 Ok(state) => state,
10977 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
10978 };
10979 let identities = self
10980 .members
10981 .iter()
10982 .map(|candidate| {
10983 selection_operation_identity(
10984 &state,
10985 &candidate.kind,
10986 candidate.base_sequence,
10987 candidate.size,
10988 )
10989 })
10990 .collect::<Vec<_>>();
10991 if let Some((position, missing)) = identities
10992 .iter()
10993 .enumerate()
10994 .find(|(_, identity)| identity.is_empty())
10995 .map(|(position, identity)| (position, identity.clone()))
10996 {
10997 let candidate = &self.members[position];
10998 return Poll::Ready(Err(invalid_recorded_history(
10999 "selection_operation_identity_missing",
11000 candidate.base_sequence,
11001 &format!(
11002 "durable {} resource identity from scheduled/open history",
11003 candidate.kind
11004 ),
11005 &missing,
11006 "selection member history is missing its canonical durable identity",
11007 )));
11008 }
11009 let expected_winner_identity = &identities[member_position];
11010 let resolution_sequence = match validated_selection_resolution_sequence(
11011 &state,
11012 &marker,
11013 &member,
11014 expected_winner_identity,
11015 ) {
11016 Ok(sequence) => sequence,
11017 Err(error) => return Poll::Ready(Err(error)),
11018 };
11019 let handles = self
11020 .members
11021 .iter()
11022 .zip(identities)
11023 .map(|(member, identity)| DurableOperationHandle {
11024 ctx: self.ctx.clone(),
11025 key: member.key.clone(),
11026 index: member.index,
11027 kind: member.kind.clone(),
11028 identity,
11029 base_sequence: member.base_sequence,
11030 size: member.size,
11031 selection_group_id: marker.selection_group_id.clone(),
11032 shape: member.shape.clone(),
11033 })
11034 .collect::<Vec<_>>();
11035 if let Err(error) = validate_selection_cancellations_for_handles(&state, &handles) {
11036 return Poll::Ready(Err(error));
11037 }
11038 state.selection_marker_cursor += 1;
11039 (handles, resolution_sequence)
11040 };
11041
11042 let mut winner_failure = None;
11043 let mut flat_results = Vec::with_capacity(member.size);
11044 if marker.outcome == "failed" {
11045 let resolution_offset = match resolution_sequence
11046 .checked_sub(member.base_sequence)
11047 .and_then(|offset| usize::try_from(offset).ok())
11048 {
11049 Some(offset) if offset < member.size => offset,
11050 _ => {
11051 return Poll::Ready(Err(invalid_recorded_history(
11052 "selection_resolution_event_mismatch",
11053 member.base_sequence,
11054 "failure event within selected member bounds",
11055 &resolution_sequence.to_string(),
11056 "selection failure event is outside the authored member",
11057 )))
11058 }
11059 };
11060 let leaf = &mut self.leaves[member.leaf_start + resolution_offset];
11061 match leaf.outcome.take() {
11062 Some(Err(error)) => winner_failure = Some(error),
11063 _ => {
11064 return Poll::Ready(Err(invalid_recorded_history(
11065 "selection_winner_outcome_mismatch",
11066 member.base_sequence,
11067 "exact failed terminal history referenced by SelectionResolved",
11068 "missing or successful resolution event",
11069 "selection winner marker disagrees with terminal operation history",
11070 )))
11071 }
11072 }
11073 } else {
11074 for leaf in &mut self.leaves[member.leaf_start..member.leaf_start + member.size] {
11075 match leaf.outcome.take() {
11076 Some(Ok(result)) => flat_results.push(result),
11077 Some(Err(_)) => {
11078 return Poll::Ready(Err(invalid_recorded_history(
11079 "selection_winner_outcome_mismatch",
11080 member.base_sequence,
11081 "fully completed nested selection member",
11082 "failed durable leaf",
11083 "completed selection winner contains a failed leaf",
11084 )))
11085 }
11086 None => {
11087 return Poll::Ready(Err(invalid_recorded_history(
11088 "selection_winner_unresolved",
11089 member.base_sequence,
11090 "terminal history for every completed winner leaf",
11091 "pending member history",
11092 "completed SelectionResolved member has an unfinished durable barrier",
11093 )))
11094 }
11095 }
11096 }
11097 }
11098 let value = if winner_failure.is_none() {
11099 let mut flat_results = flat_results.into_iter();
11100 let value = parallel_results_for_shape(&member.shape, &mut flat_results);
11101 match value.into_json_result() {
11102 Ok(value) => Some(value),
11103 Err(error) => return Poll::Ready(Err(error)),
11104 }
11105 } else {
11106 None
11107 };
11108 let winner = handles[member_position].clone();
11109 Poll::Ready(Ok(SelectionResult {
11110 key: winner.key.clone(),
11111 index: winner.index,
11112 kind: winner.kind.clone(),
11113 identity: winner.identity.clone(),
11114 value,
11115 failure: winner_failure,
11116 winner,
11117 handles,
11118 }))
11119 }
11120}
11121
11122fn selection_operation_identity(
11123 state: &WorkflowState,
11124 kind: &str,
11125 base_sequence: u64,
11126 size: usize,
11127) -> String {
11128 if kind == "group" {
11129 return format!("group:{base_sequence}:{size}");
11130 }
11131 let fields: &[&str] = match kind {
11132 "activity" => &["activity_execution_id"],
11133 "child" => &["child_workflow_run_id"],
11134 "timer" => &["timer_id"],
11135 "signal" => &["signal_wait_id"],
11136 "condition" => &["condition_wait_id"],
11137 _ => &[],
11138 };
11139 for sequence in base_sequence..base_sequence.saturating_add(size as u64) {
11140 for event in state
11141 .history_events
11142 .iter()
11143 .filter(|event| durable_event_sequence(event) == Some(sequence))
11144 {
11145 for field in fields {
11146 if let Some(identity) = event.payload.get(*field).and_then(Value::as_str) {
11147 if !identity.is_empty() {
11148 return identity.to_string();
11149 }
11150 }
11151 }
11152 }
11153 }
11154 String::new()
11155}
11156
11157fn validated_selection_resolution_sequence(
11158 state: &WorkflowState,
11159 marker: &SelectionMarker,
11160 member: &SelectionMemberPlan,
11161 expected_identity: &str,
11162) -> Result<u64> {
11163 if expected_identity.is_empty() {
11164 return Err(invalid_recorded_history(
11165 "selection_operation_identity_missing",
11166 member.base_sequence,
11167 &format!(
11168 "durable {} resource identity from scheduled/open history",
11169 member.kind
11170 ),
11171 "missing operation identity",
11172 "selection member history is missing its canonical durable identity",
11173 ));
11174 }
11175 if marker.operation_identity != expected_identity {
11176 return Err(invalid_recorded_history(
11177 "selection_operation_identity_mismatch",
11178 member.base_sequence,
11179 expected_identity,
11180 &marker.operation_identity,
11181 "selection winner identity does not match durable scheduled/open history",
11182 ));
11183 }
11184
11185 let failure_types = [
11186 "ActivityFailed",
11187 "ActivityCancelled",
11188 "ActivityTimedOut",
11189 "ChildRunFailed",
11190 "ChildRunCancelled",
11191 "ChildRunTerminated",
11192 ];
11193 let success_types = [
11194 "ActivityCompleted",
11195 "ChildRunCompleted",
11196 "TimerFired",
11197 "SignalApplied",
11198 "ConditionWaitSatisfied",
11199 "ConditionWaitTimedOut",
11200 ];
11201 let terminal_types: &[&str] = if marker.outcome == "failed" {
11202 &failure_types
11203 } else {
11204 &success_types
11205 };
11206 let mut candidates = Vec::new();
11207 for event in state.history_events.iter() {
11208 let Some(sequence) = durable_event_sequence(event) else {
11209 continue;
11210 };
11211 if sequence < member.base_sequence
11212 || sequence >= member.base_sequence.saturating_add(member.size as u64)
11213 || !terminal_types.contains(&event.event_type.as_str())
11214 {
11215 continue;
11216 }
11217 let event_id = event
11218 .raw
11219 .get("id")
11220 .or_else(|| event.raw.get("event_id"))
11221 .and_then(Value::as_str)
11222 .filter(|value| !value.is_empty())
11223 .ok_or_else(|| {
11224 invalid_recorded_history(
11225 "selection_resolution_event_id_missing",
11226 member.base_sequence,
11227 "terminal selection history with a durable event id",
11228 &event.payload.to_string(),
11229 "selection terminal history cannot be bound to its winner marker",
11230 )
11231 })?;
11232 candidates.push((event_id.to_string(), event.event_type.clone(), sequence));
11233 }
11234 let resolution = if marker.outcome == "failed" {
11235 candidates.first()
11236 } else {
11237 candidates.last()
11238 };
11239 let Some((event_id, event_type, sequence)) = resolution else {
11240 return Err(invalid_recorded_history(
11241 "selection_resolution_event_missing",
11242 member.base_sequence,
11243 "terminal history for the selected member",
11244 &format!("{:?}", marker.member_key),
11245 "selection winner marker has no matching durable terminal event",
11246 ));
11247 };
11248 if event_id != &marker.resolution_event_id || event_type != &marker.resolution_event_type {
11249 return Err(invalid_recorded_history(
11250 "selection_resolution_event_mismatch",
11251 member.base_sequence,
11252 &format!("{event_type}:{event_id}"),
11253 &format!(
11254 "{}:{}",
11255 marker.resolution_event_type, marker.resolution_event_id
11256 ),
11257 "selection winner marker does not reference the event that made its member terminal",
11258 ));
11259 }
11260 Ok(*sequence)
11261}
11262
11263fn recorded_selection_member_outcome(
11264 state: &WorkflowState,
11265 handle: &DurableOperationHandle,
11266) -> Result<Option<ParallelResult>> {
11267 for event in state.history_events.iter() {
11268 let Some(sequence) = durable_event_sequence(event) else {
11269 continue;
11270 };
11271 if sequence < handle.base_sequence
11272 || sequence >= handle.base_sequence.saturating_add(handle.size as u64)
11273 || !matches!(
11274 event.event_type.as_str(),
11275 "ActivityFailed"
11276 | "ActivityCancelled"
11277 | "ActivityTimedOut"
11278 | "ChildRunFailed"
11279 | "ChildRunCancelled"
11280 | "ChildRunTerminated"
11281 )
11282 {
11283 continue;
11284 }
11285 let Some(command) = state
11286 .recorded_commands
11287 .iter()
11288 .find(|command| command.sequence() == sequence)
11289 else {
11290 continue;
11291 };
11292 match command {
11293 RecordedCommand::Activity {
11294 outcome: Some(Err(failure)),
11295 ..
11296 } => return Err(Error::ActivityFailed(failure.clone())),
11297 RecordedCommand::ChildWorkflow {
11298 outcome: Some(Err(failure)),
11299 ..
11300 } => return Err(Error::ChildWorkflowFailed(failure.clone())),
11301 _ => {}
11302 }
11303 }
11304
11305 let mut results = Vec::with_capacity(handle.size);
11306 for sequence in handle.base_sequence..handle.base_sequence.saturating_add(handle.size as u64) {
11307 let Some(command) = state
11308 .recorded_commands
11309 .iter()
11310 .find(|command| command.sequence() == sequence)
11311 else {
11312 return Ok(None);
11313 };
11314 let result = match command {
11315 RecordedCommand::Activity { outcome, .. } => match outcome {
11316 Some(Ok(value)) => ParallelAvroResult::Activity(value.clone()),
11317 Some(Err(failure)) => return Err(Error::ActivityFailed(failure.clone())),
11318 None => return Ok(None),
11319 },
11320 RecordedCommand::Timer { fired, .. } => {
11321 if !fired {
11322 return Ok(None);
11323 }
11324 ParallelAvroResult::Timer
11325 }
11326 RecordedCommand::ChildWorkflow { outcome, .. } => match outcome {
11327 Some(Ok(value)) => ParallelAvroResult::ChildWorkflow(value.clone()),
11328 Some(Err(failure)) => return Err(Error::ChildWorkflowFailed(failure.clone())),
11329 None => return Ok(None),
11330 },
11331 RecordedCommand::SignalWait { value, .. } => match value {
11332 Some(value) => ParallelAvroResult::Signal(value.clone()),
11333 None => return Ok(None),
11334 },
11335 RecordedCommand::ConditionWait { result, .. } => match result {
11336 Some(result) => ParallelAvroResult::Condition(*result),
11337 None => return Ok(None),
11338 },
11339 other => {
11340 return Err(command_mismatch(
11341 other,
11342 format!("selected {} member", handle.kind),
11343 ))
11344 }
11345 };
11346 results.push(result);
11347 }
11348 let mut results = results.into_iter();
11349 parallel_results_for_shape(&handle.shape, &mut results)
11350 .into_json_result()
11351 .map(Some)
11352}
11353
11354fn recorded_selection_member_is_terminal(
11355 state: &WorkflowState,
11356 handle: &DurableOperationHandle,
11357) -> bool {
11358 let mut completed = 0usize;
11359 let mut all_completed = true;
11360 for sequence in handle.base_sequence..handle.base_sequence.saturating_add(handle.size as u64) {
11361 let Some(command) = state
11362 .recorded_commands
11363 .iter()
11364 .find(|command| command.sequence() == sequence)
11365 else {
11366 all_completed = false;
11367 continue;
11368 };
11369 let terminal = match command {
11370 RecordedCommand::Activity {
11371 outcome: Some(Err(_)),
11372 ..
11373 }
11374 | RecordedCommand::ChildWorkflow {
11375 outcome: Some(Err(_)),
11376 ..
11377 } => return true,
11378 RecordedCommand::Activity { outcome, .. } => outcome.is_some(),
11379 RecordedCommand::ChildWorkflow { outcome, .. } => outcome.is_some(),
11380 RecordedCommand::Timer { fired, .. } => *fired,
11381 RecordedCommand::SignalWait { value, .. } => value.is_some(),
11382 RecordedCommand::ConditionWait { result, .. } => result.is_some(),
11383 RecordedCommand::SearchAttributes { .. }
11384 | RecordedCommand::SideEffect { .. }
11385 | RecordedCommand::VersionMarker { .. }
11386 | RecordedCommand::Memo { .. } => false,
11387 };
11388 if !terminal {
11389 all_completed = false;
11390 continue;
11391 }
11392 completed += 1;
11393 }
11394 all_completed && completed == handle.size
11395}
11396
11397fn selection_cancellation_for_handle(
11398 state: &WorkflowState,
11399 handle: &DurableOperationHandle,
11400) -> Result<bool> {
11401 let Some(marker) = state.cancelled_selection_members.iter().find(|recorded| {
11402 recorded.selection_group_id == handle.selection_group_id
11403 && recorded.member_base_sequence == handle.base_sequence
11404 }) else {
11405 return Ok(false);
11406 };
11407 validate_selection_cancellation_marker(marker, handle)?;
11408 Ok(true)
11409}
11410
11411fn validate_selection_cancellations_for_handles(
11412 state: &WorkflowState,
11413 handles: &[DurableOperationHandle],
11414) -> Result<()> {
11415 let Some(group_id) = handles
11416 .first()
11417 .map(|handle| handle.selection_group_id.as_str())
11418 else {
11419 return Ok(());
11420 };
11421 for marker in state
11422 .cancelled_selection_members
11423 .iter()
11424 .filter(|marker| marker.selection_group_id == group_id)
11425 {
11426 let Some(handle) = handles
11427 .iter()
11428 .find(|handle| handle.base_sequence == marker.member_base_sequence)
11429 else {
11430 return Err(invalid_recorded_history(
11431 "selection_cancellation_member_mismatch",
11432 marker.member_base_sequence,
11433 "SelectionOperationCancelled matching an authored selection handle",
11434 &format!("{marker:?}"),
11435 "selection cancellation member base does not name an authored member",
11436 ));
11437 };
11438 validate_selection_cancellation_marker(marker, handle)?;
11439 }
11440 Ok(())
11441}
11442
11443fn validate_selection_cancellation_marker(
11444 marker: &SelectionCancellation,
11445 handle: &DurableOperationHandle,
11446) -> Result<()> {
11447 if marker.selection_group_id != handle.selection_group_id
11448 || marker.member_key != handle.key
11449 || marker.member_index != handle.index
11450 || marker.member_base_sequence != handle.base_sequence
11451 || marker.member_size != handle.size
11452 || marker.operation_kind != handle.kind
11453 || marker.operation_identity != handle.identity
11454 {
11455 return Err(invalid_recorded_history(
11456 "selection_cancellation_member_mismatch",
11457 handle.base_sequence,
11458 "SelectionOperationCancelled matching the authored selection handle",
11459 &format!("{marker:?}"),
11460 "selection cancellation history targets different authored member metadata",
11461 ));
11462 }
11463 Ok(())
11464}
11465
11466pub struct DurableOperationAwaitCall {
11468 handle: DurableOperationHandle,
11469}
11470
11471impl Future for DurableOperationAwaitCall {
11472 type Output = Result<ParallelResult>;
11473
11474 fn poll(self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11475 let state = match self.handle.ctx.state.lock() {
11476 Ok(state) => state,
11477 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11478 };
11479 match selection_cancellation_for_handle(&state, &self.handle) {
11480 Err(error) => return Poll::Ready(Err(error)),
11481 Ok(false) => {}
11482 Ok(true) => {
11483 return Poll::Ready(Err(Error::DurableOperationCancelled(
11484 DurableOperationCancelled {
11485 selection_group_id: self.handle.selection_group_id.clone(),
11486 member_key: self.handle.key.clone(),
11487 member_index: self.handle.index,
11488 operation_kind: self.handle.kind.clone(),
11489 operation_identity: self.handle.identity.clone(),
11490 },
11491 )));
11492 }
11493 }
11494 match recorded_selection_member_outcome(&state, &self.handle) {
11495 Ok(Some(result)) => Poll::Ready(Ok(result)),
11496 Ok(None) => Poll::Pending,
11497 Err(error) => Poll::Ready(Err(error)),
11498 }
11499 }
11500}
11501
11502pub struct CancelDurableOperationCall {
11504 handle: DurableOperationHandle,
11505 emitted: bool,
11506}
11507
11508impl Future for CancelDurableOperationCall {
11509 type Output = Result<()>;
11510
11511 fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11512 let ctx = self.handle.ctx.clone();
11513 let mut state = match ctx.state.lock() {
11514 Ok(state) => state,
11515 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11516 };
11517 match selection_cancellation_for_handle(&state, &self.handle) {
11518 Err(error) => return Poll::Ready(Err(error)),
11519 Ok(true) => return Poll::Ready(Ok(())),
11520 Ok(false) => {}
11521 }
11522 if recorded_selection_member_is_terminal(&state, &self.handle) {
11523 return Poll::Ready(Ok(()));
11524 }
11525 if !self.emitted {
11526 state.commands.push(json!({
11527 "type": "cancel_selection_operation",
11528 "selection_group_id": self.handle.selection_group_id,
11529 "member_key": self.handle.key,
11530 "member_index": self.handle.index,
11531 "member_base_sequence": self.handle.base_sequence,
11532 "member_size": self.handle.size,
11533 "operation_kind": self.handle.kind,
11534 "operation_identity": self.handle.identity,
11535 }));
11536 self.emitted = true;
11537 }
11538 Poll::Pending
11542 }
11543}
11544
11545trait PollNestedResultExt<T> {
11546 fn flatten_result(self) -> Poll<Result<T>>;
11547}
11548
11549impl<T> PollNestedResultExt<T> for Poll<Result<Result<T>>> {
11550 fn flatten_result(self) -> Poll<Result<T>> {
11551 match self {
11552 Poll::Ready(Ok(result)) => Poll::Ready(result),
11553 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
11554 Poll::Pending => Poll::Pending,
11555 }
11556 }
11557}
11558
11559struct SagaCompensation {
11560 activity_type: String,
11561 options: ActivityOptions,
11562 arguments: AvroValue,
11563 registration_order: usize,
11564}
11565
11566pub struct Saga {
11574 ctx: WorkflowContext,
11575 compensations: Vec<SagaCompensation>,
11576}
11577
11578impl Saga {
11579 fn new(ctx: WorkflowContext) -> Self {
11580 Self {
11581 ctx,
11582 compensations: Vec::new(),
11583 }
11584 }
11585
11586 pub fn add_compensation<T: Serialize>(
11587 &mut self,
11588 activity_type: impl Into<String>,
11589 args: T,
11590 ) -> Result<&mut Self> {
11591 self.add_compensation_with_options(activity_type, ActivityOptions::new(), args)
11592 }
11593
11594 pub fn add_compensation_with_options<T: Serialize>(
11595 &mut self,
11596 activity_type: impl Into<String>,
11597 options: ActivityOptions,
11598 args: T,
11599 ) -> Result<&mut Self> {
11600 let activity_type = activity_type.into();
11601 if activity_type.trim().is_empty() || activity_type.trim() != activity_type {
11602 return Err(Error::Codec(
11603 "saga compensation activity type must be non-empty without surrounding whitespace"
11604 .to_string(),
11605 ));
11606 }
11607 options.validate().map_err(Error::InvalidActivityOptions)?;
11608 let arguments = AvroValue::from_serialize(&args)?;
11609 let registration_order = self.compensations.len() + 1;
11610 self.compensations.push(SagaCompensation {
11611 activity_type,
11612 options,
11613 arguments,
11614 registration_order,
11615 });
11616 Ok(self)
11617 }
11618
11619 pub async fn compensate(mut self, initiating_failure: Error) -> Error {
11621 while let Some(compensation) = self.compensations.pop() {
11622 if let Err(compensation_failure) = self
11623 .ctx
11624 .activity_with_options(
11625 compensation.activity_type.clone(),
11626 compensation.options,
11627 compensation.arguments,
11628 )
11629 .await
11630 {
11631 if workflow_task_integrity_error(&compensation_failure) {
11632 return compensation_failure;
11633 }
11634 return Error::SagaCompensationFailed(SagaCompensationFailure {
11635 initiating_failure: Box::new(initiating_failure),
11636 compensation_failure: Box::new(compensation_failure),
11637 compensation_activity_type: compensation.activity_type,
11638 compensation_registration_order: compensation.registration_order,
11639 });
11640 }
11641 }
11642 initiating_failure
11643 }
11644
11645 pub async fn finish<T>(self, outcome: Result<T>) -> Result<T> {
11647 match outcome {
11648 Ok(value) => Ok(value),
11649 Err(error) => Err(self.compensate(error).await),
11650 }
11651 }
11652}
11653
11654pub struct ActivityCall {
11655 ctx: WorkflowContext,
11656 activity_type: String,
11657 options: ActivityOptions,
11658 args: Option<Result<AvroValue>>,
11659 scheduled: bool,
11660 parallel_group_path: Vec<ParallelGroupMetadata>,
11661}
11662
11663impl ActivityCall {
11664 fn poll_avro_value(
11665 mut self: Pin<&mut Self>,
11666 _cx: &mut TaskContext<'_>,
11667 ) -> Poll<Result<AvroValue>> {
11668 let ctx = self.ctx.clone();
11669 let mut state = match ctx.state.lock() {
11670 Ok(state) => state,
11671 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11672 };
11673
11674 if self.scheduled {
11675 return Poll::Pending;
11676 }
11677
11678 let options = match self.options.validate() {
11679 Ok(options) => options,
11680 Err(error) => {
11681 return Poll::Ready(Err(Error::InvalidActivityOptions(error)));
11682 }
11683 };
11684 let task_queue = options
11685 .task_queue
11686 .clone()
11687 .unwrap_or_else(|| state.task_queue.clone());
11688 let current_recorded_options = RecordedActivityOptions {
11689 task_queue: RecordedSnapshotValue::Known(Some(task_queue.clone())),
11690 execution_mode: RecordedSnapshotValue::Known(None),
11693 retry_policy: current_activity_retry_snapshot(&options),
11694 };
11695
11696 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
11697 let sequence = recorded.sequence();
11698 match recorded {
11699 RecordedCommand::Activity {
11700 activity_type,
11701 options: recorded_options,
11702 outcome,
11703 parallel_group_path,
11704 ..
11705 } => {
11706 if let Err(error) = ensure_parallel_path_matches(
11707 sequence,
11708 parallel_group_path.as_deref(),
11709 &self.parallel_group_path,
11710 ) {
11711 return Poll::Ready(Err(error));
11712 }
11713 if let Some(recorded_type) = activity_type {
11714 if recorded_type != self.activity_type {
11715 return Poll::Ready(Err(Error::NonDeterministicReplay(
11716 ReplayFailure::new(
11717 "recorded_command_detail_mismatch",
11718 Some(sequence),
11719 Some(format!("activity:{recorded_type}")),
11720 Some(format!("activity:{}", self.activity_type)),
11721 "recorded activity type differs from the current workflow command",
11722 ),
11723 )));
11724 }
11725 }
11726 if let Some(recorded_options) = recorded_options {
11727 if !recorded_options
11728 .task_queue
11729 .matches_current(¤t_recorded_options.task_queue)
11730 {
11731 return Poll::Ready(Err(Error::NonDeterministicReplay(
11732 ReplayFailure::new(
11733 "activity_task_queue_mismatch",
11734 Some(sequence),
11735 Some(activity_options_description(&recorded_options)),
11736 Some(activity_options_description(¤t_recorded_options)),
11737 "recorded activity task queue differs from the current workflow command",
11738 ),
11739 )));
11740 }
11741 if !recorded_options
11742 .execution_mode
11743 .matches_current(¤t_recorded_options.execution_mode)
11744 {
11745 return Poll::Ready(Err(Error::NonDeterministicReplay(
11746 ReplayFailure::new(
11747 "activity_execution_mode_mismatch",
11748 Some(sequence),
11749 Some(activity_options_description(&recorded_options)),
11750 Some(activity_options_description(¤t_recorded_options)),
11751 "recorded activity execution mode differs from the current workflow command",
11752 ),
11753 )));
11754 }
11755 if !recorded_options
11756 .retry_policy
11757 .matches_current(¤t_recorded_options.retry_policy)
11758 {
11759 return Poll::Ready(Err(Error::NonDeterministicReplay(
11760 ReplayFailure::new(
11761 "activity_retry_policy_mismatch",
11762 Some(sequence),
11763 Some(activity_options_description(&recorded_options)),
11764 Some(activity_options_description(¤t_recorded_options)),
11765 "recorded activity retry policy differs from the current workflow command",
11766 ),
11767 )));
11768 }
11769 }
11770 state.command_cursor += 1;
11771 if let Some(outcome) = outcome {
11772 return Poll::Ready(outcome.map_err(Error::ActivityFailed));
11773 }
11774 state.matched_recorded_pending = true;
11775 self.scheduled = true;
11776 return Poll::Pending;
11777 }
11778 other => {
11779 return Poll::Ready(Err(command_mismatch(
11780 &other,
11781 format!("activity:{}", self.activity_type),
11782 )));
11783 }
11784 }
11785 }
11786
11787 if !self.scheduled {
11788 let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
11789 Ok(args) => args,
11790 Err(error) => return Poll::Ready(Err(error)),
11791 };
11792 let arguments = normalize_avro_arguments(args);
11793 let envelope = match encode_typed_envelope(&arguments, &state.payload_codec) {
11794 Ok(envelope) => envelope,
11795 Err(error) => return Poll::Ready(Err(error)),
11796 };
11797
11798 let mut command = serde_json::Map::from_iter([
11799 ("type".to_string(), json!("schedule_activity")),
11800 (
11801 "activity_type".to_string(),
11802 json!(self.activity_type.clone()),
11803 ),
11804 ("queue".to_string(), json!(task_queue)),
11805 ("arguments".to_string(), envelope),
11806 ]);
11807 for (field, value) in [
11808 ("start_to_close_timeout", options.start_to_close_timeout),
11809 (
11810 "schedule_to_start_timeout",
11811 options.schedule_to_start_timeout,
11812 ),
11813 (
11814 "schedule_to_close_timeout",
11815 options.schedule_to_close_timeout,
11816 ),
11817 ("heartbeat_timeout", options.heartbeat_timeout),
11818 ] {
11819 if let Some(value) = value {
11820 command.insert(field.to_string(), json!(value));
11821 }
11822 }
11823 if let Some(retry_policy) = options.retry_policy {
11824 command.insert("retry_policy".to_string(), retry_policy);
11825 }
11826 apply_parallel_group_path(&mut command, &self.parallel_group_path);
11827 state.commands.push(Value::Object(command));
11828 self.scheduled = true;
11829 }
11830
11831 Poll::Pending
11832 }
11833}
11834
11835impl Future for ActivityCall {
11836 type Output = Result<Value>;
11837
11838 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11839 match self.poll_avro_value(cx) {
11840 Poll::Ready(Ok(value)) => Poll::Ready(value.into_json()),
11841 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
11842 Poll::Pending => Poll::Pending,
11843 }
11844 }
11845}
11846
11847pub struct TimerCall {
11849 ctx: WorkflowContext,
11850 delay_seconds: Option<u64>,
11851 scheduled: bool,
11852 matched_pending: bool,
11853 parallel_group_path: Vec<ParallelGroupMetadata>,
11854}
11855
11856impl Future for TimerCall {
11857 type Output = Result<()>;
11858
11859 fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11860 if self.matched_pending {
11861 return Poll::Pending;
11862 }
11863
11864 let ctx = self.ctx.clone();
11865 let Some(requested_delay) = self.delay_seconds else {
11866 return Poll::Ready(Err(Error::TimerDurationOverflow));
11867 };
11868 let mut state = match ctx.state.lock() {
11869 Ok(state) => state,
11870 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11871 };
11872
11873 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
11874 match recorded {
11875 RecordedCommand::Timer {
11876 sequence,
11877 delay_seconds,
11878 fired,
11879 parallel_group_path,
11880 ..
11881 } => {
11882 if let Err(error) = ensure_parallel_path_matches(
11883 sequence,
11884 parallel_group_path.as_deref(),
11885 &self.parallel_group_path,
11886 ) {
11887 return Poll::Ready(Err(error));
11888 }
11889 if delay_seconds != requested_delay {
11890 return Poll::Ready(Err(Error::NonDeterministicReplay(
11891 ReplayFailure::new(
11892 "timer_delay_mismatch",
11893 Some(sequence),
11894 Some(format!("timer:{delay_seconds}s")),
11895 Some(format!("timer:{requested_delay}s")),
11896 "recorded timer delay differs from the current workflow command",
11897 ),
11898 )));
11899 }
11900 state.command_cursor += 1;
11901 if fired {
11902 return Poll::Ready(Ok(()));
11903 }
11904 state.matched_recorded_pending = true;
11905 self.scheduled = true;
11906 self.matched_pending = true;
11907 return Poll::Pending;
11908 }
11909 other => return Poll::Ready(Err(command_mismatch(&other, "timer"))),
11910 }
11911 }
11912
11913 if !self.scheduled {
11914 let mut command = serde_json::Map::from_iter([
11915 ("type".to_string(), json!("start_timer")),
11916 ("delay_seconds".to_string(), json!(requested_delay)),
11917 ]);
11918 apply_parallel_group_path(&mut command, &self.parallel_group_path);
11919 state.commands.push(Value::Object(command));
11920 self.scheduled = true;
11921 }
11922
11923 Poll::Pending
11924 }
11925}
11926
11927pub struct ConditionWaitCall {
11929 ctx: WorkflowContext,
11930 options: ConditionWaitOptions,
11931 predicate: Box<dyn Fn() -> Result<bool> + Send + 'static>,
11932 occurrence_id: Option<String>,
11933 opened_wait: bool,
11934 parallel_group_path: Vec<ParallelGroupMetadata>,
11935}
11936
11937impl Future for ConditionWaitCall {
11938 type Output = Result<ConditionWaitResult>;
11939
11940 fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11941 if self.opened_wait {
11942 return Poll::Pending;
11943 }
11944
11945 let options = match self.options.validate() {
11946 Ok(options) => options,
11947 Err(error) => return Poll::Ready(Err(Error::InvalidConditionWaitOptions(error))),
11948 };
11949 let ctx = self.ctx.clone();
11950 let occurrence_id = match self.occurrence_id.as_ref() {
11951 Some(occurrence_id) => occurrence_id.clone(),
11952 None => {
11953 let mut state = match ctx.state.lock() {
11954 Ok(state) => state,
11955 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11956 };
11957 let ordinal = state.condition_wait_occurrence_counter;
11958 state.condition_wait_occurrence_counter = match ordinal.checked_add(1) {
11959 Some(next) => next,
11960 None => {
11961 return Poll::Ready(Err(Error::WorkerLoop(
11962 "condition wait occurrence counter overflowed".to_string(),
11963 )))
11964 }
11965 };
11966 let occurrence_id = format!("{CONDITION_WAIT_OCCURRENCE_PREFIX}{ordinal}");
11967 drop(state);
11968 self.occurrence_id = Some(occurrence_id.clone());
11969 occurrence_id
11970 }
11971 };
11972
11973 let recorded_result = {
11974 let mut state = match ctx.state.lock() {
11975 Ok(state) => state,
11976 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11977 };
11978 let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() else {
11979 drop(state);
11980 return self.poll_new_condition(options);
11981 };
11982 if !matches!(recorded, RecordedCommand::ConditionWait { .. }) {
11983 return Poll::Ready(Err(command_mismatch(&recorded, "condition wait")));
11984 }
11985
11986 let mut cursor = state.command_cursor;
11987 let mut result = None;
11988 loop {
11989 let Some(RecordedCommand::ConditionWait {
11990 sequence,
11991 occurrence_id: recorded_occurrence_id,
11992 condition_key,
11993 predicate_identity,
11994 timeout_seconds,
11995 result: recorded_result,
11996 parallel_group_path,
11997 ..
11998 }) = state.recorded_commands.get(cursor)
11999 else {
12000 break;
12001 };
12002
12003 if cursor > state.command_cursor && recorded_occurrence_id != &occurrence_id {
12004 break;
12005 }
12006 if let Err(error) = ensure_parallel_path_matches(
12007 *sequence,
12008 parallel_group_path.as_deref(),
12009 &self.parallel_group_path,
12010 ) {
12011 return Poll::Ready(Err(error));
12012 }
12013 if let Err(error) = validate_recorded_condition_wait(
12014 *sequence,
12015 recorded_occurrence_id,
12016 condition_key.as_deref(),
12017 predicate_identity,
12018 *timeout_seconds,
12019 &occurrence_id,
12020 &options,
12021 ) {
12022 return Poll::Ready(Err(error));
12023 }
12024 if result == Some(ConditionWaitResult::TimedOut) {
12025 return Poll::Ready(Err(Error::NonDeterministicReplay(ReplayFailure::new(
12026 "condition_wait_reopened_after_timeout",
12027 Some(*sequence),
12028 Some("timed-out condition is terminal".to_string()),
12029 Some("another physical wait-open".to_string()),
12030 "condition history reopened one logical wait after its durable timeout",
12031 ))));
12032 }
12033 result = *recorded_result;
12034 cursor += 1;
12035 }
12036 state.command_cursor = cursor;
12037 result
12038 };
12039
12040 if let Some(result) = recorded_result {
12041 return Poll::Ready(Ok(result));
12042 }
12043
12044 self.poll_open_condition(options)
12045 }
12046}
12047
12048impl ConditionWaitCall {
12049 fn poll_new_condition(
12050 self: Pin<&mut Self>,
12051 options: ValidatedConditionWaitOptions,
12052 ) -> Poll<Result<ConditionWaitResult>> {
12053 self.poll_open_condition(options)
12054 }
12055
12056 fn poll_open_condition(
12057 mut self: Pin<&mut Self>,
12058 options: ValidatedConditionWaitOptions,
12059 ) -> Poll<Result<ConditionWaitResult>> {
12060 let selection_member = self
12061 .parallel_group_path
12062 .first()
12063 .is_some_and(|entry| entry.parallel_group_mode.as_deref() == Some("select"));
12064 match (self.predicate)() {
12065 Ok(true) if !selection_member => {
12066 return Poll::Ready(Ok(ConditionWaitResult::Satisfied))
12067 }
12068 Ok(_) => {}
12069 Err(error) => return Poll::Ready(Err(error)),
12070 }
12071 if options.timeout_seconds == Some(0) && !selection_member {
12072 return Poll::Ready(Ok(ConditionWaitResult::TimedOut));
12073 }
12074
12075 let ctx = self.ctx.clone();
12076 let mut state = match ctx.state.lock() {
12077 Ok(state) => state,
12078 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
12079 };
12080 let mut command = serde_json::Map::from_iter([
12081 ("type".to_string(), json!("open_condition_wait")),
12082 (
12083 "condition_wait_occurrence_id".to_string(),
12084 json!(self.occurrence_id.as_deref().unwrap_or_default()),
12085 ),
12086 ("condition_key".to_string(), json!(options.condition_key)),
12087 (
12088 "condition_definition_fingerprint".to_string(),
12089 json!(options.predicate_identity),
12090 ),
12091 ]);
12092 if let Some(timeout_seconds) = options.timeout_seconds {
12093 command.insert("timeout_seconds".to_string(), json!(timeout_seconds));
12094 }
12095 apply_parallel_group_path(&mut command, &self.parallel_group_path);
12096 state.commands.push(Value::Object(command));
12097 drop(state);
12098 self.opened_wait = true;
12099 Poll::Pending
12100 }
12101}
12102
12103fn validate_recorded_condition_wait(
12104 sequence: u64,
12105 recorded_occurrence_id: &str,
12106 recorded_key: Option<&str>,
12107 recorded_predicate_identity: &str,
12108 recorded_timeout_seconds: Option<u64>,
12109 current_occurrence_id: &str,
12110 current: &ValidatedConditionWaitOptions,
12111) -> Result<()> {
12112 if recorded_occurrence_id != current_occurrence_id {
12113 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12114 "condition_wait_occurrence_mismatch",
12115 Some(sequence),
12116 Some(recorded_occurrence_id.to_string()),
12117 Some(current_occurrence_id.to_string()),
12118 "recorded condition occurrence differs from the current authored wait position",
12119 )));
12120 }
12121 if recorded_key != Some(current.condition_key.as_str()) {
12122 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12123 "condition_wait_key_mismatch",
12124 Some(sequence),
12125 recorded_key.map(str::to_string),
12126 Some(current.condition_key.clone()),
12127 "recorded condition identity differs from the current workflow wait",
12128 )));
12129 }
12130 if recorded_predicate_identity != current.predicate_identity {
12131 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12132 "condition_wait_predicate_mismatch",
12133 Some(sequence),
12134 Some(recorded_predicate_identity.to_string()),
12135 Some(current.predicate_identity.clone()),
12136 "recorded condition predicate behavior differs from current workflow code",
12137 )));
12138 }
12139 if recorded_timeout_seconds != current.timeout_seconds {
12140 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12141 "condition_wait_timeout_mismatch",
12142 Some(sequence),
12143 recorded_timeout_seconds.map(|seconds| format!("{seconds}s")),
12144 current.timeout_seconds.map(|seconds| format!("{seconds}s")),
12145 "recorded condition timeout differs from the current workflow wait",
12146 )));
12147 }
12148 Ok(())
12149}
12150
12151pub struct ChildWorkflowCall {
12153 ctx: WorkflowContext,
12154 workflow_type: String,
12155 options: ChildWorkflowOptions,
12156 args: Option<Result<AvroValue>>,
12157 scheduled: bool,
12158 matched_pending: bool,
12159 parallel_group_path: Vec<ParallelGroupMetadata>,
12160}
12161
12162impl ChildWorkflowCall {
12163 fn poll_avro_value(
12164 mut self: Pin<&mut Self>,
12165 _cx: &mut TaskContext<'_>,
12166 ) -> Poll<Result<ChildWorkflowAvroResult>> {
12167 if self.matched_pending {
12168 return Poll::Pending;
12169 }
12170
12171 let ctx = self.ctx.clone();
12172 let mut state = match ctx.state.lock() {
12173 Ok(state) => state,
12174 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
12175 };
12176
12177 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
12178 let sequence = recorded.sequence();
12179 match recorded {
12180 RecordedCommand::ChildWorkflow {
12181 workflow_type,
12182 outcome,
12183 parallel_group_path,
12184 ..
12185 } => {
12186 if let Err(error) = ensure_parallel_path_matches(
12187 sequence,
12188 parallel_group_path.as_deref(),
12189 &self.parallel_group_path,
12190 ) {
12191 return Poll::Ready(Err(error));
12192 }
12193 if let Some(recorded_type) = workflow_type {
12194 if recorded_type != self.workflow_type {
12195 return Poll::Ready(Err(Error::NonDeterministicReplay(
12196 ReplayFailure::new(
12197 "recorded_command_detail_mismatch",
12198 Some(sequence),
12199 Some(format!("child workflow:{recorded_type}")),
12200 Some(format!("child workflow:{}", self.workflow_type)),
12201 "recorded child workflow type differs from the current workflow command",
12202 ),
12203 )));
12204 }
12205 }
12206 state.command_cursor += 1;
12207 if let Some(outcome) = outcome {
12208 return Poll::Ready(outcome.map_err(Error::ChildWorkflowFailed));
12209 }
12210 state.matched_recorded_pending = true;
12211 self.scheduled = true;
12212 self.matched_pending = true;
12213 return Poll::Pending;
12214 }
12215 other => {
12216 return Poll::Ready(Err(command_mismatch(
12217 &other,
12218 format!("child workflow:{}", self.workflow_type),
12219 )));
12220 }
12221 }
12222 }
12223
12224 if !self.scheduled {
12225 if self.options.task_queue.trim().is_empty() {
12226 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
12227 "task_queue must not be empty".to_string(),
12228 )));
12229 }
12230 for (name, value) in [
12231 (
12232 "execution_timeout_seconds",
12233 self.options.execution_timeout_seconds,
12234 ),
12235 ("run_timeout_seconds", self.options.run_timeout_seconds),
12236 ] {
12237 if value == Some(0) {
12238 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(format!(
12239 "{name} must be at least 1"
12240 ))));
12241 }
12242 }
12243
12244 let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
12245 Ok(args) => args,
12246 Err(error) => return Poll::Ready(Err(error)),
12247 };
12248 let arguments = match encode_typed_envelope(
12249 &normalize_avro_arguments(args),
12250 &state.payload_codec,
12251 ) {
12252 Ok(arguments) => arguments,
12253 Err(error) => return Poll::Ready(Err(error)),
12254 };
12255 let mut command = json!({
12256 "type": "start_child_workflow",
12257 "workflow_type": self.workflow_type,
12258 "queue": self.options.task_queue,
12259 "parent_close_policy": self.options.parent_close_policy.as_str(),
12260 "arguments": arguments,
12261 });
12262 let object = command
12263 .as_object_mut()
12264 .expect("child workflow command is always an object");
12265 if let Some(policy) = &self.options.retry_policy {
12266 let mut retry_policy = serde_json::Map::new();
12267 if let Some(max_attempts) = policy.max_attempts {
12268 if max_attempts == 0 {
12269 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
12270 "retry_policy.max_attempts must be at least 1".to_string(),
12271 )));
12272 }
12273 retry_policy.insert("max_attempts".to_string(), json!(max_attempts));
12274 }
12275 if !policy.backoff_seconds.is_empty() {
12276 retry_policy
12277 .insert("backoff_seconds".to_string(), json!(policy.backoff_seconds));
12278 }
12279 if !policy.non_retryable_error_types.is_empty() {
12280 retry_policy.insert(
12281 "non_retryable_error_types".to_string(),
12282 json!(policy.non_retryable_error_types),
12283 );
12284 }
12285 if retry_policy.is_empty() {
12286 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
12287 "retry_policy must configure at least one field".to_string(),
12288 )));
12289 }
12290 object.insert("retry_policy".to_string(), Value::Object(retry_policy));
12291 }
12292 if let Some(seconds) = self.options.execution_timeout_seconds {
12293 object.insert("execution_timeout_seconds".to_string(), json!(seconds));
12294 }
12295 if let Some(seconds) = self.options.run_timeout_seconds {
12296 object.insert("run_timeout_seconds".to_string(), json!(seconds));
12297 }
12298 apply_parallel_group_path(object, &self.parallel_group_path);
12299 state.commands.push(command);
12300 self.scheduled = true;
12301 }
12302
12303 Poll::Pending
12304 }
12305}
12306
12307impl Future for ChildWorkflowCall {
12308 type Output = Result<ChildWorkflowResult>;
12309
12310 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
12311 match self.poll_avro_value(cx) {
12312 Poll::Ready(Ok(result)) => match result.result.into_json() {
12313 Ok(projected) => Poll::Ready(Ok(ChildWorkflowResult {
12314 parent: result.parent,
12315 child: result.child,
12316 child_workflow_type: result.child_workflow_type,
12317 result: projected,
12318 })),
12319 Err(error) => Poll::Ready(Err(error)),
12320 },
12321 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
12322 Poll::Pending => Poll::Pending,
12323 }
12324 }
12325}
12326
12327fn command_mismatch(recorded: &RecordedCommand, actual: impl Into<String>) -> Error {
12328 Error::NonDeterministicReplay(ReplayFailure::new(
12329 "recorded_command_mismatch",
12330 Some(recorded.sequence()),
12331 Some(recorded.shape().to_string()),
12332 Some(actual.into()),
12333 "current workflow command does not match the recorded durable command sequence",
12334 ))
12335}
12336
12337pub struct SignalCall {
12338 ctx: WorkflowContext,
12339 signal_name: String,
12340 runtime_reserved_allowed: bool,
12341 opened_wait: bool,
12342 matched_pending: bool,
12343 parallel_group_path: Vec<ParallelGroupMetadata>,
12344}
12345
12346impl SignalCall {
12347 fn poll_avro_value(
12348 mut self: Pin<&mut Self>,
12349 _cx: &mut TaskContext<'_>,
12350 ) -> Poll<Result<Vec<AvroValue>>> {
12351 if self.matched_pending {
12352 return Poll::Pending;
12353 }
12354 if !self.runtime_reserved_allowed {
12355 if let Err(error) = validate_user_signal_name(&self.signal_name) {
12356 return Poll::Ready(Err(error));
12357 }
12358 }
12359
12360 let ctx = self.ctx.clone();
12361 let mut state = match ctx.state.lock() {
12362 Ok(state) => state,
12363 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
12364 };
12365
12366 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
12367 match recorded {
12368 RecordedCommand::SignalWait {
12369 sequence,
12370 signal_name,
12371 value,
12372 parallel_group_path,
12373 } => {
12374 if let Err(error) = ensure_parallel_path_matches(
12375 sequence,
12376 parallel_group_path.as_deref(),
12377 &self.parallel_group_path,
12378 ) {
12379 return Poll::Ready(Err(error));
12380 }
12381 if signal_name != self.signal_name {
12382 return Poll::Ready(Err(Error::NonDeterministicReplay(
12383 ReplayFailure::new(
12384 "recorded_command_detail_mismatch",
12385 Some(sequence),
12386 Some(format!("signal wait:{signal_name}")),
12387 Some(format!("signal wait:{}", self.signal_name)),
12388 "recorded signal name differs from the current workflow command",
12389 ),
12390 )));
12391 }
12392
12393 state.command_cursor += 1;
12394 if let Some(value) = value {
12395 return Poll::Ready(Ok(value));
12396 }
12397 if state
12398 .resume_signal
12399 .as_ref()
12400 .is_some_and(|signal| signal.signal_name == self.signal_name)
12401 {
12402 let signal = state
12403 .resume_signal
12404 .take()
12405 .expect("matching resume signal is present");
12406 return Poll::Ready(Ok(signal.arguments));
12407 }
12408
12409 state.matched_recorded_pending = true;
12410 self.opened_wait = true;
12411 self.matched_pending = true;
12412 return Poll::Pending;
12413 }
12414 other => {
12415 return Poll::Ready(Err(command_mismatch(
12416 &other,
12417 format!("signal wait:{}", self.signal_name),
12418 )));
12419 }
12420 }
12421 }
12422
12423 if state
12424 .resume_signal
12425 .as_ref()
12426 .is_some_and(|signal| signal.signal_name == self.signal_name)
12427 {
12428 let signal = state
12429 .resume_signal
12430 .take()
12431 .expect("matching resume signal is present");
12432 return Poll::Ready(Ok(signal.arguments));
12433 }
12434
12435 if !self.opened_wait {
12436 let mut command = serde_json::Map::from_iter([
12437 ("type".to_string(), json!("open_signal_wait")),
12438 ("signal_name".to_string(), json!(self.signal_name)),
12439 ]);
12440 apply_parallel_group_path(&mut command, &self.parallel_group_path);
12441 state.commands.push(Value::Object(command));
12442 self.opened_wait = true;
12443 }
12444
12445 Poll::Pending
12446 }
12447}
12448
12449impl Future for SignalCall {
12450 type Output = Result<Vec<Value>>;
12451
12452 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
12453 match self.poll_avro_value(cx) {
12454 Poll::Ready(Ok(values)) => Poll::Ready(
12455 values
12456 .into_iter()
12457 .map(AvroValue::into_json)
12458 .collect::<Result<Vec<_>>>(),
12459 ),
12460 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
12461 Poll::Pending => Poll::Pending,
12462 }
12463 }
12464}
12465
12466#[derive(Clone, Debug)]
12467pub struct ActivityContext {
12468 client: Client,
12469 pub task_id: String,
12470 pub activity_attempt_id: String,
12471 pub lease_owner: String,
12472 pub activity_type: String,
12473 pub attempt_number: u64,
12474 pub task_queue: String,
12475 pub worker_id: String,
12476}
12477
12478impl ActivityContext {
12479 pub async fn heartbeat<T: Serialize>(&self, details: T) -> Result<ActivityHeartbeatResponse> {
12480 self.client
12481 .heartbeat_activity_task(
12482 &self.task_id,
12483 &self.activity_attempt_id,
12484 &self.lease_owner,
12485 details,
12486 )
12487 .await
12488 }
12489}
12490
12491fn decode_task_avro_arguments(value: Option<&Value>, codec: &str) -> Result<AvroValue> {
12492 validate_payload_codec(codec)?;
12493 match value {
12494 Some(value) => Ok(normalize_avro_arguments(decode_wire_avro_value(
12495 value, codec,
12496 )?)),
12497 None => Ok(AvroValue::Array(Vec::new())),
12498 }
12499}
12500
12501fn decode_resume_signal(task: &WorkflowTask) -> Result<Option<ResumeSignal>> {
12502 let Some(signal_name) = task
12503 .signal_name
12504 .as_deref()
12505 .filter(|value| !value.is_empty())
12506 else {
12507 return Ok(None);
12508 };
12509 let decoded = decode_task_avro_arguments(task.signal_arguments.as_ref(), &task.payload_codec)?;
12510 let AvroValue::Array(arguments) = decoded else {
12511 unreachable!("normalize_avro_arguments always returns an array");
12512 };
12513
12514 Ok(Some(ResumeSignal {
12515 signal_name: signal_name.to_string(),
12516 arguments,
12517 }))
12518}
12519
12520fn validate_workflow_task_payloads(task: &WorkflowTask) -> Result<()> {
12521 validate_payload_codec(&task.payload_codec)?;
12522 validate_optional_inbound_payload(task.arguments.as_ref(), &task.payload_codec)?;
12523 validate_optional_inbound_payload(task.signal_arguments.as_ref(), &task.payload_codec)?;
12524 for event in &task.history_events {
12525 validate_history_event_payloads(event, &task.payload_codec)?;
12526 }
12527 Ok(())
12528}
12529
12530fn validate_activity_task_payloads(task: &ActivityTask) -> Result<()> {
12531 validate_payload_codec(&task.payload_codec)?;
12532 validate_optional_inbound_payload(task.arguments.as_ref(), &task.payload_codec)
12533}
12534
12535fn validate_query_task_payloads(task: &QueryTask) -> Result<()> {
12536 validate_payload_codec(&task.payload_codec)?;
12537 validate_optional_inbound_payload(task.workflow_arguments.as_ref(), &task.payload_codec)?;
12538 validate_optional_inbound_payload(task.query_arguments.as_ref(), &task.payload_codec)?;
12539 for event in &task.history_events {
12540 validate_history_event_payloads(event, &task.payload_codec)?;
12541 }
12542
12543 let Some(export) = task.history_export.as_ref() else {
12544 return Ok(());
12545 };
12546 let export_codec = match export.get("payloads") {
12547 Some(payloads) => declared_payload_codec(payloads, "codec")?,
12548 None => None,
12549 }
12550 .unwrap_or(&task.payload_codec);
12551 validate_payload_codec(export_codec)?;
12552
12553 if let Some(events) = export.get("history_events").and_then(Value::as_array) {
12554 for event in events {
12555 let event_type = event
12556 .get("event_type")
12557 .or_else(|| event.get("type"))
12558 .and_then(Value::as_str)
12559 .unwrap_or_default();
12560 if let Some(payload) = event.get("payload") {
12561 validate_history_payloads(event_type, payload, export_codec)?;
12562 }
12563 }
12564 }
12565 for signal in export
12566 .get("signals")
12567 .and_then(Value::as_array)
12568 .into_iter()
12569 .flatten()
12570 {
12571 let codec = declared_payload_codec(signal, "payload_codec")?.unwrap_or(export_codec);
12572 validate_payload_codec(codec)?;
12573 validate_optional_inbound_payload(signal.get("arguments"), codec)?;
12574 }
12575 for activity in export
12576 .get("activities")
12577 .and_then(Value::as_array)
12578 .into_iter()
12579 .flatten()
12580 {
12581 let codec = declared_payload_codec(activity, "payload_codec")?.unwrap_or(export_codec);
12582 validate_payload_codec(codec)?;
12583 validate_optional_inbound_payload(activity.get("arguments"), codec)?;
12584 validate_optional_inbound_payload(activity.get("result"), codec)?;
12585 }
12586 Ok(())
12587}
12588
12589fn validate_history_event_payloads(event: &HistoryEvent, fallback_codec: &str) -> Result<()> {
12590 validate_history_payloads(&event.event_type, &event.payload, fallback_codec)
12591}
12592
12593fn validate_history_payloads(
12594 event_type: &str,
12595 payload: &Value,
12596 fallback_codec: &str,
12597) -> Result<()> {
12598 let codec = declared_payload_codec(payload, "payload_codec")?.unwrap_or(fallback_codec);
12599 validate_payload_codec(codec)?;
12600 for field in history_payload_fields(event_type) {
12601 validate_optional_inbound_payload(payload.get(*field), codec)?;
12602 }
12603 Ok(())
12604}
12605
12606const SIGNAL_HISTORY_PAYLOAD_FIELDS: &[&str] = &["value", "input", "arguments"];
12607
12608fn history_payload_fields(event_type: &str) -> &'static [&'static str] {
12609 match event_type {
12610 "ActivityCompleted" => &["result"],
12611 "SignalReceived" | "SignalApplied" => SIGNAL_HISTORY_PAYLOAD_FIELDS,
12612 "UpdateAccepted" | "UpdateRejected" | "UpdateApplied" => &["arguments"],
12613 "UpdateCompleted" | "SideEffectRecorded" => &["result"],
12614 "ChildRunCompleted" => &["result", "output"],
12615 "WorkflowCompleted" => &["output"],
12616 "ServiceCallStarted"
12617 | "ServiceCallCompleted"
12618 | "ServiceCallFailed"
12619 | "ServiceCallCancelled" => &["request_payload", "response_payload"],
12620 _ => &[],
12621 }
12622}
12623
12624fn signal_history_payload(payload: &Value) -> Option<&Value> {
12625 SIGNAL_HISTORY_PAYLOAD_FIELDS
12626 .iter()
12627 .find_map(|field| payload.get(*field))
12628}
12629
12630fn declared_payload_codec<'a>(value: &'a Value, field: &str) -> Result<Option<&'a str>> {
12631 match value.get(field) {
12632 None => Ok(None),
12633 Some(Value::String(codec)) => Ok(Some(codec)),
12634 Some(_) => Err(invalid_payload_envelope()),
12635 }
12636}
12637
12638fn validate_optional_inbound_payload(value: Option<&Value>, codec: &str) -> Result<()> {
12639 validate_payload_codec(codec)?;
12640 if let Some(value) = value.filter(|value| !value.is_null()) {
12641 decode_wire_avro_value(value, codec)?;
12642 }
12643 Ok(())
12644}
12645
12646fn recorded_parallel_group_entry(payload: &Value, sequence: u64) -> Result<ParallelGroupMetadata> {
12647 let group_id = payload_string(payload, "parallel_group_id").ok_or_else(|| {
12648 invalid_recorded_history(
12649 "parallel_group_metadata_invalid",
12650 sequence,
12651 "non-empty parallel_group_id",
12652 &payload.to_string(),
12653 "parallel-group history is missing its stable identity",
12654 )
12655 })?;
12656 let kind = payload_string(payload, "parallel_group_kind").ok_or_else(|| {
12657 invalid_recorded_history(
12658 "parallel_group_metadata_invalid",
12659 sequence,
12660 "activity, child, timer, signal, condition, or mixed group kind",
12661 &payload.to_string(),
12662 "parallel-group history is missing its group kind",
12663 )
12664 })?;
12665 if !matches!(
12666 kind.as_str(),
12667 "activity" | "child" | "timer" | "signal" | "condition" | "mixed"
12668 ) {
12669 return Err(invalid_recorded_history(
12670 "parallel_group_metadata_invalid",
12671 sequence,
12672 "activity, child, timer, signal, condition, or mixed group kind",
12673 &kind,
12674 "parallel-group history contains an unsupported group kind",
12675 ));
12676 }
12677 let base_sequence = payload
12678 .get("parallel_group_base_sequence")
12679 .and_then(value_as_u64)
12680 .filter(|value| *value > 0)
12681 .ok_or_else(|| {
12682 invalid_recorded_history(
12683 "parallel_group_metadata_invalid",
12684 sequence,
12685 "positive parallel_group_base_sequence",
12686 &payload.to_string(),
12687 "parallel-group history contains an invalid base sequence",
12688 )
12689 })?;
12690 let size = payload
12691 .get("parallel_group_size")
12692 .and_then(value_as_u64)
12693 .and_then(|value| usize::try_from(value).ok())
12694 .filter(|value| (1..=MAX_PARALLEL_OPERATIONS).contains(value))
12695 .ok_or_else(|| {
12696 invalid_recorded_history(
12697 "parallel_group_metadata_invalid",
12698 sequence,
12699 "bounded positive parallel_group_size",
12700 &payload.to_string(),
12701 "parallel-group history contains an invalid group size",
12702 )
12703 })?;
12704 let index = payload
12705 .get("parallel_group_index")
12706 .and_then(value_as_u64)
12707 .and_then(|value| usize::try_from(value).ok())
12708 .filter(|value| *value < size)
12709 .ok_or_else(|| {
12710 invalid_recorded_history(
12711 "parallel_group_metadata_invalid",
12712 sequence,
12713 "parallel_group_index within group bounds",
12714 &payload.to_string(),
12715 "parallel-group history contains an invalid member index",
12716 )
12717 })?;
12718 if base_sequence.checked_add(u64::try_from(index).unwrap_or(u64::MAX)) != Some(sequence) {
12719 return Err(invalid_recorded_history(
12720 "parallel_group_metadata_invalid",
12721 sequence,
12722 "base sequence plus member index equals workflow sequence",
12723 &payload.to_string(),
12724 "parallel-group path does not preserve durable workflow position",
12725 ));
12726 }
12727 let mode = payload
12728 .get("parallel_group_mode")
12729 .and_then(Value::as_str)
12730 .unwrap_or("all");
12731 if !matches!(mode, "all" | "select") {
12732 return Err(invalid_recorded_history(
12733 "parallel_group_metadata_invalid",
12734 sequence,
12735 "parallel group mode all or select",
12736 mode,
12737 "parallel-group history contains an unsupported group mode",
12738 ));
12739 }
12740 let expected_id = if mode == "select" {
12741 format!("select-calls:{base_sequence}:{size}")
12742 } else {
12743 format!("{}:{base_sequence}:{size}", parallel_group_prefix(&kind))
12744 };
12745 if group_id != expected_id {
12746 return Err(invalid_recorded_history(
12747 "parallel_group_metadata_invalid",
12748 sequence,
12749 &expected_id,
12750 &group_id,
12751 "parallel-group history contains an incompatible stable group ID",
12752 ));
12753 }
12754 let selection_member_key = if mode == "select" {
12755 Some(selection_key_from_value(
12756 payload.get("selection_member_key"),
12757 sequence,
12758 )?)
12759 } else {
12760 None
12761 };
12762 let selection_member_index = if mode == "select" {
12763 Some(required_parallel_usize(
12764 payload,
12765 "selection_member_index",
12766 sequence,
12767 )?)
12768 } else {
12769 None
12770 };
12771 let selection_member_base_sequence = if mode == "select" {
12772 Some(
12773 payload
12774 .get("selection_member_base_sequence")
12775 .and_then(value_as_u64)
12776 .filter(|value| *value >= base_sequence)
12777 .ok_or_else(|| {
12778 invalid_recorded_history(
12779 "parallel_group_metadata_invalid",
12780 sequence,
12781 "selection member base within its group",
12782 &payload.to_string(),
12783 "selection history contains an invalid member base sequence",
12784 )
12785 })?,
12786 )
12787 } else {
12788 None
12789 };
12790 let selection_member_size = if mode == "select" {
12791 let member_size = required_parallel_usize(payload, "selection_member_size", sequence)?;
12792 if member_size == 0 {
12793 return Err(invalid_recorded_history(
12794 "parallel_group_metadata_invalid",
12795 sequence,
12796 "positive selection member size",
12797 &payload.to_string(),
12798 "selection history contains an invalid member size",
12799 ));
12800 }
12801 Some(member_size)
12802 } else {
12803 None
12804 };
12805 let selection_member_kind = if mode == "select" {
12806 let kind = payload_string(payload, "selection_member_kind").ok_or_else(|| {
12807 invalid_recorded_history(
12808 "parallel_group_metadata_invalid",
12809 sequence,
12810 "selection member operation kind",
12811 &payload.to_string(),
12812 "selection history is missing its authored member kind",
12813 )
12814 })?;
12815 if !matches!(
12816 kind.as_str(),
12817 "activity" | "child" | "timer" | "signal" | "condition" | "group"
12818 ) {
12819 return Err(invalid_recorded_history(
12820 "parallel_group_metadata_invalid",
12821 sequence,
12822 "activity, child, timer, signal, condition, or group selection member kind",
12823 &kind,
12824 "selection history contains an unsupported member kind",
12825 ));
12826 }
12827 Some(kind)
12828 } else {
12829 None
12830 };
12831 if let (Some(member_base), Some(member_size)) =
12832 (selection_member_base_sequence, selection_member_size)
12833 {
12834 let member_end = member_base
12835 .checked_add(u64::try_from(member_size).unwrap_or(u64::MAX))
12836 .ok_or_else(|| {
12837 invalid_recorded_history(
12838 "parallel_group_metadata_invalid",
12839 sequence,
12840 "bounded selection member range",
12841 &payload.to_string(),
12842 "selection member range overflowed",
12843 )
12844 })?;
12845 let group_end = base_sequence
12846 .checked_add(u64::try_from(size).unwrap_or(u64::MAX))
12847 .unwrap_or(u64::MAX);
12848 if sequence < member_base || sequence >= member_end || member_end > group_end {
12849 return Err(invalid_recorded_history(
12850 "parallel_group_metadata_invalid",
12851 sequence,
12852 "workflow sequence within one bounded selection member",
12853 &payload.to_string(),
12854 "selection member range does not contain its durable leaf",
12855 ));
12856 }
12857 }
12858 Ok(ParallelGroupMetadata {
12859 parallel_group_id: group_id,
12860 parallel_group_kind: kind,
12861 parallel_group_base_sequence: base_sequence,
12862 parallel_group_size: size,
12863 parallel_group_index: index,
12864 parallel_group_mode: (mode == "select").then(|| "select".to_string()),
12865 selection_member_key,
12866 selection_member_index,
12867 selection_member_base_sequence,
12868 selection_member_size,
12869 selection_member_kind,
12870 })
12871}
12872
12873fn required_parallel_usize(payload: &Value, field: &str, sequence: u64) -> Result<usize> {
12874 payload
12875 .get(field)
12876 .and_then(value_as_u64)
12877 .and_then(|value| usize::try_from(value).ok())
12878 .ok_or_else(|| {
12879 invalid_recorded_history(
12880 "parallel_group_metadata_invalid",
12881 sequence,
12882 &format!("non-negative integer {field}"),
12883 &payload.to_string(),
12884 "selection history contains invalid member metadata",
12885 )
12886 })
12887}
12888
12889fn selection_key_from_value(value: Option<&Value>, sequence: u64) -> Result<SelectionKey> {
12890 match value {
12891 Some(Value::String(value)) if !value.is_empty() => Ok(SelectionKey::Name(value.clone())),
12892 Some(value) => value_as_u64(value)
12893 .and_then(|value| usize::try_from(value).ok())
12894 .map(SelectionKey::Index)
12895 .ok_or_else(|| {
12896 invalid_recorded_history(
12897 "selection_member_key_invalid",
12898 sequence,
12899 "non-empty string or non-negative integer member key",
12900 &value.to_string(),
12901 "selection history contains an invalid member key",
12902 )
12903 }),
12904 None => Err(invalid_recorded_history(
12905 "selection_member_key_missing",
12906 sequence,
12907 "selection_member_key",
12908 "<missing>",
12909 "selection history is missing its stable member key",
12910 )),
12911 }
12912}
12913
12914fn recorded_parallel_group_path(
12915 events: &[&HistoryEvent],
12916 sequence: u64,
12917) -> Result<Option<Vec<ParallelGroupMetadata>>> {
12918 let mut recorded: Option<Vec<ParallelGroupMetadata>> = None;
12919 for event in events {
12920 let payload = &event.payload;
12921 let has_metadata = payload.get("parallel_group_path").is_some()
12922 || payload.get("parallel_group_id").is_some()
12923 || payload.get("parallel_group_kind").is_some()
12924 || payload.get("parallel_group_base_sequence").is_some()
12925 || payload.get("parallel_group_size").is_some()
12926 || payload.get("parallel_group_index").is_some()
12927 || payload.get("parallel_group_mode").is_some()
12928 || payload.get("selection_member_key").is_some();
12929 if !has_metadata {
12930 continue;
12931 }
12932
12933 let top_level = recorded_parallel_group_entry(payload, sequence)?;
12934 let path = match payload.get("parallel_group_path") {
12935 None => vec![top_level.clone()],
12936 Some(Value::Array(entries)) if !entries.is_empty() => entries
12937 .iter()
12938 .map(|entry| recorded_parallel_group_entry(entry, sequence))
12939 .collect::<Result<Vec<_>>>()?,
12940 Some(value) => {
12941 return Err(invalid_recorded_history(
12942 "parallel_group_metadata_invalid",
12943 sequence,
12944 "non-empty parallel_group_path list",
12945 &value.to_string(),
12946 "parallel-group history contains an invalid group path",
12947 ));
12948 }
12949 };
12950 if path.last() != Some(&top_level) {
12951 return Err(invalid_recorded_history(
12952 "parallel_group_metadata_invalid",
12953 sequence,
12954 &serde_json::to_string(&path.last()).unwrap_or_default(),
12955 &serde_json::to_string(&top_level).unwrap_or_default(),
12956 "parallel-group top-level fields do not match the innermost path entry",
12957 ));
12958 }
12959 if recorded.as_ref().is_some_and(|existing| existing != &path) {
12960 return Err(invalid_recorded_history(
12961 "parallel_group_history_conflict",
12962 sequence,
12963 &serde_json::to_string(&recorded.as_ref()).unwrap_or_default(),
12964 &serde_json::to_string(&path).unwrap_or_default(),
12965 "parallel-group metadata changed between scheduling and resolution history",
12966 ));
12967 }
12968 recorded = Some(path);
12969 }
12970 Ok(recorded)
12971}
12972
12973fn recorded_commands(
12974 events: &[HistoryEvent],
12975 fallback_codec: &str,
12976 parent: WorkflowIdentity,
12977) -> Result<Vec<RecordedCommand>> {
12978 let mut events_by_sequence: BTreeMap<u64, Vec<&HistoryEvent>> = BTreeMap::new();
12979 let mut last_new_sequence = None;
12980
12981 for event in events {
12982 let is_activity = matches!(
12983 event.event_type.as_str(),
12984 "ActivityScheduled"
12985 | "ActivityStarted"
12986 | "ActivityHeartbeatRecorded"
12987 | "ActivityRetryScheduled"
12988 | "ActivityCompleted"
12989 | "ActivityFailed"
12990 | "ActivityCancelled"
12991 | "ActivityTimedOut"
12992 );
12993 let is_workflow_timer = matches!(
12994 event.event_type.as_str(),
12995 "TimerScheduled" | "TimerCancelled" | "TimerFired"
12996 ) && !is_internal_timer_event(event);
12997 let is_child_workflow = matches!(
12998 event.event_type.as_str(),
12999 "ChildWorkflowScheduled"
13000 | "ChildRunCompleted"
13001 | "ChildRunFailed"
13002 | "ChildRunCancelled"
13003 | "ChildRunTerminated"
13004 );
13005 let is_signal_wait = is_recorded_signal_wait_event(event);
13006 let is_condition_wait = is_recorded_condition_wait_event(event);
13007 let is_search_attributes = event.event_type == "SearchAttributesUpserted";
13008 let is_side_effect = event.event_type == "SideEffectRecorded";
13009 let is_version_marker = event.event_type == "VersionMarkerRecorded";
13010 let is_memo = event.event_type == "MemoUpserted";
13011 if !is_activity
13012 && !is_workflow_timer
13013 && !is_child_workflow
13014 && !is_signal_wait
13015 && !is_condition_wait
13016 && !is_search_attributes
13017 && !is_side_effect
13018 && !is_version_marker
13019 && !is_memo
13020 {
13021 continue;
13022 }
13023
13024 let sequence = durable_event_sequence(event).ok_or_else(|| {
13025 Error::NonDeterministicReplay(ReplayFailure::new(
13026 "durable_command_sequence_missing",
13027 None,
13028 Some("positive workflow sequence".to_string()),
13029 Some(event.event_type.clone()),
13030 "durable command history event has no workflow sequence",
13031 ))
13032 })?;
13033 if sequence == 0 {
13034 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
13035 "durable_command_sequence_invalid",
13036 Some(sequence),
13037 Some("positive workflow sequence".to_string()),
13038 Some(sequence.to_string()),
13039 "durable command history uses an invalid workflow sequence",
13040 )));
13041 }
13042 if !events_by_sequence.contains_key(&sequence) {
13043 if let Some(previous) = last_new_sequence {
13044 if sequence < previous {
13045 return Err(invalid_recorded_history(
13046 "durable_command_sequence_mismatch",
13047 sequence,
13048 &format!("workflow sequence greater than {previous}"),
13049 &sequence.to_string(),
13050 "durable commands are not strictly ordered by their recorded workflow sequence",
13051 ));
13052 }
13053 }
13054 last_new_sequence = Some(sequence);
13055 }
13056 events_by_sequence.entry(sequence).or_default().push(event);
13057 }
13058
13059 let commands: Vec<RecordedCommand> = events_by_sequence
13060 .into_iter()
13061 .map(|(sequence, sequence_events)| {
13062 let activity_events: Vec<_> = sequence_events
13063 .iter()
13064 .copied()
13065 .filter(|event| event.event_type.starts_with("Activity"))
13066 .collect();
13067 let timer_events: Vec<_> = sequence_events
13068 .iter()
13069 .copied()
13070 .filter(|event| event.event_type.starts_with("Timer"))
13071 .collect();
13072 let child_events: Vec<_> = sequence_events
13073 .iter()
13074 .copied()
13075 .filter(|event| {
13076 event.event_type == "ChildWorkflowScheduled"
13077 || event.event_type.starts_with("ChildRun")
13078 })
13079 .collect();
13080 let signal_wait_events: Vec<_> = sequence_events
13081 .iter()
13082 .copied()
13083 .filter(|event| is_recorded_signal_wait_event(event))
13084 .collect();
13085 let condition_wait_events: Vec<_> = sequence_events
13086 .iter()
13087 .copied()
13088 .filter(|event| is_recorded_condition_wait_event(event))
13089 .collect();
13090 let search_attribute_events: Vec<_> = sequence_events
13091 .iter()
13092 .copied()
13093 .filter(|event| event.event_type == "SearchAttributesUpserted")
13094 .collect();
13095 let side_effect_events: Vec<_> = sequence_events
13096 .iter()
13097 .copied()
13098 .filter(|event| event.event_type == "SideEffectRecorded")
13099 .collect();
13100 let version_marker_events: Vec<_> = sequence_events
13101 .iter()
13102 .copied()
13103 .filter(|event| event.event_type == "VersionMarkerRecorded")
13104 .collect();
13105 let memo_events: Vec<_> = sequence_events
13106 .iter()
13107 .copied()
13108 .filter(|event| event.event_type == "MemoUpserted")
13109 .collect();
13110
13111 let command_kind_count = usize::from(!activity_events.is_empty())
13112 + usize::from(!timer_events.is_empty())
13113 + usize::from(!child_events.is_empty())
13114 + usize::from(!signal_wait_events.is_empty())
13115 + usize::from(!condition_wait_events.is_empty())
13116 + usize::from(!search_attribute_events.is_empty())
13117 + usize::from(!side_effect_events.is_empty())
13118 + usize::from(!version_marker_events.is_empty())
13119 + usize::from(!memo_events.is_empty());
13120 if command_kind_count > 1 {
13121 let actual = [
13122 (!activity_events.is_empty()).then_some("activity"),
13123 (!timer_events.is_empty()).then_some("timer"),
13124 (!child_events.is_empty()).then_some("child workflow"),
13125 (!signal_wait_events.is_empty()).then_some("signal wait"),
13126 (!condition_wait_events.is_empty()).then_some("condition wait"),
13127 (!search_attribute_events.is_empty()).then_some("search-attribute update"),
13128 (!side_effect_events.is_empty()).then_some("side effect"),
13129 (!version_marker_events.is_empty()).then_some("version marker"),
13130 (!memo_events.is_empty()).then_some("memo upsert"),
13131 ]
13132 .into_iter()
13133 .flatten()
13134 .collect::<Vec<_>>()
13135 .join(" and ");
13136 return Err(invalid_recorded_history(
13137 "durable_command_sequence_collision",
13138 sequence,
13139 "one durable command kind",
13140 &actual,
13141 "one workflow sequence records more than one durable command kind",
13142 ));
13143 }
13144
13145 if !activity_events.is_empty() {
13146 let parallel_group_path =
13147 recorded_parallel_group_path(&activity_events, sequence)?;
13148 let scheduled_count = activity_events
13149 .iter()
13150 .filter(|event| event.event_type == "ActivityScheduled")
13151 .count();
13152 if scheduled_count > 1 {
13153 return Err(invalid_recorded_history(
13154 "duplicate_activity_schedule",
13155 sequence,
13156 "at most one ActivityScheduled event",
13157 "multiple ActivityScheduled events",
13158 "activity history schedules more than one command at one workflow sequence",
13159 ));
13160 }
13161 let activity_type = activity_events.iter().find_map(|event| {
13162 event
13163 .payload
13164 .get("activity_type")
13165 .or_else(|| event.payload.get("activity_name"))
13166 .and_then(Value::as_str)
13167 .map(str::to_string)
13168 });
13169 if activity_events.iter().filter_map(|event| {
13170 event
13171 .payload
13172 .get("activity_type")
13173 .or_else(|| event.payload.get("activity_name"))
13174 .and_then(Value::as_str)
13175 }).any(|candidate| Some(candidate) != activity_type.as_deref()) {
13176 return Err(invalid_recorded_history(
13177 "activity_identity_mismatch",
13178 sequence,
13179 activity_type.as_deref().unwrap_or("one activity identity"),
13180 "conflicting activity identities",
13181 "activity lifecycle events at one workflow sequence disagree on identity",
13182 ));
13183 }
13184 let terminal: Vec<_> = activity_events
13185 .iter()
13186 .copied()
13187 .filter(|event| {
13188 matches!(
13189 event.event_type.as_str(),
13190 "ActivityCompleted"
13191 | "ActivityFailed"
13192 | "ActivityCancelled"
13193 | "ActivityTimedOut"
13194 )
13195 })
13196 .collect();
13197 let duplicate_delivery = terminal.first().is_some_and(|first| {
13198 terminal.iter().all(|event| {
13199 event.event_type == first.event_type && event.payload == first.payload
13200 })
13201 });
13202 if terminal.len() > 1 && !duplicate_delivery {
13203 return Err(invalid_recorded_history(
13204 "duplicate_activity_terminal_event",
13205 sequence,
13206 "at most one terminal activity event",
13207 "multiple terminal activity events",
13208 "activity history settles one command more than once",
13209 ));
13210 }
13211 let outcome = terminal
13212 .first()
13213 .map(|event| activity_outcome(event, fallback_codec, activity_type.clone()))
13214 .transpose()?;
13215 let options = activity_events
13216 .iter()
13217 .find(|event| event.event_type == "ActivityScheduled")
13218 .and_then(|event| event.payload.get("activity"))
13219 .and_then(Value::as_object)
13220 .map(|activity| RecordedActivityOptions {
13221 task_queue: recorded_optional_string(activity, "queue"),
13222 execution_mode: recorded_optional_string(activity, "execution_mode"),
13223 retry_policy: recorded_activity_retry_snapshot(
13224 activity.get("retry_policy"),
13225 ),
13226 });
13227 return Ok(RecordedCommand::Activity {
13228 sequence,
13229 activity_type,
13230 options,
13231 outcome,
13232 parallel_group_path,
13233 });
13234 }
13235
13236 if !child_events.is_empty() {
13237 let parallel_group_path = recorded_parallel_group_path(&child_events, sequence)?;
13238 let scheduled: Vec<_> = child_events
13239 .iter()
13240 .copied()
13241 .filter(|event| event.event_type == "ChildWorkflowScheduled")
13242 .collect();
13243 if scheduled.len() != 1 {
13244 return Err(invalid_recorded_history(
13245 "child_workflow_schedule_missing_or_duplicate",
13246 sequence,
13247 "one ChildWorkflowScheduled event",
13248 &format!("{} ChildWorkflowScheduled events", scheduled.len()),
13249 "child workflow replay requires exactly one recorded schedule event",
13250 ));
13251 }
13252 let workflow_type = child_events.iter().find_map(|event| {
13253 event
13254 .payload
13255 .get("child_workflow_type")
13256 .or_else(|| event.payload.get("workflow_type"))
13257 .and_then(Value::as_str)
13258 .filter(|value| !value.is_empty())
13259 .map(str::to_string)
13260 });
13261 if child_events
13262 .iter()
13263 .filter_map(|event| {
13264 event
13265 .payload
13266 .get("child_workflow_type")
13267 .or_else(|| event.payload.get("workflow_type"))
13268 .and_then(Value::as_str)
13269 })
13270 .any(|candidate| Some(candidate) != workflow_type.as_deref())
13271 {
13272 return Err(invalid_recorded_history(
13273 "child_workflow_identity_mismatch",
13274 sequence,
13275 workflow_type
13276 .as_deref()
13277 .unwrap_or("one child workflow type"),
13278 "conflicting child workflow types",
13279 "child workflow lifecycle events at one sequence disagree on type",
13280 ));
13281 }
13282 let mut outcomes = child_workflow_outcomes(
13283 &child_events.iter().map(|event| (*event).clone()).collect::<Vec<_>>(),
13284 fallback_codec,
13285 parent.clone(),
13286 )?;
13287 let terminal_events = child_events
13288 .iter()
13289 .copied()
13290 .filter(|event| event.event_type.starts_with("ChildRun"))
13291 .collect::<Vec<_>>();
13292 let duplicate_delivery = terminal_events.first().is_some_and(|first| {
13293 terminal_events.iter().all(|event| {
13294 event.event_type == first.event_type && event.payload == first.payload
13295 })
13296 });
13297 if outcomes.len() > 1 && !duplicate_delivery {
13298 return Err(invalid_recorded_history(
13299 "duplicate_child_workflow_terminal_event",
13300 sequence,
13301 "at most one terminal child event",
13302 "multiple terminal child events",
13303 "child workflow history settles one command more than once",
13304 ));
13305 }
13306 return Ok(RecordedCommand::ChildWorkflow {
13307 sequence,
13308 workflow_type,
13309 outcome: outcomes.pop(),
13310 parallel_group_path,
13311 });
13312 }
13313
13314 if !signal_wait_events.is_empty() {
13315 let opened: Vec<_> = signal_wait_events
13316 .iter()
13317 .copied()
13318 .filter(|event| event.event_type == "SignalWaitOpened")
13319 .collect();
13320 if opened.len() != 1 {
13321 return Err(invalid_recorded_history(
13322 "signal_wait_open_missing_or_duplicate",
13323 sequence,
13324 "one SignalWaitOpened event",
13325 &format!("{} SignalWaitOpened events", opened.len()),
13326 "signal replay requires exactly one canonical wait-open event",
13327 ));
13328 }
13329
13330 let applied: Vec<_> = signal_wait_events
13331 .iter()
13332 .copied()
13333 .filter(|event| event.event_type == "SignalApplied")
13334 .collect();
13335 if applied.len() > 1 {
13336 return Err(invalid_recorded_history(
13337 "duplicate_signal_wait_apply",
13338 sequence,
13339 "at most one SignalApplied event",
13340 "multiple SignalApplied events",
13341 "signal history applies one durable wait more than once",
13342 ));
13343 }
13344
13345 let signal_names = signal_wait_events
13346 .iter()
13347 .map(|event| required_signal_wait_name(event, sequence))
13348 .collect::<Result<Vec<_>>>()?;
13349 let signal_name = signal_names
13350 .first()
13351 .expect("signal wait events are not empty")
13352 .clone();
13353 if signal_names.iter().any(|candidate| candidate != &signal_name) {
13354 return Err(invalid_recorded_history(
13355 "signal_wait_identity_mismatch",
13356 sequence,
13357 &signal_name,
13358 "conflicting signal names",
13359 "signal wait lifecycle events at one workflow sequence disagree on identity",
13360 ));
13361 }
13362 let value = applied
13363 .first()
13364 .map(|event| decode_signal_event_arguments(event, fallback_codec))
13365 .transpose()?;
13366 return Ok(RecordedCommand::SignalWait {
13367 sequence,
13368 signal_name,
13369 value,
13370 parallel_group_path: recorded_parallel_group_path(
13371 &signal_wait_events,
13372 sequence,
13373 )?,
13374 });
13375 }
13376
13377 if !condition_wait_events.is_empty() {
13378 return recorded_condition_wait(
13379 sequence,
13380 &condition_wait_events,
13381 events,
13382 );
13383 }
13384
13385 if !search_attribute_events.is_empty() {
13386 if search_attribute_events.len() != 1 {
13387 return Err(invalid_recorded_history(
13388 "duplicate_search_attribute_update",
13389 sequence,
13390 "one SearchAttributesUpserted event",
13391 &format!(
13392 "{} SearchAttributesUpserted events",
13393 search_attribute_events.len()
13394 ),
13395 "search-attribute history records one workflow command more than once",
13396 ));
13397 }
13398 let payload = &search_attribute_events[0].payload;
13399 let attributes = payload
13400 .get("attributes")
13401 .filter(|value| value.as_object().is_some_and(|values| !values.is_empty()))
13402 .cloned()
13403 .ok_or_else(|| {
13404 invalid_recorded_history(
13405 "search_attribute_update_missing",
13406 sequence,
13407 "non-empty attributes object",
13408 "missing or invalid attributes",
13409 "search-attribute history is missing its recorded mutation",
13410 )
13411 })?;
13412 let attribute_types =
13413 recorded_search_attribute_types(payload, &attributes, sequence)?;
13414 return Ok(RecordedCommand::SearchAttributes {
13415 sequence,
13416 attributes,
13417 attribute_types,
13418 });
13419 }
13420
13421 if !side_effect_events.is_empty() {
13422 if side_effect_events.len() != 1 {
13423 return Err(invalid_recorded_history(
13424 "duplicate_side_effect_record",
13425 sequence,
13426 "one SideEffectRecorded event",
13427 &format!("{} SideEffectRecorded events", side_effect_events.len()),
13428 "side-effect history records one workflow command more than once",
13429 ));
13430 }
13431 let event = side_effect_events[0];
13432 let result = event.payload.get("result").ok_or_else(|| {
13433 invalid_recorded_history(
13434 "side_effect_result_missing",
13435 sequence,
13436 "recorded result payload",
13437 "missing result",
13438 "side-effect history is missing its recorded value",
13439 )
13440 })?;
13441 let has_published_envelope = result.as_str().is_some()
13442 || result.as_object().is_some_and(|envelope| {
13443 envelope.get("codec").and_then(Value::as_str).is_some()
13444 && envelope.get("blob").and_then(Value::as_str).is_some()
13445 });
13446 if !has_published_envelope {
13447 return Err(invalid_recorded_history(
13448 "side_effect_payload_malformed",
13449 sequence,
13450 "payload blob or {codec, blob} envelope",
13451 &result.to_string(),
13452 "side-effect history result does not use a published payload envelope",
13453 ));
13454 }
13455 let codec = event
13456 .payload
13457 .get("payload_codec")
13458 .and_then(Value::as_str)
13459 .unwrap_or(fallback_codec);
13460 let value = decode_wire_avro_value(result, codec).map_err(|error| {
13461 if error.to_string().contains("unsupported_payload_codec") {
13462 return error;
13463 }
13464
13465 invalid_recorded_history(
13466 "side_effect_payload_incompatible",
13467 sequence,
13468 &format!("valid {codec} payload envelope"),
13469 &error.to_string(),
13470 "side-effect history payload cannot be decoded with its recorded codec",
13471 )
13472 })?;
13473 return Ok(RecordedCommand::SideEffect { sequence, value });
13474 }
13475
13476 if !version_marker_events.is_empty() {
13477 if version_marker_events.len() != 1 {
13478 return Err(invalid_recorded_history(
13479 "duplicate_version_marker_record",
13480 sequence,
13481 "one VersionMarkerRecorded event",
13482 &format!("{} VersionMarkerRecorded events", version_marker_events.len()),
13483 "version-marker history records one workflow command more than once",
13484 ));
13485 }
13486 let payload = &version_marker_events[0].payload;
13487 let change_id = payload
13488 .get("change_id")
13489 .and_then(Value::as_str)
13490 .filter(|value| !value.is_empty())
13491 .map(str::to_string)
13492 .ok_or_else(|| {
13493 invalid_recorded_history(
13494 "version_marker_field_missing",
13495 sequence,
13496 "non-empty change_id",
13497 "missing or invalid change_id",
13498 "version-marker history is missing its stable change ID",
13499 )
13500 })?;
13501 let version = required_version_i32(payload, "version", sequence)?;
13502 let min_supported = required_version_i32(payload, "min_supported", sequence)?;
13503 let max_supported = required_version_i32(payload, "max_supported", sequence)?;
13504 if min_supported > max_supported || version < min_supported || version > max_supported {
13505 return Err(invalid_recorded_history(
13506 "version_marker_history_range_invalid",
13507 sequence,
13508 "min_supported <= version <= max_supported",
13509 &format!("{min_supported} <= {version} <= {max_supported}"),
13510 "recorded version marker contains an internally incompatible range",
13511 ));
13512 }
13513 return Ok(RecordedCommand::VersionMarker {
13514 sequence,
13515 change_id,
13516 version,
13517 });
13518 }
13519
13520 if !memo_events.is_empty() {
13521 if memo_events.len() != 1 {
13522 return Err(invalid_recorded_history(
13523 "duplicate_memo_upsert_record",
13524 sequence,
13525 "one MemoUpserted event",
13526 &format!("{} MemoUpserted events", memo_events.len()),
13527 "memo history records one workflow update more than once",
13528 ));
13529 }
13530 let payload = &memo_events[0].payload;
13531 let entries = payload.get("entries").cloned().ok_or_else(|| {
13532 invalid_recorded_history(
13533 "memo_entries_missing",
13534 sequence,
13535 "memo entries object",
13536 "missing entries",
13537 "MemoUpserted history is missing replay identity entries",
13538 )
13539 })?;
13540 let entries = decode_memo_history_map(&entries, true).map_err(|error| {
13541 invalid_recorded_history(
13542 "memo_entries_invalid",
13543 sequence,
13544 "valid canonical memo entries",
13545 &error.to_string(),
13546 "MemoUpserted history contains invalid replay identity entries",
13547 )
13548 })?;
13549 let merged = payload.get("merged").cloned().ok_or_else(|| {
13550 invalid_recorded_history(
13551 "memo_merged_projection_missing",
13552 sequence,
13553 "merged memo projection",
13554 "missing merged",
13555 "MemoUpserted history is missing its merged projection",
13556 )
13557 })?;
13558 decode_memo_history_map(&merged, false).map_err(|error| {
13559 invalid_recorded_history(
13560 "memo_merged_projection_invalid",
13561 sequence,
13562 "valid merged memo projection",
13563 &error.to_string(),
13564 "MemoUpserted history contains an invalid merged projection",
13565 )
13566 })?;
13567
13568 return Ok(RecordedCommand::Memo { sequence, entries });
13569 }
13570 let scheduled: Vec<_> = timer_events
13571 .iter()
13572 .copied()
13573 .filter(|event| event.event_type == "TimerScheduled")
13574 .collect();
13575 let fired: Vec<_> = timer_events
13576 .iter()
13577 .copied()
13578 .filter(|event| event.event_type == "TimerFired")
13579 .collect();
13580 if scheduled.len() != 1 {
13581 return Err(invalid_recorded_history(
13582 "timer_schedule_missing_or_duplicate",
13583 sequence,
13584 "one TimerScheduled event",
13585 &format!("{} TimerScheduled events", scheduled.len()),
13586 "timer replay requires exactly one recorded schedule event",
13587 ));
13588 }
13589 if fired.len() > 1 {
13590 return Err(invalid_recorded_history(
13591 "duplicate_timer_fire",
13592 sequence,
13593 "at most one TimerFired event",
13594 "multiple TimerFired events",
13595 "timer history contains more than one fire event for a workflow sequence",
13596 ));
13597 }
13598
13599 let scheduled = scheduled[0];
13600 let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
13601 let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
13602 if let Some(fired) = fired.first() {
13603 let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
13604 if fired_timer_id != timer_id {
13605 return Err(invalid_recorded_history(
13606 "timer_identity_mismatch",
13607 sequence,
13608 &timer_id,
13609 &fired_timer_id,
13610 "TimerFired does not correspond to the recorded TimerScheduled event",
13611 ));
13612 }
13613 let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
13614 if fired_delay != delay_seconds {
13615 return Err(invalid_recorded_history(
13616 "timer_history_delay_mismatch",
13617 sequence,
13618 &delay_seconds.to_string(),
13619 &fired_delay.to_string(),
13620 "TimerScheduled and TimerFired record different delays",
13621 ));
13622 }
13623 }
13624
13625 Ok(RecordedCommand::Timer {
13626 sequence,
13627 delay_seconds,
13628 fired: !fired.is_empty(),
13629 parallel_group_path: recorded_parallel_group_path(&timer_events, sequence)?,
13630 })
13631 })
13632 .collect::<Result<_>>()?;
13633
13634 let mut marker_sequences = HashMap::new();
13635 for command in &commands {
13636 if let RecordedCommand::VersionMarker {
13637 sequence,
13638 change_id,
13639 ..
13640 } = command
13641 {
13642 if let Some(first_sequence) = marker_sequences.insert(change_id.clone(), *sequence) {
13643 return Err(invalid_recorded_history(
13644 "duplicate_version_marker",
13645 *sequence,
13646 &format!("one marker for change ID {change_id:?}"),
13647 &format!("markers at sequences {first_sequence} and {sequence}"),
13648 "workflow history contains duplicate markers for one stable change ID",
13649 ));
13650 }
13651 }
13652 }
13653
13654 Ok(commands)
13655}
13656
13657fn required_version_i32(payload: &Value, field: &str, sequence: u64) -> Result<i32> {
13658 payload
13659 .get(field)
13660 .and_then(Value::as_i64)
13661 .and_then(|value| i32::try_from(value).ok())
13662 .ok_or_else(|| {
13663 invalid_recorded_history(
13664 "version_marker_field_missing",
13665 sequence,
13666 &format!("integer {field}"),
13667 "missing or out-of-range integer",
13668 "version-marker history is missing a required integer field",
13669 )
13670 })
13671}
13672
13673fn durable_event_sequence(event: &HistoryEvent) -> Option<u64> {
13674 event
13675 .payload
13676 .get("sequence")
13677 .or_else(|| event.payload.get("workflow_sequence"))
13678 .or_else(|| event.raw.get("sequence"))
13679 .or_else(|| event.raw.get("workflow_sequence"))
13680 .and_then(value_as_u64)
13681}
13682
13683fn is_internal_timer_event(event: &HistoryEvent) -> bool {
13684 matches!(
13685 event
13686 .payload
13687 .get("timer_kind")
13688 .or_else(|| event.raw.get("timer_kind"))
13689 .and_then(Value::as_str),
13690 Some("condition_timeout" | "signal_timeout")
13691 )
13692}
13693
13694fn is_recorded_condition_wait_event(event: &HistoryEvent) -> bool {
13695 matches!(
13696 event.event_type.as_str(),
13697 "ConditionWaitOpened" | "ConditionWaitSatisfied" | "ConditionWaitTimedOut"
13698 )
13699}
13700
13701fn recorded_condition_wait(
13702 sequence: u64,
13703 condition_events: &[&HistoryEvent],
13704 all_events: &[HistoryEvent],
13705) -> Result<RecordedCommand> {
13706 let opened = condition_events
13707 .iter()
13708 .copied()
13709 .filter(|event| event.event_type == "ConditionWaitOpened")
13710 .collect::<Vec<_>>();
13711 if opened.len() != 1 {
13712 return Err(invalid_recorded_history(
13713 "condition_wait_open_missing_or_duplicate",
13714 sequence,
13715 "one ConditionWaitOpened event",
13716 &format!("{} ConditionWaitOpened events", opened.len()),
13717 "condition replay requires exactly one canonical wait-open event",
13718 ));
13719 }
13720 let terminal = condition_events
13721 .iter()
13722 .copied()
13723 .filter(|event| {
13724 matches!(
13725 event.event_type.as_str(),
13726 "ConditionWaitSatisfied" | "ConditionWaitTimedOut"
13727 )
13728 })
13729 .collect::<Vec<_>>();
13730 if terminal.len() > 1 {
13731 return Err(invalid_recorded_history(
13732 "duplicate_condition_wait_terminal_event",
13733 sequence,
13734 "at most one condition terminal event",
13735 "multiple condition terminal events",
13736 "condition history settles one durable wait more than once",
13737 ));
13738 }
13739
13740 let opened = opened[0];
13741 let condition_wait_id = required_condition_wait_id(opened, sequence)?;
13742 let occurrence_id = required_condition_wait_occurrence_id(opened, sequence)?;
13743 for event in condition_events
13744 .iter()
13745 .copied()
13746 .filter(|event| !std::ptr::eq(*event, opened))
13747 {
13748 let event_wait_id = required_condition_wait_id(event, sequence)?;
13749 if event_wait_id != condition_wait_id {
13750 return Err(invalid_recorded_history(
13751 "condition_wait_id_mismatch",
13752 sequence,
13753 &condition_wait_id,
13754 &event_wait_id,
13755 "condition lifecycle events at one sequence disagree on wait identity",
13756 ));
13757 }
13758 let event_occurrence_id = required_condition_wait_occurrence_id(event, sequence)?;
13759 if event_occurrence_id != occurrence_id {
13760 return Err(invalid_recorded_history(
13761 "condition_wait_occurrence_history_mismatch",
13762 sequence,
13763 &occurrence_id,
13764 &event_occurrence_id,
13765 "condition lifecycle events at one sequence disagree on authored occurrence identity",
13766 ));
13767 }
13768 }
13769
13770 let condition_key = optional_non_empty_history_string(opened, "condition_key");
13771 let predicate_identity = opened
13772 .payload
13773 .get("condition_definition_fingerprint")
13774 .and_then(Value::as_str)
13775 .filter(|value| !value.is_empty())
13776 .map(str::to_string)
13777 .ok_or_else(|| {
13778 invalid_recorded_history(
13779 "condition_wait_predicate_fingerprint_missing",
13780 sequence,
13781 "non-empty condition_definition_fingerprint",
13782 &opened.event_type,
13783 "canonical condition history is missing its predicate identity",
13784 )
13785 })?;
13786 let timeout_seconds = optional_history_u64(opened, "timeout_seconds", sequence)?;
13787 for event in condition_events
13788 .iter()
13789 .copied()
13790 .filter(|event| !std::ptr::eq(*event, opened))
13791 {
13792 for (field, opened_value) in [
13793 ("condition_key", condition_key.as_deref()),
13794 (
13795 "condition_definition_fingerprint",
13796 Some(predicate_identity.as_str()),
13797 ),
13798 ] {
13799 if let Some(value) = optional_non_empty_history_string(event, field) {
13800 if opened_value.is_some_and(|opened_value| opened_value != value) {
13801 return Err(invalid_recorded_history(
13802 "condition_wait_definition_history_mismatch",
13803 sequence,
13804 opened_value.unwrap_or_default(),
13805 &value,
13806 "condition lifecycle events disagree on the recorded definition",
13807 ));
13808 }
13809 }
13810 }
13811 if let Some(event_timeout) = optional_history_u64(event, "timeout_seconds", sequence)? {
13812 if timeout_seconds.is_some_and(|opened_timeout| opened_timeout != event_timeout) {
13813 return Err(invalid_recorded_history(
13814 "condition_wait_definition_history_mismatch",
13815 sequence,
13816 &format!("{}s", timeout_seconds.unwrap_or_default()),
13817 &format!("{event_timeout}s"),
13818 "condition lifecycle events disagree on the recorded timeout",
13819 ));
13820 }
13821 }
13822 }
13823
13824 let timeout_timer_events = all_events
13825 .iter()
13826 .filter(|event| {
13827 matches!(
13828 event.event_type.as_str(),
13829 "TimerScheduled" | "TimerCancelled" | "TimerFired"
13830 ) && event.payload.get("timer_kind").and_then(Value::as_str)
13831 == Some("condition_timeout")
13832 && event
13833 .payload
13834 .get("condition_wait_id")
13835 .and_then(Value::as_str)
13836 == Some(condition_wait_id.as_str())
13837 })
13838 .collect::<Vec<_>>();
13839 let scheduled = timeout_timer_events
13840 .iter()
13841 .copied()
13842 .filter(|event| event.event_type == "TimerScheduled")
13843 .collect::<Vec<_>>();
13844 let fired = timeout_timer_events
13845 .iter()
13846 .copied()
13847 .filter(|event| event.event_type == "TimerFired")
13848 .collect::<Vec<_>>();
13849 if scheduled.len() > 1 || fired.len() > 1 || (!fired.is_empty() && scheduled.len() != 1) {
13850 return Err(invalid_recorded_history(
13851 "condition_wait_timeout_history_invalid",
13852 sequence,
13853 "one timeout schedule and at most one fire",
13854 &format!("{} schedules and {} fires", scheduled.len(), fired.len()),
13855 "condition timeout history has a missing or duplicate lifecycle event",
13856 ));
13857 }
13858 if let Some(scheduled) = scheduled.first() {
13859 let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
13860 let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
13861 if timeout_seconds.is_some_and(|timeout| timeout != delay_seconds) {
13862 return Err(invalid_recorded_history(
13863 "condition_wait_timeout_delay_mismatch",
13864 sequence,
13865 &format!("{}s", timeout_seconds.unwrap_or_default()),
13866 &format!("{delay_seconds}s"),
13867 "condition timeout timer differs from the wait definition",
13868 ));
13869 }
13870 if let Some(fired) = fired.first() {
13871 let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
13872 let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
13873 if fired_timer_id != timer_id || fired_delay != delay_seconds {
13874 return Err(invalid_recorded_history(
13875 "condition_wait_timeout_identity_mismatch",
13876 sequence,
13877 &format!("{timer_id}:{delay_seconds}s"),
13878 &format!("{fired_timer_id}:{fired_delay}s"),
13879 "condition timeout fire does not match its durable schedule",
13880 ));
13881 }
13882 }
13883 }
13884
13885 let result = terminal.first().map(|event| {
13886 if event.event_type == "ConditionWaitTimedOut" {
13887 ConditionWaitResult::TimedOut
13888 } else {
13889 ConditionWaitResult::Satisfied
13890 }
13891 });
13892 let result = if !fired.is_empty() {
13893 if result == Some(ConditionWaitResult::Satisfied) {
13894 return Err(invalid_recorded_history(
13895 "condition_wait_terminal_conflict",
13896 sequence,
13897 "one satisfied or timed-out outcome",
13898 "satisfied event and fired timeout",
13899 "condition history records conflicting terminal outcomes",
13900 ));
13901 }
13902 Some(ConditionWaitResult::TimedOut)
13903 } else {
13904 result
13905 };
13906
13907 Ok(RecordedCommand::ConditionWait {
13908 sequence,
13909 occurrence_id,
13910 condition_key,
13911 predicate_identity,
13912 timeout_seconds,
13913 result,
13914 parallel_group_path: recorded_parallel_group_path(condition_events, sequence)?,
13915 })
13916}
13917
13918fn required_condition_wait_occurrence_id(event: &HistoryEvent, sequence: u64) -> Result<String> {
13919 event
13920 .payload
13921 .get("condition_wait_occurrence_id")
13922 .and_then(Value::as_str)
13923 .filter(|value| !value.is_empty())
13924 .map(str::to_string)
13925 .ok_or_else(|| {
13926 invalid_recorded_history(
13927 "condition_wait_occurrence_id_missing",
13928 sequence,
13929 "non-empty condition_wait_occurrence_id",
13930 &event.event_type,
13931 "condition history is missing authored occurrence identity",
13932 )
13933 })
13934}
13935
13936fn required_condition_wait_id(event: &HistoryEvent, sequence: u64) -> Result<String> {
13937 event
13938 .payload
13939 .get("condition_wait_id")
13940 .and_then(Value::as_str)
13941 .filter(|value| !value.is_empty())
13942 .map(str::to_string)
13943 .ok_or_else(|| {
13944 invalid_recorded_history(
13945 "condition_wait_id_missing",
13946 sequence,
13947 "non-empty condition_wait_id",
13948 &event.event_type,
13949 "canonical condition history is missing its durable wait identity",
13950 )
13951 })
13952}
13953
13954fn optional_non_empty_history_string(event: &HistoryEvent, field: &str) -> Option<String> {
13955 event
13956 .payload
13957 .get(field)
13958 .and_then(Value::as_str)
13959 .filter(|value| !value.is_empty())
13960 .map(str::to_string)
13961}
13962
13963fn optional_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<Option<u64>> {
13964 match event.payload.get(field) {
13965 None | Some(Value::Null) => Ok(None),
13966 Some(value) => value_as_u64(value).map(Some).ok_or_else(|| {
13967 invalid_recorded_history(
13968 "condition_wait_definition_invalid",
13969 sequence,
13970 &format!("non-negative integer {field}"),
13971 &value.to_string(),
13972 "condition history contains an invalid numeric definition field",
13973 )
13974 }),
13975 }
13976}
13977
13978fn required_signal_wait_name(event: &HistoryEvent, sequence: u64) -> Result<String> {
13979 event
13980 .payload
13981 .get("signal_name")
13982 .or_else(|| event.raw.get("signal_name"))
13983 .and_then(Value::as_str)
13984 .filter(|value| !value.is_empty())
13985 .map(str::to_string)
13986 .ok_or_else(|| {
13987 invalid_recorded_history(
13988 "signal_wait_name_missing",
13989 sequence,
13990 "non-empty signal_name",
13991 &event.event_type,
13992 "canonical signal-wait history is missing its signal identity",
13993 )
13994 })
13995}
13996
13997fn is_recorded_signal_wait_event(event: &HistoryEvent) -> bool {
13998 matches!(
13999 event.event_type.as_str(),
14000 "SignalWaitOpened" | "SignalApplied"
14001 )
14002}
14003
14004fn required_history_string(event: &HistoryEvent, field: &str, sequence: u64) -> Result<String> {
14005 event
14006 .payload
14007 .get(field)
14008 .and_then(Value::as_str)
14009 .filter(|value| !value.is_empty())
14010 .map(str::to_string)
14011 .ok_or_else(|| {
14012 invalid_recorded_history(
14013 "timer_history_field_missing",
14014 sequence,
14015 field,
14016 &event.event_type,
14017 "timer history is missing a required identity field",
14018 )
14019 })
14020}
14021
14022fn required_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<u64> {
14023 event
14024 .payload
14025 .get(field)
14026 .and_then(value_as_u64)
14027 .ok_or_else(|| {
14028 invalid_recorded_history(
14029 "timer_history_field_missing",
14030 sequence,
14031 field,
14032 &event.event_type,
14033 "timer history is missing a required numeric field",
14034 )
14035 })
14036}
14037
14038fn recorded_search_attribute_types(
14039 payload: &Value,
14040 attributes: &Value,
14041 sequence: u64,
14042) -> Result<RecordedSnapshotValue<BTreeMap<String, String>>> {
14043 let Some(raw_types) = payload.get("attribute_types") else {
14044 return Ok(RecordedSnapshotValue::Unknown);
14048 };
14049 let Some(raw_types) = raw_types.as_object() else {
14050 return Err(invalid_recorded_history(
14051 "search_attribute_types_malformed",
14052 sequence,
14053 "canonical attribute type map",
14054 &raw_types.to_string(),
14055 "search-attribute history contains malformed type identity",
14056 ));
14057 };
14058 let attribute_keys = attributes
14059 .as_object()
14060 .expect("recorded search attributes were validated as an object");
14061 let mut types = BTreeMap::new();
14062 for (key, value) in raw_types {
14063 let Some(attribute_type) = value.as_str() else {
14064 return Err(invalid_recorded_history(
14065 "search_attribute_types_malformed",
14066 sequence,
14067 "canonical string type name",
14068 &value.to_string(),
14069 "search-attribute history contains a non-string type identity",
14070 ));
14071 };
14072 if !attribute_keys.contains_key(key)
14073 || !matches!(
14074 attribute_type,
14075 "string" | "keyword" | "keyword_list" | "int" | "float" | "bool" | "datetime"
14076 )
14077 {
14078 return Err(invalid_recorded_history(
14079 "search_attribute_types_malformed",
14080 sequence,
14081 "canonical types for keys present in attributes",
14082 &format!("{key}:{attribute_type}"),
14083 "search-attribute history contains unsupported or orphaned type identity",
14084 ));
14085 }
14086 types.insert(key.clone(), attribute_type.to_string());
14087 }
14088 Ok(RecordedSnapshotValue::Known(types))
14089}
14090
14091fn invalid_recorded_history(
14092 reason: &str,
14093 sequence: u64,
14094 expected: &str,
14095 actual: &str,
14096 message: &str,
14097) -> Error {
14098 Error::NonDeterministicReplay(ReplayFailure::new(
14099 reason,
14100 Some(sequence),
14101 Some(expected.to_string()),
14102 Some(actual.to_string()),
14103 message,
14104 ))
14105}
14106
14107type ActivityOutcome = std::result::Result<AvroValue, ActivityFailure>;
14108
14109fn activity_outcome(
14110 event: &HistoryEvent,
14111 fallback_codec: &str,
14112 recorded_activity_type: Option<String>,
14113) -> Result<ActivityOutcome> {
14114 if event.event_type == "ActivityCompleted" {
14115 let codec = event
14116 .payload
14117 .get("payload_codec")
14118 .and_then(Value::as_str)
14119 .unwrap_or(fallback_codec);
14120 return Ok(Ok(decode_wire_avro_value(
14121 event.payload.get("result").unwrap_or(&Value::Null),
14122 codec,
14123 )?));
14124 }
14125
14126 let payload = &event.payload;
14127 let (kind, fallback_reason, fallback_message) = match event.event_type.as_str() {
14128 "ActivityFailed" => (ActivityFailureKind::Failed, "activity", "activity failed"),
14129 "ActivityCancelled" => (
14130 ActivityFailureKind::Cancelled,
14131 "cancelled",
14132 "activity was cancelled",
14133 ),
14134 "ActivityTimedOut" => (
14135 ActivityFailureKind::TimedOut,
14136 "timeout",
14137 "activity timed out",
14138 ),
14139 _ => unreachable!("activity_outcome is called only for terminal activity events"),
14140 };
14141 let exception = payload
14142 .get("exception")
14143 .filter(|value| !value.is_null())
14144 .cloned();
14145 let failure_category = payload_string(payload, "failure_category");
14146 let timeout_kind = payload_string(payload, "timeout_kind");
14147 let reason = payload_string(payload, "reason").unwrap_or_else(|| match kind {
14148 ActivityFailureKind::Failed => failure_category
14149 .clone()
14150 .unwrap_or_else(|| fallback_reason.to_string()),
14151 ActivityFailureKind::Cancelled => fallback_reason.to_string(),
14152 ActivityFailureKind::TimedOut => timeout_kind
14153 .clone()
14154 .unwrap_or_else(|| fallback_reason.to_string()),
14155 });
14156 let message = payload_string(payload, "message")
14157 .or_else(|| {
14158 exception
14159 .as_ref()
14160 .and_then(|value| payload_string(value, "message"))
14161 })
14162 .unwrap_or_else(|| fallback_message.to_string());
14163
14164 Ok(Err(ActivityFailure {
14165 kind,
14166 reason,
14167 message,
14168 activity_execution_id: payload_string(payload, "activity_execution_id"),
14169 activity_attempt_id: payload_string(payload, "activity_attempt_id"),
14170 activity_type: payload_string(payload, "activity_type")
14171 .or_else(|| payload_string(payload, "activity_name"))
14172 .or(recorded_activity_type),
14173 activity_class: payload_string(payload, "activity_class"),
14174 attempt_number: payload.get("attempt_number").and_then(value_as_u64),
14175 failure_id: payload_string(payload, "failure_id"),
14176 failure_category,
14177 timeout_kind,
14178 non_retryable: payload
14179 .get("non_retryable")
14180 .and_then(Value::as_bool)
14181 .unwrap_or(false),
14182 exception_type: payload_string(payload, "exception_type").or_else(|| {
14183 exception
14184 .as_ref()
14185 .and_then(|value| payload_string(value, "type"))
14186 }),
14187 exception_class: payload_string(payload, "exception_class").or_else(|| {
14188 exception
14189 .as_ref()
14190 .and_then(|value| payload_string(value, "class"))
14191 }),
14192 code: payload
14193 .get("code")
14194 .filter(|value| !value.is_null())
14195 .cloned(),
14196 exception,
14197 }))
14198}
14199
14200type ChildWorkflowOutcome = std::result::Result<ChildWorkflowAvroResult, ChildWorkflowFailure>;
14201
14202fn child_workflow_outcomes(
14203 events: &[HistoryEvent],
14204 fallback_codec: &str,
14205 parent: WorkflowIdentity,
14206) -> Result<Vec<ChildWorkflowOutcome>> {
14207 let mut outcomes = Vec::new();
14208
14209 for event in events {
14210 let kind = match event.event_type.as_str() {
14211 "ChildRunCompleted" => None,
14212 "ChildRunFailed" => Some((
14213 ChildWorkflowFailureKind::Failed,
14214 "child_workflow",
14215 "child workflow failed",
14216 )),
14217 "ChildRunCancelled" => Some((
14218 ChildWorkflowFailureKind::Cancelled,
14219 "cancelled",
14220 "child workflow was cancelled",
14221 )),
14222 "ChildRunTerminated" => Some((
14223 ChildWorkflowFailureKind::Terminated,
14224 "terminated",
14225 "child workflow was terminated",
14226 )),
14227 _ => continue,
14228 };
14229 let payload = &event.payload;
14230 let child_workflow_id = payload_string(payload, "child_workflow_instance_id");
14231 let child_workflow_run_id = payload_string(payload, "child_workflow_run_id");
14232 let child_workflow_type = payload_string(payload, "child_workflow_type");
14233
14234 if let Some((kind, reason, fallback_message)) = kind {
14235 let exception = payload
14236 .get("exception")
14237 .filter(|value| !value.is_null())
14238 .cloned();
14239 let message = payload_string(payload, "message")
14240 .or_else(|| {
14241 exception
14242 .as_ref()
14243 .and_then(|value| payload_string(value, "message"))
14244 })
14245 .unwrap_or_else(|| fallback_message.to_string());
14246 let exception_type = payload_string(payload, "exception_type").or_else(|| {
14247 exception
14248 .as_ref()
14249 .and_then(|value| payload_string(value, "type"))
14250 });
14251 let exception_class = payload_string(payload, "exception_class").or_else(|| {
14252 exception
14253 .as_ref()
14254 .and_then(|value| payload_string(value, "class"))
14255 });
14256 outcomes.push(Err(ChildWorkflowFailure {
14257 kind,
14258 reason: reason.to_string(),
14259 message,
14260 parent_workflow_id: parent.workflow_id.clone(),
14261 parent_workflow_run_id: parent.run_id.clone(),
14262 child_workflow_id,
14263 child_workflow_run_id,
14264 child_workflow_type,
14265 failure_id: payload_string(payload, "failure_id"),
14266 failure_category: payload_string(payload, "failure_category"),
14267 exception_type,
14268 exception_class,
14269 non_retryable: payload
14270 .get("non_retryable")
14271 .and_then(Value::as_bool)
14272 .unwrap_or(false),
14273 code: payload
14274 .get("code")
14275 .filter(|value| !value.is_null())
14276 .cloned(),
14277 exception,
14278 }));
14279 continue;
14280 }
14281
14282 let codec = payload
14283 .get("payload_codec")
14284 .and_then(Value::as_str)
14285 .unwrap_or(fallback_codec);
14286 let result = payload
14287 .get("result")
14288 .or_else(|| payload.get("output"))
14289 .unwrap_or(&Value::Null);
14290 outcomes.push(Ok(ChildWorkflowAvroResult {
14291 parent: parent.clone(),
14292 child: WorkflowIdentity {
14293 workflow_id: child_workflow_id,
14294 run_id: child_workflow_run_id,
14295 },
14296 child_workflow_type,
14297 result: decode_wire_avro_value(result, codec)?,
14298 }));
14299 }
14300
14301 Ok(outcomes)
14302}
14303
14304fn payload_string(payload: &Value, key: &str) -> Option<String> {
14305 payload
14306 .get(key)
14307 .and_then(Value::as_str)
14308 .filter(|value| !value.is_empty())
14309 .map(str::to_string)
14310}
14311
14312fn workflow_failure_command(error: &Error) -> Value {
14313 let (exception_type, exception_class, properties) = match error {
14314 Error::ActivityFailed(failure) => (
14315 match failure.kind {
14316 ActivityFailureKind::Failed => "ActivityFailed",
14317 ActivityFailureKind::Cancelled => "ActivityCancelled",
14318 ActivityFailureKind::TimedOut => "ActivityTimedOut",
14319 },
14320 "durable_workflow::ActivityFailure",
14321 json!({
14322 "reason": failure.reason,
14323 "activity_execution_id": failure.activity_execution_id,
14324 "activity_attempt_id": failure.activity_attempt_id,
14325 "activity_type": failure.activity_type,
14326 "activity_class": failure.activity_class,
14327 "attempt_number": failure.attempt_number,
14328 "failure_id": failure.failure_id,
14329 "failure_category": failure.failure_category,
14330 "timeout_kind": failure.timeout_kind,
14331 "activity_non_retryable": failure.non_retryable,
14332 "activity_exception_type": failure.exception_type,
14333 "activity_exception_class": failure.exception_class,
14334 "activity_code": failure.code,
14335 "activity_exception": failure.exception,
14336 }),
14337 ),
14338 Error::ChildWorkflowFailed(failure) => (
14339 match failure.kind {
14340 ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
14341 ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
14342 ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
14343 },
14344 "durable_workflow::ChildWorkflowFailure",
14345 json!({
14346 "reason": failure.reason,
14347 "parent_workflow_id": failure.parent_workflow_id,
14348 "parent_workflow_run_id": failure.parent_workflow_run_id,
14349 "child_workflow_id": failure.child_workflow_id,
14350 "child_workflow_run_id": failure.child_workflow_run_id,
14351 "child_workflow_type": failure.child_workflow_type,
14352 "failure_id": failure.failure_id,
14353 "failure_category": failure.failure_category,
14354 "child_exception_type": failure.exception_type,
14355 "child_exception_class": failure.exception_class,
14356 "child_non_retryable": failure.non_retryable,
14357 "child_code": failure.code,
14358 "child_exception": failure.exception,
14359 }),
14360 ),
14361 Error::ParallelFailed(failure) => (
14362 "ParallelFailed",
14363 "durable_workflow::ParallelFailure",
14364 json!({
14365 "parallel_group_id": failure.group_id,
14366 "parallel_member_path": failure.member_path,
14367 "parallel_group_path": failure.group_path,
14368 "completed_members": failure.completed.iter().map(|completion| &completion.member_path).collect::<Vec<_>>(),
14369 "cause_type": workflow_error_type(&failure.cause),
14370 "cause_message": failure.cause.to_string(),
14371 }),
14372 ),
14373 Error::SagaCompensationFailed(failure) => (
14374 "SagaCompensationFailed",
14375 "durable_workflow::SagaCompensationFailure",
14376 json!({
14377 "initiating_failure_type": workflow_error_type(&failure.initiating_failure),
14378 "initiating_failure_message": failure.initiating_failure.to_string(),
14379 "compensation_activity_type": failure.compensation_activity_type,
14380 "compensation_registration_order": failure.compensation_registration_order,
14381 "compensation_failure_type": workflow_error_type(&failure.compensation_failure),
14382 "compensation_failure_message": failure.compensation_failure.to_string(),
14383 }),
14384 ),
14385 Error::WorkflowCancellationRequested(_) => (
14386 "WorkflowCancellationRequested",
14387 "durable_workflow::WorkflowCancellationRequested",
14388 json!({"reason": "cancelled"}),
14389 ),
14390 Error::NonDeterministicReplay(_) => (
14391 "NonDeterministicReplay",
14392 "durable_workflow::Error",
14393 Value::Null,
14394 ),
14395 _ => ("RustWorkflowError", "durable_workflow::Error", Value::Null),
14396 };
14397 let non_retryable = match error {
14398 Error::ActivityFailed(failure) => failure.non_retryable,
14399 Error::ChildWorkflowFailed(failure) => failure.non_retryable,
14400 Error::ParallelFailed(failure) => workflow_error_non_retryable(&failure.cause),
14401 Error::SagaCompensationFailed(failure) => {
14402 workflow_error_non_retryable(&failure.compensation_failure)
14403 }
14404 Error::WorkflowCancellationRequested(_) => true,
14405 Error::NonDeterministicReplay(_) => true,
14406 _ => false,
14407 };
14408
14409 json!({
14410 "type": "fail_workflow",
14411 "message": error.to_string(),
14412 "exception_type": exception_type,
14413 "exception_class": exception_class,
14414 "non_retryable": non_retryable,
14415 "exception": {
14416 "type": exception_type,
14417 "class": exception_class,
14418 "message": error.to_string(),
14419 "properties": properties,
14420 }
14421 })
14422}
14423
14424fn workflow_error_type(error: &Error) -> &'static str {
14425 match error {
14426 Error::ActivityFailed(failure) => match failure.kind {
14427 ActivityFailureKind::Failed => "ActivityFailed",
14428 ActivityFailureKind::Cancelled => "ActivityCancelled",
14429 ActivityFailureKind::TimedOut => "ActivityTimedOut",
14430 },
14431 Error::ChildWorkflowFailed(failure) => match failure.kind {
14432 ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
14433 ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
14434 ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
14435 },
14436 Error::ParallelFailed(_) => "ParallelFailed",
14437 Error::SagaCompensationFailed(_) => "SagaCompensationFailed",
14438 Error::WorkflowCancellationRequested(_) => "WorkflowCancellationRequested",
14439 Error::NonDeterministicReplay(_) => "NonDeterministicReplay",
14440 _ => "RustWorkflowError",
14441 }
14442}
14443
14444fn workflow_error_non_retryable(error: &Error) -> bool {
14445 match error {
14446 Error::ActivityFailed(failure) => failure.non_retryable,
14447 Error::ChildWorkflowFailed(failure) => failure.non_retryable,
14448 Error::ParallelFailed(failure) => workflow_error_non_retryable(&failure.cause),
14449 Error::SagaCompensationFailed(failure) => {
14450 workflow_error_non_retryable(&failure.compensation_failure)
14451 }
14452 Error::WorkflowCancellationRequested(_) | Error::NonDeterministicReplay(_) => true,
14453 _ => false,
14454 }
14455}
14456
14457fn workflow_task_integrity_error(error: &Error) -> bool {
14458 matches!(
14459 error,
14460 Error::NonDeterministicReplay(_)
14461 | Error::Protocol(_)
14462 | Error::MissingWorkflowCommandIdentity
14463 | Error::WorkflowStatePoisoned
14464 )
14465}
14466
14467fn decode_signal_event_arguments(
14468 event: &HistoryEvent,
14469 fallback_codec: &str,
14470) -> Result<Vec<AvroValue>> {
14471 let codec = declared_payload_codec(&event.payload, "payload_codec")?.unwrap_or(fallback_codec);
14472 validate_payload_codec(codec)?;
14473 let raw = signal_history_payload(&event.payload);
14474 let decoded = match raw.filter(|value| !value.is_null()) {
14475 Some(value) => decode_wire_avro_value(value, codec)?,
14476 None => AvroValue::Array(Vec::new()),
14477 };
14478 let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
14479 unreachable!("normalize_avro_arguments always returns an array");
14480 };
14481 Ok(arguments)
14482}
14483
14484fn decode_update_event_arguments(
14485 event: &HistoryEvent,
14486 fallback_codec: &str,
14487) -> Result<Vec<AvroValue>> {
14488 let codec = declared_payload_codec(&event.payload, "payload_codec")?.unwrap_or(fallback_codec);
14489 validate_payload_codec(codec)?;
14490 let decoded = match event
14491 .payload
14492 .get("arguments")
14493 .filter(|value| !value.is_null())
14494 {
14495 Some(value) => decode_wire_avro_value(value, codec)?,
14496 None => AvroValue::Array(Vec::new()),
14497 };
14498 let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
14499 unreachable!("normalize_avro_arguments always returns an array");
14500 };
14501 Ok(arguments)
14502}
14503
14504fn hydrate_query_history_from_export(task: &mut QueryTask) -> Result<()> {
14505 let Some(export_events) = task
14506 .history_export
14507 .as_ref()
14508 .and_then(|export| export.get("history_events"))
14509 .and_then(Value::as_array)
14510 else {
14511 return Ok(());
14512 };
14513
14514 if export_events.len() > task.history_events.len() {
14515 task.history_events = serde_json::from_value(Value::Array(export_events.clone()))?;
14516 }
14517
14518 Ok(())
14519}
14520
14521fn enrich_query_history_from_export(task: &mut QueryTask) -> Result<()> {
14522 let Some(export) = task.history_export.as_ref() else {
14523 return Ok(());
14524 };
14525 let signals = export
14526 .get("signals")
14527 .and_then(Value::as_array)
14528 .cloned()
14529 .unwrap_or_default();
14530 let activities = export
14531 .get("activities")
14532 .and_then(Value::as_array)
14533 .cloned()
14534 .unwrap_or_default();
14535 let export_codec = export
14536 .get("payloads")
14537 .and_then(|payloads| payloads.get("codec"))
14538 .and_then(Value::as_str)
14539 .unwrap_or(&task.payload_codec)
14540 .to_string();
14541 let mut signal_name_offsets: HashMap<String, usize> = HashMap::new();
14542
14543 for event in &mut task.history_events {
14544 if event.event_type == "ActivityCompleted" {
14545 let sequence = event
14546 .payload
14547 .get("sequence")
14548 .or_else(|| event.payload.get("workflow_sequence"))
14549 .and_then(value_as_u64);
14550 let Some(activity) = sequence.and_then(|sequence| {
14551 activities.iter().find(|activity| {
14552 activity.get("sequence").and_then(value_as_u64) == Some(sequence)
14553 })
14554 }) else {
14555 continue;
14556 };
14557 let Some(payload) = event.payload.as_object_mut() else {
14558 continue;
14559 };
14560 if missing_payload(payload.get("result")) {
14561 if let Some(result) = activity
14562 .get("result")
14563 .filter(|value| !missing_payload(Some(value)))
14564 {
14565 payload.insert("result".to_string(), result.clone());
14566 }
14567 }
14568 for field in ["payload_codec", "activity_type"] {
14569 if payload
14570 .get(field)
14571 .and_then(Value::as_str)
14572 .unwrap_or_default()
14573 .is_empty()
14574 {
14575 if let Some(value) = activity.get(field) {
14576 payload.insert(field.to_string(), value.clone());
14577 }
14578 }
14579 }
14580 continue;
14581 }
14582
14583 if event.event_type != "SignalReceived" && event.event_type != "SignalApplied" {
14584 continue;
14585 }
14586 let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
14587 let command_id = event
14588 .payload
14589 .get("workflow_command_id")
14590 .or_else(|| event.raw.get("workflow_command_id"))
14591 .and_then(Value::as_str);
14592 let signal_name = event
14593 .payload
14594 .get("signal_name")
14595 .and_then(Value::as_str)
14596 .unwrap_or_default()
14597 .to_string();
14598 let matched = signals
14599 .iter()
14600 .find(|signal| {
14601 signal_id.is_some() && signal.get("id").and_then(Value::as_str) == signal_id
14602 })
14603 .or_else(|| {
14604 signals.iter().find(|signal| {
14605 command_id.is_some()
14606 && signal.get("command_id").and_then(Value::as_str) == command_id
14607 })
14608 })
14609 .or_else(|| {
14610 let offset = signal_name_offsets.entry(signal_name.clone()).or_default();
14611 let signal = signals
14612 .iter()
14613 .filter(|signal| {
14614 signal.get("name").and_then(Value::as_str) == Some(signal_name.as_str())
14615 })
14616 .nth(*offset);
14617 if signal.is_some() {
14618 *offset += 1;
14619 }
14620 signal
14621 });
14622 let Some(signal) = matched else {
14623 continue;
14624 };
14625 let signal_codec = signal
14626 .get("payload_codec")
14627 .and_then(Value::as_str)
14628 .unwrap_or(&export_codec);
14629 let Some(payload) = event.payload.as_object_mut() else {
14630 continue;
14631 };
14632 if missing_payload(payload.get("arguments")) {
14633 if let Some(arguments) = signal
14634 .get("arguments")
14635 .filter(|value| !missing_payload(Some(value)))
14636 {
14637 let envelope = match arguments {
14638 Value::String(blob) => json!({"codec": signal_codec, "blob": blob}),
14639 other => other.clone(),
14640 };
14641 payload.insert("arguments".to_string(), envelope);
14642 }
14643 }
14644 if payload
14645 .get("payload_codec")
14646 .and_then(Value::as_str)
14647 .unwrap_or_default()
14648 .is_empty()
14649 {
14650 payload.insert("payload_codec".to_string(), json!(signal_codec));
14651 }
14652 }
14653
14654 Ok(())
14655}
14656
14657fn missing_payload(value: Option<&Value>) -> bool {
14658 match value {
14659 None | Some(Value::Null) => true,
14660 Some(Value::String(value)) => value.is_empty(),
14661 Some(_) => false,
14662 }
14663}
14664
14665fn query_signal_events(task: &QueryTask) -> Result<Vec<QuerySignal>> {
14666 let export_signals = task
14667 .history_export
14668 .as_ref()
14669 .and_then(|export| export.get("signals"))
14670 .and_then(Value::as_array)
14671 .cloned()
14672 .unwrap_or_default();
14673 let export_codec = task
14674 .history_export
14675 .as_ref()
14676 .and_then(|export| export.get("payloads"))
14677 .and_then(|payloads| payloads.get("codec"))
14678 .and_then(Value::as_str)
14679 .unwrap_or(&task.payload_codec);
14680 let mut name_offsets: HashMap<String, usize> = HashMap::new();
14681 let mut signals = Vec::new();
14682
14683 for event in &task.history_events {
14684 if event.event_type != "SignalApplied" && event.event_type != "SignalReceived" {
14685 continue;
14686 }
14687
14688 let name = event
14689 .payload
14690 .get("signal_name")
14691 .and_then(Value::as_str)
14692 .unwrap_or_default();
14693 if name.is_empty() {
14694 continue;
14695 }
14696 let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
14697 let command_id = event
14698 .payload
14699 .get("workflow_command_id")
14700 .or_else(|| event.raw.get("workflow_command_id"))
14701 .and_then(Value::as_str);
14702 let matched_export = export_signals
14703 .iter()
14704 .find(|candidate| {
14705 signal_id.is_some() && candidate.get("id").and_then(Value::as_str) == signal_id
14706 })
14707 .or_else(|| {
14708 export_signals.iter().find(|candidate| {
14709 command_id.is_some()
14710 && candidate.get("command_id").and_then(Value::as_str) == command_id
14711 })
14712 })
14713 .or_else(|| {
14714 let offset = name_offsets.entry(name.to_string()).or_default();
14715 let candidate = export_signals
14716 .iter()
14717 .filter(|candidate| candidate.get("name").and_then(Value::as_str) == Some(name))
14718 .nth(*offset);
14719 if candidate.is_some() {
14720 *offset += 1;
14721 }
14722 candidate
14723 });
14724 let codec = event
14725 .payload
14726 .get("payload_codec")
14727 .and_then(Value::as_str)
14728 .or_else(|| {
14729 matched_export
14730 .and_then(|signal| signal.get("payload_codec"))
14731 .and_then(Value::as_str)
14732 })
14733 .unwrap_or(export_codec);
14734 let raw_arguments = signal_history_payload(&event.payload)
14735 .filter(|value| !value.is_null())
14736 .or_else(|| matched_export.and_then(|signal| signal.get("arguments")));
14737 let (arguments, avro_arguments) = decode_query_signal_arguments(raw_arguments, codec)?;
14738 let workflow_sequence = event
14739 .payload
14740 .get("workflow_sequence")
14741 .and_then(value_as_u64)
14742 .or_else(|| {
14743 matched_export
14744 .and_then(|signal| signal.get("workflow_sequence"))
14745 .and_then(value_as_u64)
14746 });
14747
14748 signals.push(QuerySignal {
14749 id: signal_id.map(str::to_string).or_else(|| {
14750 matched_export
14751 .and_then(|signal| signal.get("id"))
14752 .and_then(Value::as_str)
14753 .map(str::to_string)
14754 }),
14755 name: name.to_string(),
14756 arguments,
14757 avro_arguments,
14758 workflow_sequence,
14759 });
14760 }
14761
14762 if signals.is_empty() {
14763 for signal in export_signals {
14764 if signal.get("status").and_then(Value::as_str) == Some("rejected") {
14765 continue;
14766 }
14767 let Some(name) = signal.get("name").and_then(Value::as_str) else {
14768 continue;
14769 };
14770 let codec = signal
14771 .get("payload_codec")
14772 .and_then(Value::as_str)
14773 .unwrap_or(export_codec);
14774 let (arguments, avro_arguments) =
14775 decode_query_signal_arguments(signal.get("arguments"), codec)?;
14776 signals.push(QuerySignal {
14777 id: signal.get("id").and_then(Value::as_str).map(str::to_string),
14778 name: name.to_string(),
14779 arguments,
14780 avro_arguments,
14781 workflow_sequence: signal.get("workflow_sequence").and_then(value_as_u64),
14782 });
14783 }
14784 signals.sort_by_key(|signal| signal.workflow_sequence.unwrap_or(u64::MAX));
14785 }
14786
14787 Ok(signals)
14788}
14789
14790fn decode_query_signal_arguments(
14791 raw: Option<&Value>,
14792 codec: &str,
14793) -> Result<(Vec<Value>, Vec<AvroValue>)> {
14794 validate_payload_codec(codec)?;
14795 let decoded = match raw.filter(|value| !value.is_null()) {
14796 Some(value) => decode_wire_avro_value(value, codec)?,
14797 None => AvroValue::Array(Vec::new()),
14798 };
14799 let AvroValue::Array(avro_arguments) = normalize_avro_arguments(decoded) else {
14800 unreachable!("normalize_avro_arguments always returns an array");
14801 };
14802 let arguments = avro_arguments
14803 .iter()
14804 .cloned()
14805 .map(AvroValue::into_json)
14806 .collect::<Result<Vec<_>>>()?;
14807 Ok((arguments, avro_arguments))
14808}
14809
14810fn value_as_u64(value: &Value) -> Option<u64> {
14811 value
14812 .as_u64()
14813 .or_else(|| value.as_str().and_then(|value| value.parse().ok()))
14814}
14815
14816#[cfg(test)]
14817mod tests {
14818 use super::*;
14819 mod runtime_payloads;
14820 mod runtime_uploads;
14821 use std::{
14822 fs,
14823 io::{Read, Write},
14824 net::{SocketAddr, TcpListener, TcpStream},
14825 process::Command as ProcessCommand,
14826 sync::atomic::AtomicUsize,
14827 thread,
14828 };
14829
14830 #[derive(Clone, Copy, Debug)]
14831 enum InvalidTaskPayloadCodec {
14832 Missing,
14833 Null,
14834 NonString,
14835 }
14836
14837 impl InvalidTaskPayloadCodec {
14838 fn label(self) -> &'static str {
14839 match self {
14840 Self::Missing => "missing",
14841 Self::Null => "null",
14842 Self::NonString => "non-string",
14843 }
14844 }
14845
14846 fn apply(self, task: &mut Value) {
14847 let task = task.as_object_mut().expect("task fixture object");
14848 match self {
14849 Self::Missing => {
14850 task.remove("payload_codec");
14851 }
14852 Self::Null => {
14853 task.insert("payload_codec".to_string(), Value::Null);
14854 }
14855 Self::NonString => {
14856 task.insert("payload_codec".to_string(), json!(42));
14857 }
14858 }
14859 }
14860 }
14861
14862 fn fixture_envelope(value: Value) -> Value {
14863 encode_value_envelope(&value, DEFAULT_CODEC).expect("encode Avro test fixture")
14864 }
14865
14866 fn fixture_blob(value: Value) -> String {
14867 encode_payload(&value, DEFAULT_CODEC)
14868 .expect("encode Avro test fixture")
14869 .blob
14870 }
14871
14872 #[test]
14873 fn client_builder_rejects_the_sdk_owned_api_suffix() {
14874 for base_url in [
14875 "http://127.0.0.1:8080/api",
14876 "http://localhost:8080/api/",
14877 "https://runtime.example.test/namespaces/orders/api",
14878 ] {
14879 let error = Client::builder(base_url)
14880 .build()
14881 .expect_err("SDK-owned /api suffix must be rejected during build");
14882
14883 assert!(matches!(error, Error::InvalidBaseUrl), "{base_url}");
14884 assert!(
14885 error.to_string().contains("SDK appends /api automatically"),
14886 "the validation error must explain how to fix the endpoint"
14887 );
14888 }
14889 }
14890
14891 #[test]
14892 fn client_builder_preserves_self_hosted_and_managed_runtime_prefixes() {
14893 for (base_url, expected) in [
14894 ("http://127.0.0.1:8080", "http://127.0.0.1:8080"),
14895 (
14896 "http://localhost:8080/durable-workflow/",
14897 "http://localhost:8080/durable-workflow",
14898 ),
14899 (
14900 "https://runtime.example.test/namespaces/orders",
14901 "https://runtime.example.test/namespaces/orders",
14902 ),
14903 (
14904 "https://runtime.example.test/gateway/api/namespaces/orders",
14905 "https://runtime.example.test/gateway/api/namespaces/orders",
14906 ),
14907 (
14908 "https://api.example.test/runtime/orders/",
14909 "https://api.example.test/runtime/orders",
14910 ),
14911 ] {
14912 let client = Client::builder(base_url)
14913 .build()
14914 .expect("Server and Cloud runtime base URL must remain valid");
14915
14916 assert_eq!(client.base_url, expected);
14917 }
14918 }
14919
14920 #[test]
14921 fn workflow_completion_uses_the_additive_command_protocol_floor() {
14922 assert_eq!(
14923 workflow_completion_protocol_version(&[json!({"type": "complete_workflow"})]),
14924 WORKER_PROTOCOL_VERSION
14925 );
14926 assert_eq!(
14927 workflow_completion_protocol_version(&[json!({
14928 "type": "upsert_search_attributes",
14929 "attributes": {"OrderStatus": "waiting"},
14930 })]),
14931 SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION
14932 );
14933 assert_eq!(
14934 workflow_completion_protocol_version(&[json!({
14935 "type": "upsert_search_attributes",
14936 "attributes": {"OrderStatus": "waiting"},
14937 "attribute_types": {"OrderStatus": "keyword"},
14938 })]),
14939 TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION
14940 );
14941 assert_eq!(
14942 workflow_completion_protocol_version(&[
14943 json!({"type": "upsert_memo", "entries": {"status": "waiting"}}),
14944 json!({"type": "open_condition_wait", "condition_key": "ready"}),
14945 ]),
14946 MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION
14947 );
14948 assert_eq!(
14949 workflow_completion_protocol_version(&[
14950 json!({"type": "upsert_search_attributes", "attributes": {"State": "waiting"}}),
14951 json!({"type": "open_condition_wait", "condition_key": "ready"}),
14952 ]),
14953 CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION
14954 );
14955 assert_eq!(
14956 workflow_completion_protocol_version(&[json!({
14957 "type": "open_condition_wait",
14958 "condition_wait_occurrence_id": "rust:condition-wait:0",
14959 "condition_key": "ready",
14960 })]),
14961 CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
14962 );
14963 assert_eq!(
14964 workflow_completion_protocol_version_with_message_streams(
14965 &[json!({"type": "upsert_memo", "entries": {"status": "waiting"}})],
14966 true,
14967 ),
14968 MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION
14969 );
14970 assert_eq!(
14971 workflow_completion_protocol_version_with_message_streams(
14972 &[json!({
14973 "type": "open_condition_wait",
14974 "condition_wait_occurrence_id": "rust:condition-wait:0",
14975 "condition_key": "ready",
14976 })],
14977 true,
14978 ),
14979 CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
14980 );
14981 }
14982
14983 #[test]
14984 fn portable_worker_affinity_manifest_explicitly_refuses_unimplemented_features() {
14985 let manifest = portable_worker_affinity_capability_manifest();
14986
14987 for capability in ["local_activities", "worker_sessions", "sticky_execution"] {
14988 assert_eq!(manifest[capability]["supported"], json!(false));
14989 assert_eq!(
14990 manifest[capability]["minimum_protocol_version"],
14991 json!(PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION)
14992 );
14993 assert!(manifest[capability]["reason"]
14994 .as_str()
14995 .is_some_and(|reason| !reason.is_empty()));
14996 }
14997 }
14998
14999 fn typed_fidelity_probe() -> AvroValue {
15000 AvroValue::Map(BTreeMap::from([
15001 ("bytes".to_string(), AvroValue::Bytes(vec![0, 0xff])),
15002 ("empty".to_string(), AvroValue::Map(BTreeMap::new())),
15003 (
15004 "numeric".to_string(),
15005 AvroValue::Map(BTreeMap::from([
15006 ("0".to_string(), AvroValue::String("zero".to_string())),
15007 ("1".to_string(), AvroValue::String("one".to_string())),
15008 ])),
15009 ),
15010 (
15011 "nested".to_string(),
15012 AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([(
15013 "enabled".to_string(),
15014 AvroValue::Boolean(true),
15015 )]))]),
15016 ),
15017 (
15018 "projection_collisions".to_string(),
15019 AvroValue::Array(projection_collision_probe()),
15020 ),
15021 ]))
15022 }
15023
15024 fn projection_collision_probe() -> Vec<AvroValue> {
15025 vec![
15026 AvroValue::Map(BTreeMap::from([
15027 ("$type".to_string(), AvroValue::String("bytes".to_string())),
15028 (
15029 "base64".to_string(),
15030 AvroValue::String("ordinary user text".to_string()),
15031 ),
15032 ])),
15033 AvroValue::Map(BTreeMap::from([
15034 ("$type".to_string(), AvroValue::String("map".to_string())),
15035 (
15036 "entries".to_string(),
15037 AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([
15038 ("key".to_string(), AvroValue::String("ordinary".to_string())),
15039 (
15040 "value".to_string(),
15041 AvroValue::String("user map".to_string()),
15042 ),
15043 ]))]),
15044 ),
15045 ])),
15046 ]
15047 }
15048
15049 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
15050 struct TypedContract {
15051 nested: TypedNested,
15052 mode: TypedMode,
15053 optional: Option<String>,
15054 absent: Option<String>,
15055 items: Vec<i64>,
15056 labels: BTreeMap<String, String>,
15057 bytes: serde_bytes::ByteBuf,
15058 signed: i64,
15059 finite: f64,
15060 }
15061
15062 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
15063 struct TypedNested {
15064 enabled: bool,
15065 }
15066
15067 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
15068 enum TypedMode {
15069 Detailed { label: String },
15070 }
15071
15072 fn typed_contract() -> TypedContract {
15073 TypedContract {
15074 nested: TypedNested { enabled: true },
15075 mode: TypedMode::Detailed {
15076 label: "compiler-checked".to_string(),
15077 },
15078 optional: Some("present".to_string()),
15079 absent: None,
15080 items: vec![i64::MIN, 0, i64::MAX],
15081 labels: BTreeMap::from([
15082 ("language".to_string(), "rust".to_string()),
15083 ("wire".to_string(), "avro".to_string()),
15084 ]),
15085 bytes: serde_bytes::ByteBuf::from(vec![0, 0xff, 7]),
15086 signed: -9_223_372_036_854_775_000,
15087 finite: 12.5,
15088 }
15089 }
15090
15091 #[derive(Clone, Debug, Default, PartialEq)]
15092 struct ReplayCounterState {
15093 loaded: Option<String>,
15094 count: i64,
15095 finished: bool,
15096 }
15097
15098 fn replay_counter_worker() -> Worker {
15099 let client = Client::new("http://127.0.0.1:8080").expect("client");
15100 let mut worker = Worker::new(client, "rust-workers");
15101 worker.register_replayed_workflow(
15102 "replay-counter",
15103 ReplayCounterState::default,
15104 |ctx, _input, state| async move {
15105 let loaded = ctx.activity("load-counter", json!([])).await?;
15106 state.update(|current| {
15107 current.loaded = loaded.as_str().map(str::to_string);
15108 })?;
15109 for _ in 0..2 {
15110 let signal = ctx.wait_signal("increment").await?;
15111 let amount = signal.first().and_then(Value::as_i64).unwrap_or_default();
15112 state.update(|current| current.count += amount)?;
15113 }
15114 state.update(|current| current.finished = true)?;
15115 state.read(|current| Ok(json!(current.count)))?
15116 },
15117 );
15118 worker.register_replayed_query::<ReplayCounterState, _, _>(
15119 "replay-counter",
15120 "current",
15121 |_ctx, state, _args| async move {
15122 Ok(json!({
15123 "loaded": state.loaded,
15124 "count": state.count,
15125 "finished": state.finished,
15126 }))
15127 },
15128 );
15129 worker.register_replayed_query::<ReplayCounterState, _, _>(
15130 "replay-counter",
15131 "detached-mutation",
15132 |_ctx, state, _args| async move {
15133 let mut detached = (*state).clone();
15134 detached.count = 999;
15135 Ok(json!(detached.count))
15136 },
15137 );
15138 worker.register_replayed_query::<ReplayCounterState, _, _>(
15139 "replay-counter",
15140 "failed-mutation",
15141 |_ctx, state, _args| async move {
15142 let mut detached = (*state).clone();
15143 detached.count = 999;
15144 Err(Error::WorkerLoop("query refused".to_string()))
15145 },
15146 );
15147 worker
15148 }
15149
15150 fn replay_counter_query(
15151 query_name: &str,
15152 history_events: Value,
15153 run_status: &str,
15154 ) -> QueryTask {
15155 let arguments = fixture_envelope(json!([]));
15156 serde_json::from_value(json!({
15157 "query_task_id": format!("query-{query_name}"),
15158 "workflow_type": "replay-counter",
15159 "query_name": query_name,
15160 "payload_codec": DEFAULT_CODEC,
15161 "workflow_arguments": arguments.clone(),
15162 "query_arguments": arguments,
15163 "history_events": history_events,
15164 "run_status": run_status,
15165 }))
15166 .expect("query task")
15167 }
15168
15169 fn workflow_context(history: Vec<HistoryEvent>) -> WorkflowContext {
15170 workflow_context_with_codec(history, DEFAULT_CODEC)
15171 }
15172
15173 fn workflow_context_with_codec(
15174 history: Vec<HistoryEvent>,
15175 payload_codec: &str,
15176 ) -> WorkflowContext {
15177 WorkflowContext {
15178 state: Arc::new(Mutex::new(
15179 WorkflowState::new_with_identity(
15180 history,
15181 None,
15182 None,
15183 "rust-workers".to_string(),
15184 payload_codec.to_string(),
15185 None,
15186 )
15187 .expect("valid workflow history"),
15188 )),
15189 }
15190 }
15191
15192 fn history_event(event_type: &str, payload: Value) -> HistoryEvent {
15193 HistoryEvent {
15194 event_type: event_type.to_string(),
15195 payload,
15196 raw: HashMap::new(),
15197 }
15198 }
15199
15200 fn parallel_path_entry(
15201 kind: &str,
15202 base: u64,
15203 size: usize,
15204 index: usize,
15205 ) -> ParallelGroupMetadata {
15206 parallel_group_entry(base, size, index, kind)
15207 }
15208
15209 fn parallel_history_event(
15210 event_type: &str,
15211 sequence: u64,
15212 identity_field: &str,
15213 identity: &str,
15214 path: Vec<ParallelGroupMetadata>,
15215 result: Option<Value>,
15216 ) -> HistoryEvent {
15217 let mut payload = serde_json::Map::from_iter([
15218 ("sequence".to_string(), json!(sequence)),
15219 (identity_field.to_string(), json!(identity)),
15220 ]);
15221 let inner = path.last().expect("parallel history path");
15222 apply_parallel_group_path(&mut payload, std::slice::from_ref(inner));
15223 payload.insert("parallel_group_path".to_string(), json!(path));
15224 if let Some(result) = result {
15225 let field = if event_type == "ChildRunCompleted" {
15226 "result"
15227 } else {
15228 "result"
15229 };
15230 payload.insert(field.to_string(), fixture_envelope(result));
15231 payload.insert("payload_codec".to_string(), json!(DEFAULT_CODEC));
15232 }
15233 history_event(event_type, Value::Object(payload))
15234 }
15235
15236 fn nested_parallel_operations() -> Vec<ParallelOperation> {
15237 vec![
15238 ParallelOperation::activity("first", json!([])),
15239 ParallelOperation::group(vec![
15240 ParallelOperation::child_workflow(
15241 "second",
15242 ChildWorkflowOptions::new("child-workers"),
15243 json!([]),
15244 ),
15245 ParallelOperation::activity("third", json!([])),
15246 ]),
15247 ]
15248 }
15249
15250 fn nested_parallel_paths() -> [Vec<ParallelGroupMetadata>; 3] {
15251 let outer = [
15252 parallel_path_entry("mixed", 1, 3, 0),
15253 parallel_path_entry("mixed", 1, 3, 1),
15254 parallel_path_entry("mixed", 1, 3, 2),
15255 ];
15256 [
15257 vec![outer[0].clone()],
15258 vec![outer[1].clone(), parallel_path_entry("mixed", 2, 2, 0)],
15259 vec![outer[2].clone(), parallel_path_entry("mixed", 2, 2, 1)],
15260 ]
15261 }
15262
15263 #[test]
15264 fn parallel_schedules_every_nested_mixed_leaf_with_stable_metadata() {
15265 let ctx = workflow_context(Vec::new());
15266 let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15267 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15268
15269 assert!(matches!(
15270 call.as_mut().poll(&mut task_context),
15271 Poll::Pending
15272 ));
15273 let commands = ctx.take_commands().expect("parallel commands");
15274 assert_eq!(
15275 commands
15276 .iter()
15277 .map(|command| command["type"].as_str().unwrap_or_default())
15278 .collect::<Vec<_>>(),
15279 [
15280 "schedule_activity",
15281 "start_child_workflow",
15282 "schedule_activity"
15283 ]
15284 );
15285 let paths = nested_parallel_paths();
15286 for (command, path) in commands.iter().zip(paths) {
15287 assert_eq!(command["parallel_group_path"], json!(path));
15288 assert_eq!(
15289 command["parallel_group_id"],
15290 json!(path.last().expect("inner group").parallel_group_id)
15291 );
15292 }
15293 }
15294
15295 fn completed_nested_parallel_history() -> Vec<HistoryEvent> {
15296 let paths = nested_parallel_paths();
15297 let third = parallel_history_event(
15298 "ActivityCompleted",
15299 3,
15300 "activity_type",
15301 "third",
15302 paths[2].clone(),
15303 Some(json!("three")),
15304 );
15305 vec![
15306 parallel_history_event(
15307 "ActivityCompleted",
15308 1,
15309 "activity_type",
15310 "first",
15311 paths[0].clone(),
15312 Some(json!("one")),
15313 ),
15314 parallel_history_event(
15315 "ChildWorkflowScheduled",
15316 2,
15317 "child_workflow_type",
15318 "second",
15319 paths[1].clone(),
15320 None,
15321 ),
15322 parallel_history_event(
15323 "ChildRunCompleted",
15324 2,
15325 "child_workflow_type",
15326 "second",
15327 paths[1].clone(),
15328 Some(json!("two")),
15329 ),
15330 third.clone(),
15331 third,
15332 ]
15333 }
15334
15335 #[test]
15336 fn parallel_replay_rebuilds_input_order_and_tolerates_duplicate_delivery() {
15337 for _restart_or_completed_replay in 0..2 {
15338 let ctx = workflow_context(completed_nested_parallel_history());
15339 let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15340 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15341 let Poll::Ready(Ok(results)) = call.as_mut().poll(&mut task_context) else {
15342 panic!("completed nested parallel history must replay");
15343 };
15344 assert_eq!(
15345 results,
15346 vec![
15347 ParallelResult::Activity(json!("one")),
15348 ParallelResult::Group(vec![
15349 ParallelResult::ChildWorkflow(ChildWorkflowResult {
15350 parent: WorkflowIdentity {
15351 workflow_id: None,
15352 run_id: None,
15353 },
15354 child: WorkflowIdentity {
15355 workflow_id: None,
15356 run_id: None,
15357 },
15358 child_workflow_type: Some("second".to_string()),
15359 result: json!("two"),
15360 }),
15361 ParallelResult::Activity(json!("three")),
15362 ]),
15363 ]
15364 );
15365 assert!(ctx.take_commands().expect("commands").is_empty());
15366 ctx.ensure_history_consumed().expect("history consumed");
15367 }
15368 }
15369
15370 #[test]
15371 fn parallel_failure_keeps_typed_cause_path_and_late_completions() {
15372 let paths = nested_parallel_paths();
15373 let history = vec![
15374 parallel_history_event(
15375 "ActivityCompleted",
15376 1,
15377 "activity_type",
15378 "first",
15379 paths[0].clone(),
15380 Some(json!("one")),
15381 ),
15382 parallel_history_event(
15383 "ChildWorkflowScheduled",
15384 2,
15385 "child_workflow_type",
15386 "second",
15387 paths[1].clone(),
15388 None,
15389 ),
15390 parallel_history_event(
15391 "ChildRunFailed",
15392 2,
15393 "child_workflow_type",
15394 "second",
15395 paths[1].clone(),
15396 None,
15397 ),
15398 parallel_history_event(
15399 "ActivityCompleted",
15400 3,
15401 "activity_type",
15402 "third",
15403 paths[2].clone(),
15404 Some(json!("late")),
15405 ),
15406 ];
15407 let ctx = workflow_context(history);
15408 let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15409 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15410 let outcome = call.as_mut().poll(&mut task_context);
15411 let Poll::Ready(Err(Error::ParallelFailed(failure))) = outcome else {
15412 panic!("one failed child must return a typed partial failure: {outcome:?}");
15413 };
15414 assert_eq!(failure.member_path, [1, 0]);
15415 assert_eq!(failure.group_id, "parallel-calls:1:3");
15416 assert!(matches!(*failure.cause, Error::ChildWorkflowFailed(_)));
15417 assert_eq!(
15418 failure
15419 .completed
15420 .iter()
15421 .map(|completion| completion.member_path.clone())
15422 .collect::<Vec<_>>(),
15423 [vec![0], vec![1, 1]]
15424 );
15425 }
15426
15427 #[test]
15428 fn pending_parallel_history_restarts_without_rescheduling_any_leaf() {
15429 let paths = nested_parallel_paths();
15430 let history = vec![
15431 parallel_history_event(
15432 "ActivityScheduled",
15433 1,
15434 "activity_type",
15435 "first",
15436 paths[0].clone(),
15437 None,
15438 ),
15439 parallel_history_event(
15440 "ChildWorkflowScheduled",
15441 2,
15442 "child_workflow_type",
15443 "second",
15444 paths[1].clone(),
15445 None,
15446 ),
15447 parallel_history_event(
15448 "ActivityScheduled",
15449 3,
15450 "activity_type",
15451 "third",
15452 paths[2].clone(),
15453 None,
15454 ),
15455 ];
15456 for _restart in 0..2 {
15457 let ctx = workflow_context(history.clone());
15458 let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15459 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15460 let outcome = call.as_mut().poll(&mut task_context);
15461 assert!(matches!(outcome, Poll::Pending), "{outcome:?}");
15462 assert!(ctx.take_commands().expect("commands").is_empty());
15463 }
15464 }
15465
15466 fn selection_path(index: usize, key: &str) -> Vec<ParallelGroupMetadata> {
15467 vec![selection_group_entry(
15468 1,
15469 2,
15470 index,
15471 "activity",
15472 &SelectionMemberMetadata {
15473 key: SelectionKey::Name(key.to_string()),
15474 index,
15475 base_sequence: index as u64 + 1,
15476 size: 1,
15477 kind: "activity".to_string(),
15478 },
15479 )]
15480 }
15481
15482 fn selection_activity_event(
15483 event_type: &str,
15484 index: usize,
15485 key: &str,
15486 result: Option<Value>,
15487 ) -> HistoryEvent {
15488 let sequence = index as u64 + 1;
15489 let mut event = parallel_history_event(
15490 event_type,
15491 sequence,
15492 "activity_type",
15493 &format!("{key}-activity"),
15494 selection_path(index, key),
15495 result,
15496 );
15497 event.payload["activity_execution_id"] = json!(format!("activity-{key}"));
15498 event.raw.insert(
15499 "id".to_string(),
15500 json!(if event_type == "ActivityCompleted" {
15501 format!("event-{key}")
15502 } else {
15503 format!("{event_type}-{key}")
15504 }),
15505 );
15506 event
15507 }
15508
15509 fn selection_winner_marker() -> HistoryEvent {
15510 history_event(
15511 "SelectionResolved",
15512 json!({
15513 "selection_group_id": "select-calls:1:2",
15514 "selection_group_base_sequence": 1,
15515 "selection_group_size": 2,
15516 "member_key": "fast",
15517 "member_index": 1,
15518 "member_base_sequence": 2,
15519 "member_size": 1,
15520 "operation_kind": "activity",
15521 "operation_identity": "activity-fast",
15522 "outcome": "completed",
15523 "resolution_event_id": "event-fast",
15524 "resolution_event_type": "ActivityCompleted",
15525 }),
15526 )
15527 }
15528
15529 fn keyed_activity_selection(ctx: &WorkflowContext) -> SelectCall {
15530 ctx.select_keyed(vec![
15531 (
15532 "slow",
15533 ParallelOperation::activity_with_options(
15534 "slow-activity",
15535 ActivityOptions::new().task_queue("default"),
15536 json!([]),
15537 ),
15538 ),
15539 (
15540 "fast",
15541 ParallelOperation::activity_with_options(
15542 "fast-activity",
15543 ActivityOptions::new().task_queue("default"),
15544 json!([]),
15545 ),
15546 ),
15547 ])
15548 }
15549
15550 fn assert_persisted_selection_replay(history: Vec<HistoryEvent>) {
15551 let ctx = workflow_context(history);
15552 let mut call = Box::pin(keyed_activity_selection(&ctx));
15553 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15554 let selected = match call.as_mut().poll(&mut task_context) {
15555 Poll::Ready(Ok(selected)) => selected,
15556 Poll::Ready(Err(error)) => panic!("persisted selection winner must replay: {error:?}"),
15557 Poll::Pending => panic!("persisted selection winner must replay without pending"),
15558 };
15559 assert_eq!(selected.key, SelectionKey::Name("fast".to_string()));
15560 assert_eq!(
15561 selected.value,
15562 Some(ParallelResult::Activity(json!("winner-value")))
15563 );
15564 let slow = selected
15565 .handle(&SelectionKey::Name("slow".to_string()))
15566 .expect("slow handle")
15567 .clone();
15568 let mut await_slow = Box::pin(slow.await_result());
15569 assert!(matches!(
15570 await_slow.as_mut().poll(&mut task_context),
15571 Poll::Ready(Ok(ParallelResult::Activity(value))) if value == json!("loser-value")
15572 ));
15573 assert!(ctx.take_commands().expect("commands").is_empty());
15574 }
15575
15576 const SELECTION_COLD_REPLAY_HISTORY: &str = "DURABLE_WORKFLOW_SELECTION_COLD_REPLAY_HISTORY";
15577
15578 fn canonical_selection_history() -> Vec<HistoryEvent> {
15579 const FIXTURE: &[u8] =
15580 include_bytes!("../tests/fixtures/durable_selection_runtime_history.json");
15581 assert_eq!(
15582 format!("{:x}", Sha256::digest(FIXTURE)),
15583 "51fd8b9c16e978dcef536a5c727b9fdc0ae724d9afc17d9a7837d219f41ee3ba",
15584 );
15585 let fixture: Value = serde_json::from_slice(FIXTURE).expect("canonical selection fixture");
15586
15587 serde_json::from_value(fixture["history"].clone()).expect("canonical selection history")
15588 }
15589
15590 #[test]
15591 fn selection_fresh_process_entrypoint() {
15592 let Ok(path) = std::env::var(SELECTION_COLD_REPLAY_HISTORY) else {
15593 return;
15594 };
15595 let persisted = fs::read(path).expect("persisted selection history");
15596 assert_eq!(
15597 format!("{:x}", Sha256::digest(&persisted)),
15598 "51fd8b9c16e978dcef536a5c727b9fdc0ae724d9afc17d9a7837d219f41ee3ba",
15599 );
15600 let fixture: Value =
15601 serde_json::from_slice(&persisted).expect("valid persisted selection fixture");
15602 let history: Vec<HistoryEvent> = serde_json::from_value(fixture["history"].clone())
15603 .expect("valid persisted selection history");
15604
15605 assert_persisted_selection_replay(history);
15606 }
15607
15608 #[test]
15609 fn selection_starts_every_member_with_stable_keys_and_group_identity() {
15610 let ctx = workflow_context(Vec::new());
15611 let mut call = Box::pin(keyed_activity_selection(&ctx));
15612 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15613
15614 assert!(matches!(
15615 call.as_mut().poll(&mut task_context),
15616 Poll::Pending
15617 ));
15618 let commands = ctx.take_commands().expect("selection commands");
15619 assert_eq!(commands.len(), 2);
15620 assert_eq!(commands[0]["selection_member_key"], json!("slow"));
15621 assert_eq!(commands[1]["selection_member_key"], json!("fast"));
15622 assert!(commands.iter().all(|command| {
15623 command["parallel_group_id"] == json!("select-calls:1:2")
15624 && command["parallel_group_mode"] == json!("select")
15625 }));
15626 }
15627
15628 #[test]
15629 fn selection_key_domain_rejects_empty_authoring_and_malformed_history() {
15630 let ctx = workflow_context(Vec::new());
15631 let mut invalid = Box::pin(ctx.select_keyed(vec![(
15632 "",
15633 ParallelOperation::activity("invalid", json!([])),
15634 )]));
15635 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15636 assert!(matches!(
15637 invalid.as_mut().poll(&mut task_context),
15638 Poll::Ready(Err(Error::InvalidParallelGroup(ParallelGroupError {
15639 reason: "selection_key_invalid",
15640 ..
15641 })))
15642 ));
15643
15644 for invalid_key in [json!(""), json!(-1)] {
15645 let mut event = selection_activity_event("ActivityScheduled", 0, "slow", None);
15646 event.payload["selection_member_key"] = invalid_key.clone();
15647 event.payload["parallel_group_path"][0]["selection_member_key"] = invalid_key;
15648 assert!(matches!(
15649 WorkflowState::new_with_identity(
15650 vec![event],
15651 None,
15652 None,
15653 "rust-workers".to_string(),
15654 DEFAULT_CODEC.to_string(),
15655 None,
15656 ),
15657 Err(Error::NonDeterministicReplay(_))
15658 ));
15659 }
15660 }
15661
15662 #[test]
15663 fn selection_preserves_valid_named_and_numeric_keys() {
15664 let ctx = workflow_context(Vec::new());
15665 let mut selection = Box::pin(ctx.select_keyed(vec![
15666 (
15667 SelectionKey::Index(0),
15668 ParallelOperation::activity("numeric", json!([])),
15669 ),
15670 (
15671 SelectionKey::Name("named".to_string()),
15672 ParallelOperation::timer(Duration::from_secs(1)),
15673 ),
15674 ]));
15675 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15676
15677 assert!(matches!(
15678 selection.as_mut().poll(&mut task_context),
15679 Poll::Pending
15680 ));
15681 let commands = ctx.take_commands().expect("selection commands");
15682 assert_eq!(commands[0]["selection_member_key"], json!(0));
15683 assert_eq!(commands[1]["selection_member_key"], json!("named"));
15684 }
15685
15686 #[test]
15687 fn selection_replays_persisted_winner_and_loser_can_be_awaited_later() {
15688 let history = canonical_selection_history();
15689 assert_persisted_selection_replay(history.clone());
15690
15691 let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
15692 .join("tests/fixtures/durable_selection_runtime_history.json");
15693 let output =
15694 ProcessCommand::new(std::env::current_exe().expect("current Rust test binary"))
15695 .args([
15696 "--exact",
15697 "tests::selection_fresh_process_entrypoint",
15698 "--nocapture",
15699 ])
15700 .env(SELECTION_COLD_REPLAY_HISTORY, &path)
15701 .output()
15702 .expect("run fresh selection replay process");
15703
15704 assert!(
15705 output.status.success(),
15706 "fresh selection replay failed:\nstdout:\n{}\nstderr:\n{}",
15707 String::from_utf8_lossy(&output.stdout),
15708 String::from_utf8_lossy(&output.stderr),
15709 );
15710 }
15711
15712 #[test]
15713 fn selection_waits_durably_when_terminal_members_precede_the_winner_marker() {
15714 let mut history = canonical_selection_history();
15715 history.retain(|event| event.event_type != "SelectionResolved");
15716 let ctx = workflow_context(history);
15717 let mut selection = Box::pin(keyed_activity_selection(&ctx));
15718 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15719
15720 assert!(matches!(
15721 selection.as_mut().poll(&mut task_context),
15722 Poll::Pending
15723 ));
15724 assert!(ctx.take_commands().expect("commands").is_empty());
15725 assert!(
15726 ctx.matched_recorded_pending()
15727 .expect("selection pending state"),
15728 "terminal member history must keep the workflow durably pending until SelectionResolved commits"
15729 );
15730 }
15731
15732 #[test]
15733 fn selection_terminal_condition_history_waits_durably_for_its_winner_marker() {
15734 for (terminal_event, predicate_satisfied, timeout_seconds) in [
15735 ("ConditionWaitSatisfied", true, None),
15736 ("ConditionWaitTimedOut", false, Some(0)),
15737 ] {
15738 let member = SelectionMemberMetadata {
15739 key: SelectionKey::Name("condition".to_string()),
15740 index: 0,
15741 base_sequence: 1,
15742 size: 1,
15743 kind: "condition".to_string(),
15744 };
15745 let path = vec![selection_group_entry(1, 1, 0, "condition", &member)];
15746 let mut payload = json!({
15747 "sequence": 1,
15748 "condition_wait_id": "condition-1",
15749 "condition_wait_occurrence_id": "rust:condition-wait:0",
15750 "condition_key": "ready",
15751 "condition_definition_fingerprint": "sha256:ready-v1",
15752 "parallel_group_path": path,
15753 });
15754 payload
15755 .as_object_mut()
15756 .expect("condition history payload")
15757 .extend(
15758 serde_json::to_value(&path[0])
15759 .expect("condition selection metadata")
15760 .as_object()
15761 .expect("condition selection metadata object")
15762 .clone(),
15763 );
15764 if let Some(timeout_seconds) = timeout_seconds {
15765 payload["timeout_seconds"] = json!(timeout_seconds);
15766 }
15767 let history = vec![
15768 history_event("ConditionWaitOpened", payload.clone()),
15769 history_event(terminal_event, payload),
15770 ];
15771 let ctx = workflow_context(history);
15772 let mut options = ConditionWaitOptions::new("ready", "sha256:ready-v1");
15773 if timeout_seconds.is_some() {
15774 options = options.timeout(Duration::ZERO);
15775 }
15776 let mut selection = Box::pin(ctx.select_keyed(vec![(
15777 "condition",
15778 ParallelOperation::condition(options, move || Ok(predicate_satisfied)),
15779 )]));
15780 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15781
15782 assert!(matches!(
15783 selection.as_mut().poll(&mut task_context),
15784 Poll::Pending
15785 ));
15786 assert!(ctx.take_commands().expect("commands").is_empty());
15787 assert!(
15788 ctx.matched_recorded_pending()
15789 .expect("condition selection pending state"),
15790 "{terminal_event} must keep the workflow durably pending until SelectionResolved commits"
15791 );
15792 }
15793 }
15794
15795 #[test]
15796 fn selection_immediate_condition_members_open_a_durable_wait() {
15797 for predicate_satisfied in [true, false] {
15798 let ctx = workflow_context(Vec::new());
15799 let mut selection = Box::pin(ctx.select_keyed(vec![(
15800 "condition",
15801 ParallelOperation::condition(
15802 ConditionWaitOptions::new("ready", "sha256:ready-v1").timeout(Duration::ZERO),
15803 move || Ok(predicate_satisfied),
15804 ),
15805 )]));
15806 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15807
15808 assert!(matches!(
15809 selection.as_mut().poll(&mut task_context),
15810 Poll::Pending
15811 ));
15812 let commands = ctx.take_commands().expect("condition selection command");
15813 assert_eq!(commands.len(), 1);
15814 assert_eq!(commands[0]["type"], json!("open_condition_wait"));
15815 assert_eq!(commands[0]["timeout_seconds"], json!(0));
15816 assert_eq!(
15817 commands[0]["parallel_group_path"][0]["parallel_group_mode"],
15818 json!("select")
15819 );
15820 }
15821 }
15822
15823 #[test]
15824 fn selection_loser_cancellation_is_explicit_and_idempotent() {
15825 let history = vec![
15826 selection_activity_event("ActivityScheduled", 0, "slow", None),
15827 selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner"))),
15828 selection_winner_marker(),
15829 ];
15830 let ctx = workflow_context(history.clone());
15831 let mut call = Box::pin(keyed_activity_selection(&ctx));
15832 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15833 let Poll::Ready(Ok(selected)) = call.as_mut().poll(&mut task_context) else {
15834 panic!("winner must replay");
15835 };
15836 let slow = selected
15837 .handle(&SelectionKey::Name("slow".to_string()))
15838 .expect("slow handle")
15839 .clone();
15840 let mut cancel = Box::pin(slow.cancel());
15841 assert!(matches!(
15842 cancel.as_mut().poll(&mut task_context),
15843 Poll::Pending
15844 ));
15845 assert!(matches!(
15846 cancel.as_mut().poll(&mut task_context),
15847 Poll::Pending
15848 ));
15849 let commands = ctx.take_commands().expect("cancel command");
15850 assert_eq!(commands.len(), 1);
15851 assert_eq!(commands[0]["type"], json!("cancel_selection_operation"));
15852 assert_eq!(commands[0]["member_key"], json!("slow"));
15853
15854 let mut cancelled_history = history;
15855 cancelled_history.push(history_event(
15856 "SelectionOperationCancelled",
15857 json!({
15858 "selection_group_id": "select-calls:1:2",
15859 "member_key": "slow",
15860 "member_index": 0,
15861 "member_base_sequence": 1,
15862 "member_size": 1,
15863 "operation_kind": "activity",
15864 "operation_identity": "activity-slow",
15865 "cancelled_at": "2026-08-27T00:00:00Z",
15866 }),
15867 ));
15868 let replayed = workflow_context(cancelled_history);
15869 let mut call = Box::pin(keyed_activity_selection(&replayed));
15870 let Poll::Ready(Ok(selected)) = call.as_mut().poll(&mut task_context) else {
15871 panic!("winner must replay after cancellation");
15872 };
15873 let slow = selected
15874 .handle(&SelectionKey::Name("slow".to_string()))
15875 .expect("slow handle")
15876 .clone();
15877 let mut cancel = Box::pin(slow.cancel());
15878 assert!(matches!(
15879 cancel.as_mut().poll(&mut task_context),
15880 Poll::Ready(Ok(()))
15881 ));
15882 assert!(replayed.take_commands().expect("commands").is_empty());
15883 }
15884
15885 #[test]
15886 fn selection_cancellation_marker_is_bound_to_every_authored_handle_field() {
15887 let base_history = vec![
15888 selection_activity_event("ActivityScheduled", 0, "slow", None),
15889 selection_activity_event("ActivityScheduled", 1, "fast", None),
15890 selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner"))),
15891 selection_winner_marker(),
15892 ];
15893 for (field, corrupt) in [
15894 ("member_key", json!("fast")),
15895 ("member_index", json!(1)),
15896 ("member_base_sequence", json!(3)),
15897 ("member_size", json!(2)),
15898 ("operation_kind", json!("timer")),
15899 ("operation_identity", json!("forged")),
15900 ] {
15901 let mut cancellation = json!({
15902 "selection_group_id": "select-calls:1:2",
15903 "member_key": "slow",
15904 "member_index": 0,
15905 "member_base_sequence": 1,
15906 "member_size": 1,
15907 "operation_kind": "activity",
15908 "operation_identity": "activity-slow",
15909 });
15910 cancellation[field] = corrupt;
15911 let mut history = base_history.clone();
15912 history.push(history_event("SelectionOperationCancelled", cancellation));
15913 let ctx = workflow_context(history);
15914 let mut selection = Box::pin(keyed_activity_selection(&ctx));
15915 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15916
15917 assert!(matches!(
15918 selection.as_mut().poll(&mut task_context),
15919 Poll::Ready(Err(Error::NonDeterministicReplay(_)))
15920 ));
15921 }
15922 }
15923
15924 #[test]
15925 fn selection_child_identity_prefers_the_durable_run_id() {
15926 let ctx = workflow_context(vec![history_event(
15927 "ChildWorkflowScheduled",
15928 json!({
15929 "sequence": 1,
15930 "child_workflow_type": "child",
15931 "child_workflow_instance_id": "child-instance",
15932 "child_workflow_run_id": "child-run",
15933 }),
15934 )]);
15935 let state = ctx.state.lock().expect("workflow state");
15936
15937 assert_eq!(
15938 selection_operation_identity(&state, "child", 1, 1),
15939 "child-run"
15940 );
15941 }
15942
15943 #[test]
15944 fn selection_activity_identity_requires_canonical_execution_id() {
15945 let slow = selection_activity_event("ActivityScheduled", 0, "slow", None);
15946 let mut fast_open = selection_activity_event("ActivityScheduled", 1, "fast", None);
15947 let mut fast_completed =
15948 selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner")));
15949 for event in [&mut fast_open, &mut fast_completed] {
15950 event
15951 .payload
15952 .as_object_mut()
15953 .expect("activity payload")
15954 .remove("activity_execution_id");
15955 event.payload["activity_id"] = json!("forged-activity-id");
15956 }
15957 let mut marker = selection_winner_marker();
15958 marker.payload["operation_identity"] = json!("forged-activity-id");
15959 let ctx = workflow_context(vec![slow, fast_open, fast_completed, marker]);
15960 let mut selection = Box::pin(keyed_activity_selection(&ctx));
15961 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15962
15963 assert!(matches!(
15964 selection.as_mut().poll(&mut task_context),
15965 Poll::Ready(Err(Error::NonDeterministicReplay(_)))
15966 ));
15967 }
15968
15969 #[test]
15970 fn selection_completion_before_cancellation_remains_awaitable() {
15971 let history = vec![
15972 selection_activity_event("ActivityScheduled", 0, "slow", None),
15973 selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner"))),
15974 selection_winner_marker(),
15975 selection_activity_event(
15976 "ActivityCompleted",
15977 0,
15978 "slow",
15979 Some(json!("completed-first")),
15980 ),
15981 ];
15982 let ctx = workflow_context(history);
15983 let mut selection = Box::pin(keyed_activity_selection(&ctx));
15984 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15985 let Poll::Ready(Ok(selected)) = selection.as_mut().poll(&mut task_context) else {
15986 panic!("winner must replay");
15987 };
15988 let slow = selected
15989 .handle(&SelectionKey::Name("slow".to_string()))
15990 .expect("slow handle")
15991 .clone();
15992 let mut cancel = Box::pin(slow.cancel());
15993 assert!(matches!(
15994 cancel.as_mut().poll(&mut task_context),
15995 Poll::Ready(Ok(()))
15996 ));
15997 let mut await_slow = Box::pin(slow.await_result());
15998 assert!(matches!(
15999 await_slow.as_mut().poll(&mut task_context),
16000 Poll::Ready(Ok(ParallelResult::Activity(value))) if value == json!("completed-first")
16001 ));
16002 let commands = ctx.take_commands().expect("commands");
16003 assert!(commands.is_empty());
16004 }
16005
16006 #[test]
16007 fn selection_nested_later_failure_before_cancel_remains_the_awaited_failure() {
16008 let nested_member = SelectionMemberMetadata {
16009 key: SelectionKey::Name("nested".to_string()),
16010 index: 0,
16011 base_sequence: 1,
16012 size: 2,
16013 kind: "group".to_string(),
16014 };
16015 let deadline_member = SelectionMemberMetadata {
16016 key: SelectionKey::Name("deadline".to_string()),
16017 index: 1,
16018 base_sequence: 3,
16019 size: 1,
16020 kind: "timer".to_string(),
16021 };
16022 let nested_paths = [
16023 vec![
16024 selection_group_entry(1, 3, 0, "mixed", &nested_member),
16025 parallel_group_entry(1, 2, 0, "activity"),
16026 ],
16027 vec![
16028 selection_group_entry(1, 3, 1, "mixed", &nested_member),
16029 parallel_group_entry(1, 2, 1, "activity"),
16030 ],
16031 ];
16032 let deadline_path = vec![selection_group_entry(1, 3, 2, "mixed", &deadline_member)];
16033 let mut timer_fired = parallel_history_event(
16034 "TimerFired",
16035 3,
16036 "timer_id",
16037 "timer-3",
16038 deadline_path.clone(),
16039 None,
16040 );
16041 timer_fired.payload["delay_seconds"] = json!(0);
16042 timer_fired
16043 .raw
16044 .insert("id".to_string(), json!("timer-fired"));
16045 let mut timer_scheduled = parallel_history_event(
16046 "TimerScheduled",
16047 3,
16048 "timer_id",
16049 "timer-3",
16050 deadline_path,
16051 None,
16052 );
16053 timer_scheduled.payload["delay_seconds"] = json!(0);
16054 let history = vec![
16055 parallel_history_event(
16056 "ActivityScheduled",
16057 1,
16058 "activity_type",
16059 "nested-first",
16060 nested_paths[0].clone(),
16061 None,
16062 ),
16063 parallel_history_event(
16064 "ActivityScheduled",
16065 2,
16066 "activity_type",
16067 "nested-second",
16068 nested_paths[1].clone(),
16069 None,
16070 ),
16071 timer_scheduled,
16072 timer_fired,
16073 history_event(
16074 "SelectionResolved",
16075 json!({
16076 "selection_group_id": "select-calls:1:3",
16077 "selection_group_base_sequence": 1,
16078 "selection_group_size": 3,
16079 "member_key": "deadline",
16080 "member_index": 1,
16081 "member_base_sequence": 3,
16082 "member_size": 1,
16083 "operation_kind": "timer",
16084 "operation_identity": "timer-3",
16085 "outcome": "completed",
16086 "resolution_event_id": "timer-fired",
16087 "resolution_event_type": "TimerFired",
16088 }),
16089 ),
16090 parallel_history_event(
16091 "ActivityFailed",
16092 2,
16093 "activity_type",
16094 "nested-second",
16095 nested_paths[1].clone(),
16096 None,
16097 ),
16098 ];
16099 let ctx = workflow_context(history);
16100 let mut selection = Box::pin(ctx.select_keyed(vec![
16101 (
16102 "nested",
16103 ParallelOperation::group(vec![
16104 ParallelOperation::activity("nested-first", json!([])),
16105 ParallelOperation::activity("nested-second", json!([])),
16106 ]),
16107 ),
16108 ("deadline", ParallelOperation::timer(Duration::ZERO)),
16109 ]));
16110 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16111 let Poll::Ready(Ok(selected)) = selection.as_mut().poll(&mut task_context) else {
16112 panic!("deadline winner must replay");
16113 };
16114 let nested = selected
16115 .handle(&SelectionKey::Name("nested".to_string()))
16116 .expect("nested handle")
16117 .clone();
16118 let mut cancel = Box::pin(nested.cancel());
16119 assert!(matches!(
16120 cancel.as_mut().poll(&mut task_context),
16121 Poll::Ready(Ok(()))
16122 ));
16123 let mut await_nested = Box::pin(nested.await_result());
16124
16125 assert!(matches!(
16126 await_nested.as_mut().poll(&mut task_context),
16127 Poll::Ready(Err(Error::ActivityFailed(_)))
16128 ));
16129 assert!(ctx.take_commands().expect("commands").is_empty());
16130 }
16131
16132 #[test]
16133 fn selection_supports_child_timer_signal_condition_and_nested_groups() {
16134 let ctx = workflow_context(Vec::new());
16135 let mut call = Box::pin(ctx.select(vec![
16136 ParallelOperation::child_workflow(
16137 "child",
16138 ChildWorkflowOptions::new("children"),
16139 json!([]),
16140 ),
16141 ParallelOperation::timer(Duration::from_secs(30)),
16142 ParallelOperation::signal("approval"),
16143 ParallelOperation::condition(
16144 ConditionWaitOptions::new("ready", "sha256:ready"),
16145 || Ok(false),
16146 ),
16147 ParallelOperation::group(vec![
16148 ParallelOperation::activity("nested-one", json!([])),
16149 ParallelOperation::activity("nested-two", json!([])),
16150 ]),
16151 ]));
16152 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16153 assert!(matches!(
16154 call.as_mut().poll(&mut task_context),
16155 Poll::Pending
16156 ));
16157 let commands = ctx.take_commands().expect("selection commands");
16158 assert_eq!(
16159 commands
16160 .iter()
16161 .map(|command| command["type"].as_str().unwrap_or_default())
16162 .collect::<Vec<_>>(),
16163 [
16164 "start_child_workflow",
16165 "start_timer",
16166 "open_signal_wait",
16167 "open_condition_wait",
16168 "schedule_activity",
16169 "schedule_activity",
16170 ]
16171 );
16172 assert!(commands.iter().all(|command| {
16173 command["parallel_group_path"][0]["parallel_group_mode"] == json!("select")
16174 }));
16175 assert_eq!(
16176 commands[4]["parallel_group_path"].as_array().map(Vec::len),
16177 Some(2)
16178 );
16179 assert_eq!(
16180 commands[4]["parallel_group_path"][0]["selection_member_kind"],
16181 json!("group")
16182 );
16183 assert_eq!(
16184 commands[5]["parallel_group_path"][0]["selection_member_kind"],
16185 json!("group")
16186 );
16187
16188 let one_leaf_ctx = workflow_context(Vec::new());
16189 let mut one_leaf = Box::pin(one_leaf_ctx.select(vec![ParallelOperation::group(vec![
16190 ParallelOperation::activity("nested-only", json!([])),
16191 ])]));
16192 assert!(matches!(
16193 one_leaf.as_mut().poll(&mut task_context),
16194 Poll::Pending
16195 ));
16196 let one_leaf_commands = one_leaf_ctx.take_commands().expect("one-leaf commands");
16197 assert_eq!(one_leaf_commands.len(), 1);
16198 assert_eq!(
16199 one_leaf_commands[0]["parallel_group_path"][0]["selection_member_kind"],
16200 json!("group")
16201 );
16202 assert_eq!(
16203 one_leaf_commands[0]["parallel_group_path"][0]["selection_member_size"],
16204 json!(1)
16205 );
16206 }
16207
16208 async fn trip_saga(ctx: WorkflowContext) -> Result<Value> {
16209 let mut saga = ctx.saga();
16210 let outcome = async {
16211 let flight = ctx.activity("trip.reserve-flight", json!([])).await?;
16212 saga.add_compensation("trip.cancel-flight", json!([flight]))?;
16213 let hotel = ctx.activity("trip.reserve-hotel", json!([])).await?;
16214 saga.add_compensation("trip.cancel-hotel", json!([hotel]))?;
16215 ctx.activity("trip.charge", json!([])).await?;
16216 Ok(json!({"status": "booked"}))
16217 }
16218 .await;
16219 saga.finish(outcome).await
16220 }
16221
16222 fn saga_activity(
16223 event_type: &str,
16224 sequence: u64,
16225 activity_type: &str,
16226 result: Option<Value>,
16227 ) -> HistoryEvent {
16228 let mut payload = json!({
16229 "sequence": sequence,
16230 "activity_type": activity_type,
16231 "message": format!("{activity_type} failed"),
16232 "exception_type": "PlannedFailure",
16233 "non_retryable": true,
16234 });
16235 if let Some(result) = result {
16236 payload["result"] = fixture_envelope(result);
16237 }
16238 history_event(event_type, payload)
16239 }
16240
16241 #[test]
16242 fn saga_replays_reverse_compensation_across_restart_and_duplicate_delivery() {
16243 let completed_hotel_compensation = saga_activity(
16244 "ActivityCompleted",
16245 4,
16246 "trip.cancel-hotel",
16247 Some(Value::Null),
16248 );
16249 let history = vec![
16250 saga_activity(
16251 "ActivityCompleted",
16252 1,
16253 "trip.reserve-flight",
16254 Some(json!("flight-1")),
16255 ),
16256 saga_activity(
16257 "ActivityCompleted",
16258 2,
16259 "trip.reserve-hotel",
16260 Some(json!("hotel-1")),
16261 ),
16262 saga_activity("ActivityFailed", 3, "trip.charge", None),
16263 completed_hotel_compensation.clone(),
16264 completed_hotel_compensation,
16265 ];
16266
16267 for _restart in 0..2 {
16268 let ctx = workflow_context(history.clone());
16269 let mut future = Box::pin(trip_saga(ctx.clone()));
16270 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16271 assert!(matches!(
16272 future.as_mut().poll(&mut task_context),
16273 Poll::Pending
16274 ));
16275 let commands = ctx.take_commands().expect("compensation command");
16276 assert_eq!(commands.len(), 1);
16277 assert_eq!(commands[0]["activity_type"], "trip.cancel-flight");
16278 }
16279 }
16280
16281 #[test]
16282 fn saga_compensation_failure_preserves_both_typed_failures() {
16283 let history = vec![
16284 saga_activity(
16285 "ActivityCompleted",
16286 1,
16287 "trip.reserve-flight",
16288 Some(json!("flight-1")),
16289 ),
16290 saga_activity(
16291 "ActivityCompleted",
16292 2,
16293 "trip.reserve-hotel",
16294 Some(json!("hotel-1")),
16295 ),
16296 saga_activity("ActivityFailed", 3, "trip.charge", None),
16297 saga_activity("ActivityFailed", 4, "trip.cancel-hotel", None),
16298 ];
16299 let ctx = workflow_context(history);
16300 let mut future = Box::pin(trip_saga(ctx));
16301 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16302 let Poll::Ready(Err(Error::SagaCompensationFailed(failure))) =
16303 future.as_mut().poll(&mut task_context)
16304 else {
16305 panic!("compensation failure must remain structured");
16306 };
16307 assert!(matches!(
16308 *failure.initiating_failure,
16309 Error::ActivityFailed(_)
16310 ));
16311 assert!(matches!(
16312 *failure.compensation_failure,
16313 Error::ActivityFailed(_)
16314 ));
16315 assert_eq!(failure.compensation_activity_type, "trip.cancel-hotel");
16316 assert_eq!(failure.compensation_registration_order, 2);
16317 }
16318
16319 #[test]
16320 fn saga_compensates_cooperative_cancellation() {
16321 let ctx = workflow_context(vec![saga_activity(
16322 "ActivityCompleted",
16323 1,
16324 "trip.reserve-flight",
16325 Some(json!("flight-1")),
16326 )]);
16327 ctx.state.lock().expect("state").cancel_requested = true;
16328 let run = {
16329 let ctx = ctx.clone();
16330 async move {
16331 let mut saga = ctx.saga();
16332 let outcome = async {
16333 let flight = ctx.activity("trip.reserve-flight", json!([])).await?;
16334 saga.add_compensation("trip.cancel-flight", json!([flight]))?;
16335 ctx.throw_if_cancellation_requested()?;
16336 Ok(json!("unexpected"))
16337 }
16338 .await;
16339 saga.finish(outcome).await
16340 }
16341 };
16342 let mut future = Box::pin(run);
16343 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16344 assert!(matches!(
16345 future.as_mut().poll(&mut task_context),
16346 Poll::Pending
16347 ));
16348 let commands = ctx.take_commands().expect("cancellation compensation");
16349 assert_eq!(commands[0]["activity_type"], "trip.cancel-flight");
16350 }
16351
16352 fn workflow_task(
16353 workflow_type: &str,
16354 history_events: Vec<HistoryEvent>,
16355 payload_codec: &str,
16356 ) -> WorkflowTask {
16357 WorkflowTask {
16358 task_id: format!("wft-{workflow_type}"),
16359 workflow_command_id: None,
16360 workflow_id: Some(format!("wf-{workflow_type}")),
16361 run_id: Some(format!("run-{workflow_type}")),
16362 workflow_type: workflow_type.to_string(),
16363 cancel_requested: false,
16364 payload_codec: payload_codec.to_string(),
16365 arguments: Some(
16366 encode_value_envelope(&json!([]), payload_codec).expect("workflow arguments"),
16367 ),
16368 total_history_events: Some(history_events.len() as u64),
16369 history_size_bytes: None,
16370 continue_as_new_recommended: None,
16371 history_budget_pressure: None,
16372 history_events,
16373 next_history_page_token: None,
16374 workflow_task_attempt: 1,
16375 workflow_signal_id: None,
16376 signal_name: None,
16377 signal_arguments: None,
16378 workflow_update_id: None,
16379 update_name: None,
16380 lease_owner: Some("rust-worker".to_string()),
16381 }
16382 }
16383
16384 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
16385 struct SideEffectProbe {
16386 request_id: String,
16387 attempt: u32,
16388 }
16389
16390 #[test]
16391 fn typed_side_effect_runs_callback_once_and_replay_skips_it() {
16392 let calls = AtomicUsize::new(0);
16393 let ctx = workflow_context(Vec::new());
16394 let value = ctx
16395 .side_effect(|| {
16396 calls.fetch_add(1, Ordering::SeqCst);
16397 SideEffectProbe {
16398 request_id: "request-42".to_string(),
16399 attempt: 3,
16400 }
16401 })
16402 .expect("first side effect");
16403 assert_eq!(value.attempt, 3);
16404 assert_eq!(calls.load(Ordering::SeqCst), 1);
16405 let commands = ctx.take_commands().expect("commands");
16406 assert_eq!(commands.len(), 1);
16407 assert_eq!(commands[0]["type"], "record_side_effect");
16408 assert_eq!(
16409 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
16410 serde_json::to_value(&value).expect("value")
16411 );
16412
16413 let replay = workflow_context(vec![history_event(
16414 "SideEffectRecorded",
16415 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
16416 )]);
16417 let replayed: SideEffectProbe = replay
16418 .side_effect(|| {
16419 calls.fetch_add(1, Ordering::SeqCst);
16420 panic!("committed side-effect callbacks must not run during replay")
16421 })
16422 .expect("replayed side effect");
16423 assert_eq!(replayed, value);
16424 assert_eq!(calls.load(Ordering::SeqCst), 1);
16425 assert!(replay.take_commands().expect("commands").is_empty());
16426 replay.ensure_history_consumed().expect("history consumed");
16427 }
16428
16429 #[test]
16430 fn side_effect_uses_avro_envelope_and_uuid_is_replay_stable() {
16431 let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
16432 let value = ctx
16433 .side_effect(|| SideEffectProbe {
16434 request_id: "avro-request".to_string(),
16435 attempt: 1,
16436 })
16437 .expect("Avro side effect");
16438 let uuid = ctx.uuid_v4().expect("deterministic UUID");
16439 let commands = ctx.take_commands().expect("commands");
16440 assert_eq!(commands.len(), 2);
16441 assert_eq!(commands[0]["result"]["codec"], DEFAULT_CODEC);
16442 assert_eq!(commands[1]["result"]["codec"], DEFAULT_CODEC);
16443 assert_eq!(
16444 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
16445 serde_json::to_value(&value).expect("value")
16446 );
16447
16448 let replay = workflow_context_with_codec(
16449 vec![
16450 history_event(
16451 "SideEffectRecorded",
16452 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
16453 ),
16454 history_event(
16455 "SideEffectRecorded",
16456 json!({"sequence": 2, "result": commands[1]["result"].clone()}),
16457 ),
16458 ],
16459 DEFAULT_CODEC,
16460 );
16461 let replayed: SideEffectProbe = replay
16462 .side_effect(|| panic!("Avro callback must not run"))
16463 .expect("replayed Avro value");
16464 let replayed_uuid = replay.uuid_v4().expect("replayed UUID");
16465 assert_eq!(replayed, value);
16466 assert_eq!(replayed_uuid, uuid);
16467 assert!(replay.take_commands().expect("commands").is_empty());
16468 }
16469
16470 #[test]
16471 fn typed_side_effect_replay_preserves_bytes_and_maps() {
16472 let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
16473 let value = ctx
16474 .side_effect_avro_value(typed_fidelity_probe)
16475 .expect("typed side effect");
16476 let commands = ctx.take_commands().expect("side-effect command");
16477 assert_eq!(
16478 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
16479 .expect("recorded side effect"),
16480 value
16481 );
16482
16483 let replay = workflow_context_with_codec(
16484 vec![history_event(
16485 "SideEffectRecorded",
16486 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
16487 )],
16488 DEFAULT_CODEC,
16489 );
16490 assert_eq!(
16491 replay
16492 .side_effect_avro_value(|| panic!("replay must not invoke callback"))
16493 .expect("replayed typed side effect"),
16494 value
16495 );
16496 }
16497
16498 #[test]
16499 fn ordered_side_effects_share_the_durable_command_stream() {
16500 let first = encode_value_envelope(&json!("first"), DEFAULT_CODEC).expect("first");
16501 let second = encode_value_envelope(&json!(29), DEFAULT_CODEC).expect("second");
16502 let ctx = workflow_context(vec![
16503 history_event(
16504 "SideEffectRecorded",
16505 json!({"sequence": 1, "result": first}),
16506 ),
16507 history_event(
16508 "SideEffectRecorded",
16509 json!({"sequence": 2, "result": second}),
16510 ),
16511 ]);
16512 let first: String = ctx
16513 .side_effect(|| panic!("first callback must not run"))
16514 .expect("first replay");
16515 let second: i32 = ctx
16516 .side_effect(|| panic!("second callback must not run"))
16517 .expect("second replay");
16518 assert_eq!(first, "first");
16519 assert_eq!(second, 29);
16520 ctx.ensure_history_consumed().expect("ordered history");
16521
16522 let reordered = workflow_context(vec![history_event(
16523 "VersionMarkerRecorded",
16524 json!({
16525 "sequence": 1,
16526 "change_id": "before-side-effect",
16527 "version": 1,
16528 "min_supported": 1,
16529 "max_supported": 1,
16530 }),
16531 )]);
16532 let error = reordered
16533 .side_effect(|| "new".to_string())
16534 .expect_err("command reordering must fail");
16535 assert!(matches!(
16536 error,
16537 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16538 if reason == "recorded_command_mismatch"
16539 ));
16540 }
16541
16542 #[test]
16543 fn version_markers_replay_across_upgrades_and_do_not_duplicate() {
16544 let ctx = workflow_context(Vec::new());
16545 assert_eq!(ctx.get_version("checkout-v2", 1, 2).expect("version"), 2);
16546 assert_eq!(ctx.get_version("checkout-v2", 1, 3).expect("cached"), 2);
16547 assert!(ctx.patched("new-search").expect("patch"));
16548 ctx.deprecate_patch("new-search").expect("deprecate patch");
16549 let commands = ctx.take_commands().expect("commands");
16550 assert_eq!(commands.len(), 2);
16551 assert_eq!(commands[0]["type"], "record_version_marker");
16552 assert_eq!(commands[0]["version"], 2);
16553 assert_eq!(commands[1]["change_id"], "new-search");
16554
16555 let replay = workflow_context(vec![history_event(
16556 "VersionMarkerRecorded",
16557 json!({
16558 "sequence": 1,
16559 "change_id": "checkout-v2",
16560 "version": 2,
16561 "min_supported": 1,
16562 "max_supported": 2,
16563 }),
16564 )]);
16565 assert_eq!(replay.get_version("checkout-v2", 1, 4).expect("upgrade"), 2);
16566 assert_eq!(replay.get_version("checkout-v2", 2, 5).expect("repeat"), 2);
16567 assert!(replay.take_commands().expect("commands").is_empty());
16568 replay.ensure_history_consumed().expect("history consumed");
16569 }
16570
16571 #[test]
16572 fn version_markers_reject_incompatible_or_malformed_history() {
16573 let incompatible = workflow_context(vec![history_event(
16574 "VersionMarkerRecorded",
16575 json!({
16576 "sequence": 1,
16577 "change_id": "checkout-v2",
16578 "version": 1,
16579 "min_supported": 1,
16580 "max_supported": 2,
16581 }),
16582 )]);
16583 let error = incompatible
16584 .get_version("checkout-v2", 2, 3)
16585 .expect_err("old version is unsupported");
16586 assert!(matches!(
16587 error,
16588 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16589 if reason == "version_marker_incompatible_range"
16590 ));
16591
16592 for (history, reason) in [
16593 (
16594 vec![history_event("SideEffectRecorded", json!({"sequence": 1}))],
16595 "side_effect_result_missing",
16596 ),
16597 (
16598 vec![history_event(
16599 "SideEffectRecorded",
16600 json!({
16601 "sequence": 1,
16602 "result": {"codec": "avro", "blob": "not-base64"},
16603 }),
16604 )],
16605 "side_effect_payload_incompatible",
16606 ),
16607 (
16608 vec![history_event(
16609 "SideEffectRecorded",
16610 json!({"sequence": 1, "result": {"unwrapped": true}}),
16611 )],
16612 "side_effect_payload_malformed",
16613 ),
16614 (
16615 vec![history_event(
16616 "VersionMarkerRecorded",
16617 json!({
16618 "sequence": 1,
16619 "change_id": "change",
16620 "version": 1,
16621 "min_supported": 2,
16622 "max_supported": 1,
16623 }),
16624 )],
16625 "version_marker_history_range_invalid",
16626 ),
16627 ] {
16628 let error = WorkflowState::new(
16629 history,
16630 "rust-workers".to_string(),
16631 DEFAULT_CODEC.to_string(),
16632 None,
16633 )
16634 .expect_err("malformed history must fail");
16635 assert!(matches!(
16636 error,
16637 Error::NonDeterministicReplay(ReplayFailure { reason: actual, .. })
16638 if actual == reason
16639 ));
16640 }
16641 }
16642
16643 #[test]
16644 fn typed_search_attributes_replay_value_and_type_identity_after_restart() {
16645 let history = vec![history_event(
16646 "SearchAttributesUpserted",
16647 json!({
16648 "sequence": 1,
16649 "attributes": {"customer_tier": "gold"},
16650 "attribute_types": {"customer_tier": "keyword"},
16651 "merged": {"customer_tier": "gold"}
16652 }),
16653 )];
16654
16655 let matching = workflow_context(history.clone());
16656 matching
16657 .upsert_search_attributes(
16658 SearchAttributeUpdate::new()
16659 .keyword("customer_tier", "gold")
16660 .expect("keyword update"),
16661 )
16662 .expect("matching typed update must replay");
16663 matching
16664 .ensure_history_consumed()
16665 .expect("history consumed");
16666
16667 let changed_type = workflow_context(history.clone());
16668 let error = changed_type
16669 .upsert_search_attributes(
16670 SearchAttributeUpdate::new()
16671 .string("customer_tier", "gold")
16672 .expect("string update"),
16673 )
16674 .expect_err("same JSON value with a different declaration must be nondeterministic");
16675 let Error::NonDeterministicReplay(failure) = error else {
16676 panic!("typed identity drift must be a replay failure");
16677 };
16678 assert_eq!(failure.reason, "search_attribute_type_mismatch");
16679 assert_eq!(failure.sequence, Some(1));
16680
16681 let changed_value = workflow_context(history);
16682 let error = changed_value
16683 .upsert_search_attributes(
16684 SearchAttributeUpdate::new()
16685 .keyword("customer_tier", "platinum")
16686 .expect("keyword update"),
16687 )
16688 .expect_err("changed values must be nondeterministic");
16689 let Error::NonDeterministicReplay(failure) = error else {
16690 panic!("value drift must be a replay failure");
16691 };
16692 assert_eq!(failure.reason, "search_attribute_value_mismatch");
16693 }
16694
16695 #[test]
16696 fn legacy_search_attribute_history_keeps_type_identity_unknown() {
16697 let history = vec![history_event(
16698 "SearchAttributesUpserted",
16699 json!({
16700 "sequence": 1,
16701 "attributes": {"customer_tier": "gold"},
16702 "merged": {"customer_tier": "gold"}
16703 }),
16704 )];
16705
16706 for update in [
16707 SearchAttributeUpdate::new()
16708 .keyword("customer_tier", "gold")
16709 .expect("keyword update"),
16710 SearchAttributeUpdate::new()
16711 .string("customer_tier", "gold")
16712 .expect("string update"),
16713 ] {
16714 let restarted = workflow_context(history.clone());
16715 restarted
16716 .upsert_search_attributes(update)
16717 .expect("legacy history constrains values but has unknown type identity");
16718 restarted
16719 .ensure_history_consumed()
16720 .expect("history consumed");
16721 }
16722 }
16723
16724 #[test]
16725 fn search_attribute_command_emits_canonical_types() {
16726 let ctx = workflow_context(Vec::new());
16727 ctx.upsert_search_attributes(
16728 SearchAttributeUpdate::new()
16729 .keyword("customer_tier", "gold")
16730 .expect("keyword update")
16731 .int("attempts", 3)
16732 .expect("int update")
16733 .delete("obsolete")
16734 .expect("delete update"),
16735 )
16736 .expect("valid search attributes");
16737
16738 assert_eq!(
16739 ctx.take_commands().expect("commands"),
16740 vec![json!({
16741 "type": "upsert_search_attributes",
16742 "attributes": {
16743 "attempts": 3,
16744 "customer_tier": "gold",
16745 "obsolete": null
16746 },
16747 "attribute_types": {
16748 "attempts": "int",
16749 "customer_tier": "keyword"
16750 }
16751 })]
16752 );
16753 }
16754
16755 #[test]
16756 fn duplicate_side_effects_and_version_markers_are_rejected() {
16757 let duplicate_side_effect = WorkflowState::new(
16758 vec![
16759 history_event(
16760 "SideEffectRecorded",
16761 json!({"sequence": 1, "result": fixture_envelope(json!(1))}),
16762 ),
16763 history_event(
16764 "SideEffectRecorded",
16765 json!({"sequence": 1, "result": fixture_envelope(json!(2))}),
16766 ),
16767 ],
16768 "rust-workers".to_string(),
16769 DEFAULT_CODEC.to_string(),
16770 None,
16771 )
16772 .expect_err("duplicate side effect");
16773 assert!(matches!(
16774 duplicate_side_effect,
16775 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16776 if reason == "duplicate_side_effect_record"
16777 ));
16778
16779 let marker = |sequence| {
16780 history_event(
16781 "VersionMarkerRecorded",
16782 json!({
16783 "sequence": sequence,
16784 "change_id": "same-change",
16785 "version": 1,
16786 "min_supported": 1,
16787 "max_supported": 1,
16788 }),
16789 )
16790 };
16791 let duplicate_marker = WorkflowState::new(
16792 vec![marker(1), marker(3)],
16793 "rust-workers".to_string(),
16794 DEFAULT_CODEC.to_string(),
16795 None,
16796 )
16797 .expect_err("duplicate marker");
16798 assert!(matches!(
16799 duplicate_marker,
16800 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16801 if reason == "duplicate_version_marker"
16802 ));
16803 }
16804
16805 #[test]
16806 fn workflow_stream_authoring_derives_identity_and_replay_skips_duplicate_append() {
16807 let mut state = WorkflowState::new(
16808 Vec::new(),
16809 "rust-workers".to_string(),
16810 DEFAULT_CODEC.to_string(),
16811 None,
16812 )
16813 .expect("workflow state");
16814 state.workflow_command_identity = "command-7".to_string();
16815 let context = WorkflowContext {
16816 state: Arc::new(Mutex::new(state)),
16817 };
16818 let item =
16819 WorkflowStreamAppendItem::from_reference("s3://bucket/item.avro").item_type("receipt");
16820
16821 context
16822 .append_workflow_stream("output", &[item], Some(10))
16823 .expect("append command");
16824 context
16825 .error_workflow_stream("output", "producer failed", None)
16826 .expect("error command");
16827 let commands = context.take_commands().expect("commands");
16828
16829 assert_eq!(commands[0]["type"], "record_side_effect");
16830 assert_eq!(
16831 commands[0]["workflow_stream"]["command_identity"],
16832 "command-7"
16833 );
16834 assert_eq!(commands[0]["workflow_stream"]["command_ordinal"], 0);
16835 assert_eq!(
16836 commands[0]["workflow_stream"]["items"][0]["idempotency_key"],
16837 "dw-stream:command-7:0:0"
16838 );
16839 assert_eq!(commands[1]["workflow_stream"]["operation"], "error");
16840
16841 let recorded = history_event(
16842 "SideEffectRecorded",
16843 json!({"sequence": 1, "result": fixture_envelope(Value::Null)}),
16844 );
16845 let mut replay_state = WorkflowState::new(
16846 vec![recorded],
16847 "rust-workers".to_string(),
16848 DEFAULT_CODEC.to_string(),
16849 None,
16850 )
16851 .expect("replay state");
16852 replay_state.workflow_command_identity = "command-7".to_string();
16853 let replay_context = WorkflowContext {
16854 state: Arc::new(Mutex::new(replay_state)),
16855 };
16856 replay_context
16857 .append_workflow_stream(
16858 "output",
16859 &[WorkflowStreamAppendItem::from_reference(
16860 "s3://bucket/item.avro",
16861 )],
16862 Some(10),
16863 )
16864 .expect("replayed append");
16865 assert!(replay_context
16866 .take_commands()
16867 .expect("replayed commands")
16868 .is_empty());
16869 }
16870
16871 #[test]
16872 fn workflow_stream_authoring_requires_server_durable_command_identity() {
16873 let context = workflow_context(Vec::new());
16874 let error = context
16875 .append_workflow_stream(
16876 "output",
16877 &[WorkflowStreamAppendItem::from_reference(
16878 "s3://bucket/item.avro",
16879 )],
16880 None,
16881 )
16882 .expect_err("stream append without durable command identity must fail closed");
16883
16884 assert!(matches!(error, Error::MissingWorkflowCommandIdentity));
16885 assert!(context.take_commands().expect("commands").is_empty());
16886 }
16887
16888 #[test]
16889 fn cold_worker_replay_does_not_repeat_committed_side_effects_or_markers() {
16890 fn worker(calls: Arc<AtomicUsize>) -> Worker {
16891 let client = Client::new("http://127.0.0.1:8080").expect("client");
16892 let mut worker = Worker::new(client, "rust-workers");
16893 worker.register_workflow("rust.side-effect-version", move |ctx, _input| {
16894 let calls = Arc::clone(&calls);
16895 async move {
16896 let captured = ctx.side_effect(|| {
16897 calls.fetch_add(1, Ordering::SeqCst);
16898 "captured-once".to_string()
16899 })?;
16900 let version = ctx.get_version("cold-restart", 1, 2)?;
16901 Ok(json!({"captured": captured, "version": version}))
16902 }
16903 });
16904 worker
16905 }
16906
16907 fn task(history_events: Vec<HistoryEvent>) -> WorkflowTask {
16908 WorkflowTask {
16909 task_id: "wft-side-effect-version".to_string(),
16910 workflow_command_id: None,
16911 workflow_id: Some("wf-side-effect-version".to_string()),
16912 run_id: Some("run-side-effect-version".to_string()),
16913 workflow_type: "rust.side-effect-version".to_string(),
16914 cancel_requested: false,
16915 payload_codec: DEFAULT_CODEC.to_string(),
16916 arguments: Some(
16917 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("arguments"),
16918 ),
16919 history_events,
16920 total_history_events: None,
16921 history_size_bytes: None,
16922 continue_as_new_recommended: None,
16923 history_budget_pressure: None,
16924 next_history_page_token: None,
16925 workflow_task_attempt: 1,
16926 workflow_signal_id: None,
16927 signal_name: None,
16928 signal_arguments: None,
16929 workflow_update_id: None,
16930 update_name: None,
16931 lease_owner: Some("rust-worker".to_string()),
16932 }
16933 }
16934
16935 let calls = Arc::new(AtomicUsize::new(0));
16936 let initial = worker(Arc::clone(&calls))
16937 .execute_workflow_task(task(Vec::new()))
16938 .expect("initial execution");
16939 assert_eq!(
16940 initial
16941 .iter()
16942 .map(|command| &command["type"])
16943 .collect::<Vec<_>>(),
16944 vec![
16945 "record_side_effect",
16946 "record_version_marker",
16947 "complete_workflow"
16948 ]
16949 );
16950 assert_eq!(calls.load(Ordering::SeqCst), 1);
16951
16952 let restarted = worker(Arc::clone(&calls));
16953 let replayed = restarted
16954 .execute_workflow_task(task(vec![
16955 history_event(
16956 "SideEffectRecorded",
16957 json!({"sequence": 1, "result": initial[0]["result"].clone()}),
16958 ),
16959 history_event(
16960 "VersionMarkerRecorded",
16961 json!({
16962 "sequence": 2,
16963 "change_id": "cold-restart",
16964 "version": 2,
16965 "min_supported": 1,
16966 "max_supported": 2,
16967 }),
16968 ),
16969 ]))
16970 .expect("cold replay");
16971 assert_eq!(replayed.len(), 1);
16972 assert_eq!(replayed[0]["type"], "complete_workflow");
16973 assert_eq!(calls.load(Ordering::SeqCst), 1);
16974 }
16975
16976 #[test]
16977 fn side_effect_replay_rejects_changed_rust_value_type() {
16978 let result = encode_value_envelope(&json!({"value": 42}), DEFAULT_CODEC).expect("result");
16979 let ctx = workflow_context(vec![history_event(
16980 "SideEffectRecorded",
16981 json!({"sequence": 1, "result": result}),
16982 )]);
16983 let error = ctx
16984 .side_effect::<Vec<String>, _>(|| panic!("callback must not run"))
16985 .expect_err("changed type must fail replay");
16986 assert!(matches!(
16987 error,
16988 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16989 if reason == "side_effect_type_mismatch"
16990 ));
16991 }
16992
16993 fn completed_retry_activity_history() -> Vec<HistoryEvent> {
16994 vec![
16995 history_event(
16996 "ActivityScheduled",
16997 json!({
16998 "sequence": 1,
16999 "activity_type": "flaky",
17000 "activity_execution_id": "act-1",
17001 "activity": {
17002 "id": "act-1",
17003 "sequence": 1,
17004 "type": "flaky",
17005 "queue": "critical-activities",
17006 "execution_mode": null,
17007 "retry_policy": {
17008 "snapshot_version": 1,
17009 "max_attempts": 3,
17010 "backoff_seconds": [2, 4],
17011 "start_to_close_timeout": 30,
17012 "schedule_to_start_timeout": 5,
17013 "schedule_to_close_timeout": 90,
17014 "heartbeat_timeout": 10,
17015 "non_retryable_error_types": ["PermanentError"]
17016 }
17017 }
17018 }),
17019 ),
17020 history_event(
17021 "ActivityStarted",
17022 json!({
17023 "sequence": 1,
17024 "activity_type": "flaky",
17025 "activity_execution_id": "act-1",
17026 "activity_attempt_id": "attempt-1",
17027 "attempt_number": 1
17028 }),
17029 ),
17030 history_event(
17031 "ActivityRetryScheduled",
17032 json!({
17033 "sequence": 1,
17034 "activity_type": "flaky",
17035 "activity_execution_id": "act-1",
17036 "activity_attempt_id": "attempt-1",
17037 "attempt_number": 1,
17038 "retry_after_attempt": 1,
17039 "retry_backoff_seconds": 2,
17040 "failure_category": "activity",
17041 "exception_type": "TransientError"
17042 }),
17043 ),
17044 history_event(
17045 "ActivityStarted",
17046 json!({
17047 "sequence": 1,
17048 "activity_type": "flaky",
17049 "activity_execution_id": "act-1",
17050 "activity_attempt_id": "attempt-2",
17051 "attempt_number": 2
17052 }),
17053 ),
17054 history_event(
17055 "ActivityCompleted",
17056 json!({
17057 "sequence": 1,
17058 "activity_type": "flaky",
17059 "activity_execution_id": "act-1",
17060 "activity_attempt_id": "attempt-2",
17061 "attempt_number": 2,
17062 "payload_codec": DEFAULT_CODEC,
17063 "result": fixture_envelope(json!({"status":"recovered"}))
17064 }),
17065 ),
17066 ]
17067 }
17068
17069 fn retry_activity_options() -> ActivityOptions {
17070 ActivityOptions::new()
17071 .task_queue("critical-activities")
17072 .retry_policy(
17073 ActivityRetryPolicy::new(3)
17074 .backoff_intervals([Duration::from_secs(2), Duration::from_secs(4)])
17075 .non_retryable_error_type("PermanentError"),
17076 )
17077 .start_to_close_timeout(Duration::from_secs(30))
17078 .schedule_to_start_timeout(Duration::from_secs(5))
17079 .schedule_to_close_timeout(Duration::from_secs(90))
17080 .heartbeat_timeout(Duration::from_secs(10))
17081 }
17082
17083 #[test]
17084 fn fixed_avro_value_round_trips_json_values() {
17085 let value = json!({"greeting": "hello", "count": 3, "ok": true});
17086 let envelope = PayloadEnvelope::avro(&value).expect("encode");
17087 assert_eq!(envelope.codec, DEFAULT_CODEC);
17088 assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
17089 }
17090
17091 #[tokio::test]
17092 async fn typed_handler_adapters_round_trip_serde_contracts_on_the_fixed_wire() {
17093 let client = Client::new("http://127.0.0.1:8080").expect("client");
17094 let mut worker = Worker::new(client, "rust-workers");
17095 worker.register_typed_workflow(
17096 "typed.contract.workflow",
17097 |_ctx, input: TypedContract| async move { Ok(input) },
17098 );
17099 worker.register_typed_activity(
17100 "typed.contract.activity",
17101 |_ctx, input: TypedContract| async move { Ok(input) },
17102 );
17103
17104 let expected = typed_contract();
17105 let arguments = AvroValue::Array(vec![
17106 AvroValue::from_serialize(&expected).expect("typed request")
17107 ]);
17108 let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("arguments");
17109 let mut workflow = workflow_task("typed.contract.workflow", Vec::new(), DEFAULT_CODEC);
17110 workflow.arguments = Some(envelope.clone());
17111 let commands = worker
17112 .execute_workflow_task(workflow)
17113 .expect("typed workflow task");
17114 let workflow_result: TypedContract =
17115 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17116 .expect("workflow result envelope")
17117 .deserialize()
17118 .expect("workflow result type");
17119 assert_eq!(workflow_result, expected);
17120
17121 let activity = ActivityTask {
17122 task_id: "typed-contract-activity".to_string(),
17123 activity_attempt_id: Some("typed-contract-attempt".to_string()),
17124 attempt_id: None,
17125 activity_type: "typed.contract.activity".to_string(),
17126 payload_codec: DEFAULT_CODEC.to_string(),
17127 arguments: Some(envelope),
17128 attempt_number: 1,
17129 lease_owner: Some("rust-worker".to_string()),
17130 };
17131 let activity_result: TypedContract = worker
17132 .execute_activity_task(activity)
17133 .await
17134 .expect("typed activity task")
17135 .deserialize()
17136 .expect("activity result type");
17137 assert_eq!(activity_result, expected);
17138 }
17139
17140 #[tokio::test]
17141 async fn typed_handler_errors_include_handler_name_direction_and_rust_type() {
17142 let client = Client::new("http://127.0.0.1:8080").expect("client");
17143 let mut worker = Worker::new(client, "rust-workers");
17144 worker.register_typed_workflow(
17145 "typed.shape.workflow",
17146 |_ctx, input: TypedContract| async move { Ok(input) },
17147 );
17148 worker.register_typed_activity("typed.unsupported.activity", |_ctx, (): ()| async move {
17149 Ok(f64::NAN)
17150 });
17151
17152 let mut workflow = workflow_task("typed.shape.workflow", Vec::new(), DEFAULT_CODEC);
17153 workflow.arguments = Some(
17154 encode_typed_envelope(
17155 &AvroValue::Array(vec![
17156 AvroValue::String("first".to_string()),
17157 AvroValue::String("second".to_string()),
17158 ]),
17159 DEFAULT_CODEC,
17160 )
17161 .expect("malformed typed arguments"),
17162 );
17163 let commands = worker
17164 .execute_workflow_task(workflow)
17165 .expect("shape mismatch becomes a workflow failure");
17166 let message = commands[0]["message"].as_str().expect("failure message");
17167 assert!(message.contains("workflow handler \"typed.shape.workflow\" input type"));
17168 assert!(message.contains(type_name::<TypedContract>()));
17169 assert!(message.contains("task carried 2 arguments"));
17170
17171 let activity = ActivityTask {
17172 task_id: "typed-unsupported-activity".to_string(),
17173 activity_attempt_id: Some("typed-unsupported-attempt".to_string()),
17174 attempt_id: None,
17175 activity_type: "typed.unsupported.activity".to_string(),
17176 payload_codec: DEFAULT_CODEC.to_string(),
17177 arguments: Some(
17178 encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
17179 .expect("unit arguments"),
17180 ),
17181 attempt_number: 1,
17182 lease_owner: Some("rust-worker".to_string()),
17183 };
17184 let Error::HandlerType {
17185 handler_kind,
17186 handler_name,
17187 value_kind,
17188 rust_type,
17189 message,
17190 } = worker
17191 .execute_activity_task(activity)
17192 .await
17193 .expect_err("non-finite handler output must fail")
17194 else {
17195 panic!("expected contextual handler type failure");
17196 };
17197 assert_eq!(handler_kind, HandlerKind::Activity);
17198 assert_eq!(handler_name, "typed.unsupported.activity");
17199 assert_eq!(value_kind, HandlerValueKind::Result);
17200 assert_eq!(rust_type, type_name::<f64>());
17201 assert!(message.contains("non_finite_float"));
17202 }
17203
17204 #[tokio::test]
17205 async fn typed_replayed_workflow_decodes_input_and_activity_result_losslessly() {
17206 #[derive(Clone, Default)]
17207 struct State {
17208 observed: Option<TypedContract>,
17209 }
17210
17211 let client = Client::new("http://127.0.0.1:8080").expect("client");
17212 let mut worker = Worker::new(client, "rust-workers");
17213 worker.register_typed_replayed_workflow(
17214 "typed.contract.replayed",
17215 State::default,
17216 |ctx, input: TypedContract, state| async move {
17217 let result: TypedContract =
17218 ctx.activity_typed("typed.contract.activity", input).await?;
17219 state.update(|current| current.observed = Some(result.clone()))?;
17220 Ok(result)
17221 },
17222 );
17223 worker.register_replayed_query::<State, _, _>(
17224 "typed.contract.replayed",
17225 "observed",
17226 |_ctx, state, _args| async move {
17227 Ok(json!(state.observed.as_ref().map(|value| value.signed)))
17228 },
17229 );
17230
17231 let expected = typed_contract();
17232 let typed_value = AvroValue::from_serialize(&expected).expect("typed value");
17233 let workflow_arguments =
17234 encode_typed_envelope(&AvroValue::Array(vec![typed_value.clone()]), DEFAULT_CODEC)
17235 .expect("workflow arguments");
17236 let result = encode_typed_envelope(&typed_value, DEFAULT_CODEC).expect("activity result");
17237 let task = QueryTask {
17238 query_task_id: "typed-replay-query".to_string(),
17239 query_task_attempt: 1,
17240 lease_owner: Some("rust-worker".to_string()),
17241 workflow_id: Some("typed-replay".to_string()),
17242 run_id: Some("typed-replay-run".to_string()),
17243 workflow_type: "typed.contract.replayed".to_string(),
17244 query_name: "observed".to_string(),
17245 payload_codec: DEFAULT_CODEC.to_string(),
17246 workflow_arguments: Some(workflow_arguments),
17247 query_arguments: Some(
17248 encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
17249 .expect("query arguments"),
17250 ),
17251 history_events: vec![
17252 history_event(
17253 "ActivityScheduled",
17254 json!({
17255 "sequence": 1,
17256 "activity_type": "typed.contract.activity"
17257 }),
17258 ),
17259 history_event(
17260 "ActivityCompleted",
17261 json!({
17262 "sequence": 1,
17263 "activity_type": "typed.contract.activity",
17264 "payload_codec": DEFAULT_CODEC,
17265 "result": result
17266 }),
17267 ),
17268 ],
17269 history_export: None,
17270 run_status: Some("completed".to_string()),
17271 };
17272
17273 assert_eq!(
17274 worker
17275 .execute_query_task(task)
17276 .await
17277 .expect("typed replay query")
17278 .deserialize::<i64>()
17279 .expect("query result"),
17280 expected.signed
17281 );
17282 }
17283
17284 #[tokio::test]
17285 async fn typed_worker_surfaces_preserve_bytes_and_map_list_identity() {
17286 let client = Client::new("http://127.0.0.1:8080").expect("client");
17287 let mut worker = Worker::new(client, "rust-workers");
17288 worker.register_workflow_avro_value("typed.echo", |_ctx, input| async move { Ok(input) });
17289 worker
17290 .register_activity_avro_value("typed.activity", |_ctx, input| async move { Ok(input) });
17291 worker.register_query_avro_value("typed.echo", "inspect", |_ctx, input| async move {
17292 Ok(input)
17293 });
17294 worker.register_update_avro_value("typed.echo", "replace", |_ctx, input| async move {
17295 Ok(input)
17296 });
17297 worker.register_workflow_avro_value("typed.signal", |ctx, _input| async move {
17298 Ok(AvroValue::Array(
17299 ctx.wait_signal_avro_value("changed").await?,
17300 ))
17301 });
17302
17303 let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
17304 let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed envelope");
17305
17306 let mut workflow = workflow_task("typed.echo", Vec::new(), DEFAULT_CODEC);
17307 workflow.arguments = Some(envelope.clone());
17308 let commands = worker
17309 .execute_workflow_task(workflow)
17310 .expect("typed workflow task");
17311 assert_eq!(commands[0]["type"], "complete_workflow");
17312 assert_eq!(
17313 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17314 .expect("typed workflow result"),
17315 arguments
17316 );
17317
17318 let activity = ActivityTask {
17319 task_id: "activity-typed".to_string(),
17320 activity_attempt_id: Some("attempt-typed".to_string()),
17321 attempt_id: None,
17322 activity_type: "typed.activity".to_string(),
17323 payload_codec: DEFAULT_CODEC.to_string(),
17324 arguments: Some(envelope.clone()),
17325 attempt_number: 1,
17326 lease_owner: Some("rust-worker".to_string()),
17327 };
17328 assert_eq!(
17329 worker
17330 .execute_activity_task(activity)
17331 .await
17332 .expect("typed activity result"),
17333 arguments
17334 );
17335
17336 let query = QueryTask {
17337 query_task_id: "query-typed".to_string(),
17338 query_task_attempt: 1,
17339 lease_owner: Some("rust-worker".to_string()),
17340 workflow_id: Some("typed-1".to_string()),
17341 run_id: Some("run-typed".to_string()),
17342 workflow_type: "typed.echo".to_string(),
17343 query_name: "inspect".to_string(),
17344 payload_codec: DEFAULT_CODEC.to_string(),
17345 workflow_arguments: Some(
17346 encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
17347 .expect("workflow input"),
17348 ),
17349 query_arguments: Some(envelope.clone()),
17350 history_events: Vec::new(),
17351 history_export: None,
17352 run_status: Some("running".to_string()),
17353 };
17354 assert_eq!(
17355 worker
17356 .execute_query_task(query)
17357 .await
17358 .expect("typed query result"),
17359 arguments
17360 );
17361
17362 let mut update = workflow_task(
17363 "typed.echo",
17364 vec![history_event(
17365 "UpdateAccepted",
17366 json!({
17367 "update_id": "update-typed",
17368 "update_name": "replace",
17369 "arguments": envelope.clone(),
17370 }),
17371 )],
17372 DEFAULT_CODEC,
17373 );
17374 update.workflow_update_id = Some("update-typed".to_string());
17375 update.update_name = Some("replace".to_string());
17376 let commands = worker
17377 .execute_workflow_task(update)
17378 .expect("typed update task");
17379 assert_eq!(commands[0]["type"], "complete_update");
17380 assert_eq!(
17381 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17382 .expect("typed update result"),
17383 arguments
17384 );
17385
17386 let mut signal = workflow_task(
17387 "typed.signal",
17388 vec![history_event(
17389 "SignalReceived",
17390 json!({
17391 "signal_id": "signal-typed",
17392 "signal_name": "changed",
17393 "arguments": envelope.clone(),
17394 }),
17395 )],
17396 DEFAULT_CODEC,
17397 );
17398 signal.workflow_signal_id = Some("signal-typed".to_string());
17399 signal.signal_name = Some("changed".to_string());
17400 signal.signal_arguments = Some(envelope);
17401 let commands = worker
17402 .execute_workflow_task(signal)
17403 .expect("typed signal resume");
17404 assert_eq!(
17405 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17406 .expect("typed signal result"),
17407 arguments
17408 );
17409 }
17410
17411 #[tokio::test]
17412 async fn typed_helpers_never_parse_json_inspection_projection() {
17413 let collision_values = projection_collision_probe();
17414 let expected = AvroValue::Array(collision_values.clone());
17415 let envelope = encode_typed_envelope(&expected, DEFAULT_CODEC).expect("collision envelope");
17416
17417 let activity_context = workflow_context_with_codec(
17418 vec![history_event(
17419 "ActivityCompleted",
17420 json!({
17421 "sequence": 1,
17422 "activity_type": "collision.activity",
17423 "payload_codec": DEFAULT_CODEC,
17424 "result": envelope.clone(),
17425 }),
17426 )],
17427 DEFAULT_CODEC,
17428 );
17429 assert_eq!(
17430 activity_context
17431 .activity_avro_value("collision.activity", AvroValue::Array(Vec::new()))
17432 .await
17433 .expect("typed activity collision result"),
17434 expected
17435 );
17436
17437 let signal_context = workflow_context_with_codec(
17438 vec![
17439 history_event(
17440 "SignalWaitOpened",
17441 json!({"sequence": 1, "signal_name": "collision"}),
17442 ),
17443 history_event(
17444 "SignalApplied",
17445 json!({
17446 "sequence": 1,
17447 "signal_name": "collision",
17448 "payload_codec": DEFAULT_CODEC,
17449 "value": envelope.clone(),
17450 }),
17451 ),
17452 ],
17453 DEFAULT_CODEC,
17454 );
17455 assert_eq!(
17456 signal_context
17457 .wait_signal_avro_value("collision")
17458 .await
17459 .expect("typed signal collision arguments"),
17460 collision_values
17461 );
17462
17463 let child_context = workflow_context_with_codec(
17464 vec![
17465 history_event(
17466 "ChildWorkflowScheduled",
17467 json!({
17468 "sequence": 1,
17469 "child_workflow_instance_id": "collision-child",
17470 "child_workflow_run_id": "collision-run",
17471 "child_workflow_type": "collision.child",
17472 }),
17473 ),
17474 history_event(
17475 "ChildRunCompleted",
17476 json!({
17477 "sequence": 1,
17478 "child_workflow_instance_id": "collision-child",
17479 "child_workflow_run_id": "collision-run",
17480 "child_workflow_type": "collision.child",
17481 "payload_codec": DEFAULT_CODEC,
17482 "result": envelope,
17483 }),
17484 ),
17485 ],
17486 DEFAULT_CODEC,
17487 );
17488 let child = child_context
17489 .start_child_workflow_avro_value(
17490 "collision.child",
17491 ChildWorkflowOptions::new("collision-workers"),
17492 AvroValue::Array(Vec::new()),
17493 )
17494 .await
17495 .expect("typed child collision result");
17496 assert_eq!(child.result, expected);
17497 }
17498
17499 #[tokio::test]
17500 async fn replayed_typed_query_keeps_lossless_workflow_and_query_inputs() {
17501 let client = Client::new("http://127.0.0.1:8080").expect("client");
17502 let mut worker = Worker::new(client, "rust-workers");
17503 worker.register_replayed_workflow_avro_value(
17504 "typed.replayed",
17505 || (),
17506 |_ctx, input, _state| async move { Ok(input) },
17507 );
17508 worker.register_replayed_query_avro_value::<(), _, _>(
17509 "typed.replayed",
17510 "inspect",
17511 |ctx, _state, args| async move {
17512 let mut signals = ctx.signals_avro_value("collision");
17513 let signal = signals
17514 .pop()
17515 .map(AvroValue::Array)
17516 .unwrap_or_else(|| AvroValue::Array(Vec::new()));
17517 Ok(AvroValue::Array(vec![
17518 ctx.workflow_input_avro_value().clone(),
17519 signal,
17520 args,
17521 ]))
17522 },
17523 );
17524 let arguments = AvroValue::Array(projection_collision_probe());
17525 let signal_arguments =
17526 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed query signal arguments");
17527 let task = QueryTask {
17528 query_task_id: "query-typed-replay".to_string(),
17529 query_task_attempt: 1,
17530 lease_owner: Some("rust-worker".to_string()),
17531 workflow_id: Some("typed-replay".to_string()),
17532 run_id: Some("run-typed-replay".to_string()),
17533 workflow_type: "typed.replayed".to_string(),
17534 query_name: "inspect".to_string(),
17535 payload_codec: DEFAULT_CODEC.to_string(),
17536 workflow_arguments: Some(
17537 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("workflow arguments"),
17538 ),
17539 query_arguments: Some(
17540 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("query arguments"),
17541 ),
17542 history_events: vec![history_event(
17543 "SignalReceived",
17544 json!({
17545 "signal_id": "collision-signal",
17546 "signal_name": "collision",
17547 "workflow_sequence": 1,
17548 "payload_codec": DEFAULT_CODEC,
17549 "arguments": signal_arguments,
17550 }),
17551 )],
17552 history_export: None,
17553 run_status: Some("completed".to_string()),
17554 };
17555
17556 assert_eq!(
17557 worker
17558 .execute_query_task(task)
17559 .await
17560 .expect("typed replay query"),
17561 AvroValue::Array(vec![arguments.clone(), arguments.clone(), arguments])
17562 );
17563 }
17564
17565 #[test]
17566 fn public_avro_adapter_rejects_non_string_map_keys_before_json_conversion() {
17567 let value = BTreeMap::from([(1_i32, "integer key")]);
17568 let error = PayloadEnvelope::avro(&value)
17569 .expect_err("integer map keys must fail")
17570 .to_string();
17571
17572 assert!(error.contains("invalid_map_key"));
17573 }
17574
17575 #[test]
17576 fn json_tagged_payload_fails_closed_with_actionable_diagnostic() {
17577 let envelope = PayloadEnvelope {
17578 codec: "json".to_string(),
17579 blob: r#"{"greeting":"hello"}"#.to_string(),
17580 };
17581
17582 let error = decode_payload::<Value>(&envelope).expect_err("JSON payload must fail");
17583 let diagnostic = error.to_string();
17584 assert!(diagnostic.contains("unsupported_payload_codec"));
17585 assert!(diagnostic.contains("codec=\"avro\""));
17586 assert!(diagnostic.contains("HTTP document transport"));
17587 }
17588
17589 #[test]
17590 fn untagged_json_payload_value_fails_closed() {
17591 let error = decode_wire_value(&json!({"stale": true}), DEFAULT_CODEC)
17592 .expect_err("untagged JSON payload values must fail");
17593 let diagnostic = error.to_string();
17594 assert!(diagnostic.contains("unsupported_payload_codec"));
17595 assert!(diagnostic.contains("untagged durable payload"));
17596 assert!(diagnostic.contains("HTTP document transport"));
17597 }
17598
17599 #[test]
17600 fn prerelease_avro_payload_without_single_object_frame_is_rejected() {
17601 let envelope = PayloadEnvelope {
17602 codec: DEFAULT_CODEC.to_string(),
17603 blob: BASE64.encode([0x01]),
17604 };
17605
17606 let error = decode_payload::<Value>(&envelope).expect_err("prerelease payload must fail");
17607 assert!(error.to_string().contains("invalid_payload_framing"));
17608 }
17609
17610 #[tokio::test]
17611 async fn workflow_completion_rejects_invalid_payload_slots_without_transport() {
17612 let server = MockWorkerServer::start();
17613 let client = Client::builder(server.base_url())
17614 .timeout(Duration::from_secs(2))
17615 .build()
17616 .expect("client");
17617 let invalid_commands = [
17618 json!({
17619 "type": "complete_workflow",
17620 "result": {"codec": "json", "blob": null}
17621 }),
17622 json!({
17623 "type": "schedule_activity",
17624 "arguments": {"codec": "yaml", "blob": "ignored"}
17625 }),
17626 json!({
17627 "type": "start_child_workflow",
17628 "arguments": {"codec": DEFAULT_CODEC, "blob": null}
17629 }),
17630 json!({"type": "continue_as_new", "arguments": []}),
17631 json!({"type": "complete_update"}),
17632 json!({"type": "record_side_effect", "result": null}),
17633 json!({
17634 "type": "start_service_operation",
17635 "payload_codec": DEFAULT_CODEC,
17636 "request_payload": "raw-avro-bytes"
17637 }),
17638 ];
17639
17640 for command in invalid_commands {
17641 let error = client
17642 .complete_workflow_task("invalid-codec", "rust-worker", 1, vec![command])
17643 .await
17644 .expect_err("invalid durable payload must fail locally");
17645 let diagnostic = error.to_string();
17646 assert!(
17647 diagnostic.contains("unsupported_payload_codec")
17648 || diagnostic.contains("invalid_payload_envelope")
17649 || diagnostic.contains("untagged durable payload"),
17650 "unexpected validation diagnostic: {diagnostic}"
17651 );
17652 }
17653
17654 assert_eq!(
17655 server.request_count("/api/worker/workflow-tasks/invalid-codec/complete"),
17656 0,
17657 "invalid command payloads must not reach HTTP transport"
17658 );
17659 }
17660
17661 #[test]
17662 fn workflow_completion_validates_only_protocol_owned_payload_slots() {
17663 let envelope = fixture_envelope(json!({"codec": "customer-value"}));
17664 let commands = [
17665 json!({"type": "complete_workflow", "result": envelope.clone()}),
17666 json!({"type": "schedule_activity", "arguments": envelope.clone()}),
17667 json!({"type": "start_child_workflow", "arguments": envelope.clone()}),
17668 json!({"type": "continue_as_new", "arguments": envelope.clone()}),
17669 json!({"type": "complete_update", "result": envelope.clone()}),
17670 json!({"type": "record_side_effect", "result": envelope.clone()}),
17671 json!({
17672 "type": "start_service_operation",
17673 "payload_codec": DEFAULT_CODEC,
17674 "request_payload": envelope.clone()
17675 }),
17676 json!({
17677 "type": "complete_workflow",
17678 "result": envelope,
17679 "metadata": {
17680 "codec": "json",
17681 "payload_codec": "customer-codec",
17682 "result": {"codec": "yaml", "blob": null}
17683 }
17684 }),
17685 ];
17686
17687 validate_workflow_task_commands(&commands)
17688 .expect("customer metadata must not become a protocol codec declaration");
17689 }
17690
17691 #[test]
17692 fn valid_avro_tasks_normalize_absent_and_null_arguments_to_empty_lists() {
17693 assert_eq!(
17694 decode_task_avro_arguments(None, DEFAULT_CODEC).expect("absent arguments"),
17695 AvroValue::Array(Vec::new())
17696 );
17697 assert_eq!(
17698 decode_task_avro_arguments(Some(&Value::Null), DEFAULT_CODEC).expect("null arguments"),
17699 AvroValue::Array(Vec::new())
17700 );
17701
17702 let mut signal = workflow_task("missing", Vec::new(), DEFAULT_CODEC);
17703 signal.signal_name = Some("empty-signal".to_string());
17704 signal.signal_arguments = None;
17705 let decoded = decode_resume_signal(&signal)
17706 .expect("valid Avro signal")
17707 .expect("named signal resumes the workflow");
17708 assert!(decoded.arguments.is_empty());
17709 }
17710
17711 #[tokio::test]
17712 async fn malformed_task_level_codecs_become_pre_handler_failures() {
17713 let client = Client::new("http://127.0.0.1:8080").expect("client");
17714 let mut worker = Worker::new(client, "rust-workers");
17715 let handler_calls = Arc::new(AtomicUsize::new(0));
17716
17717 let calls = Arc::clone(&handler_calls);
17718 worker.register_workflow("codec.workflow", move |_ctx, _args| {
17719 calls.fetch_add(1, Ordering::SeqCst);
17720 async move { Ok(Value::Null) }
17721 });
17722 let calls = Arc::clone(&handler_calls);
17723 worker.register_activity("codec.activity", move |_ctx, _args| {
17724 calls.fetch_add(1, Ordering::SeqCst);
17725 async move { Ok(Value::Null) }
17726 });
17727 let calls = Arc::clone(&handler_calls);
17728 worker.register_query("codec.workflow", "known", move |_ctx, _args| {
17729 calls.fetch_add(1, Ordering::SeqCst);
17730 async move { Ok(Value::Null) }
17731 });
17732
17733 let mut failures = Vec::new();
17734 for codec_case in [
17735 InvalidTaskPayloadCodec::Missing,
17736 InvalidTaskPayloadCodec::Null,
17737 InvalidTaskPayloadCodec::NonString,
17738 ] {
17739 let mut workflow = json!({
17740 "task_id": format!("workflow-{}", codec_case.label()),
17741 "workflow_type": "codec.workflow"
17742 });
17743 codec_case.apply(&mut workflow);
17744 match serde_json::from_value::<WorkflowTask>(workflow) {
17745 Ok(task) => match worker.execute_workflow_task(task) {
17746 Err(error) if error.to_string().contains("unsupported_payload_codec") => {}
17747 outcome => failures.push(format!(
17748 "workflow {} codec returned {outcome:?}",
17749 codec_case.label()
17750 )),
17751 },
17752 Err(error) => failures.push(format!(
17753 "workflow {} codec failed transport deserialization: {error}",
17754 codec_case.label()
17755 )),
17756 }
17757
17758 let mut activity = json!({
17759 "task_id": format!("activity-{}", codec_case.label()),
17760 "activity_attempt_id": format!("attempt-{}", codec_case.label()),
17761 "activity_type": "codec.activity",
17762 "attempt_number": 1
17763 });
17764 codec_case.apply(&mut activity);
17765 match serde_json::from_value::<ActivityTask>(activity) {
17766 Ok(task) => match worker.execute_activity_task(task).await {
17767 Err(error) if error.to_string().contains("unsupported_payload_codec") => {}
17768 outcome => failures.push(format!(
17769 "activity {} codec returned {outcome:?}",
17770 codec_case.label()
17771 )),
17772 },
17773 Err(error) => failures.push(format!(
17774 "activity {} codec failed transport deserialization: {error}",
17775 codec_case.label()
17776 )),
17777 }
17778
17779 let mut query = json!({
17780 "query_task_id": format!("query-{}", codec_case.label()),
17781 "workflow_type": "codec.workflow",
17782 "query_name": "known"
17783 });
17784 codec_case.apply(&mut query);
17785 match serde_json::from_value::<QueryTask>(query) {
17786 Ok(task) => match worker.execute_query_task(task).await {
17787 Err(failure) if failure.message.contains("unsupported_payload_codec") => {}
17788 outcome => failures.push(format!(
17789 "query {} codec returned {outcome:?}",
17790 codec_case.label()
17791 )),
17792 },
17793 Err(error) => failures.push(format!(
17794 "query {} codec failed transport deserialization: {error}",
17795 codec_case.label()
17796 )),
17797 }
17798 }
17799
17800 assert!(failures.is_empty(), "{}", failures.join("\n"));
17801 assert_eq!(
17802 handler_calls.load(Ordering::SeqCst),
17803 0,
17804 "invalid task codecs must not invoke a handler"
17805 );
17806 }
17807
17808 #[tokio::test]
17809 async fn polled_malformed_task_codecs_are_settled_without_handler_execution() {
17810 for codec_case in [
17811 InvalidTaskPayloadCodec::Missing,
17812 InvalidTaskPayloadCodec::Null,
17813 InvalidTaskPayloadCodec::NonString,
17814 ] {
17815 let server = MockWorkerServer::invalid_task_payload_codec(codec_case);
17816 let client = Client::builder(server.base_url())
17817 .timeout(Duration::from_secs(2))
17818 .build()
17819 .expect("client");
17820 let mut worker = Worker::new(client, "rust-workers")
17821 .worker_id("codec-worker")
17822 .poll_timeout(Duration::from_millis(10));
17823 let handler_calls = Arc::new(AtomicUsize::new(0));
17824
17825 let calls = Arc::clone(&handler_calls);
17826 worker.register_workflow("codec.workflow", move |_ctx, _args| {
17827 calls.fetch_add(1, Ordering::SeqCst);
17828 async move { Ok(Value::Null) }
17829 });
17830 let calls = Arc::clone(&handler_calls);
17831 worker.register_activity("codec.activity", move |_ctx, _args| {
17832 calls.fetch_add(1, Ordering::SeqCst);
17833 async move { Ok(Value::Null) }
17834 });
17835 let calls = Arc::clone(&handler_calls);
17836 worker.register_query("codec.workflow", "known", move |_ctx, _args| {
17837 calls.fetch_add(1, Ordering::SeqCst);
17838 async move { Ok(Value::Null) }
17839 });
17840
17841 assert_eq!(
17842 worker.run_once().await.expect("invalid tasks are settled"),
17843 3,
17844 "all {} codec tasks must be handled",
17845 codec_case.label()
17846 );
17847 assert_eq!(
17848 handler_calls.load(Ordering::SeqCst),
17849 0,
17850 "{} task codecs must fail before every handler",
17851 codec_case.label()
17852 );
17853
17854 for path in [
17855 "/api/worker/workflow-tasks/codec-workflow/fail",
17856 "/api/worker/activity-tasks/codec-activity/fail",
17857 "/api/worker/query-tasks/codec-query/fail",
17858 ] {
17859 let body = server.request_body(path);
17860 assert!(
17861 body["failure"]["message"]
17862 .as_str()
17863 .is_some_and(|message| message.contains("unsupported_payload_codec")),
17864 "{path} must receive the stable codec diagnostic for the {} case: {body}",
17865 codec_case.label()
17866 );
17867 }
17868 assert_eq!(
17869 server.request_body("/api/worker/query-tasks/codec-query/fail")["failure"]
17870 ["reason"],
17871 "query_payload_decode_failed"
17872 );
17873 for path in [
17874 "/api/worker/workflow-tasks/codec-workflow/complete",
17875 "/api/worker/activity-tasks/codec-activity/complete",
17876 "/api/worker/query-tasks/codec-query/complete",
17877 ] {
17878 assert_eq!(
17879 server.request_count(path),
17880 0,
17881 "invalid {} codec task reached {path}",
17882 codec_case.label()
17883 );
17884 }
17885 }
17886 }
17887
17888 #[tokio::test]
17889 async fn invalid_inbound_codecs_precede_handlers_and_unrelated_outcomes() {
17890 let client = Client::new("http://127.0.0.1:8080").expect("client");
17891 let mut worker = Worker::new(client, "rust-workers");
17892 let handler_calls = Arc::new(AtomicUsize::new(0));
17893
17894 let calls = Arc::clone(&handler_calls);
17895 worker.register_workflow("codec.workflow", move |_ctx, _args| {
17896 calls.fetch_add(1, Ordering::SeqCst);
17897 async move { Ok(Value::Null) }
17898 });
17899 let calls = Arc::clone(&handler_calls);
17900 worker.register_activity("codec.activity", move |_ctx, _args| {
17901 calls.fetch_add(1, Ordering::SeqCst);
17902 async move { Ok(Value::Null) }
17903 });
17904 let calls = Arc::clone(&handler_calls);
17905 worker.register_update("codec.workflow", "known", move |_ctx, _args| {
17906 calls.fetch_add(1, Ordering::SeqCst);
17907 async move { Ok(Value::Null) }
17908 });
17909 let calls = Arc::clone(&handler_calls);
17910 worker.register_query("codec.workflow", "known", move |_ctx, _args| {
17911 calls.fetch_add(1, Ordering::SeqCst);
17912 async move { Ok(Value::Null) }
17913 });
17914
17915 let mut workflow = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
17916 workflow.payload_codec = "json".to_string();
17917 workflow.arguments = None;
17918 let error = worker
17919 .execute_workflow_task(workflow)
17920 .expect_err("task codec must be checked before workflow invocation");
17921 assert!(error.to_string().contains("unsupported_payload_codec"));
17922
17923 let activity = ActivityTask {
17924 task_id: "activity-invalid-codec".to_string(),
17925 activity_attempt_id: None,
17926 attempt_id: None,
17927 activity_type: "codec.activity".to_string(),
17928 payload_codec: "unknown".to_string(),
17929 arguments: None,
17930 attempt_number: 1,
17931 lease_owner: None,
17932 };
17933 let error = worker
17934 .execute_activity_task(activity)
17935 .await
17936 .expect_err("task codec must be checked before activity invocation");
17937 assert!(error.to_string().contains("unsupported_payload_codec"));
17938
17939 let mut update = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
17940 update.workflow_update_id = Some("update-invalid-codec".to_string());
17941 update.update_name = Some("known".to_string());
17942 update.history_events.push(history_event(
17943 "UpdateAccepted",
17944 json!({
17945 "update_id": "update-invalid-codec",
17946 "update_name": "known",
17947 "arguments": {"codec": "json", "blob": null}
17948 }),
17949 ));
17950 let error = worker
17951 .execute_workflow_task(update)
17952 .expect_err("nested update codec must be checked before handler lookup");
17953 assert!(error.to_string().contains("unsupported_payload_codec"));
17954
17955 let query: QueryTask = serde_json::from_value(json!({
17956 "query_task_id": "query-invalid-codec",
17957 "workflow_type": "codec.workflow",
17958 "query_name": "known",
17959 "payload_codec": DEFAULT_CODEC,
17960 "workflow_arguments": null,
17961 "query_arguments": null,
17962 "history_export": {
17963 "payloads": {"codec": DEFAULT_CODEC},
17964 "signals": [{
17965 "name": "empty",
17966 "payload_codec": "json",
17967 "arguments": null
17968 }]
17969 }
17970 }))
17971 .expect("query task");
17972 let failure = worker
17973 .execute_query_task(query)
17974 .await
17975 .expect_err("exported signal codec must be checked before query invocation");
17976 assert_eq!(failure.reason, "query_payload_decode_failed");
17977 assert!(failure.message.contains("unsupported_payload_codec"));
17978
17979 let exported_history: QueryTask = serde_json::from_value(json!({
17980 "query_task_id": "query-invalid-history-codec",
17981 "workflow_type": "codec.workflow",
17982 "query_name": "known",
17983 "payload_codec": DEFAULT_CODEC,
17984 "history_export": {
17985 "payloads": {"codec": DEFAULT_CODEC},
17986 "history_events": [{
17987 "type": "ActivityCompleted",
17988 "payload": {"payload_codec": "unknown", "result": null}
17989 }]
17990 }
17991 }))
17992 .expect("query task");
17993 let failure = worker
17994 .execute_query_task(exported_history)
17995 .await
17996 .expect_err("exported history codec must be checked before query invocation");
17997 assert_eq!(failure.reason, "query_payload_decode_failed");
17998 assert!(failure.message.contains("unsupported_payload_codec"));
17999 assert_eq!(handler_calls.load(Ordering::SeqCst), 0);
18000
18001 let mut unknown_workflow = workflow_task("missing", Vec::new(), DEFAULT_CODEC);
18002 unknown_workflow.arguments = None;
18003 unknown_workflow.history_events.push(history_event(
18004 "SignalReceived",
18005 json!({
18006 "signal_name": "empty",
18007 "payload_codec": "json",
18008 "arguments": null
18009 }),
18010 ));
18011 let error = worker
18012 .execute_workflow_task(unknown_workflow)
18013 .expect_err("history codec must precede unknown workflow outcome");
18014 assert!(error.to_string().contains("unsupported_payload_codec"));
18015
18016 let unknown_activity = ActivityTask {
18017 task_id: "activity-unknown".to_string(),
18018 activity_attempt_id: None,
18019 attempt_id: None,
18020 activity_type: "missing".to_string(),
18021 payload_codec: "json".to_string(),
18022 arguments: None,
18023 attempt_number: 1,
18024 lease_owner: None,
18025 };
18026 let error = worker
18027 .execute_activity_task(unknown_activity)
18028 .await
18029 .expect_err("codec must precede unknown activity outcome");
18030 assert!(error.to_string().contains("unsupported_payload_codec"));
18031
18032 let mut unknown_update = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
18033 unknown_update.payload_codec = "json".to_string();
18034 unknown_update.arguments = None;
18035 unknown_update.workflow_update_id = Some("update-unknown".to_string());
18036 unknown_update.update_name = Some("missing".to_string());
18037 let error = worker
18038 .execute_workflow_task(unknown_update)
18039 .expect_err("codec must precede fail_update shortcut");
18040 assert!(error.to_string().contains("unsupported_payload_codec"));
18041
18042 let unknown_query: QueryTask = serde_json::from_value(json!({
18043 "query_task_id": "query-unknown",
18044 "workflow_type": "missing",
18045 "query_name": "missing",
18046 "payload_codec": "json",
18047 "workflow_arguments": null,
18048 "query_arguments": null
18049 }))
18050 .expect("query task");
18051 let failure = worker
18052 .execute_query_task(unknown_query)
18053 .await
18054 .expect_err("codec must precede unknown query outcome");
18055 assert_eq!(failure.reason, "query_payload_decode_failed");
18056 assert!(failure.message.contains("unsupported_payload_codec"));
18057 }
18058
18059 #[tokio::test]
18060 async fn invalid_signal_history_payload_aliases_precede_shortcuts() {
18061 let client = Client::new("http://127.0.0.1:8080").expect("client");
18062 let worker = Worker::new(client, "rust-workers");
18063
18064 for event_type in ["SignalReceived", "SignalApplied"] {
18065 for (payload_field, codec) in [
18066 ("value", "json"),
18067 ("input", "unknown"),
18068 ("arguments", "json"),
18069 ] {
18070 let payload = json!({
18071 "signal_name": "empty",
18072 payload_field: {"codec": codec, "blob": null}
18073 });
18074 let workflow = workflow_task(
18075 "missing",
18076 vec![history_event(event_type, payload.clone())],
18077 DEFAULT_CODEC,
18078 );
18079 let error = worker
18080 .execute_workflow_task(workflow)
18081 .expect_err("signal payload codec must precede unknown workflow outcome");
18082 assert!(
18083 error.to_string().contains("unsupported_payload_codec"),
18084 "{event_type}.{payload_field} returned an unrelated workflow error: {error}"
18085 );
18086
18087 let query: QueryTask = serde_json::from_value(json!({
18088 "query_task_id": format!("query-{event_type}-{payload_field}"),
18089 "workflow_type": "missing",
18090 "query_name": "missing",
18091 "payload_codec": DEFAULT_CODEC,
18092 "workflow_arguments": null,
18093 "query_arguments": null,
18094 "history_events": [{
18095 "event_type": event_type,
18096 "payload": payload
18097 }]
18098 }))
18099 .expect("query task");
18100 let failure = worker
18101 .execute_query_task(query)
18102 .await
18103 .expect_err("signal payload codec must precede unknown query outcome");
18104 assert_eq!(
18105 failure.reason, "query_payload_decode_failed",
18106 "{event_type}.{payload_field} returned an unrelated query outcome"
18107 );
18108 assert!(
18109 failure.message.contains("unsupported_payload_codec"),
18110 "{event_type}.{payload_field} returned an unrelated query error: {}",
18111 failure.message
18112 );
18113 }
18114 }
18115 }
18116
18117 #[test]
18118 fn workflow_context_schedules_activity_until_completion_is_in_history() {
18119 let ctx = WorkflowContext {
18120 state: Arc::new(Mutex::new(
18121 WorkflowState::new_with_identity(
18122 Vec::new(),
18123 Some("wf-parent".to_string()),
18124 Some("run-parent".to_string()),
18125 "rust-workers".to_string(),
18126 DEFAULT_CODEC.to_string(),
18127 None,
18128 )
18129 .expect("workflow state"),
18130 )),
18131 };
18132
18133 let mut call = Box::pin(ctx.activity("hello.activity", json!(["Ada"])));
18134 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18135 assert!(matches!(
18136 call.as_mut().poll(&mut task_context),
18137 Poll::Pending
18138 ));
18139
18140 let commands = ctx.take_commands().expect("commands");
18141 assert_eq!(commands[0]["type"], "schedule_activity");
18142 assert_eq!(commands[0]["activity_type"], "hello.activity");
18143 }
18144
18145 #[test]
18146 fn activity_options_encode_retry_policy_queue_and_every_timeout() {
18147 let ctx = workflow_context(Vec::new());
18148 let options = ActivityOptions::new()
18149 .task_queue("payments")
18150 .retry_policy(
18151 ActivityRetryPolicy::new(4)
18152 .exponential_backoff(Duration::from_secs(1), 3, Some(Duration::from_secs(10)))
18153 .non_retryable_error_type("ValidationError"),
18154 )
18155 .start_to_close_timeout(Duration::from_secs(120))
18156 .schedule_to_start_timeout(Duration::from_secs(10))
18157 .schedule_to_close_timeout(Duration::from_secs(300))
18158 .heartbeat_timeout(Duration::from_secs(15));
18159 let mut call = Box::pin(ctx.activity_with_options(
18160 "charge-card",
18161 options,
18162 json!([{"order_id": "o-1"}]),
18163 ));
18164 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18165
18166 assert!(matches!(
18167 call.as_mut().poll(&mut task_context),
18168 Poll::Pending
18169 ));
18170 assert!(matches!(
18171 call.as_mut().poll(&mut task_context),
18172 Poll::Pending
18173 ));
18174
18175 let commands = ctx.take_commands().expect("activity command");
18176 assert_eq!(commands.len(), 1, "one future emits one logical schedule");
18177 assert_eq!(commands[0]["queue"], "payments");
18178 assert_eq!(
18179 commands[0]["retry_policy"],
18180 json!({
18181 "max_attempts": 4,
18182 "backoff_seconds": [1, 3, 9],
18183 "non_retryable_error_types": ["ValidationError"],
18184 })
18185 );
18186 assert_eq!(commands[0]["start_to_close_timeout"], 120);
18187 assert_eq!(commands[0]["schedule_to_start_timeout"], 10);
18188 assert_eq!(commands[0]["schedule_to_close_timeout"], 300);
18189 assert_eq!(commands[0]["heartbeat_timeout"], 15);
18190 }
18191
18192 #[test]
18193 fn activity_options_encode_explicit_and_rounded_backoff_intervals() {
18194 let ctx = workflow_context(Vec::new());
18195 let options = ActivityOptions::new().retry_policy(
18196 ActivityRetryPolicy::new(3)
18197 .backoff_intervals([Duration::from_millis(1), Duration::from_millis(1_001)]),
18198 );
18199 let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
18200 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18201
18202 assert!(matches!(
18203 call.as_mut().poll(&mut task_context),
18204 Poll::Pending
18205 ));
18206 assert_eq!(
18207 ctx.take_commands().expect("command")[0]["retry_policy"]["backoff_seconds"],
18208 json!([1, 2])
18209 );
18210 }
18211
18212 #[test]
18213 fn invalid_activity_options_return_typed_errors_before_emitting_commands() {
18214 let cases = [
18215 (
18216 ActivityOptions::new().task_queue(" "),
18217 ActivityOptionsErrorKind::EmptyTaskQueue,
18218 ),
18219 (
18220 ActivityOptions::new().retry_policy(ActivityRetryPolicy::default()),
18221 ActivityOptionsErrorKind::EmptyRetryPolicy,
18222 ),
18223 (
18224 ActivityOptions::new().retry_policy(ActivityRetryPolicy::new(0)),
18225 ActivityOptionsErrorKind::InvalidMaxAttempts,
18226 ),
18227 (
18228 ActivityOptions::new().retry_policy(ActivityRetryPolicy {
18229 max_attempts: None,
18230 backoff: Some(ActivityBackoff::Explicit(vec![Duration::from_secs(1)])),
18231 non_retryable_error_types: Vec::new(),
18232 }),
18233 ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
18234 ),
18235 (
18236 ActivityOptions::new().retry_policy(
18237 ActivityRetryPolicy::new(2)
18238 .backoff_intervals([Duration::from_secs(1), Duration::from_secs(2)]),
18239 ),
18240 ActivityOptionsErrorKind::TooManyBackoffIntervals,
18241 ),
18242 (
18243 ActivityOptions::new().retry_policy(
18244 ActivityRetryPolicy::new(2).exponential_backoff(
18245 Duration::from_secs(1),
18246 0,
18247 None,
18248 ),
18249 ),
18250 ActivityOptionsErrorKind::InvalidBackoffCoefficient,
18251 ),
18252 (
18253 ActivityOptions::new()
18254 .retry_policy(ActivityRetryPolicy::new(2).non_retryable_error_type(" ")),
18255 ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
18256 ),
18257 (
18258 ActivityOptions::new().retry_policy(
18259 ActivityRetryPolicy::new(10_002).exponential_backoff(
18260 Duration::from_secs(1),
18261 1,
18262 None,
18263 ),
18264 ),
18265 ActivityOptionsErrorKind::BackoffGenerationTooLarge,
18266 ),
18267 (
18268 ActivityOptions::new().retry_policy(
18269 ActivityRetryPolicy::new(2)
18270 .backoff_intervals([Duration::from_secs(i64::MAX as u64 + 1)]),
18271 ),
18272 ActivityOptionsErrorKind::BackoffOverflow,
18273 ),
18274 ];
18275
18276 for (options, expected_kind) in cases {
18277 let ctx = workflow_context(Vec::new());
18278 let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
18279 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18280 let Poll::Ready(Err(Error::InvalidActivityOptions(error))) =
18281 call.as_mut().poll(&mut task_context)
18282 else {
18283 panic!("expected typed activity validation error");
18284 };
18285 assert_eq!(error.kind, expected_kind);
18286 assert!(ctx.take_commands().expect("commands").is_empty());
18287 }
18288 }
18289
18290 #[test]
18291 fn activity_options_validate_positive_and_ordered_timeouts() {
18292 let zero_timeout_cases = [
18293 ActivityOptions::new().start_to_close_timeout(Duration::ZERO),
18294 ActivityOptions::new().schedule_to_start_timeout(Duration::ZERO),
18295 ActivityOptions::new().schedule_to_close_timeout(Duration::ZERO),
18296 ActivityOptions::new().heartbeat_timeout(Duration::ZERO),
18297 ];
18298 for options in zero_timeout_cases {
18299 assert_eq!(
18300 options.validate().expect_err("zero timeout").kind,
18301 ActivityOptionsErrorKind::TimeoutNotPositive
18302 );
18303 }
18304
18305 let ordering_cases = [
18306 ActivityOptions::new()
18307 .heartbeat_timeout(Duration::from_secs(11))
18308 .start_to_close_timeout(Duration::from_secs(10)),
18309 ActivityOptions::new()
18310 .start_to_close_timeout(Duration::from_secs(31))
18311 .schedule_to_close_timeout(Duration::from_secs(30)),
18312 ActivityOptions::new()
18313 .schedule_to_start_timeout(Duration::from_secs(31))
18314 .schedule_to_close_timeout(Duration::from_secs(30)),
18315 ];
18316 for options in ordering_cases {
18317 assert_eq!(
18318 options.validate().expect_err("timeout order").kind,
18319 ActivityOptionsErrorKind::TimeoutOrder
18320 );
18321 }
18322
18323 assert_eq!(
18324 ActivityOptions::new()
18325 .start_to_close_timeout(Duration::from_secs(i64::MAX as u64 + 1))
18326 .validate()
18327 .expect_err("protocol integer overflow")
18328 .kind,
18329 ActivityOptionsErrorKind::TimeoutOverflow
18330 );
18331 }
18332
18333 #[test]
18334 fn replayed_activity_retry_history_completes_without_duplicate_schedule() {
18335 let ctx = workflow_context(completed_retry_activity_history());
18336 let mut call =
18337 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18338 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18339
18340 assert!(matches!(
18341 call.as_mut().poll(&mut task_context),
18342 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
18343 ));
18344 assert!(ctx.take_commands().expect("commands").is_empty());
18345 ctx.ensure_history_consumed().expect("history consumed");
18346 }
18347
18348 #[test]
18349 fn duplicate_non_retryable_types_use_one_command_and_replay_representation() {
18350 let mut options = retry_activity_options();
18351 options
18352 .retry_policy
18353 .as_mut()
18354 .expect("retry policy")
18355 .non_retryable_error_types
18356 .extend([" PermanentError ".to_string(), "PermanentError".to_string()]);
18357
18358 let new_ctx = workflow_context(Vec::new());
18359 let mut new_call =
18360 Box::pin(new_ctx.activity_with_options("flaky", options.clone(), json!([])));
18361 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18362 assert!(matches!(
18363 new_call.as_mut().poll(&mut task_context),
18364 Poll::Pending
18365 ));
18366 let commands = new_ctx.take_commands().expect("commands");
18367 assert_eq!(commands.len(), 1);
18368 assert_eq!(
18369 commands[0]["retry_policy"]["non_retryable_error_types"],
18370 json!(["PermanentError"])
18371 );
18372
18373 let replay_ctx = workflow_context(completed_retry_activity_history());
18374 let mut replay_call =
18375 Box::pin(replay_ctx.activity_with_options("flaky", options, json!([])));
18376 assert!(matches!(
18377 replay_call.as_mut().poll(&mut task_context),
18378 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
18379 ));
18380 assert!(replay_ctx.take_commands().expect("commands").is_empty());
18381 replay_ctx
18382 .ensure_history_consumed()
18383 .expect("history consumed");
18384 }
18385
18386 #[test]
18387 fn replayed_intermediate_retry_remains_pending_across_restarts() {
18388 let history = completed_retry_activity_history()
18389 .into_iter()
18390 .take(3)
18391 .collect::<Vec<_>>();
18392
18393 for _restart in 0..2 {
18394 let ctx = workflow_context(history.clone());
18395 let mut call =
18396 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18397 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18398 assert!(matches!(
18399 call.as_mut().poll(&mut task_context),
18400 Poll::Pending
18401 ));
18402 assert!(ctx.take_commands().expect("commands").is_empty());
18403 }
18404 }
18405
18406 #[test]
18407 fn replayed_activity_rejects_changed_queue_retry_and_every_timeout_field() {
18408 let mut changed_queue = retry_activity_options();
18409 changed_queue.task_queue = Some("different-queue".to_string());
18410
18411 let mut changed_max_attempts = retry_activity_options();
18412 let retry_policy = changed_max_attempts
18413 .retry_policy
18414 .as_mut()
18415 .expect("retry policy");
18416 retry_policy.max_attempts = Some(4);
18417
18418 let mut changed_backoff = retry_activity_options();
18419 let retry_policy = changed_backoff.retry_policy.as_mut().expect("retry policy");
18420 retry_policy.backoff = Some(ActivityBackoff::Explicit(vec![
18421 Duration::from_secs(3),
18422 Duration::from_secs(4),
18423 ]));
18424
18425 let mut changed_non_retryable_types = retry_activity_options();
18426 let retry_policy = changed_non_retryable_types
18427 .retry_policy
18428 .as_mut()
18429 .expect("retry policy");
18430 retry_policy.non_retryable_error_types = vec!["AnotherPermanentError".to_string()];
18431
18432 let mut changed_start_to_close = retry_activity_options();
18433 changed_start_to_close.start_to_close_timeout = Some(Duration::from_secs(31));
18434 let mut changed_schedule_to_start = retry_activity_options();
18435 changed_schedule_to_start.schedule_to_start_timeout = Some(Duration::from_secs(6));
18436 let mut changed_schedule_to_close = retry_activity_options();
18437 changed_schedule_to_close.schedule_to_close_timeout = Some(Duration::from_secs(91));
18438 let mut changed_heartbeat = retry_activity_options();
18439 changed_heartbeat.heartbeat_timeout = Some(Duration::from_secs(11));
18440
18441 let cases = [
18442 (changed_queue, "activity_task_queue_mismatch"),
18443 (changed_max_attempts, "activity_retry_policy_mismatch"),
18444 (changed_backoff, "activity_retry_policy_mismatch"),
18445 (
18446 changed_non_retryable_types,
18447 "activity_retry_policy_mismatch",
18448 ),
18449 (changed_start_to_close, "activity_retry_policy_mismatch"),
18450 (changed_schedule_to_start, "activity_retry_policy_mismatch"),
18451 (changed_schedule_to_close, "activity_retry_policy_mismatch"),
18452 (changed_heartbeat, "activity_retry_policy_mismatch"),
18453 ];
18454
18455 for (options, expected_reason) in cases {
18456 let ctx = workflow_context(completed_retry_activity_history());
18457 let mut call = Box::pin(ctx.activity_with_options("flaky", options, json!([])));
18458 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18459 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
18460 call.as_mut().poll(&mut task_context)
18461 else {
18462 panic!("changed activity options must fail replay");
18463 };
18464 assert_eq!(failure.reason, expected_reason);
18465 assert_eq!(failure.sequence, Some(1));
18466 assert!(ctx.take_commands().expect("commands").is_empty());
18467 }
18468 }
18469
18470 #[test]
18471 fn replayed_activity_rejects_changed_execution_mode_and_snapshot_version() {
18472 let cases = [
18473 (
18474 "execution_mode",
18475 json!("local"),
18476 "activity_execution_mode_mismatch",
18477 ),
18478 (
18479 "snapshot_version",
18480 json!(2),
18481 "activity_retry_policy_mismatch",
18482 ),
18483 ];
18484
18485 for (field, value, expected_reason) in cases {
18486 let mut history = completed_retry_activity_history();
18487 let activity = history[0].payload["activity"]
18488 .as_object_mut()
18489 .expect("activity snapshot");
18490 if field == "execution_mode" {
18491 activity.insert(field.to_string(), value);
18492 } else {
18493 activity["retry_policy"]
18494 .as_object_mut()
18495 .expect("retry snapshot")
18496 .insert(field.to_string(), value);
18497 }
18498
18499 let ctx = workflow_context(history);
18500 let mut call =
18501 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18502 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18503 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
18504 call.as_mut().poll(&mut task_context)
18505 else {
18506 panic!("changed {field} must fail replay");
18507 };
18508 assert_eq!(failure.reason, expected_reason);
18509 assert_eq!(failure.sequence, Some(1));
18510 assert!(ctx.take_commands().expect("commands").is_empty());
18511 }
18512 }
18513
18514 #[test]
18515 fn replayed_legacy_activity_treats_missing_option_snapshot_as_unknown() {
18516 let mut history = completed_retry_activity_history();
18517 let activity = history[0].payload["activity"]
18518 .as_object_mut()
18519 .expect("activity snapshot");
18520 activity.remove("execution_mode");
18521 activity.remove("retry_policy");
18522
18523 let mut current = retry_activity_options();
18524 current.start_to_close_timeout = Some(Duration::from_secs(45));
18525 current.schedule_to_start_timeout = Some(Duration::from_secs(8));
18526 current.schedule_to_close_timeout = Some(Duration::from_secs(120));
18527 current.heartbeat_timeout = Some(Duration::from_secs(12));
18528
18529 let ctx = workflow_context(history);
18530 let mut call = Box::pin(ctx.activity_with_options("flaky", current, json!([])));
18531 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18532 assert!(matches!(
18533 call.as_mut().poll(&mut task_context),
18534 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
18535 ));
18536 assert!(ctx.take_commands().expect("commands").is_empty());
18537 ctx.ensure_history_consumed().expect("history consumed");
18538 }
18539
18540 #[test]
18541 fn terminal_activity_failed_after_start_returns_typed_failure() {
18542 let history = vec![
18543 history_event(
18544 "ActivityScheduled",
18545 json!({
18546 "sequence": 1,
18547 "activity_type": "flaky",
18548 "activity_execution_id": "act-terminal",
18549 "activity": {
18550 "id": "act-terminal",
18551 "sequence": 1,
18552 "type": "flaky",
18553 "queue": "critical-activities",
18554 "retry_policy": {
18555 "snapshot_version": 1,
18556 "max_attempts": 3,
18557 "backoff_seconds": [2, 4],
18558 "non_retryable_error_types": ["PermanentError"]
18559 }
18560 }
18561 }),
18562 ),
18563 history_event(
18564 "ActivityStarted",
18565 json!({
18566 "sequence": 1,
18567 "activity_type": "flaky",
18568 "activity_execution_id": "act-terminal",
18569 "activity_attempt_id": "attempt-1",
18570 "attempt_number": 1
18571 }),
18572 ),
18573 history_event(
18574 "ActivityFailed",
18575 json!({
18576 "sequence": 1,
18577 "activity_type": "flaky",
18578 "activity_execution_id": "act-terminal",
18579 "activity_attempt_id": "attempt-1",
18580 "attempt_number": 1,
18581 "failure_id": "failure-terminal",
18582 "failure_category": "activity",
18583 "exception_type": "PermanentError",
18584 "message": "cannot retry",
18585 "non_retryable": true
18586 }),
18587 ),
18588 ];
18589 let ctx = workflow_context(history);
18590 let mut call =
18591 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18592 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18593
18594 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
18595 call.as_mut().poll(&mut task_context)
18596 else {
18597 panic!("terminal ActivityFailed must settle the activity future");
18598 };
18599 assert_eq!(failure.kind, ActivityFailureKind::Failed);
18600 assert_eq!(
18601 failure.activity_execution_id.as_deref(),
18602 Some("act-terminal")
18603 );
18604 assert_eq!(failure.exception_type.as_deref(), Some("PermanentError"));
18605 assert!(failure.non_retryable);
18606 assert!(ctx.take_commands().expect("commands").is_empty());
18607 ctx.ensure_history_consumed().expect("history consumed");
18608 }
18609
18610 #[test]
18611 fn activity_terminal_events_return_machine_readable_failures() {
18612 let cases = [
18613 (
18614 "ActivityFailed",
18615 json!({
18616 "sequence": 1,
18617 "activity_type": "charge-card",
18618 "activity_execution_id": "act-1",
18619 "activity_attempt_id": "attempt-2",
18620 "attempt_number": 2,
18621 "failure_id": "failure-1",
18622 "failure_category": "activity",
18623 "exception_type": "PaymentDeclined",
18624 "exception_class": "payments.PaymentDeclined",
18625 "message": "card declined",
18626 "non_retryable": true
18627 }),
18628 ActivityFailureKind::Failed,
18629 "activity",
18630 ),
18631 (
18632 "ActivityCancelled",
18633 json!({
18634 "sequence": 1,
18635 "activity_type": "charge-card",
18636 "activity_execution_id": "act-1",
18637 "activity_attempt_id": "attempt-1"
18638 }),
18639 ActivityFailureKind::Cancelled,
18640 "cancelled",
18641 ),
18642 ];
18643
18644 for (event_type, payload, expected_kind, expected_reason) in cases {
18645 let ctx = workflow_context(vec![history_event(event_type, payload)]);
18646 let mut call = Box::pin(ctx.activity("charge-card", json!([])));
18647 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18648 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
18649 call.as_mut().poll(&mut task_context)
18650 else {
18651 panic!("expected terminal activity failure");
18652 };
18653 assert_eq!(failure.kind, expected_kind);
18654 assert_eq!(failure.reason, expected_reason);
18655 assert_eq!(failure.activity_execution_id.as_deref(), Some("act-1"));
18656 assert_eq!(failure.activity_type.as_deref(), Some("charge-card"));
18657 }
18658 }
18659
18660 #[test]
18661 fn every_activity_timeout_class_is_typed() {
18662 for timeout_kind in [
18663 "start_to_close",
18664 "schedule_to_start",
18665 "schedule_to_close",
18666 "heartbeat",
18667 ] {
18668 let ctx = workflow_context(vec![history_event(
18669 "ActivityTimedOut",
18670 json!({
18671 "sequence": 1,
18672 "activity_type": "slow",
18673 "activity_execution_id": "act-timeout",
18674 "activity_attempt_id": "attempt-timeout",
18675 "failure_category": "timeout",
18676 "timeout_kind": timeout_kind,
18677 "message": "deadline expired"
18678 }),
18679 )]);
18680 let mut call = Box::pin(ctx.activity("slow", json!([])));
18681 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18682 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
18683 call.as_mut().poll(&mut task_context)
18684 else {
18685 panic!("expected timeout failure");
18686 };
18687 assert_eq!(failure.kind, ActivityFailureKind::TimedOut);
18688 assert_eq!(failure.reason, timeout_kind);
18689 assert_eq!(failure.timeout_kind.as_deref(), Some(timeout_kind));
18690 assert_eq!(failure.failure_category.as_deref(), Some("timeout"));
18691 }
18692 }
18693
18694 #[test]
18695 fn workflow_sleep_emits_one_durable_timer_and_rounds_up() {
18696 let ctx = workflow_context(Vec::new());
18697 let mut sleep = Box::pin(ctx.sleep(Duration::from_millis(1_001)));
18698 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18699
18700 assert!(matches!(
18701 sleep.as_mut().poll(&mut task_context),
18702 Poll::Pending
18703 ));
18704 assert!(matches!(
18705 sleep.as_mut().poll(&mut task_context),
18706 Poll::Pending
18707 ));
18708
18709 let commands = ctx.take_commands().expect("timer command");
18710 assert_eq!(
18711 commands,
18712 vec![json!({
18713 "type": "start_timer",
18714 "delay_seconds": 2,
18715 })]
18716 );
18717 }
18718
18719 #[test]
18720 fn workflow_sleep_replays_matching_schedule_and_fire_without_a_command() {
18721 let history = vec![
18722 history_event(
18723 "TimerScheduled",
18724 json!({
18725 "sequence": 1,
18726 "timer_id": "timer-1",
18727 "delay_seconds": 5,
18728 "fire_at": "2026-07-11T12:00:05Z",
18729 }),
18730 ),
18731 history_event(
18732 "TimerFired",
18733 json!({
18734 "sequence": 1,
18735 "timer_id": "timer-1",
18736 "delay_seconds": 5,
18737 "fire_at": "2026-07-11T12:00:05Z",
18738 "fired_at": "2026-07-11T12:00:05Z",
18739 }),
18740 ),
18741 ];
18742
18743 for _restart in 0..2 {
18744 let ctx = workflow_context(history.clone());
18745 let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(5)));
18746 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18747 assert!(matches!(
18748 sleep.as_mut().poll(&mut task_context),
18749 Poll::Ready(Ok(()))
18750 ));
18751 assert!(ctx.take_commands().expect("commands").is_empty());
18752 ctx.ensure_history_consumed().expect("history consumed");
18753 }
18754 }
18755
18756 #[test]
18757 fn workflow_sleep_rejects_changed_delay_during_replay() {
18758 let ctx = workflow_context(vec![
18759 history_event(
18760 "TimerScheduled",
18761 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
18762 ),
18763 history_event(
18764 "TimerFired",
18765 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
18766 ),
18767 ]);
18768 let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(500)));
18769 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18770
18771 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
18772 sleep.as_mut().poll(&mut task_context)
18773 else {
18774 panic!("changed timer delay must be rejected");
18775 };
18776 assert_eq!(failure.reason, "timer_delay_mismatch");
18777 assert_eq!(failure.sequence, Some(1));
18778 }
18779
18780 #[test]
18781 fn workflow_condition_wait_emits_published_identity_and_timeout_contract() {
18782 let ctx = workflow_context(Vec::new());
18783 let mut wait = Box::pin(
18784 ctx.wait_condition(
18785 ConditionWaitOptions::new("approval.ready", "sha256:approval-v1")
18786 .timeout(Duration::from_millis(60_001)),
18787 || Ok(false),
18788 ),
18789 );
18790 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18791
18792 assert!(matches!(
18793 wait.as_mut().poll(&mut task_context),
18794 Poll::Pending
18795 ));
18796 assert!(matches!(
18797 wait.as_mut().poll(&mut task_context),
18798 Poll::Pending
18799 ));
18800 assert_eq!(
18801 ctx.take_commands().expect("condition command"),
18802 vec![json!({
18803 "type": "open_condition_wait",
18804 "condition_wait_occurrence_id": "rust:condition-wait:0",
18805 "condition_key": "approval.ready",
18806 "condition_definition_fingerprint": "sha256:approval-v1",
18807 "timeout_seconds": 61,
18808 })]
18809 );
18810 }
18811
18812 #[test]
18813 fn workflow_condition_wait_returns_explicit_immediate_results_without_commands() {
18814 let ctx = workflow_context(Vec::new());
18815 let mut satisfied = Box::pin(wait_condition!(ctx, "already-ready", || Ok(true)));
18816 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18817 assert!(matches!(
18818 satisfied.as_mut().poll(&mut task_context),
18819 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18820 ));
18821
18822 let mut timed_out = Box::pin(wait_condition!(
18823 ctx,
18824 "no-wait",
18825 timeout: Duration::ZERO,
18826 || Ok(false),
18827 ));
18828 assert!(matches!(
18829 timed_out.as_mut().poll(&mut task_context),
18830 Poll::Ready(Ok(ConditionWaitResult::TimedOut))
18831 ));
18832 assert!(ctx.take_commands().expect("commands").is_empty());
18833 }
18834
18835 #[test]
18836 fn signal_and_update_history_reevaluate_open_conditions_after_restart() {
18837 let signal_history = vec![
18838 history_event(
18839 "ConditionWaitOpened",
18840 json!({
18841 "sequence": 4,
18842 "condition_wait_id": "condition:4",
18843 "condition_wait_occurrence_id": "rust:condition-wait:0",
18844 "condition_key": "approval",
18845 "condition_definition_fingerprint": "sha256:approval-v1",
18846 "timeout_seconds": 30,
18847 }),
18848 ),
18849 history_event(
18850 "SignalReceived",
18851 json!({
18852 "workflow_sequence": 4,
18853 "signal_name": "approve",
18854 "arguments": fixture_envelope(json!(["Ada"])),
18855 }),
18856 ),
18857 ];
18858 for _worker_before_or_after_restart in 0..2 {
18859 let ctx = workflow_context(signal_history.clone());
18860 let predicate_ctx = ctx.clone();
18861 let mut wait = Box::pin(
18862 ctx.wait_condition(
18863 ConditionWaitOptions::new("approval", "sha256:approval-v1")
18864 .timeout(Duration::from_secs(30)),
18865 move || Ok(!predicate_ctx.signals("approve")?.is_empty()),
18866 ),
18867 );
18868 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18869 assert!(matches!(
18870 wait.as_mut().poll(&mut task_context),
18871 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18872 ));
18873 assert!(ctx.take_commands().expect("commands").is_empty());
18874 ctx.ensure_history_consumed().expect("condition consumed");
18875 }
18876
18877 let update_history = vec![
18878 history_event(
18879 "ConditionWaitOpened",
18880 json!({
18881 "sequence": 7,
18882 "condition_wait_id": "condition:7",
18883 "condition_wait_occurrence_id": "rust:condition-wait:0",
18884 "condition_key": "update-approval",
18885 "condition_definition_fingerprint": "sha256:update-approval-v1",
18886 }),
18887 ),
18888 history_event(
18889 "UpdateApplied",
18890 json!({
18891 "sequence": 7,
18892 "update_id": "update-1",
18893 "update_name": "approve",
18894 "arguments": fixture_envelope(json!([true])),
18895 }),
18896 ),
18897 ];
18898 let ctx = workflow_context(update_history);
18899 let predicate_ctx = ctx.clone();
18900 let mut wait = Box::pin(ctx.wait_condition(
18901 ConditionWaitOptions::new("update-approval", "sha256:update-approval-v1"),
18902 move || {
18903 Ok(predicate_ctx
18904 .updates("approve")?
18905 .first()
18906 .and_then(|arguments| arguments.first())
18907 .and_then(Value::as_bool)
18908 == Some(true))
18909 },
18910 ));
18911 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18912 assert!(matches!(
18913 wait.as_mut().poll(&mut task_context),
18914 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18915 ));
18916 assert!(ctx.take_commands().expect("commands").is_empty());
18917 ctx.ensure_history_consumed().expect("condition consumed");
18918 }
18919
18920 #[test]
18921 fn condition_wait_preserves_open_satisfied_and_timed_out_replay_states() {
18922 let open_history = vec![
18923 history_event(
18924 "ConditionWaitOpened",
18925 json!({
18926 "sequence": 3,
18927 "condition_wait_id": "condition:3",
18928 "condition_wait_occurrence_id": "rust:condition-wait:0",
18929 "condition_key": "two-votes",
18930 "condition_definition_fingerprint": "sha256:two-votes-v1",
18931 "timeout_seconds": 120,
18932 }),
18933 ),
18934 history_event(
18935 "SignalReceived",
18936 json!({
18937 "workflow_sequence": 3,
18938 "signal_name": "vote",
18939 "arguments": fixture_envelope(json!(["first"])),
18940 }),
18941 ),
18942 ];
18943 for _worker_before_or_after_restart in 0..2 {
18944 let ctx = workflow_context(open_history.clone());
18945 let predicate_ctx = ctx.clone();
18946 let mut wait = Box::pin(
18947 ctx.wait_condition(
18948 ConditionWaitOptions::new("two-votes", "sha256:two-votes-v1")
18949 .timeout(Duration::from_secs(120)),
18950 move || Ok(predicate_ctx.signals("vote")?.len() >= 2),
18951 ),
18952 );
18953 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18954 assert!(matches!(
18955 wait.as_mut().poll(&mut task_context),
18956 Poll::Pending
18957 ));
18958 assert_eq!(
18959 ctx.take_commands().expect("reopened condition"),
18960 vec![json!({
18961 "type": "open_condition_wait",
18962 "condition_wait_occurrence_id": "rust:condition-wait:0",
18963 "condition_key": "two-votes",
18964 "condition_definition_fingerprint": "sha256:two-votes-v1",
18965 "timeout_seconds": 120,
18966 })]
18967 );
18968 }
18969
18970 let satisfied_ctx = workflow_context(vec![
18971 history_event(
18972 "ConditionWaitOpened",
18973 json!({
18974 "sequence": 5,
18975 "condition_wait_id": "condition:5",
18976 "condition_wait_occurrence_id": "rust:condition-wait:0",
18977 "condition_key": "approval",
18978 "condition_definition_fingerprint": "sha256:approval-v1",
18979 }),
18980 ),
18981 history_event(
18982 "ConditionWaitSatisfied",
18983 json!({
18984 "sequence": 5,
18985 "condition_wait_id": "condition:5",
18986 "condition_wait_occurrence_id": "rust:condition-wait:0",
18987 "condition_key": "approval",
18988 "condition_definition_fingerprint": "sha256:approval-v1",
18989 }),
18990 ),
18991 ]);
18992 let mut satisfied = Box::pin(satisfied_ctx.wait_condition(
18993 ConditionWaitOptions::new("approval", "sha256:approval-v1"),
18994 || Ok(false),
18995 ));
18996 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18997 assert!(matches!(
18998 satisfied.as_mut().poll(&mut task_context),
18999 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19000 ));
19001
19002 let timed_out_ctx = workflow_context(vec![
19003 history_event(
19004 "ConditionWaitOpened",
19005 json!({
19006 "sequence": 8,
19007 "condition_wait_id": "condition:8",
19008 "condition_wait_occurrence_id": "rust:condition-wait:0",
19009 "condition_key": "approval-timeout",
19010 "condition_definition_fingerprint": "sha256:approval-timeout-v1",
19011 "timeout_seconds": 5,
19012 }),
19013 ),
19014 history_event(
19015 "TimerScheduled",
19016 json!({
19017 "sequence": 9,
19018 "timer_id": "condition-timer:9",
19019 "timer_kind": "condition_timeout",
19020 "condition_wait_id": "condition:8",
19021 "delay_seconds": 5,
19022 }),
19023 ),
19024 history_event(
19025 "TimerFired",
19026 json!({
19027 "sequence": 9,
19028 "timer_id": "condition-timer:9",
19029 "timer_kind": "condition_timeout",
19030 "condition_wait_id": "condition:8",
19031 "delay_seconds": 5,
19032 }),
19033 ),
19034 ]);
19035 let mut timed_out = Box::pin(
19036 timed_out_ctx.wait_condition(
19037 ConditionWaitOptions::new("approval-timeout", "sha256:approval-timeout-v1")
19038 .timeout(Duration::from_secs(5)),
19039 || Ok(true),
19040 ),
19041 );
19042 assert!(matches!(
19043 timed_out.as_mut().poll(&mut task_context),
19044 Poll::Ready(Ok(ConditionWaitResult::TimedOut))
19045 ));
19046 }
19047
19048 #[test]
19049 fn condition_wait_replays_repeated_physical_opens_as_one_logical_wait() {
19050 let history = vec![
19051 history_event(
19052 "ConditionWaitOpened",
19053 json!({
19054 "sequence": 3,
19055 "condition_wait_id": "condition:3",
19056 "condition_wait_occurrence_id": "rust:condition-wait:0",
19057 "condition_key": "two-votes",
19058 "condition_definition_fingerprint": "sha256:two-votes-v1",
19059 }),
19060 ),
19061 history_event(
19062 "SignalReceived",
19063 json!({
19064 "workflow_sequence": 3,
19065 "signal_name": "vote",
19066 "arguments": fixture_envelope(json!(["first"])),
19067 }),
19068 ),
19069 history_event(
19070 "ConditionWaitSatisfied",
19071 json!({
19072 "sequence": 3,
19073 "condition_wait_id": "condition:3",
19074 "condition_wait_occurrence_id": "rust:condition-wait:0",
19075 "condition_key": "two-votes",
19076 "condition_definition_fingerprint": "sha256:two-votes-v1",
19077 }),
19078 ),
19079 history_event(
19080 "ConditionWaitOpened",
19081 json!({
19082 "sequence": 5,
19083 "condition_wait_id": "condition:5",
19084 "condition_wait_occurrence_id": "rust:condition-wait:0",
19085 "condition_key": "two-votes",
19086 "condition_definition_fingerprint": "sha256:two-votes-v1",
19087 }),
19088 ),
19089 history_event(
19090 "SignalReceived",
19091 json!({
19092 "workflow_sequence": 5,
19093 "signal_name": "vote",
19094 "arguments": fixture_envelope(json!(["second"])),
19095 }),
19096 ),
19097 history_event(
19098 "ConditionWaitSatisfied",
19099 json!({
19100 "sequence": 5,
19101 "condition_wait_id": "condition:5",
19102 "condition_wait_occurrence_id": "rust:condition-wait:0",
19103 "condition_key": "two-votes",
19104 "condition_definition_fingerprint": "sha256:two-votes-v1",
19105 }),
19106 ),
19107 ];
19108 for _cold_worker_or_restart in 0..2 {
19109 let ctx = workflow_context(history.clone());
19110 let predicate_ctx = ctx.clone();
19111 let mut wait = Box::pin(ctx.wait_condition(
19112 ConditionWaitOptions::new("two-votes", "sha256:two-votes-v1"),
19113 move || Ok(predicate_ctx.signals("vote")?.len() >= 2),
19114 ));
19115 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19116
19117 assert!(matches!(
19118 wait.as_mut().poll(&mut task_context),
19119 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19120 ));
19121 assert!(ctx.take_commands().expect("commands").is_empty());
19122 ctx.ensure_history_consumed()
19123 .expect("every physical wait-open is consumed");
19124 }
19125 }
19126
19127 #[test]
19128 fn condition_wait_replays_update_driven_physical_opens_as_one_occurrence() {
19129 let history = vec![
19130 history_event(
19131 "ConditionWaitOpened",
19132 json!({
19133 "sequence": 3,
19134 "condition_wait_id": "condition:3",
19135 "condition_wait_occurrence_id": "rust:condition-wait:0",
19136 "condition_key": "approved",
19137 "condition_definition_fingerprint": "sha256:approved-v1",
19138 }),
19139 ),
19140 history_event(
19141 "UpdateApplied",
19142 json!({
19143 "sequence": 3,
19144 "update_id": "update-1",
19145 "update_name": "approve",
19146 "arguments": fixture_envelope(json!([false])),
19147 }),
19148 ),
19149 history_event(
19150 "ConditionWaitOpened",
19151 json!({
19152 "sequence": 5,
19153 "condition_wait_id": "condition:5",
19154 "condition_wait_occurrence_id": "rust:condition-wait:0",
19155 "condition_key": "approved",
19156 "condition_definition_fingerprint": "sha256:approved-v1",
19157 }),
19158 ),
19159 history_event(
19160 "UpdateApplied",
19161 json!({
19162 "sequence": 5,
19163 "update_id": "update-2",
19164 "update_name": "approve",
19165 "arguments": fixture_envelope(json!([true])),
19166 }),
19167 ),
19168 ];
19169
19170 for _cold_worker_or_restart in 0..2 {
19171 let ctx = workflow_context(history.clone());
19172 let predicate_ctx = ctx.clone();
19173 let mut wait = Box::pin(ctx.wait_condition(
19174 ConditionWaitOptions::new("approved", "sha256:approved-v1"),
19175 move || {
19176 Ok(predicate_ctx
19177 .updates("approve")?
19178 .last()
19179 .and_then(|arguments| arguments.first())
19180 .and_then(Value::as_bool)
19181 == Some(true))
19182 },
19183 ));
19184 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19185
19186 assert!(matches!(
19187 wait.as_mut().poll(&mut task_context),
19188 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19189 ));
19190 assert!(ctx.take_commands().expect("commands").is_empty());
19191 ctx.ensure_history_consumed()
19192 .expect("every update-driven reopen is consumed");
19193 }
19194 }
19195
19196 #[test]
19197 fn condition_wait_replay_keeps_every_adjacent_authored_occurrence_distinct() {
19198 for (first_key, first_fingerprint, second_key, second_fingerprint) in [
19199 ("shared", "sha256:first", "shared", "sha256:second"),
19200 ("first", "sha256:shared", "second", "sha256:shared"),
19201 ("shared", "sha256:shared", "shared", "sha256:shared"),
19202 ("first", "sha256:first", "second", "sha256:second"),
19203 ] {
19204 let history = vec![
19205 history_event(
19206 "ConditionWaitOpened",
19207 json!({
19208 "sequence": 3,
19209 "condition_wait_id": "condition:3",
19210 "condition_wait_occurrence_id": "rust:condition-wait:0",
19211 "condition_key": first_key,
19212 "condition_definition_fingerprint": first_fingerprint,
19213 }),
19214 ),
19215 history_event(
19216 "ConditionWaitSatisfied",
19217 json!({
19218 "sequence": 3,
19219 "condition_wait_id": "condition:3",
19220 "condition_wait_occurrence_id": "rust:condition-wait:0",
19221 "condition_key": first_key,
19222 "condition_definition_fingerprint": first_fingerprint,
19223 }),
19224 ),
19225 history_event(
19226 "ConditionWaitOpened",
19227 json!({
19228 "sequence": 4,
19229 "condition_wait_id": "condition:4",
19230 "condition_wait_occurrence_id": "rust:condition-wait:1",
19231 "condition_key": second_key,
19232 "condition_definition_fingerprint": second_fingerprint,
19233 }),
19234 ),
19235 history_event(
19236 "ConditionWaitSatisfied",
19237 json!({
19238 "sequence": 4,
19239 "condition_wait_id": "condition:4",
19240 "condition_wait_occurrence_id": "rust:condition-wait:1",
19241 "condition_key": second_key,
19242 "condition_definition_fingerprint": second_fingerprint,
19243 }),
19244 ),
19245 ];
19246 for _cold_worker_or_restart in 0..2 {
19247 let ctx = workflow_context(history.clone());
19248 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19249 let mut first = Box::pin(ctx.wait_condition(
19250 ConditionWaitOptions::new(first_key, first_fingerprint),
19251 || Ok(false),
19252 ));
19253 assert!(matches!(
19254 first.as_mut().poll(&mut task_context),
19255 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19256 ));
19257
19258 let mut second = Box::pin(ctx.wait_condition(
19259 ConditionWaitOptions::new(second_key, second_fingerprint),
19260 || Ok(false),
19261 ));
19262 assert!(matches!(
19263 second.as_mut().poll(&mut task_context),
19264 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19265 ));
19266 assert!(ctx.take_commands().expect("commands").is_empty());
19267 ctx.ensure_history_consumed()
19268 .expect("each authored wait consumes one occurrence");
19269 }
19270 }
19271 }
19272
19273 #[test]
19274 fn cold_workers_replay_adjacent_condition_waits_from_one_loop_call_site() {
19275 fn worker() -> Worker {
19276 let client = Client::new("http://127.0.0.1:8080").expect("client");
19277 let mut worker = Worker::new(client, "rust-workers");
19278 worker.register_workflow("rust.condition-loop", |ctx, _input| async move {
19279 let mut outcomes = Vec::new();
19280 for _ in 0..2 {
19281 outcomes.push(
19282 ctx.wait_condition(
19283 ConditionWaitOptions::new("shared", "sha256:shared"),
19284 || Ok(false),
19285 )
19286 .await?,
19287 );
19288 }
19289 Ok(json!(outcomes))
19290 });
19291 worker
19292 }
19293
19294 let task = workflow_task(
19295 "rust.condition-loop",
19296 vec![
19297 history_event(
19298 "ConditionWaitOpened",
19299 json!({
19300 "sequence": 1,
19301 "condition_wait_id": "condition:1",
19302 "condition_wait_occurrence_id": "rust:condition-wait:0",
19303 "condition_key": "shared",
19304 "condition_definition_fingerprint": "sha256:shared",
19305 }),
19306 ),
19307 history_event(
19308 "ConditionWaitSatisfied",
19309 json!({
19310 "sequence": 1,
19311 "condition_wait_id": "condition:1",
19312 "condition_wait_occurrence_id": "rust:condition-wait:0",
19313 "condition_key": "shared",
19314 "condition_definition_fingerprint": "sha256:shared",
19315 }),
19316 ),
19317 history_event(
19318 "ConditionWaitOpened",
19319 json!({
19320 "sequence": 2,
19321 "condition_wait_id": "condition:2",
19322 "condition_wait_occurrence_id": "rust:condition-wait:1",
19323 "condition_key": "shared",
19324 "condition_definition_fingerprint": "sha256:shared",
19325 }),
19326 ),
19327 history_event(
19328 "ConditionWaitSatisfied",
19329 json!({
19330 "sequence": 2,
19331 "condition_wait_id": "condition:2",
19332 "condition_wait_occurrence_id": "rust:condition-wait:1",
19333 "condition_key": "shared",
19334 "condition_definition_fingerprint": "sha256:shared",
19335 }),
19336 ),
19337 ],
19338 DEFAULT_CODEC,
19339 );
19340
19341 for _cold_worker_or_restart in 0..2 {
19342 let commands = worker()
19343 .execute_workflow_task(task.clone())
19344 .expect("adjacent loop waits replay deterministically");
19345 assert_eq!(commands.len(), 1);
19346 assert_eq!(commands[0]["type"], "complete_workflow");
19347 assert_eq!(
19348 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("workflow output"),
19349 json!(["satisfied", "satisfied"])
19350 );
19351 }
19352 }
19353
19354 #[test]
19355 fn condition_wait_replay_rejects_identity_predicate_and_timeout_changes() {
19356 let history = vec![history_event(
19357 "ConditionWaitOpened",
19358 json!({
19359 "sequence": 12,
19360 "condition_wait_id": "condition:12",
19361 "condition_wait_occurrence_id": "rust:condition-wait:0",
19362 "condition_key": "approval",
19363 "condition_definition_fingerprint": "sha256:approval-v1",
19364 "timeout_seconds": 30,
19365 }),
19366 )];
19367 for (options, expected_reason) in [
19368 (
19369 ConditionWaitOptions::new("changed", "sha256:approval-v1")
19370 .timeout(Duration::from_secs(30)),
19371 "condition_wait_key_mismatch",
19372 ),
19373 (
19374 ConditionWaitOptions::new("approval", "sha256:approval-v2")
19375 .timeout(Duration::from_secs(30)),
19376 "condition_wait_predicate_mismatch",
19377 ),
19378 (
19379 ConditionWaitOptions::new("approval", "sha256:approval-v1")
19380 .timeout(Duration::from_secs(29)),
19381 "condition_wait_timeout_mismatch",
19382 ),
19383 ] {
19384 let ctx = workflow_context(history.clone());
19385 let mut wait = Box::pin(ctx.wait_condition(options, || Ok(false)));
19386 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19387 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
19388 wait.as_mut().poll(&mut task_context)
19389 else {
19390 panic!("changed condition definition must fail replay");
19391 };
19392 assert_eq!(failure.reason, expected_reason);
19393 assert_eq!(failure.sequence, Some(12));
19394 }
19395 }
19396
19397 #[test]
19398 fn condition_wait_history_requires_the_canonical_predicate_fingerprint() {
19399 let error = WorkflowState::new(
19400 vec![history_event(
19401 "ConditionWaitOpened",
19402 json!({
19403 "sequence": 12,
19404 "condition_wait_id": "condition:12",
19405 "condition_wait_occurrence_id": "rust:condition-wait:0",
19406 "condition_key": "approval",
19407 }),
19408 )],
19409 "rust-workers".to_string(),
19410 DEFAULT_CODEC.to_string(),
19411 None,
19412 )
19413 .expect_err("condition history without a predicate fingerprint must fail");
19414
19415 assert!(matches!(
19416 error,
19417 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19418 if reason == "condition_wait_predicate_fingerprint_missing"
19419 ));
19420 }
19421
19422 #[test]
19423 fn condition_wait_history_requires_authored_occurrence_identity() {
19424 let error = WorkflowState::new(
19425 vec![history_event(
19426 "ConditionWaitOpened",
19427 json!({
19428 "sequence": 12,
19429 "condition_wait_id": "condition:12",
19430 "condition_key": "approval",
19431 "condition_definition_fingerprint": "sha256:approval-v1",
19432 }),
19433 )],
19434 "rust-workers".to_string(),
19435 DEFAULT_CODEC.to_string(),
19436 None,
19437 )
19438 .expect_err("condition history without occurrence identity must fail");
19439
19440 assert!(matches!(
19441 error,
19442 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19443 if reason == "condition_wait_occurrence_id_missing"
19444 ));
19445 }
19446
19447 #[test]
19448 fn typed_search_attribute_updates_validate_emit_and_replay() {
19449 let update = SearchAttributeUpdate::new()
19450 .keyword("OrderStatus", " waiting ")
19451 .expect("keyword")
19452 .int("Attempt", 3)
19453 .expect("int")
19454 .bool("Escalated", false)
19455 .expect("bool")
19456 .keyword_list("Regions", ["us-east", "eu-west"])
19457 .expect("list")
19458 .datetime("UpdatedAt", "2026-08-22T04:00:00Z")
19459 .expect("datetime")
19460 .delete("LegacyStatus")
19461 .expect("delete");
19462 let ctx = workflow_context(Vec::new());
19463 ctx.upsert_search_attributes(update.clone())
19464 .expect("typed update");
19465 assert_eq!(
19466 ctx.take_commands().expect("search-attribute command"),
19467 vec![json!({
19468 "type": "upsert_search_attributes",
19469 "attributes": {
19470 "Attempt": 3,
19471 "Escalated": false,
19472 "LegacyStatus": null,
19473 "OrderStatus": "waiting",
19474 "Regions": ["us-east", "eu-west"],
19475 "UpdatedAt": "2026-08-22T04:00:00Z",
19476 },
19477 "attribute_types": {
19478 "Attempt": "int",
19479 "Escalated": "bool",
19480 "OrderStatus": "keyword",
19481 "Regions": "keyword_list",
19482 "UpdatedAt": "datetime",
19483 },
19484 })]
19485 );
19486
19487 let replay = workflow_context(vec![history_event(
19488 "SearchAttributesUpserted",
19489 json!({
19490 "sequence": 6,
19491 "attributes": {
19492 "Attempt": 3,
19493 "Escalated": false,
19494 "LegacyStatus": null,
19495 "OrderStatus": "waiting",
19496 "Regions": ["us-east", "eu-west"],
19497 "UpdatedAt": "2026-08-22T04:00:00Z",
19498 },
19499 "attribute_types": {
19500 "Attempt": "int",
19501 "Escalated": "bool",
19502 "OrderStatus": "keyword",
19503 "Regions": "keyword_list",
19504 "UpdatedAt": "datetime",
19505 },
19506 "merged": {},
19507 }),
19508 )]);
19509 replay
19510 .upsert_search_attributes(update)
19511 .expect("matching update replays");
19512 assert!(replay.take_commands().expect("commands").is_empty());
19513 replay.ensure_history_consumed().expect("history consumed");
19514
19515 let type_drift = workflow_context(vec![history_event(
19516 "SearchAttributesUpserted",
19517 json!({
19518 "sequence": 7,
19519 "attributes": {"OrderStatus": "waiting"},
19520 "attribute_types": {"OrderStatus": "keyword"},
19521 "merged": {"OrderStatus": "waiting"},
19522 }),
19523 )]);
19524 let error = type_drift
19525 .upsert_search_attributes(
19526 SearchAttributeUpdate::new()
19527 .string("OrderStatus", "waiting")
19528 .expect("string update"),
19529 )
19530 .expect_err("same JSON value with a changed type must fail replay");
19531 assert!(matches!(
19532 error,
19533 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19534 if reason == "search_attribute_type_mismatch"
19535 ));
19536
19537 let malformed_types = WorkflowState::new(
19538 vec![history_event(
19539 "SearchAttributesUpserted",
19540 json!({
19541 "sequence": 8,
19542 "attributes": {"OrderStatus": "waiting"},
19543 "attribute_types": {"OrderStatus": "unsupported"},
19544 "merged": {"OrderStatus": "waiting"},
19545 }),
19546 )],
19547 "rust-workers".to_string(),
19548 DEFAULT_CODEC.to_string(),
19549 None,
19550 )
19551 .expect_err("unsupported search-attribute type metadata must fail");
19552 assert!(matches!(
19553 malformed_types,
19554 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19555 if reason == "search_attribute_types_malformed"
19556 ));
19557
19558 assert!(matches!(
19559 SearchAttributeUpdate::new().keyword("bad key", "value"),
19560 Err(SearchAttributeUpdateError::InvalidKey(_))
19561 ));
19562 assert!(matches!(
19563 SearchAttributeUpdate::new().float("Ratio", f64::NAN),
19564 Err(SearchAttributeUpdateError::NonFiniteFloat(_))
19565 ));
19566 assert!(matches!(
19567 SearchAttributeUpdate::new().keyword("UnicodeKeyword", "é".repeat(128)),
19568 Err(SearchAttributeUpdateError::ValueTooLong { .. })
19569 ));
19570 assert!(matches!(
19571 SearchAttributeUpdate::new().datetime("UpdatedAt", "2026-02-30T04:00:00Z"),
19572 Err(SearchAttributeUpdateError::InvalidDateTime(_))
19573 ));
19574 assert!(matches!(
19575 workflow_context(Vec::new()).upsert_search_attributes(SearchAttributeUpdate::new()),
19576 Err(Error::InvalidSearchAttributeUpdate(
19577 SearchAttributeUpdateError::Empty
19578 ))
19579 ));
19580 }
19581
19582 #[test]
19583 fn typed_search_attribute_text_uses_the_runtime_byte_limit() {
19584 let ascii = "a".repeat(MAX_SEARCH_ATTRIBUTE_STRING_LENGTH);
19585 let utf8 = "é".repeat(MAX_SEARCH_ATTRIBUTE_STRING_LENGTH / 2);
19586
19587 assert!(SearchAttributeUpdate::new()
19588 .string("AsciiDescription", ascii)
19589 .is_ok());
19590 assert!(SearchAttributeUpdate::new()
19591 .string("Utf8Description", utf8)
19592 .is_ok());
19593 assert!(matches!(
19594 SearchAttributeUpdate::new().string(
19595 "TooLongDescription",
19596 "é".repeat((MAX_SEARCH_ATTRIBUTE_STRING_LENGTH / 2) + 1),
19597 ),
19598 Err(SearchAttributeUpdateError::ValueTooLong {
19599 kind: "string",
19600 limit: MAX_SEARCH_ATTRIBUTE_STRING_LENGTH,
19601 ..
19602 })
19603 ));
19604 }
19605
19606 #[test]
19607 fn workflow_history_rejects_unpaired_or_mismatched_timer_events() {
19608 let lone_fire = WorkflowState::new(
19609 vec![history_event(
19610 "TimerFired",
19611 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19612 )],
19613 "rust-workers".to_string(),
19614 DEFAULT_CODEC.to_string(),
19615 None,
19616 )
19617 .expect_err("TimerFired requires TimerScheduled");
19618 assert!(matches!(
19619 lone_fire,
19620 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19621 if reason == "timer_schedule_missing_or_duplicate"
19622 ));
19623
19624 let wrong_identity = WorkflowState::new(
19625 vec![
19626 history_event(
19627 "TimerScheduled",
19628 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19629 ),
19630 history_event(
19631 "TimerFired",
19632 json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
19633 ),
19634 ],
19635 "rust-workers".to_string(),
19636 DEFAULT_CODEC.to_string(),
19637 None,
19638 )
19639 .expect_err("fire must match scheduled timer identity");
19640 assert!(matches!(
19641 wrong_identity,
19642 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19643 if reason == "timer_identity_mismatch"
19644 ));
19645
19646 let duplicate_fire = WorkflowState::new(
19647 vec![
19648 history_event(
19649 "TimerScheduled",
19650 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19651 ),
19652 history_event(
19653 "TimerFired",
19654 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19655 ),
19656 history_event(
19657 "TimerFired",
19658 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19659 ),
19660 ],
19661 "rust-workers".to_string(),
19662 DEFAULT_CODEC.to_string(),
19663 None,
19664 )
19665 .expect_err("a durable timer cannot fire twice");
19666 assert!(matches!(
19667 duplicate_fire,
19668 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19669 if reason == "duplicate_timer_fire"
19670 ));
19671
19672 let wrong_fired_delay = WorkflowState::new(
19673 vec![
19674 history_event(
19675 "TimerScheduled",
19676 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19677 ),
19678 history_event(
19679 "TimerFired",
19680 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 6}),
19681 ),
19682 ],
19683 "rust-workers".to_string(),
19684 DEFAULT_CODEC.to_string(),
19685 None,
19686 )
19687 .expect_err("timer schedule and fire delays must agree");
19688 assert!(matches!(
19689 wrong_fired_delay,
19690 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19691 if reason == "timer_history_delay_mismatch"
19692 ));
19693 }
19694
19695 #[test]
19696 fn replay_rejects_activity_moved_before_recorded_timer() {
19697 let ctx = workflow_context(vec![
19698 history_event(
19699 "TimerScheduled",
19700 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19701 ),
19702 history_event(
19703 "TimerFired",
19704 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19705 ),
19706 history_event(
19707 "ActivityCompleted",
19708 json!({
19709 "sequence": 2,
19710 "activity_type": "after-timer",
19711 "payload_codec": DEFAULT_CODEC,
19712 "result": fixture_envelope(json!("done")),
19713 }),
19714 ),
19715 ]);
19716 let mut activity = Box::pin(ctx.activity("after-timer", json!([])));
19717 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19718
19719 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
19720 activity.as_mut().poll(&mut task_context)
19721 else {
19722 panic!("reordered durable command must be rejected");
19723 };
19724 assert_eq!(failure.reason, "recorded_command_mismatch");
19725 assert_eq!(failure.sequence, Some(1));
19726 assert_eq!(failure.expected.as_deref(), Some("timer"));
19727 assert_eq!(failure.actual.as_deref(), Some("activity:after-timer"));
19728 }
19729
19730 #[test]
19731 fn workflow_context_emits_a_typed_named_signal_wait() {
19732 let ctx = workflow_context(Vec::new());
19733 let mut signal = Box::pin(ctx.wait_signal("finish"));
19734 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19735
19736 assert!(matches!(
19737 signal.as_mut().poll(&mut task_context),
19738 Poll::Pending
19739 ));
19740 assert_eq!(
19741 ctx.take_commands().expect("signal-wait command"),
19742 vec![json!({
19743 "type": "open_signal_wait",
19744 "signal_name": "finish",
19745 })]
19746 );
19747 }
19748
19749 #[test]
19750 fn runtime_message_stream_transport_cannot_be_opened_as_a_user_signal() {
19751 let ctx = workflow_context(Vec::new());
19752 let mut signal = Box::pin(ctx.wait_signal(MESSAGE_STREAM_SIGNAL));
19753 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19754
19755 let Poll::Ready(Err(Error::Codec(message))) = signal.as_mut().poll(&mut task_context)
19756 else {
19757 panic!("runtime-reserved signal should be rejected");
19758 };
19759 assert!(message.contains("reserved by the workflow runtime"));
19760 assert!(ctx.take_commands().expect("commands").is_empty());
19761 }
19762
19763 #[tokio::test]
19764 async fn runtime_message_stream_transport_cannot_be_sent_as_a_user_signal() {
19765 let client = Client::builder("http://127.0.0.1:9")
19766 .build()
19767 .expect("client");
19768 let error = client
19769 .signal_workflow("workflow-1", MESSAGE_STREAM_SIGNAL, json!(["forged"]))
19770 .await
19771 .expect_err("runtime-reserved signal should be rejected before transport");
19772
19773 assert!(
19774 matches!(error, Error::Codec(ref message) if message.contains("reserved by the workflow runtime"))
19775 );
19776 }
19777
19778 #[test]
19779 fn message_stream_worker_task_consumes_current_contiguous_bounded_batch() {
19780 fn delivery(message_id: &str, position: u64, value: &str) -> Value {
19781 let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
19782 value.to_string(),
19783 )]))
19784 .expect("message payload");
19785 json!({
19786 "schema": MESSAGE_STREAM_SCHEMA,
19787 "stream_name": "orders",
19788 "message_id": message_id,
19789 "position": position,
19790 "payload_envelope": payload,
19791 })
19792 }
19793
19794 fn opened(sequence: u64) -> HistoryEvent {
19795 history_event(
19796 "SignalWaitOpened",
19797 json!({
19798 "sequence": sequence,
19799 "signal_name": MESSAGE_STREAM_SIGNAL,
19800 }),
19801 )
19802 }
19803
19804 fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
19805 history_event(
19806 "SignalApplied",
19807 json!({
19808 "sequence": sequence,
19809 "signal_name": MESSAGE_STREAM_SIGNAL,
19810 "value": fixture_envelope(json!([delivery])),
19811 }),
19812 )
19813 }
19814
19815 fn received(delivery: Value) -> HistoryEvent {
19816 history_event(
19817 "SignalReceived",
19818 json!({
19819 "signal_name": MESSAGE_STREAM_SIGNAL,
19820 "arguments": fixture_envelope(json!([delivery])),
19821 "payload_codec": DEFAULT_CODEC,
19822 }),
19823 )
19824 }
19825
19826 let client = Client::new("http://127.0.0.1:8080").expect("client");
19827 let mut worker = Worker::new(client, "rust-workers");
19828 worker.register_workflow("rust.message-stream-batch", |ctx, _input| async move {
19829 let messages = ctx.message_stream("orders")?.receive(2).await?;
19830 Ok(json!(messages
19831 .into_iter()
19832 .map(|message| message.message_id)
19833 .collect::<Vec<_>>()))
19834 });
19835
19836 let first = delivery("message-1", 1, "one");
19837 let second = delivery("message-2", 2, "two");
19838 let batch = worker
19839 .execute_workflow_task_decision(workflow_task(
19840 "rust.message-stream-batch",
19841 vec![
19842 opened(1),
19843 received(first.clone()),
19844 applied(1, first.clone()),
19845 received(first.clone()),
19846 received(second),
19847 ],
19848 DEFAULT_CODEC,
19849 ))
19850 .expect("worker task consumes the available batch");
19851
19852 assert_eq!(batch.commands.len(), 1);
19853 assert_eq!(batch.commands[0]["type"], "complete_workflow");
19854 assert_eq!(
19855 decode_wire_value(&batch.commands[0]["result"], DEFAULT_CODEC)
19856 .expect("workflow result"),
19857 json!(["message-1", "message-2"])
19858 );
19859 assert_eq!(
19860 batch.message_stream_cursors,
19861 vec![json!({"stream_name": "orders", "through_position": 2})]
19862 );
19863 assert!(batch.message_stream_waits.is_empty());
19864
19865 let partial = worker
19866 .execute_workflow_task_decision(workflow_task(
19867 "rust.message-stream-batch",
19868 vec![opened(1), received(first.clone()), applied(1, first)],
19869 DEFAULT_CODEC,
19870 ))
19871 .expect("worker task returns without waiting for a missing second item");
19872 assert_eq!(partial.commands.len(), 1);
19873 assert_eq!(partial.commands[0]["type"], "complete_workflow");
19874 assert_eq!(
19875 decode_wire_value(&partial.commands[0]["result"], DEFAULT_CODEC)
19876 .expect("workflow result"),
19877 json!(["message-1"])
19878 );
19879 assert_eq!(
19880 partial.message_stream_cursors,
19881 vec![json!({"stream_name": "orders", "through_position": 1})]
19882 );
19883 assert!(partial.message_stream_waits.is_empty());
19884 }
19885
19886 #[test]
19887 fn message_stream_replay_preserves_partial_batch_boundary_before_later_wait() {
19888 fn delivery(message_id: &str, position: u64, value: &str) -> Value {
19889 let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
19890 value.to_string(),
19891 )]))
19892 .expect("message payload");
19893 json!({
19894 "schema": MESSAGE_STREAM_SCHEMA,
19895 "stream_name": "orders",
19896 "message_id": message_id,
19897 "position": position,
19898 "payload_envelope": payload,
19899 })
19900 }
19901
19902 fn opened(sequence: u64) -> HistoryEvent {
19903 history_event(
19904 "SignalWaitOpened",
19905 json!({
19906 "sequence": sequence,
19907 "signal_name": MESSAGE_STREAM_SIGNAL,
19908 }),
19909 )
19910 }
19911
19912 fn received(delivery: Value) -> HistoryEvent {
19913 history_event(
19914 "SignalReceived",
19915 json!({
19916 "signal_name": MESSAGE_STREAM_SIGNAL,
19917 "arguments": fixture_envelope(json!([delivery])),
19918 "payload_codec": DEFAULT_CODEC,
19919 }),
19920 )
19921 }
19922
19923 fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
19924 history_event(
19925 "SignalApplied",
19926 json!({
19927 "sequence": sequence,
19928 "signal_name": MESSAGE_STREAM_SIGNAL,
19929 "value": fixture_envelope(json!([delivery])),
19930 }),
19931 )
19932 }
19933
19934 let client = Client::new("http://127.0.0.1:8080").expect("client");
19935 let mut worker = Worker::new(client, "rust-workers");
19936 worker.register_workflow(
19937 "rust.message-stream-partial-batches",
19938 |ctx, _input| async move {
19939 let stream = ctx.message_stream("orders")?;
19940 let first = stream.receive(10).await?;
19941 let second = stream.receive(10).await?;
19942 Ok(json!([
19943 first
19944 .into_iter()
19945 .map(|message| message.message_id)
19946 .collect::<Vec<_>>(),
19947 second
19948 .into_iter()
19949 .map(|message| message.message_id)
19950 .collect::<Vec<_>>(),
19951 ]))
19952 },
19953 );
19954
19955 let first = delivery("message-1", 1, "one");
19956 let second = delivery("message-2", 2, "two");
19957 let decision = worker
19958 .execute_workflow_task_decision(workflow_task(
19959 "rust.message-stream-partial-batches",
19960 vec![
19961 opened(1),
19962 received(first.clone()),
19963 applied(1, first),
19964 opened(2),
19965 received(second.clone()),
19966 applied(2, second),
19967 ],
19968 DEFAULT_CODEC,
19969 ))
19970 .expect("cold replay preserves both authored receive boundaries");
19971
19972 assert_eq!(decision.commands.len(), 1);
19973 assert_eq!(decision.commands[0]["type"], "complete_workflow");
19974 assert_eq!(
19975 decode_wire_value(&decision.commands[0]["result"], DEFAULT_CODEC)
19976 .expect("workflow result"),
19977 json!([["message-1"], ["message-2"]])
19978 );
19979 assert_eq!(
19980 decision.message_stream_cursors,
19981 vec![json!({"stream_name": "orders", "through_position": 2})]
19982 );
19983 assert!(decision.message_stream_waits.is_empty());
19984 }
19985
19986 #[test]
19987 fn empty_message_stream_opens_internal_signal_wait_and_reports_position() {
19988 let ctx = workflow_context(Vec::new());
19989 let stream = ctx.message_stream("orders").expect("message stream");
19990 let mut receive = Box::pin(stream.receive(10));
19991 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19992
19993 assert!(matches!(
19994 receive.as_mut().poll(&mut task_context),
19995 Poll::Pending
19996 ));
19997 assert_eq!(
19998 ctx.take_commands().expect("message-stream wait command"),
19999 vec![json!({
20000 "type": "open_signal_wait",
20001 "signal_name": MESSAGE_STREAM_SIGNAL,
20002 })]
20003 );
20004 let (cursors, waits) = ctx.message_stream_metadata().expect("stream metadata");
20005 assert!(cursors.is_empty());
20006 assert_eq!(
20007 waits,
20008 vec![json!({"stream_name": "orders", "after_position": 0})]
20009 );
20010 }
20011
20012 #[test]
20013 fn continue_as_new_cursor_checkpoint_preserves_global_pending_position() {
20014 let ctx = workflow_context(vec![history_event(
20015 "SignalReceived",
20016 json!({
20017 "signal_name": MESSAGE_STREAM_SIGNAL,
20018 "arguments": fixture_envelope(json!([{
20019 "schema": MESSAGE_STREAM_CURSOR_SCHEMA,
20020 "stream_name": "orders",
20021 "through_position": 2,
20022 }])),
20023 "payload_codec": DEFAULT_CODEC,
20024 }),
20025 )]);
20026 let stream = ctx.message_stream("orders").expect("message stream");
20027 let mut receive = Box::pin(stream.receive(10));
20028 let mut task_context = TaskContext::from_waker(noop_waker_ref());
20029
20030 assert!(matches!(
20031 receive.as_mut().poll(&mut task_context),
20032 Poll::Pending
20033 ));
20034 let (cursors, waits) = ctx.message_stream_metadata().expect("stream metadata");
20035 assert_eq!(
20036 cursors,
20037 vec![json!({"stream_name": "orders", "through_position": 2})]
20038 );
20039 assert_eq!(
20040 waits,
20041 vec![json!({"stream_name": "orders", "after_position": 2})]
20042 );
20043 }
20044
20045 #[test]
20046 fn message_stream_delivery_preserves_typed_avro_arguments_across_replay() {
20047 let mut empty_map = BTreeMap::new();
20048 let mut nested = BTreeMap::new();
20049 nested.insert(
20050 "value".to_string(),
20051 AvroValue::Array(vec![AvroValue::Bytes(b"nested".to_vec())]),
20052 );
20053 let values = vec![
20054 AvroValue::Bytes(vec![0, 255]),
20055 AvroValue::Long(1),
20056 AvroValue::Double(1.0),
20057 AvroValue::Array(Vec::new()),
20058 AvroValue::Map(std::mem::take(&mut empty_map)),
20059 AvroValue::Map(nested),
20060 ];
20061 let payload = encode_avro_value(&AvroValue::Array(values.clone())).expect("payload");
20062 let transport = vec![json!({
20063 "schema": MESSAGE_STREAM_SCHEMA,
20064 "stream_name": "orders",
20065 "message_id": "message-1",
20066 "position": 1,
20067 "payload_envelope": payload,
20068 })];
20069
20070 for _ in 0..2 {
20071 let Some(MessageStreamDelivery::Message(message)) =
20072 decode_message_stream_delivery(transport.clone()).expect("delivery")
20073 else {
20074 panic!("message delivery expected");
20075 };
20076 assert_eq!(message.arguments, values);
20077 assert!(matches!(message.arguments[1], AvroValue::Long(1)));
20078 assert!(matches!(message.arguments[2], AvroValue::Double(1.0)));
20079 }
20080 }
20081
20082 #[test]
20083 fn cold_worker_replacement_consumes_message_stream_wait_arrivals_once_in_order() {
20084 fn delivery(message_id: &str, position: u64, value: &str) -> Value {
20085 let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
20086 value.to_string(),
20087 )]))
20088 .expect("message payload");
20089 json!({
20090 "schema": MESSAGE_STREAM_SCHEMA,
20091 "stream_name": "orders",
20092 "message_id": message_id,
20093 "position": position,
20094 "payload_envelope": payload,
20095 })
20096 }
20097
20098 fn opened(sequence: u64) -> HistoryEvent {
20099 history_event(
20100 "SignalWaitOpened",
20101 json!({
20102 "sequence": sequence,
20103 "signal_name": MESSAGE_STREAM_SIGNAL,
20104 }),
20105 )
20106 }
20107
20108 fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
20109 history_event(
20110 "SignalApplied",
20111 json!({
20112 "sequence": sequence,
20113 "signal_name": MESSAGE_STREAM_SIGNAL,
20114 "value": fixture_envelope(json!([delivery])),
20115 }),
20116 )
20117 }
20118
20119 fn worker() -> Worker {
20120 let client = Client::new("http://127.0.0.1:8080").expect("client");
20121 let mut worker = Worker::new(client, "rust-workers");
20122 worker.register_workflow("rust.message-stream", |ctx, _input| async move {
20123 let stream = ctx.message_stream("orders")?;
20124 let first = stream.receive_one().await?;
20125 let second = stream.receive_one().await?;
20126 Ok(json!([first.message_id, second.message_id]))
20127 });
20128 worker
20129 }
20130
20131 fn task_with_resume(history: Vec<HistoryEvent>, delivery: Value) -> WorkflowTask {
20132 let mut task = workflow_task("rust.message-stream", history, DEFAULT_CODEC);
20133 task.signal_name = Some(MESSAGE_STREAM_SIGNAL.to_string());
20134 task.signal_arguments = Some(fixture_envelope(json!([delivery])));
20135 task
20136 }
20137
20138 let waiting = worker()
20139 .execute_workflow_task_decision(workflow_task(
20140 "rust.message-stream",
20141 Vec::new(),
20142 DEFAULT_CODEC,
20143 ))
20144 .expect("first worker opens the stream wait");
20145 assert_eq!(
20146 waiting.commands,
20147 vec![json!({
20148 "type": "open_signal_wait",
20149 "signal_name": MESSAGE_STREAM_SIGNAL,
20150 })]
20151 );
20152 assert!(waiting.message_stream_cursors.is_empty());
20153 assert_eq!(
20154 waiting.message_stream_waits,
20155 vec![json!({"stream_name": "orders", "after_position": 0})]
20156 );
20157
20158 let first_delivery = delivery("message-1", 1, "one");
20159 let first_arrival = worker()
20160 .execute_workflow_task_decision(task_with_resume(
20161 vec![opened(1)],
20162 first_delivery.clone(),
20163 ))
20164 .expect("replacement worker consumes the first arrival");
20165 assert_eq!(
20166 first_arrival.commands,
20167 vec![json!({
20168 "type": "open_signal_wait",
20169 "signal_name": MESSAGE_STREAM_SIGNAL,
20170 })]
20171 );
20172 assert_eq!(
20173 first_arrival.message_stream_cursors,
20174 vec![json!({"stream_name": "orders", "through_position": 1})]
20175 );
20176 assert_eq!(
20177 first_arrival.message_stream_waits,
20178 vec![json!({"stream_name": "orders", "after_position": 1})]
20179 );
20180
20181 let second_delivery = delivery("message-2", 2, "two");
20182 let first_applied = applied(1, first_delivery);
20183 let completed = worker()
20184 .execute_workflow_task_decision(task_with_resume(
20185 vec![opened(1), first_applied.clone(), opened(2)],
20186 second_delivery.clone(),
20187 ))
20188 .expect("next replacement worker consumes the second arrival");
20189 assert_eq!(completed.commands.len(), 1);
20190 assert_eq!(completed.commands[0]["type"], "complete_workflow");
20191 assert_eq!(
20192 decode_wire_value(&completed.commands[0]["result"], DEFAULT_CODEC)
20193 .expect("workflow result"),
20194 json!(["message-1", "message-2"])
20195 );
20196 assert_eq!(
20197 completed.message_stream_cursors,
20198 vec![json!({"stream_name": "orders", "through_position": 2})]
20199 );
20200 assert!(completed.message_stream_waits.is_empty());
20201
20202 let replay_history = vec![
20203 opened(1),
20204 first_applied,
20205 opened(2),
20206 applied(2, second_delivery),
20207 ];
20208 for _cold_worker_or_restart in 0..2 {
20209 let replayed = worker()
20210 .execute_workflow_task_decision(workflow_task(
20211 "rust.message-stream",
20212 replay_history.clone(),
20213 DEFAULT_CODEC,
20214 ))
20215 .expect("cold worker replays each logical message exactly once");
20216 assert_eq!(replayed.commands.len(), 1);
20217 assert_eq!(
20218 decode_wire_value(&replayed.commands[0]["result"], DEFAULT_CODEC)
20219 .expect("replayed workflow result"),
20220 json!(["message-1", "message-2"])
20221 );
20222 assert_eq!(
20223 replayed.message_stream_cursors,
20224 vec![json!({"stream_name": "orders", "through_position": 2})]
20225 );
20226 assert!(replayed.message_stream_waits.is_empty());
20227 }
20228 }
20229
20230 #[test]
20231 fn message_stream_capability_and_completion_require_protocol_one_fifteen() {
20232 assert!(!worker_protocol_supports_message_streams("1.14"));
20233 assert!(worker_protocol_supports_message_streams("1.15"));
20234 assert!(worker_protocol_supports_message_streams("1.16"));
20235 assert!(worker_protocol_supports_message_streams(
20236 WORKER_PROTOCOL_VERSION
20237 ));
20238 assert_eq!(MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION, "1.15");
20239 }
20240
20241 #[test]
20242 fn condition_wait_history_cannot_be_consumed_as_a_typed_signal_wait() {
20243 let ctx = workflow_context(vec![
20244 history_event(
20245 "ConditionWaitOpened",
20246 json!({
20247 "sequence": 1,
20248 "condition_wait_id": "condition:1",
20249 "condition_wait_occurrence_id": "rust:condition-wait:0",
20250 "condition_key": "signal:finish",
20251 "condition_definition_fingerprint": "sha256:signal-finish-v1",
20252 }),
20253 ),
20254 history_event(
20255 "ConditionWaitSatisfied",
20256 json!({
20257 "sequence": 1,
20258 "condition_wait_id": "condition:1",
20259 "condition_wait_occurrence_id": "rust:condition-wait:0",
20260 "condition_key": "signal:finish",
20261 "condition_definition_fingerprint": "sha256:signal-finish-v1",
20262 }),
20263 ),
20264 history_event(
20265 "SignalReceived",
20266 json!({"signal_name": "finish", "arguments": []}),
20267 ),
20268 ]);
20269 let mut signal = Box::pin(ctx.wait_signal("finish"));
20270 let mut task_context = TaskContext::from_waker(noop_waker_ref());
20271
20272 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
20273 signal.as_mut().poll(&mut task_context)
20274 else {
20275 panic!("condition history must not resolve as a typed signal wait");
20276 };
20277 assert_eq!(failure.reason, "recorded_command_mismatch");
20278 assert_eq!(failure.expected.as_deref(), Some("condition wait"));
20279 }
20280
20281 #[test]
20282 fn replay_orders_signal_waits_and_timers_in_one_command_stream() {
20283 let signal_then_timer = vec![
20284 history_event(
20285 "SignalWaitOpened",
20286 json!({"sequence": 1, "signal_name": "go"}),
20287 ),
20288 history_event(
20289 "SignalApplied",
20290 json!({
20291 "sequence": 1,
20292 "signal_name": "go",
20293 "value": fixture_envelope(json!(["now"])),
20294 }),
20295 ),
20296 history_event(
20297 "TimerScheduled",
20298 json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
20299 ),
20300 history_event(
20301 "TimerFired",
20302 json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
20303 ),
20304 ];
20305
20306 let ctx = workflow_context(signal_then_timer.clone());
20307 let mut signal = Box::pin(ctx.wait_signal("go"));
20308 let mut task_context = TaskContext::from_waker(noop_waker_ref());
20309 assert!(matches!(
20310 signal.as_mut().poll(&mut task_context),
20311 Poll::Ready(Ok(arguments)) if arguments == vec![json!("now")]
20312 ));
20313 let mut timer = Box::pin(ctx.sleep(Duration::from_secs(5)));
20314 assert!(matches!(
20315 timer.as_mut().poll(&mut task_context),
20316 Poll::Ready(Ok(()))
20317 ));
20318 ctx.ensure_history_consumed()
20319 .expect("signal and timer history consumed in order");
20320
20321 let reordered = workflow_context(signal_then_timer);
20322 let mut timer_first = Box::pin(reordered.sleep(Duration::from_secs(5)));
20323 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
20324 timer_first.as_mut().poll(&mut task_context)
20325 else {
20326 panic!("timer cannot consume signal-wait-first history");
20327 };
20328 assert_eq!(failure.reason, "recorded_command_mismatch");
20329 assert_eq!(failure.sequence, Some(1));
20330 assert_eq!(failure.expected.as_deref(), Some("signal wait"));
20331
20332 let timer_then_signal = vec![
20333 history_event(
20334 "TimerScheduled",
20335 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20336 ),
20337 history_event(
20338 "TimerFired",
20339 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20340 ),
20341 history_event(
20342 "SignalWaitOpened",
20343 json!({"sequence": 2, "signal_name": "go"}),
20344 ),
20345 history_event(
20346 "SignalApplied",
20347 json!({
20348 "sequence": 2,
20349 "signal_name": "go",
20350 "value": fixture_envelope(json!([])),
20351 }),
20352 ),
20353 ];
20354 let reordered = workflow_context(timer_then_signal);
20355 let mut signal_first = Box::pin(reordered.wait_signal("go"));
20356 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
20357 signal_first.as_mut().poll(&mut task_context)
20358 else {
20359 panic!("signal wait cannot consume timer-first history");
20360 };
20361 assert_eq!(failure.reason, "recorded_command_mismatch");
20362 assert_eq!(failure.sequence, Some(1));
20363 assert_eq!(failure.expected.as_deref(), Some("timer"));
20364 }
20365
20366 #[test]
20367 fn workflow_history_rejects_duplicate_or_colliding_command_sequences() {
20368 let duplicate_timer = WorkflowState::new(
20369 vec![
20370 history_event(
20371 "TimerScheduled",
20372 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20373 ),
20374 history_event(
20375 "TimerScheduled",
20376 json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
20377 ),
20378 ],
20379 "rust-workers".to_string(),
20380 DEFAULT_CODEC.to_string(),
20381 None,
20382 )
20383 .expect_err("one workflow sequence cannot schedule two timers");
20384 assert!(matches!(
20385 duplicate_timer,
20386 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20387 if reason == "timer_schedule_missing_or_duplicate"
20388 ));
20389
20390 let colliding_kinds = WorkflowState::new(
20391 vec![
20392 history_event(
20393 "TimerScheduled",
20394 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20395 ),
20396 history_event(
20397 "ActivityCompleted",
20398 json!({"sequence": 1, "activity_type": "same-sequence"}),
20399 ),
20400 ],
20401 "rust-workers".to_string(),
20402 DEFAULT_CODEC.to_string(),
20403 None,
20404 )
20405 .expect_err("one workflow sequence cannot identify two command kinds");
20406 assert!(matches!(
20407 colliding_kinds,
20408 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20409 if reason == "durable_command_sequence_collision"
20410 ));
20411
20412 let duplicate_signal_wait = WorkflowState::new(
20413 vec![
20414 history_event(
20415 "SignalWaitOpened",
20416 json!({"sequence": 1, "signal_name": "go"}),
20417 ),
20418 history_event(
20419 "SignalWaitOpened",
20420 json!({"sequence": 1, "signal_name": "go"}),
20421 ),
20422 ],
20423 "rust-workers".to_string(),
20424 DEFAULT_CODEC.to_string(),
20425 None,
20426 )
20427 .expect_err("one workflow sequence cannot open two signal waits");
20428 assert!(matches!(
20429 duplicate_signal_wait,
20430 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20431 if reason == "signal_wait_open_missing_or_duplicate"
20432 ));
20433 }
20434
20435 #[test]
20436 fn workflow_history_accepts_a_first_command_after_global_sequence_gaps() {
20437 let result = encode_value_envelope(&json!({"captured": true}), DEFAULT_CODEC)
20438 .expect("side-effect result");
20439 let ctx = workflow_context(vec![history_event(
20440 "SideEffectRecorded",
20441 json!({"sequence": 99, "result": result}),
20442 )]);
20443
20444 let replayed: Value = ctx
20445 .side_effect(|| panic!("recorded side effect must not run"))
20446 .expect("positive global workflow sequence is valid");
20447 assert_eq!(replayed, json!({"captured": true}));
20448 ctx.ensure_history_consumed().expect("history consumed");
20449 }
20450
20451 #[test]
20452 fn workflow_history_rejects_zero_and_descending_command_sequences() {
20453 let result =
20454 encode_value_envelope(&json!("captured"), DEFAULT_CODEC).expect("side-effect result");
20455 let zero = WorkflowState::new(
20456 vec![history_event(
20457 "SideEffectRecorded",
20458 json!({"sequence": 0, "result": result.clone()}),
20459 )],
20460 "rust-workers".to_string(),
20461 DEFAULT_CODEC.to_string(),
20462 None,
20463 )
20464 .expect_err("durable command sequences must be positive");
20465 assert!(matches!(
20466 zero,
20467 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20468 if reason == "durable_command_sequence_invalid"
20469 ));
20470
20471 let descending = WorkflowState::new(
20472 vec![
20473 history_event(
20474 "SideEffectRecorded",
20475 json!({"sequence": 3, "result": result}),
20476 ),
20477 history_event(
20478 "VersionMarkerRecorded",
20479 json!({
20480 "sequence": 2,
20481 "change_id": "descending-marker",
20482 "version": 1,
20483 "min_supported": 1,
20484 "max_supported": 1,
20485 }),
20486 ),
20487 ],
20488 "rust-workers".to_string(),
20489 DEFAULT_CODEC.to_string(),
20490 None,
20491 )
20492 .expect_err("new durable commands must remain strictly ordered");
20493 let Error::NonDeterministicReplay(failure) = descending else {
20494 panic!("expected typed replay failure");
20495 };
20496 assert_eq!(failure.reason, "durable_command_sequence_mismatch");
20497 assert_eq!(failure.sequence, Some(2));
20498 assert_eq!(
20499 failure.expected.as_deref(),
20500 Some("workflow sequence greater than 3")
20501 );
20502 assert_eq!(failure.actual.as_deref(), Some("2"));
20503 }
20504
20505 #[test]
20506 fn workflow_task_replay_completes_after_signals_create_sequence_gaps() {
20507 fn worker() -> Worker {
20508 let client = Client::new("http://127.0.0.1:8080").expect("client");
20509 let mut worker = Worker::new(client, "rust-workers");
20510 worker.register_workflow("rust.finish-after-gaps", |ctx, _input| async move {
20511 ctx.wait_signal("finish").await?;
20512 let marker: String =
20513 ctx.side_effect(|| panic!("recorded side effect must not run"))?;
20514 assert_eq!(marker, "after-finish");
20515 Ok(json!("finished"))
20516 });
20517 worker
20518 }
20519
20520 let marker = encode_value_envelope(&json!("after-finish"), DEFAULT_CODEC)
20521 .expect("side-effect result");
20522 let task = workflow_task(
20523 "rust.finish-after-gaps",
20524 vec![
20525 history_event(
20526 "SignalWaitOpened",
20527 json!({"sequence": 1, "signal_name": "finish"}),
20528 ),
20529 history_event(
20530 "SignalReceived",
20531 json!({
20532 "signal_id": "increment-3",
20533 "signal_name": "increment",
20534 "workflow_sequence": 2,
20535 "payload_codec": DEFAULT_CODEC,
20536 "arguments": fixture_envelope(json!([3])),
20537 }),
20538 ),
20539 history_event(
20540 "SignalReceived",
20541 json!({
20542 "signal_id": "increment-5",
20543 "signal_name": "increment",
20544 "workflow_sequence": 3,
20545 "payload_codec": DEFAULT_CODEC,
20546 "arguments": fixture_envelope(json!([5])),
20547 }),
20548 ),
20549 history_event(
20550 "SignalReceived",
20551 json!({
20552 "signal_id": "finish",
20553 "signal_name": "finish",
20554 "workflow_sequence": 4,
20555 "payload_codec": DEFAULT_CODEC,
20556 "arguments": fixture_envelope(json!([])),
20557 }),
20558 ),
20559 history_event(
20560 "SignalApplied",
20561 json!({
20562 "sequence": 1,
20563 "signal_id": "finish",
20564 "signal_name": "finish",
20565 "payload_codec": DEFAULT_CODEC,
20566 "value": fixture_envelope(json!([])),
20567 }),
20568 ),
20569 history_event(
20570 "SideEffectRecorded",
20571 json!({"sequence": 5, "result": marker}),
20572 ),
20573 ],
20574 DEFAULT_CODEC,
20575 );
20576
20577 for _original_or_cold_worker in 0..2 {
20578 let commands = worker()
20579 .execute_workflow_task(task.clone())
20580 .expect("signal gaps preserve deterministic replay");
20581 assert_eq!(commands.len(), 1, "replay emits only terminal completion");
20582 assert_eq!(commands[0]["type"], "complete_workflow");
20583 assert_eq!(
20584 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("workflow output"),
20585 json!("finished")
20586 );
20587 }
20588 }
20589
20590 #[test]
20591 fn workflow_sleep_rejects_unrepresentable_rounded_duration() {
20592 let ctx = workflow_context(Vec::new());
20593 let mut sleep = Box::pin(ctx.start_timer(Duration::new(u64::MAX, 1)));
20594 let mut task_context = TaskContext::from_waker(noop_waker_ref());
20595 assert!(matches!(
20596 sleep.as_mut().poll(&mut task_context),
20597 Poll::Ready(Err(Error::TimerDurationOverflow))
20598 ));
20599 assert!(ctx.take_commands().expect("commands").is_empty());
20600 }
20601
20602 #[test]
20603 fn workflow_memo_update_emits_canonical_command_and_replays_once() {
20604 let entries = AvroValue::Map(BTreeMap::from([
20605 ("text".to_string(), AvroValue::String("same".to_string())),
20606 (
20607 "nested".to_string(),
20608 AvroValue::Map(BTreeMap::from([
20609 ("beta".to_string(), AvroValue::Long(2)),
20610 ("alpha".to_string(), AvroValue::Long(1)),
20611 ])),
20612 ),
20613 ("long".to_string(), AvroValue::Long(7)),
20614 ("double".to_string(), AvroValue::Double(7.0)),
20615 ("binary".to_string(), AvroValue::Bytes(b"same".to_vec())),
20616 ]));
20617 let ctx = workflow_context(Vec::new());
20618 ctx.upsert_memo(entries.clone()).expect("valid memo update");
20619 let commands = ctx.take_commands().expect("commands");
20620
20621 assert_eq!(commands.len(), 1);
20622 assert_eq!(commands[0]["type"], "upsert_memo");
20623 let server_entries = json!({
20624 "codec": "avro",
20625 "blob": "wwHioz3/VYAiNw4KDGJpbmFyeQgIc2FtZQxkb3VibGUGAAAAAAAAHEAIbG9uZwQODG5lc3RlZA4ECmFscGhhBAIIYmV0YQQEAAh0ZXh0CghzYW1lAA==",
20626 });
20627 assert_eq!(
20628 commands[0]["entries"]
20629 .as_object()
20630 .expect("entries envelope")
20631 .keys()
20632 .collect::<Vec<_>>(),
20633 vec!["blob", "codec"]
20634 );
20635 assert_eq!(commands[0]["entries"], server_entries);
20636 let wire_entries =
20637 decode_wire_avro_value(&commands[0]["entries"], DEFAULT_CODEC).expect("memo entries");
20638 assert_eq!(wire_entries, entries);
20639
20640 let history = vec![history_event(
20641 "MemoUpserted",
20642 json!({
20643 "sequence": 1,
20644 "entries": server_entries.clone(),
20645 "merged": server_entries,
20646 }),
20647 )];
20648 let replay = workflow_context(history.clone());
20649 replay
20650 .upsert_memo(entries.clone())
20651 .expect("matching replay identity");
20652 assert!(replay.take_commands().expect("replay commands").is_empty());
20653
20654 let changed_types = AvroValue::Map(BTreeMap::from([
20655 ("text".to_string(), AvroValue::Bytes(b"same".to_vec())),
20656 (
20657 "nested".to_string(),
20658 AvroValue::Map(BTreeMap::from([
20659 ("alpha".to_string(), AvroValue::Long(1)),
20660 ("beta".to_string(), AvroValue::Long(2)),
20661 ])),
20662 ),
20663 ("long".to_string(), AvroValue::Double(7.0)),
20664 ("double".to_string(), AvroValue::Long(7)),
20665 ("binary".to_string(), AvroValue::String("same".to_string())),
20666 ]));
20667 let error = workflow_context(history)
20668 .upsert_memo(changed_types)
20669 .expect_err("memo replay identity must preserve Avro value types");
20670 assert!(matches!(
20671 error,
20672 Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
20673 ));
20674 }
20675
20676 #[test]
20677 fn workflow_memo_update_rejects_changed_replay_identity_and_invalid_keys() {
20678 let original = encode_value_envelope(&json!({"stage": "original"}), DEFAULT_CODEC)
20679 .expect("memo envelope");
20680 let replay = workflow_context(vec![history_event(
20681 "MemoUpserted",
20682 json!({
20683 "sequence": 1,
20684 "entries": original.clone(),
20685 "merged": original
20686 }),
20687 )]);
20688 let error = replay
20689 .upsert_memo(json!({"stage": "changed"}))
20690 .expect_err("changed memo update must fail replay");
20691 assert!(matches!(
20692 error,
20693 Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
20694 ));
20695
20696 let invalid = workflow_context(Vec::new())
20697 .upsert_memo(
20698 json!({"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx": true}),
20699 )
20700 .expect_err("oversized key");
20701 assert!(matches!(invalid, Error::InvalidMemoUpdate(_)));
20702 }
20703
20704 #[test]
20705 fn workflow_memo_replay_distinguishes_signed_zero_identity() {
20706 let negative_zero = AvroValue::Map(BTreeMap::from([(
20707 "reading".to_string(),
20708 AvroValue::Double(-0.0),
20709 )]));
20710 let negative_zero_envelope =
20711 encode_typed_envelope(&negative_zero, DEFAULT_CODEC).expect("negative zero envelope");
20712 let history = vec![history_event(
20713 "MemoUpserted",
20714 json!({
20715 "sequence": 1,
20716 "entries": negative_zero_envelope.clone(),
20717 "merged": negative_zero_envelope,
20718 }),
20719 )];
20720
20721 workflow_context(history.clone())
20722 .upsert_memo(negative_zero)
20723 .expect("matching negative-zero history identity");
20724
20725 let error = workflow_context(history)
20726 .upsert_memo(AvroValue::Map(BTreeMap::from([(
20727 "reading".to_string(),
20728 AvroValue::Double(0.0),
20729 )])))
20730 .expect_err("positive zero must not consume negative-zero memo history");
20731 assert!(matches!(
20732 error,
20733 Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
20734 ));
20735 }
20736
20737 #[test]
20738 fn workflow_memo_capability_requires_flag_and_command_advertisement() {
20739 let supported = json!({
20740 "workflow_memo_updates": {"supported": true, "minimum_protocol_version": "1.14"},
20741 "supported_workflow_task_commands": ["complete_workflow", "upsert_memo"]
20742 });
20743 assert!(runtime_supports_workflow_memo_updates(Some(&supported)));
20744 assert!(!runtime_supports_workflow_memo_updates(Some(&json!({
20745 "workflow_memo_updates": {"supported": false},
20746 "supported_workflow_task_commands": ["upsert_memo"]
20747 }))));
20748 assert!(commands_use_workflow_memo_updates(&[json!({
20749 "type": "upsert_memo",
20750 "entries": {"stage": "processing"}
20751 })]));
20752 }
20753
20754 #[test]
20755 fn workflow_task_replay_completes_without_rescheduling_recorded_commands() {
20756 let client = Client::new("http://127.0.0.1:8080").expect("client");
20757 let mut worker = Worker::new(client, "rust-workers");
20758 worker.register_workflow("rust.timer", |ctx, _input| async move {
20759 ctx.sleep(Duration::from_secs(5)).await?;
20760 ctx.activity("after-timer", json!([])).await
20761 });
20762
20763 let task = |history_events| WorkflowTask {
20764 task_id: "wft-rust-timer-1".to_string(),
20765 workflow_command_id: None,
20766 workflow_id: Some("wf-rust-timer".to_string()),
20767 run_id: Some("run-rust-timer".to_string()),
20768 workflow_type: "rust.timer".to_string(),
20769 cancel_requested: false,
20770 payload_codec: DEFAULT_CODEC.to_string(),
20771 arguments: Some(
20772 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
20773 ),
20774 history_events,
20775 total_history_events: None,
20776 history_size_bytes: None,
20777 continue_as_new_recommended: None,
20778 history_budget_pressure: None,
20779 next_history_page_token: None,
20780 workflow_task_attempt: 1,
20781 workflow_signal_id: None,
20782 signal_name: None,
20783 signal_arguments: None,
20784 workflow_update_id: None,
20785 update_name: None,
20786 lease_owner: Some("rust-worker".to_string()),
20787 };
20788
20789 let initial = worker
20790 .execute_workflow_task(task(Vec::new()))
20791 .expect("initial timer task");
20792 assert_eq!(
20793 initial,
20794 vec![json!({"type": "start_timer", "delay_seconds": 5})]
20795 );
20796
20797 let activity_result =
20798 encode_value_envelope(&json!("done"), DEFAULT_CODEC).expect("activity result");
20799 let replayed = worker
20800 .execute_workflow_task(task(vec![
20801 history_event(
20802 "TimerScheduled",
20803 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20804 ),
20805 history_event(
20806 "TimerFired",
20807 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20808 ),
20809 history_event(
20810 "ActivityCompleted",
20811 json!({
20812 "sequence": 2,
20813 "activity_type": "after-timer",
20814 "payload_codec": DEFAULT_CODEC,
20815 "result": activity_result,
20816 }),
20817 ),
20818 ]))
20819 .expect("replayed workflow task");
20820 assert_eq!(replayed.len(), 1);
20821 assert_eq!(replayed[0]["type"], "complete_workflow");
20822 assert_eq!(
20823 decode_wire_value(&replayed[0]["result"], DEFAULT_CODEC).expect("result"),
20824 json!("done")
20825 );
20826 }
20827
20828 #[test]
20829 fn workflow_continue_as_new_emits_arguments_type_and_queue_once() {
20830 let client = Client::new("http://127.0.0.1:8080").expect("client");
20831 let mut worker = Worker::new(client, "rust-workers");
20832 worker.register_workflow("rust.continue", |ctx, _input| async move {
20833 ctx.continue_as_new_with_options(
20834 ContinueAsNewOptions::new()
20835 .workflow_type("rust.next")
20836 .task_queue("next-workers"),
20837 json!([2, {"cursor": "next"}]),
20838 )
20839 });
20840
20841 let commands = worker
20842 .execute_workflow_task(workflow_task("rust.continue", Vec::new(), DEFAULT_CODEC))
20843 .expect("continue-as-new command");
20844
20845 assert_eq!(commands.len(), 1);
20846 assert_eq!(commands[0]["type"], "continue_as_new");
20847 assert_eq!(commands[0]["workflow_type"], "rust.next");
20848 assert_eq!(commands[0]["queue"], "next-workers");
20849 assert_eq!(
20850 decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC)
20851 .expect("continue-as-new arguments"),
20852 json!([2, {"cursor": "next"}])
20853 );
20854 }
20855
20856 #[test]
20857 fn continue_as_new_preserves_typed_arguments() {
20858 let client = Client::new("http://127.0.0.1:8080").expect("client");
20859 let mut worker = Worker::new(client, "rust-workers");
20860 worker.register_workflow_avro_value("rust.typed-continue", |ctx, _input| async move {
20861 ctx.continue_as_new(AvroValue::Array(vec![typed_fidelity_probe()]))?;
20862 unreachable!("continue-as-new returns a control-flow error")
20863 });
20864
20865 let commands = worker
20866 .execute_workflow_task(workflow_task(
20867 "rust.typed-continue",
20868 Vec::new(),
20869 DEFAULT_CODEC,
20870 ))
20871 .expect("typed continue-as-new command");
20872
20873 assert_eq!(commands[0]["type"], "continue_as_new");
20874 assert_eq!(
20875 decode_wire_avro_value(&commands[0]["arguments"], DEFAULT_CODEC)
20876 .expect("typed continue arguments"),
20877 AvroValue::Array(vec![typed_fidelity_probe()])
20878 );
20879 }
20880
20881 #[test]
20882 fn recorded_continue_as_new_is_consumed_without_duplicate_successor_command() {
20883 let client = Client::new("http://127.0.0.1:8080").expect("client");
20884 let mut worker = Worker::new(client, "rust-workers");
20885 worker.register_workflow("rust.continue", |ctx, _input| async move {
20886 ctx.continue_as_new(json!([2]))
20887 });
20888 let task = workflow_task(
20889 "rust.continue",
20890 vec![history_event(
20891 "WorkflowContinuedAsNew",
20892 json!({"sequence": 1, "continued_to_run_id": "run-next"}),
20893 )],
20894 DEFAULT_CODEC,
20895 );
20896
20897 for _worker_restart_or_redelivery in 0..2 {
20898 let commands = worker
20899 .execute_workflow_task(task.clone())
20900 .expect("recorded transition replays");
20901 assert!(
20902 commands.is_empty(),
20903 "replay must not emit another successor"
20904 );
20905 }
20906 }
20907
20908 #[test]
20909 fn continue_as_new_rejects_invalid_overrides_before_emitting_a_command() {
20910 let ctx = workflow_context(Vec::new());
20911 let error = ctx
20912 .continue_as_new_with_options(ContinueAsNewOptions::new().task_queue(" "), json!([1]))
20913 .expect_err("blank queue must be rejected");
20914
20915 let Error::InvalidContinueAsNewOptions(error) = error else {
20916 panic!("expected typed continue-as-new validation error");
20917 };
20918 assert_eq!(error.field, "task_queue");
20919 assert!(ctx.take_commands().expect("commands").is_empty());
20920 }
20921
20922 #[test]
20923 fn workflow_context_exposes_server_history_budget() {
20924 let client = Client::new("http://127.0.0.1:8080").expect("client");
20925 let mut worker = Worker::new(client, "rust-workers");
20926 worker.register_workflow("rust.history-budget", |ctx, _input| async move {
20927 let budget = ctx.history_budget()?;
20928 Ok(json!({
20929 "events": budget.event_count,
20930 "bytes": budget.size_bytes,
20931 "recommended": budget.continue_as_new_recommended,
20932 "pressure": budget.pressure,
20933 }))
20934 });
20935 let task: WorkflowTask = serde_json::from_value(json!({
20936 "task_id": "task-history-budget",
20937 "workflow_type": "rust.history-budget",
20938 "payload_codec": DEFAULT_CODEC,
20939 "history_events": [],
20940 "total_history_events": 480,
20941 "history_size_bytes": 1_048_576,
20942 "continue_as_new_recommended": true,
20943 "history_budget_pressure": "continue_as_new_recommended",
20944 }))
20945 .expect("published workflow task");
20946
20947 let commands = worker
20948 .execute_workflow_task(task)
20949 .expect("history-budget workflow");
20950 let result = decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("result");
20951 assert_eq!(result["events"], 480);
20952 assert_eq!(result["bytes"], 1_048_576);
20953 assert_eq!(result["recommended"], true);
20954 assert_eq!(result["pressure"], "continue_as_new_recommended");
20955 }
20956
20957 #[test]
20958 fn uncaught_workflow_handler_error_emits_terminal_failure_command() {
20959 let client = Client::new("http://127.0.0.1:8080").expect("client");
20960 let mut worker = Worker::new(client, "rust-workers");
20961 worker.register_workflow("rust.failing", |_ctx, _input| async move {
20962 Err(Error::Codec("rust_conformance_failure".to_string()))
20963 });
20964 let task = WorkflowTask {
20965 task_id: "wft-rust-failing-1".to_string(),
20966 workflow_command_id: None,
20967 workflow_id: Some("wf-rust-failing".to_string()),
20968 run_id: Some("run-rust-failing".to_string()),
20969 workflow_type: "rust.failing".to_string(),
20970 cancel_requested: false,
20971 payload_codec: DEFAULT_CODEC.to_string(),
20972 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
20973 history_events: Vec::new(),
20974 total_history_events: Some(0),
20975 history_size_bytes: None,
20976 continue_as_new_recommended: None,
20977 history_budget_pressure: None,
20978 next_history_page_token: None,
20979 workflow_task_attempt: 1,
20980 workflow_signal_id: None,
20981 signal_name: None,
20982 signal_arguments: None,
20983 workflow_update_id: None,
20984 update_name: None,
20985 lease_owner: Some("rust-worker".to_string()),
20986 };
20987
20988 let commands = worker
20989 .execute_workflow_task(task)
20990 .expect("handler failure becomes a workflow command");
20991
20992 assert_eq!(commands.len(), 1);
20993 assert_eq!(commands[0]["type"], "fail_workflow");
20994 assert_eq!(commands[0]["exception_type"], "RustWorkflowError");
20995 assert_eq!(commands[0]["exception_class"], "durable_workflow::Error");
20996 assert_eq!(commands[0]["non_retryable"], false);
20997 assert_eq!(
20998 commands[0]["message"],
20999 "codec error: rust_conformance_failure"
21000 );
21001 assert_eq!(
21002 commands[0]["exception"]["message"],
21003 "codec error: rust_conformance_failure"
21004 );
21005 }
21006
21007 #[test]
21008 fn ordinary_handler_error_preserves_commands_queued_in_the_same_decision() {
21009 let client = Client::new("http://127.0.0.1:8080").expect("client");
21010 let mut worker = Worker::new(client, "rust-workers");
21011 worker.register_workflow("rust.failing-after-side-effect", |ctx, _input| async move {
21012 let _: String = ctx.side_effect(|| "captured".to_string())?;
21013 Err(Error::WorkerLoop("application failure".to_string()))
21014 });
21015
21016 let commands = worker
21017 .execute_workflow_task(workflow_task(
21018 "rust.failing-after-side-effect",
21019 Vec::new(),
21020 DEFAULT_CODEC,
21021 ))
21022 .expect("ordinary failure remains a workflow decision");
21023
21024 assert_eq!(commands.len(), 2);
21025 assert_eq!(commands[0]["type"], "record_side_effect");
21026 assert_eq!(commands[1]["type"], "fail_workflow");
21027 }
21028
21029 #[test]
21030 fn handler_error_cannot_hide_an_unconsumed_committed_side_effect() {
21031 let client = Client::new("http://127.0.0.1:8080").expect("client");
21032 let mut worker = Worker::new(client, "rust-workers");
21033 worker.register_workflow("rust.removed-side-effect", |_ctx, _input| async move {
21034 Err(Error::WorkerLoop("application failure".to_string()))
21035 });
21036 let result =
21037 encode_value_envelope(&json!("committed"), DEFAULT_CODEC).expect("side-effect result");
21038
21039 let error = worker
21040 .execute_workflow_task(workflow_task(
21041 "rust.removed-side-effect",
21042 vec![history_event(
21043 "SideEffectRecorded",
21044 json!({"sequence": 1, "result": result}),
21045 )],
21046 DEFAULT_CODEC,
21047 ))
21048 .expect_err("removed committed history must not become fail_workflow");
21049
21050 let Error::NonDeterministicReplay(failure) = error else {
21051 panic!("expected typed replay failure");
21052 };
21053 assert_eq!(failure.reason, "recorded_commands_unconsumed");
21054 assert_eq!(failure.sequence, Some(1));
21055 assert_eq!(failure.expected.as_deref(), Some("side effect"));
21056 }
21057
21058 #[test]
21059 fn replay_error_discards_side_effect_queued_before_incompatible_marker_check() {
21060 let client = Client::new("http://127.0.0.1:8080").expect("client");
21061 let mut worker = Worker::new(client, "rust-workers");
21062 worker.register_workflow(
21063 "rust.side-effect-before-marker-error",
21064 |ctx, _input| async move {
21065 assert_eq!(ctx.get_version("restart-safe", 1, 1)?, 1);
21066 let _: String = ctx.side_effect(|| "must-not-commit".to_string())?;
21067 ctx.get_version("restart-safe", 2, 2)?;
21068 Ok(Value::Null)
21069 },
21070 );
21071
21072 let error = worker
21073 .execute_workflow_task(workflow_task(
21074 "rust.side-effect-before-marker-error",
21075 vec![history_event(
21076 "VersionMarkerRecorded",
21077 json!({
21078 "sequence": 1,
21079 "change_id": "restart-safe",
21080 "version": 1,
21081 "min_supported": 1,
21082 "max_supported": 1,
21083 }),
21084 )],
21085 DEFAULT_CODEC,
21086 ))
21087 .expect_err("replay error must return no queued workflow commands");
21088
21089 let Error::NonDeterministicReplay(failure) = error else {
21090 panic!("expected typed replay failure");
21091 };
21092 assert_eq!(failure.reason, "version_marker_incompatible_range");
21093 assert_eq!(failure.sequence, Some(1));
21094 }
21095
21096 #[test]
21097 fn workflow_task_replay_keeps_recorded_unfired_timer_pending_without_rescheduling() {
21098 let client = Client::new("http://127.0.0.1:8080").expect("client");
21099 let mut worker = Worker::new(client, "rust-workers");
21100 worker.register_workflow("rust.timer.pending", |ctx, _input| async move {
21101 ctx.sleep(Duration::from_secs(5)).await?;
21102 Ok(json!({"status": "timer fired"}))
21103 });
21104
21105 let task = WorkflowTask {
21106 task_id: "wft-rust-timer-pending".to_string(),
21107 workflow_command_id: None,
21108 workflow_id: Some("wf-rust-timer".to_string()),
21109 run_id: Some("run-rust-timer".to_string()),
21110 workflow_type: "rust.timer.pending".to_string(),
21111 cancel_requested: false,
21112 payload_codec: DEFAULT_CODEC.to_string(),
21113 arguments: Some(
21114 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
21115 ),
21116 history_events: vec![history_event(
21117 "TimerScheduled",
21118 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
21119 )],
21120 total_history_events: Some(1),
21121 history_size_bytes: None,
21122 continue_as_new_recommended: None,
21123 history_budget_pressure: None,
21124 next_history_page_token: None,
21125 workflow_task_attempt: 1,
21126 workflow_signal_id: None,
21127 signal_name: None,
21128 signal_arguments: None,
21129 workflow_update_id: None,
21130 update_name: None,
21131 lease_owner: Some("rust-worker".to_string()),
21132 };
21133
21134 for _redelivery_or_restart in 0..2 {
21135 let commands = worker
21136 .execute_workflow_task(task.clone())
21137 .expect("recorded timer remains pending");
21138 assert!(
21139 commands.is_empty(),
21140 "recorded timer must not be rescheduled"
21141 );
21142 }
21143 }
21144
21145 #[test]
21146 fn workflow_task_rejects_recorded_command_removed_from_workflow_code() {
21147 let client = Client::new("http://127.0.0.1:8080").expect("client");
21148 let mut worker = Worker::new(client, "rust-workers");
21149 worker.register_workflow("rust.timer.removed", |_ctx, _input| async move {
21150 Ok(json!({"status": "completed"}))
21151 });
21152 let task = WorkflowTask {
21153 task_id: "wft-rust-timer-removed".to_string(),
21154 workflow_command_id: None,
21155 workflow_id: Some("wf-rust-timer".to_string()),
21156 run_id: Some("run-rust-timer".to_string()),
21157 workflow_type: "rust.timer.removed".to_string(),
21158 cancel_requested: false,
21159 payload_codec: DEFAULT_CODEC.to_string(),
21160 arguments: Some(
21161 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
21162 ),
21163 history_events: vec![
21164 history_event(
21165 "TimerScheduled",
21166 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
21167 ),
21168 history_event(
21169 "TimerFired",
21170 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
21171 ),
21172 ],
21173 total_history_events: Some(2),
21174 history_size_bytes: None,
21175 continue_as_new_recommended: None,
21176 history_budget_pressure: None,
21177 next_history_page_token: None,
21178 workflow_task_attempt: 1,
21179 workflow_signal_id: None,
21180 signal_name: None,
21181 signal_arguments: None,
21182 workflow_update_id: None,
21183 update_name: None,
21184 lease_owner: Some("rust-worker".to_string()),
21185 };
21186
21187 let Error::NonDeterministicReplay(failure) = worker
21188 .execute_workflow_task(task)
21189 .expect_err("removed timer must fail replay")
21190 else {
21191 panic!("expected typed replay failure");
21192 };
21193 assert_eq!(failure.reason, "recorded_commands_unconsumed");
21194 assert_eq!(failure.sequence, Some(1));
21195 }
21196
21197 #[test]
21198 fn workflow_context_emits_explicit_child_workflow_contract() {
21199 let ctx = WorkflowContext {
21200 state: Arc::new(Mutex::new(
21201 WorkflowState::new_with_identity(
21202 Vec::new(),
21203 Some("wf-parent".to_string()),
21204 Some("run-parent".to_string()),
21205 "parent-workers".to_string(),
21206 DEFAULT_CODEC.to_string(),
21207 None,
21208 )
21209 .expect("workflow state"),
21210 )),
21211 };
21212 let options = ChildWorkflowOptions::new("python-workers")
21213 .parent_close_policy(ParentClosePolicy::RequestCancel)
21214 .retry_policy(ChildWorkflowRetryPolicy {
21215 max_attempts: Some(3),
21216 backoff_seconds: vec![1, 5],
21217 non_retryable_error_types: vec!["ValidationError".to_string()],
21218 })
21219 .execution_timeout_seconds(600)
21220 .run_timeout_seconds(120);
21221 let mut call = Box::pin(ctx.start_child_workflow(
21222 "python.fulfil-order",
21223 options,
21224 json!([{"order_id": "order-42"}]),
21225 ));
21226 let mut task_context = TaskContext::from_waker(noop_waker_ref());
21227
21228 assert!(matches!(
21229 call.as_mut().poll(&mut task_context),
21230 Poll::Pending
21231 ));
21232 let commands = ctx.take_commands().expect("commands");
21233 assert_eq!(commands.len(), 1);
21234 let command = &commands[0];
21235 assert_eq!(command["type"], "start_child_workflow");
21236 assert_eq!(command["workflow_type"], "python.fulfil-order");
21237 assert_eq!(command["queue"], "python-workers");
21238 assert_eq!(command["parent_close_policy"], "request_cancel");
21239 assert_eq!(command["retry_policy"]["max_attempts"], 3);
21240 assert_eq!(command["execution_timeout_seconds"], 600);
21241 assert_eq!(command["run_timeout_seconds"], 120);
21242 assert_eq!(
21243 decode_wire_value(&command["arguments"], DEFAULT_CODEC).expect("child args"),
21244 json!([{"order_id": "order-42"}])
21245 );
21246 }
21247
21248 fn child_parent_worker() -> Worker {
21249 let client = Client::new("http://127.0.0.1:8080").expect("client");
21250 let mut worker = Worker::new(client, "rust-parent-workers");
21251 worker.register_workflow("rust.parent", |ctx, _input| async move {
21252 let child = ctx
21253 .start_child_workflow(
21254 "python.child",
21255 ChildWorkflowOptions::new("python-child-workers")
21256 .parent_close_policy(ParentClosePolicy::Terminate),
21257 json!([{"codec_probe": [1, true, "rust"]}]),
21258 )
21259 .await?;
21260 Ok(json!({
21261 "parent_workflow_id": child.parent.workflow_id,
21262 "parent_run_id": child.parent.run_id,
21263 "child_workflow_id": child.child.workflow_id,
21264 "child_run_id": child.child.run_id,
21265 "child_workflow_type": child.child_workflow_type,
21266 "result": child.result,
21267 }))
21268 });
21269 worker
21270 }
21271
21272 fn child_parent_task(event_type: &str, payload: Value) -> WorkflowTask {
21273 WorkflowTask {
21274 task_id: "wft-child-parent".to_string(),
21275 workflow_command_id: None,
21276 workflow_id: Some("wf-parent".to_string()),
21277 run_id: Some("run-parent".to_string()),
21278 workflow_type: "rust.parent".to_string(),
21279 cancel_requested: false,
21280 payload_codec: DEFAULT_CODEC.to_string(),
21281 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
21282 history_events: vec![
21283 HistoryEvent {
21284 event_type: "ChildWorkflowScheduled".to_string(),
21285 payload: json!({
21286 "sequence": 1,
21287 "child_call_id": "call-child",
21288 "child_workflow_instance_id": "wf-child",
21289 "child_workflow_run_id": "run-child",
21290 "child_workflow_type": "python.child",
21291 }),
21292 raw: HashMap::new(),
21293 },
21294 HistoryEvent {
21295 event_type: event_type.to_string(),
21296 payload,
21297 raw: HashMap::new(),
21298 },
21299 ],
21300 total_history_events: Some(2),
21301 history_size_bytes: None,
21302 continue_as_new_recommended: None,
21303 history_budget_pressure: None,
21304 next_history_page_token: None,
21305 workflow_task_attempt: 1,
21306 workflow_signal_id: None,
21307 signal_name: None,
21308 signal_arguments: None,
21309 workflow_update_id: None,
21310 update_name: None,
21311 lease_owner: Some("rust-worker".to_string()),
21312 }
21313 }
21314
21315 #[test]
21316 fn committed_child_result_replays_without_starting_a_duplicate() {
21317 let worker = child_parent_worker();
21318 let task = child_parent_task(
21319 "ChildRunCompleted",
21320 json!({
21321 "sequence": 1,
21322 "child_call_id": "call-child",
21323 "child_workflow_instance_id": "wf-child",
21324 "child_workflow_run_id": "run-child",
21325 "child_workflow_type": "python.child",
21326 "payload_codec": DEFAULT_CODEC,
21327 "result": fixture_envelope(json!({"from":"python","ok":true})),
21328 }),
21329 );
21330
21331 for _restart in 0..2 {
21332 let commands = worker
21333 .execute_workflow_task(task.clone())
21334 .expect("replayed parent task");
21335 assert_eq!(commands.len(), 1);
21336 assert_eq!(commands[0]["type"], "complete_workflow");
21337 assert!(!commands
21338 .iter()
21339 .any(|command| command["type"] == "start_child_workflow"));
21340 let output =
21341 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("parent output");
21342 assert_eq!(output["parent_workflow_id"], "wf-parent");
21343 assert_eq!(output["parent_run_id"], "run-parent");
21344 assert_eq!(output["child_workflow_id"], "wf-child");
21345 assert_eq!(output["child_run_id"], "run-child");
21346 assert_eq!(output["result"], json!({"from": "python", "ok": true}));
21347 }
21348 }
21349
21350 #[test]
21351 fn typed_child_arguments_and_results_survive_replay() {
21352 let client = Client::new("http://127.0.0.1:8080").expect("client");
21353 let mut worker = Worker::new(client, "rust-parent-workers");
21354 worker.register_workflow_avro_value("rust.typed-parent", |ctx, _input| async move {
21355 let child = ctx
21356 .start_child_workflow_avro_value(
21357 "python.typed-child",
21358 ChildWorkflowOptions::new("python-workers"),
21359 AvroValue::Array(vec![typed_fidelity_probe()]),
21360 )
21361 .await?;
21362 Ok(child.result)
21363 });
21364
21365 let initial = worker
21366 .execute_workflow_task(workflow_task(
21367 "rust.typed-parent",
21368 Vec::new(),
21369 DEFAULT_CODEC,
21370 ))
21371 .expect("typed child start");
21372 assert_eq!(initial[0]["type"], "start_child_workflow");
21373 assert_eq!(
21374 decode_wire_avro_value(&initial[0]["arguments"], DEFAULT_CODEC)
21375 .expect("typed child arguments"),
21376 AvroValue::Array(vec![typed_fidelity_probe()])
21377 );
21378
21379 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
21380 .expect("typed child result");
21381 let task = workflow_task(
21382 "rust.typed-parent",
21383 vec![
21384 history_event(
21385 "ChildWorkflowScheduled",
21386 json!({
21387 "sequence": 1,
21388 "child_call_id": "call-typed",
21389 "child_workflow_instance_id": "wf-child",
21390 "child_workflow_run_id": "run-child",
21391 "child_workflow_type": "python.typed-child",
21392 }),
21393 ),
21394 history_event(
21395 "ChildRunCompleted",
21396 json!({
21397 "sequence": 1,
21398 "child_call_id": "call-typed",
21399 "child_workflow_instance_id": "wf-child",
21400 "child_workflow_run_id": "run-child",
21401 "child_workflow_type": "python.typed-child",
21402 "payload_codec": DEFAULT_CODEC,
21403 "result": result,
21404 }),
21405 ),
21406 ],
21407 DEFAULT_CODEC,
21408 );
21409
21410 let commands = worker
21411 .execute_workflow_task(task)
21412 .expect("typed child replay");
21413 assert_eq!(commands[0]["type"], "complete_workflow");
21414 assert_eq!(
21415 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
21416 .expect("typed parent result"),
21417 typed_fidelity_probe()
21418 );
21419 }
21420
21421 #[test]
21422 fn pending_child_replays_after_restart_without_starting_a_duplicate() {
21423 let worker = child_parent_worker();
21424 let mut task = child_parent_task("unused", Value::Null);
21425 task.history_events.truncate(1);
21426 task.total_history_events = Some(1);
21427
21428 for _redelivery_or_restart in 0..2 {
21429 let commands = worker
21430 .execute_workflow_task(task.clone())
21431 .expect("recorded child remains pending");
21432 assert!(
21433 commands.is_empty(),
21434 "recorded pending child must not be started again"
21435 );
21436 }
21437 }
21438
21439 #[test]
21440 fn child_cancellation_becomes_stable_parent_failure_command() {
21441 let worker = child_parent_worker();
21442 let task = child_parent_task(
21443 "ChildRunCancelled",
21444 json!({
21445 "sequence": 1,
21446 "child_workflow_instance_id": "wf-child",
21447 "child_workflow_run_id": "run-child",
21448 "child_workflow_type": "python.child",
21449 "failure_id": "failure-child",
21450 "failure_category": "cancelled",
21451 "message": "cancelled by parent-close policy",
21452 }),
21453 );
21454
21455 let commands = worker
21456 .execute_workflow_task(task)
21457 .expect("parent settlement");
21458 assert_eq!(commands.len(), 1);
21459 assert_eq!(commands[0]["type"], "fail_workflow");
21460 assert_eq!(commands[0]["exception_type"], "ChildWorkflowCancelled");
21461 assert_eq!(
21462 commands[0]["exception"]["properties"]["reason"],
21463 "cancelled"
21464 );
21465 assert_eq!(
21466 commands[0]["exception"]["properties"]["child_workflow_run_id"],
21467 "run-child"
21468 );
21469 }
21470
21471 #[test]
21472 fn workflow_can_handle_typed_child_failure() {
21473 let client = Client::new("http://127.0.0.1:8080").expect("client");
21474 let mut worker = Worker::new(client, "rust-parent-workers");
21475 worker.register_workflow("rust.handled-parent", |ctx, _input| async move {
21476 match ctx
21477 .start_child_workflow(
21478 "python.child",
21479 ChildWorkflowOptions::new("python-child-workers"),
21480 json!([]),
21481 )
21482 .await
21483 {
21484 Err(Error::ChildWorkflowFailed(failure)) => Ok(json!({
21485 "reason": failure.reason,
21486 "failure_id": failure.failure_id,
21487 "exception_class": failure.exception_class,
21488 "child_run_id": failure.child_workflow_run_id,
21489 })),
21490 Err(error) => Err(error),
21491 Ok(_) => Err(Error::WorkerLoop(
21492 "child unexpectedly succeeded".to_string(),
21493 )),
21494 }
21495 });
21496 let mut task = child_parent_task(
21497 "ChildRunFailed",
21498 json!({
21499 "sequence": 1,
21500 "child_workflow_instance_id": "wf-child",
21501 "child_workflow_run_id": "run-child",
21502 "child_workflow_type": "python.child",
21503 "failure_id": "failure-child",
21504 "failure_category": "child_workflow",
21505 "message": "payment rejected",
21506 "exception": {
21507 "type": "PaymentRejected",
21508 "class": "payments.PaymentRejected",
21509 "message": "payment rejected"
21510 }
21511 }),
21512 );
21513 task.workflow_type = "rust.handled-parent".to_string();
21514
21515 let commands = worker.execute_workflow_task(task).expect("handled failure");
21516 assert_eq!(commands[0]["type"], "complete_workflow");
21517 let output =
21518 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("parent output");
21519 assert_eq!(output["reason"], "child_workflow");
21520 assert_eq!(output["failure_id"], "failure-child");
21521 assert_eq!(output["exception_class"], "payments.PaymentRejected");
21522 assert_eq!(output["child_run_id"], "run-child");
21523 }
21524
21525 #[test]
21526 fn rust_hello_world_uses_signal_arguments_from_resume_payload() {
21527 let client = Client::new("http://127.0.0.1:8080").expect("client");
21528 let mut worker = Worker::new(client, "rust-workers");
21529
21530 worker.register_workflow("rust.hello_workflow", |ctx, _input| async move {
21531 let signal = ctx.wait_signal("start").await?;
21532 let name = signal
21533 .first()
21534 .and_then(|value| value.as_str())
21535 .unwrap_or("world");
21536 let greeting = ctx.activity("rust.hello_activity", json!([name])).await?;
21537 Ok(json!({
21538 "greeting": greeting,
21539 "language": "rust"
21540 }))
21541 });
21542
21543 let signal_arguments =
21544 encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC).expect("signal arguments");
21545 let task = WorkflowTask {
21546 task_id: "wft-rust-signal-1".to_string(),
21547 workflow_command_id: None,
21548 workflow_id: Some("wf-rust-hello".to_string()),
21549 run_id: Some("run-rust-hello".to_string()),
21550 workflow_type: "rust.hello_workflow".to_string(),
21551 cancel_requested: false,
21552 payload_codec: DEFAULT_CODEC.to_string(),
21553 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
21554 history_events: vec![HistoryEvent {
21555 event_type: "SignalReceived".to_string(),
21556 payload: json!({
21557 "signal_id": "sig-rust-1",
21558 "signal_name": "start"
21559 }),
21560 raw: HashMap::new(),
21561 }],
21562 total_history_events: Some(1),
21563 history_size_bytes: None,
21564 continue_as_new_recommended: None,
21565 history_budget_pressure: None,
21566 next_history_page_token: None,
21567 workflow_task_attempt: 1,
21568 workflow_signal_id: Some("sig-rust-1".to_string()),
21569 signal_name: Some("start".to_string()),
21570 signal_arguments: Some(signal_arguments),
21571 workflow_update_id: None,
21572 update_name: None,
21573 lease_owner: Some("rust-worker".to_string()),
21574 };
21575
21576 let commands = worker.execute_workflow_task(task).expect("workflow task");
21577
21578 assert_eq!(commands.len(), 1);
21579 assert_eq!(commands[0]["type"], "schedule_activity");
21580 assert_eq!(commands[0]["activity_type"], "rust.hello_activity");
21581 assert_eq!(
21582 decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC).expect("activity args"),
21583 json!(["Rust"])
21584 );
21585 }
21586
21587 #[test]
21588 fn workflow_task_appends_paginated_history_events() {
21589 let mut task = WorkflowTask {
21590 task_id: "wft-rust-pages-1".to_string(),
21591 workflow_command_id: None,
21592 workflow_id: Some("wf-rust-pages".to_string()),
21593 run_id: Some("run-rust-pages".to_string()),
21594 workflow_type: "rust.hello_workflow".to_string(),
21595 cancel_requested: false,
21596 payload_codec: DEFAULT_CODEC.to_string(),
21597 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
21598 history_events: vec![HistoryEvent {
21599 event_type: "WorkflowStarted".to_string(),
21600 payload: json!({}),
21601 raw: HashMap::new(),
21602 }],
21603 total_history_events: Some(3),
21604 history_size_bytes: None,
21605 continue_as_new_recommended: None,
21606 history_budget_pressure: None,
21607 next_history_page_token: Some("MQ==".to_string()),
21608 workflow_task_attempt: 1,
21609 workflow_signal_id: None,
21610 signal_name: None,
21611 signal_arguments: None,
21612 workflow_update_id: None,
21613 update_name: None,
21614 lease_owner: Some("rust-worker".to_string()),
21615 };
21616
21617 task.append_history_page(WorkflowTaskHistoryPage {
21618 history_events: vec![
21619 HistoryEvent {
21620 event_type: "SignalReceived".to_string(),
21621 payload: json!({
21622 "signal_id": "sig-rust-1",
21623 "signal_name": "start",
21624 "arguments": encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC)
21625 .expect("signal arguments")
21626 }),
21627 raw: HashMap::new(),
21628 },
21629 HistoryEvent {
21630 event_type: "MarkerRecorded".to_string(),
21631 payload: json!({"sequence": 3}),
21632 raw: HashMap::new(),
21633 },
21634 ],
21635 total_history_events: Some(3),
21636 next_history_page_token: None,
21637 });
21638
21639 assert_eq!(task.history_events.len(), 3);
21640 assert_eq!(task.total_history_events, Some(3));
21641 assert_eq!(task.next_history_page_token, None);
21642
21643 let signal = task
21644 .history_events
21645 .iter()
21646 .find(|event| event.event_type == "SignalReceived")
21647 .expect("signal event");
21648 assert_eq!(
21649 decode_signal_event_arguments(signal, DEFAULT_CODEC).expect("signal arguments"),
21650 vec![AvroValue::String("Rust".to_string())]
21651 );
21652 }
21653
21654 #[tokio::test]
21655 async fn query_handler_reads_ordered_cross_codec_signals_without_commands() {
21656 let client = Client::new("http://127.0.0.1:8080").expect("client");
21657 let mut worker = Worker::new(client, "rust-workers");
21658 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
21659 worker.register_query("counter", "current", |ctx, _args| async move {
21660 let mut count = 0_i64;
21661 for signal in ctx.signal_events() {
21662 let value = signal
21663 .arguments
21664 .first()
21665 .and_then(Value::as_i64)
21666 .unwrap_or_default();
21667 match signal.name.as_str() {
21668 "increment" => count += value,
21669 "set" => count = value,
21670 _ => {}
21671 }
21672 }
21673 Ok(json!(count))
21674 });
21675
21676 let task = QueryTask {
21677 query_task_id: "query-rust-counter".to_string(),
21678 query_task_attempt: 1,
21679 lease_owner: Some("rust-worker".to_string()),
21680 workflow_id: Some("counter-1".to_string()),
21681 run_id: Some("run-counter-1".to_string()),
21682 workflow_type: "counter".to_string(),
21683 query_name: "current".to_string(),
21684 payload_codec: DEFAULT_CODEC.to_string(),
21685 workflow_arguments: Some(
21686 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
21687 ),
21688 query_arguments: Some(
21689 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("query arguments"),
21690 ),
21691 history_events: vec![
21692 HistoryEvent {
21693 event_type: "SignalReceived".to_string(),
21694 payload: json!({
21695 "signal_id": "php-signal-1",
21696 "signal_name": "increment",
21697 "workflow_sequence": 1,
21698 "payload_codec": DEFAULT_CODEC,
21699 "arguments": encode_value_envelope(&json!([3]), DEFAULT_CODEC).expect("php avro signal")
21700 }),
21701 raw: HashMap::new(),
21702 },
21703 HistoryEvent {
21704 event_type: "SignalReceived".to_string(),
21705 payload: json!({
21706 "signal_id": "python-signal-2",
21707 "signal_name": "increment",
21708 "workflow_sequence": 2,
21709 "payload_codec": DEFAULT_CODEC,
21710 "arguments": encode_value_envelope(&json!([5]), DEFAULT_CODEC).expect("python avro signal")
21711 }),
21712 raw: HashMap::new(),
21713 },
21714 HistoryEvent {
21715 event_type: "SignalReceived".to_string(),
21716 payload: json!({
21717 "signal_id": "rust-signal-3",
21718 "signal_name": "set",
21719 "workflow_sequence": 3,
21720 "payload_codec": DEFAULT_CODEC,
21721 "arguments": encode_value_envelope(&json!([0]), DEFAULT_CODEC).expect("rust avro signal")
21722 }),
21723 raw: HashMap::new(),
21724 },
21725 ],
21726 history_export: None,
21727 run_status: Some("completed".to_string()),
21728 };
21729
21730 let result = worker.execute_query_task(task).await.expect("query result");
21731 assert_eq!(result.into_json().expect("query projection"), json!(0));
21732 }
21733
21734 #[tokio::test]
21735 async fn replayed_queries_read_running_completed_and_cold_restarted_instance_state() {
21736 let worker = replay_counter_worker();
21737 let running_history = json!([
21738 {
21739 "type": "ActivityCompleted",
21740 "payload": {
21741 "sequence": 1,
21742 "activity_type": "load-counter",
21743 "payload_codec": DEFAULT_CODEC,
21744 "result": fixture_envelope(json!("loaded"))
21745 }
21746 },
21747 {
21748 "type": "SignalWaitOpened",
21749 "payload": {
21750 "sequence": 3,
21751 "signal_name": "increment"
21752 }
21753 },
21754 {
21755 "type": "SignalReceived",
21756 "payload": {
21757 "signal_id": "signal-3",
21758 "signal_name": "increment",
21759 "workflow_sequence": 2,
21760 "payload_codec": DEFAULT_CODEC,
21761 "arguments": fixture_envelope(json!([3]))
21762 }
21763 },
21764 {
21765 "type": "SignalApplied",
21766 "payload": {
21767 "sequence": 3,
21768 "signal_id": "signal-3",
21769 "signal_name": "increment",
21770 "payload_codec": DEFAULT_CODEC,
21771 "value": fixture_envelope(json!([3]))
21772 }
21773 }
21774 ]);
21775
21776 let running = worker
21777 .execute_query_task(replay_counter_query(
21778 "current",
21779 running_history.clone(),
21780 "running",
21781 ))
21782 .await
21783 .expect("running replay query");
21784 assert_eq!(
21785 running.clone().into_json().expect("query projection"),
21786 json!({"loaded": "loaded", "count": 3, "finished": false})
21787 );
21788
21789 let detached = worker
21790 .execute_query_task(replay_counter_query(
21791 "detached-mutation",
21792 running_history.clone(),
21793 "running",
21794 ))
21795 .await
21796 .expect("query mutates only its detached state clone");
21797 assert_eq!(detached.into_json().expect("query projection"), json!(999));
21798 let failed = worker
21799 .execute_query_task(replay_counter_query(
21800 "failed-mutation",
21801 running_history.clone(),
21802 "running",
21803 ))
21804 .await
21805 .expect_err("failed query");
21806 assert_eq!(failed.reason, "query_rejected");
21807 let unchanged = worker
21808 .execute_query_task(replay_counter_query("current", running_history, "running"))
21809 .await
21810 .expect("later query reconstructs unchanged state");
21811 assert_eq!(unchanged, running);
21812
21813 let restarted_worker = replay_counter_worker();
21814 let empty_arguments = fixture_envelope(json!([]));
21815 let loaded_result = fixture_envelope(json!("loaded"));
21816 let signal_three = fixture_blob(json!([3]));
21817 let signal_five = fixture_blob(json!([5]));
21818 let restarted_task: QueryTask = serde_json::from_value(json!({
21819 "query_task_id": "query-after-restart",
21820 "workflow_id": "counter-1",
21821 "run_id": "run-counter-1",
21822 "workflow_type": "replay-counter",
21823 "query_name": "current",
21824 "payload_codec": DEFAULT_CODEC,
21825 "workflow_arguments": empty_arguments.clone(),
21826 "query_arguments": empty_arguments,
21827 "history_events": [],
21828 "history_export": {
21829 "payloads": {"codec": DEFAULT_CODEC},
21830 "history_events": [
21831 {
21832 "type": "ActivityCompleted",
21833 "payload": {
21834 "sequence": 1,
21835 "activity_type": "load-counter",
21836 "payload_codec": DEFAULT_CODEC,
21837 "result": null
21838 }
21839 },
21840 {
21841 "type": "SignalWaitOpened",
21842 "payload": {
21843 "sequence": 3,
21844 "signal_name": "increment"
21845 }
21846 },
21847 {
21848 "type": "SignalReceived",
21849 "payload": {
21850 "signal_id": "signal-3",
21851 "signal_name": "increment",
21852 "workflow_sequence": 2
21853 }
21854 },
21855 {
21856 "type": "SignalApplied",
21857 "payload": {
21858 "sequence": 3,
21859 "signal_id": "signal-3",
21860 "signal_name": "increment"
21861 }
21862 },
21863 {
21864 "type": "SignalWaitOpened",
21865 "payload": {
21866 "sequence": 5,
21867 "signal_name": "increment"
21868 }
21869 },
21870 {
21871 "type": "SignalReceived",
21872 "payload": {
21873 "signal_id": "signal-5",
21874 "signal_name": "increment",
21875 "workflow_sequence": 4
21876 }
21877 },
21878 {
21879 "type": "SignalApplied",
21880 "payload": {
21881 "sequence": 5,
21882 "signal_id": "signal-5",
21883 "signal_name": "increment"
21884 }
21885 }
21886 ],
21887 "activities": [{
21888 "sequence": 1,
21889 "activity_type": "load-counter",
21890 "payload_codec": DEFAULT_CODEC,
21891 "result": loaded_result
21892 }],
21893 "signals": [
21894 {
21895 "id": "signal-3",
21896 "name": "increment",
21897 "workflow_sequence": 2,
21898 "payload_codec": DEFAULT_CODEC,
21899 "arguments": signal_three
21900 },
21901 {
21902 "id": "signal-5",
21903 "name": "increment",
21904 "workflow_sequence": 4,
21905 "payload_codec": DEFAULT_CODEC,
21906 "arguments": signal_five
21907 }
21908 ]
21909 },
21910 "run_status": "completed"
21911 }))
21912 .expect("cold replay query task");
21913 let completed = restarted_worker
21914 .execute_query_task(restarted_task)
21915 .await
21916 .expect("completed cold replay query");
21917 assert_eq!(
21918 completed.into_json().expect("query projection"),
21919 json!({"loaded": "loaded", "count": 8, "finished": true})
21920 );
21921 }
21922
21923 #[tokio::test]
21924 async fn replayed_query_replay_failures_are_machine_readable() {
21925 let worker = replay_counter_worker();
21926 let task = replay_counter_query(
21927 "current",
21928 json!([{
21929 "type": "ActivityCompleted",
21930 "payload": {
21931 "sequence": 1,
21932 "payload_codec": DEFAULT_CODEC,
21933 "result": {"codec": DEFAULT_CODEC, "blob": "{"}
21934 }
21935 }]),
21936 "running",
21937 );
21938 let failure = worker
21939 .execute_query_task(task)
21940 .await
21941 .expect_err("invalid replay history payload");
21942 assert_eq!(failure.reason, "query_payload_decode_failed");
21943 assert_eq!(failure.failure_type, "QueryPayloadDecodeFailed");
21944 assert!(failure.message.contains("invalid_payload_framing"));
21945 }
21946
21947 #[tokio::test]
21948 async fn query_task_restores_compact_history_from_export() {
21949 let client = Client::new("http://127.0.0.1:8080").expect("client");
21950 let mut worker = Worker::new(client, "rust-workers");
21951 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
21952 worker.register_query("counter", "current", |ctx, _args| async move {
21953 Ok(json!(ctx.signals("increment")[0][0]))
21954 });
21955 let empty_arguments = fixture_envelope(json!([]));
21956 let exported_signal = fixture_blob(json!([9]));
21957 let task: QueryTask = serde_json::from_value(json!({
21958 "query_task_id": "query-export",
21959 "workflow_type": "counter",
21960 "query_name": "current",
21961 "payload_codec": DEFAULT_CODEC,
21962 "workflow_arguments": empty_arguments.clone(),
21963 "query_arguments": empty_arguments,
21964 "history_events": [],
21965 "history_export": {
21966 "payloads": {"codec": DEFAULT_CODEC},
21967 "history_events": [{
21968 "type": "SignalReceived",
21969 "payload": {"signal_id": "signal-export", "signal_name": "increment"}
21970 }],
21971 "signals": [{
21972 "id": "signal-export",
21973 "name": "increment",
21974 "status": "applied",
21975 "workflow_sequence": 1,
21976 "payload_codec": DEFAULT_CODEC,
21977 "arguments": exported_signal
21978 }]
21979 }
21980 }))
21981 .expect("query task");
21982
21983 let result = worker.execute_query_task(task).await.expect("query result");
21984 assert_eq!(result.into_json().expect("query projection"), json!(9));
21985 }
21986
21987 #[tokio::test]
21988 async fn query_task_failures_have_stable_reasons() {
21989 let client = Client::new("http://127.0.0.1:8080").expect("client");
21990 let mut worker = Worker::new(client, "rust-workers");
21991 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
21992 worker.register_query(
21993 "counter",
21994 "current",
21995 |_ctx, _args| async move { Ok(json!(0)) },
21996 );
21997
21998 let base_task = QueryTask {
21999 query_task_id: "query-errors".to_string(),
22000 query_task_attempt: 1,
22001 lease_owner: None,
22002 workflow_id: Some("counter-errors".to_string()),
22003 run_id: Some("run-errors".to_string()),
22004 workflow_type: "counter".to_string(),
22005 query_name: "missing".to_string(),
22006 payload_codec: DEFAULT_CODEC.to_string(),
22007 workflow_arguments: Some(fixture_envelope(json!([]))),
22008 query_arguments: Some(fixture_envelope(json!([]))),
22009 history_events: Vec::new(),
22010 history_export: None,
22011 run_status: Some("running".to_string()),
22012 };
22013
22014 let unknown = worker
22015 .execute_query_task(base_task.clone())
22016 .await
22017 .expect_err("unknown query");
22018 assert_eq!(unknown.reason, "rejected_unknown_query");
22019
22020 let mut malformed = base_task;
22021 malformed.query_name = "current".to_string();
22022 malformed.query_arguments = Some(json!({"codec": DEFAULT_CODEC, "blob": "{"}));
22023 let malformed = worker
22024 .execute_query_task(malformed)
22025 .await
22026 .expect_err("malformed payload");
22027 assert_eq!(malformed.reason, "query_payload_decode_failed");
22028
22029 let client = Client::new("http://127.0.0.1:8080").expect("client");
22030 let mut unavailable_worker = Worker::new(client, "rust-workers");
22031 unavailable_worker
22032 .register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
22033 let empty_arguments = fixture_envelope(json!([]));
22034 let unavailable_task: QueryTask = serde_json::from_value(json!({
22035 "query_task_id": "query-unavailable",
22036 "workflow_type": "counter",
22037 "query_name": "current",
22038 "payload_codec": DEFAULT_CODEC,
22039 "workflow_arguments": empty_arguments.clone(),
22040 "query_arguments": empty_arguments
22041 }))
22042 .expect("query task");
22043 let unavailable = unavailable_worker
22044 .execute_query_task(unavailable_task)
22045 .await
22046 .expect_err("query handler unavailable");
22047 assert_eq!(unavailable.reason, "query_handler_unavailable");
22048 }
22049
22050 #[tokio::test]
22051 async fn client_query_decodes_result_and_typed_failure() {
22052 let server = MockWorkerServer::start();
22053 let client = Client::builder(server.base_url())
22054 .timeout(Duration::from_secs(2))
22055 .build()
22056 .expect("client");
22057
22058 let result = client
22059 .query_workflow("counter-1", "current", json!([]))
22060 .await
22061 .expect("query result");
22062 assert_eq!(result, json!({"count": 8}));
22063
22064 let error = client
22065 .query_workflow("counter-1", "missing", json!([]))
22066 .await
22067 .expect_err("unknown query");
22068 let Error::QueryFailed(failure) = error else {
22069 panic!("expected typed query failure");
22070 };
22071 assert_eq!(failure.status, 404);
22072 assert_eq!(failure.reason, "rejected_unknown_query");
22073 }
22074
22075 #[tokio::test]
22076 async fn public_client_surfaces_send_and_receive_lossless_avro_values() {
22077 let server = MockWorkerServer::start();
22078 let client = Client::builder(server.base_url())
22079 .timeout(Duration::from_secs(2))
22080 .build()
22081 .expect("client");
22082 let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
22083
22084 client
22085 .start_workflow(
22086 "typed.echo",
22087 "rust-workers",
22088 "typed-start",
22089 arguments.clone(),
22090 )
22091 .await
22092 .expect("typed workflow start");
22093 assert_eq!(
22094 decode_wire_avro_value(
22095 &server.request_body("/api/workflows")["input"],
22096 DEFAULT_CODEC,
22097 )
22098 .expect("typed start input"),
22099 arguments
22100 );
22101
22102 client
22103 .signal_workflow("typed-1", "changed", arguments.clone())
22104 .await
22105 .expect("typed signal");
22106 assert_eq!(
22107 decode_wire_avro_value(
22108 &server.request_body("/api/workflows/typed-1/signal/changed")["input"],
22109 DEFAULT_CODEC,
22110 )
22111 .expect("typed signal input"),
22112 arguments
22113 );
22114
22115 assert_eq!(
22116 client
22117 .query_workflow_avro_value("typed-1", "inspect", arguments.clone())
22118 .await
22119 .expect("typed query"),
22120 typed_fidelity_probe()
22121 );
22122 assert_eq!(
22123 decode_wire_avro_value(
22124 &server.request_body("/api/workflows/typed-1/query/inspect")["input"],
22125 DEFAULT_CODEC,
22126 )
22127 .expect("typed query input"),
22128 arguments
22129 );
22130
22131 assert_eq!(
22132 client
22133 .update_workflow_avro_value(
22134 "typed-1",
22135 "replace",
22136 arguments.clone(),
22137 Some("typed-request"),
22138 )
22139 .await
22140 .expect("typed update"),
22141 typed_fidelity_probe()
22142 );
22143 let update = server.request_body("/api/workflows/typed-1/update/replace");
22144 assert_eq!(update["request_id"], "typed-request");
22145 assert_eq!(
22146 decode_wire_avro_value(&update["input"], DEFAULT_CODEC).expect("typed update input"),
22147 arguments
22148 );
22149
22150 let handle = WorkflowHandle {
22151 client: client.clone(),
22152 workflow_id: "typed-1".to_string(),
22153 run_id: Some("run-typed-1".to_string()),
22154 workflow_type: "typed.echo".to_string(),
22155 };
22156 assert_eq!(
22157 handle
22158 .result_avro_value(WorkflowResultOptions::default())
22159 .await
22160 .expect("typed workflow result"),
22161 typed_fidelity_probe()
22162 );
22163
22164 client
22165 .complete_activity_task(
22166 "activity-typed",
22167 "attempt-typed",
22168 "rust-worker",
22169 typed_fidelity_probe(),
22170 DEFAULT_CODEC,
22171 )
22172 .await
22173 .expect("typed activity completion");
22174 assert_eq!(
22175 decode_wire_avro_value(
22176 &server.request_body("/api/worker/activity-tasks/activity-typed/complete")
22177 ["result"],
22178 DEFAULT_CODEC,
22179 )
22180 .expect("typed activity result"),
22181 typed_fidelity_probe()
22182 );
22183 client
22184 .fail_activity_task(
22185 "activity-typed",
22186 "attempt-typed",
22187 "rust-worker",
22188 "typed failure",
22189 true,
22190 )
22191 .await
22192 .expect("activity failure");
22193 }
22194
22195 #[tokio::test]
22196 async fn lifecycle_commands_support_instance_and_selected_run_targets() {
22197 let server = MockWorkerServer::start();
22198 let client = Client::builder(server.base_url())
22199 .timeout(Duration::from_secs(2))
22200 .build()
22201 .expect("client");
22202
22203 let options = WorkflowCommandOptions::new()
22204 .reason("cleanup requested")
22205 .request_id("cancel-17");
22206 let cancelled = client
22207 .cancel_workflow("wf-lifecycle", options)
22208 .await
22209 .expect("instance cancellation");
22210 assert_eq!(cancelled.command, WorkflowCommandKind::Cancel);
22211 assert_eq!(cancelled.run_id.as_deref(), Some("run-current"));
22212 assert_eq!(cancelled.outcome.as_deref(), Some("cancelled"));
22213 assert_eq!(
22214 server.request_body("/api/workflows/wf-lifecycle/cancel"),
22215 json!({"reason":"cleanup requested","request_id":"cancel-17"})
22216 );
22217
22218 let terminated = client
22219 .terminate_workflow(
22220 "wf-lifecycle",
22221 WorkflowCommandOptions::new().reason("forced stop"),
22222 )
22223 .await
22224 .expect("instance termination");
22225 assert_eq!(terminated.command, WorkflowCommandKind::Terminate);
22226 assert_eq!(terminated.outcome.as_deref(), Some("terminated"));
22227
22228 client
22229 .cancel_workflow_run(
22230 "wf-lifecycle",
22231 "run-current",
22232 WorkflowCommandOptions::default(),
22233 )
22234 .await
22235 .expect("selected run cancellation");
22236 client
22237 .terminate_workflow_run(
22238 "wf-lifecycle",
22239 "run-current",
22240 WorkflowCommandOptions::default(),
22241 )
22242 .await
22243 .expect("selected run termination");
22244
22245 for (command, error) in [
22246 (
22247 WorkflowCommandKind::Cancel,
22248 client
22249 .cancel_workflow_run(
22250 "wf-lifecycle",
22251 "run-stale",
22252 WorkflowCommandOptions::default(),
22253 )
22254 .await
22255 .expect_err("stale cancellation must be rejected"),
22256 ),
22257 (
22258 WorkflowCommandKind::Terminate,
22259 client
22260 .terminate_workflow_run(
22261 "wf-lifecycle",
22262 "run-stale",
22263 WorkflowCommandOptions::default(),
22264 )
22265 .await
22266 .expect_err("stale termination must be rejected"),
22267 ),
22268 ] {
22269 let Error::WorkflowCommandRejected(rejection) = error else {
22270 panic!("expected typed command rejection");
22271 };
22272 assert_eq!(rejection.command, command);
22273 assert_eq!(rejection.status, 409);
22274 assert_eq!(rejection.reason, "historical_run_command_rejected");
22275 assert_eq!(rejection.run_id.as_deref(), Some("run-stale"));
22276 assert_eq!(rejection.target_scope.as_deref(), Some("run"));
22277 }
22278 }
22279
22280 #[tokio::test]
22281 async fn workflow_start_options_send_server_enforced_deadlines() {
22282 let server = MockWorkerServer::start();
22283 let client = Client::builder(server.base_url())
22284 .timeout(Duration::from_secs(2))
22285 .build()
22286 .expect("client");
22287
22288 let handle = client
22289 .start_workflow_with_options(
22290 "rust.timeout",
22291 "rust-timeouts",
22292 "wf-start-options",
22293 WorkflowStartOptions::new()
22294 .execution_timeout_seconds(30)
22295 .run_timeout_seconds(1),
22296 json!([]),
22297 )
22298 .await
22299 .expect("workflow start");
22300
22301 assert_eq!(handle.run_id.as_deref(), Some("run-start-options"));
22302 let body = server.request_body("/api/workflows");
22303 assert_eq!(body["execution_timeout_seconds"], 30);
22304 assert_eq!(body["run_timeout_seconds"], 1);
22305
22306 let invalid = client
22307 .start_workflow_with_options(
22308 "rust.timeout",
22309 "rust-timeouts",
22310 "wf-invalid-options",
22311 WorkflowStartOptions::new()
22312 .execution_timeout_seconds(1)
22313 .run_timeout_seconds(2),
22314 json!([]),
22315 )
22316 .await
22317 .expect_err("invalid deadline ordering");
22318 assert!(invalid
22319 .to_string()
22320 .contains("run_timeout_seconds cannot exceed execution_timeout_seconds"));
22321 }
22322
22323 #[tokio::test]
22324 async fn workflow_result_returns_each_typed_terminal_outcome() {
22325 let server = MockWorkerServer::start();
22326 let client = Client::builder(server.base_url())
22327 .timeout(Duration::from_secs(2))
22328 .build()
22329 .expect("client");
22330 let options = WorkflowResultOptions {
22331 poll_interval: Duration::ZERO,
22332 timeout: Duration::from_secs(1),
22333 };
22334
22335 let failed = WorkflowHandle {
22336 client: client.clone(),
22337 workflow_id: "wf-failed".to_string(),
22338 run_id: Some("run-failed".to_string()),
22339 workflow_type: "failure".to_string(),
22340 }
22341 .result(options)
22342 .await
22343 .expect_err("failed outcome");
22344 let Error::WorkflowFailed(failure) = failed else {
22345 panic!("expected WorkflowFailed");
22346 };
22347 assert_eq!(failure.workflow_id, "wf-failed");
22348 assert_eq!(failure.run_id.as_deref(), Some("run-failed"));
22349 assert_eq!(failure.failure_id.as_deref(), Some("failure-17"));
22350 assert_eq!(failure.failure_category.as_deref(), Some("application"));
22351 assert_eq!(failure.exception_type.as_deref(), Some("PaymentError"));
22352 assert_eq!(
22353 failure.exception_class.as_deref(),
22354 Some("billing::PaymentError")
22355 );
22356 assert_eq!(failure.non_retryable, Some(true));
22357
22358 for (workflow_id, expected_kind, expected_reason) in [
22359 (
22360 "wf-cancelled",
22361 WorkflowTerminalKind::Cancelled,
22362 "cleanup requested",
22363 ),
22364 (
22365 "wf-terminated",
22366 WorkflowTerminalKind::Terminated,
22367 "forced stop",
22368 ),
22369 (
22370 "wf-timed-out",
22371 WorkflowTerminalKind::TimedOut,
22372 "run_timeout",
22373 ),
22374 ] {
22375 let error = WorkflowHandle {
22376 client: client.clone(),
22377 workflow_id: workflow_id.to_string(),
22378 run_id: None,
22379 workflow_type: "terminal".to_string(),
22380 }
22381 .result(options)
22382 .await
22383 .expect_err("typed terminal outcome");
22384 let outcome = match error {
22385 Error::WorkflowCancelled(outcome) => outcome,
22386 Error::WorkflowTerminated(outcome) => outcome,
22387 Error::WorkflowTimedOut(outcome) => outcome,
22388 other => panic!("unexpected terminal error: {other}"),
22389 };
22390 assert_eq!(outcome.kind, expected_kind);
22391 assert_eq!(outcome.workflow_id, workflow_id);
22392 assert_eq!(outcome.reason, expected_reason);
22393 }
22394
22395 let wait_timeout = WorkflowHandle {
22396 client,
22397 workflow_id: "wf-waiting".to_string(),
22398 run_id: Some("run-waiting".to_string()),
22399 workflow_type: "waiting".to_string(),
22400 }
22401 .result(WorkflowResultOptions {
22402 poll_interval: Duration::ZERO,
22403 timeout: Duration::ZERO,
22404 })
22405 .await
22406 .expect_err("client wait timeout");
22407 let Error::WorkflowTimedOut(timeout) = wait_timeout else {
22408 panic!("expected typed client timeout");
22409 };
22410 assert_eq!(timeout.reason, "result_wait_timeout");
22411 assert_eq!(timeout.failure_category.as_deref(), Some("client_timeout"));
22412 assert_eq!(timeout.run_id.as_deref(), Some("run-waiting"));
22413 }
22414
22415 #[tokio::test]
22416 async fn workflow_result_follows_chain_and_selected_result_preserves_history() {
22417 let server = MockWorkerServer::start();
22418 let client = Client::builder(server.base_url())
22419 .timeout(Duration::from_secs(2))
22420 .build()
22421 .expect("client");
22422
22423 let handle = WorkflowHandle {
22424 client,
22425 workflow_id: "wf-selected".to_string(),
22426 run_id: Some("run-selected".to_string()),
22427 workflow_type: "selected".to_string(),
22428 };
22429 let options = WorkflowResultOptions {
22430 poll_interval: Duration::ZERO,
22431 timeout: Duration::from_secs(1),
22432 };
22433
22434 let current = handle
22435 .result(options)
22436 .await
22437 .expect("instance result follows the current run");
22438 assert_eq!(current, json!("current run output"));
22439
22440 let error = handle
22441 .result_selected_run(options)
22442 .await
22443 .expect_err("the selected run is cancelled even though the current run completed");
22444
22445 let Error::WorkflowCancelled(outcome) = error else {
22446 panic!("expected selected run cancellation");
22447 };
22448 assert_eq!(outcome.run_id.as_deref(), Some("run-selected"));
22449 assert_eq!(outcome.reason, "selected run cancelled");
22450 assert_eq!(
22451 server.request_count("/api/workflows/wf-selected/runs/run-selected"),
22452 1
22453 );
22454 assert_eq!(server.request_count("/api/workflows/wf-selected"), 1);
22455 }
22456
22457 #[tokio::test]
22458 async fn poll_responses_decode_http_conflict_drain_as_a_stable_stop() {
22459 let server = MockWorkerServer::draining_polls();
22460 let client = Client::builder(server.base_url())
22461 .timeout(Duration::from_secs(2))
22462 .build()
22463 .expect("client");
22464
22465 let workflow = client
22466 .poll_workflow_task_response("draining-worker", "rust-workers", Duration::ZERO)
22467 .await
22468 .expect("workflow drain response");
22469 let activity = client
22470 .poll_activity_task_response("draining-worker", "rust-workers", Duration::ZERO)
22471 .await
22472 .expect("activity drain response");
22473 let query = client
22474 .poll_query_task_response("draining-worker", "rust-workers", Duration::ZERO)
22475 .await
22476 .expect("query drain response");
22477
22478 for outcome in [workflow.outcome(), activity.outcome(), query.outcome()] {
22479 assert_eq!(
22480 outcome,
22481 WorkerPollOutcome::Stop {
22482 poll_status: Some("draining".to_string()),
22483 reason: Some("worker_draining".to_string()),
22484 }
22485 );
22486 }
22487
22488 assert!(client
22489 .poll_workflow_task("draining-worker", "rust-workers", Duration::ZERO)
22490 .await
22491 .expect("compatibility poll")
22492 .is_none());
22493 }
22494
22495 #[tokio::test]
22496 async fn managed_worker_honors_drain_stop_for_every_task_family() {
22497 let server = MockWorkerServer::draining_polls();
22498 let client = Client::builder(server.base_url())
22499 .timeout(Duration::from_secs(2))
22500 .build()
22501 .expect("client");
22502
22503 let mut workflow_worker = Worker::new(client.clone(), "rust-workers")
22504 .worker_id("draining-workflow-worker")
22505 .poll_timeout(Duration::ZERO);
22506 workflow_worker.register_workflow("counter", |_ctx, _args| async { Ok(Value::Null) });
22507 workflow_worker
22508 .run()
22509 .await
22510 .expect("workflow drain is a clean stop");
22511
22512 let mut activity_worker = Worker::new(client.clone(), "rust-workers")
22513 .worker_id("draining-activity-worker")
22514 .poll_timeout(Duration::ZERO);
22515 activity_worker.register_activity("write", |_ctx, _args| async { Ok(Value::Null) });
22516 activity_worker
22517 .run()
22518 .await
22519 .expect("activity drain is a clean stop");
22520
22521 let mut query_worker = Worker::new(client, "rust-workers")
22522 .worker_id("draining-query-worker")
22523 .poll_timeout(Duration::ZERO);
22524 query_worker.register_query("counter", "current", |_ctx, _args| async {
22525 Ok(Value::Null)
22526 });
22527 query_worker
22528 .run()
22529 .await
22530 .expect("query drain is a clean stop");
22531 }
22532
22533 #[tokio::test]
22534 async fn activity_cancellation_and_late_completion_remain_machine_readable() {
22535 let server = MockWorkerServer::start();
22536 let client = Client::builder(server.base_url())
22537 .timeout(Duration::from_secs(2))
22538 .build()
22539 .expect("client");
22540
22541 let heartbeat = client
22542 .heartbeat_activity_task(
22543 "activity-cancel",
22544 "attempt-cancel",
22545 "rust-worker",
22546 typed_fidelity_probe(),
22547 )
22548 .await
22549 .expect("cancellation heartbeat");
22550 assert!(heartbeat.cancel_requested);
22551 assert!(heartbeat.should_stop());
22552 assert_eq!(heartbeat.reason.as_deref(), Some("run_cancelled"));
22553 assert_eq!(heartbeat.run_closed_reason.as_deref(), Some("cancelled"));
22554 let heartbeat_body =
22555 server.request_body("/api/worker/activity-tasks/activity-cancel/heartbeat");
22556 assert_eq!(heartbeat_body["details"]["codec"], DEFAULT_CODEC);
22557 assert_eq!(
22558 decode_wire_avro_value(&heartbeat_body["details"], DEFAULT_CODEC)
22559 .expect("typed heartbeat details"),
22560 typed_fidelity_probe()
22561 );
22562
22563 let error = client
22564 .complete_activity_task(
22565 "activity-cancel",
22566 "attempt-cancel",
22567 "rust-worker",
22568 json!({"late":true}),
22569 DEFAULT_CODEC,
22570 )
22571 .await
22572 .expect_err("late completion must be refused");
22573 assert!(activity_task_rejection_is_final(&error));
22574 let Error::ActivityTaskRejected(rejection) = error else {
22575 panic!("expected typed activity rejection");
22576 };
22577 assert_eq!(rejection.status, 409);
22578 assert_eq!(rejection.reason, "run_cancelled");
22579 assert!(rejection.cancel_requested);
22580 assert_eq!(rejection.can_continue, Some(false));
22581 }
22582
22583 #[tokio::test]
22584 async fn managed_worker_survives_late_completion_and_restart_during_cancellation() {
22585 let server = MockWorkerServer::cancelled_activity();
22586 let client = Client::builder(server.base_url())
22587 .timeout(Duration::from_secs(2))
22588 .build()
22589 .expect("client");
22590 let cancellation_observed = Arc::new(AtomicBool::new(false));
22591 let observed = Arc::clone(&cancellation_observed);
22592 let mut worker = Worker::new(client.clone(), "rust-workers")
22593 .worker_id("rust-cancel-worker")
22594 .poll_timeout(Duration::from_millis(10));
22595 worker.register_activity("cancel-aware", move |ctx, _args| {
22596 let observed = Arc::clone(&observed);
22597 async move {
22598 let heartbeat = ctx.heartbeat(json!({"stage":"running"})).await?;
22599 observed.store(heartbeat.should_stop(), Ordering::SeqCst);
22600 Ok(json!({"late":"completion"}))
22601 }
22602 });
22603
22604 assert_eq!(
22605 worker.run_once().await.expect("cancelled attempt handled"),
22606 1
22607 );
22608 assert!(cancellation_observed.load(Ordering::SeqCst));
22609 assert_eq!(
22610 server.request_count("/api/worker/activity-tasks/activity-cancel/complete"),
22611 1
22612 );
22613
22614 let mut restarted = Worker::new(client, "rust-workers")
22615 .worker_id("rust-cancel-worker-restarted")
22616 .poll_timeout(Duration::from_millis(10));
22617 restarted.register_activity("cancel-aware", |_ctx, _args| async move { Ok(Value::Null) });
22618 assert_eq!(
22619 restarted
22620 .run_once()
22621 .await
22622 .expect("replacement worker continues polling"),
22623 0
22624 );
22625 }
22626
22627 #[tokio::test]
22628 async fn managed_worker_absorbs_selected_run_terminal_timeout_completion_race() {
22629 let response = r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"outcome":"completed","recorded":false,"run_id":"run-selected-timeout","run_status":"failed","created_task_ids":[],"reason":"run_timed_out"}"#;
22630 let server = MockWorkerServer::workflow_completion("409 Conflict", response);
22631 let client = Client::builder(server.base_url())
22632 .timeout(Duration::from_secs(2))
22633 .build()
22634 .expect("client");
22635
22636 let direct_error = client
22637 .complete_workflow_task(
22638 "workflow-timeout-task",
22639 "timeout-worker",
22640 3,
22641 vec![json!({
22642 "type": "complete_workflow",
22643 "result": fixture_envelope(Value::Null)
22644 })],
22645 )
22646 .await
22647 .expect_err("the low-level client preserves the completion rejection");
22648 let Error::Http { status, body } = direct_error else {
22649 panic!("expected the original HTTP completion rejection");
22650 };
22651 assert_eq!(status, reqwest::StatusCode::CONFLICT);
22652 assert_eq!(
22653 serde_json::from_str::<Value>(&body).expect("response body")["reason"],
22654 "run_timed_out"
22655 );
22656
22657 let mut worker = Worker::new(client, "rust-workers")
22658 .worker_id("timeout-worker")
22659 .poll_timeout(Duration::from_millis(10));
22660 worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
22661 Ok(json!({"late": "result"}))
22662 });
22663
22664 assert_eq!(
22665 worker
22666 .run_once()
22667 .await
22668 .expect("authoritative selected-run timeout settles the tick"),
22669 1
22670 );
22671 assert_eq!(
22672 server.request_count("/api/worker/workflow-tasks/workflow-timeout-task/complete"),
22673 2,
22674 "both the direct client proof and managed worker must see the rejection"
22675 );
22676 }
22677
22678 #[tokio::test]
22679 async fn managed_worker_does_not_swallow_nearby_completion_errors() {
22680 for (name, status, response) in [
22681 ("bare conflict", "409 Conflict", r#"{"message":"conflict"}"#),
22682 (
22683 "command was recorded",
22684 "409 Conflict",
22685 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":true,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22686 ),
22687 (
22688 "lease conflict",
22689 "409 Conflict",
22690 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"lease_expired"}"#,
22691 ),
22692 (
22693 "nonterminal run",
22694 "409 Conflict",
22695 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"waiting","reason":"run_timed_out"}"#,
22696 ),
22697 (
22698 "different selected run",
22699 "409 Conflict",
22700 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-reused-workflow-current","run_status":"failed","reason":"run_timed_out"}"#,
22701 ),
22702 (
22703 "different task attempt",
22704 "409 Conflict",
22705 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":4,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22706 ),
22707 (
22708 "authentication failure",
22709 "401 Unauthorized",
22710 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22711 ),
22712 (
22713 "authorization failure",
22714 "403 Forbidden",
22715 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22716 ),
22717 (
22718 "protocol failure",
22719 "400 Bad Request",
22720 r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.2","requested_version":"1.3"}"#,
22721 ),
22722 (
22723 "malformed command",
22724 "422 Unprocessable Entity",
22725 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22726 ),
22727 (
22728 "transient server failure",
22729 "503 Service Unavailable",
22730 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22731 ),
22732 ] {
22733 let server = MockWorkerServer::workflow_completion(status, response);
22734 let client = Client::builder(server.base_url())
22735 .timeout(Duration::from_secs(2))
22736 .build()
22737 .expect("client");
22738 let mut worker = Worker::new(client, "rust-workers")
22739 .worker_id("timeout-worker")
22740 .poll_timeout(Duration::from_millis(10));
22741 worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
22742 Ok(json!({"late": "result"}))
22743 });
22744
22745 let error = worker
22746 .run_once()
22747 .await
22748 .expect_err(&format!("{name} must remain an error"));
22749 assert!(
22750 matches!(error, Error::Http { .. } | Error::Protocol(_)),
22751 "{name} returned an unexpected error variant: {error}"
22752 );
22753 }
22754 }
22755
22756 #[tokio::test]
22757 async fn worker_deregistration_uses_worker_plane_method_path_headers_and_result() {
22758 let server = MockWorkerServer::start();
22759 let client = Client::builder(server.base_url())
22760 .worker_token(Some("worker-secret".to_string()))
22761 .namespace("orders")
22762 .timeout(Duration::from_secs(2))
22763 .build()
22764 .expect("client");
22765 let path = "/api/worker/registrations/worker%2F%CE%B1%20space";
22766
22767 let result = client
22768 .deregister_worker_registration("worker/α space")
22769 .await
22770 .expect("deregister worker registration");
22771
22772 assert_eq!(server.method_for(path).as_deref(), Some("DELETE"));
22773 assert_eq!(
22774 server.worker_protocol_for(path).as_deref(),
22775 Some(WORKER_PROTOCOL_VERSION)
22776 );
22777 assert_eq!(server.control_protocol_for(path), None);
22778 assert_eq!(server.namespace_for(path).as_deref(), Some("orders"));
22779 assert_eq!(
22780 server.authorization_for(path).as_deref(),
22781 Some("Bearer worker-secret")
22782 );
22783 assert_eq!(
22784 result,
22785 WorkerDeregistrationEnvelope {
22786 worker_id: "deregistered-worker".to_string(),
22787 outcome: "deregistered".to_string(),
22788 recovered_workflow_task_count: 2,
22789 }
22790 );
22791 }
22792
22793 #[tokio::test]
22794 async fn low_level_registration_rejects_update_validators_before_transport() {
22795 let server = MockWorkerServer::start();
22796 let client = Client::builder(server.base_url())
22797 .timeout(Duration::from_secs(2))
22798 .build()
22799 .expect("client");
22800
22801 for update_validators in [json!(["approve"]), json!("approve")] {
22802 let error = client
22803 .register_worker_with_command_contracts(
22804 "validator-claiming-worker",
22805 "rust-workers",
22806 vec!["orders".to_string()],
22807 vec![],
22808 1,
22809 1,
22810 vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
22811 json!({
22812 "orders": {
22813 "queries": ["current"],
22814 "updates": ["approve"],
22815 "update_validators": update_validators,
22816 },
22817 }),
22818 )
22819 .await
22820 .expect_err("unsupported validator claims must fail before registration");
22821
22822 let Error::UnsupportedUpdateValidators { workflow_type } = error else {
22823 panic!("expected typed unsupported-validator failure");
22824 };
22825 assert_eq!(workflow_type, "orders");
22826 }
22827 assert_eq!(server.request_count("/api/worker/register"), 0);
22828 }
22829
22830 #[tokio::test]
22831 async fn low_level_registration_preserves_query_and_update_contracts() {
22832 let server = MockWorkerServer::start();
22833 let client = Client::builder(server.base_url())
22834 .timeout(Duration::from_secs(2))
22835 .build()
22836 .expect("client");
22837 let contracts = json!({
22838 "orders": {
22839 "queries": ["current"],
22840 "updates": ["approve"],
22841 "update_validators": [],
22842 },
22843 "payments": {
22844 "queries": ["status"],
22845 "updates": ["capture"],
22846 },
22847 });
22848
22849 client
22850 .register_worker_with_command_contracts(
22851 "command-worker",
22852 "rust-workers",
22853 vec!["orders".to_string(), "payments".to_string()],
22854 vec![],
22855 1,
22856 1,
22857 vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
22858 contracts.clone(),
22859 )
22860 .await
22861 .expect("query and update contracts must remain supported");
22862
22863 assert_eq!(
22864 server.request_body("/api/worker/register")["workflow_command_contracts"],
22865 contracts
22866 );
22867 }
22868
22869 #[tokio::test]
22870 async fn role_scoped_tokens_are_never_used_for_the_opposite_plane() {
22871 let server = MockWorkerServer::start();
22872 let control_only = Client::builder(server.base_url())
22873 .control_token(Some("control-secret".to_string()))
22874 .build()
22875 .expect("control client");
22876
22877 let error = control_only
22878 .register_worker("worker", "queue", vec![], vec![], 1, 1)
22879 .await
22880 .expect_err("control token must not authorize a worker request");
22881 assert!(matches!(
22882 error,
22883 Error::MissingRoleCredentials { role: "worker", .. }
22884 ));
22885 assert_eq!(server.request_count("/api/worker/register"), 0);
22886
22887 let worker_only = Client::builder(server.base_url())
22888 .worker_token(Some("worker-secret".to_string()))
22889 .build()
22890 .expect("worker client");
22891 let error = worker_only
22892 .health()
22893 .await
22894 .expect_err("worker token must not authorize a control request");
22895 assert!(matches!(
22896 error,
22897 Error::MissingRoleCredentials {
22898 role: "control",
22899 ..
22900 }
22901 ));
22902 assert_eq!(server.request_count("/api/health"), 0);
22903 }
22904
22905 #[tokio::test]
22906 async fn shared_token_supports_worker_and_control_planes() {
22907 let server = MockWorkerServer::start();
22908 let client = Client::builder(server.base_url())
22909 .token(Some("shared-secret".to_string()))
22910 .build()
22911 .expect("client");
22912
22913 client.health().await.expect("control request");
22914 client
22915 .register_worker("worker", "queue", vec![], vec![], 1, 1)
22916 .await
22917 .expect("worker request");
22918
22919 assert_eq!(
22920 server.authorization_for("/api/health").as_deref(),
22921 Some("Bearer shared-secret")
22922 );
22923 assert_eq!(
22924 server.control_protocol_for("/api/health").as_deref(),
22925 Some(CONTROL_PLANE_VERSION)
22926 );
22927 assert_eq!(
22928 server.authorization_for("/api/worker/register").as_deref(),
22929 Some("Bearer shared-secret")
22930 );
22931 assert_eq!(
22932 server
22933 .worker_protocol_for("/api/worker/register")
22934 .as_deref(),
22935 Some(WORKER_PROTOCOL_VERSION)
22936 );
22937 }
22938
22939 #[tokio::test]
22940 async fn baseline_worker_endpoints_send_the_baseline_protocol() {
22941 let server = MockWorkerServer::start();
22942 let client = Client::builder(server.base_url())
22943 .timeout(Duration::from_secs(2))
22944 .build()
22945 .expect("client");
22946
22947 client
22948 .register_worker("capture-worker", "capture", vec![], vec![], 1, 1)
22949 .await
22950 .expect("register");
22951 client
22952 .heartbeat_worker("capture-worker", 1, 1)
22953 .await
22954 .expect("heartbeat");
22955 client
22956 .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
22957 .await
22958 .expect("workflow poll");
22959 client
22960 .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
22961 .await
22962 .expect("activity poll");
22963
22964 for path in [
22965 "/api/worker/register",
22966 "/api/worker/heartbeat",
22967 "/api/worker/workflow-tasks/poll",
22968 "/api/worker/activity-tasks/poll",
22969 ] {
22970 assert_eq!(
22971 server.worker_protocol_for(path).as_deref(),
22972 Some(WORKER_PROTOCOL_VERSION),
22973 "unexpected protocol for {path}"
22974 );
22975 }
22976
22977 assert_eq!(
22978 server.request_body("/api/worker/workflow-tasks/poll")["timeout_seconds"],
22979 1
22980 );
22981 assert_eq!(
22982 server.request_body("/api/worker/activity-tasks/poll")["timeout_seconds"],
22983 1
22984 );
22985 assert!(
22986 server.request_body("/api/worker/workflow-tasks/poll")["poll_request_id"]
22987 .as_str()
22988 .is_some_and(|id| id.starts_with("rust-workflow-poll-"))
22989 );
22990 assert!(
22991 server.request_body("/api/worker/activity-tasks/poll")["poll_request_id"]
22992 .as_str()
22993 .is_some_and(|id| id.starts_with("rust-activity-poll-"))
22994 );
22995 }
22996
22997 #[tokio::test]
22998 async fn query_task_endpoints_send_the_query_feature_protocol() {
22999 let server = MockWorkerServer::start();
23000 let client = Client::builder(server.base_url())
23001 .timeout(Duration::from_secs(2))
23002 .build()
23003 .expect("client");
23004
23005 client
23006 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
23007 .await
23008 .expect("query poll");
23009 client
23010 .complete_query_task(
23011 "query-capture",
23012 "capture-worker",
23013 1,
23014 json!(8),
23015 DEFAULT_CODEC,
23016 )
23017 .await
23018 .expect("query complete");
23019 client
23020 .fail_query_task(
23021 "query-capture",
23022 "capture-worker",
23023 1,
23024 "failed",
23025 "query_rejected",
23026 "QueryFailed",
23027 )
23028 .await
23029 .expect("query fail");
23030
23031 for path in [
23032 "/api/worker/query-tasks/poll",
23033 "/api/worker/query-tasks/query-capture/complete",
23034 "/api/worker/query-tasks/query-capture/fail",
23035 ] {
23036 assert_eq!(
23037 server.worker_protocol_for(path).as_deref(),
23038 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
23039 "unexpected protocol for {path}"
23040 );
23041 }
23042
23043 assert_eq!(
23044 server.request_body("/api/worker/query-tasks/poll")["timeout_seconds"],
23045 1
23046 );
23047 assert!(
23048 server.request_body("/api/worker/query-tasks/poll")["poll_request_id"]
23049 .as_str()
23050 .is_some_and(|id| id.starts_with("rust-query-poll-"))
23051 );
23052 }
23053
23054 #[tokio::test]
23055 async fn disconnected_client_polls_retry_once_with_the_same_request_id() {
23056 let server = MockWorkerServer::transient_worker_failures();
23057 let client = Client::builder(server.base_url())
23058 .timeout(Duration::from_secs(2))
23059 .build()
23060 .expect("client");
23061
23062 client
23063 .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
23064 .await
23065 .expect("workflow poll retry");
23066 client
23067 .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
23068 .await
23069 .expect("activity poll retry");
23070 client
23071 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
23072 .await
23073 .expect("query poll retry");
23074
23075 for path in [
23076 "/api/worker/workflow-tasks/poll",
23077 "/api/worker/activity-tasks/poll",
23078 "/api/worker/query-tasks/poll",
23079 ] {
23080 let bodies = server.request_bodies(path);
23081 assert_eq!(bodies.len(), 2, "{path} must be retried once");
23082 assert_eq!(
23083 bodies[0]["poll_request_id"], bodies[1]["poll_request_id"],
23084 "{path} must preserve the request binding across retry"
23085 );
23086 }
23087 }
23088
23089 #[tokio::test]
23090 async fn worker_poll_retries_preserve_request_id_across_consecutive_failures() {
23091 let server = MockWorkerServer::consecutive_poll_failures(2);
23092 let client = Client::builder(server.base_url())
23093 .timeout(Duration::from_secs(2))
23094 .build()
23095 .expect("client");
23096 let mut worker = Worker::new(client, "capture")
23097 .worker_id("capture-worker")
23098 .poll_timeout(Duration::from_millis(10))
23099 .retry_policy(WorkerRetryPolicy {
23100 max_retries: 2,
23101 initial_backoff: Duration::from_millis(1),
23102 max_backoff: Duration::from_millis(1),
23103 });
23104 worker.register_workflow(
23105 "capture.workflow",
23106 |_ctx, _input| async move { Ok(Value::Null) },
23107 );
23108 worker.register_activity(
23109 "capture.activity",
23110 |_ctx, _input| async move { Ok(Value::Null) },
23111 );
23112 worker.register_query("capture.workflow", "current", |_ctx, _args| async move {
23113 Ok(Value::Null)
23114 });
23115
23116 assert_eq!(worker.run_once().await.expect("poll retries"), 0);
23117
23118 for path in [
23119 "/api/worker/workflow-tasks/poll",
23120 "/api/worker/activity-tasks/poll",
23121 "/api/worker/query-tasks/poll",
23122 ] {
23123 let bodies = server.request_bodies(path);
23124 assert_eq!(bodies.len(), 3, "{path} must use exactly two retries");
23125 assert!(
23126 bodies
23127 .iter()
23128 .all(|body| body["poll_request_id"] == bodies[0]["poll_request_id"]),
23129 "{path} must preserve one request binding across every retry"
23130 );
23131 }
23132 }
23133
23134 fn storage_refusal(poll_id: Option<&str>, unavailable: bool, mid_poll: bool) -> Value {
23135 let reason = if unavailable {
23136 "storage_admission_unavailable"
23137 } else {
23138 "storage_pressure"
23139 };
23140 let mut body = json!({
23141 "reason": reason,
23142 "storage_state": if unavailable { "fenced" } else { "draining" },
23143 "retryable": true,
23144 "retry_after_seconds": 1,
23145 });
23146 if !mid_poll {
23147 body["request_admitted"] = json!(false);
23148 }
23149 if let Some(id) = poll_id {
23150 body["task"] = Value::Null;
23151 body["poll_status"] = json!(reason);
23152 body["poll_request_id"] = json!(id);
23153 body["retry_same_poll_request_id"] = json!(true);
23154 body["claim_admitted"] = json!(false);
23155 }
23156 body
23157 }
23158
23159 fn backend_refusal(path: &str, request: &str) -> Option<Value> {
23160 let operation = match path {
23161 "/api/worker/register" => "register_worker",
23162 "/api/worker/heartbeat" => "heartbeat_worker",
23163 "/api/worker/workflow-tasks/poll" => "poll_workflow_task",
23164 "/api/worker/activity-tasks/poll" => "poll_activity_task",
23165 "/api/worker/query-tasks/poll" => "poll_query_task",
23166 "/api/worker/update-validation-tasks/poll" => "poll_update_validation_task",
23167 _ => return None,
23168 };
23169 let request: Value = serde_json::from_str(request).ok()?;
23170 let mut response = json!({
23171 "reason": "backend_unavailable",
23172 "operation": operation,
23173 "outcome": "unknown",
23174 "retryable": true,
23175 "retry_after_seconds": 1,
23176 "worker_id": request["worker_id"],
23177 "task_queue": request.get("task_queue"),
23178 });
23179 if path.ends_with("/poll") {
23180 response["task"] = Value::Null;
23181 response["poll_status"] = json!("backend_unavailable");
23182 response["poll_request_id"] = request["poll_request_id"].clone();
23183 response["retry_same_poll_request_id"] = json!(true);
23184 }
23185 Some(response)
23186 }
23187
23188 fn backend_retry_override(
23189 path: &str,
23190 request: &str,
23191 number: usize,
23192 ) -> Option<(&'static str, String)> {
23193 if number > 8 {
23194 return None;
23195 }
23196 Some((
23197 "503 Service Unavailable",
23198 backend_refusal(path, request)?.to_string(),
23199 ))
23200 }
23201
23202 fn backend_unavailable_override(
23203 path: &str,
23204 request: &str,
23205 _number: usize,
23206 ) -> Option<(&'static str, String)> {
23207 Some((
23208 "503 Service Unavailable",
23209 backend_refusal(path, request)?.to_string(),
23210 ))
23211 }
23212
23213 fn storage_worker(server: &MockWorkerServer) -> Worker {
23214 Worker::new(Client::new(server.base_url()).expect("client"), "storage")
23215 .worker_id("storage-worker")
23216 .retry_policy(WorkerRetryPolicy {
23217 max_retries: 1,
23218 initial_backoff: Duration::from_millis(1),
23219 max_backoff: Duration::from_millis(1),
23220 })
23221 }
23222
23223 fn assert_identical_requests(server: &MockWorkerServer, path: &str, count: usize) {
23224 let requests = server.requests.lock().expect("requests");
23225 let bodies: Vec<_> = requests
23226 .iter()
23227 .filter(|request| request.path == path)
23228 .map(|request| &request.body)
23229 .collect();
23230 assert_eq!(bodies.len(), count, "{path}");
23231 assert!(bodies.iter().all(|body| body == &bodies[0]), "{path}");
23232 }
23233
23234 #[test]
23235 fn backend_recovery_requires_the_explicit_worker_contract() {
23236 for (path, operation) in [
23237 ("/worker/register", "register_worker"),
23238 ("/worker/heartbeat", "heartbeat_worker"),
23239 ("/worker/workflow-tasks/poll", "poll_workflow_task"),
23240 ("/worker/activity-tasks/poll", "poll_activity_task"),
23241 ("/worker/query-tasks/poll", "poll_query_task"),
23242 (
23243 "/worker/update-validation-tasks/poll",
23244 "poll_update_validation_task",
23245 ),
23246 ] {
23247 let poll_id = path.ends_with("/poll").then_some("same-poll");
23248 let request = json!({"worker_id":"same-worker","task_queue":"same-queue","poll_request_id":poll_id});
23249 let body = backend_refusal(&format!("/api{path}"), &request.to_string())
23250 .expect("worker operation");
23251 assert_eq!(body["operation"], operation);
23252 let error = Error::Http {
23253 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23254 body: body.to_string(),
23255 };
23256 assert_eq!(
23257 worker_backend_unavailable_retry_after(&error, path, poll_id),
23258 Some(Duration::from_secs(1))
23259 );
23260 assert!(!worker_operation_is_retryable(&error));
23261 for (field, value) in [
23262 ("operation", json!("wrong_operation")),
23263 ("outcome", json!("completed")),
23264 ("retryable", json!(false)),
23265 ("retry_after_seconds", json!(0)),
23266 ("worker_id", json!("")),
23267 ] {
23268 let mut invalid = body.clone();
23269 invalid[field] = value;
23270 let error = Error::Http {
23271 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23272 body: invalid.to_string(),
23273 };
23274 assert!(
23275 worker_backend_unavailable_retry_after(&error, path, poll_id).is_none(),
23276 "{path}: {field}"
23277 );
23278 assert!(!worker_operation_is_retryable(&error));
23279 }
23280 if let Some(poll_id) = poll_id {
23281 for (field, value) in [
23282 ("poll_request_id", json!("wrong-poll")),
23283 ("poll_status", json!("empty")),
23284 ("task", json!({"task_id":"claimed"})),
23285 ("retry_same_poll_request_id", json!(false)),
23286 ] {
23287 let mut invalid = body.clone();
23288 invalid[field] = value;
23289 let error = Error::Http {
23290 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23291 body: invalid.to_string(),
23292 };
23293 assert!(
23294 worker_backend_unavailable_retry_after(&error, path, Some(poll_id))
23295 .is_none(),
23296 "{path}: {field}"
23297 );
23298 }
23299 }
23300 let error = Error::Http {
23301 status: reqwest::StatusCode::UNAUTHORIZED,
23302 body: body.to_string(),
23303 };
23304 assert!(worker_backend_unavailable_retry_after(&error, path, poll_id).is_none());
23305 assert!(!worker_operation_is_retryable(&error));
23306 }
23307 }
23308
23309 #[tokio::test]
23310 async fn backend_recovery_outlives_generic_retry_budget_and_preserves_polls() {
23311 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23312 request_override: Some(backend_retry_override),
23313 ..MockWorkerBehavior::default()
23314 });
23315 let mut worker =
23316 storage_worker(&server).with_storage_admission(Arc::new(AtomicBool::new(false)));
23317 worker.register_workflow("backend.workflow", |_, _| async { Ok(Value::Null) });
23318 worker.register_activity("backend.activity", |_, _| async { Ok(Value::Null) });
23319 worker.register_query("backend.workflow", "state", |_, _| async {
23320 Ok(Value::Null)
23321 });
23322 worker.register().await.expect("registration recovery");
23323 worker
23324 .client
23325 .heartbeat_worker("storage-worker", 1, 1)
23326 .await
23327 .expect("heartbeat recovery");
23328 assert_eq!(worker.run_once().await.expect("poll recovery"), 0);
23329 for path in [
23330 "/api/worker/register",
23331 "/api/worker/heartbeat",
23332 "/api/worker/workflow-tasks/poll",
23333 "/api/worker/activity-tasks/poll",
23334 "/api/worker/query-tasks/poll",
23335 ] {
23336 assert_identical_requests(&server, path, 9);
23337 }
23338 }
23339
23340 #[tokio::test]
23341 async fn backend_recovery_wait_stops_with_worker() {
23342 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23343 request_override: Some(backend_unavailable_override),
23344 ..MockWorkerBehavior::default()
23345 });
23346 let mut worker = storage_worker(&server).retry_policy(WorkerRetryPolicy {
23347 max_retries: 1,
23348 initial_backoff: Duration::from_secs(2),
23349 max_backoff: Duration::from_secs(2),
23350 });
23351 worker.register_workflow("backend.workflow", |_, _| async { Ok(Value::Null) });
23352 let started = tokio::time::Instant::now();
23353 let _ = tokio::time::timeout(
23354 Duration::from_secs(1),
23355 worker.run_until(tokio::time::sleep(Duration::from_millis(100))),
23356 )
23357 .await
23358 .expect("shutdown interrupts backend wait");
23359 assert!(started.elapsed() < Duration::from_secs(1));
23360 assert!(server.request_count("/api/worker/register") >= 1);
23361 }
23362
23363 #[test]
23364 fn storage_admission_requires_an_explicit_identity_preserving_contract() {
23365 for unavailable in [false, true] {
23366 for mid_poll in [false, true] {
23367 let body = storage_refusal(Some("same-poll"), unavailable, mid_poll);
23368 let error = Error::Http {
23369 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23370 body: body.to_string(),
23371 };
23372 assert_eq!(
23373 worker_storage_admission_retry_after(&error, Some("same-poll")),
23374 Some(Duration::from_secs(1))
23375 );
23376 assert!(
23377 !worker_operation_is_retryable(&error),
23378 "storage is not a bounded generic retry"
23379 );
23380 for (field, value) in [
23381 ("poll_request_id", json!("wrong-poll")),
23382 ("task", json!({"task_id":"claimed"})),
23383 ("retryable", json!(false)),
23384 ("retry_after_seconds", json!(0)),
23385 ("retry_after_seconds", json!(true)),
23386 ("retry_after_seconds", json!(1.0)),
23387 ("storage_state", json!("normal")),
23388 ("poll_status", json!("empty")),
23389 ("claim_admitted", json!(true)),
23390 ("retry_same_poll_request_id", json!(false)),
23391 ("request_admitted", json!(true)),
23392 ] {
23393 let mut invalid = body.clone();
23394 invalid[field] = value;
23395 let error = Error::Http {
23396 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23397 body: invalid.to_string(),
23398 };
23399 assert!(
23400 worker_storage_admission_retry_after(&error, Some("same-poll")).is_none(),
23401 "{field}"
23402 );
23403 }
23404 }
23405 }
23406 let body = storage_refusal(None, false, false);
23407 let error = Error::Http {
23408 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23409 body: body.to_string(),
23410 };
23411 assert!(worker_storage_admission_retry_after(&error, None).is_some());
23412 assert!(worker_storage_admission_retry_after(&error, Some("")).is_none());
23413 let error = Error::Http {
23414 status: reqwest::StatusCode::FORBIDDEN,
23415 body: body.to_string(),
23416 };
23417 assert!(worker_storage_admission_retry_after(&error, None).is_none());
23418 let body = storage_refusal(None, false, true);
23419 let error = Error::Http {
23420 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23421 body: body.to_string(),
23422 };
23423 assert!(worker_storage_admission_retry_after(&error, None).is_none());
23424 }
23425
23426 #[tokio::test]
23427 async fn storage_poll_recovery_preserves_ambiguous_claim_identity() {
23428 for unavailable in [false, true] {
23429 for mid_poll in [false, true] {
23430 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23431 poll_failures_per_path: 1,
23432 storage_refusals: 7,
23433 storage_path: Some("/poll"),
23434 storage_unavailable: unavailable,
23435 storage_mid_poll: mid_poll,
23436 ..MockWorkerBehavior::default()
23437 });
23438 let mut worker = storage_worker(&server);
23439 worker.register_query("unused", "state", |_, _| async { Ok(Value::Null) });
23440 assert_eq!(worker.run_once().await.expect("storage recovery"), 0);
23441 for path in [
23442 "/api/worker/workflow-tasks/poll",
23443 "/api/worker/activity-tasks/poll",
23444 "/api/worker/query-tasks/poll",
23445 ] {
23446 assert_identical_requests(&server, path, 9);
23447 }
23448 }
23449 }
23450 }
23451
23452 #[tokio::test]
23453 async fn storage_refused_mutations_do_not_reserialize_or_change_client_scope() {
23454 struct CountedBody(Arc<AtomicUsize>);
23455 impl Serialize for CountedBody {
23456 fn serialize<S: Serializer>(
23457 &self,
23458 serializer: S,
23459 ) -> std::result::Result<S::Ok, S::Error> {
23460 let count = self.0.fetch_add(1, Ordering::SeqCst);
23461 json!({"serialization":count,"lease_owner":"worker","attempt":7})
23462 .serialize(serializer)
23463 }
23464 }
23465 for path in [
23466 "/api/worker/register",
23467 "/api/worker/heartbeat",
23468 "/api/worker/workflow-tasks/storage-task/complete",
23469 "/api/worker/workflow-tasks/storage-task/fail",
23470 "/api/worker/activity-tasks/storage-task/complete",
23471 "/api/worker/activity-tasks/storage-task/fail",
23472 "/api/worker/activity-tasks/storage-task/heartbeat",
23473 "/api/worker/query-tasks/storage-task/complete",
23474 "/api/worker/query-tasks/storage-task/fail",
23475 ] {
23476 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23477 storage_refusals: 7,
23478 storage_path: Some(path),
23479 ..MockWorkerBehavior::default()
23480 });
23481 let worker =
23482 storage_worker(&server).with_storage_admission(Arc::new(AtomicBool::new(false)));
23483 let calls = Arc::new(AtomicUsize::new(0));
23484 let _: Value = worker
23485 .client
23486 .request_json(
23487 reqwest::Method::POST,
23488 &path[4..],
23489 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
23490 Some(&CountedBody(Arc::clone(&calls))),
23491 )
23492 .await
23493 .expect("prepared request recovery");
23494 assert_eq!(calls.load(Ordering::SeqCst), 1);
23495 assert_identical_requests(&server, path, 8);
23496 }
23497 for worker_scope in [false, true] {
23498 let path = if worker_scope {
23499 "/api/health"
23500 } else {
23501 "/api/worker/register"
23502 };
23503 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23504 storage_refusals: usize::MAX,
23505 storage_path: Some(path),
23506 ..MockWorkerBehavior::default()
23507 });
23508 let worker = storage_worker(&server);
23509 let client = worker.client.clone();
23510 let worker = worker.with_storage_admission(Arc::new(AtomicBool::new(false)));
23511 let error = if worker_scope {
23512 worker
23513 .client
23514 .health()
23515 .await
23516 .expect_err("control plane is not retried")
23517 } else {
23518 client
23519 .request_json::<Value, Value>(
23520 reqwest::Method::POST,
23521 "/worker/register",
23522 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
23523 Some(&json!({})),
23524 )
23525 .await
23526 .expect_err("direct client is not retried")
23527 };
23528 assert!(worker_storage_admission_body(&error).is_some());
23529 assert_eq!(server.request_count(path), 1);
23530 }
23531 }
23532
23533 #[tokio::test]
23534 async fn storage_activity_outcome_is_retained_without_reexecuting_handler() {
23535 for fail in [false, true] {
23536 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23537 storage_activity: true,
23538 storage_refusals: 7,
23539 storage_path: Some("/storage-activity/"),
23540 ..MockWorkerBehavior::default()
23541 });
23542 let mut worker = storage_worker(&server);
23543 let calls = Arc::new(AtomicUsize::new(0));
23544 let observed = Arc::clone(&calls);
23545 worker.register_activity("storage.activity", move |ctx, _| {
23546 let calls = Arc::clone(&observed);
23547 async move {
23548 calls.fetch_add(1, Ordering::SeqCst);
23549 ctx.heartbeat(json!({"step":2})).await?;
23550 if fail {
23551 Err(Error::WorkerLoop("intentional handler failure".to_string()))
23552 } else {
23553 Ok(json!({"receipt":true}))
23554 }
23555 }
23556 });
23557 assert_eq!(worker.run_once().await.expect("activity settled"), 1);
23558 assert_eq!(calls.load(Ordering::SeqCst), 1);
23559 assert_identical_requests(
23560 &server,
23561 "/api/worker/activity-tasks/storage-activity/heartbeat",
23562 8,
23563 );
23564 let suffix = if fail { "fail" } else { "complete" };
23565 assert_identical_requests(
23566 &server,
23567 &format!("/api/worker/activity-tasks/storage-activity/{suffix}"),
23568 8,
23569 );
23570 let other = if fail { "complete" } else { "fail" };
23571 assert_eq!(
23572 server.request_count(&format!(
23573 "/api/worker/activity-tasks/storage-activity/{other}"
23574 )),
23575 0
23576 );
23577 }
23578 }
23579
23580 #[tokio::test]
23581 async fn storage_waits_are_interruptible_without_false_activity_failure() {
23582 for path in [
23583 "/api/worker/register",
23584 "/api/worker/heartbeat",
23585 "/api/worker/activity-tasks/poll",
23586 "/api/worker/activity-tasks/storage-activity/heartbeat",
23587 "/api/worker/activity-tasks/storage-activity/complete",
23588 ] {
23589 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23590 storage_activity: true,
23591 storage_refusals: usize::MAX,
23592 storage_path: Some(path),
23593 ..MockWorkerBehavior::default()
23594 });
23595 let mut worker = storage_worker(&server).retry_policy(WorkerRetryPolicy::default());
23596 worker.register_activity("storage.activity", |ctx, _| async move {
23597 ctx.heartbeat(json!({"step":2})).await?;
23598 Ok(json!({"receipt":true}))
23599 });
23600 let shutdown = async {
23601 while server.request_count(path) == 0 {
23602 tokio::time::sleep(Duration::from_millis(1)).await;
23603 }
23604 };
23605 let result = tokio::time::timeout(Duration::from_secs(2), worker.run_until(shutdown))
23606 .await
23607 .expect("shutdown interrupts admission");
23608 assert!(
23609 result.is_err(),
23610 "a refused operation must not appear acknowledged: {path}, {result:?}"
23611 );
23612 assert_eq!(server.request_count(path), 1);
23613 assert_eq!(
23614 server.request_count("/api/worker/activity-tasks/storage-activity/fail"),
23615 0
23616 );
23617 assert_eq!(
23618 server.request_count("/api/worker/registrations/mock-worker"),
23619 usize::from(!path.ends_with("/register"))
23620 );
23621 }
23622 }
23623
23624 #[tokio::test]
23625 async fn storage_query_outcome_is_retained_without_reexecuting_handler() {
23626 for fail in [false, true] {
23627 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23628 storage_query: true,
23629 storage_refusals: 7,
23630 storage_path: Some("/storage-query/"),
23631 ..MockWorkerBehavior::default()
23632 });
23633 let mut worker = storage_worker(&server);
23634 worker.register_workflow("storage.workflow", |_, _| async { Ok(Value::Null) });
23635 let calls = Arc::new(AtomicUsize::new(0));
23636 let observed = Arc::clone(&calls);
23637 worker.register_query("storage.workflow", "state", move |_, _| {
23638 let calls = Arc::clone(&observed);
23639 async move {
23640 calls.fetch_add(1, Ordering::SeqCst);
23641 if fail {
23642 Err(Error::WorkerLoop("intentional query failure".to_string()))
23643 } else {
23644 Ok(json!({"state":"waiting"}))
23645 }
23646 }
23647 });
23648 assert_eq!(worker.run_once().await.expect("query settled"), 1);
23649 assert_eq!(calls.load(Ordering::SeqCst), 1);
23650 let suffix = if fail { "fail" } else { "complete" };
23651 assert_identical_requests(
23652 &server,
23653 &format!("/api/worker/query-tasks/storage-query/{suffix}"),
23654 8,
23655 );
23656 let other = if fail { "complete" } else { "fail" };
23657 assert_eq!(
23658 server.request_count(&format!("/api/worker/query-tasks/storage-query/{other}")),
23659 0
23660 );
23661 }
23662 }
23663
23664 #[tokio::test]
23665 async fn storage_recovery_does_not_override_auth_lease_or_invalid_contract() {
23666 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23667 storage_refusals: 7,
23668 storage_path: Some("/poll"),
23669 unauthorized_polls: true,
23670 ..MockWorkerBehavior::default()
23671 });
23672 let error = storage_worker(&server)
23673 .run_once()
23674 .await
23675 .expect_err("auth remains terminal");
23676 assert!(matches!(
23677 error,
23678 Error::Http {
23679 status: reqwest::StatusCode::UNAUTHORIZED,
23680 ..
23681 }
23682 ));
23683 assert_identical_requests(&server, "/api/worker/workflow-tasks/poll", 8);
23684
23685 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23686 storage_refusals: 7,
23687 storage_path: Some("/activity-cancel/complete"),
23688 ..MockWorkerBehavior::default()
23689 });
23690 let worker =
23691 storage_worker(&server).with_storage_admission(Arc::new(AtomicBool::new(false)));
23692 let error = worker
23693 .client
23694 .complete_activity_task(
23695 "activity-cancel",
23696 "attempt-cancel",
23697 "worker",
23698 json!({}),
23699 DEFAULT_CODEC,
23700 )
23701 .await
23702 .expect_err("cancellation remains terminal");
23703 assert!(activity_task_rejection_is_final(&error));
23704 assert_identical_requests(
23705 &server,
23706 "/api/worker/activity-tasks/activity-cancel/complete",
23707 8,
23708 );
23709
23710 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23711 storage_refusals: usize::MAX,
23712 storage_path: Some("/poll"),
23713 storage_wrong_poll_id: true,
23714 ..MockWorkerBehavior::default()
23715 });
23716 assert!(storage_worker(&server).run_once().await.is_err());
23717 assert_eq!(server.request_count("/api/worker/workflow-tasks/poll"), 1);
23718 }
23719
23720 #[tokio::test]
23721 async fn storage_pollers_stop_when_the_run_future_is_aborted() {
23722 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23723 storage_refusals: usize::MAX,
23724 storage_path: Some("/poll"),
23725 ..MockWorkerBehavior::default()
23726 });
23727 let mut worker = storage_worker(&server).retry_policy(WorkerRetryPolicy::default());
23728 worker.register_activity("unused", |_, _| async { Ok(Value::Null) });
23729 let run = tokio::spawn(async move { worker.run().await });
23730 tokio::time::timeout(Duration::from_secs(2), async {
23731 while server.request_count("/api/worker/activity-tasks/poll") == 0 {
23732 tokio::time::sleep(Duration::from_millis(1)).await;
23733 }
23734 })
23735 .await
23736 .expect("poll started");
23737 run.abort();
23738 assert!(run.await.expect_err("cancelled run").is_cancelled());
23739 tokio::time::sleep(Duration::from_millis(250)).await;
23740 assert_eq!(server.request_count("/api/worker/activity-tasks/poll"), 1);
23741 }
23742
23743 #[tokio::test]
23744 async fn query_protocol_rejection_from_older_server_is_typed() {
23745 let server = MockWorkerServer::reject_query_protocol();
23746 let client = Client::builder(server.base_url())
23747 .timeout(Duration::from_secs(2))
23748 .build()
23749 .expect("client");
23750
23751 let error = client
23752 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
23753 .await
23754 .expect_err("server below query protocol floor must reject");
23755 let Error::Protocol(failure) = error else {
23756 panic!("expected typed protocol failure");
23757 };
23758
23759 assert_eq!(failure.status, 400);
23760 assert_eq!(failure.reason, "unsupported_protocol_version");
23761 assert_eq!(failure.supported_version.as_deref(), Some("1.7"));
23762 assert_eq!(
23763 failure.requested_version.as_deref(),
23764 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
23765 );
23766 assert_eq!(
23767 server
23768 .worker_protocol_for("/api/worker/query-tasks/poll")
23769 .as_deref(),
23770 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
23771 );
23772 }
23773
23774 #[tokio::test]
23775 async fn run_once_without_query_handlers_keeps_pre_query_server_compatibility() {
23776 let server = MockWorkerServer::reject_query_protocol();
23777 let client = Client::builder(server.base_url())
23778 .timeout(Duration::from_secs(2))
23779 .build()
23780 .expect("client");
23781 let mut worker = Worker::new(client, "rust-workers")
23782 .worker_id("baseline-worker")
23783 .poll_timeout(Duration::from_millis(10));
23784
23785 worker.register_workflow("baseline.workflow", |_ctx, _input| async move {
23786 Ok(Value::Null)
23787 });
23788
23789 assert_eq!(worker.run_once().await.expect("baseline run once"), 0);
23790 assert_eq!(
23791 server
23792 .worker_protocol_for("/api/worker/workflow-tasks/poll")
23793 .as_deref(),
23794 Some(WORKER_PROTOCOL_VERSION)
23795 );
23796 assert_eq!(
23797 server.worker_protocol_for("/api/worker/query-tasks/poll"),
23798 None,
23799 "a worker without query handlers must not use the query-task endpoint"
23800 );
23801 }
23802
23803 #[tokio::test]
23804 async fn completion_time_query_rejection_is_typed_without_stopping_worker() {
23805 let server = MockWorkerServer::reject_query_completion();
23806 let client = Client::builder(server.base_url())
23807 .timeout(Duration::from_secs(2))
23808 .build()
23809 .expect("client");
23810
23811 let error = client
23812 .complete_query_task("query-late", "late-worker", 1, json!(8), DEFAULT_CODEC)
23813 .await
23814 .expect_err("expired completion must be rejected");
23815 let Error::QueryFailed(failure) = error else {
23816 panic!("expected typed query failure");
23817 };
23818 assert_eq!(failure.status, 409);
23819 assert_eq!(failure.reason, "query_task_timed_out");
23820
23821 let mut worker = Worker::new(client, "rust-workers")
23822 .worker_id("late-worker")
23823 .poll_timeout(Duration::from_millis(10));
23824 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
23825 worker.register_query(
23826 "counter",
23827 "current",
23828 |_ctx, _args| async move { Ok(json!(8)) },
23829 );
23830
23831 assert_eq!(worker.run_once().await.expect("late task is handled"), 1);
23832 assert_eq!(
23833 worker
23834 .run_once()
23835 .await
23836 .expect("worker continues after late completion"),
23837 0
23838 );
23839 assert_eq!(
23840 server.request_count("/api/worker/query-tasks/query-late/complete"),
23841 2
23842 );
23843 assert_eq!(
23844 server.request_count("/api/worker/query-tasks/query-late/fail"),
23845 0,
23846 "a server completion rejection must not be reported as an encoding failure"
23847 );
23848 }
23849
23850 #[tokio::test]
23851 async fn normal_shutdown_joins_pollers_and_deregisters_once() {
23852 let server = MockWorkerServer::start();
23853 let client = Client::builder(server.base_url())
23854 .timeout(Duration::from_secs(2))
23855 .build()
23856 .expect("client");
23857 let mut worker = Worker::new(client, "rust-workers")
23858 .worker_id("joined-worker")
23859 .poll_timeout(Duration::from_millis(10));
23860 worker.register_workflow(
23861 "joined.workflow",
23862 |_ctx, _input| async move { Ok(Value::Null) },
23863 );
23864 worker.register_activity(
23865 "joined.activity",
23866 |_ctx, _input| async move { Ok(Value::Null) },
23867 );
23868 worker.register_query("joined.workflow", "state", |_ctx, _input| async move {
23869 Ok(Value::Null)
23870 });
23871
23872 worker
23873 .run_until(tokio::time::sleep(Duration::from_millis(20)))
23874 .await
23875 .expect("normal shutdown");
23876
23877 let deregistration_path = "/api/worker/registrations/mock-worker";
23878 assert_eq!(server.request_count(deregistration_path), 1);
23879 for poll_path in [
23880 "/api/worker/workflow-tasks/poll",
23881 "/api/worker/activity-tasks/poll",
23882 "/api/worker/query-tasks/poll",
23883 ] {
23884 assert!(server.request_count(poll_path) > 0, "missing {poll_path}");
23885 }
23886 assert_eq!(
23887 server.captured_paths().last().map(String::as_str),
23888 Some(deregistration_path),
23889 "deregistration must start only after every poller has joined"
23890 );
23891 }
23892
23893 #[tokio::test]
23894 async fn registration_failure_does_not_deregister() {
23895 let server = MockWorkerServer::rejected_registration();
23896 let client = Client::builder(server.base_url())
23897 .timeout(Duration::from_secs(2))
23898 .build()
23899 .expect("client");
23900 let worker = Worker::new(client, "rust-workers").worker_id("never-registered");
23901
23902 let error = worker
23903 .run_until(async {})
23904 .await
23905 .expect_err("registration must fail");
23906 assert!(matches!(
23907 error,
23908 Error::Http {
23909 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23910 ..
23911 }
23912 ));
23913 assert!(server
23914 .captured_paths()
23915 .iter()
23916 .all(|path| !path.starts_with("/api/worker/registrations/")));
23917 }
23918
23919 #[tokio::test]
23920 async fn protocol_116_server_rejects_occurrence_identity_worker_registration() {
23921 let server = MockWorkerServer::rejected_registration_protocol();
23922 let client = Client::builder(server.base_url())
23923 .timeout(Duration::from_secs(2))
23924 .build()
23925 .expect("client");
23926 let worker = Worker::new(client, "rust-workers").worker_id("protocol-117-worker");
23927
23928 let error = worker
23929 .run_until(async {})
23930 .await
23931 .expect_err("a protocol 1.16 server must reject this worker");
23932 let Error::Protocol(failure) = error else {
23933 panic!("expected typed protocol rejection");
23934 };
23935 assert_eq!(failure.reason, "unsupported_protocol_version");
23936 assert_eq!(failure.supported_version.as_deref(), Some("1.16"));
23937 assert_eq!(failure.requested_version.as_deref(), Some("1.17"));
23938 assert_eq!(
23939 server
23940 .worker_protocol_for("/api/worker/register")
23941 .as_deref(),
23942 Some(WORKER_PROTOCOL_VERSION)
23943 );
23944 }
23945
23946 #[tokio::test]
23947 async fn declined_registration_does_not_deregister() {
23948 let server = MockWorkerServer::declined_registration();
23949 let client = Client::builder(server.base_url())
23950 .timeout(Duration::from_secs(2))
23951 .build()
23952 .expect("client");
23953 let worker = Worker::new(client, "rust-workers").worker_id("declined-worker");
23954
23955 let error = worker
23956 .run_until(async {})
23957 .await
23958 .expect_err("declined registration must fail");
23959 assert!(matches!(error, Error::WorkerLoop(_)));
23960 assert!(error.to_string().contains("was not accepted"));
23961 assert!(server
23962 .captured_paths()
23963 .iter()
23964 .all(|path| !path.starts_with("/api/worker/registrations/")));
23965 }
23966
23967 #[tokio::test]
23968 async fn deregistration_http_failure_is_returned_after_normal_shutdown() {
23969 let server = MockWorkerServer::rejected_deregistration();
23970 let client = Client::builder(server.base_url())
23971 .timeout(Duration::from_secs(2))
23972 .build()
23973 .expect("client");
23974 let worker = Worker::new(client, "rust-workers").worker_id("forbidden-cleanup");
23975
23976 let error = worker
23977 .run_until(async {})
23978 .await
23979 .expect_err("deregistration must fail");
23980 assert!(matches!(
23981 error,
23982 Error::Http {
23983 status: reqwest::StatusCode::FORBIDDEN,
23984 ..
23985 }
23986 ));
23987 assert_eq!(
23988 server.request_count("/api/worker/registrations/mock-worker"),
23989 1
23990 );
23991 }
23992
23993 #[tokio::test]
23994 async fn deregistration_protocol_failure_is_returned_after_normal_shutdown() {
23995 let server = MockWorkerServer::rejected_deregistration_protocol();
23996 let client = Client::builder(server.base_url())
23997 .timeout(Duration::from_secs(2))
23998 .build()
23999 .expect("client");
24000 let worker = Worker::new(client, "rust-workers").worker_id("protocol-cleanup");
24001
24002 let error = worker
24003 .run_until(async {})
24004 .await
24005 .expect_err("protocol rejection must fail shutdown");
24006 let Error::Protocol(failure) = error else {
24007 panic!("expected typed protocol failure");
24008 };
24009 assert_eq!(failure.reason, "unsupported_protocol_version");
24010 assert_eq!(
24011 failure.requested_version.as_deref(),
24012 Some(WORKER_PROTOCOL_VERSION)
24013 );
24014 assert_eq!(
24015 server.request_count("/api/worker/registrations/mock-worker"),
24016 1
24017 );
24018 }
24019
24020 #[tokio::test]
24021 async fn primary_poller_error_retains_deregistration_failure_context() {
24022 let server = MockWorkerServer::unauthorized_polls_and_rejected_deregistration();
24023 let client = Client::builder(server.base_url())
24024 .timeout(Duration::from_secs(2))
24025 .build()
24026 .expect("client");
24027 let mut worker = Worker::new(client, "rust-workers")
24028 .worker_id("combined-failure")
24029 .poll_timeout(Duration::from_millis(10));
24030 worker.register_workflow("combined.workflow", |_ctx, _input| async move {
24031 Ok(Value::Null)
24032 });
24033
24034 let error = worker
24035 .run()
24036 .await
24037 .expect_err("worker and cleanup must fail");
24038 let summary = error.to_string();
24039 assert!(summary.contains("authentication_failed"));
24040 assert!(summary.contains("worker cannot deregister"));
24041 let Error::WorkerShutdown {
24042 primary,
24043 deregistration,
24044 } = error
24045 else {
24046 panic!("expected combined worker shutdown error");
24047 };
24048 assert!(matches!(
24049 *primary,
24050 Error::Http {
24051 status: reqwest::StatusCode::UNAUTHORIZED,
24052 ..
24053 }
24054 ));
24055 assert!(matches!(
24056 *deregistration,
24057 Error::Http {
24058 status: reqwest::StatusCode::FORBIDDEN,
24059 ..
24060 }
24061 ));
24062 assert_eq!(
24063 server.request_count("/api/worker/registrations/mock-worker"),
24064 1
24065 );
24066 }
24067
24068 #[tokio::test]
24069 async fn activity_only_worker_can_shutdown_without_workflow_poller() {
24070 let server = MockWorkerServer::start();
24071 let client = Client::builder(server.base_url())
24072 .timeout(Duration::from_secs(2))
24073 .build()
24074 .expect("client");
24075 let mut worker = Worker::new(client, "rust-workers")
24076 .worker_id("activity-only-worker")
24077 .poll_timeout(Duration::from_millis(10));
24078
24079 worker.register_activity(
24080 "activity.only",
24081 |_ctx, _args| async move { Ok(Value::Null) },
24082 );
24083
24084 worker.run_until(async {}).await.expect("run worker");
24085 }
24086
24087 #[tokio::test]
24088 async fn workflow_only_worker_can_shutdown_without_activity_poller() {
24089 let server = MockWorkerServer::start();
24090 let client = Client::builder(server.base_url())
24091 .timeout(Duration::from_secs(2))
24092 .build()
24093 .expect("client");
24094 let mut worker = Worker::new(client, "rust-workers")
24095 .worker_id("workflow-only-worker")
24096 .poll_timeout(Duration::from_millis(10));
24097
24098 worker.register_workflow(
24099 "workflow.only",
24100 |_ctx, _input| async move { Ok(Value::Null) },
24101 );
24102
24103 worker.run_until(async {}).await.expect("run worker");
24104 }
24105
24106 #[tokio::test]
24107 async fn worker_heartbeat_observer_receives_server_acknowledgements() {
24108 let server = MockWorkerServer::start();
24109 let client = Client::builder(server.base_url())
24110 .timeout(Duration::from_secs(2))
24111 .build()
24112 .expect("client");
24113 let observations = Arc::new(Mutex::new(Vec::new()));
24114 let observed = Arc::clone(&observations);
24115 let mut worker = Worker::new(client, "rust-workers")
24116 .worker_id("observed-heartbeat-worker")
24117 .poll_timeout(Duration::from_millis(10))
24118 .on_worker_heartbeat(move |observation| {
24119 observed
24120 .lock()
24121 .expect("heartbeat observations")
24122 .push(observation.clone());
24123 });
24124
24125 worker.register_workflow("workflow.observed", |_ctx, _input| async move {
24126 Ok(Value::Null)
24127 });
24128 let acknowledged = Arc::clone(&observations);
24129 worker
24130 .run_until(async move {
24131 tokio::time::timeout(Duration::from_secs(2), async move {
24132 loop {
24133 if !acknowledged
24134 .lock()
24135 .expect("heartbeat observations")
24136 .is_empty()
24137 {
24138 break;
24139 }
24140 tokio::time::sleep(Duration::from_millis(1)).await;
24141 }
24142 })
24143 .await
24144 .expect("heartbeat acknowledgement within timeout");
24145 })
24146 .await
24147 .expect("run worker");
24148
24149 let observations = observations.lock().expect("heartbeat observations");
24150 let first = observations.first().expect("heartbeat acknowledgement");
24151 assert_eq!(first.worker_id, "observed-heartbeat-worker");
24152 assert_eq!(first.task_queue, "rust-workers");
24153 assert!(first.acknowledged_at_unix_millis > 0);
24154 assert_eq!(first.acknowledgement, json!({}));
24155 }
24156
24157 #[tokio::test]
24158 async fn delayed_worker_heartbeat_keeps_cadence_and_pollers_live() {
24159 let server = MockWorkerServer::delayed_heartbeat_worker();
24160 let client = Client::builder(server.base_url())
24161 .timeout(Duration::from_secs(3))
24162 .build()
24163 .expect("client");
24164 let observations = Arc::new(Mutex::new(Vec::new()));
24165 let observed = Arc::clone(&observations);
24166 let mut worker = Worker::new(client, "rust-snapshot-workers")
24167 .worker_id("rust-snapshot-worker")
24168 .poll_timeout(Duration::from_millis(10))
24169 .on_worker_heartbeat(move |observation| {
24170 observed
24171 .lock()
24172 .expect("heartbeat observations")
24173 .push(observation.clone());
24174 });
24175
24176 worker.register_workflow("snapshot", |ctx, _input| async move {
24177 ctx.wait_signal("finish").await?;
24178 Ok(json!({"status": "finished"}))
24179 });
24180 worker.register_query("snapshot", "current", |ctx, _args| async move {
24181 Ok(json!(ctx
24182 .signals("increment")
24183 .iter()
24184 .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
24185 .sum::<i64>()))
24186 });
24187 worker.register_activity("cancel-aware", |_ctx, _args| async move {
24188 Ok(json!({"late": "completion"}))
24189 });
24190
24191 worker
24192 .run_until(tokio::time::sleep(Duration::from_millis(3_800)))
24193 .await
24194 .expect("delayed heartbeat must allow a clean worker shutdown");
24195
24196 let observations = observations.lock().expect("heartbeat observations");
24197 assert!(
24198 observations.len() >= 3,
24199 "the immediate heartbeat, delayed acknowledgement, and next cadence heartbeat must complete"
24200 );
24201 assert!(
24202 observations.windows(2).all(|pair| {
24203 pair[1].acknowledged_at_unix_millis
24204 .saturating_sub(pair[0].acknowledged_at_unix_millis)
24205 >= 850
24206 }),
24207 "successful acknowledgements must not catch up faster than the advertised one-second cadence: {observations:?}"
24208 );
24209 drop(observations);
24210
24211 let heartbeat_times = server.request_times("/api/worker/heartbeat");
24212 let delayed_request_at = *heartbeat_times
24213 .get(1)
24214 .expect("intentionally delayed heartbeat request");
24215 let delay_window_start = delayed_request_at + Duration::from_millis(100);
24216 let delay_window_end = delayed_request_at + Duration::from_millis(1_400);
24217 for path in [
24218 "/api/worker/workflow-tasks/poll",
24219 "/api/worker/activity-tasks/poll",
24220 "/api/worker/query-tasks/poll",
24221 ] {
24222 assert!(
24223 server
24224 .request_times(path)
24225 .iter()
24226 .any(|received_at| *received_at >= delay_window_start
24227 && *received_at <= delay_window_end),
24228 "{path} must keep polling while a heartbeat acknowledgement is delayed"
24229 );
24230 }
24231 assert!(
24232 server.request_count("/api/worker/workflow-tasks/snapshot-wait-3/fail") >= 1,
24233 "workflow work must be settled"
24234 );
24235 assert!(
24236 server.request_count("/api/worker/activity-tasks/activity-cancel/complete") >= 1,
24237 "activity work must be settled"
24238 );
24239 assert!(
24240 server.request_count("/api/worker/query-tasks/snapshot-current/complete") >= 1,
24241 "query work must be settled"
24242 );
24243 }
24244
24245 #[tokio::test]
24246 async fn retried_worker_heartbeat_restarts_the_advertised_cadence() {
24247 let server = MockWorkerServer::heartbeat_retry_worker();
24248 let client = Client::builder(server.base_url())
24249 .timeout(Duration::from_secs(2))
24250 .build()
24251 .expect("client");
24252 let observations = Arc::new(Mutex::new(Vec::new()));
24253 let observed = Arc::clone(&observations);
24254 let worker = Worker::new(client, "rust-workers")
24255 .worker_id("heartbeat-retry-worker")
24256 .retry_policy(WorkerRetryPolicy {
24257 max_retries: 1,
24258 initial_backoff: Duration::from_millis(300),
24259 max_backoff: Duration::from_millis(300),
24260 })
24261 .on_worker_heartbeat(move |observation| {
24262 observed
24263 .lock()
24264 .expect("heartbeat observations")
24265 .push(observation.clone());
24266 });
24267
24268 worker
24269 .run_until(tokio::time::sleep(Duration::from_millis(2_700)))
24270 .await
24271 .expect("retryable heartbeat failure must remain bounded and recover");
24272
24273 let observations = observations.lock().expect("heartbeat observations");
24274 assert!(observations.len() >= 3, "heartbeat retry must recover");
24275 assert!(
24276 observations.windows(2).all(|pair| {
24277 pair[1]
24278 .acknowledged_at_unix_millis
24279 .saturating_sub(pair[0].acknowledged_at_unix_millis)
24280 >= 850
24281 }),
24282 "a successful retry must start a fresh advertised cadence: {observations:?}"
24283 );
24284 assert_eq!(
24285 server.request_count("/api/worker/heartbeat"),
24286 observations.len() + 1,
24287 "one retryable failure must add exactly one bounded request"
24288 );
24289 }
24290
24291 #[tokio::test]
24292 async fn query_enabled_worker_ignores_unmatched_signals_then_completes_once() {
24293 let server = MockWorkerServer::waiting_query_worker();
24294 let client = Client::builder(server.base_url())
24295 .timeout(Duration::from_secs(2))
24296 .build()
24297 .expect("client");
24298 let observations = Arc::new(Mutex::new(Vec::new()));
24299 let observed = Arc::clone(&observations);
24300 let mut worker = Worker::new(client, "rust-snapshot-workers")
24301 .worker_id("rust-snapshot-worker")
24302 .poll_timeout(Duration::from_millis(10))
24303 .on_worker_heartbeat(move |observation| {
24304 observed
24305 .lock()
24306 .expect("heartbeat observations")
24307 .push(observation.clone());
24308 });
24309
24310 worker.register_workflow("snapshot", |ctx, _input| async move {
24311 ctx.wait_signal("finish").await?;
24312 Ok(json!({"status": "finished"}))
24313 });
24314 worker.register_query("snapshot", "current", |ctx, _args| async move {
24315 let current = ctx
24316 .signals("increment")
24317 .iter()
24318 .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
24319 .sum::<i64>();
24320 Ok(json!(current))
24321 });
24322 worker.register_update("snapshot", "replace", |_ctx, args| async move { Ok(args) });
24323
24324 worker
24325 .run_until(tokio::time::sleep(Duration::from_millis(3_200)))
24326 .await
24327 .expect("pending workflow and query poller must remain live until shutdown");
24328
24329 assert!(
24330 observations.lock().expect("heartbeat observations").len() >= 4,
24331 "the immediate heartbeat and at least three advertised one-second intervals must be acknowledged"
24332 );
24333 assert!(
24334 server.request_count("/api/worker/workflow-tasks/poll") >= 3,
24335 "workflow polling must continue after empty replay acknowledgements"
24336 );
24337 assert!(
24338 server.request_count("/api/worker/query-tasks/poll") >= 2,
24339 "query polling must continue after serving the current query"
24340 );
24341 assert_eq!(
24342 server.request_body("/api/worker/register")["capabilities"],
24343 json!([
24344 CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY,
24345 DURABLE_SELECTION_CAPABILITY,
24346 MEMO_UPSERTS_CAPABILITY,
24347 TYPED_SEARCH_ATTRIBUTES_CAPABILITY,
24348 QUERY_TASKS_CAPABILITY,
24349 WORKFLOW_UPDATES_CAPABILITY,
24350 MESSAGE_STREAMS_CAPABILITY
24351 ])
24352 );
24353 assert_eq!(
24354 server.request_body("/api/worker/register")["workflow_command_contracts"]["snapshot"],
24355 json!({
24356 "queries": ["current"],
24357 "query_contracts": [],
24358 "signals": [],
24359 "signal_contracts": [],
24360 "updates": ["replace"],
24361 "update_contracts": [],
24362 "update_validators": [],
24363 })
24364 );
24365
24366 let opened = server.request_body("/api/worker/workflow-tasks/snapshot-open/complete");
24367 assert_eq!(
24368 opened["commands"],
24369 json!([{
24370 "type": "open_signal_wait",
24371 "signal_name": "finish",
24372 }])
24373 );
24374
24375 for task_id in ["snapshot-wait-3", "snapshot-wait-5"] {
24376 let fail_path = format!("/api/worker/workflow-tasks/{task_id}/fail");
24377 let completion_path = format!("/api/worker/workflow-tasks/{task_id}/complete");
24378 let failure = server.request_body(&fail_path);
24379 assert_eq!(
24380 failure["failure"]["type"],
24381 WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE
24382 );
24383 assert_eq!(server.request_count(&completion_path), 0);
24384 }
24385
24386 let query_completion =
24387 server.request_body("/api/worker/query-tasks/snapshot-current/complete");
24388 assert_eq!(query_completion["result"], json!(8));
24389
24390 let terminal_path = "/api/worker/workflow-tasks/snapshot-finish/complete";
24391 assert_eq!(
24392 server.request_count(terminal_path),
24393 1,
24394 "the matching signal must settle the workflow exactly once"
24395 );
24396 let terminal = server.request_body(terminal_path);
24397 assert_eq!(terminal["commands"].as_array().map(Vec::len), Some(1));
24398 assert_eq!(terminal["commands"][0]["type"], "complete_workflow");
24399 assert_eq!(
24400 decode_wire_value(&terminal["commands"][0]["result"], DEFAULT_CODEC)
24401 .expect("terminal workflow result"),
24402 json!({"status": "finished"})
24403 );
24404 }
24405
24406 #[tokio::test]
24407 async fn worker_retries_poll_and_heartbeat_transport_failures_independently() {
24408 let server = MockWorkerServer::transient_worker_failures();
24409 let client = Client::builder(server.base_url())
24410 .timeout(Duration::from_secs(2))
24411 .build()
24412 .expect("client");
24413 let mut worker = Worker::new(client, "rust-workers")
24414 .worker_id("retry-worker")
24415 .poll_timeout(Duration::from_millis(10))
24416 .retry_policy(WorkerRetryPolicy {
24417 max_retries: 2,
24418 initial_backoff: Duration::from_millis(1),
24419 max_backoff: Duration::from_millis(1),
24420 });
24421 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
24422 worker.register_activity(
24423 "counter.activity",
24424 |_ctx, _input| async move { Ok(Value::Null) },
24425 );
24426 worker.register_query(
24427 "counter",
24428 "current",
24429 |_ctx, _args| async move { Ok(json!(8)) },
24430 );
24431
24432 worker
24433 .run_until(tokio::time::sleep(Duration::from_millis(75)))
24434 .await
24435 .expect("transient failures must not stop the worker");
24436
24437 for path in [
24438 "/api/worker/heartbeat",
24439 "/api/worker/workflow-tasks/poll",
24440 "/api/worker/activity-tasks/poll",
24441 "/api/worker/query-tasks/poll",
24442 ] {
24443 assert!(
24444 server.request_count(path) >= 2,
24445 "{path} must continue after its transient failure"
24446 );
24447 }
24448 }
24449
24450 #[tokio::test]
24451 async fn worker_continues_after_long_poll_capacity_backpressure() {
24452 let server = MockWorkerServer::capacity_limited_activity_poll();
24453 let client = Client::builder(server.base_url())
24454 .timeout(Duration::from_secs(2))
24455 .build()
24456 .expect("client");
24457 let mut worker = Worker::new(client, "rust-workers")
24458 .worker_id("capacity-worker")
24459 .poll_timeout(Duration::from_millis(10))
24460 .retry_policy(WorkerRetryPolicy {
24461 max_retries: 0,
24462 initial_backoff: Duration::from_millis(1),
24463 max_backoff: Duration::from_millis(1),
24464 });
24465 worker.register_activity("capacity.activity", |_ctx, _input| async move {
24466 Ok(json!({"handled": true}))
24467 });
24468
24469 worker
24470 .run_until(tokio::time::sleep(Duration::from_millis(50)))
24471 .await
24472 .expect("capacity backpressure must not stop the worker");
24473
24474 assert!(
24475 server.request_count("/api/worker/activity-tasks/poll") >= 2,
24476 "the activity poller must continue after capacity backpressure"
24477 );
24478 assert_eq!(
24479 server.request_count("/api/worker/activity-tasks/capacity-activity/complete"),
24480 1,
24481 "the worker must complete work returned after capacity recovers"
24482 );
24483 }
24484
24485 #[test]
24486 fn worker_poll_capacity_backpressure_requires_the_typed_retryable_contract() {
24487 let capacity = Error::Http {
24488 status: reqwest::StatusCode::TOO_MANY_REQUESTS,
24489 body: r#"{"poll_status":"long_poll_capacity_exhausted","retryable":true,"retry_after_seconds":3}"#.to_string(),
24490 };
24491 assert_eq!(
24492 worker_poll_capacity_retry_after(&capacity),
24493 Some(Duration::from_secs(3))
24494 );
24495
24496 let rejected_capacity = Error::Http {
24497 status: reqwest::StatusCode::TOO_MANY_REQUESTS,
24498 body: r#"{"reason":"long_poll_capacity_exhausted","retryable":false,"retry_after_seconds":3}"#.to_string(),
24499 };
24500 assert_eq!(worker_poll_capacity_retry_after(&rejected_capacity), None);
24501 assert!(!worker_operation_is_retryable(&rejected_capacity));
24502
24503 let ordinary_rate_limit = Error::Http {
24504 status: reqwest::StatusCode::TOO_MANY_REQUESTS,
24505 body: r#"{"reason":"rate_limited","retryable":true,"retry_after_seconds":3}"#
24506 .to_string(),
24507 };
24508 assert_eq!(worker_poll_capacity_retry_after(&ordinary_rate_limit), None);
24509 assert!(worker_operation_is_retryable(&ordinary_rate_limit));
24510 }
24511
24512 #[tokio::test]
24513 async fn worker_bounds_transport_retries() {
24514 let server = MockWorkerServer::unavailable_polls();
24515 let client = Client::builder(server.base_url())
24516 .timeout(Duration::from_secs(2))
24517 .build()
24518 .expect("client");
24519 let mut worker = Worker::new(client, "rust-workers")
24520 .worker_id("bounded-retry-worker")
24521 .poll_timeout(Duration::from_millis(10))
24522 .retry_policy(WorkerRetryPolicy {
24523 max_retries: 2,
24524 initial_backoff: Duration::from_millis(1),
24525 max_backoff: Duration::from_millis(1),
24526 });
24527 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
24528
24529 let error = worker.run().await.expect_err("retry bound must terminate");
24530 assert!(matches!(error, Error::Transport(_)));
24531 assert_eq!(
24532 server.request_count("/api/worker/workflow-tasks/poll"),
24533 3,
24534 "one initial request plus exactly two retries"
24535 );
24536 }
24537
24538 #[tokio::test]
24539 async fn worker_retry_policy_can_disable_poll_retries() {
24540 let server = MockWorkerServer::unavailable_polls();
24541 let client = Client::builder(server.base_url())
24542 .timeout(Duration::from_secs(2))
24543 .build()
24544 .expect("client");
24545 let mut worker = Worker::new(client, "rust-workers")
24546 .worker_id("no-retry-worker")
24547 .poll_timeout(Duration::from_millis(10))
24548 .retry_policy(WorkerRetryPolicy {
24549 max_retries: 0,
24550 initial_backoff: Duration::from_millis(1),
24551 max_backoff: Duration::from_millis(1),
24552 });
24553 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
24554
24555 let error = worker
24556 .run_once()
24557 .await
24558 .expect_err("disabled retries must return the first transport failure");
24559 assert!(matches!(error, Error::Transport(_)));
24560 assert_eq!(
24561 server.request_count("/api/worker/workflow-tasks/poll"),
24562 1,
24563 "max_retries=0 must send only the initial request"
24564 );
24565 }
24566
24567 #[tokio::test]
24568 async fn worker_does_not_retry_authentication_failures() {
24569 let server = MockWorkerServer::unauthorized_polls();
24570 let client = Client::builder(server.base_url())
24571 .timeout(Duration::from_secs(2))
24572 .build()
24573 .expect("client");
24574 let mut worker = Worker::new(client, "rust-workers")
24575 .worker_id("unauthorized-worker")
24576 .poll_timeout(Duration::from_millis(10));
24577 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
24578
24579 let error = worker
24580 .run()
24581 .await
24582 .expect_err("authentication must terminate");
24583 let Error::Http { status, body } = error else {
24584 panic!("expected stable HTTP authentication error");
24585 };
24586 assert_eq!(status, reqwest::StatusCode::UNAUTHORIZED);
24587 assert!(body.contains("authentication_failed"));
24588 assert_eq!(
24589 server.request_count("/api/worker/workflow-tasks/poll"),
24590 1,
24591 "authentication failures must not be retried"
24592 );
24593 }
24594
24595 #[derive(Clone, Debug)]
24596 struct CapturedRequest {
24597 headers: String,
24598 method: String,
24599 path: String,
24600 authorization: Option<String>,
24601 namespace: Option<String>,
24602 worker_protocol: Option<String>,
24603 control_protocol: Option<String>,
24604 body: String,
24605 received_at: Instant,
24606 }
24607
24608 struct MockWorkerServer {
24609 addr: SocketAddr,
24610 stop: Arc<AtomicBool>,
24611 requests: Arc<Mutex<Vec<CapturedRequest>>>,
24612 thread: Option<thread::JoinHandle<()>>,
24613 }
24614
24615 type RequestOverride = fn(&str, &str, usize) -> Option<(&'static str, String)>;
24616
24617 #[derive(Clone, Copy, Default)]
24618 struct MockWorkerBehavior {
24619 response_override: Option<fn(&str) -> Option<(&'static str, String)>>,
24620 request_override: Option<RequestOverride>,
24621 storage_refusals: usize,
24622 storage_path: Option<&'static str>,
24623 storage_unavailable: bool,
24624 storage_mid_poll: bool,
24625 storage_activity: bool,
24626 storage_query: bool,
24627 storage_wrong_poll_id: bool,
24628 reject_query_protocol: bool,
24629 reject_query_completion: bool,
24630 waiting_query_worker: bool,
24631 decline_registration: bool,
24632 complete_named_signal: bool,
24633 poll_failures_per_path: usize,
24634 long_poll_capacity_responses_per_path: usize,
24635 heartbeat_failures: usize,
24636 heartbeat_failure_request: Option<usize>,
24637 delayed_heartbeat_request: Option<usize>,
24638 heartbeat_response_delay: Duration,
24639 concurrent_requests: bool,
24640 unauthorized_polls: bool,
24641 reject_registration: bool,
24642 reject_registration_protocol: bool,
24643 reject_deregistration: bool,
24644 reject_deregistration_protocol: bool,
24645 cancelled_activity: bool,
24646 draining_polls: bool,
24647 invalid_task_payload_codec: Option<InvalidTaskPayloadCodec>,
24648 workflow_completion_status: Option<&'static str>,
24649 workflow_completion_body: Option<&'static str>,
24650 }
24651
24652 impl MockWorkerServer {
24653 fn start() -> Self {
24654 Self::start_with_behavior(MockWorkerBehavior::default())
24655 }
24656
24657 fn reject_query_protocol() -> Self {
24658 Self::start_with_behavior(MockWorkerBehavior {
24659 reject_query_protocol: true,
24660 ..MockWorkerBehavior::default()
24661 })
24662 }
24663
24664 fn reject_query_completion() -> Self {
24665 Self::start_with_behavior(MockWorkerBehavior {
24666 reject_query_completion: true,
24667 ..MockWorkerBehavior::default()
24668 })
24669 }
24670
24671 fn waiting_query_worker() -> Self {
24672 Self::start_with_behavior(MockWorkerBehavior {
24673 waiting_query_worker: true,
24674 complete_named_signal: true,
24675 ..MockWorkerBehavior::default()
24676 })
24677 }
24678
24679 fn transient_worker_failures() -> Self {
24680 Self::start_with_behavior(MockWorkerBehavior {
24681 poll_failures_per_path: 1,
24682 heartbeat_failures: 1,
24683 ..MockWorkerBehavior::default()
24684 })
24685 }
24686
24687 fn consecutive_poll_failures(count: usize) -> Self {
24688 Self::start_with_behavior(MockWorkerBehavior {
24689 poll_failures_per_path: count,
24690 ..MockWorkerBehavior::default()
24691 })
24692 }
24693
24694 fn capacity_limited_activity_poll() -> Self {
24695 Self::start_with_behavior(MockWorkerBehavior {
24696 long_poll_capacity_responses_per_path: 1,
24697 ..MockWorkerBehavior::default()
24698 })
24699 }
24700
24701 fn delayed_heartbeat_worker() -> Self {
24702 Self::start_with_behavior(MockWorkerBehavior {
24703 waiting_query_worker: true,
24704 delayed_heartbeat_request: Some(2),
24705 heartbeat_response_delay: Duration::from_millis(1_500),
24706 concurrent_requests: true,
24707 cancelled_activity: true,
24708 ..MockWorkerBehavior::default()
24709 })
24710 }
24711
24712 fn heartbeat_retry_worker() -> Self {
24713 Self::start_with_behavior(MockWorkerBehavior {
24714 waiting_query_worker: true,
24715 heartbeat_failure_request: Some(2),
24716 concurrent_requests: true,
24717 ..MockWorkerBehavior::default()
24718 })
24719 }
24720
24721 fn unavailable_polls() -> Self {
24722 Self::start_with_behavior(MockWorkerBehavior {
24723 poll_failures_per_path: usize::MAX,
24724 ..MockWorkerBehavior::default()
24725 })
24726 }
24727
24728 fn unauthorized_polls() -> Self {
24729 Self::start_with_behavior(MockWorkerBehavior {
24730 unauthorized_polls: true,
24731 ..MockWorkerBehavior::default()
24732 })
24733 }
24734
24735 fn rejected_registration() -> Self {
24736 Self::start_with_behavior(MockWorkerBehavior {
24737 reject_registration: true,
24738 ..MockWorkerBehavior::default()
24739 })
24740 }
24741
24742 fn rejected_registration_protocol() -> Self {
24743 Self::start_with_behavior(MockWorkerBehavior {
24744 reject_registration_protocol: true,
24745 ..MockWorkerBehavior::default()
24746 })
24747 }
24748
24749 fn declined_registration() -> Self {
24750 Self::start_with_behavior(MockWorkerBehavior {
24751 decline_registration: true,
24752 ..MockWorkerBehavior::default()
24753 })
24754 }
24755
24756 fn rejected_deregistration() -> Self {
24757 Self::start_with_behavior(MockWorkerBehavior {
24758 reject_deregistration: true,
24759 ..MockWorkerBehavior::default()
24760 })
24761 }
24762
24763 fn rejected_deregistration_protocol() -> Self {
24764 Self::start_with_behavior(MockWorkerBehavior {
24765 reject_deregistration_protocol: true,
24766 ..MockWorkerBehavior::default()
24767 })
24768 }
24769
24770 fn unauthorized_polls_and_rejected_deregistration() -> Self {
24771 Self::start_with_behavior(MockWorkerBehavior {
24772 unauthorized_polls: true,
24773 reject_deregistration: true,
24774 ..MockWorkerBehavior::default()
24775 })
24776 }
24777
24778 fn cancelled_activity() -> Self {
24779 Self::start_with_behavior(MockWorkerBehavior {
24780 cancelled_activity: true,
24781 ..MockWorkerBehavior::default()
24782 })
24783 }
24784
24785 fn draining_polls() -> Self {
24786 Self::start_with_behavior(MockWorkerBehavior {
24787 draining_polls: true,
24788 ..MockWorkerBehavior::default()
24789 })
24790 }
24791
24792 fn invalid_task_payload_codec(codec: InvalidTaskPayloadCodec) -> Self {
24793 Self::start_with_behavior(MockWorkerBehavior {
24794 invalid_task_payload_codec: Some(codec),
24795 ..MockWorkerBehavior::default()
24796 })
24797 }
24798
24799 fn workflow_completion(status: &'static str, body: &'static str) -> Self {
24800 Self::start_with_behavior(MockWorkerBehavior {
24801 workflow_completion_status: Some(status),
24802 workflow_completion_body: Some(body),
24803 ..MockWorkerBehavior::default()
24804 })
24805 }
24806
24807 fn start_with_behavior(behavior: MockWorkerBehavior) -> Self {
24808 let listener = TcpListener::bind("127.0.0.1:0").expect("bind mock server");
24809 listener
24810 .set_nonblocking(true)
24811 .expect("configure mock listener");
24812 let addr = listener.local_addr().expect("mock server address");
24813 let stop = Arc::new(AtomicBool::new(false));
24814 let server_stop = Arc::clone(&stop);
24815 let requests = Arc::new(Mutex::new(Vec::new()));
24816 let server_requests = Arc::clone(&requests);
24817 let thread = thread::spawn(move || {
24818 let mut request_threads = Vec::new();
24819 while !server_stop.load(Ordering::SeqCst) {
24820 match listener.accept() {
24821 Ok((mut stream, _)) => {
24822 if behavior.concurrent_requests {
24823 let requests = Arc::clone(&server_requests);
24824 request_threads.push(thread::spawn(move || {
24825 handle_mock_worker_request(&mut stream, &requests, behavior)
24826 }));
24827 } else {
24828 handle_mock_worker_request(&mut stream, &server_requests, behavior);
24829 }
24830 }
24831 Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
24832 let mut index = 0;
24833 while index < request_threads.len() {
24834 if request_threads[index].is_finished() {
24835 request_threads
24836 .swap_remove(index)
24837 .join()
24838 .expect("join mock request");
24839 } else {
24840 index += 1;
24841 }
24842 }
24843 thread::sleep(Duration::from_millis(5));
24844 }
24845 Err(_) => break,
24846 }
24847 }
24848 for request_thread in request_threads {
24849 request_thread.join().expect("join mock request");
24850 }
24851 });
24852
24853 Self {
24854 addr,
24855 stop,
24856 requests,
24857 thread: Some(thread),
24858 }
24859 }
24860
24861 fn base_url(&self) -> String {
24862 format!("http://{}", self.addr)
24863 }
24864
24865 fn worker_protocol_for(&self, path: &str) -> Option<String> {
24866 self.requests
24867 .lock()
24868 .expect("captured requests")
24869 .iter()
24870 .find(|request| request.path == path)
24871 .and_then(|request| request.worker_protocol.clone())
24872 }
24873
24874 fn control_protocol_for(&self, path: &str) -> Option<String> {
24875 self.requests
24876 .lock()
24877 .expect("captured requests")
24878 .iter()
24879 .find(|request| request.path == path)
24880 .and_then(|request| request.control_protocol.clone())
24881 }
24882
24883 fn method_for(&self, path: &str) -> Option<String> {
24884 self.requests
24885 .lock()
24886 .expect("captured requests")
24887 .iter()
24888 .find(|request| request.path == path)
24889 .map(|request| request.method.clone())
24890 }
24891
24892 fn authorization_for(&self, path: &str) -> Option<String> {
24893 self.requests
24894 .lock()
24895 .expect("captured requests")
24896 .iter()
24897 .find(|request| request.path == path)
24898 .and_then(|request| request.authorization.clone())
24899 }
24900
24901 fn namespace_for(&self, path: &str) -> Option<String> {
24902 self.requests
24903 .lock()
24904 .expect("captured requests")
24905 .iter()
24906 .find(|request| request.path == path)
24907 .and_then(|request| request.namespace.clone())
24908 }
24909
24910 fn request_count(&self, path: &str) -> usize {
24911 self.requests
24912 .lock()
24913 .expect("captured requests")
24914 .iter()
24915 .filter(|request| request.path == path)
24916 .count()
24917 }
24918
24919 fn captured_paths(&self) -> Vec<String> {
24920 self.requests
24921 .lock()
24922 .expect("captured requests")
24923 .iter()
24924 .map(|request| request.path.clone())
24925 .collect()
24926 }
24927
24928 fn request_times(&self, path: &str) -> Vec<Instant> {
24929 self.requests
24930 .lock()
24931 .expect("captured requests")
24932 .iter()
24933 .filter(|request| request.path == path)
24934 .map(|request| request.received_at)
24935 .collect()
24936 }
24937
24938 fn request_body(&self, path: &str) -> Value {
24939 let requests = self.requests.lock().expect("captured requests");
24940 let body = &requests
24941 .iter()
24942 .find(|request| request.path == path)
24943 .unwrap_or_else(|| panic!("missing request for {path}"))
24944 .body;
24945 serde_json::from_str(body).unwrap_or_else(|error| {
24946 panic!("invalid JSON request body for {path}: {error}: {body:?}")
24947 })
24948 }
24949
24950 fn request_bodies(&self, path: &str) -> Vec<Value> {
24951 self.requests
24952 .lock()
24953 .expect("captured requests")
24954 .iter()
24955 .filter(|request| request.path == path)
24956 .map(|request| {
24957 serde_json::from_str(&request.body).unwrap_or_else(|error| {
24958 panic!(
24959 "invalid JSON request body for {path}: {error}: {:?}",
24960 request.body
24961 )
24962 })
24963 })
24964 .collect()
24965 }
24966 }
24967
24968 impl Drop for MockWorkerServer {
24969 fn drop(&mut self) {
24970 self.stop.store(true, Ordering::SeqCst);
24971 let _ = TcpStream::connect(self.addr);
24972
24973 if let Some(thread) = self.thread.take() {
24974 thread.join().expect("join mock server");
24975 }
24976 }
24977 }
24978
24979 fn handle_mock_worker_request(
24980 stream: &mut TcpStream,
24981 requests: &Arc<Mutex<Vec<CapturedRequest>>>,
24982 behavior: MockWorkerBehavior,
24983 ) {
24984 let _ = stream.set_read_timeout(Some(Duration::from_millis(200)));
24985 let mut buffer = [0_u8; 8192];
24986 let mut request = Vec::new();
24987
24988 loop {
24989 match stream.read(&mut buffer) {
24990 Ok(0) => break,
24991 Ok(read) => {
24992 request.extend_from_slice(&buffer[..read]);
24993 if mock_request_is_complete(&request) {
24994 break;
24995 }
24996 }
24997 Err(error)
24998 if matches!(
24999 error.kind(),
25000 std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut
25001 ) =>
25002 {
25003 break;
25004 }
25005 Err(_) => return,
25006 }
25007 }
25008
25009 let request = String::from_utf8_lossy(&request);
25010 let body = request
25011 .split_once("\r\n\r\n")
25012 .map(|(_, body)| body)
25013 .unwrap_or_default();
25014 let path = request
25015 .lines()
25016 .next()
25017 .and_then(|line| line.split_whitespace().nth(1))
25018 .unwrap_or_default();
25019 let method = request
25020 .lines()
25021 .next()
25022 .and_then(|line| line.split_whitespace().next())
25023 .unwrap_or_default();
25024 let authorization = request.lines().find_map(|line| {
25025 let (name, value) = line.split_once(':')?;
25026 name.eq_ignore_ascii_case("Authorization")
25027 .then(|| value.trim().to_string())
25028 });
25029 let namespace = request.lines().find_map(|line| {
25030 let (name, value) = line.split_once(':')?;
25031 name.eq_ignore_ascii_case("X-Namespace")
25032 .then(|| value.trim().to_string())
25033 });
25034 let worker_protocol = request.lines().find_map(|line| {
25035 let (name, value) = line.split_once(':')?;
25036 name.eq_ignore_ascii_case("X-Durable-Workflow-Protocol-Version")
25037 .then(|| value.trim().to_string())
25038 });
25039 let control_protocol = request.lines().find_map(|line| {
25040 let (name, value) = line.split_once(':')?;
25041 name.eq_ignore_ascii_case("X-Durable-Workflow-Control-Plane-Version")
25042 .then(|| value.trim().to_string())
25043 });
25044 let request_number = {
25045 let mut requests = requests.lock().expect("captured requests");
25046 requests.push(CapturedRequest {
25047 headers: request
25048 .split_once("\r\n\r\n")
25049 .map_or("", |(headers, _)| headers)
25050 .to_owned(),
25051 method: method.to_string(),
25052 path: path.to_string(),
25053 authorization,
25054 namespace,
25055 worker_protocol: worker_protocol.clone(),
25056 control_protocol,
25057 body: body.to_string(),
25058 received_at: Instant::now(),
25059 });
25060 requests
25061 .iter()
25062 .filter(|request| request.path == path)
25063 .count()
25064 };
25065
25066 if let Some(response) = behavior
25067 .request_override
25068 .and_then(|handler| handler(path, body, request_number))
25069 {
25070 write_mock_response(stream, response.0, &response.1);
25071 return;
25072 }
25073 if let Some(response) = behavior.response_override.and_then(|handler| handler(path)) {
25074 write_mock_response(stream, response.0, &response.1);
25075 return;
25076 }
25077 if path.ends_with("/poll") && request_number <= behavior.poll_failures_per_path {
25078 return;
25079 }
25080 let pressure_path = behavior
25081 .storage_path
25082 .is_some_and(|part| path.contains(part));
25083 let prior_failures = if path.ends_with("/poll") {
25084 behavior.poll_failures_per_path
25085 } else {
25086 0
25087 };
25088 if pressure_path
25089 && request_number.saturating_sub(prior_failures) <= behavior.storage_refusals
25090 {
25091 let request_body: Value = serde_json::from_str(body).unwrap_or(Value::Null);
25092 let poll_id = path
25093 .ends_with("/poll")
25094 .then(|| request_body["poll_request_id"].as_str().unwrap_or(""));
25095 let mut refusal = storage_refusal(
25096 poll_id,
25097 behavior.storage_unavailable,
25098 behavior.storage_mid_poll,
25099 );
25100 if behavior.storage_wrong_poll_id {
25101 refusal["poll_request_id"] = json!("wrong-poll");
25102 }
25103 write_mock_response(stream, "503 Service Unavailable", &refusal.to_string());
25104 return;
25105 }
25106 if path.contains("/storage-task/")
25107 || path.contains("/storage-activity/")
25108 || path.contains("/storage-query/")
25109 {
25110 write_mock_response(stream, "200 OK", "{}");
25111 return;
25112 }
25113 if behavior.storage_query && path == "/api/worker/query-tasks/poll" && request_number == 1 {
25114 write_mock_response(stream, "200 OK", &json!({"task":{
25115 "query_task_id":"storage-query", "query_task_attempt":7, "workflow_type":"storage.workflow",
25116 "query_name":"state", "workflow_id":"workflow", "run_id":"run", "payload_codec":"avro",
25117 "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC).unwrap(),
25118 "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC).unwrap(),
25119 "history_events":[], "run_status":"waiting", "lease_owner":"storage-worker"
25120 }}).to_string());
25121 return;
25122 }
25123 if behavior.storage_activity
25124 && path == "/api/worker/activity-tasks/poll"
25125 && request_number == 1
25126 {
25127 write_mock_response(stream, "200 OK", &json!({"task":{
25128 "task_id":"storage-activity", "activity_attempt_id":"storage-attempt", "activity_type":"storage.activity",
25129 "payload_codec":"avro", "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC).unwrap(),
25130 "attempt_number":7, "lease_owner":"storage-worker"
25131 }}).to_string());
25132 return;
25133 }
25134
25135 if path == "/api/worker/register" {
25136 if behavior.reject_registration_protocol {
25137 write_mock_response(
25138 stream,
25139 "400 Bad Request",
25140 r#"{"reason":"unsupported_protocol_version","message":"condition-wait occurrence identity requires worker protocol 1.17","supported_version":"1.16","requested_version":"1.17"}"#,
25141 );
25142 return;
25143 }
25144 if behavior.reject_registration {
25145 write_mock_response(
25146 stream,
25147 "503 Service Unavailable",
25148 r#"{"reason":"registration_unavailable","message":"registration failed"}"#,
25149 );
25150 return;
25151 }
25152 }
25153
25154 if path.starts_with("/api/worker/registrations/") {
25155 if behavior.reject_deregistration_protocol {
25156 write_mock_response(
25157 stream,
25158 "400 Bad Request",
25159 r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.17","requested_version":"1.19"}"#,
25160 );
25161 } else if behavior.reject_deregistration {
25162 write_mock_response(
25163 stream,
25164 "403 Forbidden",
25165 r#"{"reason":"authorization_failed","message":"worker cannot deregister"}"#,
25166 );
25167 } else {
25168 write_mock_response(
25169 stream,
25170 "200 OK",
25171 r#"{"worker_id":"deregistered-worker","outcome":"deregistered","recovered_workflow_task_count":2}"#,
25172 );
25173 }
25174 return;
25175 }
25176
25177 let is_poll = matches!(
25178 path,
25179 "/api/worker/workflow-tasks/poll"
25180 | "/api/worker/activity-tasks/poll"
25181 | "/api/worker/query-tasks/poll"
25182 );
25183 if is_poll && request_number <= behavior.long_poll_capacity_responses_per_path {
25184 write_mock_response(
25185 stream,
25186 "429 Too Many Requests",
25187 r#"{"task":null,"poll_status":"long_poll_capacity_exhausted","reason":"long_poll_capacity_exhausted","retryable":true,"retry_after_seconds":1}"#,
25188 );
25189 return;
25190 }
25191 if is_poll && request_number <= behavior.poll_failures_per_path {
25192 return;
25193 }
25194 if path == "/api/worker/heartbeat" && request_number <= behavior.heartbeat_failures {
25195 return;
25196 }
25197 if path == "/api/worker/heartbeat"
25198 && behavior.heartbeat_failure_request == Some(request_number)
25199 {
25200 return;
25201 }
25202 if path == "/api/worker/heartbeat"
25203 && behavior.delayed_heartbeat_request == Some(request_number)
25204 {
25205 thread::sleep(behavior.heartbeat_response_delay);
25206 }
25207 if behavior.unauthorized_polls && is_poll {
25208 write_mock_response(
25209 stream,
25210 "401 Unauthorized",
25211 r#"{"reason":"authentication_failed","message":"invalid worker token"}"#,
25212 );
25213 return;
25214 }
25215 if behavior.draining_polls && is_poll {
25216 write_mock_response(
25217 stream,
25218 "409 Conflict",
25219 r#"{"task":null,"poll_status":"draining","reason":"worker_draining","worker_status":"draining","drain_intent":"draining"}"#,
25220 );
25221 return;
25222 }
25223
25224 if let Some(codec_case) = behavior.invalid_task_payload_codec {
25225 if is_poll && request_number == 1 {
25226 let mut task = match path {
25227 "/api/worker/workflow-tasks/poll" => json!({
25228 "task_id": "codec-workflow",
25229 "workflow_type": "codec.workflow",
25230 "payload_codec": DEFAULT_CODEC,
25231 "workflow_task_attempt": 1,
25232 "lease_owner": "codec-worker"
25233 }),
25234 "/api/worker/activity-tasks/poll" => json!({
25235 "task_id": "codec-activity",
25236 "activity_attempt_id": "codec-activity-attempt",
25237 "activity_type": "codec.activity",
25238 "payload_codec": DEFAULT_CODEC,
25239 "attempt_number": 1,
25240 "lease_owner": "codec-worker"
25241 }),
25242 "/api/worker/query-tasks/poll" => json!({
25243 "query_task_id": "codec-query",
25244 "query_task_attempt": 1,
25245 "workflow_type": "codec.workflow",
25246 "query_name": "known",
25247 "payload_codec": DEFAULT_CODEC,
25248 "lease_owner": "codec-worker"
25249 }),
25250 _ => unreachable!("is_poll limits task codec probe paths"),
25251 };
25252 codec_case.apply(&mut task);
25253 write_mock_response(stream, "200 OK", &json!({"task": task}).to_string());
25254 return;
25255 }
25256
25257 if matches!(
25258 path,
25259 "/api/worker/workflow-tasks/codec-workflow/fail"
25260 | "/api/worker/activity-tasks/codec-activity/fail"
25261 | "/api/worker/query-tasks/codec-query/fail"
25262 ) {
25263 write_mock_response(stream, "200 OK", r#"{"outcome":"failed"}"#);
25264 return;
25265 }
25266 }
25267
25268 if behavior.reject_query_protocol && path.starts_with("/api/worker/query-tasks/") {
25269 let requested_version = worker_protocol.as_deref().unwrap_or("missing");
25270 let body = format!(
25271 r#"{{"reason":"unsupported_protocol_version","message":"Query tasks require worker protocol 1.8 or newer.","supported_version":"1.7","requested_version":"{requested_version}"}}"#
25272 );
25273 write_mock_response(stream, "400 Bad Request", &body);
25274 return;
25275 }
25276
25277 if behavior.reject_query_completion && path == "/api/worker/query-tasks/query-late/complete"
25278 {
25279 write_mock_response(
25280 stream,
25281 "409 Conflict",
25282 r#"{"reason":"query_task_timed_out","message":"query task timed out before completion"}"#,
25283 );
25284 return;
25285 }
25286
25287 if behavior.workflow_completion_status.is_some()
25288 && path == "/api/worker/workflow-tasks/poll"
25289 && request_number == 1
25290 {
25291 write_mock_response(
25292 stream,
25293 "200 OK",
25294 r#"{"task":{"task_id":"workflow-timeout-task","workflow_id":"reused-workflow-id","run_id":"run-selected-timeout","workflow_type":"timeout.workflow","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"history_events":[],"workflow_task_attempt":3,"lease_owner":"timeout-worker"}}"#,
25295 );
25296 return;
25297 }
25298
25299 if path == "/api/worker/workflow-tasks/workflow-timeout-task/complete" {
25300 if let (Some(status), Some(body)) = (
25301 behavior.workflow_completion_status,
25302 behavior.workflow_completion_body,
25303 ) {
25304 write_mock_response(stream, status, body);
25305 return;
25306 }
25307 }
25308
25309 if behavior.waiting_query_worker {
25310 if behavior.complete_named_signal
25311 && path == "/api/worker/workflow-tasks/poll"
25312 && request_number == 1
25313 {
25314 let body = json!({
25315 "task": {
25316 "task_id": "snapshot-open",
25317 "workflow_id": "snapshot-1",
25318 "run_id": "snapshot-run-1",
25319 "workflow_type": "snapshot",
25320 "payload_codec": DEFAULT_CODEC,
25321 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25322 .expect("Avro workflow arguments"),
25323 "history_events": [],
25324 "workflow_task_attempt": 1,
25325 "lease_owner": "rust-snapshot-worker"
25326 }
25327 })
25328 .to_string();
25329 write_mock_response(stream, "200 OK", &body);
25330 return;
25331 }
25332
25333 let signal_request = request_number - usize::from(behavior.complete_named_signal);
25334 let signal_request_limit = 2 + usize::from(behavior.complete_named_signal);
25335 if path == "/api/worker/workflow-tasks/poll"
25336 && signal_request >= 1
25337 && signal_request <= signal_request_limit
25338 {
25339 let finish = behavior.complete_named_signal && signal_request == 3;
25340 let amounts = if signal_request == 1 {
25341 vec![3]
25342 } else {
25343 vec![3, 5]
25344 };
25345 let task_id = if signal_request == 1 {
25346 "snapshot-wait-3"
25347 } else if finish {
25348 "snapshot-finish"
25349 } else {
25350 "snapshot-wait-5"
25351 };
25352 let mut history_events = std::iter::once(json!({
25353 "event_type": "SignalWaitOpened",
25354 "payload": {"sequence": 1, "signal_name": "finish"}
25355 }))
25356 .chain(amounts.iter().enumerate().map(|(index, amount)| {
25357 json!({
25358 "event_type": "SignalReceived",
25359 "payload": {
25360 "signal_id": format!("increment-{amount}"),
25361 "signal_name": "increment",
25362 "workflow_sequence": index + 2,
25363 "payload_codec": DEFAULT_CODEC,
25364 "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
25365 .expect("Avro signal envelope")
25366 }
25367 })
25368 }))
25369 .collect::<Vec<_>>();
25370 let (resume_id, resume_name, resume_arguments) = if finish {
25371 history_events.push(json!({
25372 "event_type": "SignalReceived",
25373 "payload": {
25374 "signal_id": "finish",
25375 "signal_name": "finish",
25376 "workflow_sequence": 4,
25377 "payload_codec": DEFAULT_CODEC,
25378 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25379 .expect("Avro finish signal envelope")
25380 }
25381 }));
25382 (
25383 "finish".to_string(),
25384 "finish".to_string(),
25385 encode_value_envelope(&json!([]), DEFAULT_CODEC)
25386 .expect("Avro finish resume signal"),
25387 )
25388 } else {
25389 let amount = amounts.last().expect("amount");
25390 (
25391 format!("increment-{amount}"),
25392 "increment".to_string(),
25393 encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
25394 .expect("Avro increment resume signal"),
25395 )
25396 };
25397 let body = json!({
25398 "task": {
25399 "task_id": task_id,
25400 "workflow_id": "snapshot-1",
25401 "run_id": "snapshot-run-1",
25402 "workflow_type": "snapshot",
25403 "payload_codec": DEFAULT_CODEC,
25404 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25405 .expect("Avro workflow arguments"),
25406 "history_events": history_events,
25407 "workflow_task_attempt": 1,
25408 "workflow_signal_id": resume_id,
25409 "signal_name": resume_name,
25410 "signal_arguments": resume_arguments,
25411 "lease_owner": "rust-snapshot-worker"
25412 }
25413 })
25414 .to_string();
25415 write_mock_response(stream, "200 OK", &body);
25416 return;
25417 }
25418
25419 if path == "/api/worker/query-tasks/poll" && request_number == 1 {
25420 let history_events = [3, 5]
25421 .into_iter()
25422 .enumerate()
25423 .map(|(index, amount)| {
25424 json!({
25425 "event_type": "SignalReceived",
25426 "payload": {
25427 "signal_id": format!("increment-{amount}"),
25428 "signal_name": "increment",
25429 "workflow_sequence": index + 2,
25430 "payload_codec": DEFAULT_CODEC,
25431 "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
25432 .expect("Avro query signal envelope")
25433 }
25434 })
25435 })
25436 .collect::<Vec<_>>();
25437 let body = json!({
25438 "task": {
25439 "query_task_id": "snapshot-current",
25440 "query_task_attempt": 1,
25441 "lease_owner": "rust-snapshot-worker",
25442 "workflow_id": "snapshot-1",
25443 "run_id": "snapshot-run-1",
25444 "workflow_type": "snapshot",
25445 "query_name": "current",
25446 "payload_codec": DEFAULT_CODEC,
25447 "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25448 .expect("Avro workflow arguments"),
25449 "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25450 .expect("Avro query arguments"),
25451 "history_events": history_events,
25452 "run_status": "waiting"
25453 }
25454 })
25455 .to_string();
25456 write_mock_response(stream, "200 OK", &body);
25457 return;
25458 }
25459
25460 if path == "/api/worker/workflow-tasks/snapshot-wait-3/fail"
25461 || path == "/api/worker/workflow-tasks/snapshot-wait-5/fail"
25462 {
25463 write_mock_response(
25464 stream,
25465 "200 OK",
25466 r#"{"outcome":"waiting_for_history","recorded":true}"#,
25467 );
25468 return;
25469 }
25470
25471 if path == "/api/worker/workflow-tasks/snapshot-open/complete" {
25472 write_mock_response(stream, "200 OK", r#"{"outcome":"waiting","recorded":true}"#);
25473 return;
25474 }
25475
25476 if path == "/api/worker/workflow-tasks/snapshot-finish/complete" {
25477 write_mock_response(
25478 stream,
25479 "200 OK",
25480 r#"{"outcome":"completed","run_status":"completed","recorded":true}"#,
25481 );
25482 return;
25483 }
25484
25485 if path == "/api/worker/query-tasks/snapshot-current/complete" {
25486 write_mock_response(stream, "200 OK", r#"{"outcome":"completed"}"#);
25487 return;
25488 }
25489 }
25490
25491 if matches!(
25492 path,
25493 "/api/workflows/typed-1/query/inspect" | "/api/workflows/typed-1/update/replace"
25494 ) {
25495 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
25496 .expect("typed mock result");
25497 let body = json!({
25498 "result": typed_fidelity_probe().into_json().expect("result projection"),
25499 "result_envelope": result,
25500 })
25501 .to_string();
25502 write_mock_response(stream, "200 OK", &body);
25503 return;
25504 }
25505
25506 if path == "/api/workflows/typed-1" {
25507 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
25508 .expect("typed mock result");
25509 let body = json!({
25510 "workflow_id": "typed-1",
25511 "run_id": "run-typed-1",
25512 "workflow_type": "typed.echo",
25513 "status": "completed",
25514 "output": typed_fidelity_probe().into_json().expect("output projection"),
25515 "output_envelope": result,
25516 })
25517 .to_string();
25518 write_mock_response(stream, "200 OK", &body);
25519 return;
25520 }
25521
25522 let (status, body) = match path {
25523 "/api/cluster/info" => ("200 OK", r#"{"limits":{"max_payload_bytes":2097152}}"#),
25524 "/api/health" => ("200 OK", r#"{"status":"ok"}"#),
25525 "/api/workflows" => (
25526 "201 Created",
25527 r#"{"workflow_id":"wf-start-options","run_id":"run-start-options","workflow_type":"rust.timeout"}"#,
25528 ),
25529 "/api/worker/register" if behavior.decline_registration => (
25530 "200 OK",
25531 r#"{"worker_id":"declined-worker","registered":false}"#,
25532 ),
25533 "/api/worker/register" if behavior.waiting_query_worker => (
25534 "200 OK",
25535 r#"{"worker_id":"rust-snapshot-worker","registered":true,"heartbeat_interval_seconds":1}"#,
25536 ),
25537 "/api/worker/register" => (
25538 "200 OK",
25539 r#"{"worker_id":"mock-worker","registered":true,"heartbeat_interval_seconds":3600}"#,
25540 ),
25541 "/api/worker/heartbeat" => ("200 OK", "{}"),
25542 "/api/worker/activity-tasks/poll"
25543 if behavior.cancelled_activity && request_number == 1 =>
25544 {
25545 (
25546 "200 OK",
25547 r#"{"task":{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","activity_type":"cancel-aware","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"attempt_number":1,"lease_owner":"rust-cancel-worker"}}"#,
25548 )
25549 }
25550 "/api/worker/activity-tasks/poll"
25551 if behavior.long_poll_capacity_responses_per_path > 0
25552 && request_number
25553 == behavior
25554 .long_poll_capacity_responses_per_path
25555 .saturating_add(1) =>
25556 {
25557 (
25558 "200 OK",
25559 r#"{"task":{"task_id":"capacity-activity","activity_attempt_id":"capacity-attempt","activity_type":"capacity.activity","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"attempt_number":1,"lease_owner":"capacity-worker"}}"#,
25560 )
25561 }
25562 "/api/worker/activity-tasks/poll" | "/api/worker/workflow-tasks/poll" => {
25563 ("200 OK", r#"{"task":null}"#)
25564 }
25565 "/api/worker/query-tasks/poll"
25566 if behavior.reject_query_completion && request_number == 1 =>
25567 {
25568 (
25569 "200 OK",
25570 r#"{"task":{"query_task_id":"query-late","query_task_attempt":1,"lease_owner":"late-worker","workflow_id":"counter-late","run_id":"run-late","workflow_type":"counter","query_name":"current","payload_codec":"avro","workflow_arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"query_arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"history_events":[],"run_status":"running"}}"#,
25571 )
25572 }
25573 "/api/worker/query-tasks/poll" => ("200 OK", r#"{"task":null}"#),
25574 "/api/worker/query-tasks/query-capture/complete"
25575 | "/api/worker/query-tasks/query-capture/fail" => ("200 OK", "{}"),
25576 "/api/worker/activity-tasks/activity-cancel/heartbeat" => (
25577 "200 OK",
25578 r#"{"activity_attempt_id":"attempt-cancel","cancel_requested":true,"can_continue":false,"reason":"run_cancelled","run_closed_reason":"cancelled","heartbeat_recorded":false}"#,
25579 ),
25580 "/api/worker/activity-tasks/activity-cancel/complete" => (
25581 "409 Conflict",
25582 r#"{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","reason":"run_cancelled","cancel_requested":true,"can_continue":false,"run_closed_reason":"cancelled"}"#,
25583 ),
25584 "/api/worker/activity-tasks/activity-typed/complete"
25585 | "/api/worker/activity-tasks/activity-typed/fail"
25586 | "/api/worker/activity-tasks/capacity-activity/complete"
25587 | "/api/workflows/typed-1/signal/changed" => ("200 OK", "{}"),
25588 "/api/workflows/counter-1/query/current" => (
25589 "200 OK",
25590 r#"{"workflow_id":"counter-1","query_name":"current","result":{"count":8},"result_envelope":{"codec":"avro","blob":"wwHioz3/VYAiNw4CCmNvdW50BBAA"}}"#,
25591 ),
25592 "/api/workflows/counter-1/query/missing" => (
25593 "404 Not Found",
25594 r#"{"workflow_id":"counter-1","query_name":"missing","reason":"rejected_unknown_query","message":"unknown query"}"#,
25595 ),
25596 "/api/workflows/wf-lifecycle/cancel" => (
25597 "200 OK",
25598 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","reason":"cleanup requested","command_status":"accepted"}"#,
25599 ),
25600 "/api/workflows/wf-lifecycle/terminate" => (
25601 "200 OK",
25602 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","reason":"forced stop","command_status":"accepted"}"#,
25603 ),
25604 "/api/workflows/wf-lifecycle/runs/run-current/cancel" => (
25605 "200 OK",
25606 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","command_status":"accepted"}"#,
25607 ),
25608 "/api/workflows/wf-lifecycle/runs/run-current/terminate" => (
25609 "200 OK",
25610 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","command_status":"accepted"}"#,
25611 ),
25612 "/api/workflows/wf-lifecycle/runs/run-stale/cancel"
25613 | "/api/workflows/wf-lifecycle/runs/run-stale/terminate" => (
25614 "409 Conflict",
25615 r#"{"workflow_id":"wf-lifecycle","run_id":"run-stale","reason":"historical_run_command_rejected","target_scope":"run","message":"Commands cannot target historical runs."}"#,
25616 ),
25617 "/api/workflows/wf-failed" | "/api/workflows/wf-failed/runs/run-failed" => (
25618 "200 OK",
25619 r#"{"workflow_id":"wf-failed","run_id":"run-failed","status":"failed","closed_reason":"failed","error":"payment failed","failure":{"message":"payment failed","failure_category":"application","exception_type":"PaymentError","exception_class":"billing::PaymentError","non_retryable":true,"exception":{"type":"PaymentError","class":"billing::PaymentError","message":"payment failed"},"failures":[{"id":"failure-17","failure_category":"application"}]}}"#,
25620 ),
25621 "/api/workflows/wf-cancelled" => (
25622 "200 OK",
25623 r#"{"workflow_id":"wf-cancelled","run_id":"run-cancelled","status":"cancelled","closed_reason":"cancelled","reason":"cleanup requested"}"#,
25624 ),
25625 "/api/workflows/wf-terminated" => (
25626 "200 OK",
25627 r#"{"workflow_id":"wf-terminated","run_id":"run-terminated","status":"terminated","closed_reason":"terminated","reason":"forced stop"}"#,
25628 ),
25629 "/api/workflows/wf-timed-out" => (
25630 "200 OK",
25631 r#"{"workflow_id":"wf-timed-out","run_id":"run-timed-out","status":"failed","closed_reason":"timed_out","reason":"run_timeout"}"#,
25632 ),
25633 "/api/workflows/wf-waiting" | "/api/workflows/wf-waiting/runs/run-waiting" => (
25634 "200 OK",
25635 r#"{"workflow_id":"wf-waiting","run_id":"run-waiting","status":"waiting"}"#,
25636 ),
25637 "/api/workflows/wf-selected" => (
25638 "200 OK",
25639 r#"{"workflow_id":"wf-selected","run_id":"run-current","status":"completed","output":"current run output"}"#,
25640 ),
25641 "/api/workflows/wf-selected/runs/run-selected" => (
25642 "200 OK",
25643 r#"{"workflow_id":"wf-selected","run_id":"run-selected","status":"cancelled","closed_reason":"cancelled","reason":"selected run cancelled"}"#,
25644 ),
25645 _ => ("404 Not Found", r#"{"message":"not found"}"#),
25646 };
25647 write_mock_response(stream, status, body);
25648 }
25649
25650 fn mock_request_is_complete(request: &[u8]) -> bool {
25651 let Some(header_end) = request
25652 .windows(4)
25653 .position(|window| window == b"\r\n\r\n")
25654 .map(|position| position + 4)
25655 else {
25656 return false;
25657 };
25658 let headers = String::from_utf8_lossy(&request[..header_end]);
25659 let content_length = headers.lines().find_map(|line| {
25660 let (name, value) = line.split_once(':')?;
25661 name.eq_ignore_ascii_case("content-length")
25662 .then(|| value.trim().parse::<usize>().ok())
25663 .flatten()
25664 });
25665
25666 request.len() >= header_end + content_length.unwrap_or(0)
25667 }
25668
25669 fn write_mock_response(stream: &mut TcpStream, status: &str, body: &str) {
25670 let response = format!(
25671 "HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
25672 body.len()
25673 );
25674
25675 let _ = stream.write_all(response.as_bytes());
25676 let _ = stream.flush();
25677 }
25678}