1#![doc = include_str!("../README.md")]
2
3mod runtime_payloads;
4
5use std::{
6 any::{type_name, Any, TypeId},
7 collections::{BTreeMap, HashMap},
8 future::Future,
9 io::{self, Read},
10 pin::Pin,
11 sync::{
12 atomic::{AtomicBool, Ordering},
13 Arc, Mutex, OnceLock,
14 },
15 task::{Context as TaskContext, Poll},
16 time::{Duration, Instant, SystemTime, UNIX_EPOCH},
17};
18
19use apache_avro::{from_avro_datum, to_avro_datum, types::Value as AvroDatum, Schema};
20use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
21use chrono::DateTime;
22use futures_util::{future::OptionFuture, task::noop_waker_ref};
23use serde::{
24 de::DeserializeOwned,
25 ser::{SerializeMap, SerializeSeq},
26 Deserialize, Deserializer, Serialize, Serializer,
27};
28pub use serde_json::{json, Value};
29use sha2::{Digest, Sha256};
30use thiserror::Error;
31pub use uuid::Uuid;
32
33pub const WORKER_PROTOCOL_VERSION: &str = "1.19";
34pub const PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION: &str = "1.18";
36pub const CONTROL_PLANE_VERSION: &str = "2";
37pub const DEFAULT_CODEC: &str = "avro";
38pub const SDK_VERSION: &str = concat!("durable-workflow-rust/", env!("CARGO_PKG_VERSION"));
39pub const CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY: &str =
41 "condition_wait_occurrence_identity";
42pub const MEMO_UPSERTS_CAPABILITY: &str = "memo_upserts";
44pub const QUERY_TASKS_CAPABILITY: &str = "query_tasks";
46pub const TYPED_SEARCH_ATTRIBUTES_CAPABILITY: &str = "typed_search_attributes";
48pub const WORKFLOW_UPDATES_CAPABILITY: &str = "workflow_updates";
50pub const MESSAGE_STREAMS_CAPABILITY: &str = "message_streams";
52pub const DURABLE_SELECTION_CAPABILITY: &str = "durable_selection";
54pub const MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.15";
55pub const MESSAGE_STREAM_SIGNAL: &str = "__durable_workflow_message_stream";
56pub const MESSAGE_STREAM_SCHEMA: &str = "durable-workflow.v2.message-stream.message";
57pub const MESSAGE_STREAM_CURSOR_SCHEMA: &str = "durable-workflow.v2.message-stream.cursor";
58pub const MESSAGE_STREAM_MAX_BATCH: usize = 100;
59pub const QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
61pub const SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
63pub const MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.14";
65pub const TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.16";
67pub const CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.9";
69pub const CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.17";
71pub const DURABLE_SELECTION_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.19";
73
74pub fn worker_protocol_supports_message_streams(version: &str) -> bool {
75 let Some((major, minor)) = version.split_once('.') else {
76 return false;
77 };
78 major == "1" && minor.parse::<u64>().is_ok_and(|minor| minor >= 15)
79}
80
81fn validate_user_signal_name(signal_name: &str) -> Result<()> {
82 if signal_name == MESSAGE_STREAM_SIGNAL {
83 return Err(Error::Codec(format!(
84 "signal name {MESSAGE_STREAM_SIGNAL:?} is reserved by the workflow runtime"
85 )));
86 }
87 Ok(())
88}
89
90const MAX_LONG_POLL_TIMEOUT_SECONDS: u64 = 60;
91const WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE: &str =
92 "Workflow task waiting for scheduled history.";
93const WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE: &str = "WorkflowTaskWaitingForHistory";
94const MISSING_TASK_PAYLOAD_CODEC: &str = "\0missing-task-payload-codec";
95const NULL_TASK_PAYLOAD_CODEC: &str = "\0null-task-payload-codec";
96const NON_STRING_TASK_PAYLOAD_CODEC: &str = "\0non-string-task-payload-codec";
97const MAX_MEMO_ENTRIES: usize = 100;
98const MAX_MEMO_VALUE_SIZE_BYTES: usize = 10_240;
99const MAX_MEMO_TOTAL_SIZE_BYTES: usize = 65_536;
100
101const QUERY_TASK_FINAL_REJECTION_REASONS: &[&str] = &[
102 "lease_expired",
103 "query_task_not_found",
104 "query_task_not_leased",
105 "query_task_timed_out",
106];
107
108pub fn portable_worker_affinity_capability_manifest() -> Value {
110 json!({
111 "local_activities": {
112 "supported": false,
113 "minimum_protocol_version": PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
114 "reason": "rust_worker_does_not_execute_record_local_activity",
115 },
116 "worker_sessions": {
117 "supported": false,
118 "minimum_protocol_version": PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
119 "reason": "rust_worker_has_no_typed_session_lifecycle",
120 },
121 "sticky_execution": {
122 "supported": false,
123 "minimum_protocol_version": PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
124 "reason": "rust_worker_uses_complete_durable_history_replay",
125 },
126 })
127}
128
129pub const AVRO_VALUE_SCHEMA_JSON: &str =
131 include_str!("../schema/durable_workflow.protocol.Value.v1.avsc");
132pub const AVRO_VALUE_SCHEMA_FINGERPRINT_HEX: &str = "e2a33dff55802237";
133pub const AVRO_VALUE_SCHEMA_FINGERPRINT: [u8; 8] = [0xe2, 0xa3, 0x3d, 0xff, 0x55, 0x80, 0x22, 0x37];
134const AVRO_SINGLE_OBJECT_MAGIC: [u8; 2] = [0xc3, 0x01];
135
136static AVRO_VALUE_SCHEMA: OnceLock<std::result::Result<Schema, String>> = OnceLock::new();
137static AVRO_VALUE_ORDERED_MAP_ENCODING_SCHEMA: OnceLock<std::result::Result<Schema, String>> =
138 OnceLock::new();
139
140#[derive(Clone, Copy)]
141enum RequestProtocol {
142 ControlPlane,
143 Worker(&'static str),
144}
145
146pub type Result<T> = std::result::Result<T, Error>;
147
148#[derive(Debug, Error)]
149pub enum Error {
150 #[error("transport error: {0}")]
151 Transport(#[from] reqwest::Error),
152 #[error(
153 "invalid Durable Workflow base URL: omit the SDK-owned /api suffix and pass the Server or Cloud runtime base URL; the SDK appends /api automatically"
154 )]
155 InvalidBaseUrl,
156 #[error("json error: {0}")]
157 Json(#[from] serde_json::Error),
158 #[error("http {status}: {body}")]
159 Http {
160 status: reqwest::StatusCode,
161 body: String,
162 },
163 #[error("codec error: {0}")]
164 Codec(String),
165 #[error(transparent)]
166 QueryFailed(QueryFailure),
167 #[error(transparent)]
168 Protocol(ProtocolFailure),
169 #[error(transparent)]
170 NonDeterministicReplay(ReplayFailure),
171 #[error(transparent)]
172 ChildWorkflowFailed(ChildWorkflowFailure),
173 #[error(transparent)]
174 ActivityFailed(ActivityFailure),
175 #[error(transparent)]
176 ParallelFailed(ParallelFailure),
177 #[error(transparent)]
178 SagaCompensationFailed(SagaCompensationFailure),
179 #[error(transparent)]
180 InvalidParallelGroup(ParallelGroupError),
181 #[error(transparent)]
182 DurableOperationCancelled(DurableOperationCancelled),
183 #[error(transparent)]
184 WorkflowCancellationRequested(WorkflowCancellationRequested),
185 #[error(transparent)]
186 WorkflowCommandRejected(WorkflowCommandRejection),
187 #[error(transparent)]
188 WorkflowFailed(WorkflowTerminalOutcome),
189 #[error(transparent)]
190 WorkflowCancelled(WorkflowTerminalOutcome),
191 #[error(transparent)]
192 WorkflowTerminated(WorkflowTerminalOutcome),
193 #[error(transparent)]
194 WorkflowTimedOut(WorkflowTerminalOutcome),
195 #[error(transparent)]
196 ActivityTaskRejected(ActivityTaskRejection),
197 #[error("workflow handler {0:?} is not registered")]
198 WorkflowNotRegistered(String),
199 #[error("activity handler {0:?} is not registered")]
200 ActivityNotRegistered(String),
201 #[error(
202 "{handler_kind} handler {handler_name:?} {value_kind} type {rust_type} is incompatible with the fixed Avro Value codec: {message}"
203 )]
204 HandlerType {
205 handler_kind: HandlerKind,
206 handler_name: String,
207 value_kind: HandlerValueKind,
208 rust_type: &'static str,
209 message: String,
210 },
211 #[error("workflow future yielded without emitting a durable command")]
212 WorkflowYieldedWithoutCommand,
213 #[error(
214 "workflow_stream_command_identity_missing: workflow stream authoring requires a non-empty server-provided workflow_command_id"
215 )]
216 MissingWorkflowCommandIdentity,
217 #[error("workflow state lock is poisoned")]
218 WorkflowStatePoisoned,
219 #[error("timer duration is too large for the worker protocol")]
220 TimerDurationOverflow,
221 #[error(transparent)]
222 InvalidConditionWaitOptions(#[from] ConditionWaitOptionsError),
223 #[error(transparent)]
224 InvalidSearchAttributeUpdate(#[from] SearchAttributeUpdateError),
225 #[error("operation timed out")]
226 Timeout,
227 #[error(
228 "missing {role}-plane credentials: configure ClientBuilder::{role}_token or ClientBuilder::token; a {opposite_role}-plane token cannot authorize this request"
229 )]
230 MissingRoleCredentials {
231 role: &'static str,
232 opposite_role: &'static str,
233 },
234 #[error("worker loop error: {0}")]
235 WorkerLoop(String),
236 #[error(
237 "workflow command contract for {workflow_type:?} declares update validators, but this Rust SDK cannot execute synchronous pre-accept update validation"
238 )]
239 UnsupportedUpdateValidators { workflow_type: String },
240 #[error("{primary}; worker deregistration also failed: {deregistration}")]
241 WorkerShutdown {
242 primary: Box<Error>,
243 deregistration: Box<Error>,
244 },
245 #[error("invalid child workflow options: {0}")]
246 InvalidChildWorkflowOptions(String),
247 #[error("invalid workflow memo update: {0}")]
248 InvalidMemoUpdate(String),
249 #[error(
250 "workflow_memo_updates_unavailable: the connected runtime did not advertise workflow memo update support"
251 )]
252 WorkflowMemoUpdatesUnavailable,
253 #[error(transparent)]
254 InvalidActivityOptions(ActivityOptionsError),
255 #[error(transparent)]
256 InvalidContinueAsNewOptions(#[from] ContinueAsNewOptionsError),
257 #[doc(hidden)]
258 #[error("workflow requested continue as new")]
259 ContinueAsNew(ContinueAsNewRequest),
260}
261
262#[derive(Clone, Debug, Error, PartialEq, Eq)]
264pub enum ConditionWaitOptionsError {
265 #[error("condition_key must be non-empty")]
266 EmptyKey,
267 #[error("condition_definition_fingerprint must be non-empty")]
268 EmptyPredicateIdentity,
269 #[error("condition timeout is too large for the worker protocol")]
270 TimeoutOverflow,
271}
272
273#[derive(Clone, Debug, PartialEq, Eq)]
280pub struct ConditionWaitOptions {
281 condition_key: String,
282 predicate_identity: String,
283 timeout: Option<Duration>,
284}
285
286impl ConditionWaitOptions {
287 pub fn new(condition_key: impl Into<String>, predicate_identity: impl Into<String>) -> Self {
288 Self {
289 condition_key: condition_key.into(),
290 predicate_identity: predicate_identity.into(),
291 timeout: None,
292 }
293 }
294
295 pub fn timeout(mut self, timeout: Duration) -> Self {
296 self.timeout = Some(timeout);
297 self
298 }
299
300 fn validate(
301 &self,
302 ) -> std::result::Result<ValidatedConditionWaitOptions, ConditionWaitOptionsError> {
303 let condition_key = self.condition_key.trim();
304 if condition_key.is_empty() {
305 return Err(ConditionWaitOptionsError::EmptyKey);
306 }
307 let predicate_identity = self.predicate_identity.trim();
308 if predicate_identity.is_empty() {
309 return Err(ConditionWaitOptionsError::EmptyPredicateIdentity);
310 }
311 let timeout_seconds = self
312 .timeout
313 .map(|timeout| {
314 timeout
315 .as_secs()
316 .checked_add(u64::from(timeout.subsec_nanos() > 0))
317 .ok_or(ConditionWaitOptionsError::TimeoutOverflow)
318 })
319 .transpose()?;
320
321 Ok(ValidatedConditionWaitOptions {
322 condition_key: condition_key.to_string(),
323 predicate_identity: predicate_identity.to_string(),
324 timeout_seconds,
325 })
326 }
327}
328
329#[derive(Clone, Debug, PartialEq, Eq)]
330struct ValidatedConditionWaitOptions {
331 condition_key: String,
332 predicate_identity: String,
333 timeout_seconds: Option<u64>,
334}
335
336const CONDITION_WAIT_OCCURRENCE_PREFIX: &str = "rust:condition-wait:";
337
338#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
340#[serde(rename_all = "snake_case")]
341pub enum ConditionWaitResult {
342 Satisfied,
343 TimedOut,
344}
345
346impl ConditionWaitResult {
347 pub fn is_satisfied(self) -> bool {
348 self == Self::Satisfied
349 }
350
351 pub fn is_timed_out(self) -> bool {
352 self == Self::TimedOut
353 }
354}
355
356#[doc(hidden)]
358pub fn __condition_definition_fingerprint(source: &str) -> String {
359 let mut digest = Sha256::new();
360 digest.update(b"durable-workflow-rust.wait-condition.v1\0");
361 digest.update(source.as_bytes());
362 format!("sha256:{:x}", digest.finalize())
363}
364
365#[macro_export]
371macro_rules! wait_condition {
372 ($ctx:expr, $key:expr, timeout: $timeout:expr, $predicate:expr $(,)?) => {{
373 $ctx.wait_condition(
374 $crate::ConditionWaitOptions::new(
375 $key,
376 $crate::__condition_definition_fingerprint(concat!(
377 module_path!(),
378 "\0",
379 stringify!($predicate)
380 )),
381 )
382 .timeout($timeout),
383 $predicate,
384 )
385 }};
386 ($ctx:expr, $key:expr, $predicate:expr $(,)?) => {{
387 $ctx.wait_condition(
388 $crate::ConditionWaitOptions::new(
389 $key,
390 $crate::__condition_definition_fingerprint(concat!(
391 module_path!(),
392 "\0",
393 stringify!($predicate)
394 )),
395 ),
396 $predicate,
397 )
398 }};
399}
400
401const MAX_SEARCH_ATTRIBUTES_PER_UPDATE: usize = 100;
402const MAX_SEARCH_ATTRIBUTE_KEY_LENGTH: usize = 64;
403const MAX_SEARCH_ATTRIBUTE_STRING_LENGTH: usize = 2_048;
404const MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH: usize = 255;
405const MAX_SEARCH_ATTRIBUTE_UPDATE_BYTES: usize = 65_536;
406
407#[derive(Clone, Debug, Error, PartialEq, Eq)]
409pub enum SearchAttributeUpdateError {
410 #[error("search-attribute update requires at least one attribute")]
411 Empty,
412 #[error("search attribute key {0:?} must be 1-64 URL-safe ASCII characters")]
413 InvalidKey(String),
414 #[error("search-attribute update exceeds the limit of 100 attributes")]
415 TooManyAttributes,
416 #[error("search attribute {key:?} {kind} value exceeds {limit} bytes")]
417 ValueTooLong {
418 key: String,
419 kind: &'static str,
420 limit: usize,
421 },
422 #[error(
423 "search attribute {0:?} must not contain an empty string value; use delete() to remove it"
424 )]
425 EmptyString(String),
426 #[error("search attribute {0:?} has a non-finite float value")]
427 NonFiniteFloat(String),
428 #[error("search attribute {0:?} must use an RFC 3339 datetime with an explicit timezone")]
429 InvalidDateTime(String),
430 #[error("search-attribute update exceeds the 65536-byte protocol limit")]
431 PayloadTooLarge,
432}
433
434#[derive(Clone, Debug, PartialEq)]
436pub enum SearchAttributeValue {
437 String(String),
438 Keyword(String),
439 KeywordList(Vec<String>),
440 Int(i64),
441 Float(f64),
442 Bool(bool),
443 DateTime(String),
444 Delete,
445}
446
447impl SearchAttributeValue {
448 fn type_name(&self) -> Option<&'static str> {
449 match self {
450 Self::String(_) => Some("string"),
451 Self::Keyword(_) => Some("keyword"),
452 Self::KeywordList(_) => Some("keyword_list"),
453 Self::Int(_) => Some("int"),
454 Self::Float(_) => Some("float"),
455 Self::Bool(_) => Some("bool"),
456 Self::DateTime(_) => Some("datetime"),
457 Self::Delete => None,
458 }
459 }
460
461 fn normalized(self, key: &str) -> std::result::Result<Self, SearchAttributeUpdateError> {
462 let normalize_string = |value: String, kind: &'static str, limit: usize| {
463 let value = value.trim().to_string();
464 if value.is_empty() {
465 return Err(SearchAttributeUpdateError::EmptyString(key.to_string()));
466 }
467 if value.len() > limit {
468 return Err(SearchAttributeUpdateError::ValueTooLong {
469 key: key.to_string(),
470 kind,
471 limit,
472 });
473 }
474 Ok(value)
475 };
476
477 match self {
478 Self::String(value) => Ok(Self::String(normalize_string(
479 value,
480 "string",
481 MAX_SEARCH_ATTRIBUTE_STRING_LENGTH,
482 )?)),
483 Self::Keyword(value) => Ok(Self::Keyword(normalize_string(
484 value,
485 "keyword",
486 MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH,
487 )?)),
488 Self::KeywordList(values) => {
489 let values = values
490 .into_iter()
491 .map(|value| {
492 let value = value.trim().to_string();
493 if value.len() > MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH {
494 return Err(SearchAttributeUpdateError::ValueTooLong {
495 key: key.to_string(),
496 kind: "keyword-list entry",
497 limit: MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH,
498 });
499 }
500 Ok(value)
501 })
502 .collect::<std::result::Result<Vec<_>, _>>()?;
503 Ok(Self::KeywordList(values))
504 }
505 Self::Float(value) if !value.is_finite() => {
506 Err(SearchAttributeUpdateError::NonFiniteFloat(key.to_string()))
507 }
508 Self::DateTime(value) => {
509 let value =
510 normalize_string(value, "datetime", MAX_SEARCH_ATTRIBUTE_STRING_LENGTH)?;
511 if DateTime::parse_from_rfc3339(&value).is_err() {
512 return Err(SearchAttributeUpdateError::InvalidDateTime(key.to_string()));
513 }
514 Ok(Self::DateTime(value))
515 }
516 value => Ok(value),
517 }
518 }
519
520 fn into_json(self) -> Value {
521 match self {
522 Self::String(value) | Self::Keyword(value) | Self::DateTime(value) => {
523 Value::String(value)
524 }
525 Self::KeywordList(values) => {
526 Value::Array(values.into_iter().map(Value::String).collect())
527 }
528 Self::Int(value) => json!(value),
529 Self::Float(value) => json!(value),
530 Self::Bool(value) => json!(value),
531 Self::Delete => Value::Null,
532 }
533 }
534}
535
536#[derive(Clone, Debug, Default, PartialEq)]
538pub struct SearchAttributeUpdate {
539 attributes: BTreeMap<String, SearchAttributeValue>,
540}
541
542impl SearchAttributeUpdate {
543 pub fn new() -> Self {
544 Self::default()
545 }
546
547 pub fn set(
548 mut self,
549 key: impl Into<String>,
550 value: SearchAttributeValue,
551 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
552 let key = key.into();
553 validate_search_attribute_key(&key)?;
554 if !self.attributes.contains_key(&key)
555 && self.attributes.len() >= MAX_SEARCH_ATTRIBUTES_PER_UPDATE
556 {
557 return Err(SearchAttributeUpdateError::TooManyAttributes);
558 }
559 self.attributes.insert(key.clone(), value.normalized(&key)?);
560 self.validate_size()?;
561 Ok(self)
562 }
563
564 pub fn string(
565 self,
566 key: impl Into<String>,
567 value: impl Into<String>,
568 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
569 self.set(key, SearchAttributeValue::String(value.into()))
570 }
571
572 pub fn keyword(
573 self,
574 key: impl Into<String>,
575 value: impl Into<String>,
576 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
577 self.set(key, SearchAttributeValue::Keyword(value.into()))
578 }
579
580 pub fn keyword_list<I, V>(
581 self,
582 key: impl Into<String>,
583 values: I,
584 ) -> std::result::Result<Self, SearchAttributeUpdateError>
585 where
586 I: IntoIterator<Item = V>,
587 V: Into<String>,
588 {
589 self.set(
590 key,
591 SearchAttributeValue::KeywordList(values.into_iter().map(Into::into).collect()),
592 )
593 }
594
595 pub fn int(
596 self,
597 key: impl Into<String>,
598 value: i64,
599 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
600 self.set(key, SearchAttributeValue::Int(value))
601 }
602
603 pub fn float(
604 self,
605 key: impl Into<String>,
606 value: f64,
607 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
608 self.set(key, SearchAttributeValue::Float(value))
609 }
610
611 pub fn bool(
612 self,
613 key: impl Into<String>,
614 value: bool,
615 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
616 self.set(key, SearchAttributeValue::Bool(value))
617 }
618
619 pub fn datetime(
620 self,
621 key: impl Into<String>,
622 value: impl Into<String>,
623 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
624 self.set(key, SearchAttributeValue::DateTime(value.into()))
625 }
626
627 pub fn delete(
628 self,
629 key: impl Into<String>,
630 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
631 self.set(key, SearchAttributeValue::Delete)
632 }
633
634 fn validate_size(&self) -> std::result::Result<(), SearchAttributeUpdateError> {
635 let (attributes, _) = self.clone().into_wire_parts();
636 if serde_json::to_vec(&attributes)
637 .map(|payload| payload.len() > MAX_SEARCH_ATTRIBUTE_UPDATE_BYTES)
638 .unwrap_or(true)
639 {
640 return Err(SearchAttributeUpdateError::PayloadTooLarge);
641 }
642 Ok(())
643 }
644
645 fn into_wire_parts(self) -> (Value, BTreeMap<String, String>) {
646 let mut attributes = serde_json::Map::new();
647 let mut attribute_types = BTreeMap::new();
648 for (key, value) in self.attributes {
649 if let Some(type_name) = value.type_name() {
650 attribute_types.insert(key.clone(), type_name.to_string());
651 }
652 attributes.insert(key, value.into_json());
653 }
654 (Value::Object(attributes), attribute_types)
655 }
656
657 fn validate(&self) -> std::result::Result<(), SearchAttributeUpdateError> {
658 if self.attributes.is_empty() {
659 return Err(SearchAttributeUpdateError::Empty);
660 }
661 self.validate_size()
662 }
663}
664
665fn validate_search_attribute_key(key: &str) -> std::result::Result<(), SearchAttributeUpdateError> {
666 let valid = !key.is_empty()
667 && key.len() <= MAX_SEARCH_ATTRIBUTE_KEY_LENGTH
668 && key
669 .bytes()
670 .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-' | b':'));
671 if valid {
672 Ok(())
673 } else {
674 Err(SearchAttributeUpdateError::InvalidKey(key.to_string()))
675 }
676}
677
678#[derive(Clone, Copy, Debug, PartialEq, Eq)]
680pub enum HandlerKind {
681 Workflow,
682 Activity,
683}
684
685impl std::fmt::Display for HandlerKind {
686 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
687 formatter.write_str(match self {
688 Self::Workflow => "workflow",
689 Self::Activity => "activity",
690 })
691 }
692}
693
694#[derive(Clone, Copy, Debug, PartialEq, Eq)]
696pub enum HandlerValueKind {
697 Input,
698 Result,
699}
700
701impl std::fmt::Display for HandlerValueKind {
702 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
703 formatter.write_str(match self {
704 Self::Input => "input",
705 Self::Result => "result",
706 })
707 }
708}
709
710#[derive(Clone, Copy, Debug, PartialEq, Eq)]
712pub enum WorkflowCommandKind {
713 Cancel,
714 Terminate,
715}
716
717impl WorkflowCommandKind {
718 fn as_str(self) -> &'static str {
719 match self {
720 Self::Cancel => "cancel",
721 Self::Terminate => "terminate",
722 }
723 }
724}
725
726#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
728pub struct WorkflowCommandOptions {
729 #[serde(skip_serializing_if = "Option::is_none")]
730 pub reason: Option<String>,
731 #[serde(skip_serializing_if = "Option::is_none")]
732 pub request_id: Option<String>,
733}
734
735#[derive(Clone, Debug, PartialEq, Eq)]
742pub struct WorkflowStartOptions {
743 pub execution_timeout_seconds: u64,
744 pub run_timeout_seconds: u64,
745}
746
747impl Default for WorkflowStartOptions {
748 fn default() -> Self {
749 Self {
750 execution_timeout_seconds: 3600,
751 run_timeout_seconds: 600,
752 }
753 }
754}
755
756impl WorkflowStartOptions {
757 pub fn new() -> Self {
758 Self::default()
759 }
760
761 pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
762 self.execution_timeout_seconds = seconds;
763 self
764 }
765
766 pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
767 self.run_timeout_seconds = seconds;
768 self
769 }
770
771 fn validate(&self) -> Result<()> {
772 if self.execution_timeout_seconds == 0 {
773 return Err(Error::Codec(
774 "execution_timeout_seconds must be at least 1".to_string(),
775 ));
776 }
777 if self.run_timeout_seconds == 0 {
778 return Err(Error::Codec(
779 "run_timeout_seconds must be at least 1".to_string(),
780 ));
781 }
782 if self.run_timeout_seconds > self.execution_timeout_seconds {
783 return Err(Error::Codec(
784 "run_timeout_seconds cannot exceed execution_timeout_seconds".to_string(),
785 ));
786 }
787
788 Ok(())
789 }
790}
791
792#[derive(Clone, Debug, Default, PartialEq, Eq)]
797pub struct ContinueAsNewOptions {
798 pub workflow_type: Option<String>,
799 pub task_queue: Option<String>,
800}
801
802impl ContinueAsNewOptions {
803 pub fn new() -> Self {
804 Self::default()
805 }
806
807 pub fn workflow_type(mut self, workflow_type: impl Into<String>) -> Self {
808 self.workflow_type = Some(workflow_type.into());
809 self
810 }
811
812 pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
813 self.task_queue = Some(task_queue.into());
814 self
815 }
816
817 fn validate(&self) -> std::result::Result<(), ContinueAsNewOptionsError> {
818 for (field, value) in [
819 ("workflow_type", self.workflow_type.as_deref()),
820 ("task_queue", self.task_queue.as_deref()),
821 ] {
822 if value.is_some_and(|value| value.trim().is_empty()) {
823 return Err(ContinueAsNewOptionsError {
824 field,
825 message: format!("{field} must not be empty"),
826 });
827 }
828 }
829 Ok(())
830 }
831}
832
833#[derive(Clone, Debug, Error, PartialEq, Eq)]
835#[error("invalid continue-as-new option {field}: {message}")]
836pub struct ContinueAsNewOptionsError {
837 pub field: &'static str,
838 pub message: String,
839}
840
841#[derive(Clone, Debug, Default, PartialEq, Eq)]
843pub struct WorkflowHistoryBudget {
844 pub event_count: u64,
845 pub size_bytes: Option<u64>,
846 pub continue_as_new_recommended: bool,
847 pub pressure: Option<String>,
848}
849
850#[doc(hidden)]
851#[derive(Clone, Debug)]
852pub struct ContinueAsNewRequest {
853 arguments: AvroValue,
854 options: ContinueAsNewOptions,
855}
856
857impl WorkflowCommandOptions {
858 pub fn new() -> Self {
859 Self::default()
860 }
861
862 pub fn reason(mut self, reason: impl Into<String>) -> Self {
863 self.reason = Some(reason.into());
864 self
865 }
866
867 pub fn request_id(mut self, request_id: impl Into<String>) -> Self {
868 self.request_id = Some(request_id.into());
869 self
870 }
871}
872
873#[derive(Clone, Debug, PartialEq)]
875pub struct WorkflowCommandResult {
876 pub command: WorkflowCommandKind,
877 pub workflow_id: String,
878 pub run_id: Option<String>,
879 pub outcome: Option<String>,
880 pub reason: Option<String>,
881 pub command_status: Option<String>,
882 pub raw: Value,
883}
884
885#[derive(Clone, Debug, Error)]
887#[error("workflow {command:?} rejected ({reason}, HTTP {status}): {message}")]
888pub struct WorkflowCommandRejection {
889 pub command: WorkflowCommandKind,
890 pub status: u16,
891 pub reason: String,
892 pub message: String,
893 pub workflow_id: String,
894 pub run_id: Option<String>,
895 pub target_scope: Option<String>,
896 pub body: Value,
897}
898
899#[derive(Clone, Copy, Debug, PartialEq, Eq)]
901pub enum WorkflowTerminalKind {
902 Failed,
903 Cancelled,
904 Terminated,
905 TimedOut,
906}
907
908#[derive(Clone, Debug, Error)]
914#[error("workflow {workflow_id} run {run_id:?} ended as {kind:?} ({reason})")]
915pub struct WorkflowTerminalOutcome {
916 pub kind: WorkflowTerminalKind,
917 pub workflow_id: String,
918 pub run_id: Option<String>,
919 pub reason: String,
920 pub failure_category: Option<String>,
921 pub failure_id: Option<String>,
922 pub exception_type: Option<String>,
923 pub exception_class: Option<String>,
924 pub non_retryable: Option<bool>,
925 pub message: Option<String>,
926 pub exception: Option<Value>,
927 pub raw: Value,
928}
929
930#[derive(Clone, Debug, Error)]
932#[error("activity task {operation} rejected ({reason}, HTTP {status})")]
933pub struct ActivityTaskRejection {
934 pub operation: String,
935 pub status: u16,
936 pub reason: String,
937 pub task_id: String,
938 pub activity_attempt_id: String,
939 pub cancel_requested: bool,
940 pub can_continue: Option<bool>,
941 pub run_closed_reason: Option<String>,
942 pub body: Value,
943}
944
945#[derive(Clone, Copy, Debug, PartialEq, Eq)]
947pub enum ActivityOptionsErrorKind {
948 EmptyTaskQueue,
949 EmptyRetryPolicy,
950 InvalidMaxAttempts,
951 BackoffWithoutRetryBudget,
952 TooManyBackoffIntervals,
953 InvalidBackoffCoefficient,
954 BackoffGenerationTooLarge,
955 BackoffOverflow,
956 EmptyNonRetryableErrorType,
957 TimeoutNotPositive,
958 TimeoutOverflow,
959 TimeoutOrder,
960}
961
962#[derive(Clone, Debug, Error, PartialEq, Eq)]
964#[error("invalid activity options ({kind:?}, {field:?}): {message}")]
965pub struct ActivityOptionsError {
966 pub kind: ActivityOptionsErrorKind,
967 pub field: Option<&'static str>,
968 pub message: String,
969}
970
971impl ActivityOptionsError {
972 fn new(
973 kind: ActivityOptionsErrorKind,
974 field: Option<&'static str>,
975 message: impl Into<String>,
976 ) -> Self {
977 Self {
978 kind,
979 field,
980 message: message.into(),
981 }
982 }
983}
984
985#[derive(Clone, Copy, Debug, PartialEq, Eq)]
987pub enum ActivityFailureKind {
988 Failed,
989 Cancelled,
990 TimedOut,
991}
992
993#[derive(Clone, Debug, Error)]
998#[error("activity failed ({reason}): {message}")]
999pub struct ActivityFailure {
1000 pub kind: ActivityFailureKind,
1001 pub reason: String,
1002 pub message: String,
1003 pub activity_execution_id: Option<String>,
1004 pub activity_attempt_id: Option<String>,
1005 pub activity_type: Option<String>,
1006 pub activity_class: Option<String>,
1007 pub attempt_number: Option<u64>,
1008 pub failure_id: Option<String>,
1009 pub failure_category: Option<String>,
1010 pub timeout_kind: Option<String>,
1011 pub non_retryable: bool,
1012 pub exception_type: Option<String>,
1013 pub exception_class: Option<String>,
1014 pub code: Option<Value>,
1015 pub exception: Option<Value>,
1016}
1017
1018#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1020pub enum ChildWorkflowFailureKind {
1021 Failed,
1022 Cancelled,
1023 Terminated,
1024}
1025
1026#[derive(Clone, Debug, Error)]
1032#[error("child workflow failed ({reason}): {message}")]
1033pub struct ChildWorkflowFailure {
1034 pub kind: ChildWorkflowFailureKind,
1035 pub reason: String,
1036 pub message: String,
1037 pub parent_workflow_id: Option<String>,
1038 pub parent_workflow_run_id: Option<String>,
1039 pub child_workflow_id: Option<String>,
1040 pub child_workflow_run_id: Option<String>,
1041 pub child_workflow_type: Option<String>,
1042 pub failure_id: Option<String>,
1043 pub failure_category: Option<String>,
1044 pub exception_type: Option<String>,
1045 pub exception_class: Option<String>,
1046 pub non_retryable: bool,
1047 pub code: Option<Value>,
1048 pub exception: Option<Value>,
1049}
1050
1051#[derive(Clone, Debug, PartialEq, Eq)]
1053pub struct WorkflowIdentity {
1054 pub workflow_id: Option<String>,
1055 pub run_id: Option<String>,
1056}
1057
1058#[derive(Clone, Debug, PartialEq)]
1060pub struct ChildWorkflowResult {
1061 pub parent: WorkflowIdentity,
1062 pub child: WorkflowIdentity,
1063 pub child_workflow_type: Option<String>,
1064 pub result: Value,
1065}
1066
1067#[derive(Clone, Debug, PartialEq)]
1069pub struct ChildWorkflowAvroResult {
1070 pub parent: WorkflowIdentity,
1071 pub child: WorkflowIdentity,
1072 pub child_workflow_type: Option<String>,
1073 pub result: AvroValue,
1074}
1075
1076#[derive(Clone, Debug, Deserialize, Hash, PartialEq, Eq, Serialize)]
1078#[serde(untagged)]
1079pub enum SelectionKey {
1080 Index(usize),
1081 Name(String),
1082}
1083
1084impl From<usize> for SelectionKey {
1085 fn from(value: usize) -> Self {
1086 Self::Index(value)
1087 }
1088}
1089
1090impl From<String> for SelectionKey {
1091 fn from(value: String) -> Self {
1092 Self::Name(value)
1093 }
1094}
1095
1096impl From<&str> for SelectionKey {
1097 fn from(value: &str) -> Self {
1098 Self::Name(value.to_string())
1099 }
1100}
1101
1102#[derive(Clone, Debug, Error, PartialEq, Eq)]
1104#[error("selected {operation_kind} operation {operation_identity} was explicitly cancelled")]
1105pub struct DurableOperationCancelled {
1106 pub selection_group_id: String,
1107 pub member_key: SelectionKey,
1108 pub member_index: usize,
1109 pub operation_kind: String,
1110 pub operation_identity: String,
1111}
1112
1113#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
1119pub struct ParallelGroupMetadata {
1120 pub parallel_group_id: String,
1121 pub parallel_group_kind: String,
1122 pub parallel_group_base_sequence: u64,
1123 pub parallel_group_size: usize,
1124 pub parallel_group_index: usize,
1125 #[serde(default, skip_serializing_if = "Option::is_none")]
1126 pub parallel_group_mode: Option<String>,
1127 #[serde(default, skip_serializing_if = "Option::is_none")]
1128 pub selection_member_key: Option<SelectionKey>,
1129 #[serde(default, skip_serializing_if = "Option::is_none")]
1130 pub selection_member_index: Option<usize>,
1131 #[serde(default, skip_serializing_if = "Option::is_none")]
1132 pub selection_member_base_sequence: Option<u64>,
1133 #[serde(default, skip_serializing_if = "Option::is_none")]
1134 pub selection_member_size: Option<usize>,
1135 #[serde(default, skip_serializing_if = "Option::is_none")]
1136 pub selection_member_kind: Option<String>,
1137}
1138
1139#[derive(Clone, Debug, PartialEq)]
1141pub enum ParallelResult {
1142 Activity(Value),
1143 ChildWorkflow(ChildWorkflowResult),
1144 Timer,
1145 Signal(Vec<Value>),
1146 Condition(ConditionWaitResult),
1147 Group(Vec<ParallelResult>),
1148}
1149
1150#[derive(Clone, Debug, PartialEq)]
1152pub enum ParallelAvroResult {
1153 Activity(AvroValue),
1154 ChildWorkflow(ChildWorkflowAvroResult),
1155 Timer,
1156 Signal(Vec<AvroValue>),
1157 Condition(ConditionWaitResult),
1158 Group(Vec<ParallelAvroResult>),
1159}
1160
1161impl ParallelAvroResult {
1162 fn into_json_result(self) -> Result<ParallelResult> {
1163 match self {
1164 Self::Activity(value) => Ok(ParallelResult::Activity(value.into_json()?)),
1165 Self::ChildWorkflow(result) => Ok(ParallelResult::ChildWorkflow(ChildWorkflowResult {
1166 parent: result.parent,
1167 child: result.child,
1168 child_workflow_type: result.child_workflow_type,
1169 result: result.result.into_json()?,
1170 })),
1171 Self::Timer => Ok(ParallelResult::Timer),
1172 Self::Signal(values) => Ok(ParallelResult::Signal(
1173 values
1174 .into_iter()
1175 .map(AvroValue::into_json)
1176 .collect::<Result<Vec<_>>>()?,
1177 )),
1178 Self::Condition(result) => Ok(ParallelResult::Condition(result)),
1179 Self::Group(results) => Ok(ParallelResult::Group(
1180 results
1181 .into_iter()
1182 .map(Self::into_json_result)
1183 .collect::<Result<Vec<_>>>()?,
1184 )),
1185 }
1186 }
1187}
1188
1189#[derive(Clone, Debug, PartialEq)]
1191pub struct ParallelCompletion {
1192 pub member_path: Vec<usize>,
1193 pub result: ParallelResult,
1194}
1195
1196#[derive(Debug, Error)]
1204#[error("parallel group {group_id} member {member_path:?} failed: {cause}")]
1205pub struct ParallelFailure {
1206 pub group_id: String,
1207 pub member_path: Vec<usize>,
1208 pub group_path: Vec<ParallelGroupMetadata>,
1209 pub completed: Vec<ParallelCompletion>,
1210 #[source]
1211 pub cause: Box<Error>,
1212}
1213
1214#[derive(Clone, Debug, Error, PartialEq, Eq)]
1216#[error("invalid deterministic parallel group ({reason}): {message}")]
1217pub struct ParallelGroupError {
1218 pub reason: &'static str,
1219 pub member_path: Vec<usize>,
1220 pub message: String,
1221}
1222
1223#[derive(Clone, Debug, Error, PartialEq, Eq)]
1225#[error("workflow cancellation was requested")]
1226pub struct WorkflowCancellationRequested;
1227
1228#[derive(Debug, Error)]
1230#[error(
1231 "saga forward execution failed; compensation activity {compensation_activity_type} (registration {compensation_registration_order}) also failed: {compensation_failure}"
1232)]
1233pub struct SagaCompensationFailure {
1234 pub initiating_failure: Box<Error>,
1235 pub compensation_failure: Box<Error>,
1236 pub compensation_activity_type: String,
1237 pub compensation_registration_order: usize,
1238}
1239
1240#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
1242pub enum ParentClosePolicy {
1243 #[default]
1244 Abandon,
1245 RequestCancel,
1246 Terminate,
1247}
1248
1249impl ParentClosePolicy {
1250 fn as_str(self) -> &'static str {
1251 match self {
1252 Self::Abandon => "abandon",
1253 Self::RequestCancel => "request_cancel",
1254 Self::Terminate => "terminate",
1255 }
1256 }
1257}
1258
1259#[derive(Clone, Debug, Default, PartialEq, Eq)]
1261pub struct ChildWorkflowRetryPolicy {
1262 pub max_attempts: Option<u32>,
1263 pub backoff_seconds: Vec<u64>,
1264 pub non_retryable_error_types: Vec<String>,
1265}
1266
1267#[derive(Clone, Debug, PartialEq, Eq)]
1271pub struct ChildWorkflowOptions {
1272 pub task_queue: String,
1273 pub parent_close_policy: ParentClosePolicy,
1274 pub retry_policy: Option<ChildWorkflowRetryPolicy>,
1275 pub execution_timeout_seconds: Option<u64>,
1276 pub run_timeout_seconds: Option<u64>,
1277}
1278
1279impl ChildWorkflowOptions {
1280 pub fn new(task_queue: impl Into<String>) -> Self {
1281 Self {
1282 task_queue: task_queue.into(),
1283 parent_close_policy: ParentClosePolicy::Abandon,
1284 retry_policy: None,
1285 execution_timeout_seconds: None,
1286 run_timeout_seconds: None,
1287 }
1288 }
1289
1290 pub fn parent_close_policy(mut self, policy: ParentClosePolicy) -> Self {
1291 self.parent_close_policy = policy;
1292 self
1293 }
1294
1295 pub fn retry_policy(mut self, policy: ChildWorkflowRetryPolicy) -> Self {
1296 self.retry_policy = Some(policy);
1297 self
1298 }
1299
1300 pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
1301 self.execution_timeout_seconds = Some(seconds);
1302 self
1303 }
1304
1305 pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
1306 self.run_timeout_seconds = Some(seconds);
1307 self
1308 }
1309}
1310
1311#[derive(Clone, Debug, PartialEq, Eq)]
1313pub enum ActivityBackoff {
1314 Explicit(Vec<Duration>),
1317 Exponential {
1319 initial_interval: Duration,
1320 coefficient: u32,
1321 maximum_interval: Option<Duration>,
1322 },
1323}
1324
1325#[derive(Clone, Debug, Default, PartialEq, Eq)]
1327pub struct ActivityRetryPolicy {
1328 pub max_attempts: Option<u32>,
1329 pub backoff: Option<ActivityBackoff>,
1330 pub non_retryable_error_types: Vec<String>,
1331}
1332
1333impl ActivityRetryPolicy {
1334 pub fn new(max_attempts: u32) -> Self {
1336 Self {
1337 max_attempts: Some(max_attempts),
1338 ..Self::default()
1339 }
1340 }
1341
1342 pub fn backoff_intervals(mut self, intervals: impl IntoIterator<Item = Duration>) -> Self {
1343 self.backoff = Some(ActivityBackoff::Explicit(intervals.into_iter().collect()));
1344 self
1345 }
1346
1347 pub fn exponential_backoff(
1348 mut self,
1349 initial_interval: Duration,
1350 coefficient: u32,
1351 maximum_interval: Option<Duration>,
1352 ) -> Self {
1353 self.backoff = Some(ActivityBackoff::Exponential {
1354 initial_interval,
1355 coefficient,
1356 maximum_interval,
1357 });
1358 self
1359 }
1360
1361 pub fn non_retryable_error_type(mut self, error_type: impl Into<String>) -> Self {
1362 self.non_retryable_error_types.push(error_type.into());
1363 self
1364 }
1365
1366 pub fn non_retryable_error_types(
1367 mut self,
1368 error_types: impl IntoIterator<Item = impl Into<String>>,
1369 ) -> Self {
1370 self.non_retryable_error_types
1371 .extend(error_types.into_iter().map(Into::into));
1372 self
1373 }
1374}
1375
1376#[derive(Clone, Debug, Default, PartialEq, Eq)]
1381pub struct ActivityOptions {
1382 pub task_queue: Option<String>,
1383 pub retry_policy: Option<ActivityRetryPolicy>,
1384 pub start_to_close_timeout: Option<Duration>,
1385 pub schedule_to_start_timeout: Option<Duration>,
1386 pub schedule_to_close_timeout: Option<Duration>,
1387 pub heartbeat_timeout: Option<Duration>,
1388}
1389
1390impl ActivityOptions {
1391 pub fn new() -> Self {
1392 Self::default()
1393 }
1394
1395 pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
1396 self.task_queue = Some(task_queue.into());
1397 self
1398 }
1399
1400 pub fn retry_policy(mut self, policy: ActivityRetryPolicy) -> Self {
1401 self.retry_policy = Some(policy);
1402 self
1403 }
1404
1405 pub fn start_to_close_timeout(mut self, timeout: Duration) -> Self {
1406 self.start_to_close_timeout = Some(timeout);
1407 self
1408 }
1409
1410 pub fn schedule_to_start_timeout(mut self, timeout: Duration) -> Self {
1411 self.schedule_to_start_timeout = Some(timeout);
1412 self
1413 }
1414
1415 pub fn schedule_to_close_timeout(mut self, timeout: Duration) -> Self {
1416 self.schedule_to_close_timeout = Some(timeout);
1417 self
1418 }
1419
1420 pub fn heartbeat_timeout(mut self, timeout: Duration) -> Self {
1421 self.heartbeat_timeout = Some(timeout);
1422 self
1423 }
1424
1425 fn validate(&self) -> std::result::Result<ValidatedActivityOptions, ActivityOptionsError> {
1426 if self
1427 .task_queue
1428 .as_deref()
1429 .is_some_and(|queue| queue.trim().is_empty())
1430 {
1431 return Err(ActivityOptionsError::new(
1432 ActivityOptionsErrorKind::EmptyTaskQueue,
1433 Some("task_queue"),
1434 "task_queue must not be empty",
1435 ));
1436 }
1437
1438 for (field, value) in [
1439 ("start_to_close_timeout", self.start_to_close_timeout),
1440 ("schedule_to_start_timeout", self.schedule_to_start_timeout),
1441 ("schedule_to_close_timeout", self.schedule_to_close_timeout),
1442 ("heartbeat_timeout", self.heartbeat_timeout),
1443 ] {
1444 if value.is_some_and(|value| value.is_zero()) {
1445 return Err(ActivityOptionsError::new(
1446 ActivityOptionsErrorKind::TimeoutNotPositive,
1447 Some(field),
1448 format!("{field} must be positive"),
1449 ));
1450 }
1451 }
1452
1453 validate_timeout_order(
1454 "heartbeat_timeout",
1455 self.heartbeat_timeout,
1456 "start_to_close_timeout",
1457 self.start_to_close_timeout,
1458 )?;
1459 validate_timeout_order(
1460 "start_to_close_timeout",
1461 self.start_to_close_timeout,
1462 "schedule_to_close_timeout",
1463 self.schedule_to_close_timeout,
1464 )?;
1465 validate_timeout_order(
1466 "schedule_to_start_timeout",
1467 self.schedule_to_start_timeout,
1468 "schedule_to_close_timeout",
1469 self.schedule_to_close_timeout,
1470 )?;
1471
1472 Ok(ValidatedActivityOptions {
1473 task_queue: self.task_queue.clone(),
1474 retry_policy: self
1475 .retry_policy
1476 .as_ref()
1477 .map(validate_activity_retry_policy)
1478 .transpose()?,
1479 start_to_close_timeout: timeout_seconds(
1480 "start_to_close_timeout",
1481 self.start_to_close_timeout,
1482 )?,
1483 schedule_to_start_timeout: timeout_seconds(
1484 "schedule_to_start_timeout",
1485 self.schedule_to_start_timeout,
1486 )?,
1487 schedule_to_close_timeout: timeout_seconds(
1488 "schedule_to_close_timeout",
1489 self.schedule_to_close_timeout,
1490 )?,
1491 heartbeat_timeout: timeout_seconds("heartbeat_timeout", self.heartbeat_timeout)?,
1492 })
1493 }
1494}
1495
1496pub enum ParallelOperation {
1502 Activity {
1503 activity_type: String,
1504 options: ActivityOptions,
1505 arguments: Result<AvroValue>,
1506 },
1507 ChildWorkflow {
1508 workflow_type: String,
1509 options: ChildWorkflowOptions,
1510 arguments: Result<AvroValue>,
1511 },
1512 Timer(Duration),
1513 Signal(String),
1514 Condition {
1515 options: ConditionWaitOptions,
1516 predicate: Box<dyn Fn() -> Result<bool> + Send + 'static>,
1517 },
1518 Group(Vec<ParallelOperation>),
1519}
1520
1521impl ParallelOperation {
1522 pub fn activity<T: Serialize>(activity_type: impl Into<String>, args: T) -> Self {
1523 Self::activity_with_options(activity_type, ActivityOptions::new(), args)
1524 }
1525
1526 pub fn activity_with_options<T: Serialize>(
1527 activity_type: impl Into<String>,
1528 options: ActivityOptions,
1529 args: T,
1530 ) -> Self {
1531 Self::Activity {
1532 activity_type: activity_type.into(),
1533 options,
1534 arguments: AvroValue::from_serialize(&args),
1535 }
1536 }
1537
1538 pub fn child_workflow<T: Serialize>(
1539 workflow_type: impl Into<String>,
1540 options: ChildWorkflowOptions,
1541 args: T,
1542 ) -> Self {
1543 Self::ChildWorkflow {
1544 workflow_type: workflow_type.into(),
1545 options,
1546 arguments: AvroValue::from_serialize(&args),
1547 }
1548 }
1549
1550 pub fn timer(duration: Duration) -> Self {
1551 Self::Timer(duration)
1552 }
1553
1554 pub fn signal(signal_name: impl Into<String>) -> Self {
1555 Self::Signal(signal_name.into())
1556 }
1557
1558 pub fn condition<F>(options: ConditionWaitOptions, predicate: F) -> Self
1559 where
1560 F: Fn() -> Result<bool> + Send + 'static,
1561 {
1562 Self::Condition {
1563 options,
1564 predicate: Box::new(predicate),
1565 }
1566 }
1567
1568 pub fn group(operations: Vec<ParallelOperation>) -> Self {
1569 Self::Group(operations)
1570 }
1571}
1572
1573#[derive(Clone, Debug)]
1574struct ValidatedActivityOptions {
1575 task_queue: Option<String>,
1576 retry_policy: Option<Value>,
1577 start_to_close_timeout: Option<u64>,
1578 schedule_to_start_timeout: Option<u64>,
1579 schedule_to_close_timeout: Option<u64>,
1580 heartbeat_timeout: Option<u64>,
1581}
1582
1583fn validate_timeout_order(
1584 smaller_name: &'static str,
1585 smaller: Option<Duration>,
1586 larger_name: &'static str,
1587 larger: Option<Duration>,
1588) -> std::result::Result<(), ActivityOptionsError> {
1589 if matches!((smaller, larger), (Some(smaller), Some(larger)) if smaller > larger) {
1590 return Err(ActivityOptionsError::new(
1591 ActivityOptionsErrorKind::TimeoutOrder,
1592 Some(smaller_name),
1593 format!("{smaller_name} must be <= {larger_name}"),
1594 ));
1595 }
1596 Ok(())
1597}
1598
1599fn timeout_seconds(
1600 field: &'static str,
1601 value: Option<Duration>,
1602) -> std::result::Result<Option<u64>, ActivityOptionsError> {
1603 value
1604 .map(|value| {
1605 activity_protocol_seconds(value).ok_or_else(|| {
1606 ActivityOptionsError::new(
1607 ActivityOptionsErrorKind::TimeoutOverflow,
1608 Some(field),
1609 format!("{field} is too large for the worker protocol"),
1610 )
1611 })
1612 })
1613 .transpose()
1614}
1615
1616fn duration_seconds_ceil(value: Duration) -> Option<u64> {
1617 value
1618 .as_secs()
1619 .checked_add(u64::from(value.subsec_nanos() > 0))
1620}
1621
1622fn activity_protocol_seconds(value: Duration) -> Option<u64> {
1623 duration_seconds_ceil(value).filter(|seconds| *seconds <= i64::MAX as u64)
1624}
1625
1626fn validate_activity_retry_policy(
1627 policy: &ActivityRetryPolicy,
1628) -> std::result::Result<Value, ActivityOptionsError> {
1629 if policy.max_attempts.is_none()
1630 && policy.backoff.is_none()
1631 && policy.non_retryable_error_types.is_empty()
1632 {
1633 return Err(ActivityOptionsError::new(
1634 ActivityOptionsErrorKind::EmptyRetryPolicy,
1635 Some("retry_policy"),
1636 "retry_policy must configure at least one field",
1637 ));
1638 }
1639 if policy.max_attempts == Some(0) {
1640 return Err(ActivityOptionsError::new(
1641 ActivityOptionsErrorKind::InvalidMaxAttempts,
1642 Some("retry_policy.max_attempts"),
1643 "max_attempts must be >= 1",
1644 ));
1645 }
1646 if policy
1647 .non_retryable_error_types
1648 .iter()
1649 .any(|error_type| error_type.trim().is_empty())
1650 {
1651 return Err(ActivityOptionsError::new(
1652 ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
1653 Some("retry_policy.non_retryable_error_types"),
1654 "non_retryable_error_types must not contain empty values",
1655 ));
1656 }
1657
1658 let backoff_seconds = match &policy.backoff {
1659 None => None,
1660 Some(backoff) => {
1661 let max_attempts = policy.max_attempts.ok_or_else(|| {
1662 ActivityOptionsError::new(
1663 ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
1664 Some("retry_policy.backoff"),
1665 "backoff requires max_attempts",
1666 )
1667 })?;
1668 let retry_count = max_attempts.saturating_sub(1) as usize;
1669 let intervals = match backoff {
1670 ActivityBackoff::Explicit(intervals) => {
1671 if intervals.len() > retry_count {
1672 return Err(ActivityOptionsError::new(
1673 ActivityOptionsErrorKind::TooManyBackoffIntervals,
1674 Some("retry_policy.backoff"),
1675 "backoff interval count must not exceed max_attempts - 1",
1676 ));
1677 }
1678 intervals.clone()
1679 }
1680 ActivityBackoff::Exponential {
1681 initial_interval,
1682 coefficient,
1683 maximum_interval,
1684 } => {
1685 if *coefficient < 1 {
1686 return Err(ActivityOptionsError::new(
1687 ActivityOptionsErrorKind::InvalidBackoffCoefficient,
1688 Some("retry_policy.backoff.coefficient"),
1689 "backoff coefficient must be >= 1",
1690 ));
1691 }
1692 if retry_count > 10_000 {
1693 return Err(ActivityOptionsError::new(
1694 ActivityOptionsErrorKind::BackoffGenerationTooLarge,
1695 Some("retry_policy.max_attempts"),
1696 "generated backoff supports at most 10000 retry intervals",
1697 ));
1698 }
1699 let mut current = *initial_interval;
1700 let mut intervals = Vec::with_capacity(retry_count);
1701 for _ in 0..retry_count {
1702 let interval = maximum_interval
1703 .map(|maximum| current.min(maximum))
1704 .unwrap_or(current);
1705 intervals.push(interval);
1706 if maximum_interval.is_some_and(|maximum| interval == maximum) {
1707 break;
1708 }
1709 current = current.checked_mul(*coefficient).ok_or_else(|| {
1710 ActivityOptionsError::new(
1711 ActivityOptionsErrorKind::BackoffOverflow,
1712 Some("retry_policy.backoff"),
1713 "generated backoff interval overflowed",
1714 )
1715 })?;
1716 }
1717 intervals
1718 }
1719 };
1720 Some(
1721 intervals
1722 .into_iter()
1723 .map(|interval| {
1724 activity_protocol_seconds(interval).ok_or_else(|| {
1725 ActivityOptionsError::new(
1726 ActivityOptionsErrorKind::BackoffOverflow,
1727 Some("retry_policy.backoff"),
1728 "backoff interval is too large for the worker protocol",
1729 )
1730 })
1731 })
1732 .collect::<std::result::Result<Vec<_>, _>>()?,
1733 )
1734 }
1735 };
1736
1737 let mut encoded = serde_json::Map::new();
1738 if let Some(max_attempts) = policy.max_attempts {
1739 encoded.insert("max_attempts".to_string(), json!(max_attempts));
1740 }
1741 if let Some(backoff_seconds) = backoff_seconds {
1742 encoded.insert("backoff_seconds".to_string(), json!(backoff_seconds));
1743 }
1744 if !policy.non_retryable_error_types.is_empty() {
1745 let mut canonical_error_types = Vec::new();
1746 for error_type in policy
1747 .non_retryable_error_types
1748 .iter()
1749 .map(|error_type| error_type.trim())
1750 {
1751 if !canonical_error_types.contains(&error_type) {
1752 canonical_error_types.push(error_type);
1753 }
1754 }
1755 encoded.insert(
1756 "non_retryable_error_types".to_string(),
1757 json!(canonical_error_types),
1758 );
1759 }
1760 Ok(Value::Object(encoded))
1761}
1762
1763#[derive(Clone, Debug, Error)]
1766#[error("non-deterministic workflow replay ({reason}) at sequence {sequence:?}: {message}")]
1767pub struct ReplayFailure {
1768 pub reason: String,
1769 pub sequence: Option<u64>,
1770 pub expected: Option<String>,
1771 pub actual: Option<String>,
1772 pub message: String,
1773}
1774
1775impl ReplayFailure {
1776 fn new(
1777 reason: impl Into<String>,
1778 sequence: Option<u64>,
1779 expected: Option<String>,
1780 actual: Option<String>,
1781 message: impl Into<String>,
1782 ) -> Self {
1783 Self {
1784 reason: reason.into(),
1785 sequence,
1786 expected,
1787 actual,
1788 message: message.into(),
1789 }
1790 }
1791}
1792
1793#[derive(Clone, Debug, Error)]
1795#[error("query failed ({reason}, HTTP {status}): {message}")]
1796pub struct QueryFailure {
1797 pub status: u16,
1798 pub reason: String,
1799 pub message: String,
1800 pub body: Value,
1801}
1802
1803#[derive(Clone, Debug, Error)]
1805#[error("protocol rejected ({reason}, HTTP {status}): {message}")]
1806pub struct ProtocolFailure {
1807 pub status: u16,
1808 pub reason: String,
1809 pub message: String,
1810 pub supported_version: Option<String>,
1811 pub requested_version: Option<String>,
1812 pub body: Value,
1813}
1814
1815#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
1816pub struct PayloadEnvelope {
1817 pub codec: String,
1818 pub blob: String,
1819}
1820
1821impl PayloadEnvelope {
1822 pub fn avro<T: Serialize>(value: &T) -> Result<Self> {
1823 encode_payload(value, DEFAULT_CODEC)
1824 }
1825
1826 pub fn avro_value(value: &AvroValue) -> Result<Self> {
1829 encode_avro_value(value)
1830 }
1831}
1832
1833#[derive(Clone, Debug)]
1835pub enum AvroValue {
1836 Null,
1837 Boolean(bool),
1838 Long(i64),
1839 Double(f64),
1840 Bytes(Vec<u8>),
1841 String(String),
1842 Array(Vec<AvroValue>),
1843 Map(BTreeMap<String, AvroValue>),
1844}
1845
1846impl PartialEq for AvroValue {
1847 fn eq(&self, other: &Self) -> bool {
1848 match (self, other) {
1849 (Self::Null, Self::Null) => true,
1850 (Self::Boolean(left), Self::Boolean(right)) => left == right,
1851 (Self::Long(left), Self::Long(right)) => left == right,
1852 (Self::Double(left), Self::Double(right)) => left.to_bits() == right.to_bits(),
1853 (Self::Bytes(left), Self::Bytes(right)) => left == right,
1854 (Self::String(left), Self::String(right)) => left == right,
1855 (Self::Array(left), Self::Array(right)) => left == right,
1856 (Self::Map(left), Self::Map(right)) => left == right,
1857 _ => false,
1858 }
1859 }
1860}
1861
1862impl AvroValue {
1863 fn from_serialize<T: Serialize>(value: &T) -> Result<Self> {
1864 Self::from_serde_value(
1865 serde_value::to_value(value).map_err(|error| {
1866 Error::Codec(format!("could not adapt value for Avro: {error}"))
1867 })?,
1868 )
1869 }
1870
1871 fn from_serde_value(value: serde_value::Value) -> Result<Self> {
1872 use serde_value::Value as SerdeValue;
1873
1874 match value {
1875 SerdeValue::Unit => Ok(Self::Null),
1876 SerdeValue::Bool(value) => Ok(Self::Boolean(value)),
1877 SerdeValue::I8(value) => Ok(Self::Long(i64::from(value))),
1878 SerdeValue::I16(value) => Ok(Self::Long(i64::from(value))),
1879 SerdeValue::I32(value) => Ok(Self::Long(i64::from(value))),
1880 SerdeValue::I64(value) => Ok(Self::Long(value)),
1881 SerdeValue::U8(value) => Ok(Self::Long(i64::from(value))),
1882 SerdeValue::U16(value) => Ok(Self::Long(i64::from(value))),
1883 SerdeValue::U32(value) => Ok(Self::Long(i64::from(value))),
1884 SerdeValue::U64(value) => i64::try_from(value).map(Self::Long).map_err(|_| {
1885 Error::Codec(
1886 "integer_overflow: Avro Value long must be within signed 64-bit range"
1887 .to_string(),
1888 )
1889 }),
1890 SerdeValue::F32(value) => Self::finite_double(f64::from(value)),
1891 SerdeValue::F64(value) => Self::finite_double(value),
1892 SerdeValue::Char(value) => Ok(Self::String(value.to_string())),
1893 SerdeValue::String(value) => Ok(Self::String(value)),
1894 SerdeValue::Bytes(value) => Ok(Self::Bytes(value)),
1895 SerdeValue::Option(None) => Ok(Self::Null),
1896 SerdeValue::Option(Some(value)) | SerdeValue::Newtype(value) => {
1897 Self::from_serde_value(*value)
1898 }
1899 SerdeValue::Seq(values) => values
1900 .into_iter()
1901 .map(Self::from_serde_value)
1902 .collect::<Result<Vec<_>>>()
1903 .map(Self::Array),
1904 SerdeValue::Map(values) => values
1905 .into_iter()
1906 .map(|(key, value)| {
1907 let SerdeValue::String(key) = key else {
1908 return Err(Error::Codec(
1909 "invalid_map_key: Avro Value map keys must be strings".to_string(),
1910 ));
1911 };
1912
1913 Ok((key, Self::from_serde_value(value)?))
1914 })
1915 .collect::<Result<BTreeMap<_, _>>>()
1916 .map(Self::Map),
1917 }
1918 }
1919
1920 fn finite_double(value: f64) -> Result<Self> {
1921 if !value.is_finite() {
1922 return Err(Error::Codec(
1923 "non_finite_float: Avro Value doubles must be finite".to_string(),
1924 ));
1925 }
1926
1927 Ok(Self::Double(value))
1928 }
1929
1930 fn into_json(self) -> Result<Value> {
1931 match self {
1932 Self::Null => Ok(Value::Null),
1933 Self::Boolean(value) => Ok(Value::Bool(value)),
1934 Self::Long(value) => Ok(Value::Number(value.into())),
1935 Self::Double(value) => serde_json::Number::from_f64(value)
1936 .map(Value::Number)
1937 .ok_or_else(|| {
1938 Error::Codec(
1939 "non_finite_float: decoded Avro Value double is not finite".to_string(),
1940 )
1941 }),
1942 Self::Bytes(value) => Ok(json!({
1943 "$type": "bytes",
1944 "base64": BASE64.encode(value),
1945 })),
1946 Self::String(value) => Ok(Value::String(value)),
1947 Self::Array(values) => values
1948 .into_iter()
1949 .map(Self::into_json)
1950 .collect::<Result<Vec<_>>>()
1951 .map(Value::Array),
1952 Self::Map(values) => values
1953 .into_iter()
1954 .map(|(key, value)| Ok((key, value.into_json()?)))
1955 .collect::<Result<serde_json::Map<_, _>>>()
1956 .map(Value::Object),
1957 }
1958 }
1959
1960 fn into_serde_value(self) -> serde_value::Value {
1961 use serde_value::Value as SerdeValue;
1962
1963 match self {
1964 Self::Null => SerdeValue::Unit,
1965 Self::Boolean(value) => SerdeValue::Bool(value),
1966 Self::Long(value) => SerdeValue::I64(value),
1967 Self::Double(value) => SerdeValue::F64(value),
1968 Self::Bytes(value) => SerdeValue::Bytes(value),
1969 Self::String(value) => SerdeValue::String(value),
1970 Self::Array(values) => {
1971 SerdeValue::Seq(values.into_iter().map(Self::into_serde_value).collect())
1972 }
1973 Self::Map(values) => SerdeValue::Map(
1974 values
1975 .into_iter()
1976 .map(|(key, value)| (SerdeValue::String(key), value.into_serde_value()))
1977 .collect(),
1978 ),
1979 }
1980 }
1981
1982 pub fn deserialize<T: DeserializeOwned>(self) -> Result<T> {
1983 self.into_serde_value().deserialize_into().map_err(|error| {
1984 Error::Codec(format!(
1985 "avro_value_type_mismatch: could not adapt decoded value: {error}"
1986 ))
1987 })
1988 }
1989}
1990
1991impl Serialize for AvroValue {
1992 fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
1993 where
1994 S: Serializer,
1995 {
1996 match self {
1997 Self::Null => serializer.serialize_unit(),
1998 Self::Boolean(value) => serializer.serialize_bool(*value),
1999 Self::Long(value) => serializer.serialize_i64(*value),
2000 Self::Double(value) => serializer.serialize_f64(*value),
2001 Self::Bytes(value) => serializer.serialize_bytes(value),
2002 Self::String(value) => serializer.serialize_str(value),
2003 Self::Array(values) => {
2004 let mut sequence = serializer.serialize_seq(Some(values.len()))?;
2005 for value in values {
2006 sequence.serialize_element(value)?;
2007 }
2008 sequence.end()
2009 }
2010 Self::Map(values) => {
2011 let mut map = serializer.serialize_map(Some(values.len()))?;
2012 for (key, value) in values {
2013 map.serialize_entry(key, value)?;
2014 }
2015 map.end()
2016 }
2017 }
2018 }
2019}
2020
2021pub fn encode_avro_value(value: &AvroValue) -> Result<PayloadEnvelope> {
2022 let datum = avro_value_to_datum(value)?;
2023 let datum = to_avro_datum(avro_value_ordered_map_encoding_schema()?, datum)
2024 .map_err(|err| Error::Codec(format!("avro_value_encode_failed: {err}")))?;
2025 let mut bytes = Vec::with_capacity(datum.len() + 10);
2026 bytes.extend_from_slice(&AVRO_SINGLE_OBJECT_MAGIC);
2027 bytes.extend_from_slice(&AVRO_VALUE_SCHEMA_FINGERPRINT);
2028 bytes.extend_from_slice(&datum);
2029 Ok(PayloadEnvelope {
2030 codec: DEFAULT_CODEC.to_string(),
2031 blob: BASE64.encode(bytes),
2032 })
2033}
2034
2035pub fn decode_avro_value(envelope: &PayloadEnvelope) -> Result<AvroValue> {
2036 if envelope.codec != DEFAULT_CODEC {
2037 return Err(unsupported_payload_codec(&envelope.codec));
2038 }
2039 decode_avro_value_blob(&envelope.blob)
2040}
2041
2042pub fn encode_payload<T: Serialize>(value: &T, codec: &str) -> Result<PayloadEnvelope> {
2043 let blob = match codec {
2044 DEFAULT_CODEC => encode_avro_value(&AvroValue::from_serialize(value)?)?.blob,
2045 other => return Err(unsupported_payload_codec(other)),
2046 };
2047
2048 Ok(PayloadEnvelope {
2049 codec: codec.to_string(),
2050 blob,
2051 })
2052}
2053
2054pub fn decode_payload<T: DeserializeOwned>(envelope: &PayloadEnvelope) -> Result<T> {
2055 match envelope.codec.as_str() {
2056 DEFAULT_CODEC => decode_avro_value(envelope)?.deserialize(),
2057 other => Err(unsupported_payload_codec(other)),
2058 }
2059}
2060
2061fn handler_type_error<T>(
2062 handler_kind: HandlerKind,
2063 handler_name: &str,
2064 value_kind: HandlerValueKind,
2065 message: impl Into<String>,
2066) -> Error {
2067 Error::HandlerType {
2068 handler_kind,
2069 handler_name: handler_name.to_string(),
2070 value_kind,
2071 rust_type: type_name::<T>(),
2072 message: message.into(),
2073 }
2074}
2075
2076fn decode_handler_input<T: DeserializeOwned>(
2077 arguments: AvroValue,
2078 handler_kind: HandlerKind,
2079 handler_name: &str,
2080) -> Result<T> {
2081 let argument = match arguments {
2082 AvroValue::Array(mut arguments) if arguments.len() == 1 => {
2083 arguments.pop().expect("one typed handler argument")
2084 }
2085 AvroValue::Array(arguments) if arguments.is_empty() => AvroValue::Null,
2086 AvroValue::Array(arguments) => {
2087 return Err(handler_type_error::<T>(
2088 handler_kind,
2089 handler_name,
2090 HandlerValueKind::Input,
2091 format!(
2092 "typed handlers accept one request value, but the task carried {} arguments",
2093 arguments.len()
2094 ),
2095 ));
2096 }
2097 argument => argument,
2098 };
2099
2100 argument.deserialize().map_err(|error| {
2101 handler_type_error::<T>(
2102 handler_kind,
2103 handler_name,
2104 HandlerValueKind::Input,
2105 error.to_string(),
2106 )
2107 })
2108}
2109
2110fn encode_handler_result<T: Serialize>(
2111 result: &T,
2112 handler_kind: HandlerKind,
2113 handler_name: &str,
2114) -> Result<AvroValue> {
2115 AvroValue::from_serialize(result).map_err(|error| {
2116 handler_type_error::<T>(
2117 handler_kind,
2118 handler_name,
2119 HandlerValueKind::Result,
2120 error.to_string(),
2121 )
2122 })
2123}
2124
2125fn decode_handler_result<T: DeserializeOwned>(
2126 result: AvroValue,
2127 handler_kind: HandlerKind,
2128 handler_name: &str,
2129) -> Result<T> {
2130 result.deserialize().map_err(|error| {
2131 handler_type_error::<T>(
2132 handler_kind,
2133 handler_name,
2134 HandlerValueKind::Result,
2135 error.to_string(),
2136 )
2137 })
2138}
2139
2140#[cfg(test)]
2141fn encode_value_envelope(value: &Value, codec: &str) -> Result<Value> {
2142 Ok(serde_json::to_value(encode_payload(value, codec)?)?)
2143}
2144
2145fn decode_wire_value(value: &Value, fallback_codec: &str) -> Result<Value> {
2146 validate_payload_codec(fallback_codec)?;
2147
2148 if value.is_null() {
2149 return Ok(Value::Null);
2150 }
2151
2152 if let Some((codec, blob)) = payload_envelope_parts(value)? {
2153 return decode_blob(blob, codec);
2154 }
2155
2156 if let Some(blob) = value.as_str() {
2157 return decode_blob(blob, fallback_codec);
2158 }
2159
2160 Err(untagged_payload_value())
2161}
2162
2163fn encode_typed_envelope(value: &AvroValue, codec: &str) -> Result<Value> {
2164 let envelope = match codec {
2165 DEFAULT_CODEC => encode_avro_value(value)?,
2166 other => return Err(unsupported_payload_codec(other)),
2167 };
2168 Ok(serde_json::to_value(envelope)?)
2169}
2170
2171fn decode_wire_avro_value(value: &Value, fallback_codec: &str) -> Result<AvroValue> {
2172 validate_payload_codec(fallback_codec)?;
2173
2174 if value.is_null() {
2175 return Ok(AvroValue::Null);
2176 }
2177
2178 if let Some((codec, blob)) = payload_envelope_parts(value)? {
2179 validate_payload_codec(codec)?;
2180 return decode_avro_value_blob(blob);
2181 }
2182
2183 if let Some(blob) = value.as_str() {
2184 return match fallback_codec {
2185 DEFAULT_CODEC => decode_avro_value_blob(blob),
2186 other => Err(unsupported_payload_codec(other)),
2187 };
2188 }
2189
2190 Err(untagged_payload_value())
2191}
2192
2193fn normalize_avro_arguments(value: AvroValue) -> AvroValue {
2194 match value {
2195 AvroValue::Null => AvroValue::Array(Vec::new()),
2196 AvroValue::Array(_) => value,
2197 other => AvroValue::Array(vec![other]),
2198 }
2199}
2200
2201fn decode_blob(blob: &str, codec: &str) -> Result<Value> {
2202 match codec {
2203 DEFAULT_CODEC => decode_avro_value_blob(blob)?.into_json(),
2204 other => Err(unsupported_payload_codec(other)),
2205 }
2206}
2207
2208fn validate_payload_codec(codec: &str) -> Result<()> {
2209 match codec {
2210 DEFAULT_CODEC => Ok(()),
2211 MISSING_TASK_PAYLOAD_CODEC => {
2212 Err(invalid_task_payload_codec("task payload_codec is missing"))
2213 }
2214 NULL_TASK_PAYLOAD_CODEC => Err(invalid_task_payload_codec("task payload_codec is null")),
2215 NON_STRING_TASK_PAYLOAD_CODEC => Err(invalid_task_payload_codec(
2216 "task payload_codec must be a string",
2217 )),
2218 other => Err(unsupported_payload_codec(other)),
2219 }
2220}
2221
2222fn invalid_task_payload_codec(reason: &str) -> Error {
2223 Error::Codec(format!(
2224 "unsupported_payload_codec: {reason}; Durable Workflow 2.0 requires an explicit string payload_codec=\"avro\" before worker task execution"
2225 ))
2226}
2227
2228fn payload_envelope_parts(value: &Value) -> Result<Option<(&str, &str)>> {
2229 let Some(object) = value.as_object() else {
2230 return Ok(None);
2231 };
2232 if !object.contains_key("codec") && !object.contains_key("blob") {
2233 return Ok(None);
2234 }
2235
2236 let codec = object
2237 .get("codec")
2238 .and_then(Value::as_str)
2239 .ok_or_else(invalid_payload_envelope)?;
2240 validate_payload_codec(codec)?;
2241 let blob = object
2242 .get("blob")
2243 .and_then(Value::as_str)
2244 .ok_or_else(invalid_payload_envelope)?;
2245 Ok(Some((codec, blob)))
2246}
2247
2248fn invalid_payload_envelope() -> Error {
2249 Error::Codec(
2250 "invalid_payload_envelope: durable payloads must use an object with string codec=\"avro\" and blob fields"
2251 .to_string(),
2252 )
2253}
2254
2255fn validate_workflow_task_commands(commands: &[Value]) -> Result<()> {
2256 for command in commands {
2257 let Some(command) = command.as_object() else {
2258 continue;
2259 };
2260 let Some(command_type) = command.get("type").and_then(Value::as_str) else {
2261 continue;
2262 };
2263 let Some(payload_field) = workflow_command_payload_field(command_type) else {
2264 continue;
2265 };
2266
2267 if let Some(codec) = command.get("payload_codec") {
2268 let codec = codec.as_str().ok_or_else(invalid_payload_envelope)?;
2269 validate_payload_codec(codec)?;
2270 }
2271
2272 let payload = command
2273 .get(payload_field)
2274 .ok_or_else(invalid_payload_envelope)?;
2275 validate_outbound_payload_envelope(payload)?;
2276 }
2277 Ok(())
2278}
2279
2280fn workflow_completion_protocol_version(commands: &[Value]) -> &'static str {
2281 if commands.iter().any(|command| {
2282 command.get("type").and_then(Value::as_str) == Some("open_condition_wait")
2283 && command
2284 .get("condition_wait_occurrence_id")
2285 .and_then(Value::as_str)
2286 .is_some_and(|occurrence_id| !occurrence_id.is_empty())
2287 }) {
2288 CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
2289 } else if commands.iter().any(|command| {
2290 command.get("type").and_then(Value::as_str) == Some("upsert_search_attributes")
2291 && command.get("attribute_types").is_some()
2292 }) {
2293 TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION
2294 } else if commands
2295 .iter()
2296 .any(|command| command.get("type").and_then(Value::as_str) == Some("upsert_memo"))
2297 {
2298 MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION
2299 } else if commands
2300 .iter()
2301 .any(|command| command.get("type").and_then(Value::as_str) == Some("open_condition_wait"))
2302 {
2303 CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION
2304 } else if commands.iter().any(|command| {
2305 command.get("type").and_then(Value::as_str) == Some("upsert_search_attributes")
2306 }) {
2307 SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION
2308 } else {
2309 WORKER_PROTOCOL_VERSION
2310 }
2311}
2312
2313fn workflow_completion_protocol_version_with_message_streams(
2314 commands: &[Value],
2315 has_message_stream_metadata: bool,
2316) -> &'static str {
2317 let command_protocol = workflow_completion_protocol_version(commands);
2318 if has_message_stream_metadata && !worker_protocol_supports_message_streams(command_protocol) {
2319 MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION
2320 } else {
2321 command_protocol
2322 }
2323}
2324
2325fn workflow_command_payload_field(command_type: &str) -> Option<&'static str> {
2326 match command_type {
2327 "complete_workflow" | "complete_update" | "record_side_effect" => Some("result"),
2328 "schedule_activity" | "start_child_workflow" | "continue_as_new" => Some("arguments"),
2329 "start_service_operation" => Some("request_payload"),
2330 "upsert_memo" => Some("entries"),
2331 _ => None,
2332 }
2333}
2334
2335fn validate_outbound_payload_envelope(value: &Value) -> Result<()> {
2336 let Some((codec, blob)) = payload_envelope_parts(value)? else {
2337 return Err(untagged_payload_value());
2338 };
2339 validate_payload_codec(codec)?;
2340 decode_avro_value_blob(blob)?;
2341 Ok(())
2342}
2343
2344fn unsupported_payload_codec(codec: &str) -> Error {
2345 Error::Codec(format!(
2346 "unsupported_payload_codec: workflow payload codec {codec:?} is not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec"
2347 ))
2348}
2349
2350fn untagged_payload_value() -> Error {
2351 Error::Codec(
2352 "unsupported_payload_codec: untagged durable payload values are not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec"
2353 .to_string(),
2354 )
2355}
2356
2357fn decode_avro_value_blob(blob: &str) -> Result<AvroValue> {
2358 let bytes = BASE64.decode(blob).map_err(|err| {
2359 Error::Codec(format!(
2360 "invalid_payload_framing: expected strict base64 Avro single-object bytes: {err}"
2361 ))
2362 })?;
2363
2364 if serde_json::from_slice::<Value>(&bytes).is_ok() {
2365 return Err(unsupported_payload_codec("json"));
2366 }
2367
2368 if bytes.len() < 10 || bytes[..2] != AVRO_SINGLE_OBJECT_MAGIC {
2369 return Err(Error::Codec(
2370 "invalid_payload_framing: expected Avro single-object magic c301".to_string(),
2371 ));
2372 }
2373
2374 let fingerprint: [u8; 8] = bytes[2..10]
2375 .try_into()
2376 .map_err(|_| Error::Codec("invalid Avro fingerprint length".to_string()))?;
2377 if fingerprint != AVRO_VALUE_SCHEMA_FINGERPRINT {
2378 return Err(Error::Codec(format!(
2379 "unsupported_payload_schema: unknown CRC-64-AVRO fingerprint {}",
2380 fingerprint
2381 .iter()
2382 .map(|byte| format!("{byte:02x}"))
2383 .collect::<String>()
2384 )));
2385 }
2386
2387 let mut datum_reader = StrictAvroDatumReader::new(&bytes[10..]);
2388 let datum = from_avro_datum(avro_value_schema()?, &mut datum_reader, None);
2392 if datum_reader.truncated {
2393 return Err(Error::Codec(
2394 "invalid_payload_framing: truncated Avro Value datum".to_string(),
2395 ));
2396 }
2397 let datum = datum.map_err(|err| {
2398 Error::Codec(format!(
2399 "invalid_payload_framing: malformed Avro Value datum: {err}"
2400 ))
2401 })?;
2402 if datum_reader.remaining() != 0 {
2403 return Err(Error::Codec(format!(
2404 "invalid_payload_framing: {} trailing bytes after Avro Value datum",
2405 datum_reader.remaining()
2406 )));
2407 }
2408 avro_value_from_datum(datum)
2409}
2410
2411struct StrictAvroDatumReader<'a> {
2412 bytes: &'a [u8],
2413 offset: usize,
2414 truncated: bool,
2415}
2416
2417impl<'a> StrictAvroDatumReader<'a> {
2418 fn new(bytes: &'a [u8]) -> Self {
2419 Self {
2420 bytes,
2421 offset: 0,
2422 truncated: false,
2423 }
2424 }
2425
2426 fn remaining(&self) -> usize {
2427 self.bytes.len() - self.offset
2428 }
2429}
2430
2431impl Read for StrictAvroDatumReader<'_> {
2432 fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
2433 let count = buffer.len().min(self.remaining());
2434 buffer[..count].copy_from_slice(&self.bytes[self.offset..self.offset + count]);
2435 self.offset += count;
2436 if count < buffer.len() {
2437 self.truncated = true;
2438 }
2439
2440 Ok(count)
2441 }
2442}
2443
2444fn avro_value_to_datum(value: &AvroValue) -> Result<AvroDatum> {
2445 let branch = match value {
2446 AvroValue::Null => AvroDatum::Union(0, Box::new(AvroDatum::Null)),
2447 AvroValue::Boolean(value) => AvroDatum::Union(
2448 1,
2449 Box::new(AvroDatum::Record(vec![(
2450 "boolean".to_string(),
2451 AvroDatum::Boolean(*value),
2452 )])),
2453 ),
2454 AvroValue::Long(value) => AvroDatum::Union(
2455 2,
2456 Box::new(AvroDatum::Record(vec![(
2457 "long".to_string(),
2458 AvroDatum::Long(*value),
2459 )])),
2460 ),
2461 AvroValue::Double(value) => {
2462 if !value.is_finite() {
2463 return Err(Error::Codec(
2464 "non_finite_float: Avro Value doubles must be finite".to_string(),
2465 ));
2466 }
2467 AvroDatum::Union(
2468 3,
2469 Box::new(AvroDatum::Record(vec![(
2470 "double".to_string(),
2471 AvroDatum::Double(*value),
2472 )])),
2473 )
2474 }
2475 AvroValue::Bytes(value) => AvroDatum::Union(
2476 4,
2477 Box::new(AvroDatum::Record(vec![(
2478 "bytes".to_string(),
2479 AvroDatum::Bytes(value.clone()),
2480 )])),
2481 ),
2482 AvroValue::String(value) => AvroDatum::Union(
2483 5,
2484 Box::new(AvroDatum::Record(vec![(
2485 "string".to_string(),
2486 AvroDatum::String(value.clone()),
2487 )])),
2488 ),
2489 AvroValue::Array(values) => AvroDatum::Union(
2490 6,
2491 Box::new(AvroDatum::Record(vec![(
2492 "items".to_string(),
2493 AvroDatum::Array(
2494 values
2495 .iter()
2496 .map(avro_value_to_datum)
2497 .collect::<Result<Vec<_>>>()?,
2498 ),
2499 )])),
2500 ),
2501 AvroValue::Map(values) => AvroDatum::Union(
2502 7,
2503 Box::new(AvroDatum::Record(vec![(
2504 "entries".to_string(),
2505 AvroDatum::Array(
2506 values
2507 .iter()
2508 .map(|(key, value)| {
2509 Ok(AvroDatum::Record(vec![
2510 ("key".to_string(), AvroDatum::String(key.clone())),
2511 ("value".to_string(), avro_value_to_datum(value)?),
2512 ]))
2513 })
2514 .collect::<Result<Vec<_>>>()?,
2515 ),
2516 )])),
2517 ),
2518 };
2519 Ok(AvroDatum::Record(vec![("value".to_string(), branch)]))
2520}
2521
2522fn avro_value_from_datum(datum: AvroDatum) -> Result<AvroValue> {
2523 let AvroDatum::Record(mut outer) = datum else {
2524 return Err(Error::Codec(
2525 "invalid_payload_framing: datum is not a Value record".to_string(),
2526 ));
2527 };
2528 let (_, branch) = outer
2529 .pop()
2530 .filter(|(name, _)| name == "value")
2531 .ok_or_else(|| Error::Codec("invalid_payload_framing: Value field missing".to_string()))?;
2532 let AvroDatum::Union(_, branch) = branch else {
2533 return Err(Error::Codec(
2534 "invalid_payload_framing: invalid Value union".to_string(),
2535 ));
2536 };
2537 match *branch {
2538 AvroDatum::Null => Ok(AvroValue::Null),
2539 AvroDatum::Record(mut fields) => {
2540 let (name, value) = fields.pop().ok_or_else(|| {
2541 Error::Codec("invalid_payload_framing: empty Value branch".to_string())
2542 })?;
2543 match (name.as_str(), value) {
2544 ("boolean", AvroDatum::Boolean(value)) => Ok(AvroValue::Boolean(value)),
2545 ("long", AvroDatum::Long(value)) => Ok(AvroValue::Long(value)),
2546 ("double", AvroDatum::Double(value)) if value.is_finite() => {
2547 Ok(AvroValue::Double(value))
2548 }
2549 ("bytes", AvroDatum::Bytes(value)) => Ok(AvroValue::Bytes(value)),
2550 ("string", AvroDatum::String(value)) => Ok(AvroValue::String(value)),
2551 ("items", AvroDatum::Array(values)) => values
2552 .into_iter()
2553 .map(avro_value_from_datum)
2554 .collect::<Result<Vec<_>>>()
2555 .map(AvroValue::Array),
2556 ("entries", AvroDatum::Map(values)) => values
2557 .into_iter()
2558 .map(|(key, value)| Ok((key, avro_value_from_datum(value)?)))
2559 .collect::<Result<BTreeMap<_, _>>>()
2560 .map(AvroValue::Map),
2561 _ => Err(Error::Codec(
2562 "invalid_payload_framing: unknown Value branch".to_string(),
2563 )),
2564 }
2565 }
2566 _ => Err(Error::Codec(
2567 "invalid_payload_framing: invalid Value branch".to_string(),
2568 )),
2569 }
2570}
2571
2572fn avro_value_schema() -> Result<&'static Schema> {
2573 match AVRO_VALUE_SCHEMA.get_or_init(|| {
2574 Schema::parse_str(AVRO_VALUE_SCHEMA_JSON)
2575 .map_err(|err| format!("could not parse Avro Value schema: {err}"))
2576 }) {
2577 Ok(schema) => Ok(schema),
2578 Err(message) => Err(Error::Codec(message.clone())),
2579 }
2580}
2581
2582fn avro_value_ordered_map_encoding_schema() -> Result<&'static Schema> {
2583 match AVRO_VALUE_ORDERED_MAP_ENCODING_SCHEMA.get_or_init(|| {
2584 let mut schema: Value = serde_json::from_str(AVRO_VALUE_SCHEMA_JSON)
2589 .map_err(|err| format!("could not read packaged Avro Value schema: {err}"))?;
2590 let entries_schema = schema
2591 .pointer_mut("/fields/0/type/7/fields/0/type")
2592 .ok_or_else(|| "packaged Avro Value map schema is missing".to_string())?;
2593 if *entries_schema != json!({"type": "map", "values": "Value"}) {
2594 return Err("packaged Avro Value map schema changed unexpectedly".to_string());
2595 }
2596 *entries_schema = json!({
2597 "type": "array",
2598 "items": {
2599 "type": "record",
2600 "name": "MapEntry",
2601 "fields": [
2602 {"name": "key", "type": "string"},
2603 {"name": "value", "type": "Value"}
2604 ]
2605 }
2606 });
2607 Schema::parse_str(&schema.to_string())
2608 .map_err(|err| format!("could not parse ordered-map Avro Value schema: {err}"))
2609 }) {
2610 Ok(schema) => Ok(schema),
2611 Err(message) => Err(Error::Codec(message.clone())),
2612 }
2613}
2614
2615#[derive(Clone, Debug)]
2616pub struct Client {
2617 http: reqwest::Client,
2618 base_url: String,
2619 token: Option<String>,
2620 control_token: Option<String>,
2621 worker_token: Option<String>,
2622 namespace: String,
2623 max_external_payload_bytes: usize,
2624 worker_storage_admission: Option<WorkerStorageAdmission>,
2625}
2626
2627impl Client {
2628 pub fn new(base_url: impl Into<String>) -> Result<Self> {
2629 Self::builder(base_url).build()
2630 }
2631
2632 pub fn builder(base_url: impl Into<String>) -> ClientBuilder {
2633 ClientBuilder {
2634 base_url: base_url.into(),
2635 token: None,
2636 control_token: None,
2637 worker_token: None,
2638 namespace: "default".to_string(),
2639 timeout: Duration::from_secs(60),
2640 max_external_payload_bytes: 64 * 1024 * 1024,
2641 }
2642 }
2643
2644 pub async fn health(&self) -> Result<Value> {
2645 self.request_json(
2646 reqwest::Method::GET,
2647 "/health",
2648 RequestProtocol::ControlPlane,
2649 Option::<&Value>::None,
2650 )
2651 .await
2652 }
2653
2654 pub async fn cluster_info(&self) -> Result<Value> {
2655 self.request_json(
2656 reqwest::Method::GET,
2657 "/cluster/info",
2658 RequestProtocol::ControlPlane,
2659 Option::<&Value>::None,
2660 )
2661 .await
2662 }
2663
2664 pub async fn start_workflow<T: Serialize>(
2665 &self,
2666 workflow_type: &str,
2667 task_queue: &str,
2668 workflow_id: &str,
2669 input: T,
2670 ) -> Result<WorkflowHandle> {
2671 self.start_workflow_with_options(
2672 workflow_type,
2673 task_queue,
2674 workflow_id,
2675 WorkflowStartOptions::default(),
2676 input,
2677 )
2678 .await
2679 }
2680
2681 pub async fn start_workflow_with_options<T: Serialize>(
2684 &self,
2685 workflow_type: &str,
2686 task_queue: &str,
2687 workflow_id: &str,
2688 options: WorkflowStartOptions,
2689 input: T,
2690 ) -> Result<WorkflowHandle> {
2691 options.validate()?;
2692 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2693 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2694 let body = json!({
2695 "workflow_id": workflow_id,
2696 "workflow_type": workflow_type,
2697 "task_queue": task_queue,
2698 "input": input_envelope,
2699 "execution_timeout_seconds": options.execution_timeout_seconds,
2700 "run_timeout_seconds": options.run_timeout_seconds
2701 });
2702
2703 let data: Value = self
2704 .request_json(
2705 reqwest::Method::POST,
2706 "/workflows",
2707 RequestProtocol::ControlPlane,
2708 Some(&body),
2709 )
2710 .await?;
2711
2712 Ok(WorkflowHandle {
2713 client: self.clone(),
2714 workflow_id: data
2715 .get("workflow_id")
2716 .and_then(Value::as_str)
2717 .unwrap_or(workflow_id)
2718 .to_string(),
2719 run_id: data
2720 .get("run_id")
2721 .and_then(Value::as_str)
2722 .map(str::to_string),
2723 workflow_type: data
2724 .get("workflow_type")
2725 .and_then(Value::as_str)
2726 .unwrap_or(workflow_type)
2727 .to_string(),
2728 })
2729 }
2730
2731 pub async fn signal_workflow<T: Serialize>(
2732 &self,
2733 workflow_id: &str,
2734 signal_name: &str,
2735 input: T,
2736 ) -> Result<Value> {
2737 self.signal_workflow_target(workflow_id, None, signal_name, input)
2738 .await
2739 }
2740
2741 pub async fn append_message_stream<T: Serialize>(
2743 &self,
2744 workflow_id: &str,
2745 stream_name: &str,
2746 message_id: &str,
2747 input: T,
2748 ) -> Result<Value> {
2749 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2750 let body = json!({
2751 "message_id": message_id,
2752 "input": encode_typed_envelope(&input, DEFAULT_CODEC)?
2753 });
2754 self.request_json(
2755 reqwest::Method::POST,
2756 &format!("/workflows/{workflow_id}/message-streams/{stream_name}/messages"),
2757 RequestProtocol::ControlPlane,
2758 Some(&body),
2759 )
2760 .await
2761 }
2762
2763 pub async fn signal_workflow_run<T: Serialize>(
2765 &self,
2766 workflow_id: &str,
2767 run_id: &str,
2768 signal_name: &str,
2769 input: T,
2770 ) -> Result<Value> {
2771 self.signal_workflow_target(workflow_id, Some(run_id), signal_name, input)
2772 .await
2773 }
2774
2775 async fn signal_workflow_target<T: Serialize>(
2776 &self,
2777 workflow_id: &str,
2778 run_id: Option<&str>,
2779 signal_name: &str,
2780 input: T,
2781 ) -> Result<Value> {
2782 validate_user_signal_name(signal_name)?;
2783 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2784 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2785 let body = json!({
2786 "input": input_envelope
2787 });
2788 let path = match run_id {
2789 Some(run_id) => {
2790 format!("/workflows/{workflow_id}/runs/{run_id}/signal/{signal_name}")
2791 }
2792 None => format!("/workflows/{workflow_id}/signal/{signal_name}"),
2793 };
2794 self.request_json(
2795 reqwest::Method::POST,
2796 &path,
2797 RequestProtocol::ControlPlane,
2798 Some(&body),
2799 )
2800 .await
2801 }
2802
2803 pub async fn cancel_workflow(
2805 &self,
2806 workflow_id: &str,
2807 options: WorkflowCommandOptions,
2808 ) -> Result<WorkflowCommandResult> {
2809 self.workflow_command(workflow_id, None, WorkflowCommandKind::Cancel, options)
2810 .await
2811 }
2812
2813 pub async fn cancel_workflow_run(
2815 &self,
2816 workflow_id: &str,
2817 run_id: &str,
2818 options: WorkflowCommandOptions,
2819 ) -> Result<WorkflowCommandResult> {
2820 self.workflow_command(
2821 workflow_id,
2822 Some(run_id),
2823 WorkflowCommandKind::Cancel,
2824 options,
2825 )
2826 .await
2827 }
2828
2829 pub async fn terminate_workflow(
2831 &self,
2832 workflow_id: &str,
2833 options: WorkflowCommandOptions,
2834 ) -> Result<WorkflowCommandResult> {
2835 self.workflow_command(workflow_id, None, WorkflowCommandKind::Terminate, options)
2836 .await
2837 }
2838
2839 pub async fn terminate_workflow_run(
2841 &self,
2842 workflow_id: &str,
2843 run_id: &str,
2844 options: WorkflowCommandOptions,
2845 ) -> Result<WorkflowCommandResult> {
2846 self.workflow_command(
2847 workflow_id,
2848 Some(run_id),
2849 WorkflowCommandKind::Terminate,
2850 options,
2851 )
2852 .await
2853 }
2854
2855 async fn workflow_command(
2856 &self,
2857 workflow_id: &str,
2858 run_id: Option<&str>,
2859 command: WorkflowCommandKind,
2860 options: WorkflowCommandOptions,
2861 ) -> Result<WorkflowCommandResult> {
2862 let path = match run_id {
2863 Some(run_id) => format!(
2864 "/workflows/{workflow_id}/runs/{run_id}/{}",
2865 command.as_str()
2866 ),
2867 None => format!("/workflows/{workflow_id}/{}", command.as_str()),
2868 };
2869 let data = match self
2870 .request_json(
2871 reqwest::Method::POST,
2872 &path,
2873 RequestProtocol::ControlPlane,
2874 Some(&options),
2875 )
2876 .await
2877 {
2878 Ok(data) => data,
2879 Err(Error::Http { status, body }) => {
2880 return Err(Error::WorkflowCommandRejected(workflow_command_rejection(
2881 command,
2882 status,
2883 body,
2884 workflow_id,
2885 run_id,
2886 )));
2887 }
2888 Err(error) => return Err(error),
2889 };
2890
2891 Ok(workflow_command_result(command, data, workflow_id, run_id))
2892 }
2893
2894 pub async fn query_workflow<T: Serialize>(
2900 &self,
2901 workflow_id: &str,
2902 query_name: &str,
2903 input: T,
2904 ) -> Result<Value> {
2905 self.query_workflow_target(workflow_id, None, query_name, input)
2906 .await
2907 }
2908
2909 pub async fn query_workflow_run<T: Serialize>(
2911 &self,
2912 workflow_id: &str,
2913 run_id: &str,
2914 query_name: &str,
2915 input: T,
2916 ) -> Result<Value> {
2917 self.query_workflow_target(workflow_id, Some(run_id), query_name, input)
2918 .await
2919 }
2920
2921 pub async fn query_workflow_avro_value<T: Serialize>(
2923 &self,
2924 workflow_id: &str,
2925 query_name: &str,
2926 input: T,
2927 ) -> Result<AvroValue> {
2928 self.query_workflow_avro_value_target(workflow_id, None, query_name, input)
2929 .await
2930 }
2931
2932 pub async fn query_workflow_run_avro_value<T: Serialize>(
2934 &self,
2935 workflow_id: &str,
2936 run_id: &str,
2937 query_name: &str,
2938 input: T,
2939 ) -> Result<AvroValue> {
2940 self.query_workflow_avro_value_target(workflow_id, Some(run_id), query_name, input)
2941 .await
2942 }
2943
2944 async fn query_workflow_avro_value_target<T: Serialize>(
2945 &self,
2946 workflow_id: &str,
2947 run_id: Option<&str>,
2948 query_name: &str,
2949 input: T,
2950 ) -> Result<AvroValue> {
2951 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2952 let body = json!({"input": encode_typed_envelope(&input, DEFAULT_CODEC)?});
2953 let path = match run_id {
2954 Some(run_id) => {
2955 format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
2956 }
2957 None => format!("/workflows/{workflow_id}/query/{query_name}"),
2958 };
2959 let response: Value = match self
2960 .request_json(
2961 reqwest::Method::POST,
2962 &path,
2963 RequestProtocol::ControlPlane,
2964 Some(&body),
2965 )
2966 .await
2967 {
2968 Ok(response) => response,
2969 Err(Error::Http { status, body }) => {
2970 return Err(Error::QueryFailed(query_failure(status, body)));
2971 }
2972 Err(error) => return Err(error),
2973 };
2974
2975 let envelope = response
2976 .get("result_envelope")
2977 .filter(|envelope| !envelope.is_null())
2978 .ok_or_else(|| {
2979 Error::Codec(
2980 "missing_payload_envelope: typed query result requires result_envelope"
2981 .to_string(),
2982 )
2983 })?;
2984 decode_wire_avro_value(envelope, DEFAULT_CODEC)
2985 }
2986
2987 async fn query_workflow_target<T: Serialize>(
2988 &self,
2989 workflow_id: &str,
2990 run_id: Option<&str>,
2991 query_name: &str,
2992 input: T,
2993 ) -> Result<Value> {
2994 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2995 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2996 let body = json!({
2997 "input": input_envelope
2998 });
2999 let path = match run_id {
3000 Some(run_id) => {
3001 format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
3002 }
3003 None => format!("/workflows/{workflow_id}/query/{query_name}"),
3004 };
3005 let response: Value = match self
3006 .request_json(
3007 reqwest::Method::POST,
3008 &path,
3009 RequestProtocol::ControlPlane,
3010 Some(&body),
3011 )
3012 .await
3013 {
3014 Ok(response) => response,
3015 Err(Error::Http { status, body }) => {
3016 return Err(Error::QueryFailed(query_failure(status, body)));
3017 }
3018 Err(error) => return Err(error),
3019 };
3020
3021 if let Some(envelope) = response
3022 .get("result_envelope")
3023 .filter(|envelope| !envelope.is_null())
3024 {
3025 return decode_wire_value(envelope, DEFAULT_CODEC);
3026 }
3027
3028 Ok(response.get("result").cloned().unwrap_or(Value::Null))
3029 }
3030
3031 pub async fn update_workflow<T: Serialize>(
3033 &self,
3034 workflow_id: &str,
3035 update_name: &str,
3036 input: T,
3037 request_id: Option<&str>,
3038 ) -> Result<Value> {
3039 let response = self
3040 .update_workflow_response(workflow_id, update_name, input, request_id)
3041 .await?;
3042 if let Some(envelope) = response
3043 .get("result_envelope")
3044 .filter(|envelope| !envelope.is_null())
3045 {
3046 return decode_wire_value(envelope, DEFAULT_CODEC);
3047 }
3048 Ok(response.get("result").cloned().unwrap_or(response))
3049 }
3050
3051 pub async fn update_workflow_avro_value<T: Serialize>(
3053 &self,
3054 workflow_id: &str,
3055 update_name: &str,
3056 input: T,
3057 request_id: Option<&str>,
3058 ) -> Result<AvroValue> {
3059 let response = self
3060 .update_workflow_response(workflow_id, update_name, input, request_id)
3061 .await?;
3062 let envelope = response
3063 .get("result_envelope")
3064 .filter(|envelope| !envelope.is_null())
3065 .ok_or_else(|| {
3066 Error::Codec(
3067 "missing_payload_envelope: typed update result requires result_envelope"
3068 .to_string(),
3069 )
3070 })?;
3071 decode_wire_avro_value(envelope, DEFAULT_CODEC)
3072 }
3073
3074 async fn update_workflow_response<T: Serialize>(
3075 &self,
3076 workflow_id: &str,
3077 update_name: &str,
3078 input: T,
3079 request_id: Option<&str>,
3080 ) -> Result<Value> {
3081 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
3082 let mut body = json!({
3083 "input": encode_typed_envelope(&input, DEFAULT_CODEC)?,
3084 "wait_for": "completed",
3085 });
3086 if let Some(request_id) = request_id {
3087 body["request_id"] = json!(request_id);
3088 }
3089 self.request_json(
3090 reqwest::Method::POST,
3091 &format!("/workflows/{workflow_id}/update/{update_name}"),
3092 RequestProtocol::ControlPlane,
3093 Some(&body),
3094 )
3095 .await
3096 }
3097
3098 pub async fn describe_workflow(&self, workflow_id: &str) -> Result<WorkflowDescription> {
3099 let path = format!("/workflows/{workflow_id}");
3100 let mut data: WorkflowDescription = self
3101 .request_json(
3102 reqwest::Method::GET,
3103 &path,
3104 RequestProtocol::ControlPlane,
3105 Option::<&Value>::None,
3106 )
3107 .await?;
3108 data.decode_payloads()?;
3109 Ok(data)
3110 }
3111
3112 pub async fn describe_workflow_run(
3114 &self,
3115 workflow_id: &str,
3116 run_id: &str,
3117 ) -> Result<WorkflowDescription> {
3118 let path = format!("/workflows/{workflow_id}/runs/{run_id}");
3119 let mut data: WorkflowDescription = self
3120 .request_json(
3121 reqwest::Method::GET,
3122 &path,
3123 RequestProtocol::ControlPlane,
3124 Option::<&Value>::None,
3125 )
3126 .await?;
3127 data.decode_payloads()?;
3128 Ok(data)
3129 }
3130
3131 fn workflow_stream_path(workflow_id: &str, run_id: &str, stream_name: Option<&str>) -> String {
3132 let mut path = format!(
3133 "/workflows/{}/runs/{}/streams",
3134 percent_encode_path_segment(workflow_id),
3135 percent_encode_path_segment(run_id),
3136 );
3137 if let Some(stream_name) = stream_name {
3138 path.push('/');
3139 path.push_str(&percent_encode_path_segment(stream_name));
3140 }
3141 path
3142 }
3143
3144 pub async fn list_workflow_streams(
3146 &self,
3147 workflow_id: &str,
3148 run_id: &str,
3149 ) -> Result<Vec<WorkflowStreamDescription>> {
3150 let response: WorkflowStreamListResponse = self
3151 .request_json(
3152 reqwest::Method::GET,
3153 &Self::workflow_stream_path(workflow_id, run_id, None),
3154 RequestProtocol::ControlPlane,
3155 Option::<&Value>::None,
3156 )
3157 .await?;
3158 Ok(response.streams)
3159 }
3160
3161 pub async fn describe_workflow_stream(
3163 &self,
3164 workflow_id: &str,
3165 run_id: &str,
3166 stream_name: &str,
3167 ) -> Result<WorkflowStreamDescription> {
3168 let response: WorkflowStreamDescriptionResponse = self
3169 .request_json(
3170 reqwest::Method::GET,
3171 &Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3172 RequestProtocol::ControlPlane,
3173 Option::<&Value>::None,
3174 )
3175 .await?;
3176 Ok(response.stream)
3177 }
3178
3179 pub async fn subscribe_workflow_stream(
3186 &self,
3187 workflow_id: &str,
3188 run_id: &str,
3189 stream_name: &str,
3190 from_offset: u64,
3191 max_items: usize,
3192 wait: Duration,
3193 ) -> Result<WorkflowStreamPage> {
3194 let max_items = max_items.clamp(1, 500);
3195 let wait_seconds = wait.as_secs().min(MAX_LONG_POLL_TIMEOUT_SECONDS);
3196 let path = format!(
3197 "{}/items?from={from_offset}&max_items={max_items}&wait_seconds={wait_seconds}",
3198 Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3199 );
3200 let response: WorkflowStreamPageResponse = self
3201 .request_json_with_timeout(
3202 reqwest::Method::GET,
3203 &path,
3204 RequestProtocol::ControlPlane,
3205 Option::<&Value>::None,
3206 Duration::from_secs(wait_seconds.saturating_add(5).max(5)),
3207 )
3208 .await?;
3209
3210 let items = response
3211 .items
3212 .into_iter()
3213 .map(|raw| {
3214 let offset = raw.get("offset").and_then(Value::as_u64).unwrap_or(0);
3215 let envelope = raw.get("payload").cloned();
3216 let payload = envelope
3217 .as_ref()
3218 .filter(|value| value.get("blob").is_some())
3219 .map(|value| decode_wire_avro_value(value, DEFAULT_CODEC))
3220 .transpose()?
3221 .map(AvroValue::into_json)
3222 .transpose()?;
3223 Ok(WorkflowStreamItem {
3224 offset,
3225 payload,
3226 payload_envelope: envelope,
3227 payload_reference: raw
3228 .get("payload_reference")
3229 .and_then(Value::as_str)
3230 .map(str::to_string),
3231 payload_codec: raw
3232 .get("payload_codec")
3233 .and_then(Value::as_str)
3234 .map(str::to_string),
3235 idempotency_key: raw
3236 .get("idempotency_key")
3237 .and_then(Value::as_str)
3238 .map(str::to_string),
3239 item_type: raw
3240 .get("item_type")
3241 .and_then(Value::as_str)
3242 .map(str::to_string),
3243 content_type: raw
3244 .get("content_type")
3245 .and_then(Value::as_str)
3246 .map(str::to_string),
3247 origin: raw
3248 .get("origin")
3249 .and_then(Value::as_str)
3250 .map(str::to_string),
3251 origin_reference: raw
3252 .get("origin_reference")
3253 .and_then(Value::as_str)
3254 .map(str::to_string),
3255 emitted_at: raw
3256 .get("emitted_at")
3257 .and_then(Value::as_str)
3258 .map(str::to_string),
3259 raw,
3260 })
3261 })
3262 .collect::<Result<Vec<_>>>()?;
3263 Ok(WorkflowStreamPage {
3264 stream: response.stream,
3265 items,
3266 next_offset: response.next_offset,
3267 terminal: response.terminal,
3268 })
3269 }
3270
3271 pub async fn append_workflow_stream(
3273 &self,
3274 workflow_id: &str,
3275 run_id: &str,
3276 stream_name: &str,
3277 items: &[WorkflowStreamAppendItem],
3278 max_pending_items: Option<u64>,
3279 ) -> Result<WorkflowStreamAppendResult> {
3280 if items.is_empty() {
3281 return Err(Error::Codec(
3282 "workflow_stream_items_empty: append requires at least one item".to_string(),
3283 ));
3284 }
3285 let mut body = json!({
3286 "items": items
3287 .iter()
3288 .map(|item| item.wire_value(None))
3289 .collect::<Vec<_>>(),
3290 });
3291 if let Some(max_pending_items) = max_pending_items {
3292 if max_pending_items == 0 {
3293 return Err(Error::Codec(
3294 "workflow_stream_pending_limit_invalid: max_pending_items must be positive"
3295 .to_string(),
3296 ));
3297 }
3298 body["max_pending_items"] = json!(max_pending_items);
3299 }
3300 let response: WorkflowStreamAppendResponse = self
3301 .request_json(
3302 reqwest::Method::POST,
3303 &format!(
3304 "{}/items",
3305 Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3306 ),
3307 RequestProtocol::ControlPlane,
3308 Some(&body),
3309 )
3310 .await?;
3311 Ok(WorkflowStreamAppendResult {
3312 stream: response.stream,
3313 accepted_offsets: response.accepted_offsets,
3314 accepted: response.accepted,
3315 deduped: response.deduped,
3316 })
3317 }
3318
3319 pub async fn close_workflow_stream(
3321 &self,
3322 workflow_id: &str,
3323 run_id: &str,
3324 stream_name: &str,
3325 error_reason: Option<&str>,
3326 retention_seconds: Option<u64>,
3327 ) -> Result<WorkflowStreamDescription> {
3328 let mut body = json!({});
3329 if let Some(error_reason) = error_reason {
3330 body["error_reason"] = json!(error_reason);
3331 }
3332 if let Some(retention_seconds) = retention_seconds {
3333 if retention_seconds == 0 {
3334 return Err(Error::Codec(
3335 "workflow_stream_retention_invalid: retention_seconds must be positive"
3336 .to_string(),
3337 ));
3338 }
3339 body["retention_seconds"] = json!(retention_seconds);
3340 }
3341 let response: WorkflowStreamDescriptionResponse = self
3342 .request_json(
3343 reqwest::Method::POST,
3344 &format!(
3345 "{}/close",
3346 Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3347 ),
3348 RequestProtocol::ControlPlane,
3349 Some(&body),
3350 )
3351 .await?;
3352 Ok(response.stream)
3353 }
3354
3355 pub async fn register_worker(
3356 &self,
3357 worker_id: &str,
3358 task_queue: &str,
3359 supported_workflow_types: Vec<String>,
3360 supported_activity_types: Vec<String>,
3361 max_concurrent_workflow_tasks: usize,
3362 max_concurrent_activity_tasks: usize,
3363 ) -> Result<RegisterWorkerResponse> {
3364 self.register_worker_with_capabilities(
3365 worker_id,
3366 task_queue,
3367 supported_workflow_types,
3368 supported_activity_types,
3369 max_concurrent_workflow_tasks,
3370 max_concurrent_activity_tasks,
3371 Vec::new(),
3372 )
3373 .await
3374 }
3375
3376 pub async fn register_worker_with_capabilities(
3378 &self,
3379 worker_id: &str,
3380 task_queue: &str,
3381 supported_workflow_types: Vec<String>,
3382 supported_activity_types: Vec<String>,
3383 max_concurrent_workflow_tasks: usize,
3384 max_concurrent_activity_tasks: usize,
3385 capabilities: Vec<String>,
3386 ) -> Result<RegisterWorkerResponse> {
3387 self.register_worker_with_command_contracts(
3388 worker_id,
3389 task_queue,
3390 supported_workflow_types,
3391 supported_activity_types,
3392 max_concurrent_workflow_tasks,
3393 max_concurrent_activity_tasks,
3394 capabilities,
3395 Value::Object(serde_json::Map::new()),
3396 )
3397 .await
3398 }
3399
3400 #[allow(clippy::too_many_arguments)]
3407 pub async fn register_worker_with_command_contracts(
3408 &self,
3409 worker_id: &str,
3410 task_queue: &str,
3411 supported_workflow_types: Vec<String>,
3412 supported_activity_types: Vec<String>,
3413 max_concurrent_workflow_tasks: usize,
3414 max_concurrent_activity_tasks: usize,
3415 capabilities: Vec<String>,
3416 workflow_command_contracts: Value,
3417 ) -> Result<RegisterWorkerResponse> {
3418 if let Some(contracts) = workflow_command_contracts.as_object() {
3419 for (workflow_type, contract) in contracts {
3420 let Some(update_validators) = contract.get("update_validators") else {
3421 continue;
3422 };
3423 if !update_validators
3424 .as_array()
3425 .is_some_and(|validators| validators.is_empty())
3426 {
3427 return Err(Error::UnsupportedUpdateValidators {
3428 workflow_type: workflow_type.clone(),
3429 });
3430 }
3431 }
3432 }
3433
3434 let mut body = json!({
3435 "worker_id": worker_id,
3436 "task_queue": task_queue,
3437 "runtime": "rust",
3438 "sdk_version": SDK_VERSION,
3439 "supported_workflow_types": supported_workflow_types,
3440 "supported_activity_types": supported_activity_types,
3441 "capabilities": capabilities,
3442 "capability_manifest": portable_worker_affinity_capability_manifest(),
3443 "max_concurrent_workflow_tasks": max_concurrent_workflow_tasks,
3444 "max_concurrent_activity_tasks": max_concurrent_activity_tasks
3445 });
3446 if workflow_command_contracts
3447 .as_object()
3448 .is_some_and(|contracts| !contracts.is_empty())
3449 {
3450 body["workflow_command_contracts"] = workflow_command_contracts;
3451 }
3452
3453 self.request_json(
3454 reqwest::Method::POST,
3455 "/worker/register",
3456 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3457 Some(&body),
3458 )
3459 .await
3460 }
3461
3462 pub async fn deregister_worker_registration(
3468 &self,
3469 worker_id: &str,
3470 ) -> Result<WorkerDeregistrationEnvelope> {
3471 let path = format!(
3472 "/worker/registrations/{}",
3473 percent_encode_path_segment(worker_id)
3474 );
3475 self.request_json(
3476 reqwest::Method::DELETE,
3477 &path,
3478 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3479 Option::<&Value>::None,
3480 )
3481 .await
3482 }
3483
3484 pub async fn poll_query_task(
3486 &self,
3487 worker_id: &str,
3488 task_queue: &str,
3489 timeout: Duration,
3490 ) -> Result<Option<QueryTask>> {
3491 Ok(self
3492 .poll_query_task_response(worker_id, task_queue, timeout)
3493 .await?
3494 .task)
3495 }
3496
3497 pub async fn poll_query_task_response(
3499 &self,
3500 worker_id: &str,
3501 task_queue: &str,
3502 timeout: Duration,
3503 ) -> Result<PollQueryTaskResponse> {
3504 let poll_request_id = unique_request_id("rust-query-poll");
3505 self.poll_query_task_response_with_request_id(
3506 worker_id,
3507 task_queue,
3508 timeout,
3509 &poll_request_id,
3510 1,
3511 )
3512 .await
3513 }
3514
3515 async fn poll_query_task_response_with_request_id(
3516 &self,
3517 worker_id: &str,
3518 task_queue: &str,
3519 timeout: Duration,
3520 poll_request_id: &str,
3521 transport_retries: usize,
3522 ) -> Result<PollQueryTaskResponse> {
3523 let timeout_seconds = long_poll_timeout_seconds(timeout);
3524 let body = json!({
3525 "worker_id": worker_id,
3526 "task_queue": task_queue,
3527 "poll_request_id": poll_request_id,
3528 "timeout_seconds": timeout_seconds,
3529 });
3530 self.poll_request_json(
3531 "/worker/query-tasks/poll",
3532 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3533 &body,
3534 timeout + Duration::from_secs(5),
3535 transport_retries,
3536 )
3537 .await
3538 }
3539
3540 pub async fn complete_query_task<T: Serialize>(
3542 &self,
3543 query_task_id: &str,
3544 lease_owner: &str,
3545 query_task_attempt: u64,
3546 result: T,
3547 codec: &str,
3548 ) -> Result<Value> {
3549 let typed_result = AvroValue::from_serialize(&result)?;
3550 let result_envelope = encode_typed_envelope(&typed_result, codec)?;
3551 self.complete_query_task_with_envelope(
3552 query_task_id,
3553 lease_owner,
3554 query_task_attempt,
3555 typed_result.into_json()?,
3556 result_envelope,
3557 )
3558 .await
3559 }
3560
3561 async fn complete_query_task_with_envelope(
3562 &self,
3563 query_task_id: &str,
3564 lease_owner: &str,
3565 query_task_attempt: u64,
3566 result: Value,
3567 result_envelope: Value,
3568 ) -> Result<Value> {
3569 let body = json!({
3570 "lease_owner": lease_owner,
3571 "query_task_attempt": query_task_attempt,
3572 "result": result,
3573 "result_envelope": result_envelope,
3574 });
3575 let path = format!("/worker/query-tasks/{query_task_id}/complete");
3576 let response = self
3577 .request_json(
3578 reqwest::Method::POST,
3579 &path,
3580 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3581 Some(&body),
3582 )
3583 .await;
3584 query_task_response(response)
3585 }
3586
3587 pub async fn fail_query_task(
3589 &self,
3590 query_task_id: &str,
3591 lease_owner: &str,
3592 query_task_attempt: u64,
3593 message: impl Into<String>,
3594 reason: impl Into<String>,
3595 failure_type: impl Into<String>,
3596 ) -> Result<Value> {
3597 let body = json!({
3598 "lease_owner": lease_owner,
3599 "query_task_attempt": query_task_attempt,
3600 "failure": {
3601 "message": message.into(),
3602 "reason": reason.into(),
3603 "type": failure_type.into(),
3604 }
3605 });
3606 let path = format!("/worker/query-tasks/{query_task_id}/fail");
3607 let response = self
3608 .request_json(
3609 reqwest::Method::POST,
3610 &path,
3611 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3612 Some(&body),
3613 )
3614 .await;
3615 query_task_response(response)
3616 }
3617
3618 pub async fn heartbeat_worker(
3619 &self,
3620 worker_id: &str,
3621 workflow_available: usize,
3622 activity_available: usize,
3623 ) -> Result<Value> {
3624 let body = json!({
3625 "worker_id": worker_id,
3626 "task_slots": {
3627 "workflow_available": workflow_available,
3628 "activity_available": activity_available
3629 },
3630 "process_metrics": {
3631 "process_id": std::process::id(),
3632 "process_uptime_seconds": 0
3633 }
3634 });
3635
3636 self.request_json(
3637 reqwest::Method::POST,
3638 "/worker/heartbeat",
3639 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3640 Some(&body),
3641 )
3642 .await
3643 }
3644
3645 pub async fn poll_workflow_task(
3646 &self,
3647 worker_id: &str,
3648 task_queue: &str,
3649 timeout: Duration,
3650 ) -> Result<Option<WorkflowTask>> {
3651 Ok(self
3652 .poll_workflow_task_response(worker_id, task_queue, timeout)
3653 .await?
3654 .task)
3655 }
3656
3657 pub async fn poll_workflow_task_response(
3658 &self,
3659 worker_id: &str,
3660 task_queue: &str,
3661 timeout: Duration,
3662 ) -> Result<PollWorkflowTaskResponse> {
3663 let poll_request_id = unique_request_id("rust-workflow-poll");
3664 self.poll_workflow_task_response_with_request_id(
3665 worker_id,
3666 task_queue,
3667 timeout,
3668 &poll_request_id,
3669 1,
3670 )
3671 .await
3672 }
3673
3674 async fn poll_workflow_task_response_with_request_id(
3675 &self,
3676 worker_id: &str,
3677 task_queue: &str,
3678 timeout: Duration,
3679 poll_request_id: &str,
3680 transport_retries: usize,
3681 ) -> Result<PollWorkflowTaskResponse> {
3682 let body = json!({
3683 "worker_id": worker_id,
3684 "task_queue": task_queue,
3685 "poll_request_id": poll_request_id,
3686 "timeout_seconds": long_poll_timeout_seconds(timeout),
3687 });
3688 let mut data: PollWorkflowTaskResponse = self
3689 .poll_request_json(
3690 "/worker/workflow-tasks/poll",
3691 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3692 &body,
3693 timeout + Duration::from_secs(5),
3694 transport_retries,
3695 )
3696 .await?;
3697
3698 if let Some(task) = data.task.as_mut() {
3699 self.fetch_remaining_workflow_history(worker_id, task)
3700 .await?;
3701 }
3702
3703 Ok(data)
3704 }
3705
3706 async fn fetch_remaining_workflow_history(
3707 &self,
3708 worker_id: &str,
3709 task: &mut WorkflowTask,
3710 ) -> Result<()> {
3711 let mut next_token = task.next_history_page_token.clone();
3712
3713 while let Some(token) = next_token.take().filter(|token| !token.is_empty()) {
3714 let lease_owner = task
3715 .lease_owner
3716 .clone()
3717 .unwrap_or_else(|| worker_id.to_string());
3718 let page = self
3719 .workflow_task_history_page(
3720 &task.task_id,
3721 &lease_owner,
3722 task.workflow_task_attempt,
3723 &token,
3724 )
3725 .await?;
3726
3727 task.append_history_page(page);
3728
3729 if task.next_history_page_token.as_deref() == Some(token.as_str()) {
3730 return Err(Error::Codec(
3731 "workflow history pagination returned the same page token".to_string(),
3732 ));
3733 }
3734
3735 next_token = task.next_history_page_token.clone();
3736 }
3737
3738 Ok(())
3739 }
3740
3741 async fn workflow_task_history_page(
3742 &self,
3743 task_id: &str,
3744 lease_owner: &str,
3745 workflow_task_attempt: u64,
3746 next_history_page_token: &str,
3747 ) -> Result<WorkflowTaskHistoryPage> {
3748 let body = json!({
3749 "lease_owner": lease_owner,
3750 "workflow_task_attempt": workflow_task_attempt,
3751 "next_history_page_token": next_history_page_token
3752 });
3753 let path = format!("/worker/workflow-tasks/{task_id}/history");
3754
3755 self.request_json(
3756 reqwest::Method::POST,
3757 &path,
3758 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3759 Some(&body),
3760 )
3761 .await
3762 }
3763
3764 pub async fn complete_workflow_task(
3765 &self,
3766 task_id: &str,
3767 lease_owner: &str,
3768 workflow_task_attempt: u64,
3769 commands: Vec<Value>,
3770 ) -> Result<Value> {
3771 self.complete_workflow_task_with_message_streams(
3772 task_id,
3773 lease_owner,
3774 workflow_task_attempt,
3775 commands,
3776 Vec::new(),
3777 Vec::new(),
3778 )
3779 .await
3780 }
3781
3782 async fn complete_workflow_task_with_message_streams(
3783 &self,
3784 task_id: &str,
3785 lease_owner: &str,
3786 workflow_task_attempt: u64,
3787 commands: Vec<Value>,
3788 message_stream_cursors: Vec<Value>,
3789 message_stream_waits: Vec<Value>,
3790 ) -> Result<Value> {
3791 validate_workflow_task_commands(&commands)?;
3792 let has_message_stream_metadata =
3793 !message_stream_cursors.is_empty() || !message_stream_waits.is_empty();
3794 if has_message_stream_metadata
3795 && !worker_protocol_supports_message_streams(WORKER_PROTOCOL_VERSION)
3796 {
3797 return Err(Error::Codec(
3798 "message_streams_unavailable: message stream completion metadata requires worker protocol 1.15 or newer"
3799 .to_string(),
3800 ));
3801 }
3802 let protocol_version = workflow_completion_protocol_version_with_message_streams(
3803 &commands,
3804 has_message_stream_metadata,
3805 );
3806 let mut body = json!({
3807 "lease_owner": lease_owner,
3808 "workflow_task_attempt": workflow_task_attempt,
3809 "commands": commands
3810 });
3811 if !message_stream_cursors.is_empty() {
3812 body["message_stream_cursors"] = Value::Array(message_stream_cursors);
3813 }
3814 if !message_stream_waits.is_empty() {
3815 body["message_stream_waits"] = Value::Array(message_stream_waits);
3816 }
3817 let path = format!("/worker/workflow-tasks/{task_id}/complete");
3818 self.request_json(
3819 reqwest::Method::POST,
3820 &path,
3821 RequestProtocol::Worker(protocol_version),
3822 Some(&body),
3823 )
3824 .await
3825 }
3826
3827 pub async fn fail_workflow_task(
3828 &self,
3829 task_id: &str,
3830 lease_owner: &str,
3831 workflow_task_attempt: u64,
3832 message: impl Into<String>,
3833 ) -> Result<Value> {
3834 self.fail_workflow_task_with_type(
3835 task_id,
3836 lease_owner,
3837 workflow_task_attempt,
3838 message,
3839 "RustWorkflowTaskFailure",
3840 )
3841 .await
3842 }
3843
3844 async fn fail_workflow_task_with_type(
3845 &self,
3846 task_id: &str,
3847 lease_owner: &str,
3848 workflow_task_attempt: u64,
3849 message: impl Into<String>,
3850 failure_type: &str,
3851 ) -> Result<Value> {
3852 let body = json!({
3853 "lease_owner": lease_owner,
3854 "workflow_task_attempt": workflow_task_attempt,
3855 "failure": {
3856 "message": message.into(),
3857 "type": failure_type
3858 }
3859 });
3860 let path = format!("/worker/workflow-tasks/{task_id}/fail");
3861 self.request_json(
3862 reqwest::Method::POST,
3863 &path,
3864 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3865 Some(&body),
3866 )
3867 .await
3868 }
3869
3870 pub async fn poll_activity_task(
3871 &self,
3872 worker_id: &str,
3873 task_queue: &str,
3874 timeout: Duration,
3875 ) -> Result<Option<ActivityTask>> {
3876 Ok(self
3877 .poll_activity_task_response(worker_id, task_queue, timeout)
3878 .await?
3879 .task)
3880 }
3881
3882 pub async fn poll_activity_task_response(
3884 &self,
3885 worker_id: &str,
3886 task_queue: &str,
3887 timeout: Duration,
3888 ) -> Result<PollActivityTaskResponse> {
3889 let poll_request_id = unique_request_id("rust-activity-poll");
3890 self.poll_activity_task_response_with_request_id(
3891 worker_id,
3892 task_queue,
3893 timeout,
3894 &poll_request_id,
3895 1,
3896 )
3897 .await
3898 }
3899
3900 async fn poll_activity_task_response_with_request_id(
3901 &self,
3902 worker_id: &str,
3903 task_queue: &str,
3904 timeout: Duration,
3905 poll_request_id: &str,
3906 transport_retries: usize,
3907 ) -> Result<PollActivityTaskResponse> {
3908 let body = json!({
3909 "worker_id": worker_id,
3910 "task_queue": task_queue,
3911 "poll_request_id": poll_request_id,
3912 "timeout_seconds": long_poll_timeout_seconds(timeout),
3913 });
3914 let data: PollActivityTaskResponse = self
3915 .poll_request_json(
3916 "/worker/activity-tasks/poll",
3917 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3918 &body,
3919 timeout + Duration::from_secs(5),
3920 transport_retries,
3921 )
3922 .await?;
3923 Ok(data)
3924 }
3925
3926 pub async fn complete_activity_task<T: Serialize>(
3927 &self,
3928 task_id: &str,
3929 activity_attempt_id: &str,
3930 lease_owner: &str,
3931 result: T,
3932 codec: &str,
3933 ) -> Result<Value> {
3934 let result = encode_typed_envelope(&AvroValue::from_serialize(&result)?, codec)?;
3935 let body = json!({
3936 "activity_attempt_id": activity_attempt_id,
3937 "lease_owner": lease_owner,
3938 "result": result
3939 });
3940 let path = format!("/worker/activity-tasks/{task_id}/complete");
3941 activity_task_response(
3942 self.request_json(
3943 reqwest::Method::POST,
3944 &path,
3945 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3946 Some(&body),
3947 )
3948 .await,
3949 "complete",
3950 task_id,
3951 activity_attempt_id,
3952 )
3953 }
3954
3955 pub async fn fail_activity_task(
3956 &self,
3957 task_id: &str,
3958 activity_attempt_id: &str,
3959 lease_owner: &str,
3960 message: impl Into<String>,
3961 non_retryable: bool,
3962 ) -> Result<Value> {
3963 let body = json!({
3964 "activity_attempt_id": activity_attempt_id,
3965 "lease_owner": lease_owner,
3966 "failure": {
3967 "message": message.into(),
3968 "type": "RustActivityFailure",
3969 "non_retryable": non_retryable
3970 }
3971 });
3972 let path = format!("/worker/activity-tasks/{task_id}/fail");
3973 activity_task_response(
3974 self.request_json(
3975 reqwest::Method::POST,
3976 &path,
3977 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3978 Some(&body),
3979 )
3980 .await,
3981 "fail",
3982 task_id,
3983 activity_attempt_id,
3984 )
3985 }
3986
3987 pub async fn heartbeat_activity_task<T: Serialize>(
3988 &self,
3989 task_id: &str,
3990 activity_attempt_id: &str,
3991 lease_owner: &str,
3992 details: T,
3993 ) -> Result<ActivityHeartbeatResponse> {
3994 let details = encode_typed_envelope(&AvroValue::from_serialize(&details)?, DEFAULT_CODEC)?;
3995 let body = json!({
3996 "activity_attempt_id": activity_attempt_id,
3997 "lease_owner": lease_owner,
3998 "details": details
3999 });
4000 let path = format!("/worker/activity-tasks/{task_id}/heartbeat");
4001 activity_task_response(
4002 self.request_json(
4003 reqwest::Method::POST,
4004 &path,
4005 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
4006 Some(&body),
4007 )
4008 .await,
4009 "heartbeat",
4010 task_id,
4011 activity_attempt_id,
4012 )
4013 }
4014
4015 async fn request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
4016 &self,
4017 method: reqwest::Method,
4018 path: &str,
4019 protocol: RequestProtocol,
4020 body: Option<&B>,
4021 ) -> Result<T> {
4022 self.request_json_with_timeout(method, path, protocol, body, Duration::from_secs(60))
4023 .await
4024 }
4025
4026 async fn request_json_with_timeout<T: DeserializeOwned, B: Serialize + ?Sized>(
4027 &self,
4028 method: reqwest::Method,
4029 path: &str,
4030 protocol: RequestProtocol,
4031 body: Option<&B>,
4032 timeout: Duration,
4033 ) -> Result<T> {
4034 let auth_token = self.auth_token(protocol)?;
4035 let mut request = self
4036 .http
4037 .request(method, format!("{}/api{}", self.base_url, path))
4038 .timeout(timeout)
4039 .header(reqwest::header::ACCEPT, "application/json")
4040 .header(reqwest::header::CONTENT_TYPE, "application/json")
4041 .header("X-Namespace", &self.namespace);
4042
4043 match protocol {
4044 RequestProtocol::Worker(version) => {
4045 request = request.header("X-Durable-Workflow-Protocol-Version", version);
4046 }
4047 RequestProtocol::ControlPlane => {
4048 request = request.header(
4049 "X-Durable-Workflow-Control-Plane-Version",
4050 CONTROL_PLANE_VERSION,
4051 );
4052 }
4053 }
4054
4055 if let Some(token) = auth_token {
4056 request = request.bearer_auth(token);
4057 }
4058
4059 if let Some(body) = body {
4060 request = request.json(body);
4061 }
4062
4063 let request = request.build()?;
4064 let admission = self
4065 .worker_storage_admission
4066 .as_ref()
4067 .filter(|_| matches!(protocol, RequestProtocol::Worker(_)));
4068 let poll_request_id = path.ends_with("/poll").then(|| {
4069 request
4070 .body()
4071 .and_then(reqwest::Body::as_bytes)
4072 .and_then(|body| serde_json::from_slice::<Value>(body).ok())
4073 .and_then(|body| body.get("poll_request_id")?.as_str().map(str::to_owned))
4074 .unwrap_or_default()
4075 });
4076 let mut storage_retries = 0_usize;
4077
4078 loop {
4079 let response = self
4081 .http
4082 .execute(request.try_clone().ok_or_else(|| {
4083 Error::WorkerLoop("worker request body cannot be retried".to_string())
4084 })?)
4085 .await?;
4086 let status = response.status();
4087 let bytes = response.bytes().await?;
4088
4089 if !status.is_success() {
4090 let body = String::from_utf8_lossy(&bytes).to_string();
4091 if let Some(protocol) = protocol_failure(status, &body) {
4092 return Err(Error::Protocol(protocol));
4093 }
4094 let error = Error::Http { status, body };
4095 if let Some(admission) = admission {
4096 if let Some(advertised_delay) =
4097 worker_storage_admission_retry_after(&error, poll_request_id.as_deref())
4098 {
4099 storage_retries = storage_retries.saturating_add(1);
4100 let delay = worker_retry_delay(admission.policy, storage_retries)
4101 .max(advertised_delay)
4102 .min(admission.policy.max_backoff.max(Duration::from_millis(1)));
4103 let deadline = tokio::time::Instant::now() + delay;
4104 loop {
4105 if admission.stop.load(Ordering::SeqCst) {
4106 return Err(error);
4107 }
4108 let remaining =
4109 deadline.saturating_duration_since(tokio::time::Instant::now());
4110 if remaining.is_zero() {
4111 break;
4112 }
4113 tokio::time::sleep(remaining.min(Duration::from_millis(100))).await;
4114 }
4115 continue;
4116 }
4117 }
4118 return Err(error);
4119 }
4120
4121 if bytes.is_empty() {
4122 return Ok(serde_json::from_value(Value::Null)?);
4123 }
4124
4125 let mut value: Value = serde_json::from_slice(&bytes)?;
4126 self.resolve_runtime_payloads(&mut value, path, protocol)
4127 .await?;
4128 return Ok(serde_json::from_value(value)?);
4129 }
4130 }
4131
4132 async fn poll_request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
4133 &self,
4134 path: &str,
4135 protocol: RequestProtocol,
4136 body: &B,
4137 timeout: Duration,
4138 max_retries: usize,
4139 ) -> Result<T> {
4140 let mut retries = 0;
4141
4142 loop {
4143 let response = self
4144 .request_json_with_timeout(
4145 reqwest::Method::POST,
4146 path,
4147 protocol,
4148 Some(body),
4149 timeout,
4150 )
4151 .await;
4152
4153 match response {
4154 Err(Error::Transport(_)) if retries < max_retries => retries += 1,
4155 response => return worker_poll_response(response),
4156 }
4157 }
4158 }
4159
4160 fn auth_token(&self, protocol: RequestProtocol) -> Result<Option<&str>> {
4161 match protocol {
4162 RequestProtocol::Worker(_) => {
4163 if let Some(token) = self.worker_token.as_deref().or(self.token.as_deref()) {
4164 return Ok(Some(token));
4165 }
4166 if self.control_token.is_some() {
4167 return Err(Error::MissingRoleCredentials {
4168 role: "worker",
4169 opposite_role: "control",
4170 });
4171 }
4172 Ok(None)
4173 }
4174 RequestProtocol::ControlPlane => {
4175 if let Some(token) = self.control_token.as_deref().or(self.token.as_deref()) {
4176 return Ok(Some(token));
4177 }
4178 if self.worker_token.is_some() {
4179 return Err(Error::MissingRoleCredentials {
4180 role: "control",
4181 opposite_role: "worker",
4182 });
4183 }
4184 Ok(None)
4185 }
4186 }
4187 }
4188}
4189
4190fn query_failure(status: reqwest::StatusCode, raw_body: String) -> QueryFailure {
4191 let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
4192 let reason = body
4193 .get("reason")
4194 .and_then(Value::as_str)
4195 .unwrap_or("query_rejected")
4196 .to_string();
4197 let message = body
4198 .get("message")
4199 .or_else(|| body.get("error"))
4200 .and_then(Value::as_str)
4201 .unwrap_or("workflow query was rejected")
4202 .to_string();
4203
4204 QueryFailure {
4205 status: status.as_u16(),
4206 reason,
4207 message,
4208 body,
4209 }
4210}
4211
4212fn workflow_command_result(
4213 command: WorkflowCommandKind,
4214 data: Value,
4215 workflow_id: &str,
4216 run_id: Option<&str>,
4217) -> WorkflowCommandResult {
4218 WorkflowCommandResult {
4219 command,
4220 workflow_id: data
4221 .get("workflow_id")
4222 .and_then(Value::as_str)
4223 .unwrap_or(workflow_id)
4224 .to_string(),
4225 run_id: data
4226 .get("run_id")
4227 .and_then(Value::as_str)
4228 .or(run_id)
4229 .map(str::to_string),
4230 outcome: data
4231 .get("outcome")
4232 .and_then(Value::as_str)
4233 .map(str::to_string),
4234 reason: data
4235 .get("reason")
4236 .and_then(Value::as_str)
4237 .map(str::to_string),
4238 command_status: data
4239 .get("command_status")
4240 .and_then(Value::as_str)
4241 .map(str::to_string),
4242 raw: data,
4243 }
4244}
4245
4246fn workflow_command_rejection(
4247 command: WorkflowCommandKind,
4248 status: reqwest::StatusCode,
4249 raw_body: String,
4250 workflow_id: &str,
4251 run_id: Option<&str>,
4252) -> WorkflowCommandRejection {
4253 let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
4254 WorkflowCommandRejection {
4255 command,
4256 status: status.as_u16(),
4257 reason: body
4258 .get("reason")
4259 .and_then(Value::as_str)
4260 .unwrap_or("workflow_command_rejected")
4261 .to_string(),
4262 message: body
4263 .get("message")
4264 .or_else(|| body.get("error"))
4265 .and_then(Value::as_str)
4266 .unwrap_or("workflow lifecycle command was rejected")
4267 .to_string(),
4268 workflow_id: body
4269 .get("workflow_id")
4270 .and_then(Value::as_str)
4271 .unwrap_or(workflow_id)
4272 .to_string(),
4273 run_id: body
4274 .get("run_id")
4275 .and_then(Value::as_str)
4276 .or(run_id)
4277 .map(str::to_string),
4278 target_scope: body
4279 .get("target_scope")
4280 .and_then(Value::as_str)
4281 .map(str::to_string),
4282 body,
4283 }
4284}
4285
4286fn query_task_response(response: Result<Value>) -> Result<Value> {
4287 match response {
4288 Err(Error::Http { status, body }) => Err(Error::QueryFailed(query_failure(status, body))),
4289 response => response,
4290 }
4291}
4292
4293fn worker_poll_response<T: DeserializeOwned>(response: Result<T>) -> Result<T> {
4294 match response {
4295 Err(Error::Http { status, body })
4296 if status == reqwest::StatusCode::CONFLICT && worker_poll_body_is_stop(&body) =>
4297 {
4298 Ok(serde_json::from_str(&body)?)
4299 }
4300 response => response,
4301 }
4302}
4303
4304fn worker_poll_body_is_stop(body: &str) -> bool {
4305 serde_json::from_str::<Value>(body)
4306 .ok()
4307 .is_some_and(|body| {
4308 worker_poll_is_stop(
4309 body.get("poll_status").and_then(Value::as_str),
4310 body.get("reason").and_then(Value::as_str),
4311 )
4312 })
4313}
4314
4315fn worker_poll_is_stop(poll_status: Option<&str>, reason: Option<&str>) -> bool {
4316 matches!(poll_status, Some("draining" | "stopped"))
4317 || matches!(reason, Some("worker_draining" | "worker_stopped"))
4318}
4319
4320fn query_task_rejection_is_final(error: &Error) -> bool {
4321 matches!(
4322 error,
4323 Error::QueryFailed(failure)
4324 if QUERY_TASK_FINAL_REJECTION_REASONS.contains(&failure.reason.as_str())
4325 )
4326}
4327
4328fn activity_task_response<T>(
4329 response: Result<T>,
4330 operation: &str,
4331 task_id: &str,
4332 activity_attempt_id: &str,
4333) -> Result<T> {
4334 match response {
4335 Err(Error::Http { status, body }) => {
4336 let body = serde_json::from_str(&body).unwrap_or_else(|_| json!({"message": body}));
4337 Err(Error::ActivityTaskRejected(ActivityTaskRejection {
4338 operation: operation.to_string(),
4339 status: status.as_u16(),
4340 reason: body
4341 .get("reason")
4342 .and_then(Value::as_str)
4343 .unwrap_or("activity_task_rejected")
4344 .to_string(),
4345 task_id: body
4346 .get("task_id")
4347 .and_then(Value::as_str)
4348 .unwrap_or(task_id)
4349 .to_string(),
4350 activity_attempt_id: body
4351 .get("activity_attempt_id")
4352 .and_then(Value::as_str)
4353 .unwrap_or(activity_attempt_id)
4354 .to_string(),
4355 cancel_requested: body
4356 .get("cancel_requested")
4357 .and_then(Value::as_bool)
4358 .unwrap_or(false),
4359 can_continue: body.get("can_continue").and_then(Value::as_bool),
4360 run_closed_reason: body
4361 .get("run_closed_reason")
4362 .and_then(Value::as_str)
4363 .map(str::to_string),
4364 body,
4365 }))
4366 }
4367 response => response,
4368 }
4369}
4370
4371fn activity_task_rejection_is_final(error: &Error) -> bool {
4372 matches!(
4373 error,
4374 Error::ActivityTaskRejected(rejection)
4375 if matches!(
4376 rejection.reason.as_str(),
4377 "run_cancelled"
4378 | "run_terminated"
4379 | "attempt_closed"
4380 | "stale_attempt"
4381 | "activity_cancelled"
4382 | "task_cancelled"
4383 | "run_closed"
4384 | "activity_not_running"
4385 | "attempt_not_found"
4386 )
4387 )
4388}
4389
4390fn workflow_task_completion_is_terminal_timeout(
4391 error: &Error,
4392 task_id: &str,
4393 workflow_task_attempt: u64,
4394 run_id: Option<&str>,
4395) -> bool {
4396 let Error::Http { status, body } = error else {
4397 return false;
4398 };
4399 if *status != reqwest::StatusCode::CONFLICT {
4400 return false;
4401 }
4402
4403 let Some(run_id) = run_id else {
4404 return false;
4405 };
4406 let Ok(body) = serde_json::from_str::<Value>(body) else {
4407 return false;
4408 };
4409
4410 body.get("recorded").and_then(Value::as_bool) == Some(false)
4411 && body.get("reason").and_then(Value::as_str) == Some("run_timed_out")
4412 && body.get("run_status").and_then(Value::as_str) == Some("failed")
4413 && body.get("run_id").and_then(Value::as_str) == Some(run_id)
4414 && body.get("task_id").and_then(Value::as_str) == Some(task_id)
4415 && body.get("workflow_task_attempt").and_then(Value::as_u64) == Some(workflow_task_attempt)
4416}
4417
4418fn protocol_failure(status: reqwest::StatusCode, raw_body: &str) -> Option<ProtocolFailure> {
4419 let body: Value = serde_json::from_str(raw_body).ok()?;
4420 let reason = body.get("reason")?.as_str()?;
4421 if !matches!(
4422 reason,
4423 "missing_protocol_version"
4424 | "unsupported_protocol_version"
4425 | "missing_control_plane_version"
4426 | "unsupported_control_plane_version"
4427 ) {
4428 return None;
4429 }
4430
4431 Some(ProtocolFailure {
4432 status: status.as_u16(),
4433 reason: reason.to_string(),
4434 message: body
4435 .get("message")
4436 .or_else(|| body.get("error"))
4437 .and_then(Value::as_str)
4438 .unwrap_or("protocol version rejected")
4439 .to_string(),
4440 supported_version: body
4441 .get("supported_version")
4442 .and_then(Value::as_str)
4443 .map(str::to_string),
4444 requested_version: body
4445 .get("requested_version")
4446 .and_then(Value::as_str)
4447 .map(str::to_string),
4448 body,
4449 })
4450}
4451
4452fn long_poll_timeout_seconds(timeout: Duration) -> u64 {
4453 timeout
4454 .as_secs()
4455 .saturating_add(u64::from(timeout.subsec_nanos() > 0))
4456 .min(MAX_LONG_POLL_TIMEOUT_SECONDS)
4457}
4458
4459fn worker_operation_is_retryable(error: &Error) -> bool {
4460 if worker_poll_capacity_retry_after(error).is_some()
4461 || worker_storage_admission_body(error).is_some()
4462 || worker_operation_is_explicitly_non_retryable(error)
4463 {
4464 return false;
4465 }
4466
4467 match error {
4468 Error::Transport(error) => {
4469 error.is_timeout() || error.is_connect() || error.is_request() || error.is_body()
4470 }
4471 Error::Http { status, .. } => {
4472 matches!(
4473 *status,
4474 reqwest::StatusCode::REQUEST_TIMEOUT | reqwest::StatusCode::TOO_MANY_REQUESTS
4475 ) || status.is_server_error()
4476 }
4477 _ => false,
4478 }
4479}
4480
4481fn worker_storage_admission_body(error: &Error) -> Option<Value> {
4482 let body: Value = match error {
4483 Error::Http { body, .. } => serde_json::from_str(body).ok()?,
4484 Error::ActivityTaskRejected(rejection) => rejection.body.clone(),
4485 _ => return None,
4486 };
4487 matches!(
4488 body.get("reason").and_then(Value::as_str),
4489 Some("storage_pressure" | "storage_admission_unavailable")
4490 )
4491 .then_some(body)
4492}
4493
4494fn worker_storage_admission_retry_after(
4495 error: &Error,
4496 poll_request_id: Option<&str>,
4497) -> Option<Duration> {
4498 let Error::Http { status, .. } = error else {
4499 return None;
4500 };
4501 let body = worker_storage_admission_body(error)?;
4502 let delay = body.get("retry_after_seconds")?.as_u64()?;
4503 if *status != reqwest::StatusCode::SERVICE_UNAVAILABLE
4504 || delay == 0
4505 || body.get("retryable") != Some(&Value::Bool(true))
4506 || !matches!(body.get("storage_state")?.as_str()?, "draining" | "fenced")
4507 || (body["reason"] == "storage_admission_unavailable" && body["storage_state"] != "fenced")
4508 || body
4509 .get("request_admitted")
4510 .is_some_and(|admitted| admitted != &Value::Bool(false))
4511 {
4512 return None;
4513 }
4514 match poll_request_id {
4515 Some(id) => {
4516 if id.is_empty()
4517 || body.get("task") != Some(&Value::Null)
4518 || body.get("poll_request_id").and_then(Value::as_str) != Some(id)
4519 || body.get("poll_status") != body.get("reason")
4520 || body.get("retry_same_poll_request_id") != Some(&Value::Bool(true))
4521 || body.get("claim_admitted") != Some(&Value::Bool(false))
4522 {
4523 return None;
4524 }
4525 }
4526 None if body.get("request_admitted") != Some(&Value::Bool(false)) => return None,
4527 None => {}
4528 }
4529 Some(Duration::from_secs(delay))
4530}
4531
4532fn worker_operation_is_explicitly_non_retryable(error: &Error) -> bool {
4533 let Error::Http { body, .. } = error else {
4534 return false;
4535 };
4536
4537 serde_json::from_str::<Value>(body)
4538 .ok()
4539 .and_then(|body| body.get("retryable").and_then(Value::as_bool))
4540 == Some(false)
4541}
4542
4543fn worker_poll_capacity_retry_after(error: &Error) -> Option<Duration> {
4544 let Error::Http { status, body } = error else {
4545 return None;
4546 };
4547 if *status != reqwest::StatusCode::TOO_MANY_REQUESTS {
4548 return None;
4549 }
4550
4551 let body = serde_json::from_str::<Value>(body).ok()?;
4552 let capacity_exhausted = body.get("poll_status").and_then(Value::as_str)
4553 == Some("long_poll_capacity_exhausted")
4554 || body.get("reason").and_then(Value::as_str) == Some("long_poll_capacity_exhausted");
4555 if !capacity_exhausted || body.get("retryable").and_then(Value::as_bool) != Some(true) {
4556 return None;
4557 }
4558
4559 Some(Duration::from_secs(
4560 body.get("retry_after_seconds")
4561 .and_then(Value::as_u64)
4562 .unwrap_or_default(),
4563 ))
4564}
4565
4566fn worker_retry_delay(policy: WorkerRetryPolicy, retry: usize) -> Duration {
4567 let exponent = retry.saturating_sub(1).min(31) as u32;
4568 policy
4569 .initial_backoff
4570 .saturating_mul(1_u32 << exponent)
4571 .min(policy.max_backoff)
4572}
4573
4574#[derive(Debug)]
4575pub struct ClientBuilder {
4576 base_url: String,
4577 token: Option<String>,
4578 control_token: Option<String>,
4579 worker_token: Option<String>,
4580 namespace: String,
4581 timeout: Duration,
4582 max_external_payload_bytes: usize,
4583}
4584
4585impl ClientBuilder {
4586 pub fn token(mut self, token: Option<String>) -> Self {
4587 self.token = token;
4588 self
4589 }
4590
4591 pub fn control_token(mut self, token: Option<String>) -> Self {
4592 self.control_token = token;
4593 self
4594 }
4595
4596 pub fn worker_token(mut self, token: Option<String>) -> Self {
4597 self.worker_token = token;
4598 self
4599 }
4600
4601 pub fn namespace(mut self, namespace: impl Into<String>) -> Self {
4602 self.namespace = namespace.into();
4603 self
4604 }
4605
4606 pub fn timeout(mut self, timeout: Duration) -> Self {
4607 self.timeout = timeout;
4608 self
4609 }
4610
4611 pub fn max_external_payload_bytes(mut self, bytes: usize) -> Self {
4614 self.max_external_payload_bytes = bytes;
4615 self
4616 }
4617
4618 pub fn build(self) -> Result<Client> {
4619 let base_url = self.base_url.trim_end_matches('/').to_string();
4620 let has_sdk_api_suffix = reqwest::Url::parse(&base_url)
4621 .map(|url| url.path().trim_end_matches('/').ends_with("/api"))
4622 .unwrap_or_else(|_| base_url.ends_with("/api"));
4623
4624 if has_sdk_api_suffix {
4625 return Err(Error::InvalidBaseUrl);
4626 }
4627
4628 Ok(Client {
4629 http: reqwest::Client::builder()
4630 .timeout(self.timeout)
4631 .redirect(reqwest::redirect::Policy::none())
4632 .build()?,
4633 base_url,
4634 token: self.token,
4635 control_token: self.control_token,
4636 worker_token: self.worker_token,
4637 namespace: self.namespace,
4638 max_external_payload_bytes: self.max_external_payload_bytes,
4639 worker_storage_admission: None,
4640 })
4641 }
4642}
4643
4644#[derive(Clone, Debug)]
4645pub struct WorkflowHandle {
4646 client: Client,
4647 pub workflow_id: String,
4648 pub run_id: Option<String>,
4649 pub workflow_type: String,
4650}
4651
4652impl WorkflowHandle {
4653 pub async fn describe(&self) -> Result<WorkflowDescription> {
4655 self.client.describe_workflow(&self.workflow_id).await
4656 }
4657
4658 pub async fn describe_selected_run(&self) -> Result<WorkflowDescription> {
4660 let run_id = self.run_id.as_deref().ok_or_else(|| {
4661 Error::Codec("run_id is required for selected-run description".to_string())
4662 })?;
4663 self.client
4664 .describe_workflow_run(&self.workflow_id, run_id)
4665 .await
4666 }
4667
4668 pub async fn signal<T: Serialize>(&self, signal_name: &str, input: T) -> Result<Value> {
4669 self.client
4670 .signal_workflow(&self.workflow_id, signal_name, input)
4671 .await
4672 }
4673
4674 pub async fn append_message<T: Serialize>(
4675 &self,
4676 stream_name: &str,
4677 message_id: &str,
4678 input: T,
4679 ) -> Result<Value> {
4680 self.client
4681 .append_message_stream(&self.workflow_id, stream_name, message_id, input)
4682 .await
4683 }
4684
4685 pub async fn signal_selected_run<T: Serialize>(
4687 &self,
4688 signal_name: &str,
4689 input: T,
4690 ) -> Result<Value> {
4691 let run_id = self.run_id.as_deref().ok_or_else(|| {
4692 Error::Codec("run_id is required for selected-run signaling".to_string())
4693 })?;
4694 self.client
4695 .signal_workflow_run(&self.workflow_id, run_id, signal_name, input)
4696 .await
4697 }
4698
4699 pub async fn cancel(&self, options: WorkflowCommandOptions) -> Result<WorkflowCommandResult> {
4701 self.client
4702 .cancel_workflow(&self.workflow_id, options)
4703 .await
4704 }
4705
4706 pub async fn cancel_selected_run(
4708 &self,
4709 options: WorkflowCommandOptions,
4710 ) -> Result<WorkflowCommandResult> {
4711 let run_id = self.run_id.as_deref().ok_or_else(|| {
4712 Error::Codec("run_id is required for selected-run cancellation".to_string())
4713 })?;
4714 self.client
4715 .cancel_workflow_run(&self.workflow_id, run_id, options)
4716 .await
4717 }
4718
4719 pub async fn terminate(
4721 &self,
4722 options: WorkflowCommandOptions,
4723 ) -> Result<WorkflowCommandResult> {
4724 self.client
4725 .terminate_workflow(&self.workflow_id, options)
4726 .await
4727 }
4728
4729 pub async fn terminate_selected_run(
4731 &self,
4732 options: WorkflowCommandOptions,
4733 ) -> Result<WorkflowCommandResult> {
4734 let run_id = self.run_id.as_deref().ok_or_else(|| {
4735 Error::Codec("run_id is required for selected-run termination".to_string())
4736 })?;
4737 self.client
4738 .terminate_workflow_run(&self.workflow_id, run_id, options)
4739 .await
4740 }
4741
4742 pub async fn query<T: Serialize>(&self, query_name: &str, input: T) -> Result<Value> {
4744 self.client
4745 .query_workflow(&self.workflow_id, query_name, input)
4746 .await
4747 }
4748
4749 pub async fn query_avro_value<T: Serialize>(
4750 &self,
4751 query_name: &str,
4752 input: T,
4753 ) -> Result<AvroValue> {
4754 self.client
4755 .query_workflow_avro_value(&self.workflow_id, query_name, input)
4756 .await
4757 }
4758
4759 pub async fn update<T: Serialize>(
4760 &self,
4761 update_name: &str,
4762 input: T,
4763 request_id: Option<&str>,
4764 ) -> Result<Value> {
4765 self.client
4766 .update_workflow(&self.workflow_id, update_name, input, request_id)
4767 .await
4768 }
4769
4770 pub async fn update_avro_value<T: Serialize>(
4771 &self,
4772 update_name: &str,
4773 input: T,
4774 request_id: Option<&str>,
4775 ) -> Result<AvroValue> {
4776 self.client
4777 .update_workflow_avro_value(&self.workflow_id, update_name, input, request_id)
4778 .await
4779 }
4780
4781 pub async fn query_selected_run<T: Serialize>(
4783 &self,
4784 query_name: &str,
4785 input: T,
4786 ) -> Result<Value> {
4787 let run_id = self
4788 .run_id
4789 .as_deref()
4790 .ok_or_else(|| Error::Codec("run_id is required for selected-run query".to_string()))?;
4791 self.client
4792 .query_workflow_run(&self.workflow_id, run_id, query_name, input)
4793 .await
4794 }
4795
4796 pub async fn result(&self, options: WorkflowResultOptions) -> Result<Value> {
4798 self.result_target(options, None).await
4799 }
4800
4801 pub async fn result_avro_value(&self, options: WorkflowResultOptions) -> Result<AvroValue> {
4803 self.result_avro_value_target(options, None).await
4804 }
4805
4806 pub async fn result_typed<T: DeserializeOwned>(
4808 &self,
4809 options: WorkflowResultOptions,
4810 ) -> Result<T> {
4811 let result = self.result_avro_value(options).await?;
4812 decode_handler_result(result, HandlerKind::Workflow, &self.workflow_type)
4813 }
4814
4815 pub async fn result_selected_run(&self, options: WorkflowResultOptions) -> Result<Value> {
4817 let run_id = self.run_id.as_deref().ok_or_else(|| {
4818 Error::Codec("run_id is required for selected-run result".to_string())
4819 })?;
4820 self.result_target(options, Some(run_id)).await
4821 }
4822
4823 pub async fn result_selected_run_avro_value(
4825 &self,
4826 options: WorkflowResultOptions,
4827 ) -> Result<AvroValue> {
4828 let run_id = self.run_id.as_deref().ok_or_else(|| {
4829 Error::Codec("run_id is required for selected-run result".to_string())
4830 })?;
4831 self.result_avro_value_target(options, Some(run_id)).await
4832 }
4833
4834 pub async fn result_selected_run_typed<T: DeserializeOwned>(
4836 &self,
4837 options: WorkflowResultOptions,
4838 ) -> Result<T> {
4839 let result = self.result_selected_run_avro_value(options).await?;
4840 decode_handler_result(result, HandlerKind::Workflow, &self.workflow_type)
4841 }
4842
4843 async fn result_avro_value_target(
4844 &self,
4845 options: WorkflowResultOptions,
4846 selected_run_id: Option<&str>,
4847 ) -> Result<AvroValue> {
4848 let started = Instant::now();
4849
4850 loop {
4851 let description = match selected_run_id {
4852 Some(run_id) => {
4853 self.client
4854 .describe_workflow_run(&self.workflow_id, run_id)
4855 .await?
4856 }
4857 None => self.describe().await?,
4858 };
4859 if description.is_completed() {
4860 return description.output_avro_value.ok_or_else(|| {
4861 Error::Codec(
4862 "missing_payload_envelope: typed workflow result requires output_envelope"
4863 .to_string(),
4864 )
4865 });
4866 }
4867 if description.is_terminal() {
4868 let outcome =
4869 workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
4870 return Err(match outcome.kind {
4871 WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
4872 WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
4873 WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
4874 WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
4875 });
4876 }
4877 if started.elapsed() >= options.timeout {
4878 return Err(Error::Timeout);
4879 }
4880 tokio::time::sleep(options.poll_interval).await;
4881 }
4882 }
4883
4884 async fn result_target(
4885 &self,
4886 options: WorkflowResultOptions,
4887 selected_run_id: Option<&str>,
4888 ) -> Result<Value> {
4889 let started = Instant::now();
4890
4891 loop {
4892 let description = match selected_run_id {
4893 Some(run_id) => {
4894 self.client
4895 .describe_workflow_run(&self.workflow_id, run_id)
4896 .await?
4897 }
4898 None => self.describe().await?,
4899 };
4900 if description.is_completed() {
4901 return Ok(description.output.unwrap_or(Value::Null));
4902 }
4903
4904 if description.is_terminal() {
4905 let outcome =
4906 workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
4907 return Err(match outcome.kind {
4908 WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
4909 WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
4910 WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
4911 WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
4912 });
4913 }
4914
4915 if started.elapsed() >= options.timeout {
4916 return Err(Error::WorkflowTimedOut(WorkflowTerminalOutcome {
4917 kind: WorkflowTerminalKind::TimedOut,
4918 workflow_id: description
4919 .workflow_id
4920 .clone()
4921 .unwrap_or_else(|| self.workflow_id.clone()),
4922 run_id: description
4923 .run_id
4924 .clone()
4925 .or_else(|| selected_run_id.map(str::to_string)),
4926 reason: "result_wait_timeout".to_string(),
4927 failure_category: Some("client_timeout".to_string()),
4928 failure_id: None,
4929 exception_type: None,
4930 exception_class: None,
4931 non_retryable: None,
4932 message: Some(format!(
4933 "workflow result was not terminal within {:?}",
4934 options.timeout
4935 )),
4936 exception: None,
4937 raw: description.raw_value(),
4938 }));
4939 }
4940
4941 tokio::time::sleep(options.poll_interval).await;
4942 }
4943 }
4944}
4945
4946#[derive(Clone, Copy, Debug)]
4947pub struct WorkflowResultOptions {
4948 pub poll_interval: Duration,
4949 pub timeout: Duration,
4950}
4951
4952impl Default for WorkflowResultOptions {
4953 fn default() -> Self {
4954 Self {
4955 poll_interval: Duration::from_millis(500),
4956 timeout: Duration::from_secs(30),
4957 }
4958 }
4959}
4960
4961#[derive(Clone, Debug, Deserialize)]
4962pub struct WorkflowDescription {
4963 pub workflow_id: Option<String>,
4964 pub run_id: Option<String>,
4965 pub workflow_type: Option<String>,
4966 pub status: Option<String>,
4967 #[serde(default)]
4968 pub closed_reason: Option<String>,
4969 #[serde(default)]
4970 pub error: Option<String>,
4971 #[serde(default)]
4972 pub failure: Option<Value>,
4973 #[serde(default)]
4974 pub exception: Option<Value>,
4975 #[serde(default)]
4976 pub failures: Vec<Value>,
4977 #[serde(default)]
4978 pub output: Option<Value>,
4979 #[serde(default)]
4980 pub output_envelope: Option<Value>,
4981 #[serde(skip)]
4982 pub output_avro_value: Option<AvroValue>,
4983 #[serde(flatten)]
4984 pub raw: HashMap<String, Value>,
4985}
4986
4987#[derive(Clone, Debug, Deserialize)]
4989pub struct WorkflowStreamDescription {
4990 pub stream_name: String,
4991 pub status: String,
4992 pub last_offset: i64,
4993 pub total_items: u64,
4994 pub pending_items: u64,
4995 #[serde(default)]
4996 pub opened_at: Option<String>,
4997 #[serde(default)]
4998 pub last_appended_at: Option<String>,
4999 #[serde(default)]
5000 pub closed_at: Option<String>,
5001 #[serde(default)]
5002 pub error_reason: Option<String>,
5003 #[serde(default)]
5004 pub retention_seconds: Option<u64>,
5005 #[serde(flatten)]
5006 pub raw: HashMap<String, Value>,
5007}
5008
5009impl WorkflowStreamDescription {
5010 pub fn is_terminal(&self) -> bool {
5011 matches!(self.status.as_str(), "closed" | "errored")
5012 }
5013}
5014
5015#[derive(Clone, Debug, Default)]
5017pub struct WorkflowStreamAppendItem {
5018 pub payload_envelope: Option<Value>,
5019 pub payload_reference: Option<String>,
5020 pub item_type: Option<String>,
5021 pub content_type: Option<String>,
5022 pub idempotency_key: Option<String>,
5023}
5024
5025impl WorkflowStreamAppendItem {
5026 pub fn new<T: Serialize>(payload: T) -> Result<Self> {
5028 let value = AvroValue::from_serialize(&payload)?;
5029 Ok(Self {
5030 payload_envelope: Some(encode_typed_envelope(&value, DEFAULT_CODEC)?),
5031 ..Self::default()
5032 })
5033 }
5034
5035 pub fn from_reference(reference: impl Into<String>) -> Self {
5037 Self {
5038 payload_reference: Some(reference.into()),
5039 ..Self::default()
5040 }
5041 }
5042
5043 pub fn item_type(mut self, item_type: impl Into<String>) -> Self {
5044 self.item_type = Some(item_type.into());
5045 self
5046 }
5047
5048 pub fn content_type(mut self, content_type: impl Into<String>) -> Self {
5049 self.content_type = Some(content_type.into());
5050 self
5051 }
5052
5053 pub fn idempotency_key(mut self, idempotency_key: impl Into<String>) -> Self {
5054 self.idempotency_key = Some(idempotency_key.into());
5055 self
5056 }
5057
5058 fn wire_value(&self, derived_idempotency_key: Option<String>) -> Value {
5059 let mut item = serde_json::Map::new();
5060 if let Some(payload) = &self.payload_envelope {
5061 item.insert("payload".to_string(), payload.clone());
5062 item.insert("payload_codec".to_string(), json!(DEFAULT_CODEC));
5063 }
5064 if let Some(reference) = &self.payload_reference {
5065 item.insert("payload_reference".to_string(), json!(reference));
5066 }
5067 if let Some(item_type) = &self.item_type {
5068 item.insert("item_type".to_string(), json!(item_type));
5069 }
5070 if let Some(content_type) = &self.content_type {
5071 item.insert("content_type".to_string(), json!(content_type));
5072 }
5073 if let Some(key) = derived_idempotency_key
5074 .as_ref()
5075 .or(self.idempotency_key.as_ref())
5076 {
5077 item.insert("idempotency_key".to_string(), json!(key));
5078 }
5079 Value::Object(item)
5080 }
5081}
5082
5083#[derive(Clone, Debug)]
5085pub struct WorkflowStreamItem {
5086 pub offset: u64,
5087 pub payload: Option<Value>,
5088 pub payload_envelope: Option<Value>,
5089 pub payload_reference: Option<String>,
5090 pub payload_codec: Option<String>,
5091 pub idempotency_key: Option<String>,
5092 pub item_type: Option<String>,
5093 pub content_type: Option<String>,
5094 pub origin: Option<String>,
5095 pub origin_reference: Option<String>,
5096 pub emitted_at: Option<String>,
5097 pub raw: Value,
5098}
5099
5100#[derive(Clone, Debug)]
5102pub struct WorkflowStreamPage {
5103 pub stream: WorkflowStreamDescription,
5104 pub items: Vec<WorkflowStreamItem>,
5105 pub next_offset: u64,
5106 pub terminal: bool,
5107}
5108
5109#[derive(Clone, Debug)]
5111pub struct WorkflowStreamAppendResult {
5112 pub stream: WorkflowStreamDescription,
5113 pub accepted_offsets: Vec<u64>,
5114 pub accepted: u64,
5115 pub deduped: u64,
5116}
5117
5118#[derive(Deserialize)]
5119struct WorkflowStreamListResponse {
5120 #[serde(default)]
5121 streams: Vec<WorkflowStreamDescription>,
5122}
5123
5124#[derive(Deserialize)]
5125struct WorkflowStreamDescriptionResponse {
5126 stream: WorkflowStreamDescription,
5127}
5128
5129#[derive(Deserialize)]
5130struct WorkflowStreamPageResponse {
5131 stream: WorkflowStreamDescription,
5132 #[serde(default)]
5133 items: Vec<Value>,
5134 next_offset: u64,
5135 terminal: bool,
5136}
5137
5138#[derive(Deserialize)]
5139struct WorkflowStreamAppendResponse {
5140 stream: WorkflowStreamDescription,
5141 #[serde(default)]
5142 accepted_offsets: Vec<u64>,
5143 accepted: u64,
5144 deduped: u64,
5145}
5146
5147impl WorkflowDescription {
5148 pub fn is_completed(&self) -> bool {
5149 matches!(self.status.as_deref(), Some("completed" | "Completed"))
5150 }
5151
5152 pub fn is_terminal(&self) -> bool {
5153 matches!(
5154 self.status.as_deref(),
5155 Some(
5156 "completed"
5157 | "Completed"
5158 | "failed"
5159 | "Failed"
5160 | "cancelled"
5161 | "Cancelled"
5162 | "terminated"
5163 | "Terminated"
5164 | "timed_out"
5165 | "TimedOut",
5166 )
5167 )
5168 }
5169
5170 fn decode_payloads(&mut self) -> Result<()> {
5171 if let Some(envelope) = &self.output_envelope {
5172 let value = decode_wire_avro_value(envelope, DEFAULT_CODEC)?;
5173 self.output = Some(value.clone().into_json()?);
5174 self.output_avro_value = Some(value);
5175 }
5176
5177 Ok(())
5178 }
5179
5180 fn raw_value(&self) -> Value {
5181 let mut data = self.raw.clone();
5182 data.insert(
5183 "workflow_id".to_string(),
5184 self.workflow_id
5185 .clone()
5186 .map(Value::String)
5187 .unwrap_or(Value::Null),
5188 );
5189 data.insert(
5190 "run_id".to_string(),
5191 self.run_id
5192 .clone()
5193 .map(Value::String)
5194 .unwrap_or(Value::Null),
5195 );
5196 data.insert(
5197 "workflow_type".to_string(),
5198 self.workflow_type
5199 .clone()
5200 .map(Value::String)
5201 .unwrap_or(Value::Null),
5202 );
5203 data.insert(
5204 "status".to_string(),
5205 self.status
5206 .clone()
5207 .map(Value::String)
5208 .unwrap_or(Value::Null),
5209 );
5210 data.insert(
5211 "closed_reason".to_string(),
5212 self.closed_reason
5213 .clone()
5214 .map(Value::String)
5215 .unwrap_or(Value::Null),
5216 );
5217 if let Some(failure) = &self.failure {
5218 data.insert("failure".to_string(), failure.clone());
5219 }
5220 if let Some(exception) = &self.exception {
5221 data.insert("exception".to_string(), exception.clone());
5222 }
5223 Value::Object(data.into_iter().collect())
5224 }
5225}
5226
5227fn workflow_terminal_outcome(
5228 description: &WorkflowDescription,
5229 workflow_id: &str,
5230 run_id: Option<&str>,
5231) -> WorkflowTerminalOutcome {
5232 let terminal_kind = description
5233 .closed_reason
5234 .as_deref()
5235 .or(description.status.as_deref())
5236 .unwrap_or("failed")
5237 .to_ascii_lowercase();
5238 let kind = match terminal_kind.as_str() {
5239 "cancelled" | "canceled" => WorkflowTerminalKind::Cancelled,
5240 "terminated" => WorkflowTerminalKind::Terminated,
5241 "timed_out" | "timedout" => WorkflowTerminalKind::TimedOut,
5242 _ => WorkflowTerminalKind::Failed,
5243 };
5244 let default_reason = match kind {
5245 WorkflowTerminalKind::Failed => "workflow_failed",
5246 WorkflowTerminalKind::Cancelled => "cancelled",
5247 WorkflowTerminalKind::Terminated => "terminated",
5248 WorkflowTerminalKind::TimedOut => "timed_out",
5249 };
5250 let failure = description
5251 .failure
5252 .as_ref()
5253 .filter(|value| value.is_object());
5254 let nested_failure = failure
5255 .and_then(|value| value.get("failures"))
5256 .and_then(Value::as_array)
5257 .and_then(|failures| failures.last())
5258 .or_else(|| description.failures.last());
5259 let exception = description
5260 .exception
5261 .clone()
5262 .or_else(|| failure.and_then(|value| value.get("exception")).cloned())
5263 .or_else(|| {
5264 nested_failure
5265 .and_then(|value| value.get("exception_payload"))
5266 .cloned()
5267 });
5268 let string_field = |name: &str| {
5269 failure
5270 .and_then(|value| value.get(name))
5271 .and_then(Value::as_str)
5272 .or_else(|| {
5273 nested_failure
5274 .and_then(|value| value.get(name))
5275 .and_then(Value::as_str)
5276 })
5277 .map(str::to_string)
5278 };
5279 let exception_field = |name: &str| {
5280 exception
5281 .as_ref()
5282 .and_then(|value| value.get(name))
5283 .and_then(Value::as_str)
5284 .map(str::to_string)
5285 };
5286 let message = description
5287 .error
5288 .clone()
5289 .or_else(|| string_field("message"))
5290 .or_else(|| exception_field("message"));
5291 let reason = description
5292 .raw
5293 .get("reason")
5294 .and_then(Value::as_str)
5295 .map(str::to_string)
5296 .or_else(|| {
5297 failure
5298 .and_then(|value| value.get("reason"))
5299 .and_then(Value::as_str)
5300 .map(str::to_string)
5301 })
5302 .or_else(|| description.closed_reason.clone())
5303 .unwrap_or_else(|| default_reason.to_string());
5304 let failure_id = string_field("failure_id").or_else(|| {
5305 nested_failure
5306 .and_then(|value| value.get("id"))
5307 .and_then(Value::as_str)
5308 .map(str::to_string)
5309 });
5310
5311 WorkflowTerminalOutcome {
5312 kind,
5313 workflow_id: description
5314 .workflow_id
5315 .clone()
5316 .unwrap_or_else(|| workflow_id.to_string()),
5317 run_id: description
5318 .run_id
5319 .clone()
5320 .or_else(|| run_id.map(str::to_string)),
5321 reason,
5322 failure_category: string_field("failure_category")
5323 .or_else(|| Some(default_reason.to_string())),
5324 failure_id,
5325 exception_type: string_field("exception_type").or_else(|| exception_field("type")),
5326 exception_class: string_field("exception_class").or_else(|| exception_field("class")),
5327 non_retryable: failure
5328 .and_then(|value| value.get("non_retryable"))
5329 .and_then(Value::as_bool)
5330 .or_else(|| {
5331 nested_failure
5332 .and_then(|value| value.get("non_retryable"))
5333 .and_then(Value::as_bool)
5334 }),
5335 message,
5336 exception,
5337 raw: description.raw_value(),
5338 }
5339}
5340
5341#[derive(Clone, Debug, Deserialize)]
5342pub struct RegisterWorkerResponse {
5343 pub worker_id: String,
5344 pub registered: bool,
5345 #[serde(default)]
5346 pub heartbeat_interval_seconds: Option<u64>,
5347 #[serde(default)]
5348 pub protocol_version: Option<String>,
5349 #[serde(default)]
5350 pub server_capabilities: Option<Value>,
5351}
5352
5353#[derive(Clone, Debug, Deserialize, PartialEq, Eq)]
5355pub struct WorkerDeregistrationEnvelope {
5356 pub worker_id: String,
5357 pub outcome: String,
5358 pub recovered_workflow_task_count: u64,
5359}
5360
5361#[derive(Clone, Debug, Deserialize)]
5362pub struct PollWorkflowTaskResponse {
5363 #[serde(default)]
5364 pub task: Option<WorkflowTask>,
5365 #[serde(default)]
5366 pub poll_status: Option<String>,
5367 #[serde(default)]
5368 pub reason: Option<String>,
5369 #[serde(default)]
5370 pub protocol_version: Option<String>,
5371 #[serde(default)]
5372 pub server_capabilities: Option<Value>,
5373}
5374
5375impl PollWorkflowTaskResponse {
5376 pub fn outcome(&self) -> WorkerPollOutcome {
5378 worker_poll_outcome(
5379 self.task.is_some(),
5380 self.poll_status.as_deref(),
5381 self.reason.as_deref(),
5382 )
5383 }
5384}
5385
5386fn runtime_supports_workflow_memo_updates(capabilities: Option<&Value>) -> bool {
5387 let Some(capabilities) = capabilities.and_then(Value::as_object) else {
5388 return false;
5389 };
5390 let supported = capabilities
5391 .get("workflow_memo_updates")
5392 .and_then(Value::as_object)
5393 .and_then(|memo| memo.get("supported"))
5394 .and_then(Value::as_bool)
5395 == Some(true);
5396 let command_advertised = capabilities
5397 .get("supported_workflow_task_commands")
5398 .and_then(Value::as_array)
5399 .is_some_and(|commands| {
5400 commands
5401 .iter()
5402 .any(|command| command.as_str() == Some("upsert_memo"))
5403 });
5404 supported && command_advertised
5405}
5406
5407fn commands_use_workflow_memo_updates(commands: &[Value]) -> bool {
5408 commands
5409 .iter()
5410 .any(|command| command.get("type").and_then(Value::as_str) == Some("upsert_memo"))
5411}
5412
5413#[derive(Clone, Debug, Deserialize)]
5414pub struct PollActivityTaskResponse {
5415 #[serde(default)]
5416 pub task: Option<ActivityTask>,
5417 #[serde(default)]
5418 pub poll_status: Option<String>,
5419 #[serde(default)]
5420 pub reason: Option<String>,
5421}
5422
5423impl PollActivityTaskResponse {
5424 pub fn outcome(&self) -> WorkerPollOutcome {
5426 worker_poll_outcome(
5427 self.task.is_some(),
5428 self.poll_status.as_deref(),
5429 self.reason.as_deref(),
5430 )
5431 }
5432}
5433
5434#[derive(Clone, Debug, Deserialize)]
5435pub struct PollQueryTaskResponse {
5436 #[serde(default)]
5437 pub task: Option<QueryTask>,
5438 #[serde(default)]
5439 pub poll_status: Option<String>,
5440 #[serde(default)]
5441 pub reason: Option<String>,
5442}
5443
5444impl PollQueryTaskResponse {
5445 pub fn outcome(&self) -> WorkerPollOutcome {
5447 worker_poll_outcome(
5448 self.task.is_some(),
5449 self.poll_status.as_deref(),
5450 self.reason.as_deref(),
5451 )
5452 }
5453}
5454
5455#[derive(Clone, Debug, PartialEq, Eq)]
5457pub enum WorkerPollOutcome {
5458 Task,
5460 Idle {
5462 poll_status: Option<String>,
5463 reason: Option<String>,
5464 },
5465 Stop {
5467 poll_status: Option<String>,
5468 reason: Option<String>,
5469 },
5470}
5471
5472impl WorkerPollOutcome {
5473 pub fn should_stop(&self) -> bool {
5474 matches!(self, Self::Stop { .. })
5475 }
5476}
5477
5478fn worker_poll_outcome(
5479 has_task: bool,
5480 poll_status: Option<&str>,
5481 reason: Option<&str>,
5482) -> WorkerPollOutcome {
5483 if worker_poll_is_stop(poll_status, reason) {
5484 return WorkerPollOutcome::Stop {
5485 poll_status: poll_status.map(str::to_string),
5486 reason: reason.map(str::to_string),
5487 };
5488 }
5489
5490 if has_task {
5491 WorkerPollOutcome::Task
5492 } else {
5493 WorkerPollOutcome::Idle {
5494 poll_status: poll_status.map(str::to_string),
5495 reason: reason.map(str::to_string),
5496 }
5497 }
5498}
5499
5500#[derive(Clone, Debug, Deserialize)]
5502pub struct QueryTask {
5503 pub query_task_id: String,
5504 #[serde(default = "default_workflow_task_attempt")]
5505 pub query_task_attempt: u64,
5506 #[serde(default)]
5507 pub lease_owner: Option<String>,
5508 #[serde(default)]
5509 pub workflow_id: Option<String>,
5510 #[serde(default)]
5511 pub run_id: Option<String>,
5512 pub workflow_type: String,
5513 pub query_name: String,
5514 #[serde(
5515 default = "missing_task_payload_codec",
5516 deserialize_with = "deserialize_task_payload_codec"
5517 )]
5518 pub payload_codec: String,
5519 #[serde(default)]
5520 pub workflow_arguments: Option<Value>,
5521 #[serde(default)]
5522 pub query_arguments: Option<Value>,
5523 #[serde(default)]
5524 pub history_events: Vec<HistoryEvent>,
5525 #[serde(default)]
5526 pub history_export: Option<Value>,
5527 #[serde(default)]
5528 pub run_status: Option<String>,
5529}
5530
5531#[derive(Clone, Debug, Deserialize)]
5532pub struct WorkflowTask {
5533 pub task_id: String,
5534 #[serde(default)]
5535 pub workflow_command_id: Option<String>,
5536 #[serde(default)]
5537 pub workflow_id: Option<String>,
5538 #[serde(default)]
5539 pub run_id: Option<String>,
5540 pub workflow_type: String,
5541 #[serde(default)]
5542 pub cancel_requested: bool,
5543 #[serde(
5544 default = "missing_task_payload_codec",
5545 deserialize_with = "deserialize_task_payload_codec"
5546 )]
5547 pub payload_codec: String,
5548 #[serde(default)]
5549 pub arguments: Option<Value>,
5550 #[serde(default)]
5551 pub history_events: Vec<HistoryEvent>,
5552 #[serde(default)]
5553 pub total_history_events: Option<u64>,
5554 #[serde(default)]
5555 pub history_size_bytes: Option<u64>,
5556 #[serde(default)]
5557 pub continue_as_new_recommended: Option<bool>,
5558 #[serde(default)]
5559 pub history_budget_pressure: Option<String>,
5560 #[serde(default)]
5561 pub next_history_page_token: Option<String>,
5562 #[serde(default = "default_workflow_task_attempt")]
5563 pub workflow_task_attempt: u64,
5564 #[serde(default)]
5565 pub workflow_signal_id: Option<String>,
5566 #[serde(default)]
5567 pub signal_name: Option<String>,
5568 #[serde(default)]
5569 pub signal_arguments: Option<Value>,
5570 #[serde(default)]
5571 pub workflow_update_id: Option<String>,
5572 #[serde(default)]
5573 pub update_name: Option<String>,
5574 #[serde(default)]
5575 pub lease_owner: Option<String>,
5576}
5577
5578impl WorkflowTask {
5579 fn append_history_page(&mut self, page: WorkflowTaskHistoryPage) {
5580 self.history_events.extend(page.history_events);
5581
5582 if page.total_history_events.is_some() {
5583 self.total_history_events = page.total_history_events;
5584 }
5585
5586 self.next_history_page_token = page
5587 .next_history_page_token
5588 .filter(|token| !token.is_empty());
5589 }
5590}
5591
5592#[derive(Clone, Debug, Deserialize)]
5593struct WorkflowTaskHistoryPage {
5594 #[serde(default)]
5595 history_events: Vec<HistoryEvent>,
5596 #[serde(default)]
5597 total_history_events: Option<u64>,
5598 #[serde(default)]
5599 next_history_page_token: Option<String>,
5600}
5601
5602#[derive(Clone, Debug, Deserialize)]
5603pub struct ActivityTask {
5604 pub task_id: String,
5605 #[serde(default)]
5606 pub activity_attempt_id: Option<String>,
5607 #[serde(default)]
5608 pub attempt_id: Option<String>,
5609 pub activity_type: String,
5610 #[serde(
5611 default = "missing_task_payload_codec",
5612 deserialize_with = "deserialize_task_payload_codec"
5613 )]
5614 pub payload_codec: String,
5615 #[serde(default)]
5616 pub arguments: Option<Value>,
5617 #[serde(default = "default_attempt_number")]
5618 pub attempt_number: u64,
5619 #[serde(default)]
5620 pub lease_owner: Option<String>,
5621}
5622
5623#[derive(Clone, Debug, Deserialize)]
5624pub struct HistoryEvent {
5625 #[serde(alias = "type")]
5626 pub event_type: String,
5627 #[serde(default)]
5628 pub payload: Value,
5629 #[serde(flatten)]
5630 pub raw: HashMap<String, Value>,
5631}
5632
5633#[derive(Clone, Debug, PartialEq)]
5635pub struct QuerySignal {
5636 pub id: Option<String>,
5637 pub name: String,
5638 pub arguments: Vec<Value>,
5639 avro_arguments: Vec<AvroValue>,
5640 pub workflow_sequence: Option<u64>,
5641}
5642
5643impl QuerySignal {
5644 pub fn arguments_avro_value(&self) -> &[AvroValue] {
5646 &self.avro_arguments
5647 }
5648}
5649
5650#[derive(Clone, Debug)]
5656pub struct QueryContext {
5657 pub workflow_id: Option<String>,
5658 pub run_id: Option<String>,
5659 pub workflow_type: String,
5660 pub run_status: Option<String>,
5661 workflow_input: Value,
5662 workflow_input_avro_value: AvroValue,
5663 history_events: Arc<Vec<HistoryEvent>>,
5664 signal_events: Arc<Vec<QuerySignal>>,
5665}
5666
5667impl QueryContext {
5668 pub fn workflow_input(&self) -> &Value {
5670 &self.workflow_input
5671 }
5672
5673 pub fn workflow_input_avro_value(&self) -> &AvroValue {
5675 &self.workflow_input_avro_value
5676 }
5677
5678 pub fn history_events(&self) -> &[HistoryEvent] {
5680 self.history_events.as_slice()
5681 }
5682
5683 pub fn signal_events(&self) -> &[QuerySignal] {
5685 self.signal_events.as_slice()
5686 }
5687
5688 pub fn signals(&self, signal_name: &str) -> Vec<Vec<Value>> {
5690 self.signal_events
5691 .iter()
5692 .filter(|signal| signal.name == signal_name)
5693 .map(|signal| signal.arguments.clone())
5694 .collect()
5695 }
5696
5697 pub fn signals_avro_value(&self, signal_name: &str) -> Vec<Vec<AvroValue>> {
5699 self.signal_events
5700 .iter()
5701 .filter(|signal| signal.name == signal_name)
5702 .map(|signal| signal.avro_arguments.clone())
5703 .collect()
5704 }
5705}
5706
5707#[derive(Clone, Debug, Deserialize)]
5708pub struct ActivityHeartbeatResponse {
5709 #[serde(default)]
5710 pub cancel_requested: bool,
5711 #[serde(default)]
5712 pub heartbeat_recorded: bool,
5713 #[serde(default)]
5714 pub can_continue: Option<bool>,
5715 #[serde(default)]
5716 pub reason: Option<String>,
5717 #[serde(default)]
5718 pub run_closed_reason: Option<String>,
5719 #[serde(default)]
5720 pub run_closed_at: Option<String>,
5721 #[serde(default)]
5722 pub lease_expires_at: Option<String>,
5723 #[serde(default)]
5724 pub last_heartbeat_at: Option<String>,
5725}
5726
5727impl ActivityHeartbeatResponse {
5728 pub fn should_stop(&self) -> bool {
5730 self.cancel_requested || self.can_continue == Some(false)
5731 }
5732}
5733
5734fn missing_task_payload_codec() -> String {
5735 MISSING_TASK_PAYLOAD_CODEC.to_string()
5736}
5737
5738fn deserialize_task_payload_codec<'de, D>(deserializer: D) -> std::result::Result<String, D::Error>
5739where
5740 D: Deserializer<'de>,
5741{
5742 Ok(match Value::deserialize(deserializer)? {
5743 Value::String(codec) => codec,
5744 Value::Null => NULL_TASK_PAYLOAD_CODEC.to_string(),
5745 _ => NON_STRING_TASK_PAYLOAD_CODEC.to_string(),
5746 })
5747}
5748
5749fn default_workflow_task_attempt() -> u64 {
5750 1
5751}
5752
5753fn default_attempt_number() -> u64 {
5754 1
5755}
5756
5757type WorkflowFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
5758type WorkflowHandler = Arc<dyn Fn(WorkflowContext, AvroValue) -> WorkflowFuture + Send + Sync>;
5759type ErasedWorkflowState = Arc<dyn Any + Send + Sync>;
5760type WorkflowStateSnapshot = Arc<dyn Fn() -> Result<ErasedWorkflowState> + Send + Sync>;
5761type ReplayedWorkflowHandler =
5762 Arc<dyn Fn(WorkflowContext, AvroValue) -> ReplayedWorkflowInvocation + Send + Sync>;
5763type ActivityFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
5764type ActivityHandler = Arc<dyn Fn(ActivityContext, AvroValue) -> ActivityFuture + Send + Sync>;
5765type QueryFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
5766type QueryHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
5767type UpdateHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
5768type ReplayedQueryHandler = Arc<
5769 dyn Fn(QueryContext, ErasedWorkflowState, AvroValue) -> std::result::Result<QueryFuture, String>
5770 + Send
5771 + Sync,
5772>;
5773type WorkerHeartbeatObserver = Arc<dyn Fn(&WorkerHeartbeatObservation) + Send + Sync>;
5774
5775struct ReplayedWorkflowInvocation {
5776 future: WorkflowFuture,
5777 snapshot: WorkflowStateSnapshot,
5778}
5779
5780#[derive(Clone)]
5781struct RegisteredWorkflow {
5782 execute: WorkflowHandler,
5783 replay: Option<ReplayedWorkflowHandler>,
5784 state_type: Option<TypeId>,
5785}
5786
5787#[derive(Debug)]
5788struct WorkflowTaskDecision {
5789 commands: Vec<Value>,
5790 message_stream_cursors: Vec<Value>,
5791 message_stream_waits: Vec<Value>,
5792}
5793
5794impl WorkflowTaskDecision {
5795 fn without_message_streams(commands: Vec<Value>) -> Self {
5796 Self {
5797 commands,
5798 message_stream_cursors: Vec::new(),
5799 message_stream_waits: Vec::new(),
5800 }
5801 }
5802}
5803
5804#[derive(Clone)]
5805enum RegisteredQuery {
5806 Snapshot(QueryHandler),
5807 Replayed {
5808 state_type: TypeId,
5809 handler: ReplayedQueryHandler,
5810 },
5811}
5812
5813#[derive(Clone, Debug)]
5814pub struct WorkerHeartbeatObservation {
5815 pub worker_id: String,
5816 pub task_queue: String,
5817 pub acknowledged_at_unix_millis: u64,
5818 pub acknowledgement: Value,
5819}
5820
5821#[derive(Clone, Copy, Debug)]
5833pub struct WorkerRetryPolicy {
5834 pub max_retries: usize,
5836 pub initial_backoff: Duration,
5838 pub max_backoff: Duration,
5840}
5841
5842impl Default for WorkerRetryPolicy {
5843 fn default() -> Self {
5844 Self {
5845 max_retries: 5,
5846 initial_backoff: Duration::from_millis(100),
5847 max_backoff: Duration::from_secs(5),
5848 }
5849 }
5850}
5851
5852#[derive(Clone, Debug)]
5853struct WorkerStorageAdmission {
5854 policy: WorkerRetryPolicy,
5855 stop: Arc<AtomicBool>,
5856}
5857
5858struct StopWorkerOnDrop(Arc<AtomicBool>);
5859
5860impl Drop for StopWorkerOnDrop {
5861 fn drop(&mut self) {
5862 self.0.store(true, Ordering::SeqCst);
5863 }
5864}
5865
5866async fn wait_for_worker_stop(stop: &AtomicBool) {
5867 while !stop.load(Ordering::SeqCst) {
5868 tokio::time::sleep(Duration::from_millis(100)).await;
5869 }
5870}
5871
5872#[derive(Clone, Copy, Debug, PartialEq, Eq)]
5873enum ManagedPollOutcome {
5874 Idle,
5875 Handled,
5876 Stop,
5877}
5878
5879#[derive(Clone)]
5880pub struct Worker {
5881 client: Client,
5882 worker_id: String,
5883 task_queue: String,
5884 workflows: HashMap<String, RegisteredWorkflow>,
5885 activities: HashMap<String, ActivityHandler>,
5886 queries: HashMap<String, HashMap<String, RegisteredQuery>>,
5887 updates: HashMap<String, HashMap<String, UpdateHandler>>,
5888 max_concurrent_workflow_tasks: usize,
5889 max_concurrent_activity_tasks: usize,
5890 poll_timeout: Duration,
5891 heartbeat_interval: Duration,
5892 retry_policy: WorkerRetryPolicy,
5893 heartbeat_observer: Option<WorkerHeartbeatObserver>,
5894}
5895
5896impl Worker {
5897 pub fn new(client: Client, task_queue: impl Into<String>) -> Self {
5898 Self {
5899 client,
5900 worker_id: default_worker_id(),
5901 task_queue: task_queue.into(),
5902 workflows: HashMap::new(),
5903 activities: HashMap::new(),
5904 queries: HashMap::new(),
5905 updates: HashMap::new(),
5906 max_concurrent_workflow_tasks: 10,
5907 max_concurrent_activity_tasks: 10,
5908 poll_timeout: Duration::from_secs(30),
5909 heartbeat_interval: Duration::from_secs(60),
5910 retry_policy: WorkerRetryPolicy::default(),
5911 heartbeat_observer: None,
5912 }
5913 }
5914
5915 pub fn worker_id(mut self, worker_id: impl Into<String>) -> Self {
5916 self.worker_id = worker_id.into();
5917 self
5918 }
5919
5920 pub fn poll_timeout(mut self, timeout: Duration) -> Self {
5921 self.poll_timeout = timeout;
5922 self
5923 }
5924
5925 pub fn heartbeat_interval(mut self, interval: Duration) -> Self {
5926 self.heartbeat_interval = interval;
5927 self
5928 }
5929
5930 pub fn retry_policy(mut self, policy: WorkerRetryPolicy) -> Self {
5932 self.retry_policy = policy;
5933 self
5934 }
5935
5936 pub fn on_worker_heartbeat<F>(mut self, observer: F) -> Self
5937 where
5938 F: Fn(&WorkerHeartbeatObservation) + Send + Sync + 'static,
5939 {
5940 self.heartbeat_observer = Some(Arc::new(observer));
5941 self
5942 }
5943
5944 pub fn max_concurrent_workflow_tasks(mut self, count: usize) -> Self {
5945 self.max_concurrent_workflow_tasks = count.max(1);
5946 self
5947 }
5948
5949 pub fn max_concurrent_activity_tasks(mut self, count: usize) -> Self {
5950 self.max_concurrent_activity_tasks = count.max(1);
5951 self
5952 }
5953
5954 pub fn register_workflow<F, Fut>(&mut self, workflow_type: impl Into<String>, handler: F)
5961 where
5962 F: Fn(WorkflowContext, Value) -> Fut + Send + Sync + 'static,
5963 Fut: Future<Output = Result<Value>> + Send + 'static,
5964 {
5965 let handler = Arc::new(handler);
5966 self.workflows.insert(
5967 workflow_type.into(),
5968 RegisteredWorkflow {
5969 execute: Arc::new(move |ctx, input| {
5970 let handler = Arc::clone(&handler);
5971 Box::pin(async move {
5972 let result = handler(ctx, input.into_json()?).await?;
5973 AvroValue::from_serialize(&result)
5974 })
5975 }),
5976 replay: None,
5977 state_type: None,
5978 },
5979 );
5980 }
5981
5982 pub fn register_typed_workflow<I, O, F, Fut>(
5993 &mut self,
5994 workflow_type: impl Into<String>,
5995 handler: F,
5996 ) where
5997 I: DeserializeOwned + Send + 'static,
5998 O: Serialize + Send + 'static,
5999 F: Fn(WorkflowContext, I) -> Fut + Send + Sync + 'static,
6000 Fut: Future<Output = Result<O>> + Send + 'static,
6001 {
6002 let workflow_type = workflow_type.into();
6003 let handler_name = workflow_type.clone();
6004 let handler = Arc::new(handler);
6005 self.workflows.insert(
6006 workflow_type,
6007 RegisteredWorkflow {
6008 execute: Arc::new(move |ctx, input| {
6009 let handler = Arc::clone(&handler);
6010 let handler_name = handler_name.clone();
6011 Box::pin(async move {
6012 let input =
6013 decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
6014 let result = handler(ctx, input).await?;
6015 encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
6016 })
6017 }),
6018 replay: None,
6019 state_type: None,
6020 },
6021 );
6022 }
6023
6024 pub fn register_workflow_avro_value<F, Fut>(
6026 &mut self,
6027 workflow_type: impl Into<String>,
6028 handler: F,
6029 ) where
6030 F: Fn(WorkflowContext, AvroValue) -> Fut + Send + Sync + 'static,
6031 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6032 {
6033 self.workflows.insert(
6034 workflow_type.into(),
6035 RegisteredWorkflow {
6036 execute: Arc::new(move |ctx, input| Box::pin(handler(ctx, input))),
6037 replay: None,
6038 state_type: None,
6039 },
6040 );
6041 }
6042
6043 pub fn register_replayed_workflow<S, Factory, F, Fut>(
6051 &mut self,
6052 workflow_type: impl Into<String>,
6053 state_factory: Factory,
6054 handler: F,
6055 ) where
6056 S: Clone + Send + Sync + 'static,
6057 Factory: Fn() -> S + Send + Sync + 'static,
6058 F: Fn(WorkflowContext, Value, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
6059 Fut: Future<Output = Result<Value>> + Send + 'static,
6060 {
6061 let state_factory = Arc::new(state_factory);
6062 let handler = Arc::new(handler);
6063
6064 let execute_factory = Arc::clone(&state_factory);
6065 let execute_handler = Arc::clone(&handler);
6066 let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6067 let state = WorkflowInstance::new(execute_factory());
6068 let handler = Arc::clone(&execute_handler);
6069 Box::pin(async move {
6070 let result = handler(ctx, input.into_json()?, state).await?;
6071 AvroValue::from_serialize(&result)
6072 }) as WorkflowFuture
6073 });
6074
6075 let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6076 let state = WorkflowInstance::new(state_factory());
6077 let snapshot_state = state.clone();
6078 let snapshot: WorkflowStateSnapshot =
6079 Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
6080 let replay_handler = Arc::clone(&handler);
6081 let future = async move {
6082 let result = replay_handler(ctx, input.into_json()?, state).await?;
6083 AvroValue::from_serialize(&result)
6084 };
6085 ReplayedWorkflowInvocation {
6086 future: Box::pin(future),
6087 snapshot,
6088 }
6089 });
6090
6091 self.workflows.insert(
6092 workflow_type.into(),
6093 RegisteredWorkflow {
6094 execute,
6095 replay: Some(replay),
6096 state_type: Some(TypeId::of::<S>()),
6097 },
6098 );
6099 }
6100
6101 pub fn register_typed_replayed_workflow<I, O, S, Factory, F, Fut>(
6107 &mut self,
6108 workflow_type: impl Into<String>,
6109 state_factory: Factory,
6110 handler: F,
6111 ) where
6112 I: DeserializeOwned + Send + 'static,
6113 O: Serialize + Send + 'static,
6114 S: Clone + Send + Sync + 'static,
6115 Factory: Fn() -> S + Send + Sync + 'static,
6116 F: Fn(WorkflowContext, I, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
6117 Fut: Future<Output = Result<O>> + Send + 'static,
6118 {
6119 let workflow_type = workflow_type.into();
6120 let state_factory = Arc::new(state_factory);
6121 let handler = Arc::new(handler);
6122
6123 let execute_name = workflow_type.clone();
6124 let execute_factory = Arc::clone(&state_factory);
6125 let execute_handler = Arc::clone(&handler);
6126 let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6127 let state = WorkflowInstance::new(execute_factory());
6128 let handler = Arc::clone(&execute_handler);
6129 let handler_name = execute_name.clone();
6130 Box::pin(async move {
6131 let input = decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
6132 let result = handler(ctx, input, state).await?;
6133 encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
6134 }) as WorkflowFuture
6135 });
6136
6137 let replay_name = workflow_type.clone();
6138 let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6139 let state = WorkflowInstance::new(state_factory());
6140 let snapshot_state = state.clone();
6141 let snapshot: WorkflowStateSnapshot =
6142 Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
6143 let handler = Arc::clone(&handler);
6144 let handler_name = replay_name.clone();
6145 let future = async move {
6146 let input = decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
6147 let result = handler(ctx, input, state).await?;
6148 encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
6149 };
6150 ReplayedWorkflowInvocation {
6151 future: Box::pin(future),
6152 snapshot,
6153 }
6154 });
6155
6156 self.workflows.insert(
6157 workflow_type,
6158 RegisteredWorkflow {
6159 execute,
6160 replay: Some(replay),
6161 state_type: Some(TypeId::of::<S>()),
6162 },
6163 );
6164 }
6165
6166 pub fn register_replayed_workflow_avro_value<S, Factory, F, Fut>(
6168 &mut self,
6169 workflow_type: impl Into<String>,
6170 state_factory: Factory,
6171 handler: F,
6172 ) where
6173 S: Clone + Send + Sync + 'static,
6174 Factory: Fn() -> S + Send + Sync + 'static,
6175 F: Fn(WorkflowContext, AvroValue, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
6176 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6177 {
6178 let state_factory = Arc::new(state_factory);
6179 let handler = Arc::new(handler);
6180
6181 let execute_factory = Arc::clone(&state_factory);
6182 let execute_handler = Arc::clone(&handler);
6183 let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6184 let state = WorkflowInstance::new(execute_factory());
6185 Box::pin(execute_handler(ctx, input, state)) as WorkflowFuture
6186 });
6187
6188 let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6189 let state = WorkflowInstance::new(state_factory());
6190 let snapshot_state = state.clone();
6191 let snapshot: WorkflowStateSnapshot =
6192 Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
6193 ReplayedWorkflowInvocation {
6194 future: Box::pin(handler(ctx, input, state)),
6195 snapshot,
6196 }
6197 });
6198
6199 self.workflows.insert(
6200 workflow_type.into(),
6201 RegisteredWorkflow {
6202 execute,
6203 replay: Some(replay),
6204 state_type: Some(TypeId::of::<S>()),
6205 },
6206 );
6207 }
6208
6209 pub fn register_activity<F, Fut>(&mut self, activity_type: impl Into<String>, handler: F)
6210 where
6211 F: Fn(ActivityContext, Value) -> Fut + Send + Sync + 'static,
6212 Fut: Future<Output = Result<Value>> + Send + 'static,
6213 {
6214 let handler = Arc::new(handler);
6215 self.activities.insert(
6216 activity_type.into(),
6217 Arc::new(move |ctx, args| {
6218 let handler = Arc::clone(&handler);
6219 Box::pin(async move {
6220 let result = handler(ctx, args.into_json()?).await?;
6221 AvroValue::from_serialize(&result)
6222 })
6223 }),
6224 );
6225 }
6226
6227 pub fn register_typed_activity<I, O, F, Fut>(
6233 &mut self,
6234 activity_type: impl Into<String>,
6235 handler: F,
6236 ) where
6237 I: DeserializeOwned + Send + 'static,
6238 O: Serialize + Send + 'static,
6239 F: Fn(ActivityContext, I) -> Fut + Send + Sync + 'static,
6240 Fut: Future<Output = Result<O>> + Send + 'static,
6241 {
6242 let activity_type = activity_type.into();
6243 let handler_name = activity_type.clone();
6244 let handler = Arc::new(handler);
6245 self.activities.insert(
6246 activity_type,
6247 Arc::new(move |ctx, input| {
6248 let handler = Arc::clone(&handler);
6249 let handler_name = handler_name.clone();
6250 Box::pin(async move {
6251 let input =
6252 decode_handler_input::<I>(input, HandlerKind::Activity, &handler_name)?;
6253 let result = handler(ctx, input).await?;
6254 encode_handler_result(&result, HandlerKind::Activity, &handler_name)
6255 })
6256 }),
6257 );
6258 }
6259
6260 pub fn register_activity_avro_value<F, Fut>(
6262 &mut self,
6263 activity_type: impl Into<String>,
6264 handler: F,
6265 ) where
6266 F: Fn(ActivityContext, AvroValue) -> Fut + Send + Sync + 'static,
6267 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6268 {
6269 self.activities.insert(
6270 activity_type.into(),
6271 Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
6272 );
6273 }
6274
6275 pub fn register_query<F, Fut>(
6281 &mut self,
6282 workflow_type: impl Into<String>,
6283 query_name: impl Into<String>,
6284 handler: F,
6285 ) where
6286 F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
6287 Fut: Future<Output = Result<Value>> + Send + 'static,
6288 {
6289 let handler = Arc::new(handler);
6290 self.queries
6291 .entry(workflow_type.into())
6292 .or_default()
6293 .insert(
6294 query_name.into(),
6295 RegisteredQuery::Snapshot(Arc::new(move |ctx, args| {
6296 let handler = Arc::clone(&handler);
6297 Box::pin(async move {
6298 let result = handler(ctx, args.into_json()?).await?;
6299 AvroValue::from_serialize(&result)
6300 })
6301 })),
6302 );
6303 }
6304
6305 pub fn register_query_avro_value<F, Fut>(
6307 &mut self,
6308 workflow_type: impl Into<String>,
6309 query_name: impl Into<String>,
6310 handler: F,
6311 ) where
6312 F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
6313 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6314 {
6315 self.queries
6316 .entry(workflow_type.into())
6317 .or_default()
6318 .insert(
6319 query_name.into(),
6320 RegisteredQuery::Snapshot(Arc::new(move |ctx, args| Box::pin(handler(ctx, args)))),
6321 );
6322 }
6323
6324 pub fn register_replayed_query<S, F, Fut>(
6331 &mut self,
6332 workflow_type: impl Into<String>,
6333 query_name: impl Into<String>,
6334 handler: F,
6335 ) where
6336 S: Clone + Send + Sync + 'static,
6337 F: Fn(QueryContext, Arc<S>, Value) -> Fut + Send + Sync + 'static,
6338 Fut: Future<Output = Result<Value>> + Send + 'static,
6339 {
6340 let handler = Arc::new(handler);
6341 let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
6342 let state = state.downcast::<S>().map_err(|_| {
6343 "registered query state type does not match the replayed workflow state".to_string()
6344 })?;
6345 let handler = Arc::clone(&handler);
6346 Ok(Box::pin(async move {
6347 let result = handler(ctx, state, args.into_json()?).await?;
6348 AvroValue::from_serialize(&result)
6349 }))
6350 });
6351
6352 self.queries
6353 .entry(workflow_type.into())
6354 .or_default()
6355 .insert(
6356 query_name.into(),
6357 RegisteredQuery::Replayed {
6358 state_type: TypeId::of::<S>(),
6359 handler: erased_handler,
6360 },
6361 );
6362 }
6363
6364 pub fn register_replayed_query_avro_value<S, F, Fut>(
6366 &mut self,
6367 workflow_type: impl Into<String>,
6368 query_name: impl Into<String>,
6369 handler: F,
6370 ) where
6371 S: Clone + Send + Sync + 'static,
6372 F: Fn(QueryContext, Arc<S>, AvroValue) -> Fut + Send + Sync + 'static,
6373 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6374 {
6375 let handler = Arc::new(handler);
6376 let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
6377 let state = state.downcast::<S>().map_err(|_| {
6378 "registered query state type does not match the replayed workflow state".to_string()
6379 })?;
6380 Ok(Box::pin(handler(ctx, state, args)))
6381 });
6382
6383 self.queries
6384 .entry(workflow_type.into())
6385 .or_default()
6386 .insert(
6387 query_name.into(),
6388 RegisteredQuery::Replayed {
6389 state_type: TypeId::of::<S>(),
6390 handler: erased_handler,
6391 },
6392 );
6393 }
6394
6395 pub fn register_update<F, Fut>(
6397 &mut self,
6398 workflow_type: impl Into<String>,
6399 update_name: impl Into<String>,
6400 handler: F,
6401 ) where
6402 F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
6403 Fut: Future<Output = Result<Value>> + Send + 'static,
6404 {
6405 let handler = Arc::new(handler);
6406 self.updates
6407 .entry(workflow_type.into())
6408 .or_default()
6409 .insert(
6410 update_name.into(),
6411 Arc::new(move |ctx, args| {
6412 let handler = Arc::clone(&handler);
6413 Box::pin(async move {
6414 let result = handler(ctx, args.into_json()?).await?;
6415 AvroValue::from_serialize(&result)
6416 })
6417 }),
6418 );
6419 }
6420
6421 pub fn register_update_avro_value<F, Fut>(
6423 &mut self,
6424 workflow_type: impl Into<String>,
6425 update_name: impl Into<String>,
6426 handler: F,
6427 ) where
6428 F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
6429 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6430 {
6431 self.updates
6432 .entry(workflow_type.into())
6433 .or_default()
6434 .insert(
6435 update_name.into(),
6436 Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
6437 );
6438 }
6439
6440 pub async fn register(&self) -> Result<RegisterWorkerResponse> {
6441 let mut command_contracts = serde_json::Map::new();
6442 for workflow_type in self.workflows.keys() {
6443 let mut queries = self
6444 .queries
6445 .get(workflow_type)
6446 .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
6447 .unwrap_or_default();
6448 queries.sort();
6449 let mut updates = self
6450 .updates
6451 .get(workflow_type)
6452 .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
6453 .unwrap_or_default();
6454 updates.sort();
6455 command_contracts.insert(
6456 workflow_type.clone(),
6457 json!({
6458 "queries": queries,
6459 "query_contracts": [],
6460 "signals": [],
6461 "signal_contracts": [],
6462 "updates": updates,
6463 "update_contracts": [],
6464 "update_validators": [],
6465 }),
6466 );
6467 }
6468
6469 self.client
6470 .register_worker_with_command_contracts(
6471 &self.worker_id,
6472 &self.task_queue,
6473 self.workflows.keys().cloned().collect(),
6474 self.activities.keys().cloned().collect(),
6475 self.max_concurrent_workflow_tasks,
6476 self.max_concurrent_activity_tasks,
6477 [
6478 Some(CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY.to_string()),
6479 Some(DURABLE_SELECTION_CAPABILITY.to_string()),
6480 Some(MEMO_UPSERTS_CAPABILITY.to_string()),
6481 Some(TYPED_SEARCH_ATTRIBUTES_CAPABILITY.to_string()),
6482 (!self.queries.is_empty()).then(|| QUERY_TASKS_CAPABILITY.to_string()),
6483 (!self.updates.is_empty()).then(|| WORKFLOW_UPDATES_CAPABILITY.to_string()),
6484 worker_protocol_supports_message_streams(WORKER_PROTOCOL_VERSION)
6485 .then(|| MESSAGE_STREAMS_CAPABILITY.to_string()),
6486 ]
6487 .into_iter()
6488 .flatten()
6489 .collect(),
6490 Value::Object(command_contracts),
6491 )
6492 .await
6493 }
6494
6495 pub async fn run(&self) -> Result<()> {
6501 self.run_until(std::future::pending::<()>()).await
6502 }
6503
6504 pub async fn run_until<F>(&self, shutdown: F) -> Result<()>
6508 where
6509 F: Future<Output = ()>,
6510 {
6511 let stop = Arc::new(AtomicBool::new(false));
6512 let _stop_on_drop = StopWorkerOnDrop(Arc::clone(&stop));
6513 let worker = self.with_storage_admission(Arc::clone(&stop));
6514 let run = worker.run_with_storage_admission(Arc::clone(&stop));
6515 tokio::pin!(run);
6516 tokio::pin!(shutdown);
6517 tokio::select! {
6518 result = &mut run => result,
6519 _ = &mut shutdown => {
6520 stop.store(true, Ordering::SeqCst);
6521 run.await
6522 }
6523 }
6524 }
6525
6526 fn with_storage_admission(&self, stop: Arc<AtomicBool>) -> Self {
6527 let mut worker = self.clone();
6528 worker.client.worker_storage_admission = Some(WorkerStorageAdmission {
6529 policy: self.retry_policy,
6530 stop,
6531 });
6532 worker
6533 }
6534
6535 async fn run_with_storage_admission(&self, stop: Arc<AtomicBool>) -> Result<()> {
6536 let registration = self.register().await?;
6537 if !registration.registered {
6538 return Err(Error::WorkerLoop(format!(
6539 "worker registration for {:?} was not accepted",
6540 self.worker_id
6541 )));
6542 }
6543 let registered_worker_id = registration.worker_id.clone();
6544 let primary = self.run_registered_until(stop, registration).await;
6545 let deregistration = self
6546 .client
6547 .deregister_worker_registration(®istered_worker_id)
6548 .await;
6549
6550 match (primary, deregistration) {
6551 (Ok(()), Ok(_)) => Ok(()),
6552 (Ok(()), Err(deregistration)) => Err(deregistration),
6553 (Err(primary), Ok(_)) => Err(primary),
6554 (Err(primary), Err(deregistration)) => Err(Error::WorkerShutdown {
6555 primary: Box::new(primary),
6556 deregistration: Box::new(deregistration),
6557 }),
6558 }
6559 }
6560
6561 async fn run_registered_until(
6562 &self,
6563 stop: Arc<AtomicBool>,
6564 registration: RegisterWorkerResponse,
6565 ) -> Result<()> {
6566 let heartbeat_interval = Duration::from_secs(
6567 registration
6568 .heartbeat_interval_seconds
6569 .unwrap_or(self.heartbeat_interval.as_secs().max(1)),
6570 );
6571 let heartbeat = tokio::time::sleep(Duration::ZERO);
6577 tokio::pin!(heartbeat);
6578 let mut workflow_poller = (!self.workflows.is_empty()).then(|| {
6582 let worker = self.clone();
6583 let stop = Arc::clone(&stop);
6584 tokio::spawn(async move { worker.poll_workflows_until_stopped(stop).await })
6585 });
6586 let mut activity_poller = (!self.activities.is_empty()).then(|| {
6587 let worker = self.clone();
6588 let stop = Arc::clone(&stop);
6589 tokio::spawn(async move { worker.poll_activities_until_stopped(stop).await })
6590 });
6591 let mut query_poller = (!self.queries.is_empty()).then(|| {
6592 let worker = self.clone();
6593 let stop = Arc::clone(&stop);
6594 tokio::spawn(async move { worker.poll_queries_until_stopped(stop).await })
6595 });
6596
6597 loop {
6598 tokio::select! {
6599 _ = wait_for_worker_stop(&stop) => {
6600 stop.store(true, Ordering::SeqCst);
6601 break;
6602 }
6603 _ = &mut heartbeat => {
6604 let result = self.retry_worker_operation(|| {
6605 self.client.heartbeat_worker(
6606 &self.worker_id,
6607 self.max_concurrent_workflow_tasks,
6608 self.max_concurrent_activity_tasks,
6609 )
6610 }).await;
6611 heartbeat
6612 .as_mut()
6613 .reset(tokio::time::Instant::now() + heartbeat_interval);
6614 match result {
6615 Ok(acknowledgement) => {
6616 if let Some(observer) = &self.heartbeat_observer {
6617 observer(&WorkerHeartbeatObservation {
6618 worker_id: self.worker_id.clone(),
6619 task_queue: self.task_queue.clone(),
6620 acknowledged_at_unix_millis: SystemTime::now()
6621 .duration_since(UNIX_EPOCH)
6622 .unwrap_or_default()
6623 .as_millis()
6624 .min(u64::MAX as u128)
6625 as u64,
6626 acknowledgement,
6627 });
6628 }
6629 }
6630 Err(error) => {
6631 stop.store(true, Ordering::SeqCst);
6632 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await?;
6633 return Err(error);
6634 }
6635 }
6636 }
6637 result = OptionFuture::from(workflow_poller.as_mut()), if workflow_poller.is_some() => {
6638 workflow_poller = None;
6639 let stopped_by_server = stop.load(Ordering::SeqCst);
6640 stop.store(true, Ordering::SeqCst);
6641 let poller_result = optional_poller_result("workflow", result);
6642 let join_result =
6643 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6644 poller_result?;
6645 join_result?;
6646 if stopped_by_server {
6647 return Ok(());
6648 }
6649 return Err(Error::WorkerLoop(
6650 "workflow poller stopped unexpectedly".to_string(),
6651 ));
6652 }
6653 result = OptionFuture::from(activity_poller.as_mut()), if activity_poller.is_some() => {
6654 activity_poller = None;
6655 let stopped_by_server = stop.load(Ordering::SeqCst);
6656 stop.store(true, Ordering::SeqCst);
6657 let poller_result = optional_poller_result("activity", result);
6658 let join_result =
6659 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6660 poller_result?;
6661 join_result?;
6662 if stopped_by_server {
6663 return Ok(());
6664 }
6665 return Err(Error::WorkerLoop(
6666 "activity poller stopped unexpectedly".to_string(),
6667 ));
6668 }
6669 result = OptionFuture::from(query_poller.as_mut()), if query_poller.is_some() => {
6670 query_poller = None;
6671 let stopped_by_server = stop.load(Ordering::SeqCst);
6672 stop.store(true, Ordering::SeqCst);
6673 let poller_result = optional_poller_result("query", result);
6674 let join_result =
6675 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6676 poller_result?;
6677 join_result?;
6678 if stopped_by_server {
6679 return Ok(());
6680 }
6681 return Err(Error::WorkerLoop(
6682 "query poller stopped unexpectedly".to_string(),
6683 ));
6684 }
6685 }
6686 }
6687
6688 join_pollers(
6689 workflow_poller.take(),
6690 activity_poller.take(),
6691 query_poller.take(),
6692 )
6693 .await
6694 }
6695
6696 pub async fn run_once(&self) -> Result<usize> {
6710 let worker = self.with_storage_admission(Arc::new(AtomicBool::new(false)));
6711 let mut handled = 0;
6712 match worker.poll_workflow_once().await? {
6713 ManagedPollOutcome::Handled => handled += 1,
6714 ManagedPollOutcome::Stop => return Ok(handled),
6715 ManagedPollOutcome::Idle => {}
6716 }
6717 match worker.poll_activity_once().await? {
6718 ManagedPollOutcome::Handled => handled += 1,
6719 ManagedPollOutcome::Stop => return Ok(handled),
6720 ManagedPollOutcome::Idle => {}
6721 }
6722 if !self.queries.is_empty() {
6723 match worker.poll_query_once().await? {
6724 ManagedPollOutcome::Handled => handled += 1,
6725 ManagedPollOutcome::Stop => return Ok(handled),
6726 ManagedPollOutcome::Idle => {}
6727 }
6728 }
6729 Ok(handled)
6730 }
6731
6732 async fn poll_workflow_once(&self) -> Result<ManagedPollOutcome> {
6733 let poll_request_id = unique_request_id("rust-workflow-poll");
6734 let response = self
6735 .retry_worker_operation(|| {
6736 self.client.poll_workflow_task_response_with_request_id(
6737 &self.worker_id,
6738 &self.task_queue,
6739 self.poll_timeout,
6740 &poll_request_id,
6741 0,
6742 )
6743 })
6744 .await;
6745 let Some(response) = self.settle_worker_poll_response(response).await? else {
6746 return Ok(ManagedPollOutcome::Idle);
6747 };
6748 if response.outcome().should_stop() {
6749 return Ok(ManagedPollOutcome::Stop);
6750 }
6751 let memo_updates_supported =
6752 runtime_supports_workflow_memo_updates(response.server_capabilities.as_ref());
6753 let Some(task) = response.task else {
6754 return Ok(ManagedPollOutcome::Idle);
6755 };
6756
6757 let task_id = task.task_id.clone();
6758 let attempt = task.workflow_task_attempt;
6759 let run_id = task.run_id.clone();
6760 let lease_owner = task
6761 .lease_owner
6762 .clone()
6763 .unwrap_or_else(|| self.worker_id.clone());
6764
6765 match self.execute_workflow_task_decision(task) {
6766 Ok(decision)
6767 if commands_use_workflow_memo_updates(&decision.commands)
6768 && !memo_updates_supported =>
6769 {
6770 self.client
6771 .fail_workflow_task(
6772 &task_id,
6773 &lease_owner,
6774 attempt,
6775 Error::WorkflowMemoUpdatesUnavailable.to_string(),
6776 )
6777 .await?;
6778 }
6779 Ok(decision) if decision.commands.is_empty() => {
6780 self.client
6786 .fail_workflow_task_with_type(
6787 &task_id,
6788 &lease_owner,
6789 attempt,
6790 WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE,
6791 WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE,
6792 )
6793 .await?;
6794 }
6795 Ok(decision) => {
6796 let completion = self
6797 .client
6798 .complete_workflow_task_with_message_streams(
6799 &task_id,
6800 &lease_owner,
6801 attempt,
6802 decision.commands,
6803 decision.message_stream_cursors,
6804 decision.message_stream_waits,
6805 )
6806 .await;
6807 if let Err(error) = completion {
6808 if !workflow_task_completion_is_terminal_timeout(
6809 &error,
6810 &task_id,
6811 attempt,
6812 run_id.as_deref(),
6813 ) {
6814 return Err(error);
6815 }
6816 }
6817 }
6818 Err(error) => {
6819 self.client
6820 .fail_workflow_task(&task_id, &lease_owner, attempt, error.to_string())
6821 .await?;
6822 }
6823 }
6824
6825 Ok(ManagedPollOutcome::Handled)
6826 }
6827
6828 async fn poll_workflows_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
6829 while !stop.load(Ordering::SeqCst) {
6830 if self.poll_workflow_once().await? == ManagedPollOutcome::Stop {
6831 stop.store(true, Ordering::SeqCst);
6832 break;
6833 }
6834 }
6835
6836 Ok(())
6837 }
6838
6839 async fn poll_activity_once(&self) -> Result<ManagedPollOutcome> {
6840 let poll_request_id = unique_request_id("rust-activity-poll");
6841 let response = self
6842 .retry_worker_operation(|| {
6843 self.client.poll_activity_task_response_with_request_id(
6844 &self.worker_id,
6845 &self.task_queue,
6846 self.poll_timeout,
6847 &poll_request_id,
6848 0,
6849 )
6850 })
6851 .await;
6852 let Some(response) = self.settle_worker_poll_response(response).await? else {
6853 return Ok(ManagedPollOutcome::Idle);
6854 };
6855 if response.outcome().should_stop() {
6856 return Ok(ManagedPollOutcome::Stop);
6857 }
6858 let Some(task) = response.task else {
6859 return Ok(ManagedPollOutcome::Idle);
6860 };
6861
6862 let task_id = task.task_id.clone();
6863 let attempt_id = task
6864 .activity_attempt_id
6865 .clone()
6866 .or(task.attempt_id.clone())
6867 .unwrap_or_default();
6868 let lease_owner = task
6869 .lease_owner
6870 .clone()
6871 .unwrap_or_else(|| self.worker_id.clone());
6872 let codec = task.payload_codec.clone();
6873 let result = self.execute_activity_task(task).await;
6874 match result {
6875 Err(error) if worker_storage_admission_body(&error).is_some() => return Err(error),
6876 Ok(value) => {
6877 let completion = self
6878 .client
6879 .complete_activity_task(&task_id, &attempt_id, &lease_owner, value, &codec)
6880 .await;
6881 if let Err(error) = completion {
6882 if !activity_task_rejection_is_final(&error) {
6883 return Err(error);
6884 }
6885 }
6886 }
6887 Err(error) => {
6888 let failure = self
6889 .client
6890 .fail_activity_task(
6891 &task_id,
6892 &attempt_id,
6893 &lease_owner,
6894 error.to_string(),
6895 false,
6896 )
6897 .await;
6898 if let Err(error) = failure {
6899 if !activity_task_rejection_is_final(&error) {
6900 return Err(error);
6901 }
6902 }
6903 }
6904 }
6905
6906 Ok(ManagedPollOutcome::Handled)
6907 }
6908
6909 async fn poll_activities_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
6910 while !stop.load(Ordering::SeqCst) {
6911 if self.poll_activity_once().await? == ManagedPollOutcome::Stop {
6912 stop.store(true, Ordering::SeqCst);
6913 break;
6914 }
6915 }
6916
6917 Ok(())
6918 }
6919
6920 async fn poll_query_once(&self) -> Result<ManagedPollOutcome> {
6921 let poll_request_id = unique_request_id("rust-query-poll");
6922 let response = self
6923 .retry_worker_operation(|| {
6924 self.client.poll_query_task_response_with_request_id(
6925 &self.worker_id,
6926 &self.task_queue,
6927 self.poll_timeout,
6928 &poll_request_id,
6929 0,
6930 )
6931 })
6932 .await;
6933 let Some(response) = self.settle_worker_poll_response(response).await? else {
6934 return Ok(ManagedPollOutcome::Idle);
6935 };
6936 if response.outcome().should_stop() {
6937 return Ok(ManagedPollOutcome::Stop);
6938 }
6939 let Some(task) = response.task else {
6940 return Ok(ManagedPollOutcome::Idle);
6941 };
6942
6943 let query_task_id = task.query_task_id.clone();
6944 let attempt = task.query_task_attempt;
6945 let lease_owner = task
6946 .lease_owner
6947 .clone()
6948 .unwrap_or_else(|| self.worker_id.clone());
6949 let codec = task.payload_codec.clone();
6950
6951 match self.execute_query_task(task).await {
6952 Ok(value) => {
6953 let result_envelope = match encode_typed_envelope(&value, &codec) {
6954 Ok(result_envelope) => result_envelope,
6955 Err(error) => {
6956 let failure = self
6957 .client
6958 .fail_query_task(
6959 &query_task_id,
6960 &lease_owner,
6961 attempt,
6962 error.to_string(),
6963 "query_result_encode_failed",
6964 "QueryResultEncodeFailed",
6965 )
6966 .await;
6967 if let Err(error) = failure {
6968 if !query_task_rejection_is_final(&error) {
6969 return Err(error);
6970 }
6971 }
6972 return Ok(ManagedPollOutcome::Handled);
6973 }
6974 };
6975
6976 if let Err(error) = self
6977 .client
6978 .complete_query_task_with_envelope(
6979 &query_task_id,
6980 &lease_owner,
6981 attempt,
6982 value.clone().into_json()?,
6983 result_envelope,
6984 )
6985 .await
6986 {
6987 if !query_task_rejection_is_final(&error) {
6988 return Err(error);
6989 }
6990 }
6991 }
6992 Err(failure) => {
6993 let result = self
6994 .client
6995 .fail_query_task(
6996 &query_task_id,
6997 &lease_owner,
6998 attempt,
6999 failure.message,
7000 failure.reason,
7001 failure.failure_type,
7002 )
7003 .await;
7004 if let Err(error) = result {
7005 if !query_task_rejection_is_final(&error) {
7006 return Err(error);
7007 }
7008 }
7009 }
7010 }
7011
7012 Ok(ManagedPollOutcome::Handled)
7013 }
7014
7015 async fn poll_queries_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
7016 while !stop.load(Ordering::SeqCst) {
7017 if self.poll_query_once().await? == ManagedPollOutcome::Stop {
7018 stop.store(true, Ordering::SeqCst);
7019 break;
7020 }
7021 }
7022
7023 Ok(())
7024 }
7025
7026 async fn retry_worker_operation<T, F, Fut>(&self, mut operation: F) -> Result<T>
7027 where
7028 F: FnMut() -> Fut,
7029 Fut: Future<Output = Result<T>>,
7030 {
7031 let mut retries = 0;
7032
7033 loop {
7034 match operation().await {
7035 Err(error)
7036 if worker_operation_is_retryable(&error)
7037 && retries < self.retry_policy.max_retries =>
7038 {
7039 retries += 1;
7040 tokio::time::sleep(worker_retry_delay(self.retry_policy, retries)).await;
7041 }
7042 result => return result,
7043 }
7044 }
7045 }
7046
7047 async fn settle_worker_poll_response<T>(&self, response: Result<T>) -> Result<Option<T>> {
7048 match response {
7049 Ok(response) => Ok(Some(response)),
7050 Err(error) => {
7051 let Some(advertised_delay) = worker_poll_capacity_retry_after(&error) else {
7052 return Err(error);
7053 };
7054 let minimum_delay = self
7055 .retry_policy
7056 .initial_backoff
7057 .max(Duration::from_millis(1));
7058 let maximum_delay = self.retry_policy.max_backoff.max(minimum_delay);
7059 tokio::time::sleep(advertised_delay.max(minimum_delay).min(maximum_delay)).await;
7060 Ok(None)
7061 }
7062 }
7063 }
7064
7065 async fn execute_query_task(
7066 &self,
7067 mut task: QueryTask,
7068 ) -> std::result::Result<AvroValue, QueryTaskExecutionFailure> {
7069 validate_query_task_payloads(&task).map_err(|error| {
7070 QueryTaskExecutionFailure::new(
7071 "query_payload_decode_failed",
7072 error.to_string(),
7073 "QueryPayloadDecodeFailed",
7074 )
7075 })?;
7076
7077 if !self.workflows.contains_key(&task.workflow_type) {
7078 return Err(QueryTaskExecutionFailure::new(
7079 "query_workflow_type_not_registered",
7080 format!("no workflow registered for type {:?}", task.workflow_type),
7081 "WorkflowTypeNotRegistered",
7082 ));
7083 }
7084
7085 let Some(handlers) = self.queries.get(&task.workflow_type) else {
7086 return Err(QueryTaskExecutionFailure::new(
7087 "query_handler_unavailable",
7088 format!(
7089 "query handlers are unavailable for workflow type {:?}",
7090 task.workflow_type
7091 ),
7092 "QueryHandlerUnavailable",
7093 ));
7094 };
7095 let Some(query) = handlers.get(&task.query_name) else {
7096 return Err(QueryTaskExecutionFailure::new(
7097 "rejected_unknown_query",
7098 format!("unknown query {:?}", task.query_name),
7099 "QueryFailed",
7100 ));
7101 };
7102
7103 let args = decode_task_avro_arguments(task.query_arguments.as_ref(), &task.payload_codec)
7104 .map_err(|error| {
7105 QueryTaskExecutionFailure::new(
7106 "query_payload_decode_failed",
7107 format!("cannot decode query arguments: {error}"),
7108 "QueryPayloadDecodeFailed",
7109 )
7110 })?;
7111 let workflow_input_typed =
7112 decode_task_avro_arguments(task.workflow_arguments.as_ref(), &task.payload_codec)
7113 .map_err(|error| {
7114 QueryTaskExecutionFailure::new(
7115 "query_workflow_state_unavailable",
7116 format!("cannot decode workflow start input: {error}"),
7117 "QueryWorkflowStateUnavailable",
7118 )
7119 })?;
7120 let workflow_input = workflow_input_typed.clone().into_json().map_err(|error| {
7121 QueryTaskExecutionFailure::new(
7122 "query_workflow_state_unavailable",
7123 format!("cannot project workflow start input: {error}"),
7124 "QueryWorkflowStateUnavailable",
7125 )
7126 })?;
7127 hydrate_query_history_from_export(&mut task).map_err(|error| {
7128 QueryTaskExecutionFailure::new(
7129 "query_workflow_state_unavailable",
7130 format!("cannot restore query history snapshot: {error}"),
7131 "QueryWorkflowStateUnavailable",
7132 )
7133 })?;
7134 enrich_query_history_from_export(&mut task).map_err(|error| {
7135 QueryTaskExecutionFailure::new(
7136 "query_workflow_state_unavailable",
7137 format!("cannot restore compact query history payloads: {error}"),
7138 "QueryWorkflowStateUnavailable",
7139 )
7140 })?;
7141 let signal_events = query_signal_events(&task).map_err(|error| {
7142 QueryTaskExecutionFailure::new(
7143 "query_workflow_state_unavailable",
7144 format!("cannot decode committed workflow signals: {error}"),
7145 "QueryWorkflowStateUnavailable",
7146 )
7147 })?;
7148 let history_events = Arc::new(std::mem::take(&mut task.history_events));
7149 let context = QueryContext {
7150 workflow_id: task.workflow_id,
7151 run_id: task.run_id,
7152 workflow_type: task.workflow_type.clone(),
7153 run_status: task.run_status,
7154 workflow_input,
7155 workflow_input_avro_value: workflow_input_typed.clone(),
7156 history_events: Arc::clone(&history_events),
7157 signal_events: Arc::new(signal_events),
7158 };
7159
7160 let future = match query {
7161 RegisteredQuery::Snapshot(handler) => handler(context, args),
7162 RegisteredQuery::Replayed {
7163 state_type,
7164 handler,
7165 } => {
7166 let workflow = self
7167 .workflows
7168 .get(&task.workflow_type)
7169 .expect("workflow registration was checked above");
7170 if workflow.state_type != Some(*state_type) {
7171 return Err(QueryTaskExecutionFailure::new(
7172 "query_workflow_state_unavailable",
7173 "replayed query state type does not match its workflow registration",
7174 "QueryWorkflowStateUnavailable",
7175 ));
7176 }
7177 let replay = workflow.replay.as_ref().ok_or_else(|| {
7178 QueryTaskExecutionFailure::new(
7179 "query_workflow_state_unavailable",
7180 format!(
7181 "workflow type {:?} is not registered for instance-state replay",
7182 task.workflow_type
7183 ),
7184 "QueryWorkflowStateUnavailable",
7185 )
7186 })?;
7187 let workflow_state = Arc::new(Mutex::new(
7188 WorkflowState::new_with_identity(
7189 history_events.as_ref().clone(),
7190 context.workflow_id.clone(),
7191 context.run_id.clone(),
7192 self.task_queue.clone(),
7193 task.payload_codec,
7194 None,
7195 )
7196 .map_err(|error| {
7197 QueryTaskExecutionFailure::new(
7198 "query_workflow_state_unavailable",
7199 format!("workflow replay failed before query: {error}"),
7200 "QueryWorkflowStateUnavailable",
7201 )
7202 })?,
7203 ));
7204 let workflow_context = WorkflowContext {
7205 state: workflow_state,
7206 };
7207 let mut invocation = replay(workflow_context.clone(), workflow_input_typed.clone());
7208 let mut cx = TaskContext::from_waker(noop_waker_ref());
7209 match invocation.future.as_mut().poll(&mut cx) {
7210 Poll::Ready(Ok(_)) => {
7211 workflow_context
7212 .ensure_history_consumed()
7213 .map_err(|error| {
7214 QueryTaskExecutionFailure::new(
7215 "query_workflow_state_unavailable",
7216 format!("workflow replay failed before query: {error}"),
7217 "QueryWorkflowStateUnavailable",
7218 )
7219 })?;
7220 }
7221 Poll::Ready(Err(error)) => {
7222 return Err(QueryTaskExecutionFailure::new(
7223 "query_workflow_state_unavailable",
7224 format!("workflow replay failed before query: {error}"),
7225 "QueryWorkflowStateUnavailable",
7226 ));
7227 }
7228 Poll::Pending => {
7229 let commands = workflow_context.take_commands().map_err(|error| {
7230 QueryTaskExecutionFailure::new(
7231 "query_workflow_state_unavailable",
7232 format!("workflow replay failed before query: {error}"),
7233 "QueryWorkflowStateUnavailable",
7234 )
7235 })?;
7236 if commands.is_empty()
7237 && !workflow_context
7238 .matched_recorded_pending()
7239 .map_err(|error| {
7240 QueryTaskExecutionFailure::new(
7241 "query_workflow_state_unavailable",
7242 format!("workflow replay failed before query: {error}"),
7243 "QueryWorkflowStateUnavailable",
7244 )
7245 })?
7246 {
7247 return Err(QueryTaskExecutionFailure::new(
7248 "query_workflow_state_unavailable",
7249 "workflow replay yielded without a durable command",
7250 "QueryWorkflowStateUnavailable",
7251 ));
7252 }
7253 }
7254 }
7255 let state = (invocation.snapshot)().map_err(|error| {
7256 QueryTaskExecutionFailure::new(
7257 "query_workflow_state_unavailable",
7258 format!("cannot snapshot replayed workflow state: {error}"),
7259 "QueryWorkflowStateUnavailable",
7260 )
7261 })?;
7262 handler(context, state, args).map_err(|message| {
7263 QueryTaskExecutionFailure::new(
7264 "query_workflow_state_unavailable",
7265 message,
7266 "QueryWorkflowStateUnavailable",
7267 )
7268 })?
7269 }
7270 };
7271
7272 future.await.map_err(|error| {
7273 QueryTaskExecutionFailure::new("query_rejected", error.to_string(), "QueryFailed")
7274 })
7275 }
7276
7277 #[cfg(test)]
7278 fn execute_workflow_task(&self, task: WorkflowTask) -> Result<Vec<Value>> {
7279 Ok(self.execute_workflow_task_decision(task)?.commands)
7280 }
7281
7282 fn execute_workflow_task_decision(&self, task: WorkflowTask) -> Result<WorkflowTaskDecision> {
7283 validate_workflow_task_payloads(&task)?;
7284
7285 if let Some(update_id) = task
7286 .workflow_update_id
7287 .as_deref()
7288 .filter(|update_id| !update_id.is_empty())
7289 {
7290 return self
7291 .execute_update_task(&task, update_id)
7292 .map(WorkflowTaskDecision::without_message_streams);
7293 }
7294
7295 let workflow = self
7296 .workflows
7297 .get(&task.workflow_type)
7298 .ok_or_else(|| Error::WorkflowNotRegistered(task.workflow_type.clone()))?;
7299 let input = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
7300 let resume_signal = decode_resume_signal(&task)?;
7301 let history_budget = WorkflowHistoryBudget {
7302 event_count: task
7303 .total_history_events
7304 .unwrap_or_else(|| u64::try_from(task.history_events.len()).unwrap_or(u64::MAX)),
7305 size_bytes: task.history_size_bytes,
7306 continue_as_new_recommended: task.continue_as_new_recommended.unwrap_or(false),
7307 pressure: task.history_budget_pressure.clone(),
7308 };
7309 let workflow_command_identity = task
7310 .workflow_command_id
7311 .clone()
7312 .filter(|identity| !identity.is_empty())
7313 .unwrap_or_default();
7314 let mut workflow_state = WorkflowState::new_with_identity(
7315 task.history_events,
7316 task.workflow_id,
7317 task.run_id,
7318 self.task_queue.clone(),
7319 task.payload_codec.clone(),
7320 resume_signal,
7321 )?;
7322 workflow_state.history_budget = history_budget;
7323 workflow_state.workflow_command_identity = workflow_command_identity;
7324 workflow_state.cancel_requested = task.cancel_requested;
7325 let state = Arc::new(Mutex::new(workflow_state));
7326 let ctx = WorkflowContext { state };
7327 let mut future = (workflow.execute)(ctx.clone(), input);
7328 let mut cx = TaskContext::from_waker(noop_waker_ref());
7329
7330 match future.as_mut().poll(&mut cx) {
7331 Poll::Ready(Ok(result)) => {
7332 ctx.ensure_history_consumed()?;
7333 let result = encode_typed_envelope(&result, &task.payload_codec)?;
7334 let mut commands = ctx.take_commands()?;
7335 commands.push(json!({
7336 "type": "complete_workflow",
7337 "result": result
7338 }));
7339 self.message_stream_decision(&ctx, commands)
7340 }
7341 Poll::Ready(Err(error)) => {
7342 if let Error::ContinueAsNew(request) = error {
7343 let mut commands = ctx.take_commands()?;
7344 if let Some(command) = ctx.continue_as_new_command(request)? {
7345 commands.push(command);
7346 }
7347 ctx.ensure_history_consumed()?;
7348 return self.message_stream_decision(&ctx, commands);
7349 }
7350 if workflow_task_integrity_error(&error) {
7351 return Err(error);
7356 }
7357 ctx.ensure_history_consumed()?;
7360 let mut commands = ctx.take_commands()?;
7361 commands.push(workflow_failure_command(&error));
7362 self.message_stream_decision(&ctx, commands)
7363 }
7364 Poll::Pending => {
7365 let commands = ctx.take_commands()?;
7366 if commands.is_empty() && !ctx.matched_recorded_pending()? {
7367 Err(Error::WorkflowYieldedWithoutCommand)
7368 } else {
7369 self.message_stream_decision(&ctx, commands)
7370 }
7371 }
7372 }
7373 }
7374
7375 fn message_stream_decision(
7376 &self,
7377 ctx: &WorkflowContext,
7378 commands: Vec<Value>,
7379 ) -> Result<WorkflowTaskDecision> {
7380 let (message_stream_cursors, message_stream_waits) = ctx.message_stream_metadata()?;
7381 Ok(WorkflowTaskDecision {
7382 commands,
7383 message_stream_cursors,
7384 message_stream_waits,
7385 })
7386 }
7387
7388 fn execute_update_task(&self, task: &WorkflowTask, update_id: &str) -> Result<Vec<Value>> {
7389 if !self.workflows.contains_key(&task.workflow_type) {
7390 return Err(Error::WorkflowNotRegistered(task.workflow_type.clone()));
7391 }
7392
7393 let accepted = task.history_events.iter().rev().find_map(|event| {
7394 (event.event_type == "UpdateAccepted"
7395 && event.payload.get("update_id").and_then(Value::as_str) == Some(update_id))
7396 .then_some(&event.payload)
7397 });
7398 let update_name = accepted
7399 .and_then(|payload| payload.get("update_name"))
7400 .and_then(Value::as_str)
7401 .or(task.update_name.as_deref())
7402 .unwrap_or_default();
7403 let Some(handler) = self
7404 .updates
7405 .get(&task.workflow_type)
7406 .and_then(|handlers| handlers.get(update_name))
7407 else {
7408 return Ok(vec![json!({
7409 "type": "fail_update",
7410 "update_id": update_id,
7411 "message": format!(
7412 "no update handler is registered for {}.{update_name}",
7413 task.workflow_type
7414 ),
7415 "exception_type": "UnknownUpdate",
7416 "non_retryable": true,
7417 })]);
7418 };
7419 let arguments = accepted
7420 .and_then(|payload| payload.get("arguments"))
7421 .or(task.arguments.as_ref());
7422 let arguments = decode_task_avro_arguments(arguments, &task.payload_codec)?;
7423 let context = QueryContext {
7424 workflow_id: task.workflow_id.clone(),
7425 run_id: task.run_id.clone(),
7426 workflow_type: task.workflow_type.clone(),
7427 run_status: Some("running".to_string()),
7428 workflow_input: Value::Null,
7429 workflow_input_avro_value: AvroValue::Null,
7430 history_events: Arc::new(task.history_events.clone()),
7431 signal_events: Arc::new(Vec::new()),
7432 };
7433 let mut future = handler(context, arguments);
7434 let mut cx = TaskContext::from_waker(noop_waker_ref());
7435
7436 match future.as_mut().poll(&mut cx) {
7437 Poll::Ready(Ok(result)) => Ok(vec![json!({
7438 "type": "complete_update",
7439 "update_id": update_id,
7440 "result": encode_typed_envelope(&result, &task.payload_codec)?,
7441 })]),
7442 Poll::Ready(Err(error)) => Ok(vec![json!({
7443 "type": "fail_update",
7444 "update_id": update_id,
7445 "message": error.to_string(),
7446 "exception_type": "UpdateFailed",
7447 "non_retryable": true,
7448 })]),
7449 Poll::Pending => Err(Error::WorkflowYieldedWithoutCommand),
7450 }
7451 }
7452
7453 async fn execute_activity_task(&self, task: ActivityTask) -> Result<AvroValue> {
7454 validate_activity_task_payloads(&task)?;
7455
7456 let handler = self
7457 .activities
7458 .get(&task.activity_type)
7459 .ok_or_else(|| Error::ActivityNotRegistered(task.activity_type.clone()))?;
7460 let args = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
7461 let attempt_id = task
7462 .activity_attempt_id
7463 .clone()
7464 .or(task.attempt_id.clone())
7465 .unwrap_or_default();
7466 let lease_owner = task
7467 .lease_owner
7468 .clone()
7469 .unwrap_or_else(|| self.worker_id.clone());
7470 let ctx = ActivityContext {
7471 client: self.client.clone(),
7472 task_id: task.task_id,
7473 activity_attempt_id: attempt_id,
7474 lease_owner,
7475 activity_type: task.activity_type,
7476 attempt_number: task.attempt_number,
7477 task_queue: self.task_queue.clone(),
7478 worker_id: self.worker_id.clone(),
7479 };
7480
7481 handler(ctx, args).await
7482 }
7483}
7484
7485fn poller_result(
7486 kind: &str,
7487 result: std::result::Result<Result<()>, tokio::task::JoinError>,
7488) -> Result<()> {
7489 match result {
7490 Ok(result) => result,
7491 Err(error) => Err(Error::WorkerLoop(format!(
7492 "{kind} poller join error: {error}"
7493 ))),
7494 }
7495}
7496
7497fn optional_poller_result(
7498 kind: &str,
7499 result: Option<std::result::Result<Result<()>, tokio::task::JoinError>>,
7500) -> Result<()> {
7501 match result {
7502 Some(result) => poller_result(kind, result),
7503 None => Ok(()),
7504 }
7505}
7506
7507async fn join_pollers(
7508 workflow_poller: Option<tokio::task::JoinHandle<Result<()>>>,
7509 activity_poller: Option<tokio::task::JoinHandle<Result<()>>>,
7510 query_poller: Option<tokio::task::JoinHandle<Result<()>>>,
7511) -> Result<()> {
7512 let mut first_error = None;
7513
7514 if let Some(handle) = workflow_poller {
7515 if let Err(error) = poller_result("workflow", handle.await) {
7516 first_error.get_or_insert(error);
7517 }
7518 }
7519
7520 if let Some(handle) = activity_poller {
7521 if let Err(error) = poller_result("activity", handle.await) {
7522 first_error.get_or_insert(error);
7523 }
7524 }
7525
7526 if let Some(handle) = query_poller {
7527 if let Err(error) = poller_result("query", handle.await) {
7528 first_error.get_or_insert(error);
7529 }
7530 }
7531
7532 if let Some(error) = first_error {
7533 Err(error)
7534 } else {
7535 Ok(())
7536 }
7537}
7538
7539fn default_worker_id() -> String {
7540 let millis = SystemTime::now()
7541 .duration_since(UNIX_EPOCH)
7542 .unwrap_or_default()
7543 .as_millis();
7544 format!("rust-worker-{}-{millis}", std::process::id())
7545}
7546
7547fn percent_encode_path_segment(segment: &str) -> String {
7548 const HEX: &[u8; 16] = b"0123456789ABCDEF";
7549 let mut encoded = String::with_capacity(segment.len());
7550
7551 for byte in segment.bytes() {
7552 if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') {
7553 encoded.push(char::from(byte));
7554 } else {
7555 encoded.push('%');
7556 encoded.push(char::from(HEX[(byte >> 4) as usize]));
7557 encoded.push(char::from(HEX[(byte & 0x0f) as usize]));
7558 }
7559 }
7560
7561 encoded
7562}
7563
7564fn unique_request_id(prefix: &str) -> String {
7565 let nanos = SystemTime::now()
7566 .duration_since(UNIX_EPOCH)
7567 .unwrap_or_default()
7568 .as_nanos();
7569 format!("{prefix}-{}-{nanos}", std::process::id())
7570}
7571
7572#[derive(Debug)]
7573struct QueryTaskExecutionFailure {
7574 reason: String,
7575 message: String,
7576 failure_type: String,
7577}
7578
7579impl QueryTaskExecutionFailure {
7580 fn new(
7581 reason: impl Into<String>,
7582 message: impl Into<String>,
7583 failure_type: impl Into<String>,
7584 ) -> Self {
7585 Self {
7586 reason: reason.into(),
7587 message: message.into(),
7588 failure_type: failure_type.into(),
7589 }
7590 }
7591}
7592
7593#[derive(Clone, Debug)]
7599pub struct WorkflowInstance<S> {
7600 state: Arc<Mutex<S>>,
7601}
7602
7603impl<S> WorkflowInstance<S> {
7604 fn new(state: S) -> Self {
7605 Self {
7606 state: Arc::new(Mutex::new(state)),
7607 }
7608 }
7609
7610 pub fn read<R>(&self, reader: impl FnOnce(&S) -> R) -> Result<R> {
7612 let state = self
7613 .state
7614 .lock()
7615 .map_err(|_| Error::WorkflowStatePoisoned)?;
7616 Ok(reader(&state))
7617 }
7618
7619 pub fn update<R>(&self, transition: impl FnOnce(&mut S) -> R) -> Result<R> {
7621 let mut state = self
7622 .state
7623 .lock()
7624 .map_err(|_| Error::WorkflowStatePoisoned)?;
7625 Ok(transition(&mut state))
7626 }
7627}
7628
7629impl<S: Clone> WorkflowInstance<S> {
7630 fn snapshot(&self) -> Result<S> {
7631 self.read(Clone::clone)
7632 }
7633}
7634
7635#[derive(Clone, Debug, PartialEq)]
7636pub struct MessageStreamMessage {
7637 pub stream_name: String,
7638 pub message_id: String,
7639 pub position: u64,
7640 pub arguments: Vec<AvroValue>,
7641}
7642
7643#[derive(Clone, Debug)]
7644pub struct MessageStream {
7645 ctx: WorkflowContext,
7646 name: String,
7647}
7648
7649impl MessageStream {
7650 pub async fn receive(&self, max_items: usize) -> Result<Vec<MessageStreamMessage>> {
7652 if !(1..=MESSAGE_STREAM_MAX_BATCH).contains(&max_items) {
7653 return Err(Error::Codec(format!(
7654 "message stream max_items must be between 1 and {MESSAGE_STREAM_MAX_BATCH}"
7655 )));
7656 }
7657 loop {
7658 if let Some(batch) = self.ctx.take_message_stream_batch(&self.name, max_items)? {
7659 return Ok(batch);
7660 }
7661
7662 self.ctx.record_message_stream_wait(&self.name)?;
7663 let replay_wait_sequence = self.ctx.next_message_stream_wait_sequence()?;
7664 let arguments = self.ctx.wait_runtime_signal(MESSAGE_STREAM_SIGNAL).await?;
7665 self.ctx.buffer_message_stream_delivery(arguments)?;
7666 if let Some(sequence) = replay_wait_sequence {
7667 self.ctx.buffer_message_stream_history_for_wait(sequence)?;
7668 }
7669 }
7670 }
7671
7672 pub async fn receive_one(&self) -> Result<MessageStreamMessage> {
7673 self.receive(1)
7674 .await?
7675 .into_iter()
7676 .next()
7677 .ok_or_else(|| Error::Codec("message stream resumed without a message".to_string()))
7678 }
7679}
7680
7681#[derive(Clone, Debug)]
7682pub struct WorkflowContext {
7683 state: Arc<Mutex<WorkflowState>>,
7684}
7685
7686fn valid_memo_key(key: &str) -> bool {
7687 let numeric_candidate = key.strip_prefix('-').unwrap_or(key);
7688
7689 !key.is_empty()
7690 && key.len() <= 64
7691 && (numeric_candidate.is_empty()
7692 || !numeric_candidate.bytes().all(|byte| byte.is_ascii_digit()))
7693 && key
7694 .bytes()
7695 .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'.' | b':' | b'-'))
7696}
7697
7698fn avro_encoded_size(value: &AvroValue) -> Result<usize> {
7699 BASE64
7700 .decode(encode_avro_value(value)?.blob)
7701 .map(|bytes| bytes.len())
7702 .map_err(|error| Error::Codec(format!("memo Avro encoding was not strict base64: {error}")))
7703}
7704
7705fn canonical_memo_entries(value: AvroValue, require_entries: bool) -> Result<AvroValue> {
7706 let AvroValue::Map(entries) = value else {
7707 return Err(Error::InvalidMemoUpdate(
7708 "entries must serialize to an Avro string-keyed map".to_string(),
7709 ));
7710 };
7711 if require_entries && entries.is_empty() {
7712 return Err(Error::InvalidMemoUpdate(
7713 "at least one entry is required".to_string(),
7714 ));
7715 }
7716 if entries.len() > MAX_MEMO_ENTRIES {
7717 return Err(Error::InvalidMemoUpdate(format!(
7718 "at most {MAX_MEMO_ENTRIES} entries are allowed"
7719 )));
7720 }
7721
7722 for (key, value) in &entries {
7723 if !valid_memo_key(&key) {
7724 return Err(Error::InvalidMemoUpdate(
7725 "keys must match ^(?!-?[0-9]+$)[A-Za-z0-9_.:-]{1,64}$".to_string(),
7726 ));
7727 }
7728 if avro_encoded_size(value)? > MAX_MEMO_VALUE_SIZE_BYTES {
7729 return Err(Error::InvalidMemoUpdate(format!(
7730 "value {key:?} exceeds the {MAX_MEMO_VALUE_SIZE_BYTES}-byte limit"
7731 )));
7732 }
7733 }
7734
7735 let value = AvroValue::Map(entries);
7736 if avro_encoded_size(&value)? > MAX_MEMO_TOTAL_SIZE_BYTES {
7737 return Err(Error::InvalidMemoUpdate(format!(
7738 "update exceeds the {MAX_MEMO_TOTAL_SIZE_BYTES}-byte total limit"
7739 )));
7740 }
7741 Ok(value)
7742}
7743
7744fn decode_memo_history_map(envelope: &Value, require_entries: bool) -> Result<AvroValue> {
7745 let object = envelope.as_object().ok_or_else(|| {
7746 Error::InvalidMemoUpdate(
7747 "history field must use the public {codec, blob} payload envelope".to_string(),
7748 )
7749 })?;
7750 if object.len() != 2 || !object.contains_key("codec") || !object.contains_key("blob") {
7751 return Err(Error::InvalidMemoUpdate(
7752 "history field must use exactly the public {codec, blob} payload envelope".to_string(),
7753 ));
7754 }
7755
7756 canonical_memo_entries(
7757 decode_wire_avro_value(envelope, DEFAULT_CODEC)?,
7758 require_entries,
7759 )
7760}
7761
7762impl WorkflowContext {
7763 pub fn message_stream(&self, name: impl Into<String>) -> Result<MessageStream> {
7764 let name = name.into();
7765 if name.is_empty()
7766 || name.len() > 128
7767 || !name.bytes().all(|byte| {
7768 byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b':' | b'-')
7769 })
7770 {
7771 return Err(Error::Codec(
7772 "message stream names must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens"
7773 .to_string(),
7774 ));
7775 }
7776 Ok(MessageStream {
7777 ctx: self.clone(),
7778 name,
7779 })
7780 }
7781
7782 fn record_message_stream_wait(&self, name: &str) -> Result<()> {
7783 let mut state = self
7784 .state
7785 .lock()
7786 .map_err(|_| Error::WorkflowStatePoisoned)?;
7787 let position = state.message_stream_cursors.get(name).copied().unwrap_or(0);
7788 state
7789 .message_stream_waits
7790 .insert(name.to_string(), position);
7791 Ok(())
7792 }
7793
7794 fn buffer_message_stream(&self, message: MessageStreamMessage) -> Result<()> {
7795 let mut state = self
7796 .state
7797 .lock()
7798 .map_err(|_| Error::WorkflowStatePoisoned)?;
7799 let cursor = state
7800 .message_stream_cursors
7801 .get(&message.stream_name)
7802 .copied()
7803 .unwrap_or(0);
7804 if message.position <= cursor {
7805 return Ok(());
7806 }
7807 let pending = state
7808 .message_stream_messages
7809 .entry(message.stream_name.clone())
7810 .or_default();
7811 if pending.iter().any(|candidate| {
7812 candidate.position == message.position || candidate.message_id == message.message_id
7813 }) {
7814 return Ok(());
7815 }
7816 pending.push(message);
7817 pending.sort_by_key(|candidate| candidate.position);
7818 Ok(())
7819 }
7820
7821 fn buffer_message_stream_delivery(&self, arguments: Vec<Value>) -> Result<Option<String>> {
7822 if let Some(delivery) = decode_message_stream_delivery(arguments)? {
7823 match delivery {
7824 MessageStreamDelivery::Message(message) => {
7825 let stream_name = message.stream_name.clone();
7826 self.buffer_message_stream(message)?;
7827 return Ok(Some(stream_name));
7828 }
7829 MessageStreamDelivery::Cursor {
7830 stream_name,
7831 through_position,
7832 } => self.apply_message_stream_cursor(&stream_name, through_position)?,
7833 }
7834 }
7835 Ok(None)
7836 }
7837
7838 fn next_message_stream_wait_sequence(&self) -> Result<Option<u64>> {
7839 let state = self
7840 .state
7841 .lock()
7842 .map_err(|_| Error::WorkflowStatePoisoned)?;
7843 Ok(match state.recorded_commands.get(state.command_cursor) {
7844 Some(RecordedCommand::SignalWait {
7845 sequence,
7846 signal_name,
7847 ..
7848 }) if signal_name == MESSAGE_STREAM_SIGNAL => Some(*sequence),
7849 _ => None,
7850 })
7851 }
7852
7853 fn buffer_message_stream_history_for_wait(&self, wait_sequence: u64) -> Result<()> {
7854 let (history, payload_codec) = {
7855 let state = self
7856 .state
7857 .lock()
7858 .map_err(|_| Error::WorkflowStatePoisoned)?;
7859 (
7860 Arc::clone(&state.history_events),
7861 state.payload_codec.clone(),
7862 )
7863 };
7864
7865 let Some(opened_index) = history.iter().position(|event| {
7866 event.event_type == "SignalWaitOpened"
7867 && durable_event_sequence(event) == Some(wait_sequence)
7868 && event.payload.get("signal_name").and_then(Value::as_str)
7869 == Some(MESSAGE_STREAM_SIGNAL)
7870 }) else {
7871 return Ok(());
7872 };
7873 let boundary_index = history
7874 .iter()
7875 .enumerate()
7876 .skip(opened_index + 1)
7877 .find_map(|(index, event)| {
7878 (durable_event_sequence(event).is_some_and(|sequence| sequence > wait_sequence)
7879 && is_authored_command_open_event(event))
7880 .then_some(index)
7881 })
7882 .unwrap_or(history.len());
7883
7884 for event in history[opened_index + 1..boundary_index]
7885 .iter()
7886 .filter(|event| {
7887 event.event_type == "SignalReceived"
7888 && event.payload.get("signal_name").and_then(Value::as_str)
7889 == Some(MESSAGE_STREAM_SIGNAL)
7890 })
7891 {
7892 let arguments = decode_signal_event_arguments(event, &payload_codec)?
7893 .into_iter()
7894 .map(AvroValue::into_json)
7895 .collect::<Result<Vec<_>>>()?;
7896 self.buffer_message_stream_delivery(arguments)?;
7897 }
7898 Ok(())
7899 }
7900
7901 fn apply_message_stream_cursor(&self, name: &str, through_position: u64) -> Result<()> {
7902 let mut state = self
7903 .state
7904 .lock()
7905 .map_err(|_| Error::WorkflowStatePoisoned)?;
7906 let cursor = state
7907 .message_stream_cursors
7908 .entry(name.to_string())
7909 .or_default();
7910 *cursor = (*cursor).max(through_position);
7911 if let Some(pending) = state.message_stream_messages.get_mut(name) {
7912 pending.retain(|message| message.position > through_position);
7913 }
7914 Ok(())
7915 }
7916
7917 fn take_message_stream_batch(
7918 &self,
7919 name: &str,
7920 max_items: usize,
7921 ) -> Result<Option<Vec<MessageStreamMessage>>> {
7922 let mut state = self
7923 .state
7924 .lock()
7925 .map_err(|_| Error::WorkflowStatePoisoned)?;
7926 let cursor = state.message_stream_cursors.get(name).copied().unwrap_or(0);
7927 let pending = state
7928 .message_stream_messages
7929 .entry(name.to_string())
7930 .or_default();
7931 let count = contiguous_message_stream_count(pending, cursor, max_items);
7932 if count == 0 {
7933 return Ok(None);
7934 }
7935 let batch = pending.drain(..count).collect::<Vec<_>>();
7936 let position = batch.last().map(|message| message.position).unwrap_or(0);
7937 state
7938 .message_stream_cursors
7939 .insert(name.to_string(), position);
7940 state.message_stream_waits.remove(name);
7941 Ok(Some(batch))
7942 }
7943
7944 fn message_stream_metadata(&self) -> Result<(Vec<Value>, Vec<Value>)> {
7945 let state = self
7946 .state
7947 .lock()
7948 .map_err(|_| Error::WorkflowStatePoisoned)?;
7949 let mut cursors = state.message_stream_cursors.iter().collect::<Vec<_>>();
7950 cursors.sort_by_key(|(name, _)| *name);
7951 let mut waits = state.message_stream_waits.iter().collect::<Vec<_>>();
7952 waits.sort_by_key(|(name, _)| *name);
7953 Ok((
7954 cursors
7955 .into_iter()
7956 .map(|(name, position)| json!({"stream_name": name, "through_position": position}))
7957 .collect(),
7958 waits
7959 .into_iter()
7960 .map(|(name, position)| json!({"stream_name": name, "after_position": position}))
7961 .collect(),
7962 ))
7963 }
7964 pub fn workflow_identity(&self) -> Result<WorkflowIdentity> {
7966 let state = self
7967 .state
7968 .lock()
7969 .map_err(|_| Error::WorkflowStatePoisoned)?;
7970 Ok(WorkflowIdentity {
7971 workflow_id: state.workflow_id.clone(),
7972 run_id: state.run_id.clone(),
7973 })
7974 }
7975
7976 pub fn history_budget(&self) -> Result<WorkflowHistoryBudget> {
7978 let state = self
7979 .state
7980 .lock()
7981 .map_err(|_| Error::WorkflowStatePoisoned)?;
7982 Ok(state.history_budget.clone())
7983 }
7984
7985 pub fn continue_as_new<T: Serialize>(&self, args: T) -> Result<Value> {
7991 self.continue_as_new_with_options(ContinueAsNewOptions::new(), args)
7992 }
7993
7994 pub fn continue_as_new_with_options<T: Serialize>(
7996 &self,
7997 options: ContinueAsNewOptions,
7998 args: T,
7999 ) -> Result<Value> {
8000 options.validate()?;
8001 Err(Error::ContinueAsNew(ContinueAsNewRequest {
8002 arguments: normalize_avro_arguments(AvroValue::from_serialize(&args)?),
8003 options,
8004 }))
8005 }
8006
8007 pub fn activity<T: Serialize>(
8008 &self,
8009 activity_type: impl Into<String>,
8010 args: T,
8011 ) -> ActivityCall {
8012 self.activity_with_options(activity_type, ActivityOptions::new(), args)
8013 }
8014
8015 pub fn activity_on_queue<T, Q>(
8016 &self,
8017 activity_type: impl Into<String>,
8018 task_queue: Option<Q>,
8019 args: T,
8020 ) -> ActivityCall
8021 where
8022 T: Serialize,
8023 Q: Into<String>,
8024 {
8025 let mut options = ActivityOptions::new();
8026 options.task_queue = task_queue.map(Into::into);
8027 self.activity_with_options(activity_type, options, args)
8028 }
8029
8030 pub fn activity_with_options<T: Serialize>(
8068 &self,
8069 activity_type: impl Into<String>,
8070 options: ActivityOptions,
8071 args: T,
8072 ) -> ActivityCall {
8073 ActivityCall {
8074 ctx: self.clone(),
8075 activity_type: activity_type.into(),
8076 options,
8077 args: Some(AvroValue::from_serialize(&args)),
8078 scheduled: false,
8079 parallel_group_path: Vec::new(),
8080 }
8081 }
8082
8083 pub async fn activity_avro_value<T: Serialize>(
8084 &self,
8085 activity_type: impl Into<String>,
8086 args: T,
8087 ) -> Result<AvroValue> {
8088 let mut call = self.activity(activity_type, args);
8089 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8090 }
8091
8092 pub async fn activity_avro_value_with_options<T: Serialize>(
8093 &self,
8094 activity_type: impl Into<String>,
8095 options: ActivityOptions,
8096 args: T,
8097 ) -> Result<AvroValue> {
8098 let mut call = self.activity_with_options(activity_type, options, args);
8099 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8100 }
8101
8102 pub async fn activity_typed<I, O>(&self, activity_type: impl Into<String>, args: I) -> Result<O>
8104 where
8105 I: Serialize,
8106 O: DeserializeOwned,
8107 {
8108 self.activity_typed_with_options(activity_type, ActivityOptions::new(), args)
8109 .await
8110 }
8111
8112 pub async fn activity_typed_with_options<I, O>(
8118 &self,
8119 activity_type: impl Into<String>,
8120 options: ActivityOptions,
8121 args: I,
8122 ) -> Result<O>
8123 where
8124 I: Serialize,
8125 O: DeserializeOwned,
8126 {
8127 let activity_type = activity_type.into();
8128 let encoded = AvroValue::from_serialize(&args).map_err(|error| {
8129 handler_type_error::<I>(
8130 HandlerKind::Activity,
8131 &activity_type,
8132 HandlerValueKind::Input,
8133 error.to_string(),
8134 )
8135 });
8136 let mut call = ActivityCall {
8137 ctx: self.clone(),
8138 activity_type: activity_type.clone(),
8139 options,
8140 args: Some(encoded),
8141 scheduled: false,
8142 parallel_group_path: Vec::new(),
8143 };
8144 let result = std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await?;
8145 decode_handler_result(result, HandlerKind::Activity, &activity_type)
8146 }
8147
8148 pub fn parallel(&self, operations: Vec<ParallelOperation>) -> ParallelCall {
8155 ParallelCall::new(self.clone(), operations)
8156 }
8157
8158 pub fn join(&self, operations: Vec<ParallelOperation>) -> ParallelCall {
8160 self.parallel(operations)
8161 }
8162
8163 pub async fn parallel_avro_value(
8165 &self,
8166 operations: Vec<ParallelOperation>,
8167 ) -> Result<Vec<ParallelAvroResult>> {
8168 let mut call = self.parallel(operations);
8169 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8170 }
8171
8172 pub fn select(&self, operations: Vec<ParallelOperation>) -> SelectCall {
8175 let operations = operations
8176 .into_iter()
8177 .enumerate()
8178 .map(|(index, operation)| (SelectionKey::Index(index), operation))
8179 .collect();
8180 SelectCall::new(self.clone(), operations)
8181 }
8182
8183 pub fn select_keyed<K>(&self, operations: Vec<(K, ParallelOperation)>) -> SelectCall
8185 where
8186 K: Into<SelectionKey>,
8187 {
8188 SelectCall::new(
8189 self.clone(),
8190 operations
8191 .into_iter()
8192 .map(|(key, operation)| (key.into(), operation))
8193 .collect(),
8194 )
8195 }
8196
8197 pub fn saga(&self) -> Saga {
8199 Saga::new(self.clone())
8200 }
8201
8202 pub fn is_cancellation_requested(&self) -> Result<bool> {
8204 let state = self
8205 .state
8206 .lock()
8207 .map_err(|_| Error::WorkflowStatePoisoned)?;
8208 Ok(state.cancel_requested)
8209 }
8210
8211 pub fn throw_if_cancellation_requested(&self) -> Result<()> {
8216 if self.is_cancellation_requested()? {
8217 return Err(Error::WorkflowCancellationRequested(
8218 WorkflowCancellationRequested,
8219 ));
8220 }
8221 Ok(())
8222 }
8223
8224 pub fn wait_signal(&self, signal_name: impl Into<String>) -> SignalCall {
8225 SignalCall {
8226 ctx: self.clone(),
8227 signal_name: signal_name.into(),
8228 runtime_reserved_allowed: false,
8229 opened_wait: false,
8230 matched_pending: false,
8231 parallel_group_path: Vec::new(),
8232 }
8233 }
8234
8235 fn wait_runtime_signal(&self, signal_name: impl Into<String>) -> SignalCall {
8236 SignalCall {
8237 ctx: self.clone(),
8238 signal_name: signal_name.into(),
8239 runtime_reserved_allowed: true,
8240 opened_wait: false,
8241 matched_pending: false,
8242 parallel_group_path: Vec::new(),
8243 }
8244 }
8245
8246 pub async fn wait_signal_avro_value(
8247 &self,
8248 signal_name: impl Into<String>,
8249 ) -> Result<Vec<AvroValue>> {
8250 let mut call = self.wait_signal(signal_name);
8251 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8252 }
8253
8254 pub fn signals(&self, signal_name: &str) -> Result<Vec<Vec<Value>>> {
8260 self.signals_avro_value(signal_name)?
8261 .into_iter()
8262 .map(|arguments| {
8263 arguments
8264 .into_iter()
8265 .map(AvroValue::into_json)
8266 .collect::<Result<Vec<_>>>()
8267 })
8268 .collect()
8269 }
8270
8271 pub fn signals_avro_value(&self, signal_name: &str) -> Result<Vec<Vec<AvroValue>>> {
8273 let state = self
8274 .state
8275 .lock()
8276 .map_err(|_| Error::WorkflowStatePoisoned)?;
8277 state
8278 .history_events
8279 .iter()
8280 .filter(|event| {
8281 event.event_type == "SignalReceived"
8282 && event.payload.get("signal_name").and_then(Value::as_str) == Some(signal_name)
8283 })
8284 .map(|event| decode_signal_event_arguments(event, &state.payload_codec))
8285 .collect()
8286 }
8287
8288 pub fn updates(&self, update_name: &str) -> Result<Vec<Vec<Value>>> {
8294 self.updates_avro_value(update_name)?
8295 .into_iter()
8296 .map(|arguments| {
8297 arguments
8298 .into_iter()
8299 .map(AvroValue::into_json)
8300 .collect::<Result<Vec<_>>>()
8301 })
8302 .collect()
8303 }
8304
8305 pub fn updates_avro_value(&self, update_name: &str) -> Result<Vec<Vec<AvroValue>>> {
8307 let state = self
8308 .state
8309 .lock()
8310 .map_err(|_| Error::WorkflowStatePoisoned)?;
8311 let mut seen = Vec::new();
8312 let mut updates = Vec::new();
8313 for event in state.history_events.iter() {
8314 if !matches!(
8315 event.event_type.as_str(),
8316 "UpdateAccepted" | "UpdateApplied"
8317 ) || event.payload.get("update_name").and_then(Value::as_str) != Some(update_name)
8318 || event.payload.get("arguments").is_none()
8319 {
8320 continue;
8321 }
8322 if let Some(update_id) = event.payload.get("update_id").and_then(Value::as_str) {
8323 if seen.iter().any(|recorded| recorded == update_id) {
8324 continue;
8325 }
8326 seen.push(update_id.to_string());
8327 }
8328 updates.push(decode_update_event_arguments(event, &state.payload_codec)?);
8329 }
8330 Ok(updates)
8331 }
8332
8333 pub fn wait_condition<F>(
8341 &self,
8342 options: ConditionWaitOptions,
8343 predicate: F,
8344 ) -> ConditionWaitCall
8345 where
8346 F: Fn() -> Result<bool> + Send + 'static,
8347 {
8348 ConditionWaitCall {
8349 ctx: self.clone(),
8350 options,
8351 predicate: Box::new(predicate),
8352 occurrence_id: None,
8353 opened_wait: false,
8354 parallel_group_path: Vec::new(),
8355 }
8356 }
8357
8358 pub fn sleep(&self, duration: Duration) -> TimerCall {
8379 let delay_seconds = duration
8380 .as_secs()
8381 .checked_add(u64::from(duration.subsec_nanos() > 0));
8382 TimerCall {
8383 ctx: self.clone(),
8384 delay_seconds,
8385 scheduled: false,
8386 matched_pending: false,
8387 parallel_group_path: Vec::new(),
8388 }
8389 }
8390
8391 pub fn start_timer(&self, duration: Duration) -> TimerCall {
8393 self.sleep(duration)
8394 }
8395
8396 pub fn side_effect<T, F>(&self, callback: F) -> Result<T>
8403 where
8404 T: Serialize + DeserializeOwned,
8405 F: FnOnce() -> T,
8406 {
8407 {
8408 let mut state = self
8409 .state
8410 .lock()
8411 .map_err(|_| Error::WorkflowStatePoisoned)?;
8412 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8413 return match recorded {
8414 RecordedCommand::SideEffect { sequence, value } => {
8415 state.command_cursor += 1;
8416 value.deserialize().map_err(|error| {
8417 Error::NonDeterministicReplay(ReplayFailure::new(
8418 "side_effect_type_mismatch",
8419 Some(sequence),
8420 Some(std::any::type_name::<T>().to_string()),
8421 Some(error.to_string()),
8422 "recorded side-effect value is incompatible with the requested Rust type",
8423 ))
8424 })
8425 }
8426 other => Err(command_mismatch(&other, "side effect")),
8427 };
8428 }
8429 }
8430
8431 let value = callback();
8432 let avro_value = AvroValue::from_serialize(&value)?;
8433 let mut state = self
8434 .state
8435 .lock()
8436 .map_err(|_| Error::WorkflowStatePoisoned)?;
8437 let result = encode_typed_envelope(&avro_value, &state.payload_codec)?;
8438 state.commands.push(json!({
8439 "type": "record_side_effect",
8440 "result": result,
8441 }));
8442 Ok(value)
8443 }
8444
8445 pub fn side_effect_avro_value<F>(&self, callback: F) -> Result<AvroValue>
8447 where
8448 F: FnOnce() -> AvroValue,
8449 {
8450 {
8451 let mut state = self
8452 .state
8453 .lock()
8454 .map_err(|_| Error::WorkflowStatePoisoned)?;
8455 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8456 return match recorded {
8457 RecordedCommand::SideEffect { value, .. } => {
8458 state.command_cursor += 1;
8459 Ok(value)
8460 }
8461 other => Err(command_mismatch(&other, "side effect")),
8462 };
8463 }
8464 }
8465
8466 let value = callback();
8467 let mut state = self
8468 .state
8469 .lock()
8470 .map_err(|_| Error::WorkflowStatePoisoned)?;
8471 let result = encode_typed_envelope(&value, &state.payload_codec)?;
8472 state.commands.push(json!({
8473 "type": "record_side_effect",
8474 "result": result,
8475 }));
8476 Ok(value)
8477 }
8478
8479 pub fn append_workflow_stream(
8485 &self,
8486 stream_name: impl Into<String>,
8487 items: &[WorkflowStreamAppendItem],
8488 max_pending_items: Option<u64>,
8489 ) -> Result<()> {
8490 if items.is_empty() {
8491 return Err(Error::Codec(
8492 "workflow_stream_items_empty: append requires at least one item".to_string(),
8493 ));
8494 }
8495 if max_pending_items == Some(0) {
8496 return Err(Error::Codec(
8497 "workflow_stream_pending_limit_invalid: max_pending_items must be positive"
8498 .to_string(),
8499 ));
8500 }
8501 let stream_name = stream_name.into();
8502 if stream_name.is_empty() {
8503 return Err(Error::Codec(
8504 "workflow_stream_name_invalid: stream name must not be empty".to_string(),
8505 ));
8506 }
8507
8508 let mut state = self
8509 .state
8510 .lock()
8511 .map_err(|_| Error::WorkflowStatePoisoned)?;
8512 let command_ordinal = state.workflow_stream_command_counter;
8513 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8514 state.workflow_stream_command_counter += 1;
8515 return match recorded {
8516 RecordedCommand::SideEffect { .. } => {
8517 state.command_cursor += 1;
8518 Ok(())
8519 }
8520 other => Err(command_mismatch(&other, "workflow stream append")),
8521 };
8522 }
8523
8524 let identity = Self::workflow_stream_command_identity(&state)?.to_string();
8525 state.workflow_stream_command_counter += 1;
8526 let wire_items = items
8527 .iter()
8528 .enumerate()
8529 .map(|(item_index, item)| {
8530 item.wire_value(Some(format!(
8531 "dw-stream:{identity}:{command_ordinal}:{item_index}"
8532 )))
8533 })
8534 .collect::<Vec<_>>();
8535 let mut directive = json!({
8536 "operation": "append",
8537 "stream_name": stream_name,
8538 "command_identity": identity,
8539 "command_ordinal": command_ordinal,
8540 "items": wire_items,
8541 });
8542 if let Some(max_pending_items) = max_pending_items {
8543 directive["max_pending_items"] = json!(max_pending_items);
8544 }
8545 let result = encode_typed_envelope(&AvroValue::Null, &state.payload_codec)?;
8546 state.commands.push(json!({
8547 "type": "record_side_effect",
8548 "result": result,
8549 "workflow_stream": directive,
8550 }));
8551 Ok(())
8552 }
8553
8554 pub fn close_workflow_stream(
8556 &self,
8557 stream_name: impl Into<String>,
8558 retention_seconds: Option<u64>,
8559 ) -> Result<()> {
8560 self.finish_workflow_stream(stream_name.into(), None, retention_seconds)
8561 }
8562
8563 pub fn error_workflow_stream(
8565 &self,
8566 stream_name: impl Into<String>,
8567 error_reason: impl Into<String>,
8568 retention_seconds: Option<u64>,
8569 ) -> Result<()> {
8570 let error_reason = error_reason.into();
8571 if error_reason.is_empty() {
8572 return Err(Error::Codec(
8573 "workflow_stream_error_invalid: error reason must not be empty".to_string(),
8574 ));
8575 }
8576 self.finish_workflow_stream(stream_name.into(), Some(error_reason), retention_seconds)
8577 }
8578
8579 fn finish_workflow_stream(
8580 &self,
8581 stream_name: String,
8582 error_reason: Option<String>,
8583 retention_seconds: Option<u64>,
8584 ) -> Result<()> {
8585 if stream_name.is_empty() {
8586 return Err(Error::Codec(
8587 "workflow_stream_name_invalid: stream name must not be empty".to_string(),
8588 ));
8589 }
8590 if retention_seconds == Some(0) {
8591 return Err(Error::Codec(
8592 "workflow_stream_retention_invalid: retention_seconds must be positive".to_string(),
8593 ));
8594 }
8595 let mut state = self
8596 .state
8597 .lock()
8598 .map_err(|_| Error::WorkflowStatePoisoned)?;
8599 let command_ordinal = state.workflow_stream_command_counter;
8600 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8601 state.workflow_stream_command_counter += 1;
8602 return match recorded {
8603 RecordedCommand::SideEffect { .. } => {
8604 state.command_cursor += 1;
8605 Ok(())
8606 }
8607 other => Err(command_mismatch(&other, "workflow stream close")),
8608 };
8609 }
8610 let identity = Self::workflow_stream_command_identity(&state)?.to_string();
8611 state.workflow_stream_command_counter += 1;
8612 let mut directive = json!({
8613 "operation": if error_reason.is_some() { "error" } else { "close" },
8614 "stream_name": stream_name,
8615 "command_identity": identity,
8616 "command_ordinal": command_ordinal,
8617 });
8618 if let Some(error_reason) = error_reason {
8619 directive["error_reason"] = json!(error_reason);
8620 }
8621 if let Some(retention_seconds) = retention_seconds {
8622 directive["retention_seconds"] = json!(retention_seconds);
8623 }
8624 let result = encode_typed_envelope(&AvroValue::Null, &state.payload_codec)?;
8625 state.commands.push(json!({
8626 "type": "record_side_effect",
8627 "result": result,
8628 "workflow_stream": directive,
8629 }));
8630 Ok(())
8631 }
8632
8633 fn workflow_stream_command_identity(state: &WorkflowState) -> Result<&str> {
8634 let identity = state.workflow_command_identity.as_str();
8635 if identity.is_empty() {
8636 return Err(Error::MissingWorkflowCommandIdentity);
8637 }
8638 Ok(identity)
8639 }
8640
8641 pub fn upsert_search_attributes(&self, update: SearchAttributeUpdate) -> Result<()> {
8647 update.validate()?;
8648 let (attributes, attribute_types) = update.into_wire_parts();
8649 let mut state = self
8650 .state
8651 .lock()
8652 .map_err(|_| Error::WorkflowStatePoisoned)?;
8653
8654 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8655 return match recorded {
8656 RecordedCommand::SearchAttributes {
8657 sequence,
8658 attributes: recorded_attributes,
8659 attribute_types: recorded_attribute_types,
8660 } => {
8661 if recorded_attributes != attributes {
8662 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8663 "search_attribute_value_mismatch",
8664 Some(sequence),
8665 Some(recorded_attributes.to_string()),
8666 Some(attributes.to_string()),
8667 "search-attribute values differ from the recorded durable command",
8668 )));
8669 }
8670 if let RecordedSnapshotValue::Known(recorded_types) = recorded_attribute_types {
8671 if recorded_types != attribute_types {
8672 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8673 "search_attribute_type_mismatch",
8674 Some(sequence),
8675 Some(json!(recorded_types).to_string()),
8676 Some(json!(attribute_types).to_string()),
8677 "search-attribute declared types differ from the recorded durable command",
8678 )));
8679 }
8680 }
8681 state.command_cursor += 1;
8682 Ok(())
8683 }
8684 other => Err(command_mismatch(&other, "search-attribute update")),
8685 };
8686 }
8687
8688 let mut command = serde_json::Map::from_iter([
8689 ("type".to_string(), json!("upsert_search_attributes")),
8690 ("attributes".to_string(), attributes),
8691 ]);
8692 if !attribute_types.is_empty() {
8693 command.insert("attribute_types".to_string(), json!(attribute_types));
8694 }
8695 state.commands.push(Value::Object(command));
8696 Ok(())
8697 }
8698
8699 pub fn uuid_v4(&self) -> Result<Uuid> {
8701 self.side_effect(Uuid::new_v4)
8702 }
8703
8704 pub fn get_version(
8707 &self,
8708 change_id: impl Into<String>,
8709 min_supported: i32,
8710 max_supported: i32,
8711 ) -> Result<i32> {
8712 let change_id = change_id.into();
8713 if change_id.trim().is_empty() {
8714 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8715 "version_change_id_invalid",
8716 None,
8717 Some("non-empty change ID".to_string()),
8718 Some(change_id),
8719 "version markers require a stable non-empty change ID",
8720 )));
8721 }
8722 if min_supported > max_supported {
8723 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8724 "version_range_invalid",
8725 None,
8726 Some("min_supported <= max_supported".to_string()),
8727 Some(format!("{min_supported}..={max_supported}")),
8728 "version marker supported range is invalid",
8729 )));
8730 }
8731
8732 let mut state = self
8733 .state
8734 .lock()
8735 .map_err(|_| Error::WorkflowStatePoisoned)?;
8736 if let Some((version, sequence)) = state.version_markers.get(&change_id).copied() {
8737 ensure_version_supported(&change_id, version, min_supported, max_supported, sequence)?;
8738 return Ok(version);
8739 }
8740
8741 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8742 return match recorded {
8743 RecordedCommand::VersionMarker {
8744 sequence,
8745 change_id: recorded_change_id,
8746 version,
8747 ..
8748 } => {
8749 if recorded_change_id != change_id {
8750 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8751 "version_change_id_mismatch",
8752 Some(sequence),
8753 Some(recorded_change_id),
8754 Some(change_id),
8755 "recorded version marker change ID differs from current workflow code",
8756 )));
8757 }
8758 ensure_version_supported(
8759 &change_id,
8760 version,
8761 min_supported,
8762 max_supported,
8763 sequence,
8764 )?;
8765 state.command_cursor += 1;
8766 state.version_markers.insert(change_id, (version, sequence));
8767 Ok(version)
8768 }
8769 other => Err(command_mismatch(
8770 &other,
8771 format!("version marker:{change_id}"),
8772 )),
8773 };
8774 }
8775
8776 let version = max_supported;
8777 state.commands.push(json!({
8778 "type": "record_version_marker",
8779 "change_id": change_id,
8780 "version": version,
8781 "min_supported": min_supported,
8782 "max_supported": max_supported,
8783 }));
8784 state.version_markers.insert(change_id, (version, 0));
8787 Ok(version)
8788 }
8789
8790 pub fn patched(&self, change_id: impl Into<String>) -> Result<bool> {
8792 Ok(self.get_version(change_id, -1, 1)? == 1)
8793 }
8794
8795 pub fn deprecate_patch(&self, change_id: impl Into<String>) -> Result<()> {
8797 self.get_version(change_id, -1, 1).map(|_| ())
8798 }
8799
8800 pub fn upsert_memo<T: Serialize>(&self, entries: T) -> Result<()> {
8805 let entries = canonical_memo_entries(AvroValue::from_serialize(&entries)?, true)?;
8806 let mut state = self
8807 .state
8808 .lock()
8809 .map_err(|_| Error::WorkflowStatePoisoned)?;
8810
8811 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8812 return match recorded {
8813 RecordedCommand::Memo {
8814 sequence,
8815 entries: recorded_entries,
8816 } => {
8817 if recorded_entries != entries {
8818 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8819 "memo_update_mismatch",
8820 Some(sequence),
8821 Some(format!("{recorded_entries:?}")),
8822 Some(format!("{entries:?}")),
8823 "recorded memo entries differ from the current workflow update",
8824 )));
8825 }
8826 state.command_cursor += 1;
8827 Ok(())
8828 }
8829 other => Err(command_mismatch(&other, "memo upsert")),
8830 };
8831 }
8832
8833 let entries_envelope = encode_typed_envelope(&entries, DEFAULT_CODEC)?;
8834 state.commands.push(json!({
8835 "type": "upsert_memo",
8836 "entries": entries_envelope,
8837 }));
8838 Ok(())
8839 }
8840
8841 pub fn start_child_workflow<T: Serialize>(
8868 &self,
8869 workflow_type: impl Into<String>,
8870 options: ChildWorkflowOptions,
8871 args: T,
8872 ) -> ChildWorkflowCall {
8873 ChildWorkflowCall {
8874 ctx: self.clone(),
8875 workflow_type: workflow_type.into(),
8876 options,
8877 args: Some(AvroValue::from_serialize(&args)),
8878 scheduled: false,
8879 matched_pending: false,
8880 parallel_group_path: Vec::new(),
8881 }
8882 }
8883
8884 pub async fn start_child_workflow_avro_value<T: Serialize>(
8885 &self,
8886 workflow_type: impl Into<String>,
8887 options: ChildWorkflowOptions,
8888 args: T,
8889 ) -> Result<ChildWorkflowAvroResult> {
8890 let mut call = self.start_child_workflow(workflow_type, options, args);
8891 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8892 }
8893
8894 fn take_commands(&self) -> Result<Vec<Value>> {
8895 let mut state = self
8896 .state
8897 .lock()
8898 .map_err(|_| Error::WorkflowStatePoisoned)?;
8899 Ok(std::mem::take(&mut state.commands))
8900 }
8901
8902 fn continue_as_new_command(&self, request: ContinueAsNewRequest) -> Result<Option<Value>> {
8903 let mut state = self
8904 .state
8905 .lock()
8906 .map_err(|_| Error::WorkflowStatePoisoned)?;
8907
8908 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8909 return Err(command_mismatch(&recorded, "continue as new"));
8910 }
8911 if state.recorded_continue_as_new_sequence.is_some() {
8912 state.continue_as_new_consumed = true;
8913 return Ok(None);
8914 }
8915
8916 let arguments = encode_typed_envelope(&request.arguments, &state.payload_codec)?;
8917 let mut command = serde_json::Map::from_iter([
8918 ("type".to_string(), json!("continue_as_new")),
8919 ("arguments".to_string(), arguments),
8920 ("queue".to_string(), json!(state.task_queue.clone())),
8921 ]);
8922 if let Some(workflow_type) = request.options.workflow_type {
8923 command.insert("workflow_type".to_string(), json!(workflow_type));
8924 }
8925 if let Some(task_queue) = request.options.task_queue {
8926 command.insert("queue".to_string(), json!(task_queue));
8927 }
8928 Ok(Some(Value::Object(command)))
8929 }
8930
8931 fn matched_recorded_pending(&self) -> Result<bool> {
8932 let state = self
8933 .state
8934 .lock()
8935 .map_err(|_| Error::WorkflowStatePoisoned)?;
8936 Ok(state.matched_recorded_pending)
8937 }
8938
8939 fn ensure_history_consumed(&self) -> Result<()> {
8940 let state = self
8941 .state
8942 .lock()
8943 .map_err(|_| Error::WorkflowStatePoisoned)?;
8944 if let Some(command) = state.recorded_commands.get(state.command_cursor) {
8945 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8946 "recorded_commands_unconsumed",
8947 Some(command.sequence()),
8948 Some(command.shape().to_string()),
8949 Some("workflow completion".to_string()),
8950 "workflow completed before consuming all recorded durable commands",
8951 )));
8952 }
8953 if let Some(sequence) = state
8954 .recorded_continue_as_new_sequence
8955 .filter(|_| !state.continue_as_new_consumed)
8956 {
8957 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8958 "recorded_continue_as_new_unconsumed",
8959 Some(sequence),
8960 Some("continue as new".to_string()),
8961 Some("workflow completion".to_string()),
8962 "workflow completed without consuming its recorded continue-as-new transition",
8963 )));
8964 }
8965 Ok(())
8966 }
8967}
8968
8969fn contiguous_message_stream_count(
8970 pending: &[MessageStreamMessage],
8971 cursor: u64,
8972 max_items: usize,
8973) -> usize {
8974 pending
8975 .iter()
8976 .take(max_items)
8977 .enumerate()
8978 .take_while(|(offset, message)| {
8979 u64::try_from(*offset)
8980 .ok()
8981 .and_then(|offset| cursor.checked_add(offset + 1))
8982 == Some(message.position)
8983 })
8984 .count()
8985}
8986
8987fn is_authored_command_open_event(event: &HistoryEvent) -> bool {
8988 matches!(
8989 event.event_type.as_str(),
8990 "ActivityScheduled"
8991 | "TimerScheduled"
8992 | "ChildWorkflowScheduled"
8993 | "SignalWaitOpened"
8994 | "ConditionWaitOpened"
8995 | "SearchAttributesUpserted"
8996 | "SideEffectRecorded"
8997 | "VersionMarkerRecorded"
8998 | "MemoUpserted"
8999 | "WorkflowContinuedAsNew"
9000 )
9001}
9002
9003#[derive(Debug)]
9004struct WorkflowState {
9005 workflow_id: Option<String>,
9006 run_id: Option<String>,
9007 task_queue: String,
9008 payload_codec: String,
9009 history_events: Arc<Vec<HistoryEvent>>,
9010 history_budget: WorkflowHistoryBudget,
9011 cancel_requested: bool,
9012 resume_signal: Option<ResumeSignal>,
9013 recorded_commands: Vec<RecordedCommand>,
9014 selection_markers: Vec<SelectionMarker>,
9015 selection_marker_cursor: usize,
9016 cancelled_selection_members: Vec<SelectionCancellation>,
9017 recorded_continue_as_new_sequence: Option<u64>,
9018 continue_as_new_consumed: bool,
9019 command_cursor: usize,
9020 condition_wait_occurrence_counter: u64,
9021 matched_recorded_pending: bool,
9022 version_markers: HashMap<String, (i32, u64)>,
9023 workflow_command_identity: String,
9024 workflow_stream_command_counter: u64,
9025 commands: Vec<Value>,
9026 message_stream_messages: HashMap<String, Vec<MessageStreamMessage>>,
9027 message_stream_cursors: HashMap<String, u64>,
9028 message_stream_waits: HashMap<String, u64>,
9029}
9030
9031impl WorkflowState {
9032 #[cfg(test)]
9033 fn new(
9034 history: Vec<HistoryEvent>,
9035 task_queue: String,
9036 payload_codec: String,
9037 resume_signal: Option<ResumeSignal>,
9038 ) -> Result<Self> {
9039 Self::new_with_identity(
9040 history,
9041 None,
9042 None,
9043 task_queue,
9044 payload_codec,
9045 resume_signal,
9046 )
9047 }
9048
9049 fn new_with_identity(
9050 history: Vec<HistoryEvent>,
9051 workflow_id: Option<String>,
9052 run_id: Option<String>,
9053 task_queue: String,
9054 payload_codec: String,
9055 resume_signal: Option<ResumeSignal>,
9056 ) -> Result<Self> {
9057 let recorded_commands = recorded_commands(
9058 &history,
9059 &payload_codec,
9060 WorkflowIdentity {
9061 workflow_id: workflow_id.clone(),
9062 run_id: run_id.clone(),
9063 },
9064 )?;
9065 let selection_markers = recorded_selection_markers(&history)?;
9066 let cancelled_selection_members = recorded_selection_cancellations(&history)?;
9067 let recorded_continue_as_new = history
9068 .iter()
9069 .filter(|event| event.event_type == "WorkflowContinuedAsNew")
9070 .collect::<Vec<_>>();
9071 if recorded_continue_as_new.len() > 1 {
9072 return Err(invalid_recorded_history(
9073 "duplicate_continue_as_new_transition",
9074 recorded_continue_as_new
9075 .last()
9076 .and_then(|event| durable_event_sequence(event))
9077 .unwrap_or(0),
9078 "one WorkflowContinuedAsNew event",
9079 &format!(
9080 "{} WorkflowContinuedAsNew events",
9081 recorded_continue_as_new.len()
9082 ),
9083 "workflow history records one continue-as-new transition more than once",
9084 ));
9085 }
9086 let recorded_continue_as_new_sequence = recorded_continue_as_new
9087 .first()
9088 .map(|event| {
9089 durable_event_sequence(event).ok_or_else(|| {
9090 Error::NonDeterministicReplay(ReplayFailure::new(
9091 "continue_as_new_sequence_missing",
9092 None,
9093 Some("recorded transition sequence".to_string()),
9094 Some("missing sequence".to_string()),
9095 "WorkflowContinuedAsNew history is missing its recorded sequence",
9096 ))
9097 })
9098 })
9099 .transpose()?;
9100 let mut message_stream_cursors = HashMap::new();
9101 for event in &history {
9102 if !matches!(
9103 event.event_type.as_str(),
9104 "SignalReceived" | "SignalApplied"
9105 ) || event.payload.get("signal_name").and_then(Value::as_str)
9106 != Some(MESSAGE_STREAM_SIGNAL)
9107 {
9108 continue;
9109 }
9110 let arguments = decode_signal_event_arguments(event, &payload_codec)?;
9111 if arguments.len() != 1 {
9112 continue;
9113 }
9114 let envelope = arguments[0].clone().into_json()?;
9115 let Some(envelope) = envelope.as_object() else {
9116 continue;
9117 };
9118 if envelope.get("schema").and_then(Value::as_str) != Some(MESSAGE_STREAM_CURSOR_SCHEMA)
9119 {
9120 continue;
9121 }
9122 let Some(stream_name) = envelope.get("stream_name").and_then(Value::as_str) else {
9123 continue;
9124 };
9125 let Some(through_position) = envelope.get("through_position").and_then(Value::as_u64)
9126 else {
9127 continue;
9128 };
9129 let cursor = message_stream_cursors
9130 .entry(stream_name.to_string())
9131 .or_insert(0);
9132 *cursor = (*cursor).max(through_position);
9133 }
9134 let event_count = u64::try_from(history.len()).unwrap_or(u64::MAX);
9135 let cancel_requested = history.iter().any(|event| {
9136 matches!(
9137 event.event_type.as_str(),
9138 "WorkflowCancellationRequested" | "WorkflowCancelRequested"
9139 )
9140 });
9141 Ok(Self {
9142 workflow_command_identity: String::new(),
9143 workflow_stream_command_counter: 0,
9144 workflow_id,
9145 run_id,
9146 task_queue,
9147 payload_codec,
9148 history_events: Arc::new(history),
9149 history_budget: WorkflowHistoryBudget {
9150 event_count,
9151 ..WorkflowHistoryBudget::default()
9152 },
9153 cancel_requested,
9154 resume_signal,
9155 recorded_commands,
9156 selection_markers,
9157 selection_marker_cursor: 0,
9158 cancelled_selection_members,
9159 recorded_continue_as_new_sequence,
9160 continue_as_new_consumed: false,
9161 command_cursor: 0,
9162 condition_wait_occurrence_counter: 0,
9163 matched_recorded_pending: false,
9164 version_markers: HashMap::new(),
9165 commands: Vec::new(),
9166 message_stream_messages: HashMap::new(),
9167 message_stream_cursors,
9168 message_stream_waits: HashMap::new(),
9169 })
9170 }
9171}
9172
9173enum MessageStreamDelivery {
9174 Message(MessageStreamMessage),
9175 Cursor {
9176 stream_name: String,
9177 through_position: u64,
9178 },
9179}
9180
9181fn decode_message_stream_delivery(arguments: Vec<Value>) -> Result<Option<MessageStreamDelivery>> {
9182 if arguments.len() != 1 {
9183 return Ok(None);
9184 }
9185 let envelope = arguments
9186 .into_iter()
9187 .next()
9188 .expect("one argument was checked");
9189 let Some(envelope) = envelope.as_object() else {
9190 return Ok(None);
9191 };
9192 let Some(stream_name) = envelope.get("stream_name").and_then(Value::as_str) else {
9193 return Ok(None);
9194 };
9195 if envelope.get("schema").and_then(Value::as_str) == Some(MESSAGE_STREAM_CURSOR_SCHEMA) {
9196 let Some(through_position) = envelope.get("through_position").and_then(Value::as_u64)
9197 else {
9198 return Ok(None);
9199 };
9200 return Ok(Some(MessageStreamDelivery::Cursor {
9201 stream_name: stream_name.to_string(),
9202 through_position,
9203 }));
9204 }
9205 if envelope.get("schema").and_then(Value::as_str) != Some(MESSAGE_STREAM_SCHEMA) {
9206 return Ok(None);
9207 }
9208 let Some(message_id) = envelope.get("message_id").and_then(Value::as_str) else {
9209 return Ok(None);
9210 };
9211 let Some(position) = envelope
9212 .get("position")
9213 .and_then(Value::as_u64)
9214 .filter(|value| *value > 0)
9215 else {
9216 return Ok(None);
9217 };
9218 let Some(payload_envelope) = envelope.get("payload_envelope") else {
9219 return Ok(None);
9220 };
9221 let Ok(payload_envelope) = serde_json::from_value::<PayloadEnvelope>(payload_envelope.clone())
9222 else {
9223 return Ok(None);
9224 };
9225 let decoded = decode_avro_value(&payload_envelope)?;
9226 let AvroValue::Array(values) = decoded else {
9227 return Ok(None);
9228 };
9229 Ok(Some(MessageStreamDelivery::Message(MessageStreamMessage {
9230 stream_name: stream_name.to_string(),
9231 message_id: message_id.to_string(),
9232 position,
9233 arguments: values,
9234 })))
9235}
9236
9237#[derive(Clone, Debug)]
9238enum RecordedCommand {
9239 Activity {
9240 sequence: u64,
9241 activity_type: Option<String>,
9242 options: Option<RecordedActivityOptions>,
9243 outcome: Option<ActivityOutcome>,
9244 parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9245 },
9246 Timer {
9247 sequence: u64,
9248 delay_seconds: u64,
9249 fired: bool,
9250 parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9251 },
9252 ChildWorkflow {
9253 sequence: u64,
9254 workflow_type: Option<String>,
9255 outcome: Option<ChildWorkflowOutcome>,
9256 parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9257 },
9258 SignalWait {
9259 sequence: u64,
9260 signal_name: String,
9261 value: Option<Vec<AvroValue>>,
9262 parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9263 },
9264 ConditionWait {
9265 sequence: u64,
9266 occurrence_id: String,
9267 condition_key: Option<String>,
9268 predicate_identity: String,
9269 timeout_seconds: Option<u64>,
9270 result: Option<ConditionWaitResult>,
9271 parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9272 },
9273 SearchAttributes {
9274 sequence: u64,
9275 attributes: Value,
9276 attribute_types: RecordedSnapshotValue<BTreeMap<String, String>>,
9277 },
9278 SideEffect {
9279 sequence: u64,
9280 value: AvroValue,
9281 },
9282 VersionMarker {
9283 sequence: u64,
9284 change_id: String,
9285 version: i32,
9286 },
9287 Memo {
9288 sequence: u64,
9289 entries: AvroValue,
9290 },
9291}
9292
9293#[derive(Clone, Debug, PartialEq, Eq)]
9294struct SelectionMarker {
9295 selection_group_id: String,
9296 selection_group_base_sequence: u64,
9297 selection_group_size: usize,
9298 member_key: SelectionKey,
9299 member_index: usize,
9300 member_base_sequence: u64,
9301 member_size: usize,
9302 operation_kind: String,
9303 operation_identity: String,
9304 outcome: String,
9305 resolution_event_id: String,
9306 resolution_event_type: String,
9307}
9308
9309#[derive(Clone, Debug, PartialEq, Eq)]
9310struct SelectionCancellation {
9311 selection_group_id: String,
9312 member_key: SelectionKey,
9313 member_index: usize,
9314 member_base_sequence: u64,
9315 member_size: usize,
9316 operation_kind: String,
9317 operation_identity: String,
9318}
9319
9320fn recorded_selection_markers(events: &[HistoryEvent]) -> Result<Vec<SelectionMarker>> {
9321 let mut markers: Vec<SelectionMarker> = Vec::new();
9322 for event in events
9323 .iter()
9324 .filter(|event| event.event_type == "SelectionResolved")
9325 {
9326 let payload = &event.payload;
9327 let base_sequence = required_selection_u64(payload, "selection_group_base_sequence")?;
9328 let group_size = required_selection_usize(payload, "selection_group_size")?;
9329 let member_base_sequence = required_selection_u64(payload, "member_base_sequence")?;
9330 let member_size = required_selection_usize(payload, "member_size")?;
9331 let member_index = required_selection_usize_allow_zero(payload, "member_index")?;
9332 let group_id = payload_string(payload, "selection_group_id").ok_or_else(|| {
9333 invalid_recorded_history(
9334 "selection_marker_invalid",
9335 base_sequence,
9336 "non-empty selection_group_id",
9337 &payload.to_string(),
9338 "selection winner history is missing its durable group identity",
9339 )
9340 })?;
9341 let expected_group_id = format!("select-calls:{base_sequence}:{group_size}");
9342 if group_id != expected_group_id {
9343 return Err(invalid_recorded_history(
9344 "selection_marker_invalid",
9345 base_sequence,
9346 &expected_group_id,
9347 &group_id,
9348 "selection winner history contains an incompatible group identity",
9349 ));
9350 }
9351 let group_end = base_sequence
9352 .checked_add(u64::try_from(group_size).unwrap_or(u64::MAX))
9353 .unwrap_or(u64::MAX);
9354 let member_end = member_base_sequence
9355 .checked_add(u64::try_from(member_size).unwrap_or(u64::MAX))
9356 .unwrap_or(u64::MAX);
9357 if member_index >= group_size
9358 || member_base_sequence < base_sequence
9359 || member_end > group_end
9360 {
9361 return Err(invalid_recorded_history(
9362 "selection_marker_invalid",
9363 base_sequence,
9364 "winner member within selection group bounds",
9365 &payload.to_string(),
9366 "selection winner history contains an invalid member range",
9367 ));
9368 }
9369 let operation_kind = payload_string(payload, "operation_kind").ok_or_else(|| {
9370 invalid_recorded_history(
9371 "selection_marker_invalid",
9372 base_sequence,
9373 "selection operation kind",
9374 &payload.to_string(),
9375 "selection winner history is missing its operation kind",
9376 )
9377 })?;
9378 if !matches!(
9379 operation_kind.as_str(),
9380 "activity" | "child" | "timer" | "signal" | "condition" | "group"
9381 ) {
9382 return Err(invalid_recorded_history(
9383 "selection_marker_invalid",
9384 base_sequence,
9385 "activity, child, timer, signal, condition, or group",
9386 &operation_kind,
9387 "selection winner history contains an unsupported operation kind",
9388 ));
9389 }
9390 let operation_identity =
9391 payload_string(payload, "operation_identity").ok_or_else(|| {
9392 invalid_recorded_history(
9393 "selection_marker_invalid",
9394 base_sequence,
9395 "non-empty operation identity",
9396 &payload.to_string(),
9397 "selection winner history is missing its durable operation identity",
9398 )
9399 })?;
9400 let outcome = payload_string(payload, "outcome").ok_or_else(|| {
9401 invalid_recorded_history(
9402 "selection_marker_invalid",
9403 base_sequence,
9404 "completed or failed selection outcome",
9405 &payload.to_string(),
9406 "selection winner history is missing its outcome",
9407 )
9408 })?;
9409 if !matches!(outcome.as_str(), "completed" | "failed") {
9410 return Err(invalid_recorded_history(
9411 "selection_marker_invalid",
9412 base_sequence,
9413 "completed or failed selection outcome",
9414 &outcome,
9415 "selection winner history contains an unsupported outcome",
9416 ));
9417 }
9418 let marker = SelectionMarker {
9419 selection_group_id: group_id,
9420 selection_group_base_sequence: base_sequence,
9421 selection_group_size: group_size,
9422 member_key: selection_key_from_value(payload.get("member_key"), base_sequence)?,
9423 member_index,
9424 member_base_sequence,
9425 member_size,
9426 operation_kind,
9427 operation_identity,
9428 outcome,
9429 resolution_event_id: payload_string(payload, "resolution_event_id").ok_or_else(
9430 || {
9431 invalid_recorded_history(
9432 "selection_marker_invalid",
9433 base_sequence,
9434 "durable resolution_event_id",
9435 &payload.to_string(),
9436 "selection winner history is missing its terminal event identity",
9437 )
9438 },
9439 )?,
9440 resolution_event_type: payload_string(payload, "resolution_event_type").ok_or_else(
9441 || {
9442 invalid_recorded_history(
9443 "selection_marker_invalid",
9444 base_sequence,
9445 "durable resolution_event_type",
9446 &payload.to_string(),
9447 "selection winner history is missing its terminal event type",
9448 )
9449 },
9450 )?,
9451 };
9452 if let Some(existing) = markers
9453 .iter()
9454 .find(|existing| existing.selection_group_id == marker.selection_group_id)
9455 {
9456 if existing != &marker {
9457 return Err(invalid_recorded_history(
9458 "selection_marker_conflict",
9459 base_sequence,
9460 &format!("one winner for {}", marker.selection_group_id),
9461 &payload.to_string(),
9462 "selection history records conflicting winners for one durable group",
9463 ));
9464 }
9465 continue;
9466 }
9467 markers.push(marker);
9468 }
9469 Ok(markers)
9470}
9471
9472fn recorded_selection_cancellations(events: &[HistoryEvent]) -> Result<Vec<SelectionCancellation>> {
9473 let mut cancelled: Vec<SelectionCancellation> = Vec::new();
9474 for event in events
9475 .iter()
9476 .filter(|event| event.event_type == "SelectionOperationCancelled")
9477 {
9478 let group_id = payload_string(&event.payload, "selection_group_id").ok_or_else(|| {
9479 invalid_recorded_history(
9480 "selection_cancellation_invalid",
9481 0,
9482 "non-empty selection_group_id",
9483 &event.payload.to_string(),
9484 "selection cancellation history is missing its group identity",
9485 )
9486 })?;
9487 let member_base_sequence = required_selection_u64(&event.payload, "member_base_sequence")?;
9488 let marker = SelectionCancellation {
9489 selection_group_id: group_id,
9490 member_key: selection_key_from_value(
9491 event.payload.get("member_key"),
9492 member_base_sequence,
9493 )?,
9494 member_index: required_selection_usize_allow_zero(&event.payload, "member_index")?,
9495 member_base_sequence,
9496 member_size: required_selection_usize(&event.payload, "member_size")?,
9497 operation_kind: payload_string(&event.payload, "operation_kind").ok_or_else(|| {
9498 invalid_recorded_history(
9499 "selection_cancellation_invalid",
9500 member_base_sequence,
9501 "selection operation kind",
9502 &event.payload.to_string(),
9503 "selection cancellation is missing its operation kind",
9504 )
9505 })?,
9506 operation_identity: payload_string(&event.payload, "operation_identity").ok_or_else(
9507 || {
9508 invalid_recorded_history(
9509 "selection_cancellation_invalid",
9510 member_base_sequence,
9511 "selection operation identity",
9512 &event.payload.to_string(),
9513 "selection cancellation is missing its operation identity",
9514 )
9515 },
9516 )?,
9517 };
9518 if let Some(existing) = cancelled.iter().find(|recorded| {
9519 recorded.selection_group_id == marker.selection_group_id
9520 && recorded.member_base_sequence == marker.member_base_sequence
9521 }) {
9522 if existing != &marker {
9523 return Err(invalid_recorded_history(
9524 "selection_cancellation_conflict",
9525 member_base_sequence,
9526 "one stable SelectionOperationCancelled marker",
9527 &event.payload.to_string(),
9528 "selection cancellation history contains conflicting member metadata",
9529 ));
9530 }
9531 } else {
9532 cancelled.push(marker);
9533 }
9534 }
9535 Ok(cancelled)
9536}
9537
9538fn required_selection_u64(payload: &Value, field: &str) -> Result<u64> {
9539 payload
9540 .get(field)
9541 .and_then(value_as_u64)
9542 .filter(|value| *value > 0)
9543 .ok_or_else(|| {
9544 invalid_recorded_history(
9545 "selection_marker_invalid",
9546 0,
9547 &format!("positive integer {field}"),
9548 &payload.to_string(),
9549 "selection history contains invalid durable identity metadata",
9550 )
9551 })
9552}
9553
9554fn required_selection_usize(payload: &Value, field: &str) -> Result<usize> {
9555 required_selection_usize_allow_zero(payload, field).and_then(|value| {
9556 if value > 0 {
9557 Ok(value)
9558 } else {
9559 Err(invalid_recorded_history(
9560 "selection_marker_invalid",
9561 0,
9562 &format!("positive integer {field}"),
9563 &payload.to_string(),
9564 "selection history contains invalid durable identity metadata",
9565 ))
9566 }
9567 })
9568}
9569
9570fn required_selection_usize_allow_zero(payload: &Value, field: &str) -> Result<usize> {
9571 payload
9572 .get(field)
9573 .and_then(value_as_u64)
9574 .and_then(|value| usize::try_from(value).ok())
9575 .ok_or_else(|| {
9576 invalid_recorded_history(
9577 "selection_marker_invalid",
9578 0,
9579 &format!("non-negative integer {field}"),
9580 &payload.to_string(),
9581 "selection history contains invalid durable identity metadata",
9582 )
9583 })
9584}
9585
9586#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
9587struct RecordedActivityOptions {
9588 task_queue: RecordedSnapshotValue<Option<String>>,
9589 execution_mode: RecordedSnapshotValue<Option<String>>,
9590 retry_policy: ActivityRetrySnapshot,
9591}
9592
9593#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
9594enum RecordedSnapshotValue<T> {
9595 Unknown,
9597 Known(T),
9598}
9599
9600impl<T: PartialEq> RecordedSnapshotValue<T> {
9601 fn matches_current(&self, current: &Self) -> bool {
9602 match self {
9603 Self::Unknown => true,
9604 Self::Known(recorded) => matches!(current, Self::Known(value) if value == recorded),
9605 }
9606 }
9607}
9608
9609#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
9610struct ActivityRetrySnapshot {
9611 snapshot_version: RecordedSnapshotValue<Option<u64>>,
9612 max_attempts: RecordedSnapshotValue<Option<u64>>,
9613 backoff_seconds: RecordedSnapshotValue<Vec<u64>>,
9614 start_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
9615 schedule_to_start_timeout: RecordedSnapshotValue<Option<u64>>,
9616 schedule_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
9617 heartbeat_timeout: RecordedSnapshotValue<Option<u64>>,
9618 non_retryable_error_types: RecordedSnapshotValue<Vec<String>>,
9619}
9620
9621impl ActivityRetrySnapshot {
9622 fn matches_current(&self, current: &Self) -> bool {
9623 self.snapshot_version
9624 .matches_current(¤t.snapshot_version)
9625 && self.max_attempts.matches_current(¤t.max_attempts)
9626 && self
9627 .backoff_seconds
9628 .matches_current(¤t.backoff_seconds)
9629 && self
9630 .start_to_close_timeout
9631 .matches_current(¤t.start_to_close_timeout)
9632 && self
9633 .schedule_to_start_timeout
9634 .matches_current(¤t.schedule_to_start_timeout)
9635 && self
9636 .schedule_to_close_timeout
9637 .matches_current(¤t.schedule_to_close_timeout)
9638 && self
9639 .heartbeat_timeout
9640 .matches_current(¤t.heartbeat_timeout)
9641 && self
9642 .non_retryable_error_types
9643 .matches_current(¤t.non_retryable_error_types)
9644 }
9645}
9646
9647fn recorded_optional_u64(
9648 object: Option<&serde_json::Map<String, Value>>,
9649 field: &str,
9650) -> RecordedSnapshotValue<Option<u64>> {
9651 match object.and_then(|object| object.get(field)) {
9652 None => RecordedSnapshotValue::Unknown,
9653 Some(Value::Null) => RecordedSnapshotValue::Known(None),
9654 Some(value) => RecordedSnapshotValue::Known(value_as_u64(value)),
9655 }
9656}
9657
9658fn recorded_optional_string(
9659 object: &serde_json::Map<String, Value>,
9660 field: &str,
9661) -> RecordedSnapshotValue<Option<String>> {
9662 match object.get(field) {
9663 None => RecordedSnapshotValue::Unknown,
9664 Some(Value::Null) => RecordedSnapshotValue::Known(None),
9665 Some(value) => RecordedSnapshotValue::Known(value.as_str().map(str::to_string)),
9666 }
9667}
9668
9669fn recorded_activity_retry_snapshot(policy: Option<&Value>) -> ActivityRetrySnapshot {
9670 let policy = policy.and_then(Value::as_object);
9671 let backoff_seconds = policy
9672 .and_then(|policy| policy.get("backoff_seconds"))
9673 .and_then(Value::as_array)
9674 .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
9675 .map_or(RecordedSnapshotValue::Unknown, RecordedSnapshotValue::Known);
9676 let mut non_retryable_error_types = Vec::new();
9677 for error_type in policy
9678 .and_then(|policy| policy.get("non_retryable_error_types"))
9679 .and_then(Value::as_array)
9680 .into_iter()
9681 .flatten()
9682 .filter_map(Value::as_str)
9683 .map(str::trim)
9684 .filter(|error_type| !error_type.is_empty())
9685 {
9686 if !non_retryable_error_types
9687 .iter()
9688 .any(|recorded| recorded == error_type)
9689 {
9690 non_retryable_error_types.push(error_type.to_string());
9691 }
9692 }
9693
9694 ActivityRetrySnapshot {
9695 snapshot_version: recorded_optional_u64(policy, "snapshot_version"),
9696 max_attempts: recorded_optional_u64(policy, "max_attempts"),
9697 backoff_seconds,
9698 start_to_close_timeout: recorded_optional_u64(policy, "start_to_close_timeout"),
9699 schedule_to_start_timeout: recorded_optional_u64(policy, "schedule_to_start_timeout"),
9700 schedule_to_close_timeout: recorded_optional_u64(policy, "schedule_to_close_timeout"),
9701 heartbeat_timeout: recorded_optional_u64(policy, "heartbeat_timeout"),
9702 non_retryable_error_types: if policy
9703 .is_some_and(|policy| policy.contains_key("non_retryable_error_types"))
9704 {
9705 RecordedSnapshotValue::Known(non_retryable_error_types)
9706 } else {
9707 RecordedSnapshotValue::Unknown
9708 },
9709 }
9710}
9711
9712fn current_activity_retry_snapshot(options: &ValidatedActivityOptions) -> ActivityRetrySnapshot {
9713 let policy = options.retry_policy.as_ref();
9714 let max_attempts = match policy.and_then(|policy| policy.get("max_attempts")) {
9715 Some(Value::Null) => None,
9716 Some(value) => value_as_u64(value),
9717 None => Some(1),
9718 };
9719 let backoff_seconds = policy
9720 .and_then(|policy| policy.get("backoff_seconds"))
9721 .and_then(Value::as_array)
9722 .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
9723 .unwrap_or_default();
9724 let non_retryable_error_types = policy
9725 .and_then(|policy| policy.get("non_retryable_error_types"))
9726 .and_then(Value::as_array)
9727 .into_iter()
9728 .flatten()
9729 .filter_map(Value::as_str)
9730 .map(str::to_string)
9731 .collect();
9732
9733 ActivityRetrySnapshot {
9734 snapshot_version: RecordedSnapshotValue::Known(Some(1)),
9735 max_attempts: RecordedSnapshotValue::Known(max_attempts),
9736 backoff_seconds: RecordedSnapshotValue::Known(backoff_seconds),
9737 start_to_close_timeout: RecordedSnapshotValue::Known(options.start_to_close_timeout),
9738 schedule_to_start_timeout: RecordedSnapshotValue::Known(options.schedule_to_start_timeout),
9739 schedule_to_close_timeout: RecordedSnapshotValue::Known(options.schedule_to_close_timeout),
9740 heartbeat_timeout: RecordedSnapshotValue::Known(options.heartbeat_timeout),
9741 non_retryable_error_types: RecordedSnapshotValue::Known(non_retryable_error_types),
9742 }
9743}
9744
9745fn activity_options_description(options: &RecordedActivityOptions) -> String {
9746 serde_json::to_string(options).unwrap_or_else(|_| format!("{options:?}"))
9747}
9748
9749impl RecordedCommand {
9750 fn sequence(&self) -> u64 {
9751 match self {
9752 Self::Activity { sequence, .. }
9753 | Self::Timer { sequence, .. }
9754 | Self::ChildWorkflow { sequence, .. }
9755 | Self::SignalWait { sequence, .. }
9756 | Self::ConditionWait { sequence, .. }
9757 | Self::SearchAttributes { sequence, .. }
9758 | Self::SideEffect { sequence, .. }
9759 | Self::VersionMarker { sequence, .. }
9760 | Self::Memo { sequence, .. } => *sequence,
9761 }
9762 }
9763
9764 fn shape(&self) -> &'static str {
9765 match self {
9766 Self::Activity { .. } => "activity",
9767 Self::Timer { .. } => "timer",
9768 Self::ChildWorkflow { .. } => "child workflow",
9769 Self::SignalWait { .. } => "signal wait",
9770 Self::ConditionWait { .. } => "condition wait",
9771 Self::SearchAttributes { .. } => "search-attribute update",
9772 Self::SideEffect { .. } => "side effect",
9773 Self::VersionMarker { .. } => "version marker",
9774 Self::Memo { .. } => "memo upsert",
9775 }
9776 }
9777}
9778
9779fn ensure_version_supported(
9780 change_id: &str,
9781 version: i32,
9782 min_supported: i32,
9783 max_supported: i32,
9784 sequence: u64,
9785) -> Result<()> {
9786 if (min_supported..=max_supported).contains(&version) {
9787 return Ok(());
9788 }
9789 Err(Error::NonDeterministicReplay(ReplayFailure::new(
9790 "version_marker_incompatible_range",
9791 (sequence != 0).then_some(sequence),
9792 Some(format!("{min_supported}..={max_supported}")),
9793 Some(format!("{change_id}:{version}")),
9794 "recorded workflow version is outside the range supported by current code",
9795 )))
9796}
9797
9798#[derive(Clone, Debug)]
9799struct ResumeSignal {
9800 signal_name: String,
9801 arguments: Vec<AvroValue>,
9802}
9803
9804const MAX_PARALLEL_OPERATIONS: usize = 1000;
9805
9806fn parallel_group_prefix(kind: &str) -> &'static str {
9807 match kind {
9808 "activity" => "parallel-activities",
9809 "child" => "parallel-children",
9810 "timer" => "parallel-timers",
9811 _ => "parallel-calls",
9812 }
9813}
9814
9815fn parallel_group_entry(
9816 base_sequence: u64,
9817 size: usize,
9818 index: usize,
9819 kind: &str,
9820) -> ParallelGroupMetadata {
9821 ParallelGroupMetadata {
9822 parallel_group_id: format!("{}:{base_sequence}:{size}", parallel_group_prefix(kind)),
9823 parallel_group_kind: kind.to_string(),
9824 parallel_group_base_sequence: base_sequence,
9825 parallel_group_size: size,
9826 parallel_group_index: index,
9827 parallel_group_mode: None,
9828 selection_member_key: None,
9829 selection_member_index: None,
9830 selection_member_base_sequence: None,
9831 selection_member_size: None,
9832 selection_member_kind: None,
9833 }
9834}
9835
9836struct SelectionMemberMetadata {
9837 key: SelectionKey,
9838 index: usize,
9839 base_sequence: u64,
9840 size: usize,
9841 kind: String,
9842}
9843
9844fn selection_group_entry(
9845 base_sequence: u64,
9846 size: usize,
9847 index: usize,
9848 kind: &str,
9849 member: &SelectionMemberMetadata,
9850) -> ParallelGroupMetadata {
9851 ParallelGroupMetadata {
9852 parallel_group_id: format!("select-calls:{base_sequence}:{size}"),
9853 parallel_group_kind: kind.to_string(),
9854 parallel_group_base_sequence: base_sequence,
9855 parallel_group_size: size,
9856 parallel_group_index: index,
9857 parallel_group_mode: Some("select".to_string()),
9858 selection_member_key: Some(member.key.clone()),
9859 selection_member_index: Some(member.index),
9860 selection_member_base_sequence: Some(member.base_sequence),
9861 selection_member_size: Some(member.size),
9862 selection_member_kind: Some(member.kind.clone()),
9863 }
9864}
9865
9866fn apply_parallel_group_path(
9867 command: &mut serde_json::Map<String, Value>,
9868 path: &[ParallelGroupMetadata],
9869) {
9870 let Some(inner) = path.last() else {
9871 return;
9872 };
9873 command.insert(
9874 "parallel_group_id".to_string(),
9875 json!(inner.parallel_group_id),
9876 );
9877 command.insert(
9878 "parallel_group_kind".to_string(),
9879 json!(inner.parallel_group_kind),
9880 );
9881 command.insert(
9882 "parallel_group_base_sequence".to_string(),
9883 json!(inner.parallel_group_base_sequence),
9884 );
9885 command.insert(
9886 "parallel_group_size".to_string(),
9887 json!(inner.parallel_group_size),
9888 );
9889 command.insert(
9890 "parallel_group_index".to_string(),
9891 json!(inner.parallel_group_index),
9892 );
9893 if let Some(mode) = &inner.parallel_group_mode {
9894 command.insert("parallel_group_mode".to_string(), json!(mode));
9895 }
9896 if let Some(key) = &inner.selection_member_key {
9897 command.insert("selection_member_key".to_string(), json!(key));
9898 }
9899 if let Some(index) = inner.selection_member_index {
9900 command.insert("selection_member_index".to_string(), json!(index));
9901 }
9902 if let Some(base_sequence) = inner.selection_member_base_sequence {
9903 command.insert(
9904 "selection_member_base_sequence".to_string(),
9905 json!(base_sequence),
9906 );
9907 }
9908 if let Some(size) = inner.selection_member_size {
9909 command.insert("selection_member_size".to_string(), json!(size));
9910 }
9911 if let Some(kind) = &inner.selection_member_kind {
9912 command.insert("selection_member_kind".to_string(), json!(kind));
9913 }
9914 command.insert("parallel_group_path".to_string(), json!(path));
9915}
9916
9917fn ensure_parallel_path_matches(
9918 sequence: u64,
9919 recorded: Option<&[ParallelGroupMetadata]>,
9920 expected: &[ParallelGroupMetadata],
9921) -> Result<()> {
9922 match (recorded, expected.is_empty()) {
9923 (None, true) => Ok(()),
9924 (Some(recorded), false) if recorded == expected => Ok(()),
9925 (None, false) => Err(invalid_recorded_history(
9926 "parallel_group_metadata_missing",
9927 sequence,
9928 &serde_json::to_string(expected).unwrap_or_default(),
9929 "<missing>",
9930 "recorded parallel member is missing its durable group path",
9931 )),
9932 (Some(recorded), true) => Err(invalid_recorded_history(
9933 "parallel_group_shape_mismatch",
9934 sequence,
9935 "sequential command",
9936 &serde_json::to_string(recorded).unwrap_or_default(),
9937 "recorded command belonged to a parallel group but current code schedules it sequentially",
9938 )),
9939 (Some(recorded), false) => Err(invalid_recorded_history(
9940 "parallel_group_shape_mismatch",
9941 sequence,
9942 &serde_json::to_string(recorded).unwrap_or_default(),
9943 &serde_json::to_string(expected).unwrap_or_default(),
9944 "recorded parallel-group identity or path changed during replay",
9945 )),
9946 }
9947}
9948
9949#[derive(Clone, Debug)]
9950enum ParallelShape {
9951 Leaf,
9952 Group(Vec<ParallelShape>),
9953}
9954
9955struct ParallelDescriptor {
9956 operation: ParallelOperation,
9957 offset: usize,
9958 member_path: Vec<usize>,
9959 group_path: Vec<ParallelGroupMetadata>,
9960}
9961
9962fn parallel_leaf_count(operations: &[ParallelOperation]) -> usize {
9963 operations
9964 .iter()
9965 .map(|operation| match operation {
9966 ParallelOperation::Group(children) => parallel_leaf_count(children),
9967 _ => 1,
9968 })
9969 .sum()
9970}
9971
9972fn parallel_operation_kind(operation: &ParallelOperation) -> Option<&'static str> {
9973 match operation {
9974 ParallelOperation::Activity { .. } => Some("activity"),
9975 ParallelOperation::ChildWorkflow { .. } => Some("child"),
9976 ParallelOperation::Timer(_) => Some("timer"),
9977 ParallelOperation::Signal(_) => Some("signal"),
9978 ParallelOperation::Condition { .. } => Some("condition"),
9979 ParallelOperation::Group(children) => parallel_group_kind(children),
9980 }
9981}
9982
9983fn parallel_group_kind(operations: &[ParallelOperation]) -> Option<&'static str> {
9984 let mut kind = None;
9985 for operation in operations {
9986 let Some(operation_kind) = parallel_operation_kind(operation) else {
9987 continue;
9988 };
9989 match kind {
9990 None => kind = Some(operation_kind),
9991 Some(current) if current == operation_kind => {}
9992 Some(_) => return Some("mixed"),
9993 }
9994 }
9995 kind
9996}
9997
9998fn validate_parallel_operations(
9999 operations: &[ParallelOperation],
10000 member_path: &mut Vec<usize>,
10001 root: bool,
10002) -> Result<()> {
10003 let leaves = parallel_leaf_count(operations);
10004 if leaves > MAX_PARALLEL_OPERATIONS {
10005 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10006 reason: "fan_out_limit_exceeded",
10007 member_path: member_path.clone(),
10008 message: format!(
10009 "group contains {leaves} durable leaves; the limit is {MAX_PARALLEL_OPERATIONS}"
10010 ),
10011 }));
10012 }
10013 if !root && operations.is_empty() {
10014 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10015 reason: "nested_group_empty",
10016 member_path: member_path.clone(),
10017 message: "a nested group must contain at least one durable leaf".to_string(),
10018 }));
10019 }
10020
10021 for (index, operation) in operations.iter().enumerate() {
10022 member_path.push(index);
10023 match operation {
10024 ParallelOperation::Activity {
10025 options, arguments, ..
10026 } => {
10027 options
10028 .validate()
10029 .map_err(|error| Error::InvalidActivityOptions(error))?;
10030 if let Err(error) = arguments {
10031 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10032 reason: "arguments_invalid",
10033 member_path: member_path.clone(),
10034 message: error.to_string(),
10035 }));
10036 }
10037 }
10038 ParallelOperation::ChildWorkflow {
10039 options, arguments, ..
10040 } => {
10041 validate_parallel_child_options(options)?;
10042 if let Err(error) = arguments {
10043 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10044 reason: "arguments_invalid",
10045 member_path: member_path.clone(),
10046 message: error.to_string(),
10047 }));
10048 }
10049 }
10050 ParallelOperation::Timer(duration)
10051 if duration.as_secs() == u64::MAX && duration.subsec_nanos() > 0 =>
10052 {
10053 return Err(Error::TimerDurationOverflow);
10054 }
10055 ParallelOperation::Timer(_) => {}
10056 ParallelOperation::Signal(signal_name) => {
10057 validate_user_signal_name(signal_name)?;
10058 if signal_name.trim().is_empty() {
10059 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10060 reason: "signal_name_empty",
10061 member_path: member_path.clone(),
10062 message: "signal wait name must not be empty".to_string(),
10063 }));
10064 }
10065 }
10066 ParallelOperation::Condition { options, .. } => {
10067 options.validate()?;
10068 }
10069 ParallelOperation::Group(children) => {
10070 validate_parallel_operations(children, member_path, false)?;
10071 }
10072 }
10073 member_path.pop();
10074 }
10075 Ok(())
10076}
10077
10078fn validate_parallel_child_options(options: &ChildWorkflowOptions) -> Result<()> {
10079 if options.task_queue.trim().is_empty() {
10080 return Err(Error::InvalidChildWorkflowOptions(
10081 "task_queue must not be empty".to_string(),
10082 ));
10083 }
10084 for (name, value) in [
10085 (
10086 "execution_timeout_seconds",
10087 options.execution_timeout_seconds,
10088 ),
10089 ("run_timeout_seconds", options.run_timeout_seconds),
10090 ] {
10091 if value == Some(0) {
10092 return Err(Error::InvalidChildWorkflowOptions(format!(
10093 "{name} must be at least 1"
10094 )));
10095 }
10096 }
10097 if options
10098 .retry_policy
10099 .as_ref()
10100 .is_some_and(|policy| policy.max_attempts == Some(0))
10101 {
10102 return Err(Error::InvalidChildWorkflowOptions(
10103 "retry_policy.max_attempts must be at least 1".to_string(),
10104 ));
10105 }
10106 Ok(())
10107}
10108
10109fn parallel_shape(operations: &[ParallelOperation]) -> ParallelShape {
10110 ParallelShape::Group(
10111 operations
10112 .iter()
10113 .map(|operation| match operation {
10114 ParallelOperation::Group(children) => parallel_shape(children),
10115 _ => ParallelShape::Leaf,
10116 })
10117 .collect(),
10118 )
10119}
10120
10121fn parallel_descriptors(
10122 operations: Vec<ParallelOperation>,
10123 base_sequence: u64,
10124) -> Result<Vec<ParallelDescriptor>> {
10125 let size = parallel_leaf_count(&operations);
10126 let kind = parallel_group_kind(&operations).unwrap_or("activity");
10127 let mut descriptors = Vec::with_capacity(size);
10128 let mut cursor = 0;
10129
10130 for (index, operation) in operations.into_iter().enumerate() {
10131 match operation {
10132 ParallelOperation::Group(children) => {
10133 let child_base = base_sequence
10134 .checked_add(u64::try_from(cursor).unwrap_or(u64::MAX))
10135 .ok_or(Error::TimerDurationOverflow)?;
10136 for mut descriptor in parallel_descriptors(children, child_base)? {
10137 let outer_index = cursor + descriptor.offset;
10138 descriptor.group_path.insert(
10139 0,
10140 parallel_group_entry(base_sequence, size, outer_index, kind),
10141 );
10142 descriptor.member_path.insert(0, index);
10143 descriptor.offset = outer_index;
10144 descriptors.push(descriptor);
10145 }
10146 cursor = descriptors.len();
10147 }
10148 operation => {
10149 descriptors.push(ParallelDescriptor {
10150 operation,
10151 offset: cursor,
10152 member_path: vec![index],
10153 group_path: vec![parallel_group_entry(base_sequence, size, cursor, kind)],
10154 });
10155 cursor += 1;
10156 }
10157 }
10158 }
10159 Ok(descriptors)
10160}
10161
10162enum ParallelLeafCall {
10163 Activity(ActivityCall),
10164 ChildWorkflow(ChildWorkflowCall),
10165 Timer(TimerCall),
10166 Signal(SignalCall),
10167 Condition(ConditionWaitCall),
10168}
10169
10170fn parallel_leaf_call(
10171 ctx: &WorkflowContext,
10172 operation: ParallelOperation,
10173 parallel_group_path: Vec<ParallelGroupMetadata>,
10174) -> ParallelLeafCall {
10175 match operation {
10176 ParallelOperation::Activity {
10177 activity_type,
10178 options,
10179 arguments,
10180 } => ParallelLeafCall::Activity(ActivityCall {
10181 ctx: ctx.clone(),
10182 activity_type,
10183 options,
10184 args: Some(arguments),
10185 scheduled: false,
10186 parallel_group_path,
10187 }),
10188 ParallelOperation::ChildWorkflow {
10189 workflow_type,
10190 options,
10191 arguments,
10192 } => ParallelLeafCall::ChildWorkflow(ChildWorkflowCall {
10193 ctx: ctx.clone(),
10194 workflow_type,
10195 options,
10196 args: Some(arguments),
10197 scheduled: false,
10198 matched_pending: false,
10199 parallel_group_path,
10200 }),
10201 ParallelOperation::Timer(duration) => {
10202 let delay_seconds = duration
10203 .as_secs()
10204 .checked_add(u64::from(duration.subsec_nanos() > 0));
10205 ParallelLeafCall::Timer(TimerCall {
10206 ctx: ctx.clone(),
10207 delay_seconds,
10208 scheduled: false,
10209 matched_pending: false,
10210 parallel_group_path,
10211 })
10212 }
10213 ParallelOperation::Signal(signal_name) => ParallelLeafCall::Signal(SignalCall {
10214 ctx: ctx.clone(),
10215 signal_name,
10216 runtime_reserved_allowed: false,
10217 opened_wait: false,
10218 matched_pending: false,
10219 parallel_group_path,
10220 }),
10221 ParallelOperation::Condition { options, predicate } => {
10222 ParallelLeafCall::Condition(ConditionWaitCall {
10223 ctx: ctx.clone(),
10224 options,
10225 predicate,
10226 occurrence_id: None,
10227 opened_wait: false,
10228 parallel_group_path,
10229 })
10230 }
10231 ParallelOperation::Group(_) => {
10232 unreachable!("parallel descriptors contain only durable leaves")
10233 }
10234 }
10235}
10236
10237impl ParallelLeafCall {
10238 fn poll_avro_value(&mut self, cx: &mut TaskContext<'_>) -> Poll<Result<ParallelAvroResult>> {
10239 match self {
10240 Self::Activity(call) => Pin::new(call)
10241 .poll_avro_value(cx)
10242 .map_ok(ParallelAvroResult::Activity),
10243 Self::ChildWorkflow(call) => Pin::new(call)
10244 .poll_avro_value(cx)
10245 .map_ok(ParallelAvroResult::ChildWorkflow),
10246 Self::Timer(call) => Pin::new(call)
10247 .poll(cx)
10248 .map_ok(|()| ParallelAvroResult::Timer),
10249 Self::Signal(call) => Pin::new(call)
10250 .poll_avro_value(cx)
10251 .map_ok(ParallelAvroResult::Signal),
10252 Self::Condition(call) => Pin::new(call)
10253 .poll(cx)
10254 .map_ok(ParallelAvroResult::Condition),
10255 }
10256 }
10257}
10258
10259struct ParallelLeaf {
10260 call: ParallelLeafCall,
10261 member_path: Vec<usize>,
10262 group_path: Vec<ParallelGroupMetadata>,
10263 result: Option<ParallelAvroResult>,
10264}
10265
10266pub struct ParallelCall {
10268 ctx: WorkflowContext,
10269 operations: Option<Vec<ParallelOperation>>,
10270 shape: Option<ParallelShape>,
10271 leaves: Vec<ParallelLeaf>,
10272}
10273
10274impl ParallelCall {
10275 fn new(ctx: WorkflowContext, operations: Vec<ParallelOperation>) -> Self {
10276 Self {
10277 ctx,
10278 operations: Some(operations),
10279 shape: None,
10280 leaves: Vec::new(),
10281 }
10282 }
10283
10284 fn initialize(&mut self) -> Result<()> {
10285 let operations = self.operations.take().unwrap_or_default();
10286 validate_parallel_operations(&operations, &mut Vec::new(), true)?;
10287 self.shape = Some(parallel_shape(&operations));
10288 if operations.is_empty() {
10289 return Ok(());
10290 }
10291
10292 let base_sequence = {
10293 let state = self
10294 .ctx
10295 .state
10296 .lock()
10297 .map_err(|_| Error::WorkflowStatePoisoned)?;
10298 if let Some(recorded) = state.recorded_commands.get(state.command_cursor) {
10299 recorded.sequence()
10300 } else {
10301 let last = state
10302 .recorded_commands
10303 .last()
10304 .map(RecordedCommand::sequence)
10305 .unwrap_or(0);
10306 last.checked_add(u64::try_from(state.commands.len()).unwrap_or(u64::MAX))
10307 .and_then(|sequence| sequence.checked_add(1))
10308 .ok_or_else(|| {
10309 Error::InvalidParallelGroup(ParallelGroupError {
10310 reason: "sequence_overflow",
10311 member_path: Vec::new(),
10312 message: "parallel group sequence identity overflowed u64".to_string(),
10313 })
10314 })?
10315 }
10316 };
10317
10318 self.leaves = parallel_descriptors(operations, base_sequence)?
10319 .into_iter()
10320 .map(|descriptor| {
10321 let call = parallel_leaf_call(
10322 &self.ctx,
10323 descriptor.operation,
10324 descriptor.group_path.clone(),
10325 );
10326 ParallelLeaf {
10327 call,
10328 member_path: descriptor.member_path,
10329 group_path: descriptor.group_path,
10330 result: None,
10331 }
10332 })
10333 .collect();
10334 Ok(())
10335 }
10336
10337 fn poll_avro_value(
10338 mut self: Pin<&mut Self>,
10339 cx: &mut TaskContext<'_>,
10340 ) -> Poll<Result<Vec<ParallelAvroResult>>> {
10341 if self.operations.is_some() {
10342 if let Err(error) = self.initialize() {
10343 return Poll::Ready(Err(error));
10344 }
10345 }
10346 if self.leaves.is_empty() {
10347 return Poll::Ready(Ok(Vec::new()));
10348 }
10349
10350 let mut failures = Vec::new();
10351 let mut pending = false;
10352 for (index, leaf) in self.leaves.iter_mut().enumerate() {
10353 if leaf.result.is_some() {
10354 continue;
10355 }
10356 match leaf.call.poll_avro_value(cx) {
10357 Poll::Ready(Ok(result)) => leaf.result = Some(result),
10358 Poll::Ready(Err(error)) => failures.push((index, error)),
10359 Poll::Pending => pending = true,
10360 }
10361 }
10362
10363 if !failures.is_empty() {
10364 if let Some(position) = failures
10365 .iter()
10366 .position(|(_, error)| workflow_task_integrity_error(error))
10367 {
10368 return Poll::Ready(Err(failures.remove(position).1));
10369 }
10370 failures.sort_by_key(|(index, _)| *index);
10371 let (failed_index, cause) = failures.remove(0);
10372 let failed = &self.leaves[failed_index];
10373 let completed = self
10374 .leaves
10375 .iter()
10376 .filter_map(|leaf| {
10377 leaf.result
10378 .clone()
10379 .and_then(|result| result.into_json_result().ok())
10380 .map(|result| ParallelCompletion {
10381 member_path: leaf.member_path.clone(),
10382 result,
10383 })
10384 })
10385 .collect();
10386 let group_id = failed
10387 .group_path
10388 .first()
10389 .map(|entry| entry.parallel_group_id.clone())
10390 .unwrap_or_default();
10391 return Poll::Ready(Err(Error::ParallelFailed(ParallelFailure {
10392 group_id,
10393 member_path: failed.member_path.clone(),
10394 group_path: failed.group_path.clone(),
10395 completed,
10396 cause: Box::new(cause),
10397 })));
10398 }
10399 if pending {
10400 return Poll::Pending;
10401 }
10402
10403 let mut flat_results = self
10404 .leaves
10405 .iter_mut()
10406 .map(|leaf| leaf.result.take().expect("completed parallel leaf"))
10407 .collect::<Vec<_>>()
10408 .into_iter();
10409 let results = parallel_results_for_shape(
10410 self.shape.as_ref().expect("initialized parallel shape"),
10411 &mut flat_results,
10412 );
10413 Poll::Ready(Ok(match results {
10414 ParallelAvroResult::Group(results) => results,
10415 ParallelAvroResult::Activity(_)
10416 | ParallelAvroResult::ChildWorkflow(_)
10417 | ParallelAvroResult::Timer
10418 | ParallelAvroResult::Signal(_)
10419 | ParallelAvroResult::Condition(_) => {
10420 unreachable!("root parallel shape is a group")
10421 }
10422 }))
10423 }
10424}
10425
10426fn parallel_results_for_shape(
10427 shape: &ParallelShape,
10428 flat_results: &mut impl Iterator<Item = ParallelAvroResult>,
10429) -> ParallelAvroResult {
10430 match shape {
10431 ParallelShape::Leaf => flat_results.next().expect("one result per parallel leaf"),
10432 ParallelShape::Group(children) => ParallelAvroResult::Group(
10433 children
10434 .iter()
10435 .map(|child| parallel_results_for_shape(child, flat_results))
10436 .collect(),
10437 ),
10438 }
10439}
10440
10441impl Future for ParallelCall {
10442 type Output = Result<Vec<ParallelResult>>;
10443
10444 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
10445 self.poll_avro_value(cx)
10446 .map_ok(|results| {
10447 results
10448 .into_iter()
10449 .map(ParallelAvroResult::into_json_result)
10450 .collect::<Result<Vec<_>>>()
10451 })
10452 .map_ok(|result| result)
10453 .flatten_result()
10454 }
10455}
10456
10457#[derive(Clone, Debug)]
10458struct SelectionMemberPlan {
10459 key: SelectionKey,
10460 index: usize,
10461 base_sequence: u64,
10462 size: usize,
10463 kind: String,
10464 shape: ParallelShape,
10465 leaf_start: usize,
10466}
10467
10468fn selection_operation_kind(operation: &ParallelOperation) -> &'static str {
10469 match operation {
10470 ParallelOperation::Activity { .. } => "activity",
10471 ParallelOperation::ChildWorkflow { .. } => "child",
10472 ParallelOperation::Timer(_) => "timer",
10473 ParallelOperation::Signal(_) => "signal",
10474 ParallelOperation::Condition { .. } => "condition",
10475 ParallelOperation::Group(_) => "group",
10476 }
10477}
10478
10479fn selection_operation_shape(operation: &ParallelOperation) -> ParallelShape {
10480 match operation {
10481 ParallelOperation::Group(children) => parallel_shape(children),
10482 _ => ParallelShape::Leaf,
10483 }
10484}
10485
10486fn selection_descriptors(
10487 operations: Vec<(SelectionKey, ParallelOperation)>,
10488 base_sequence: u64,
10489) -> Result<(Vec<ParallelDescriptor>, Vec<SelectionMemberPlan>)> {
10490 if operations.is_empty() {
10491 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10492 reason: "selection_empty",
10493 member_path: Vec::new(),
10494 message: "durable selection requires at least one operation".to_string(),
10495 }));
10496 }
10497 let operation_refs = operations
10498 .iter()
10499 .map(|(_, operation)| operation)
10500 .collect::<Vec<_>>();
10501 let total_size = operation_refs
10502 .iter()
10503 .map(|operation| match operation {
10504 ParallelOperation::Group(children) => parallel_leaf_count(children),
10505 _ => 1,
10506 })
10507 .sum::<usize>();
10508 if total_size > MAX_PARALLEL_OPERATIONS {
10509 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10510 reason: "fan_out_limit_exceeded",
10511 member_path: Vec::new(),
10512 message: format!(
10513 "selection contains {total_size} durable leaves; the limit is {MAX_PARALLEL_OPERATIONS}"
10514 ),
10515 }));
10516 }
10517 let group_kind = {
10518 let mut kind = None;
10519 for operation in &operation_refs {
10520 let operation_kind = parallel_operation_kind(operation).unwrap_or("mixed");
10521 match kind {
10522 None => kind = Some(operation_kind),
10523 Some(current) if current == operation_kind => {}
10524 Some(_) => {
10525 kind = Some("mixed");
10526 break;
10527 }
10528 }
10529 }
10530 kind.unwrap_or("mixed")
10531 };
10532
10533 let mut descriptors = Vec::with_capacity(total_size);
10534 let mut members = Vec::with_capacity(operations.len());
10535 let mut cursor = 0usize;
10536 let mut seen_keys: Vec<SelectionKey> = Vec::new();
10537 for (member_index, (key, operation)) in operations.into_iter().enumerate() {
10538 if matches!(&key, SelectionKey::Name(value) if value.is_empty()) {
10539 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10540 reason: "selection_key_invalid",
10541 member_path: vec![member_index],
10542 message: "selection member keys must be non-empty strings or non-negative integers"
10543 .to_string(),
10544 }));
10545 }
10546 if seen_keys.contains(&key) {
10547 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10548 reason: "selection_key_duplicate",
10549 member_path: vec![member_index],
10550 message: format!("selection member key {key:?} is duplicated"),
10551 }));
10552 }
10553 seen_keys.push(key.clone());
10554 let member_size = match &operation {
10555 ParallelOperation::Group(children) => parallel_leaf_count(children),
10556 _ => 1,
10557 };
10558 if member_size == 0 {
10559 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10560 reason: "selection_member_empty",
10561 member_path: vec![member_index],
10562 message: "a selection member must contain at least one durable leaf".to_string(),
10563 }));
10564 }
10565 let member_base = base_sequence
10566 .checked_add(u64::try_from(cursor).unwrap_or(u64::MAX))
10567 .ok_or(Error::TimerDurationOverflow)?;
10568 let member_kind = selection_operation_kind(&operation).to_string();
10569 let member_shape = selection_operation_shape(&operation);
10570 let leaf_start = descriptors.len();
10571 match operation {
10572 ParallelOperation::Group(children) => {
10573 validate_parallel_operations(&children, &mut vec![member_index], false)?;
10574 for mut descriptor in parallel_descriptors(children, member_base)? {
10575 let flat_index = cursor + descriptor.offset;
10576 descriptor.group_path.insert(
10577 0,
10578 selection_group_entry(
10579 base_sequence,
10580 total_size,
10581 flat_index,
10582 group_kind,
10583 &SelectionMemberMetadata {
10584 key: key.clone(),
10585 index: member_index,
10586 base_sequence: member_base,
10587 size: member_size,
10588 kind: member_kind.clone(),
10589 },
10590 ),
10591 );
10592 descriptor.member_path.insert(0, member_index);
10593 descriptor.offset = flat_index;
10594 descriptors.push(descriptor);
10595 }
10596 }
10597 operation => {
10598 validate_parallel_operations(
10599 std::slice::from_ref(&operation),
10600 &mut Vec::new(),
10601 true,
10602 )?;
10603 descriptors.push(ParallelDescriptor {
10604 operation,
10605 offset: cursor,
10606 member_path: vec![member_index],
10607 group_path: vec![selection_group_entry(
10608 base_sequence,
10609 total_size,
10610 cursor,
10611 group_kind,
10612 &SelectionMemberMetadata {
10613 key: key.clone(),
10614 index: member_index,
10615 base_sequence: member_base,
10616 size: member_size,
10617 kind: member_kind.clone(),
10618 },
10619 )],
10620 });
10621 }
10622 }
10623 members.push(SelectionMemberPlan {
10624 key,
10625 index: member_index,
10626 base_sequence: member_base,
10627 size: member_size,
10628 kind: member_kind,
10629 shape: member_shape,
10630 leaf_start,
10631 });
10632 cursor += member_size;
10633 }
10634 Ok((descriptors, members))
10635}
10636
10637struct SelectionLeaf {
10638 call: ParallelLeafCall,
10639 outcome: Option<Result<ParallelAvroResult>>,
10640}
10641
10642#[derive(Clone)]
10644pub struct DurableOperationHandle {
10645 ctx: WorkflowContext,
10646 pub key: SelectionKey,
10647 pub index: usize,
10648 pub kind: String,
10649 pub identity: String,
10650 pub base_sequence: u64,
10651 pub size: usize,
10652 pub selection_group_id: String,
10653 shape: ParallelShape,
10654}
10655
10656impl std::fmt::Debug for DurableOperationHandle {
10657 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10658 formatter
10659 .debug_struct("DurableOperationHandle")
10660 .field("key", &self.key)
10661 .field("index", &self.index)
10662 .field("kind", &self.kind)
10663 .field("identity", &self.identity)
10664 .field("base_sequence", &self.base_sequence)
10665 .field("size", &self.size)
10666 .field("selection_group_id", &self.selection_group_id)
10667 .finish()
10668 }
10669}
10670
10671impl DurableOperationHandle {
10672 pub fn await_result(&self) -> DurableOperationAwaitCall {
10674 DurableOperationAwaitCall {
10675 handle: self.clone(),
10676 }
10677 }
10678
10679 pub fn cancel(&self) -> CancelDurableOperationCall {
10683 CancelDurableOperationCall {
10684 handle: self.clone(),
10685 emitted: false,
10686 }
10687 }
10688}
10689
10690#[derive(Debug)]
10692pub struct SelectionResult {
10693 pub key: SelectionKey,
10694 pub index: usize,
10695 pub kind: String,
10696 pub identity: String,
10697 pub value: Option<ParallelResult>,
10698 pub failure: Option<Error>,
10699 pub winner: DurableOperationHandle,
10700 pub handles: Vec<DurableOperationHandle>,
10701}
10702
10703impl SelectionResult {
10704 pub fn succeeded(&self) -> bool {
10705 self.failure.is_none()
10706 }
10707
10708 pub fn handle(&self, key: &SelectionKey) -> Option<&DurableOperationHandle> {
10709 self.handles.iter().find(|handle| &handle.key == key)
10710 }
10711
10712 pub fn remaining(&self) -> Vec<&DurableOperationHandle> {
10713 self.handles
10714 .iter()
10715 .filter(|handle| handle.index != self.index)
10716 .collect()
10717 }
10718
10719 pub fn into_result(self) -> Result<ParallelResult> {
10720 match (self.value, self.failure) {
10721 (Some(value), None) => Ok(value),
10722 (_, Some(error)) => Err(error),
10723 _ => Err(Error::WorkerLoop(
10724 "selection result contained neither a value nor a failure".to_string(),
10725 )),
10726 }
10727 }
10728}
10729
10730pub struct SelectCall {
10732 ctx: WorkflowContext,
10733 operations: Option<Vec<(SelectionKey, ParallelOperation)>>,
10734 members: Vec<SelectionMemberPlan>,
10735 leaves: Vec<SelectionLeaf>,
10736 group_id: Option<String>,
10737}
10738
10739impl SelectCall {
10740 fn new(ctx: WorkflowContext, operations: Vec<(SelectionKey, ParallelOperation)>) -> Self {
10741 Self {
10742 ctx,
10743 operations: Some(operations),
10744 members: Vec::new(),
10745 leaves: Vec::new(),
10746 group_id: None,
10747 }
10748 }
10749
10750 fn initialize(&mut self) -> Result<()> {
10751 let operations = self.operations.take().unwrap_or_default();
10752 let base_sequence = {
10753 let state = self
10754 .ctx
10755 .state
10756 .lock()
10757 .map_err(|_| Error::WorkflowStatePoisoned)?;
10758 if let Some(marker) = state.selection_markers.get(state.selection_marker_cursor) {
10759 marker.selection_group_base_sequence
10760 } else if let Some(recorded) = state.recorded_commands.get(state.command_cursor) {
10761 recorded.sequence()
10762 } else {
10763 let last = state
10764 .recorded_commands
10765 .last()
10766 .map(RecordedCommand::sequence)
10767 .unwrap_or(0);
10768 last.checked_add(u64::try_from(state.commands.len()).unwrap_or(u64::MAX))
10769 .and_then(|sequence| sequence.checked_add(1))
10770 .ok_or(Error::TimerDurationOverflow)?
10771 }
10772 };
10773 let (descriptors, members) = selection_descriptors(operations, base_sequence)?;
10774 let group_id = format!("select-calls:{base_sequence}:{}", descriptors.len());
10775 self.leaves = descriptors
10776 .into_iter()
10777 .map(|descriptor| SelectionLeaf {
10778 call: parallel_leaf_call(&self.ctx, descriptor.operation, descriptor.group_path),
10779 outcome: None,
10780 })
10781 .collect();
10782 self.members = members;
10783 self.group_id = Some(group_id);
10784 Ok(())
10785 }
10786}
10787
10788impl Future for SelectCall {
10789 type Output = Result<SelectionResult>;
10790
10791 fn poll(mut self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
10792 if self.operations.is_some() {
10793 if let Err(error) = self.initialize() {
10794 return Poll::Ready(Err(error));
10795 }
10796 }
10797
10798 for leaf in &mut self.leaves {
10799 if leaf.outcome.is_some() {
10800 continue;
10801 }
10802 if let Poll::Ready(outcome) = leaf.call.poll_avro_value(cx) {
10803 if outcome
10804 .as_ref()
10805 .err()
10806 .is_some_and(workflow_task_integrity_error)
10807 {
10808 return Poll::Ready(outcome.map(|_| unreachable!()));
10809 }
10810 leaf.outcome = Some(outcome);
10811 }
10812 }
10813
10814 let all_members_terminal = self.leaves.iter().all(|leaf| leaf.outcome.is_some());
10815 let selection_member_range = self
10816 .members
10817 .first()
10818 .map(|member| member.base_sequence)
10819 .zip(self.leaves.len().try_into().ok())
10820 .map(|(base_sequence, size): (u64, u64)| {
10821 base_sequence..base_sequence.saturating_add(size)
10822 });
10823 let marker = {
10824 let mut state = match self.ctx.state.lock() {
10825 Ok(state) => state,
10826 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
10827 };
10828 let marker = state
10829 .selection_markers
10830 .get(state.selection_marker_cursor)
10831 .cloned();
10832 if marker.is_none()
10833 && all_members_terminal
10834 && selection_member_range.as_ref().is_some_and(|member_range| {
10835 state
10836 .recorded_commands
10837 .iter()
10838 .any(|command| member_range.contains(&command.sequence()))
10839 })
10840 {
10841 state.matched_recorded_pending = true;
10847 }
10848 marker
10849 };
10850 let Some(marker) = marker else {
10851 return Poll::Pending;
10852 };
10853 if self.group_id.as_deref() != Some(marker.selection_group_id.as_str())
10854 || marker.selection_group_size != self.leaves.len()
10855 || self.members.first().map(|member| member.base_sequence)
10856 != Some(marker.selection_group_base_sequence)
10857 {
10858 return Poll::Ready(Err(invalid_recorded_history(
10859 "selection_group_shape_mismatch",
10860 marker.selection_group_base_sequence,
10861 self.group_id
10862 .as_deref()
10863 .unwrap_or("initialized selection group"),
10864 &marker.selection_group_id,
10865 "recorded selection group differs from current workflow code",
10866 )));
10867 }
10868 let Some(member_position) = self.members.iter().position(|member| {
10869 member.key == marker.member_key
10870 && member.index == marker.member_index
10871 && member.base_sequence == marker.member_base_sequence
10872 && member.size == marker.member_size
10873 && member.kind == marker.operation_kind
10874 }) else {
10875 return Poll::Ready(Err(invalid_recorded_history(
10876 "selection_member_shape_mismatch",
10877 marker.member_base_sequence,
10878 "winner member matching current workflow code",
10879 &format!("{:?}", marker.member_key),
10880 "recorded selection winner differs from the authored member identity",
10881 )));
10882 };
10883 let member = self.members[member_position].clone();
10884 let (handles, resolution_sequence) = {
10885 let mut state = match self.ctx.state.lock() {
10886 Ok(state) => state,
10887 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
10888 };
10889 let identities = self
10890 .members
10891 .iter()
10892 .map(|candidate| {
10893 selection_operation_identity(
10894 &state,
10895 &candidate.kind,
10896 candidate.base_sequence,
10897 candidate.size,
10898 )
10899 })
10900 .collect::<Vec<_>>();
10901 if let Some((position, missing)) = identities
10902 .iter()
10903 .enumerate()
10904 .find(|(_, identity)| identity.is_empty())
10905 .map(|(position, identity)| (position, identity.clone()))
10906 {
10907 let candidate = &self.members[position];
10908 return Poll::Ready(Err(invalid_recorded_history(
10909 "selection_operation_identity_missing",
10910 candidate.base_sequence,
10911 &format!(
10912 "durable {} resource identity from scheduled/open history",
10913 candidate.kind
10914 ),
10915 &missing,
10916 "selection member history is missing its canonical durable identity",
10917 )));
10918 }
10919 let expected_winner_identity = &identities[member_position];
10920 let resolution_sequence = match validated_selection_resolution_sequence(
10921 &state,
10922 &marker,
10923 &member,
10924 expected_winner_identity,
10925 ) {
10926 Ok(sequence) => sequence,
10927 Err(error) => return Poll::Ready(Err(error)),
10928 };
10929 let handles = self
10930 .members
10931 .iter()
10932 .zip(identities)
10933 .map(|(member, identity)| DurableOperationHandle {
10934 ctx: self.ctx.clone(),
10935 key: member.key.clone(),
10936 index: member.index,
10937 kind: member.kind.clone(),
10938 identity,
10939 base_sequence: member.base_sequence,
10940 size: member.size,
10941 selection_group_id: marker.selection_group_id.clone(),
10942 shape: member.shape.clone(),
10943 })
10944 .collect::<Vec<_>>();
10945 if let Err(error) = validate_selection_cancellations_for_handles(&state, &handles) {
10946 return Poll::Ready(Err(error));
10947 }
10948 state.selection_marker_cursor += 1;
10949 (handles, resolution_sequence)
10950 };
10951
10952 let mut winner_failure = None;
10953 let mut flat_results = Vec::with_capacity(member.size);
10954 if marker.outcome == "failed" {
10955 let resolution_offset = match resolution_sequence
10956 .checked_sub(member.base_sequence)
10957 .and_then(|offset| usize::try_from(offset).ok())
10958 {
10959 Some(offset) if offset < member.size => offset,
10960 _ => {
10961 return Poll::Ready(Err(invalid_recorded_history(
10962 "selection_resolution_event_mismatch",
10963 member.base_sequence,
10964 "failure event within selected member bounds",
10965 &resolution_sequence.to_string(),
10966 "selection failure event is outside the authored member",
10967 )))
10968 }
10969 };
10970 let leaf = &mut self.leaves[member.leaf_start + resolution_offset];
10971 match leaf.outcome.take() {
10972 Some(Err(error)) => winner_failure = Some(error),
10973 _ => {
10974 return Poll::Ready(Err(invalid_recorded_history(
10975 "selection_winner_outcome_mismatch",
10976 member.base_sequence,
10977 "exact failed terminal history referenced by SelectionResolved",
10978 "missing or successful resolution event",
10979 "selection winner marker disagrees with terminal operation history",
10980 )))
10981 }
10982 }
10983 } else {
10984 for leaf in &mut self.leaves[member.leaf_start..member.leaf_start + member.size] {
10985 match leaf.outcome.take() {
10986 Some(Ok(result)) => flat_results.push(result),
10987 Some(Err(_)) => {
10988 return Poll::Ready(Err(invalid_recorded_history(
10989 "selection_winner_outcome_mismatch",
10990 member.base_sequence,
10991 "fully completed nested selection member",
10992 "failed durable leaf",
10993 "completed selection winner contains a failed leaf",
10994 )))
10995 }
10996 None => {
10997 return Poll::Ready(Err(invalid_recorded_history(
10998 "selection_winner_unresolved",
10999 member.base_sequence,
11000 "terminal history for every completed winner leaf",
11001 "pending member history",
11002 "completed SelectionResolved member has an unfinished durable barrier",
11003 )))
11004 }
11005 }
11006 }
11007 }
11008 let value = if winner_failure.is_none() {
11009 let mut flat_results = flat_results.into_iter();
11010 let value = parallel_results_for_shape(&member.shape, &mut flat_results);
11011 match value.into_json_result() {
11012 Ok(value) => Some(value),
11013 Err(error) => return Poll::Ready(Err(error)),
11014 }
11015 } else {
11016 None
11017 };
11018 let winner = handles[member_position].clone();
11019 Poll::Ready(Ok(SelectionResult {
11020 key: winner.key.clone(),
11021 index: winner.index,
11022 kind: winner.kind.clone(),
11023 identity: winner.identity.clone(),
11024 value,
11025 failure: winner_failure,
11026 winner,
11027 handles,
11028 }))
11029 }
11030}
11031
11032fn selection_operation_identity(
11033 state: &WorkflowState,
11034 kind: &str,
11035 base_sequence: u64,
11036 size: usize,
11037) -> String {
11038 if kind == "group" {
11039 return format!("group:{base_sequence}:{size}");
11040 }
11041 let fields: &[&str] = match kind {
11042 "activity" => &["activity_execution_id"],
11043 "child" => &["child_workflow_run_id"],
11044 "timer" => &["timer_id"],
11045 "signal" => &["signal_wait_id"],
11046 "condition" => &["condition_wait_id"],
11047 _ => &[],
11048 };
11049 for sequence in base_sequence..base_sequence.saturating_add(size as u64) {
11050 for event in state
11051 .history_events
11052 .iter()
11053 .filter(|event| durable_event_sequence(event) == Some(sequence))
11054 {
11055 for field in fields {
11056 if let Some(identity) = event.payload.get(*field).and_then(Value::as_str) {
11057 if !identity.is_empty() {
11058 return identity.to_string();
11059 }
11060 }
11061 }
11062 }
11063 }
11064 String::new()
11065}
11066
11067fn validated_selection_resolution_sequence(
11068 state: &WorkflowState,
11069 marker: &SelectionMarker,
11070 member: &SelectionMemberPlan,
11071 expected_identity: &str,
11072) -> Result<u64> {
11073 if expected_identity.is_empty() {
11074 return Err(invalid_recorded_history(
11075 "selection_operation_identity_missing",
11076 member.base_sequence,
11077 &format!(
11078 "durable {} resource identity from scheduled/open history",
11079 member.kind
11080 ),
11081 "missing operation identity",
11082 "selection member history is missing its canonical durable identity",
11083 ));
11084 }
11085 if marker.operation_identity != expected_identity {
11086 return Err(invalid_recorded_history(
11087 "selection_operation_identity_mismatch",
11088 member.base_sequence,
11089 expected_identity,
11090 &marker.operation_identity,
11091 "selection winner identity does not match durable scheduled/open history",
11092 ));
11093 }
11094
11095 let failure_types = [
11096 "ActivityFailed",
11097 "ActivityCancelled",
11098 "ActivityTimedOut",
11099 "ChildRunFailed",
11100 "ChildRunCancelled",
11101 "ChildRunTerminated",
11102 ];
11103 let success_types = [
11104 "ActivityCompleted",
11105 "ChildRunCompleted",
11106 "TimerFired",
11107 "SignalApplied",
11108 "ConditionWaitSatisfied",
11109 "ConditionWaitTimedOut",
11110 ];
11111 let terminal_types: &[&str] = if marker.outcome == "failed" {
11112 &failure_types
11113 } else {
11114 &success_types
11115 };
11116 let mut candidates = Vec::new();
11117 for event in state.history_events.iter() {
11118 let Some(sequence) = durable_event_sequence(event) else {
11119 continue;
11120 };
11121 if sequence < member.base_sequence
11122 || sequence >= member.base_sequence.saturating_add(member.size as u64)
11123 || !terminal_types.contains(&event.event_type.as_str())
11124 {
11125 continue;
11126 }
11127 let event_id = event
11128 .raw
11129 .get("id")
11130 .or_else(|| event.raw.get("event_id"))
11131 .and_then(Value::as_str)
11132 .filter(|value| !value.is_empty())
11133 .ok_or_else(|| {
11134 invalid_recorded_history(
11135 "selection_resolution_event_id_missing",
11136 member.base_sequence,
11137 "terminal selection history with a durable event id",
11138 &event.payload.to_string(),
11139 "selection terminal history cannot be bound to its winner marker",
11140 )
11141 })?;
11142 candidates.push((event_id.to_string(), event.event_type.clone(), sequence));
11143 }
11144 let resolution = if marker.outcome == "failed" {
11145 candidates.first()
11146 } else {
11147 candidates.last()
11148 };
11149 let Some((event_id, event_type, sequence)) = resolution else {
11150 return Err(invalid_recorded_history(
11151 "selection_resolution_event_missing",
11152 member.base_sequence,
11153 "terminal history for the selected member",
11154 &format!("{:?}", marker.member_key),
11155 "selection winner marker has no matching durable terminal event",
11156 ));
11157 };
11158 if event_id != &marker.resolution_event_id || event_type != &marker.resolution_event_type {
11159 return Err(invalid_recorded_history(
11160 "selection_resolution_event_mismatch",
11161 member.base_sequence,
11162 &format!("{event_type}:{event_id}"),
11163 &format!(
11164 "{}:{}",
11165 marker.resolution_event_type, marker.resolution_event_id
11166 ),
11167 "selection winner marker does not reference the event that made its member terminal",
11168 ));
11169 }
11170 Ok(*sequence)
11171}
11172
11173fn recorded_selection_member_outcome(
11174 state: &WorkflowState,
11175 handle: &DurableOperationHandle,
11176) -> Result<Option<ParallelResult>> {
11177 for event in state.history_events.iter() {
11178 let Some(sequence) = durable_event_sequence(event) else {
11179 continue;
11180 };
11181 if sequence < handle.base_sequence
11182 || sequence >= handle.base_sequence.saturating_add(handle.size as u64)
11183 || !matches!(
11184 event.event_type.as_str(),
11185 "ActivityFailed"
11186 | "ActivityCancelled"
11187 | "ActivityTimedOut"
11188 | "ChildRunFailed"
11189 | "ChildRunCancelled"
11190 | "ChildRunTerminated"
11191 )
11192 {
11193 continue;
11194 }
11195 let Some(command) = state
11196 .recorded_commands
11197 .iter()
11198 .find(|command| command.sequence() == sequence)
11199 else {
11200 continue;
11201 };
11202 match command {
11203 RecordedCommand::Activity {
11204 outcome: Some(Err(failure)),
11205 ..
11206 } => return Err(Error::ActivityFailed(failure.clone())),
11207 RecordedCommand::ChildWorkflow {
11208 outcome: Some(Err(failure)),
11209 ..
11210 } => return Err(Error::ChildWorkflowFailed(failure.clone())),
11211 _ => {}
11212 }
11213 }
11214
11215 let mut results = Vec::with_capacity(handle.size);
11216 for sequence in handle.base_sequence..handle.base_sequence.saturating_add(handle.size as u64) {
11217 let Some(command) = state
11218 .recorded_commands
11219 .iter()
11220 .find(|command| command.sequence() == sequence)
11221 else {
11222 return Ok(None);
11223 };
11224 let result = match command {
11225 RecordedCommand::Activity { outcome, .. } => match outcome {
11226 Some(Ok(value)) => ParallelAvroResult::Activity(value.clone()),
11227 Some(Err(failure)) => return Err(Error::ActivityFailed(failure.clone())),
11228 None => return Ok(None),
11229 },
11230 RecordedCommand::Timer { fired, .. } => {
11231 if !fired {
11232 return Ok(None);
11233 }
11234 ParallelAvroResult::Timer
11235 }
11236 RecordedCommand::ChildWorkflow { outcome, .. } => match outcome {
11237 Some(Ok(value)) => ParallelAvroResult::ChildWorkflow(value.clone()),
11238 Some(Err(failure)) => return Err(Error::ChildWorkflowFailed(failure.clone())),
11239 None => return Ok(None),
11240 },
11241 RecordedCommand::SignalWait { value, .. } => match value {
11242 Some(value) => ParallelAvroResult::Signal(value.clone()),
11243 None => return Ok(None),
11244 },
11245 RecordedCommand::ConditionWait { result, .. } => match result {
11246 Some(result) => ParallelAvroResult::Condition(*result),
11247 None => return Ok(None),
11248 },
11249 other => {
11250 return Err(command_mismatch(
11251 other,
11252 format!("selected {} member", handle.kind),
11253 ))
11254 }
11255 };
11256 results.push(result);
11257 }
11258 let mut results = results.into_iter();
11259 parallel_results_for_shape(&handle.shape, &mut results)
11260 .into_json_result()
11261 .map(Some)
11262}
11263
11264fn recorded_selection_member_is_terminal(
11265 state: &WorkflowState,
11266 handle: &DurableOperationHandle,
11267) -> bool {
11268 let mut completed = 0usize;
11269 let mut all_completed = true;
11270 for sequence in handle.base_sequence..handle.base_sequence.saturating_add(handle.size as u64) {
11271 let Some(command) = state
11272 .recorded_commands
11273 .iter()
11274 .find(|command| command.sequence() == sequence)
11275 else {
11276 all_completed = false;
11277 continue;
11278 };
11279 let terminal = match command {
11280 RecordedCommand::Activity {
11281 outcome: Some(Err(_)),
11282 ..
11283 }
11284 | RecordedCommand::ChildWorkflow {
11285 outcome: Some(Err(_)),
11286 ..
11287 } => return true,
11288 RecordedCommand::Activity { outcome, .. } => outcome.is_some(),
11289 RecordedCommand::ChildWorkflow { outcome, .. } => outcome.is_some(),
11290 RecordedCommand::Timer { fired, .. } => *fired,
11291 RecordedCommand::SignalWait { value, .. } => value.is_some(),
11292 RecordedCommand::ConditionWait { result, .. } => result.is_some(),
11293 RecordedCommand::SearchAttributes { .. }
11294 | RecordedCommand::SideEffect { .. }
11295 | RecordedCommand::VersionMarker { .. }
11296 | RecordedCommand::Memo { .. } => false,
11297 };
11298 if !terminal {
11299 all_completed = false;
11300 continue;
11301 }
11302 completed += 1;
11303 }
11304 all_completed && completed == handle.size
11305}
11306
11307fn selection_cancellation_for_handle(
11308 state: &WorkflowState,
11309 handle: &DurableOperationHandle,
11310) -> Result<bool> {
11311 let Some(marker) = state.cancelled_selection_members.iter().find(|recorded| {
11312 recorded.selection_group_id == handle.selection_group_id
11313 && recorded.member_base_sequence == handle.base_sequence
11314 }) else {
11315 return Ok(false);
11316 };
11317 validate_selection_cancellation_marker(marker, handle)?;
11318 Ok(true)
11319}
11320
11321fn validate_selection_cancellations_for_handles(
11322 state: &WorkflowState,
11323 handles: &[DurableOperationHandle],
11324) -> Result<()> {
11325 let Some(group_id) = handles
11326 .first()
11327 .map(|handle| handle.selection_group_id.as_str())
11328 else {
11329 return Ok(());
11330 };
11331 for marker in state
11332 .cancelled_selection_members
11333 .iter()
11334 .filter(|marker| marker.selection_group_id == group_id)
11335 {
11336 let Some(handle) = handles
11337 .iter()
11338 .find(|handle| handle.base_sequence == marker.member_base_sequence)
11339 else {
11340 return Err(invalid_recorded_history(
11341 "selection_cancellation_member_mismatch",
11342 marker.member_base_sequence,
11343 "SelectionOperationCancelled matching an authored selection handle",
11344 &format!("{marker:?}"),
11345 "selection cancellation member base does not name an authored member",
11346 ));
11347 };
11348 validate_selection_cancellation_marker(marker, handle)?;
11349 }
11350 Ok(())
11351}
11352
11353fn validate_selection_cancellation_marker(
11354 marker: &SelectionCancellation,
11355 handle: &DurableOperationHandle,
11356) -> Result<()> {
11357 if marker.selection_group_id != handle.selection_group_id
11358 || marker.member_key != handle.key
11359 || marker.member_index != handle.index
11360 || marker.member_base_sequence != handle.base_sequence
11361 || marker.member_size != handle.size
11362 || marker.operation_kind != handle.kind
11363 || marker.operation_identity != handle.identity
11364 {
11365 return Err(invalid_recorded_history(
11366 "selection_cancellation_member_mismatch",
11367 handle.base_sequence,
11368 "SelectionOperationCancelled matching the authored selection handle",
11369 &format!("{marker:?}"),
11370 "selection cancellation history targets different authored member metadata",
11371 ));
11372 }
11373 Ok(())
11374}
11375
11376pub struct DurableOperationAwaitCall {
11378 handle: DurableOperationHandle,
11379}
11380
11381impl Future for DurableOperationAwaitCall {
11382 type Output = Result<ParallelResult>;
11383
11384 fn poll(self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11385 let state = match self.handle.ctx.state.lock() {
11386 Ok(state) => state,
11387 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11388 };
11389 match selection_cancellation_for_handle(&state, &self.handle) {
11390 Err(error) => return Poll::Ready(Err(error)),
11391 Ok(false) => {}
11392 Ok(true) => {
11393 return Poll::Ready(Err(Error::DurableOperationCancelled(
11394 DurableOperationCancelled {
11395 selection_group_id: self.handle.selection_group_id.clone(),
11396 member_key: self.handle.key.clone(),
11397 member_index: self.handle.index,
11398 operation_kind: self.handle.kind.clone(),
11399 operation_identity: self.handle.identity.clone(),
11400 },
11401 )));
11402 }
11403 }
11404 match recorded_selection_member_outcome(&state, &self.handle) {
11405 Ok(Some(result)) => Poll::Ready(Ok(result)),
11406 Ok(None) => Poll::Pending,
11407 Err(error) => Poll::Ready(Err(error)),
11408 }
11409 }
11410}
11411
11412pub struct CancelDurableOperationCall {
11414 handle: DurableOperationHandle,
11415 emitted: bool,
11416}
11417
11418impl Future for CancelDurableOperationCall {
11419 type Output = Result<()>;
11420
11421 fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11422 let ctx = self.handle.ctx.clone();
11423 let mut state = match ctx.state.lock() {
11424 Ok(state) => state,
11425 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11426 };
11427 match selection_cancellation_for_handle(&state, &self.handle) {
11428 Err(error) => return Poll::Ready(Err(error)),
11429 Ok(true) => return Poll::Ready(Ok(())),
11430 Ok(false) => {}
11431 }
11432 if recorded_selection_member_is_terminal(&state, &self.handle) {
11433 return Poll::Ready(Ok(()));
11434 }
11435 if !self.emitted {
11436 state.commands.push(json!({
11437 "type": "cancel_selection_operation",
11438 "selection_group_id": self.handle.selection_group_id,
11439 "member_key": self.handle.key,
11440 "member_index": self.handle.index,
11441 "member_base_sequence": self.handle.base_sequence,
11442 "member_size": self.handle.size,
11443 "operation_kind": self.handle.kind,
11444 "operation_identity": self.handle.identity,
11445 }));
11446 self.emitted = true;
11447 }
11448 Poll::Pending
11452 }
11453}
11454
11455trait PollNestedResultExt<T> {
11456 fn flatten_result(self) -> Poll<Result<T>>;
11457}
11458
11459impl<T> PollNestedResultExt<T> for Poll<Result<Result<T>>> {
11460 fn flatten_result(self) -> Poll<Result<T>> {
11461 match self {
11462 Poll::Ready(Ok(result)) => Poll::Ready(result),
11463 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
11464 Poll::Pending => Poll::Pending,
11465 }
11466 }
11467}
11468
11469struct SagaCompensation {
11470 activity_type: String,
11471 options: ActivityOptions,
11472 arguments: AvroValue,
11473 registration_order: usize,
11474}
11475
11476pub struct Saga {
11484 ctx: WorkflowContext,
11485 compensations: Vec<SagaCompensation>,
11486}
11487
11488impl Saga {
11489 fn new(ctx: WorkflowContext) -> Self {
11490 Self {
11491 ctx,
11492 compensations: Vec::new(),
11493 }
11494 }
11495
11496 pub fn add_compensation<T: Serialize>(
11497 &mut self,
11498 activity_type: impl Into<String>,
11499 args: T,
11500 ) -> Result<&mut Self> {
11501 self.add_compensation_with_options(activity_type, ActivityOptions::new(), args)
11502 }
11503
11504 pub fn add_compensation_with_options<T: Serialize>(
11505 &mut self,
11506 activity_type: impl Into<String>,
11507 options: ActivityOptions,
11508 args: T,
11509 ) -> Result<&mut Self> {
11510 let activity_type = activity_type.into();
11511 if activity_type.trim().is_empty() || activity_type.trim() != activity_type {
11512 return Err(Error::Codec(
11513 "saga compensation activity type must be non-empty without surrounding whitespace"
11514 .to_string(),
11515 ));
11516 }
11517 options.validate().map_err(Error::InvalidActivityOptions)?;
11518 let arguments = AvroValue::from_serialize(&args)?;
11519 let registration_order = self.compensations.len() + 1;
11520 self.compensations.push(SagaCompensation {
11521 activity_type,
11522 options,
11523 arguments,
11524 registration_order,
11525 });
11526 Ok(self)
11527 }
11528
11529 pub async fn compensate(mut self, initiating_failure: Error) -> Error {
11531 while let Some(compensation) = self.compensations.pop() {
11532 if let Err(compensation_failure) = self
11533 .ctx
11534 .activity_with_options(
11535 compensation.activity_type.clone(),
11536 compensation.options,
11537 compensation.arguments,
11538 )
11539 .await
11540 {
11541 if workflow_task_integrity_error(&compensation_failure) {
11542 return compensation_failure;
11543 }
11544 return Error::SagaCompensationFailed(SagaCompensationFailure {
11545 initiating_failure: Box::new(initiating_failure),
11546 compensation_failure: Box::new(compensation_failure),
11547 compensation_activity_type: compensation.activity_type,
11548 compensation_registration_order: compensation.registration_order,
11549 });
11550 }
11551 }
11552 initiating_failure
11553 }
11554
11555 pub async fn finish<T>(self, outcome: Result<T>) -> Result<T> {
11557 match outcome {
11558 Ok(value) => Ok(value),
11559 Err(error) => Err(self.compensate(error).await),
11560 }
11561 }
11562}
11563
11564pub struct ActivityCall {
11565 ctx: WorkflowContext,
11566 activity_type: String,
11567 options: ActivityOptions,
11568 args: Option<Result<AvroValue>>,
11569 scheduled: bool,
11570 parallel_group_path: Vec<ParallelGroupMetadata>,
11571}
11572
11573impl ActivityCall {
11574 fn poll_avro_value(
11575 mut self: Pin<&mut Self>,
11576 _cx: &mut TaskContext<'_>,
11577 ) -> Poll<Result<AvroValue>> {
11578 let ctx = self.ctx.clone();
11579 let mut state = match ctx.state.lock() {
11580 Ok(state) => state,
11581 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11582 };
11583
11584 if self.scheduled {
11585 return Poll::Pending;
11586 }
11587
11588 let options = match self.options.validate() {
11589 Ok(options) => options,
11590 Err(error) => {
11591 return Poll::Ready(Err(Error::InvalidActivityOptions(error)));
11592 }
11593 };
11594 let task_queue = options
11595 .task_queue
11596 .clone()
11597 .unwrap_or_else(|| state.task_queue.clone());
11598 let current_recorded_options = RecordedActivityOptions {
11599 task_queue: RecordedSnapshotValue::Known(Some(task_queue.clone())),
11600 execution_mode: RecordedSnapshotValue::Known(None),
11603 retry_policy: current_activity_retry_snapshot(&options),
11604 };
11605
11606 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
11607 let sequence = recorded.sequence();
11608 match recorded {
11609 RecordedCommand::Activity {
11610 activity_type,
11611 options: recorded_options,
11612 outcome,
11613 parallel_group_path,
11614 ..
11615 } => {
11616 if let Err(error) = ensure_parallel_path_matches(
11617 sequence,
11618 parallel_group_path.as_deref(),
11619 &self.parallel_group_path,
11620 ) {
11621 return Poll::Ready(Err(error));
11622 }
11623 if let Some(recorded_type) = activity_type {
11624 if recorded_type != self.activity_type {
11625 return Poll::Ready(Err(Error::NonDeterministicReplay(
11626 ReplayFailure::new(
11627 "recorded_command_detail_mismatch",
11628 Some(sequence),
11629 Some(format!("activity:{recorded_type}")),
11630 Some(format!("activity:{}", self.activity_type)),
11631 "recorded activity type differs from the current workflow command",
11632 ),
11633 )));
11634 }
11635 }
11636 if let Some(recorded_options) = recorded_options {
11637 if !recorded_options
11638 .task_queue
11639 .matches_current(¤t_recorded_options.task_queue)
11640 {
11641 return Poll::Ready(Err(Error::NonDeterministicReplay(
11642 ReplayFailure::new(
11643 "activity_task_queue_mismatch",
11644 Some(sequence),
11645 Some(activity_options_description(&recorded_options)),
11646 Some(activity_options_description(¤t_recorded_options)),
11647 "recorded activity task queue differs from the current workflow command",
11648 ),
11649 )));
11650 }
11651 if !recorded_options
11652 .execution_mode
11653 .matches_current(¤t_recorded_options.execution_mode)
11654 {
11655 return Poll::Ready(Err(Error::NonDeterministicReplay(
11656 ReplayFailure::new(
11657 "activity_execution_mode_mismatch",
11658 Some(sequence),
11659 Some(activity_options_description(&recorded_options)),
11660 Some(activity_options_description(¤t_recorded_options)),
11661 "recorded activity execution mode differs from the current workflow command",
11662 ),
11663 )));
11664 }
11665 if !recorded_options
11666 .retry_policy
11667 .matches_current(¤t_recorded_options.retry_policy)
11668 {
11669 return Poll::Ready(Err(Error::NonDeterministicReplay(
11670 ReplayFailure::new(
11671 "activity_retry_policy_mismatch",
11672 Some(sequence),
11673 Some(activity_options_description(&recorded_options)),
11674 Some(activity_options_description(¤t_recorded_options)),
11675 "recorded activity retry policy differs from the current workflow command",
11676 ),
11677 )));
11678 }
11679 }
11680 state.command_cursor += 1;
11681 if let Some(outcome) = outcome {
11682 return Poll::Ready(outcome.map_err(Error::ActivityFailed));
11683 }
11684 state.matched_recorded_pending = true;
11685 self.scheduled = true;
11686 return Poll::Pending;
11687 }
11688 other => {
11689 return Poll::Ready(Err(command_mismatch(
11690 &other,
11691 format!("activity:{}", self.activity_type),
11692 )));
11693 }
11694 }
11695 }
11696
11697 if !self.scheduled {
11698 let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
11699 Ok(args) => args,
11700 Err(error) => return Poll::Ready(Err(error)),
11701 };
11702 let arguments = normalize_avro_arguments(args);
11703 let envelope = match encode_typed_envelope(&arguments, &state.payload_codec) {
11704 Ok(envelope) => envelope,
11705 Err(error) => return Poll::Ready(Err(error)),
11706 };
11707
11708 let mut command = serde_json::Map::from_iter([
11709 ("type".to_string(), json!("schedule_activity")),
11710 (
11711 "activity_type".to_string(),
11712 json!(self.activity_type.clone()),
11713 ),
11714 ("queue".to_string(), json!(task_queue)),
11715 ("arguments".to_string(), envelope),
11716 ]);
11717 for (field, value) in [
11718 ("start_to_close_timeout", options.start_to_close_timeout),
11719 (
11720 "schedule_to_start_timeout",
11721 options.schedule_to_start_timeout,
11722 ),
11723 (
11724 "schedule_to_close_timeout",
11725 options.schedule_to_close_timeout,
11726 ),
11727 ("heartbeat_timeout", options.heartbeat_timeout),
11728 ] {
11729 if let Some(value) = value {
11730 command.insert(field.to_string(), json!(value));
11731 }
11732 }
11733 if let Some(retry_policy) = options.retry_policy {
11734 command.insert("retry_policy".to_string(), retry_policy);
11735 }
11736 apply_parallel_group_path(&mut command, &self.parallel_group_path);
11737 state.commands.push(Value::Object(command));
11738 self.scheduled = true;
11739 }
11740
11741 Poll::Pending
11742 }
11743}
11744
11745impl Future for ActivityCall {
11746 type Output = Result<Value>;
11747
11748 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11749 match self.poll_avro_value(cx) {
11750 Poll::Ready(Ok(value)) => Poll::Ready(value.into_json()),
11751 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
11752 Poll::Pending => Poll::Pending,
11753 }
11754 }
11755}
11756
11757pub struct TimerCall {
11759 ctx: WorkflowContext,
11760 delay_seconds: Option<u64>,
11761 scheduled: bool,
11762 matched_pending: bool,
11763 parallel_group_path: Vec<ParallelGroupMetadata>,
11764}
11765
11766impl Future for TimerCall {
11767 type Output = Result<()>;
11768
11769 fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11770 if self.matched_pending {
11771 return Poll::Pending;
11772 }
11773
11774 let ctx = self.ctx.clone();
11775 let Some(requested_delay) = self.delay_seconds else {
11776 return Poll::Ready(Err(Error::TimerDurationOverflow));
11777 };
11778 let mut state = match ctx.state.lock() {
11779 Ok(state) => state,
11780 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11781 };
11782
11783 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
11784 match recorded {
11785 RecordedCommand::Timer {
11786 sequence,
11787 delay_seconds,
11788 fired,
11789 parallel_group_path,
11790 ..
11791 } => {
11792 if let Err(error) = ensure_parallel_path_matches(
11793 sequence,
11794 parallel_group_path.as_deref(),
11795 &self.parallel_group_path,
11796 ) {
11797 return Poll::Ready(Err(error));
11798 }
11799 if delay_seconds != requested_delay {
11800 return Poll::Ready(Err(Error::NonDeterministicReplay(
11801 ReplayFailure::new(
11802 "timer_delay_mismatch",
11803 Some(sequence),
11804 Some(format!("timer:{delay_seconds}s")),
11805 Some(format!("timer:{requested_delay}s")),
11806 "recorded timer delay differs from the current workflow command",
11807 ),
11808 )));
11809 }
11810 state.command_cursor += 1;
11811 if fired {
11812 return Poll::Ready(Ok(()));
11813 }
11814 state.matched_recorded_pending = true;
11815 self.scheduled = true;
11816 self.matched_pending = true;
11817 return Poll::Pending;
11818 }
11819 other => return Poll::Ready(Err(command_mismatch(&other, "timer"))),
11820 }
11821 }
11822
11823 if !self.scheduled {
11824 let mut command = serde_json::Map::from_iter([
11825 ("type".to_string(), json!("start_timer")),
11826 ("delay_seconds".to_string(), json!(requested_delay)),
11827 ]);
11828 apply_parallel_group_path(&mut command, &self.parallel_group_path);
11829 state.commands.push(Value::Object(command));
11830 self.scheduled = true;
11831 }
11832
11833 Poll::Pending
11834 }
11835}
11836
11837pub struct ConditionWaitCall {
11839 ctx: WorkflowContext,
11840 options: ConditionWaitOptions,
11841 predicate: Box<dyn Fn() -> Result<bool> + Send + 'static>,
11842 occurrence_id: Option<String>,
11843 opened_wait: bool,
11844 parallel_group_path: Vec<ParallelGroupMetadata>,
11845}
11846
11847impl Future for ConditionWaitCall {
11848 type Output = Result<ConditionWaitResult>;
11849
11850 fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11851 if self.opened_wait {
11852 return Poll::Pending;
11853 }
11854
11855 let options = match self.options.validate() {
11856 Ok(options) => options,
11857 Err(error) => return Poll::Ready(Err(Error::InvalidConditionWaitOptions(error))),
11858 };
11859 let ctx = self.ctx.clone();
11860 let occurrence_id = match self.occurrence_id.as_ref() {
11861 Some(occurrence_id) => occurrence_id.clone(),
11862 None => {
11863 let mut state = match ctx.state.lock() {
11864 Ok(state) => state,
11865 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11866 };
11867 let ordinal = state.condition_wait_occurrence_counter;
11868 state.condition_wait_occurrence_counter = match ordinal.checked_add(1) {
11869 Some(next) => next,
11870 None => {
11871 return Poll::Ready(Err(Error::WorkerLoop(
11872 "condition wait occurrence counter overflowed".to_string(),
11873 )))
11874 }
11875 };
11876 let occurrence_id = format!("{CONDITION_WAIT_OCCURRENCE_PREFIX}{ordinal}");
11877 drop(state);
11878 self.occurrence_id = Some(occurrence_id.clone());
11879 occurrence_id
11880 }
11881 };
11882
11883 let recorded_result = {
11884 let mut state = match ctx.state.lock() {
11885 Ok(state) => state,
11886 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11887 };
11888 let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() else {
11889 drop(state);
11890 return self.poll_new_condition(options);
11891 };
11892 if !matches!(recorded, RecordedCommand::ConditionWait { .. }) {
11893 return Poll::Ready(Err(command_mismatch(&recorded, "condition wait")));
11894 }
11895
11896 let mut cursor = state.command_cursor;
11897 let mut result = None;
11898 loop {
11899 let Some(RecordedCommand::ConditionWait {
11900 sequence,
11901 occurrence_id: recorded_occurrence_id,
11902 condition_key,
11903 predicate_identity,
11904 timeout_seconds,
11905 result: recorded_result,
11906 parallel_group_path,
11907 ..
11908 }) = state.recorded_commands.get(cursor)
11909 else {
11910 break;
11911 };
11912
11913 if cursor > state.command_cursor && recorded_occurrence_id != &occurrence_id {
11914 break;
11915 }
11916 if let Err(error) = ensure_parallel_path_matches(
11917 *sequence,
11918 parallel_group_path.as_deref(),
11919 &self.parallel_group_path,
11920 ) {
11921 return Poll::Ready(Err(error));
11922 }
11923 if let Err(error) = validate_recorded_condition_wait(
11924 *sequence,
11925 recorded_occurrence_id,
11926 condition_key.as_deref(),
11927 predicate_identity,
11928 *timeout_seconds,
11929 &occurrence_id,
11930 &options,
11931 ) {
11932 return Poll::Ready(Err(error));
11933 }
11934 if result == Some(ConditionWaitResult::TimedOut) {
11935 return Poll::Ready(Err(Error::NonDeterministicReplay(ReplayFailure::new(
11936 "condition_wait_reopened_after_timeout",
11937 Some(*sequence),
11938 Some("timed-out condition is terminal".to_string()),
11939 Some("another physical wait-open".to_string()),
11940 "condition history reopened one logical wait after its durable timeout",
11941 ))));
11942 }
11943 result = *recorded_result;
11944 cursor += 1;
11945 }
11946 state.command_cursor = cursor;
11947 result
11948 };
11949
11950 if let Some(result) = recorded_result {
11951 return Poll::Ready(Ok(result));
11952 }
11953
11954 self.poll_open_condition(options)
11955 }
11956}
11957
11958impl ConditionWaitCall {
11959 fn poll_new_condition(
11960 self: Pin<&mut Self>,
11961 options: ValidatedConditionWaitOptions,
11962 ) -> Poll<Result<ConditionWaitResult>> {
11963 self.poll_open_condition(options)
11964 }
11965
11966 fn poll_open_condition(
11967 mut self: Pin<&mut Self>,
11968 options: ValidatedConditionWaitOptions,
11969 ) -> Poll<Result<ConditionWaitResult>> {
11970 let selection_member = self
11971 .parallel_group_path
11972 .first()
11973 .is_some_and(|entry| entry.parallel_group_mode.as_deref() == Some("select"));
11974 match (self.predicate)() {
11975 Ok(true) if !selection_member => {
11976 return Poll::Ready(Ok(ConditionWaitResult::Satisfied))
11977 }
11978 Ok(_) => {}
11979 Err(error) => return Poll::Ready(Err(error)),
11980 }
11981 if options.timeout_seconds == Some(0) && !selection_member {
11982 return Poll::Ready(Ok(ConditionWaitResult::TimedOut));
11983 }
11984
11985 let ctx = self.ctx.clone();
11986 let mut state = match ctx.state.lock() {
11987 Ok(state) => state,
11988 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11989 };
11990 let mut command = serde_json::Map::from_iter([
11991 ("type".to_string(), json!("open_condition_wait")),
11992 (
11993 "condition_wait_occurrence_id".to_string(),
11994 json!(self.occurrence_id.as_deref().unwrap_or_default()),
11995 ),
11996 ("condition_key".to_string(), json!(options.condition_key)),
11997 (
11998 "condition_definition_fingerprint".to_string(),
11999 json!(options.predicate_identity),
12000 ),
12001 ]);
12002 if let Some(timeout_seconds) = options.timeout_seconds {
12003 command.insert("timeout_seconds".to_string(), json!(timeout_seconds));
12004 }
12005 apply_parallel_group_path(&mut command, &self.parallel_group_path);
12006 state.commands.push(Value::Object(command));
12007 drop(state);
12008 self.opened_wait = true;
12009 Poll::Pending
12010 }
12011}
12012
12013fn validate_recorded_condition_wait(
12014 sequence: u64,
12015 recorded_occurrence_id: &str,
12016 recorded_key: Option<&str>,
12017 recorded_predicate_identity: &str,
12018 recorded_timeout_seconds: Option<u64>,
12019 current_occurrence_id: &str,
12020 current: &ValidatedConditionWaitOptions,
12021) -> Result<()> {
12022 if recorded_occurrence_id != current_occurrence_id {
12023 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12024 "condition_wait_occurrence_mismatch",
12025 Some(sequence),
12026 Some(recorded_occurrence_id.to_string()),
12027 Some(current_occurrence_id.to_string()),
12028 "recorded condition occurrence differs from the current authored wait position",
12029 )));
12030 }
12031 if recorded_key != Some(current.condition_key.as_str()) {
12032 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12033 "condition_wait_key_mismatch",
12034 Some(sequence),
12035 recorded_key.map(str::to_string),
12036 Some(current.condition_key.clone()),
12037 "recorded condition identity differs from the current workflow wait",
12038 )));
12039 }
12040 if recorded_predicate_identity != current.predicate_identity {
12041 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12042 "condition_wait_predicate_mismatch",
12043 Some(sequence),
12044 Some(recorded_predicate_identity.to_string()),
12045 Some(current.predicate_identity.clone()),
12046 "recorded condition predicate behavior differs from current workflow code",
12047 )));
12048 }
12049 if recorded_timeout_seconds != current.timeout_seconds {
12050 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12051 "condition_wait_timeout_mismatch",
12052 Some(sequence),
12053 recorded_timeout_seconds.map(|seconds| format!("{seconds}s")),
12054 current.timeout_seconds.map(|seconds| format!("{seconds}s")),
12055 "recorded condition timeout differs from the current workflow wait",
12056 )));
12057 }
12058 Ok(())
12059}
12060
12061pub struct ChildWorkflowCall {
12063 ctx: WorkflowContext,
12064 workflow_type: String,
12065 options: ChildWorkflowOptions,
12066 args: Option<Result<AvroValue>>,
12067 scheduled: bool,
12068 matched_pending: bool,
12069 parallel_group_path: Vec<ParallelGroupMetadata>,
12070}
12071
12072impl ChildWorkflowCall {
12073 fn poll_avro_value(
12074 mut self: Pin<&mut Self>,
12075 _cx: &mut TaskContext<'_>,
12076 ) -> Poll<Result<ChildWorkflowAvroResult>> {
12077 if self.matched_pending {
12078 return Poll::Pending;
12079 }
12080
12081 let ctx = self.ctx.clone();
12082 let mut state = match ctx.state.lock() {
12083 Ok(state) => state,
12084 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
12085 };
12086
12087 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
12088 let sequence = recorded.sequence();
12089 match recorded {
12090 RecordedCommand::ChildWorkflow {
12091 workflow_type,
12092 outcome,
12093 parallel_group_path,
12094 ..
12095 } => {
12096 if let Err(error) = ensure_parallel_path_matches(
12097 sequence,
12098 parallel_group_path.as_deref(),
12099 &self.parallel_group_path,
12100 ) {
12101 return Poll::Ready(Err(error));
12102 }
12103 if let Some(recorded_type) = workflow_type {
12104 if recorded_type != self.workflow_type {
12105 return Poll::Ready(Err(Error::NonDeterministicReplay(
12106 ReplayFailure::new(
12107 "recorded_command_detail_mismatch",
12108 Some(sequence),
12109 Some(format!("child workflow:{recorded_type}")),
12110 Some(format!("child workflow:{}", self.workflow_type)),
12111 "recorded child workflow type differs from the current workflow command",
12112 ),
12113 )));
12114 }
12115 }
12116 state.command_cursor += 1;
12117 if let Some(outcome) = outcome {
12118 return Poll::Ready(outcome.map_err(Error::ChildWorkflowFailed));
12119 }
12120 state.matched_recorded_pending = true;
12121 self.scheduled = true;
12122 self.matched_pending = true;
12123 return Poll::Pending;
12124 }
12125 other => {
12126 return Poll::Ready(Err(command_mismatch(
12127 &other,
12128 format!("child workflow:{}", self.workflow_type),
12129 )));
12130 }
12131 }
12132 }
12133
12134 if !self.scheduled {
12135 if self.options.task_queue.trim().is_empty() {
12136 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
12137 "task_queue must not be empty".to_string(),
12138 )));
12139 }
12140 for (name, value) in [
12141 (
12142 "execution_timeout_seconds",
12143 self.options.execution_timeout_seconds,
12144 ),
12145 ("run_timeout_seconds", self.options.run_timeout_seconds),
12146 ] {
12147 if value == Some(0) {
12148 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(format!(
12149 "{name} must be at least 1"
12150 ))));
12151 }
12152 }
12153
12154 let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
12155 Ok(args) => args,
12156 Err(error) => return Poll::Ready(Err(error)),
12157 };
12158 let arguments = match encode_typed_envelope(
12159 &normalize_avro_arguments(args),
12160 &state.payload_codec,
12161 ) {
12162 Ok(arguments) => arguments,
12163 Err(error) => return Poll::Ready(Err(error)),
12164 };
12165 let mut command = json!({
12166 "type": "start_child_workflow",
12167 "workflow_type": self.workflow_type,
12168 "queue": self.options.task_queue,
12169 "parent_close_policy": self.options.parent_close_policy.as_str(),
12170 "arguments": arguments,
12171 });
12172 let object = command
12173 .as_object_mut()
12174 .expect("child workflow command is always an object");
12175 if let Some(policy) = &self.options.retry_policy {
12176 let mut retry_policy = serde_json::Map::new();
12177 if let Some(max_attempts) = policy.max_attempts {
12178 if max_attempts == 0 {
12179 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
12180 "retry_policy.max_attempts must be at least 1".to_string(),
12181 )));
12182 }
12183 retry_policy.insert("max_attempts".to_string(), json!(max_attempts));
12184 }
12185 if !policy.backoff_seconds.is_empty() {
12186 retry_policy
12187 .insert("backoff_seconds".to_string(), json!(policy.backoff_seconds));
12188 }
12189 if !policy.non_retryable_error_types.is_empty() {
12190 retry_policy.insert(
12191 "non_retryable_error_types".to_string(),
12192 json!(policy.non_retryable_error_types),
12193 );
12194 }
12195 if retry_policy.is_empty() {
12196 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
12197 "retry_policy must configure at least one field".to_string(),
12198 )));
12199 }
12200 object.insert("retry_policy".to_string(), Value::Object(retry_policy));
12201 }
12202 if let Some(seconds) = self.options.execution_timeout_seconds {
12203 object.insert("execution_timeout_seconds".to_string(), json!(seconds));
12204 }
12205 if let Some(seconds) = self.options.run_timeout_seconds {
12206 object.insert("run_timeout_seconds".to_string(), json!(seconds));
12207 }
12208 apply_parallel_group_path(object, &self.parallel_group_path);
12209 state.commands.push(command);
12210 self.scheduled = true;
12211 }
12212
12213 Poll::Pending
12214 }
12215}
12216
12217impl Future for ChildWorkflowCall {
12218 type Output = Result<ChildWorkflowResult>;
12219
12220 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
12221 match self.poll_avro_value(cx) {
12222 Poll::Ready(Ok(result)) => match result.result.into_json() {
12223 Ok(projected) => Poll::Ready(Ok(ChildWorkflowResult {
12224 parent: result.parent,
12225 child: result.child,
12226 child_workflow_type: result.child_workflow_type,
12227 result: projected,
12228 })),
12229 Err(error) => Poll::Ready(Err(error)),
12230 },
12231 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
12232 Poll::Pending => Poll::Pending,
12233 }
12234 }
12235}
12236
12237fn command_mismatch(recorded: &RecordedCommand, actual: impl Into<String>) -> Error {
12238 Error::NonDeterministicReplay(ReplayFailure::new(
12239 "recorded_command_mismatch",
12240 Some(recorded.sequence()),
12241 Some(recorded.shape().to_string()),
12242 Some(actual.into()),
12243 "current workflow command does not match the recorded durable command sequence",
12244 ))
12245}
12246
12247pub struct SignalCall {
12248 ctx: WorkflowContext,
12249 signal_name: String,
12250 runtime_reserved_allowed: bool,
12251 opened_wait: bool,
12252 matched_pending: bool,
12253 parallel_group_path: Vec<ParallelGroupMetadata>,
12254}
12255
12256impl SignalCall {
12257 fn poll_avro_value(
12258 mut self: Pin<&mut Self>,
12259 _cx: &mut TaskContext<'_>,
12260 ) -> Poll<Result<Vec<AvroValue>>> {
12261 if self.matched_pending {
12262 return Poll::Pending;
12263 }
12264 if !self.runtime_reserved_allowed {
12265 if let Err(error) = validate_user_signal_name(&self.signal_name) {
12266 return Poll::Ready(Err(error));
12267 }
12268 }
12269
12270 let ctx = self.ctx.clone();
12271 let mut state = match ctx.state.lock() {
12272 Ok(state) => state,
12273 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
12274 };
12275
12276 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
12277 match recorded {
12278 RecordedCommand::SignalWait {
12279 sequence,
12280 signal_name,
12281 value,
12282 parallel_group_path,
12283 } => {
12284 if let Err(error) = ensure_parallel_path_matches(
12285 sequence,
12286 parallel_group_path.as_deref(),
12287 &self.parallel_group_path,
12288 ) {
12289 return Poll::Ready(Err(error));
12290 }
12291 if signal_name != self.signal_name {
12292 return Poll::Ready(Err(Error::NonDeterministicReplay(
12293 ReplayFailure::new(
12294 "recorded_command_detail_mismatch",
12295 Some(sequence),
12296 Some(format!("signal wait:{signal_name}")),
12297 Some(format!("signal wait:{}", self.signal_name)),
12298 "recorded signal name differs from the current workflow command",
12299 ),
12300 )));
12301 }
12302
12303 state.command_cursor += 1;
12304 if let Some(value) = value {
12305 return Poll::Ready(Ok(value));
12306 }
12307 if state
12308 .resume_signal
12309 .as_ref()
12310 .is_some_and(|signal| signal.signal_name == self.signal_name)
12311 {
12312 let signal = state
12313 .resume_signal
12314 .take()
12315 .expect("matching resume signal is present");
12316 return Poll::Ready(Ok(signal.arguments));
12317 }
12318
12319 state.matched_recorded_pending = true;
12320 self.opened_wait = true;
12321 self.matched_pending = true;
12322 return Poll::Pending;
12323 }
12324 other => {
12325 return Poll::Ready(Err(command_mismatch(
12326 &other,
12327 format!("signal wait:{}", self.signal_name),
12328 )));
12329 }
12330 }
12331 }
12332
12333 if state
12334 .resume_signal
12335 .as_ref()
12336 .is_some_and(|signal| signal.signal_name == self.signal_name)
12337 {
12338 let signal = state
12339 .resume_signal
12340 .take()
12341 .expect("matching resume signal is present");
12342 return Poll::Ready(Ok(signal.arguments));
12343 }
12344
12345 if !self.opened_wait {
12346 let mut command = serde_json::Map::from_iter([
12347 ("type".to_string(), json!("open_signal_wait")),
12348 ("signal_name".to_string(), json!(self.signal_name)),
12349 ]);
12350 apply_parallel_group_path(&mut command, &self.parallel_group_path);
12351 state.commands.push(Value::Object(command));
12352 self.opened_wait = true;
12353 }
12354
12355 Poll::Pending
12356 }
12357}
12358
12359impl Future for SignalCall {
12360 type Output = Result<Vec<Value>>;
12361
12362 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
12363 match self.poll_avro_value(cx) {
12364 Poll::Ready(Ok(values)) => Poll::Ready(
12365 values
12366 .into_iter()
12367 .map(AvroValue::into_json)
12368 .collect::<Result<Vec<_>>>(),
12369 ),
12370 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
12371 Poll::Pending => Poll::Pending,
12372 }
12373 }
12374}
12375
12376#[derive(Clone, Debug)]
12377pub struct ActivityContext {
12378 client: Client,
12379 pub task_id: String,
12380 pub activity_attempt_id: String,
12381 pub lease_owner: String,
12382 pub activity_type: String,
12383 pub attempt_number: u64,
12384 pub task_queue: String,
12385 pub worker_id: String,
12386}
12387
12388impl ActivityContext {
12389 pub async fn heartbeat<T: Serialize>(&self, details: T) -> Result<ActivityHeartbeatResponse> {
12390 self.client
12391 .heartbeat_activity_task(
12392 &self.task_id,
12393 &self.activity_attempt_id,
12394 &self.lease_owner,
12395 details,
12396 )
12397 .await
12398 }
12399}
12400
12401fn decode_task_avro_arguments(value: Option<&Value>, codec: &str) -> Result<AvroValue> {
12402 validate_payload_codec(codec)?;
12403 match value {
12404 Some(value) => Ok(normalize_avro_arguments(decode_wire_avro_value(
12405 value, codec,
12406 )?)),
12407 None => Ok(AvroValue::Array(Vec::new())),
12408 }
12409}
12410
12411fn decode_resume_signal(task: &WorkflowTask) -> Result<Option<ResumeSignal>> {
12412 let Some(signal_name) = task
12413 .signal_name
12414 .as_deref()
12415 .filter(|value| !value.is_empty())
12416 else {
12417 return Ok(None);
12418 };
12419 let decoded = decode_task_avro_arguments(task.signal_arguments.as_ref(), &task.payload_codec)?;
12420 let AvroValue::Array(arguments) = decoded else {
12421 unreachable!("normalize_avro_arguments always returns an array");
12422 };
12423
12424 Ok(Some(ResumeSignal {
12425 signal_name: signal_name.to_string(),
12426 arguments,
12427 }))
12428}
12429
12430fn validate_workflow_task_payloads(task: &WorkflowTask) -> Result<()> {
12431 validate_payload_codec(&task.payload_codec)?;
12432 validate_optional_inbound_payload(task.arguments.as_ref(), &task.payload_codec)?;
12433 validate_optional_inbound_payload(task.signal_arguments.as_ref(), &task.payload_codec)?;
12434 for event in &task.history_events {
12435 validate_history_event_payloads(event, &task.payload_codec)?;
12436 }
12437 Ok(())
12438}
12439
12440fn validate_activity_task_payloads(task: &ActivityTask) -> Result<()> {
12441 validate_payload_codec(&task.payload_codec)?;
12442 validate_optional_inbound_payload(task.arguments.as_ref(), &task.payload_codec)
12443}
12444
12445fn validate_query_task_payloads(task: &QueryTask) -> Result<()> {
12446 validate_payload_codec(&task.payload_codec)?;
12447 validate_optional_inbound_payload(task.workflow_arguments.as_ref(), &task.payload_codec)?;
12448 validate_optional_inbound_payload(task.query_arguments.as_ref(), &task.payload_codec)?;
12449 for event in &task.history_events {
12450 validate_history_event_payloads(event, &task.payload_codec)?;
12451 }
12452
12453 let Some(export) = task.history_export.as_ref() else {
12454 return Ok(());
12455 };
12456 let export_codec = match export.get("payloads") {
12457 Some(payloads) => declared_payload_codec(payloads, "codec")?,
12458 None => None,
12459 }
12460 .unwrap_or(&task.payload_codec);
12461 validate_payload_codec(export_codec)?;
12462
12463 if let Some(events) = export.get("history_events").and_then(Value::as_array) {
12464 for event in events {
12465 let event_type = event
12466 .get("event_type")
12467 .or_else(|| event.get("type"))
12468 .and_then(Value::as_str)
12469 .unwrap_or_default();
12470 if let Some(payload) = event.get("payload") {
12471 validate_history_payloads(event_type, payload, export_codec)?;
12472 }
12473 }
12474 }
12475 for signal in export
12476 .get("signals")
12477 .and_then(Value::as_array)
12478 .into_iter()
12479 .flatten()
12480 {
12481 let codec = declared_payload_codec(signal, "payload_codec")?.unwrap_or(export_codec);
12482 validate_payload_codec(codec)?;
12483 validate_optional_inbound_payload(signal.get("arguments"), codec)?;
12484 }
12485 for activity in export
12486 .get("activities")
12487 .and_then(Value::as_array)
12488 .into_iter()
12489 .flatten()
12490 {
12491 let codec = declared_payload_codec(activity, "payload_codec")?.unwrap_or(export_codec);
12492 validate_payload_codec(codec)?;
12493 validate_optional_inbound_payload(activity.get("arguments"), codec)?;
12494 validate_optional_inbound_payload(activity.get("result"), codec)?;
12495 }
12496 Ok(())
12497}
12498
12499fn validate_history_event_payloads(event: &HistoryEvent, fallback_codec: &str) -> Result<()> {
12500 validate_history_payloads(&event.event_type, &event.payload, fallback_codec)
12501}
12502
12503fn validate_history_payloads(
12504 event_type: &str,
12505 payload: &Value,
12506 fallback_codec: &str,
12507) -> Result<()> {
12508 let codec = declared_payload_codec(payload, "payload_codec")?.unwrap_or(fallback_codec);
12509 validate_payload_codec(codec)?;
12510 for field in history_payload_fields(event_type) {
12511 validate_optional_inbound_payload(payload.get(*field), codec)?;
12512 }
12513 Ok(())
12514}
12515
12516const SIGNAL_HISTORY_PAYLOAD_FIELDS: &[&str] = &["value", "input", "arguments"];
12517
12518fn history_payload_fields(event_type: &str) -> &'static [&'static str] {
12519 match event_type {
12520 "ActivityCompleted" => &["result"],
12521 "SignalReceived" | "SignalApplied" => SIGNAL_HISTORY_PAYLOAD_FIELDS,
12522 "UpdateAccepted" | "UpdateRejected" | "UpdateApplied" => &["arguments"],
12523 "UpdateCompleted" | "SideEffectRecorded" => &["result"],
12524 "ChildRunCompleted" => &["result", "output"],
12525 "WorkflowCompleted" => &["output"],
12526 "ServiceCallStarted"
12527 | "ServiceCallCompleted"
12528 | "ServiceCallFailed"
12529 | "ServiceCallCancelled" => &["request_payload", "response_payload"],
12530 _ => &[],
12531 }
12532}
12533
12534fn signal_history_payload(payload: &Value) -> Option<&Value> {
12535 SIGNAL_HISTORY_PAYLOAD_FIELDS
12536 .iter()
12537 .find_map(|field| payload.get(*field))
12538}
12539
12540fn declared_payload_codec<'a>(value: &'a Value, field: &str) -> Result<Option<&'a str>> {
12541 match value.get(field) {
12542 None => Ok(None),
12543 Some(Value::String(codec)) => Ok(Some(codec)),
12544 Some(_) => Err(invalid_payload_envelope()),
12545 }
12546}
12547
12548fn validate_optional_inbound_payload(value: Option<&Value>, codec: &str) -> Result<()> {
12549 validate_payload_codec(codec)?;
12550 if let Some(value) = value.filter(|value| !value.is_null()) {
12551 decode_wire_avro_value(value, codec)?;
12552 }
12553 Ok(())
12554}
12555
12556fn recorded_parallel_group_entry(payload: &Value, sequence: u64) -> Result<ParallelGroupMetadata> {
12557 let group_id = payload_string(payload, "parallel_group_id").ok_or_else(|| {
12558 invalid_recorded_history(
12559 "parallel_group_metadata_invalid",
12560 sequence,
12561 "non-empty parallel_group_id",
12562 &payload.to_string(),
12563 "parallel-group history is missing its stable identity",
12564 )
12565 })?;
12566 let kind = payload_string(payload, "parallel_group_kind").ok_or_else(|| {
12567 invalid_recorded_history(
12568 "parallel_group_metadata_invalid",
12569 sequence,
12570 "activity, child, timer, signal, condition, or mixed group kind",
12571 &payload.to_string(),
12572 "parallel-group history is missing its group kind",
12573 )
12574 })?;
12575 if !matches!(
12576 kind.as_str(),
12577 "activity" | "child" | "timer" | "signal" | "condition" | "mixed"
12578 ) {
12579 return Err(invalid_recorded_history(
12580 "parallel_group_metadata_invalid",
12581 sequence,
12582 "activity, child, timer, signal, condition, or mixed group kind",
12583 &kind,
12584 "parallel-group history contains an unsupported group kind",
12585 ));
12586 }
12587 let base_sequence = payload
12588 .get("parallel_group_base_sequence")
12589 .and_then(value_as_u64)
12590 .filter(|value| *value > 0)
12591 .ok_or_else(|| {
12592 invalid_recorded_history(
12593 "parallel_group_metadata_invalid",
12594 sequence,
12595 "positive parallel_group_base_sequence",
12596 &payload.to_string(),
12597 "parallel-group history contains an invalid base sequence",
12598 )
12599 })?;
12600 let size = payload
12601 .get("parallel_group_size")
12602 .and_then(value_as_u64)
12603 .and_then(|value| usize::try_from(value).ok())
12604 .filter(|value| (1..=MAX_PARALLEL_OPERATIONS).contains(value))
12605 .ok_or_else(|| {
12606 invalid_recorded_history(
12607 "parallel_group_metadata_invalid",
12608 sequence,
12609 "bounded positive parallel_group_size",
12610 &payload.to_string(),
12611 "parallel-group history contains an invalid group size",
12612 )
12613 })?;
12614 let index = payload
12615 .get("parallel_group_index")
12616 .and_then(value_as_u64)
12617 .and_then(|value| usize::try_from(value).ok())
12618 .filter(|value| *value < size)
12619 .ok_or_else(|| {
12620 invalid_recorded_history(
12621 "parallel_group_metadata_invalid",
12622 sequence,
12623 "parallel_group_index within group bounds",
12624 &payload.to_string(),
12625 "parallel-group history contains an invalid member index",
12626 )
12627 })?;
12628 if base_sequence.checked_add(u64::try_from(index).unwrap_or(u64::MAX)) != Some(sequence) {
12629 return Err(invalid_recorded_history(
12630 "parallel_group_metadata_invalid",
12631 sequence,
12632 "base sequence plus member index equals workflow sequence",
12633 &payload.to_string(),
12634 "parallel-group path does not preserve durable workflow position",
12635 ));
12636 }
12637 let mode = payload
12638 .get("parallel_group_mode")
12639 .and_then(Value::as_str)
12640 .unwrap_or("all");
12641 if !matches!(mode, "all" | "select") {
12642 return Err(invalid_recorded_history(
12643 "parallel_group_metadata_invalid",
12644 sequence,
12645 "parallel group mode all or select",
12646 mode,
12647 "parallel-group history contains an unsupported group mode",
12648 ));
12649 }
12650 let expected_id = if mode == "select" {
12651 format!("select-calls:{base_sequence}:{size}")
12652 } else {
12653 format!("{}:{base_sequence}:{size}", parallel_group_prefix(&kind))
12654 };
12655 if group_id != expected_id {
12656 return Err(invalid_recorded_history(
12657 "parallel_group_metadata_invalid",
12658 sequence,
12659 &expected_id,
12660 &group_id,
12661 "parallel-group history contains an incompatible stable group ID",
12662 ));
12663 }
12664 let selection_member_key = if mode == "select" {
12665 Some(selection_key_from_value(
12666 payload.get("selection_member_key"),
12667 sequence,
12668 )?)
12669 } else {
12670 None
12671 };
12672 let selection_member_index = if mode == "select" {
12673 Some(required_parallel_usize(
12674 payload,
12675 "selection_member_index",
12676 sequence,
12677 )?)
12678 } else {
12679 None
12680 };
12681 let selection_member_base_sequence = if mode == "select" {
12682 Some(
12683 payload
12684 .get("selection_member_base_sequence")
12685 .and_then(value_as_u64)
12686 .filter(|value| *value >= base_sequence)
12687 .ok_or_else(|| {
12688 invalid_recorded_history(
12689 "parallel_group_metadata_invalid",
12690 sequence,
12691 "selection member base within its group",
12692 &payload.to_string(),
12693 "selection history contains an invalid member base sequence",
12694 )
12695 })?,
12696 )
12697 } else {
12698 None
12699 };
12700 let selection_member_size = if mode == "select" {
12701 let member_size = required_parallel_usize(payload, "selection_member_size", sequence)?;
12702 if member_size == 0 {
12703 return Err(invalid_recorded_history(
12704 "parallel_group_metadata_invalid",
12705 sequence,
12706 "positive selection member size",
12707 &payload.to_string(),
12708 "selection history contains an invalid member size",
12709 ));
12710 }
12711 Some(member_size)
12712 } else {
12713 None
12714 };
12715 let selection_member_kind = if mode == "select" {
12716 let kind = payload_string(payload, "selection_member_kind").ok_or_else(|| {
12717 invalid_recorded_history(
12718 "parallel_group_metadata_invalid",
12719 sequence,
12720 "selection member operation kind",
12721 &payload.to_string(),
12722 "selection history is missing its authored member kind",
12723 )
12724 })?;
12725 if !matches!(
12726 kind.as_str(),
12727 "activity" | "child" | "timer" | "signal" | "condition" | "group"
12728 ) {
12729 return Err(invalid_recorded_history(
12730 "parallel_group_metadata_invalid",
12731 sequence,
12732 "activity, child, timer, signal, condition, or group selection member kind",
12733 &kind,
12734 "selection history contains an unsupported member kind",
12735 ));
12736 }
12737 Some(kind)
12738 } else {
12739 None
12740 };
12741 if let (Some(member_base), Some(member_size)) =
12742 (selection_member_base_sequence, selection_member_size)
12743 {
12744 let member_end = member_base
12745 .checked_add(u64::try_from(member_size).unwrap_or(u64::MAX))
12746 .ok_or_else(|| {
12747 invalid_recorded_history(
12748 "parallel_group_metadata_invalid",
12749 sequence,
12750 "bounded selection member range",
12751 &payload.to_string(),
12752 "selection member range overflowed",
12753 )
12754 })?;
12755 let group_end = base_sequence
12756 .checked_add(u64::try_from(size).unwrap_or(u64::MAX))
12757 .unwrap_or(u64::MAX);
12758 if sequence < member_base || sequence >= member_end || member_end > group_end {
12759 return Err(invalid_recorded_history(
12760 "parallel_group_metadata_invalid",
12761 sequence,
12762 "workflow sequence within one bounded selection member",
12763 &payload.to_string(),
12764 "selection member range does not contain its durable leaf",
12765 ));
12766 }
12767 }
12768 Ok(ParallelGroupMetadata {
12769 parallel_group_id: group_id,
12770 parallel_group_kind: kind,
12771 parallel_group_base_sequence: base_sequence,
12772 parallel_group_size: size,
12773 parallel_group_index: index,
12774 parallel_group_mode: (mode == "select").then(|| "select".to_string()),
12775 selection_member_key,
12776 selection_member_index,
12777 selection_member_base_sequence,
12778 selection_member_size,
12779 selection_member_kind,
12780 })
12781}
12782
12783fn required_parallel_usize(payload: &Value, field: &str, sequence: u64) -> Result<usize> {
12784 payload
12785 .get(field)
12786 .and_then(value_as_u64)
12787 .and_then(|value| usize::try_from(value).ok())
12788 .ok_or_else(|| {
12789 invalid_recorded_history(
12790 "parallel_group_metadata_invalid",
12791 sequence,
12792 &format!("non-negative integer {field}"),
12793 &payload.to_string(),
12794 "selection history contains invalid member metadata",
12795 )
12796 })
12797}
12798
12799fn selection_key_from_value(value: Option<&Value>, sequence: u64) -> Result<SelectionKey> {
12800 match value {
12801 Some(Value::String(value)) if !value.is_empty() => Ok(SelectionKey::Name(value.clone())),
12802 Some(value) => value_as_u64(value)
12803 .and_then(|value| usize::try_from(value).ok())
12804 .map(SelectionKey::Index)
12805 .ok_or_else(|| {
12806 invalid_recorded_history(
12807 "selection_member_key_invalid",
12808 sequence,
12809 "non-empty string or non-negative integer member key",
12810 &value.to_string(),
12811 "selection history contains an invalid member key",
12812 )
12813 }),
12814 None => Err(invalid_recorded_history(
12815 "selection_member_key_missing",
12816 sequence,
12817 "selection_member_key",
12818 "<missing>",
12819 "selection history is missing its stable member key",
12820 )),
12821 }
12822}
12823
12824fn recorded_parallel_group_path(
12825 events: &[&HistoryEvent],
12826 sequence: u64,
12827) -> Result<Option<Vec<ParallelGroupMetadata>>> {
12828 let mut recorded: Option<Vec<ParallelGroupMetadata>> = None;
12829 for event in events {
12830 let payload = &event.payload;
12831 let has_metadata = payload.get("parallel_group_path").is_some()
12832 || payload.get("parallel_group_id").is_some()
12833 || payload.get("parallel_group_kind").is_some()
12834 || payload.get("parallel_group_base_sequence").is_some()
12835 || payload.get("parallel_group_size").is_some()
12836 || payload.get("parallel_group_index").is_some()
12837 || payload.get("parallel_group_mode").is_some()
12838 || payload.get("selection_member_key").is_some();
12839 if !has_metadata {
12840 continue;
12841 }
12842
12843 let top_level = recorded_parallel_group_entry(payload, sequence)?;
12844 let path = match payload.get("parallel_group_path") {
12845 None => vec![top_level.clone()],
12846 Some(Value::Array(entries)) if !entries.is_empty() => entries
12847 .iter()
12848 .map(|entry| recorded_parallel_group_entry(entry, sequence))
12849 .collect::<Result<Vec<_>>>()?,
12850 Some(value) => {
12851 return Err(invalid_recorded_history(
12852 "parallel_group_metadata_invalid",
12853 sequence,
12854 "non-empty parallel_group_path list",
12855 &value.to_string(),
12856 "parallel-group history contains an invalid group path",
12857 ));
12858 }
12859 };
12860 if path.last() != Some(&top_level) {
12861 return Err(invalid_recorded_history(
12862 "parallel_group_metadata_invalid",
12863 sequence,
12864 &serde_json::to_string(&path.last()).unwrap_or_default(),
12865 &serde_json::to_string(&top_level).unwrap_or_default(),
12866 "parallel-group top-level fields do not match the innermost path entry",
12867 ));
12868 }
12869 if recorded.as_ref().is_some_and(|existing| existing != &path) {
12870 return Err(invalid_recorded_history(
12871 "parallel_group_history_conflict",
12872 sequence,
12873 &serde_json::to_string(&recorded.as_ref()).unwrap_or_default(),
12874 &serde_json::to_string(&path).unwrap_or_default(),
12875 "parallel-group metadata changed between scheduling and resolution history",
12876 ));
12877 }
12878 recorded = Some(path);
12879 }
12880 Ok(recorded)
12881}
12882
12883fn recorded_commands(
12884 events: &[HistoryEvent],
12885 fallback_codec: &str,
12886 parent: WorkflowIdentity,
12887) -> Result<Vec<RecordedCommand>> {
12888 let mut events_by_sequence: BTreeMap<u64, Vec<&HistoryEvent>> = BTreeMap::new();
12889 let mut last_new_sequence = None;
12890
12891 for event in events {
12892 let is_activity = matches!(
12893 event.event_type.as_str(),
12894 "ActivityScheduled"
12895 | "ActivityStarted"
12896 | "ActivityHeartbeatRecorded"
12897 | "ActivityRetryScheduled"
12898 | "ActivityCompleted"
12899 | "ActivityFailed"
12900 | "ActivityCancelled"
12901 | "ActivityTimedOut"
12902 );
12903 let is_workflow_timer = matches!(
12904 event.event_type.as_str(),
12905 "TimerScheduled" | "TimerCancelled" | "TimerFired"
12906 ) && !is_internal_timer_event(event);
12907 let is_child_workflow = matches!(
12908 event.event_type.as_str(),
12909 "ChildWorkflowScheduled"
12910 | "ChildRunCompleted"
12911 | "ChildRunFailed"
12912 | "ChildRunCancelled"
12913 | "ChildRunTerminated"
12914 );
12915 let is_signal_wait = is_recorded_signal_wait_event(event);
12916 let is_condition_wait = is_recorded_condition_wait_event(event);
12917 let is_search_attributes = event.event_type == "SearchAttributesUpserted";
12918 let is_side_effect = event.event_type == "SideEffectRecorded";
12919 let is_version_marker = event.event_type == "VersionMarkerRecorded";
12920 let is_memo = event.event_type == "MemoUpserted";
12921 if !is_activity
12922 && !is_workflow_timer
12923 && !is_child_workflow
12924 && !is_signal_wait
12925 && !is_condition_wait
12926 && !is_search_attributes
12927 && !is_side_effect
12928 && !is_version_marker
12929 && !is_memo
12930 {
12931 continue;
12932 }
12933
12934 let sequence = durable_event_sequence(event).ok_or_else(|| {
12935 Error::NonDeterministicReplay(ReplayFailure::new(
12936 "durable_command_sequence_missing",
12937 None,
12938 Some("positive workflow sequence".to_string()),
12939 Some(event.event_type.clone()),
12940 "durable command history event has no workflow sequence",
12941 ))
12942 })?;
12943 if sequence == 0 {
12944 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12945 "durable_command_sequence_invalid",
12946 Some(sequence),
12947 Some("positive workflow sequence".to_string()),
12948 Some(sequence.to_string()),
12949 "durable command history uses an invalid workflow sequence",
12950 )));
12951 }
12952 if !events_by_sequence.contains_key(&sequence) {
12953 if let Some(previous) = last_new_sequence {
12954 if sequence < previous {
12955 return Err(invalid_recorded_history(
12956 "durable_command_sequence_mismatch",
12957 sequence,
12958 &format!("workflow sequence greater than {previous}"),
12959 &sequence.to_string(),
12960 "durable commands are not strictly ordered by their recorded workflow sequence",
12961 ));
12962 }
12963 }
12964 last_new_sequence = Some(sequence);
12965 }
12966 events_by_sequence.entry(sequence).or_default().push(event);
12967 }
12968
12969 let commands: Vec<RecordedCommand> = events_by_sequence
12970 .into_iter()
12971 .map(|(sequence, sequence_events)| {
12972 let activity_events: Vec<_> = sequence_events
12973 .iter()
12974 .copied()
12975 .filter(|event| event.event_type.starts_with("Activity"))
12976 .collect();
12977 let timer_events: Vec<_> = sequence_events
12978 .iter()
12979 .copied()
12980 .filter(|event| event.event_type.starts_with("Timer"))
12981 .collect();
12982 let child_events: Vec<_> = sequence_events
12983 .iter()
12984 .copied()
12985 .filter(|event| {
12986 event.event_type == "ChildWorkflowScheduled"
12987 || event.event_type.starts_with("ChildRun")
12988 })
12989 .collect();
12990 let signal_wait_events: Vec<_> = sequence_events
12991 .iter()
12992 .copied()
12993 .filter(|event| is_recorded_signal_wait_event(event))
12994 .collect();
12995 let condition_wait_events: Vec<_> = sequence_events
12996 .iter()
12997 .copied()
12998 .filter(|event| is_recorded_condition_wait_event(event))
12999 .collect();
13000 let search_attribute_events: Vec<_> = sequence_events
13001 .iter()
13002 .copied()
13003 .filter(|event| event.event_type == "SearchAttributesUpserted")
13004 .collect();
13005 let side_effect_events: Vec<_> = sequence_events
13006 .iter()
13007 .copied()
13008 .filter(|event| event.event_type == "SideEffectRecorded")
13009 .collect();
13010 let version_marker_events: Vec<_> = sequence_events
13011 .iter()
13012 .copied()
13013 .filter(|event| event.event_type == "VersionMarkerRecorded")
13014 .collect();
13015 let memo_events: Vec<_> = sequence_events
13016 .iter()
13017 .copied()
13018 .filter(|event| event.event_type == "MemoUpserted")
13019 .collect();
13020
13021 let command_kind_count = usize::from(!activity_events.is_empty())
13022 + usize::from(!timer_events.is_empty())
13023 + usize::from(!child_events.is_empty())
13024 + usize::from(!signal_wait_events.is_empty())
13025 + usize::from(!condition_wait_events.is_empty())
13026 + usize::from(!search_attribute_events.is_empty())
13027 + usize::from(!side_effect_events.is_empty())
13028 + usize::from(!version_marker_events.is_empty())
13029 + usize::from(!memo_events.is_empty());
13030 if command_kind_count > 1 {
13031 let actual = [
13032 (!activity_events.is_empty()).then_some("activity"),
13033 (!timer_events.is_empty()).then_some("timer"),
13034 (!child_events.is_empty()).then_some("child workflow"),
13035 (!signal_wait_events.is_empty()).then_some("signal wait"),
13036 (!condition_wait_events.is_empty()).then_some("condition wait"),
13037 (!search_attribute_events.is_empty()).then_some("search-attribute update"),
13038 (!side_effect_events.is_empty()).then_some("side effect"),
13039 (!version_marker_events.is_empty()).then_some("version marker"),
13040 (!memo_events.is_empty()).then_some("memo upsert"),
13041 ]
13042 .into_iter()
13043 .flatten()
13044 .collect::<Vec<_>>()
13045 .join(" and ");
13046 return Err(invalid_recorded_history(
13047 "durable_command_sequence_collision",
13048 sequence,
13049 "one durable command kind",
13050 &actual,
13051 "one workflow sequence records more than one durable command kind",
13052 ));
13053 }
13054
13055 if !activity_events.is_empty() {
13056 let parallel_group_path =
13057 recorded_parallel_group_path(&activity_events, sequence)?;
13058 let scheduled_count = activity_events
13059 .iter()
13060 .filter(|event| event.event_type == "ActivityScheduled")
13061 .count();
13062 if scheduled_count > 1 {
13063 return Err(invalid_recorded_history(
13064 "duplicate_activity_schedule",
13065 sequence,
13066 "at most one ActivityScheduled event",
13067 "multiple ActivityScheduled events",
13068 "activity history schedules more than one command at one workflow sequence",
13069 ));
13070 }
13071 let activity_type = activity_events.iter().find_map(|event| {
13072 event
13073 .payload
13074 .get("activity_type")
13075 .or_else(|| event.payload.get("activity_name"))
13076 .and_then(Value::as_str)
13077 .map(str::to_string)
13078 });
13079 if activity_events.iter().filter_map(|event| {
13080 event
13081 .payload
13082 .get("activity_type")
13083 .or_else(|| event.payload.get("activity_name"))
13084 .and_then(Value::as_str)
13085 }).any(|candidate| Some(candidate) != activity_type.as_deref()) {
13086 return Err(invalid_recorded_history(
13087 "activity_identity_mismatch",
13088 sequence,
13089 activity_type.as_deref().unwrap_or("one activity identity"),
13090 "conflicting activity identities",
13091 "activity lifecycle events at one workflow sequence disagree on identity",
13092 ));
13093 }
13094 let terminal: Vec<_> = activity_events
13095 .iter()
13096 .copied()
13097 .filter(|event| {
13098 matches!(
13099 event.event_type.as_str(),
13100 "ActivityCompleted"
13101 | "ActivityFailed"
13102 | "ActivityCancelled"
13103 | "ActivityTimedOut"
13104 )
13105 })
13106 .collect();
13107 let duplicate_delivery = terminal.first().is_some_and(|first| {
13108 terminal.iter().all(|event| {
13109 event.event_type == first.event_type && event.payload == first.payload
13110 })
13111 });
13112 if terminal.len() > 1 && !duplicate_delivery {
13113 return Err(invalid_recorded_history(
13114 "duplicate_activity_terminal_event",
13115 sequence,
13116 "at most one terminal activity event",
13117 "multiple terminal activity events",
13118 "activity history settles one command more than once",
13119 ));
13120 }
13121 let outcome = terminal
13122 .first()
13123 .map(|event| activity_outcome(event, fallback_codec, activity_type.clone()))
13124 .transpose()?;
13125 let options = activity_events
13126 .iter()
13127 .find(|event| event.event_type == "ActivityScheduled")
13128 .and_then(|event| event.payload.get("activity"))
13129 .and_then(Value::as_object)
13130 .map(|activity| RecordedActivityOptions {
13131 task_queue: recorded_optional_string(activity, "queue"),
13132 execution_mode: recorded_optional_string(activity, "execution_mode"),
13133 retry_policy: recorded_activity_retry_snapshot(
13134 activity.get("retry_policy"),
13135 ),
13136 });
13137 return Ok(RecordedCommand::Activity {
13138 sequence,
13139 activity_type,
13140 options,
13141 outcome,
13142 parallel_group_path,
13143 });
13144 }
13145
13146 if !child_events.is_empty() {
13147 let parallel_group_path = recorded_parallel_group_path(&child_events, sequence)?;
13148 let scheduled: Vec<_> = child_events
13149 .iter()
13150 .copied()
13151 .filter(|event| event.event_type == "ChildWorkflowScheduled")
13152 .collect();
13153 if scheduled.len() != 1 {
13154 return Err(invalid_recorded_history(
13155 "child_workflow_schedule_missing_or_duplicate",
13156 sequence,
13157 "one ChildWorkflowScheduled event",
13158 &format!("{} ChildWorkflowScheduled events", scheduled.len()),
13159 "child workflow replay requires exactly one recorded schedule event",
13160 ));
13161 }
13162 let workflow_type = child_events.iter().find_map(|event| {
13163 event
13164 .payload
13165 .get("child_workflow_type")
13166 .or_else(|| event.payload.get("workflow_type"))
13167 .and_then(Value::as_str)
13168 .filter(|value| !value.is_empty())
13169 .map(str::to_string)
13170 });
13171 if child_events
13172 .iter()
13173 .filter_map(|event| {
13174 event
13175 .payload
13176 .get("child_workflow_type")
13177 .or_else(|| event.payload.get("workflow_type"))
13178 .and_then(Value::as_str)
13179 })
13180 .any(|candidate| Some(candidate) != workflow_type.as_deref())
13181 {
13182 return Err(invalid_recorded_history(
13183 "child_workflow_identity_mismatch",
13184 sequence,
13185 workflow_type
13186 .as_deref()
13187 .unwrap_or("one child workflow type"),
13188 "conflicting child workflow types",
13189 "child workflow lifecycle events at one sequence disagree on type",
13190 ));
13191 }
13192 let mut outcomes = child_workflow_outcomes(
13193 &child_events.iter().map(|event| (*event).clone()).collect::<Vec<_>>(),
13194 fallback_codec,
13195 parent.clone(),
13196 )?;
13197 let terminal_events = child_events
13198 .iter()
13199 .copied()
13200 .filter(|event| event.event_type.starts_with("ChildRun"))
13201 .collect::<Vec<_>>();
13202 let duplicate_delivery = terminal_events.first().is_some_and(|first| {
13203 terminal_events.iter().all(|event| {
13204 event.event_type == first.event_type && event.payload == first.payload
13205 })
13206 });
13207 if outcomes.len() > 1 && !duplicate_delivery {
13208 return Err(invalid_recorded_history(
13209 "duplicate_child_workflow_terminal_event",
13210 sequence,
13211 "at most one terminal child event",
13212 "multiple terminal child events",
13213 "child workflow history settles one command more than once",
13214 ));
13215 }
13216 return Ok(RecordedCommand::ChildWorkflow {
13217 sequence,
13218 workflow_type,
13219 outcome: outcomes.pop(),
13220 parallel_group_path,
13221 });
13222 }
13223
13224 if !signal_wait_events.is_empty() {
13225 let opened: Vec<_> = signal_wait_events
13226 .iter()
13227 .copied()
13228 .filter(|event| event.event_type == "SignalWaitOpened")
13229 .collect();
13230 if opened.len() != 1 {
13231 return Err(invalid_recorded_history(
13232 "signal_wait_open_missing_or_duplicate",
13233 sequence,
13234 "one SignalWaitOpened event",
13235 &format!("{} SignalWaitOpened events", opened.len()),
13236 "signal replay requires exactly one canonical wait-open event",
13237 ));
13238 }
13239
13240 let applied: Vec<_> = signal_wait_events
13241 .iter()
13242 .copied()
13243 .filter(|event| event.event_type == "SignalApplied")
13244 .collect();
13245 if applied.len() > 1 {
13246 return Err(invalid_recorded_history(
13247 "duplicate_signal_wait_apply",
13248 sequence,
13249 "at most one SignalApplied event",
13250 "multiple SignalApplied events",
13251 "signal history applies one durable wait more than once",
13252 ));
13253 }
13254
13255 let signal_names = signal_wait_events
13256 .iter()
13257 .map(|event| required_signal_wait_name(event, sequence))
13258 .collect::<Result<Vec<_>>>()?;
13259 let signal_name = signal_names
13260 .first()
13261 .expect("signal wait events are not empty")
13262 .clone();
13263 if signal_names.iter().any(|candidate| candidate != &signal_name) {
13264 return Err(invalid_recorded_history(
13265 "signal_wait_identity_mismatch",
13266 sequence,
13267 &signal_name,
13268 "conflicting signal names",
13269 "signal wait lifecycle events at one workflow sequence disagree on identity",
13270 ));
13271 }
13272 let value = applied
13273 .first()
13274 .map(|event| decode_signal_event_arguments(event, fallback_codec))
13275 .transpose()?;
13276 return Ok(RecordedCommand::SignalWait {
13277 sequence,
13278 signal_name,
13279 value,
13280 parallel_group_path: recorded_parallel_group_path(
13281 &signal_wait_events,
13282 sequence,
13283 )?,
13284 });
13285 }
13286
13287 if !condition_wait_events.is_empty() {
13288 return recorded_condition_wait(
13289 sequence,
13290 &condition_wait_events,
13291 events,
13292 );
13293 }
13294
13295 if !search_attribute_events.is_empty() {
13296 if search_attribute_events.len() != 1 {
13297 return Err(invalid_recorded_history(
13298 "duplicate_search_attribute_update",
13299 sequence,
13300 "one SearchAttributesUpserted event",
13301 &format!(
13302 "{} SearchAttributesUpserted events",
13303 search_attribute_events.len()
13304 ),
13305 "search-attribute history records one workflow command more than once",
13306 ));
13307 }
13308 let payload = &search_attribute_events[0].payload;
13309 let attributes = payload
13310 .get("attributes")
13311 .filter(|value| value.as_object().is_some_and(|values| !values.is_empty()))
13312 .cloned()
13313 .ok_or_else(|| {
13314 invalid_recorded_history(
13315 "search_attribute_update_missing",
13316 sequence,
13317 "non-empty attributes object",
13318 "missing or invalid attributes",
13319 "search-attribute history is missing its recorded mutation",
13320 )
13321 })?;
13322 let attribute_types =
13323 recorded_search_attribute_types(payload, &attributes, sequence)?;
13324 return Ok(RecordedCommand::SearchAttributes {
13325 sequence,
13326 attributes,
13327 attribute_types,
13328 });
13329 }
13330
13331 if !side_effect_events.is_empty() {
13332 if side_effect_events.len() != 1 {
13333 return Err(invalid_recorded_history(
13334 "duplicate_side_effect_record",
13335 sequence,
13336 "one SideEffectRecorded event",
13337 &format!("{} SideEffectRecorded events", side_effect_events.len()),
13338 "side-effect history records one workflow command more than once",
13339 ));
13340 }
13341 let event = side_effect_events[0];
13342 let result = event.payload.get("result").ok_or_else(|| {
13343 invalid_recorded_history(
13344 "side_effect_result_missing",
13345 sequence,
13346 "recorded result payload",
13347 "missing result",
13348 "side-effect history is missing its recorded value",
13349 )
13350 })?;
13351 let has_published_envelope = result.as_str().is_some()
13352 || result.as_object().is_some_and(|envelope| {
13353 envelope.get("codec").and_then(Value::as_str).is_some()
13354 && envelope.get("blob").and_then(Value::as_str).is_some()
13355 });
13356 if !has_published_envelope {
13357 return Err(invalid_recorded_history(
13358 "side_effect_payload_malformed",
13359 sequence,
13360 "payload blob or {codec, blob} envelope",
13361 &result.to_string(),
13362 "side-effect history result does not use a published payload envelope",
13363 ));
13364 }
13365 let codec = event
13366 .payload
13367 .get("payload_codec")
13368 .and_then(Value::as_str)
13369 .unwrap_or(fallback_codec);
13370 let value = decode_wire_avro_value(result, codec).map_err(|error| {
13371 if error.to_string().contains("unsupported_payload_codec") {
13372 return error;
13373 }
13374
13375 invalid_recorded_history(
13376 "side_effect_payload_incompatible",
13377 sequence,
13378 &format!("valid {codec} payload envelope"),
13379 &error.to_string(),
13380 "side-effect history payload cannot be decoded with its recorded codec",
13381 )
13382 })?;
13383 return Ok(RecordedCommand::SideEffect { sequence, value });
13384 }
13385
13386 if !version_marker_events.is_empty() {
13387 if version_marker_events.len() != 1 {
13388 return Err(invalid_recorded_history(
13389 "duplicate_version_marker_record",
13390 sequence,
13391 "one VersionMarkerRecorded event",
13392 &format!("{} VersionMarkerRecorded events", version_marker_events.len()),
13393 "version-marker history records one workflow command more than once",
13394 ));
13395 }
13396 let payload = &version_marker_events[0].payload;
13397 let change_id = payload
13398 .get("change_id")
13399 .and_then(Value::as_str)
13400 .filter(|value| !value.is_empty())
13401 .map(str::to_string)
13402 .ok_or_else(|| {
13403 invalid_recorded_history(
13404 "version_marker_field_missing",
13405 sequence,
13406 "non-empty change_id",
13407 "missing or invalid change_id",
13408 "version-marker history is missing its stable change ID",
13409 )
13410 })?;
13411 let version = required_version_i32(payload, "version", sequence)?;
13412 let min_supported = required_version_i32(payload, "min_supported", sequence)?;
13413 let max_supported = required_version_i32(payload, "max_supported", sequence)?;
13414 if min_supported > max_supported || version < min_supported || version > max_supported {
13415 return Err(invalid_recorded_history(
13416 "version_marker_history_range_invalid",
13417 sequence,
13418 "min_supported <= version <= max_supported",
13419 &format!("{min_supported} <= {version} <= {max_supported}"),
13420 "recorded version marker contains an internally incompatible range",
13421 ));
13422 }
13423 return Ok(RecordedCommand::VersionMarker {
13424 sequence,
13425 change_id,
13426 version,
13427 });
13428 }
13429
13430 if !memo_events.is_empty() {
13431 if memo_events.len() != 1 {
13432 return Err(invalid_recorded_history(
13433 "duplicate_memo_upsert_record",
13434 sequence,
13435 "one MemoUpserted event",
13436 &format!("{} MemoUpserted events", memo_events.len()),
13437 "memo history records one workflow update more than once",
13438 ));
13439 }
13440 let payload = &memo_events[0].payload;
13441 let entries = payload.get("entries").cloned().ok_or_else(|| {
13442 invalid_recorded_history(
13443 "memo_entries_missing",
13444 sequence,
13445 "memo entries object",
13446 "missing entries",
13447 "MemoUpserted history is missing replay identity entries",
13448 )
13449 })?;
13450 let entries = decode_memo_history_map(&entries, true).map_err(|error| {
13451 invalid_recorded_history(
13452 "memo_entries_invalid",
13453 sequence,
13454 "valid canonical memo entries",
13455 &error.to_string(),
13456 "MemoUpserted history contains invalid replay identity entries",
13457 )
13458 })?;
13459 let merged = payload.get("merged").cloned().ok_or_else(|| {
13460 invalid_recorded_history(
13461 "memo_merged_projection_missing",
13462 sequence,
13463 "merged memo projection",
13464 "missing merged",
13465 "MemoUpserted history is missing its merged projection",
13466 )
13467 })?;
13468 decode_memo_history_map(&merged, false).map_err(|error| {
13469 invalid_recorded_history(
13470 "memo_merged_projection_invalid",
13471 sequence,
13472 "valid merged memo projection",
13473 &error.to_string(),
13474 "MemoUpserted history contains an invalid merged projection",
13475 )
13476 })?;
13477
13478 return Ok(RecordedCommand::Memo { sequence, entries });
13479 }
13480 let scheduled: Vec<_> = timer_events
13481 .iter()
13482 .copied()
13483 .filter(|event| event.event_type == "TimerScheduled")
13484 .collect();
13485 let fired: Vec<_> = timer_events
13486 .iter()
13487 .copied()
13488 .filter(|event| event.event_type == "TimerFired")
13489 .collect();
13490 if scheduled.len() != 1 {
13491 return Err(invalid_recorded_history(
13492 "timer_schedule_missing_or_duplicate",
13493 sequence,
13494 "one TimerScheduled event",
13495 &format!("{} TimerScheduled events", scheduled.len()),
13496 "timer replay requires exactly one recorded schedule event",
13497 ));
13498 }
13499 if fired.len() > 1 {
13500 return Err(invalid_recorded_history(
13501 "duplicate_timer_fire",
13502 sequence,
13503 "at most one TimerFired event",
13504 "multiple TimerFired events",
13505 "timer history contains more than one fire event for a workflow sequence",
13506 ));
13507 }
13508
13509 let scheduled = scheduled[0];
13510 let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
13511 let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
13512 if let Some(fired) = fired.first() {
13513 let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
13514 if fired_timer_id != timer_id {
13515 return Err(invalid_recorded_history(
13516 "timer_identity_mismatch",
13517 sequence,
13518 &timer_id,
13519 &fired_timer_id,
13520 "TimerFired does not correspond to the recorded TimerScheduled event",
13521 ));
13522 }
13523 let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
13524 if fired_delay != delay_seconds {
13525 return Err(invalid_recorded_history(
13526 "timer_history_delay_mismatch",
13527 sequence,
13528 &delay_seconds.to_string(),
13529 &fired_delay.to_string(),
13530 "TimerScheduled and TimerFired record different delays",
13531 ));
13532 }
13533 }
13534
13535 Ok(RecordedCommand::Timer {
13536 sequence,
13537 delay_seconds,
13538 fired: !fired.is_empty(),
13539 parallel_group_path: recorded_parallel_group_path(&timer_events, sequence)?,
13540 })
13541 })
13542 .collect::<Result<_>>()?;
13543
13544 let mut marker_sequences = HashMap::new();
13545 for command in &commands {
13546 if let RecordedCommand::VersionMarker {
13547 sequence,
13548 change_id,
13549 ..
13550 } = command
13551 {
13552 if let Some(first_sequence) = marker_sequences.insert(change_id.clone(), *sequence) {
13553 return Err(invalid_recorded_history(
13554 "duplicate_version_marker",
13555 *sequence,
13556 &format!("one marker for change ID {change_id:?}"),
13557 &format!("markers at sequences {first_sequence} and {sequence}"),
13558 "workflow history contains duplicate markers for one stable change ID",
13559 ));
13560 }
13561 }
13562 }
13563
13564 Ok(commands)
13565}
13566
13567fn required_version_i32(payload: &Value, field: &str, sequence: u64) -> Result<i32> {
13568 payload
13569 .get(field)
13570 .and_then(Value::as_i64)
13571 .and_then(|value| i32::try_from(value).ok())
13572 .ok_or_else(|| {
13573 invalid_recorded_history(
13574 "version_marker_field_missing",
13575 sequence,
13576 &format!("integer {field}"),
13577 "missing or out-of-range integer",
13578 "version-marker history is missing a required integer field",
13579 )
13580 })
13581}
13582
13583fn durable_event_sequence(event: &HistoryEvent) -> Option<u64> {
13584 event
13585 .payload
13586 .get("sequence")
13587 .or_else(|| event.payload.get("workflow_sequence"))
13588 .or_else(|| event.raw.get("sequence"))
13589 .or_else(|| event.raw.get("workflow_sequence"))
13590 .and_then(value_as_u64)
13591}
13592
13593fn is_internal_timer_event(event: &HistoryEvent) -> bool {
13594 matches!(
13595 event
13596 .payload
13597 .get("timer_kind")
13598 .or_else(|| event.raw.get("timer_kind"))
13599 .and_then(Value::as_str),
13600 Some("condition_timeout" | "signal_timeout")
13601 )
13602}
13603
13604fn is_recorded_condition_wait_event(event: &HistoryEvent) -> bool {
13605 matches!(
13606 event.event_type.as_str(),
13607 "ConditionWaitOpened" | "ConditionWaitSatisfied" | "ConditionWaitTimedOut"
13608 )
13609}
13610
13611fn recorded_condition_wait(
13612 sequence: u64,
13613 condition_events: &[&HistoryEvent],
13614 all_events: &[HistoryEvent],
13615) -> Result<RecordedCommand> {
13616 let opened = condition_events
13617 .iter()
13618 .copied()
13619 .filter(|event| event.event_type == "ConditionWaitOpened")
13620 .collect::<Vec<_>>();
13621 if opened.len() != 1 {
13622 return Err(invalid_recorded_history(
13623 "condition_wait_open_missing_or_duplicate",
13624 sequence,
13625 "one ConditionWaitOpened event",
13626 &format!("{} ConditionWaitOpened events", opened.len()),
13627 "condition replay requires exactly one canonical wait-open event",
13628 ));
13629 }
13630 let terminal = condition_events
13631 .iter()
13632 .copied()
13633 .filter(|event| {
13634 matches!(
13635 event.event_type.as_str(),
13636 "ConditionWaitSatisfied" | "ConditionWaitTimedOut"
13637 )
13638 })
13639 .collect::<Vec<_>>();
13640 if terminal.len() > 1 {
13641 return Err(invalid_recorded_history(
13642 "duplicate_condition_wait_terminal_event",
13643 sequence,
13644 "at most one condition terminal event",
13645 "multiple condition terminal events",
13646 "condition history settles one durable wait more than once",
13647 ));
13648 }
13649
13650 let opened = opened[0];
13651 let condition_wait_id = required_condition_wait_id(opened, sequence)?;
13652 let occurrence_id = required_condition_wait_occurrence_id(opened, sequence)?;
13653 for event in condition_events
13654 .iter()
13655 .copied()
13656 .filter(|event| !std::ptr::eq(*event, opened))
13657 {
13658 let event_wait_id = required_condition_wait_id(event, sequence)?;
13659 if event_wait_id != condition_wait_id {
13660 return Err(invalid_recorded_history(
13661 "condition_wait_id_mismatch",
13662 sequence,
13663 &condition_wait_id,
13664 &event_wait_id,
13665 "condition lifecycle events at one sequence disagree on wait identity",
13666 ));
13667 }
13668 let event_occurrence_id = required_condition_wait_occurrence_id(event, sequence)?;
13669 if event_occurrence_id != occurrence_id {
13670 return Err(invalid_recorded_history(
13671 "condition_wait_occurrence_history_mismatch",
13672 sequence,
13673 &occurrence_id,
13674 &event_occurrence_id,
13675 "condition lifecycle events at one sequence disagree on authored occurrence identity",
13676 ));
13677 }
13678 }
13679
13680 let condition_key = optional_non_empty_history_string(opened, "condition_key");
13681 let predicate_identity = opened
13682 .payload
13683 .get("condition_definition_fingerprint")
13684 .and_then(Value::as_str)
13685 .filter(|value| !value.is_empty())
13686 .map(str::to_string)
13687 .ok_or_else(|| {
13688 invalid_recorded_history(
13689 "condition_wait_predicate_fingerprint_missing",
13690 sequence,
13691 "non-empty condition_definition_fingerprint",
13692 &opened.event_type,
13693 "canonical condition history is missing its predicate identity",
13694 )
13695 })?;
13696 let timeout_seconds = optional_history_u64(opened, "timeout_seconds", sequence)?;
13697 for event in condition_events
13698 .iter()
13699 .copied()
13700 .filter(|event| !std::ptr::eq(*event, opened))
13701 {
13702 for (field, opened_value) in [
13703 ("condition_key", condition_key.as_deref()),
13704 (
13705 "condition_definition_fingerprint",
13706 Some(predicate_identity.as_str()),
13707 ),
13708 ] {
13709 if let Some(value) = optional_non_empty_history_string(event, field) {
13710 if opened_value.is_some_and(|opened_value| opened_value != value) {
13711 return Err(invalid_recorded_history(
13712 "condition_wait_definition_history_mismatch",
13713 sequence,
13714 opened_value.unwrap_or_default(),
13715 &value,
13716 "condition lifecycle events disagree on the recorded definition",
13717 ));
13718 }
13719 }
13720 }
13721 if let Some(event_timeout) = optional_history_u64(event, "timeout_seconds", sequence)? {
13722 if timeout_seconds.is_some_and(|opened_timeout| opened_timeout != event_timeout) {
13723 return Err(invalid_recorded_history(
13724 "condition_wait_definition_history_mismatch",
13725 sequence,
13726 &format!("{}s", timeout_seconds.unwrap_or_default()),
13727 &format!("{event_timeout}s"),
13728 "condition lifecycle events disagree on the recorded timeout",
13729 ));
13730 }
13731 }
13732 }
13733
13734 let timeout_timer_events = all_events
13735 .iter()
13736 .filter(|event| {
13737 matches!(
13738 event.event_type.as_str(),
13739 "TimerScheduled" | "TimerCancelled" | "TimerFired"
13740 ) && event.payload.get("timer_kind").and_then(Value::as_str)
13741 == Some("condition_timeout")
13742 && event
13743 .payload
13744 .get("condition_wait_id")
13745 .and_then(Value::as_str)
13746 == Some(condition_wait_id.as_str())
13747 })
13748 .collect::<Vec<_>>();
13749 let scheduled = timeout_timer_events
13750 .iter()
13751 .copied()
13752 .filter(|event| event.event_type == "TimerScheduled")
13753 .collect::<Vec<_>>();
13754 let fired = timeout_timer_events
13755 .iter()
13756 .copied()
13757 .filter(|event| event.event_type == "TimerFired")
13758 .collect::<Vec<_>>();
13759 if scheduled.len() > 1 || fired.len() > 1 || (!fired.is_empty() && scheduled.len() != 1) {
13760 return Err(invalid_recorded_history(
13761 "condition_wait_timeout_history_invalid",
13762 sequence,
13763 "one timeout schedule and at most one fire",
13764 &format!("{} schedules and {} fires", scheduled.len(), fired.len()),
13765 "condition timeout history has a missing or duplicate lifecycle event",
13766 ));
13767 }
13768 if let Some(scheduled) = scheduled.first() {
13769 let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
13770 let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
13771 if timeout_seconds.is_some_and(|timeout| timeout != delay_seconds) {
13772 return Err(invalid_recorded_history(
13773 "condition_wait_timeout_delay_mismatch",
13774 sequence,
13775 &format!("{}s", timeout_seconds.unwrap_or_default()),
13776 &format!("{delay_seconds}s"),
13777 "condition timeout timer differs from the wait definition",
13778 ));
13779 }
13780 if let Some(fired) = fired.first() {
13781 let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
13782 let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
13783 if fired_timer_id != timer_id || fired_delay != delay_seconds {
13784 return Err(invalid_recorded_history(
13785 "condition_wait_timeout_identity_mismatch",
13786 sequence,
13787 &format!("{timer_id}:{delay_seconds}s"),
13788 &format!("{fired_timer_id}:{fired_delay}s"),
13789 "condition timeout fire does not match its durable schedule",
13790 ));
13791 }
13792 }
13793 }
13794
13795 let result = terminal.first().map(|event| {
13796 if event.event_type == "ConditionWaitTimedOut" {
13797 ConditionWaitResult::TimedOut
13798 } else {
13799 ConditionWaitResult::Satisfied
13800 }
13801 });
13802 let result = if !fired.is_empty() {
13803 if result == Some(ConditionWaitResult::Satisfied) {
13804 return Err(invalid_recorded_history(
13805 "condition_wait_terminal_conflict",
13806 sequence,
13807 "one satisfied or timed-out outcome",
13808 "satisfied event and fired timeout",
13809 "condition history records conflicting terminal outcomes",
13810 ));
13811 }
13812 Some(ConditionWaitResult::TimedOut)
13813 } else {
13814 result
13815 };
13816
13817 Ok(RecordedCommand::ConditionWait {
13818 sequence,
13819 occurrence_id,
13820 condition_key,
13821 predicate_identity,
13822 timeout_seconds,
13823 result,
13824 parallel_group_path: recorded_parallel_group_path(condition_events, sequence)?,
13825 })
13826}
13827
13828fn required_condition_wait_occurrence_id(event: &HistoryEvent, sequence: u64) -> Result<String> {
13829 event
13830 .payload
13831 .get("condition_wait_occurrence_id")
13832 .and_then(Value::as_str)
13833 .filter(|value| !value.is_empty())
13834 .map(str::to_string)
13835 .ok_or_else(|| {
13836 invalid_recorded_history(
13837 "condition_wait_occurrence_id_missing",
13838 sequence,
13839 "non-empty condition_wait_occurrence_id",
13840 &event.event_type,
13841 "condition history is missing authored occurrence identity",
13842 )
13843 })
13844}
13845
13846fn required_condition_wait_id(event: &HistoryEvent, sequence: u64) -> Result<String> {
13847 event
13848 .payload
13849 .get("condition_wait_id")
13850 .and_then(Value::as_str)
13851 .filter(|value| !value.is_empty())
13852 .map(str::to_string)
13853 .ok_or_else(|| {
13854 invalid_recorded_history(
13855 "condition_wait_id_missing",
13856 sequence,
13857 "non-empty condition_wait_id",
13858 &event.event_type,
13859 "canonical condition history is missing its durable wait identity",
13860 )
13861 })
13862}
13863
13864fn optional_non_empty_history_string(event: &HistoryEvent, field: &str) -> Option<String> {
13865 event
13866 .payload
13867 .get(field)
13868 .and_then(Value::as_str)
13869 .filter(|value| !value.is_empty())
13870 .map(str::to_string)
13871}
13872
13873fn optional_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<Option<u64>> {
13874 match event.payload.get(field) {
13875 None | Some(Value::Null) => Ok(None),
13876 Some(value) => value_as_u64(value).map(Some).ok_or_else(|| {
13877 invalid_recorded_history(
13878 "condition_wait_definition_invalid",
13879 sequence,
13880 &format!("non-negative integer {field}"),
13881 &value.to_string(),
13882 "condition history contains an invalid numeric definition field",
13883 )
13884 }),
13885 }
13886}
13887
13888fn required_signal_wait_name(event: &HistoryEvent, sequence: u64) -> Result<String> {
13889 event
13890 .payload
13891 .get("signal_name")
13892 .or_else(|| event.raw.get("signal_name"))
13893 .and_then(Value::as_str)
13894 .filter(|value| !value.is_empty())
13895 .map(str::to_string)
13896 .ok_or_else(|| {
13897 invalid_recorded_history(
13898 "signal_wait_name_missing",
13899 sequence,
13900 "non-empty signal_name",
13901 &event.event_type,
13902 "canonical signal-wait history is missing its signal identity",
13903 )
13904 })
13905}
13906
13907fn is_recorded_signal_wait_event(event: &HistoryEvent) -> bool {
13908 matches!(
13909 event.event_type.as_str(),
13910 "SignalWaitOpened" | "SignalApplied"
13911 )
13912}
13913
13914fn required_history_string(event: &HistoryEvent, field: &str, sequence: u64) -> Result<String> {
13915 event
13916 .payload
13917 .get(field)
13918 .and_then(Value::as_str)
13919 .filter(|value| !value.is_empty())
13920 .map(str::to_string)
13921 .ok_or_else(|| {
13922 invalid_recorded_history(
13923 "timer_history_field_missing",
13924 sequence,
13925 field,
13926 &event.event_type,
13927 "timer history is missing a required identity field",
13928 )
13929 })
13930}
13931
13932fn required_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<u64> {
13933 event
13934 .payload
13935 .get(field)
13936 .and_then(value_as_u64)
13937 .ok_or_else(|| {
13938 invalid_recorded_history(
13939 "timer_history_field_missing",
13940 sequence,
13941 field,
13942 &event.event_type,
13943 "timer history is missing a required numeric field",
13944 )
13945 })
13946}
13947
13948fn recorded_search_attribute_types(
13949 payload: &Value,
13950 attributes: &Value,
13951 sequence: u64,
13952) -> Result<RecordedSnapshotValue<BTreeMap<String, String>>> {
13953 let Some(raw_types) = payload.get("attribute_types") else {
13954 return Ok(RecordedSnapshotValue::Unknown);
13958 };
13959 let Some(raw_types) = raw_types.as_object() else {
13960 return Err(invalid_recorded_history(
13961 "search_attribute_types_malformed",
13962 sequence,
13963 "canonical attribute type map",
13964 &raw_types.to_string(),
13965 "search-attribute history contains malformed type identity",
13966 ));
13967 };
13968 let attribute_keys = attributes
13969 .as_object()
13970 .expect("recorded search attributes were validated as an object");
13971 let mut types = BTreeMap::new();
13972 for (key, value) in raw_types {
13973 let Some(attribute_type) = value.as_str() else {
13974 return Err(invalid_recorded_history(
13975 "search_attribute_types_malformed",
13976 sequence,
13977 "canonical string type name",
13978 &value.to_string(),
13979 "search-attribute history contains a non-string type identity",
13980 ));
13981 };
13982 if !attribute_keys.contains_key(key)
13983 || !matches!(
13984 attribute_type,
13985 "string" | "keyword" | "keyword_list" | "int" | "float" | "bool" | "datetime"
13986 )
13987 {
13988 return Err(invalid_recorded_history(
13989 "search_attribute_types_malformed",
13990 sequence,
13991 "canonical types for keys present in attributes",
13992 &format!("{key}:{attribute_type}"),
13993 "search-attribute history contains unsupported or orphaned type identity",
13994 ));
13995 }
13996 types.insert(key.clone(), attribute_type.to_string());
13997 }
13998 Ok(RecordedSnapshotValue::Known(types))
13999}
14000
14001fn invalid_recorded_history(
14002 reason: &str,
14003 sequence: u64,
14004 expected: &str,
14005 actual: &str,
14006 message: &str,
14007) -> Error {
14008 Error::NonDeterministicReplay(ReplayFailure::new(
14009 reason,
14010 Some(sequence),
14011 Some(expected.to_string()),
14012 Some(actual.to_string()),
14013 message,
14014 ))
14015}
14016
14017type ActivityOutcome = std::result::Result<AvroValue, ActivityFailure>;
14018
14019fn activity_outcome(
14020 event: &HistoryEvent,
14021 fallback_codec: &str,
14022 recorded_activity_type: Option<String>,
14023) -> Result<ActivityOutcome> {
14024 if event.event_type == "ActivityCompleted" {
14025 let codec = event
14026 .payload
14027 .get("payload_codec")
14028 .and_then(Value::as_str)
14029 .unwrap_or(fallback_codec);
14030 return Ok(Ok(decode_wire_avro_value(
14031 event.payload.get("result").unwrap_or(&Value::Null),
14032 codec,
14033 )?));
14034 }
14035
14036 let payload = &event.payload;
14037 let (kind, fallback_reason, fallback_message) = match event.event_type.as_str() {
14038 "ActivityFailed" => (ActivityFailureKind::Failed, "activity", "activity failed"),
14039 "ActivityCancelled" => (
14040 ActivityFailureKind::Cancelled,
14041 "cancelled",
14042 "activity was cancelled",
14043 ),
14044 "ActivityTimedOut" => (
14045 ActivityFailureKind::TimedOut,
14046 "timeout",
14047 "activity timed out",
14048 ),
14049 _ => unreachable!("activity_outcome is called only for terminal activity events"),
14050 };
14051 let exception = payload
14052 .get("exception")
14053 .filter(|value| !value.is_null())
14054 .cloned();
14055 let failure_category = payload_string(payload, "failure_category");
14056 let timeout_kind = payload_string(payload, "timeout_kind");
14057 let reason = payload_string(payload, "reason").unwrap_or_else(|| match kind {
14058 ActivityFailureKind::Failed => failure_category
14059 .clone()
14060 .unwrap_or_else(|| fallback_reason.to_string()),
14061 ActivityFailureKind::Cancelled => fallback_reason.to_string(),
14062 ActivityFailureKind::TimedOut => timeout_kind
14063 .clone()
14064 .unwrap_or_else(|| fallback_reason.to_string()),
14065 });
14066 let message = payload_string(payload, "message")
14067 .or_else(|| {
14068 exception
14069 .as_ref()
14070 .and_then(|value| payload_string(value, "message"))
14071 })
14072 .unwrap_or_else(|| fallback_message.to_string());
14073
14074 Ok(Err(ActivityFailure {
14075 kind,
14076 reason,
14077 message,
14078 activity_execution_id: payload_string(payload, "activity_execution_id"),
14079 activity_attempt_id: payload_string(payload, "activity_attempt_id"),
14080 activity_type: payload_string(payload, "activity_type")
14081 .or_else(|| payload_string(payload, "activity_name"))
14082 .or(recorded_activity_type),
14083 activity_class: payload_string(payload, "activity_class"),
14084 attempt_number: payload.get("attempt_number").and_then(value_as_u64),
14085 failure_id: payload_string(payload, "failure_id"),
14086 failure_category,
14087 timeout_kind,
14088 non_retryable: payload
14089 .get("non_retryable")
14090 .and_then(Value::as_bool)
14091 .unwrap_or(false),
14092 exception_type: payload_string(payload, "exception_type").or_else(|| {
14093 exception
14094 .as_ref()
14095 .and_then(|value| payload_string(value, "type"))
14096 }),
14097 exception_class: payload_string(payload, "exception_class").or_else(|| {
14098 exception
14099 .as_ref()
14100 .and_then(|value| payload_string(value, "class"))
14101 }),
14102 code: payload
14103 .get("code")
14104 .filter(|value| !value.is_null())
14105 .cloned(),
14106 exception,
14107 }))
14108}
14109
14110type ChildWorkflowOutcome = std::result::Result<ChildWorkflowAvroResult, ChildWorkflowFailure>;
14111
14112fn child_workflow_outcomes(
14113 events: &[HistoryEvent],
14114 fallback_codec: &str,
14115 parent: WorkflowIdentity,
14116) -> Result<Vec<ChildWorkflowOutcome>> {
14117 let mut outcomes = Vec::new();
14118
14119 for event in events {
14120 let kind = match event.event_type.as_str() {
14121 "ChildRunCompleted" => None,
14122 "ChildRunFailed" => Some((
14123 ChildWorkflowFailureKind::Failed,
14124 "child_workflow",
14125 "child workflow failed",
14126 )),
14127 "ChildRunCancelled" => Some((
14128 ChildWorkflowFailureKind::Cancelled,
14129 "cancelled",
14130 "child workflow was cancelled",
14131 )),
14132 "ChildRunTerminated" => Some((
14133 ChildWorkflowFailureKind::Terminated,
14134 "terminated",
14135 "child workflow was terminated",
14136 )),
14137 _ => continue,
14138 };
14139 let payload = &event.payload;
14140 let child_workflow_id = payload_string(payload, "child_workflow_instance_id");
14141 let child_workflow_run_id = payload_string(payload, "child_workflow_run_id");
14142 let child_workflow_type = payload_string(payload, "child_workflow_type");
14143
14144 if let Some((kind, reason, fallback_message)) = kind {
14145 let exception = payload
14146 .get("exception")
14147 .filter(|value| !value.is_null())
14148 .cloned();
14149 let message = payload_string(payload, "message")
14150 .or_else(|| {
14151 exception
14152 .as_ref()
14153 .and_then(|value| payload_string(value, "message"))
14154 })
14155 .unwrap_or_else(|| fallback_message.to_string());
14156 let exception_type = payload_string(payload, "exception_type").or_else(|| {
14157 exception
14158 .as_ref()
14159 .and_then(|value| payload_string(value, "type"))
14160 });
14161 let exception_class = payload_string(payload, "exception_class").or_else(|| {
14162 exception
14163 .as_ref()
14164 .and_then(|value| payload_string(value, "class"))
14165 });
14166 outcomes.push(Err(ChildWorkflowFailure {
14167 kind,
14168 reason: reason.to_string(),
14169 message,
14170 parent_workflow_id: parent.workflow_id.clone(),
14171 parent_workflow_run_id: parent.run_id.clone(),
14172 child_workflow_id,
14173 child_workflow_run_id,
14174 child_workflow_type,
14175 failure_id: payload_string(payload, "failure_id"),
14176 failure_category: payload_string(payload, "failure_category"),
14177 exception_type,
14178 exception_class,
14179 non_retryable: payload
14180 .get("non_retryable")
14181 .and_then(Value::as_bool)
14182 .unwrap_or(false),
14183 code: payload
14184 .get("code")
14185 .filter(|value| !value.is_null())
14186 .cloned(),
14187 exception,
14188 }));
14189 continue;
14190 }
14191
14192 let codec = payload
14193 .get("payload_codec")
14194 .and_then(Value::as_str)
14195 .unwrap_or(fallback_codec);
14196 let result = payload
14197 .get("result")
14198 .or_else(|| payload.get("output"))
14199 .unwrap_or(&Value::Null);
14200 outcomes.push(Ok(ChildWorkflowAvroResult {
14201 parent: parent.clone(),
14202 child: WorkflowIdentity {
14203 workflow_id: child_workflow_id,
14204 run_id: child_workflow_run_id,
14205 },
14206 child_workflow_type,
14207 result: decode_wire_avro_value(result, codec)?,
14208 }));
14209 }
14210
14211 Ok(outcomes)
14212}
14213
14214fn payload_string(payload: &Value, key: &str) -> Option<String> {
14215 payload
14216 .get(key)
14217 .and_then(Value::as_str)
14218 .filter(|value| !value.is_empty())
14219 .map(str::to_string)
14220}
14221
14222fn workflow_failure_command(error: &Error) -> Value {
14223 let (exception_type, exception_class, properties) = match error {
14224 Error::ActivityFailed(failure) => (
14225 match failure.kind {
14226 ActivityFailureKind::Failed => "ActivityFailed",
14227 ActivityFailureKind::Cancelled => "ActivityCancelled",
14228 ActivityFailureKind::TimedOut => "ActivityTimedOut",
14229 },
14230 "durable_workflow::ActivityFailure",
14231 json!({
14232 "reason": failure.reason,
14233 "activity_execution_id": failure.activity_execution_id,
14234 "activity_attempt_id": failure.activity_attempt_id,
14235 "activity_type": failure.activity_type,
14236 "activity_class": failure.activity_class,
14237 "attempt_number": failure.attempt_number,
14238 "failure_id": failure.failure_id,
14239 "failure_category": failure.failure_category,
14240 "timeout_kind": failure.timeout_kind,
14241 "activity_non_retryable": failure.non_retryable,
14242 "activity_exception_type": failure.exception_type,
14243 "activity_exception_class": failure.exception_class,
14244 "activity_code": failure.code,
14245 "activity_exception": failure.exception,
14246 }),
14247 ),
14248 Error::ChildWorkflowFailed(failure) => (
14249 match failure.kind {
14250 ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
14251 ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
14252 ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
14253 },
14254 "durable_workflow::ChildWorkflowFailure",
14255 json!({
14256 "reason": failure.reason,
14257 "parent_workflow_id": failure.parent_workflow_id,
14258 "parent_workflow_run_id": failure.parent_workflow_run_id,
14259 "child_workflow_id": failure.child_workflow_id,
14260 "child_workflow_run_id": failure.child_workflow_run_id,
14261 "child_workflow_type": failure.child_workflow_type,
14262 "failure_id": failure.failure_id,
14263 "failure_category": failure.failure_category,
14264 "child_exception_type": failure.exception_type,
14265 "child_exception_class": failure.exception_class,
14266 "child_non_retryable": failure.non_retryable,
14267 "child_code": failure.code,
14268 "child_exception": failure.exception,
14269 }),
14270 ),
14271 Error::ParallelFailed(failure) => (
14272 "ParallelFailed",
14273 "durable_workflow::ParallelFailure",
14274 json!({
14275 "parallel_group_id": failure.group_id,
14276 "parallel_member_path": failure.member_path,
14277 "parallel_group_path": failure.group_path,
14278 "completed_members": failure.completed.iter().map(|completion| &completion.member_path).collect::<Vec<_>>(),
14279 "cause_type": workflow_error_type(&failure.cause),
14280 "cause_message": failure.cause.to_string(),
14281 }),
14282 ),
14283 Error::SagaCompensationFailed(failure) => (
14284 "SagaCompensationFailed",
14285 "durable_workflow::SagaCompensationFailure",
14286 json!({
14287 "initiating_failure_type": workflow_error_type(&failure.initiating_failure),
14288 "initiating_failure_message": failure.initiating_failure.to_string(),
14289 "compensation_activity_type": failure.compensation_activity_type,
14290 "compensation_registration_order": failure.compensation_registration_order,
14291 "compensation_failure_type": workflow_error_type(&failure.compensation_failure),
14292 "compensation_failure_message": failure.compensation_failure.to_string(),
14293 }),
14294 ),
14295 Error::WorkflowCancellationRequested(_) => (
14296 "WorkflowCancellationRequested",
14297 "durable_workflow::WorkflowCancellationRequested",
14298 json!({"reason": "cancelled"}),
14299 ),
14300 Error::NonDeterministicReplay(_) => (
14301 "NonDeterministicReplay",
14302 "durable_workflow::Error",
14303 Value::Null,
14304 ),
14305 _ => ("RustWorkflowError", "durable_workflow::Error", Value::Null),
14306 };
14307 let non_retryable = match error {
14308 Error::ActivityFailed(failure) => failure.non_retryable,
14309 Error::ChildWorkflowFailed(failure) => failure.non_retryable,
14310 Error::ParallelFailed(failure) => workflow_error_non_retryable(&failure.cause),
14311 Error::SagaCompensationFailed(failure) => {
14312 workflow_error_non_retryable(&failure.compensation_failure)
14313 }
14314 Error::WorkflowCancellationRequested(_) => true,
14315 Error::NonDeterministicReplay(_) => true,
14316 _ => false,
14317 };
14318
14319 json!({
14320 "type": "fail_workflow",
14321 "message": error.to_string(),
14322 "exception_type": exception_type,
14323 "exception_class": exception_class,
14324 "non_retryable": non_retryable,
14325 "exception": {
14326 "type": exception_type,
14327 "class": exception_class,
14328 "message": error.to_string(),
14329 "properties": properties,
14330 }
14331 })
14332}
14333
14334fn workflow_error_type(error: &Error) -> &'static str {
14335 match error {
14336 Error::ActivityFailed(failure) => match failure.kind {
14337 ActivityFailureKind::Failed => "ActivityFailed",
14338 ActivityFailureKind::Cancelled => "ActivityCancelled",
14339 ActivityFailureKind::TimedOut => "ActivityTimedOut",
14340 },
14341 Error::ChildWorkflowFailed(failure) => match failure.kind {
14342 ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
14343 ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
14344 ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
14345 },
14346 Error::ParallelFailed(_) => "ParallelFailed",
14347 Error::SagaCompensationFailed(_) => "SagaCompensationFailed",
14348 Error::WorkflowCancellationRequested(_) => "WorkflowCancellationRequested",
14349 Error::NonDeterministicReplay(_) => "NonDeterministicReplay",
14350 _ => "RustWorkflowError",
14351 }
14352}
14353
14354fn workflow_error_non_retryable(error: &Error) -> bool {
14355 match error {
14356 Error::ActivityFailed(failure) => failure.non_retryable,
14357 Error::ChildWorkflowFailed(failure) => failure.non_retryable,
14358 Error::ParallelFailed(failure) => workflow_error_non_retryable(&failure.cause),
14359 Error::SagaCompensationFailed(failure) => {
14360 workflow_error_non_retryable(&failure.compensation_failure)
14361 }
14362 Error::WorkflowCancellationRequested(_) | Error::NonDeterministicReplay(_) => true,
14363 _ => false,
14364 }
14365}
14366
14367fn workflow_task_integrity_error(error: &Error) -> bool {
14368 matches!(
14369 error,
14370 Error::NonDeterministicReplay(_)
14371 | Error::Protocol(_)
14372 | Error::MissingWorkflowCommandIdentity
14373 | Error::WorkflowStatePoisoned
14374 )
14375}
14376
14377fn decode_signal_event_arguments(
14378 event: &HistoryEvent,
14379 fallback_codec: &str,
14380) -> Result<Vec<AvroValue>> {
14381 let codec = declared_payload_codec(&event.payload, "payload_codec")?.unwrap_or(fallback_codec);
14382 validate_payload_codec(codec)?;
14383 let raw = signal_history_payload(&event.payload);
14384 let decoded = match raw.filter(|value| !value.is_null()) {
14385 Some(value) => decode_wire_avro_value(value, codec)?,
14386 None => AvroValue::Array(Vec::new()),
14387 };
14388 let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
14389 unreachable!("normalize_avro_arguments always returns an array");
14390 };
14391 Ok(arguments)
14392}
14393
14394fn decode_update_event_arguments(
14395 event: &HistoryEvent,
14396 fallback_codec: &str,
14397) -> Result<Vec<AvroValue>> {
14398 let codec = declared_payload_codec(&event.payload, "payload_codec")?.unwrap_or(fallback_codec);
14399 validate_payload_codec(codec)?;
14400 let decoded = match event
14401 .payload
14402 .get("arguments")
14403 .filter(|value| !value.is_null())
14404 {
14405 Some(value) => decode_wire_avro_value(value, codec)?,
14406 None => AvroValue::Array(Vec::new()),
14407 };
14408 let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
14409 unreachable!("normalize_avro_arguments always returns an array");
14410 };
14411 Ok(arguments)
14412}
14413
14414fn hydrate_query_history_from_export(task: &mut QueryTask) -> Result<()> {
14415 let Some(export_events) = task
14416 .history_export
14417 .as_ref()
14418 .and_then(|export| export.get("history_events"))
14419 .and_then(Value::as_array)
14420 else {
14421 return Ok(());
14422 };
14423
14424 if export_events.len() > task.history_events.len() {
14425 task.history_events = serde_json::from_value(Value::Array(export_events.clone()))?;
14426 }
14427
14428 Ok(())
14429}
14430
14431fn enrich_query_history_from_export(task: &mut QueryTask) -> Result<()> {
14432 let Some(export) = task.history_export.as_ref() else {
14433 return Ok(());
14434 };
14435 let signals = export
14436 .get("signals")
14437 .and_then(Value::as_array)
14438 .cloned()
14439 .unwrap_or_default();
14440 let activities = export
14441 .get("activities")
14442 .and_then(Value::as_array)
14443 .cloned()
14444 .unwrap_or_default();
14445 let export_codec = export
14446 .get("payloads")
14447 .and_then(|payloads| payloads.get("codec"))
14448 .and_then(Value::as_str)
14449 .unwrap_or(&task.payload_codec)
14450 .to_string();
14451 let mut signal_name_offsets: HashMap<String, usize> = HashMap::new();
14452
14453 for event in &mut task.history_events {
14454 if event.event_type == "ActivityCompleted" {
14455 let sequence = event
14456 .payload
14457 .get("sequence")
14458 .or_else(|| event.payload.get("workflow_sequence"))
14459 .and_then(value_as_u64);
14460 let Some(activity) = sequence.and_then(|sequence| {
14461 activities.iter().find(|activity| {
14462 activity.get("sequence").and_then(value_as_u64) == Some(sequence)
14463 })
14464 }) else {
14465 continue;
14466 };
14467 let Some(payload) = event.payload.as_object_mut() else {
14468 continue;
14469 };
14470 if missing_payload(payload.get("result")) {
14471 if let Some(result) = activity
14472 .get("result")
14473 .filter(|value| !missing_payload(Some(value)))
14474 {
14475 payload.insert("result".to_string(), result.clone());
14476 }
14477 }
14478 for field in ["payload_codec", "activity_type"] {
14479 if payload
14480 .get(field)
14481 .and_then(Value::as_str)
14482 .unwrap_or_default()
14483 .is_empty()
14484 {
14485 if let Some(value) = activity.get(field) {
14486 payload.insert(field.to_string(), value.clone());
14487 }
14488 }
14489 }
14490 continue;
14491 }
14492
14493 if event.event_type != "SignalReceived" && event.event_type != "SignalApplied" {
14494 continue;
14495 }
14496 let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
14497 let command_id = event
14498 .payload
14499 .get("workflow_command_id")
14500 .or_else(|| event.raw.get("workflow_command_id"))
14501 .and_then(Value::as_str);
14502 let signal_name = event
14503 .payload
14504 .get("signal_name")
14505 .and_then(Value::as_str)
14506 .unwrap_or_default()
14507 .to_string();
14508 let matched = signals
14509 .iter()
14510 .find(|signal| {
14511 signal_id.is_some() && signal.get("id").and_then(Value::as_str) == signal_id
14512 })
14513 .or_else(|| {
14514 signals.iter().find(|signal| {
14515 command_id.is_some()
14516 && signal.get("command_id").and_then(Value::as_str) == command_id
14517 })
14518 })
14519 .or_else(|| {
14520 let offset = signal_name_offsets.entry(signal_name.clone()).or_default();
14521 let signal = signals
14522 .iter()
14523 .filter(|signal| {
14524 signal.get("name").and_then(Value::as_str) == Some(signal_name.as_str())
14525 })
14526 .nth(*offset);
14527 if signal.is_some() {
14528 *offset += 1;
14529 }
14530 signal
14531 });
14532 let Some(signal) = matched else {
14533 continue;
14534 };
14535 let signal_codec = signal
14536 .get("payload_codec")
14537 .and_then(Value::as_str)
14538 .unwrap_or(&export_codec);
14539 let Some(payload) = event.payload.as_object_mut() else {
14540 continue;
14541 };
14542 if missing_payload(payload.get("arguments")) {
14543 if let Some(arguments) = signal
14544 .get("arguments")
14545 .filter(|value| !missing_payload(Some(value)))
14546 {
14547 let envelope = match arguments {
14548 Value::String(blob) => json!({"codec": signal_codec, "blob": blob}),
14549 other => other.clone(),
14550 };
14551 payload.insert("arguments".to_string(), envelope);
14552 }
14553 }
14554 if payload
14555 .get("payload_codec")
14556 .and_then(Value::as_str)
14557 .unwrap_or_default()
14558 .is_empty()
14559 {
14560 payload.insert("payload_codec".to_string(), json!(signal_codec));
14561 }
14562 }
14563
14564 Ok(())
14565}
14566
14567fn missing_payload(value: Option<&Value>) -> bool {
14568 match value {
14569 None | Some(Value::Null) => true,
14570 Some(Value::String(value)) => value.is_empty(),
14571 Some(_) => false,
14572 }
14573}
14574
14575fn query_signal_events(task: &QueryTask) -> Result<Vec<QuerySignal>> {
14576 let export_signals = task
14577 .history_export
14578 .as_ref()
14579 .and_then(|export| export.get("signals"))
14580 .and_then(Value::as_array)
14581 .cloned()
14582 .unwrap_or_default();
14583 let export_codec = task
14584 .history_export
14585 .as_ref()
14586 .and_then(|export| export.get("payloads"))
14587 .and_then(|payloads| payloads.get("codec"))
14588 .and_then(Value::as_str)
14589 .unwrap_or(&task.payload_codec);
14590 let mut name_offsets: HashMap<String, usize> = HashMap::new();
14591 let mut signals = Vec::new();
14592
14593 for event in &task.history_events {
14594 if event.event_type != "SignalApplied" && event.event_type != "SignalReceived" {
14595 continue;
14596 }
14597
14598 let name = event
14599 .payload
14600 .get("signal_name")
14601 .and_then(Value::as_str)
14602 .unwrap_or_default();
14603 if name.is_empty() {
14604 continue;
14605 }
14606 let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
14607 let command_id = event
14608 .payload
14609 .get("workflow_command_id")
14610 .or_else(|| event.raw.get("workflow_command_id"))
14611 .and_then(Value::as_str);
14612 let matched_export = export_signals
14613 .iter()
14614 .find(|candidate| {
14615 signal_id.is_some() && candidate.get("id").and_then(Value::as_str) == signal_id
14616 })
14617 .or_else(|| {
14618 export_signals.iter().find(|candidate| {
14619 command_id.is_some()
14620 && candidate.get("command_id").and_then(Value::as_str) == command_id
14621 })
14622 })
14623 .or_else(|| {
14624 let offset = name_offsets.entry(name.to_string()).or_default();
14625 let candidate = export_signals
14626 .iter()
14627 .filter(|candidate| candidate.get("name").and_then(Value::as_str) == Some(name))
14628 .nth(*offset);
14629 if candidate.is_some() {
14630 *offset += 1;
14631 }
14632 candidate
14633 });
14634 let codec = event
14635 .payload
14636 .get("payload_codec")
14637 .and_then(Value::as_str)
14638 .or_else(|| {
14639 matched_export
14640 .and_then(|signal| signal.get("payload_codec"))
14641 .and_then(Value::as_str)
14642 })
14643 .unwrap_or(export_codec);
14644 let raw_arguments = signal_history_payload(&event.payload)
14645 .filter(|value| !value.is_null())
14646 .or_else(|| matched_export.and_then(|signal| signal.get("arguments")));
14647 let (arguments, avro_arguments) = decode_query_signal_arguments(raw_arguments, codec)?;
14648 let workflow_sequence = event
14649 .payload
14650 .get("workflow_sequence")
14651 .and_then(value_as_u64)
14652 .or_else(|| {
14653 matched_export
14654 .and_then(|signal| signal.get("workflow_sequence"))
14655 .and_then(value_as_u64)
14656 });
14657
14658 signals.push(QuerySignal {
14659 id: signal_id.map(str::to_string).or_else(|| {
14660 matched_export
14661 .and_then(|signal| signal.get("id"))
14662 .and_then(Value::as_str)
14663 .map(str::to_string)
14664 }),
14665 name: name.to_string(),
14666 arguments,
14667 avro_arguments,
14668 workflow_sequence,
14669 });
14670 }
14671
14672 if signals.is_empty() {
14673 for signal in export_signals {
14674 if signal.get("status").and_then(Value::as_str) == Some("rejected") {
14675 continue;
14676 }
14677 let Some(name) = signal.get("name").and_then(Value::as_str) else {
14678 continue;
14679 };
14680 let codec = signal
14681 .get("payload_codec")
14682 .and_then(Value::as_str)
14683 .unwrap_or(export_codec);
14684 let (arguments, avro_arguments) =
14685 decode_query_signal_arguments(signal.get("arguments"), codec)?;
14686 signals.push(QuerySignal {
14687 id: signal.get("id").and_then(Value::as_str).map(str::to_string),
14688 name: name.to_string(),
14689 arguments,
14690 avro_arguments,
14691 workflow_sequence: signal.get("workflow_sequence").and_then(value_as_u64),
14692 });
14693 }
14694 signals.sort_by_key(|signal| signal.workflow_sequence.unwrap_or(u64::MAX));
14695 }
14696
14697 Ok(signals)
14698}
14699
14700fn decode_query_signal_arguments(
14701 raw: Option<&Value>,
14702 codec: &str,
14703) -> Result<(Vec<Value>, Vec<AvroValue>)> {
14704 validate_payload_codec(codec)?;
14705 let decoded = match raw.filter(|value| !value.is_null()) {
14706 Some(value) => decode_wire_avro_value(value, codec)?,
14707 None => AvroValue::Array(Vec::new()),
14708 };
14709 let AvroValue::Array(avro_arguments) = normalize_avro_arguments(decoded) else {
14710 unreachable!("normalize_avro_arguments always returns an array");
14711 };
14712 let arguments = avro_arguments
14713 .iter()
14714 .cloned()
14715 .map(AvroValue::into_json)
14716 .collect::<Result<Vec<_>>>()?;
14717 Ok((arguments, avro_arguments))
14718}
14719
14720fn value_as_u64(value: &Value) -> Option<u64> {
14721 value
14722 .as_u64()
14723 .or_else(|| value.as_str().and_then(|value| value.parse().ok()))
14724}
14725
14726#[cfg(test)]
14727mod tests {
14728 use super::*;
14729 mod runtime_payloads;
14730 use std::{
14731 fs,
14732 io::{Read, Write},
14733 net::{SocketAddr, TcpListener, TcpStream},
14734 process::Command as ProcessCommand,
14735 sync::atomic::AtomicUsize,
14736 thread,
14737 };
14738
14739 #[derive(Clone, Copy, Debug)]
14740 enum InvalidTaskPayloadCodec {
14741 Missing,
14742 Null,
14743 NonString,
14744 }
14745
14746 impl InvalidTaskPayloadCodec {
14747 fn label(self) -> &'static str {
14748 match self {
14749 Self::Missing => "missing",
14750 Self::Null => "null",
14751 Self::NonString => "non-string",
14752 }
14753 }
14754
14755 fn apply(self, task: &mut Value) {
14756 let task = task.as_object_mut().expect("task fixture object");
14757 match self {
14758 Self::Missing => {
14759 task.remove("payload_codec");
14760 }
14761 Self::Null => {
14762 task.insert("payload_codec".to_string(), Value::Null);
14763 }
14764 Self::NonString => {
14765 task.insert("payload_codec".to_string(), json!(42));
14766 }
14767 }
14768 }
14769 }
14770
14771 fn fixture_envelope(value: Value) -> Value {
14772 encode_value_envelope(&value, DEFAULT_CODEC).expect("encode Avro test fixture")
14773 }
14774
14775 fn fixture_blob(value: Value) -> String {
14776 encode_payload(&value, DEFAULT_CODEC)
14777 .expect("encode Avro test fixture")
14778 .blob
14779 }
14780
14781 #[test]
14782 fn client_builder_rejects_the_sdk_owned_api_suffix() {
14783 for base_url in [
14784 "http://127.0.0.1:8080/api",
14785 "http://localhost:8080/api/",
14786 "https://runtime.example.test/namespaces/orders/api",
14787 ] {
14788 let error = Client::builder(base_url)
14789 .build()
14790 .expect_err("SDK-owned /api suffix must be rejected during build");
14791
14792 assert!(matches!(error, Error::InvalidBaseUrl), "{base_url}");
14793 assert!(
14794 error.to_string().contains("SDK appends /api automatically"),
14795 "the validation error must explain how to fix the endpoint"
14796 );
14797 }
14798 }
14799
14800 #[test]
14801 fn client_builder_preserves_self_hosted_and_managed_runtime_prefixes() {
14802 for (base_url, expected) in [
14803 ("http://127.0.0.1:8080", "http://127.0.0.1:8080"),
14804 (
14805 "http://localhost:8080/durable-workflow/",
14806 "http://localhost:8080/durable-workflow",
14807 ),
14808 (
14809 "https://runtime.example.test/namespaces/orders",
14810 "https://runtime.example.test/namespaces/orders",
14811 ),
14812 (
14813 "https://runtime.example.test/gateway/api/namespaces/orders",
14814 "https://runtime.example.test/gateway/api/namespaces/orders",
14815 ),
14816 (
14817 "https://api.example.test/runtime/orders/",
14818 "https://api.example.test/runtime/orders",
14819 ),
14820 ] {
14821 let client = Client::builder(base_url)
14822 .build()
14823 .expect("Server and Cloud runtime base URL must remain valid");
14824
14825 assert_eq!(client.base_url, expected);
14826 }
14827 }
14828
14829 #[test]
14830 fn workflow_completion_uses_the_additive_command_protocol_floor() {
14831 assert_eq!(
14832 workflow_completion_protocol_version(&[json!({"type": "complete_workflow"})]),
14833 WORKER_PROTOCOL_VERSION
14834 );
14835 assert_eq!(
14836 workflow_completion_protocol_version(&[json!({
14837 "type": "upsert_search_attributes",
14838 "attributes": {"OrderStatus": "waiting"},
14839 })]),
14840 SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION
14841 );
14842 assert_eq!(
14843 workflow_completion_protocol_version(&[json!({
14844 "type": "upsert_search_attributes",
14845 "attributes": {"OrderStatus": "waiting"},
14846 "attribute_types": {"OrderStatus": "keyword"},
14847 })]),
14848 TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION
14849 );
14850 assert_eq!(
14851 workflow_completion_protocol_version(&[
14852 json!({"type": "upsert_memo", "entries": {"status": "waiting"}}),
14853 json!({"type": "open_condition_wait", "condition_key": "ready"}),
14854 ]),
14855 MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION
14856 );
14857 assert_eq!(
14858 workflow_completion_protocol_version(&[
14859 json!({"type": "upsert_search_attributes", "attributes": {"State": "waiting"}}),
14860 json!({"type": "open_condition_wait", "condition_key": "ready"}),
14861 ]),
14862 CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION
14863 );
14864 assert_eq!(
14865 workflow_completion_protocol_version(&[json!({
14866 "type": "open_condition_wait",
14867 "condition_wait_occurrence_id": "rust:condition-wait:0",
14868 "condition_key": "ready",
14869 })]),
14870 CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
14871 );
14872 assert_eq!(
14873 workflow_completion_protocol_version_with_message_streams(
14874 &[json!({"type": "upsert_memo", "entries": {"status": "waiting"}})],
14875 true,
14876 ),
14877 MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION
14878 );
14879 assert_eq!(
14880 workflow_completion_protocol_version_with_message_streams(
14881 &[json!({
14882 "type": "open_condition_wait",
14883 "condition_wait_occurrence_id": "rust:condition-wait:0",
14884 "condition_key": "ready",
14885 })],
14886 true,
14887 ),
14888 CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
14889 );
14890 }
14891
14892 #[test]
14893 fn portable_worker_affinity_manifest_explicitly_refuses_unimplemented_features() {
14894 let manifest = portable_worker_affinity_capability_manifest();
14895
14896 for capability in ["local_activities", "worker_sessions", "sticky_execution"] {
14897 assert_eq!(manifest[capability]["supported"], json!(false));
14898 assert_eq!(
14899 manifest[capability]["minimum_protocol_version"],
14900 json!(PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION)
14901 );
14902 assert!(manifest[capability]["reason"]
14903 .as_str()
14904 .is_some_and(|reason| !reason.is_empty()));
14905 }
14906 }
14907
14908 fn typed_fidelity_probe() -> AvroValue {
14909 AvroValue::Map(BTreeMap::from([
14910 ("bytes".to_string(), AvroValue::Bytes(vec![0, 0xff])),
14911 ("empty".to_string(), AvroValue::Map(BTreeMap::new())),
14912 (
14913 "numeric".to_string(),
14914 AvroValue::Map(BTreeMap::from([
14915 ("0".to_string(), AvroValue::String("zero".to_string())),
14916 ("1".to_string(), AvroValue::String("one".to_string())),
14917 ])),
14918 ),
14919 (
14920 "nested".to_string(),
14921 AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([(
14922 "enabled".to_string(),
14923 AvroValue::Boolean(true),
14924 )]))]),
14925 ),
14926 (
14927 "projection_collisions".to_string(),
14928 AvroValue::Array(projection_collision_probe()),
14929 ),
14930 ]))
14931 }
14932
14933 fn projection_collision_probe() -> Vec<AvroValue> {
14934 vec![
14935 AvroValue::Map(BTreeMap::from([
14936 ("$type".to_string(), AvroValue::String("bytes".to_string())),
14937 (
14938 "base64".to_string(),
14939 AvroValue::String("ordinary user text".to_string()),
14940 ),
14941 ])),
14942 AvroValue::Map(BTreeMap::from([
14943 ("$type".to_string(), AvroValue::String("map".to_string())),
14944 (
14945 "entries".to_string(),
14946 AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([
14947 ("key".to_string(), AvroValue::String("ordinary".to_string())),
14948 (
14949 "value".to_string(),
14950 AvroValue::String("user map".to_string()),
14951 ),
14952 ]))]),
14953 ),
14954 ])),
14955 ]
14956 }
14957
14958 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
14959 struct TypedContract {
14960 nested: TypedNested,
14961 mode: TypedMode,
14962 optional: Option<String>,
14963 absent: Option<String>,
14964 items: Vec<i64>,
14965 labels: BTreeMap<String, String>,
14966 bytes: serde_bytes::ByteBuf,
14967 signed: i64,
14968 finite: f64,
14969 }
14970
14971 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
14972 struct TypedNested {
14973 enabled: bool,
14974 }
14975
14976 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
14977 enum TypedMode {
14978 Detailed { label: String },
14979 }
14980
14981 fn typed_contract() -> TypedContract {
14982 TypedContract {
14983 nested: TypedNested { enabled: true },
14984 mode: TypedMode::Detailed {
14985 label: "compiler-checked".to_string(),
14986 },
14987 optional: Some("present".to_string()),
14988 absent: None,
14989 items: vec![i64::MIN, 0, i64::MAX],
14990 labels: BTreeMap::from([
14991 ("language".to_string(), "rust".to_string()),
14992 ("wire".to_string(), "avro".to_string()),
14993 ]),
14994 bytes: serde_bytes::ByteBuf::from(vec![0, 0xff, 7]),
14995 signed: -9_223_372_036_854_775_000,
14996 finite: 12.5,
14997 }
14998 }
14999
15000 #[derive(Clone, Debug, Default, PartialEq)]
15001 struct ReplayCounterState {
15002 loaded: Option<String>,
15003 count: i64,
15004 finished: bool,
15005 }
15006
15007 fn replay_counter_worker() -> Worker {
15008 let client = Client::new("http://127.0.0.1:8080").expect("client");
15009 let mut worker = Worker::new(client, "rust-workers");
15010 worker.register_replayed_workflow(
15011 "replay-counter",
15012 ReplayCounterState::default,
15013 |ctx, _input, state| async move {
15014 let loaded = ctx.activity("load-counter", json!([])).await?;
15015 state.update(|current| {
15016 current.loaded = loaded.as_str().map(str::to_string);
15017 })?;
15018 for _ in 0..2 {
15019 let signal = ctx.wait_signal("increment").await?;
15020 let amount = signal.first().and_then(Value::as_i64).unwrap_or_default();
15021 state.update(|current| current.count += amount)?;
15022 }
15023 state.update(|current| current.finished = true)?;
15024 state.read(|current| Ok(json!(current.count)))?
15025 },
15026 );
15027 worker.register_replayed_query::<ReplayCounterState, _, _>(
15028 "replay-counter",
15029 "current",
15030 |_ctx, state, _args| async move {
15031 Ok(json!({
15032 "loaded": state.loaded,
15033 "count": state.count,
15034 "finished": state.finished,
15035 }))
15036 },
15037 );
15038 worker.register_replayed_query::<ReplayCounterState, _, _>(
15039 "replay-counter",
15040 "detached-mutation",
15041 |_ctx, state, _args| async move {
15042 let mut detached = (*state).clone();
15043 detached.count = 999;
15044 Ok(json!(detached.count))
15045 },
15046 );
15047 worker.register_replayed_query::<ReplayCounterState, _, _>(
15048 "replay-counter",
15049 "failed-mutation",
15050 |_ctx, state, _args| async move {
15051 let mut detached = (*state).clone();
15052 detached.count = 999;
15053 Err(Error::WorkerLoop("query refused".to_string()))
15054 },
15055 );
15056 worker
15057 }
15058
15059 fn replay_counter_query(
15060 query_name: &str,
15061 history_events: Value,
15062 run_status: &str,
15063 ) -> QueryTask {
15064 let arguments = fixture_envelope(json!([]));
15065 serde_json::from_value(json!({
15066 "query_task_id": format!("query-{query_name}"),
15067 "workflow_type": "replay-counter",
15068 "query_name": query_name,
15069 "payload_codec": DEFAULT_CODEC,
15070 "workflow_arguments": arguments.clone(),
15071 "query_arguments": arguments,
15072 "history_events": history_events,
15073 "run_status": run_status,
15074 }))
15075 .expect("query task")
15076 }
15077
15078 fn workflow_context(history: Vec<HistoryEvent>) -> WorkflowContext {
15079 workflow_context_with_codec(history, DEFAULT_CODEC)
15080 }
15081
15082 fn workflow_context_with_codec(
15083 history: Vec<HistoryEvent>,
15084 payload_codec: &str,
15085 ) -> WorkflowContext {
15086 WorkflowContext {
15087 state: Arc::new(Mutex::new(
15088 WorkflowState::new_with_identity(
15089 history,
15090 None,
15091 None,
15092 "rust-workers".to_string(),
15093 payload_codec.to_string(),
15094 None,
15095 )
15096 .expect("valid workflow history"),
15097 )),
15098 }
15099 }
15100
15101 fn history_event(event_type: &str, payload: Value) -> HistoryEvent {
15102 HistoryEvent {
15103 event_type: event_type.to_string(),
15104 payload,
15105 raw: HashMap::new(),
15106 }
15107 }
15108
15109 fn parallel_path_entry(
15110 kind: &str,
15111 base: u64,
15112 size: usize,
15113 index: usize,
15114 ) -> ParallelGroupMetadata {
15115 parallel_group_entry(base, size, index, kind)
15116 }
15117
15118 fn parallel_history_event(
15119 event_type: &str,
15120 sequence: u64,
15121 identity_field: &str,
15122 identity: &str,
15123 path: Vec<ParallelGroupMetadata>,
15124 result: Option<Value>,
15125 ) -> HistoryEvent {
15126 let mut payload = serde_json::Map::from_iter([
15127 ("sequence".to_string(), json!(sequence)),
15128 (identity_field.to_string(), json!(identity)),
15129 ]);
15130 let inner = path.last().expect("parallel history path");
15131 apply_parallel_group_path(&mut payload, std::slice::from_ref(inner));
15132 payload.insert("parallel_group_path".to_string(), json!(path));
15133 if let Some(result) = result {
15134 let field = if event_type == "ChildRunCompleted" {
15135 "result"
15136 } else {
15137 "result"
15138 };
15139 payload.insert(field.to_string(), fixture_envelope(result));
15140 payload.insert("payload_codec".to_string(), json!(DEFAULT_CODEC));
15141 }
15142 history_event(event_type, Value::Object(payload))
15143 }
15144
15145 fn nested_parallel_operations() -> Vec<ParallelOperation> {
15146 vec![
15147 ParallelOperation::activity("first", json!([])),
15148 ParallelOperation::group(vec![
15149 ParallelOperation::child_workflow(
15150 "second",
15151 ChildWorkflowOptions::new("child-workers"),
15152 json!([]),
15153 ),
15154 ParallelOperation::activity("third", json!([])),
15155 ]),
15156 ]
15157 }
15158
15159 fn nested_parallel_paths() -> [Vec<ParallelGroupMetadata>; 3] {
15160 let outer = [
15161 parallel_path_entry("mixed", 1, 3, 0),
15162 parallel_path_entry("mixed", 1, 3, 1),
15163 parallel_path_entry("mixed", 1, 3, 2),
15164 ];
15165 [
15166 vec![outer[0].clone()],
15167 vec![outer[1].clone(), parallel_path_entry("mixed", 2, 2, 0)],
15168 vec![outer[2].clone(), parallel_path_entry("mixed", 2, 2, 1)],
15169 ]
15170 }
15171
15172 #[test]
15173 fn parallel_schedules_every_nested_mixed_leaf_with_stable_metadata() {
15174 let ctx = workflow_context(Vec::new());
15175 let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15176 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15177
15178 assert!(matches!(
15179 call.as_mut().poll(&mut task_context),
15180 Poll::Pending
15181 ));
15182 let commands = ctx.take_commands().expect("parallel commands");
15183 assert_eq!(
15184 commands
15185 .iter()
15186 .map(|command| command["type"].as_str().unwrap_or_default())
15187 .collect::<Vec<_>>(),
15188 [
15189 "schedule_activity",
15190 "start_child_workflow",
15191 "schedule_activity"
15192 ]
15193 );
15194 let paths = nested_parallel_paths();
15195 for (command, path) in commands.iter().zip(paths) {
15196 assert_eq!(command["parallel_group_path"], json!(path));
15197 assert_eq!(
15198 command["parallel_group_id"],
15199 json!(path.last().expect("inner group").parallel_group_id)
15200 );
15201 }
15202 }
15203
15204 fn completed_nested_parallel_history() -> Vec<HistoryEvent> {
15205 let paths = nested_parallel_paths();
15206 let third = parallel_history_event(
15207 "ActivityCompleted",
15208 3,
15209 "activity_type",
15210 "third",
15211 paths[2].clone(),
15212 Some(json!("three")),
15213 );
15214 vec![
15215 parallel_history_event(
15216 "ActivityCompleted",
15217 1,
15218 "activity_type",
15219 "first",
15220 paths[0].clone(),
15221 Some(json!("one")),
15222 ),
15223 parallel_history_event(
15224 "ChildWorkflowScheduled",
15225 2,
15226 "child_workflow_type",
15227 "second",
15228 paths[1].clone(),
15229 None,
15230 ),
15231 parallel_history_event(
15232 "ChildRunCompleted",
15233 2,
15234 "child_workflow_type",
15235 "second",
15236 paths[1].clone(),
15237 Some(json!("two")),
15238 ),
15239 third.clone(),
15240 third,
15241 ]
15242 }
15243
15244 #[test]
15245 fn parallel_replay_rebuilds_input_order_and_tolerates_duplicate_delivery() {
15246 for _restart_or_completed_replay in 0..2 {
15247 let ctx = workflow_context(completed_nested_parallel_history());
15248 let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15249 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15250 let Poll::Ready(Ok(results)) = call.as_mut().poll(&mut task_context) else {
15251 panic!("completed nested parallel history must replay");
15252 };
15253 assert_eq!(
15254 results,
15255 vec![
15256 ParallelResult::Activity(json!("one")),
15257 ParallelResult::Group(vec![
15258 ParallelResult::ChildWorkflow(ChildWorkflowResult {
15259 parent: WorkflowIdentity {
15260 workflow_id: None,
15261 run_id: None,
15262 },
15263 child: WorkflowIdentity {
15264 workflow_id: None,
15265 run_id: None,
15266 },
15267 child_workflow_type: Some("second".to_string()),
15268 result: json!("two"),
15269 }),
15270 ParallelResult::Activity(json!("three")),
15271 ]),
15272 ]
15273 );
15274 assert!(ctx.take_commands().expect("commands").is_empty());
15275 ctx.ensure_history_consumed().expect("history consumed");
15276 }
15277 }
15278
15279 #[test]
15280 fn parallel_failure_keeps_typed_cause_path_and_late_completions() {
15281 let paths = nested_parallel_paths();
15282 let history = vec![
15283 parallel_history_event(
15284 "ActivityCompleted",
15285 1,
15286 "activity_type",
15287 "first",
15288 paths[0].clone(),
15289 Some(json!("one")),
15290 ),
15291 parallel_history_event(
15292 "ChildWorkflowScheduled",
15293 2,
15294 "child_workflow_type",
15295 "second",
15296 paths[1].clone(),
15297 None,
15298 ),
15299 parallel_history_event(
15300 "ChildRunFailed",
15301 2,
15302 "child_workflow_type",
15303 "second",
15304 paths[1].clone(),
15305 None,
15306 ),
15307 parallel_history_event(
15308 "ActivityCompleted",
15309 3,
15310 "activity_type",
15311 "third",
15312 paths[2].clone(),
15313 Some(json!("late")),
15314 ),
15315 ];
15316 let ctx = workflow_context(history);
15317 let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15318 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15319 let outcome = call.as_mut().poll(&mut task_context);
15320 let Poll::Ready(Err(Error::ParallelFailed(failure))) = outcome else {
15321 panic!("one failed child must return a typed partial failure: {outcome:?}");
15322 };
15323 assert_eq!(failure.member_path, [1, 0]);
15324 assert_eq!(failure.group_id, "parallel-calls:1:3");
15325 assert!(matches!(*failure.cause, Error::ChildWorkflowFailed(_)));
15326 assert_eq!(
15327 failure
15328 .completed
15329 .iter()
15330 .map(|completion| completion.member_path.clone())
15331 .collect::<Vec<_>>(),
15332 [vec![0], vec![1, 1]]
15333 );
15334 }
15335
15336 #[test]
15337 fn pending_parallel_history_restarts_without_rescheduling_any_leaf() {
15338 let paths = nested_parallel_paths();
15339 let history = vec![
15340 parallel_history_event(
15341 "ActivityScheduled",
15342 1,
15343 "activity_type",
15344 "first",
15345 paths[0].clone(),
15346 None,
15347 ),
15348 parallel_history_event(
15349 "ChildWorkflowScheduled",
15350 2,
15351 "child_workflow_type",
15352 "second",
15353 paths[1].clone(),
15354 None,
15355 ),
15356 parallel_history_event(
15357 "ActivityScheduled",
15358 3,
15359 "activity_type",
15360 "third",
15361 paths[2].clone(),
15362 None,
15363 ),
15364 ];
15365 for _restart in 0..2 {
15366 let ctx = workflow_context(history.clone());
15367 let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15368 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15369 let outcome = call.as_mut().poll(&mut task_context);
15370 assert!(matches!(outcome, Poll::Pending), "{outcome:?}");
15371 assert!(ctx.take_commands().expect("commands").is_empty());
15372 }
15373 }
15374
15375 fn selection_path(index: usize, key: &str) -> Vec<ParallelGroupMetadata> {
15376 vec![selection_group_entry(
15377 1,
15378 2,
15379 index,
15380 "activity",
15381 &SelectionMemberMetadata {
15382 key: SelectionKey::Name(key.to_string()),
15383 index,
15384 base_sequence: index as u64 + 1,
15385 size: 1,
15386 kind: "activity".to_string(),
15387 },
15388 )]
15389 }
15390
15391 fn selection_activity_event(
15392 event_type: &str,
15393 index: usize,
15394 key: &str,
15395 result: Option<Value>,
15396 ) -> HistoryEvent {
15397 let sequence = index as u64 + 1;
15398 let mut event = parallel_history_event(
15399 event_type,
15400 sequence,
15401 "activity_type",
15402 &format!("{key}-activity"),
15403 selection_path(index, key),
15404 result,
15405 );
15406 event.payload["activity_execution_id"] = json!(format!("activity-{key}"));
15407 event.raw.insert(
15408 "id".to_string(),
15409 json!(if event_type == "ActivityCompleted" {
15410 format!("event-{key}")
15411 } else {
15412 format!("{event_type}-{key}")
15413 }),
15414 );
15415 event
15416 }
15417
15418 fn selection_winner_marker() -> HistoryEvent {
15419 history_event(
15420 "SelectionResolved",
15421 json!({
15422 "selection_group_id": "select-calls:1:2",
15423 "selection_group_base_sequence": 1,
15424 "selection_group_size": 2,
15425 "member_key": "fast",
15426 "member_index": 1,
15427 "member_base_sequence": 2,
15428 "member_size": 1,
15429 "operation_kind": "activity",
15430 "operation_identity": "activity-fast",
15431 "outcome": "completed",
15432 "resolution_event_id": "event-fast",
15433 "resolution_event_type": "ActivityCompleted",
15434 }),
15435 )
15436 }
15437
15438 fn keyed_activity_selection(ctx: &WorkflowContext) -> SelectCall {
15439 ctx.select_keyed(vec![
15440 (
15441 "slow",
15442 ParallelOperation::activity_with_options(
15443 "slow-activity",
15444 ActivityOptions::new().task_queue("default"),
15445 json!([]),
15446 ),
15447 ),
15448 (
15449 "fast",
15450 ParallelOperation::activity_with_options(
15451 "fast-activity",
15452 ActivityOptions::new().task_queue("default"),
15453 json!([]),
15454 ),
15455 ),
15456 ])
15457 }
15458
15459 fn assert_persisted_selection_replay(history: Vec<HistoryEvent>) {
15460 let ctx = workflow_context(history);
15461 let mut call = Box::pin(keyed_activity_selection(&ctx));
15462 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15463 let selected = match call.as_mut().poll(&mut task_context) {
15464 Poll::Ready(Ok(selected)) => selected,
15465 Poll::Ready(Err(error)) => panic!("persisted selection winner must replay: {error:?}"),
15466 Poll::Pending => panic!("persisted selection winner must replay without pending"),
15467 };
15468 assert_eq!(selected.key, SelectionKey::Name("fast".to_string()));
15469 assert_eq!(
15470 selected.value,
15471 Some(ParallelResult::Activity(json!("winner-value")))
15472 );
15473 let slow = selected
15474 .handle(&SelectionKey::Name("slow".to_string()))
15475 .expect("slow handle")
15476 .clone();
15477 let mut await_slow = Box::pin(slow.await_result());
15478 assert!(matches!(
15479 await_slow.as_mut().poll(&mut task_context),
15480 Poll::Ready(Ok(ParallelResult::Activity(value))) if value == json!("loser-value")
15481 ));
15482 assert!(ctx.take_commands().expect("commands").is_empty());
15483 }
15484
15485 const SELECTION_COLD_REPLAY_HISTORY: &str = "DURABLE_WORKFLOW_SELECTION_COLD_REPLAY_HISTORY";
15486
15487 fn canonical_selection_history() -> Vec<HistoryEvent> {
15488 const FIXTURE: &[u8] =
15489 include_bytes!("../tests/fixtures/durable_selection_runtime_history.json");
15490 assert_eq!(
15491 format!("{:x}", Sha256::digest(FIXTURE)),
15492 "51fd8b9c16e978dcef536a5c727b9fdc0ae724d9afc17d9a7837d219f41ee3ba",
15493 );
15494 let fixture: Value = serde_json::from_slice(FIXTURE).expect("canonical selection fixture");
15495
15496 serde_json::from_value(fixture["history"].clone()).expect("canonical selection history")
15497 }
15498
15499 #[test]
15500 fn selection_fresh_process_entrypoint() {
15501 let Ok(path) = std::env::var(SELECTION_COLD_REPLAY_HISTORY) else {
15502 return;
15503 };
15504 let persisted = fs::read(path).expect("persisted selection history");
15505 assert_eq!(
15506 format!("{:x}", Sha256::digest(&persisted)),
15507 "51fd8b9c16e978dcef536a5c727b9fdc0ae724d9afc17d9a7837d219f41ee3ba",
15508 );
15509 let fixture: Value =
15510 serde_json::from_slice(&persisted).expect("valid persisted selection fixture");
15511 let history: Vec<HistoryEvent> = serde_json::from_value(fixture["history"].clone())
15512 .expect("valid persisted selection history");
15513
15514 assert_persisted_selection_replay(history);
15515 }
15516
15517 #[test]
15518 fn selection_starts_every_member_with_stable_keys_and_group_identity() {
15519 let ctx = workflow_context(Vec::new());
15520 let mut call = Box::pin(keyed_activity_selection(&ctx));
15521 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15522
15523 assert!(matches!(
15524 call.as_mut().poll(&mut task_context),
15525 Poll::Pending
15526 ));
15527 let commands = ctx.take_commands().expect("selection commands");
15528 assert_eq!(commands.len(), 2);
15529 assert_eq!(commands[0]["selection_member_key"], json!("slow"));
15530 assert_eq!(commands[1]["selection_member_key"], json!("fast"));
15531 assert!(commands.iter().all(|command| {
15532 command["parallel_group_id"] == json!("select-calls:1:2")
15533 && command["parallel_group_mode"] == json!("select")
15534 }));
15535 }
15536
15537 #[test]
15538 fn selection_key_domain_rejects_empty_authoring_and_malformed_history() {
15539 let ctx = workflow_context(Vec::new());
15540 let mut invalid = Box::pin(ctx.select_keyed(vec![(
15541 "",
15542 ParallelOperation::activity("invalid", json!([])),
15543 )]));
15544 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15545 assert!(matches!(
15546 invalid.as_mut().poll(&mut task_context),
15547 Poll::Ready(Err(Error::InvalidParallelGroup(ParallelGroupError {
15548 reason: "selection_key_invalid",
15549 ..
15550 })))
15551 ));
15552
15553 for invalid_key in [json!(""), json!(-1)] {
15554 let mut event = selection_activity_event("ActivityScheduled", 0, "slow", None);
15555 event.payload["selection_member_key"] = invalid_key.clone();
15556 event.payload["parallel_group_path"][0]["selection_member_key"] = invalid_key;
15557 assert!(matches!(
15558 WorkflowState::new_with_identity(
15559 vec![event],
15560 None,
15561 None,
15562 "rust-workers".to_string(),
15563 DEFAULT_CODEC.to_string(),
15564 None,
15565 ),
15566 Err(Error::NonDeterministicReplay(_))
15567 ));
15568 }
15569 }
15570
15571 #[test]
15572 fn selection_preserves_valid_named_and_numeric_keys() {
15573 let ctx = workflow_context(Vec::new());
15574 let mut selection = Box::pin(ctx.select_keyed(vec![
15575 (
15576 SelectionKey::Index(0),
15577 ParallelOperation::activity("numeric", json!([])),
15578 ),
15579 (
15580 SelectionKey::Name("named".to_string()),
15581 ParallelOperation::timer(Duration::from_secs(1)),
15582 ),
15583 ]));
15584 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15585
15586 assert!(matches!(
15587 selection.as_mut().poll(&mut task_context),
15588 Poll::Pending
15589 ));
15590 let commands = ctx.take_commands().expect("selection commands");
15591 assert_eq!(commands[0]["selection_member_key"], json!(0));
15592 assert_eq!(commands[1]["selection_member_key"], json!("named"));
15593 }
15594
15595 #[test]
15596 fn selection_replays_persisted_winner_and_loser_can_be_awaited_later() {
15597 let history = canonical_selection_history();
15598 assert_persisted_selection_replay(history.clone());
15599
15600 let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
15601 .join("tests/fixtures/durable_selection_runtime_history.json");
15602 let output =
15603 ProcessCommand::new(std::env::current_exe().expect("current Rust test binary"))
15604 .args([
15605 "--exact",
15606 "tests::selection_fresh_process_entrypoint",
15607 "--nocapture",
15608 ])
15609 .env(SELECTION_COLD_REPLAY_HISTORY, &path)
15610 .output()
15611 .expect("run fresh selection replay process");
15612
15613 assert!(
15614 output.status.success(),
15615 "fresh selection replay failed:\nstdout:\n{}\nstderr:\n{}",
15616 String::from_utf8_lossy(&output.stdout),
15617 String::from_utf8_lossy(&output.stderr),
15618 );
15619 }
15620
15621 #[test]
15622 fn selection_waits_durably_when_terminal_members_precede_the_winner_marker() {
15623 let mut history = canonical_selection_history();
15624 history.retain(|event| event.event_type != "SelectionResolved");
15625 let ctx = workflow_context(history);
15626 let mut selection = Box::pin(keyed_activity_selection(&ctx));
15627 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15628
15629 assert!(matches!(
15630 selection.as_mut().poll(&mut task_context),
15631 Poll::Pending
15632 ));
15633 assert!(ctx.take_commands().expect("commands").is_empty());
15634 assert!(
15635 ctx.matched_recorded_pending()
15636 .expect("selection pending state"),
15637 "terminal member history must keep the workflow durably pending until SelectionResolved commits"
15638 );
15639 }
15640
15641 #[test]
15642 fn selection_terminal_condition_history_waits_durably_for_its_winner_marker() {
15643 for (terminal_event, predicate_satisfied, timeout_seconds) in [
15644 ("ConditionWaitSatisfied", true, None),
15645 ("ConditionWaitTimedOut", false, Some(0)),
15646 ] {
15647 let member = SelectionMemberMetadata {
15648 key: SelectionKey::Name("condition".to_string()),
15649 index: 0,
15650 base_sequence: 1,
15651 size: 1,
15652 kind: "condition".to_string(),
15653 };
15654 let path = vec![selection_group_entry(1, 1, 0, "condition", &member)];
15655 let mut payload = json!({
15656 "sequence": 1,
15657 "condition_wait_id": "condition-1",
15658 "condition_wait_occurrence_id": "rust:condition-wait:0",
15659 "condition_key": "ready",
15660 "condition_definition_fingerprint": "sha256:ready-v1",
15661 "parallel_group_path": path,
15662 });
15663 payload
15664 .as_object_mut()
15665 .expect("condition history payload")
15666 .extend(
15667 serde_json::to_value(&path[0])
15668 .expect("condition selection metadata")
15669 .as_object()
15670 .expect("condition selection metadata object")
15671 .clone(),
15672 );
15673 if let Some(timeout_seconds) = timeout_seconds {
15674 payload["timeout_seconds"] = json!(timeout_seconds);
15675 }
15676 let history = vec![
15677 history_event("ConditionWaitOpened", payload.clone()),
15678 history_event(terminal_event, payload),
15679 ];
15680 let ctx = workflow_context(history);
15681 let mut options = ConditionWaitOptions::new("ready", "sha256:ready-v1");
15682 if timeout_seconds.is_some() {
15683 options = options.timeout(Duration::ZERO);
15684 }
15685 let mut selection = Box::pin(ctx.select_keyed(vec![(
15686 "condition",
15687 ParallelOperation::condition(options, move || Ok(predicate_satisfied)),
15688 )]));
15689 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15690
15691 assert!(matches!(
15692 selection.as_mut().poll(&mut task_context),
15693 Poll::Pending
15694 ));
15695 assert!(ctx.take_commands().expect("commands").is_empty());
15696 assert!(
15697 ctx.matched_recorded_pending()
15698 .expect("condition selection pending state"),
15699 "{terminal_event} must keep the workflow durably pending until SelectionResolved commits"
15700 );
15701 }
15702 }
15703
15704 #[test]
15705 fn selection_immediate_condition_members_open_a_durable_wait() {
15706 for predicate_satisfied in [true, false] {
15707 let ctx = workflow_context(Vec::new());
15708 let mut selection = Box::pin(ctx.select_keyed(vec![(
15709 "condition",
15710 ParallelOperation::condition(
15711 ConditionWaitOptions::new("ready", "sha256:ready-v1").timeout(Duration::ZERO),
15712 move || Ok(predicate_satisfied),
15713 ),
15714 )]));
15715 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15716
15717 assert!(matches!(
15718 selection.as_mut().poll(&mut task_context),
15719 Poll::Pending
15720 ));
15721 let commands = ctx.take_commands().expect("condition selection command");
15722 assert_eq!(commands.len(), 1);
15723 assert_eq!(commands[0]["type"], json!("open_condition_wait"));
15724 assert_eq!(commands[0]["timeout_seconds"], json!(0));
15725 assert_eq!(
15726 commands[0]["parallel_group_path"][0]["parallel_group_mode"],
15727 json!("select")
15728 );
15729 }
15730 }
15731
15732 #[test]
15733 fn selection_loser_cancellation_is_explicit_and_idempotent() {
15734 let history = vec![
15735 selection_activity_event("ActivityScheduled", 0, "slow", None),
15736 selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner"))),
15737 selection_winner_marker(),
15738 ];
15739 let ctx = workflow_context(history.clone());
15740 let mut call = Box::pin(keyed_activity_selection(&ctx));
15741 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15742 let Poll::Ready(Ok(selected)) = call.as_mut().poll(&mut task_context) else {
15743 panic!("winner must replay");
15744 };
15745 let slow = selected
15746 .handle(&SelectionKey::Name("slow".to_string()))
15747 .expect("slow handle")
15748 .clone();
15749 let mut cancel = Box::pin(slow.cancel());
15750 assert!(matches!(
15751 cancel.as_mut().poll(&mut task_context),
15752 Poll::Pending
15753 ));
15754 assert!(matches!(
15755 cancel.as_mut().poll(&mut task_context),
15756 Poll::Pending
15757 ));
15758 let commands = ctx.take_commands().expect("cancel command");
15759 assert_eq!(commands.len(), 1);
15760 assert_eq!(commands[0]["type"], json!("cancel_selection_operation"));
15761 assert_eq!(commands[0]["member_key"], json!("slow"));
15762
15763 let mut cancelled_history = history;
15764 cancelled_history.push(history_event(
15765 "SelectionOperationCancelled",
15766 json!({
15767 "selection_group_id": "select-calls:1:2",
15768 "member_key": "slow",
15769 "member_index": 0,
15770 "member_base_sequence": 1,
15771 "member_size": 1,
15772 "operation_kind": "activity",
15773 "operation_identity": "activity-slow",
15774 "cancelled_at": "2026-08-27T00:00:00Z",
15775 }),
15776 ));
15777 let replayed = workflow_context(cancelled_history);
15778 let mut call = Box::pin(keyed_activity_selection(&replayed));
15779 let Poll::Ready(Ok(selected)) = call.as_mut().poll(&mut task_context) else {
15780 panic!("winner must replay after cancellation");
15781 };
15782 let slow = selected
15783 .handle(&SelectionKey::Name("slow".to_string()))
15784 .expect("slow handle")
15785 .clone();
15786 let mut cancel = Box::pin(slow.cancel());
15787 assert!(matches!(
15788 cancel.as_mut().poll(&mut task_context),
15789 Poll::Ready(Ok(()))
15790 ));
15791 assert!(replayed.take_commands().expect("commands").is_empty());
15792 }
15793
15794 #[test]
15795 fn selection_cancellation_marker_is_bound_to_every_authored_handle_field() {
15796 let base_history = vec![
15797 selection_activity_event("ActivityScheduled", 0, "slow", None),
15798 selection_activity_event("ActivityScheduled", 1, "fast", None),
15799 selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner"))),
15800 selection_winner_marker(),
15801 ];
15802 for (field, corrupt) in [
15803 ("member_key", json!("fast")),
15804 ("member_index", json!(1)),
15805 ("member_base_sequence", json!(3)),
15806 ("member_size", json!(2)),
15807 ("operation_kind", json!("timer")),
15808 ("operation_identity", json!("forged")),
15809 ] {
15810 let mut cancellation = json!({
15811 "selection_group_id": "select-calls:1:2",
15812 "member_key": "slow",
15813 "member_index": 0,
15814 "member_base_sequence": 1,
15815 "member_size": 1,
15816 "operation_kind": "activity",
15817 "operation_identity": "activity-slow",
15818 });
15819 cancellation[field] = corrupt;
15820 let mut history = base_history.clone();
15821 history.push(history_event("SelectionOperationCancelled", cancellation));
15822 let ctx = workflow_context(history);
15823 let mut selection = Box::pin(keyed_activity_selection(&ctx));
15824 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15825
15826 assert!(matches!(
15827 selection.as_mut().poll(&mut task_context),
15828 Poll::Ready(Err(Error::NonDeterministicReplay(_)))
15829 ));
15830 }
15831 }
15832
15833 #[test]
15834 fn selection_child_identity_prefers_the_durable_run_id() {
15835 let ctx = workflow_context(vec![history_event(
15836 "ChildWorkflowScheduled",
15837 json!({
15838 "sequence": 1,
15839 "child_workflow_type": "child",
15840 "child_workflow_instance_id": "child-instance",
15841 "child_workflow_run_id": "child-run",
15842 }),
15843 )]);
15844 let state = ctx.state.lock().expect("workflow state");
15845
15846 assert_eq!(
15847 selection_operation_identity(&state, "child", 1, 1),
15848 "child-run"
15849 );
15850 }
15851
15852 #[test]
15853 fn selection_activity_identity_requires_canonical_execution_id() {
15854 let slow = selection_activity_event("ActivityScheduled", 0, "slow", None);
15855 let mut fast_open = selection_activity_event("ActivityScheduled", 1, "fast", None);
15856 let mut fast_completed =
15857 selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner")));
15858 for event in [&mut fast_open, &mut fast_completed] {
15859 event
15860 .payload
15861 .as_object_mut()
15862 .expect("activity payload")
15863 .remove("activity_execution_id");
15864 event.payload["activity_id"] = json!("forged-activity-id");
15865 }
15866 let mut marker = selection_winner_marker();
15867 marker.payload["operation_identity"] = json!("forged-activity-id");
15868 let ctx = workflow_context(vec![slow, fast_open, fast_completed, marker]);
15869 let mut selection = Box::pin(keyed_activity_selection(&ctx));
15870 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15871
15872 assert!(matches!(
15873 selection.as_mut().poll(&mut task_context),
15874 Poll::Ready(Err(Error::NonDeterministicReplay(_)))
15875 ));
15876 }
15877
15878 #[test]
15879 fn selection_completion_before_cancellation_remains_awaitable() {
15880 let history = vec![
15881 selection_activity_event("ActivityScheduled", 0, "slow", None),
15882 selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner"))),
15883 selection_winner_marker(),
15884 selection_activity_event(
15885 "ActivityCompleted",
15886 0,
15887 "slow",
15888 Some(json!("completed-first")),
15889 ),
15890 ];
15891 let ctx = workflow_context(history);
15892 let mut selection = Box::pin(keyed_activity_selection(&ctx));
15893 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15894 let Poll::Ready(Ok(selected)) = selection.as_mut().poll(&mut task_context) else {
15895 panic!("winner must replay");
15896 };
15897 let slow = selected
15898 .handle(&SelectionKey::Name("slow".to_string()))
15899 .expect("slow handle")
15900 .clone();
15901 let mut cancel = Box::pin(slow.cancel());
15902 assert!(matches!(
15903 cancel.as_mut().poll(&mut task_context),
15904 Poll::Ready(Ok(()))
15905 ));
15906 let mut await_slow = Box::pin(slow.await_result());
15907 assert!(matches!(
15908 await_slow.as_mut().poll(&mut task_context),
15909 Poll::Ready(Ok(ParallelResult::Activity(value))) if value == json!("completed-first")
15910 ));
15911 let commands = ctx.take_commands().expect("commands");
15912 assert!(commands.is_empty());
15913 }
15914
15915 #[test]
15916 fn selection_nested_later_failure_before_cancel_remains_the_awaited_failure() {
15917 let nested_member = SelectionMemberMetadata {
15918 key: SelectionKey::Name("nested".to_string()),
15919 index: 0,
15920 base_sequence: 1,
15921 size: 2,
15922 kind: "group".to_string(),
15923 };
15924 let deadline_member = SelectionMemberMetadata {
15925 key: SelectionKey::Name("deadline".to_string()),
15926 index: 1,
15927 base_sequence: 3,
15928 size: 1,
15929 kind: "timer".to_string(),
15930 };
15931 let nested_paths = [
15932 vec![
15933 selection_group_entry(1, 3, 0, "mixed", &nested_member),
15934 parallel_group_entry(1, 2, 0, "activity"),
15935 ],
15936 vec![
15937 selection_group_entry(1, 3, 1, "mixed", &nested_member),
15938 parallel_group_entry(1, 2, 1, "activity"),
15939 ],
15940 ];
15941 let deadline_path = vec![selection_group_entry(1, 3, 2, "mixed", &deadline_member)];
15942 let mut timer_fired = parallel_history_event(
15943 "TimerFired",
15944 3,
15945 "timer_id",
15946 "timer-3",
15947 deadline_path.clone(),
15948 None,
15949 );
15950 timer_fired.payload["delay_seconds"] = json!(0);
15951 timer_fired
15952 .raw
15953 .insert("id".to_string(), json!("timer-fired"));
15954 let mut timer_scheduled = parallel_history_event(
15955 "TimerScheduled",
15956 3,
15957 "timer_id",
15958 "timer-3",
15959 deadline_path,
15960 None,
15961 );
15962 timer_scheduled.payload["delay_seconds"] = json!(0);
15963 let history = vec![
15964 parallel_history_event(
15965 "ActivityScheduled",
15966 1,
15967 "activity_type",
15968 "nested-first",
15969 nested_paths[0].clone(),
15970 None,
15971 ),
15972 parallel_history_event(
15973 "ActivityScheduled",
15974 2,
15975 "activity_type",
15976 "nested-second",
15977 nested_paths[1].clone(),
15978 None,
15979 ),
15980 timer_scheduled,
15981 timer_fired,
15982 history_event(
15983 "SelectionResolved",
15984 json!({
15985 "selection_group_id": "select-calls:1:3",
15986 "selection_group_base_sequence": 1,
15987 "selection_group_size": 3,
15988 "member_key": "deadline",
15989 "member_index": 1,
15990 "member_base_sequence": 3,
15991 "member_size": 1,
15992 "operation_kind": "timer",
15993 "operation_identity": "timer-3",
15994 "outcome": "completed",
15995 "resolution_event_id": "timer-fired",
15996 "resolution_event_type": "TimerFired",
15997 }),
15998 ),
15999 parallel_history_event(
16000 "ActivityFailed",
16001 2,
16002 "activity_type",
16003 "nested-second",
16004 nested_paths[1].clone(),
16005 None,
16006 ),
16007 ];
16008 let ctx = workflow_context(history);
16009 let mut selection = Box::pin(ctx.select_keyed(vec![
16010 (
16011 "nested",
16012 ParallelOperation::group(vec![
16013 ParallelOperation::activity("nested-first", json!([])),
16014 ParallelOperation::activity("nested-second", json!([])),
16015 ]),
16016 ),
16017 ("deadline", ParallelOperation::timer(Duration::ZERO)),
16018 ]));
16019 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16020 let Poll::Ready(Ok(selected)) = selection.as_mut().poll(&mut task_context) else {
16021 panic!("deadline winner must replay");
16022 };
16023 let nested = selected
16024 .handle(&SelectionKey::Name("nested".to_string()))
16025 .expect("nested handle")
16026 .clone();
16027 let mut cancel = Box::pin(nested.cancel());
16028 assert!(matches!(
16029 cancel.as_mut().poll(&mut task_context),
16030 Poll::Ready(Ok(()))
16031 ));
16032 let mut await_nested = Box::pin(nested.await_result());
16033
16034 assert!(matches!(
16035 await_nested.as_mut().poll(&mut task_context),
16036 Poll::Ready(Err(Error::ActivityFailed(_)))
16037 ));
16038 assert!(ctx.take_commands().expect("commands").is_empty());
16039 }
16040
16041 #[test]
16042 fn selection_supports_child_timer_signal_condition_and_nested_groups() {
16043 let ctx = workflow_context(Vec::new());
16044 let mut call = Box::pin(ctx.select(vec![
16045 ParallelOperation::child_workflow(
16046 "child",
16047 ChildWorkflowOptions::new("children"),
16048 json!([]),
16049 ),
16050 ParallelOperation::timer(Duration::from_secs(30)),
16051 ParallelOperation::signal("approval"),
16052 ParallelOperation::condition(
16053 ConditionWaitOptions::new("ready", "sha256:ready"),
16054 || Ok(false),
16055 ),
16056 ParallelOperation::group(vec![
16057 ParallelOperation::activity("nested-one", json!([])),
16058 ParallelOperation::activity("nested-two", json!([])),
16059 ]),
16060 ]));
16061 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16062 assert!(matches!(
16063 call.as_mut().poll(&mut task_context),
16064 Poll::Pending
16065 ));
16066 let commands = ctx.take_commands().expect("selection commands");
16067 assert_eq!(
16068 commands
16069 .iter()
16070 .map(|command| command["type"].as_str().unwrap_or_default())
16071 .collect::<Vec<_>>(),
16072 [
16073 "start_child_workflow",
16074 "start_timer",
16075 "open_signal_wait",
16076 "open_condition_wait",
16077 "schedule_activity",
16078 "schedule_activity",
16079 ]
16080 );
16081 assert!(commands.iter().all(|command| {
16082 command["parallel_group_path"][0]["parallel_group_mode"] == json!("select")
16083 }));
16084 assert_eq!(
16085 commands[4]["parallel_group_path"].as_array().map(Vec::len),
16086 Some(2)
16087 );
16088 assert_eq!(
16089 commands[4]["parallel_group_path"][0]["selection_member_kind"],
16090 json!("group")
16091 );
16092 assert_eq!(
16093 commands[5]["parallel_group_path"][0]["selection_member_kind"],
16094 json!("group")
16095 );
16096
16097 let one_leaf_ctx = workflow_context(Vec::new());
16098 let mut one_leaf = Box::pin(one_leaf_ctx.select(vec![ParallelOperation::group(vec![
16099 ParallelOperation::activity("nested-only", json!([])),
16100 ])]));
16101 assert!(matches!(
16102 one_leaf.as_mut().poll(&mut task_context),
16103 Poll::Pending
16104 ));
16105 let one_leaf_commands = one_leaf_ctx.take_commands().expect("one-leaf commands");
16106 assert_eq!(one_leaf_commands.len(), 1);
16107 assert_eq!(
16108 one_leaf_commands[0]["parallel_group_path"][0]["selection_member_kind"],
16109 json!("group")
16110 );
16111 assert_eq!(
16112 one_leaf_commands[0]["parallel_group_path"][0]["selection_member_size"],
16113 json!(1)
16114 );
16115 }
16116
16117 async fn trip_saga(ctx: WorkflowContext) -> Result<Value> {
16118 let mut saga = ctx.saga();
16119 let outcome = async {
16120 let flight = ctx.activity("trip.reserve-flight", json!([])).await?;
16121 saga.add_compensation("trip.cancel-flight", json!([flight]))?;
16122 let hotel = ctx.activity("trip.reserve-hotel", json!([])).await?;
16123 saga.add_compensation("trip.cancel-hotel", json!([hotel]))?;
16124 ctx.activity("trip.charge", json!([])).await?;
16125 Ok(json!({"status": "booked"}))
16126 }
16127 .await;
16128 saga.finish(outcome).await
16129 }
16130
16131 fn saga_activity(
16132 event_type: &str,
16133 sequence: u64,
16134 activity_type: &str,
16135 result: Option<Value>,
16136 ) -> HistoryEvent {
16137 let mut payload = json!({
16138 "sequence": sequence,
16139 "activity_type": activity_type,
16140 "message": format!("{activity_type} failed"),
16141 "exception_type": "PlannedFailure",
16142 "non_retryable": true,
16143 });
16144 if let Some(result) = result {
16145 payload["result"] = fixture_envelope(result);
16146 }
16147 history_event(event_type, payload)
16148 }
16149
16150 #[test]
16151 fn saga_replays_reverse_compensation_across_restart_and_duplicate_delivery() {
16152 let completed_hotel_compensation = saga_activity(
16153 "ActivityCompleted",
16154 4,
16155 "trip.cancel-hotel",
16156 Some(Value::Null),
16157 );
16158 let history = vec![
16159 saga_activity(
16160 "ActivityCompleted",
16161 1,
16162 "trip.reserve-flight",
16163 Some(json!("flight-1")),
16164 ),
16165 saga_activity(
16166 "ActivityCompleted",
16167 2,
16168 "trip.reserve-hotel",
16169 Some(json!("hotel-1")),
16170 ),
16171 saga_activity("ActivityFailed", 3, "trip.charge", None),
16172 completed_hotel_compensation.clone(),
16173 completed_hotel_compensation,
16174 ];
16175
16176 for _restart in 0..2 {
16177 let ctx = workflow_context(history.clone());
16178 let mut future = Box::pin(trip_saga(ctx.clone()));
16179 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16180 assert!(matches!(
16181 future.as_mut().poll(&mut task_context),
16182 Poll::Pending
16183 ));
16184 let commands = ctx.take_commands().expect("compensation command");
16185 assert_eq!(commands.len(), 1);
16186 assert_eq!(commands[0]["activity_type"], "trip.cancel-flight");
16187 }
16188 }
16189
16190 #[test]
16191 fn saga_compensation_failure_preserves_both_typed_failures() {
16192 let history = vec![
16193 saga_activity(
16194 "ActivityCompleted",
16195 1,
16196 "trip.reserve-flight",
16197 Some(json!("flight-1")),
16198 ),
16199 saga_activity(
16200 "ActivityCompleted",
16201 2,
16202 "trip.reserve-hotel",
16203 Some(json!("hotel-1")),
16204 ),
16205 saga_activity("ActivityFailed", 3, "trip.charge", None),
16206 saga_activity("ActivityFailed", 4, "trip.cancel-hotel", None),
16207 ];
16208 let ctx = workflow_context(history);
16209 let mut future = Box::pin(trip_saga(ctx));
16210 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16211 let Poll::Ready(Err(Error::SagaCompensationFailed(failure))) =
16212 future.as_mut().poll(&mut task_context)
16213 else {
16214 panic!("compensation failure must remain structured");
16215 };
16216 assert!(matches!(
16217 *failure.initiating_failure,
16218 Error::ActivityFailed(_)
16219 ));
16220 assert!(matches!(
16221 *failure.compensation_failure,
16222 Error::ActivityFailed(_)
16223 ));
16224 assert_eq!(failure.compensation_activity_type, "trip.cancel-hotel");
16225 assert_eq!(failure.compensation_registration_order, 2);
16226 }
16227
16228 #[test]
16229 fn saga_compensates_cooperative_cancellation() {
16230 let ctx = workflow_context(vec![saga_activity(
16231 "ActivityCompleted",
16232 1,
16233 "trip.reserve-flight",
16234 Some(json!("flight-1")),
16235 )]);
16236 ctx.state.lock().expect("state").cancel_requested = true;
16237 let run = {
16238 let ctx = ctx.clone();
16239 async move {
16240 let mut saga = ctx.saga();
16241 let outcome = async {
16242 let flight = ctx.activity("trip.reserve-flight", json!([])).await?;
16243 saga.add_compensation("trip.cancel-flight", json!([flight]))?;
16244 ctx.throw_if_cancellation_requested()?;
16245 Ok(json!("unexpected"))
16246 }
16247 .await;
16248 saga.finish(outcome).await
16249 }
16250 };
16251 let mut future = Box::pin(run);
16252 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16253 assert!(matches!(
16254 future.as_mut().poll(&mut task_context),
16255 Poll::Pending
16256 ));
16257 let commands = ctx.take_commands().expect("cancellation compensation");
16258 assert_eq!(commands[0]["activity_type"], "trip.cancel-flight");
16259 }
16260
16261 fn workflow_task(
16262 workflow_type: &str,
16263 history_events: Vec<HistoryEvent>,
16264 payload_codec: &str,
16265 ) -> WorkflowTask {
16266 WorkflowTask {
16267 task_id: format!("wft-{workflow_type}"),
16268 workflow_command_id: None,
16269 workflow_id: Some(format!("wf-{workflow_type}")),
16270 run_id: Some(format!("run-{workflow_type}")),
16271 workflow_type: workflow_type.to_string(),
16272 cancel_requested: false,
16273 payload_codec: payload_codec.to_string(),
16274 arguments: Some(
16275 encode_value_envelope(&json!([]), payload_codec).expect("workflow arguments"),
16276 ),
16277 total_history_events: Some(history_events.len() as u64),
16278 history_size_bytes: None,
16279 continue_as_new_recommended: None,
16280 history_budget_pressure: None,
16281 history_events,
16282 next_history_page_token: None,
16283 workflow_task_attempt: 1,
16284 workflow_signal_id: None,
16285 signal_name: None,
16286 signal_arguments: None,
16287 workflow_update_id: None,
16288 update_name: None,
16289 lease_owner: Some("rust-worker".to_string()),
16290 }
16291 }
16292
16293 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
16294 struct SideEffectProbe {
16295 request_id: String,
16296 attempt: u32,
16297 }
16298
16299 #[test]
16300 fn typed_side_effect_runs_callback_once_and_replay_skips_it() {
16301 let calls = AtomicUsize::new(0);
16302 let ctx = workflow_context(Vec::new());
16303 let value = ctx
16304 .side_effect(|| {
16305 calls.fetch_add(1, Ordering::SeqCst);
16306 SideEffectProbe {
16307 request_id: "request-42".to_string(),
16308 attempt: 3,
16309 }
16310 })
16311 .expect("first side effect");
16312 assert_eq!(value.attempt, 3);
16313 assert_eq!(calls.load(Ordering::SeqCst), 1);
16314 let commands = ctx.take_commands().expect("commands");
16315 assert_eq!(commands.len(), 1);
16316 assert_eq!(commands[0]["type"], "record_side_effect");
16317 assert_eq!(
16318 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
16319 serde_json::to_value(&value).expect("value")
16320 );
16321
16322 let replay = workflow_context(vec![history_event(
16323 "SideEffectRecorded",
16324 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
16325 )]);
16326 let replayed: SideEffectProbe = replay
16327 .side_effect(|| {
16328 calls.fetch_add(1, Ordering::SeqCst);
16329 panic!("committed side-effect callbacks must not run during replay")
16330 })
16331 .expect("replayed side effect");
16332 assert_eq!(replayed, value);
16333 assert_eq!(calls.load(Ordering::SeqCst), 1);
16334 assert!(replay.take_commands().expect("commands").is_empty());
16335 replay.ensure_history_consumed().expect("history consumed");
16336 }
16337
16338 #[test]
16339 fn side_effect_uses_avro_envelope_and_uuid_is_replay_stable() {
16340 let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
16341 let value = ctx
16342 .side_effect(|| SideEffectProbe {
16343 request_id: "avro-request".to_string(),
16344 attempt: 1,
16345 })
16346 .expect("Avro side effect");
16347 let uuid = ctx.uuid_v4().expect("deterministic UUID");
16348 let commands = ctx.take_commands().expect("commands");
16349 assert_eq!(commands.len(), 2);
16350 assert_eq!(commands[0]["result"]["codec"], DEFAULT_CODEC);
16351 assert_eq!(commands[1]["result"]["codec"], DEFAULT_CODEC);
16352 assert_eq!(
16353 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
16354 serde_json::to_value(&value).expect("value")
16355 );
16356
16357 let replay = workflow_context_with_codec(
16358 vec![
16359 history_event(
16360 "SideEffectRecorded",
16361 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
16362 ),
16363 history_event(
16364 "SideEffectRecorded",
16365 json!({"sequence": 2, "result": commands[1]["result"].clone()}),
16366 ),
16367 ],
16368 DEFAULT_CODEC,
16369 );
16370 let replayed: SideEffectProbe = replay
16371 .side_effect(|| panic!("Avro callback must not run"))
16372 .expect("replayed Avro value");
16373 let replayed_uuid = replay.uuid_v4().expect("replayed UUID");
16374 assert_eq!(replayed, value);
16375 assert_eq!(replayed_uuid, uuid);
16376 assert!(replay.take_commands().expect("commands").is_empty());
16377 }
16378
16379 #[test]
16380 fn typed_side_effect_replay_preserves_bytes_and_maps() {
16381 let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
16382 let value = ctx
16383 .side_effect_avro_value(typed_fidelity_probe)
16384 .expect("typed side effect");
16385 let commands = ctx.take_commands().expect("side-effect command");
16386 assert_eq!(
16387 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
16388 .expect("recorded side effect"),
16389 value
16390 );
16391
16392 let replay = workflow_context_with_codec(
16393 vec![history_event(
16394 "SideEffectRecorded",
16395 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
16396 )],
16397 DEFAULT_CODEC,
16398 );
16399 assert_eq!(
16400 replay
16401 .side_effect_avro_value(|| panic!("replay must not invoke callback"))
16402 .expect("replayed typed side effect"),
16403 value
16404 );
16405 }
16406
16407 #[test]
16408 fn ordered_side_effects_share_the_durable_command_stream() {
16409 let first = encode_value_envelope(&json!("first"), DEFAULT_CODEC).expect("first");
16410 let second = encode_value_envelope(&json!(29), DEFAULT_CODEC).expect("second");
16411 let ctx = workflow_context(vec![
16412 history_event(
16413 "SideEffectRecorded",
16414 json!({"sequence": 1, "result": first}),
16415 ),
16416 history_event(
16417 "SideEffectRecorded",
16418 json!({"sequence": 2, "result": second}),
16419 ),
16420 ]);
16421 let first: String = ctx
16422 .side_effect(|| panic!("first callback must not run"))
16423 .expect("first replay");
16424 let second: i32 = ctx
16425 .side_effect(|| panic!("second callback must not run"))
16426 .expect("second replay");
16427 assert_eq!(first, "first");
16428 assert_eq!(second, 29);
16429 ctx.ensure_history_consumed().expect("ordered history");
16430
16431 let reordered = workflow_context(vec![history_event(
16432 "VersionMarkerRecorded",
16433 json!({
16434 "sequence": 1,
16435 "change_id": "before-side-effect",
16436 "version": 1,
16437 "min_supported": 1,
16438 "max_supported": 1,
16439 }),
16440 )]);
16441 let error = reordered
16442 .side_effect(|| "new".to_string())
16443 .expect_err("command reordering must fail");
16444 assert!(matches!(
16445 error,
16446 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16447 if reason == "recorded_command_mismatch"
16448 ));
16449 }
16450
16451 #[test]
16452 fn version_markers_replay_across_upgrades_and_do_not_duplicate() {
16453 let ctx = workflow_context(Vec::new());
16454 assert_eq!(ctx.get_version("checkout-v2", 1, 2).expect("version"), 2);
16455 assert_eq!(ctx.get_version("checkout-v2", 1, 3).expect("cached"), 2);
16456 assert!(ctx.patched("new-search").expect("patch"));
16457 ctx.deprecate_patch("new-search").expect("deprecate patch");
16458 let commands = ctx.take_commands().expect("commands");
16459 assert_eq!(commands.len(), 2);
16460 assert_eq!(commands[0]["type"], "record_version_marker");
16461 assert_eq!(commands[0]["version"], 2);
16462 assert_eq!(commands[1]["change_id"], "new-search");
16463
16464 let replay = workflow_context(vec![history_event(
16465 "VersionMarkerRecorded",
16466 json!({
16467 "sequence": 1,
16468 "change_id": "checkout-v2",
16469 "version": 2,
16470 "min_supported": 1,
16471 "max_supported": 2,
16472 }),
16473 )]);
16474 assert_eq!(replay.get_version("checkout-v2", 1, 4).expect("upgrade"), 2);
16475 assert_eq!(replay.get_version("checkout-v2", 2, 5).expect("repeat"), 2);
16476 assert!(replay.take_commands().expect("commands").is_empty());
16477 replay.ensure_history_consumed().expect("history consumed");
16478 }
16479
16480 #[test]
16481 fn version_markers_reject_incompatible_or_malformed_history() {
16482 let incompatible = workflow_context(vec![history_event(
16483 "VersionMarkerRecorded",
16484 json!({
16485 "sequence": 1,
16486 "change_id": "checkout-v2",
16487 "version": 1,
16488 "min_supported": 1,
16489 "max_supported": 2,
16490 }),
16491 )]);
16492 let error = incompatible
16493 .get_version("checkout-v2", 2, 3)
16494 .expect_err("old version is unsupported");
16495 assert!(matches!(
16496 error,
16497 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16498 if reason == "version_marker_incompatible_range"
16499 ));
16500
16501 for (history, reason) in [
16502 (
16503 vec![history_event("SideEffectRecorded", json!({"sequence": 1}))],
16504 "side_effect_result_missing",
16505 ),
16506 (
16507 vec![history_event(
16508 "SideEffectRecorded",
16509 json!({
16510 "sequence": 1,
16511 "result": {"codec": "avro", "blob": "not-base64"},
16512 }),
16513 )],
16514 "side_effect_payload_incompatible",
16515 ),
16516 (
16517 vec![history_event(
16518 "SideEffectRecorded",
16519 json!({"sequence": 1, "result": {"unwrapped": true}}),
16520 )],
16521 "side_effect_payload_malformed",
16522 ),
16523 (
16524 vec![history_event(
16525 "VersionMarkerRecorded",
16526 json!({
16527 "sequence": 1,
16528 "change_id": "change",
16529 "version": 1,
16530 "min_supported": 2,
16531 "max_supported": 1,
16532 }),
16533 )],
16534 "version_marker_history_range_invalid",
16535 ),
16536 ] {
16537 let error = WorkflowState::new(
16538 history,
16539 "rust-workers".to_string(),
16540 DEFAULT_CODEC.to_string(),
16541 None,
16542 )
16543 .expect_err("malformed history must fail");
16544 assert!(matches!(
16545 error,
16546 Error::NonDeterministicReplay(ReplayFailure { reason: actual, .. })
16547 if actual == reason
16548 ));
16549 }
16550 }
16551
16552 #[test]
16553 fn typed_search_attributes_replay_value_and_type_identity_after_restart() {
16554 let history = vec![history_event(
16555 "SearchAttributesUpserted",
16556 json!({
16557 "sequence": 1,
16558 "attributes": {"customer_tier": "gold"},
16559 "attribute_types": {"customer_tier": "keyword"},
16560 "merged": {"customer_tier": "gold"}
16561 }),
16562 )];
16563
16564 let matching = workflow_context(history.clone());
16565 matching
16566 .upsert_search_attributes(
16567 SearchAttributeUpdate::new()
16568 .keyword("customer_tier", "gold")
16569 .expect("keyword update"),
16570 )
16571 .expect("matching typed update must replay");
16572 matching
16573 .ensure_history_consumed()
16574 .expect("history consumed");
16575
16576 let changed_type = workflow_context(history.clone());
16577 let error = changed_type
16578 .upsert_search_attributes(
16579 SearchAttributeUpdate::new()
16580 .string("customer_tier", "gold")
16581 .expect("string update"),
16582 )
16583 .expect_err("same JSON value with a different declaration must be nondeterministic");
16584 let Error::NonDeterministicReplay(failure) = error else {
16585 panic!("typed identity drift must be a replay failure");
16586 };
16587 assert_eq!(failure.reason, "search_attribute_type_mismatch");
16588 assert_eq!(failure.sequence, Some(1));
16589
16590 let changed_value = workflow_context(history);
16591 let error = changed_value
16592 .upsert_search_attributes(
16593 SearchAttributeUpdate::new()
16594 .keyword("customer_tier", "platinum")
16595 .expect("keyword update"),
16596 )
16597 .expect_err("changed values must be nondeterministic");
16598 let Error::NonDeterministicReplay(failure) = error else {
16599 panic!("value drift must be a replay failure");
16600 };
16601 assert_eq!(failure.reason, "search_attribute_value_mismatch");
16602 }
16603
16604 #[test]
16605 fn legacy_search_attribute_history_keeps_type_identity_unknown() {
16606 let history = vec![history_event(
16607 "SearchAttributesUpserted",
16608 json!({
16609 "sequence": 1,
16610 "attributes": {"customer_tier": "gold"},
16611 "merged": {"customer_tier": "gold"}
16612 }),
16613 )];
16614
16615 for update in [
16616 SearchAttributeUpdate::new()
16617 .keyword("customer_tier", "gold")
16618 .expect("keyword update"),
16619 SearchAttributeUpdate::new()
16620 .string("customer_tier", "gold")
16621 .expect("string update"),
16622 ] {
16623 let restarted = workflow_context(history.clone());
16624 restarted
16625 .upsert_search_attributes(update)
16626 .expect("legacy history constrains values but has unknown type identity");
16627 restarted
16628 .ensure_history_consumed()
16629 .expect("history consumed");
16630 }
16631 }
16632
16633 #[test]
16634 fn search_attribute_command_emits_canonical_types() {
16635 let ctx = workflow_context(Vec::new());
16636 ctx.upsert_search_attributes(
16637 SearchAttributeUpdate::new()
16638 .keyword("customer_tier", "gold")
16639 .expect("keyword update")
16640 .int("attempts", 3)
16641 .expect("int update")
16642 .delete("obsolete")
16643 .expect("delete update"),
16644 )
16645 .expect("valid search attributes");
16646
16647 assert_eq!(
16648 ctx.take_commands().expect("commands"),
16649 vec![json!({
16650 "type": "upsert_search_attributes",
16651 "attributes": {
16652 "attempts": 3,
16653 "customer_tier": "gold",
16654 "obsolete": null
16655 },
16656 "attribute_types": {
16657 "attempts": "int",
16658 "customer_tier": "keyword"
16659 }
16660 })]
16661 );
16662 }
16663
16664 #[test]
16665 fn duplicate_side_effects_and_version_markers_are_rejected() {
16666 let duplicate_side_effect = WorkflowState::new(
16667 vec![
16668 history_event(
16669 "SideEffectRecorded",
16670 json!({"sequence": 1, "result": fixture_envelope(json!(1))}),
16671 ),
16672 history_event(
16673 "SideEffectRecorded",
16674 json!({"sequence": 1, "result": fixture_envelope(json!(2))}),
16675 ),
16676 ],
16677 "rust-workers".to_string(),
16678 DEFAULT_CODEC.to_string(),
16679 None,
16680 )
16681 .expect_err("duplicate side effect");
16682 assert!(matches!(
16683 duplicate_side_effect,
16684 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16685 if reason == "duplicate_side_effect_record"
16686 ));
16687
16688 let marker = |sequence| {
16689 history_event(
16690 "VersionMarkerRecorded",
16691 json!({
16692 "sequence": sequence,
16693 "change_id": "same-change",
16694 "version": 1,
16695 "min_supported": 1,
16696 "max_supported": 1,
16697 }),
16698 )
16699 };
16700 let duplicate_marker = WorkflowState::new(
16701 vec![marker(1), marker(3)],
16702 "rust-workers".to_string(),
16703 DEFAULT_CODEC.to_string(),
16704 None,
16705 )
16706 .expect_err("duplicate marker");
16707 assert!(matches!(
16708 duplicate_marker,
16709 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16710 if reason == "duplicate_version_marker"
16711 ));
16712 }
16713
16714 #[test]
16715 fn workflow_stream_authoring_derives_identity_and_replay_skips_duplicate_append() {
16716 let mut state = WorkflowState::new(
16717 Vec::new(),
16718 "rust-workers".to_string(),
16719 DEFAULT_CODEC.to_string(),
16720 None,
16721 )
16722 .expect("workflow state");
16723 state.workflow_command_identity = "command-7".to_string();
16724 let context = WorkflowContext {
16725 state: Arc::new(Mutex::new(state)),
16726 };
16727 let item =
16728 WorkflowStreamAppendItem::from_reference("s3://bucket/item.avro").item_type("receipt");
16729
16730 context
16731 .append_workflow_stream("output", &[item], Some(10))
16732 .expect("append command");
16733 context
16734 .error_workflow_stream("output", "producer failed", None)
16735 .expect("error command");
16736 let commands = context.take_commands().expect("commands");
16737
16738 assert_eq!(commands[0]["type"], "record_side_effect");
16739 assert_eq!(
16740 commands[0]["workflow_stream"]["command_identity"],
16741 "command-7"
16742 );
16743 assert_eq!(commands[0]["workflow_stream"]["command_ordinal"], 0);
16744 assert_eq!(
16745 commands[0]["workflow_stream"]["items"][0]["idempotency_key"],
16746 "dw-stream:command-7:0:0"
16747 );
16748 assert_eq!(commands[1]["workflow_stream"]["operation"], "error");
16749
16750 let recorded = history_event(
16751 "SideEffectRecorded",
16752 json!({"sequence": 1, "result": fixture_envelope(Value::Null)}),
16753 );
16754 let mut replay_state = WorkflowState::new(
16755 vec![recorded],
16756 "rust-workers".to_string(),
16757 DEFAULT_CODEC.to_string(),
16758 None,
16759 )
16760 .expect("replay state");
16761 replay_state.workflow_command_identity = "command-7".to_string();
16762 let replay_context = WorkflowContext {
16763 state: Arc::new(Mutex::new(replay_state)),
16764 };
16765 replay_context
16766 .append_workflow_stream(
16767 "output",
16768 &[WorkflowStreamAppendItem::from_reference(
16769 "s3://bucket/item.avro",
16770 )],
16771 Some(10),
16772 )
16773 .expect("replayed append");
16774 assert!(replay_context
16775 .take_commands()
16776 .expect("replayed commands")
16777 .is_empty());
16778 }
16779
16780 #[test]
16781 fn workflow_stream_authoring_requires_server_durable_command_identity() {
16782 let context = workflow_context(Vec::new());
16783 let error = context
16784 .append_workflow_stream(
16785 "output",
16786 &[WorkflowStreamAppendItem::from_reference(
16787 "s3://bucket/item.avro",
16788 )],
16789 None,
16790 )
16791 .expect_err("stream append without durable command identity must fail closed");
16792
16793 assert!(matches!(error, Error::MissingWorkflowCommandIdentity));
16794 assert!(context.take_commands().expect("commands").is_empty());
16795 }
16796
16797 #[test]
16798 fn cold_worker_replay_does_not_repeat_committed_side_effects_or_markers() {
16799 fn worker(calls: Arc<AtomicUsize>) -> Worker {
16800 let client = Client::new("http://127.0.0.1:8080").expect("client");
16801 let mut worker = Worker::new(client, "rust-workers");
16802 worker.register_workflow("rust.side-effect-version", move |ctx, _input| {
16803 let calls = Arc::clone(&calls);
16804 async move {
16805 let captured = ctx.side_effect(|| {
16806 calls.fetch_add(1, Ordering::SeqCst);
16807 "captured-once".to_string()
16808 })?;
16809 let version = ctx.get_version("cold-restart", 1, 2)?;
16810 Ok(json!({"captured": captured, "version": version}))
16811 }
16812 });
16813 worker
16814 }
16815
16816 fn task(history_events: Vec<HistoryEvent>) -> WorkflowTask {
16817 WorkflowTask {
16818 task_id: "wft-side-effect-version".to_string(),
16819 workflow_command_id: None,
16820 workflow_id: Some("wf-side-effect-version".to_string()),
16821 run_id: Some("run-side-effect-version".to_string()),
16822 workflow_type: "rust.side-effect-version".to_string(),
16823 cancel_requested: false,
16824 payload_codec: DEFAULT_CODEC.to_string(),
16825 arguments: Some(
16826 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("arguments"),
16827 ),
16828 history_events,
16829 total_history_events: None,
16830 history_size_bytes: None,
16831 continue_as_new_recommended: None,
16832 history_budget_pressure: None,
16833 next_history_page_token: None,
16834 workflow_task_attempt: 1,
16835 workflow_signal_id: None,
16836 signal_name: None,
16837 signal_arguments: None,
16838 workflow_update_id: None,
16839 update_name: None,
16840 lease_owner: Some("rust-worker".to_string()),
16841 }
16842 }
16843
16844 let calls = Arc::new(AtomicUsize::new(0));
16845 let initial = worker(Arc::clone(&calls))
16846 .execute_workflow_task(task(Vec::new()))
16847 .expect("initial execution");
16848 assert_eq!(
16849 initial
16850 .iter()
16851 .map(|command| &command["type"])
16852 .collect::<Vec<_>>(),
16853 vec![
16854 "record_side_effect",
16855 "record_version_marker",
16856 "complete_workflow"
16857 ]
16858 );
16859 assert_eq!(calls.load(Ordering::SeqCst), 1);
16860
16861 let restarted = worker(Arc::clone(&calls));
16862 let replayed = restarted
16863 .execute_workflow_task(task(vec![
16864 history_event(
16865 "SideEffectRecorded",
16866 json!({"sequence": 1, "result": initial[0]["result"].clone()}),
16867 ),
16868 history_event(
16869 "VersionMarkerRecorded",
16870 json!({
16871 "sequence": 2,
16872 "change_id": "cold-restart",
16873 "version": 2,
16874 "min_supported": 1,
16875 "max_supported": 2,
16876 }),
16877 ),
16878 ]))
16879 .expect("cold replay");
16880 assert_eq!(replayed.len(), 1);
16881 assert_eq!(replayed[0]["type"], "complete_workflow");
16882 assert_eq!(calls.load(Ordering::SeqCst), 1);
16883 }
16884
16885 #[test]
16886 fn side_effect_replay_rejects_changed_rust_value_type() {
16887 let result = encode_value_envelope(&json!({"value": 42}), DEFAULT_CODEC).expect("result");
16888 let ctx = workflow_context(vec![history_event(
16889 "SideEffectRecorded",
16890 json!({"sequence": 1, "result": result}),
16891 )]);
16892 let error = ctx
16893 .side_effect::<Vec<String>, _>(|| panic!("callback must not run"))
16894 .expect_err("changed type must fail replay");
16895 assert!(matches!(
16896 error,
16897 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16898 if reason == "side_effect_type_mismatch"
16899 ));
16900 }
16901
16902 fn completed_retry_activity_history() -> Vec<HistoryEvent> {
16903 vec![
16904 history_event(
16905 "ActivityScheduled",
16906 json!({
16907 "sequence": 1,
16908 "activity_type": "flaky",
16909 "activity_execution_id": "act-1",
16910 "activity": {
16911 "id": "act-1",
16912 "sequence": 1,
16913 "type": "flaky",
16914 "queue": "critical-activities",
16915 "execution_mode": null,
16916 "retry_policy": {
16917 "snapshot_version": 1,
16918 "max_attempts": 3,
16919 "backoff_seconds": [2, 4],
16920 "start_to_close_timeout": 30,
16921 "schedule_to_start_timeout": 5,
16922 "schedule_to_close_timeout": 90,
16923 "heartbeat_timeout": 10,
16924 "non_retryable_error_types": ["PermanentError"]
16925 }
16926 }
16927 }),
16928 ),
16929 history_event(
16930 "ActivityStarted",
16931 json!({
16932 "sequence": 1,
16933 "activity_type": "flaky",
16934 "activity_execution_id": "act-1",
16935 "activity_attempt_id": "attempt-1",
16936 "attempt_number": 1
16937 }),
16938 ),
16939 history_event(
16940 "ActivityRetryScheduled",
16941 json!({
16942 "sequence": 1,
16943 "activity_type": "flaky",
16944 "activity_execution_id": "act-1",
16945 "activity_attempt_id": "attempt-1",
16946 "attempt_number": 1,
16947 "retry_after_attempt": 1,
16948 "retry_backoff_seconds": 2,
16949 "failure_category": "activity",
16950 "exception_type": "TransientError"
16951 }),
16952 ),
16953 history_event(
16954 "ActivityStarted",
16955 json!({
16956 "sequence": 1,
16957 "activity_type": "flaky",
16958 "activity_execution_id": "act-1",
16959 "activity_attempt_id": "attempt-2",
16960 "attempt_number": 2
16961 }),
16962 ),
16963 history_event(
16964 "ActivityCompleted",
16965 json!({
16966 "sequence": 1,
16967 "activity_type": "flaky",
16968 "activity_execution_id": "act-1",
16969 "activity_attempt_id": "attempt-2",
16970 "attempt_number": 2,
16971 "payload_codec": DEFAULT_CODEC,
16972 "result": fixture_envelope(json!({"status":"recovered"}))
16973 }),
16974 ),
16975 ]
16976 }
16977
16978 fn retry_activity_options() -> ActivityOptions {
16979 ActivityOptions::new()
16980 .task_queue("critical-activities")
16981 .retry_policy(
16982 ActivityRetryPolicy::new(3)
16983 .backoff_intervals([Duration::from_secs(2), Duration::from_secs(4)])
16984 .non_retryable_error_type("PermanentError"),
16985 )
16986 .start_to_close_timeout(Duration::from_secs(30))
16987 .schedule_to_start_timeout(Duration::from_secs(5))
16988 .schedule_to_close_timeout(Duration::from_secs(90))
16989 .heartbeat_timeout(Duration::from_secs(10))
16990 }
16991
16992 #[test]
16993 fn fixed_avro_value_round_trips_json_values() {
16994 let value = json!({"greeting": "hello", "count": 3, "ok": true});
16995 let envelope = PayloadEnvelope::avro(&value).expect("encode");
16996 assert_eq!(envelope.codec, DEFAULT_CODEC);
16997 assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
16998 }
16999
17000 #[tokio::test]
17001 async fn typed_handler_adapters_round_trip_serde_contracts_on_the_fixed_wire() {
17002 let client = Client::new("http://127.0.0.1:8080").expect("client");
17003 let mut worker = Worker::new(client, "rust-workers");
17004 worker.register_typed_workflow(
17005 "typed.contract.workflow",
17006 |_ctx, input: TypedContract| async move { Ok(input) },
17007 );
17008 worker.register_typed_activity(
17009 "typed.contract.activity",
17010 |_ctx, input: TypedContract| async move { Ok(input) },
17011 );
17012
17013 let expected = typed_contract();
17014 let arguments = AvroValue::Array(vec![
17015 AvroValue::from_serialize(&expected).expect("typed request")
17016 ]);
17017 let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("arguments");
17018 let mut workflow = workflow_task("typed.contract.workflow", Vec::new(), DEFAULT_CODEC);
17019 workflow.arguments = Some(envelope.clone());
17020 let commands = worker
17021 .execute_workflow_task(workflow)
17022 .expect("typed workflow task");
17023 let workflow_result: TypedContract =
17024 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17025 .expect("workflow result envelope")
17026 .deserialize()
17027 .expect("workflow result type");
17028 assert_eq!(workflow_result, expected);
17029
17030 let activity = ActivityTask {
17031 task_id: "typed-contract-activity".to_string(),
17032 activity_attempt_id: Some("typed-contract-attempt".to_string()),
17033 attempt_id: None,
17034 activity_type: "typed.contract.activity".to_string(),
17035 payload_codec: DEFAULT_CODEC.to_string(),
17036 arguments: Some(envelope),
17037 attempt_number: 1,
17038 lease_owner: Some("rust-worker".to_string()),
17039 };
17040 let activity_result: TypedContract = worker
17041 .execute_activity_task(activity)
17042 .await
17043 .expect("typed activity task")
17044 .deserialize()
17045 .expect("activity result type");
17046 assert_eq!(activity_result, expected);
17047 }
17048
17049 #[tokio::test]
17050 async fn typed_handler_errors_include_handler_name_direction_and_rust_type() {
17051 let client = Client::new("http://127.0.0.1:8080").expect("client");
17052 let mut worker = Worker::new(client, "rust-workers");
17053 worker.register_typed_workflow(
17054 "typed.shape.workflow",
17055 |_ctx, input: TypedContract| async move { Ok(input) },
17056 );
17057 worker.register_typed_activity("typed.unsupported.activity", |_ctx, (): ()| async move {
17058 Ok(f64::NAN)
17059 });
17060
17061 let mut workflow = workflow_task("typed.shape.workflow", Vec::new(), DEFAULT_CODEC);
17062 workflow.arguments = Some(
17063 encode_typed_envelope(
17064 &AvroValue::Array(vec![
17065 AvroValue::String("first".to_string()),
17066 AvroValue::String("second".to_string()),
17067 ]),
17068 DEFAULT_CODEC,
17069 )
17070 .expect("malformed typed arguments"),
17071 );
17072 let commands = worker
17073 .execute_workflow_task(workflow)
17074 .expect("shape mismatch becomes a workflow failure");
17075 let message = commands[0]["message"].as_str().expect("failure message");
17076 assert!(message.contains("workflow handler \"typed.shape.workflow\" input type"));
17077 assert!(message.contains(type_name::<TypedContract>()));
17078 assert!(message.contains("task carried 2 arguments"));
17079
17080 let activity = ActivityTask {
17081 task_id: "typed-unsupported-activity".to_string(),
17082 activity_attempt_id: Some("typed-unsupported-attempt".to_string()),
17083 attempt_id: None,
17084 activity_type: "typed.unsupported.activity".to_string(),
17085 payload_codec: DEFAULT_CODEC.to_string(),
17086 arguments: Some(
17087 encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
17088 .expect("unit arguments"),
17089 ),
17090 attempt_number: 1,
17091 lease_owner: Some("rust-worker".to_string()),
17092 };
17093 let Error::HandlerType {
17094 handler_kind,
17095 handler_name,
17096 value_kind,
17097 rust_type,
17098 message,
17099 } = worker
17100 .execute_activity_task(activity)
17101 .await
17102 .expect_err("non-finite handler output must fail")
17103 else {
17104 panic!("expected contextual handler type failure");
17105 };
17106 assert_eq!(handler_kind, HandlerKind::Activity);
17107 assert_eq!(handler_name, "typed.unsupported.activity");
17108 assert_eq!(value_kind, HandlerValueKind::Result);
17109 assert_eq!(rust_type, type_name::<f64>());
17110 assert!(message.contains("non_finite_float"));
17111 }
17112
17113 #[tokio::test]
17114 async fn typed_replayed_workflow_decodes_input_and_activity_result_losslessly() {
17115 #[derive(Clone, Default)]
17116 struct State {
17117 observed: Option<TypedContract>,
17118 }
17119
17120 let client = Client::new("http://127.0.0.1:8080").expect("client");
17121 let mut worker = Worker::new(client, "rust-workers");
17122 worker.register_typed_replayed_workflow(
17123 "typed.contract.replayed",
17124 State::default,
17125 |ctx, input: TypedContract, state| async move {
17126 let result: TypedContract =
17127 ctx.activity_typed("typed.contract.activity", input).await?;
17128 state.update(|current| current.observed = Some(result.clone()))?;
17129 Ok(result)
17130 },
17131 );
17132 worker.register_replayed_query::<State, _, _>(
17133 "typed.contract.replayed",
17134 "observed",
17135 |_ctx, state, _args| async move {
17136 Ok(json!(state.observed.as_ref().map(|value| value.signed)))
17137 },
17138 );
17139
17140 let expected = typed_contract();
17141 let typed_value = AvroValue::from_serialize(&expected).expect("typed value");
17142 let workflow_arguments =
17143 encode_typed_envelope(&AvroValue::Array(vec![typed_value.clone()]), DEFAULT_CODEC)
17144 .expect("workflow arguments");
17145 let result = encode_typed_envelope(&typed_value, DEFAULT_CODEC).expect("activity result");
17146 let task = QueryTask {
17147 query_task_id: "typed-replay-query".to_string(),
17148 query_task_attempt: 1,
17149 lease_owner: Some("rust-worker".to_string()),
17150 workflow_id: Some("typed-replay".to_string()),
17151 run_id: Some("typed-replay-run".to_string()),
17152 workflow_type: "typed.contract.replayed".to_string(),
17153 query_name: "observed".to_string(),
17154 payload_codec: DEFAULT_CODEC.to_string(),
17155 workflow_arguments: Some(workflow_arguments),
17156 query_arguments: Some(
17157 encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
17158 .expect("query arguments"),
17159 ),
17160 history_events: vec![
17161 history_event(
17162 "ActivityScheduled",
17163 json!({
17164 "sequence": 1,
17165 "activity_type": "typed.contract.activity"
17166 }),
17167 ),
17168 history_event(
17169 "ActivityCompleted",
17170 json!({
17171 "sequence": 1,
17172 "activity_type": "typed.contract.activity",
17173 "payload_codec": DEFAULT_CODEC,
17174 "result": result
17175 }),
17176 ),
17177 ],
17178 history_export: None,
17179 run_status: Some("completed".to_string()),
17180 };
17181
17182 assert_eq!(
17183 worker
17184 .execute_query_task(task)
17185 .await
17186 .expect("typed replay query")
17187 .deserialize::<i64>()
17188 .expect("query result"),
17189 expected.signed
17190 );
17191 }
17192
17193 #[tokio::test]
17194 async fn typed_worker_surfaces_preserve_bytes_and_map_list_identity() {
17195 let client = Client::new("http://127.0.0.1:8080").expect("client");
17196 let mut worker = Worker::new(client, "rust-workers");
17197 worker.register_workflow_avro_value("typed.echo", |_ctx, input| async move { Ok(input) });
17198 worker
17199 .register_activity_avro_value("typed.activity", |_ctx, input| async move { Ok(input) });
17200 worker.register_query_avro_value("typed.echo", "inspect", |_ctx, input| async move {
17201 Ok(input)
17202 });
17203 worker.register_update_avro_value("typed.echo", "replace", |_ctx, input| async move {
17204 Ok(input)
17205 });
17206 worker.register_workflow_avro_value("typed.signal", |ctx, _input| async move {
17207 Ok(AvroValue::Array(
17208 ctx.wait_signal_avro_value("changed").await?,
17209 ))
17210 });
17211
17212 let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
17213 let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed envelope");
17214
17215 let mut workflow = workflow_task("typed.echo", Vec::new(), DEFAULT_CODEC);
17216 workflow.arguments = Some(envelope.clone());
17217 let commands = worker
17218 .execute_workflow_task(workflow)
17219 .expect("typed workflow task");
17220 assert_eq!(commands[0]["type"], "complete_workflow");
17221 assert_eq!(
17222 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17223 .expect("typed workflow result"),
17224 arguments
17225 );
17226
17227 let activity = ActivityTask {
17228 task_id: "activity-typed".to_string(),
17229 activity_attempt_id: Some("attempt-typed".to_string()),
17230 attempt_id: None,
17231 activity_type: "typed.activity".to_string(),
17232 payload_codec: DEFAULT_CODEC.to_string(),
17233 arguments: Some(envelope.clone()),
17234 attempt_number: 1,
17235 lease_owner: Some("rust-worker".to_string()),
17236 };
17237 assert_eq!(
17238 worker
17239 .execute_activity_task(activity)
17240 .await
17241 .expect("typed activity result"),
17242 arguments
17243 );
17244
17245 let query = QueryTask {
17246 query_task_id: "query-typed".to_string(),
17247 query_task_attempt: 1,
17248 lease_owner: Some("rust-worker".to_string()),
17249 workflow_id: Some("typed-1".to_string()),
17250 run_id: Some("run-typed".to_string()),
17251 workflow_type: "typed.echo".to_string(),
17252 query_name: "inspect".to_string(),
17253 payload_codec: DEFAULT_CODEC.to_string(),
17254 workflow_arguments: Some(
17255 encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
17256 .expect("workflow input"),
17257 ),
17258 query_arguments: Some(envelope.clone()),
17259 history_events: Vec::new(),
17260 history_export: None,
17261 run_status: Some("running".to_string()),
17262 };
17263 assert_eq!(
17264 worker
17265 .execute_query_task(query)
17266 .await
17267 .expect("typed query result"),
17268 arguments
17269 );
17270
17271 let mut update = workflow_task(
17272 "typed.echo",
17273 vec![history_event(
17274 "UpdateAccepted",
17275 json!({
17276 "update_id": "update-typed",
17277 "update_name": "replace",
17278 "arguments": envelope.clone(),
17279 }),
17280 )],
17281 DEFAULT_CODEC,
17282 );
17283 update.workflow_update_id = Some("update-typed".to_string());
17284 update.update_name = Some("replace".to_string());
17285 let commands = worker
17286 .execute_workflow_task(update)
17287 .expect("typed update task");
17288 assert_eq!(commands[0]["type"], "complete_update");
17289 assert_eq!(
17290 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17291 .expect("typed update result"),
17292 arguments
17293 );
17294
17295 let mut signal = workflow_task(
17296 "typed.signal",
17297 vec![history_event(
17298 "SignalReceived",
17299 json!({
17300 "signal_id": "signal-typed",
17301 "signal_name": "changed",
17302 "arguments": envelope.clone(),
17303 }),
17304 )],
17305 DEFAULT_CODEC,
17306 );
17307 signal.workflow_signal_id = Some("signal-typed".to_string());
17308 signal.signal_name = Some("changed".to_string());
17309 signal.signal_arguments = Some(envelope);
17310 let commands = worker
17311 .execute_workflow_task(signal)
17312 .expect("typed signal resume");
17313 assert_eq!(
17314 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17315 .expect("typed signal result"),
17316 arguments
17317 );
17318 }
17319
17320 #[tokio::test]
17321 async fn typed_helpers_never_parse_json_inspection_projection() {
17322 let collision_values = projection_collision_probe();
17323 let expected = AvroValue::Array(collision_values.clone());
17324 let envelope = encode_typed_envelope(&expected, DEFAULT_CODEC).expect("collision envelope");
17325
17326 let activity_context = workflow_context_with_codec(
17327 vec![history_event(
17328 "ActivityCompleted",
17329 json!({
17330 "sequence": 1,
17331 "activity_type": "collision.activity",
17332 "payload_codec": DEFAULT_CODEC,
17333 "result": envelope.clone(),
17334 }),
17335 )],
17336 DEFAULT_CODEC,
17337 );
17338 assert_eq!(
17339 activity_context
17340 .activity_avro_value("collision.activity", AvroValue::Array(Vec::new()))
17341 .await
17342 .expect("typed activity collision result"),
17343 expected
17344 );
17345
17346 let signal_context = workflow_context_with_codec(
17347 vec![
17348 history_event(
17349 "SignalWaitOpened",
17350 json!({"sequence": 1, "signal_name": "collision"}),
17351 ),
17352 history_event(
17353 "SignalApplied",
17354 json!({
17355 "sequence": 1,
17356 "signal_name": "collision",
17357 "payload_codec": DEFAULT_CODEC,
17358 "value": envelope.clone(),
17359 }),
17360 ),
17361 ],
17362 DEFAULT_CODEC,
17363 );
17364 assert_eq!(
17365 signal_context
17366 .wait_signal_avro_value("collision")
17367 .await
17368 .expect("typed signal collision arguments"),
17369 collision_values
17370 );
17371
17372 let child_context = workflow_context_with_codec(
17373 vec![
17374 history_event(
17375 "ChildWorkflowScheduled",
17376 json!({
17377 "sequence": 1,
17378 "child_workflow_instance_id": "collision-child",
17379 "child_workflow_run_id": "collision-run",
17380 "child_workflow_type": "collision.child",
17381 }),
17382 ),
17383 history_event(
17384 "ChildRunCompleted",
17385 json!({
17386 "sequence": 1,
17387 "child_workflow_instance_id": "collision-child",
17388 "child_workflow_run_id": "collision-run",
17389 "child_workflow_type": "collision.child",
17390 "payload_codec": DEFAULT_CODEC,
17391 "result": envelope,
17392 }),
17393 ),
17394 ],
17395 DEFAULT_CODEC,
17396 );
17397 let child = child_context
17398 .start_child_workflow_avro_value(
17399 "collision.child",
17400 ChildWorkflowOptions::new("collision-workers"),
17401 AvroValue::Array(Vec::new()),
17402 )
17403 .await
17404 .expect("typed child collision result");
17405 assert_eq!(child.result, expected);
17406 }
17407
17408 #[tokio::test]
17409 async fn replayed_typed_query_keeps_lossless_workflow_and_query_inputs() {
17410 let client = Client::new("http://127.0.0.1:8080").expect("client");
17411 let mut worker = Worker::new(client, "rust-workers");
17412 worker.register_replayed_workflow_avro_value(
17413 "typed.replayed",
17414 || (),
17415 |_ctx, input, _state| async move { Ok(input) },
17416 );
17417 worker.register_replayed_query_avro_value::<(), _, _>(
17418 "typed.replayed",
17419 "inspect",
17420 |ctx, _state, args| async move {
17421 let mut signals = ctx.signals_avro_value("collision");
17422 let signal = signals
17423 .pop()
17424 .map(AvroValue::Array)
17425 .unwrap_or_else(|| AvroValue::Array(Vec::new()));
17426 Ok(AvroValue::Array(vec![
17427 ctx.workflow_input_avro_value().clone(),
17428 signal,
17429 args,
17430 ]))
17431 },
17432 );
17433 let arguments = AvroValue::Array(projection_collision_probe());
17434 let signal_arguments =
17435 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed query signal arguments");
17436 let task = QueryTask {
17437 query_task_id: "query-typed-replay".to_string(),
17438 query_task_attempt: 1,
17439 lease_owner: Some("rust-worker".to_string()),
17440 workflow_id: Some("typed-replay".to_string()),
17441 run_id: Some("run-typed-replay".to_string()),
17442 workflow_type: "typed.replayed".to_string(),
17443 query_name: "inspect".to_string(),
17444 payload_codec: DEFAULT_CODEC.to_string(),
17445 workflow_arguments: Some(
17446 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("workflow arguments"),
17447 ),
17448 query_arguments: Some(
17449 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("query arguments"),
17450 ),
17451 history_events: vec![history_event(
17452 "SignalReceived",
17453 json!({
17454 "signal_id": "collision-signal",
17455 "signal_name": "collision",
17456 "workflow_sequence": 1,
17457 "payload_codec": DEFAULT_CODEC,
17458 "arguments": signal_arguments,
17459 }),
17460 )],
17461 history_export: None,
17462 run_status: Some("completed".to_string()),
17463 };
17464
17465 assert_eq!(
17466 worker
17467 .execute_query_task(task)
17468 .await
17469 .expect("typed replay query"),
17470 AvroValue::Array(vec![arguments.clone(), arguments.clone(), arguments])
17471 );
17472 }
17473
17474 #[test]
17475 fn public_avro_adapter_rejects_non_string_map_keys_before_json_conversion() {
17476 let value = BTreeMap::from([(1_i32, "integer key")]);
17477 let error = PayloadEnvelope::avro(&value)
17478 .expect_err("integer map keys must fail")
17479 .to_string();
17480
17481 assert!(error.contains("invalid_map_key"));
17482 }
17483
17484 #[test]
17485 fn json_tagged_payload_fails_closed_with_actionable_diagnostic() {
17486 let envelope = PayloadEnvelope {
17487 codec: "json".to_string(),
17488 blob: r#"{"greeting":"hello"}"#.to_string(),
17489 };
17490
17491 let error = decode_payload::<Value>(&envelope).expect_err("JSON payload must fail");
17492 let diagnostic = error.to_string();
17493 assert!(diagnostic.contains("unsupported_payload_codec"));
17494 assert!(diagnostic.contains("codec=\"avro\""));
17495 assert!(diagnostic.contains("HTTP document transport"));
17496 }
17497
17498 #[test]
17499 fn untagged_json_payload_value_fails_closed() {
17500 let error = decode_wire_value(&json!({"stale": true}), DEFAULT_CODEC)
17501 .expect_err("untagged JSON payload values must fail");
17502 let diagnostic = error.to_string();
17503 assert!(diagnostic.contains("unsupported_payload_codec"));
17504 assert!(diagnostic.contains("untagged durable payload"));
17505 assert!(diagnostic.contains("HTTP document transport"));
17506 }
17507
17508 #[test]
17509 fn prerelease_avro_payload_without_single_object_frame_is_rejected() {
17510 let envelope = PayloadEnvelope {
17511 codec: DEFAULT_CODEC.to_string(),
17512 blob: BASE64.encode([0x01]),
17513 };
17514
17515 let error = decode_payload::<Value>(&envelope).expect_err("prerelease payload must fail");
17516 assert!(error.to_string().contains("invalid_payload_framing"));
17517 }
17518
17519 #[tokio::test]
17520 async fn workflow_completion_rejects_invalid_payload_slots_without_transport() {
17521 let server = MockWorkerServer::start();
17522 let client = Client::builder(server.base_url())
17523 .timeout(Duration::from_secs(2))
17524 .build()
17525 .expect("client");
17526 let invalid_commands = [
17527 json!({
17528 "type": "complete_workflow",
17529 "result": {"codec": "json", "blob": null}
17530 }),
17531 json!({
17532 "type": "schedule_activity",
17533 "arguments": {"codec": "yaml", "blob": "ignored"}
17534 }),
17535 json!({
17536 "type": "start_child_workflow",
17537 "arguments": {"codec": DEFAULT_CODEC, "blob": null}
17538 }),
17539 json!({"type": "continue_as_new", "arguments": []}),
17540 json!({"type": "complete_update"}),
17541 json!({"type": "record_side_effect", "result": null}),
17542 json!({
17543 "type": "start_service_operation",
17544 "payload_codec": DEFAULT_CODEC,
17545 "request_payload": "raw-avro-bytes"
17546 }),
17547 ];
17548
17549 for command in invalid_commands {
17550 let error = client
17551 .complete_workflow_task("invalid-codec", "rust-worker", 1, vec![command])
17552 .await
17553 .expect_err("invalid durable payload must fail locally");
17554 let diagnostic = error.to_string();
17555 assert!(
17556 diagnostic.contains("unsupported_payload_codec")
17557 || diagnostic.contains("invalid_payload_envelope")
17558 || diagnostic.contains("untagged durable payload"),
17559 "unexpected validation diagnostic: {diagnostic}"
17560 );
17561 }
17562
17563 assert_eq!(
17564 server.request_count("/api/worker/workflow-tasks/invalid-codec/complete"),
17565 0,
17566 "invalid command payloads must not reach HTTP transport"
17567 );
17568 }
17569
17570 #[test]
17571 fn workflow_completion_validates_only_protocol_owned_payload_slots() {
17572 let envelope = fixture_envelope(json!({"codec": "customer-value"}));
17573 let commands = [
17574 json!({"type": "complete_workflow", "result": envelope.clone()}),
17575 json!({"type": "schedule_activity", "arguments": envelope.clone()}),
17576 json!({"type": "start_child_workflow", "arguments": envelope.clone()}),
17577 json!({"type": "continue_as_new", "arguments": envelope.clone()}),
17578 json!({"type": "complete_update", "result": envelope.clone()}),
17579 json!({"type": "record_side_effect", "result": envelope.clone()}),
17580 json!({
17581 "type": "start_service_operation",
17582 "payload_codec": DEFAULT_CODEC,
17583 "request_payload": envelope.clone()
17584 }),
17585 json!({
17586 "type": "complete_workflow",
17587 "result": envelope,
17588 "metadata": {
17589 "codec": "json",
17590 "payload_codec": "customer-codec",
17591 "result": {"codec": "yaml", "blob": null}
17592 }
17593 }),
17594 ];
17595
17596 validate_workflow_task_commands(&commands)
17597 .expect("customer metadata must not become a protocol codec declaration");
17598 }
17599
17600 #[test]
17601 fn valid_avro_tasks_normalize_absent_and_null_arguments_to_empty_lists() {
17602 assert_eq!(
17603 decode_task_avro_arguments(None, DEFAULT_CODEC).expect("absent arguments"),
17604 AvroValue::Array(Vec::new())
17605 );
17606 assert_eq!(
17607 decode_task_avro_arguments(Some(&Value::Null), DEFAULT_CODEC).expect("null arguments"),
17608 AvroValue::Array(Vec::new())
17609 );
17610
17611 let mut signal = workflow_task("missing", Vec::new(), DEFAULT_CODEC);
17612 signal.signal_name = Some("empty-signal".to_string());
17613 signal.signal_arguments = None;
17614 let decoded = decode_resume_signal(&signal)
17615 .expect("valid Avro signal")
17616 .expect("named signal resumes the workflow");
17617 assert!(decoded.arguments.is_empty());
17618 }
17619
17620 #[tokio::test]
17621 async fn malformed_task_level_codecs_become_pre_handler_failures() {
17622 let client = Client::new("http://127.0.0.1:8080").expect("client");
17623 let mut worker = Worker::new(client, "rust-workers");
17624 let handler_calls = Arc::new(AtomicUsize::new(0));
17625
17626 let calls = Arc::clone(&handler_calls);
17627 worker.register_workflow("codec.workflow", move |_ctx, _args| {
17628 calls.fetch_add(1, Ordering::SeqCst);
17629 async move { Ok(Value::Null) }
17630 });
17631 let calls = Arc::clone(&handler_calls);
17632 worker.register_activity("codec.activity", move |_ctx, _args| {
17633 calls.fetch_add(1, Ordering::SeqCst);
17634 async move { Ok(Value::Null) }
17635 });
17636 let calls = Arc::clone(&handler_calls);
17637 worker.register_query("codec.workflow", "known", move |_ctx, _args| {
17638 calls.fetch_add(1, Ordering::SeqCst);
17639 async move { Ok(Value::Null) }
17640 });
17641
17642 let mut failures = Vec::new();
17643 for codec_case in [
17644 InvalidTaskPayloadCodec::Missing,
17645 InvalidTaskPayloadCodec::Null,
17646 InvalidTaskPayloadCodec::NonString,
17647 ] {
17648 let mut workflow = json!({
17649 "task_id": format!("workflow-{}", codec_case.label()),
17650 "workflow_type": "codec.workflow"
17651 });
17652 codec_case.apply(&mut workflow);
17653 match serde_json::from_value::<WorkflowTask>(workflow) {
17654 Ok(task) => match worker.execute_workflow_task(task) {
17655 Err(error) if error.to_string().contains("unsupported_payload_codec") => {}
17656 outcome => failures.push(format!(
17657 "workflow {} codec returned {outcome:?}",
17658 codec_case.label()
17659 )),
17660 },
17661 Err(error) => failures.push(format!(
17662 "workflow {} codec failed transport deserialization: {error}",
17663 codec_case.label()
17664 )),
17665 }
17666
17667 let mut activity = json!({
17668 "task_id": format!("activity-{}", codec_case.label()),
17669 "activity_attempt_id": format!("attempt-{}", codec_case.label()),
17670 "activity_type": "codec.activity",
17671 "attempt_number": 1
17672 });
17673 codec_case.apply(&mut activity);
17674 match serde_json::from_value::<ActivityTask>(activity) {
17675 Ok(task) => match worker.execute_activity_task(task).await {
17676 Err(error) if error.to_string().contains("unsupported_payload_codec") => {}
17677 outcome => failures.push(format!(
17678 "activity {} codec returned {outcome:?}",
17679 codec_case.label()
17680 )),
17681 },
17682 Err(error) => failures.push(format!(
17683 "activity {} codec failed transport deserialization: {error}",
17684 codec_case.label()
17685 )),
17686 }
17687
17688 let mut query = json!({
17689 "query_task_id": format!("query-{}", codec_case.label()),
17690 "workflow_type": "codec.workflow",
17691 "query_name": "known"
17692 });
17693 codec_case.apply(&mut query);
17694 match serde_json::from_value::<QueryTask>(query) {
17695 Ok(task) => match worker.execute_query_task(task).await {
17696 Err(failure) if failure.message.contains("unsupported_payload_codec") => {}
17697 outcome => failures.push(format!(
17698 "query {} codec returned {outcome:?}",
17699 codec_case.label()
17700 )),
17701 },
17702 Err(error) => failures.push(format!(
17703 "query {} codec failed transport deserialization: {error}",
17704 codec_case.label()
17705 )),
17706 }
17707 }
17708
17709 assert!(failures.is_empty(), "{}", failures.join("\n"));
17710 assert_eq!(
17711 handler_calls.load(Ordering::SeqCst),
17712 0,
17713 "invalid task codecs must not invoke a handler"
17714 );
17715 }
17716
17717 #[tokio::test]
17718 async fn polled_malformed_task_codecs_are_settled_without_handler_execution() {
17719 for codec_case in [
17720 InvalidTaskPayloadCodec::Missing,
17721 InvalidTaskPayloadCodec::Null,
17722 InvalidTaskPayloadCodec::NonString,
17723 ] {
17724 let server = MockWorkerServer::invalid_task_payload_codec(codec_case);
17725 let client = Client::builder(server.base_url())
17726 .timeout(Duration::from_secs(2))
17727 .build()
17728 .expect("client");
17729 let mut worker = Worker::new(client, "rust-workers")
17730 .worker_id("codec-worker")
17731 .poll_timeout(Duration::from_millis(10));
17732 let handler_calls = Arc::new(AtomicUsize::new(0));
17733
17734 let calls = Arc::clone(&handler_calls);
17735 worker.register_workflow("codec.workflow", move |_ctx, _args| {
17736 calls.fetch_add(1, Ordering::SeqCst);
17737 async move { Ok(Value::Null) }
17738 });
17739 let calls = Arc::clone(&handler_calls);
17740 worker.register_activity("codec.activity", move |_ctx, _args| {
17741 calls.fetch_add(1, Ordering::SeqCst);
17742 async move { Ok(Value::Null) }
17743 });
17744 let calls = Arc::clone(&handler_calls);
17745 worker.register_query("codec.workflow", "known", move |_ctx, _args| {
17746 calls.fetch_add(1, Ordering::SeqCst);
17747 async move { Ok(Value::Null) }
17748 });
17749
17750 assert_eq!(
17751 worker.run_once().await.expect("invalid tasks are settled"),
17752 3,
17753 "all {} codec tasks must be handled",
17754 codec_case.label()
17755 );
17756 assert_eq!(
17757 handler_calls.load(Ordering::SeqCst),
17758 0,
17759 "{} task codecs must fail before every handler",
17760 codec_case.label()
17761 );
17762
17763 for path in [
17764 "/api/worker/workflow-tasks/codec-workflow/fail",
17765 "/api/worker/activity-tasks/codec-activity/fail",
17766 "/api/worker/query-tasks/codec-query/fail",
17767 ] {
17768 let body = server.request_body(path);
17769 assert!(
17770 body["failure"]["message"]
17771 .as_str()
17772 .is_some_and(|message| message.contains("unsupported_payload_codec")),
17773 "{path} must receive the stable codec diagnostic for the {} case: {body}",
17774 codec_case.label()
17775 );
17776 }
17777 assert_eq!(
17778 server.request_body("/api/worker/query-tasks/codec-query/fail")["failure"]
17779 ["reason"],
17780 "query_payload_decode_failed"
17781 );
17782 for path in [
17783 "/api/worker/workflow-tasks/codec-workflow/complete",
17784 "/api/worker/activity-tasks/codec-activity/complete",
17785 "/api/worker/query-tasks/codec-query/complete",
17786 ] {
17787 assert_eq!(
17788 server.request_count(path),
17789 0,
17790 "invalid {} codec task reached {path}",
17791 codec_case.label()
17792 );
17793 }
17794 }
17795 }
17796
17797 #[tokio::test]
17798 async fn invalid_inbound_codecs_precede_handlers_and_unrelated_outcomes() {
17799 let client = Client::new("http://127.0.0.1:8080").expect("client");
17800 let mut worker = Worker::new(client, "rust-workers");
17801 let handler_calls = Arc::new(AtomicUsize::new(0));
17802
17803 let calls = Arc::clone(&handler_calls);
17804 worker.register_workflow("codec.workflow", move |_ctx, _args| {
17805 calls.fetch_add(1, Ordering::SeqCst);
17806 async move { Ok(Value::Null) }
17807 });
17808 let calls = Arc::clone(&handler_calls);
17809 worker.register_activity("codec.activity", move |_ctx, _args| {
17810 calls.fetch_add(1, Ordering::SeqCst);
17811 async move { Ok(Value::Null) }
17812 });
17813 let calls = Arc::clone(&handler_calls);
17814 worker.register_update("codec.workflow", "known", move |_ctx, _args| {
17815 calls.fetch_add(1, Ordering::SeqCst);
17816 async move { Ok(Value::Null) }
17817 });
17818 let calls = Arc::clone(&handler_calls);
17819 worker.register_query("codec.workflow", "known", move |_ctx, _args| {
17820 calls.fetch_add(1, Ordering::SeqCst);
17821 async move { Ok(Value::Null) }
17822 });
17823
17824 let mut workflow = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
17825 workflow.payload_codec = "json".to_string();
17826 workflow.arguments = None;
17827 let error = worker
17828 .execute_workflow_task(workflow)
17829 .expect_err("task codec must be checked before workflow invocation");
17830 assert!(error.to_string().contains("unsupported_payload_codec"));
17831
17832 let activity = ActivityTask {
17833 task_id: "activity-invalid-codec".to_string(),
17834 activity_attempt_id: None,
17835 attempt_id: None,
17836 activity_type: "codec.activity".to_string(),
17837 payload_codec: "unknown".to_string(),
17838 arguments: None,
17839 attempt_number: 1,
17840 lease_owner: None,
17841 };
17842 let error = worker
17843 .execute_activity_task(activity)
17844 .await
17845 .expect_err("task codec must be checked before activity invocation");
17846 assert!(error.to_string().contains("unsupported_payload_codec"));
17847
17848 let mut update = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
17849 update.workflow_update_id = Some("update-invalid-codec".to_string());
17850 update.update_name = Some("known".to_string());
17851 update.history_events.push(history_event(
17852 "UpdateAccepted",
17853 json!({
17854 "update_id": "update-invalid-codec",
17855 "update_name": "known",
17856 "arguments": {"codec": "json", "blob": null}
17857 }),
17858 ));
17859 let error = worker
17860 .execute_workflow_task(update)
17861 .expect_err("nested update codec must be checked before handler lookup");
17862 assert!(error.to_string().contains("unsupported_payload_codec"));
17863
17864 let query: QueryTask = serde_json::from_value(json!({
17865 "query_task_id": "query-invalid-codec",
17866 "workflow_type": "codec.workflow",
17867 "query_name": "known",
17868 "payload_codec": DEFAULT_CODEC,
17869 "workflow_arguments": null,
17870 "query_arguments": null,
17871 "history_export": {
17872 "payloads": {"codec": DEFAULT_CODEC},
17873 "signals": [{
17874 "name": "empty",
17875 "payload_codec": "json",
17876 "arguments": null
17877 }]
17878 }
17879 }))
17880 .expect("query task");
17881 let failure = worker
17882 .execute_query_task(query)
17883 .await
17884 .expect_err("exported signal codec must be checked before query invocation");
17885 assert_eq!(failure.reason, "query_payload_decode_failed");
17886 assert!(failure.message.contains("unsupported_payload_codec"));
17887
17888 let exported_history: QueryTask = serde_json::from_value(json!({
17889 "query_task_id": "query-invalid-history-codec",
17890 "workflow_type": "codec.workflow",
17891 "query_name": "known",
17892 "payload_codec": DEFAULT_CODEC,
17893 "history_export": {
17894 "payloads": {"codec": DEFAULT_CODEC},
17895 "history_events": [{
17896 "type": "ActivityCompleted",
17897 "payload": {"payload_codec": "unknown", "result": null}
17898 }]
17899 }
17900 }))
17901 .expect("query task");
17902 let failure = worker
17903 .execute_query_task(exported_history)
17904 .await
17905 .expect_err("exported history codec must be checked before query invocation");
17906 assert_eq!(failure.reason, "query_payload_decode_failed");
17907 assert!(failure.message.contains("unsupported_payload_codec"));
17908 assert_eq!(handler_calls.load(Ordering::SeqCst), 0);
17909
17910 let mut unknown_workflow = workflow_task("missing", Vec::new(), DEFAULT_CODEC);
17911 unknown_workflow.arguments = None;
17912 unknown_workflow.history_events.push(history_event(
17913 "SignalReceived",
17914 json!({
17915 "signal_name": "empty",
17916 "payload_codec": "json",
17917 "arguments": null
17918 }),
17919 ));
17920 let error = worker
17921 .execute_workflow_task(unknown_workflow)
17922 .expect_err("history codec must precede unknown workflow outcome");
17923 assert!(error.to_string().contains("unsupported_payload_codec"));
17924
17925 let unknown_activity = ActivityTask {
17926 task_id: "activity-unknown".to_string(),
17927 activity_attempt_id: None,
17928 attempt_id: None,
17929 activity_type: "missing".to_string(),
17930 payload_codec: "json".to_string(),
17931 arguments: None,
17932 attempt_number: 1,
17933 lease_owner: None,
17934 };
17935 let error = worker
17936 .execute_activity_task(unknown_activity)
17937 .await
17938 .expect_err("codec must precede unknown activity outcome");
17939 assert!(error.to_string().contains("unsupported_payload_codec"));
17940
17941 let mut unknown_update = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
17942 unknown_update.payload_codec = "json".to_string();
17943 unknown_update.arguments = None;
17944 unknown_update.workflow_update_id = Some("update-unknown".to_string());
17945 unknown_update.update_name = Some("missing".to_string());
17946 let error = worker
17947 .execute_workflow_task(unknown_update)
17948 .expect_err("codec must precede fail_update shortcut");
17949 assert!(error.to_string().contains("unsupported_payload_codec"));
17950
17951 let unknown_query: QueryTask = serde_json::from_value(json!({
17952 "query_task_id": "query-unknown",
17953 "workflow_type": "missing",
17954 "query_name": "missing",
17955 "payload_codec": "json",
17956 "workflow_arguments": null,
17957 "query_arguments": null
17958 }))
17959 .expect("query task");
17960 let failure = worker
17961 .execute_query_task(unknown_query)
17962 .await
17963 .expect_err("codec must precede unknown query outcome");
17964 assert_eq!(failure.reason, "query_payload_decode_failed");
17965 assert!(failure.message.contains("unsupported_payload_codec"));
17966 }
17967
17968 #[tokio::test]
17969 async fn invalid_signal_history_payload_aliases_precede_shortcuts() {
17970 let client = Client::new("http://127.0.0.1:8080").expect("client");
17971 let worker = Worker::new(client, "rust-workers");
17972
17973 for event_type in ["SignalReceived", "SignalApplied"] {
17974 for (payload_field, codec) in [
17975 ("value", "json"),
17976 ("input", "unknown"),
17977 ("arguments", "json"),
17978 ] {
17979 let payload = json!({
17980 "signal_name": "empty",
17981 payload_field: {"codec": codec, "blob": null}
17982 });
17983 let workflow = workflow_task(
17984 "missing",
17985 vec![history_event(event_type, payload.clone())],
17986 DEFAULT_CODEC,
17987 );
17988 let error = worker
17989 .execute_workflow_task(workflow)
17990 .expect_err("signal payload codec must precede unknown workflow outcome");
17991 assert!(
17992 error.to_string().contains("unsupported_payload_codec"),
17993 "{event_type}.{payload_field} returned an unrelated workflow error: {error}"
17994 );
17995
17996 let query: QueryTask = serde_json::from_value(json!({
17997 "query_task_id": format!("query-{event_type}-{payload_field}"),
17998 "workflow_type": "missing",
17999 "query_name": "missing",
18000 "payload_codec": DEFAULT_CODEC,
18001 "workflow_arguments": null,
18002 "query_arguments": null,
18003 "history_events": [{
18004 "event_type": event_type,
18005 "payload": payload
18006 }]
18007 }))
18008 .expect("query task");
18009 let failure = worker
18010 .execute_query_task(query)
18011 .await
18012 .expect_err("signal payload codec must precede unknown query outcome");
18013 assert_eq!(
18014 failure.reason, "query_payload_decode_failed",
18015 "{event_type}.{payload_field} returned an unrelated query outcome"
18016 );
18017 assert!(
18018 failure.message.contains("unsupported_payload_codec"),
18019 "{event_type}.{payload_field} returned an unrelated query error: {}",
18020 failure.message
18021 );
18022 }
18023 }
18024 }
18025
18026 #[test]
18027 fn workflow_context_schedules_activity_until_completion_is_in_history() {
18028 let ctx = WorkflowContext {
18029 state: Arc::new(Mutex::new(
18030 WorkflowState::new_with_identity(
18031 Vec::new(),
18032 Some("wf-parent".to_string()),
18033 Some("run-parent".to_string()),
18034 "rust-workers".to_string(),
18035 DEFAULT_CODEC.to_string(),
18036 None,
18037 )
18038 .expect("workflow state"),
18039 )),
18040 };
18041
18042 let mut call = Box::pin(ctx.activity("hello.activity", json!(["Ada"])));
18043 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18044 assert!(matches!(
18045 call.as_mut().poll(&mut task_context),
18046 Poll::Pending
18047 ));
18048
18049 let commands = ctx.take_commands().expect("commands");
18050 assert_eq!(commands[0]["type"], "schedule_activity");
18051 assert_eq!(commands[0]["activity_type"], "hello.activity");
18052 }
18053
18054 #[test]
18055 fn activity_options_encode_retry_policy_queue_and_every_timeout() {
18056 let ctx = workflow_context(Vec::new());
18057 let options = ActivityOptions::new()
18058 .task_queue("payments")
18059 .retry_policy(
18060 ActivityRetryPolicy::new(4)
18061 .exponential_backoff(Duration::from_secs(1), 3, Some(Duration::from_secs(10)))
18062 .non_retryable_error_type("ValidationError"),
18063 )
18064 .start_to_close_timeout(Duration::from_secs(120))
18065 .schedule_to_start_timeout(Duration::from_secs(10))
18066 .schedule_to_close_timeout(Duration::from_secs(300))
18067 .heartbeat_timeout(Duration::from_secs(15));
18068 let mut call = Box::pin(ctx.activity_with_options(
18069 "charge-card",
18070 options,
18071 json!([{"order_id": "o-1"}]),
18072 ));
18073 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18074
18075 assert!(matches!(
18076 call.as_mut().poll(&mut task_context),
18077 Poll::Pending
18078 ));
18079 assert!(matches!(
18080 call.as_mut().poll(&mut task_context),
18081 Poll::Pending
18082 ));
18083
18084 let commands = ctx.take_commands().expect("activity command");
18085 assert_eq!(commands.len(), 1, "one future emits one logical schedule");
18086 assert_eq!(commands[0]["queue"], "payments");
18087 assert_eq!(
18088 commands[0]["retry_policy"],
18089 json!({
18090 "max_attempts": 4,
18091 "backoff_seconds": [1, 3, 9],
18092 "non_retryable_error_types": ["ValidationError"],
18093 })
18094 );
18095 assert_eq!(commands[0]["start_to_close_timeout"], 120);
18096 assert_eq!(commands[0]["schedule_to_start_timeout"], 10);
18097 assert_eq!(commands[0]["schedule_to_close_timeout"], 300);
18098 assert_eq!(commands[0]["heartbeat_timeout"], 15);
18099 }
18100
18101 #[test]
18102 fn activity_options_encode_explicit_and_rounded_backoff_intervals() {
18103 let ctx = workflow_context(Vec::new());
18104 let options = ActivityOptions::new().retry_policy(
18105 ActivityRetryPolicy::new(3)
18106 .backoff_intervals([Duration::from_millis(1), Duration::from_millis(1_001)]),
18107 );
18108 let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
18109 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18110
18111 assert!(matches!(
18112 call.as_mut().poll(&mut task_context),
18113 Poll::Pending
18114 ));
18115 assert_eq!(
18116 ctx.take_commands().expect("command")[0]["retry_policy"]["backoff_seconds"],
18117 json!([1, 2])
18118 );
18119 }
18120
18121 #[test]
18122 fn invalid_activity_options_return_typed_errors_before_emitting_commands() {
18123 let cases = [
18124 (
18125 ActivityOptions::new().task_queue(" "),
18126 ActivityOptionsErrorKind::EmptyTaskQueue,
18127 ),
18128 (
18129 ActivityOptions::new().retry_policy(ActivityRetryPolicy::default()),
18130 ActivityOptionsErrorKind::EmptyRetryPolicy,
18131 ),
18132 (
18133 ActivityOptions::new().retry_policy(ActivityRetryPolicy::new(0)),
18134 ActivityOptionsErrorKind::InvalidMaxAttempts,
18135 ),
18136 (
18137 ActivityOptions::new().retry_policy(ActivityRetryPolicy {
18138 max_attempts: None,
18139 backoff: Some(ActivityBackoff::Explicit(vec![Duration::from_secs(1)])),
18140 non_retryable_error_types: Vec::new(),
18141 }),
18142 ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
18143 ),
18144 (
18145 ActivityOptions::new().retry_policy(
18146 ActivityRetryPolicy::new(2)
18147 .backoff_intervals([Duration::from_secs(1), Duration::from_secs(2)]),
18148 ),
18149 ActivityOptionsErrorKind::TooManyBackoffIntervals,
18150 ),
18151 (
18152 ActivityOptions::new().retry_policy(
18153 ActivityRetryPolicy::new(2).exponential_backoff(
18154 Duration::from_secs(1),
18155 0,
18156 None,
18157 ),
18158 ),
18159 ActivityOptionsErrorKind::InvalidBackoffCoefficient,
18160 ),
18161 (
18162 ActivityOptions::new()
18163 .retry_policy(ActivityRetryPolicy::new(2).non_retryable_error_type(" ")),
18164 ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
18165 ),
18166 (
18167 ActivityOptions::new().retry_policy(
18168 ActivityRetryPolicy::new(10_002).exponential_backoff(
18169 Duration::from_secs(1),
18170 1,
18171 None,
18172 ),
18173 ),
18174 ActivityOptionsErrorKind::BackoffGenerationTooLarge,
18175 ),
18176 (
18177 ActivityOptions::new().retry_policy(
18178 ActivityRetryPolicy::new(2)
18179 .backoff_intervals([Duration::from_secs(i64::MAX as u64 + 1)]),
18180 ),
18181 ActivityOptionsErrorKind::BackoffOverflow,
18182 ),
18183 ];
18184
18185 for (options, expected_kind) in cases {
18186 let ctx = workflow_context(Vec::new());
18187 let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
18188 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18189 let Poll::Ready(Err(Error::InvalidActivityOptions(error))) =
18190 call.as_mut().poll(&mut task_context)
18191 else {
18192 panic!("expected typed activity validation error");
18193 };
18194 assert_eq!(error.kind, expected_kind);
18195 assert!(ctx.take_commands().expect("commands").is_empty());
18196 }
18197 }
18198
18199 #[test]
18200 fn activity_options_validate_positive_and_ordered_timeouts() {
18201 let zero_timeout_cases = [
18202 ActivityOptions::new().start_to_close_timeout(Duration::ZERO),
18203 ActivityOptions::new().schedule_to_start_timeout(Duration::ZERO),
18204 ActivityOptions::new().schedule_to_close_timeout(Duration::ZERO),
18205 ActivityOptions::new().heartbeat_timeout(Duration::ZERO),
18206 ];
18207 for options in zero_timeout_cases {
18208 assert_eq!(
18209 options.validate().expect_err("zero timeout").kind,
18210 ActivityOptionsErrorKind::TimeoutNotPositive
18211 );
18212 }
18213
18214 let ordering_cases = [
18215 ActivityOptions::new()
18216 .heartbeat_timeout(Duration::from_secs(11))
18217 .start_to_close_timeout(Duration::from_secs(10)),
18218 ActivityOptions::new()
18219 .start_to_close_timeout(Duration::from_secs(31))
18220 .schedule_to_close_timeout(Duration::from_secs(30)),
18221 ActivityOptions::new()
18222 .schedule_to_start_timeout(Duration::from_secs(31))
18223 .schedule_to_close_timeout(Duration::from_secs(30)),
18224 ];
18225 for options in ordering_cases {
18226 assert_eq!(
18227 options.validate().expect_err("timeout order").kind,
18228 ActivityOptionsErrorKind::TimeoutOrder
18229 );
18230 }
18231
18232 assert_eq!(
18233 ActivityOptions::new()
18234 .start_to_close_timeout(Duration::from_secs(i64::MAX as u64 + 1))
18235 .validate()
18236 .expect_err("protocol integer overflow")
18237 .kind,
18238 ActivityOptionsErrorKind::TimeoutOverflow
18239 );
18240 }
18241
18242 #[test]
18243 fn replayed_activity_retry_history_completes_without_duplicate_schedule() {
18244 let ctx = workflow_context(completed_retry_activity_history());
18245 let mut call =
18246 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18247 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18248
18249 assert!(matches!(
18250 call.as_mut().poll(&mut task_context),
18251 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
18252 ));
18253 assert!(ctx.take_commands().expect("commands").is_empty());
18254 ctx.ensure_history_consumed().expect("history consumed");
18255 }
18256
18257 #[test]
18258 fn duplicate_non_retryable_types_use_one_command_and_replay_representation() {
18259 let mut options = retry_activity_options();
18260 options
18261 .retry_policy
18262 .as_mut()
18263 .expect("retry policy")
18264 .non_retryable_error_types
18265 .extend([" PermanentError ".to_string(), "PermanentError".to_string()]);
18266
18267 let new_ctx = workflow_context(Vec::new());
18268 let mut new_call =
18269 Box::pin(new_ctx.activity_with_options("flaky", options.clone(), json!([])));
18270 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18271 assert!(matches!(
18272 new_call.as_mut().poll(&mut task_context),
18273 Poll::Pending
18274 ));
18275 let commands = new_ctx.take_commands().expect("commands");
18276 assert_eq!(commands.len(), 1);
18277 assert_eq!(
18278 commands[0]["retry_policy"]["non_retryable_error_types"],
18279 json!(["PermanentError"])
18280 );
18281
18282 let replay_ctx = workflow_context(completed_retry_activity_history());
18283 let mut replay_call =
18284 Box::pin(replay_ctx.activity_with_options("flaky", options, json!([])));
18285 assert!(matches!(
18286 replay_call.as_mut().poll(&mut task_context),
18287 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
18288 ));
18289 assert!(replay_ctx.take_commands().expect("commands").is_empty());
18290 replay_ctx
18291 .ensure_history_consumed()
18292 .expect("history consumed");
18293 }
18294
18295 #[test]
18296 fn replayed_intermediate_retry_remains_pending_across_restarts() {
18297 let history = completed_retry_activity_history()
18298 .into_iter()
18299 .take(3)
18300 .collect::<Vec<_>>();
18301
18302 for _restart in 0..2 {
18303 let ctx = workflow_context(history.clone());
18304 let mut call =
18305 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18306 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18307 assert!(matches!(
18308 call.as_mut().poll(&mut task_context),
18309 Poll::Pending
18310 ));
18311 assert!(ctx.take_commands().expect("commands").is_empty());
18312 }
18313 }
18314
18315 #[test]
18316 fn replayed_activity_rejects_changed_queue_retry_and_every_timeout_field() {
18317 let mut changed_queue = retry_activity_options();
18318 changed_queue.task_queue = Some("different-queue".to_string());
18319
18320 let mut changed_max_attempts = retry_activity_options();
18321 let retry_policy = changed_max_attempts
18322 .retry_policy
18323 .as_mut()
18324 .expect("retry policy");
18325 retry_policy.max_attempts = Some(4);
18326
18327 let mut changed_backoff = retry_activity_options();
18328 let retry_policy = changed_backoff.retry_policy.as_mut().expect("retry policy");
18329 retry_policy.backoff = Some(ActivityBackoff::Explicit(vec![
18330 Duration::from_secs(3),
18331 Duration::from_secs(4),
18332 ]));
18333
18334 let mut changed_non_retryable_types = retry_activity_options();
18335 let retry_policy = changed_non_retryable_types
18336 .retry_policy
18337 .as_mut()
18338 .expect("retry policy");
18339 retry_policy.non_retryable_error_types = vec!["AnotherPermanentError".to_string()];
18340
18341 let mut changed_start_to_close = retry_activity_options();
18342 changed_start_to_close.start_to_close_timeout = Some(Duration::from_secs(31));
18343 let mut changed_schedule_to_start = retry_activity_options();
18344 changed_schedule_to_start.schedule_to_start_timeout = Some(Duration::from_secs(6));
18345 let mut changed_schedule_to_close = retry_activity_options();
18346 changed_schedule_to_close.schedule_to_close_timeout = Some(Duration::from_secs(91));
18347 let mut changed_heartbeat = retry_activity_options();
18348 changed_heartbeat.heartbeat_timeout = Some(Duration::from_secs(11));
18349
18350 let cases = [
18351 (changed_queue, "activity_task_queue_mismatch"),
18352 (changed_max_attempts, "activity_retry_policy_mismatch"),
18353 (changed_backoff, "activity_retry_policy_mismatch"),
18354 (
18355 changed_non_retryable_types,
18356 "activity_retry_policy_mismatch",
18357 ),
18358 (changed_start_to_close, "activity_retry_policy_mismatch"),
18359 (changed_schedule_to_start, "activity_retry_policy_mismatch"),
18360 (changed_schedule_to_close, "activity_retry_policy_mismatch"),
18361 (changed_heartbeat, "activity_retry_policy_mismatch"),
18362 ];
18363
18364 for (options, expected_reason) in cases {
18365 let ctx = workflow_context(completed_retry_activity_history());
18366 let mut call = Box::pin(ctx.activity_with_options("flaky", options, json!([])));
18367 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18368 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
18369 call.as_mut().poll(&mut task_context)
18370 else {
18371 panic!("changed activity options must fail replay");
18372 };
18373 assert_eq!(failure.reason, expected_reason);
18374 assert_eq!(failure.sequence, Some(1));
18375 assert!(ctx.take_commands().expect("commands").is_empty());
18376 }
18377 }
18378
18379 #[test]
18380 fn replayed_activity_rejects_changed_execution_mode_and_snapshot_version() {
18381 let cases = [
18382 (
18383 "execution_mode",
18384 json!("local"),
18385 "activity_execution_mode_mismatch",
18386 ),
18387 (
18388 "snapshot_version",
18389 json!(2),
18390 "activity_retry_policy_mismatch",
18391 ),
18392 ];
18393
18394 for (field, value, expected_reason) in cases {
18395 let mut history = completed_retry_activity_history();
18396 let activity = history[0].payload["activity"]
18397 .as_object_mut()
18398 .expect("activity snapshot");
18399 if field == "execution_mode" {
18400 activity.insert(field.to_string(), value);
18401 } else {
18402 activity["retry_policy"]
18403 .as_object_mut()
18404 .expect("retry snapshot")
18405 .insert(field.to_string(), value);
18406 }
18407
18408 let ctx = workflow_context(history);
18409 let mut call =
18410 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18411 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18412 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
18413 call.as_mut().poll(&mut task_context)
18414 else {
18415 panic!("changed {field} must fail replay");
18416 };
18417 assert_eq!(failure.reason, expected_reason);
18418 assert_eq!(failure.sequence, Some(1));
18419 assert!(ctx.take_commands().expect("commands").is_empty());
18420 }
18421 }
18422
18423 #[test]
18424 fn replayed_legacy_activity_treats_missing_option_snapshot_as_unknown() {
18425 let mut history = completed_retry_activity_history();
18426 let activity = history[0].payload["activity"]
18427 .as_object_mut()
18428 .expect("activity snapshot");
18429 activity.remove("execution_mode");
18430 activity.remove("retry_policy");
18431
18432 let mut current = retry_activity_options();
18433 current.start_to_close_timeout = Some(Duration::from_secs(45));
18434 current.schedule_to_start_timeout = Some(Duration::from_secs(8));
18435 current.schedule_to_close_timeout = Some(Duration::from_secs(120));
18436 current.heartbeat_timeout = Some(Duration::from_secs(12));
18437
18438 let ctx = workflow_context(history);
18439 let mut call = Box::pin(ctx.activity_with_options("flaky", current, json!([])));
18440 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18441 assert!(matches!(
18442 call.as_mut().poll(&mut task_context),
18443 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
18444 ));
18445 assert!(ctx.take_commands().expect("commands").is_empty());
18446 ctx.ensure_history_consumed().expect("history consumed");
18447 }
18448
18449 #[test]
18450 fn terminal_activity_failed_after_start_returns_typed_failure() {
18451 let history = vec![
18452 history_event(
18453 "ActivityScheduled",
18454 json!({
18455 "sequence": 1,
18456 "activity_type": "flaky",
18457 "activity_execution_id": "act-terminal",
18458 "activity": {
18459 "id": "act-terminal",
18460 "sequence": 1,
18461 "type": "flaky",
18462 "queue": "critical-activities",
18463 "retry_policy": {
18464 "snapshot_version": 1,
18465 "max_attempts": 3,
18466 "backoff_seconds": [2, 4],
18467 "non_retryable_error_types": ["PermanentError"]
18468 }
18469 }
18470 }),
18471 ),
18472 history_event(
18473 "ActivityStarted",
18474 json!({
18475 "sequence": 1,
18476 "activity_type": "flaky",
18477 "activity_execution_id": "act-terminal",
18478 "activity_attempt_id": "attempt-1",
18479 "attempt_number": 1
18480 }),
18481 ),
18482 history_event(
18483 "ActivityFailed",
18484 json!({
18485 "sequence": 1,
18486 "activity_type": "flaky",
18487 "activity_execution_id": "act-terminal",
18488 "activity_attempt_id": "attempt-1",
18489 "attempt_number": 1,
18490 "failure_id": "failure-terminal",
18491 "failure_category": "activity",
18492 "exception_type": "PermanentError",
18493 "message": "cannot retry",
18494 "non_retryable": true
18495 }),
18496 ),
18497 ];
18498 let ctx = workflow_context(history);
18499 let mut call =
18500 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18501 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18502
18503 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
18504 call.as_mut().poll(&mut task_context)
18505 else {
18506 panic!("terminal ActivityFailed must settle the activity future");
18507 };
18508 assert_eq!(failure.kind, ActivityFailureKind::Failed);
18509 assert_eq!(
18510 failure.activity_execution_id.as_deref(),
18511 Some("act-terminal")
18512 );
18513 assert_eq!(failure.exception_type.as_deref(), Some("PermanentError"));
18514 assert!(failure.non_retryable);
18515 assert!(ctx.take_commands().expect("commands").is_empty());
18516 ctx.ensure_history_consumed().expect("history consumed");
18517 }
18518
18519 #[test]
18520 fn activity_terminal_events_return_machine_readable_failures() {
18521 let cases = [
18522 (
18523 "ActivityFailed",
18524 json!({
18525 "sequence": 1,
18526 "activity_type": "charge-card",
18527 "activity_execution_id": "act-1",
18528 "activity_attempt_id": "attempt-2",
18529 "attempt_number": 2,
18530 "failure_id": "failure-1",
18531 "failure_category": "activity",
18532 "exception_type": "PaymentDeclined",
18533 "exception_class": "payments.PaymentDeclined",
18534 "message": "card declined",
18535 "non_retryable": true
18536 }),
18537 ActivityFailureKind::Failed,
18538 "activity",
18539 ),
18540 (
18541 "ActivityCancelled",
18542 json!({
18543 "sequence": 1,
18544 "activity_type": "charge-card",
18545 "activity_execution_id": "act-1",
18546 "activity_attempt_id": "attempt-1"
18547 }),
18548 ActivityFailureKind::Cancelled,
18549 "cancelled",
18550 ),
18551 ];
18552
18553 for (event_type, payload, expected_kind, expected_reason) in cases {
18554 let ctx = workflow_context(vec![history_event(event_type, payload)]);
18555 let mut call = Box::pin(ctx.activity("charge-card", json!([])));
18556 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18557 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
18558 call.as_mut().poll(&mut task_context)
18559 else {
18560 panic!("expected terminal activity failure");
18561 };
18562 assert_eq!(failure.kind, expected_kind);
18563 assert_eq!(failure.reason, expected_reason);
18564 assert_eq!(failure.activity_execution_id.as_deref(), Some("act-1"));
18565 assert_eq!(failure.activity_type.as_deref(), Some("charge-card"));
18566 }
18567 }
18568
18569 #[test]
18570 fn every_activity_timeout_class_is_typed() {
18571 for timeout_kind in [
18572 "start_to_close",
18573 "schedule_to_start",
18574 "schedule_to_close",
18575 "heartbeat",
18576 ] {
18577 let ctx = workflow_context(vec![history_event(
18578 "ActivityTimedOut",
18579 json!({
18580 "sequence": 1,
18581 "activity_type": "slow",
18582 "activity_execution_id": "act-timeout",
18583 "activity_attempt_id": "attempt-timeout",
18584 "failure_category": "timeout",
18585 "timeout_kind": timeout_kind,
18586 "message": "deadline expired"
18587 }),
18588 )]);
18589 let mut call = Box::pin(ctx.activity("slow", json!([])));
18590 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18591 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
18592 call.as_mut().poll(&mut task_context)
18593 else {
18594 panic!("expected timeout failure");
18595 };
18596 assert_eq!(failure.kind, ActivityFailureKind::TimedOut);
18597 assert_eq!(failure.reason, timeout_kind);
18598 assert_eq!(failure.timeout_kind.as_deref(), Some(timeout_kind));
18599 assert_eq!(failure.failure_category.as_deref(), Some("timeout"));
18600 }
18601 }
18602
18603 #[test]
18604 fn workflow_sleep_emits_one_durable_timer_and_rounds_up() {
18605 let ctx = workflow_context(Vec::new());
18606 let mut sleep = Box::pin(ctx.sleep(Duration::from_millis(1_001)));
18607 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18608
18609 assert!(matches!(
18610 sleep.as_mut().poll(&mut task_context),
18611 Poll::Pending
18612 ));
18613 assert!(matches!(
18614 sleep.as_mut().poll(&mut task_context),
18615 Poll::Pending
18616 ));
18617
18618 let commands = ctx.take_commands().expect("timer command");
18619 assert_eq!(
18620 commands,
18621 vec![json!({
18622 "type": "start_timer",
18623 "delay_seconds": 2,
18624 })]
18625 );
18626 }
18627
18628 #[test]
18629 fn workflow_sleep_replays_matching_schedule_and_fire_without_a_command() {
18630 let history = vec![
18631 history_event(
18632 "TimerScheduled",
18633 json!({
18634 "sequence": 1,
18635 "timer_id": "timer-1",
18636 "delay_seconds": 5,
18637 "fire_at": "2026-07-11T12:00:05Z",
18638 }),
18639 ),
18640 history_event(
18641 "TimerFired",
18642 json!({
18643 "sequence": 1,
18644 "timer_id": "timer-1",
18645 "delay_seconds": 5,
18646 "fire_at": "2026-07-11T12:00:05Z",
18647 "fired_at": "2026-07-11T12:00:05Z",
18648 }),
18649 ),
18650 ];
18651
18652 for _restart in 0..2 {
18653 let ctx = workflow_context(history.clone());
18654 let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(5)));
18655 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18656 assert!(matches!(
18657 sleep.as_mut().poll(&mut task_context),
18658 Poll::Ready(Ok(()))
18659 ));
18660 assert!(ctx.take_commands().expect("commands").is_empty());
18661 ctx.ensure_history_consumed().expect("history consumed");
18662 }
18663 }
18664
18665 #[test]
18666 fn workflow_sleep_rejects_changed_delay_during_replay() {
18667 let ctx = workflow_context(vec![
18668 history_event(
18669 "TimerScheduled",
18670 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
18671 ),
18672 history_event(
18673 "TimerFired",
18674 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
18675 ),
18676 ]);
18677 let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(500)));
18678 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18679
18680 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
18681 sleep.as_mut().poll(&mut task_context)
18682 else {
18683 panic!("changed timer delay must be rejected");
18684 };
18685 assert_eq!(failure.reason, "timer_delay_mismatch");
18686 assert_eq!(failure.sequence, Some(1));
18687 }
18688
18689 #[test]
18690 fn workflow_condition_wait_emits_published_identity_and_timeout_contract() {
18691 let ctx = workflow_context(Vec::new());
18692 let mut wait = Box::pin(
18693 ctx.wait_condition(
18694 ConditionWaitOptions::new("approval.ready", "sha256:approval-v1")
18695 .timeout(Duration::from_millis(60_001)),
18696 || Ok(false),
18697 ),
18698 );
18699 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18700
18701 assert!(matches!(
18702 wait.as_mut().poll(&mut task_context),
18703 Poll::Pending
18704 ));
18705 assert!(matches!(
18706 wait.as_mut().poll(&mut task_context),
18707 Poll::Pending
18708 ));
18709 assert_eq!(
18710 ctx.take_commands().expect("condition command"),
18711 vec![json!({
18712 "type": "open_condition_wait",
18713 "condition_wait_occurrence_id": "rust:condition-wait:0",
18714 "condition_key": "approval.ready",
18715 "condition_definition_fingerprint": "sha256:approval-v1",
18716 "timeout_seconds": 61,
18717 })]
18718 );
18719 }
18720
18721 #[test]
18722 fn workflow_condition_wait_returns_explicit_immediate_results_without_commands() {
18723 let ctx = workflow_context(Vec::new());
18724 let mut satisfied = Box::pin(wait_condition!(ctx, "already-ready", || Ok(true)));
18725 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18726 assert!(matches!(
18727 satisfied.as_mut().poll(&mut task_context),
18728 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18729 ));
18730
18731 let mut timed_out = Box::pin(wait_condition!(
18732 ctx,
18733 "no-wait",
18734 timeout: Duration::ZERO,
18735 || Ok(false),
18736 ));
18737 assert!(matches!(
18738 timed_out.as_mut().poll(&mut task_context),
18739 Poll::Ready(Ok(ConditionWaitResult::TimedOut))
18740 ));
18741 assert!(ctx.take_commands().expect("commands").is_empty());
18742 }
18743
18744 #[test]
18745 fn signal_and_update_history_reevaluate_open_conditions_after_restart() {
18746 let signal_history = vec![
18747 history_event(
18748 "ConditionWaitOpened",
18749 json!({
18750 "sequence": 4,
18751 "condition_wait_id": "condition:4",
18752 "condition_wait_occurrence_id": "rust:condition-wait:0",
18753 "condition_key": "approval",
18754 "condition_definition_fingerprint": "sha256:approval-v1",
18755 "timeout_seconds": 30,
18756 }),
18757 ),
18758 history_event(
18759 "SignalReceived",
18760 json!({
18761 "workflow_sequence": 4,
18762 "signal_name": "approve",
18763 "arguments": fixture_envelope(json!(["Ada"])),
18764 }),
18765 ),
18766 ];
18767 for _worker_before_or_after_restart in 0..2 {
18768 let ctx = workflow_context(signal_history.clone());
18769 let predicate_ctx = ctx.clone();
18770 let mut wait = Box::pin(
18771 ctx.wait_condition(
18772 ConditionWaitOptions::new("approval", "sha256:approval-v1")
18773 .timeout(Duration::from_secs(30)),
18774 move || Ok(!predicate_ctx.signals("approve")?.is_empty()),
18775 ),
18776 );
18777 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18778 assert!(matches!(
18779 wait.as_mut().poll(&mut task_context),
18780 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18781 ));
18782 assert!(ctx.take_commands().expect("commands").is_empty());
18783 ctx.ensure_history_consumed().expect("condition consumed");
18784 }
18785
18786 let update_history = vec![
18787 history_event(
18788 "ConditionWaitOpened",
18789 json!({
18790 "sequence": 7,
18791 "condition_wait_id": "condition:7",
18792 "condition_wait_occurrence_id": "rust:condition-wait:0",
18793 "condition_key": "update-approval",
18794 "condition_definition_fingerprint": "sha256:update-approval-v1",
18795 }),
18796 ),
18797 history_event(
18798 "UpdateApplied",
18799 json!({
18800 "sequence": 7,
18801 "update_id": "update-1",
18802 "update_name": "approve",
18803 "arguments": fixture_envelope(json!([true])),
18804 }),
18805 ),
18806 ];
18807 let ctx = workflow_context(update_history);
18808 let predicate_ctx = ctx.clone();
18809 let mut wait = Box::pin(ctx.wait_condition(
18810 ConditionWaitOptions::new("update-approval", "sha256:update-approval-v1"),
18811 move || {
18812 Ok(predicate_ctx
18813 .updates("approve")?
18814 .first()
18815 .and_then(|arguments| arguments.first())
18816 .and_then(Value::as_bool)
18817 == Some(true))
18818 },
18819 ));
18820 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18821 assert!(matches!(
18822 wait.as_mut().poll(&mut task_context),
18823 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18824 ));
18825 assert!(ctx.take_commands().expect("commands").is_empty());
18826 ctx.ensure_history_consumed().expect("condition consumed");
18827 }
18828
18829 #[test]
18830 fn condition_wait_preserves_open_satisfied_and_timed_out_replay_states() {
18831 let open_history = vec![
18832 history_event(
18833 "ConditionWaitOpened",
18834 json!({
18835 "sequence": 3,
18836 "condition_wait_id": "condition:3",
18837 "condition_wait_occurrence_id": "rust:condition-wait:0",
18838 "condition_key": "two-votes",
18839 "condition_definition_fingerprint": "sha256:two-votes-v1",
18840 "timeout_seconds": 120,
18841 }),
18842 ),
18843 history_event(
18844 "SignalReceived",
18845 json!({
18846 "workflow_sequence": 3,
18847 "signal_name": "vote",
18848 "arguments": fixture_envelope(json!(["first"])),
18849 }),
18850 ),
18851 ];
18852 for _worker_before_or_after_restart in 0..2 {
18853 let ctx = workflow_context(open_history.clone());
18854 let predicate_ctx = ctx.clone();
18855 let mut wait = Box::pin(
18856 ctx.wait_condition(
18857 ConditionWaitOptions::new("two-votes", "sha256:two-votes-v1")
18858 .timeout(Duration::from_secs(120)),
18859 move || Ok(predicate_ctx.signals("vote")?.len() >= 2),
18860 ),
18861 );
18862 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18863 assert!(matches!(
18864 wait.as_mut().poll(&mut task_context),
18865 Poll::Pending
18866 ));
18867 assert_eq!(
18868 ctx.take_commands().expect("reopened condition"),
18869 vec![json!({
18870 "type": "open_condition_wait",
18871 "condition_wait_occurrence_id": "rust:condition-wait:0",
18872 "condition_key": "two-votes",
18873 "condition_definition_fingerprint": "sha256:two-votes-v1",
18874 "timeout_seconds": 120,
18875 })]
18876 );
18877 }
18878
18879 let satisfied_ctx = workflow_context(vec![
18880 history_event(
18881 "ConditionWaitOpened",
18882 json!({
18883 "sequence": 5,
18884 "condition_wait_id": "condition:5",
18885 "condition_wait_occurrence_id": "rust:condition-wait:0",
18886 "condition_key": "approval",
18887 "condition_definition_fingerprint": "sha256:approval-v1",
18888 }),
18889 ),
18890 history_event(
18891 "ConditionWaitSatisfied",
18892 json!({
18893 "sequence": 5,
18894 "condition_wait_id": "condition:5",
18895 "condition_wait_occurrence_id": "rust:condition-wait:0",
18896 "condition_key": "approval",
18897 "condition_definition_fingerprint": "sha256:approval-v1",
18898 }),
18899 ),
18900 ]);
18901 let mut satisfied = Box::pin(satisfied_ctx.wait_condition(
18902 ConditionWaitOptions::new("approval", "sha256:approval-v1"),
18903 || Ok(false),
18904 ));
18905 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18906 assert!(matches!(
18907 satisfied.as_mut().poll(&mut task_context),
18908 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18909 ));
18910
18911 let timed_out_ctx = workflow_context(vec![
18912 history_event(
18913 "ConditionWaitOpened",
18914 json!({
18915 "sequence": 8,
18916 "condition_wait_id": "condition:8",
18917 "condition_wait_occurrence_id": "rust:condition-wait:0",
18918 "condition_key": "approval-timeout",
18919 "condition_definition_fingerprint": "sha256:approval-timeout-v1",
18920 "timeout_seconds": 5,
18921 }),
18922 ),
18923 history_event(
18924 "TimerScheduled",
18925 json!({
18926 "sequence": 9,
18927 "timer_id": "condition-timer:9",
18928 "timer_kind": "condition_timeout",
18929 "condition_wait_id": "condition:8",
18930 "delay_seconds": 5,
18931 }),
18932 ),
18933 history_event(
18934 "TimerFired",
18935 json!({
18936 "sequence": 9,
18937 "timer_id": "condition-timer:9",
18938 "timer_kind": "condition_timeout",
18939 "condition_wait_id": "condition:8",
18940 "delay_seconds": 5,
18941 }),
18942 ),
18943 ]);
18944 let mut timed_out = Box::pin(
18945 timed_out_ctx.wait_condition(
18946 ConditionWaitOptions::new("approval-timeout", "sha256:approval-timeout-v1")
18947 .timeout(Duration::from_secs(5)),
18948 || Ok(true),
18949 ),
18950 );
18951 assert!(matches!(
18952 timed_out.as_mut().poll(&mut task_context),
18953 Poll::Ready(Ok(ConditionWaitResult::TimedOut))
18954 ));
18955 }
18956
18957 #[test]
18958 fn condition_wait_replays_repeated_physical_opens_as_one_logical_wait() {
18959 let history = vec![
18960 history_event(
18961 "ConditionWaitOpened",
18962 json!({
18963 "sequence": 3,
18964 "condition_wait_id": "condition:3",
18965 "condition_wait_occurrence_id": "rust:condition-wait:0",
18966 "condition_key": "two-votes",
18967 "condition_definition_fingerprint": "sha256:two-votes-v1",
18968 }),
18969 ),
18970 history_event(
18971 "SignalReceived",
18972 json!({
18973 "workflow_sequence": 3,
18974 "signal_name": "vote",
18975 "arguments": fixture_envelope(json!(["first"])),
18976 }),
18977 ),
18978 history_event(
18979 "ConditionWaitSatisfied",
18980 json!({
18981 "sequence": 3,
18982 "condition_wait_id": "condition:3",
18983 "condition_wait_occurrence_id": "rust:condition-wait:0",
18984 "condition_key": "two-votes",
18985 "condition_definition_fingerprint": "sha256:two-votes-v1",
18986 }),
18987 ),
18988 history_event(
18989 "ConditionWaitOpened",
18990 json!({
18991 "sequence": 5,
18992 "condition_wait_id": "condition:5",
18993 "condition_wait_occurrence_id": "rust:condition-wait:0",
18994 "condition_key": "two-votes",
18995 "condition_definition_fingerprint": "sha256:two-votes-v1",
18996 }),
18997 ),
18998 history_event(
18999 "SignalReceived",
19000 json!({
19001 "workflow_sequence": 5,
19002 "signal_name": "vote",
19003 "arguments": fixture_envelope(json!(["second"])),
19004 }),
19005 ),
19006 history_event(
19007 "ConditionWaitSatisfied",
19008 json!({
19009 "sequence": 5,
19010 "condition_wait_id": "condition:5",
19011 "condition_wait_occurrence_id": "rust:condition-wait:0",
19012 "condition_key": "two-votes",
19013 "condition_definition_fingerprint": "sha256:two-votes-v1",
19014 }),
19015 ),
19016 ];
19017 for _cold_worker_or_restart in 0..2 {
19018 let ctx = workflow_context(history.clone());
19019 let predicate_ctx = ctx.clone();
19020 let mut wait = Box::pin(ctx.wait_condition(
19021 ConditionWaitOptions::new("two-votes", "sha256:two-votes-v1"),
19022 move || Ok(predicate_ctx.signals("vote")?.len() >= 2),
19023 ));
19024 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19025
19026 assert!(matches!(
19027 wait.as_mut().poll(&mut task_context),
19028 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19029 ));
19030 assert!(ctx.take_commands().expect("commands").is_empty());
19031 ctx.ensure_history_consumed()
19032 .expect("every physical wait-open is consumed");
19033 }
19034 }
19035
19036 #[test]
19037 fn condition_wait_replays_update_driven_physical_opens_as_one_occurrence() {
19038 let history = vec![
19039 history_event(
19040 "ConditionWaitOpened",
19041 json!({
19042 "sequence": 3,
19043 "condition_wait_id": "condition:3",
19044 "condition_wait_occurrence_id": "rust:condition-wait:0",
19045 "condition_key": "approved",
19046 "condition_definition_fingerprint": "sha256:approved-v1",
19047 }),
19048 ),
19049 history_event(
19050 "UpdateApplied",
19051 json!({
19052 "sequence": 3,
19053 "update_id": "update-1",
19054 "update_name": "approve",
19055 "arguments": fixture_envelope(json!([false])),
19056 }),
19057 ),
19058 history_event(
19059 "ConditionWaitOpened",
19060 json!({
19061 "sequence": 5,
19062 "condition_wait_id": "condition:5",
19063 "condition_wait_occurrence_id": "rust:condition-wait:0",
19064 "condition_key": "approved",
19065 "condition_definition_fingerprint": "sha256:approved-v1",
19066 }),
19067 ),
19068 history_event(
19069 "UpdateApplied",
19070 json!({
19071 "sequence": 5,
19072 "update_id": "update-2",
19073 "update_name": "approve",
19074 "arguments": fixture_envelope(json!([true])),
19075 }),
19076 ),
19077 ];
19078
19079 for _cold_worker_or_restart in 0..2 {
19080 let ctx = workflow_context(history.clone());
19081 let predicate_ctx = ctx.clone();
19082 let mut wait = Box::pin(ctx.wait_condition(
19083 ConditionWaitOptions::new("approved", "sha256:approved-v1"),
19084 move || {
19085 Ok(predicate_ctx
19086 .updates("approve")?
19087 .last()
19088 .and_then(|arguments| arguments.first())
19089 .and_then(Value::as_bool)
19090 == Some(true))
19091 },
19092 ));
19093 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19094
19095 assert!(matches!(
19096 wait.as_mut().poll(&mut task_context),
19097 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19098 ));
19099 assert!(ctx.take_commands().expect("commands").is_empty());
19100 ctx.ensure_history_consumed()
19101 .expect("every update-driven reopen is consumed");
19102 }
19103 }
19104
19105 #[test]
19106 fn condition_wait_replay_keeps_every_adjacent_authored_occurrence_distinct() {
19107 for (first_key, first_fingerprint, second_key, second_fingerprint) in [
19108 ("shared", "sha256:first", "shared", "sha256:second"),
19109 ("first", "sha256:shared", "second", "sha256:shared"),
19110 ("shared", "sha256:shared", "shared", "sha256:shared"),
19111 ("first", "sha256:first", "second", "sha256:second"),
19112 ] {
19113 let history = vec![
19114 history_event(
19115 "ConditionWaitOpened",
19116 json!({
19117 "sequence": 3,
19118 "condition_wait_id": "condition:3",
19119 "condition_wait_occurrence_id": "rust:condition-wait:0",
19120 "condition_key": first_key,
19121 "condition_definition_fingerprint": first_fingerprint,
19122 }),
19123 ),
19124 history_event(
19125 "ConditionWaitSatisfied",
19126 json!({
19127 "sequence": 3,
19128 "condition_wait_id": "condition:3",
19129 "condition_wait_occurrence_id": "rust:condition-wait:0",
19130 "condition_key": first_key,
19131 "condition_definition_fingerprint": first_fingerprint,
19132 }),
19133 ),
19134 history_event(
19135 "ConditionWaitOpened",
19136 json!({
19137 "sequence": 4,
19138 "condition_wait_id": "condition:4",
19139 "condition_wait_occurrence_id": "rust:condition-wait:1",
19140 "condition_key": second_key,
19141 "condition_definition_fingerprint": second_fingerprint,
19142 }),
19143 ),
19144 history_event(
19145 "ConditionWaitSatisfied",
19146 json!({
19147 "sequence": 4,
19148 "condition_wait_id": "condition:4",
19149 "condition_wait_occurrence_id": "rust:condition-wait:1",
19150 "condition_key": second_key,
19151 "condition_definition_fingerprint": second_fingerprint,
19152 }),
19153 ),
19154 ];
19155 for _cold_worker_or_restart in 0..2 {
19156 let ctx = workflow_context(history.clone());
19157 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19158 let mut first = Box::pin(ctx.wait_condition(
19159 ConditionWaitOptions::new(first_key, first_fingerprint),
19160 || Ok(false),
19161 ));
19162 assert!(matches!(
19163 first.as_mut().poll(&mut task_context),
19164 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19165 ));
19166
19167 let mut second = Box::pin(ctx.wait_condition(
19168 ConditionWaitOptions::new(second_key, second_fingerprint),
19169 || Ok(false),
19170 ));
19171 assert!(matches!(
19172 second.as_mut().poll(&mut task_context),
19173 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19174 ));
19175 assert!(ctx.take_commands().expect("commands").is_empty());
19176 ctx.ensure_history_consumed()
19177 .expect("each authored wait consumes one occurrence");
19178 }
19179 }
19180 }
19181
19182 #[test]
19183 fn cold_workers_replay_adjacent_condition_waits_from_one_loop_call_site() {
19184 fn worker() -> Worker {
19185 let client = Client::new("http://127.0.0.1:8080").expect("client");
19186 let mut worker = Worker::new(client, "rust-workers");
19187 worker.register_workflow("rust.condition-loop", |ctx, _input| async move {
19188 let mut outcomes = Vec::new();
19189 for _ in 0..2 {
19190 outcomes.push(
19191 ctx.wait_condition(
19192 ConditionWaitOptions::new("shared", "sha256:shared"),
19193 || Ok(false),
19194 )
19195 .await?,
19196 );
19197 }
19198 Ok(json!(outcomes))
19199 });
19200 worker
19201 }
19202
19203 let task = workflow_task(
19204 "rust.condition-loop",
19205 vec![
19206 history_event(
19207 "ConditionWaitOpened",
19208 json!({
19209 "sequence": 1,
19210 "condition_wait_id": "condition:1",
19211 "condition_wait_occurrence_id": "rust:condition-wait:0",
19212 "condition_key": "shared",
19213 "condition_definition_fingerprint": "sha256:shared",
19214 }),
19215 ),
19216 history_event(
19217 "ConditionWaitSatisfied",
19218 json!({
19219 "sequence": 1,
19220 "condition_wait_id": "condition:1",
19221 "condition_wait_occurrence_id": "rust:condition-wait:0",
19222 "condition_key": "shared",
19223 "condition_definition_fingerprint": "sha256:shared",
19224 }),
19225 ),
19226 history_event(
19227 "ConditionWaitOpened",
19228 json!({
19229 "sequence": 2,
19230 "condition_wait_id": "condition:2",
19231 "condition_wait_occurrence_id": "rust:condition-wait:1",
19232 "condition_key": "shared",
19233 "condition_definition_fingerprint": "sha256:shared",
19234 }),
19235 ),
19236 history_event(
19237 "ConditionWaitSatisfied",
19238 json!({
19239 "sequence": 2,
19240 "condition_wait_id": "condition:2",
19241 "condition_wait_occurrence_id": "rust:condition-wait:1",
19242 "condition_key": "shared",
19243 "condition_definition_fingerprint": "sha256:shared",
19244 }),
19245 ),
19246 ],
19247 DEFAULT_CODEC,
19248 );
19249
19250 for _cold_worker_or_restart in 0..2 {
19251 let commands = worker()
19252 .execute_workflow_task(task.clone())
19253 .expect("adjacent loop waits replay deterministically");
19254 assert_eq!(commands.len(), 1);
19255 assert_eq!(commands[0]["type"], "complete_workflow");
19256 assert_eq!(
19257 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("workflow output"),
19258 json!(["satisfied", "satisfied"])
19259 );
19260 }
19261 }
19262
19263 #[test]
19264 fn condition_wait_replay_rejects_identity_predicate_and_timeout_changes() {
19265 let history = vec![history_event(
19266 "ConditionWaitOpened",
19267 json!({
19268 "sequence": 12,
19269 "condition_wait_id": "condition:12",
19270 "condition_wait_occurrence_id": "rust:condition-wait:0",
19271 "condition_key": "approval",
19272 "condition_definition_fingerprint": "sha256:approval-v1",
19273 "timeout_seconds": 30,
19274 }),
19275 )];
19276 for (options, expected_reason) in [
19277 (
19278 ConditionWaitOptions::new("changed", "sha256:approval-v1")
19279 .timeout(Duration::from_secs(30)),
19280 "condition_wait_key_mismatch",
19281 ),
19282 (
19283 ConditionWaitOptions::new("approval", "sha256:approval-v2")
19284 .timeout(Duration::from_secs(30)),
19285 "condition_wait_predicate_mismatch",
19286 ),
19287 (
19288 ConditionWaitOptions::new("approval", "sha256:approval-v1")
19289 .timeout(Duration::from_secs(29)),
19290 "condition_wait_timeout_mismatch",
19291 ),
19292 ] {
19293 let ctx = workflow_context(history.clone());
19294 let mut wait = Box::pin(ctx.wait_condition(options, || Ok(false)));
19295 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19296 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
19297 wait.as_mut().poll(&mut task_context)
19298 else {
19299 panic!("changed condition definition must fail replay");
19300 };
19301 assert_eq!(failure.reason, expected_reason);
19302 assert_eq!(failure.sequence, Some(12));
19303 }
19304 }
19305
19306 #[test]
19307 fn condition_wait_history_requires_the_canonical_predicate_fingerprint() {
19308 let error = WorkflowState::new(
19309 vec![history_event(
19310 "ConditionWaitOpened",
19311 json!({
19312 "sequence": 12,
19313 "condition_wait_id": "condition:12",
19314 "condition_wait_occurrence_id": "rust:condition-wait:0",
19315 "condition_key": "approval",
19316 }),
19317 )],
19318 "rust-workers".to_string(),
19319 DEFAULT_CODEC.to_string(),
19320 None,
19321 )
19322 .expect_err("condition history without a predicate fingerprint must fail");
19323
19324 assert!(matches!(
19325 error,
19326 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19327 if reason == "condition_wait_predicate_fingerprint_missing"
19328 ));
19329 }
19330
19331 #[test]
19332 fn condition_wait_history_requires_authored_occurrence_identity() {
19333 let error = WorkflowState::new(
19334 vec![history_event(
19335 "ConditionWaitOpened",
19336 json!({
19337 "sequence": 12,
19338 "condition_wait_id": "condition:12",
19339 "condition_key": "approval",
19340 "condition_definition_fingerprint": "sha256:approval-v1",
19341 }),
19342 )],
19343 "rust-workers".to_string(),
19344 DEFAULT_CODEC.to_string(),
19345 None,
19346 )
19347 .expect_err("condition history without occurrence identity must fail");
19348
19349 assert!(matches!(
19350 error,
19351 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19352 if reason == "condition_wait_occurrence_id_missing"
19353 ));
19354 }
19355
19356 #[test]
19357 fn typed_search_attribute_updates_validate_emit_and_replay() {
19358 let update = SearchAttributeUpdate::new()
19359 .keyword("OrderStatus", " waiting ")
19360 .expect("keyword")
19361 .int("Attempt", 3)
19362 .expect("int")
19363 .bool("Escalated", false)
19364 .expect("bool")
19365 .keyword_list("Regions", ["us-east", "eu-west"])
19366 .expect("list")
19367 .datetime("UpdatedAt", "2026-08-22T04:00:00Z")
19368 .expect("datetime")
19369 .delete("LegacyStatus")
19370 .expect("delete");
19371 let ctx = workflow_context(Vec::new());
19372 ctx.upsert_search_attributes(update.clone())
19373 .expect("typed update");
19374 assert_eq!(
19375 ctx.take_commands().expect("search-attribute command"),
19376 vec![json!({
19377 "type": "upsert_search_attributes",
19378 "attributes": {
19379 "Attempt": 3,
19380 "Escalated": false,
19381 "LegacyStatus": null,
19382 "OrderStatus": "waiting",
19383 "Regions": ["us-east", "eu-west"],
19384 "UpdatedAt": "2026-08-22T04:00:00Z",
19385 },
19386 "attribute_types": {
19387 "Attempt": "int",
19388 "Escalated": "bool",
19389 "OrderStatus": "keyword",
19390 "Regions": "keyword_list",
19391 "UpdatedAt": "datetime",
19392 },
19393 })]
19394 );
19395
19396 let replay = workflow_context(vec![history_event(
19397 "SearchAttributesUpserted",
19398 json!({
19399 "sequence": 6,
19400 "attributes": {
19401 "Attempt": 3,
19402 "Escalated": false,
19403 "LegacyStatus": null,
19404 "OrderStatus": "waiting",
19405 "Regions": ["us-east", "eu-west"],
19406 "UpdatedAt": "2026-08-22T04:00:00Z",
19407 },
19408 "attribute_types": {
19409 "Attempt": "int",
19410 "Escalated": "bool",
19411 "OrderStatus": "keyword",
19412 "Regions": "keyword_list",
19413 "UpdatedAt": "datetime",
19414 },
19415 "merged": {},
19416 }),
19417 )]);
19418 replay
19419 .upsert_search_attributes(update)
19420 .expect("matching update replays");
19421 assert!(replay.take_commands().expect("commands").is_empty());
19422 replay.ensure_history_consumed().expect("history consumed");
19423
19424 let type_drift = workflow_context(vec![history_event(
19425 "SearchAttributesUpserted",
19426 json!({
19427 "sequence": 7,
19428 "attributes": {"OrderStatus": "waiting"},
19429 "attribute_types": {"OrderStatus": "keyword"},
19430 "merged": {"OrderStatus": "waiting"},
19431 }),
19432 )]);
19433 let error = type_drift
19434 .upsert_search_attributes(
19435 SearchAttributeUpdate::new()
19436 .string("OrderStatus", "waiting")
19437 .expect("string update"),
19438 )
19439 .expect_err("same JSON value with a changed type must fail replay");
19440 assert!(matches!(
19441 error,
19442 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19443 if reason == "search_attribute_type_mismatch"
19444 ));
19445
19446 let malformed_types = WorkflowState::new(
19447 vec![history_event(
19448 "SearchAttributesUpserted",
19449 json!({
19450 "sequence": 8,
19451 "attributes": {"OrderStatus": "waiting"},
19452 "attribute_types": {"OrderStatus": "unsupported"},
19453 "merged": {"OrderStatus": "waiting"},
19454 }),
19455 )],
19456 "rust-workers".to_string(),
19457 DEFAULT_CODEC.to_string(),
19458 None,
19459 )
19460 .expect_err("unsupported search-attribute type metadata must fail");
19461 assert!(matches!(
19462 malformed_types,
19463 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19464 if reason == "search_attribute_types_malformed"
19465 ));
19466
19467 assert!(matches!(
19468 SearchAttributeUpdate::new().keyword("bad key", "value"),
19469 Err(SearchAttributeUpdateError::InvalidKey(_))
19470 ));
19471 assert!(matches!(
19472 SearchAttributeUpdate::new().float("Ratio", f64::NAN),
19473 Err(SearchAttributeUpdateError::NonFiniteFloat(_))
19474 ));
19475 assert!(matches!(
19476 SearchAttributeUpdate::new().keyword("UnicodeKeyword", "é".repeat(128)),
19477 Err(SearchAttributeUpdateError::ValueTooLong { .. })
19478 ));
19479 assert!(matches!(
19480 SearchAttributeUpdate::new().datetime("UpdatedAt", "2026-02-30T04:00:00Z"),
19481 Err(SearchAttributeUpdateError::InvalidDateTime(_))
19482 ));
19483 assert!(matches!(
19484 workflow_context(Vec::new()).upsert_search_attributes(SearchAttributeUpdate::new()),
19485 Err(Error::InvalidSearchAttributeUpdate(
19486 SearchAttributeUpdateError::Empty
19487 ))
19488 ));
19489 }
19490
19491 #[test]
19492 fn typed_search_attribute_text_uses_the_runtime_byte_limit() {
19493 let ascii = "a".repeat(MAX_SEARCH_ATTRIBUTE_STRING_LENGTH);
19494 let utf8 = "é".repeat(MAX_SEARCH_ATTRIBUTE_STRING_LENGTH / 2);
19495
19496 assert!(SearchAttributeUpdate::new()
19497 .string("AsciiDescription", ascii)
19498 .is_ok());
19499 assert!(SearchAttributeUpdate::new()
19500 .string("Utf8Description", utf8)
19501 .is_ok());
19502 assert!(matches!(
19503 SearchAttributeUpdate::new().string(
19504 "TooLongDescription",
19505 "é".repeat((MAX_SEARCH_ATTRIBUTE_STRING_LENGTH / 2) + 1),
19506 ),
19507 Err(SearchAttributeUpdateError::ValueTooLong {
19508 kind: "string",
19509 limit: MAX_SEARCH_ATTRIBUTE_STRING_LENGTH,
19510 ..
19511 })
19512 ));
19513 }
19514
19515 #[test]
19516 fn workflow_history_rejects_unpaired_or_mismatched_timer_events() {
19517 let lone_fire = WorkflowState::new(
19518 vec![history_event(
19519 "TimerFired",
19520 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19521 )],
19522 "rust-workers".to_string(),
19523 DEFAULT_CODEC.to_string(),
19524 None,
19525 )
19526 .expect_err("TimerFired requires TimerScheduled");
19527 assert!(matches!(
19528 lone_fire,
19529 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19530 if reason == "timer_schedule_missing_or_duplicate"
19531 ));
19532
19533 let wrong_identity = WorkflowState::new(
19534 vec![
19535 history_event(
19536 "TimerScheduled",
19537 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19538 ),
19539 history_event(
19540 "TimerFired",
19541 json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
19542 ),
19543 ],
19544 "rust-workers".to_string(),
19545 DEFAULT_CODEC.to_string(),
19546 None,
19547 )
19548 .expect_err("fire must match scheduled timer identity");
19549 assert!(matches!(
19550 wrong_identity,
19551 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19552 if reason == "timer_identity_mismatch"
19553 ));
19554
19555 let duplicate_fire = WorkflowState::new(
19556 vec![
19557 history_event(
19558 "TimerScheduled",
19559 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19560 ),
19561 history_event(
19562 "TimerFired",
19563 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19564 ),
19565 history_event(
19566 "TimerFired",
19567 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19568 ),
19569 ],
19570 "rust-workers".to_string(),
19571 DEFAULT_CODEC.to_string(),
19572 None,
19573 )
19574 .expect_err("a durable timer cannot fire twice");
19575 assert!(matches!(
19576 duplicate_fire,
19577 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19578 if reason == "duplicate_timer_fire"
19579 ));
19580
19581 let wrong_fired_delay = WorkflowState::new(
19582 vec![
19583 history_event(
19584 "TimerScheduled",
19585 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19586 ),
19587 history_event(
19588 "TimerFired",
19589 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 6}),
19590 ),
19591 ],
19592 "rust-workers".to_string(),
19593 DEFAULT_CODEC.to_string(),
19594 None,
19595 )
19596 .expect_err("timer schedule and fire delays must agree");
19597 assert!(matches!(
19598 wrong_fired_delay,
19599 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19600 if reason == "timer_history_delay_mismatch"
19601 ));
19602 }
19603
19604 #[test]
19605 fn replay_rejects_activity_moved_before_recorded_timer() {
19606 let ctx = workflow_context(vec![
19607 history_event(
19608 "TimerScheduled",
19609 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19610 ),
19611 history_event(
19612 "TimerFired",
19613 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19614 ),
19615 history_event(
19616 "ActivityCompleted",
19617 json!({
19618 "sequence": 2,
19619 "activity_type": "after-timer",
19620 "payload_codec": DEFAULT_CODEC,
19621 "result": fixture_envelope(json!("done")),
19622 }),
19623 ),
19624 ]);
19625 let mut activity = Box::pin(ctx.activity("after-timer", json!([])));
19626 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19627
19628 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
19629 activity.as_mut().poll(&mut task_context)
19630 else {
19631 panic!("reordered durable command must be rejected");
19632 };
19633 assert_eq!(failure.reason, "recorded_command_mismatch");
19634 assert_eq!(failure.sequence, Some(1));
19635 assert_eq!(failure.expected.as_deref(), Some("timer"));
19636 assert_eq!(failure.actual.as_deref(), Some("activity:after-timer"));
19637 }
19638
19639 #[test]
19640 fn workflow_context_emits_a_typed_named_signal_wait() {
19641 let ctx = workflow_context(Vec::new());
19642 let mut signal = Box::pin(ctx.wait_signal("finish"));
19643 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19644
19645 assert!(matches!(
19646 signal.as_mut().poll(&mut task_context),
19647 Poll::Pending
19648 ));
19649 assert_eq!(
19650 ctx.take_commands().expect("signal-wait command"),
19651 vec![json!({
19652 "type": "open_signal_wait",
19653 "signal_name": "finish",
19654 })]
19655 );
19656 }
19657
19658 #[test]
19659 fn runtime_message_stream_transport_cannot_be_opened_as_a_user_signal() {
19660 let ctx = workflow_context(Vec::new());
19661 let mut signal = Box::pin(ctx.wait_signal(MESSAGE_STREAM_SIGNAL));
19662 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19663
19664 let Poll::Ready(Err(Error::Codec(message))) = signal.as_mut().poll(&mut task_context)
19665 else {
19666 panic!("runtime-reserved signal should be rejected");
19667 };
19668 assert!(message.contains("reserved by the workflow runtime"));
19669 assert!(ctx.take_commands().expect("commands").is_empty());
19670 }
19671
19672 #[tokio::test]
19673 async fn runtime_message_stream_transport_cannot_be_sent_as_a_user_signal() {
19674 let client = Client::builder("http://127.0.0.1:9")
19675 .build()
19676 .expect("client");
19677 let error = client
19678 .signal_workflow("workflow-1", MESSAGE_STREAM_SIGNAL, json!(["forged"]))
19679 .await
19680 .expect_err("runtime-reserved signal should be rejected before transport");
19681
19682 assert!(
19683 matches!(error, Error::Codec(ref message) if message.contains("reserved by the workflow runtime"))
19684 );
19685 }
19686
19687 #[test]
19688 fn message_stream_worker_task_consumes_current_contiguous_bounded_batch() {
19689 fn delivery(message_id: &str, position: u64, value: &str) -> Value {
19690 let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
19691 value.to_string(),
19692 )]))
19693 .expect("message payload");
19694 json!({
19695 "schema": MESSAGE_STREAM_SCHEMA,
19696 "stream_name": "orders",
19697 "message_id": message_id,
19698 "position": position,
19699 "payload_envelope": payload,
19700 })
19701 }
19702
19703 fn opened(sequence: u64) -> HistoryEvent {
19704 history_event(
19705 "SignalWaitOpened",
19706 json!({
19707 "sequence": sequence,
19708 "signal_name": MESSAGE_STREAM_SIGNAL,
19709 }),
19710 )
19711 }
19712
19713 fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
19714 history_event(
19715 "SignalApplied",
19716 json!({
19717 "sequence": sequence,
19718 "signal_name": MESSAGE_STREAM_SIGNAL,
19719 "value": fixture_envelope(json!([delivery])),
19720 }),
19721 )
19722 }
19723
19724 fn received(delivery: Value) -> HistoryEvent {
19725 history_event(
19726 "SignalReceived",
19727 json!({
19728 "signal_name": MESSAGE_STREAM_SIGNAL,
19729 "arguments": fixture_envelope(json!([delivery])),
19730 "payload_codec": DEFAULT_CODEC,
19731 }),
19732 )
19733 }
19734
19735 let client = Client::new("http://127.0.0.1:8080").expect("client");
19736 let mut worker = Worker::new(client, "rust-workers");
19737 worker.register_workflow("rust.message-stream-batch", |ctx, _input| async move {
19738 let messages = ctx.message_stream("orders")?.receive(2).await?;
19739 Ok(json!(messages
19740 .into_iter()
19741 .map(|message| message.message_id)
19742 .collect::<Vec<_>>()))
19743 });
19744
19745 let first = delivery("message-1", 1, "one");
19746 let second = delivery("message-2", 2, "two");
19747 let batch = worker
19748 .execute_workflow_task_decision(workflow_task(
19749 "rust.message-stream-batch",
19750 vec![
19751 opened(1),
19752 received(first.clone()),
19753 applied(1, first.clone()),
19754 received(first.clone()),
19755 received(second),
19756 ],
19757 DEFAULT_CODEC,
19758 ))
19759 .expect("worker task consumes the available batch");
19760
19761 assert_eq!(batch.commands.len(), 1);
19762 assert_eq!(batch.commands[0]["type"], "complete_workflow");
19763 assert_eq!(
19764 decode_wire_value(&batch.commands[0]["result"], DEFAULT_CODEC)
19765 .expect("workflow result"),
19766 json!(["message-1", "message-2"])
19767 );
19768 assert_eq!(
19769 batch.message_stream_cursors,
19770 vec![json!({"stream_name": "orders", "through_position": 2})]
19771 );
19772 assert!(batch.message_stream_waits.is_empty());
19773
19774 let partial = worker
19775 .execute_workflow_task_decision(workflow_task(
19776 "rust.message-stream-batch",
19777 vec![opened(1), received(first.clone()), applied(1, first)],
19778 DEFAULT_CODEC,
19779 ))
19780 .expect("worker task returns without waiting for a missing second item");
19781 assert_eq!(partial.commands.len(), 1);
19782 assert_eq!(partial.commands[0]["type"], "complete_workflow");
19783 assert_eq!(
19784 decode_wire_value(&partial.commands[0]["result"], DEFAULT_CODEC)
19785 .expect("workflow result"),
19786 json!(["message-1"])
19787 );
19788 assert_eq!(
19789 partial.message_stream_cursors,
19790 vec![json!({"stream_name": "orders", "through_position": 1})]
19791 );
19792 assert!(partial.message_stream_waits.is_empty());
19793 }
19794
19795 #[test]
19796 fn message_stream_replay_preserves_partial_batch_boundary_before_later_wait() {
19797 fn delivery(message_id: &str, position: u64, value: &str) -> Value {
19798 let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
19799 value.to_string(),
19800 )]))
19801 .expect("message payload");
19802 json!({
19803 "schema": MESSAGE_STREAM_SCHEMA,
19804 "stream_name": "orders",
19805 "message_id": message_id,
19806 "position": position,
19807 "payload_envelope": payload,
19808 })
19809 }
19810
19811 fn opened(sequence: u64) -> HistoryEvent {
19812 history_event(
19813 "SignalWaitOpened",
19814 json!({
19815 "sequence": sequence,
19816 "signal_name": MESSAGE_STREAM_SIGNAL,
19817 }),
19818 )
19819 }
19820
19821 fn received(delivery: Value) -> HistoryEvent {
19822 history_event(
19823 "SignalReceived",
19824 json!({
19825 "signal_name": MESSAGE_STREAM_SIGNAL,
19826 "arguments": fixture_envelope(json!([delivery])),
19827 "payload_codec": DEFAULT_CODEC,
19828 }),
19829 )
19830 }
19831
19832 fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
19833 history_event(
19834 "SignalApplied",
19835 json!({
19836 "sequence": sequence,
19837 "signal_name": MESSAGE_STREAM_SIGNAL,
19838 "value": fixture_envelope(json!([delivery])),
19839 }),
19840 )
19841 }
19842
19843 let client = Client::new("http://127.0.0.1:8080").expect("client");
19844 let mut worker = Worker::new(client, "rust-workers");
19845 worker.register_workflow(
19846 "rust.message-stream-partial-batches",
19847 |ctx, _input| async move {
19848 let stream = ctx.message_stream("orders")?;
19849 let first = stream.receive(10).await?;
19850 let second = stream.receive(10).await?;
19851 Ok(json!([
19852 first
19853 .into_iter()
19854 .map(|message| message.message_id)
19855 .collect::<Vec<_>>(),
19856 second
19857 .into_iter()
19858 .map(|message| message.message_id)
19859 .collect::<Vec<_>>(),
19860 ]))
19861 },
19862 );
19863
19864 let first = delivery("message-1", 1, "one");
19865 let second = delivery("message-2", 2, "two");
19866 let decision = worker
19867 .execute_workflow_task_decision(workflow_task(
19868 "rust.message-stream-partial-batches",
19869 vec![
19870 opened(1),
19871 received(first.clone()),
19872 applied(1, first),
19873 opened(2),
19874 received(second.clone()),
19875 applied(2, second),
19876 ],
19877 DEFAULT_CODEC,
19878 ))
19879 .expect("cold replay preserves both authored receive boundaries");
19880
19881 assert_eq!(decision.commands.len(), 1);
19882 assert_eq!(decision.commands[0]["type"], "complete_workflow");
19883 assert_eq!(
19884 decode_wire_value(&decision.commands[0]["result"], DEFAULT_CODEC)
19885 .expect("workflow result"),
19886 json!([["message-1"], ["message-2"]])
19887 );
19888 assert_eq!(
19889 decision.message_stream_cursors,
19890 vec![json!({"stream_name": "orders", "through_position": 2})]
19891 );
19892 assert!(decision.message_stream_waits.is_empty());
19893 }
19894
19895 #[test]
19896 fn empty_message_stream_opens_internal_signal_wait_and_reports_position() {
19897 let ctx = workflow_context(Vec::new());
19898 let stream = ctx.message_stream("orders").expect("message stream");
19899 let mut receive = Box::pin(stream.receive(10));
19900 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19901
19902 assert!(matches!(
19903 receive.as_mut().poll(&mut task_context),
19904 Poll::Pending
19905 ));
19906 assert_eq!(
19907 ctx.take_commands().expect("message-stream wait command"),
19908 vec![json!({
19909 "type": "open_signal_wait",
19910 "signal_name": MESSAGE_STREAM_SIGNAL,
19911 })]
19912 );
19913 let (cursors, waits) = ctx.message_stream_metadata().expect("stream metadata");
19914 assert!(cursors.is_empty());
19915 assert_eq!(
19916 waits,
19917 vec![json!({"stream_name": "orders", "after_position": 0})]
19918 );
19919 }
19920
19921 #[test]
19922 fn continue_as_new_cursor_checkpoint_preserves_global_pending_position() {
19923 let ctx = workflow_context(vec![history_event(
19924 "SignalReceived",
19925 json!({
19926 "signal_name": MESSAGE_STREAM_SIGNAL,
19927 "arguments": fixture_envelope(json!([{
19928 "schema": MESSAGE_STREAM_CURSOR_SCHEMA,
19929 "stream_name": "orders",
19930 "through_position": 2,
19931 }])),
19932 "payload_codec": DEFAULT_CODEC,
19933 }),
19934 )]);
19935 let stream = ctx.message_stream("orders").expect("message stream");
19936 let mut receive = Box::pin(stream.receive(10));
19937 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19938
19939 assert!(matches!(
19940 receive.as_mut().poll(&mut task_context),
19941 Poll::Pending
19942 ));
19943 let (cursors, waits) = ctx.message_stream_metadata().expect("stream metadata");
19944 assert_eq!(
19945 cursors,
19946 vec![json!({"stream_name": "orders", "through_position": 2})]
19947 );
19948 assert_eq!(
19949 waits,
19950 vec![json!({"stream_name": "orders", "after_position": 2})]
19951 );
19952 }
19953
19954 #[test]
19955 fn message_stream_delivery_preserves_typed_avro_arguments_across_replay() {
19956 let mut empty_map = BTreeMap::new();
19957 let mut nested = BTreeMap::new();
19958 nested.insert(
19959 "value".to_string(),
19960 AvroValue::Array(vec![AvroValue::Bytes(b"nested".to_vec())]),
19961 );
19962 let values = vec![
19963 AvroValue::Bytes(vec![0, 255]),
19964 AvroValue::Long(1),
19965 AvroValue::Double(1.0),
19966 AvroValue::Array(Vec::new()),
19967 AvroValue::Map(std::mem::take(&mut empty_map)),
19968 AvroValue::Map(nested),
19969 ];
19970 let payload = encode_avro_value(&AvroValue::Array(values.clone())).expect("payload");
19971 let transport = vec![json!({
19972 "schema": MESSAGE_STREAM_SCHEMA,
19973 "stream_name": "orders",
19974 "message_id": "message-1",
19975 "position": 1,
19976 "payload_envelope": payload,
19977 })];
19978
19979 for _ in 0..2 {
19980 let Some(MessageStreamDelivery::Message(message)) =
19981 decode_message_stream_delivery(transport.clone()).expect("delivery")
19982 else {
19983 panic!("message delivery expected");
19984 };
19985 assert_eq!(message.arguments, values);
19986 assert!(matches!(message.arguments[1], AvroValue::Long(1)));
19987 assert!(matches!(message.arguments[2], AvroValue::Double(1.0)));
19988 }
19989 }
19990
19991 #[test]
19992 fn cold_worker_replacement_consumes_message_stream_wait_arrivals_once_in_order() {
19993 fn delivery(message_id: &str, position: u64, value: &str) -> Value {
19994 let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
19995 value.to_string(),
19996 )]))
19997 .expect("message payload");
19998 json!({
19999 "schema": MESSAGE_STREAM_SCHEMA,
20000 "stream_name": "orders",
20001 "message_id": message_id,
20002 "position": position,
20003 "payload_envelope": payload,
20004 })
20005 }
20006
20007 fn opened(sequence: u64) -> HistoryEvent {
20008 history_event(
20009 "SignalWaitOpened",
20010 json!({
20011 "sequence": sequence,
20012 "signal_name": MESSAGE_STREAM_SIGNAL,
20013 }),
20014 )
20015 }
20016
20017 fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
20018 history_event(
20019 "SignalApplied",
20020 json!({
20021 "sequence": sequence,
20022 "signal_name": MESSAGE_STREAM_SIGNAL,
20023 "value": fixture_envelope(json!([delivery])),
20024 }),
20025 )
20026 }
20027
20028 fn worker() -> Worker {
20029 let client = Client::new("http://127.0.0.1:8080").expect("client");
20030 let mut worker = Worker::new(client, "rust-workers");
20031 worker.register_workflow("rust.message-stream", |ctx, _input| async move {
20032 let stream = ctx.message_stream("orders")?;
20033 let first = stream.receive_one().await?;
20034 let second = stream.receive_one().await?;
20035 Ok(json!([first.message_id, second.message_id]))
20036 });
20037 worker
20038 }
20039
20040 fn task_with_resume(history: Vec<HistoryEvent>, delivery: Value) -> WorkflowTask {
20041 let mut task = workflow_task("rust.message-stream", history, DEFAULT_CODEC);
20042 task.signal_name = Some(MESSAGE_STREAM_SIGNAL.to_string());
20043 task.signal_arguments = Some(fixture_envelope(json!([delivery])));
20044 task
20045 }
20046
20047 let waiting = worker()
20048 .execute_workflow_task_decision(workflow_task(
20049 "rust.message-stream",
20050 Vec::new(),
20051 DEFAULT_CODEC,
20052 ))
20053 .expect("first worker opens the stream wait");
20054 assert_eq!(
20055 waiting.commands,
20056 vec![json!({
20057 "type": "open_signal_wait",
20058 "signal_name": MESSAGE_STREAM_SIGNAL,
20059 })]
20060 );
20061 assert!(waiting.message_stream_cursors.is_empty());
20062 assert_eq!(
20063 waiting.message_stream_waits,
20064 vec![json!({"stream_name": "orders", "after_position": 0})]
20065 );
20066
20067 let first_delivery = delivery("message-1", 1, "one");
20068 let first_arrival = worker()
20069 .execute_workflow_task_decision(task_with_resume(
20070 vec![opened(1)],
20071 first_delivery.clone(),
20072 ))
20073 .expect("replacement worker consumes the first arrival");
20074 assert_eq!(
20075 first_arrival.commands,
20076 vec![json!({
20077 "type": "open_signal_wait",
20078 "signal_name": MESSAGE_STREAM_SIGNAL,
20079 })]
20080 );
20081 assert_eq!(
20082 first_arrival.message_stream_cursors,
20083 vec![json!({"stream_name": "orders", "through_position": 1})]
20084 );
20085 assert_eq!(
20086 first_arrival.message_stream_waits,
20087 vec![json!({"stream_name": "orders", "after_position": 1})]
20088 );
20089
20090 let second_delivery = delivery("message-2", 2, "two");
20091 let first_applied = applied(1, first_delivery);
20092 let completed = worker()
20093 .execute_workflow_task_decision(task_with_resume(
20094 vec![opened(1), first_applied.clone(), opened(2)],
20095 second_delivery.clone(),
20096 ))
20097 .expect("next replacement worker consumes the second arrival");
20098 assert_eq!(completed.commands.len(), 1);
20099 assert_eq!(completed.commands[0]["type"], "complete_workflow");
20100 assert_eq!(
20101 decode_wire_value(&completed.commands[0]["result"], DEFAULT_CODEC)
20102 .expect("workflow result"),
20103 json!(["message-1", "message-2"])
20104 );
20105 assert_eq!(
20106 completed.message_stream_cursors,
20107 vec![json!({"stream_name": "orders", "through_position": 2})]
20108 );
20109 assert!(completed.message_stream_waits.is_empty());
20110
20111 let replay_history = vec![
20112 opened(1),
20113 first_applied,
20114 opened(2),
20115 applied(2, second_delivery),
20116 ];
20117 for _cold_worker_or_restart in 0..2 {
20118 let replayed = worker()
20119 .execute_workflow_task_decision(workflow_task(
20120 "rust.message-stream",
20121 replay_history.clone(),
20122 DEFAULT_CODEC,
20123 ))
20124 .expect("cold worker replays each logical message exactly once");
20125 assert_eq!(replayed.commands.len(), 1);
20126 assert_eq!(
20127 decode_wire_value(&replayed.commands[0]["result"], DEFAULT_CODEC)
20128 .expect("replayed workflow result"),
20129 json!(["message-1", "message-2"])
20130 );
20131 assert_eq!(
20132 replayed.message_stream_cursors,
20133 vec![json!({"stream_name": "orders", "through_position": 2})]
20134 );
20135 assert!(replayed.message_stream_waits.is_empty());
20136 }
20137 }
20138
20139 #[test]
20140 fn message_stream_capability_and_completion_require_protocol_one_fifteen() {
20141 assert!(!worker_protocol_supports_message_streams("1.14"));
20142 assert!(worker_protocol_supports_message_streams("1.15"));
20143 assert!(worker_protocol_supports_message_streams("1.16"));
20144 assert!(worker_protocol_supports_message_streams(
20145 WORKER_PROTOCOL_VERSION
20146 ));
20147 assert_eq!(MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION, "1.15");
20148 }
20149
20150 #[test]
20151 fn condition_wait_history_cannot_be_consumed_as_a_typed_signal_wait() {
20152 let ctx = workflow_context(vec![
20153 history_event(
20154 "ConditionWaitOpened",
20155 json!({
20156 "sequence": 1,
20157 "condition_wait_id": "condition:1",
20158 "condition_wait_occurrence_id": "rust:condition-wait:0",
20159 "condition_key": "signal:finish",
20160 "condition_definition_fingerprint": "sha256:signal-finish-v1",
20161 }),
20162 ),
20163 history_event(
20164 "ConditionWaitSatisfied",
20165 json!({
20166 "sequence": 1,
20167 "condition_wait_id": "condition:1",
20168 "condition_wait_occurrence_id": "rust:condition-wait:0",
20169 "condition_key": "signal:finish",
20170 "condition_definition_fingerprint": "sha256:signal-finish-v1",
20171 }),
20172 ),
20173 history_event(
20174 "SignalReceived",
20175 json!({"signal_name": "finish", "arguments": []}),
20176 ),
20177 ]);
20178 let mut signal = Box::pin(ctx.wait_signal("finish"));
20179 let mut task_context = TaskContext::from_waker(noop_waker_ref());
20180
20181 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
20182 signal.as_mut().poll(&mut task_context)
20183 else {
20184 panic!("condition history must not resolve as a typed signal wait");
20185 };
20186 assert_eq!(failure.reason, "recorded_command_mismatch");
20187 assert_eq!(failure.expected.as_deref(), Some("condition wait"));
20188 }
20189
20190 #[test]
20191 fn replay_orders_signal_waits_and_timers_in_one_command_stream() {
20192 let signal_then_timer = vec![
20193 history_event(
20194 "SignalWaitOpened",
20195 json!({"sequence": 1, "signal_name": "go"}),
20196 ),
20197 history_event(
20198 "SignalApplied",
20199 json!({
20200 "sequence": 1,
20201 "signal_name": "go",
20202 "value": fixture_envelope(json!(["now"])),
20203 }),
20204 ),
20205 history_event(
20206 "TimerScheduled",
20207 json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
20208 ),
20209 history_event(
20210 "TimerFired",
20211 json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
20212 ),
20213 ];
20214
20215 let ctx = workflow_context(signal_then_timer.clone());
20216 let mut signal = Box::pin(ctx.wait_signal("go"));
20217 let mut task_context = TaskContext::from_waker(noop_waker_ref());
20218 assert!(matches!(
20219 signal.as_mut().poll(&mut task_context),
20220 Poll::Ready(Ok(arguments)) if arguments == vec![json!("now")]
20221 ));
20222 let mut timer = Box::pin(ctx.sleep(Duration::from_secs(5)));
20223 assert!(matches!(
20224 timer.as_mut().poll(&mut task_context),
20225 Poll::Ready(Ok(()))
20226 ));
20227 ctx.ensure_history_consumed()
20228 .expect("signal and timer history consumed in order");
20229
20230 let reordered = workflow_context(signal_then_timer);
20231 let mut timer_first = Box::pin(reordered.sleep(Duration::from_secs(5)));
20232 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
20233 timer_first.as_mut().poll(&mut task_context)
20234 else {
20235 panic!("timer cannot consume signal-wait-first history");
20236 };
20237 assert_eq!(failure.reason, "recorded_command_mismatch");
20238 assert_eq!(failure.sequence, Some(1));
20239 assert_eq!(failure.expected.as_deref(), Some("signal wait"));
20240
20241 let timer_then_signal = vec![
20242 history_event(
20243 "TimerScheduled",
20244 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20245 ),
20246 history_event(
20247 "TimerFired",
20248 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20249 ),
20250 history_event(
20251 "SignalWaitOpened",
20252 json!({"sequence": 2, "signal_name": "go"}),
20253 ),
20254 history_event(
20255 "SignalApplied",
20256 json!({
20257 "sequence": 2,
20258 "signal_name": "go",
20259 "value": fixture_envelope(json!([])),
20260 }),
20261 ),
20262 ];
20263 let reordered = workflow_context(timer_then_signal);
20264 let mut signal_first = Box::pin(reordered.wait_signal("go"));
20265 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
20266 signal_first.as_mut().poll(&mut task_context)
20267 else {
20268 panic!("signal wait cannot consume timer-first history");
20269 };
20270 assert_eq!(failure.reason, "recorded_command_mismatch");
20271 assert_eq!(failure.sequence, Some(1));
20272 assert_eq!(failure.expected.as_deref(), Some("timer"));
20273 }
20274
20275 #[test]
20276 fn workflow_history_rejects_duplicate_or_colliding_command_sequences() {
20277 let duplicate_timer = WorkflowState::new(
20278 vec![
20279 history_event(
20280 "TimerScheduled",
20281 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20282 ),
20283 history_event(
20284 "TimerScheduled",
20285 json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
20286 ),
20287 ],
20288 "rust-workers".to_string(),
20289 DEFAULT_CODEC.to_string(),
20290 None,
20291 )
20292 .expect_err("one workflow sequence cannot schedule two timers");
20293 assert!(matches!(
20294 duplicate_timer,
20295 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20296 if reason == "timer_schedule_missing_or_duplicate"
20297 ));
20298
20299 let colliding_kinds = WorkflowState::new(
20300 vec![
20301 history_event(
20302 "TimerScheduled",
20303 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20304 ),
20305 history_event(
20306 "ActivityCompleted",
20307 json!({"sequence": 1, "activity_type": "same-sequence"}),
20308 ),
20309 ],
20310 "rust-workers".to_string(),
20311 DEFAULT_CODEC.to_string(),
20312 None,
20313 )
20314 .expect_err("one workflow sequence cannot identify two command kinds");
20315 assert!(matches!(
20316 colliding_kinds,
20317 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20318 if reason == "durable_command_sequence_collision"
20319 ));
20320
20321 let duplicate_signal_wait = WorkflowState::new(
20322 vec![
20323 history_event(
20324 "SignalWaitOpened",
20325 json!({"sequence": 1, "signal_name": "go"}),
20326 ),
20327 history_event(
20328 "SignalWaitOpened",
20329 json!({"sequence": 1, "signal_name": "go"}),
20330 ),
20331 ],
20332 "rust-workers".to_string(),
20333 DEFAULT_CODEC.to_string(),
20334 None,
20335 )
20336 .expect_err("one workflow sequence cannot open two signal waits");
20337 assert!(matches!(
20338 duplicate_signal_wait,
20339 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20340 if reason == "signal_wait_open_missing_or_duplicate"
20341 ));
20342 }
20343
20344 #[test]
20345 fn workflow_history_accepts_a_first_command_after_global_sequence_gaps() {
20346 let result = encode_value_envelope(&json!({"captured": true}), DEFAULT_CODEC)
20347 .expect("side-effect result");
20348 let ctx = workflow_context(vec![history_event(
20349 "SideEffectRecorded",
20350 json!({"sequence": 99, "result": result}),
20351 )]);
20352
20353 let replayed: Value = ctx
20354 .side_effect(|| panic!("recorded side effect must not run"))
20355 .expect("positive global workflow sequence is valid");
20356 assert_eq!(replayed, json!({"captured": true}));
20357 ctx.ensure_history_consumed().expect("history consumed");
20358 }
20359
20360 #[test]
20361 fn workflow_history_rejects_zero_and_descending_command_sequences() {
20362 let result =
20363 encode_value_envelope(&json!("captured"), DEFAULT_CODEC).expect("side-effect result");
20364 let zero = WorkflowState::new(
20365 vec![history_event(
20366 "SideEffectRecorded",
20367 json!({"sequence": 0, "result": result.clone()}),
20368 )],
20369 "rust-workers".to_string(),
20370 DEFAULT_CODEC.to_string(),
20371 None,
20372 )
20373 .expect_err("durable command sequences must be positive");
20374 assert!(matches!(
20375 zero,
20376 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20377 if reason == "durable_command_sequence_invalid"
20378 ));
20379
20380 let descending = WorkflowState::new(
20381 vec![
20382 history_event(
20383 "SideEffectRecorded",
20384 json!({"sequence": 3, "result": result}),
20385 ),
20386 history_event(
20387 "VersionMarkerRecorded",
20388 json!({
20389 "sequence": 2,
20390 "change_id": "descending-marker",
20391 "version": 1,
20392 "min_supported": 1,
20393 "max_supported": 1,
20394 }),
20395 ),
20396 ],
20397 "rust-workers".to_string(),
20398 DEFAULT_CODEC.to_string(),
20399 None,
20400 )
20401 .expect_err("new durable commands must remain strictly ordered");
20402 let Error::NonDeterministicReplay(failure) = descending else {
20403 panic!("expected typed replay failure");
20404 };
20405 assert_eq!(failure.reason, "durable_command_sequence_mismatch");
20406 assert_eq!(failure.sequence, Some(2));
20407 assert_eq!(
20408 failure.expected.as_deref(),
20409 Some("workflow sequence greater than 3")
20410 );
20411 assert_eq!(failure.actual.as_deref(), Some("2"));
20412 }
20413
20414 #[test]
20415 fn workflow_task_replay_completes_after_signals_create_sequence_gaps() {
20416 fn worker() -> Worker {
20417 let client = Client::new("http://127.0.0.1:8080").expect("client");
20418 let mut worker = Worker::new(client, "rust-workers");
20419 worker.register_workflow("rust.finish-after-gaps", |ctx, _input| async move {
20420 ctx.wait_signal("finish").await?;
20421 let marker: String =
20422 ctx.side_effect(|| panic!("recorded side effect must not run"))?;
20423 assert_eq!(marker, "after-finish");
20424 Ok(json!("finished"))
20425 });
20426 worker
20427 }
20428
20429 let marker = encode_value_envelope(&json!("after-finish"), DEFAULT_CODEC)
20430 .expect("side-effect result");
20431 let task = workflow_task(
20432 "rust.finish-after-gaps",
20433 vec![
20434 history_event(
20435 "SignalWaitOpened",
20436 json!({"sequence": 1, "signal_name": "finish"}),
20437 ),
20438 history_event(
20439 "SignalReceived",
20440 json!({
20441 "signal_id": "increment-3",
20442 "signal_name": "increment",
20443 "workflow_sequence": 2,
20444 "payload_codec": DEFAULT_CODEC,
20445 "arguments": fixture_envelope(json!([3])),
20446 }),
20447 ),
20448 history_event(
20449 "SignalReceived",
20450 json!({
20451 "signal_id": "increment-5",
20452 "signal_name": "increment",
20453 "workflow_sequence": 3,
20454 "payload_codec": DEFAULT_CODEC,
20455 "arguments": fixture_envelope(json!([5])),
20456 }),
20457 ),
20458 history_event(
20459 "SignalReceived",
20460 json!({
20461 "signal_id": "finish",
20462 "signal_name": "finish",
20463 "workflow_sequence": 4,
20464 "payload_codec": DEFAULT_CODEC,
20465 "arguments": fixture_envelope(json!([])),
20466 }),
20467 ),
20468 history_event(
20469 "SignalApplied",
20470 json!({
20471 "sequence": 1,
20472 "signal_id": "finish",
20473 "signal_name": "finish",
20474 "payload_codec": DEFAULT_CODEC,
20475 "value": fixture_envelope(json!([])),
20476 }),
20477 ),
20478 history_event(
20479 "SideEffectRecorded",
20480 json!({"sequence": 5, "result": marker}),
20481 ),
20482 ],
20483 DEFAULT_CODEC,
20484 );
20485
20486 for _original_or_cold_worker in 0..2 {
20487 let commands = worker()
20488 .execute_workflow_task(task.clone())
20489 .expect("signal gaps preserve deterministic replay");
20490 assert_eq!(commands.len(), 1, "replay emits only terminal completion");
20491 assert_eq!(commands[0]["type"], "complete_workflow");
20492 assert_eq!(
20493 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("workflow output"),
20494 json!("finished")
20495 );
20496 }
20497 }
20498
20499 #[test]
20500 fn workflow_sleep_rejects_unrepresentable_rounded_duration() {
20501 let ctx = workflow_context(Vec::new());
20502 let mut sleep = Box::pin(ctx.start_timer(Duration::new(u64::MAX, 1)));
20503 let mut task_context = TaskContext::from_waker(noop_waker_ref());
20504 assert!(matches!(
20505 sleep.as_mut().poll(&mut task_context),
20506 Poll::Ready(Err(Error::TimerDurationOverflow))
20507 ));
20508 assert!(ctx.take_commands().expect("commands").is_empty());
20509 }
20510
20511 #[test]
20512 fn workflow_memo_update_emits_canonical_command_and_replays_once() {
20513 let entries = AvroValue::Map(BTreeMap::from([
20514 ("text".to_string(), AvroValue::String("same".to_string())),
20515 (
20516 "nested".to_string(),
20517 AvroValue::Map(BTreeMap::from([
20518 ("beta".to_string(), AvroValue::Long(2)),
20519 ("alpha".to_string(), AvroValue::Long(1)),
20520 ])),
20521 ),
20522 ("long".to_string(), AvroValue::Long(7)),
20523 ("double".to_string(), AvroValue::Double(7.0)),
20524 ("binary".to_string(), AvroValue::Bytes(b"same".to_vec())),
20525 ]));
20526 let ctx = workflow_context(Vec::new());
20527 ctx.upsert_memo(entries.clone()).expect("valid memo update");
20528 let commands = ctx.take_commands().expect("commands");
20529
20530 assert_eq!(commands.len(), 1);
20531 assert_eq!(commands[0]["type"], "upsert_memo");
20532 let server_entries = json!({
20533 "codec": "avro",
20534 "blob": "wwHioz3/VYAiNw4KDGJpbmFyeQgIc2FtZQxkb3VibGUGAAAAAAAAHEAIbG9uZwQODG5lc3RlZA4ECmFscGhhBAIIYmV0YQQEAAh0ZXh0CghzYW1lAA==",
20535 });
20536 assert_eq!(
20537 commands[0]["entries"]
20538 .as_object()
20539 .expect("entries envelope")
20540 .keys()
20541 .collect::<Vec<_>>(),
20542 vec!["blob", "codec"]
20543 );
20544 assert_eq!(commands[0]["entries"], server_entries);
20545 let wire_entries =
20546 decode_wire_avro_value(&commands[0]["entries"], DEFAULT_CODEC).expect("memo entries");
20547 assert_eq!(wire_entries, entries);
20548
20549 let history = vec![history_event(
20550 "MemoUpserted",
20551 json!({
20552 "sequence": 1,
20553 "entries": server_entries.clone(),
20554 "merged": server_entries,
20555 }),
20556 )];
20557 let replay = workflow_context(history.clone());
20558 replay
20559 .upsert_memo(entries.clone())
20560 .expect("matching replay identity");
20561 assert!(replay.take_commands().expect("replay commands").is_empty());
20562
20563 let changed_types = AvroValue::Map(BTreeMap::from([
20564 ("text".to_string(), AvroValue::Bytes(b"same".to_vec())),
20565 (
20566 "nested".to_string(),
20567 AvroValue::Map(BTreeMap::from([
20568 ("alpha".to_string(), AvroValue::Long(1)),
20569 ("beta".to_string(), AvroValue::Long(2)),
20570 ])),
20571 ),
20572 ("long".to_string(), AvroValue::Double(7.0)),
20573 ("double".to_string(), AvroValue::Long(7)),
20574 ("binary".to_string(), AvroValue::String("same".to_string())),
20575 ]));
20576 let error = workflow_context(history)
20577 .upsert_memo(changed_types)
20578 .expect_err("memo replay identity must preserve Avro value types");
20579 assert!(matches!(
20580 error,
20581 Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
20582 ));
20583 }
20584
20585 #[test]
20586 fn workflow_memo_update_rejects_changed_replay_identity_and_invalid_keys() {
20587 let original = encode_value_envelope(&json!({"stage": "original"}), DEFAULT_CODEC)
20588 .expect("memo envelope");
20589 let replay = workflow_context(vec![history_event(
20590 "MemoUpserted",
20591 json!({
20592 "sequence": 1,
20593 "entries": original.clone(),
20594 "merged": original
20595 }),
20596 )]);
20597 let error = replay
20598 .upsert_memo(json!({"stage": "changed"}))
20599 .expect_err("changed memo update must fail replay");
20600 assert!(matches!(
20601 error,
20602 Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
20603 ));
20604
20605 let invalid = workflow_context(Vec::new())
20606 .upsert_memo(
20607 json!({"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx": true}),
20608 )
20609 .expect_err("oversized key");
20610 assert!(matches!(invalid, Error::InvalidMemoUpdate(_)));
20611 }
20612
20613 #[test]
20614 fn workflow_memo_replay_distinguishes_signed_zero_identity() {
20615 let negative_zero = AvroValue::Map(BTreeMap::from([(
20616 "reading".to_string(),
20617 AvroValue::Double(-0.0),
20618 )]));
20619 let negative_zero_envelope =
20620 encode_typed_envelope(&negative_zero, DEFAULT_CODEC).expect("negative zero envelope");
20621 let history = vec![history_event(
20622 "MemoUpserted",
20623 json!({
20624 "sequence": 1,
20625 "entries": negative_zero_envelope.clone(),
20626 "merged": negative_zero_envelope,
20627 }),
20628 )];
20629
20630 workflow_context(history.clone())
20631 .upsert_memo(negative_zero)
20632 .expect("matching negative-zero history identity");
20633
20634 let error = workflow_context(history)
20635 .upsert_memo(AvroValue::Map(BTreeMap::from([(
20636 "reading".to_string(),
20637 AvroValue::Double(0.0),
20638 )])))
20639 .expect_err("positive zero must not consume negative-zero memo history");
20640 assert!(matches!(
20641 error,
20642 Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
20643 ));
20644 }
20645
20646 #[test]
20647 fn workflow_memo_capability_requires_flag_and_command_advertisement() {
20648 let supported = json!({
20649 "workflow_memo_updates": {"supported": true, "minimum_protocol_version": "1.14"},
20650 "supported_workflow_task_commands": ["complete_workflow", "upsert_memo"]
20651 });
20652 assert!(runtime_supports_workflow_memo_updates(Some(&supported)));
20653 assert!(!runtime_supports_workflow_memo_updates(Some(&json!({
20654 "workflow_memo_updates": {"supported": false},
20655 "supported_workflow_task_commands": ["upsert_memo"]
20656 }))));
20657 assert!(commands_use_workflow_memo_updates(&[json!({
20658 "type": "upsert_memo",
20659 "entries": {"stage": "processing"}
20660 })]));
20661 }
20662
20663 #[test]
20664 fn workflow_task_replay_completes_without_rescheduling_recorded_commands() {
20665 let client = Client::new("http://127.0.0.1:8080").expect("client");
20666 let mut worker = Worker::new(client, "rust-workers");
20667 worker.register_workflow("rust.timer", |ctx, _input| async move {
20668 ctx.sleep(Duration::from_secs(5)).await?;
20669 ctx.activity("after-timer", json!([])).await
20670 });
20671
20672 let task = |history_events| WorkflowTask {
20673 task_id: "wft-rust-timer-1".to_string(),
20674 workflow_command_id: None,
20675 workflow_id: Some("wf-rust-timer".to_string()),
20676 run_id: Some("run-rust-timer".to_string()),
20677 workflow_type: "rust.timer".to_string(),
20678 cancel_requested: false,
20679 payload_codec: DEFAULT_CODEC.to_string(),
20680 arguments: Some(
20681 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
20682 ),
20683 history_events,
20684 total_history_events: None,
20685 history_size_bytes: None,
20686 continue_as_new_recommended: None,
20687 history_budget_pressure: None,
20688 next_history_page_token: None,
20689 workflow_task_attempt: 1,
20690 workflow_signal_id: None,
20691 signal_name: None,
20692 signal_arguments: None,
20693 workflow_update_id: None,
20694 update_name: None,
20695 lease_owner: Some("rust-worker".to_string()),
20696 };
20697
20698 let initial = worker
20699 .execute_workflow_task(task(Vec::new()))
20700 .expect("initial timer task");
20701 assert_eq!(
20702 initial,
20703 vec![json!({"type": "start_timer", "delay_seconds": 5})]
20704 );
20705
20706 let activity_result =
20707 encode_value_envelope(&json!("done"), DEFAULT_CODEC).expect("activity result");
20708 let replayed = worker
20709 .execute_workflow_task(task(vec![
20710 history_event(
20711 "TimerScheduled",
20712 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20713 ),
20714 history_event(
20715 "TimerFired",
20716 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20717 ),
20718 history_event(
20719 "ActivityCompleted",
20720 json!({
20721 "sequence": 2,
20722 "activity_type": "after-timer",
20723 "payload_codec": DEFAULT_CODEC,
20724 "result": activity_result,
20725 }),
20726 ),
20727 ]))
20728 .expect("replayed workflow task");
20729 assert_eq!(replayed.len(), 1);
20730 assert_eq!(replayed[0]["type"], "complete_workflow");
20731 assert_eq!(
20732 decode_wire_value(&replayed[0]["result"], DEFAULT_CODEC).expect("result"),
20733 json!("done")
20734 );
20735 }
20736
20737 #[test]
20738 fn workflow_continue_as_new_emits_arguments_type_and_queue_once() {
20739 let client = Client::new("http://127.0.0.1:8080").expect("client");
20740 let mut worker = Worker::new(client, "rust-workers");
20741 worker.register_workflow("rust.continue", |ctx, _input| async move {
20742 ctx.continue_as_new_with_options(
20743 ContinueAsNewOptions::new()
20744 .workflow_type("rust.next")
20745 .task_queue("next-workers"),
20746 json!([2, {"cursor": "next"}]),
20747 )
20748 });
20749
20750 let commands = worker
20751 .execute_workflow_task(workflow_task("rust.continue", Vec::new(), DEFAULT_CODEC))
20752 .expect("continue-as-new command");
20753
20754 assert_eq!(commands.len(), 1);
20755 assert_eq!(commands[0]["type"], "continue_as_new");
20756 assert_eq!(commands[0]["workflow_type"], "rust.next");
20757 assert_eq!(commands[0]["queue"], "next-workers");
20758 assert_eq!(
20759 decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC)
20760 .expect("continue-as-new arguments"),
20761 json!([2, {"cursor": "next"}])
20762 );
20763 }
20764
20765 #[test]
20766 fn continue_as_new_preserves_typed_arguments() {
20767 let client = Client::new("http://127.0.0.1:8080").expect("client");
20768 let mut worker = Worker::new(client, "rust-workers");
20769 worker.register_workflow_avro_value("rust.typed-continue", |ctx, _input| async move {
20770 ctx.continue_as_new(AvroValue::Array(vec![typed_fidelity_probe()]))?;
20771 unreachable!("continue-as-new returns a control-flow error")
20772 });
20773
20774 let commands = worker
20775 .execute_workflow_task(workflow_task(
20776 "rust.typed-continue",
20777 Vec::new(),
20778 DEFAULT_CODEC,
20779 ))
20780 .expect("typed continue-as-new command");
20781
20782 assert_eq!(commands[0]["type"], "continue_as_new");
20783 assert_eq!(
20784 decode_wire_avro_value(&commands[0]["arguments"], DEFAULT_CODEC)
20785 .expect("typed continue arguments"),
20786 AvroValue::Array(vec![typed_fidelity_probe()])
20787 );
20788 }
20789
20790 #[test]
20791 fn recorded_continue_as_new_is_consumed_without_duplicate_successor_command() {
20792 let client = Client::new("http://127.0.0.1:8080").expect("client");
20793 let mut worker = Worker::new(client, "rust-workers");
20794 worker.register_workflow("rust.continue", |ctx, _input| async move {
20795 ctx.continue_as_new(json!([2]))
20796 });
20797 let task = workflow_task(
20798 "rust.continue",
20799 vec![history_event(
20800 "WorkflowContinuedAsNew",
20801 json!({"sequence": 1, "continued_to_run_id": "run-next"}),
20802 )],
20803 DEFAULT_CODEC,
20804 );
20805
20806 for _worker_restart_or_redelivery in 0..2 {
20807 let commands = worker
20808 .execute_workflow_task(task.clone())
20809 .expect("recorded transition replays");
20810 assert!(
20811 commands.is_empty(),
20812 "replay must not emit another successor"
20813 );
20814 }
20815 }
20816
20817 #[test]
20818 fn continue_as_new_rejects_invalid_overrides_before_emitting_a_command() {
20819 let ctx = workflow_context(Vec::new());
20820 let error = ctx
20821 .continue_as_new_with_options(ContinueAsNewOptions::new().task_queue(" "), json!([1]))
20822 .expect_err("blank queue must be rejected");
20823
20824 let Error::InvalidContinueAsNewOptions(error) = error else {
20825 panic!("expected typed continue-as-new validation error");
20826 };
20827 assert_eq!(error.field, "task_queue");
20828 assert!(ctx.take_commands().expect("commands").is_empty());
20829 }
20830
20831 #[test]
20832 fn workflow_context_exposes_server_history_budget() {
20833 let client = Client::new("http://127.0.0.1:8080").expect("client");
20834 let mut worker = Worker::new(client, "rust-workers");
20835 worker.register_workflow("rust.history-budget", |ctx, _input| async move {
20836 let budget = ctx.history_budget()?;
20837 Ok(json!({
20838 "events": budget.event_count,
20839 "bytes": budget.size_bytes,
20840 "recommended": budget.continue_as_new_recommended,
20841 "pressure": budget.pressure,
20842 }))
20843 });
20844 let task: WorkflowTask = serde_json::from_value(json!({
20845 "task_id": "task-history-budget",
20846 "workflow_type": "rust.history-budget",
20847 "payload_codec": DEFAULT_CODEC,
20848 "history_events": [],
20849 "total_history_events": 480,
20850 "history_size_bytes": 1_048_576,
20851 "continue_as_new_recommended": true,
20852 "history_budget_pressure": "continue_as_new_recommended",
20853 }))
20854 .expect("published workflow task");
20855
20856 let commands = worker
20857 .execute_workflow_task(task)
20858 .expect("history-budget workflow");
20859 let result = decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("result");
20860 assert_eq!(result["events"], 480);
20861 assert_eq!(result["bytes"], 1_048_576);
20862 assert_eq!(result["recommended"], true);
20863 assert_eq!(result["pressure"], "continue_as_new_recommended");
20864 }
20865
20866 #[test]
20867 fn uncaught_workflow_handler_error_emits_terminal_failure_command() {
20868 let client = Client::new("http://127.0.0.1:8080").expect("client");
20869 let mut worker = Worker::new(client, "rust-workers");
20870 worker.register_workflow("rust.failing", |_ctx, _input| async move {
20871 Err(Error::Codec("rust_conformance_failure".to_string()))
20872 });
20873 let task = WorkflowTask {
20874 task_id: "wft-rust-failing-1".to_string(),
20875 workflow_command_id: None,
20876 workflow_id: Some("wf-rust-failing".to_string()),
20877 run_id: Some("run-rust-failing".to_string()),
20878 workflow_type: "rust.failing".to_string(),
20879 cancel_requested: false,
20880 payload_codec: DEFAULT_CODEC.to_string(),
20881 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
20882 history_events: Vec::new(),
20883 total_history_events: Some(0),
20884 history_size_bytes: None,
20885 continue_as_new_recommended: None,
20886 history_budget_pressure: None,
20887 next_history_page_token: None,
20888 workflow_task_attempt: 1,
20889 workflow_signal_id: None,
20890 signal_name: None,
20891 signal_arguments: None,
20892 workflow_update_id: None,
20893 update_name: None,
20894 lease_owner: Some("rust-worker".to_string()),
20895 };
20896
20897 let commands = worker
20898 .execute_workflow_task(task)
20899 .expect("handler failure becomes a workflow command");
20900
20901 assert_eq!(commands.len(), 1);
20902 assert_eq!(commands[0]["type"], "fail_workflow");
20903 assert_eq!(commands[0]["exception_type"], "RustWorkflowError");
20904 assert_eq!(commands[0]["exception_class"], "durable_workflow::Error");
20905 assert_eq!(commands[0]["non_retryable"], false);
20906 assert_eq!(
20907 commands[0]["message"],
20908 "codec error: rust_conformance_failure"
20909 );
20910 assert_eq!(
20911 commands[0]["exception"]["message"],
20912 "codec error: rust_conformance_failure"
20913 );
20914 }
20915
20916 #[test]
20917 fn ordinary_handler_error_preserves_commands_queued_in_the_same_decision() {
20918 let client = Client::new("http://127.0.0.1:8080").expect("client");
20919 let mut worker = Worker::new(client, "rust-workers");
20920 worker.register_workflow("rust.failing-after-side-effect", |ctx, _input| async move {
20921 let _: String = ctx.side_effect(|| "captured".to_string())?;
20922 Err(Error::WorkerLoop("application failure".to_string()))
20923 });
20924
20925 let commands = worker
20926 .execute_workflow_task(workflow_task(
20927 "rust.failing-after-side-effect",
20928 Vec::new(),
20929 DEFAULT_CODEC,
20930 ))
20931 .expect("ordinary failure remains a workflow decision");
20932
20933 assert_eq!(commands.len(), 2);
20934 assert_eq!(commands[0]["type"], "record_side_effect");
20935 assert_eq!(commands[1]["type"], "fail_workflow");
20936 }
20937
20938 #[test]
20939 fn handler_error_cannot_hide_an_unconsumed_committed_side_effect() {
20940 let client = Client::new("http://127.0.0.1:8080").expect("client");
20941 let mut worker = Worker::new(client, "rust-workers");
20942 worker.register_workflow("rust.removed-side-effect", |_ctx, _input| async move {
20943 Err(Error::WorkerLoop("application failure".to_string()))
20944 });
20945 let result =
20946 encode_value_envelope(&json!("committed"), DEFAULT_CODEC).expect("side-effect result");
20947
20948 let error = worker
20949 .execute_workflow_task(workflow_task(
20950 "rust.removed-side-effect",
20951 vec![history_event(
20952 "SideEffectRecorded",
20953 json!({"sequence": 1, "result": result}),
20954 )],
20955 DEFAULT_CODEC,
20956 ))
20957 .expect_err("removed committed history must not become fail_workflow");
20958
20959 let Error::NonDeterministicReplay(failure) = error else {
20960 panic!("expected typed replay failure");
20961 };
20962 assert_eq!(failure.reason, "recorded_commands_unconsumed");
20963 assert_eq!(failure.sequence, Some(1));
20964 assert_eq!(failure.expected.as_deref(), Some("side effect"));
20965 }
20966
20967 #[test]
20968 fn replay_error_discards_side_effect_queued_before_incompatible_marker_check() {
20969 let client = Client::new("http://127.0.0.1:8080").expect("client");
20970 let mut worker = Worker::new(client, "rust-workers");
20971 worker.register_workflow(
20972 "rust.side-effect-before-marker-error",
20973 |ctx, _input| async move {
20974 assert_eq!(ctx.get_version("restart-safe", 1, 1)?, 1);
20975 let _: String = ctx.side_effect(|| "must-not-commit".to_string())?;
20976 ctx.get_version("restart-safe", 2, 2)?;
20977 Ok(Value::Null)
20978 },
20979 );
20980
20981 let error = worker
20982 .execute_workflow_task(workflow_task(
20983 "rust.side-effect-before-marker-error",
20984 vec![history_event(
20985 "VersionMarkerRecorded",
20986 json!({
20987 "sequence": 1,
20988 "change_id": "restart-safe",
20989 "version": 1,
20990 "min_supported": 1,
20991 "max_supported": 1,
20992 }),
20993 )],
20994 DEFAULT_CODEC,
20995 ))
20996 .expect_err("replay error must return no queued workflow commands");
20997
20998 let Error::NonDeterministicReplay(failure) = error else {
20999 panic!("expected typed replay failure");
21000 };
21001 assert_eq!(failure.reason, "version_marker_incompatible_range");
21002 assert_eq!(failure.sequence, Some(1));
21003 }
21004
21005 #[test]
21006 fn workflow_task_replay_keeps_recorded_unfired_timer_pending_without_rescheduling() {
21007 let client = Client::new("http://127.0.0.1:8080").expect("client");
21008 let mut worker = Worker::new(client, "rust-workers");
21009 worker.register_workflow("rust.timer.pending", |ctx, _input| async move {
21010 ctx.sleep(Duration::from_secs(5)).await?;
21011 Ok(json!({"status": "timer fired"}))
21012 });
21013
21014 let task = WorkflowTask {
21015 task_id: "wft-rust-timer-pending".to_string(),
21016 workflow_command_id: None,
21017 workflow_id: Some("wf-rust-timer".to_string()),
21018 run_id: Some("run-rust-timer".to_string()),
21019 workflow_type: "rust.timer.pending".to_string(),
21020 cancel_requested: false,
21021 payload_codec: DEFAULT_CODEC.to_string(),
21022 arguments: Some(
21023 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
21024 ),
21025 history_events: vec![history_event(
21026 "TimerScheduled",
21027 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
21028 )],
21029 total_history_events: Some(1),
21030 history_size_bytes: None,
21031 continue_as_new_recommended: None,
21032 history_budget_pressure: None,
21033 next_history_page_token: None,
21034 workflow_task_attempt: 1,
21035 workflow_signal_id: None,
21036 signal_name: None,
21037 signal_arguments: None,
21038 workflow_update_id: None,
21039 update_name: None,
21040 lease_owner: Some("rust-worker".to_string()),
21041 };
21042
21043 for _redelivery_or_restart in 0..2 {
21044 let commands = worker
21045 .execute_workflow_task(task.clone())
21046 .expect("recorded timer remains pending");
21047 assert!(
21048 commands.is_empty(),
21049 "recorded timer must not be rescheduled"
21050 );
21051 }
21052 }
21053
21054 #[test]
21055 fn workflow_task_rejects_recorded_command_removed_from_workflow_code() {
21056 let client = Client::new("http://127.0.0.1:8080").expect("client");
21057 let mut worker = Worker::new(client, "rust-workers");
21058 worker.register_workflow("rust.timer.removed", |_ctx, _input| async move {
21059 Ok(json!({"status": "completed"}))
21060 });
21061 let task = WorkflowTask {
21062 task_id: "wft-rust-timer-removed".to_string(),
21063 workflow_command_id: None,
21064 workflow_id: Some("wf-rust-timer".to_string()),
21065 run_id: Some("run-rust-timer".to_string()),
21066 workflow_type: "rust.timer.removed".to_string(),
21067 cancel_requested: false,
21068 payload_codec: DEFAULT_CODEC.to_string(),
21069 arguments: Some(
21070 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
21071 ),
21072 history_events: vec![
21073 history_event(
21074 "TimerScheduled",
21075 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
21076 ),
21077 history_event(
21078 "TimerFired",
21079 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
21080 ),
21081 ],
21082 total_history_events: Some(2),
21083 history_size_bytes: None,
21084 continue_as_new_recommended: None,
21085 history_budget_pressure: None,
21086 next_history_page_token: None,
21087 workflow_task_attempt: 1,
21088 workflow_signal_id: None,
21089 signal_name: None,
21090 signal_arguments: None,
21091 workflow_update_id: None,
21092 update_name: None,
21093 lease_owner: Some("rust-worker".to_string()),
21094 };
21095
21096 let Error::NonDeterministicReplay(failure) = worker
21097 .execute_workflow_task(task)
21098 .expect_err("removed timer must fail replay")
21099 else {
21100 panic!("expected typed replay failure");
21101 };
21102 assert_eq!(failure.reason, "recorded_commands_unconsumed");
21103 assert_eq!(failure.sequence, Some(1));
21104 }
21105
21106 #[test]
21107 fn workflow_context_emits_explicit_child_workflow_contract() {
21108 let ctx = WorkflowContext {
21109 state: Arc::new(Mutex::new(
21110 WorkflowState::new_with_identity(
21111 Vec::new(),
21112 Some("wf-parent".to_string()),
21113 Some("run-parent".to_string()),
21114 "parent-workers".to_string(),
21115 DEFAULT_CODEC.to_string(),
21116 None,
21117 )
21118 .expect("workflow state"),
21119 )),
21120 };
21121 let options = ChildWorkflowOptions::new("python-workers")
21122 .parent_close_policy(ParentClosePolicy::RequestCancel)
21123 .retry_policy(ChildWorkflowRetryPolicy {
21124 max_attempts: Some(3),
21125 backoff_seconds: vec![1, 5],
21126 non_retryable_error_types: vec!["ValidationError".to_string()],
21127 })
21128 .execution_timeout_seconds(600)
21129 .run_timeout_seconds(120);
21130 let mut call = Box::pin(ctx.start_child_workflow(
21131 "python.fulfil-order",
21132 options,
21133 json!([{"order_id": "order-42"}]),
21134 ));
21135 let mut task_context = TaskContext::from_waker(noop_waker_ref());
21136
21137 assert!(matches!(
21138 call.as_mut().poll(&mut task_context),
21139 Poll::Pending
21140 ));
21141 let commands = ctx.take_commands().expect("commands");
21142 assert_eq!(commands.len(), 1);
21143 let command = &commands[0];
21144 assert_eq!(command["type"], "start_child_workflow");
21145 assert_eq!(command["workflow_type"], "python.fulfil-order");
21146 assert_eq!(command["queue"], "python-workers");
21147 assert_eq!(command["parent_close_policy"], "request_cancel");
21148 assert_eq!(command["retry_policy"]["max_attempts"], 3);
21149 assert_eq!(command["execution_timeout_seconds"], 600);
21150 assert_eq!(command["run_timeout_seconds"], 120);
21151 assert_eq!(
21152 decode_wire_value(&command["arguments"], DEFAULT_CODEC).expect("child args"),
21153 json!([{"order_id": "order-42"}])
21154 );
21155 }
21156
21157 fn child_parent_worker() -> Worker {
21158 let client = Client::new("http://127.0.0.1:8080").expect("client");
21159 let mut worker = Worker::new(client, "rust-parent-workers");
21160 worker.register_workflow("rust.parent", |ctx, _input| async move {
21161 let child = ctx
21162 .start_child_workflow(
21163 "python.child",
21164 ChildWorkflowOptions::new("python-child-workers")
21165 .parent_close_policy(ParentClosePolicy::Terminate),
21166 json!([{"codec_probe": [1, true, "rust"]}]),
21167 )
21168 .await?;
21169 Ok(json!({
21170 "parent_workflow_id": child.parent.workflow_id,
21171 "parent_run_id": child.parent.run_id,
21172 "child_workflow_id": child.child.workflow_id,
21173 "child_run_id": child.child.run_id,
21174 "child_workflow_type": child.child_workflow_type,
21175 "result": child.result,
21176 }))
21177 });
21178 worker
21179 }
21180
21181 fn child_parent_task(event_type: &str, payload: Value) -> WorkflowTask {
21182 WorkflowTask {
21183 task_id: "wft-child-parent".to_string(),
21184 workflow_command_id: None,
21185 workflow_id: Some("wf-parent".to_string()),
21186 run_id: Some("run-parent".to_string()),
21187 workflow_type: "rust.parent".to_string(),
21188 cancel_requested: false,
21189 payload_codec: DEFAULT_CODEC.to_string(),
21190 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
21191 history_events: vec![
21192 HistoryEvent {
21193 event_type: "ChildWorkflowScheduled".to_string(),
21194 payload: json!({
21195 "sequence": 1,
21196 "child_call_id": "call-child",
21197 "child_workflow_instance_id": "wf-child",
21198 "child_workflow_run_id": "run-child",
21199 "child_workflow_type": "python.child",
21200 }),
21201 raw: HashMap::new(),
21202 },
21203 HistoryEvent {
21204 event_type: event_type.to_string(),
21205 payload,
21206 raw: HashMap::new(),
21207 },
21208 ],
21209 total_history_events: Some(2),
21210 history_size_bytes: None,
21211 continue_as_new_recommended: None,
21212 history_budget_pressure: None,
21213 next_history_page_token: None,
21214 workflow_task_attempt: 1,
21215 workflow_signal_id: None,
21216 signal_name: None,
21217 signal_arguments: None,
21218 workflow_update_id: None,
21219 update_name: None,
21220 lease_owner: Some("rust-worker".to_string()),
21221 }
21222 }
21223
21224 #[test]
21225 fn committed_child_result_replays_without_starting_a_duplicate() {
21226 let worker = child_parent_worker();
21227 let task = child_parent_task(
21228 "ChildRunCompleted",
21229 json!({
21230 "sequence": 1,
21231 "child_call_id": "call-child",
21232 "child_workflow_instance_id": "wf-child",
21233 "child_workflow_run_id": "run-child",
21234 "child_workflow_type": "python.child",
21235 "payload_codec": DEFAULT_CODEC,
21236 "result": fixture_envelope(json!({"from":"python","ok":true})),
21237 }),
21238 );
21239
21240 for _restart in 0..2 {
21241 let commands = worker
21242 .execute_workflow_task(task.clone())
21243 .expect("replayed parent task");
21244 assert_eq!(commands.len(), 1);
21245 assert_eq!(commands[0]["type"], "complete_workflow");
21246 assert!(!commands
21247 .iter()
21248 .any(|command| command["type"] == "start_child_workflow"));
21249 let output =
21250 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("parent output");
21251 assert_eq!(output["parent_workflow_id"], "wf-parent");
21252 assert_eq!(output["parent_run_id"], "run-parent");
21253 assert_eq!(output["child_workflow_id"], "wf-child");
21254 assert_eq!(output["child_run_id"], "run-child");
21255 assert_eq!(output["result"], json!({"from": "python", "ok": true}));
21256 }
21257 }
21258
21259 #[test]
21260 fn typed_child_arguments_and_results_survive_replay() {
21261 let client = Client::new("http://127.0.0.1:8080").expect("client");
21262 let mut worker = Worker::new(client, "rust-parent-workers");
21263 worker.register_workflow_avro_value("rust.typed-parent", |ctx, _input| async move {
21264 let child = ctx
21265 .start_child_workflow_avro_value(
21266 "python.typed-child",
21267 ChildWorkflowOptions::new("python-workers"),
21268 AvroValue::Array(vec![typed_fidelity_probe()]),
21269 )
21270 .await?;
21271 Ok(child.result)
21272 });
21273
21274 let initial = worker
21275 .execute_workflow_task(workflow_task(
21276 "rust.typed-parent",
21277 Vec::new(),
21278 DEFAULT_CODEC,
21279 ))
21280 .expect("typed child start");
21281 assert_eq!(initial[0]["type"], "start_child_workflow");
21282 assert_eq!(
21283 decode_wire_avro_value(&initial[0]["arguments"], DEFAULT_CODEC)
21284 .expect("typed child arguments"),
21285 AvroValue::Array(vec![typed_fidelity_probe()])
21286 );
21287
21288 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
21289 .expect("typed child result");
21290 let task = workflow_task(
21291 "rust.typed-parent",
21292 vec![
21293 history_event(
21294 "ChildWorkflowScheduled",
21295 json!({
21296 "sequence": 1,
21297 "child_call_id": "call-typed",
21298 "child_workflow_instance_id": "wf-child",
21299 "child_workflow_run_id": "run-child",
21300 "child_workflow_type": "python.typed-child",
21301 }),
21302 ),
21303 history_event(
21304 "ChildRunCompleted",
21305 json!({
21306 "sequence": 1,
21307 "child_call_id": "call-typed",
21308 "child_workflow_instance_id": "wf-child",
21309 "child_workflow_run_id": "run-child",
21310 "child_workflow_type": "python.typed-child",
21311 "payload_codec": DEFAULT_CODEC,
21312 "result": result,
21313 }),
21314 ),
21315 ],
21316 DEFAULT_CODEC,
21317 );
21318
21319 let commands = worker
21320 .execute_workflow_task(task)
21321 .expect("typed child replay");
21322 assert_eq!(commands[0]["type"], "complete_workflow");
21323 assert_eq!(
21324 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
21325 .expect("typed parent result"),
21326 typed_fidelity_probe()
21327 );
21328 }
21329
21330 #[test]
21331 fn pending_child_replays_after_restart_without_starting_a_duplicate() {
21332 let worker = child_parent_worker();
21333 let mut task = child_parent_task("unused", Value::Null);
21334 task.history_events.truncate(1);
21335 task.total_history_events = Some(1);
21336
21337 for _redelivery_or_restart in 0..2 {
21338 let commands = worker
21339 .execute_workflow_task(task.clone())
21340 .expect("recorded child remains pending");
21341 assert!(
21342 commands.is_empty(),
21343 "recorded pending child must not be started again"
21344 );
21345 }
21346 }
21347
21348 #[test]
21349 fn child_cancellation_becomes_stable_parent_failure_command() {
21350 let worker = child_parent_worker();
21351 let task = child_parent_task(
21352 "ChildRunCancelled",
21353 json!({
21354 "sequence": 1,
21355 "child_workflow_instance_id": "wf-child",
21356 "child_workflow_run_id": "run-child",
21357 "child_workflow_type": "python.child",
21358 "failure_id": "failure-child",
21359 "failure_category": "cancelled",
21360 "message": "cancelled by parent-close policy",
21361 }),
21362 );
21363
21364 let commands = worker
21365 .execute_workflow_task(task)
21366 .expect("parent settlement");
21367 assert_eq!(commands.len(), 1);
21368 assert_eq!(commands[0]["type"], "fail_workflow");
21369 assert_eq!(commands[0]["exception_type"], "ChildWorkflowCancelled");
21370 assert_eq!(
21371 commands[0]["exception"]["properties"]["reason"],
21372 "cancelled"
21373 );
21374 assert_eq!(
21375 commands[0]["exception"]["properties"]["child_workflow_run_id"],
21376 "run-child"
21377 );
21378 }
21379
21380 #[test]
21381 fn workflow_can_handle_typed_child_failure() {
21382 let client = Client::new("http://127.0.0.1:8080").expect("client");
21383 let mut worker = Worker::new(client, "rust-parent-workers");
21384 worker.register_workflow("rust.handled-parent", |ctx, _input| async move {
21385 match ctx
21386 .start_child_workflow(
21387 "python.child",
21388 ChildWorkflowOptions::new("python-child-workers"),
21389 json!([]),
21390 )
21391 .await
21392 {
21393 Err(Error::ChildWorkflowFailed(failure)) => Ok(json!({
21394 "reason": failure.reason,
21395 "failure_id": failure.failure_id,
21396 "exception_class": failure.exception_class,
21397 "child_run_id": failure.child_workflow_run_id,
21398 })),
21399 Err(error) => Err(error),
21400 Ok(_) => Err(Error::WorkerLoop(
21401 "child unexpectedly succeeded".to_string(),
21402 )),
21403 }
21404 });
21405 let mut task = child_parent_task(
21406 "ChildRunFailed",
21407 json!({
21408 "sequence": 1,
21409 "child_workflow_instance_id": "wf-child",
21410 "child_workflow_run_id": "run-child",
21411 "child_workflow_type": "python.child",
21412 "failure_id": "failure-child",
21413 "failure_category": "child_workflow",
21414 "message": "payment rejected",
21415 "exception": {
21416 "type": "PaymentRejected",
21417 "class": "payments.PaymentRejected",
21418 "message": "payment rejected"
21419 }
21420 }),
21421 );
21422 task.workflow_type = "rust.handled-parent".to_string();
21423
21424 let commands = worker.execute_workflow_task(task).expect("handled failure");
21425 assert_eq!(commands[0]["type"], "complete_workflow");
21426 let output =
21427 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("parent output");
21428 assert_eq!(output["reason"], "child_workflow");
21429 assert_eq!(output["failure_id"], "failure-child");
21430 assert_eq!(output["exception_class"], "payments.PaymentRejected");
21431 assert_eq!(output["child_run_id"], "run-child");
21432 }
21433
21434 #[test]
21435 fn rust_hello_world_uses_signal_arguments_from_resume_payload() {
21436 let client = Client::new("http://127.0.0.1:8080").expect("client");
21437 let mut worker = Worker::new(client, "rust-workers");
21438
21439 worker.register_workflow("rust.hello_workflow", |ctx, _input| async move {
21440 let signal = ctx.wait_signal("start").await?;
21441 let name = signal
21442 .first()
21443 .and_then(|value| value.as_str())
21444 .unwrap_or("world");
21445 let greeting = ctx.activity("rust.hello_activity", json!([name])).await?;
21446 Ok(json!({
21447 "greeting": greeting,
21448 "language": "rust"
21449 }))
21450 });
21451
21452 let signal_arguments =
21453 encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC).expect("signal arguments");
21454 let task = WorkflowTask {
21455 task_id: "wft-rust-signal-1".to_string(),
21456 workflow_command_id: None,
21457 workflow_id: Some("wf-rust-hello".to_string()),
21458 run_id: Some("run-rust-hello".to_string()),
21459 workflow_type: "rust.hello_workflow".to_string(),
21460 cancel_requested: false,
21461 payload_codec: DEFAULT_CODEC.to_string(),
21462 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
21463 history_events: vec![HistoryEvent {
21464 event_type: "SignalReceived".to_string(),
21465 payload: json!({
21466 "signal_id": "sig-rust-1",
21467 "signal_name": "start"
21468 }),
21469 raw: HashMap::new(),
21470 }],
21471 total_history_events: Some(1),
21472 history_size_bytes: None,
21473 continue_as_new_recommended: None,
21474 history_budget_pressure: None,
21475 next_history_page_token: None,
21476 workflow_task_attempt: 1,
21477 workflow_signal_id: Some("sig-rust-1".to_string()),
21478 signal_name: Some("start".to_string()),
21479 signal_arguments: Some(signal_arguments),
21480 workflow_update_id: None,
21481 update_name: None,
21482 lease_owner: Some("rust-worker".to_string()),
21483 };
21484
21485 let commands = worker.execute_workflow_task(task).expect("workflow task");
21486
21487 assert_eq!(commands.len(), 1);
21488 assert_eq!(commands[0]["type"], "schedule_activity");
21489 assert_eq!(commands[0]["activity_type"], "rust.hello_activity");
21490 assert_eq!(
21491 decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC).expect("activity args"),
21492 json!(["Rust"])
21493 );
21494 }
21495
21496 #[test]
21497 fn workflow_task_appends_paginated_history_events() {
21498 let mut task = WorkflowTask {
21499 task_id: "wft-rust-pages-1".to_string(),
21500 workflow_command_id: None,
21501 workflow_id: Some("wf-rust-pages".to_string()),
21502 run_id: Some("run-rust-pages".to_string()),
21503 workflow_type: "rust.hello_workflow".to_string(),
21504 cancel_requested: false,
21505 payload_codec: DEFAULT_CODEC.to_string(),
21506 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
21507 history_events: vec![HistoryEvent {
21508 event_type: "WorkflowStarted".to_string(),
21509 payload: json!({}),
21510 raw: HashMap::new(),
21511 }],
21512 total_history_events: Some(3),
21513 history_size_bytes: None,
21514 continue_as_new_recommended: None,
21515 history_budget_pressure: None,
21516 next_history_page_token: Some("MQ==".to_string()),
21517 workflow_task_attempt: 1,
21518 workflow_signal_id: None,
21519 signal_name: None,
21520 signal_arguments: None,
21521 workflow_update_id: None,
21522 update_name: None,
21523 lease_owner: Some("rust-worker".to_string()),
21524 };
21525
21526 task.append_history_page(WorkflowTaskHistoryPage {
21527 history_events: vec![
21528 HistoryEvent {
21529 event_type: "SignalReceived".to_string(),
21530 payload: json!({
21531 "signal_id": "sig-rust-1",
21532 "signal_name": "start",
21533 "arguments": encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC)
21534 .expect("signal arguments")
21535 }),
21536 raw: HashMap::new(),
21537 },
21538 HistoryEvent {
21539 event_type: "MarkerRecorded".to_string(),
21540 payload: json!({"sequence": 3}),
21541 raw: HashMap::new(),
21542 },
21543 ],
21544 total_history_events: Some(3),
21545 next_history_page_token: None,
21546 });
21547
21548 assert_eq!(task.history_events.len(), 3);
21549 assert_eq!(task.total_history_events, Some(3));
21550 assert_eq!(task.next_history_page_token, None);
21551
21552 let signal = task
21553 .history_events
21554 .iter()
21555 .find(|event| event.event_type == "SignalReceived")
21556 .expect("signal event");
21557 assert_eq!(
21558 decode_signal_event_arguments(signal, DEFAULT_CODEC).expect("signal arguments"),
21559 vec![AvroValue::String("Rust".to_string())]
21560 );
21561 }
21562
21563 #[tokio::test]
21564 async fn query_handler_reads_ordered_cross_codec_signals_without_commands() {
21565 let client = Client::new("http://127.0.0.1:8080").expect("client");
21566 let mut worker = Worker::new(client, "rust-workers");
21567 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
21568 worker.register_query("counter", "current", |ctx, _args| async move {
21569 let mut count = 0_i64;
21570 for signal in ctx.signal_events() {
21571 let value = signal
21572 .arguments
21573 .first()
21574 .and_then(Value::as_i64)
21575 .unwrap_or_default();
21576 match signal.name.as_str() {
21577 "increment" => count += value,
21578 "set" => count = value,
21579 _ => {}
21580 }
21581 }
21582 Ok(json!(count))
21583 });
21584
21585 let task = QueryTask {
21586 query_task_id: "query-rust-counter".to_string(),
21587 query_task_attempt: 1,
21588 lease_owner: Some("rust-worker".to_string()),
21589 workflow_id: Some("counter-1".to_string()),
21590 run_id: Some("run-counter-1".to_string()),
21591 workflow_type: "counter".to_string(),
21592 query_name: "current".to_string(),
21593 payload_codec: DEFAULT_CODEC.to_string(),
21594 workflow_arguments: Some(
21595 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
21596 ),
21597 query_arguments: Some(
21598 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("query arguments"),
21599 ),
21600 history_events: vec![
21601 HistoryEvent {
21602 event_type: "SignalReceived".to_string(),
21603 payload: json!({
21604 "signal_id": "php-signal-1",
21605 "signal_name": "increment",
21606 "workflow_sequence": 1,
21607 "payload_codec": DEFAULT_CODEC,
21608 "arguments": encode_value_envelope(&json!([3]), DEFAULT_CODEC).expect("php avro signal")
21609 }),
21610 raw: HashMap::new(),
21611 },
21612 HistoryEvent {
21613 event_type: "SignalReceived".to_string(),
21614 payload: json!({
21615 "signal_id": "python-signal-2",
21616 "signal_name": "increment",
21617 "workflow_sequence": 2,
21618 "payload_codec": DEFAULT_CODEC,
21619 "arguments": encode_value_envelope(&json!([5]), DEFAULT_CODEC).expect("python avro signal")
21620 }),
21621 raw: HashMap::new(),
21622 },
21623 HistoryEvent {
21624 event_type: "SignalReceived".to_string(),
21625 payload: json!({
21626 "signal_id": "rust-signal-3",
21627 "signal_name": "set",
21628 "workflow_sequence": 3,
21629 "payload_codec": DEFAULT_CODEC,
21630 "arguments": encode_value_envelope(&json!([0]), DEFAULT_CODEC).expect("rust avro signal")
21631 }),
21632 raw: HashMap::new(),
21633 },
21634 ],
21635 history_export: None,
21636 run_status: Some("completed".to_string()),
21637 };
21638
21639 let result = worker.execute_query_task(task).await.expect("query result");
21640 assert_eq!(result.into_json().expect("query projection"), json!(0));
21641 }
21642
21643 #[tokio::test]
21644 async fn replayed_queries_read_running_completed_and_cold_restarted_instance_state() {
21645 let worker = replay_counter_worker();
21646 let running_history = json!([
21647 {
21648 "type": "ActivityCompleted",
21649 "payload": {
21650 "sequence": 1,
21651 "activity_type": "load-counter",
21652 "payload_codec": DEFAULT_CODEC,
21653 "result": fixture_envelope(json!("loaded"))
21654 }
21655 },
21656 {
21657 "type": "SignalWaitOpened",
21658 "payload": {
21659 "sequence": 3,
21660 "signal_name": "increment"
21661 }
21662 },
21663 {
21664 "type": "SignalReceived",
21665 "payload": {
21666 "signal_id": "signal-3",
21667 "signal_name": "increment",
21668 "workflow_sequence": 2,
21669 "payload_codec": DEFAULT_CODEC,
21670 "arguments": fixture_envelope(json!([3]))
21671 }
21672 },
21673 {
21674 "type": "SignalApplied",
21675 "payload": {
21676 "sequence": 3,
21677 "signal_id": "signal-3",
21678 "signal_name": "increment",
21679 "payload_codec": DEFAULT_CODEC,
21680 "value": fixture_envelope(json!([3]))
21681 }
21682 }
21683 ]);
21684
21685 let running = worker
21686 .execute_query_task(replay_counter_query(
21687 "current",
21688 running_history.clone(),
21689 "running",
21690 ))
21691 .await
21692 .expect("running replay query");
21693 assert_eq!(
21694 running.clone().into_json().expect("query projection"),
21695 json!({"loaded": "loaded", "count": 3, "finished": false})
21696 );
21697
21698 let detached = worker
21699 .execute_query_task(replay_counter_query(
21700 "detached-mutation",
21701 running_history.clone(),
21702 "running",
21703 ))
21704 .await
21705 .expect("query mutates only its detached state clone");
21706 assert_eq!(detached.into_json().expect("query projection"), json!(999));
21707 let failed = worker
21708 .execute_query_task(replay_counter_query(
21709 "failed-mutation",
21710 running_history.clone(),
21711 "running",
21712 ))
21713 .await
21714 .expect_err("failed query");
21715 assert_eq!(failed.reason, "query_rejected");
21716 let unchanged = worker
21717 .execute_query_task(replay_counter_query("current", running_history, "running"))
21718 .await
21719 .expect("later query reconstructs unchanged state");
21720 assert_eq!(unchanged, running);
21721
21722 let restarted_worker = replay_counter_worker();
21723 let empty_arguments = fixture_envelope(json!([]));
21724 let loaded_result = fixture_envelope(json!("loaded"));
21725 let signal_three = fixture_blob(json!([3]));
21726 let signal_five = fixture_blob(json!([5]));
21727 let restarted_task: QueryTask = serde_json::from_value(json!({
21728 "query_task_id": "query-after-restart",
21729 "workflow_id": "counter-1",
21730 "run_id": "run-counter-1",
21731 "workflow_type": "replay-counter",
21732 "query_name": "current",
21733 "payload_codec": DEFAULT_CODEC,
21734 "workflow_arguments": empty_arguments.clone(),
21735 "query_arguments": empty_arguments,
21736 "history_events": [],
21737 "history_export": {
21738 "payloads": {"codec": DEFAULT_CODEC},
21739 "history_events": [
21740 {
21741 "type": "ActivityCompleted",
21742 "payload": {
21743 "sequence": 1,
21744 "activity_type": "load-counter",
21745 "payload_codec": DEFAULT_CODEC,
21746 "result": null
21747 }
21748 },
21749 {
21750 "type": "SignalWaitOpened",
21751 "payload": {
21752 "sequence": 3,
21753 "signal_name": "increment"
21754 }
21755 },
21756 {
21757 "type": "SignalReceived",
21758 "payload": {
21759 "signal_id": "signal-3",
21760 "signal_name": "increment",
21761 "workflow_sequence": 2
21762 }
21763 },
21764 {
21765 "type": "SignalApplied",
21766 "payload": {
21767 "sequence": 3,
21768 "signal_id": "signal-3",
21769 "signal_name": "increment"
21770 }
21771 },
21772 {
21773 "type": "SignalWaitOpened",
21774 "payload": {
21775 "sequence": 5,
21776 "signal_name": "increment"
21777 }
21778 },
21779 {
21780 "type": "SignalReceived",
21781 "payload": {
21782 "signal_id": "signal-5",
21783 "signal_name": "increment",
21784 "workflow_sequence": 4
21785 }
21786 },
21787 {
21788 "type": "SignalApplied",
21789 "payload": {
21790 "sequence": 5,
21791 "signal_id": "signal-5",
21792 "signal_name": "increment"
21793 }
21794 }
21795 ],
21796 "activities": [{
21797 "sequence": 1,
21798 "activity_type": "load-counter",
21799 "payload_codec": DEFAULT_CODEC,
21800 "result": loaded_result
21801 }],
21802 "signals": [
21803 {
21804 "id": "signal-3",
21805 "name": "increment",
21806 "workflow_sequence": 2,
21807 "payload_codec": DEFAULT_CODEC,
21808 "arguments": signal_three
21809 },
21810 {
21811 "id": "signal-5",
21812 "name": "increment",
21813 "workflow_sequence": 4,
21814 "payload_codec": DEFAULT_CODEC,
21815 "arguments": signal_five
21816 }
21817 ]
21818 },
21819 "run_status": "completed"
21820 }))
21821 .expect("cold replay query task");
21822 let completed = restarted_worker
21823 .execute_query_task(restarted_task)
21824 .await
21825 .expect("completed cold replay query");
21826 assert_eq!(
21827 completed.into_json().expect("query projection"),
21828 json!({"loaded": "loaded", "count": 8, "finished": true})
21829 );
21830 }
21831
21832 #[tokio::test]
21833 async fn replayed_query_replay_failures_are_machine_readable() {
21834 let worker = replay_counter_worker();
21835 let task = replay_counter_query(
21836 "current",
21837 json!([{
21838 "type": "ActivityCompleted",
21839 "payload": {
21840 "sequence": 1,
21841 "payload_codec": DEFAULT_CODEC,
21842 "result": {"codec": DEFAULT_CODEC, "blob": "{"}
21843 }
21844 }]),
21845 "running",
21846 );
21847 let failure = worker
21848 .execute_query_task(task)
21849 .await
21850 .expect_err("invalid replay history payload");
21851 assert_eq!(failure.reason, "query_payload_decode_failed");
21852 assert_eq!(failure.failure_type, "QueryPayloadDecodeFailed");
21853 assert!(failure.message.contains("invalid_payload_framing"));
21854 }
21855
21856 #[tokio::test]
21857 async fn query_task_restores_compact_history_from_export() {
21858 let client = Client::new("http://127.0.0.1:8080").expect("client");
21859 let mut worker = Worker::new(client, "rust-workers");
21860 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
21861 worker.register_query("counter", "current", |ctx, _args| async move {
21862 Ok(json!(ctx.signals("increment")[0][0]))
21863 });
21864 let empty_arguments = fixture_envelope(json!([]));
21865 let exported_signal = fixture_blob(json!([9]));
21866 let task: QueryTask = serde_json::from_value(json!({
21867 "query_task_id": "query-export",
21868 "workflow_type": "counter",
21869 "query_name": "current",
21870 "payload_codec": DEFAULT_CODEC,
21871 "workflow_arguments": empty_arguments.clone(),
21872 "query_arguments": empty_arguments,
21873 "history_events": [],
21874 "history_export": {
21875 "payloads": {"codec": DEFAULT_CODEC},
21876 "history_events": [{
21877 "type": "SignalReceived",
21878 "payload": {"signal_id": "signal-export", "signal_name": "increment"}
21879 }],
21880 "signals": [{
21881 "id": "signal-export",
21882 "name": "increment",
21883 "status": "applied",
21884 "workflow_sequence": 1,
21885 "payload_codec": DEFAULT_CODEC,
21886 "arguments": exported_signal
21887 }]
21888 }
21889 }))
21890 .expect("query task");
21891
21892 let result = worker.execute_query_task(task).await.expect("query result");
21893 assert_eq!(result.into_json().expect("query projection"), json!(9));
21894 }
21895
21896 #[tokio::test]
21897 async fn query_task_failures_have_stable_reasons() {
21898 let client = Client::new("http://127.0.0.1:8080").expect("client");
21899 let mut worker = Worker::new(client, "rust-workers");
21900 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
21901 worker.register_query(
21902 "counter",
21903 "current",
21904 |_ctx, _args| async move { Ok(json!(0)) },
21905 );
21906
21907 let base_task = QueryTask {
21908 query_task_id: "query-errors".to_string(),
21909 query_task_attempt: 1,
21910 lease_owner: None,
21911 workflow_id: Some("counter-errors".to_string()),
21912 run_id: Some("run-errors".to_string()),
21913 workflow_type: "counter".to_string(),
21914 query_name: "missing".to_string(),
21915 payload_codec: DEFAULT_CODEC.to_string(),
21916 workflow_arguments: Some(fixture_envelope(json!([]))),
21917 query_arguments: Some(fixture_envelope(json!([]))),
21918 history_events: Vec::new(),
21919 history_export: None,
21920 run_status: Some("running".to_string()),
21921 };
21922
21923 let unknown = worker
21924 .execute_query_task(base_task.clone())
21925 .await
21926 .expect_err("unknown query");
21927 assert_eq!(unknown.reason, "rejected_unknown_query");
21928
21929 let mut malformed = base_task;
21930 malformed.query_name = "current".to_string();
21931 malformed.query_arguments = Some(json!({"codec": DEFAULT_CODEC, "blob": "{"}));
21932 let malformed = worker
21933 .execute_query_task(malformed)
21934 .await
21935 .expect_err("malformed payload");
21936 assert_eq!(malformed.reason, "query_payload_decode_failed");
21937
21938 let client = Client::new("http://127.0.0.1:8080").expect("client");
21939 let mut unavailable_worker = Worker::new(client, "rust-workers");
21940 unavailable_worker
21941 .register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
21942 let empty_arguments = fixture_envelope(json!([]));
21943 let unavailable_task: QueryTask = serde_json::from_value(json!({
21944 "query_task_id": "query-unavailable",
21945 "workflow_type": "counter",
21946 "query_name": "current",
21947 "payload_codec": DEFAULT_CODEC,
21948 "workflow_arguments": empty_arguments.clone(),
21949 "query_arguments": empty_arguments
21950 }))
21951 .expect("query task");
21952 let unavailable = unavailable_worker
21953 .execute_query_task(unavailable_task)
21954 .await
21955 .expect_err("query handler unavailable");
21956 assert_eq!(unavailable.reason, "query_handler_unavailable");
21957 }
21958
21959 #[tokio::test]
21960 async fn client_query_decodes_result_and_typed_failure() {
21961 let server = MockWorkerServer::start();
21962 let client = Client::builder(server.base_url())
21963 .timeout(Duration::from_secs(2))
21964 .build()
21965 .expect("client");
21966
21967 let result = client
21968 .query_workflow("counter-1", "current", json!([]))
21969 .await
21970 .expect("query result");
21971 assert_eq!(result, json!({"count": 8}));
21972
21973 let error = client
21974 .query_workflow("counter-1", "missing", json!([]))
21975 .await
21976 .expect_err("unknown query");
21977 let Error::QueryFailed(failure) = error else {
21978 panic!("expected typed query failure");
21979 };
21980 assert_eq!(failure.status, 404);
21981 assert_eq!(failure.reason, "rejected_unknown_query");
21982 }
21983
21984 #[tokio::test]
21985 async fn public_client_surfaces_send_and_receive_lossless_avro_values() {
21986 let server = MockWorkerServer::start();
21987 let client = Client::builder(server.base_url())
21988 .timeout(Duration::from_secs(2))
21989 .build()
21990 .expect("client");
21991 let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
21992
21993 client
21994 .start_workflow(
21995 "typed.echo",
21996 "rust-workers",
21997 "typed-start",
21998 arguments.clone(),
21999 )
22000 .await
22001 .expect("typed workflow start");
22002 assert_eq!(
22003 decode_wire_avro_value(
22004 &server.request_body("/api/workflows")["input"],
22005 DEFAULT_CODEC,
22006 )
22007 .expect("typed start input"),
22008 arguments
22009 );
22010
22011 client
22012 .signal_workflow("typed-1", "changed", arguments.clone())
22013 .await
22014 .expect("typed signal");
22015 assert_eq!(
22016 decode_wire_avro_value(
22017 &server.request_body("/api/workflows/typed-1/signal/changed")["input"],
22018 DEFAULT_CODEC,
22019 )
22020 .expect("typed signal input"),
22021 arguments
22022 );
22023
22024 assert_eq!(
22025 client
22026 .query_workflow_avro_value("typed-1", "inspect", arguments.clone())
22027 .await
22028 .expect("typed query"),
22029 typed_fidelity_probe()
22030 );
22031 assert_eq!(
22032 decode_wire_avro_value(
22033 &server.request_body("/api/workflows/typed-1/query/inspect")["input"],
22034 DEFAULT_CODEC,
22035 )
22036 .expect("typed query input"),
22037 arguments
22038 );
22039
22040 assert_eq!(
22041 client
22042 .update_workflow_avro_value(
22043 "typed-1",
22044 "replace",
22045 arguments.clone(),
22046 Some("typed-request"),
22047 )
22048 .await
22049 .expect("typed update"),
22050 typed_fidelity_probe()
22051 );
22052 let update = server.request_body("/api/workflows/typed-1/update/replace");
22053 assert_eq!(update["request_id"], "typed-request");
22054 assert_eq!(
22055 decode_wire_avro_value(&update["input"], DEFAULT_CODEC).expect("typed update input"),
22056 arguments
22057 );
22058
22059 let handle = WorkflowHandle {
22060 client: client.clone(),
22061 workflow_id: "typed-1".to_string(),
22062 run_id: Some("run-typed-1".to_string()),
22063 workflow_type: "typed.echo".to_string(),
22064 };
22065 assert_eq!(
22066 handle
22067 .result_avro_value(WorkflowResultOptions::default())
22068 .await
22069 .expect("typed workflow result"),
22070 typed_fidelity_probe()
22071 );
22072
22073 client
22074 .complete_activity_task(
22075 "activity-typed",
22076 "attempt-typed",
22077 "rust-worker",
22078 typed_fidelity_probe(),
22079 DEFAULT_CODEC,
22080 )
22081 .await
22082 .expect("typed activity completion");
22083 assert_eq!(
22084 decode_wire_avro_value(
22085 &server.request_body("/api/worker/activity-tasks/activity-typed/complete")
22086 ["result"],
22087 DEFAULT_CODEC,
22088 )
22089 .expect("typed activity result"),
22090 typed_fidelity_probe()
22091 );
22092 client
22093 .fail_activity_task(
22094 "activity-typed",
22095 "attempt-typed",
22096 "rust-worker",
22097 "typed failure",
22098 true,
22099 )
22100 .await
22101 .expect("activity failure");
22102 }
22103
22104 #[tokio::test]
22105 async fn lifecycle_commands_support_instance_and_selected_run_targets() {
22106 let server = MockWorkerServer::start();
22107 let client = Client::builder(server.base_url())
22108 .timeout(Duration::from_secs(2))
22109 .build()
22110 .expect("client");
22111
22112 let options = WorkflowCommandOptions::new()
22113 .reason("cleanup requested")
22114 .request_id("cancel-17");
22115 let cancelled = client
22116 .cancel_workflow("wf-lifecycle", options)
22117 .await
22118 .expect("instance cancellation");
22119 assert_eq!(cancelled.command, WorkflowCommandKind::Cancel);
22120 assert_eq!(cancelled.run_id.as_deref(), Some("run-current"));
22121 assert_eq!(cancelled.outcome.as_deref(), Some("cancelled"));
22122 assert_eq!(
22123 server.request_body("/api/workflows/wf-lifecycle/cancel"),
22124 json!({"reason":"cleanup requested","request_id":"cancel-17"})
22125 );
22126
22127 let terminated = client
22128 .terminate_workflow(
22129 "wf-lifecycle",
22130 WorkflowCommandOptions::new().reason("forced stop"),
22131 )
22132 .await
22133 .expect("instance termination");
22134 assert_eq!(terminated.command, WorkflowCommandKind::Terminate);
22135 assert_eq!(terminated.outcome.as_deref(), Some("terminated"));
22136
22137 client
22138 .cancel_workflow_run(
22139 "wf-lifecycle",
22140 "run-current",
22141 WorkflowCommandOptions::default(),
22142 )
22143 .await
22144 .expect("selected run cancellation");
22145 client
22146 .terminate_workflow_run(
22147 "wf-lifecycle",
22148 "run-current",
22149 WorkflowCommandOptions::default(),
22150 )
22151 .await
22152 .expect("selected run termination");
22153
22154 for (command, error) in [
22155 (
22156 WorkflowCommandKind::Cancel,
22157 client
22158 .cancel_workflow_run(
22159 "wf-lifecycle",
22160 "run-stale",
22161 WorkflowCommandOptions::default(),
22162 )
22163 .await
22164 .expect_err("stale cancellation must be rejected"),
22165 ),
22166 (
22167 WorkflowCommandKind::Terminate,
22168 client
22169 .terminate_workflow_run(
22170 "wf-lifecycle",
22171 "run-stale",
22172 WorkflowCommandOptions::default(),
22173 )
22174 .await
22175 .expect_err("stale termination must be rejected"),
22176 ),
22177 ] {
22178 let Error::WorkflowCommandRejected(rejection) = error else {
22179 panic!("expected typed command rejection");
22180 };
22181 assert_eq!(rejection.command, command);
22182 assert_eq!(rejection.status, 409);
22183 assert_eq!(rejection.reason, "historical_run_command_rejected");
22184 assert_eq!(rejection.run_id.as_deref(), Some("run-stale"));
22185 assert_eq!(rejection.target_scope.as_deref(), Some("run"));
22186 }
22187 }
22188
22189 #[tokio::test]
22190 async fn workflow_start_options_send_server_enforced_deadlines() {
22191 let server = MockWorkerServer::start();
22192 let client = Client::builder(server.base_url())
22193 .timeout(Duration::from_secs(2))
22194 .build()
22195 .expect("client");
22196
22197 let handle = client
22198 .start_workflow_with_options(
22199 "rust.timeout",
22200 "rust-timeouts",
22201 "wf-start-options",
22202 WorkflowStartOptions::new()
22203 .execution_timeout_seconds(30)
22204 .run_timeout_seconds(1),
22205 json!([]),
22206 )
22207 .await
22208 .expect("workflow start");
22209
22210 assert_eq!(handle.run_id.as_deref(), Some("run-start-options"));
22211 let body = server.request_body("/api/workflows");
22212 assert_eq!(body["execution_timeout_seconds"], 30);
22213 assert_eq!(body["run_timeout_seconds"], 1);
22214
22215 let invalid = client
22216 .start_workflow_with_options(
22217 "rust.timeout",
22218 "rust-timeouts",
22219 "wf-invalid-options",
22220 WorkflowStartOptions::new()
22221 .execution_timeout_seconds(1)
22222 .run_timeout_seconds(2),
22223 json!([]),
22224 )
22225 .await
22226 .expect_err("invalid deadline ordering");
22227 assert!(invalid
22228 .to_string()
22229 .contains("run_timeout_seconds cannot exceed execution_timeout_seconds"));
22230 }
22231
22232 #[tokio::test]
22233 async fn workflow_result_returns_each_typed_terminal_outcome() {
22234 let server = MockWorkerServer::start();
22235 let client = Client::builder(server.base_url())
22236 .timeout(Duration::from_secs(2))
22237 .build()
22238 .expect("client");
22239 let options = WorkflowResultOptions {
22240 poll_interval: Duration::ZERO,
22241 timeout: Duration::from_secs(1),
22242 };
22243
22244 let failed = WorkflowHandle {
22245 client: client.clone(),
22246 workflow_id: "wf-failed".to_string(),
22247 run_id: Some("run-failed".to_string()),
22248 workflow_type: "failure".to_string(),
22249 }
22250 .result(options)
22251 .await
22252 .expect_err("failed outcome");
22253 let Error::WorkflowFailed(failure) = failed else {
22254 panic!("expected WorkflowFailed");
22255 };
22256 assert_eq!(failure.workflow_id, "wf-failed");
22257 assert_eq!(failure.run_id.as_deref(), Some("run-failed"));
22258 assert_eq!(failure.failure_id.as_deref(), Some("failure-17"));
22259 assert_eq!(failure.failure_category.as_deref(), Some("application"));
22260 assert_eq!(failure.exception_type.as_deref(), Some("PaymentError"));
22261 assert_eq!(
22262 failure.exception_class.as_deref(),
22263 Some("billing::PaymentError")
22264 );
22265 assert_eq!(failure.non_retryable, Some(true));
22266
22267 for (workflow_id, expected_kind, expected_reason) in [
22268 (
22269 "wf-cancelled",
22270 WorkflowTerminalKind::Cancelled,
22271 "cleanup requested",
22272 ),
22273 (
22274 "wf-terminated",
22275 WorkflowTerminalKind::Terminated,
22276 "forced stop",
22277 ),
22278 (
22279 "wf-timed-out",
22280 WorkflowTerminalKind::TimedOut,
22281 "run_timeout",
22282 ),
22283 ] {
22284 let error = WorkflowHandle {
22285 client: client.clone(),
22286 workflow_id: workflow_id.to_string(),
22287 run_id: None,
22288 workflow_type: "terminal".to_string(),
22289 }
22290 .result(options)
22291 .await
22292 .expect_err("typed terminal outcome");
22293 let outcome = match error {
22294 Error::WorkflowCancelled(outcome) => outcome,
22295 Error::WorkflowTerminated(outcome) => outcome,
22296 Error::WorkflowTimedOut(outcome) => outcome,
22297 other => panic!("unexpected terminal error: {other}"),
22298 };
22299 assert_eq!(outcome.kind, expected_kind);
22300 assert_eq!(outcome.workflow_id, workflow_id);
22301 assert_eq!(outcome.reason, expected_reason);
22302 }
22303
22304 let wait_timeout = WorkflowHandle {
22305 client,
22306 workflow_id: "wf-waiting".to_string(),
22307 run_id: Some("run-waiting".to_string()),
22308 workflow_type: "waiting".to_string(),
22309 }
22310 .result(WorkflowResultOptions {
22311 poll_interval: Duration::ZERO,
22312 timeout: Duration::ZERO,
22313 })
22314 .await
22315 .expect_err("client wait timeout");
22316 let Error::WorkflowTimedOut(timeout) = wait_timeout else {
22317 panic!("expected typed client timeout");
22318 };
22319 assert_eq!(timeout.reason, "result_wait_timeout");
22320 assert_eq!(timeout.failure_category.as_deref(), Some("client_timeout"));
22321 assert_eq!(timeout.run_id.as_deref(), Some("run-waiting"));
22322 }
22323
22324 #[tokio::test]
22325 async fn workflow_result_follows_chain_and_selected_result_preserves_history() {
22326 let server = MockWorkerServer::start();
22327 let client = Client::builder(server.base_url())
22328 .timeout(Duration::from_secs(2))
22329 .build()
22330 .expect("client");
22331
22332 let handle = WorkflowHandle {
22333 client,
22334 workflow_id: "wf-selected".to_string(),
22335 run_id: Some("run-selected".to_string()),
22336 workflow_type: "selected".to_string(),
22337 };
22338 let options = WorkflowResultOptions {
22339 poll_interval: Duration::ZERO,
22340 timeout: Duration::from_secs(1),
22341 };
22342
22343 let current = handle
22344 .result(options)
22345 .await
22346 .expect("instance result follows the current run");
22347 assert_eq!(current, json!("current run output"));
22348
22349 let error = handle
22350 .result_selected_run(options)
22351 .await
22352 .expect_err("the selected run is cancelled even though the current run completed");
22353
22354 let Error::WorkflowCancelled(outcome) = error else {
22355 panic!("expected selected run cancellation");
22356 };
22357 assert_eq!(outcome.run_id.as_deref(), Some("run-selected"));
22358 assert_eq!(outcome.reason, "selected run cancelled");
22359 assert_eq!(
22360 server.request_count("/api/workflows/wf-selected/runs/run-selected"),
22361 1
22362 );
22363 assert_eq!(server.request_count("/api/workflows/wf-selected"), 1);
22364 }
22365
22366 #[tokio::test]
22367 async fn poll_responses_decode_http_conflict_drain_as_a_stable_stop() {
22368 let server = MockWorkerServer::draining_polls();
22369 let client = Client::builder(server.base_url())
22370 .timeout(Duration::from_secs(2))
22371 .build()
22372 .expect("client");
22373
22374 let workflow = client
22375 .poll_workflow_task_response("draining-worker", "rust-workers", Duration::ZERO)
22376 .await
22377 .expect("workflow drain response");
22378 let activity = client
22379 .poll_activity_task_response("draining-worker", "rust-workers", Duration::ZERO)
22380 .await
22381 .expect("activity drain response");
22382 let query = client
22383 .poll_query_task_response("draining-worker", "rust-workers", Duration::ZERO)
22384 .await
22385 .expect("query drain response");
22386
22387 for outcome in [workflow.outcome(), activity.outcome(), query.outcome()] {
22388 assert_eq!(
22389 outcome,
22390 WorkerPollOutcome::Stop {
22391 poll_status: Some("draining".to_string()),
22392 reason: Some("worker_draining".to_string()),
22393 }
22394 );
22395 }
22396
22397 assert!(client
22398 .poll_workflow_task("draining-worker", "rust-workers", Duration::ZERO)
22399 .await
22400 .expect("compatibility poll")
22401 .is_none());
22402 }
22403
22404 #[tokio::test]
22405 async fn managed_worker_honors_drain_stop_for_every_task_family() {
22406 let server = MockWorkerServer::draining_polls();
22407 let client = Client::builder(server.base_url())
22408 .timeout(Duration::from_secs(2))
22409 .build()
22410 .expect("client");
22411
22412 let mut workflow_worker = Worker::new(client.clone(), "rust-workers")
22413 .worker_id("draining-workflow-worker")
22414 .poll_timeout(Duration::ZERO);
22415 workflow_worker.register_workflow("counter", |_ctx, _args| async { Ok(Value::Null) });
22416 workflow_worker
22417 .run()
22418 .await
22419 .expect("workflow drain is a clean stop");
22420
22421 let mut activity_worker = Worker::new(client.clone(), "rust-workers")
22422 .worker_id("draining-activity-worker")
22423 .poll_timeout(Duration::ZERO);
22424 activity_worker.register_activity("write", |_ctx, _args| async { Ok(Value::Null) });
22425 activity_worker
22426 .run()
22427 .await
22428 .expect("activity drain is a clean stop");
22429
22430 let mut query_worker = Worker::new(client, "rust-workers")
22431 .worker_id("draining-query-worker")
22432 .poll_timeout(Duration::ZERO);
22433 query_worker.register_query("counter", "current", |_ctx, _args| async {
22434 Ok(Value::Null)
22435 });
22436 query_worker
22437 .run()
22438 .await
22439 .expect("query drain is a clean stop");
22440 }
22441
22442 #[tokio::test]
22443 async fn activity_cancellation_and_late_completion_remain_machine_readable() {
22444 let server = MockWorkerServer::start();
22445 let client = Client::builder(server.base_url())
22446 .timeout(Duration::from_secs(2))
22447 .build()
22448 .expect("client");
22449
22450 let heartbeat = client
22451 .heartbeat_activity_task(
22452 "activity-cancel",
22453 "attempt-cancel",
22454 "rust-worker",
22455 typed_fidelity_probe(),
22456 )
22457 .await
22458 .expect("cancellation heartbeat");
22459 assert!(heartbeat.cancel_requested);
22460 assert!(heartbeat.should_stop());
22461 assert_eq!(heartbeat.reason.as_deref(), Some("run_cancelled"));
22462 assert_eq!(heartbeat.run_closed_reason.as_deref(), Some("cancelled"));
22463 let heartbeat_body =
22464 server.request_body("/api/worker/activity-tasks/activity-cancel/heartbeat");
22465 assert_eq!(heartbeat_body["details"]["codec"], DEFAULT_CODEC);
22466 assert_eq!(
22467 decode_wire_avro_value(&heartbeat_body["details"], DEFAULT_CODEC)
22468 .expect("typed heartbeat details"),
22469 typed_fidelity_probe()
22470 );
22471
22472 let error = client
22473 .complete_activity_task(
22474 "activity-cancel",
22475 "attempt-cancel",
22476 "rust-worker",
22477 json!({"late":true}),
22478 DEFAULT_CODEC,
22479 )
22480 .await
22481 .expect_err("late completion must be refused");
22482 assert!(activity_task_rejection_is_final(&error));
22483 let Error::ActivityTaskRejected(rejection) = error else {
22484 panic!("expected typed activity rejection");
22485 };
22486 assert_eq!(rejection.status, 409);
22487 assert_eq!(rejection.reason, "run_cancelled");
22488 assert!(rejection.cancel_requested);
22489 assert_eq!(rejection.can_continue, Some(false));
22490 }
22491
22492 #[tokio::test]
22493 async fn managed_worker_survives_late_completion_and_restart_during_cancellation() {
22494 let server = MockWorkerServer::cancelled_activity();
22495 let client = Client::builder(server.base_url())
22496 .timeout(Duration::from_secs(2))
22497 .build()
22498 .expect("client");
22499 let cancellation_observed = Arc::new(AtomicBool::new(false));
22500 let observed = Arc::clone(&cancellation_observed);
22501 let mut worker = Worker::new(client.clone(), "rust-workers")
22502 .worker_id("rust-cancel-worker")
22503 .poll_timeout(Duration::from_millis(10));
22504 worker.register_activity("cancel-aware", move |ctx, _args| {
22505 let observed = Arc::clone(&observed);
22506 async move {
22507 let heartbeat = ctx.heartbeat(json!({"stage":"running"})).await?;
22508 observed.store(heartbeat.should_stop(), Ordering::SeqCst);
22509 Ok(json!({"late":"completion"}))
22510 }
22511 });
22512
22513 assert_eq!(
22514 worker.run_once().await.expect("cancelled attempt handled"),
22515 1
22516 );
22517 assert!(cancellation_observed.load(Ordering::SeqCst));
22518 assert_eq!(
22519 server.request_count("/api/worker/activity-tasks/activity-cancel/complete"),
22520 1
22521 );
22522
22523 let mut restarted = Worker::new(client, "rust-workers")
22524 .worker_id("rust-cancel-worker-restarted")
22525 .poll_timeout(Duration::from_millis(10));
22526 restarted.register_activity("cancel-aware", |_ctx, _args| async move { Ok(Value::Null) });
22527 assert_eq!(
22528 restarted
22529 .run_once()
22530 .await
22531 .expect("replacement worker continues polling"),
22532 0
22533 );
22534 }
22535
22536 #[tokio::test]
22537 async fn managed_worker_absorbs_selected_run_terminal_timeout_completion_race() {
22538 let response = r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"outcome":"completed","recorded":false,"run_id":"run-selected-timeout","run_status":"failed","created_task_ids":[],"reason":"run_timed_out"}"#;
22539 let server = MockWorkerServer::workflow_completion("409 Conflict", response);
22540 let client = Client::builder(server.base_url())
22541 .timeout(Duration::from_secs(2))
22542 .build()
22543 .expect("client");
22544
22545 let direct_error = client
22546 .complete_workflow_task(
22547 "workflow-timeout-task",
22548 "timeout-worker",
22549 3,
22550 vec![json!({
22551 "type": "complete_workflow",
22552 "result": fixture_envelope(Value::Null)
22553 })],
22554 )
22555 .await
22556 .expect_err("the low-level client preserves the completion rejection");
22557 let Error::Http { status, body } = direct_error else {
22558 panic!("expected the original HTTP completion rejection");
22559 };
22560 assert_eq!(status, reqwest::StatusCode::CONFLICT);
22561 assert_eq!(
22562 serde_json::from_str::<Value>(&body).expect("response body")["reason"],
22563 "run_timed_out"
22564 );
22565
22566 let mut worker = Worker::new(client, "rust-workers")
22567 .worker_id("timeout-worker")
22568 .poll_timeout(Duration::from_millis(10));
22569 worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
22570 Ok(json!({"late": "result"}))
22571 });
22572
22573 assert_eq!(
22574 worker
22575 .run_once()
22576 .await
22577 .expect("authoritative selected-run timeout settles the tick"),
22578 1
22579 );
22580 assert_eq!(
22581 server.request_count("/api/worker/workflow-tasks/workflow-timeout-task/complete"),
22582 2,
22583 "both the direct client proof and managed worker must see the rejection"
22584 );
22585 }
22586
22587 #[tokio::test]
22588 async fn managed_worker_does_not_swallow_nearby_completion_errors() {
22589 for (name, status, response) in [
22590 ("bare conflict", "409 Conflict", r#"{"message":"conflict"}"#),
22591 (
22592 "command was recorded",
22593 "409 Conflict",
22594 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":true,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22595 ),
22596 (
22597 "lease conflict",
22598 "409 Conflict",
22599 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"lease_expired"}"#,
22600 ),
22601 (
22602 "nonterminal run",
22603 "409 Conflict",
22604 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"waiting","reason":"run_timed_out"}"#,
22605 ),
22606 (
22607 "different selected run",
22608 "409 Conflict",
22609 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-reused-workflow-current","run_status":"failed","reason":"run_timed_out"}"#,
22610 ),
22611 (
22612 "different task attempt",
22613 "409 Conflict",
22614 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":4,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22615 ),
22616 (
22617 "authentication failure",
22618 "401 Unauthorized",
22619 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22620 ),
22621 (
22622 "authorization failure",
22623 "403 Forbidden",
22624 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22625 ),
22626 (
22627 "protocol failure",
22628 "400 Bad Request",
22629 r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.2","requested_version":"1.3"}"#,
22630 ),
22631 (
22632 "malformed command",
22633 "422 Unprocessable Entity",
22634 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22635 ),
22636 (
22637 "transient server failure",
22638 "503 Service Unavailable",
22639 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22640 ),
22641 ] {
22642 let server = MockWorkerServer::workflow_completion(status, response);
22643 let client = Client::builder(server.base_url())
22644 .timeout(Duration::from_secs(2))
22645 .build()
22646 .expect("client");
22647 let mut worker = Worker::new(client, "rust-workers")
22648 .worker_id("timeout-worker")
22649 .poll_timeout(Duration::from_millis(10));
22650 worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
22651 Ok(json!({"late": "result"}))
22652 });
22653
22654 let error = worker
22655 .run_once()
22656 .await
22657 .expect_err(&format!("{name} must remain an error"));
22658 assert!(
22659 matches!(error, Error::Http { .. } | Error::Protocol(_)),
22660 "{name} returned an unexpected error variant: {error}"
22661 );
22662 }
22663 }
22664
22665 #[tokio::test]
22666 async fn worker_deregistration_uses_worker_plane_method_path_headers_and_result() {
22667 let server = MockWorkerServer::start();
22668 let client = Client::builder(server.base_url())
22669 .worker_token(Some("worker-secret".to_string()))
22670 .namespace("orders")
22671 .timeout(Duration::from_secs(2))
22672 .build()
22673 .expect("client");
22674 let path = "/api/worker/registrations/worker%2F%CE%B1%20space";
22675
22676 let result = client
22677 .deregister_worker_registration("worker/α space")
22678 .await
22679 .expect("deregister worker registration");
22680
22681 assert_eq!(server.method_for(path).as_deref(), Some("DELETE"));
22682 assert_eq!(
22683 server.worker_protocol_for(path).as_deref(),
22684 Some(WORKER_PROTOCOL_VERSION)
22685 );
22686 assert_eq!(server.control_protocol_for(path), None);
22687 assert_eq!(server.namespace_for(path).as_deref(), Some("orders"));
22688 assert_eq!(
22689 server.authorization_for(path).as_deref(),
22690 Some("Bearer worker-secret")
22691 );
22692 assert_eq!(
22693 result,
22694 WorkerDeregistrationEnvelope {
22695 worker_id: "deregistered-worker".to_string(),
22696 outcome: "deregistered".to_string(),
22697 recovered_workflow_task_count: 2,
22698 }
22699 );
22700 }
22701
22702 #[tokio::test]
22703 async fn low_level_registration_rejects_update_validators_before_transport() {
22704 let server = MockWorkerServer::start();
22705 let client = Client::builder(server.base_url())
22706 .timeout(Duration::from_secs(2))
22707 .build()
22708 .expect("client");
22709
22710 for update_validators in [json!(["approve"]), json!("approve")] {
22711 let error = client
22712 .register_worker_with_command_contracts(
22713 "validator-claiming-worker",
22714 "rust-workers",
22715 vec!["orders".to_string()],
22716 vec![],
22717 1,
22718 1,
22719 vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
22720 json!({
22721 "orders": {
22722 "queries": ["current"],
22723 "updates": ["approve"],
22724 "update_validators": update_validators,
22725 },
22726 }),
22727 )
22728 .await
22729 .expect_err("unsupported validator claims must fail before registration");
22730
22731 let Error::UnsupportedUpdateValidators { workflow_type } = error else {
22732 panic!("expected typed unsupported-validator failure");
22733 };
22734 assert_eq!(workflow_type, "orders");
22735 }
22736 assert_eq!(server.request_count("/api/worker/register"), 0);
22737 }
22738
22739 #[tokio::test]
22740 async fn low_level_registration_preserves_query_and_update_contracts() {
22741 let server = MockWorkerServer::start();
22742 let client = Client::builder(server.base_url())
22743 .timeout(Duration::from_secs(2))
22744 .build()
22745 .expect("client");
22746 let contracts = json!({
22747 "orders": {
22748 "queries": ["current"],
22749 "updates": ["approve"],
22750 "update_validators": [],
22751 },
22752 "payments": {
22753 "queries": ["status"],
22754 "updates": ["capture"],
22755 },
22756 });
22757
22758 client
22759 .register_worker_with_command_contracts(
22760 "command-worker",
22761 "rust-workers",
22762 vec!["orders".to_string(), "payments".to_string()],
22763 vec![],
22764 1,
22765 1,
22766 vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
22767 contracts.clone(),
22768 )
22769 .await
22770 .expect("query and update contracts must remain supported");
22771
22772 assert_eq!(
22773 server.request_body("/api/worker/register")["workflow_command_contracts"],
22774 contracts
22775 );
22776 }
22777
22778 #[tokio::test]
22779 async fn role_scoped_tokens_are_never_used_for_the_opposite_plane() {
22780 let server = MockWorkerServer::start();
22781 let control_only = Client::builder(server.base_url())
22782 .control_token(Some("control-secret".to_string()))
22783 .build()
22784 .expect("control client");
22785
22786 let error = control_only
22787 .register_worker("worker", "queue", vec![], vec![], 1, 1)
22788 .await
22789 .expect_err("control token must not authorize a worker request");
22790 assert!(matches!(
22791 error,
22792 Error::MissingRoleCredentials { role: "worker", .. }
22793 ));
22794 assert_eq!(server.request_count("/api/worker/register"), 0);
22795
22796 let worker_only = Client::builder(server.base_url())
22797 .worker_token(Some("worker-secret".to_string()))
22798 .build()
22799 .expect("worker client");
22800 let error = worker_only
22801 .health()
22802 .await
22803 .expect_err("worker token must not authorize a control request");
22804 assert!(matches!(
22805 error,
22806 Error::MissingRoleCredentials {
22807 role: "control",
22808 ..
22809 }
22810 ));
22811 assert_eq!(server.request_count("/api/health"), 0);
22812 }
22813
22814 #[tokio::test]
22815 async fn shared_token_supports_worker_and_control_planes() {
22816 let server = MockWorkerServer::start();
22817 let client = Client::builder(server.base_url())
22818 .token(Some("shared-secret".to_string()))
22819 .build()
22820 .expect("client");
22821
22822 client.health().await.expect("control request");
22823 client
22824 .register_worker("worker", "queue", vec![], vec![], 1, 1)
22825 .await
22826 .expect("worker request");
22827
22828 assert_eq!(
22829 server.authorization_for("/api/health").as_deref(),
22830 Some("Bearer shared-secret")
22831 );
22832 assert_eq!(
22833 server.control_protocol_for("/api/health").as_deref(),
22834 Some(CONTROL_PLANE_VERSION)
22835 );
22836 assert_eq!(
22837 server.authorization_for("/api/worker/register").as_deref(),
22838 Some("Bearer shared-secret")
22839 );
22840 assert_eq!(
22841 server
22842 .worker_protocol_for("/api/worker/register")
22843 .as_deref(),
22844 Some(WORKER_PROTOCOL_VERSION)
22845 );
22846 }
22847
22848 #[tokio::test]
22849 async fn baseline_worker_endpoints_send_the_baseline_protocol() {
22850 let server = MockWorkerServer::start();
22851 let client = Client::builder(server.base_url())
22852 .timeout(Duration::from_secs(2))
22853 .build()
22854 .expect("client");
22855
22856 client
22857 .register_worker("capture-worker", "capture", vec![], vec![], 1, 1)
22858 .await
22859 .expect("register");
22860 client
22861 .heartbeat_worker("capture-worker", 1, 1)
22862 .await
22863 .expect("heartbeat");
22864 client
22865 .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
22866 .await
22867 .expect("workflow poll");
22868 client
22869 .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
22870 .await
22871 .expect("activity poll");
22872
22873 for path in [
22874 "/api/worker/register",
22875 "/api/worker/heartbeat",
22876 "/api/worker/workflow-tasks/poll",
22877 "/api/worker/activity-tasks/poll",
22878 ] {
22879 assert_eq!(
22880 server.worker_protocol_for(path).as_deref(),
22881 Some(WORKER_PROTOCOL_VERSION),
22882 "unexpected protocol for {path}"
22883 );
22884 }
22885
22886 assert_eq!(
22887 server.request_body("/api/worker/workflow-tasks/poll")["timeout_seconds"],
22888 1
22889 );
22890 assert_eq!(
22891 server.request_body("/api/worker/activity-tasks/poll")["timeout_seconds"],
22892 1
22893 );
22894 assert!(
22895 server.request_body("/api/worker/workflow-tasks/poll")["poll_request_id"]
22896 .as_str()
22897 .is_some_and(|id| id.starts_with("rust-workflow-poll-"))
22898 );
22899 assert!(
22900 server.request_body("/api/worker/activity-tasks/poll")["poll_request_id"]
22901 .as_str()
22902 .is_some_and(|id| id.starts_with("rust-activity-poll-"))
22903 );
22904 }
22905
22906 #[tokio::test]
22907 async fn query_task_endpoints_send_the_query_feature_protocol() {
22908 let server = MockWorkerServer::start();
22909 let client = Client::builder(server.base_url())
22910 .timeout(Duration::from_secs(2))
22911 .build()
22912 .expect("client");
22913
22914 client
22915 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
22916 .await
22917 .expect("query poll");
22918 client
22919 .complete_query_task(
22920 "query-capture",
22921 "capture-worker",
22922 1,
22923 json!(8),
22924 DEFAULT_CODEC,
22925 )
22926 .await
22927 .expect("query complete");
22928 client
22929 .fail_query_task(
22930 "query-capture",
22931 "capture-worker",
22932 1,
22933 "failed",
22934 "query_rejected",
22935 "QueryFailed",
22936 )
22937 .await
22938 .expect("query fail");
22939
22940 for path in [
22941 "/api/worker/query-tasks/poll",
22942 "/api/worker/query-tasks/query-capture/complete",
22943 "/api/worker/query-tasks/query-capture/fail",
22944 ] {
22945 assert_eq!(
22946 server.worker_protocol_for(path).as_deref(),
22947 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
22948 "unexpected protocol for {path}"
22949 );
22950 }
22951
22952 assert_eq!(
22953 server.request_body("/api/worker/query-tasks/poll")["timeout_seconds"],
22954 1
22955 );
22956 assert!(
22957 server.request_body("/api/worker/query-tasks/poll")["poll_request_id"]
22958 .as_str()
22959 .is_some_and(|id| id.starts_with("rust-query-poll-"))
22960 );
22961 }
22962
22963 #[tokio::test]
22964 async fn disconnected_client_polls_retry_once_with_the_same_request_id() {
22965 let server = MockWorkerServer::transient_worker_failures();
22966 let client = Client::builder(server.base_url())
22967 .timeout(Duration::from_secs(2))
22968 .build()
22969 .expect("client");
22970
22971 client
22972 .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
22973 .await
22974 .expect("workflow poll retry");
22975 client
22976 .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
22977 .await
22978 .expect("activity poll retry");
22979 client
22980 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
22981 .await
22982 .expect("query poll retry");
22983
22984 for path in [
22985 "/api/worker/workflow-tasks/poll",
22986 "/api/worker/activity-tasks/poll",
22987 "/api/worker/query-tasks/poll",
22988 ] {
22989 let bodies = server.request_bodies(path);
22990 assert_eq!(bodies.len(), 2, "{path} must be retried once");
22991 assert_eq!(
22992 bodies[0]["poll_request_id"], bodies[1]["poll_request_id"],
22993 "{path} must preserve the request binding across retry"
22994 );
22995 }
22996 }
22997
22998 #[tokio::test]
22999 async fn worker_poll_retries_preserve_request_id_across_consecutive_failures() {
23000 let server = MockWorkerServer::consecutive_poll_failures(2);
23001 let client = Client::builder(server.base_url())
23002 .timeout(Duration::from_secs(2))
23003 .build()
23004 .expect("client");
23005 let mut worker = Worker::new(client, "capture")
23006 .worker_id("capture-worker")
23007 .poll_timeout(Duration::from_millis(10))
23008 .retry_policy(WorkerRetryPolicy {
23009 max_retries: 2,
23010 initial_backoff: Duration::from_millis(1),
23011 max_backoff: Duration::from_millis(1),
23012 });
23013 worker.register_workflow(
23014 "capture.workflow",
23015 |_ctx, _input| async move { Ok(Value::Null) },
23016 );
23017 worker.register_activity(
23018 "capture.activity",
23019 |_ctx, _input| async move { Ok(Value::Null) },
23020 );
23021 worker.register_query("capture.workflow", "current", |_ctx, _args| async move {
23022 Ok(Value::Null)
23023 });
23024
23025 assert_eq!(worker.run_once().await.expect("poll retries"), 0);
23026
23027 for path in [
23028 "/api/worker/workflow-tasks/poll",
23029 "/api/worker/activity-tasks/poll",
23030 "/api/worker/query-tasks/poll",
23031 ] {
23032 let bodies = server.request_bodies(path);
23033 assert_eq!(bodies.len(), 3, "{path} must use exactly two retries");
23034 assert!(
23035 bodies
23036 .iter()
23037 .all(|body| body["poll_request_id"] == bodies[0]["poll_request_id"]),
23038 "{path} must preserve one request binding across every retry"
23039 );
23040 }
23041 }
23042
23043 fn storage_refusal(poll_id: Option<&str>, unavailable: bool, mid_poll: bool) -> Value {
23044 let reason = if unavailable {
23045 "storage_admission_unavailable"
23046 } else {
23047 "storage_pressure"
23048 };
23049 let mut body = json!({
23050 "reason": reason,
23051 "storage_state": if unavailable { "fenced" } else { "draining" },
23052 "retryable": true,
23053 "retry_after_seconds": 1,
23054 });
23055 if !mid_poll {
23056 body["request_admitted"] = json!(false);
23057 }
23058 if let Some(id) = poll_id {
23059 body["task"] = Value::Null;
23060 body["poll_status"] = json!(reason);
23061 body["poll_request_id"] = json!(id);
23062 body["retry_same_poll_request_id"] = json!(true);
23063 body["claim_admitted"] = json!(false);
23064 }
23065 body
23066 }
23067
23068 fn storage_worker(server: &MockWorkerServer) -> Worker {
23069 Worker::new(Client::new(server.base_url()).expect("client"), "storage")
23070 .worker_id("storage-worker")
23071 .retry_policy(WorkerRetryPolicy {
23072 max_retries: 1,
23073 initial_backoff: Duration::from_millis(1),
23074 max_backoff: Duration::from_millis(1),
23075 })
23076 }
23077
23078 fn assert_identical_requests(server: &MockWorkerServer, path: &str, count: usize) {
23079 let requests = server.requests.lock().expect("requests");
23080 let bodies: Vec<_> = requests
23081 .iter()
23082 .filter(|request| request.path == path)
23083 .map(|request| &request.body)
23084 .collect();
23085 assert_eq!(bodies.len(), count, "{path}");
23086 assert!(bodies.iter().all(|body| body == &bodies[0]), "{path}");
23087 }
23088
23089 #[test]
23090 fn storage_admission_requires_an_explicit_identity_preserving_contract() {
23091 for unavailable in [false, true] {
23092 for mid_poll in [false, true] {
23093 let body = storage_refusal(Some("same-poll"), unavailable, mid_poll);
23094 let error = Error::Http {
23095 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23096 body: body.to_string(),
23097 };
23098 assert_eq!(
23099 worker_storage_admission_retry_after(&error, Some("same-poll")),
23100 Some(Duration::from_secs(1))
23101 );
23102 assert!(
23103 !worker_operation_is_retryable(&error),
23104 "storage is not a bounded generic retry"
23105 );
23106 for (field, value) in [
23107 ("poll_request_id", json!("wrong-poll")),
23108 ("task", json!({"task_id":"claimed"})),
23109 ("retryable", json!(false)),
23110 ("retry_after_seconds", json!(0)),
23111 ("retry_after_seconds", json!(true)),
23112 ("retry_after_seconds", json!(1.0)),
23113 ("storage_state", json!("normal")),
23114 ("poll_status", json!("empty")),
23115 ("claim_admitted", json!(true)),
23116 ("retry_same_poll_request_id", json!(false)),
23117 ("request_admitted", json!(true)),
23118 ] {
23119 let mut invalid = body.clone();
23120 invalid[field] = value;
23121 let error = Error::Http {
23122 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23123 body: invalid.to_string(),
23124 };
23125 assert!(
23126 worker_storage_admission_retry_after(&error, Some("same-poll")).is_none(),
23127 "{field}"
23128 );
23129 }
23130 }
23131 }
23132 let body = storage_refusal(None, false, false);
23133 let error = Error::Http {
23134 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23135 body: body.to_string(),
23136 };
23137 assert!(worker_storage_admission_retry_after(&error, None).is_some());
23138 assert!(worker_storage_admission_retry_after(&error, Some("")).is_none());
23139 let error = Error::Http {
23140 status: reqwest::StatusCode::FORBIDDEN,
23141 body: body.to_string(),
23142 };
23143 assert!(worker_storage_admission_retry_after(&error, None).is_none());
23144 let body = storage_refusal(None, false, true);
23145 let error = Error::Http {
23146 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23147 body: body.to_string(),
23148 };
23149 assert!(worker_storage_admission_retry_after(&error, None).is_none());
23150 }
23151
23152 #[tokio::test]
23153 async fn storage_poll_recovery_preserves_ambiguous_claim_identity() {
23154 for unavailable in [false, true] {
23155 for mid_poll in [false, true] {
23156 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23157 poll_failures_per_path: 1,
23158 storage_refusals: 7,
23159 storage_path: Some("/poll"),
23160 storage_unavailable: unavailable,
23161 storage_mid_poll: mid_poll,
23162 ..MockWorkerBehavior::default()
23163 });
23164 let mut worker = storage_worker(&server);
23165 worker.register_query("unused", "state", |_, _| async { Ok(Value::Null) });
23166 assert_eq!(worker.run_once().await.expect("storage recovery"), 0);
23167 for path in [
23168 "/api/worker/workflow-tasks/poll",
23169 "/api/worker/activity-tasks/poll",
23170 "/api/worker/query-tasks/poll",
23171 ] {
23172 assert_identical_requests(&server, path, 9);
23173 }
23174 }
23175 }
23176 }
23177
23178 #[tokio::test]
23179 async fn storage_refused_mutations_do_not_reserialize_or_change_client_scope() {
23180 struct CountedBody(Arc<AtomicUsize>);
23181 impl Serialize for CountedBody {
23182 fn serialize<S: Serializer>(
23183 &self,
23184 serializer: S,
23185 ) -> std::result::Result<S::Ok, S::Error> {
23186 let count = self.0.fetch_add(1, Ordering::SeqCst);
23187 json!({"serialization":count,"lease_owner":"worker","attempt":7})
23188 .serialize(serializer)
23189 }
23190 }
23191 for path in [
23192 "/api/worker/register",
23193 "/api/worker/heartbeat",
23194 "/api/worker/workflow-tasks/storage-task/complete",
23195 "/api/worker/workflow-tasks/storage-task/fail",
23196 "/api/worker/activity-tasks/storage-task/complete",
23197 "/api/worker/activity-tasks/storage-task/fail",
23198 "/api/worker/activity-tasks/storage-task/heartbeat",
23199 "/api/worker/query-tasks/storage-task/complete",
23200 "/api/worker/query-tasks/storage-task/fail",
23201 ] {
23202 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23203 storage_refusals: 7,
23204 storage_path: Some(path),
23205 ..MockWorkerBehavior::default()
23206 });
23207 let worker =
23208 storage_worker(&server).with_storage_admission(Arc::new(AtomicBool::new(false)));
23209 let calls = Arc::new(AtomicUsize::new(0));
23210 let _: Value = worker
23211 .client
23212 .request_json(
23213 reqwest::Method::POST,
23214 &path[4..],
23215 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
23216 Some(&CountedBody(Arc::clone(&calls))),
23217 )
23218 .await
23219 .expect("prepared request recovery");
23220 assert_eq!(calls.load(Ordering::SeqCst), 1);
23221 assert_identical_requests(&server, path, 8);
23222 }
23223 for worker_scope in [false, true] {
23224 let path = if worker_scope {
23225 "/api/health"
23226 } else {
23227 "/api/worker/register"
23228 };
23229 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23230 storage_refusals: usize::MAX,
23231 storage_path: Some(path),
23232 ..MockWorkerBehavior::default()
23233 });
23234 let worker = storage_worker(&server);
23235 let client = worker.client.clone();
23236 let worker = worker.with_storage_admission(Arc::new(AtomicBool::new(false)));
23237 let error = if worker_scope {
23238 worker
23239 .client
23240 .health()
23241 .await
23242 .expect_err("control plane is not retried")
23243 } else {
23244 client
23245 .request_json::<Value, Value>(
23246 reqwest::Method::POST,
23247 "/worker/register",
23248 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
23249 Some(&json!({})),
23250 )
23251 .await
23252 .expect_err("direct client is not retried")
23253 };
23254 assert!(worker_storage_admission_body(&error).is_some());
23255 assert_eq!(server.request_count(path), 1);
23256 }
23257 }
23258
23259 #[tokio::test]
23260 async fn storage_activity_outcome_is_retained_without_reexecuting_handler() {
23261 for fail in [false, true] {
23262 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23263 storage_activity: true,
23264 storage_refusals: 7,
23265 storage_path: Some("/storage-activity/"),
23266 ..MockWorkerBehavior::default()
23267 });
23268 let mut worker = storage_worker(&server);
23269 let calls = Arc::new(AtomicUsize::new(0));
23270 let observed = Arc::clone(&calls);
23271 worker.register_activity("storage.activity", move |ctx, _| {
23272 let calls = Arc::clone(&observed);
23273 async move {
23274 calls.fetch_add(1, Ordering::SeqCst);
23275 ctx.heartbeat(json!({"step":2})).await?;
23276 if fail {
23277 Err(Error::WorkerLoop("intentional handler failure".to_string()))
23278 } else {
23279 Ok(json!({"receipt":true}))
23280 }
23281 }
23282 });
23283 assert_eq!(worker.run_once().await.expect("activity settled"), 1);
23284 assert_eq!(calls.load(Ordering::SeqCst), 1);
23285 assert_identical_requests(
23286 &server,
23287 "/api/worker/activity-tasks/storage-activity/heartbeat",
23288 8,
23289 );
23290 let suffix = if fail { "fail" } else { "complete" };
23291 assert_identical_requests(
23292 &server,
23293 &format!("/api/worker/activity-tasks/storage-activity/{suffix}"),
23294 8,
23295 );
23296 let other = if fail { "complete" } else { "fail" };
23297 assert_eq!(
23298 server.request_count(&format!(
23299 "/api/worker/activity-tasks/storage-activity/{other}"
23300 )),
23301 0
23302 );
23303 }
23304 }
23305
23306 #[tokio::test]
23307 async fn storage_waits_are_interruptible_without_false_activity_failure() {
23308 for path in [
23309 "/api/worker/register",
23310 "/api/worker/heartbeat",
23311 "/api/worker/activity-tasks/poll",
23312 "/api/worker/activity-tasks/storage-activity/heartbeat",
23313 "/api/worker/activity-tasks/storage-activity/complete",
23314 ] {
23315 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23316 storage_activity: true,
23317 storage_refusals: usize::MAX,
23318 storage_path: Some(path),
23319 ..MockWorkerBehavior::default()
23320 });
23321 let mut worker = storage_worker(&server).retry_policy(WorkerRetryPolicy::default());
23322 worker.register_activity("storage.activity", |ctx, _| async move {
23323 ctx.heartbeat(json!({"step":2})).await?;
23324 Ok(json!({"receipt":true}))
23325 });
23326 let shutdown = async {
23327 while server.request_count(path) == 0 {
23328 tokio::time::sleep(Duration::from_millis(1)).await;
23329 }
23330 };
23331 let result = tokio::time::timeout(Duration::from_secs(2), worker.run_until(shutdown))
23332 .await
23333 .expect("shutdown interrupts admission");
23334 assert!(
23335 result.is_err(),
23336 "a refused operation must not appear acknowledged: {path}, {result:?}"
23337 );
23338 assert_eq!(server.request_count(path), 1);
23339 assert_eq!(
23340 server.request_count("/api/worker/activity-tasks/storage-activity/fail"),
23341 0
23342 );
23343 assert_eq!(
23344 server.request_count("/api/worker/registrations/mock-worker"),
23345 usize::from(!path.ends_with("/register"))
23346 );
23347 }
23348 }
23349
23350 #[tokio::test]
23351 async fn storage_query_outcome_is_retained_without_reexecuting_handler() {
23352 for fail in [false, true] {
23353 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23354 storage_query: true,
23355 storage_refusals: 7,
23356 storage_path: Some("/storage-query/"),
23357 ..MockWorkerBehavior::default()
23358 });
23359 let mut worker = storage_worker(&server);
23360 worker.register_workflow("storage.workflow", |_, _| async { Ok(Value::Null) });
23361 let calls = Arc::new(AtomicUsize::new(0));
23362 let observed = Arc::clone(&calls);
23363 worker.register_query("storage.workflow", "state", move |_, _| {
23364 let calls = Arc::clone(&observed);
23365 async move {
23366 calls.fetch_add(1, Ordering::SeqCst);
23367 if fail {
23368 Err(Error::WorkerLoop("intentional query failure".to_string()))
23369 } else {
23370 Ok(json!({"state":"waiting"}))
23371 }
23372 }
23373 });
23374 assert_eq!(worker.run_once().await.expect("query settled"), 1);
23375 assert_eq!(calls.load(Ordering::SeqCst), 1);
23376 let suffix = if fail { "fail" } else { "complete" };
23377 assert_identical_requests(
23378 &server,
23379 &format!("/api/worker/query-tasks/storage-query/{suffix}"),
23380 8,
23381 );
23382 let other = if fail { "complete" } else { "fail" };
23383 assert_eq!(
23384 server.request_count(&format!("/api/worker/query-tasks/storage-query/{other}")),
23385 0
23386 );
23387 }
23388 }
23389
23390 #[tokio::test]
23391 async fn storage_recovery_does_not_override_auth_lease_or_invalid_contract() {
23392 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23393 storage_refusals: 7,
23394 storage_path: Some("/poll"),
23395 unauthorized_polls: true,
23396 ..MockWorkerBehavior::default()
23397 });
23398 let error = storage_worker(&server)
23399 .run_once()
23400 .await
23401 .expect_err("auth remains terminal");
23402 assert!(matches!(
23403 error,
23404 Error::Http {
23405 status: reqwest::StatusCode::UNAUTHORIZED,
23406 ..
23407 }
23408 ));
23409 assert_identical_requests(&server, "/api/worker/workflow-tasks/poll", 8);
23410
23411 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23412 storage_refusals: 7,
23413 storage_path: Some("/activity-cancel/complete"),
23414 ..MockWorkerBehavior::default()
23415 });
23416 let worker =
23417 storage_worker(&server).with_storage_admission(Arc::new(AtomicBool::new(false)));
23418 let error = worker
23419 .client
23420 .complete_activity_task(
23421 "activity-cancel",
23422 "attempt-cancel",
23423 "worker",
23424 json!({}),
23425 DEFAULT_CODEC,
23426 )
23427 .await
23428 .expect_err("cancellation remains terminal");
23429 assert!(activity_task_rejection_is_final(&error));
23430 assert_identical_requests(
23431 &server,
23432 "/api/worker/activity-tasks/activity-cancel/complete",
23433 8,
23434 );
23435
23436 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23437 storage_refusals: usize::MAX,
23438 storage_path: Some("/poll"),
23439 storage_wrong_poll_id: true,
23440 ..MockWorkerBehavior::default()
23441 });
23442 assert!(storage_worker(&server).run_once().await.is_err());
23443 assert_eq!(server.request_count("/api/worker/workflow-tasks/poll"), 1);
23444 }
23445
23446 #[tokio::test]
23447 async fn storage_pollers_stop_when_the_run_future_is_aborted() {
23448 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23449 storage_refusals: usize::MAX,
23450 storage_path: Some("/poll"),
23451 ..MockWorkerBehavior::default()
23452 });
23453 let mut worker = storage_worker(&server).retry_policy(WorkerRetryPolicy::default());
23454 worker.register_activity("unused", |_, _| async { Ok(Value::Null) });
23455 let run = tokio::spawn(async move { worker.run().await });
23456 tokio::time::timeout(Duration::from_secs(2), async {
23457 while server.request_count("/api/worker/activity-tasks/poll") == 0 {
23458 tokio::time::sleep(Duration::from_millis(1)).await;
23459 }
23460 })
23461 .await
23462 .expect("poll started");
23463 run.abort();
23464 assert!(run.await.expect_err("cancelled run").is_cancelled());
23465 tokio::time::sleep(Duration::from_millis(250)).await;
23466 assert_eq!(server.request_count("/api/worker/activity-tasks/poll"), 1);
23467 }
23468
23469 #[tokio::test]
23470 async fn query_protocol_rejection_from_older_server_is_typed() {
23471 let server = MockWorkerServer::reject_query_protocol();
23472 let client = Client::builder(server.base_url())
23473 .timeout(Duration::from_secs(2))
23474 .build()
23475 .expect("client");
23476
23477 let error = client
23478 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
23479 .await
23480 .expect_err("server below query protocol floor must reject");
23481 let Error::Protocol(failure) = error else {
23482 panic!("expected typed protocol failure");
23483 };
23484
23485 assert_eq!(failure.status, 400);
23486 assert_eq!(failure.reason, "unsupported_protocol_version");
23487 assert_eq!(failure.supported_version.as_deref(), Some("1.7"));
23488 assert_eq!(
23489 failure.requested_version.as_deref(),
23490 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
23491 );
23492 assert_eq!(
23493 server
23494 .worker_protocol_for("/api/worker/query-tasks/poll")
23495 .as_deref(),
23496 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
23497 );
23498 }
23499
23500 #[tokio::test]
23501 async fn run_once_without_query_handlers_keeps_pre_query_server_compatibility() {
23502 let server = MockWorkerServer::reject_query_protocol();
23503 let client = Client::builder(server.base_url())
23504 .timeout(Duration::from_secs(2))
23505 .build()
23506 .expect("client");
23507 let mut worker = Worker::new(client, "rust-workers")
23508 .worker_id("baseline-worker")
23509 .poll_timeout(Duration::from_millis(10));
23510
23511 worker.register_workflow("baseline.workflow", |_ctx, _input| async move {
23512 Ok(Value::Null)
23513 });
23514
23515 assert_eq!(worker.run_once().await.expect("baseline run once"), 0);
23516 assert_eq!(
23517 server
23518 .worker_protocol_for("/api/worker/workflow-tasks/poll")
23519 .as_deref(),
23520 Some(WORKER_PROTOCOL_VERSION)
23521 );
23522 assert_eq!(
23523 server.worker_protocol_for("/api/worker/query-tasks/poll"),
23524 None,
23525 "a worker without query handlers must not use the query-task endpoint"
23526 );
23527 }
23528
23529 #[tokio::test]
23530 async fn completion_time_query_rejection_is_typed_without_stopping_worker() {
23531 let server = MockWorkerServer::reject_query_completion();
23532 let client = Client::builder(server.base_url())
23533 .timeout(Duration::from_secs(2))
23534 .build()
23535 .expect("client");
23536
23537 let error = client
23538 .complete_query_task("query-late", "late-worker", 1, json!(8), DEFAULT_CODEC)
23539 .await
23540 .expect_err("expired completion must be rejected");
23541 let Error::QueryFailed(failure) = error else {
23542 panic!("expected typed query failure");
23543 };
23544 assert_eq!(failure.status, 409);
23545 assert_eq!(failure.reason, "query_task_timed_out");
23546
23547 let mut worker = Worker::new(client, "rust-workers")
23548 .worker_id("late-worker")
23549 .poll_timeout(Duration::from_millis(10));
23550 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
23551 worker.register_query(
23552 "counter",
23553 "current",
23554 |_ctx, _args| async move { Ok(json!(8)) },
23555 );
23556
23557 assert_eq!(worker.run_once().await.expect("late task is handled"), 1);
23558 assert_eq!(
23559 worker
23560 .run_once()
23561 .await
23562 .expect("worker continues after late completion"),
23563 0
23564 );
23565 assert_eq!(
23566 server.request_count("/api/worker/query-tasks/query-late/complete"),
23567 2
23568 );
23569 assert_eq!(
23570 server.request_count("/api/worker/query-tasks/query-late/fail"),
23571 0,
23572 "a server completion rejection must not be reported as an encoding failure"
23573 );
23574 }
23575
23576 #[tokio::test]
23577 async fn normal_shutdown_joins_pollers_and_deregisters_once() {
23578 let server = MockWorkerServer::start();
23579 let client = Client::builder(server.base_url())
23580 .timeout(Duration::from_secs(2))
23581 .build()
23582 .expect("client");
23583 let mut worker = Worker::new(client, "rust-workers")
23584 .worker_id("joined-worker")
23585 .poll_timeout(Duration::from_millis(10));
23586 worker.register_workflow(
23587 "joined.workflow",
23588 |_ctx, _input| async move { Ok(Value::Null) },
23589 );
23590 worker.register_activity(
23591 "joined.activity",
23592 |_ctx, _input| async move { Ok(Value::Null) },
23593 );
23594 worker.register_query("joined.workflow", "state", |_ctx, _input| async move {
23595 Ok(Value::Null)
23596 });
23597
23598 worker
23599 .run_until(tokio::time::sleep(Duration::from_millis(20)))
23600 .await
23601 .expect("normal shutdown");
23602
23603 let deregistration_path = "/api/worker/registrations/mock-worker";
23604 assert_eq!(server.request_count(deregistration_path), 1);
23605 for poll_path in [
23606 "/api/worker/workflow-tasks/poll",
23607 "/api/worker/activity-tasks/poll",
23608 "/api/worker/query-tasks/poll",
23609 ] {
23610 assert!(server.request_count(poll_path) > 0, "missing {poll_path}");
23611 }
23612 assert_eq!(
23613 server.captured_paths().last().map(String::as_str),
23614 Some(deregistration_path),
23615 "deregistration must start only after every poller has joined"
23616 );
23617 }
23618
23619 #[tokio::test]
23620 async fn registration_failure_does_not_deregister() {
23621 let server = MockWorkerServer::rejected_registration();
23622 let client = Client::builder(server.base_url())
23623 .timeout(Duration::from_secs(2))
23624 .build()
23625 .expect("client");
23626 let worker = Worker::new(client, "rust-workers").worker_id("never-registered");
23627
23628 let error = worker
23629 .run_until(async {})
23630 .await
23631 .expect_err("registration must fail");
23632 assert!(matches!(
23633 error,
23634 Error::Http {
23635 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23636 ..
23637 }
23638 ));
23639 assert!(server
23640 .captured_paths()
23641 .iter()
23642 .all(|path| !path.starts_with("/api/worker/registrations/")));
23643 }
23644
23645 #[tokio::test]
23646 async fn protocol_116_server_rejects_occurrence_identity_worker_registration() {
23647 let server = MockWorkerServer::rejected_registration_protocol();
23648 let client = Client::builder(server.base_url())
23649 .timeout(Duration::from_secs(2))
23650 .build()
23651 .expect("client");
23652 let worker = Worker::new(client, "rust-workers").worker_id("protocol-117-worker");
23653
23654 let error = worker
23655 .run_until(async {})
23656 .await
23657 .expect_err("a protocol 1.16 server must reject this worker");
23658 let Error::Protocol(failure) = error else {
23659 panic!("expected typed protocol rejection");
23660 };
23661 assert_eq!(failure.reason, "unsupported_protocol_version");
23662 assert_eq!(failure.supported_version.as_deref(), Some("1.16"));
23663 assert_eq!(failure.requested_version.as_deref(), Some("1.17"));
23664 assert_eq!(
23665 server
23666 .worker_protocol_for("/api/worker/register")
23667 .as_deref(),
23668 Some(WORKER_PROTOCOL_VERSION)
23669 );
23670 }
23671
23672 #[tokio::test]
23673 async fn declined_registration_does_not_deregister() {
23674 let server = MockWorkerServer::declined_registration();
23675 let client = Client::builder(server.base_url())
23676 .timeout(Duration::from_secs(2))
23677 .build()
23678 .expect("client");
23679 let worker = Worker::new(client, "rust-workers").worker_id("declined-worker");
23680
23681 let error = worker
23682 .run_until(async {})
23683 .await
23684 .expect_err("declined registration must fail");
23685 assert!(matches!(error, Error::WorkerLoop(_)));
23686 assert!(error.to_string().contains("was not accepted"));
23687 assert!(server
23688 .captured_paths()
23689 .iter()
23690 .all(|path| !path.starts_with("/api/worker/registrations/")));
23691 }
23692
23693 #[tokio::test]
23694 async fn deregistration_http_failure_is_returned_after_normal_shutdown() {
23695 let server = MockWorkerServer::rejected_deregistration();
23696 let client = Client::builder(server.base_url())
23697 .timeout(Duration::from_secs(2))
23698 .build()
23699 .expect("client");
23700 let worker = Worker::new(client, "rust-workers").worker_id("forbidden-cleanup");
23701
23702 let error = worker
23703 .run_until(async {})
23704 .await
23705 .expect_err("deregistration must fail");
23706 assert!(matches!(
23707 error,
23708 Error::Http {
23709 status: reqwest::StatusCode::FORBIDDEN,
23710 ..
23711 }
23712 ));
23713 assert_eq!(
23714 server.request_count("/api/worker/registrations/mock-worker"),
23715 1
23716 );
23717 }
23718
23719 #[tokio::test]
23720 async fn deregistration_protocol_failure_is_returned_after_normal_shutdown() {
23721 let server = MockWorkerServer::rejected_deregistration_protocol();
23722 let client = Client::builder(server.base_url())
23723 .timeout(Duration::from_secs(2))
23724 .build()
23725 .expect("client");
23726 let worker = Worker::new(client, "rust-workers").worker_id("protocol-cleanup");
23727
23728 let error = worker
23729 .run_until(async {})
23730 .await
23731 .expect_err("protocol rejection must fail shutdown");
23732 let Error::Protocol(failure) = error else {
23733 panic!("expected typed protocol failure");
23734 };
23735 assert_eq!(failure.reason, "unsupported_protocol_version");
23736 assert_eq!(
23737 failure.requested_version.as_deref(),
23738 Some(WORKER_PROTOCOL_VERSION)
23739 );
23740 assert_eq!(
23741 server.request_count("/api/worker/registrations/mock-worker"),
23742 1
23743 );
23744 }
23745
23746 #[tokio::test]
23747 async fn primary_poller_error_retains_deregistration_failure_context() {
23748 let server = MockWorkerServer::unauthorized_polls_and_rejected_deregistration();
23749 let client = Client::builder(server.base_url())
23750 .timeout(Duration::from_secs(2))
23751 .build()
23752 .expect("client");
23753 let mut worker = Worker::new(client, "rust-workers")
23754 .worker_id("combined-failure")
23755 .poll_timeout(Duration::from_millis(10));
23756 worker.register_workflow("combined.workflow", |_ctx, _input| async move {
23757 Ok(Value::Null)
23758 });
23759
23760 let error = worker
23761 .run()
23762 .await
23763 .expect_err("worker and cleanup must fail");
23764 let summary = error.to_string();
23765 assert!(summary.contains("authentication_failed"));
23766 assert!(summary.contains("worker cannot deregister"));
23767 let Error::WorkerShutdown {
23768 primary,
23769 deregistration,
23770 } = error
23771 else {
23772 panic!("expected combined worker shutdown error");
23773 };
23774 assert!(matches!(
23775 *primary,
23776 Error::Http {
23777 status: reqwest::StatusCode::UNAUTHORIZED,
23778 ..
23779 }
23780 ));
23781 assert!(matches!(
23782 *deregistration,
23783 Error::Http {
23784 status: reqwest::StatusCode::FORBIDDEN,
23785 ..
23786 }
23787 ));
23788 assert_eq!(
23789 server.request_count("/api/worker/registrations/mock-worker"),
23790 1
23791 );
23792 }
23793
23794 #[tokio::test]
23795 async fn activity_only_worker_can_shutdown_without_workflow_poller() {
23796 let server = MockWorkerServer::start();
23797 let client = Client::builder(server.base_url())
23798 .timeout(Duration::from_secs(2))
23799 .build()
23800 .expect("client");
23801 let mut worker = Worker::new(client, "rust-workers")
23802 .worker_id("activity-only-worker")
23803 .poll_timeout(Duration::from_millis(10));
23804
23805 worker.register_activity(
23806 "activity.only",
23807 |_ctx, _args| async move { Ok(Value::Null) },
23808 );
23809
23810 worker.run_until(async {}).await.expect("run worker");
23811 }
23812
23813 #[tokio::test]
23814 async fn workflow_only_worker_can_shutdown_without_activity_poller() {
23815 let server = MockWorkerServer::start();
23816 let client = Client::builder(server.base_url())
23817 .timeout(Duration::from_secs(2))
23818 .build()
23819 .expect("client");
23820 let mut worker = Worker::new(client, "rust-workers")
23821 .worker_id("workflow-only-worker")
23822 .poll_timeout(Duration::from_millis(10));
23823
23824 worker.register_workflow(
23825 "workflow.only",
23826 |_ctx, _input| async move { Ok(Value::Null) },
23827 );
23828
23829 worker.run_until(async {}).await.expect("run worker");
23830 }
23831
23832 #[tokio::test]
23833 async fn worker_heartbeat_observer_receives_server_acknowledgements() {
23834 let server = MockWorkerServer::start();
23835 let client = Client::builder(server.base_url())
23836 .timeout(Duration::from_secs(2))
23837 .build()
23838 .expect("client");
23839 let observations = Arc::new(Mutex::new(Vec::new()));
23840 let observed = Arc::clone(&observations);
23841 let mut worker = Worker::new(client, "rust-workers")
23842 .worker_id("observed-heartbeat-worker")
23843 .poll_timeout(Duration::from_millis(10))
23844 .on_worker_heartbeat(move |observation| {
23845 observed
23846 .lock()
23847 .expect("heartbeat observations")
23848 .push(observation.clone());
23849 });
23850
23851 worker.register_workflow("workflow.observed", |_ctx, _input| async move {
23852 Ok(Value::Null)
23853 });
23854 let acknowledged = Arc::clone(&observations);
23855 worker
23856 .run_until(async move {
23857 tokio::time::timeout(Duration::from_secs(2), async move {
23858 loop {
23859 if !acknowledged
23860 .lock()
23861 .expect("heartbeat observations")
23862 .is_empty()
23863 {
23864 break;
23865 }
23866 tokio::time::sleep(Duration::from_millis(1)).await;
23867 }
23868 })
23869 .await
23870 .expect("heartbeat acknowledgement within timeout");
23871 })
23872 .await
23873 .expect("run worker");
23874
23875 let observations = observations.lock().expect("heartbeat observations");
23876 let first = observations.first().expect("heartbeat acknowledgement");
23877 assert_eq!(first.worker_id, "observed-heartbeat-worker");
23878 assert_eq!(first.task_queue, "rust-workers");
23879 assert!(first.acknowledged_at_unix_millis > 0);
23880 assert_eq!(first.acknowledgement, json!({}));
23881 }
23882
23883 #[tokio::test]
23884 async fn delayed_worker_heartbeat_keeps_cadence_and_pollers_live() {
23885 let server = MockWorkerServer::delayed_heartbeat_worker();
23886 let client = Client::builder(server.base_url())
23887 .timeout(Duration::from_secs(3))
23888 .build()
23889 .expect("client");
23890 let observations = Arc::new(Mutex::new(Vec::new()));
23891 let observed = Arc::clone(&observations);
23892 let mut worker = Worker::new(client, "rust-snapshot-workers")
23893 .worker_id("rust-snapshot-worker")
23894 .poll_timeout(Duration::from_millis(10))
23895 .on_worker_heartbeat(move |observation| {
23896 observed
23897 .lock()
23898 .expect("heartbeat observations")
23899 .push(observation.clone());
23900 });
23901
23902 worker.register_workflow("snapshot", |ctx, _input| async move {
23903 ctx.wait_signal("finish").await?;
23904 Ok(json!({"status": "finished"}))
23905 });
23906 worker.register_query("snapshot", "current", |ctx, _args| async move {
23907 Ok(json!(ctx
23908 .signals("increment")
23909 .iter()
23910 .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
23911 .sum::<i64>()))
23912 });
23913 worker.register_activity("cancel-aware", |_ctx, _args| async move {
23914 Ok(json!({"late": "completion"}))
23915 });
23916
23917 worker
23918 .run_until(tokio::time::sleep(Duration::from_millis(3_800)))
23919 .await
23920 .expect("delayed heartbeat must allow a clean worker shutdown");
23921
23922 let observations = observations.lock().expect("heartbeat observations");
23923 assert!(
23924 observations.len() >= 3,
23925 "the immediate heartbeat, delayed acknowledgement, and next cadence heartbeat must complete"
23926 );
23927 assert!(
23928 observations.windows(2).all(|pair| {
23929 pair[1].acknowledged_at_unix_millis
23930 .saturating_sub(pair[0].acknowledged_at_unix_millis)
23931 >= 850
23932 }),
23933 "successful acknowledgements must not catch up faster than the advertised one-second cadence: {observations:?}"
23934 );
23935 drop(observations);
23936
23937 let heartbeat_times = server.request_times("/api/worker/heartbeat");
23938 let delayed_request_at = *heartbeat_times
23939 .get(1)
23940 .expect("intentionally delayed heartbeat request");
23941 let delay_window_start = delayed_request_at + Duration::from_millis(100);
23942 let delay_window_end = delayed_request_at + Duration::from_millis(1_400);
23943 for path in [
23944 "/api/worker/workflow-tasks/poll",
23945 "/api/worker/activity-tasks/poll",
23946 "/api/worker/query-tasks/poll",
23947 ] {
23948 assert!(
23949 server
23950 .request_times(path)
23951 .iter()
23952 .any(|received_at| *received_at >= delay_window_start
23953 && *received_at <= delay_window_end),
23954 "{path} must keep polling while a heartbeat acknowledgement is delayed"
23955 );
23956 }
23957 assert!(
23958 server.request_count("/api/worker/workflow-tasks/snapshot-wait-3/fail") >= 1,
23959 "workflow work must be settled"
23960 );
23961 assert!(
23962 server.request_count("/api/worker/activity-tasks/activity-cancel/complete") >= 1,
23963 "activity work must be settled"
23964 );
23965 assert!(
23966 server.request_count("/api/worker/query-tasks/snapshot-current/complete") >= 1,
23967 "query work must be settled"
23968 );
23969 }
23970
23971 #[tokio::test]
23972 async fn retried_worker_heartbeat_restarts_the_advertised_cadence() {
23973 let server = MockWorkerServer::heartbeat_retry_worker();
23974 let client = Client::builder(server.base_url())
23975 .timeout(Duration::from_secs(2))
23976 .build()
23977 .expect("client");
23978 let observations = Arc::new(Mutex::new(Vec::new()));
23979 let observed = Arc::clone(&observations);
23980 let worker = Worker::new(client, "rust-workers")
23981 .worker_id("heartbeat-retry-worker")
23982 .retry_policy(WorkerRetryPolicy {
23983 max_retries: 1,
23984 initial_backoff: Duration::from_millis(300),
23985 max_backoff: Duration::from_millis(300),
23986 })
23987 .on_worker_heartbeat(move |observation| {
23988 observed
23989 .lock()
23990 .expect("heartbeat observations")
23991 .push(observation.clone());
23992 });
23993
23994 worker
23995 .run_until(tokio::time::sleep(Duration::from_millis(2_700)))
23996 .await
23997 .expect("retryable heartbeat failure must remain bounded and recover");
23998
23999 let observations = observations.lock().expect("heartbeat observations");
24000 assert!(observations.len() >= 3, "heartbeat retry must recover");
24001 assert!(
24002 observations.windows(2).all(|pair| {
24003 pair[1]
24004 .acknowledged_at_unix_millis
24005 .saturating_sub(pair[0].acknowledged_at_unix_millis)
24006 >= 850
24007 }),
24008 "a successful retry must start a fresh advertised cadence: {observations:?}"
24009 );
24010 assert_eq!(
24011 server.request_count("/api/worker/heartbeat"),
24012 observations.len() + 1,
24013 "one retryable failure must add exactly one bounded request"
24014 );
24015 }
24016
24017 #[tokio::test]
24018 async fn query_enabled_worker_ignores_unmatched_signals_then_completes_once() {
24019 let server = MockWorkerServer::waiting_query_worker();
24020 let client = Client::builder(server.base_url())
24021 .timeout(Duration::from_secs(2))
24022 .build()
24023 .expect("client");
24024 let observations = Arc::new(Mutex::new(Vec::new()));
24025 let observed = Arc::clone(&observations);
24026 let mut worker = Worker::new(client, "rust-snapshot-workers")
24027 .worker_id("rust-snapshot-worker")
24028 .poll_timeout(Duration::from_millis(10))
24029 .on_worker_heartbeat(move |observation| {
24030 observed
24031 .lock()
24032 .expect("heartbeat observations")
24033 .push(observation.clone());
24034 });
24035
24036 worker.register_workflow("snapshot", |ctx, _input| async move {
24037 ctx.wait_signal("finish").await?;
24038 Ok(json!({"status": "finished"}))
24039 });
24040 worker.register_query("snapshot", "current", |ctx, _args| async move {
24041 let current = ctx
24042 .signals("increment")
24043 .iter()
24044 .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
24045 .sum::<i64>();
24046 Ok(json!(current))
24047 });
24048 worker.register_update("snapshot", "replace", |_ctx, args| async move { Ok(args) });
24049
24050 worker
24051 .run_until(tokio::time::sleep(Duration::from_millis(3_200)))
24052 .await
24053 .expect("pending workflow and query poller must remain live until shutdown");
24054
24055 assert!(
24056 observations.lock().expect("heartbeat observations").len() >= 4,
24057 "the immediate heartbeat and at least three advertised one-second intervals must be acknowledged"
24058 );
24059 assert!(
24060 server.request_count("/api/worker/workflow-tasks/poll") >= 3,
24061 "workflow polling must continue after empty replay acknowledgements"
24062 );
24063 assert!(
24064 server.request_count("/api/worker/query-tasks/poll") >= 2,
24065 "query polling must continue after serving the current query"
24066 );
24067 assert_eq!(
24068 server.request_body("/api/worker/register")["capabilities"],
24069 json!([
24070 CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY,
24071 DURABLE_SELECTION_CAPABILITY,
24072 MEMO_UPSERTS_CAPABILITY,
24073 TYPED_SEARCH_ATTRIBUTES_CAPABILITY,
24074 QUERY_TASKS_CAPABILITY,
24075 WORKFLOW_UPDATES_CAPABILITY,
24076 MESSAGE_STREAMS_CAPABILITY
24077 ])
24078 );
24079 assert_eq!(
24080 server.request_body("/api/worker/register")["workflow_command_contracts"]["snapshot"],
24081 json!({
24082 "queries": ["current"],
24083 "query_contracts": [],
24084 "signals": [],
24085 "signal_contracts": [],
24086 "updates": ["replace"],
24087 "update_contracts": [],
24088 "update_validators": [],
24089 })
24090 );
24091
24092 let opened = server.request_body("/api/worker/workflow-tasks/snapshot-open/complete");
24093 assert_eq!(
24094 opened["commands"],
24095 json!([{
24096 "type": "open_signal_wait",
24097 "signal_name": "finish",
24098 }])
24099 );
24100
24101 for task_id in ["snapshot-wait-3", "snapshot-wait-5"] {
24102 let fail_path = format!("/api/worker/workflow-tasks/{task_id}/fail");
24103 let completion_path = format!("/api/worker/workflow-tasks/{task_id}/complete");
24104 let failure = server.request_body(&fail_path);
24105 assert_eq!(
24106 failure["failure"]["type"],
24107 WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE
24108 );
24109 assert_eq!(server.request_count(&completion_path), 0);
24110 }
24111
24112 let query_completion =
24113 server.request_body("/api/worker/query-tasks/snapshot-current/complete");
24114 assert_eq!(query_completion["result"], json!(8));
24115
24116 let terminal_path = "/api/worker/workflow-tasks/snapshot-finish/complete";
24117 assert_eq!(
24118 server.request_count(terminal_path),
24119 1,
24120 "the matching signal must settle the workflow exactly once"
24121 );
24122 let terminal = server.request_body(terminal_path);
24123 assert_eq!(terminal["commands"].as_array().map(Vec::len), Some(1));
24124 assert_eq!(terminal["commands"][0]["type"], "complete_workflow");
24125 assert_eq!(
24126 decode_wire_value(&terminal["commands"][0]["result"], DEFAULT_CODEC)
24127 .expect("terminal workflow result"),
24128 json!({"status": "finished"})
24129 );
24130 }
24131
24132 #[tokio::test]
24133 async fn worker_retries_poll_and_heartbeat_transport_failures_independently() {
24134 let server = MockWorkerServer::transient_worker_failures();
24135 let client = Client::builder(server.base_url())
24136 .timeout(Duration::from_secs(2))
24137 .build()
24138 .expect("client");
24139 let mut worker = Worker::new(client, "rust-workers")
24140 .worker_id("retry-worker")
24141 .poll_timeout(Duration::from_millis(10))
24142 .retry_policy(WorkerRetryPolicy {
24143 max_retries: 2,
24144 initial_backoff: Duration::from_millis(1),
24145 max_backoff: Duration::from_millis(1),
24146 });
24147 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
24148 worker.register_activity(
24149 "counter.activity",
24150 |_ctx, _input| async move { Ok(Value::Null) },
24151 );
24152 worker.register_query(
24153 "counter",
24154 "current",
24155 |_ctx, _args| async move { Ok(json!(8)) },
24156 );
24157
24158 worker
24159 .run_until(tokio::time::sleep(Duration::from_millis(75)))
24160 .await
24161 .expect("transient failures must not stop the worker");
24162
24163 for path in [
24164 "/api/worker/heartbeat",
24165 "/api/worker/workflow-tasks/poll",
24166 "/api/worker/activity-tasks/poll",
24167 "/api/worker/query-tasks/poll",
24168 ] {
24169 assert!(
24170 server.request_count(path) >= 2,
24171 "{path} must continue after its transient failure"
24172 );
24173 }
24174 }
24175
24176 #[tokio::test]
24177 async fn worker_continues_after_long_poll_capacity_backpressure() {
24178 let server = MockWorkerServer::capacity_limited_activity_poll();
24179 let client = Client::builder(server.base_url())
24180 .timeout(Duration::from_secs(2))
24181 .build()
24182 .expect("client");
24183 let mut worker = Worker::new(client, "rust-workers")
24184 .worker_id("capacity-worker")
24185 .poll_timeout(Duration::from_millis(10))
24186 .retry_policy(WorkerRetryPolicy {
24187 max_retries: 0,
24188 initial_backoff: Duration::from_millis(1),
24189 max_backoff: Duration::from_millis(1),
24190 });
24191 worker.register_activity("capacity.activity", |_ctx, _input| async move {
24192 Ok(json!({"handled": true}))
24193 });
24194
24195 worker
24196 .run_until(tokio::time::sleep(Duration::from_millis(50)))
24197 .await
24198 .expect("capacity backpressure must not stop the worker");
24199
24200 assert!(
24201 server.request_count("/api/worker/activity-tasks/poll") >= 2,
24202 "the activity poller must continue after capacity backpressure"
24203 );
24204 assert_eq!(
24205 server.request_count("/api/worker/activity-tasks/capacity-activity/complete"),
24206 1,
24207 "the worker must complete work returned after capacity recovers"
24208 );
24209 }
24210
24211 #[test]
24212 fn worker_poll_capacity_backpressure_requires_the_typed_retryable_contract() {
24213 let capacity = Error::Http {
24214 status: reqwest::StatusCode::TOO_MANY_REQUESTS,
24215 body: r#"{"poll_status":"long_poll_capacity_exhausted","retryable":true,"retry_after_seconds":3}"#.to_string(),
24216 };
24217 assert_eq!(
24218 worker_poll_capacity_retry_after(&capacity),
24219 Some(Duration::from_secs(3))
24220 );
24221
24222 let rejected_capacity = Error::Http {
24223 status: reqwest::StatusCode::TOO_MANY_REQUESTS,
24224 body: r#"{"reason":"long_poll_capacity_exhausted","retryable":false,"retry_after_seconds":3}"#.to_string(),
24225 };
24226 assert_eq!(worker_poll_capacity_retry_after(&rejected_capacity), None);
24227 assert!(!worker_operation_is_retryable(&rejected_capacity));
24228
24229 let ordinary_rate_limit = Error::Http {
24230 status: reqwest::StatusCode::TOO_MANY_REQUESTS,
24231 body: r#"{"reason":"rate_limited","retryable":true,"retry_after_seconds":3}"#
24232 .to_string(),
24233 };
24234 assert_eq!(worker_poll_capacity_retry_after(&ordinary_rate_limit), None);
24235 assert!(worker_operation_is_retryable(&ordinary_rate_limit));
24236 }
24237
24238 #[tokio::test]
24239 async fn worker_bounds_transport_retries() {
24240 let server = MockWorkerServer::unavailable_polls();
24241 let client = Client::builder(server.base_url())
24242 .timeout(Duration::from_secs(2))
24243 .build()
24244 .expect("client");
24245 let mut worker = Worker::new(client, "rust-workers")
24246 .worker_id("bounded-retry-worker")
24247 .poll_timeout(Duration::from_millis(10))
24248 .retry_policy(WorkerRetryPolicy {
24249 max_retries: 2,
24250 initial_backoff: Duration::from_millis(1),
24251 max_backoff: Duration::from_millis(1),
24252 });
24253 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
24254
24255 let error = worker.run().await.expect_err("retry bound must terminate");
24256 assert!(matches!(error, Error::Transport(_)));
24257 assert_eq!(
24258 server.request_count("/api/worker/workflow-tasks/poll"),
24259 3,
24260 "one initial request plus exactly two retries"
24261 );
24262 }
24263
24264 #[tokio::test]
24265 async fn worker_retry_policy_can_disable_poll_retries() {
24266 let server = MockWorkerServer::unavailable_polls();
24267 let client = Client::builder(server.base_url())
24268 .timeout(Duration::from_secs(2))
24269 .build()
24270 .expect("client");
24271 let mut worker = Worker::new(client, "rust-workers")
24272 .worker_id("no-retry-worker")
24273 .poll_timeout(Duration::from_millis(10))
24274 .retry_policy(WorkerRetryPolicy {
24275 max_retries: 0,
24276 initial_backoff: Duration::from_millis(1),
24277 max_backoff: Duration::from_millis(1),
24278 });
24279 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
24280
24281 let error = worker
24282 .run_once()
24283 .await
24284 .expect_err("disabled retries must return the first transport failure");
24285 assert!(matches!(error, Error::Transport(_)));
24286 assert_eq!(
24287 server.request_count("/api/worker/workflow-tasks/poll"),
24288 1,
24289 "max_retries=0 must send only the initial request"
24290 );
24291 }
24292
24293 #[tokio::test]
24294 async fn worker_does_not_retry_authentication_failures() {
24295 let server = MockWorkerServer::unauthorized_polls();
24296 let client = Client::builder(server.base_url())
24297 .timeout(Duration::from_secs(2))
24298 .build()
24299 .expect("client");
24300 let mut worker = Worker::new(client, "rust-workers")
24301 .worker_id("unauthorized-worker")
24302 .poll_timeout(Duration::from_millis(10));
24303 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
24304
24305 let error = worker
24306 .run()
24307 .await
24308 .expect_err("authentication must terminate");
24309 let Error::Http { status, body } = error else {
24310 panic!("expected stable HTTP authentication error");
24311 };
24312 assert_eq!(status, reqwest::StatusCode::UNAUTHORIZED);
24313 assert!(body.contains("authentication_failed"));
24314 assert_eq!(
24315 server.request_count("/api/worker/workflow-tasks/poll"),
24316 1,
24317 "authentication failures must not be retried"
24318 );
24319 }
24320
24321 #[derive(Clone, Debug)]
24322 struct CapturedRequest {
24323 method: String,
24324 path: String,
24325 authorization: Option<String>,
24326 namespace: Option<String>,
24327 worker_protocol: Option<String>,
24328 control_protocol: Option<String>,
24329 body: String,
24330 received_at: Instant,
24331 }
24332
24333 struct MockWorkerServer {
24334 addr: SocketAddr,
24335 stop: Arc<AtomicBool>,
24336 requests: Arc<Mutex<Vec<CapturedRequest>>>,
24337 thread: Option<thread::JoinHandle<()>>,
24338 }
24339
24340 #[derive(Clone, Copy, Default)]
24341 struct MockWorkerBehavior {
24342 response_override: Option<fn(&str) -> Option<(&'static str, String)>>,
24343 storage_refusals: usize,
24344 storage_path: Option<&'static str>,
24345 storage_unavailable: bool,
24346 storage_mid_poll: bool,
24347 storage_activity: bool,
24348 storage_query: bool,
24349 storage_wrong_poll_id: bool,
24350 reject_query_protocol: bool,
24351 reject_query_completion: bool,
24352 waiting_query_worker: bool,
24353 decline_registration: bool,
24354 complete_named_signal: bool,
24355 poll_failures_per_path: usize,
24356 long_poll_capacity_responses_per_path: usize,
24357 heartbeat_failures: usize,
24358 heartbeat_failure_request: Option<usize>,
24359 delayed_heartbeat_request: Option<usize>,
24360 heartbeat_response_delay: Duration,
24361 concurrent_requests: bool,
24362 unauthorized_polls: bool,
24363 reject_registration: bool,
24364 reject_registration_protocol: bool,
24365 reject_deregistration: bool,
24366 reject_deregistration_protocol: bool,
24367 cancelled_activity: bool,
24368 draining_polls: bool,
24369 invalid_task_payload_codec: Option<InvalidTaskPayloadCodec>,
24370 workflow_completion_status: Option<&'static str>,
24371 workflow_completion_body: Option<&'static str>,
24372 }
24373
24374 impl MockWorkerServer {
24375 fn start() -> Self {
24376 Self::start_with_behavior(MockWorkerBehavior::default())
24377 }
24378
24379 fn reject_query_protocol() -> Self {
24380 Self::start_with_behavior(MockWorkerBehavior {
24381 reject_query_protocol: true,
24382 ..MockWorkerBehavior::default()
24383 })
24384 }
24385
24386 fn reject_query_completion() -> Self {
24387 Self::start_with_behavior(MockWorkerBehavior {
24388 reject_query_completion: true,
24389 ..MockWorkerBehavior::default()
24390 })
24391 }
24392
24393 fn waiting_query_worker() -> Self {
24394 Self::start_with_behavior(MockWorkerBehavior {
24395 waiting_query_worker: true,
24396 complete_named_signal: true,
24397 ..MockWorkerBehavior::default()
24398 })
24399 }
24400
24401 fn transient_worker_failures() -> Self {
24402 Self::start_with_behavior(MockWorkerBehavior {
24403 poll_failures_per_path: 1,
24404 heartbeat_failures: 1,
24405 ..MockWorkerBehavior::default()
24406 })
24407 }
24408
24409 fn consecutive_poll_failures(count: usize) -> Self {
24410 Self::start_with_behavior(MockWorkerBehavior {
24411 poll_failures_per_path: count,
24412 ..MockWorkerBehavior::default()
24413 })
24414 }
24415
24416 fn capacity_limited_activity_poll() -> Self {
24417 Self::start_with_behavior(MockWorkerBehavior {
24418 long_poll_capacity_responses_per_path: 1,
24419 ..MockWorkerBehavior::default()
24420 })
24421 }
24422
24423 fn delayed_heartbeat_worker() -> Self {
24424 Self::start_with_behavior(MockWorkerBehavior {
24425 waiting_query_worker: true,
24426 delayed_heartbeat_request: Some(2),
24427 heartbeat_response_delay: Duration::from_millis(1_500),
24428 concurrent_requests: true,
24429 cancelled_activity: true,
24430 ..MockWorkerBehavior::default()
24431 })
24432 }
24433
24434 fn heartbeat_retry_worker() -> Self {
24435 Self::start_with_behavior(MockWorkerBehavior {
24436 waiting_query_worker: true,
24437 heartbeat_failure_request: Some(2),
24438 concurrent_requests: true,
24439 ..MockWorkerBehavior::default()
24440 })
24441 }
24442
24443 fn unavailable_polls() -> Self {
24444 Self::start_with_behavior(MockWorkerBehavior {
24445 poll_failures_per_path: usize::MAX,
24446 ..MockWorkerBehavior::default()
24447 })
24448 }
24449
24450 fn unauthorized_polls() -> Self {
24451 Self::start_with_behavior(MockWorkerBehavior {
24452 unauthorized_polls: true,
24453 ..MockWorkerBehavior::default()
24454 })
24455 }
24456
24457 fn rejected_registration() -> Self {
24458 Self::start_with_behavior(MockWorkerBehavior {
24459 reject_registration: true,
24460 ..MockWorkerBehavior::default()
24461 })
24462 }
24463
24464 fn rejected_registration_protocol() -> Self {
24465 Self::start_with_behavior(MockWorkerBehavior {
24466 reject_registration_protocol: true,
24467 ..MockWorkerBehavior::default()
24468 })
24469 }
24470
24471 fn declined_registration() -> Self {
24472 Self::start_with_behavior(MockWorkerBehavior {
24473 decline_registration: true,
24474 ..MockWorkerBehavior::default()
24475 })
24476 }
24477
24478 fn rejected_deregistration() -> Self {
24479 Self::start_with_behavior(MockWorkerBehavior {
24480 reject_deregistration: true,
24481 ..MockWorkerBehavior::default()
24482 })
24483 }
24484
24485 fn rejected_deregistration_protocol() -> Self {
24486 Self::start_with_behavior(MockWorkerBehavior {
24487 reject_deregistration_protocol: true,
24488 ..MockWorkerBehavior::default()
24489 })
24490 }
24491
24492 fn unauthorized_polls_and_rejected_deregistration() -> Self {
24493 Self::start_with_behavior(MockWorkerBehavior {
24494 unauthorized_polls: true,
24495 reject_deregistration: true,
24496 ..MockWorkerBehavior::default()
24497 })
24498 }
24499
24500 fn cancelled_activity() -> Self {
24501 Self::start_with_behavior(MockWorkerBehavior {
24502 cancelled_activity: true,
24503 ..MockWorkerBehavior::default()
24504 })
24505 }
24506
24507 fn draining_polls() -> Self {
24508 Self::start_with_behavior(MockWorkerBehavior {
24509 draining_polls: true,
24510 ..MockWorkerBehavior::default()
24511 })
24512 }
24513
24514 fn invalid_task_payload_codec(codec: InvalidTaskPayloadCodec) -> Self {
24515 Self::start_with_behavior(MockWorkerBehavior {
24516 invalid_task_payload_codec: Some(codec),
24517 ..MockWorkerBehavior::default()
24518 })
24519 }
24520
24521 fn workflow_completion(status: &'static str, body: &'static str) -> Self {
24522 Self::start_with_behavior(MockWorkerBehavior {
24523 workflow_completion_status: Some(status),
24524 workflow_completion_body: Some(body),
24525 ..MockWorkerBehavior::default()
24526 })
24527 }
24528
24529 fn start_with_behavior(behavior: MockWorkerBehavior) -> Self {
24530 let listener = TcpListener::bind("127.0.0.1:0").expect("bind mock server");
24531 listener
24532 .set_nonblocking(true)
24533 .expect("configure mock listener");
24534 let addr = listener.local_addr().expect("mock server address");
24535 let stop = Arc::new(AtomicBool::new(false));
24536 let server_stop = Arc::clone(&stop);
24537 let requests = Arc::new(Mutex::new(Vec::new()));
24538 let server_requests = Arc::clone(&requests);
24539 let thread = thread::spawn(move || {
24540 let mut request_threads = Vec::new();
24541 while !server_stop.load(Ordering::SeqCst) {
24542 match listener.accept() {
24543 Ok((mut stream, _)) => {
24544 if behavior.concurrent_requests {
24545 let requests = Arc::clone(&server_requests);
24546 request_threads.push(thread::spawn(move || {
24547 handle_mock_worker_request(&mut stream, &requests, behavior)
24548 }));
24549 } else {
24550 handle_mock_worker_request(&mut stream, &server_requests, behavior);
24551 }
24552 }
24553 Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
24554 let mut index = 0;
24555 while index < request_threads.len() {
24556 if request_threads[index].is_finished() {
24557 request_threads
24558 .swap_remove(index)
24559 .join()
24560 .expect("join mock request");
24561 } else {
24562 index += 1;
24563 }
24564 }
24565 thread::sleep(Duration::from_millis(5));
24566 }
24567 Err(_) => break,
24568 }
24569 }
24570 for request_thread in request_threads {
24571 request_thread.join().expect("join mock request");
24572 }
24573 });
24574
24575 Self {
24576 addr,
24577 stop,
24578 requests,
24579 thread: Some(thread),
24580 }
24581 }
24582
24583 fn base_url(&self) -> String {
24584 format!("http://{}", self.addr)
24585 }
24586
24587 fn worker_protocol_for(&self, path: &str) -> Option<String> {
24588 self.requests
24589 .lock()
24590 .expect("captured requests")
24591 .iter()
24592 .find(|request| request.path == path)
24593 .and_then(|request| request.worker_protocol.clone())
24594 }
24595
24596 fn control_protocol_for(&self, path: &str) -> Option<String> {
24597 self.requests
24598 .lock()
24599 .expect("captured requests")
24600 .iter()
24601 .find(|request| request.path == path)
24602 .and_then(|request| request.control_protocol.clone())
24603 }
24604
24605 fn method_for(&self, path: &str) -> Option<String> {
24606 self.requests
24607 .lock()
24608 .expect("captured requests")
24609 .iter()
24610 .find(|request| request.path == path)
24611 .map(|request| request.method.clone())
24612 }
24613
24614 fn authorization_for(&self, path: &str) -> Option<String> {
24615 self.requests
24616 .lock()
24617 .expect("captured requests")
24618 .iter()
24619 .find(|request| request.path == path)
24620 .and_then(|request| request.authorization.clone())
24621 }
24622
24623 fn namespace_for(&self, path: &str) -> Option<String> {
24624 self.requests
24625 .lock()
24626 .expect("captured requests")
24627 .iter()
24628 .find(|request| request.path == path)
24629 .and_then(|request| request.namespace.clone())
24630 }
24631
24632 fn request_count(&self, path: &str) -> usize {
24633 self.requests
24634 .lock()
24635 .expect("captured requests")
24636 .iter()
24637 .filter(|request| request.path == path)
24638 .count()
24639 }
24640
24641 fn captured_paths(&self) -> Vec<String> {
24642 self.requests
24643 .lock()
24644 .expect("captured requests")
24645 .iter()
24646 .map(|request| request.path.clone())
24647 .collect()
24648 }
24649
24650 fn request_times(&self, path: &str) -> Vec<Instant> {
24651 self.requests
24652 .lock()
24653 .expect("captured requests")
24654 .iter()
24655 .filter(|request| request.path == path)
24656 .map(|request| request.received_at)
24657 .collect()
24658 }
24659
24660 fn request_body(&self, path: &str) -> Value {
24661 let requests = self.requests.lock().expect("captured requests");
24662 let body = &requests
24663 .iter()
24664 .find(|request| request.path == path)
24665 .unwrap_or_else(|| panic!("missing request for {path}"))
24666 .body;
24667 serde_json::from_str(body).unwrap_or_else(|error| {
24668 panic!("invalid JSON request body for {path}: {error}: {body:?}")
24669 })
24670 }
24671
24672 fn request_bodies(&self, path: &str) -> Vec<Value> {
24673 self.requests
24674 .lock()
24675 .expect("captured requests")
24676 .iter()
24677 .filter(|request| request.path == path)
24678 .map(|request| {
24679 serde_json::from_str(&request.body).unwrap_or_else(|error| {
24680 panic!(
24681 "invalid JSON request body for {path}: {error}: {:?}",
24682 request.body
24683 )
24684 })
24685 })
24686 .collect()
24687 }
24688 }
24689
24690 impl Drop for MockWorkerServer {
24691 fn drop(&mut self) {
24692 self.stop.store(true, Ordering::SeqCst);
24693 let _ = TcpStream::connect(self.addr);
24694
24695 if let Some(thread) = self.thread.take() {
24696 thread.join().expect("join mock server");
24697 }
24698 }
24699 }
24700
24701 fn handle_mock_worker_request(
24702 stream: &mut TcpStream,
24703 requests: &Arc<Mutex<Vec<CapturedRequest>>>,
24704 behavior: MockWorkerBehavior,
24705 ) {
24706 let _ = stream.set_read_timeout(Some(Duration::from_millis(200)));
24707 let mut buffer = [0_u8; 8192];
24708 let mut request = Vec::new();
24709
24710 loop {
24711 match stream.read(&mut buffer) {
24712 Ok(0) => break,
24713 Ok(read) => {
24714 request.extend_from_slice(&buffer[..read]);
24715 if mock_request_is_complete(&request) {
24716 break;
24717 }
24718 }
24719 Err(error)
24720 if matches!(
24721 error.kind(),
24722 std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut
24723 ) =>
24724 {
24725 break;
24726 }
24727 Err(_) => return,
24728 }
24729 }
24730
24731 let request = String::from_utf8_lossy(&request);
24732 let body = request
24733 .split_once("\r\n\r\n")
24734 .map(|(_, body)| body)
24735 .unwrap_or_default();
24736 let path = request
24737 .lines()
24738 .next()
24739 .and_then(|line| line.split_whitespace().nth(1))
24740 .unwrap_or_default();
24741 let method = request
24742 .lines()
24743 .next()
24744 .and_then(|line| line.split_whitespace().next())
24745 .unwrap_or_default();
24746 let authorization = request.lines().find_map(|line| {
24747 let (name, value) = line.split_once(':')?;
24748 name.eq_ignore_ascii_case("Authorization")
24749 .then(|| value.trim().to_string())
24750 });
24751 let namespace = request.lines().find_map(|line| {
24752 let (name, value) = line.split_once(':')?;
24753 name.eq_ignore_ascii_case("X-Namespace")
24754 .then(|| value.trim().to_string())
24755 });
24756 let worker_protocol = request.lines().find_map(|line| {
24757 let (name, value) = line.split_once(':')?;
24758 name.eq_ignore_ascii_case("X-Durable-Workflow-Protocol-Version")
24759 .then(|| value.trim().to_string())
24760 });
24761 let control_protocol = request.lines().find_map(|line| {
24762 let (name, value) = line.split_once(':')?;
24763 name.eq_ignore_ascii_case("X-Durable-Workflow-Control-Plane-Version")
24764 .then(|| value.trim().to_string())
24765 });
24766 let request_number = {
24767 let mut requests = requests.lock().expect("captured requests");
24768 requests.push(CapturedRequest {
24769 method: method.to_string(),
24770 path: path.to_string(),
24771 authorization,
24772 namespace,
24773 worker_protocol: worker_protocol.clone(),
24774 control_protocol,
24775 body: body.to_string(),
24776 received_at: Instant::now(),
24777 });
24778 requests
24779 .iter()
24780 .filter(|request| request.path == path)
24781 .count()
24782 };
24783
24784 if let Some(response) = behavior.response_override.and_then(|handler| handler(path)) {
24785 write_mock_response(stream, response.0, &response.1);
24786 return;
24787 }
24788 if path.ends_with("/poll") && request_number <= behavior.poll_failures_per_path {
24789 return;
24790 }
24791 let pressure_path = behavior
24792 .storage_path
24793 .is_some_and(|part| path.contains(part));
24794 let prior_failures = if path.ends_with("/poll") {
24795 behavior.poll_failures_per_path
24796 } else {
24797 0
24798 };
24799 if pressure_path
24800 && request_number.saturating_sub(prior_failures) <= behavior.storage_refusals
24801 {
24802 let request_body: Value = serde_json::from_str(body).unwrap_or(Value::Null);
24803 let poll_id = path
24804 .ends_with("/poll")
24805 .then(|| request_body["poll_request_id"].as_str().unwrap_or(""));
24806 let mut refusal = storage_refusal(
24807 poll_id,
24808 behavior.storage_unavailable,
24809 behavior.storage_mid_poll,
24810 );
24811 if behavior.storage_wrong_poll_id {
24812 refusal["poll_request_id"] = json!("wrong-poll");
24813 }
24814 write_mock_response(stream, "503 Service Unavailable", &refusal.to_string());
24815 return;
24816 }
24817 if path.contains("/storage-task/")
24818 || path.contains("/storage-activity/")
24819 || path.contains("/storage-query/")
24820 {
24821 write_mock_response(stream, "200 OK", "{}");
24822 return;
24823 }
24824 if behavior.storage_query && path == "/api/worker/query-tasks/poll" && request_number == 1 {
24825 write_mock_response(stream, "200 OK", &json!({"task":{
24826 "query_task_id":"storage-query", "query_task_attempt":7, "workflow_type":"storage.workflow",
24827 "query_name":"state", "workflow_id":"workflow", "run_id":"run", "payload_codec":"avro",
24828 "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC).unwrap(),
24829 "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC).unwrap(),
24830 "history_events":[], "run_status":"waiting", "lease_owner":"storage-worker"
24831 }}).to_string());
24832 return;
24833 }
24834 if behavior.storage_activity
24835 && path == "/api/worker/activity-tasks/poll"
24836 && request_number == 1
24837 {
24838 write_mock_response(stream, "200 OK", &json!({"task":{
24839 "task_id":"storage-activity", "activity_attempt_id":"storage-attempt", "activity_type":"storage.activity",
24840 "payload_codec":"avro", "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC).unwrap(),
24841 "attempt_number":7, "lease_owner":"storage-worker"
24842 }}).to_string());
24843 return;
24844 }
24845
24846 if path == "/api/worker/register" {
24847 if behavior.reject_registration_protocol {
24848 write_mock_response(
24849 stream,
24850 "400 Bad Request",
24851 r#"{"reason":"unsupported_protocol_version","message":"condition-wait occurrence identity requires worker protocol 1.17","supported_version":"1.16","requested_version":"1.17"}"#,
24852 );
24853 return;
24854 }
24855 if behavior.reject_registration {
24856 write_mock_response(
24857 stream,
24858 "503 Service Unavailable",
24859 r#"{"reason":"registration_unavailable","message":"registration failed"}"#,
24860 );
24861 return;
24862 }
24863 }
24864
24865 if path.starts_with("/api/worker/registrations/") {
24866 if behavior.reject_deregistration_protocol {
24867 write_mock_response(
24868 stream,
24869 "400 Bad Request",
24870 r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.17","requested_version":"1.19"}"#,
24871 );
24872 } else if behavior.reject_deregistration {
24873 write_mock_response(
24874 stream,
24875 "403 Forbidden",
24876 r#"{"reason":"authorization_failed","message":"worker cannot deregister"}"#,
24877 );
24878 } else {
24879 write_mock_response(
24880 stream,
24881 "200 OK",
24882 r#"{"worker_id":"deregistered-worker","outcome":"deregistered","recovered_workflow_task_count":2}"#,
24883 );
24884 }
24885 return;
24886 }
24887
24888 let is_poll = matches!(
24889 path,
24890 "/api/worker/workflow-tasks/poll"
24891 | "/api/worker/activity-tasks/poll"
24892 | "/api/worker/query-tasks/poll"
24893 );
24894 if is_poll && request_number <= behavior.long_poll_capacity_responses_per_path {
24895 write_mock_response(
24896 stream,
24897 "429 Too Many Requests",
24898 r#"{"task":null,"poll_status":"long_poll_capacity_exhausted","reason":"long_poll_capacity_exhausted","retryable":true,"retry_after_seconds":1}"#,
24899 );
24900 return;
24901 }
24902 if is_poll && request_number <= behavior.poll_failures_per_path {
24903 return;
24904 }
24905 if path == "/api/worker/heartbeat" && request_number <= behavior.heartbeat_failures {
24906 return;
24907 }
24908 if path == "/api/worker/heartbeat"
24909 && behavior.heartbeat_failure_request == Some(request_number)
24910 {
24911 return;
24912 }
24913 if path == "/api/worker/heartbeat"
24914 && behavior.delayed_heartbeat_request == Some(request_number)
24915 {
24916 thread::sleep(behavior.heartbeat_response_delay);
24917 }
24918 if behavior.unauthorized_polls && is_poll {
24919 write_mock_response(
24920 stream,
24921 "401 Unauthorized",
24922 r#"{"reason":"authentication_failed","message":"invalid worker token"}"#,
24923 );
24924 return;
24925 }
24926 if behavior.draining_polls && is_poll {
24927 write_mock_response(
24928 stream,
24929 "409 Conflict",
24930 r#"{"task":null,"poll_status":"draining","reason":"worker_draining","worker_status":"draining","drain_intent":"draining"}"#,
24931 );
24932 return;
24933 }
24934
24935 if let Some(codec_case) = behavior.invalid_task_payload_codec {
24936 if is_poll && request_number == 1 {
24937 let mut task = match path {
24938 "/api/worker/workflow-tasks/poll" => json!({
24939 "task_id": "codec-workflow",
24940 "workflow_type": "codec.workflow",
24941 "payload_codec": DEFAULT_CODEC,
24942 "workflow_task_attempt": 1,
24943 "lease_owner": "codec-worker"
24944 }),
24945 "/api/worker/activity-tasks/poll" => json!({
24946 "task_id": "codec-activity",
24947 "activity_attempt_id": "codec-activity-attempt",
24948 "activity_type": "codec.activity",
24949 "payload_codec": DEFAULT_CODEC,
24950 "attempt_number": 1,
24951 "lease_owner": "codec-worker"
24952 }),
24953 "/api/worker/query-tasks/poll" => json!({
24954 "query_task_id": "codec-query",
24955 "query_task_attempt": 1,
24956 "workflow_type": "codec.workflow",
24957 "query_name": "known",
24958 "payload_codec": DEFAULT_CODEC,
24959 "lease_owner": "codec-worker"
24960 }),
24961 _ => unreachable!("is_poll limits task codec probe paths"),
24962 };
24963 codec_case.apply(&mut task);
24964 write_mock_response(stream, "200 OK", &json!({"task": task}).to_string());
24965 return;
24966 }
24967
24968 if matches!(
24969 path,
24970 "/api/worker/workflow-tasks/codec-workflow/fail"
24971 | "/api/worker/activity-tasks/codec-activity/fail"
24972 | "/api/worker/query-tasks/codec-query/fail"
24973 ) {
24974 write_mock_response(stream, "200 OK", r#"{"outcome":"failed"}"#);
24975 return;
24976 }
24977 }
24978
24979 if behavior.reject_query_protocol && path.starts_with("/api/worker/query-tasks/") {
24980 let requested_version = worker_protocol.as_deref().unwrap_or("missing");
24981 let body = format!(
24982 r#"{{"reason":"unsupported_protocol_version","message":"Query tasks require worker protocol 1.8 or newer.","supported_version":"1.7","requested_version":"{requested_version}"}}"#
24983 );
24984 write_mock_response(stream, "400 Bad Request", &body);
24985 return;
24986 }
24987
24988 if behavior.reject_query_completion && path == "/api/worker/query-tasks/query-late/complete"
24989 {
24990 write_mock_response(
24991 stream,
24992 "409 Conflict",
24993 r#"{"reason":"query_task_timed_out","message":"query task timed out before completion"}"#,
24994 );
24995 return;
24996 }
24997
24998 if behavior.workflow_completion_status.is_some()
24999 && path == "/api/worker/workflow-tasks/poll"
25000 && request_number == 1
25001 {
25002 write_mock_response(
25003 stream,
25004 "200 OK",
25005 r#"{"task":{"task_id":"workflow-timeout-task","workflow_id":"reused-workflow-id","run_id":"run-selected-timeout","workflow_type":"timeout.workflow","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"history_events":[],"workflow_task_attempt":3,"lease_owner":"timeout-worker"}}"#,
25006 );
25007 return;
25008 }
25009
25010 if path == "/api/worker/workflow-tasks/workflow-timeout-task/complete" {
25011 if let (Some(status), Some(body)) = (
25012 behavior.workflow_completion_status,
25013 behavior.workflow_completion_body,
25014 ) {
25015 write_mock_response(stream, status, body);
25016 return;
25017 }
25018 }
25019
25020 if behavior.waiting_query_worker {
25021 if behavior.complete_named_signal
25022 && path == "/api/worker/workflow-tasks/poll"
25023 && request_number == 1
25024 {
25025 let body = json!({
25026 "task": {
25027 "task_id": "snapshot-open",
25028 "workflow_id": "snapshot-1",
25029 "run_id": "snapshot-run-1",
25030 "workflow_type": "snapshot",
25031 "payload_codec": DEFAULT_CODEC,
25032 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25033 .expect("Avro workflow arguments"),
25034 "history_events": [],
25035 "workflow_task_attempt": 1,
25036 "lease_owner": "rust-snapshot-worker"
25037 }
25038 })
25039 .to_string();
25040 write_mock_response(stream, "200 OK", &body);
25041 return;
25042 }
25043
25044 let signal_request = request_number - usize::from(behavior.complete_named_signal);
25045 let signal_request_limit = 2 + usize::from(behavior.complete_named_signal);
25046 if path == "/api/worker/workflow-tasks/poll"
25047 && signal_request >= 1
25048 && signal_request <= signal_request_limit
25049 {
25050 let finish = behavior.complete_named_signal && signal_request == 3;
25051 let amounts = if signal_request == 1 {
25052 vec![3]
25053 } else {
25054 vec![3, 5]
25055 };
25056 let task_id = if signal_request == 1 {
25057 "snapshot-wait-3"
25058 } else if finish {
25059 "snapshot-finish"
25060 } else {
25061 "snapshot-wait-5"
25062 };
25063 let mut history_events = std::iter::once(json!({
25064 "event_type": "SignalWaitOpened",
25065 "payload": {"sequence": 1, "signal_name": "finish"}
25066 }))
25067 .chain(amounts.iter().enumerate().map(|(index, amount)| {
25068 json!({
25069 "event_type": "SignalReceived",
25070 "payload": {
25071 "signal_id": format!("increment-{amount}"),
25072 "signal_name": "increment",
25073 "workflow_sequence": index + 2,
25074 "payload_codec": DEFAULT_CODEC,
25075 "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
25076 .expect("Avro signal envelope")
25077 }
25078 })
25079 }))
25080 .collect::<Vec<_>>();
25081 let (resume_id, resume_name, resume_arguments) = if finish {
25082 history_events.push(json!({
25083 "event_type": "SignalReceived",
25084 "payload": {
25085 "signal_id": "finish",
25086 "signal_name": "finish",
25087 "workflow_sequence": 4,
25088 "payload_codec": DEFAULT_CODEC,
25089 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25090 .expect("Avro finish signal envelope")
25091 }
25092 }));
25093 (
25094 "finish".to_string(),
25095 "finish".to_string(),
25096 encode_value_envelope(&json!([]), DEFAULT_CODEC)
25097 .expect("Avro finish resume signal"),
25098 )
25099 } else {
25100 let amount = amounts.last().expect("amount");
25101 (
25102 format!("increment-{amount}"),
25103 "increment".to_string(),
25104 encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
25105 .expect("Avro increment resume signal"),
25106 )
25107 };
25108 let body = json!({
25109 "task": {
25110 "task_id": task_id,
25111 "workflow_id": "snapshot-1",
25112 "run_id": "snapshot-run-1",
25113 "workflow_type": "snapshot",
25114 "payload_codec": DEFAULT_CODEC,
25115 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25116 .expect("Avro workflow arguments"),
25117 "history_events": history_events,
25118 "workflow_task_attempt": 1,
25119 "workflow_signal_id": resume_id,
25120 "signal_name": resume_name,
25121 "signal_arguments": resume_arguments,
25122 "lease_owner": "rust-snapshot-worker"
25123 }
25124 })
25125 .to_string();
25126 write_mock_response(stream, "200 OK", &body);
25127 return;
25128 }
25129
25130 if path == "/api/worker/query-tasks/poll" && request_number == 1 {
25131 let history_events = [3, 5]
25132 .into_iter()
25133 .enumerate()
25134 .map(|(index, amount)| {
25135 json!({
25136 "event_type": "SignalReceived",
25137 "payload": {
25138 "signal_id": format!("increment-{amount}"),
25139 "signal_name": "increment",
25140 "workflow_sequence": index + 2,
25141 "payload_codec": DEFAULT_CODEC,
25142 "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
25143 .expect("Avro query signal envelope")
25144 }
25145 })
25146 })
25147 .collect::<Vec<_>>();
25148 let body = json!({
25149 "task": {
25150 "query_task_id": "snapshot-current",
25151 "query_task_attempt": 1,
25152 "lease_owner": "rust-snapshot-worker",
25153 "workflow_id": "snapshot-1",
25154 "run_id": "snapshot-run-1",
25155 "workflow_type": "snapshot",
25156 "query_name": "current",
25157 "payload_codec": DEFAULT_CODEC,
25158 "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25159 .expect("Avro workflow arguments"),
25160 "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25161 .expect("Avro query arguments"),
25162 "history_events": history_events,
25163 "run_status": "waiting"
25164 }
25165 })
25166 .to_string();
25167 write_mock_response(stream, "200 OK", &body);
25168 return;
25169 }
25170
25171 if path == "/api/worker/workflow-tasks/snapshot-wait-3/fail"
25172 || path == "/api/worker/workflow-tasks/snapshot-wait-5/fail"
25173 {
25174 write_mock_response(
25175 stream,
25176 "200 OK",
25177 r#"{"outcome":"waiting_for_history","recorded":true}"#,
25178 );
25179 return;
25180 }
25181
25182 if path == "/api/worker/workflow-tasks/snapshot-open/complete" {
25183 write_mock_response(stream, "200 OK", r#"{"outcome":"waiting","recorded":true}"#);
25184 return;
25185 }
25186
25187 if path == "/api/worker/workflow-tasks/snapshot-finish/complete" {
25188 write_mock_response(
25189 stream,
25190 "200 OK",
25191 r#"{"outcome":"completed","run_status":"completed","recorded":true}"#,
25192 );
25193 return;
25194 }
25195
25196 if path == "/api/worker/query-tasks/snapshot-current/complete" {
25197 write_mock_response(stream, "200 OK", r#"{"outcome":"completed"}"#);
25198 return;
25199 }
25200 }
25201
25202 if matches!(
25203 path,
25204 "/api/workflows/typed-1/query/inspect" | "/api/workflows/typed-1/update/replace"
25205 ) {
25206 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
25207 .expect("typed mock result");
25208 let body = json!({
25209 "result": typed_fidelity_probe().into_json().expect("result projection"),
25210 "result_envelope": result,
25211 })
25212 .to_string();
25213 write_mock_response(stream, "200 OK", &body);
25214 return;
25215 }
25216
25217 if path == "/api/workflows/typed-1" {
25218 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
25219 .expect("typed mock result");
25220 let body = json!({
25221 "workflow_id": "typed-1",
25222 "run_id": "run-typed-1",
25223 "workflow_type": "typed.echo",
25224 "status": "completed",
25225 "output": typed_fidelity_probe().into_json().expect("output projection"),
25226 "output_envelope": result,
25227 })
25228 .to_string();
25229 write_mock_response(stream, "200 OK", &body);
25230 return;
25231 }
25232
25233 let (status, body) = match path {
25234 "/api/health" => ("200 OK", r#"{"status":"ok"}"#),
25235 "/api/workflows" => (
25236 "201 Created",
25237 r#"{"workflow_id":"wf-start-options","run_id":"run-start-options","workflow_type":"rust.timeout"}"#,
25238 ),
25239 "/api/worker/register" if behavior.decline_registration => (
25240 "200 OK",
25241 r#"{"worker_id":"declined-worker","registered":false}"#,
25242 ),
25243 "/api/worker/register" if behavior.waiting_query_worker => (
25244 "200 OK",
25245 r#"{"worker_id":"rust-snapshot-worker","registered":true,"heartbeat_interval_seconds":1}"#,
25246 ),
25247 "/api/worker/register" => (
25248 "200 OK",
25249 r#"{"worker_id":"mock-worker","registered":true,"heartbeat_interval_seconds":3600}"#,
25250 ),
25251 "/api/worker/heartbeat" => ("200 OK", "{}"),
25252 "/api/worker/activity-tasks/poll"
25253 if behavior.cancelled_activity && request_number == 1 =>
25254 {
25255 (
25256 "200 OK",
25257 r#"{"task":{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","activity_type":"cancel-aware","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"attempt_number":1,"lease_owner":"rust-cancel-worker"}}"#,
25258 )
25259 }
25260 "/api/worker/activity-tasks/poll"
25261 if behavior.long_poll_capacity_responses_per_path > 0
25262 && request_number
25263 == behavior
25264 .long_poll_capacity_responses_per_path
25265 .saturating_add(1) =>
25266 {
25267 (
25268 "200 OK",
25269 r#"{"task":{"task_id":"capacity-activity","activity_attempt_id":"capacity-attempt","activity_type":"capacity.activity","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"attempt_number":1,"lease_owner":"capacity-worker"}}"#,
25270 )
25271 }
25272 "/api/worker/activity-tasks/poll" | "/api/worker/workflow-tasks/poll" => {
25273 ("200 OK", r#"{"task":null}"#)
25274 }
25275 "/api/worker/query-tasks/poll"
25276 if behavior.reject_query_completion && request_number == 1 =>
25277 {
25278 (
25279 "200 OK",
25280 r#"{"task":{"query_task_id":"query-late","query_task_attempt":1,"lease_owner":"late-worker","workflow_id":"counter-late","run_id":"run-late","workflow_type":"counter","query_name":"current","payload_codec":"avro","workflow_arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"query_arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"history_events":[],"run_status":"running"}}"#,
25281 )
25282 }
25283 "/api/worker/query-tasks/poll" => ("200 OK", r#"{"task":null}"#),
25284 "/api/worker/query-tasks/query-capture/complete"
25285 | "/api/worker/query-tasks/query-capture/fail" => ("200 OK", "{}"),
25286 "/api/worker/activity-tasks/activity-cancel/heartbeat" => (
25287 "200 OK",
25288 r#"{"activity_attempt_id":"attempt-cancel","cancel_requested":true,"can_continue":false,"reason":"run_cancelled","run_closed_reason":"cancelled","heartbeat_recorded":false}"#,
25289 ),
25290 "/api/worker/activity-tasks/activity-cancel/complete" => (
25291 "409 Conflict",
25292 r#"{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","reason":"run_cancelled","cancel_requested":true,"can_continue":false,"run_closed_reason":"cancelled"}"#,
25293 ),
25294 "/api/worker/activity-tasks/activity-typed/complete"
25295 | "/api/worker/activity-tasks/activity-typed/fail"
25296 | "/api/worker/activity-tasks/capacity-activity/complete"
25297 | "/api/workflows/typed-1/signal/changed" => ("200 OK", "{}"),
25298 "/api/workflows/counter-1/query/current" => (
25299 "200 OK",
25300 r#"{"workflow_id":"counter-1","query_name":"current","result":{"count":8},"result_envelope":{"codec":"avro","blob":"wwHioz3/VYAiNw4CCmNvdW50BBAA"}}"#,
25301 ),
25302 "/api/workflows/counter-1/query/missing" => (
25303 "404 Not Found",
25304 r#"{"workflow_id":"counter-1","query_name":"missing","reason":"rejected_unknown_query","message":"unknown query"}"#,
25305 ),
25306 "/api/workflows/wf-lifecycle/cancel" => (
25307 "200 OK",
25308 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","reason":"cleanup requested","command_status":"accepted"}"#,
25309 ),
25310 "/api/workflows/wf-lifecycle/terminate" => (
25311 "200 OK",
25312 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","reason":"forced stop","command_status":"accepted"}"#,
25313 ),
25314 "/api/workflows/wf-lifecycle/runs/run-current/cancel" => (
25315 "200 OK",
25316 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","command_status":"accepted"}"#,
25317 ),
25318 "/api/workflows/wf-lifecycle/runs/run-current/terminate" => (
25319 "200 OK",
25320 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","command_status":"accepted"}"#,
25321 ),
25322 "/api/workflows/wf-lifecycle/runs/run-stale/cancel"
25323 | "/api/workflows/wf-lifecycle/runs/run-stale/terminate" => (
25324 "409 Conflict",
25325 r#"{"workflow_id":"wf-lifecycle","run_id":"run-stale","reason":"historical_run_command_rejected","target_scope":"run","message":"Commands cannot target historical runs."}"#,
25326 ),
25327 "/api/workflows/wf-failed" | "/api/workflows/wf-failed/runs/run-failed" => (
25328 "200 OK",
25329 r#"{"workflow_id":"wf-failed","run_id":"run-failed","status":"failed","closed_reason":"failed","error":"payment failed","failure":{"message":"payment failed","failure_category":"application","exception_type":"PaymentError","exception_class":"billing::PaymentError","non_retryable":true,"exception":{"type":"PaymentError","class":"billing::PaymentError","message":"payment failed"},"failures":[{"id":"failure-17","failure_category":"application"}]}}"#,
25330 ),
25331 "/api/workflows/wf-cancelled" => (
25332 "200 OK",
25333 r#"{"workflow_id":"wf-cancelled","run_id":"run-cancelled","status":"cancelled","closed_reason":"cancelled","reason":"cleanup requested"}"#,
25334 ),
25335 "/api/workflows/wf-terminated" => (
25336 "200 OK",
25337 r#"{"workflow_id":"wf-terminated","run_id":"run-terminated","status":"terminated","closed_reason":"terminated","reason":"forced stop"}"#,
25338 ),
25339 "/api/workflows/wf-timed-out" => (
25340 "200 OK",
25341 r#"{"workflow_id":"wf-timed-out","run_id":"run-timed-out","status":"failed","closed_reason":"timed_out","reason":"run_timeout"}"#,
25342 ),
25343 "/api/workflows/wf-waiting" | "/api/workflows/wf-waiting/runs/run-waiting" => (
25344 "200 OK",
25345 r#"{"workflow_id":"wf-waiting","run_id":"run-waiting","status":"waiting"}"#,
25346 ),
25347 "/api/workflows/wf-selected" => (
25348 "200 OK",
25349 r#"{"workflow_id":"wf-selected","run_id":"run-current","status":"completed","output":"current run output"}"#,
25350 ),
25351 "/api/workflows/wf-selected/runs/run-selected" => (
25352 "200 OK",
25353 r#"{"workflow_id":"wf-selected","run_id":"run-selected","status":"cancelled","closed_reason":"cancelled","reason":"selected run cancelled"}"#,
25354 ),
25355 _ => ("404 Not Found", r#"{"message":"not found"}"#),
25356 };
25357 write_mock_response(stream, status, body);
25358 }
25359
25360 fn mock_request_is_complete(request: &[u8]) -> bool {
25361 let Some(header_end) = request
25362 .windows(4)
25363 .position(|window| window == b"\r\n\r\n")
25364 .map(|position| position + 4)
25365 else {
25366 return false;
25367 };
25368 let headers = String::from_utf8_lossy(&request[..header_end]);
25369 let content_length = headers.lines().find_map(|line| {
25370 let (name, value) = line.split_once(':')?;
25371 name.eq_ignore_ascii_case("content-length")
25372 .then(|| value.trim().parse::<usize>().ok())
25373 .flatten()
25374 });
25375
25376 request.len() >= header_end + content_length.unwrap_or(0)
25377 }
25378
25379 fn write_mock_response(stream: &mut TcpStream, status: &str, body: &str) {
25380 let response = format!(
25381 "HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
25382 body.len()
25383 );
25384
25385 let _ = stream.write_all(response.as_bytes());
25386 let _ = stream.flush();
25387 }
25388}