1use std::{
17 sync::{Arc, RwLock},
18 time::{Duration, Instant},
19};
20
21use nainbox_core::{InboxClaim, InboxStore};
22use naoutbox_core::DurableOutboxAppend;
23use nasaga_backend::{
24 AttemptConflictFact, AttemptOutcomeRecord, CasOutcome, ControlCasOutcome,
25 ControlTransitionSpec, ManagementAuditOutcome, NewSagaInstance, SagaAuditEventCursor,
26 SagaAuditEventRecord, SagaAuditStore, SagaBackend, SagaBackendErrorKind, SagaBackendFactory,
27 SagaConflictKind, SagaCreation, SagaGovernanceStore, SagaInstanceQuery, SagaInstanceRow,
28 SagaInstanceStore, SagaInstanceSummary, SagaJournalStore, SagaStepRow, SagaTimerRow,
29 SagaTimerStore, StepJournalPatch, TimerFencing, TimerFencingToken, TimerFencingTokenIssuer,
30 TimerScope, TimerSpec, TransitionSpec,
31};
32use nasaga_core::{
33 freeze_compensation_plan, AttemptNo, BusinessKey, CancelMode, CommandId, DefinitionVersion,
34 Direction, EffectId, ResolutionMode, SagaId, SagaStatus, ServiceIdentity, StepAttemptStatus,
35 StepCancelStatus, StepCompensationStatus, StepDefinition, StepForwardStatus, StepJournalEntry,
36 StepName, StepPhase, StepResolutionStatus, TenantId, TimeoutPolicy, TriggerKind,
37 WorkflowDefinition, WorkflowName,
38};
39use natelemetry::TraceContext;
40use sha2::{Digest as _, Sha256};
41use uuid::Uuid;
42
43use crate::envelope::{canonical_bytes, SagaCommandEnvelope, SagaResultEnvelope, VerifiedIdentity};
44use crate::management::{
45 SagaAuditPage, SagaAuditPageCursor, SagaAuditRecord, SagaAuditTrail, SagaManagementContext,
46 SagaManagementError, SagaManagementExpectation, SagaManagementPermission,
47};
48use crate::observability::{
49 record_action_rate_rejection, record_quota_rejection, SagaOperationalMetrics,
50};
51use crate::registry::DefinitionRegistry;
52use crate::timers::{
53 derive_timer_id, parse_step_scope, phase_timeout_kind, resolution_budget_kind,
54 KIND_CANCEL_TIMEOUT, KIND_COMPENSATE_TIMEOUT, KIND_COMPENSATION_RESOLUTION_BUDGET,
55 KIND_FORWARD_RESOLUTION_BUDGET, KIND_INSTANCE_DEADLINE, KIND_RESOLUTION_BUDGET,
56 KIND_RESOLVE_TIMEOUT, KIND_STEP_TIMEOUT,
57};
58
59const CONVERGE_TRIGGER_NAMESPACE: Uuid = Uuid::from_bytes(*b"nasasaga-v1-cvns");
64
65#[derive(Debug, Clone)]
67pub struct OrchestratorConfig {
68 pub inbox_consumer: String,
70 pub cancel_max_attempts: u32,
72 pub compensate_max_attempts: u32,
74 pub resolve_max_attempts: u32,
76 pub timer_claim_limit: u32,
78 pub timer_lease_ms: i64,
80 pub pause_backoff_ms: i64,
82 pub startup_scan_limit: u32,
84 pub tenant_quotas: std::collections::BTreeMap<String, u64>,
90 pub tenant_action_rates: std::collections::BTreeMap<String, TenantActionRate>,
98 pub enable_manual_close: bool,
104}
105
106impl Default for OrchestratorConfig {
107 fn default() -> Self {
113 Self {
114 inbox_consumer: "nasaga-orchestrator".to_string(),
115 cancel_max_attempts: 5,
116 compensate_max_attempts: 5,
117 resolve_max_attempts: 10,
118 timer_claim_limit: 32,
119 timer_lease_ms: 60_000,
120 pause_backoff_ms: 30_000,
121 startup_scan_limit: 1_000,
122 tenant_quotas: std::collections::BTreeMap::new(),
123 tenant_action_rates: std::collections::BTreeMap::new(),
124 enable_manual_close: false,
126 }
127 }
128}
129
130#[derive(Debug, Clone, Copy, PartialEq, Eq)]
135pub struct TenantActionRate {
136 pub max_actions: u64,
138 pub window_ms: i64,
140}
141
142#[derive(Debug, Clone, PartialEq, Eq)]
144pub enum StartOutcome {
145 Started(SagaInstanceRow),
147 AlreadyExists(SagaInstanceRow),
149}
150
151#[derive(Debug, Clone, Copy, PartialEq, Eq)]
153#[non_exhaustive]
154pub enum StartSagaError {
155 DefinitionInactive,
157 InvalidPayload,
159 DefinitionEmpty,
161 RequestConflict,
163 TenantQuotaExceeded,
165}
166
167impl StartSagaError {
168 pub fn from_error(error: &anyhow::Error) -> Option<Self> {
175 error
176 .chain()
177 .find_map(|cause| cause.downcast_ref::<Self>().copied())
178 }
179}
180
181impl std::fmt::Display for StartSagaError {
182 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
189 formatter.write_str(match self {
190 Self::InvalidPayload => "Saga payload contract is invalid",
191 Self::DefinitionInactive => "Saga workflow definition is not active",
192 Self::DefinitionEmpty => "Saga workflow definition has no steps",
193 Self::RequestConflict => "Saga identity is already bound to a different start request",
194 Self::TenantQuotaExceeded => "saga_tenant_quota_exceeded",
195 })
196 }
197}
198
199impl std::error::Error for StartSagaError {}
200
201#[derive(Debug, Clone, Copy, PartialEq, Eq)]
203#[non_exhaustive]
204pub enum SagaConcurrencyError {
205 StaleSnapshot,
207 DuplicateTrigger,
209}
210
211impl SagaConcurrencyError {
212 pub fn from_error(error: &anyhow::Error) -> Option<Self> {
219 error
220 .chain()
221 .find_map(|cause| cause.downcast_ref::<Self>().copied())
222 }
223}
224
225impl std::fmt::Display for SagaConcurrencyError {
226 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
233 formatter.write_str(match self {
234 Self::StaleSnapshot => "Saga state snapshot is stale",
235 Self::DuplicateTrigger => "Saga transition trigger was already applied",
236 })
237 }
238}
239
240impl std::error::Error for SagaConcurrencyError {}
241
242#[derive(Debug, Clone, PartialEq, Eq)]
244pub enum HandleOutcome {
245 Applied {
247 status: SagaStatus,
249 },
250 Duplicate,
252}
253
254#[derive(Debug, Clone, PartialEq, Eq)]
256pub enum TimerOutcome {
257 Applied {
259 status: SagaStatus,
261 },
262 SkippedPaused,
264 SkippedFencingLost,
266 SkippedStale,
268}
269
270#[derive(Debug, Clone)]
272pub struct StartSagaRequest<'a> {
273 pub saga_id: &'a SagaId,
275 pub tenant: &'a TenantId,
277 pub workflow: &'a WorkflowName,
279 pub version: DefinitionVersion,
281 pub business_key: &'a BusinessKey,
283 pub deadline_at_ms: Option<i64>,
285 pub trigger_kind: TriggerKind,
287 pub trigger_id: &'a str,
289 pub first_command_payload: Option<serde_json::Value>,
291 pub first_command_raw_payload: Option<nasaga_core::SagaPayload>,
293 pub now_ms: i64,
295}
296
297enum CommandInput {
299 Json(serde_json::Value),
300 Raw(nasaga_core::SagaPayload),
301}
302
303pub struct Orchestrator<B: SagaBackend> {
308 backend: B,
309 registry: RwLock<Arc<DefinitionRegistry>>,
310 config: OrchestratorConfig,
311 fencing_tokens: TimerFencingTokenIssuer,
313}
314
315struct ResultContext<'a> {
317 instance: &'a SagaInstanceRow,
318 definition: &'a WorkflowDefinition,
319 step_def: &'a StepDefinition,
320 identity: &'a VerifiedIdentity,
321 status: StepAttemptStatus,
322 terminal: Option<StepForwardStatus>,
323 reason_code: Option<&'a str>,
324 event_id: &'a str,
325 now_ms: i64,
326}
327
328fn validate_config(config: &OrchestratorConfig) -> anyhow::Result<()> {
335 if config.inbox_consumer.is_empty()
336 || config.inbox_consumer.trim() != config.inbox_consumer
337 || config.inbox_consumer.len() > 128
338 || config.inbox_consumer.contains('\0')
339 || !config.inbox_consumer.bytes().all(|byte| {
340 byte.is_ascii_lowercase() || byte.is_ascii_digit() || matches!(byte, b'_' | b'-' | b'.')
341 })
342 || config.cancel_max_attempts == 0
343 || config.compensate_max_attempts == 0
344 || config.resolve_max_attempts == 0
345 || config.timer_claim_limit == 0
346 || config.timer_lease_ms <= 0
347 || config.pause_backoff_ms <= 0
348 || config.startup_scan_limit == 0
349 {
350 anyhow::bail!("orchestrator config violates the Inbox key contract or positive budgets");
351 }
352 for tenant in config.tenant_quotas.keys() {
355 if TenantId::new(tenant.as_str()).is_err() {
356 anyhow::bail!("tenant quota key is not a valid tenant id");
357 }
358 }
359 for (tenant, rate) in &config.tenant_action_rates {
360 if TenantId::new(tenant.as_str()).is_err() {
361 anyhow::bail!("tenant action rate key is not a valid tenant id");
362 }
363 if rate.window_ms <= 0 {
364 anyhow::bail!("tenant action rate window_ms must be positive");
365 }
366 }
367 Ok(())
368}
369
370fn duration_millis(duration: Duration) -> anyhow::Result<i64> {
377 let millis = i64::try_from(duration.as_millis())
378 .map_err(|_| anyhow::anyhow!("saga duration exceeds the supported millisecond range"))?;
379 if millis <= 0 {
380 anyhow::bail!("saga duration must be at least one millisecond");
381 }
382 Ok(millis)
383}
384
385fn checked_deadline(now_ms: i64, delay_ms: i64) -> anyhow::Result<i64> {
393 if delay_ms <= 0 {
394 anyhow::bail!("saga timer delay must be positive");
395 }
396 now_ms
397 .checked_add(delay_ms)
398 .ok_or_else(|| anyhow::anyhow!("saga timer deadline overflow"))
399}
400
401fn management_precondition(message: &'static str) -> anyhow::Error {
408 anyhow::Error::new(SagaManagementError::PreconditionFailed).context(message)
409}
410
411fn hash_canonical_json(hasher: &mut Sha256, value: &serde_json::Value) {
419 match value {
420 serde_json::Value::Null => hasher.update(canonical_bytes(&[b"null"])),
421 serde_json::Value::Bool(value) => {
422 hasher.update(canonical_bytes(&[b"bool", &[u8::from(*value)]]));
423 }
424 serde_json::Value::Number(value) => {
425 hasher.update(canonical_bytes(&[b"number", value.to_string().as_bytes()]));
426 }
427 serde_json::Value::String(value) => {
428 hasher.update(canonical_bytes(&[b"string", value.as_bytes()]));
429 }
430 serde_json::Value::Array(values) => {
431 hasher.update(canonical_bytes(&[b"array", &values.len().to_be_bytes()]));
432 for value in values {
433 hash_canonical_json(hasher, value);
434 }
435 }
436 serde_json::Value::Object(values) => {
437 let mut keys: Vec<_> = values.keys().collect();
438 keys.sort_unstable();
439 hasher.update(canonical_bytes(&[b"object", &keys.len().to_be_bytes()]));
440 for key in keys {
441 hasher.update(canonical_bytes(&[b"key", key.as_bytes()]));
442 hash_canonical_json(hasher, &values[key]);
443 }
444 }
445 }
446}
447
448fn derive_legacy_start_request_digest(
460 request: &StartSagaRequest<'_>,
461 definition_digest: &str,
462 first_step: &StepName,
463) -> String {
464 let version = request.version.get().to_be_bytes();
465 let deadline = request.deadline_at_ms.unwrap_or_default().to_be_bytes();
466 let deadline_present = [u8::from(request.deadline_at_ms.is_some())];
467 let payload_present = [u8::from(request.first_command_payload.is_some())];
468 let mut hasher = Sha256::new();
469 hasher.update(canonical_bytes(&[
470 request.tenant.as_str().as_bytes(),
471 request.workflow.as_str().as_bytes(),
472 &version,
473 definition_digest.as_bytes(),
474 request.business_key.as_str().as_bytes(),
475 first_step.as_str().as_bytes(),
476 &deadline_present,
477 &deadline,
478 &payload_present,
479 ]));
480 if let Some(payload) = request.first_command_payload.as_ref() {
481 hash_canonical_json(&mut hasher, payload);
482 }
483 hasher
484 .finalize()
485 .iter()
486 .map(|byte| format!("{byte:02x}"))
487 .collect()
488}
489
490fn derive_start_request_digest(
500 request: &StartSagaRequest<'_>,
501 definition_digest: &str,
502 first_step: &StepName,
503 payload: Option<&nasaga_core::SagaPayload>,
504) -> String {
505 let mut empty = request.clone();
506 empty.first_command_payload = None;
507 empty.first_command_raw_payload = None;
508 let domain = derive_legacy_start_request_digest(&empty, definition_digest, first_step);
509 let Some(payload) = payload else {
510 return domain;
511 };
512 let mut hasher = Sha256::new();
513 hasher.update(canonical_bytes(&[
514 b"saga-start-payload",
515 domain.as_bytes(),
516 payload.content_type.as_bytes(),
517 payload.schema_id.as_bytes(),
518 &payload.body,
519 ]));
520 hex::encode(hasher.finalize())
521}
522
523fn derive_converge_trigger_id(cause_id: &str) -> String {
530 Uuid::new_v5(
531 &CONVERGE_TRIGGER_NAMESPACE,
532 &canonical_bytes(&[cause_id.as_bytes(), b"converge"]),
533 )
534 .to_string()
535}
536
537fn elapsed_epoch_ms(base_now_ms: i64, claimed_at: Instant) -> anyhow::Result<i64> {
548 let elapsed_ms = i64::try_from(claimed_at.elapsed().as_millis())
549 .map_err(|_| anyhow::anyhow!("timer lease elapsed duration overflow"))?;
550 base_now_ms
551 .checked_add(elapsed_ms)
552 .ok_or_else(|| anyhow::anyhow!("timer lease clock overflow"))
553}
554
555impl<B> Orchestrator<B>
556where
557 B: SagaBackend + SagaBackendFactory,
558{
559 pub fn new(registry: DefinitionRegistry, config: OrchestratorConfig) -> anyhow::Result<Self> {
567 validate_config(&config)?;
568 Ok(Self {
569 backend: B::default_backend()?,
570 registry: RwLock::new(Arc::new(registry)),
571 config,
572 fencing_tokens: TimerFencingTokenIssuer::new(),
573 })
574 }
575
576 pub fn with_datasource(
585 registry: DefinitionRegistry,
586 config: OrchestratorConfig,
587 datasource: impl AsRef<str>,
588 ) -> anyhow::Result<Self> {
589 validate_config(&config)?;
590 Ok(Self {
591 backend: B::backend_for(datasource.as_ref())?,
592 registry: RwLock::new(Arc::new(registry)),
593 config,
594 fencing_tokens: TimerFencingTokenIssuer::new(),
595 })
596 }
597
598 pub fn datasource_ref(&self) -> &natx_core::DatasourceRef {
604 self.backend.datasource_ref()
605 }
606
607 fn registry_snapshot(&self) -> Arc<DefinitionRegistry> {
613 self.registry
614 .read()
615 .unwrap_or_else(std::sync::PoisonError::into_inner)
616 .clone()
617 }
618
619 pub fn replace_registry(&self, registry: DefinitionRegistry) {
626 *self
627 .registry
628 .write()
629 .unwrap_or_else(std::sync::PoisonError::into_inner) = Arc::new(registry);
630 }
631
632 pub async fn verify_registry_snapshot(
639 &self,
640 registry: &DefinitionRegistry,
641 ) -> anyhow::Result<()> {
642 registry
643 .verify_non_terminal(self.backend.store(), self.config.startup_scan_limit)
644 .await
645 }
646
647 pub async fn verify_startup(&self) -> anyhow::Result<()> {
653 let registry = self.registry_snapshot();
656 crate::verify_descriptors(®istry)?;
657 for tenant in self.config.tenant_quotas.keys() {
661 let tenant = TenantId::new(tenant.as_str())
662 .map_err(|_| anyhow::anyhow!("tenant quota key is not a valid tenant id"))?;
663 if !self
664 .backend
665 .store()
666 .tenant_quota_initialized(&tenant)
667 .await?
668 {
669 anyhow::bail!(
670 "tenant quota ledger for a capped tenant is not initialized; \
671 run reconcile_tenant_quota after every writer runs the accounting binary"
672 );
673 }
674 }
675 registry
676 .verify_non_terminal(self.backend.store(), self.config.startup_scan_limit)
677 .await
678 }
679
680 pub fn definition_digest(
688 &self,
689 workflow: &WorkflowName,
690 version: DefinitionVersion,
691 ) -> Option<String> {
692 self.registry_snapshot()
693 .get(workflow, version)
694 .map(WorkflowDefinition::digest)
695 }
696
697 pub fn definition_digest_for_tenant(
706 &self,
707 tenant: &TenantId,
708 workflow: &WorkflowName,
709 version: DefinitionVersion,
710 ) -> Option<String> {
711 self.registry_snapshot()
712 .get_for_tenant(tenant, workflow, version)
713 .map(WorkflowDefinition::digest)
714 }
715
716 pub async fn load_instance(
725 &self,
726 tenant: &TenantId,
727 saga_id: &SagaId,
728 ) -> anyhow::Result<Option<SagaInstanceRow>> {
729 Ok(self
730 .backend
731 .store()
732 .load_instance(saga_id)
733 .await?
734 .filter(|instance| &instance.tenant == tenant))
735 }
736
737 pub async fn load_audit_trail(
750 &self,
751 management: &SagaManagementContext,
752 tenant: &TenantId,
753 saga_id: &SagaId,
754 after_transition_seq: u64,
755 after_control_seq: u64,
756 limit: u32,
757 ) -> anyhow::Result<SagaAuditTrail> {
758 management.require(SagaManagementPermission::ReadAudit)?;
760 self.backend
761 .store()
762 .load_instance(saga_id)
763 .await?
764 .filter(|instance| &instance.tenant == tenant)
765 .ok_or(SagaManagementError::NotFound)?;
766 Ok(SagaAuditTrail {
767 attempts: self
768 .backend
769 .store()
770 .load_attempt_audit(saga_id, None, limit)
771 .await?
772 .into_iter()
773 .map(|row| row.attempt)
774 .collect(),
775 transitions: self
776 .backend
777 .store()
778 .load_transition_audit(saga_id, after_transition_seq, limit)
779 .await?,
780 controls: self
781 .backend
782 .store()
783 .load_control_audit(saga_id, after_control_seq, limit)
784 .await?,
785 management_operations: self
786 .backend
787 .store()
788 .load_management_audit(saga_id, None, limit)
789 .await?,
790 conflicts: self
791 .backend
792 .store()
793 .load_conflict_audit(saga_id, None, limit)
794 .await?,
795 })
796 }
797
798 pub async fn load_audit_page(
811 &self,
812 management: &SagaManagementContext,
813 tenant: &TenantId,
814 saga_id: &SagaId,
815 cursor: Option<&SagaAuditPageCursor>,
816 limit: u32,
817 ) -> anyhow::Result<SagaAuditPage> {
818 if limit == 0 || limit > 1_000 {
819 return Err(SagaManagementError::InvalidContext.into());
820 }
821 management.require(SagaManagementPermission::ReadAudit)?;
823 self.backend
824 .store()
825 .load_instance(saga_id)
826 .await?
827 .filter(|instance| &instance.tenant == tenant)
828 .ok_or(SagaManagementError::NotFound)?;
829 let after = SagaAuditEventCursor {
830 audit_seq: cursor.map_or(0, |cursor| cursor.audit_seq),
831 };
832 let events = self
833 .backend
834 .store()
835 .load_audit_events(saga_id, after, limit)
836 .await?;
837 let next_cursor = events.last().map(|event| SagaAuditPageCursor {
840 audit_seq: event.audit_seq,
841 });
842 let records = events
843 .into_iter()
844 .map(|event| match event.record {
845 SagaAuditEventRecord::Attempt(row) => SagaAuditRecord::Attempt(row),
846 SagaAuditEventRecord::Transition(row) => SagaAuditRecord::Transition(row),
847 SagaAuditEventRecord::Control(row) => SagaAuditRecord::Control(row),
848 SagaAuditEventRecord::Management(row) => SagaAuditRecord::Management(row),
849 SagaAuditEventRecord::Conflict(row) => SagaAuditRecord::Conflict(row),
850 })
851 .collect();
852 Ok(SagaAuditPage {
853 records,
854 next_cursor,
855 })
856 }
857
858 pub async fn list_instances(
872 &self,
873 management: &SagaManagementContext,
874 query: &SagaInstanceQuery<'_>,
875 ) -> anyhow::Result<Vec<SagaInstanceSummary>> {
876 management.require(SagaManagementPermission::ListInstances)?;
878 Ok(self.backend.store().list_instances(query).await?)
879 }
880
881 pub async fn tenant_quota_usage(
889 &self,
890 management: &SagaManagementContext,
891 tenant: &TenantId,
892 ) -> anyhow::Result<u64> {
893 management.require(SagaManagementPermission::ReadAudit)?;
894 Ok(self.backend.store().tenant_quota_usage(tenant).await?)
895 }
896
897 pub async fn tenant_action_rate_usage(
905 &self,
906 management: &SagaManagementContext,
907 tenant: &TenantId,
908 ) -> anyhow::Result<Option<(u64, u64)>> {
909 management.require(SagaManagementPermission::ReadAudit)?;
910 let Some(rate) = self.config.tenant_action_rates.get(tenant.as_str()) else {
911 return Ok(None);
912 };
913 Ok(Some(
914 self.backend
915 .store()
916 .tenant_action_rate_usage(tenant, rate.window_ms)
917 .await?,
918 ))
919 }
920
921 async fn reserve_action_rate_budget(&self, tenant: &TenantId) -> anyhow::Result<()> {
932 let Some(rate) = self.config.tenant_action_rates.get(tenant.as_str()) else {
933 return Ok(());
934 };
935 if matches!(
936 self.backend
937 .store()
938 .reserve_tenant_action_rate(tenant, rate.max_actions, rate.window_ms)
939 .await?,
940 nasaga_backend::ActionRateReservation::Exceeded
941 ) {
942 record_action_rate_rejection();
943 return Err(SagaManagementError::RateLimitExceeded.into());
945 }
946 Ok(())
947 }
948
949 pub async fn load_operational_metrics(
959 &self,
960 now_ms: i64,
961 ) -> anyhow::Result<SagaOperationalMetrics> {
962 Ok(self
963 .backend
964 .store()
965 .load_operational_metrics(now_ms)
966 .await?
967 .into())
968 }
969
970 pub fn verify_result_producer(
984 &self,
985 envelope: &SagaResultEnvelope,
986 producer: &ServiceIdentity,
987 ) -> anyhow::Result<()> {
988 let identity = envelope.verified_for_delivery()?;
989 let registry = self.registry_snapshot();
990 let definition = registry
991 .get_for_tenant(
992 &identity.tenant,
993 &identity.workflow,
994 identity.definition_version,
995 )
996 .ok_or(crate::SagaResultProcessingError::ContractInvalid)?;
997 if definition.digest() != identity.definition_digest {
998 return Err(crate::SagaResultProcessingError::ContractInvalid.into());
999 }
1000 let step = definition
1001 .step(&identity.step)
1002 .ok_or(crate::SagaResultProcessingError::ContractInvalid)?;
1003 if step.owner() != producer {
1005 return Err(crate::SagaResultProcessingError::ProducerUnauthorized.into());
1006 }
1007 Ok(())
1008 }
1009
1010 pub async fn handle_authenticated_result(
1020 &self,
1021 envelope: &SagaResultEnvelope,
1022 producer: &ServiceIdentity,
1023 now_ms: i64,
1024 ) -> anyhow::Result<HandleOutcome> {
1025 self.handle_authenticated_result_traced(envelope, producer, None, now_ms)
1026 .await
1027 }
1028
1029 pub async fn handle_authenticated_result_traced(
1044 &self,
1045 envelope: &SagaResultEnvelope,
1046 producer: &ServiceIdentity,
1047 receipt_trace: Option<&TraceContext>,
1048 now_ms: i64,
1049 ) -> anyhow::Result<HandleOutcome> {
1050 self.handle_authenticated_result_authorized_traced(
1051 envelope,
1052 producer,
1053 receipt_trace,
1054 now_ms,
1055 &|| Ok(()),
1056 )
1057 .await
1058 }
1059
1060 pub async fn handle_authenticated_result_authorized_traced(
1071 &self,
1072 envelope: &SagaResultEnvelope,
1073 producer: &ServiceIdentity,
1074 receipt_trace: Option<&TraceContext>,
1075 now_ms: i64,
1076 authorize: &(dyn Fn() -> anyhow::Result<()> + Send + Sync),
1077 ) -> anyhow::Result<HandleOutcome> {
1078 authorize()?;
1080 self.verify_result_producer(envelope, producer)?;
1082 let _latency =
1083 crate::latency::LatencyGuard::new(crate::latency::LatencyStage::TransitionTransaction);
1084 self.handle_verified_result(envelope, receipt_trace, now_ms, authorize)
1085 .await
1086 }
1087
1088 pub async fn start_saga(&self, request: &StartSagaRequest<'_>) -> anyhow::Result<StartOutcome> {
1100 self.start_saga_traced(request, None).await
1101 }
1102
1103 pub async fn start_saga_traced(
1117 &self,
1118 request: &StartSagaRequest<'_>,
1119 trace: Option<&TraceContext>,
1120 ) -> anyhow::Result<StartOutcome> {
1121 self.start_saga_authorized_traced(request, trace, &|| Ok(()))
1122 .await
1123 }
1124
1125 pub async fn start_saga_authorized_traced(
1136 &self,
1137 request: &StartSagaRequest<'_>,
1138 trace: Option<&TraceContext>,
1139 authorize: &(dyn Fn() -> anyhow::Result<()> + Send + Sync),
1140 ) -> anyhow::Result<StartOutcome> {
1141 authorize()?;
1143 let registry = self.registry_snapshot();
1144 let definition = registry
1146 .get_for_tenant(request.tenant, request.workflow, request.version)
1147 .ok_or(StartSagaError::DefinitionInactive)?;
1148 let digest = definition.digest();
1149 let first_step = definition
1150 .steps()
1151 .first()
1152 .ok_or(StartSagaError::DefinitionEmpty)?;
1153 let _latency =
1154 crate::latency::LatencyGuard::new(crate::latency::LatencyStage::StartTransaction);
1155 if request.first_command_payload.is_some() && request.first_command_raw_payload.is_some() {
1156 return Err(StartSagaError::InvalidPayload.into());
1157 }
1158 let payload = request.first_command_raw_payload.clone().or_else(|| {
1159 request
1160 .first_command_payload
1161 .clone()
1162 .map(nasaga_core::SagaPayload::json)
1163 });
1164 if let Some(payload) = &payload {
1165 payload
1166 .validate()
1167 .map_err(|_| StartSagaError::InvalidPayload)?;
1168 if &payload.contract() != first_step.payload_contract() {
1169 return Err(StartSagaError::InvalidPayload.into());
1170 }
1171 }
1172 let start_request_digest =
1173 derive_start_request_digest(request, &digest, first_step.name(), payload.as_ref());
1174 let legacy_start_request_digest = match payload.as_ref() {
1175 Some(payload) if payload.contract() == nasaga_core::SagaPayloadContract::default() => {
1176 let mut legacy = request.clone();
1177 legacy.first_command_payload = Some(
1178 serde_json::from_slice(&payload.body)
1179 .map_err(|_| StartSagaError::InvalidPayload)?,
1180 );
1181 Some(derive_legacy_start_request_digest(
1182 &legacy,
1183 &digest,
1184 first_step.name(),
1185 ))
1186 }
1187 _ => None,
1188 };
1189 let canonical_trace = trace.map(TraceContext::to_traceparent);
1190
1191 let outcome = crate::transaction::run_authorized_for(&self.backend, authorize, async {
1192 let creation = self
1193 .backend
1194 .store()
1195 .create_instance(&NewSagaInstance {
1196 saga_id: request.saga_id,
1197 tenant: request.tenant,
1198 workflow: request.workflow,
1199 business_key: request.business_key,
1200 definition_version: request.version,
1201 definition_digest: &digest,
1202 start_request_digest: &start_request_digest,
1203 legacy_start_request_digest: legacy_start_request_digest.as_deref(),
1204 deadline_at_ms: request.deadline_at_ms,
1205 current_step: Some(first_step.name()),
1207 trigger_kind: request.trigger_kind,
1208 trigger_id: request.trigger_id,
1209 traceparent: canonical_trace.as_deref(),
1212 })
1213 .await
1214 .map_err(|error| {
1215 if error.kind() == SagaBackendErrorKind::Conflict {
1216 anyhow::Error::new(StartSagaError::RequestConflict)
1217 } else {
1218 error.into()
1219 }
1220 })?;
1221 let instance = match creation {
1222 SagaCreation::Existing(row) => return Ok(StartOutcome::AlreadyExists(row)),
1224 SagaCreation::Created(row) => row,
1225 };
1226 if registry
1229 .get_for_start(request.tenant, request.workflow, request.version)
1230 .is_none()
1231 {
1232 return Err(StartSagaError::DefinitionInactive.into());
1233 }
1234 let cap = self
1237 .config
1238 .tenant_quotas
1239 .get(request.tenant.as_str())
1240 .copied();
1241 if matches!(
1242 self.backend
1243 .store()
1244 .reserve_tenant_quota(request.tenant, cap)
1245 .await?,
1246 nasaga_backend::QuotaReservation::Exceeded
1247 ) {
1248 record_quota_rejection();
1249 return Err(StartSagaError::TenantQuotaExceeded.into());
1251 }
1252 self.backend
1253 .store()
1254 .register_steps(request.saga_id, definition)
1255 .await?;
1256 self.issue_command(
1257 &instance,
1258 first_step,
1259 StepPhase::Execute,
1260 AttemptNo::FIRST,
1261 request
1262 .first_command_raw_payload
1263 .clone()
1264 .map(CommandInput::Raw)
1265 .or_else(|| {
1266 request
1267 .first_command_payload
1268 .clone()
1269 .map(CommandInput::Json)
1270 }),
1271 None,
1272 request.now_ms,
1273 instance.version,
1274 )
1275 .await?;
1276 if let Some(deadline) = request.deadline_at_ms {
1278 self.schedule_timer(
1279 request.saga_id,
1280 TimerScope::Instance,
1281 KIND_INSTANCE_DEADLINE,
1282 AttemptNo::FIRST,
1283 deadline,
1284 instance.version,
1285 )
1286 .await?;
1287 }
1288 Ok(StartOutcome::Started(instance))
1289 })
1290 .await?;
1291 match &outcome {
1292 StartOutcome::Started(instance) => tracing::info!(
1293 saga_id = instance.saga_id.as_str(),
1294 tenant = instance.tenant.as_str(),
1295 workflow = instance.workflow.as_str(),
1296 definition_version = instance.definition_version.get(),
1297 saga_status = instance.status.as_str(),
1298 "Saga 创建事务已提交"
1299 ),
1300 StartOutcome::AlreadyExists(instance) => tracing::info!(
1301 saga_id = instance.saga_id.as_str(),
1302 tenant = instance.tenant.as_str(),
1303 workflow = instance.workflow.as_str(),
1304 saga_status = instance.status.as_str(),
1305 "Saga 启动重放已幂等吸收"
1306 ),
1307 }
1308 Ok(outcome)
1309 }
1310
1311 async fn handle_verified_result(
1321 &self,
1322 envelope: &SagaResultEnvelope,
1323 receipt_trace: Option<&TraceContext>,
1324 now_ms: i64,
1325 authorize: &(dyn Fn() -> anyhow::Result<()> + Send + Sync),
1326 ) -> anyhow::Result<HandleOutcome> {
1327 let identity = envelope.verified_for_delivery()?;
1329 let status = envelope
1330 .parsed_status()
1331 .map_err(|_| crate::SagaResultProcessingError::ContractInvalid)?;
1332 let terminal = envelope
1333 .parsed_terminal()
1334 .map_err(|_| crate::SagaResultProcessingError::ContractInvalid)?;
1335 let registry = self.registry_snapshot();
1336
1337 let outcome = crate::transaction::run_authorized_for(&self.backend, authorize, async {
1338 if matches!(
1339 self.backend
1340 .inbox()
1341 .claim(&self.config.inbox_consumer, &envelope.event_id)
1342 .await?,
1343 InboxClaim::Duplicate
1344 ) {
1345 return Ok(HandleOutcome::Duplicate);
1346 }
1347 let instance = self
1348 .backend
1349 .store()
1350 .load_instance(&identity.saga_id)
1351 .await?
1352 .ok_or(crate::SagaResultProcessingError::ContractInvalid)?;
1353 verify_instance_contract(&instance, &identity)
1354 .map_err(|_| crate::SagaResultProcessingError::ContractInvalid)?;
1355 authorize()?;
1357 if !instance.control_state.allows_automatic_actions() {
1358 return Err(crate::SagaResultProcessingError::Paused.into());
1361 }
1362 let definition = registry
1363 .get_for_tenant(
1364 &instance.tenant,
1365 &instance.workflow,
1366 instance.definition_version,
1367 )
1368 .ok_or_else(|| anyhow::anyhow!("instance definition is not registered"))?;
1369 let step_def = definition.step(&identity.step).ok_or_else(|| {
1370 anyhow::Error::from(crate::SagaResultProcessingError::ContractInvalid)
1371 })?;
1372
1373 let recorded = self
1376 .backend
1377 .store()
1378 .record_attempt_outcome(
1379 &identity.saga_id,
1380 &identity.step,
1381 identity.phase,
1382 identity.attempt,
1383 status,
1384 Some(&envelope.event_id),
1385 )
1386 .await?;
1387 match recorded {
1388 AttemptOutcomeRecord::AlreadyRecorded => {
1392 return Ok(HandleOutcome::Applied {
1393 status: instance.status,
1394 });
1395 }
1396 AttemptOutcomeRecord::Conflicting {
1397 existing,
1398 } => {
1399 self.backend
1402 .store()
1403 .record_attempt_conflict(&AttemptConflictFact {
1404 saga_id: &identity.saga_id,
1405 step: &identity.step,
1406 phase: identity.phase,
1407 attempt: identity.attempt,
1408 existing_status: existing,
1409 incoming_status: status,
1410 incoming_event_id: &envelope.event_id,
1411 conflict_kind: SagaConflictKind::AttemptTerminal,
1412 })
1413 .await?;
1414 let status = if instance.status.is_terminal()
1415 || instance.status == SagaStatus::ManualIntervention
1416 {
1417 instance.status
1420 } else {
1421 self.escalate_manual(
1422 &instance,
1423 TriggerKind::Event,
1424 &envelope.event_id,
1425 "attempt_outcome_conflict",
1426 )
1427 .await?
1428 };
1429 return Ok(HandleOutcome::Applied {
1430 status,
1431 });
1432 }
1433 AttemptOutcomeRecord::Recorded => {}
1434 }
1435
1436 let instance = match receipt_trace {
1440 Some(trace) => {
1441 let canonical = trace.to_traceparent();
1442 self.backend
1443 .store()
1444 .update_trace_context(&identity.saga_id, &canonical)
1445 .await?;
1446 SagaInstanceRow {
1447 traceparent: Some(canonical),
1448 ..instance
1449 }
1450 }
1451 None => instance,
1452 };
1453
1454 let context = ResultContext {
1455 instance: &instance,
1456 definition,
1457 step_def,
1458 identity: &identity,
1459 status,
1460 terminal,
1461 reason_code: envelope.reason_code.as_deref(),
1462 event_id: &envelope.event_id,
1463 now_ms,
1464 };
1465 let new_status = match identity.phase {
1469 StepPhase::Execute => Box::pin(self.on_execute_result(&context)).await?,
1470 StepPhase::Cancel => Box::pin(self.on_cancel_result(&context)).await?,
1471 StepPhase::Compensate => Box::pin(self.on_compensate_result(&context)).await?,
1472 StepPhase::Resolve => Box::pin(self.on_resolve_result(&context)).await?,
1473 };
1474 Ok(HandleOutcome::Applied {
1475 status: new_status,
1476 })
1477 })
1478 .await?;
1479 match &outcome {
1480 HandleOutcome::Applied {
1481 status,
1482 } => tracing::info!(
1483 saga_id = identity.saga_id.as_str(),
1484 workflow = identity.workflow.as_str(),
1485 step = identity.step.as_str(),
1486 phase = identity.phase.as_str(),
1487 attempt = identity.attempt.get(),
1488 saga_status = status.as_str(),
1489 "Saga 结果事务已提交"
1490 ),
1491 HandleOutcome::Duplicate => tracing::info!(
1492 saga_id = identity.saga_id.as_str(),
1493 workflow = identity.workflow.as_str(),
1494 step = identity.step.as_str(),
1495 phase = identity.phase.as_str(),
1496 attempt = identity.attempt.get(),
1497 "Saga 重复结果已由 Inbox 吸收"
1498 ),
1499 }
1500 Ok(outcome)
1501 }
1502
1503 pub async fn run_due_timers(&self, owner: &str, now_ms: i64) -> anyhow::Result<u32> {
1516 let claimed_at = Instant::now();
1519 let token = self.fencing_tokens.issue(owner, now_ms);
1522 let claimed = self
1523 .backend
1524 .store()
1525 .claim_due_timers(
1526 owner,
1527 token,
1528 now_ms,
1529 self.config.timer_lease_ms,
1530 self.config.timer_claim_limit,
1531 )
1532 .await?;
1533 let mut applied = 0u32;
1534 for timer in claimed.timers() {
1535 match self
1536 .fire_timer_inner(timer, claimed.token(), now_ms, Some(claimed_at))
1537 .await?
1538 {
1539 TimerOutcome::Applied {
1540 status,
1541 } => {
1542 applied += 1;
1543 tracing::info!(
1544 saga_id = timer.saga_id.as_str(),
1545 timer_kind = %timer.kind,
1546 timer_scope = %timer.scope_kind,
1547 attempt = timer.attempt.get(),
1548 saga_status = status.as_str(),
1549 "Saga durable timer 裁决事务已提交"
1550 );
1551 }
1552 TimerOutcome::SkippedPaused => {
1553 let release_now = elapsed_epoch_ms(now_ms, claimed_at)?;
1557 let _ = self
1558 .backend
1559 .store()
1560 .release_timer(
1561 &timer.timer_id,
1562 claimed.token(),
1563 release_now,
1564 checked_deadline(release_now, self.config.pause_backoff_ms)?,
1565 )
1566 .await?;
1567 }
1568 TimerOutcome::SkippedFencingLost => tracing::warn!(
1569 saga_id = timer.saga_id.as_str(),
1570 timer_kind = %timer.kind,
1571 "Saga timer 已失去 fencing 权威,旧 owner 停止动作"
1572 ),
1573 TimerOutcome::SkippedStale => tracing::debug!(
1574 saga_id = timer.saga_id.as_str(),
1575 timer_kind = %timer.kind,
1576 "Saga 过期 timer 已幂等吸收"
1577 ),
1578 }
1579 }
1580 Ok(applied)
1581 }
1582
1583 pub async fn fire_timer(
1593 &self,
1594 timer: &SagaTimerRow,
1595 token: &TimerFencingToken,
1596 now_ms: i64,
1597 ) -> anyhow::Result<TimerOutcome> {
1598 self.fire_timer_inner(timer, token, now_ms, None).await
1599 }
1600
1601 async fn fire_timer_inner(
1612 &self,
1613 timer: &SagaTimerRow,
1614 token: &TimerFencingToken,
1615 base_now_ms: i64,
1616 claimed_at: Option<Instant>,
1617 ) -> anyhow::Result<TimerOutcome> {
1618 let _latency =
1619 crate::latency::LatencyGuard::new(crate::latency::LatencyStage::TransitionTransaction);
1620 let step_scope = parse_step_scope(&timer.scope_kind, &timer.scope_key)?;
1621 let registry = self.registry_snapshot();
1622 crate::transaction::run_for(&self.backend, async {
1623 let Some(instance) = self.backend.store().load_instance(&timer.saga_id).await? else {
1624 let fencing_now = match claimed_at {
1626 Some(started) => elapsed_epoch_ms(base_now_ms, started)?,
1627 None => base_now_ms,
1628 };
1629 return Ok(
1630 match self
1631 .backend
1632 .store()
1633 .complete_timer(&timer.timer_id, token, fencing_now)
1634 .await?
1635 {
1636 TimerFencing::Applied => TimerOutcome::SkippedStale,
1637 TimerFencing::Lost => TimerOutcome::SkippedFencingLost,
1638 },
1639 );
1640 };
1641 if !instance.control_state.allows_automatic_actions() {
1642 return Ok(TimerOutcome::SkippedPaused);
1643 }
1644 let now_ms = match claimed_at {
1645 Some(started) => elapsed_epoch_ms(base_now_ms, started)?,
1646 None => base_now_ms,
1647 };
1648 if matches!(
1650 self.backend
1651 .store()
1652 .complete_timer(&timer.timer_id, token, now_ms)
1653 .await?,
1654 TimerFencing::Lost
1655 ) {
1656 return Ok(TimerOutcome::SkippedFencingLost);
1657 }
1658 crate::latency::observe(
1659 crate::latency::LatencyStage::TimerLateness,
1660 Duration::from_millis(now_ms.saturating_sub(timer.due_at_ms).max(0) as u64),
1661 );
1662 if matches!(
1666 timer.kind.as_str(),
1667 KIND_STEP_TIMEOUT
1668 | KIND_CANCEL_TIMEOUT
1669 | KIND_COMPENSATE_TIMEOUT
1670 | KIND_RESOLVE_TIMEOUT
1671 ) && timer.expected_saga_version != instance.version
1672 {
1673 return Ok(TimerOutcome::SkippedStale);
1674 }
1675 let definition = registry
1676 .get_for_tenant(
1677 &instance.tenant,
1678 &instance.workflow,
1679 instance.definition_version,
1680 )
1681 .ok_or_else(|| anyhow::anyhow!("instance definition is not registered"))?;
1682
1683 let status = match timer.kind.as_str() {
1685 KIND_STEP_TIMEOUT => {
1686 Box::pin(self.on_step_timeout(
1687 &instance,
1688 definition,
1689 step_scope.as_ref(),
1690 timer,
1691 now_ms,
1692 ))
1693 .await?
1694 }
1695 KIND_CANCEL_TIMEOUT => {
1696 Box::pin(self.on_phase_retry_timeout(
1697 &instance,
1698 definition,
1699 step_scope.as_ref(),
1700 timer,
1701 StepPhase::Cancel,
1702 SagaStatus::Cancelling,
1703 self.config.cancel_max_attempts,
1704 "cancel_attempts_exhausted",
1705 now_ms,
1706 ))
1707 .await?
1708 }
1709 KIND_COMPENSATE_TIMEOUT => {
1710 Box::pin(self.on_phase_retry_timeout(
1711 &instance,
1712 definition,
1713 step_scope.as_ref(),
1714 timer,
1715 StepPhase::Compensate,
1716 SagaStatus::Compensating,
1717 self.config.compensate_max_attempts,
1718 "compensation_attempts_exhausted",
1719 now_ms,
1720 ))
1721 .await?
1722 }
1723 KIND_RESOLVE_TIMEOUT => {
1724 Box::pin(self.on_phase_retry_timeout(
1725 &instance,
1726 definition,
1727 step_scope.as_ref(),
1728 timer,
1729 StepPhase::Resolve,
1730 SagaStatus::WaitingResolution,
1731 self.config.resolve_max_attempts,
1732 "resolve_attempts_exhausted",
1733 now_ms,
1734 ))
1735 .await?
1736 }
1737 KIND_RESOLUTION_BUDGET
1738 | KIND_FORWARD_RESOLUTION_BUDGET
1739 | KIND_COMPENSATION_RESOLUTION_BUDGET => {
1740 if instance.status == SagaStatus::WaitingResolution {
1743 Some(
1744 self.escalate_manual(
1745 &instance,
1746 TriggerKind::Timer,
1747 &timer.timer_id,
1748 "resolution_budget_exhausted",
1749 )
1750 .await?,
1751 )
1752 } else {
1753 None
1754 }
1755 }
1756 KIND_INSTANCE_DEADLINE => {
1757 Box::pin(self.on_instance_deadline(&instance, definition, timer, now_ms))
1758 .await?
1759 }
1760 _ => anyhow::bail!("unknown timer kind"),
1761 };
1762 Ok(match status {
1763 Some(status) => TimerOutcome::Applied {
1764 status,
1765 },
1766 None => TimerOutcome::SkippedStale,
1768 })
1769 })
1770 .await
1771 }
1772
1773 pub async fn pause(
1783 &self,
1784 management: &SagaManagementContext,
1785 tenant: &TenantId,
1786 saga_id: &SagaId,
1787 operation_id: &str,
1788 ) -> anyhow::Result<()> {
1789 self.pause_with_expectation(
1790 management,
1791 tenant,
1792 saga_id,
1793 operation_id,
1794 SagaManagementExpectation::default(),
1795 )
1796 .await
1797 }
1798
1799 pub async fn pause_with_expectation(
1810 &self,
1811 management: &SagaManagementContext,
1812 tenant: &TenantId,
1813 saga_id: &SagaId,
1814 operation_id: &str,
1815 expectation: SagaManagementExpectation,
1816 ) -> anyhow::Result<()> {
1817 management.require(SagaManagementPermission::Pause)?;
1819 crate::transaction::run_for(&self.backend, async {
1820 let instance = self
1821 .backend
1822 .store()
1823 .load_instance(saga_id)
1824 .await?
1825 .filter(|instance| &instance.tenant == tenant)
1826 .ok_or(SagaManagementError::NotFound)?;
1827 match self
1830 .backend
1831 .store()
1832 .set_control_state(&ControlTransitionSpec {
1833 saga_id,
1834 expected_version: instance.version,
1835 expected_control_version: instance.control_version,
1836 from: nasaga_core::ControlState::Active,
1837 to: nasaga_core::ControlState::Paused,
1838 operation_id,
1839 actor: management.actor(),
1840 reason: management.reason(),
1841 })
1842 .await?
1843 {
1844 ControlCasOutcome::Applied => {
1845 expectation.verify(instance.version, instance.control_version)?;
1846 self.reserve_action_rate_budget(tenant).await?;
1849 Ok(())
1850 }
1851 ControlCasOutcome::AlreadyApplied => Ok(()),
1852 ControlCasOutcome::Conflict => {
1853 expectation.verify(instance.version, instance.control_version)?;
1854 if instance.control_state == nasaga_core::ControlState::Active {
1855 Err(anyhow::Error::new(SagaConcurrencyError::StaleSnapshot)
1858 .context("pause lost the control-state race; reload and retry"))
1859 } else {
1860 Err(management_precondition(
1861 "pause requires an active control state",
1862 ))
1863 }
1864 }
1865 }
1866 })
1867 .await?;
1868 tracing::info!(
1869 saga_id = saga_id.as_str(),
1870 tenant = tenant.as_str(),
1871 actor = management.actor(),
1872 operation_id,
1873 control_state = "PAUSED",
1874 "Saga 暂停操作已提交"
1875 );
1876 Ok(())
1877 }
1878
1879 pub async fn resume(
1890 &self,
1891 management: &SagaManagementContext,
1892 tenant: &TenantId,
1893 saga_id: &SagaId,
1894 operation_id: &str,
1895 ) -> anyhow::Result<()> {
1896 self.resume_with_expectation(
1897 management,
1898 tenant,
1899 saga_id,
1900 operation_id,
1901 SagaManagementExpectation::default(),
1902 )
1903 .await
1904 }
1905
1906 pub async fn resume_with_expectation(
1917 &self,
1918 management: &SagaManagementContext,
1919 tenant: &TenantId,
1920 saga_id: &SagaId,
1921 operation_id: &str,
1922 expectation: SagaManagementExpectation,
1923 ) -> anyhow::Result<()> {
1924 management.require(SagaManagementPermission::Resume)?;
1926 crate::transaction::run_for(&self.backend, async {
1927 let instance = self
1928 .backend
1929 .store()
1930 .load_instance(saga_id)
1931 .await?
1932 .filter(|instance| &instance.tenant == tenant)
1933 .ok_or(SagaManagementError::NotFound)?;
1934 match self
1937 .backend
1938 .store()
1939 .set_control_state(&ControlTransitionSpec {
1940 saga_id,
1941 expected_version: instance.version,
1942 expected_control_version: instance.control_version,
1943 from: nasaga_core::ControlState::Paused,
1944 to: nasaga_core::ControlState::Active,
1945 operation_id,
1946 actor: management.actor(),
1947 reason: management.reason(),
1948 })
1949 .await?
1950 {
1951 ControlCasOutcome::Applied => {
1952 expectation.verify(instance.version, instance.control_version)?;
1953 self.reserve_action_rate_budget(tenant).await?;
1956 self.backend.store().wake_saga_timers(saga_id).await?;
1959 Ok(())
1960 }
1961 ControlCasOutcome::AlreadyApplied => Ok(()),
1962 ControlCasOutcome::Conflict => {
1963 expectation.verify(instance.version, instance.control_version)?;
1964 if instance.control_state == nasaga_core::ControlState::Paused {
1965 Err(anyhow::Error::new(SagaConcurrencyError::StaleSnapshot)
1968 .context("resume lost the control-state race; reload and retry"))
1969 } else {
1970 Err(management_precondition(
1971 "resume requires a paused control state",
1972 ))
1973 }
1974 }
1975 }
1976 })
1977 .await?;
1978 tracing::info!(
1979 saga_id = saga_id.as_str(),
1980 tenant = tenant.as_str(),
1981 actor = management.actor(),
1982 operation_id,
1983 control_state = "ACTIVE",
1984 "Saga 恢复操作已提交"
1985 );
1986 Ok(())
1987 }
1988
1989 pub async fn retry_compensation(
2004 &self,
2005 management: &SagaManagementContext,
2006 tenant: &TenantId,
2007 saga_id: &SagaId,
2008 operation_id: &str,
2009 now_ms: i64,
2010 ) -> anyhow::Result<SagaStatus> {
2011 self.retry_compensation_with_expectation(
2012 management,
2013 tenant,
2014 saga_id,
2015 operation_id,
2016 now_ms,
2017 SagaManagementExpectation::default(),
2018 )
2019 .await
2020 }
2021
2022 pub async fn retry_compensation_with_expectation(
2034 &self,
2035 management: &SagaManagementContext,
2036 tenant: &TenantId,
2037 saga_id: &SagaId,
2038 operation_id: &str,
2039 now_ms: i64,
2040 expectation: SagaManagementExpectation,
2041 ) -> anyhow::Result<SagaStatus> {
2042 management.require(SagaManagementPermission::RetryCompensation)?;
2044 let registry = self.registry_snapshot();
2045 let status = crate::transaction::run_for(&self.backend, async {
2046 let instance = self
2047 .backend
2048 .store()
2049 .load_instance(saga_id)
2050 .await?
2051 .filter(|instance| &instance.tenant == tenant)
2052 .ok_or(SagaManagementError::NotFound)?;
2053 match self
2054 .backend
2055 .store()
2056 .record_management_operation(
2057 saga_id,
2058 operation_id,
2059 "retry_frozen_compensation",
2060 management.actor(),
2061 management.reason(),
2062 )
2063 .await?
2064 {
2065 ManagementAuditOutcome::AlreadyRecorded => return Ok(instance.status),
2068 ManagementAuditOutcome::Recorded => {}
2069 }
2070 self.reserve_action_rate_budget(tenant).await?;
2073 expectation.verify(instance.version, instance.control_version)?;
2076 if !instance.control_state.allows_automatic_actions() {
2077 return Err(management_precondition(
2078 "retry_compensation requires ACTIVE control state; resume first",
2079 ));
2080 }
2081 if instance.status != SagaStatus::ManualIntervention {
2082 return Err(management_precondition(
2083 "retry_compensation requires MANUAL_INTERVENTION",
2084 ));
2085 }
2086 if instance.compensation_plan_version.is_none() {
2087 return Err(management_precondition(
2088 "no frozen compensation plan to retry",
2089 ));
2090 }
2091 let definition = registry
2092 .get_for_tenant(
2093 &instance.tenant,
2094 &instance.workflow,
2095 instance.definition_version,
2096 )
2097 .ok_or_else(|| management_precondition("instance definition is not registered"))?;
2098 let steps = self.backend.store().load_steps(saga_id).await?;
2099 match next_pending_compensation(&instance, definition, &steps, true)? {
2100 Some(step) => {
2101 let recovering_halted = steps.iter().any(|row| {
2102 row.step == step
2103 && row.compensation_status == StepCompensationStatus::Halted
2104 });
2105 if recovering_halted {
2106 self.backend
2109 .store()
2110 .reopen_halted_compensation(saga_id, &step, operation_id)
2111 .await?;
2112 }
2113 let version = self
2114 .advance(
2115 &instance,
2116 SagaStatus::Compensating,
2117 Direction::Compensating,
2118 Some(&step),
2119 TriggerKind::Admin,
2120 operation_id,
2121 None,
2122 None,
2123 )
2124 .await?;
2125 let step_def = definition.step(&step).ok_or_else(|| {
2126 management_precondition("plan step absent from definition")
2127 })?;
2128 let attempt = self
2129 .next_attempt(saga_id, &step, StepPhase::Compensate)
2130 .await?;
2131 self.issue_command(
2132 &instance,
2133 step_def,
2134 StepPhase::Compensate,
2135 attempt,
2136 None,
2137 Some(operation_id),
2138 now_ms,
2139 version,
2140 )
2141 .await?;
2142 Ok(SagaStatus::Compensating)
2143 }
2144 None => {
2145 let version = self
2148 .advance(
2149 &instance,
2150 SagaStatus::Compensating,
2151 Direction::Compensating,
2152 None,
2153 TriggerKind::Admin,
2154 operation_id,
2155 None,
2156 None,
2157 )
2158 .await?;
2159 self.converge_compensated(&instance, version, TriggerKind::Admin, operation_id)
2160 .await?;
2161 Ok(SagaStatus::Compensated)
2162 }
2163 }
2164 })
2165 .await?;
2166 tracing::info!(
2167 saga_id = saga_id.as_str(),
2168 tenant = tenant.as_str(),
2169 actor = management.actor(),
2170 operation_id,
2171 saga_status = status.as_str(),
2172 "Saga 人工补偿恢复操作已提交"
2173 );
2174 Ok(status)
2175 }
2176
2177 pub async fn retry_resolution(
2192 &self,
2193 management: &SagaManagementContext,
2194 tenant: &TenantId,
2195 saga_id: &SagaId,
2196 operation_id: &str,
2197 now_ms: i64,
2198 ) -> anyhow::Result<SagaStatus> {
2199 self.retry_resolution_with_expectation(
2200 management,
2201 tenant,
2202 saga_id,
2203 operation_id,
2204 now_ms,
2205 SagaManagementExpectation::default(),
2206 )
2207 .await
2208 }
2209
2210 pub async fn retry_resolution_with_expectation(
2222 &self,
2223 management: &SagaManagementContext,
2224 tenant: &TenantId,
2225 saga_id: &SagaId,
2226 operation_id: &str,
2227 now_ms: i64,
2228 expectation: SagaManagementExpectation,
2229 ) -> anyhow::Result<SagaStatus> {
2230 management.require(SagaManagementPermission::RetryResolution)?;
2231 let registry = self.registry_snapshot();
2232 let status = crate::transaction::run_for(&self.backend, async {
2233 let instance = self
2234 .backend
2235 .store()
2236 .load_instance(saga_id)
2237 .await?
2238 .filter(|instance| &instance.tenant == tenant)
2239 .ok_or(SagaManagementError::NotFound)?;
2240 match self
2241 .backend
2242 .store()
2243 .record_management_operation(
2244 saga_id,
2245 operation_id,
2246 "retry_resolution",
2247 management.actor(),
2248 management.reason(),
2249 )
2250 .await?
2251 {
2252 ManagementAuditOutcome::AlreadyRecorded => return Ok(instance.status),
2253 ManagementAuditOutcome::Recorded => {}
2254 }
2255 self.reserve_action_rate_budget(tenant).await?;
2258 expectation.verify(instance.version, instance.control_version)?;
2261 if !instance.control_state.allows_automatic_actions() {
2262 return Err(management_precondition(
2263 "retry_resolution requires ACTIVE control state; resume first",
2264 ));
2265 }
2266 if instance.status != SagaStatus::ManualIntervention {
2267 return Err(management_precondition(
2268 "retry_resolution requires MANUAL_INTERVENTION",
2269 ));
2270 }
2271 let step = instance
2272 .current_step
2273 .as_ref()
2274 .ok_or_else(|| management_precondition("manual resolution has no current step"))?;
2275 let definition = registry
2276 .get_for_tenant(
2277 &instance.tenant,
2278 &instance.workflow,
2279 instance.definition_version,
2280 )
2281 .ok_or_else(|| management_precondition("instance definition is not registered"))?;
2282 let step_def = definition
2283 .step(step)
2284 .ok_or_else(|| management_precondition("resolution step absent from definition"))?;
2285 if step_def.resolution().mode() != Some(ResolutionMode::Poll) {
2286 return Err(management_precondition(
2287 "retry_resolution requires a Poll resolver",
2288 ));
2289 }
2290 let row = self
2291 .backend
2292 .store()
2293 .load_steps(saga_id)
2294 .await?
2295 .into_iter()
2296 .find(|row| row.step == *step)
2297 .ok_or_else(|| management_precondition("resolution step journal is missing"))?;
2298 let has_unknown_target = match instance.direction {
2299 Direction::Forward => row.forward_status == StepForwardStatus::Unknown,
2300 Direction::Compensating => {
2301 row.compensation_status == StepCompensationStatus::Unknown
2302 }
2303 };
2304 if !has_unknown_target
2305 || !matches!(
2306 row.resolution_status,
2307 StepResolutionStatus::Pending | StepResolutionStatus::Halted
2308 )
2309 {
2310 return Err(management_precondition(
2311 "retry_resolution requires an unresolved Unknown effect",
2312 ));
2313 }
2314
2315 let attempt = self.next_attempt(saga_id, step, StepPhase::Resolve).await?;
2316 if row.resolution_status == StepResolutionStatus::Halted {
2317 self.backend
2320 .store()
2321 .reopen_halted_resolution(saga_id, step, operation_id)
2322 .await?;
2323 }
2324 let version = self
2325 .advance(
2326 &instance,
2327 SagaStatus::WaitingResolution,
2328 instance.direction,
2329 Some(step),
2330 TriggerKind::Admin,
2331 operation_id,
2332 None,
2333 None,
2334 )
2335 .await?;
2336 self.schedule_timer(
2337 saga_id,
2338 TimerScope::Step(step),
2339 resolution_budget_kind(instance.direction),
2340 attempt,
2341 checked_deadline(now_ms, duration_millis(step_def.resolution().budget())?)?,
2342 version,
2343 )
2344 .await?;
2345 self.issue_command(
2346 &instance,
2347 step_def,
2348 StepPhase::Resolve,
2349 attempt,
2350 None,
2351 Some(operation_id),
2352 now_ms,
2353 version,
2354 )
2355 .await?;
2356 Ok(SagaStatus::WaitingResolution)
2357 })
2358 .await?;
2359 tracing::info!(
2360 saga_id = saga_id.as_str(),
2361 tenant = tenant.as_str(),
2362 actor = management.actor(),
2363 operation_id,
2364 saga_status = status.as_str(),
2365 "Saga 人工解决查询恢复操作已提交"
2366 );
2367 Ok(status)
2368 }
2369
2370 pub async fn manual_close(
2387 &self,
2388 management: &SagaManagementContext,
2389 tenant: &TenantId,
2390 saga_id: &SagaId,
2391 operation_id: &str,
2392 ) -> anyhow::Result<SagaStatus> {
2393 self.manual_close_with_expectation(
2394 management,
2395 tenant,
2396 saga_id,
2397 operation_id,
2398 SagaManagementExpectation::default(),
2399 )
2400 .await
2401 }
2402
2403 pub async fn manual_close_with_expectation(
2414 &self,
2415 management: &SagaManagementContext,
2416 tenant: &TenantId,
2417 saga_id: &SagaId,
2418 operation_id: &str,
2419 expectation: SagaManagementExpectation,
2420 ) -> anyhow::Result<SagaStatus> {
2421 management.require(SagaManagementPermission::ManualClose)?;
2422 if !self.config.enable_manual_close {
2425 return Err(management_precondition(
2426 "manual close is disabled: deploy MANUALLY_CLOSED-aware readers to every replica, then enable it explicitly",
2427 ));
2428 }
2429 let status = crate::transaction::run_for(&self.backend, async {
2430 let instance = self
2431 .backend
2432 .store()
2433 .load_instance(saga_id)
2434 .await?
2435 .filter(|instance| &instance.tenant == tenant)
2436 .ok_or(SagaManagementError::NotFound)?;
2437 match self
2440 .backend
2441 .store()
2442 .record_management_operation(
2443 saga_id,
2444 operation_id,
2445 nasaga_backend::MANUAL_CLOSE_ACTION,
2446 management.actor(),
2447 management.reason(),
2448 )
2449 .await?
2450 {
2451 ManagementAuditOutcome::AlreadyRecorded => return Ok(instance.status),
2453 ManagementAuditOutcome::Recorded => {}
2454 }
2455 self.reserve_action_rate_budget(tenant).await?;
2458 expectation.verify(instance.version, instance.control_version)?;
2461 if !instance.control_state.allows_automatic_actions() {
2462 return Err(management_precondition(
2463 "manual_close requires ACTIVE control state; resume first",
2464 ));
2465 }
2466 if instance.status != SagaStatus::ManualIntervention {
2467 return Err(management_precondition(
2468 "manual_close requires MANUAL_INTERVENTION",
2469 ));
2470 }
2471 self.advance(
2472 &instance,
2473 SagaStatus::ManuallyClosed,
2474 instance.direction,
2475 None,
2476 TriggerKind::Admin,
2477 operation_id,
2478 None,
2479 None,
2480 )
2481 .await?;
2482 self.backend
2485 .store()
2486 .cancel_scope_timers(saga_id, TimerScope::Instance, None)
2487 .await?;
2488 Ok(SagaStatus::ManuallyClosed)
2489 })
2490 .await?;
2491 tracing::info!(
2492 saga_id = saga_id.as_str(),
2493 tenant = tenant.as_str(),
2494 actor = management.actor(),
2495 operation_id,
2496 saga_status = status.as_str(),
2497 "Saga 人工关闭操作已提交"
2498 );
2499 Ok(status)
2500 }
2501
2502 async fn project_deferred_result(
2517 &self,
2518 context: &ResultContext<'_>,
2519 ) -> anyhow::Result<SagaStatus> {
2520 let saga_id = &context.identity.saga_id;
2521 let step = &context.identity.step;
2522 let row = self
2523 .backend
2524 .store()
2525 .load_steps(saga_id)
2526 .await?
2527 .into_iter()
2528 .find(|row| row.step == *step)
2529 .ok_or_else(|| anyhow::anyhow!("step journal row is missing"))?;
2530 let current_attempt = match context.identity.phase {
2531 StepPhase::Execute => row.execute_attempt,
2532 StepPhase::Cancel => row.cancel_attempt,
2533 StepPhase::Compensate => row.compensate_attempt,
2534 StepPhase::Resolve => row.resolve_attempt,
2535 };
2536 if current_attempt != Some(context.identity.attempt.get()) {
2537 return Ok(context.instance.status);
2538 }
2539
2540 let mut patch = StepJournalPatch {
2541 last_error_code: context.reason_code,
2542 ..Default::default()
2543 };
2544 let timeout_kind = match context.identity.phase {
2545 StepPhase::Execute => {
2546 patch.forward_status = Some(match context.status {
2547 StepAttemptStatus::Succeeded => StepForwardStatus::Succeeded,
2548 StepAttemptStatus::Rejected => StepForwardStatus::Rejected,
2549 StepAttemptStatus::Unknown => StepForwardStatus::Unknown,
2550 StepAttemptStatus::Halted => StepForwardStatus::Halted,
2551 _ => anyhow::bail!("invalid deferred execute status"),
2552 });
2553 patch.mark_finished = !matches!(context.status, StepAttemptStatus::Unknown);
2554 KIND_STEP_TIMEOUT
2555 }
2556 StepPhase::Cancel => {
2557 match context.status {
2558 StepAttemptStatus::CancelConfirmed => {
2559 patch.forward_status = Some(StepForwardStatus::Cancelled);
2560 patch.cancel_status = Some(StepCancelStatus::Confirmed);
2561 patch.mark_finished = true;
2562 }
2563 StepAttemptStatus::AlreadyTerminal => {
2564 patch.forward_status = context.terminal;
2565 patch.cancel_status = Some(StepCancelStatus::AlreadyTerminal);
2566 patch.mark_finished = true;
2567 }
2568 StepAttemptStatus::ResolutionPending => {
2569 patch.cancel_status = Some(StepCancelStatus::ResolutionPending);
2570 }
2571 _ => anyhow::bail!("invalid deferred cancel status"),
2572 }
2573 KIND_CANCEL_TIMEOUT
2574 }
2575 StepPhase::Compensate => {
2576 patch.compensation_status = Some(match context.status {
2577 StepAttemptStatus::Succeeded => StepCompensationStatus::Succeeded,
2578 StepAttemptStatus::Unknown => StepCompensationStatus::Unknown,
2579 StepAttemptStatus::Halted => StepCompensationStatus::Halted,
2580 _ => anyhow::bail!("invalid deferred compensation status"),
2581 });
2582 KIND_COMPENSATE_TIMEOUT
2583 }
2584 StepPhase::Resolve => {
2585 patch.resolution_status = Some(match context.status {
2586 StepAttemptStatus::Succeeded => StepResolutionStatus::Succeeded,
2587 StepAttemptStatus::Rejected => StepResolutionStatus::Rejected,
2588 StepAttemptStatus::Unknown => StepResolutionStatus::Pending,
2589 StepAttemptStatus::Halted => StepResolutionStatus::Halted,
2590 _ => anyhow::bail!("invalid deferred resolution status"),
2591 });
2592 if context.instance.direction == Direction::Forward {
2593 patch.forward_status = match context.status {
2594 StepAttemptStatus::Succeeded => Some(StepForwardStatus::Succeeded),
2595 StepAttemptStatus::Rejected => Some(StepForwardStatus::Rejected),
2596 StepAttemptStatus::Halted => Some(StepForwardStatus::Halted),
2597 StepAttemptStatus::Unknown => Some(StepForwardStatus::Unknown),
2598 _ => None,
2599 };
2600 } else {
2601 patch.compensation_status = match context.status {
2602 StepAttemptStatus::Succeeded => Some(StepCompensationStatus::Succeeded),
2603 StepAttemptStatus::Rejected => Some(StepCompensationStatus::Pending),
2604 StepAttemptStatus::Halted => Some(StepCompensationStatus::Halted),
2605 StepAttemptStatus::Unknown => Some(StepCompensationStatus::Unknown),
2606 _ => None,
2607 };
2608 }
2609 KIND_RESOLVE_TIMEOUT
2610 }
2611 };
2612
2613 let resolve_after_compensation_success = context.identity.phase == StepPhase::Resolve
2617 && context.instance.direction == Direction::Compensating
2618 && context.status == StepAttemptStatus::Rejected
2619 && row.compensation_status == StepCompensationStatus::Succeeded
2620 && patch.compensation_status == Some(StepCompensationStatus::Pending);
2621 if resolve_after_compensation_success
2622 || merge_deferred_projection(&row, &mut patch).is_err()
2623 {
2624 let existing = conflicting_projection_status(&row, context)
2628 .ok_or_else(|| anyhow::anyhow!("conflicting projection has no settled fact"))?;
2629 self.backend
2630 .store()
2631 .record_attempt_conflict(&AttemptConflictFact {
2632 saga_id,
2633 step,
2634 phase: context.identity.phase,
2635 attempt: context.identity.attempt,
2636 existing_status: existing,
2637 incoming_status: context.status,
2638 incoming_event_id: context.event_id,
2639 conflict_kind: SagaConflictKind::CrossPhaseFact,
2640 })
2641 .await?;
2642 if !context.instance.status.is_terminal()
2643 && context.instance.status != SagaStatus::ManualIntervention
2644 {
2645 return self
2646 .escalate_manual(
2647 context.instance,
2648 TriggerKind::Event,
2649 context.event_id,
2650 "cross_phase_fact_conflict",
2651 )
2652 .await;
2653 }
2654 return Ok(context.instance.status);
2655 }
2656 self.backend
2657 .store()
2658 .cancel_scope_timers(saga_id, TimerScope::Step(step), Some(timeout_kind))
2659 .await?;
2660 self.patch_step(saga_id, step, patch).await?;
2661 Ok(context.instance.status)
2662 }
2663
2664 async fn apply_monotonic_projection(
2676 &self,
2677 context: &ResultContext<'_>,
2678 mut patch: StepJournalPatch<'_>,
2679 ) -> anyhow::Result<Option<SagaStatus>> {
2680 let saga_id = &context.identity.saga_id;
2681 let step = &context.identity.step;
2682 let row = self
2683 .backend
2684 .store()
2685 .load_steps(saga_id)
2686 .await?
2687 .into_iter()
2688 .find(|row| row.step == *step)
2689 .ok_or_else(|| anyhow::anyhow!("step journal row is missing"))?;
2690
2691 let resolve_after_compensation_success = context.identity.phase == StepPhase::Resolve
2694 && context.instance.direction == Direction::Compensating
2695 && context.status == StepAttemptStatus::Rejected
2696 && row.compensation_status == StepCompensationStatus::Succeeded
2697 && patch.compensation_status == Some(StepCompensationStatus::Pending);
2698 if resolve_after_compensation_success
2699 || merge_deferred_projection(&row, &mut patch).is_err()
2700 {
2701 let existing = conflicting_projection_status(&row, context)
2702 .ok_or_else(|| anyhow::anyhow!("conflicting projection has no settled fact"))?;
2703 self.backend
2704 .store()
2705 .record_attempt_conflict(&AttemptConflictFact {
2706 saga_id,
2707 step,
2708 phase: context.identity.phase,
2709 attempt: context.identity.attempt,
2710 existing_status: existing,
2711 incoming_status: context.status,
2712 incoming_event_id: context.event_id,
2713 conflict_kind: SagaConflictKind::CrossPhaseFact,
2714 })
2715 .await?;
2716 if !context.instance.status.is_terminal()
2717 && context.instance.status != SagaStatus::ManualIntervention
2718 {
2719 let status = self
2720 .escalate_manual(
2721 context.instance,
2722 TriggerKind::Event,
2723 context.event_id,
2724 "cross_phase_fact_conflict",
2725 )
2726 .await?;
2727 return Ok(Some(status));
2728 }
2729 return Ok(Some(context.instance.status));
2730 }
2731 self.patch_step(saga_id, step, patch).await?;
2732 Ok(None)
2733 }
2734
2735 async fn on_execute_result(&self, context: &ResultContext<'_>) -> anyhow::Result<SagaStatus> {
2742 if context.instance.status != SagaStatus::Running
2745 || context.instance.current_step.as_ref() != Some(&context.identity.step)
2746 {
2747 return self.project_deferred_result(context).await;
2748 }
2749 let saga_id = &context.identity.saga_id;
2750 let step = &context.identity.step;
2751 match context.status {
2752 StepAttemptStatus::Succeeded => {
2753 self.patch_step(
2754 saga_id,
2755 step,
2756 StepJournalPatch {
2757 forward_status: Some(StepForwardStatus::Succeeded),
2758 mark_finished: true,
2759 ..Default::default()
2760 },
2761 )
2762 .await?;
2763 self.backend
2764 .store()
2765 .cancel_scope_timers(saga_id, TimerScope::Step(step), Some(KIND_STEP_TIMEOUT))
2766 .await?;
2767 self.proceed_forward(context).await
2768 }
2769 StepAttemptStatus::Rejected => {
2770 self.patch_step(
2771 saga_id,
2772 step,
2773 StepJournalPatch {
2774 forward_status: Some(StepForwardStatus::Rejected),
2775 last_error_code: context.reason_code,
2776 mark_finished: true,
2777 ..Default::default()
2778 },
2779 )
2780 .await?;
2781 self.backend
2782 .store()
2783 .cancel_scope_timers(saga_id, TimerScope::Step(step), Some(KIND_STEP_TIMEOUT))
2784 .await?;
2785 self.enter_compensation(context).await
2787 }
2788 StepAttemptStatus::Unknown => {
2789 self.enter_waiting_resolution(context, Direction::Forward)
2790 .await
2791 }
2792 StepAttemptStatus::Halted => {
2793 self.patch_step(
2794 saga_id,
2795 step,
2796 StepJournalPatch {
2797 forward_status: Some(StepForwardStatus::Halted),
2798 last_error_code: context.reason_code,
2799 mark_finished: true,
2800 ..Default::default()
2801 },
2802 )
2803 .await?;
2804 self.backend
2805 .store()
2806 .cancel_scope_timers(saga_id, TimerScope::Step(step), None)
2807 .await?;
2808 self.escalate_manual(
2809 context.instance,
2810 TriggerKind::Event,
2811 context.event_id,
2812 "step_halted",
2813 )
2814 .await
2815 }
2816 _ => anyhow::bail!("cancel-vocabulary status on an execute result"),
2817 }
2818 }
2819
2820 async fn on_cancel_result(&self, context: &ResultContext<'_>) -> anyhow::Result<SagaStatus> {
2827 if context.instance.status != SagaStatus::Cancelling
2828 || context.instance.current_step.as_ref() != Some(&context.identity.step)
2829 {
2830 return self.project_deferred_result(context).await;
2831 }
2832 let saga_id = &context.identity.saga_id;
2833 let step = &context.identity.step;
2834 match context.status {
2835 StepAttemptStatus::CancelConfirmed => {
2836 if let Some(status) = self
2838 .apply_monotonic_projection(
2839 context,
2840 StepJournalPatch {
2841 forward_status: Some(StepForwardStatus::Cancelled),
2842 cancel_status: Some(StepCancelStatus::Confirmed),
2843 mark_finished: true,
2844 ..Default::default()
2845 },
2846 )
2847 .await?
2848 {
2849 return Ok(status);
2850 }
2851 self.backend
2852 .store()
2853 .cancel_scope_timers(saga_id, TimerScope::Step(step), Some(KIND_CANCEL_TIMEOUT))
2854 .await?;
2855 self.enter_compensation(context).await
2856 }
2857 StepAttemptStatus::AlreadyTerminal => {
2858 let terminal = context.terminal.ok_or_else(|| {
2859 anyhow::anyhow!("ALREADY_TERMINAL requires a terminal status")
2860 })?;
2861 if let Some(status) = self
2862 .apply_monotonic_projection(
2863 context,
2864 StepJournalPatch {
2865 forward_status: Some(terminal),
2866 cancel_status: Some(StepCancelStatus::AlreadyTerminal),
2867 mark_finished: true,
2868 ..Default::default()
2869 },
2870 )
2871 .await?
2872 {
2873 return Ok(status);
2874 }
2875 self.backend
2876 .store()
2877 .cancel_scope_timers(saga_id, TimerScope::Step(step), Some(KIND_CANCEL_TIMEOUT))
2878 .await?;
2879 match terminal {
2880 StepForwardStatus::Succeeded => match context.step_def.timeout_policy() {
2881 TimeoutPolicy::AcceptLateSuccess => self.proceed_forward(context).await,
2883 TimeoutPolicy::CompensateLateSuccess => {
2885 self.enter_compensation(context).await
2886 }
2887 },
2888 StepForwardStatus::Rejected => self.enter_compensation(context).await,
2889 StepForwardStatus::Halted => {
2890 self.escalate_manual(
2891 context.instance,
2892 TriggerKind::Event,
2893 context.event_id,
2894 "step_halted",
2895 )
2896 .await
2897 }
2898 _ => anyhow::bail!("ALREADY_TERMINAL carried a non-terminal forward status"),
2899 }
2900 }
2901 StepAttemptStatus::ResolutionPending => {
2902 if let Some(status) = self
2904 .apply_monotonic_projection(
2905 context,
2906 StepJournalPatch {
2907 cancel_status: Some(StepCancelStatus::ResolutionPending),
2908 ..Default::default()
2909 },
2910 )
2911 .await?
2912 {
2913 return Ok(status);
2914 }
2915 self.backend
2916 .store()
2917 .cancel_scope_timers(saga_id, TimerScope::Step(step), Some(KIND_CANCEL_TIMEOUT))
2918 .await?;
2919 self.enter_waiting_resolution(context, Direction::Forward)
2920 .await
2921 }
2922 _ => anyhow::bail!("non-cancel status on a cancel result"),
2923 }
2924 }
2925
2926 async fn on_compensate_result(
2933 &self,
2934 context: &ResultContext<'_>,
2935 ) -> anyhow::Result<SagaStatus> {
2936 if context.instance.status != SagaStatus::Compensating
2937 || context.instance.current_step.as_ref() != Some(&context.identity.step)
2938 {
2939 return self.project_deferred_result(context).await;
2940 }
2941 let saga_id = &context.identity.saga_id;
2942 let step = &context.identity.step;
2943 self.backend
2944 .store()
2945 .cancel_scope_timers(
2946 saga_id,
2947 TimerScope::Step(step),
2948 Some(KIND_COMPENSATE_TIMEOUT),
2949 )
2950 .await?;
2951 match context.status {
2952 StepAttemptStatus::Succeeded => {
2953 self.patch_step(
2954 saga_id,
2955 step,
2956 StepJournalPatch {
2957 compensation_status: Some(StepCompensationStatus::Succeeded),
2958 ..Default::default()
2959 },
2960 )
2961 .await?;
2962 self.continue_compensation(context).await
2963 }
2964 StepAttemptStatus::Unknown => {
2965 self.patch_step(
2966 saga_id,
2967 step,
2968 StepJournalPatch {
2969 compensation_status: Some(StepCompensationStatus::Unknown),
2970 ..Default::default()
2971 },
2972 )
2973 .await?;
2974 self.enter_waiting_resolution(context, Direction::Compensating)
2975 .await
2976 }
2977 StepAttemptStatus::Halted => {
2978 self.patch_step(
2979 saga_id,
2980 step,
2981 StepJournalPatch {
2982 compensation_status: Some(StepCompensationStatus::Halted),
2983 last_error_code: context.reason_code,
2984 ..Default::default()
2985 },
2986 )
2987 .await?;
2988 self.escalate_manual(
2989 context.instance,
2990 TriggerKind::Event,
2991 context.event_id,
2992 "compensation_halted",
2993 )
2994 .await
2995 }
2996 _ => anyhow::bail!("invalid status on a compensate result"),
2997 }
2998 }
2999
3000 async fn settle_resolution_from_committed_success(
3012 &self,
3013 context: &ResultContext<'_>,
3014 ) -> anyhow::Result<Option<SagaStatus>> {
3015 let saga_id = &context.identity.saga_id;
3016 let step = &context.identity.step;
3017 let row = self
3018 .backend
3019 .store()
3020 .load_steps(saga_id)
3021 .await?
3022 .into_iter()
3023 .find(|row| row.step == *step)
3024 .ok_or_else(|| anyhow::anyhow!("step journal row is missing"))?;
3025 let committed_success = match context.instance.direction {
3026 Direction::Forward => row.forward_status == StepForwardStatus::Succeeded,
3027 Direction::Compensating => row.compensation_status == StepCompensationStatus::Succeeded,
3028 };
3029 if !committed_success {
3030 return Ok(None);
3031 }
3032
3033 let mut patch = StepJournalPatch {
3034 resolution_status: Some(StepResolutionStatus::Succeeded),
3035 ..Default::default()
3036 };
3037 match context.instance.direction {
3038 Direction::Forward => {
3039 patch.forward_status = Some(StepForwardStatus::Succeeded);
3040 patch.mark_finished = true;
3041 }
3042 Direction::Compensating => {
3043 patch.compensation_status = Some(StepCompensationStatus::Succeeded);
3044 }
3045 }
3046 if let Some(status) = self.apply_monotonic_projection(context, patch).await? {
3047 return Ok(Some(status));
3048 }
3049
3050 self.cancel_resolution_timers(saga_id, step).await?;
3053 let status = match context.instance.direction {
3054 Direction::Forward => match context.step_def.timeout_policy() {
3055 TimeoutPolicy::AcceptLateSuccess => self.proceed_forward(context).await?,
3056 TimeoutPolicy::CompensateLateSuccess => self.enter_compensation(context).await?,
3057 },
3058 Direction::Compensating => self.continue_compensation(context).await?,
3059 };
3060 Ok(Some(status))
3061 }
3062
3063 async fn on_resolve_result(&self, context: &ResultContext<'_>) -> anyhow::Result<SagaStatus> {
3070 if context.instance.status != SagaStatus::WaitingResolution
3071 || context.instance.current_step.as_ref() != Some(&context.identity.step)
3072 {
3073 return self.project_deferred_result(context).await;
3074 }
3075 if let Some(status) = self
3076 .settle_resolution_from_committed_success(context)
3077 .await?
3078 {
3079 return Ok(status);
3080 }
3081 let saga_id = &context.identity.saga_id;
3082 let step = &context.identity.step;
3083 match context.instance.direction {
3084 Direction::Forward => match context.status {
3085 StepAttemptStatus::Succeeded => {
3086 if let Some(status) = self
3087 .apply_monotonic_projection(
3088 context,
3089 StepJournalPatch {
3090 forward_status: Some(StepForwardStatus::Succeeded),
3091 resolution_status: Some(StepResolutionStatus::Succeeded),
3092 mark_finished: true,
3093 ..Default::default()
3094 },
3095 )
3096 .await?
3097 {
3098 return Ok(status);
3099 }
3100 self.cancel_resolution_timers(saga_id, step).await?;
3101 self.proceed_forward(context).await
3102 }
3103 StepAttemptStatus::Rejected => {
3104 if let Some(status) = self
3107 .apply_monotonic_projection(
3108 context,
3109 StepJournalPatch {
3110 forward_status: Some(StepForwardStatus::Rejected),
3111 resolution_status: Some(StepResolutionStatus::Rejected),
3112 last_error_code: context.reason_code,
3113 mark_finished: true,
3114 ..Default::default()
3115 },
3116 )
3117 .await?
3118 {
3119 return Ok(status);
3120 }
3121 self.cancel_resolution_timers(saga_id, step).await?;
3122 self.enter_compensation(context).await
3123 }
3124 StepAttemptStatus::Unknown => self.reissue_resolve(context).await,
3125 StepAttemptStatus::Halted => {
3126 if let Some(status) = self
3127 .apply_monotonic_projection(
3128 context,
3129 StepJournalPatch {
3130 resolution_status: Some(StepResolutionStatus::Halted),
3131 last_error_code: context.reason_code,
3132 ..Default::default()
3133 },
3134 )
3135 .await?
3136 {
3137 return Ok(status);
3138 }
3139 self.escalate_manual(
3140 context.instance,
3141 TriggerKind::Event,
3142 context.event_id,
3143 "resolution_halted",
3144 )
3145 .await
3146 }
3147 _ => anyhow::bail!("invalid status on a resolve result"),
3148 },
3149 Direction::Compensating => match context.status {
3150 StepAttemptStatus::Succeeded => {
3151 if let Some(status) = self
3153 .apply_monotonic_projection(
3154 context,
3155 StepJournalPatch {
3156 compensation_status: Some(StepCompensationStatus::Succeeded),
3157 resolution_status: Some(StepResolutionStatus::Succeeded),
3158 ..Default::default()
3159 },
3160 )
3161 .await?
3162 {
3163 return Ok(status);
3164 }
3165 self.cancel_resolution_timers(saga_id, step).await?;
3166 self.continue_compensation(context).await
3167 }
3168 StepAttemptStatus::Rejected => {
3169 if let Some(status) = self
3171 .apply_monotonic_projection(
3172 context,
3173 StepJournalPatch {
3174 compensation_status: Some(StepCompensationStatus::Pending),
3175 resolution_status: Some(StepResolutionStatus::Rejected),
3176 ..Default::default()
3177 },
3178 )
3179 .await?
3180 {
3181 return Ok(status);
3182 }
3183 self.cancel_resolution_timers(saga_id, step).await?;
3184 let attempt = self
3185 .next_attempt(saga_id, step, StepPhase::Compensate)
3186 .await?;
3187 if attempt.get() > self.config.compensate_max_attempts {
3188 return self
3189 .escalate_manual(
3190 context.instance,
3191 TriggerKind::Event,
3192 context.event_id,
3193 "compensation_attempts_exhausted",
3194 )
3195 .await;
3196 }
3197 let version = self
3198 .advance(
3199 context.instance,
3200 SagaStatus::Compensating,
3201 Direction::Compensating,
3202 Some(step),
3203 TriggerKind::Event,
3204 context.event_id,
3205 None,
3206 None,
3207 )
3208 .await?;
3209 self.issue_command(
3210 context.instance,
3211 context.step_def,
3212 StepPhase::Compensate,
3213 attempt,
3214 None,
3215 None,
3216 context.now_ms,
3217 version,
3218 )
3219 .await?;
3220 Ok(SagaStatus::Compensating)
3221 }
3222 StepAttemptStatus::Unknown => self.reissue_resolve(context).await,
3223 StepAttemptStatus::Halted => {
3224 if let Some(status) = self
3225 .apply_monotonic_projection(
3226 context,
3227 StepJournalPatch {
3228 resolution_status: Some(StepResolutionStatus::Halted),
3229 last_error_code: context.reason_code,
3230 ..Default::default()
3231 },
3232 )
3233 .await?
3234 {
3235 return Ok(status);
3236 }
3237 self.escalate_manual(
3238 context.instance,
3239 TriggerKind::Event,
3240 context.event_id,
3241 "resolution_halted",
3242 )
3243 .await
3244 }
3245 _ => anyhow::bail!("invalid status on a resolve result"),
3246 },
3247 }
3248 }
3249
3250 async fn on_step_timeout(
3263 &self,
3264 instance: &SagaInstanceRow,
3265 definition: &WorkflowDefinition,
3266 step_scope: Option<&StepName>,
3267 timer: &SagaTimerRow,
3268 now_ms: i64,
3269 ) -> anyhow::Result<Option<SagaStatus>> {
3270 let Some(step) = step_scope else {
3271 anyhow::bail!("step-timeout timer without a step scope");
3272 };
3273 if instance.status != SagaStatus::Running || instance.current_step.as_ref() != Some(step) {
3275 return Ok(None);
3276 }
3277 let step_def = definition
3278 .step(step)
3279 .ok_or_else(|| anyhow::anyhow!("timer step absent from definition"))?;
3280 if step_def.cancel_mode() == CancelMode::ResolveOnly {
3281 self.patch_step(
3284 &instance.saga_id,
3285 step,
3286 StepJournalPatch {
3287 forward_status: Some(StepForwardStatus::Unknown),
3288 resolution_status: Some(StepResolutionStatus::Pending),
3289 ..Default::default()
3290 },
3291 )
3292 .await?;
3293 let version = self
3294 .advance(
3295 instance,
3296 SagaStatus::WaitingResolution,
3297 Direction::Forward,
3298 Some(step),
3299 TriggerKind::Timer,
3300 &timer.timer_id,
3301 None,
3302 None,
3303 )
3304 .await?;
3305 let resolution = step_def.resolution();
3306 self.schedule_timer(
3307 &instance.saga_id,
3308 TimerScope::Step(step),
3309 resolution_budget_kind(Direction::Forward),
3310 timer.attempt,
3311 checked_deadline(now_ms, duration_millis(resolution.budget())?)?,
3312 version,
3313 )
3314 .await?;
3315 if resolution.mode() == Some(ResolutionMode::Poll) {
3316 let attempt = self
3317 .next_attempt(&instance.saga_id, step, StepPhase::Resolve)
3318 .await?;
3319 self.issue_command(
3320 instance,
3321 step_def,
3322 StepPhase::Resolve,
3323 attempt,
3324 None,
3325 None,
3326 now_ms,
3327 version,
3328 )
3329 .await?;
3330 }
3331 return Ok(Some(SagaStatus::WaitingResolution));
3332 }
3333
3334 self.patch_step(
3337 &instance.saga_id,
3338 step,
3339 StepJournalPatch {
3340 forward_status: Some(StepForwardStatus::TimedOut),
3341 ..Default::default()
3342 },
3343 )
3344 .await?;
3345 let version = self
3346 .advance(
3347 instance,
3348 SagaStatus::Cancelling,
3349 Direction::Forward,
3350 Some(step),
3351 TriggerKind::Timer,
3352 &timer.timer_id,
3353 None,
3354 None,
3355 )
3356 .await?;
3357 let attempt = self
3358 .next_attempt(&instance.saga_id, step, StepPhase::Cancel)
3359 .await?;
3360 self.issue_command(
3361 instance,
3362 step_def,
3363 StepPhase::Cancel,
3364 attempt,
3365 None,
3366 None,
3367 now_ms,
3368 version,
3369 )
3370 .await?;
3371 Ok(Some(SagaStatus::Cancelling))
3372 }
3373
3374 #[allow(clippy::too_many_arguments)]
3389 async fn on_phase_retry_timeout(
3390 &self,
3391 instance: &SagaInstanceRow,
3392 definition: &WorkflowDefinition,
3393 step_scope: Option<&StepName>,
3394 timer: &SagaTimerRow,
3395 phase: StepPhase,
3396 required_status: SagaStatus,
3397 max_attempts: u32,
3398 exhausted_code: &str,
3399 now_ms: i64,
3400 ) -> anyhow::Result<Option<SagaStatus>> {
3401 let Some(step) = step_scope else {
3402 anyhow::bail!("phase retry timer without a step scope");
3403 };
3404 if instance.status != required_status || instance.current_step.as_ref() != Some(step) {
3405 return Ok(None);
3406 }
3407 let step_def = definition
3408 .step(step)
3409 .ok_or_else(|| anyhow::anyhow!("timer step absent from definition"))?;
3410 let exhausted = if phase == StepPhase::Resolve {
3411 self.backend
3414 .store()
3415 .count_resolution_attempts(&instance.saga_id, step, instance.direction)
3416 .await?
3417 >= max_attempts
3418 } else {
3419 self.next_attempt(&instance.saga_id, step, phase)
3420 .await?
3421 .get()
3422 > max_attempts
3423 };
3424 if exhausted {
3427 return Ok(Some(
3428 self.escalate_manual(
3429 instance,
3430 TriggerKind::Timer,
3431 &timer.timer_id,
3432 exhausted_code,
3433 )
3434 .await?,
3435 ));
3436 }
3437 let attempt = self.next_attempt(&instance.saga_id, step, phase).await?;
3439 let version = self
3440 .advance(
3441 instance,
3442 required_status,
3443 instance.direction,
3444 Some(step),
3445 TriggerKind::Timer,
3446 &timer.timer_id,
3447 None,
3448 None,
3449 )
3450 .await?;
3451 self.issue_command(
3452 instance, step_def, phase, attempt, None, None, now_ms, version,
3453 )
3454 .await?;
3455 Ok(Some(required_status))
3456 }
3457
3458 async fn on_instance_deadline(
3468 &self,
3469 instance: &SagaInstanceRow,
3470 definition: &WorkflowDefinition,
3471 timer: &SagaTimerRow,
3472 now_ms: i64,
3473 ) -> anyhow::Result<Option<SagaStatus>> {
3474 match instance.status {
3475 SagaStatus::Running => {
3477 let Some(step) = instance.current_step.clone() else {
3478 return Ok(None);
3479 };
3480 self.on_step_timeout(instance, definition, Some(&step), timer, now_ms)
3481 .await
3482 }
3483 SagaStatus::Cancelling | SagaStatus::WaitingResolution | SagaStatus::Compensating => {
3486 Ok(Some(
3487 self.escalate_manual(
3488 instance,
3489 TriggerKind::Timer,
3490 &timer.timer_id,
3491 "deadline_exceeded",
3492 )
3493 .await?,
3494 ))
3495 }
3496 _ => Ok(None),
3497 }
3498 }
3499
3500 async fn proceed_forward(&self, context: &ResultContext<'_>) -> anyhow::Result<SagaStatus> {
3512 let saga_id = &context.identity.saga_id;
3513 let steps = self.backend.store().load_steps(saga_id).await?;
3514 match next_pending_forward(context.definition, &steps)? {
3515 Some(next) => {
3516 let version = self
3517 .advance(
3518 context.instance,
3519 SagaStatus::Running,
3520 Direction::Forward,
3521 Some(&next),
3522 TriggerKind::Event,
3523 context.event_id,
3524 None,
3525 None,
3526 )
3527 .await?;
3528 let next_def = context
3529 .definition
3530 .step(&next)
3531 .ok_or_else(|| anyhow::anyhow!("next step absent from definition"))?;
3532 let attempt = self
3533 .next_attempt(saga_id, &next, StepPhase::Execute)
3534 .await?;
3535 self.issue_command(
3537 context.instance,
3538 next_def,
3539 StepPhase::Execute,
3540 attempt,
3541 None,
3542 None,
3543 context.now_ms,
3544 version,
3545 )
3546 .await?;
3547 Ok(SagaStatus::Running)
3548 }
3549 None => {
3550 self.advance(
3551 context.instance,
3552 SagaStatus::Completed,
3553 Direction::Forward,
3554 None,
3555 TriggerKind::Event,
3556 context.event_id,
3557 None,
3558 None,
3559 )
3560 .await?;
3561 self.backend
3563 .store()
3564 .cancel_scope_timers(saga_id, TimerScope::Instance, None)
3565 .await?;
3566 Ok(SagaStatus::Completed)
3567 }
3568 }
3569 }
3570
3571 async fn enter_compensation(&self, context: &ResultContext<'_>) -> anyhow::Result<SagaStatus> {
3582 let saga_id = &context.identity.saga_id;
3583 let steps = self.backend.store().load_steps(saga_id).await?;
3584 let journal: Vec<StepJournalEntry> = steps
3585 .iter()
3586 .map(|row| StepJournalEntry::new(row.step.clone(), row.forward_status))
3587 .collect();
3588 let plan = match freeze_compensation_plan(context.definition, &journal) {
3591 Ok(plan) => plan,
3592 Err(violation) => {
3593 return self
3594 .escalate_manual(
3595 context.instance,
3596 TriggerKind::Event,
3597 context.event_id,
3598 violation.code(),
3599 )
3600 .await;
3601 }
3602 };
3603 let first = plan.entries().first().map(|entry| entry.step().clone());
3604 let version = self
3605 .advance(
3606 context.instance,
3607 SagaStatus::Compensating,
3608 Direction::Compensating,
3609 first.as_ref(),
3610 TriggerKind::Event,
3611 context.event_id,
3612 None,
3613 Some(plan.version()),
3614 )
3615 .await?;
3616 self.backend
3617 .store()
3618 .mark_compensation_plan(saga_id, &plan)
3619 .await?;
3620 match first {
3621 Some(step) => {
3622 let step_def = context
3623 .definition
3624 .step(&step)
3625 .ok_or_else(|| anyhow::anyhow!("plan step absent from definition"))?;
3626 let attempt = self
3627 .next_attempt(saga_id, &step, StepPhase::Compensate)
3628 .await?;
3629 self.issue_command(
3630 context.instance,
3631 step_def,
3632 StepPhase::Compensate,
3633 attempt,
3634 None,
3635 None,
3636 context.now_ms,
3637 version,
3638 )
3639 .await?;
3640 Ok(SagaStatus::Compensating)
3641 }
3642 None => {
3643 self.converge_compensated(
3645 context.instance,
3646 version,
3647 TriggerKind::Event,
3648 context.event_id,
3649 )
3650 .await?;
3651 Ok(SagaStatus::Compensated)
3652 }
3653 }
3654 }
3655
3656 async fn continue_compensation(
3669 &self,
3670 context: &ResultContext<'_>,
3671 ) -> anyhow::Result<SagaStatus> {
3672 let saga_id = &context.identity.saga_id;
3673 let steps = self.backend.store().load_steps(saga_id).await?;
3674 let next =
3675 match next_pending_compensation(context.instance, context.definition, &steps, false) {
3676 Ok(next) => next,
3677 Err(_) => {
3678 return self
3681 .escalate_manual(
3682 context.instance,
3683 TriggerKind::Event,
3684 context.event_id,
3685 "compensation_plan_drift",
3686 )
3687 .await;
3688 }
3689 };
3690 let version = self
3691 .advance(
3692 context.instance,
3693 SagaStatus::Compensating,
3694 Direction::Compensating,
3695 next.as_ref(),
3696 TriggerKind::Event,
3697 context.event_id,
3698 None,
3699 None,
3700 )
3701 .await?;
3702 match next {
3703 Some(step) => {
3704 let step_def = context
3705 .definition
3706 .step(&step)
3707 .ok_or_else(|| anyhow::anyhow!("plan step absent from definition"))?;
3708 let attempt = self
3709 .next_attempt(saga_id, &step, StepPhase::Compensate)
3710 .await?;
3711 self.issue_command(
3712 context.instance,
3713 step_def,
3714 StepPhase::Compensate,
3715 attempt,
3716 None,
3717 None,
3718 context.now_ms,
3719 version,
3720 )
3721 .await?;
3722 Ok(SagaStatus::Compensating)
3723 }
3724 None => {
3725 self.converge_compensated_at(
3726 saga_id,
3727 version,
3728 TriggerKind::Event,
3729 context.event_id,
3730 )
3731 .await?;
3732 Ok(SagaStatus::Compensated)
3733 }
3734 }
3735 }
3736
3737 async fn enter_waiting_resolution(
3745 &self,
3746 context: &ResultContext<'_>,
3747 direction: Direction,
3748 ) -> anyhow::Result<SagaStatus> {
3749 let saga_id = &context.identity.saga_id;
3750 let step = &context.identity.step;
3751 let resolution = context.step_def.resolution();
3752 if !resolution.allow_unknown() {
3755 if context.identity.phase == StepPhase::Execute {
3756 self.patch_step(
3757 saga_id,
3758 step,
3759 StepJournalPatch {
3760 forward_status: Some(StepForwardStatus::Unknown),
3761 ..Default::default()
3762 },
3763 )
3764 .await?;
3765 }
3766 return self
3769 .escalate_manual(
3770 context.instance,
3771 TriggerKind::Event,
3772 context.event_id,
3773 "unknown_not_allowed",
3774 )
3775 .await;
3776 }
3777 if context.identity.phase == StepPhase::Execute {
3778 self.patch_step(
3779 saga_id,
3780 step,
3781 StepJournalPatch {
3782 forward_status: Some(StepForwardStatus::Unknown),
3783 resolution_status: Some(StepResolutionStatus::Pending),
3784 ..Default::default()
3785 },
3786 )
3787 .await?;
3788 } else {
3789 self.patch_step(
3790 saga_id,
3791 step,
3792 StepJournalPatch {
3793 resolution_status: Some(StepResolutionStatus::Pending),
3794 ..Default::default()
3795 },
3796 )
3797 .await?;
3798 }
3799 self.backend
3800 .store()
3801 .cancel_scope_timers(saga_id, TimerScope::Step(step), Some(KIND_STEP_TIMEOUT))
3802 .await?;
3803 let version = self
3804 .advance(
3805 context.instance,
3806 SagaStatus::WaitingResolution,
3807 direction,
3808 Some(step),
3809 TriggerKind::Event,
3810 context.event_id,
3811 None,
3812 None,
3813 )
3814 .await?;
3815 self.schedule_timer(
3817 saga_id,
3818 TimerScope::Step(step),
3819 resolution_budget_kind(direction),
3820 context.identity.attempt,
3821 checked_deadline(context.now_ms, duration_millis(resolution.budget())?)?,
3822 version,
3823 )
3824 .await?;
3825 if resolution.mode() == Some(ResolutionMode::Poll) {
3826 let attempt = self.next_attempt(saga_id, step, StepPhase::Resolve).await?;
3827 self.issue_command(
3828 context.instance,
3829 context.step_def,
3830 StepPhase::Resolve,
3831 attempt,
3832 None,
3833 None,
3834 context.now_ms,
3835 version,
3836 )
3837 .await?;
3838 }
3839 Ok(SagaStatus::WaitingResolution)
3840 }
3841
3842 async fn reissue_resolve(&self, context: &ResultContext<'_>) -> anyhow::Result<SagaStatus> {
3849 let saga_id = &context.identity.saga_id;
3850 let step = &context.identity.step;
3851 let used = self
3852 .backend
3853 .store()
3854 .count_resolution_attempts(saga_id, step, context.instance.direction)
3855 .await?;
3856 if used >= self.config.resolve_max_attempts {
3857 return self
3858 .escalate_manual(
3859 context.instance,
3860 TriggerKind::Event,
3861 context.event_id,
3862 "resolve_attempts_exhausted",
3863 )
3864 .await;
3865 }
3866 let attempt = self.next_attempt(saga_id, step, StepPhase::Resolve).await?;
3869 let version = self
3870 .advance(
3871 context.instance,
3872 SagaStatus::WaitingResolution,
3873 context.instance.direction,
3874 Some(step),
3875 TriggerKind::Event,
3876 context.event_id,
3877 None,
3878 None,
3879 )
3880 .await?;
3881 self.issue_command(
3882 context.instance,
3883 context.step_def,
3884 StepPhase::Resolve,
3885 attempt,
3886 None,
3887 None,
3888 context.now_ms,
3889 version,
3890 )
3891 .await?;
3892 Ok(SagaStatus::WaitingResolution)
3893 }
3894
3895 async fn escalate_manual(
3905 &self,
3906 instance: &SagaInstanceRow,
3907 trigger_kind: TriggerKind,
3908 trigger_id: &str,
3909 failure_code: &str,
3910 ) -> anyhow::Result<SagaStatus> {
3911 self.advance(
3912 instance,
3913 SagaStatus::ManualIntervention,
3914 instance.direction,
3915 instance.current_step.as_ref(),
3916 trigger_kind,
3917 trigger_id,
3918 Some(failure_code),
3919 None,
3920 )
3921 .await?;
3922 Ok(SagaStatus::ManualIntervention)
3923 }
3924
3925 async fn converge_compensated(
3935 &self,
3936 instance: &SagaInstanceRow,
3937 version: u64,
3938 trigger_kind: TriggerKind,
3939 cause_id: &str,
3940 ) -> anyhow::Result<()> {
3941 self.converge_compensated_at(&instance.saga_id, version, trigger_kind, cause_id)
3942 .await
3943 }
3944
3945 async fn converge_compensated_at(
3955 &self,
3956 saga_id: &SagaId,
3957 version: u64,
3958 trigger_kind: TriggerKind,
3959 cause_id: &str,
3960 ) -> anyhow::Result<()> {
3961 let definition_version = self
3964 .verify_compensation_completion(saga_id, version)
3965 .await?;
3966 let converge_trigger = derive_converge_trigger_id(cause_id);
3969 let outcome = self
3970 .backend
3971 .store()
3972 .advance(
3973 saga_id,
3974 version,
3975 SagaStatus::Compensating,
3976 &TransitionSpec {
3977 to_status: SagaStatus::Compensated,
3978 direction: Direction::Compensating,
3979 current_step: None,
3980 trigger_kind,
3981 trigger_id: &converge_trigger,
3982 definition_version,
3983 failure_code: None,
3984 compensation_plan_version: None,
3985 },
3986 )
3987 .await?;
3988 let _ = ensure_applied(outcome)?;
3989 self.backend
3990 .store()
3991 .cancel_scope_timers(saga_id, TimerScope::Instance, None)
3992 .await?;
3993 Ok(())
3994 }
3995
3996 async fn verify_compensation_completion(
4005 &self,
4006 saga_id: &SagaId,
4007 expected_version: u64,
4008 ) -> anyhow::Result<DefinitionVersion> {
4009 let instance = self
4010 .backend
4011 .store()
4012 .load_instance(saga_id)
4013 .await?
4014 .ok_or_else(|| anyhow::anyhow!("saga instance not found while converging"))?;
4015 if instance.status != SagaStatus::Compensating || instance.version != expected_version {
4016 anyhow::bail!("compensation completion proof used a stale instance snapshot");
4017 }
4018 let frozen_version = instance
4019 .compensation_plan_version
4020 .as_deref()
4021 .ok_or_else(|| anyhow::anyhow!("compensating instance has no frozen plan"))?;
4022 let registry = self.registry_snapshot();
4023 let definition = registry
4024 .get_for_tenant(
4025 &instance.tenant,
4026 &instance.workflow,
4027 instance.definition_version,
4028 )
4029 .ok_or_else(|| anyhow::anyhow!("instance definition is not registered"))?;
4030 let steps = self.backend.store().load_steps(saga_id).await?;
4031 let journal: Vec<StepJournalEntry> = steps
4032 .iter()
4033 .map(|row| StepJournalEntry::new(row.step.clone(), row.forward_status))
4034 .collect();
4035 let rebuilt = freeze_compensation_plan(definition, &journal)
4036 .map_err(|violation| anyhow::anyhow!("{}", violation.code()))?;
4037 if rebuilt.version() != frozen_version {
4038 anyhow::bail!("frozen compensation plan no longer matches durable forward facts");
4039 }
4040 for entry in rebuilt.entries() {
4041 let row = steps
4042 .iter()
4043 .find(|row| row.step == *entry.step())
4044 .ok_or_else(|| anyhow::anyhow!("frozen compensation member is missing"))?;
4045 if row.compensation_plan_version.as_deref() != Some(frozen_version)
4046 || row.compensation_order != Some(entry.order())
4047 || row.compensation_status != StepCompensationStatus::Succeeded
4048 {
4049 anyhow::bail!("frozen compensation member is not durably succeeded");
4050 }
4051 }
4052 let marked_count = steps
4053 .iter()
4054 .filter(|row| row.compensation_plan_version.as_deref() == Some(frozen_version))
4055 .count();
4056 if marked_count != rebuilt.entries().len() {
4057 anyhow::bail!("durable compensation plan contains unexpected members");
4058 }
4059 Ok(instance.definition_version)
4060 }
4061
4062 #[allow(clippy::too_many_arguments)]
4076 async fn advance(
4077 &self,
4078 instance: &SagaInstanceRow,
4079 to: SagaStatus,
4080 direction: Direction,
4081 current_step: Option<&StepName>,
4082 trigger_kind: TriggerKind,
4083 trigger_id: &str,
4084 failure_code: Option<&str>,
4085 plan_version: Option<&str>,
4086 ) -> anyhow::Result<u64> {
4087 self.advance_from(
4088 instance,
4089 instance.status,
4090 to,
4091 direction,
4092 current_step,
4093 trigger_kind,
4094 trigger_id,
4095 failure_code,
4096 plan_version,
4097 )
4098 .await
4099 }
4100
4101 #[allow(clippy::too_many_arguments)]
4116 async fn advance_from(
4117 &self,
4118 instance: &SagaInstanceRow,
4119 from: SagaStatus,
4120 to: SagaStatus,
4121 direction: Direction,
4122 current_step: Option<&StepName>,
4123 trigger_kind: TriggerKind,
4124 trigger_id: &str,
4125 failure_code: Option<&str>,
4126 plan_version: Option<&str>,
4127 ) -> anyhow::Result<u64> {
4128 let outcome = self
4129 .backend
4130 .store()
4131 .advance(
4132 &instance.saga_id,
4133 instance.version,
4134 from,
4135 &TransitionSpec {
4136 to_status: to,
4137 direction,
4138 current_step,
4139 trigger_kind,
4140 trigger_id,
4141 definition_version: instance.definition_version,
4142 failure_code,
4143 compensation_plan_version: plan_version,
4144 },
4145 )
4146 .await?;
4147 ensure_applied(outcome)
4148 }
4149
4150 #[allow(clippy::too_many_arguments)]
4164 async fn issue_command(
4165 &self,
4166 instance: &SagaInstanceRow,
4167 step_def: &StepDefinition,
4168 phase: StepPhase,
4169 attempt: AttemptNo,
4170 payload: Option<CommandInput>,
4171 recovery_operation_id: Option<&str>,
4172 now_ms: i64,
4173 expected_version: u64,
4174 ) -> anyhow::Result<()> {
4175 let step = step_def.name();
4176 let effect = EffectId::derive(&instance.saga_id, instance.definition_version, step, phase);
4177 let command = CommandId::derive(&effect, attempt);
4178 self.backend
4180 .store()
4181 .record_attempt_started(&instance.saga_id, step, phase, attempt, &effect, &command)
4182 .await?;
4183 let (payload, raw_payload) = match payload {
4185 Some(CommandInput::Json(value)) => (Some(value), None),
4186 Some(CommandInput::Raw(value)) => (None, Some(value)),
4187 None => (None, None),
4188 };
4189 let envelope = SagaCommandEnvelope {
4190 saga_id: instance.saga_id.as_str().to_string(),
4191 tenant_id: instance.tenant.as_str().to_string(),
4192 workflow: instance.workflow.as_str().to_string(),
4193 definition_version: instance.definition_version.get(),
4194 definition_digest: instance.definition_digest.clone(),
4195 step: step.as_str().to_string(),
4196 phase: phase.as_str().to_string(),
4197 attempt: attempt.get(),
4198 effect_id: effect.to_string(),
4199 command_id: command.to_string(),
4200 recovery_operation_id: recovery_operation_id.map(str::to_owned),
4201 payload,
4202 raw_payload,
4203 };
4204 let event = match instance
4208 .traceparent
4209 .as_deref()
4210 .and_then(TraceContext::parse_traceparent)
4211 {
4212 Some(base) => envelope
4213 .to_outbox_event()?
4214 .with_traceparent(base.child(natelemetry::random_span_id()).to_traceparent()),
4215 None => envelope.to_outbox_event()?,
4216 };
4217 let write_context = naoutbox_core::OutboxWriteContext::new(instance.tenant.as_str())
4221 .map_err(|reason| anyhow::anyhow!("outbox write context rejected: {reason}"))?;
4222 self.backend
4223 .outbox()
4224 .append_transactional_with_context(&write_context, &event)
4225 .await?;
4226 let timeout_ms = duration_millis(step_def.timeout())?;
4228 self.schedule_timer(
4229 &instance.saga_id,
4230 TimerScope::Step(step),
4231 phase_timeout_kind(phase),
4232 attempt,
4233 checked_deadline(now_ms, timeout_ms)?,
4234 expected_version,
4235 )
4236 .await?;
4237 Ok(())
4238 }
4239
4240 async fn schedule_timer(
4252 &self,
4253 saga_id: &SagaId,
4254 scope: TimerScope<'_>,
4255 kind: &str,
4256 attempt: AttemptNo,
4257 due_at_ms: i64,
4258 expected_version: u64,
4259 ) -> anyhow::Result<()> {
4260 let timer_id = derive_timer_id(saga_id, &scope, kind, attempt);
4261 self.backend
4262 .store()
4263 .schedule_timer(&TimerSpec {
4264 timer_id: &timer_id,
4265 saga_id,
4266 scope,
4267 kind,
4268 due_at_ms,
4269 attempt,
4270 expected_saga_version: expected_version,
4271 })
4272 .await?;
4273 Ok(())
4274 }
4275
4276 async fn cancel_resolution_timers(
4284 &self,
4285 saga_id: &SagaId,
4286 step: &StepName,
4287 ) -> anyhow::Result<()> {
4288 for kind in [
4291 KIND_RESOLUTION_BUDGET,
4292 KIND_FORWARD_RESOLUTION_BUDGET,
4293 KIND_COMPENSATION_RESOLUTION_BUDGET,
4294 ] {
4295 self.backend
4296 .store()
4297 .cancel_scope_timers(saga_id, TimerScope::Step(step), Some(kind))
4298 .await?;
4299 }
4300 self.backend
4301 .store()
4302 .cancel_scope_timers(saga_id, TimerScope::Step(step), Some(KIND_RESOLVE_TIMEOUT))
4303 .await?;
4304 Ok(())
4305 }
4306
4307 async fn patch_step(
4316 &self,
4317 saga_id: &SagaId,
4318 step: &StepName,
4319 patch: StepJournalPatch<'_>,
4320 ) -> anyhow::Result<()> {
4321 self.backend
4322 .store()
4323 .update_step_journal(saga_id, step, &patch)
4324 .await?;
4325 Ok(())
4326 }
4327
4328 async fn next_attempt(
4337 &self,
4338 saga_id: &SagaId,
4339 step: &StepName,
4340 phase: StepPhase,
4341 ) -> anyhow::Result<AttemptNo> {
4342 let attempts = self.backend.store().load_attempts(saga_id, step).await?;
4343 let max = attempts
4344 .iter()
4345 .filter(|row| row.phase == phase)
4346 .map(|row| row.attempt.get())
4347 .max();
4348 match max {
4349 None => Ok(AttemptNo::FIRST),
4350 Some(value) => AttemptNo::new(value)
4351 .and_then(|attempt| attempt.next())
4352 .map_err(|violation| anyhow::anyhow!("attempt overflow: {}", violation.code())),
4353 }
4354 }
4355}
4356
4357fn verify_instance_contract(
4368 instance: &SagaInstanceRow,
4369 identity: &VerifiedIdentity,
4370) -> anyhow::Result<()> {
4371 if instance.tenant != identity.tenant {
4372 anyhow::bail!("envelope tenant does not match the instance tenant");
4373 }
4374 if instance.workflow != identity.workflow
4375 || instance.definition_version != identity.definition_version
4376 {
4377 anyhow::bail!("envelope workflow/version does not match the instance");
4378 }
4379 if instance.definition_digest != identity.definition_digest {
4380 anyhow::bail!("envelope definition digest does not match the instance");
4381 }
4382 Ok(())
4383}
4384
4385fn conflicting_projection_status(
4397 row: &SagaStepRow,
4398 context: &ResultContext<'_>,
4399) -> Option<StepAttemptStatus> {
4400 let forward = || match row.forward_status {
4401 StepForwardStatus::Succeeded => Some(StepAttemptStatus::Succeeded),
4402 StepForwardStatus::Rejected => Some(StepAttemptStatus::Rejected),
4403 StepForwardStatus::Cancelled => Some(StepAttemptStatus::CancelConfirmed),
4404 StepForwardStatus::Halted => Some(StepAttemptStatus::Halted),
4405 StepForwardStatus::Pending | StepForwardStatus::Unknown | StepForwardStatus::TimedOut => {
4406 None
4407 }
4408 };
4409 let cancellation = || match row.cancel_status {
4410 StepCancelStatus::Confirmed => Some(StepAttemptStatus::CancelConfirmed),
4411 StepCancelStatus::AlreadyTerminal => Some(StepAttemptStatus::AlreadyTerminal),
4412 StepCancelStatus::None | StepCancelStatus::ResolutionPending => None,
4413 };
4414 let compensation = || match row.compensation_status {
4415 StepCompensationStatus::Succeeded => Some(StepAttemptStatus::Succeeded),
4416 StepCompensationStatus::Halted => Some(StepAttemptStatus::Halted),
4417 StepCompensationStatus::None
4418 | StepCompensationStatus::Pending
4419 | StepCompensationStatus::Unknown => None,
4420 };
4421 let resolution = || match row.resolution_status {
4422 StepResolutionStatus::Succeeded => Some(StepAttemptStatus::Succeeded),
4423 StepResolutionStatus::Rejected => Some(StepAttemptStatus::Rejected),
4424 StepResolutionStatus::Halted => Some(StepAttemptStatus::Halted),
4425 StepResolutionStatus::None | StepResolutionStatus::Pending => None,
4426 };
4427
4428 match context.identity.phase {
4429 StepPhase::Execute => forward().or_else(cancellation),
4430 StepPhase::Cancel => cancellation().or_else(forward),
4431 StepPhase::Compensate => compensation().or_else(resolution),
4432 StepPhase::Resolve => resolution().or_else(|| {
4433 if context.instance.direction == Direction::Compensating {
4434 compensation()
4435 } else {
4436 forward()
4437 }
4438 }),
4439 }
4440}
4441
4442fn merge_deferred_projection(
4454 row: &SagaStepRow,
4455 patch: &mut StepJournalPatch<'_>,
4456) -> Result<(), ()> {
4457 if let Some(next) = patch.forward_status {
4458 patch.forward_status = if row.forward_status == next {
4459 None
4460 } else {
4461 match (row.forward_status, next) {
4462 (
4463 StepForwardStatus::Pending
4464 | StepForwardStatus::Unknown
4465 | StepForwardStatus::TimedOut,
4466 _,
4467 ) => Some(next),
4468 (_, StepForwardStatus::Unknown | StepForwardStatus::TimedOut) => None,
4470 _ => return Err(()),
4471 }
4472 };
4473 }
4474
4475 if let Some(next) = patch.cancel_status {
4476 patch.cancel_status = if row.cancel_status == next {
4477 None
4478 } else {
4479 match (row.cancel_status, next) {
4480 (StepCancelStatus::None | StepCancelStatus::ResolutionPending, _) => Some(next),
4481 (_, StepCancelStatus::ResolutionPending) => None,
4483 _ => return Err(()),
4484 }
4485 };
4486 }
4487
4488 if let Some(next) = patch.compensation_status {
4489 patch.compensation_status = if row.compensation_status == next {
4490 None
4491 } else {
4492 match (row.compensation_status, next) {
4493 (
4494 StepCompensationStatus::None
4495 | StepCompensationStatus::Pending
4496 | StepCompensationStatus::Unknown,
4497 _,
4498 ) => Some(next),
4499 (_, StepCompensationStatus::Unknown | StepCompensationStatus::Pending) => None,
4501 _ => return Err(()),
4502 }
4503 };
4504 }
4505
4506 if let Some(next) = patch.resolution_status {
4507 patch.resolution_status = if row.resolution_status == next {
4508 None
4509 } else {
4510 match (row.resolution_status, next) {
4511 (StepResolutionStatus::None | StepResolutionStatus::Pending, _) => Some(next),
4512 (_, StepResolutionStatus::Pending) => None,
4514 _ => return Err(()),
4515 }
4516 };
4517 }
4518 Ok(())
4519}
4520
4521fn ensure_applied(outcome: CasOutcome) -> anyhow::Result<u64> {
4531 match outcome {
4532 CasOutcome::Applied {
4533 new_version,
4534 } => Ok(new_version),
4535 CasOutcome::Conflict => Err(SagaConcurrencyError::StaleSnapshot.into()),
4536 CasOutcome::DuplicateTrigger => Err(SagaConcurrencyError::DuplicateTrigger.into()),
4537 }
4538}
4539
4540fn next_pending_forward(
4549 definition: &WorkflowDefinition,
4550 steps: &[SagaStepRow],
4551) -> anyhow::Result<Option<StepName>> {
4552 if steps.len() != definition.steps().len() {
4553 anyhow::bail!("step journal does not completely cover the workflow definition");
4554 }
4555 let mut next = None;
4556 for (index, step_def) in definition.steps().iter().enumerate() {
4557 let row = &steps[index];
4558 if row.step != *step_def.name() || row.ordinal != index as u32 + 1 {
4559 anyhow::bail!("step journal order does not match the workflow definition");
4560 }
4561 match (next.is_some(), row.forward_status) {
4562 (false, StepForwardStatus::Succeeded) => {}
4563 (false, StepForwardStatus::Pending) => next = Some(row.step.clone()),
4564 (true, StepForwardStatus::Pending) => {}
4565 _ => anyhow::bail!("forward journal is not a succeeded-prefix/pending-suffix"),
4568 }
4569 }
4570 Ok(next)
4571}
4572
4573fn next_pending_compensation(
4585 instance: &SagaInstanceRow,
4586 definition: &WorkflowDefinition,
4587 steps: &[SagaStepRow],
4588 allow_halted_recovery: bool,
4589) -> anyhow::Result<Option<StepName>> {
4590 let frozen_version = instance
4591 .compensation_plan_version
4592 .as_deref()
4593 .ok_or_else(|| anyhow::anyhow!("compensating instance has no frozen plan"))?;
4594 let journal: Vec<StepJournalEntry> = steps
4595 .iter()
4596 .map(|row| StepJournalEntry::new(row.step.clone(), row.forward_status))
4597 .collect();
4598 let rebuilt = freeze_compensation_plan(definition, &journal)
4599 .map_err(|violation| anyhow::anyhow!("{}", violation.code()))?;
4600 if rebuilt.version() != frozen_version {
4601 anyhow::bail!("frozen compensation plan digest drifted");
4602 }
4603
4604 let mut next = None;
4605 for entry in rebuilt.entries() {
4606 let row = steps
4607 .iter()
4608 .find(|row| row.step == *entry.step())
4609 .ok_or_else(|| anyhow::anyhow!("frozen compensation member is missing"))?;
4610 if row.compensation_plan_version.as_deref() != Some(frozen_version)
4611 || row.compensation_order != Some(entry.order())
4612 {
4613 anyhow::bail!("durable compensation member metadata drifted");
4614 }
4615 match (next.is_some(), row.compensation_status) {
4616 (false, StepCompensationStatus::Succeeded) => {}
4617 (false, StepCompensationStatus::Pending) => next = Some(row.step.clone()),
4618 (false, StepCompensationStatus::Halted) if allow_halted_recovery => {
4619 next = Some(row.step.clone())
4620 }
4621 (true, StepCompensationStatus::Pending) => {}
4622 _ => anyhow::bail!("compensation journal is not a succeeded-prefix/pending-suffix"),
4623 }
4624 }
4625 let marked_count = steps
4626 .iter()
4627 .filter(|row| row.compensation_plan_version.as_deref() == Some(frozen_version))
4628 .count();
4629 if marked_count != rebuilt.entries().len() {
4630 anyhow::bail!("durable compensation plan contains unexpected members");
4631 }
4632 Ok(next)
4633}