1use std::{
17 sync::{Arc, RwLock},
18 time::{Duration, Instant},
19};
20
21use nainbox_core::{InboxClaim, InboxStore};
22use naoutbox_core::DurableOutboxAppend;
23use nasaga_backend::{
24 AttemptConflictFact, AttemptOutcomeRecord, CasOutcome, ControlCasOutcome,
25 ControlTransitionSpec, ManagementAuditOutcome, NewSagaInstance, SagaAuditEventCursor,
26 SagaAuditEventRecord, SagaAuditStore, SagaBackend, SagaBackendErrorKind, SagaBackendFactory,
27 SagaConflictKind, SagaCreation, SagaGovernanceStore, SagaInstanceQuery, SagaInstanceRow,
28 SagaInstanceStore, SagaInstanceSummary, SagaJournalStore, SagaStepRow, SagaTimerRow,
29 SagaTimerStore, StepJournalPatch, TimerFencing, TimerFencingToken, TimerFencingTokenIssuer,
30 TimerScope, TimerSpec, TransitionSpec,
31};
32use nasaga_core::{
33 freeze_compensation_plan, AttemptNo, BusinessKey, CancelMode, CommandId, DefinitionVersion,
34 Direction, EffectId, ResolutionMode, SagaId, SagaStatus, ServiceIdentity, StepAttemptStatus,
35 StepCancelStatus, StepCompensationStatus, StepDefinition, StepForwardStatus, StepJournalEntry,
36 StepName, StepPhase, StepResolutionStatus, TenantId, TimeoutPolicy, TriggerKind,
37 WorkflowDefinition, WorkflowName,
38};
39use natelemetry::TraceContext;
40use sha2::{Digest as _, Sha256};
41use uuid::Uuid;
42
43use crate::envelope::{canonical_bytes, SagaCommandEnvelope, SagaResultEnvelope, VerifiedIdentity};
44use crate::management::{
45 SagaAuditPage, SagaAuditPageCursor, SagaAuditRecord, SagaAuditTrail, SagaManagementContext,
46 SagaManagementError, SagaManagementExpectation, SagaManagementPermission,
47};
48use crate::observability::{
49 record_action_rate_rejection, record_quota_rejection, SagaOperationalMetrics,
50};
51use crate::registry::DefinitionRegistry;
52use crate::timers::{
53 derive_timer_id, parse_step_scope, phase_timeout_kind, resolution_budget_kind,
54 KIND_CANCEL_TIMEOUT, KIND_COMPENSATE_TIMEOUT, KIND_COMPENSATION_RESOLUTION_BUDGET,
55 KIND_FORWARD_RESOLUTION_BUDGET, KIND_INSTANCE_DEADLINE, KIND_RESOLUTION_BUDGET,
56 KIND_RESOLVE_TIMEOUT, KIND_STEP_TIMEOUT,
57};
58
59const CONVERGE_TRIGGER_NAMESPACE: Uuid = Uuid::from_bytes(*b"nasasaga-v1-cvns");
64
65#[derive(Debug, Clone)]
67pub struct OrchestratorConfig {
68 pub inbox_consumer: String,
70 pub cancel_max_attempts: u32,
72 pub compensate_max_attempts: u32,
74 pub resolve_max_attempts: u32,
76 pub timer_claim_limit: u32,
78 pub timer_lease_ms: i64,
80 pub pause_backoff_ms: i64,
82 pub startup_scan_limit: u32,
84 pub tenant_quotas: std::collections::BTreeMap<String, u64>,
90 pub tenant_action_rates: std::collections::BTreeMap<String, TenantActionRate>,
98 pub enable_manual_close: bool,
104}
105
106impl Default for OrchestratorConfig {
107 fn default() -> Self {
113 Self {
114 inbox_consumer: "nasaga-orchestrator".to_string(),
115 cancel_max_attempts: 5,
116 compensate_max_attempts: 5,
117 resolve_max_attempts: 10,
118 timer_claim_limit: 32,
119 timer_lease_ms: 60_000,
120 pause_backoff_ms: 30_000,
121 startup_scan_limit: 1_000,
122 tenant_quotas: std::collections::BTreeMap::new(),
123 tenant_action_rates: std::collections::BTreeMap::new(),
124 enable_manual_close: false,
126 }
127 }
128}
129
130#[derive(Debug, Clone, Copy, PartialEq, Eq)]
135pub struct TenantActionRate {
136 pub max_actions: u64,
138 pub window_ms: i64,
140}
141
142#[derive(Debug, Clone, PartialEq, Eq)]
144pub enum StartOutcome {
145 Started(SagaInstanceRow),
147 AlreadyExists(SagaInstanceRow),
149}
150
151#[derive(Debug, Clone, Copy, PartialEq, Eq)]
153#[non_exhaustive]
154pub enum StartSagaError {
155 DefinitionInactive,
157 InvalidPayload,
159 DefinitionEmpty,
161 RequestConflict,
163 TenantQuotaExceeded,
165}
166
167impl StartSagaError {
168 pub fn from_error(error: &anyhow::Error) -> Option<Self> {
174 error
175 .chain()
176 .find_map(|cause| cause.downcast_ref::<Self>().copied())
177 }
178}
179
180impl std::fmt::Display for StartSagaError {
181 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
187 formatter.write_str(match self {
188 Self::InvalidPayload => "Saga payload contract is invalid",
189 Self::DefinitionInactive => "Saga workflow definition is not active",
190 Self::DefinitionEmpty => "Saga workflow definition has no steps",
191 Self::RequestConflict => "Saga identity is already bound to a different start request",
192 Self::TenantQuotaExceeded => "saga_tenant_quota_exceeded",
193 })
194 }
195}
196
197impl std::error::Error for StartSagaError {}
198
199#[derive(Debug, Clone, Copy, PartialEq, Eq)]
201#[non_exhaustive]
202pub enum SagaConcurrencyError {
203 StaleSnapshot,
205 DuplicateTrigger,
207}
208
209impl SagaConcurrencyError {
210 pub fn from_error(error: &anyhow::Error) -> Option<Self> {
216 error
217 .chain()
218 .find_map(|cause| cause.downcast_ref::<Self>().copied())
219 }
220}
221
222impl std::fmt::Display for SagaConcurrencyError {
223 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
229 formatter.write_str(match self {
230 Self::StaleSnapshot => "Saga state snapshot is stale",
231 Self::DuplicateTrigger => "Saga transition trigger was already applied",
232 })
233 }
234}
235
236impl std::error::Error for SagaConcurrencyError {}
237
238#[derive(Debug, Clone, PartialEq, Eq)]
240pub enum HandleOutcome {
241 Applied {
243 status: SagaStatus,
245 },
246 Duplicate,
248}
249
250#[derive(Debug, Clone, PartialEq, Eq)]
252pub enum TimerOutcome {
253 Applied {
255 status: SagaStatus,
257 },
258 SkippedPaused,
260 SkippedFencingLost,
262 SkippedStale,
264}
265
266#[derive(Debug, Clone)]
268pub struct StartSagaRequest<'a> {
269 pub saga_id: &'a SagaId,
271 pub tenant: &'a TenantId,
273 pub workflow: &'a WorkflowName,
275 pub version: DefinitionVersion,
277 pub business_key: &'a BusinessKey,
279 pub deadline_at_ms: Option<i64>,
281 pub trigger_kind: TriggerKind,
283 pub trigger_id: &'a str,
285 pub first_command_payload: Option<serde_json::Value>,
287 pub first_command_raw_payload: Option<nasaga_core::SagaPayload>,
289 pub now_ms: i64,
291}
292
293enum CommandInput {
295 Json(serde_json::Value),
296 Raw(nasaga_core::SagaPayload),
297}
298
299pub struct Orchestrator<B: SagaBackend> {
304 backend: B,
305 registry: RwLock<Arc<DefinitionRegistry>>,
306 config: OrchestratorConfig,
307 fencing_tokens: TimerFencingTokenIssuer,
309}
310
311struct ResultContext<'a> {
313 instance: &'a SagaInstanceRow,
314 definition: &'a WorkflowDefinition,
315 step_def: &'a StepDefinition,
316 identity: &'a VerifiedIdentity,
317 status: StepAttemptStatus,
318 terminal: Option<StepForwardStatus>,
319 reason_code: Option<&'a str>,
320 event_id: &'a str,
321 now_ms: i64,
322}
323
324fn validate_config(config: &OrchestratorConfig) -> anyhow::Result<()> {
331 if config.inbox_consumer.is_empty()
332 || config.inbox_consumer.trim() != config.inbox_consumer
333 || config.inbox_consumer.len() > 128
334 || config.inbox_consumer.contains('\0')
335 || !config.inbox_consumer.bytes().all(|byte| {
336 byte.is_ascii_lowercase() || byte.is_ascii_digit() || matches!(byte, b'_' | b'-' | b'.')
337 })
338 || config.cancel_max_attempts == 0
339 || config.compensate_max_attempts == 0
340 || config.resolve_max_attempts == 0
341 || config.timer_claim_limit == 0
342 || config.timer_lease_ms <= 0
343 || config.pause_backoff_ms <= 0
344 || config.startup_scan_limit == 0
345 {
346 anyhow::bail!("orchestrator config violates the Inbox key contract or positive budgets");
347 }
348 for tenant in config.tenant_quotas.keys() {
351 if TenantId::new(tenant.as_str()).is_err() {
352 anyhow::bail!("tenant quota key is not a valid tenant id");
353 }
354 }
355 for (tenant, rate) in &config.tenant_action_rates {
356 if TenantId::new(tenant.as_str()).is_err() {
357 anyhow::bail!("tenant action rate key is not a valid tenant id");
358 }
359 if rate.window_ms <= 0 {
360 anyhow::bail!("tenant action rate window_ms must be positive");
361 }
362 }
363 Ok(())
364}
365
366fn duration_millis(duration: Duration) -> anyhow::Result<i64> {
373 let millis = i64::try_from(duration.as_millis())
374 .map_err(|_| anyhow::anyhow!("saga duration exceeds the supported millisecond range"))?;
375 if millis <= 0 {
376 anyhow::bail!("saga duration must be at least one millisecond");
377 }
378 Ok(millis)
379}
380
381fn checked_deadline(now_ms: i64, delay_ms: i64) -> anyhow::Result<i64> {
389 if delay_ms <= 0 {
390 anyhow::bail!("saga timer delay must be positive");
391 }
392 now_ms
393 .checked_add(delay_ms)
394 .ok_or_else(|| anyhow::anyhow!("saga timer deadline overflow"))
395}
396
397fn management_precondition(message: &'static str) -> anyhow::Error {
403 anyhow::Error::new(SagaManagementError::PreconditionFailed).context(message)
404}
405
406fn hash_canonical_json(hasher: &mut Sha256, value: &serde_json::Value) {
414 match value {
415 serde_json::Value::Null => hasher.update(canonical_bytes(&[b"null"])),
416 serde_json::Value::Bool(value) => {
417 hasher.update(canonical_bytes(&[b"bool", &[u8::from(*value)]]));
418 }
419 serde_json::Value::Number(value) => {
420 hasher.update(canonical_bytes(&[b"number", value.to_string().as_bytes()]));
421 }
422 serde_json::Value::String(value) => {
423 hasher.update(canonical_bytes(&[b"string", value.as_bytes()]));
424 }
425 serde_json::Value::Array(values) => {
426 hasher.update(canonical_bytes(&[b"array", &values.len().to_be_bytes()]));
427 for value in values {
428 hash_canonical_json(hasher, value);
429 }
430 }
431 serde_json::Value::Object(values) => {
432 let mut keys: Vec<_> = values.keys().collect();
433 keys.sort_unstable();
434 hasher.update(canonical_bytes(&[b"object", &keys.len().to_be_bytes()]));
435 for key in keys {
436 hasher.update(canonical_bytes(&[b"key", key.as_bytes()]));
437 hash_canonical_json(hasher, &values[key]);
438 }
439 }
440 }
441}
442
443fn derive_legacy_start_request_digest(
455 request: &StartSagaRequest<'_>,
456 definition_digest: &str,
457 first_step: &StepName,
458) -> String {
459 let version = request.version.get().to_be_bytes();
460 let deadline = request.deadline_at_ms.unwrap_or_default().to_be_bytes();
461 let deadline_present = [u8::from(request.deadline_at_ms.is_some())];
462 let payload_present = [u8::from(request.first_command_payload.is_some())];
463 let mut hasher = Sha256::new();
464 hasher.update(canonical_bytes(&[
465 request.tenant.as_str().as_bytes(),
466 request.workflow.as_str().as_bytes(),
467 &version,
468 definition_digest.as_bytes(),
469 request.business_key.as_str().as_bytes(),
470 first_step.as_str().as_bytes(),
471 &deadline_present,
472 &deadline,
473 &payload_present,
474 ]));
475 if let Some(payload) = request.first_command_payload.as_ref() {
476 hash_canonical_json(&mut hasher, payload);
477 }
478 hasher
479 .finalize()
480 .iter()
481 .map(|byte| format!("{byte:02x}"))
482 .collect()
483}
484
485fn derive_start_request_digest(
491 request: &StartSagaRequest<'_>,
492 definition_digest: &str,
493 first_step: &StepName,
494 payload: Option<&nasaga_core::SagaPayload>,
495) -> String {
496 let mut empty = request.clone();
497 empty.first_command_payload = None;
498 empty.first_command_raw_payload = None;
499 let domain = derive_legacy_start_request_digest(&empty, definition_digest, first_step);
500 let Some(payload) = payload else {
501 return domain;
502 };
503 let mut hasher = Sha256::new();
504 hasher.update(canonical_bytes(&[
505 b"saga-start-payload",
506 domain.as_bytes(),
507 payload.content_type.as_bytes(),
508 payload.schema_id.as_bytes(),
509 &payload.body,
510 ]));
511 hex::encode(hasher.finalize())
512}
513
514fn derive_converge_trigger_id(cause_id: &str) -> String {
521 Uuid::new_v5(
522 &CONVERGE_TRIGGER_NAMESPACE,
523 &canonical_bytes(&[cause_id.as_bytes(), b"converge"]),
524 )
525 .to_string()
526}
527
528fn elapsed_epoch_ms(base_now_ms: i64, claimed_at: Instant) -> anyhow::Result<i64> {
539 let elapsed_ms = i64::try_from(claimed_at.elapsed().as_millis())
540 .map_err(|_| anyhow::anyhow!("timer lease elapsed duration overflow"))?;
541 base_now_ms
542 .checked_add(elapsed_ms)
543 .ok_or_else(|| anyhow::anyhow!("timer lease clock overflow"))
544}
545
546impl<B> Orchestrator<B>
547where
548 B: SagaBackend + SagaBackendFactory,
549{
550 pub fn new(registry: DefinitionRegistry, config: OrchestratorConfig) -> anyhow::Result<Self> {
558 validate_config(&config)?;
559 Ok(Self {
560 backend: B::default_backend()?,
561 registry: RwLock::new(Arc::new(registry)),
562 config,
563 fencing_tokens: TimerFencingTokenIssuer::new(),
564 })
565 }
566
567 pub fn with_datasource(
576 registry: DefinitionRegistry,
577 config: OrchestratorConfig,
578 datasource: impl AsRef<str>,
579 ) -> anyhow::Result<Self> {
580 validate_config(&config)?;
581 Ok(Self {
582 backend: B::backend_for(datasource.as_ref())?,
583 registry: RwLock::new(Arc::new(registry)),
584 config,
585 fencing_tokens: TimerFencingTokenIssuer::new(),
586 })
587 }
588
589 pub fn datasource_ref(&self) -> &natx_core::DatasourceRef {
595 self.backend.datasource_ref()
596 }
597
598 fn registry_snapshot(&self) -> Arc<DefinitionRegistry> {
604 self.registry
605 .read()
606 .unwrap_or_else(std::sync::PoisonError::into_inner)
607 .clone()
608 }
609
610 pub fn replace_registry(&self, registry: DefinitionRegistry) {
616 *self
617 .registry
618 .write()
619 .unwrap_or_else(std::sync::PoisonError::into_inner) = Arc::new(registry);
620 }
621
622 pub async fn verify_registry_snapshot(
628 &self,
629 registry: &DefinitionRegistry,
630 ) -> anyhow::Result<()> {
631 registry
632 .verify_non_terminal(self.backend.store(), self.config.startup_scan_limit)
633 .await
634 }
635
636 pub async fn verify_startup(&self) -> anyhow::Result<()> {
642 let registry = self.registry_snapshot();
645 crate::verify_descriptors(®istry)?;
646 for tenant in self.config.tenant_quotas.keys() {
650 let tenant = TenantId::new(tenant.as_str())
651 .map_err(|_| anyhow::anyhow!("tenant quota key is not a valid tenant id"))?;
652 if !self
653 .backend
654 .store()
655 .tenant_quota_initialized(&tenant)
656 .await?
657 {
658 anyhow::bail!(
659 "tenant quota ledger for a capped tenant is not initialized; \
660 run reconcile_tenant_quota after every writer runs the accounting binary"
661 );
662 }
663 }
664 registry
665 .verify_non_terminal(self.backend.store(), self.config.startup_scan_limit)
666 .await
667 }
668
669 pub fn definition_digest(
675 &self,
676 workflow: &WorkflowName,
677 version: DefinitionVersion,
678 ) -> Option<String> {
679 self.registry_snapshot()
680 .get(workflow, version)
681 .map(WorkflowDefinition::digest)
682 }
683
684 pub fn definition_digest_for_tenant(
690 &self,
691 tenant: &TenantId,
692 workflow: &WorkflowName,
693 version: DefinitionVersion,
694 ) -> Option<String> {
695 self.registry_snapshot()
696 .get_for_tenant(tenant, workflow, version)
697 .map(WorkflowDefinition::digest)
698 }
699
700 pub async fn load_instance(
709 &self,
710 tenant: &TenantId,
711 saga_id: &SagaId,
712 ) -> anyhow::Result<Option<SagaInstanceRow>> {
713 Ok(self
714 .backend
715 .store()
716 .load_instance(saga_id)
717 .await?
718 .filter(|instance| &instance.tenant == tenant))
719 }
720
721 pub async fn load_audit_trail(
734 &self,
735 management: &SagaManagementContext,
736 tenant: &TenantId,
737 saga_id: &SagaId,
738 after_transition_seq: u64,
739 after_control_seq: u64,
740 limit: u32,
741 ) -> anyhow::Result<SagaAuditTrail> {
742 management.require(SagaManagementPermission::ReadAudit)?;
744 self.backend
745 .store()
746 .load_instance(saga_id)
747 .await?
748 .filter(|instance| &instance.tenant == tenant)
749 .ok_or(SagaManagementError::NotFound)?;
750 Ok(SagaAuditTrail {
751 attempts: self
752 .backend
753 .store()
754 .load_attempt_audit(saga_id, None, limit)
755 .await?
756 .into_iter()
757 .map(|row| row.attempt)
758 .collect(),
759 transitions: self
760 .backend
761 .store()
762 .load_transition_audit(saga_id, after_transition_seq, limit)
763 .await?,
764 controls: self
765 .backend
766 .store()
767 .load_control_audit(saga_id, after_control_seq, limit)
768 .await?,
769 management_operations: self
770 .backend
771 .store()
772 .load_management_audit(saga_id, None, limit)
773 .await?,
774 conflicts: self
775 .backend
776 .store()
777 .load_conflict_audit(saga_id, None, limit)
778 .await?,
779 })
780 }
781
782 pub async fn load_audit_page(
795 &self,
796 management: &SagaManagementContext,
797 tenant: &TenantId,
798 saga_id: &SagaId,
799 cursor: Option<&SagaAuditPageCursor>,
800 limit: u32,
801 ) -> anyhow::Result<SagaAuditPage> {
802 if limit == 0 || limit > 1_000 {
803 return Err(SagaManagementError::InvalidContext.into());
804 }
805 management.require(SagaManagementPermission::ReadAudit)?;
807 self.backend
808 .store()
809 .load_instance(saga_id)
810 .await?
811 .filter(|instance| &instance.tenant == tenant)
812 .ok_or(SagaManagementError::NotFound)?;
813 let after = SagaAuditEventCursor {
814 audit_seq: cursor.map_or(0, |cursor| cursor.audit_seq),
815 };
816 let events = self
817 .backend
818 .store()
819 .load_audit_events(saga_id, after, limit)
820 .await?;
821 let next_cursor = events.last().map(|event| SagaAuditPageCursor {
824 audit_seq: event.audit_seq,
825 });
826 let records = events
827 .into_iter()
828 .map(|event| match event.record {
829 SagaAuditEventRecord::Attempt(row) => SagaAuditRecord::Attempt(row),
830 SagaAuditEventRecord::Transition(row) => SagaAuditRecord::Transition(row),
831 SagaAuditEventRecord::Control(row) => SagaAuditRecord::Control(row),
832 SagaAuditEventRecord::Management(row) => SagaAuditRecord::Management(row),
833 SagaAuditEventRecord::Conflict(row) => SagaAuditRecord::Conflict(row),
834 })
835 .collect();
836 Ok(SagaAuditPage {
837 records,
838 next_cursor,
839 })
840 }
841
842 pub async fn list_instances(
856 &self,
857 management: &SagaManagementContext,
858 query: &SagaInstanceQuery<'_>,
859 ) -> anyhow::Result<Vec<SagaInstanceSummary>> {
860 management.require(SagaManagementPermission::ListInstances)?;
862 Ok(self.backend.store().list_instances(query).await?)
863 }
864
865 pub async fn tenant_quota_usage(
873 &self,
874 management: &SagaManagementContext,
875 tenant: &TenantId,
876 ) -> anyhow::Result<u64> {
877 management.require(SagaManagementPermission::ReadAudit)?;
878 Ok(self.backend.store().tenant_quota_usage(tenant).await?)
879 }
880
881 pub async fn tenant_action_rate_usage(
889 &self,
890 management: &SagaManagementContext,
891 tenant: &TenantId,
892 ) -> anyhow::Result<Option<(u64, u64)>> {
893 management.require(SagaManagementPermission::ReadAudit)?;
894 let Some(rate) = self.config.tenant_action_rates.get(tenant.as_str()) else {
895 return Ok(None);
896 };
897 Ok(Some(
898 self.backend
899 .store()
900 .tenant_action_rate_usage(tenant, rate.window_ms)
901 .await?,
902 ))
903 }
904
905 async fn reserve_action_rate_budget(&self, tenant: &TenantId) -> anyhow::Result<()> {
916 let Some(rate) = self.config.tenant_action_rates.get(tenant.as_str()) else {
917 return Ok(());
918 };
919 if matches!(
920 self.backend
921 .store()
922 .reserve_tenant_action_rate(tenant, rate.max_actions, rate.window_ms)
923 .await?,
924 nasaga_backend::ActionRateReservation::Exceeded
925 ) {
926 record_action_rate_rejection();
927 return Err(SagaManagementError::RateLimitExceeded.into());
929 }
930 Ok(())
931 }
932
933 pub async fn load_operational_metrics(
943 &self,
944 now_ms: i64,
945 ) -> anyhow::Result<SagaOperationalMetrics> {
946 Ok(self
947 .backend
948 .store()
949 .load_operational_metrics(now_ms)
950 .await?
951 .into())
952 }
953
954 pub fn verify_result_producer(
968 &self,
969 envelope: &SagaResultEnvelope,
970 producer: &ServiceIdentity,
971 ) -> anyhow::Result<()> {
972 let identity = envelope.verified_for_delivery()?;
973 let registry = self.registry_snapshot();
974 let definition = registry
975 .get_for_tenant(
976 &identity.tenant,
977 &identity.workflow,
978 identity.definition_version,
979 )
980 .ok_or(crate::SagaResultProcessingError::ContractInvalid)?;
981 if definition.digest() != identity.definition_digest {
982 return Err(crate::SagaResultProcessingError::ContractInvalid.into());
983 }
984 let step = definition
985 .step(&identity.step)
986 .ok_or(crate::SagaResultProcessingError::ContractInvalid)?;
987 if step.owner() != producer {
989 return Err(crate::SagaResultProcessingError::ProducerUnauthorized.into());
990 }
991 Ok(())
992 }
993
994 pub async fn handle_authenticated_result(
1004 &self,
1005 envelope: &SagaResultEnvelope,
1006 producer: &ServiceIdentity,
1007 now_ms: i64,
1008 ) -> anyhow::Result<HandleOutcome> {
1009 self.handle_authenticated_result_traced(envelope, producer, None, now_ms)
1010 .await
1011 }
1012
1013 pub async fn handle_authenticated_result_traced(
1028 &self,
1029 envelope: &SagaResultEnvelope,
1030 producer: &ServiceIdentity,
1031 receipt_trace: Option<&TraceContext>,
1032 now_ms: i64,
1033 ) -> anyhow::Result<HandleOutcome> {
1034 self.handle_authenticated_result_authorized_traced(
1035 envelope,
1036 producer,
1037 receipt_trace,
1038 now_ms,
1039 &|| Ok(()),
1040 )
1041 .await
1042 }
1043
1044 pub async fn handle_authenticated_result_authorized_traced(
1052 &self,
1053 envelope: &SagaResultEnvelope,
1054 producer: &ServiceIdentity,
1055 receipt_trace: Option<&TraceContext>,
1056 now_ms: i64,
1057 authorize: &(dyn Fn() -> anyhow::Result<()> + Send + Sync),
1058 ) -> anyhow::Result<HandleOutcome> {
1059 authorize()?;
1061 self.verify_result_producer(envelope, producer)?;
1063 let _latency =
1064 crate::latency::LatencyGuard::new(crate::latency::LatencyStage::TransitionTransaction);
1065 self.handle_verified_result(envelope, receipt_trace, now_ms, authorize)
1066 .await
1067 }
1068
1069 pub async fn start_saga(&self, request: &StartSagaRequest<'_>) -> anyhow::Result<StartOutcome> {
1081 self.start_saga_traced(request, None).await
1082 }
1083
1084 pub async fn start_saga_traced(
1098 &self,
1099 request: &StartSagaRequest<'_>,
1100 trace: Option<&TraceContext>,
1101 ) -> anyhow::Result<StartOutcome> {
1102 self.start_saga_authorized_traced(request, trace, &|| Ok(()))
1103 .await
1104 }
1105
1106 pub async fn start_saga_authorized_traced(
1114 &self,
1115 request: &StartSagaRequest<'_>,
1116 trace: Option<&TraceContext>,
1117 authorize: &(dyn Fn() -> anyhow::Result<()> + Send + Sync),
1118 ) -> anyhow::Result<StartOutcome> {
1119 authorize()?;
1121 let registry = self.registry_snapshot();
1122 let definition = registry
1124 .get_for_tenant(request.tenant, request.workflow, request.version)
1125 .ok_or(StartSagaError::DefinitionInactive)?;
1126 let digest = definition.digest();
1127 let first_step = definition
1128 .steps()
1129 .first()
1130 .ok_or(StartSagaError::DefinitionEmpty)?;
1131 let _latency =
1132 crate::latency::LatencyGuard::new(crate::latency::LatencyStage::StartTransaction);
1133 if request.first_command_payload.is_some() && request.first_command_raw_payload.is_some() {
1134 return Err(StartSagaError::InvalidPayload.into());
1135 }
1136 let payload = request.first_command_raw_payload.clone().or_else(|| {
1137 request
1138 .first_command_payload
1139 .clone()
1140 .map(nasaga_core::SagaPayload::json)
1141 });
1142 if let Some(payload) = &payload {
1143 payload
1144 .validate()
1145 .map_err(|_| StartSagaError::InvalidPayload)?;
1146 if &payload.contract() != first_step.payload_contract() {
1147 return Err(StartSagaError::InvalidPayload.into());
1148 }
1149 }
1150 let start_request_digest =
1151 derive_start_request_digest(request, &digest, first_step.name(), payload.as_ref());
1152 let legacy_start_request_digest = match payload.as_ref() {
1153 Some(payload) if payload.contract() == nasaga_core::SagaPayloadContract::default() => {
1154 let mut legacy = request.clone();
1155 legacy.first_command_payload = Some(
1156 serde_json::from_slice(&payload.body)
1157 .map_err(|_| StartSagaError::InvalidPayload)?,
1158 );
1159 Some(derive_legacy_start_request_digest(
1160 &legacy,
1161 &digest,
1162 first_step.name(),
1163 ))
1164 }
1165 _ => None,
1166 };
1167 let canonical_trace = trace.map(TraceContext::to_traceparent);
1168
1169 let outcome = crate::transaction::run_authorized_for(&self.backend, authorize, async {
1170 let creation = self
1171 .backend
1172 .store()
1173 .create_instance(&NewSagaInstance {
1174 saga_id: request.saga_id,
1175 tenant: request.tenant,
1176 workflow: request.workflow,
1177 business_key: request.business_key,
1178 definition_version: request.version,
1179 definition_digest: &digest,
1180 start_request_digest: &start_request_digest,
1181 legacy_start_request_digest: legacy_start_request_digest.as_deref(),
1182 deadline_at_ms: request.deadline_at_ms,
1183 current_step: Some(first_step.name()),
1185 trigger_kind: request.trigger_kind,
1186 trigger_id: request.trigger_id,
1187 traceparent: canonical_trace.as_deref(),
1190 })
1191 .await
1192 .map_err(|error| {
1193 if error.kind() == SagaBackendErrorKind::Conflict {
1194 anyhow::Error::new(StartSagaError::RequestConflict)
1195 } else {
1196 error.into()
1197 }
1198 })?;
1199 let instance = match creation {
1200 SagaCreation::Existing(row) => return Ok(StartOutcome::AlreadyExists(row)),
1202 SagaCreation::Created(row) => row,
1203 };
1204 if registry
1207 .get_for_start(request.tenant, request.workflow, request.version)
1208 .is_none()
1209 {
1210 return Err(StartSagaError::DefinitionInactive.into());
1211 }
1212 let cap = self
1215 .config
1216 .tenant_quotas
1217 .get(request.tenant.as_str())
1218 .copied();
1219 if matches!(
1220 self.backend
1221 .store()
1222 .reserve_tenant_quota(request.tenant, cap)
1223 .await?,
1224 nasaga_backend::QuotaReservation::Exceeded
1225 ) {
1226 record_quota_rejection();
1227 return Err(StartSagaError::TenantQuotaExceeded.into());
1229 }
1230 self.backend
1231 .store()
1232 .register_steps(request.saga_id, definition)
1233 .await?;
1234 self.issue_command(
1235 &instance,
1236 first_step,
1237 StepPhase::Execute,
1238 AttemptNo::FIRST,
1239 request
1240 .first_command_raw_payload
1241 .clone()
1242 .map(CommandInput::Raw)
1243 .or_else(|| {
1244 request
1245 .first_command_payload
1246 .clone()
1247 .map(CommandInput::Json)
1248 }),
1249 None,
1250 request.now_ms,
1251 instance.version,
1252 )
1253 .await?;
1254 if let Some(deadline) = request.deadline_at_ms {
1256 self.schedule_timer(
1257 request.saga_id,
1258 TimerScope::Instance,
1259 KIND_INSTANCE_DEADLINE,
1260 AttemptNo::FIRST,
1261 deadline,
1262 instance.version,
1263 )
1264 .await?;
1265 }
1266 Ok(StartOutcome::Started(instance))
1267 })
1268 .await?;
1269 match &outcome {
1270 StartOutcome::Started(instance) => tracing::info!(
1271 saga_id = instance.saga_id.as_str(),
1272 tenant = instance.tenant.as_str(),
1273 workflow = instance.workflow.as_str(),
1274 definition_version = instance.definition_version.get(),
1275 saga_status = instance.status.as_str(),
1276 "Saga 创建事务已提交"
1277 ),
1278 StartOutcome::AlreadyExists(instance) => tracing::info!(
1279 saga_id = instance.saga_id.as_str(),
1280 tenant = instance.tenant.as_str(),
1281 workflow = instance.workflow.as_str(),
1282 saga_status = instance.status.as_str(),
1283 "Saga 启动重放已幂等吸收"
1284 ),
1285 }
1286 Ok(outcome)
1287 }
1288
1289 async fn handle_verified_result(
1304 &self,
1305 envelope: &SagaResultEnvelope,
1306 receipt_trace: Option<&TraceContext>,
1307 now_ms: i64,
1308 authorize: &(dyn Fn() -> anyhow::Result<()> + Send + Sync),
1309 ) -> anyhow::Result<HandleOutcome> {
1310 let identity = envelope.verified_for_delivery()?;
1312 let status = envelope
1313 .parsed_status()
1314 .map_err(|_| crate::SagaResultProcessingError::ContractInvalid)?;
1315 let terminal = envelope
1316 .parsed_terminal()
1317 .map_err(|_| crate::SagaResultProcessingError::ContractInvalid)?;
1318 let registry = self.registry_snapshot();
1319
1320 let outcome = crate::transaction::run_authorized_for(&self.backend, authorize, async {
1321 if matches!(
1322 self.backend
1323 .inbox()
1324 .claim(&self.config.inbox_consumer, &envelope.event_id)
1325 .await?,
1326 InboxClaim::Duplicate
1327 ) {
1328 return Ok(HandleOutcome::Duplicate);
1329 }
1330 let instance = self
1331 .backend
1332 .store()
1333 .load_instance(&identity.saga_id)
1334 .await?
1335 .ok_or(crate::SagaResultProcessingError::ContractInvalid)?;
1336 verify_instance_contract(&instance, &identity)
1337 .map_err(|_| crate::SagaResultProcessingError::ContractInvalid)?;
1338 authorize()?;
1340 if !instance.control_state.allows_automatic_actions() {
1341 return Err(crate::SagaResultProcessingError::Paused.into());
1344 }
1345 let definition = registry
1346 .get_for_tenant(
1347 &instance.tenant,
1348 &instance.workflow,
1349 instance.definition_version,
1350 )
1351 .ok_or_else(|| anyhow::anyhow!("instance definition is not registered"))?;
1352 let step_def = definition.step(&identity.step).ok_or_else(|| {
1353 anyhow::Error::from(crate::SagaResultProcessingError::ContractInvalid)
1354 })?;
1355
1356 let recorded = self
1359 .backend
1360 .store()
1361 .record_attempt_outcome(
1362 &identity.saga_id,
1363 &identity.step,
1364 identity.phase,
1365 identity.attempt,
1366 status,
1367 Some(&envelope.event_id),
1368 )
1369 .await?;
1370 match recorded {
1371 AttemptOutcomeRecord::AlreadyRecorded => {
1375 return Ok(HandleOutcome::Applied {
1376 status: instance.status,
1377 });
1378 }
1379 AttemptOutcomeRecord::Conflicting { existing } => {
1380 self.backend
1383 .store()
1384 .record_attempt_conflict(&AttemptConflictFact {
1385 saga_id: &identity.saga_id,
1386 step: &identity.step,
1387 phase: identity.phase,
1388 attempt: identity.attempt,
1389 existing_status: existing,
1390 incoming_status: status,
1391 incoming_event_id: &envelope.event_id,
1392 conflict_kind: SagaConflictKind::AttemptTerminal,
1393 })
1394 .await?;
1395 let status = if instance.status.is_terminal()
1396 || instance.status == SagaStatus::ManualIntervention
1397 {
1398 instance.status
1401 } else {
1402 self.escalate_manual(
1403 &instance,
1404 TriggerKind::Event,
1405 &envelope.event_id,
1406 "attempt_outcome_conflict",
1407 )
1408 .await?
1409 };
1410 return Ok(HandleOutcome::Applied { status });
1411 }
1412 AttemptOutcomeRecord::Recorded => {}
1413 }
1414
1415 let instance = match receipt_trace {
1419 Some(trace) => {
1420 let canonical = trace.to_traceparent();
1421 self.backend
1422 .store()
1423 .update_trace_context(&identity.saga_id, &canonical)
1424 .await?;
1425 SagaInstanceRow {
1426 traceparent: Some(canonical),
1427 ..instance
1428 }
1429 }
1430 None => instance,
1431 };
1432
1433 let context = ResultContext {
1434 instance: &instance,
1435 definition,
1436 step_def,
1437 identity: &identity,
1438 status,
1439 terminal,
1440 reason_code: envelope.reason_code.as_deref(),
1441 event_id: &envelope.event_id,
1442 now_ms,
1443 };
1444 let new_status = match identity.phase {
1448 StepPhase::Execute => Box::pin(self.on_execute_result(&context)).await?,
1449 StepPhase::Cancel => Box::pin(self.on_cancel_result(&context)).await?,
1450 StepPhase::Compensate => Box::pin(self.on_compensate_result(&context)).await?,
1451 StepPhase::Resolve => Box::pin(self.on_resolve_result(&context)).await?,
1452 };
1453 Ok(HandleOutcome::Applied { status: new_status })
1454 })
1455 .await?;
1456 match &outcome {
1457 HandleOutcome::Applied { status } => tracing::info!(
1458 saga_id = identity.saga_id.as_str(),
1459 workflow = identity.workflow.as_str(),
1460 step = identity.step.as_str(),
1461 phase = identity.phase.as_str(),
1462 attempt = identity.attempt.get(),
1463 saga_status = status.as_str(),
1464 "Saga 结果事务已提交"
1465 ),
1466 HandleOutcome::Duplicate => tracing::info!(
1467 saga_id = identity.saga_id.as_str(),
1468 workflow = identity.workflow.as_str(),
1469 step = identity.step.as_str(),
1470 phase = identity.phase.as_str(),
1471 attempt = identity.attempt.get(),
1472 "Saga 重复结果已由 Inbox 吸收"
1473 ),
1474 }
1475 Ok(outcome)
1476 }
1477
1478 pub async fn run_due_timers(&self, owner: &str, now_ms: i64) -> anyhow::Result<u32> {
1491 let claimed_at = Instant::now();
1494 let token = self.fencing_tokens.issue(owner, now_ms);
1497 let claimed = self
1498 .backend
1499 .store()
1500 .claim_due_timers(
1501 owner,
1502 token,
1503 now_ms,
1504 self.config.timer_lease_ms,
1505 self.config.timer_claim_limit,
1506 )
1507 .await?;
1508 let mut applied = 0u32;
1509 for timer in claimed.timers() {
1510 match self
1511 .fire_timer_inner(timer, claimed.token(), now_ms, Some(claimed_at))
1512 .await?
1513 {
1514 TimerOutcome::Applied { status } => {
1515 applied += 1;
1516 tracing::info!(
1517 saga_id = timer.saga_id.as_str(),
1518 timer_kind = %timer.kind,
1519 timer_scope = %timer.scope_kind,
1520 attempt = timer.attempt.get(),
1521 saga_status = status.as_str(),
1522 "Saga durable timer 裁决事务已提交"
1523 );
1524 }
1525 TimerOutcome::SkippedPaused => {
1526 let release_now = elapsed_epoch_ms(now_ms, claimed_at)?;
1530 let _ = self
1531 .backend
1532 .store()
1533 .release_timer(
1534 &timer.timer_id,
1535 claimed.token(),
1536 release_now,
1537 checked_deadline(release_now, self.config.pause_backoff_ms)?,
1538 )
1539 .await?;
1540 }
1541 TimerOutcome::SkippedFencingLost => tracing::warn!(
1542 saga_id = timer.saga_id.as_str(),
1543 timer_kind = %timer.kind,
1544 "Saga timer 已失去 fencing 权威,旧 owner 停止动作"
1545 ),
1546 TimerOutcome::SkippedStale => tracing::debug!(
1547 saga_id = timer.saga_id.as_str(),
1548 timer_kind = %timer.kind,
1549 "Saga 过期 timer 已幂等吸收"
1550 ),
1551 }
1552 }
1553 Ok(applied)
1554 }
1555
1556 pub async fn fire_timer(
1566 &self,
1567 timer: &SagaTimerRow,
1568 token: &TimerFencingToken,
1569 now_ms: i64,
1570 ) -> anyhow::Result<TimerOutcome> {
1571 self.fire_timer_inner(timer, token, now_ms, None).await
1572 }
1573
1574 async fn fire_timer_inner(
1585 &self,
1586 timer: &SagaTimerRow,
1587 token: &TimerFencingToken,
1588 base_now_ms: i64,
1589 claimed_at: Option<Instant>,
1590 ) -> anyhow::Result<TimerOutcome> {
1591 let _latency =
1592 crate::latency::LatencyGuard::new(crate::latency::LatencyStage::TransitionTransaction);
1593 let step_scope = parse_step_scope(&timer.scope_kind, &timer.scope_key)?;
1594 let registry = self.registry_snapshot();
1595 crate::transaction::run_for(&self.backend, async {
1596 let Some(instance) = self.backend.store().load_instance(&timer.saga_id).await? else {
1597 let fencing_now = match claimed_at {
1599 Some(started) => elapsed_epoch_ms(base_now_ms, started)?,
1600 None => base_now_ms,
1601 };
1602 return Ok(
1603 match self
1604 .backend
1605 .store()
1606 .complete_timer(&timer.timer_id, token, fencing_now)
1607 .await?
1608 {
1609 TimerFencing::Applied => TimerOutcome::SkippedStale,
1610 TimerFencing::Lost => TimerOutcome::SkippedFencingLost,
1611 },
1612 );
1613 };
1614 if !instance.control_state.allows_automatic_actions() {
1615 return Ok(TimerOutcome::SkippedPaused);
1616 }
1617 let now_ms = match claimed_at {
1618 Some(started) => elapsed_epoch_ms(base_now_ms, started)?,
1619 None => base_now_ms,
1620 };
1621 if matches!(
1623 self.backend
1624 .store()
1625 .complete_timer(&timer.timer_id, token, now_ms)
1626 .await?,
1627 TimerFencing::Lost
1628 ) {
1629 return Ok(TimerOutcome::SkippedFencingLost);
1630 }
1631 crate::latency::observe(
1632 crate::latency::LatencyStage::TimerLateness,
1633 Duration::from_millis(now_ms.saturating_sub(timer.due_at_ms).max(0) as u64),
1634 );
1635 if matches!(
1639 timer.kind.as_str(),
1640 KIND_STEP_TIMEOUT
1641 | KIND_CANCEL_TIMEOUT
1642 | KIND_COMPENSATE_TIMEOUT
1643 | KIND_RESOLVE_TIMEOUT
1644 ) && timer.expected_saga_version != instance.version
1645 {
1646 return Ok(TimerOutcome::SkippedStale);
1647 }
1648 let definition = registry
1649 .get_for_tenant(
1650 &instance.tenant,
1651 &instance.workflow,
1652 instance.definition_version,
1653 )
1654 .ok_or_else(|| anyhow::anyhow!("instance definition is not registered"))?;
1655
1656 let status = match timer.kind.as_str() {
1658 KIND_STEP_TIMEOUT => {
1659 Box::pin(self.on_step_timeout(
1660 &instance,
1661 definition,
1662 step_scope.as_ref(),
1663 timer,
1664 now_ms,
1665 ))
1666 .await?
1667 }
1668 KIND_CANCEL_TIMEOUT => {
1669 Box::pin(self.on_phase_retry_timeout(
1670 &instance,
1671 definition,
1672 step_scope.as_ref(),
1673 timer,
1674 StepPhase::Cancel,
1675 SagaStatus::Cancelling,
1676 self.config.cancel_max_attempts,
1677 "cancel_attempts_exhausted",
1678 now_ms,
1679 ))
1680 .await?
1681 }
1682 KIND_COMPENSATE_TIMEOUT => {
1683 Box::pin(self.on_phase_retry_timeout(
1684 &instance,
1685 definition,
1686 step_scope.as_ref(),
1687 timer,
1688 StepPhase::Compensate,
1689 SagaStatus::Compensating,
1690 self.config.compensate_max_attempts,
1691 "compensation_attempts_exhausted",
1692 now_ms,
1693 ))
1694 .await?
1695 }
1696 KIND_RESOLVE_TIMEOUT => {
1697 Box::pin(self.on_phase_retry_timeout(
1698 &instance,
1699 definition,
1700 step_scope.as_ref(),
1701 timer,
1702 StepPhase::Resolve,
1703 SagaStatus::WaitingResolution,
1704 self.config.resolve_max_attempts,
1705 "resolve_attempts_exhausted",
1706 now_ms,
1707 ))
1708 .await?
1709 }
1710 KIND_RESOLUTION_BUDGET
1711 | KIND_FORWARD_RESOLUTION_BUDGET
1712 | KIND_COMPENSATION_RESOLUTION_BUDGET => {
1713 if instance.status == SagaStatus::WaitingResolution {
1716 Some(
1717 self.escalate_manual(
1718 &instance,
1719 TriggerKind::Timer,
1720 &timer.timer_id,
1721 "resolution_budget_exhausted",
1722 )
1723 .await?,
1724 )
1725 } else {
1726 None
1727 }
1728 }
1729 KIND_INSTANCE_DEADLINE => {
1730 Box::pin(self.on_instance_deadline(&instance, definition, timer, now_ms))
1731 .await?
1732 }
1733 _ => anyhow::bail!("unknown timer kind"),
1734 };
1735 Ok(match status {
1736 Some(status) => TimerOutcome::Applied { status },
1737 None => TimerOutcome::SkippedStale,
1739 })
1740 })
1741 .await
1742 }
1743
1744 pub async fn pause(
1754 &self,
1755 management: &SagaManagementContext,
1756 tenant: &TenantId,
1757 saga_id: &SagaId,
1758 operation_id: &str,
1759 ) -> anyhow::Result<()> {
1760 self.pause_with_expectation(
1761 management,
1762 tenant,
1763 saga_id,
1764 operation_id,
1765 SagaManagementExpectation::default(),
1766 )
1767 .await
1768 }
1769
1770 pub async fn pause_with_expectation(
1777 &self,
1778 management: &SagaManagementContext,
1779 tenant: &TenantId,
1780 saga_id: &SagaId,
1781 operation_id: &str,
1782 expectation: SagaManagementExpectation,
1783 ) -> anyhow::Result<()> {
1784 management.require(SagaManagementPermission::Pause)?;
1786 crate::transaction::run_for(&self.backend, async {
1787 let instance = self
1788 .backend
1789 .store()
1790 .load_instance(saga_id)
1791 .await?
1792 .filter(|instance| &instance.tenant == tenant)
1793 .ok_or(SagaManagementError::NotFound)?;
1794 match self
1797 .backend
1798 .store()
1799 .set_control_state(&ControlTransitionSpec {
1800 saga_id,
1801 expected_version: instance.version,
1802 expected_control_version: instance.control_version,
1803 from: nasaga_core::ControlState::Active,
1804 to: nasaga_core::ControlState::Paused,
1805 operation_id,
1806 actor: management.actor(),
1807 reason: management.reason(),
1808 })
1809 .await?
1810 {
1811 ControlCasOutcome::Applied => {
1812 expectation.verify(instance.version, instance.control_version)?;
1813 self.reserve_action_rate_budget(tenant).await?;
1816 Ok(())
1817 }
1818 ControlCasOutcome::AlreadyApplied => Ok(()),
1819 ControlCasOutcome::Conflict => {
1820 expectation.verify(instance.version, instance.control_version)?;
1821 if instance.control_state == nasaga_core::ControlState::Active {
1822 Err(anyhow::Error::new(SagaConcurrencyError::StaleSnapshot)
1825 .context("pause lost the control-state race; reload and retry"))
1826 } else {
1827 Err(management_precondition(
1828 "pause requires an active control state",
1829 ))
1830 }
1831 }
1832 }
1833 })
1834 .await?;
1835 tracing::info!(
1836 saga_id = saga_id.as_str(),
1837 tenant = tenant.as_str(),
1838 actor = management.actor(),
1839 operation_id,
1840 control_state = "PAUSED",
1841 "Saga 暂停操作已提交"
1842 );
1843 Ok(())
1844 }
1845
1846 pub async fn resume(
1857 &self,
1858 management: &SagaManagementContext,
1859 tenant: &TenantId,
1860 saga_id: &SagaId,
1861 operation_id: &str,
1862 ) -> anyhow::Result<()> {
1863 self.resume_with_expectation(
1864 management,
1865 tenant,
1866 saga_id,
1867 operation_id,
1868 SagaManagementExpectation::default(),
1869 )
1870 .await
1871 }
1872
1873 pub async fn resume_with_expectation(
1880 &self,
1881 management: &SagaManagementContext,
1882 tenant: &TenantId,
1883 saga_id: &SagaId,
1884 operation_id: &str,
1885 expectation: SagaManagementExpectation,
1886 ) -> anyhow::Result<()> {
1887 management.require(SagaManagementPermission::Resume)?;
1889 crate::transaction::run_for(&self.backend, async {
1890 let instance = self
1891 .backend
1892 .store()
1893 .load_instance(saga_id)
1894 .await?
1895 .filter(|instance| &instance.tenant == tenant)
1896 .ok_or(SagaManagementError::NotFound)?;
1897 match self
1900 .backend
1901 .store()
1902 .set_control_state(&ControlTransitionSpec {
1903 saga_id,
1904 expected_version: instance.version,
1905 expected_control_version: instance.control_version,
1906 from: nasaga_core::ControlState::Paused,
1907 to: nasaga_core::ControlState::Active,
1908 operation_id,
1909 actor: management.actor(),
1910 reason: management.reason(),
1911 })
1912 .await?
1913 {
1914 ControlCasOutcome::Applied => {
1915 expectation.verify(instance.version, instance.control_version)?;
1916 self.reserve_action_rate_budget(tenant).await?;
1919 self.backend.store().wake_saga_timers(saga_id).await?;
1922 Ok(())
1923 }
1924 ControlCasOutcome::AlreadyApplied => Ok(()),
1925 ControlCasOutcome::Conflict => {
1926 expectation.verify(instance.version, instance.control_version)?;
1927 if instance.control_state == nasaga_core::ControlState::Paused {
1928 Err(anyhow::Error::new(SagaConcurrencyError::StaleSnapshot)
1931 .context("resume lost the control-state race; reload and retry"))
1932 } else {
1933 Err(management_precondition(
1934 "resume requires a paused control state",
1935 ))
1936 }
1937 }
1938 }
1939 })
1940 .await?;
1941 tracing::info!(
1942 saga_id = saga_id.as_str(),
1943 tenant = tenant.as_str(),
1944 actor = management.actor(),
1945 operation_id,
1946 control_state = "ACTIVE",
1947 "Saga 恢复操作已提交"
1948 );
1949 Ok(())
1950 }
1951
1952 pub async fn retry_compensation(
1967 &self,
1968 management: &SagaManagementContext,
1969 tenant: &TenantId,
1970 saga_id: &SagaId,
1971 operation_id: &str,
1972 now_ms: i64,
1973 ) -> anyhow::Result<SagaStatus> {
1974 self.retry_compensation_with_expectation(
1975 management,
1976 tenant,
1977 saga_id,
1978 operation_id,
1979 now_ms,
1980 SagaManagementExpectation::default(),
1981 )
1982 .await
1983 }
1984
1985 pub async fn retry_compensation_with_expectation(
1991 &self,
1992 management: &SagaManagementContext,
1993 tenant: &TenantId,
1994 saga_id: &SagaId,
1995 operation_id: &str,
1996 now_ms: i64,
1997 expectation: SagaManagementExpectation,
1998 ) -> anyhow::Result<SagaStatus> {
1999 management.require(SagaManagementPermission::RetryCompensation)?;
2001 let registry = self.registry_snapshot();
2002 let status = crate::transaction::run_for(&self.backend, async {
2003 let instance = self
2004 .backend
2005 .store()
2006 .load_instance(saga_id)
2007 .await?
2008 .filter(|instance| &instance.tenant == tenant)
2009 .ok_or(SagaManagementError::NotFound)?;
2010 match self
2011 .backend
2012 .store()
2013 .record_management_operation(
2014 saga_id,
2015 operation_id,
2016 "retry_frozen_compensation",
2017 management.actor(),
2018 management.reason(),
2019 )
2020 .await?
2021 {
2022 ManagementAuditOutcome::AlreadyRecorded => return Ok(instance.status),
2025 ManagementAuditOutcome::Recorded => {}
2026 }
2027 self.reserve_action_rate_budget(tenant).await?;
2030 expectation.verify(instance.version, instance.control_version)?;
2033 if !instance.control_state.allows_automatic_actions() {
2034 return Err(management_precondition(
2035 "retry_compensation requires ACTIVE control state; resume first",
2036 ));
2037 }
2038 if instance.status != SagaStatus::ManualIntervention {
2039 return Err(management_precondition(
2040 "retry_compensation requires MANUAL_INTERVENTION",
2041 ));
2042 }
2043 if instance.compensation_plan_version.is_none() {
2044 return Err(management_precondition(
2045 "no frozen compensation plan to retry",
2046 ));
2047 }
2048 let definition = registry
2049 .get_for_tenant(
2050 &instance.tenant,
2051 &instance.workflow,
2052 instance.definition_version,
2053 )
2054 .ok_or_else(|| management_precondition("instance definition is not registered"))?;
2055 let steps = self.backend.store().load_steps(saga_id).await?;
2056 match next_pending_compensation(&instance, definition, &steps, true)? {
2057 Some(step) => {
2058 let recovering_halted = steps.iter().any(|row| {
2059 row.step == step
2060 && row.compensation_status == StepCompensationStatus::Halted
2061 });
2062 if recovering_halted {
2063 self.backend
2066 .store()
2067 .reopen_halted_compensation(saga_id, &step, operation_id)
2068 .await?;
2069 }
2070 let version = self
2071 .advance(
2072 &instance,
2073 SagaStatus::Compensating,
2074 Direction::Compensating,
2075 Some(&step),
2076 TriggerKind::Admin,
2077 operation_id,
2078 None,
2079 None,
2080 )
2081 .await?;
2082 let step_def = definition.step(&step).ok_or_else(|| {
2083 management_precondition("plan step absent from definition")
2084 })?;
2085 let attempt = self
2086 .next_attempt(saga_id, &step, StepPhase::Compensate)
2087 .await?;
2088 self.issue_command(
2089 &instance,
2090 step_def,
2091 StepPhase::Compensate,
2092 attempt,
2093 None,
2094 Some(operation_id),
2095 now_ms,
2096 version,
2097 )
2098 .await?;
2099 Ok(SagaStatus::Compensating)
2100 }
2101 None => {
2102 let version = self
2105 .advance(
2106 &instance,
2107 SagaStatus::Compensating,
2108 Direction::Compensating,
2109 None,
2110 TriggerKind::Admin,
2111 operation_id,
2112 None,
2113 None,
2114 )
2115 .await?;
2116 self.converge_compensated(&instance, version, TriggerKind::Admin, operation_id)
2117 .await?;
2118 Ok(SagaStatus::Compensated)
2119 }
2120 }
2121 })
2122 .await?;
2123 tracing::info!(
2124 saga_id = saga_id.as_str(),
2125 tenant = tenant.as_str(),
2126 actor = management.actor(),
2127 operation_id,
2128 saga_status = status.as_str(),
2129 "Saga 人工补偿恢复操作已提交"
2130 );
2131 Ok(status)
2132 }
2133
2134 pub async fn retry_resolution(
2149 &self,
2150 management: &SagaManagementContext,
2151 tenant: &TenantId,
2152 saga_id: &SagaId,
2153 operation_id: &str,
2154 now_ms: i64,
2155 ) -> anyhow::Result<SagaStatus> {
2156 self.retry_resolution_with_expectation(
2157 management,
2158 tenant,
2159 saga_id,
2160 operation_id,
2161 now_ms,
2162 SagaManagementExpectation::default(),
2163 )
2164 .await
2165 }
2166
2167 pub async fn retry_resolution_with_expectation(
2173 &self,
2174 management: &SagaManagementContext,
2175 tenant: &TenantId,
2176 saga_id: &SagaId,
2177 operation_id: &str,
2178 now_ms: i64,
2179 expectation: SagaManagementExpectation,
2180 ) -> anyhow::Result<SagaStatus> {
2181 management.require(SagaManagementPermission::RetryResolution)?;
2182 let registry = self.registry_snapshot();
2183 let status = crate::transaction::run_for(&self.backend, async {
2184 let instance = self
2185 .backend
2186 .store()
2187 .load_instance(saga_id)
2188 .await?
2189 .filter(|instance| &instance.tenant == tenant)
2190 .ok_or(SagaManagementError::NotFound)?;
2191 match self
2192 .backend
2193 .store()
2194 .record_management_operation(
2195 saga_id,
2196 operation_id,
2197 "retry_resolution",
2198 management.actor(),
2199 management.reason(),
2200 )
2201 .await?
2202 {
2203 ManagementAuditOutcome::AlreadyRecorded => return Ok(instance.status),
2204 ManagementAuditOutcome::Recorded => {}
2205 }
2206 self.reserve_action_rate_budget(tenant).await?;
2209 expectation.verify(instance.version, instance.control_version)?;
2212 if !instance.control_state.allows_automatic_actions() {
2213 return Err(management_precondition(
2214 "retry_resolution requires ACTIVE control state; resume first",
2215 ));
2216 }
2217 if instance.status != SagaStatus::ManualIntervention {
2218 return Err(management_precondition(
2219 "retry_resolution requires MANUAL_INTERVENTION",
2220 ));
2221 }
2222 let step = instance
2223 .current_step
2224 .as_ref()
2225 .ok_or_else(|| management_precondition("manual resolution has no current step"))?;
2226 let definition = registry
2227 .get_for_tenant(
2228 &instance.tenant,
2229 &instance.workflow,
2230 instance.definition_version,
2231 )
2232 .ok_or_else(|| management_precondition("instance definition is not registered"))?;
2233 let step_def = definition
2234 .step(step)
2235 .ok_or_else(|| management_precondition("resolution step absent from definition"))?;
2236 if step_def.resolution().mode() != Some(ResolutionMode::Poll) {
2237 return Err(management_precondition(
2238 "retry_resolution requires a Poll resolver",
2239 ));
2240 }
2241 let row = self
2242 .backend
2243 .store()
2244 .load_steps(saga_id)
2245 .await?
2246 .into_iter()
2247 .find(|row| row.step == *step)
2248 .ok_or_else(|| management_precondition("resolution step journal is missing"))?;
2249 let has_unknown_target = match instance.direction {
2250 Direction::Forward => row.forward_status == StepForwardStatus::Unknown,
2251 Direction::Compensating => {
2252 row.compensation_status == StepCompensationStatus::Unknown
2253 }
2254 };
2255 if !has_unknown_target
2256 || !matches!(
2257 row.resolution_status,
2258 StepResolutionStatus::Pending | StepResolutionStatus::Halted
2259 )
2260 {
2261 return Err(management_precondition(
2262 "retry_resolution requires an unresolved Unknown effect",
2263 ));
2264 }
2265
2266 let attempt = self.next_attempt(saga_id, step, StepPhase::Resolve).await?;
2267 if row.resolution_status == StepResolutionStatus::Halted {
2268 self.backend
2271 .store()
2272 .reopen_halted_resolution(saga_id, step, operation_id)
2273 .await?;
2274 }
2275 let version = self
2276 .advance(
2277 &instance,
2278 SagaStatus::WaitingResolution,
2279 instance.direction,
2280 Some(step),
2281 TriggerKind::Admin,
2282 operation_id,
2283 None,
2284 None,
2285 )
2286 .await?;
2287 self.schedule_timer(
2288 saga_id,
2289 TimerScope::Step(step),
2290 resolution_budget_kind(instance.direction),
2291 attempt,
2292 checked_deadline(now_ms, duration_millis(step_def.resolution().budget())?)?,
2293 version,
2294 )
2295 .await?;
2296 self.issue_command(
2297 &instance,
2298 step_def,
2299 StepPhase::Resolve,
2300 attempt,
2301 None,
2302 Some(operation_id),
2303 now_ms,
2304 version,
2305 )
2306 .await?;
2307 Ok(SagaStatus::WaitingResolution)
2308 })
2309 .await?;
2310 tracing::info!(
2311 saga_id = saga_id.as_str(),
2312 tenant = tenant.as_str(),
2313 actor = management.actor(),
2314 operation_id,
2315 saga_status = status.as_str(),
2316 "Saga 人工解决查询恢复操作已提交"
2317 );
2318 Ok(status)
2319 }
2320
2321 pub async fn manual_close(
2338 &self,
2339 management: &SagaManagementContext,
2340 tenant: &TenantId,
2341 saga_id: &SagaId,
2342 operation_id: &str,
2343 ) -> anyhow::Result<SagaStatus> {
2344 self.manual_close_with_expectation(
2345 management,
2346 tenant,
2347 saga_id,
2348 operation_id,
2349 SagaManagementExpectation::default(),
2350 )
2351 .await
2352 }
2353
2354 pub async fn manual_close_with_expectation(
2360 &self,
2361 management: &SagaManagementContext,
2362 tenant: &TenantId,
2363 saga_id: &SagaId,
2364 operation_id: &str,
2365 expectation: SagaManagementExpectation,
2366 ) -> anyhow::Result<SagaStatus> {
2367 management.require(SagaManagementPermission::ManualClose)?;
2368 if !self.config.enable_manual_close {
2371 return Err(management_precondition(
2372 "manual close is disabled: deploy MANUALLY_CLOSED-aware readers to every replica, then enable it explicitly",
2373 ));
2374 }
2375 let status = crate::transaction::run_for(&self.backend, async {
2376 let instance = self
2377 .backend
2378 .store()
2379 .load_instance(saga_id)
2380 .await?
2381 .filter(|instance| &instance.tenant == tenant)
2382 .ok_or(SagaManagementError::NotFound)?;
2383 match self
2386 .backend
2387 .store()
2388 .record_management_operation(
2389 saga_id,
2390 operation_id,
2391 nasaga_backend::MANUAL_CLOSE_ACTION,
2392 management.actor(),
2393 management.reason(),
2394 )
2395 .await?
2396 {
2397 ManagementAuditOutcome::AlreadyRecorded => return Ok(instance.status),
2399 ManagementAuditOutcome::Recorded => {}
2400 }
2401 self.reserve_action_rate_budget(tenant).await?;
2404 expectation.verify(instance.version, instance.control_version)?;
2407 if !instance.control_state.allows_automatic_actions() {
2408 return Err(management_precondition(
2409 "manual_close requires ACTIVE control state; resume first",
2410 ));
2411 }
2412 if instance.status != SagaStatus::ManualIntervention {
2413 return Err(management_precondition(
2414 "manual_close requires MANUAL_INTERVENTION",
2415 ));
2416 }
2417 self.advance(
2418 &instance,
2419 SagaStatus::ManuallyClosed,
2420 instance.direction,
2421 None,
2422 TriggerKind::Admin,
2423 operation_id,
2424 None,
2425 None,
2426 )
2427 .await?;
2428 self.backend
2431 .store()
2432 .cancel_scope_timers(saga_id, TimerScope::Instance, None)
2433 .await?;
2434 Ok(SagaStatus::ManuallyClosed)
2435 })
2436 .await?;
2437 tracing::info!(
2438 saga_id = saga_id.as_str(),
2439 tenant = tenant.as_str(),
2440 actor = management.actor(),
2441 operation_id,
2442 saga_status = status.as_str(),
2443 "Saga 人工关闭操作已提交"
2444 );
2445 Ok(status)
2446 }
2447
2448 async fn project_deferred_result(
2463 &self,
2464 context: &ResultContext<'_>,
2465 ) -> anyhow::Result<SagaStatus> {
2466 let saga_id = &context.identity.saga_id;
2467 let step = &context.identity.step;
2468 let row = self
2469 .backend
2470 .store()
2471 .load_steps(saga_id)
2472 .await?
2473 .into_iter()
2474 .find(|row| row.step == *step)
2475 .ok_or_else(|| anyhow::anyhow!("step journal row is missing"))?;
2476 let current_attempt = match context.identity.phase {
2477 StepPhase::Execute => row.execute_attempt,
2478 StepPhase::Cancel => row.cancel_attempt,
2479 StepPhase::Compensate => row.compensate_attempt,
2480 StepPhase::Resolve => row.resolve_attempt,
2481 };
2482 if current_attempt != Some(context.identity.attempt.get()) {
2483 return Ok(context.instance.status);
2484 }
2485
2486 let mut patch = StepJournalPatch {
2487 last_error_code: context.reason_code,
2488 ..Default::default()
2489 };
2490 let timeout_kind = match context.identity.phase {
2491 StepPhase::Execute => {
2492 patch.forward_status = Some(match context.status {
2493 StepAttemptStatus::Succeeded => StepForwardStatus::Succeeded,
2494 StepAttemptStatus::Rejected => StepForwardStatus::Rejected,
2495 StepAttemptStatus::Unknown => StepForwardStatus::Unknown,
2496 StepAttemptStatus::Halted => StepForwardStatus::Halted,
2497 _ => anyhow::bail!("invalid deferred execute status"),
2498 });
2499 patch.mark_finished = !matches!(context.status, StepAttemptStatus::Unknown);
2500 KIND_STEP_TIMEOUT
2501 }
2502 StepPhase::Cancel => {
2503 match context.status {
2504 StepAttemptStatus::CancelConfirmed => {
2505 patch.forward_status = Some(StepForwardStatus::Cancelled);
2506 patch.cancel_status = Some(StepCancelStatus::Confirmed);
2507 patch.mark_finished = true;
2508 }
2509 StepAttemptStatus::AlreadyTerminal => {
2510 patch.forward_status = context.terminal;
2511 patch.cancel_status = Some(StepCancelStatus::AlreadyTerminal);
2512 patch.mark_finished = true;
2513 }
2514 StepAttemptStatus::ResolutionPending => {
2515 patch.cancel_status = Some(StepCancelStatus::ResolutionPending);
2516 }
2517 _ => anyhow::bail!("invalid deferred cancel status"),
2518 }
2519 KIND_CANCEL_TIMEOUT
2520 }
2521 StepPhase::Compensate => {
2522 patch.compensation_status = Some(match context.status {
2523 StepAttemptStatus::Succeeded => StepCompensationStatus::Succeeded,
2524 StepAttemptStatus::Unknown => StepCompensationStatus::Unknown,
2525 StepAttemptStatus::Halted => StepCompensationStatus::Halted,
2526 _ => anyhow::bail!("invalid deferred compensation status"),
2527 });
2528 KIND_COMPENSATE_TIMEOUT
2529 }
2530 StepPhase::Resolve => {
2531 patch.resolution_status = Some(match context.status {
2532 StepAttemptStatus::Succeeded => StepResolutionStatus::Succeeded,
2533 StepAttemptStatus::Rejected => StepResolutionStatus::Rejected,
2534 StepAttemptStatus::Unknown => StepResolutionStatus::Pending,
2535 StepAttemptStatus::Halted => StepResolutionStatus::Halted,
2536 _ => anyhow::bail!("invalid deferred resolution status"),
2537 });
2538 if context.instance.direction == Direction::Forward {
2539 patch.forward_status = match context.status {
2540 StepAttemptStatus::Succeeded => Some(StepForwardStatus::Succeeded),
2541 StepAttemptStatus::Rejected => Some(StepForwardStatus::Rejected),
2542 StepAttemptStatus::Halted => Some(StepForwardStatus::Halted),
2543 StepAttemptStatus::Unknown => Some(StepForwardStatus::Unknown),
2544 _ => None,
2545 };
2546 } else {
2547 patch.compensation_status = match context.status {
2548 StepAttemptStatus::Succeeded => Some(StepCompensationStatus::Succeeded),
2549 StepAttemptStatus::Rejected => Some(StepCompensationStatus::Pending),
2550 StepAttemptStatus::Halted => Some(StepCompensationStatus::Halted),
2551 StepAttemptStatus::Unknown => Some(StepCompensationStatus::Unknown),
2552 _ => None,
2553 };
2554 }
2555 KIND_RESOLVE_TIMEOUT
2556 }
2557 };
2558
2559 let resolve_after_compensation_success = context.identity.phase == StepPhase::Resolve
2563 && context.instance.direction == Direction::Compensating
2564 && context.status == StepAttemptStatus::Rejected
2565 && row.compensation_status == StepCompensationStatus::Succeeded
2566 && patch.compensation_status == Some(StepCompensationStatus::Pending);
2567 if resolve_after_compensation_success
2568 || merge_deferred_projection(&row, &mut patch).is_err()
2569 {
2570 let existing = conflicting_projection_status(&row, context)
2574 .ok_or_else(|| anyhow::anyhow!("conflicting projection has no settled fact"))?;
2575 self.backend
2576 .store()
2577 .record_attempt_conflict(&AttemptConflictFact {
2578 saga_id,
2579 step,
2580 phase: context.identity.phase,
2581 attempt: context.identity.attempt,
2582 existing_status: existing,
2583 incoming_status: context.status,
2584 incoming_event_id: context.event_id,
2585 conflict_kind: SagaConflictKind::CrossPhaseFact,
2586 })
2587 .await?;
2588 if !context.instance.status.is_terminal()
2589 && context.instance.status != SagaStatus::ManualIntervention
2590 {
2591 return self
2592 .escalate_manual(
2593 context.instance,
2594 TriggerKind::Event,
2595 context.event_id,
2596 "cross_phase_fact_conflict",
2597 )
2598 .await;
2599 }
2600 return Ok(context.instance.status);
2601 }
2602 self.backend
2603 .store()
2604 .cancel_scope_timers(saga_id, TimerScope::Step(step), Some(timeout_kind))
2605 .await?;
2606 self.patch_step(saga_id, step, patch).await?;
2607 Ok(context.instance.status)
2608 }
2609
2610 async fn apply_monotonic_projection(
2622 &self,
2623 context: &ResultContext<'_>,
2624 mut patch: StepJournalPatch<'_>,
2625 ) -> anyhow::Result<Option<SagaStatus>> {
2626 let saga_id = &context.identity.saga_id;
2627 let step = &context.identity.step;
2628 let row = self
2629 .backend
2630 .store()
2631 .load_steps(saga_id)
2632 .await?
2633 .into_iter()
2634 .find(|row| row.step == *step)
2635 .ok_or_else(|| anyhow::anyhow!("step journal row is missing"))?;
2636
2637 let resolve_after_compensation_success = context.identity.phase == StepPhase::Resolve
2640 && context.instance.direction == Direction::Compensating
2641 && context.status == StepAttemptStatus::Rejected
2642 && row.compensation_status == StepCompensationStatus::Succeeded
2643 && patch.compensation_status == Some(StepCompensationStatus::Pending);
2644 if resolve_after_compensation_success
2645 || merge_deferred_projection(&row, &mut patch).is_err()
2646 {
2647 let existing = conflicting_projection_status(&row, context)
2648 .ok_or_else(|| anyhow::anyhow!("conflicting projection has no settled fact"))?;
2649 self.backend
2650 .store()
2651 .record_attempt_conflict(&AttemptConflictFact {
2652 saga_id,
2653 step,
2654 phase: context.identity.phase,
2655 attempt: context.identity.attempt,
2656 existing_status: existing,
2657 incoming_status: context.status,
2658 incoming_event_id: context.event_id,
2659 conflict_kind: SagaConflictKind::CrossPhaseFact,
2660 })
2661 .await?;
2662 if !context.instance.status.is_terminal()
2663 && context.instance.status != SagaStatus::ManualIntervention
2664 {
2665 let status = self
2666 .escalate_manual(
2667 context.instance,
2668 TriggerKind::Event,
2669 context.event_id,
2670 "cross_phase_fact_conflict",
2671 )
2672 .await?;
2673 return Ok(Some(status));
2674 }
2675 return Ok(Some(context.instance.status));
2676 }
2677 self.patch_step(saga_id, step, patch).await?;
2678 Ok(None)
2679 }
2680
2681 async fn on_execute_result(&self, context: &ResultContext<'_>) -> anyhow::Result<SagaStatus> {
2688 if context.instance.status != SagaStatus::Running
2691 || context.instance.current_step.as_ref() != Some(&context.identity.step)
2692 {
2693 return self.project_deferred_result(context).await;
2694 }
2695 let saga_id = &context.identity.saga_id;
2696 let step = &context.identity.step;
2697 match context.status {
2698 StepAttemptStatus::Succeeded => {
2699 self.patch_step(
2700 saga_id,
2701 step,
2702 StepJournalPatch {
2703 forward_status: Some(StepForwardStatus::Succeeded),
2704 mark_finished: true,
2705 ..Default::default()
2706 },
2707 )
2708 .await?;
2709 self.backend
2710 .store()
2711 .cancel_scope_timers(saga_id, TimerScope::Step(step), Some(KIND_STEP_TIMEOUT))
2712 .await?;
2713 self.proceed_forward(context).await
2714 }
2715 StepAttemptStatus::Rejected => {
2716 self.patch_step(
2717 saga_id,
2718 step,
2719 StepJournalPatch {
2720 forward_status: Some(StepForwardStatus::Rejected),
2721 last_error_code: context.reason_code,
2722 mark_finished: true,
2723 ..Default::default()
2724 },
2725 )
2726 .await?;
2727 self.backend
2728 .store()
2729 .cancel_scope_timers(saga_id, TimerScope::Step(step), Some(KIND_STEP_TIMEOUT))
2730 .await?;
2731 self.enter_compensation(context).await
2733 }
2734 StepAttemptStatus::Unknown => {
2735 self.enter_waiting_resolution(context, Direction::Forward)
2736 .await
2737 }
2738 StepAttemptStatus::Halted => {
2739 self.patch_step(
2740 saga_id,
2741 step,
2742 StepJournalPatch {
2743 forward_status: Some(StepForwardStatus::Halted),
2744 last_error_code: context.reason_code,
2745 mark_finished: true,
2746 ..Default::default()
2747 },
2748 )
2749 .await?;
2750 self.backend
2751 .store()
2752 .cancel_scope_timers(saga_id, TimerScope::Step(step), None)
2753 .await?;
2754 self.escalate_manual(
2755 context.instance,
2756 TriggerKind::Event,
2757 context.event_id,
2758 "step_halted",
2759 )
2760 .await
2761 }
2762 _ => anyhow::bail!("cancel-vocabulary status on an execute result"),
2763 }
2764 }
2765
2766 async fn on_cancel_result(&self, context: &ResultContext<'_>) -> anyhow::Result<SagaStatus> {
2773 if context.instance.status != SagaStatus::Cancelling
2774 || context.instance.current_step.as_ref() != Some(&context.identity.step)
2775 {
2776 return self.project_deferred_result(context).await;
2777 }
2778 let saga_id = &context.identity.saga_id;
2779 let step = &context.identity.step;
2780 match context.status {
2781 StepAttemptStatus::CancelConfirmed => {
2782 if let Some(status) = self
2784 .apply_monotonic_projection(
2785 context,
2786 StepJournalPatch {
2787 forward_status: Some(StepForwardStatus::Cancelled),
2788 cancel_status: Some(StepCancelStatus::Confirmed),
2789 mark_finished: true,
2790 ..Default::default()
2791 },
2792 )
2793 .await?
2794 {
2795 return Ok(status);
2796 }
2797 self.backend
2798 .store()
2799 .cancel_scope_timers(saga_id, TimerScope::Step(step), Some(KIND_CANCEL_TIMEOUT))
2800 .await?;
2801 self.enter_compensation(context).await
2802 }
2803 StepAttemptStatus::AlreadyTerminal => {
2804 let terminal = context.terminal.ok_or_else(|| {
2805 anyhow::anyhow!("ALREADY_TERMINAL requires a terminal status")
2806 })?;
2807 if let Some(status) = self
2808 .apply_monotonic_projection(
2809 context,
2810 StepJournalPatch {
2811 forward_status: Some(terminal),
2812 cancel_status: Some(StepCancelStatus::AlreadyTerminal),
2813 mark_finished: true,
2814 ..Default::default()
2815 },
2816 )
2817 .await?
2818 {
2819 return Ok(status);
2820 }
2821 self.backend
2822 .store()
2823 .cancel_scope_timers(saga_id, TimerScope::Step(step), Some(KIND_CANCEL_TIMEOUT))
2824 .await?;
2825 match terminal {
2826 StepForwardStatus::Succeeded => match context.step_def.timeout_policy() {
2827 TimeoutPolicy::AcceptLateSuccess => self.proceed_forward(context).await,
2829 TimeoutPolicy::CompensateLateSuccess => {
2831 self.enter_compensation(context).await
2832 }
2833 },
2834 StepForwardStatus::Rejected => self.enter_compensation(context).await,
2835 StepForwardStatus::Halted => {
2836 self.escalate_manual(
2837 context.instance,
2838 TriggerKind::Event,
2839 context.event_id,
2840 "step_halted",
2841 )
2842 .await
2843 }
2844 _ => anyhow::bail!("ALREADY_TERMINAL carried a non-terminal forward status"),
2845 }
2846 }
2847 StepAttemptStatus::ResolutionPending => {
2848 if let Some(status) = self
2850 .apply_monotonic_projection(
2851 context,
2852 StepJournalPatch {
2853 cancel_status: Some(StepCancelStatus::ResolutionPending),
2854 ..Default::default()
2855 },
2856 )
2857 .await?
2858 {
2859 return Ok(status);
2860 }
2861 self.backend
2862 .store()
2863 .cancel_scope_timers(saga_id, TimerScope::Step(step), Some(KIND_CANCEL_TIMEOUT))
2864 .await?;
2865 self.enter_waiting_resolution(context, Direction::Forward)
2866 .await
2867 }
2868 _ => anyhow::bail!("non-cancel status on a cancel result"),
2869 }
2870 }
2871
2872 async fn on_compensate_result(
2879 &self,
2880 context: &ResultContext<'_>,
2881 ) -> anyhow::Result<SagaStatus> {
2882 if context.instance.status != SagaStatus::Compensating
2883 || context.instance.current_step.as_ref() != Some(&context.identity.step)
2884 {
2885 return self.project_deferred_result(context).await;
2886 }
2887 let saga_id = &context.identity.saga_id;
2888 let step = &context.identity.step;
2889 self.backend
2890 .store()
2891 .cancel_scope_timers(
2892 saga_id,
2893 TimerScope::Step(step),
2894 Some(KIND_COMPENSATE_TIMEOUT),
2895 )
2896 .await?;
2897 match context.status {
2898 StepAttemptStatus::Succeeded => {
2899 self.patch_step(
2900 saga_id,
2901 step,
2902 StepJournalPatch {
2903 compensation_status: Some(StepCompensationStatus::Succeeded),
2904 ..Default::default()
2905 },
2906 )
2907 .await?;
2908 self.continue_compensation(context).await
2909 }
2910 StepAttemptStatus::Unknown => {
2911 self.patch_step(
2912 saga_id,
2913 step,
2914 StepJournalPatch {
2915 compensation_status: Some(StepCompensationStatus::Unknown),
2916 ..Default::default()
2917 },
2918 )
2919 .await?;
2920 self.enter_waiting_resolution(context, Direction::Compensating)
2921 .await
2922 }
2923 StepAttemptStatus::Halted => {
2924 self.patch_step(
2925 saga_id,
2926 step,
2927 StepJournalPatch {
2928 compensation_status: Some(StepCompensationStatus::Halted),
2929 last_error_code: context.reason_code,
2930 ..Default::default()
2931 },
2932 )
2933 .await?;
2934 self.escalate_manual(
2935 context.instance,
2936 TriggerKind::Event,
2937 context.event_id,
2938 "compensation_halted",
2939 )
2940 .await
2941 }
2942 _ => anyhow::bail!("invalid status on a compensate result"),
2943 }
2944 }
2945
2946 async fn settle_resolution_from_committed_success(
2958 &self,
2959 context: &ResultContext<'_>,
2960 ) -> anyhow::Result<Option<SagaStatus>> {
2961 let saga_id = &context.identity.saga_id;
2962 let step = &context.identity.step;
2963 let row = self
2964 .backend
2965 .store()
2966 .load_steps(saga_id)
2967 .await?
2968 .into_iter()
2969 .find(|row| row.step == *step)
2970 .ok_or_else(|| anyhow::anyhow!("step journal row is missing"))?;
2971 let committed_success = match context.instance.direction {
2972 Direction::Forward => row.forward_status == StepForwardStatus::Succeeded,
2973 Direction::Compensating => row.compensation_status == StepCompensationStatus::Succeeded,
2974 };
2975 if !committed_success {
2976 return Ok(None);
2977 }
2978
2979 let mut patch = StepJournalPatch {
2980 resolution_status: Some(StepResolutionStatus::Succeeded),
2981 ..Default::default()
2982 };
2983 match context.instance.direction {
2984 Direction::Forward => {
2985 patch.forward_status = Some(StepForwardStatus::Succeeded);
2986 patch.mark_finished = true;
2987 }
2988 Direction::Compensating => {
2989 patch.compensation_status = Some(StepCompensationStatus::Succeeded);
2990 }
2991 }
2992 if let Some(status) = self.apply_monotonic_projection(context, patch).await? {
2993 return Ok(Some(status));
2994 }
2995
2996 self.cancel_resolution_timers(saga_id, step).await?;
2999 let status = match context.instance.direction {
3000 Direction::Forward => match context.step_def.timeout_policy() {
3001 TimeoutPolicy::AcceptLateSuccess => self.proceed_forward(context).await?,
3002 TimeoutPolicy::CompensateLateSuccess => self.enter_compensation(context).await?,
3003 },
3004 Direction::Compensating => self.continue_compensation(context).await?,
3005 };
3006 Ok(Some(status))
3007 }
3008
3009 async fn on_resolve_result(&self, context: &ResultContext<'_>) -> anyhow::Result<SagaStatus> {
3016 if context.instance.status != SagaStatus::WaitingResolution
3017 || context.instance.current_step.as_ref() != Some(&context.identity.step)
3018 {
3019 return self.project_deferred_result(context).await;
3020 }
3021 if let Some(status) = self
3022 .settle_resolution_from_committed_success(context)
3023 .await?
3024 {
3025 return Ok(status);
3026 }
3027 let saga_id = &context.identity.saga_id;
3028 let step = &context.identity.step;
3029 match context.instance.direction {
3030 Direction::Forward => match context.status {
3031 StepAttemptStatus::Succeeded => {
3032 if let Some(status) = self
3033 .apply_monotonic_projection(
3034 context,
3035 StepJournalPatch {
3036 forward_status: Some(StepForwardStatus::Succeeded),
3037 resolution_status: Some(StepResolutionStatus::Succeeded),
3038 mark_finished: true,
3039 ..Default::default()
3040 },
3041 )
3042 .await?
3043 {
3044 return Ok(status);
3045 }
3046 self.cancel_resolution_timers(saga_id, step).await?;
3047 self.proceed_forward(context).await
3048 }
3049 StepAttemptStatus::Rejected => {
3050 if let Some(status) = self
3053 .apply_monotonic_projection(
3054 context,
3055 StepJournalPatch {
3056 forward_status: Some(StepForwardStatus::Rejected),
3057 resolution_status: Some(StepResolutionStatus::Rejected),
3058 last_error_code: context.reason_code,
3059 mark_finished: true,
3060 ..Default::default()
3061 },
3062 )
3063 .await?
3064 {
3065 return Ok(status);
3066 }
3067 self.cancel_resolution_timers(saga_id, step).await?;
3068 self.enter_compensation(context).await
3069 }
3070 StepAttemptStatus::Unknown => self.reissue_resolve(context).await,
3071 StepAttemptStatus::Halted => {
3072 if let Some(status) = self
3073 .apply_monotonic_projection(
3074 context,
3075 StepJournalPatch {
3076 resolution_status: Some(StepResolutionStatus::Halted),
3077 last_error_code: context.reason_code,
3078 ..Default::default()
3079 },
3080 )
3081 .await?
3082 {
3083 return Ok(status);
3084 }
3085 self.escalate_manual(
3086 context.instance,
3087 TriggerKind::Event,
3088 context.event_id,
3089 "resolution_halted",
3090 )
3091 .await
3092 }
3093 _ => anyhow::bail!("invalid status on a resolve result"),
3094 },
3095 Direction::Compensating => match context.status {
3096 StepAttemptStatus::Succeeded => {
3097 if let Some(status) = self
3099 .apply_monotonic_projection(
3100 context,
3101 StepJournalPatch {
3102 compensation_status: Some(StepCompensationStatus::Succeeded),
3103 resolution_status: Some(StepResolutionStatus::Succeeded),
3104 ..Default::default()
3105 },
3106 )
3107 .await?
3108 {
3109 return Ok(status);
3110 }
3111 self.cancel_resolution_timers(saga_id, step).await?;
3112 self.continue_compensation(context).await
3113 }
3114 StepAttemptStatus::Rejected => {
3115 if let Some(status) = self
3117 .apply_monotonic_projection(
3118 context,
3119 StepJournalPatch {
3120 compensation_status: Some(StepCompensationStatus::Pending),
3121 resolution_status: Some(StepResolutionStatus::Rejected),
3122 ..Default::default()
3123 },
3124 )
3125 .await?
3126 {
3127 return Ok(status);
3128 }
3129 self.cancel_resolution_timers(saga_id, step).await?;
3130 let attempt = self
3131 .next_attempt(saga_id, step, StepPhase::Compensate)
3132 .await?;
3133 if attempt.get() > self.config.compensate_max_attempts {
3134 return self
3135 .escalate_manual(
3136 context.instance,
3137 TriggerKind::Event,
3138 context.event_id,
3139 "compensation_attempts_exhausted",
3140 )
3141 .await;
3142 }
3143 let version = self
3144 .advance(
3145 context.instance,
3146 SagaStatus::Compensating,
3147 Direction::Compensating,
3148 Some(step),
3149 TriggerKind::Event,
3150 context.event_id,
3151 None,
3152 None,
3153 )
3154 .await?;
3155 self.issue_command(
3156 context.instance,
3157 context.step_def,
3158 StepPhase::Compensate,
3159 attempt,
3160 None,
3161 None,
3162 context.now_ms,
3163 version,
3164 )
3165 .await?;
3166 Ok(SagaStatus::Compensating)
3167 }
3168 StepAttemptStatus::Unknown => self.reissue_resolve(context).await,
3169 StepAttemptStatus::Halted => {
3170 if let Some(status) = self
3171 .apply_monotonic_projection(
3172 context,
3173 StepJournalPatch {
3174 resolution_status: Some(StepResolutionStatus::Halted),
3175 last_error_code: context.reason_code,
3176 ..Default::default()
3177 },
3178 )
3179 .await?
3180 {
3181 return Ok(status);
3182 }
3183 self.escalate_manual(
3184 context.instance,
3185 TriggerKind::Event,
3186 context.event_id,
3187 "resolution_halted",
3188 )
3189 .await
3190 }
3191 _ => anyhow::bail!("invalid status on a resolve result"),
3192 },
3193 }
3194 }
3195
3196 async fn on_step_timeout(
3209 &self,
3210 instance: &SagaInstanceRow,
3211 definition: &WorkflowDefinition,
3212 step_scope: Option<&StepName>,
3213 timer: &SagaTimerRow,
3214 now_ms: i64,
3215 ) -> anyhow::Result<Option<SagaStatus>> {
3216 let Some(step) = step_scope else {
3217 anyhow::bail!("step-timeout timer without a step scope");
3218 };
3219 if instance.status != SagaStatus::Running || instance.current_step.as_ref() != Some(step) {
3221 return Ok(None);
3222 }
3223 let step_def = definition
3224 .step(step)
3225 .ok_or_else(|| anyhow::anyhow!("timer step absent from definition"))?;
3226 if step_def.cancel_mode() == CancelMode::ResolveOnly {
3227 self.patch_step(
3230 &instance.saga_id,
3231 step,
3232 StepJournalPatch {
3233 forward_status: Some(StepForwardStatus::Unknown),
3234 resolution_status: Some(StepResolutionStatus::Pending),
3235 ..Default::default()
3236 },
3237 )
3238 .await?;
3239 let version = self
3240 .advance(
3241 instance,
3242 SagaStatus::WaitingResolution,
3243 Direction::Forward,
3244 Some(step),
3245 TriggerKind::Timer,
3246 &timer.timer_id,
3247 None,
3248 None,
3249 )
3250 .await?;
3251 let resolution = step_def.resolution();
3252 self.schedule_timer(
3253 &instance.saga_id,
3254 TimerScope::Step(step),
3255 resolution_budget_kind(Direction::Forward),
3256 timer.attempt,
3257 checked_deadline(now_ms, duration_millis(resolution.budget())?)?,
3258 version,
3259 )
3260 .await?;
3261 if resolution.mode() == Some(ResolutionMode::Poll) {
3262 let attempt = self
3263 .next_attempt(&instance.saga_id, step, StepPhase::Resolve)
3264 .await?;
3265 self.issue_command(
3266 instance,
3267 step_def,
3268 StepPhase::Resolve,
3269 attempt,
3270 None,
3271 None,
3272 now_ms,
3273 version,
3274 )
3275 .await?;
3276 }
3277 return Ok(Some(SagaStatus::WaitingResolution));
3278 }
3279
3280 self.patch_step(
3283 &instance.saga_id,
3284 step,
3285 StepJournalPatch {
3286 forward_status: Some(StepForwardStatus::TimedOut),
3287 ..Default::default()
3288 },
3289 )
3290 .await?;
3291 let version = self
3292 .advance(
3293 instance,
3294 SagaStatus::Cancelling,
3295 Direction::Forward,
3296 Some(step),
3297 TriggerKind::Timer,
3298 &timer.timer_id,
3299 None,
3300 None,
3301 )
3302 .await?;
3303 let attempt = self
3304 .next_attempt(&instance.saga_id, step, StepPhase::Cancel)
3305 .await?;
3306 self.issue_command(
3307 instance,
3308 step_def,
3309 StepPhase::Cancel,
3310 attempt,
3311 None,
3312 None,
3313 now_ms,
3314 version,
3315 )
3316 .await?;
3317 Ok(Some(SagaStatus::Cancelling))
3318 }
3319
3320 #[allow(clippy::too_many_arguments)]
3335 async fn on_phase_retry_timeout(
3336 &self,
3337 instance: &SagaInstanceRow,
3338 definition: &WorkflowDefinition,
3339 step_scope: Option<&StepName>,
3340 timer: &SagaTimerRow,
3341 phase: StepPhase,
3342 required_status: SagaStatus,
3343 max_attempts: u32,
3344 exhausted_code: &str,
3345 now_ms: i64,
3346 ) -> anyhow::Result<Option<SagaStatus>> {
3347 let Some(step) = step_scope else {
3348 anyhow::bail!("phase retry timer without a step scope");
3349 };
3350 if instance.status != required_status || instance.current_step.as_ref() != Some(step) {
3351 return Ok(None);
3352 }
3353 let step_def = definition
3354 .step(step)
3355 .ok_or_else(|| anyhow::anyhow!("timer step absent from definition"))?;
3356 let exhausted = if phase == StepPhase::Resolve {
3357 self.backend
3360 .store()
3361 .count_resolution_attempts(&instance.saga_id, step, instance.direction)
3362 .await?
3363 >= max_attempts
3364 } else {
3365 self.next_attempt(&instance.saga_id, step, phase)
3366 .await?
3367 .get()
3368 > max_attempts
3369 };
3370 if exhausted {
3373 return Ok(Some(
3374 self.escalate_manual(
3375 instance,
3376 TriggerKind::Timer,
3377 &timer.timer_id,
3378 exhausted_code,
3379 )
3380 .await?,
3381 ));
3382 }
3383 let attempt = self.next_attempt(&instance.saga_id, step, phase).await?;
3385 let version = self
3386 .advance(
3387 instance,
3388 required_status,
3389 instance.direction,
3390 Some(step),
3391 TriggerKind::Timer,
3392 &timer.timer_id,
3393 None,
3394 None,
3395 )
3396 .await?;
3397 self.issue_command(
3398 instance, step_def, phase, attempt, None, None, now_ms, version,
3399 )
3400 .await?;
3401 Ok(Some(required_status))
3402 }
3403
3404 async fn on_instance_deadline(
3414 &self,
3415 instance: &SagaInstanceRow,
3416 definition: &WorkflowDefinition,
3417 timer: &SagaTimerRow,
3418 now_ms: i64,
3419 ) -> anyhow::Result<Option<SagaStatus>> {
3420 match instance.status {
3421 SagaStatus::Running => {
3423 let Some(step) = instance.current_step.clone() else {
3424 return Ok(None);
3425 };
3426 self.on_step_timeout(instance, definition, Some(&step), timer, now_ms)
3427 .await
3428 }
3429 SagaStatus::Cancelling | SagaStatus::WaitingResolution | SagaStatus::Compensating => {
3432 Ok(Some(
3433 self.escalate_manual(
3434 instance,
3435 TriggerKind::Timer,
3436 &timer.timer_id,
3437 "deadline_exceeded",
3438 )
3439 .await?,
3440 ))
3441 }
3442 _ => Ok(None),
3443 }
3444 }
3445
3446 async fn proceed_forward(&self, context: &ResultContext<'_>) -> anyhow::Result<SagaStatus> {
3458 let saga_id = &context.identity.saga_id;
3459 let steps = self.backend.store().load_steps(saga_id).await?;
3460 match next_pending_forward(context.definition, &steps)? {
3461 Some(next) => {
3462 let version = self
3463 .advance(
3464 context.instance,
3465 SagaStatus::Running,
3466 Direction::Forward,
3467 Some(&next),
3468 TriggerKind::Event,
3469 context.event_id,
3470 None,
3471 None,
3472 )
3473 .await?;
3474 let next_def = context
3475 .definition
3476 .step(&next)
3477 .ok_or_else(|| anyhow::anyhow!("next step absent from definition"))?;
3478 let attempt = self
3479 .next_attempt(saga_id, &next, StepPhase::Execute)
3480 .await?;
3481 self.issue_command(
3483 context.instance,
3484 next_def,
3485 StepPhase::Execute,
3486 attempt,
3487 None,
3488 None,
3489 context.now_ms,
3490 version,
3491 )
3492 .await?;
3493 Ok(SagaStatus::Running)
3494 }
3495 None => {
3496 self.advance(
3497 context.instance,
3498 SagaStatus::Completed,
3499 Direction::Forward,
3500 None,
3501 TriggerKind::Event,
3502 context.event_id,
3503 None,
3504 None,
3505 )
3506 .await?;
3507 self.backend
3509 .store()
3510 .cancel_scope_timers(saga_id, TimerScope::Instance, None)
3511 .await?;
3512 Ok(SagaStatus::Completed)
3513 }
3514 }
3515 }
3516
3517 async fn enter_compensation(&self, context: &ResultContext<'_>) -> anyhow::Result<SagaStatus> {
3528 let saga_id = &context.identity.saga_id;
3529 let steps = self.backend.store().load_steps(saga_id).await?;
3530 let journal: Vec<StepJournalEntry> = steps
3531 .iter()
3532 .map(|row| StepJournalEntry::new(row.step.clone(), row.forward_status))
3533 .collect();
3534 let plan = match freeze_compensation_plan(context.definition, &journal) {
3537 Ok(plan) => plan,
3538 Err(violation) => {
3539 return self
3540 .escalate_manual(
3541 context.instance,
3542 TriggerKind::Event,
3543 context.event_id,
3544 violation.code(),
3545 )
3546 .await;
3547 }
3548 };
3549 let first = plan.entries().first().map(|entry| entry.step().clone());
3550 let version = self
3551 .advance(
3552 context.instance,
3553 SagaStatus::Compensating,
3554 Direction::Compensating,
3555 first.as_ref(),
3556 TriggerKind::Event,
3557 context.event_id,
3558 None,
3559 Some(plan.version()),
3560 )
3561 .await?;
3562 self.backend
3563 .store()
3564 .mark_compensation_plan(saga_id, &plan)
3565 .await?;
3566 match first {
3567 Some(step) => {
3568 let step_def = context
3569 .definition
3570 .step(&step)
3571 .ok_or_else(|| anyhow::anyhow!("plan step absent from definition"))?;
3572 let attempt = self
3573 .next_attempt(saga_id, &step, StepPhase::Compensate)
3574 .await?;
3575 self.issue_command(
3576 context.instance,
3577 step_def,
3578 StepPhase::Compensate,
3579 attempt,
3580 None,
3581 None,
3582 context.now_ms,
3583 version,
3584 )
3585 .await?;
3586 Ok(SagaStatus::Compensating)
3587 }
3588 None => {
3589 self.converge_compensated(
3591 context.instance,
3592 version,
3593 TriggerKind::Event,
3594 context.event_id,
3595 )
3596 .await?;
3597 Ok(SagaStatus::Compensated)
3598 }
3599 }
3600 }
3601
3602 async fn continue_compensation(
3615 &self,
3616 context: &ResultContext<'_>,
3617 ) -> anyhow::Result<SagaStatus> {
3618 let saga_id = &context.identity.saga_id;
3619 let steps = self.backend.store().load_steps(saga_id).await?;
3620 let next =
3621 match next_pending_compensation(context.instance, context.definition, &steps, false) {
3622 Ok(next) => next,
3623 Err(_) => {
3624 return self
3627 .escalate_manual(
3628 context.instance,
3629 TriggerKind::Event,
3630 context.event_id,
3631 "compensation_plan_drift",
3632 )
3633 .await;
3634 }
3635 };
3636 let version = self
3637 .advance(
3638 context.instance,
3639 SagaStatus::Compensating,
3640 Direction::Compensating,
3641 next.as_ref(),
3642 TriggerKind::Event,
3643 context.event_id,
3644 None,
3645 None,
3646 )
3647 .await?;
3648 match next {
3649 Some(step) => {
3650 let step_def = context
3651 .definition
3652 .step(&step)
3653 .ok_or_else(|| anyhow::anyhow!("plan step absent from definition"))?;
3654 let attempt = self
3655 .next_attempt(saga_id, &step, StepPhase::Compensate)
3656 .await?;
3657 self.issue_command(
3658 context.instance,
3659 step_def,
3660 StepPhase::Compensate,
3661 attempt,
3662 None,
3663 None,
3664 context.now_ms,
3665 version,
3666 )
3667 .await?;
3668 Ok(SagaStatus::Compensating)
3669 }
3670 None => {
3671 self.converge_compensated_at(
3672 saga_id,
3673 version,
3674 TriggerKind::Event,
3675 context.event_id,
3676 )
3677 .await?;
3678 Ok(SagaStatus::Compensated)
3679 }
3680 }
3681 }
3682
3683 async fn enter_waiting_resolution(
3691 &self,
3692 context: &ResultContext<'_>,
3693 direction: Direction,
3694 ) -> anyhow::Result<SagaStatus> {
3695 let saga_id = &context.identity.saga_id;
3696 let step = &context.identity.step;
3697 let resolution = context.step_def.resolution();
3698 if !resolution.allow_unknown() {
3701 if context.identity.phase == StepPhase::Execute {
3702 self.patch_step(
3703 saga_id,
3704 step,
3705 StepJournalPatch {
3706 forward_status: Some(StepForwardStatus::Unknown),
3707 ..Default::default()
3708 },
3709 )
3710 .await?;
3711 }
3712 return self
3715 .escalate_manual(
3716 context.instance,
3717 TriggerKind::Event,
3718 context.event_id,
3719 "unknown_not_allowed",
3720 )
3721 .await;
3722 }
3723 if context.identity.phase == StepPhase::Execute {
3724 self.patch_step(
3725 saga_id,
3726 step,
3727 StepJournalPatch {
3728 forward_status: Some(StepForwardStatus::Unknown),
3729 resolution_status: Some(StepResolutionStatus::Pending),
3730 ..Default::default()
3731 },
3732 )
3733 .await?;
3734 } else {
3735 self.patch_step(
3736 saga_id,
3737 step,
3738 StepJournalPatch {
3739 resolution_status: Some(StepResolutionStatus::Pending),
3740 ..Default::default()
3741 },
3742 )
3743 .await?;
3744 }
3745 self.backend
3746 .store()
3747 .cancel_scope_timers(saga_id, TimerScope::Step(step), Some(KIND_STEP_TIMEOUT))
3748 .await?;
3749 let version = self
3750 .advance(
3751 context.instance,
3752 SagaStatus::WaitingResolution,
3753 direction,
3754 Some(step),
3755 TriggerKind::Event,
3756 context.event_id,
3757 None,
3758 None,
3759 )
3760 .await?;
3761 self.schedule_timer(
3763 saga_id,
3764 TimerScope::Step(step),
3765 resolution_budget_kind(direction),
3766 context.identity.attempt,
3767 checked_deadline(context.now_ms, duration_millis(resolution.budget())?)?,
3768 version,
3769 )
3770 .await?;
3771 if resolution.mode() == Some(ResolutionMode::Poll) {
3772 let attempt = self.next_attempt(saga_id, step, StepPhase::Resolve).await?;
3773 self.issue_command(
3774 context.instance,
3775 context.step_def,
3776 StepPhase::Resolve,
3777 attempt,
3778 None,
3779 None,
3780 context.now_ms,
3781 version,
3782 )
3783 .await?;
3784 }
3785 Ok(SagaStatus::WaitingResolution)
3786 }
3787
3788 async fn reissue_resolve(&self, context: &ResultContext<'_>) -> anyhow::Result<SagaStatus> {
3795 let saga_id = &context.identity.saga_id;
3796 let step = &context.identity.step;
3797 let used = self
3798 .backend
3799 .store()
3800 .count_resolution_attempts(saga_id, step, context.instance.direction)
3801 .await?;
3802 if used >= self.config.resolve_max_attempts {
3803 return self
3804 .escalate_manual(
3805 context.instance,
3806 TriggerKind::Event,
3807 context.event_id,
3808 "resolve_attempts_exhausted",
3809 )
3810 .await;
3811 }
3812 let attempt = self.next_attempt(saga_id, step, StepPhase::Resolve).await?;
3815 let version = self
3816 .advance(
3817 context.instance,
3818 SagaStatus::WaitingResolution,
3819 context.instance.direction,
3820 Some(step),
3821 TriggerKind::Event,
3822 context.event_id,
3823 None,
3824 None,
3825 )
3826 .await?;
3827 self.issue_command(
3828 context.instance,
3829 context.step_def,
3830 StepPhase::Resolve,
3831 attempt,
3832 None,
3833 None,
3834 context.now_ms,
3835 version,
3836 )
3837 .await?;
3838 Ok(SagaStatus::WaitingResolution)
3839 }
3840
3841 async fn escalate_manual(
3851 &self,
3852 instance: &SagaInstanceRow,
3853 trigger_kind: TriggerKind,
3854 trigger_id: &str,
3855 failure_code: &str,
3856 ) -> anyhow::Result<SagaStatus> {
3857 self.advance(
3858 instance,
3859 SagaStatus::ManualIntervention,
3860 instance.direction,
3861 instance.current_step.as_ref(),
3862 trigger_kind,
3863 trigger_id,
3864 Some(failure_code),
3865 None,
3866 )
3867 .await?;
3868 Ok(SagaStatus::ManualIntervention)
3869 }
3870
3871 async fn converge_compensated(
3880 &self,
3881 instance: &SagaInstanceRow,
3882 version: u64,
3883 trigger_kind: TriggerKind,
3884 cause_id: &str,
3885 ) -> anyhow::Result<()> {
3886 self.converge_compensated_at(&instance.saga_id, version, trigger_kind, cause_id)
3887 .await
3888 }
3889
3890 async fn converge_compensated_at(
3899 &self,
3900 saga_id: &SagaId,
3901 version: u64,
3902 trigger_kind: TriggerKind,
3903 cause_id: &str,
3904 ) -> anyhow::Result<()> {
3905 let definition_version = self
3908 .verify_compensation_completion(saga_id, version)
3909 .await?;
3910 let converge_trigger = derive_converge_trigger_id(cause_id);
3913 let outcome = self
3914 .backend
3915 .store()
3916 .advance(
3917 saga_id,
3918 version,
3919 SagaStatus::Compensating,
3920 &TransitionSpec {
3921 to_status: SagaStatus::Compensated,
3922 direction: Direction::Compensating,
3923 current_step: None,
3924 trigger_kind,
3925 trigger_id: &converge_trigger,
3926 definition_version,
3927 failure_code: None,
3928 compensation_plan_version: None,
3929 },
3930 )
3931 .await?;
3932 let _ = ensure_applied(outcome)?;
3933 self.backend
3934 .store()
3935 .cancel_scope_timers(saga_id, TimerScope::Instance, None)
3936 .await?;
3937 Ok(())
3938 }
3939
3940 async fn verify_compensation_completion(
3949 &self,
3950 saga_id: &SagaId,
3951 expected_version: u64,
3952 ) -> anyhow::Result<DefinitionVersion> {
3953 let instance = self
3954 .backend
3955 .store()
3956 .load_instance(saga_id)
3957 .await?
3958 .ok_or_else(|| anyhow::anyhow!("saga instance not found while converging"))?;
3959 if instance.status != SagaStatus::Compensating || instance.version != expected_version {
3960 anyhow::bail!("compensation completion proof used a stale instance snapshot");
3961 }
3962 let frozen_version = instance
3963 .compensation_plan_version
3964 .as_deref()
3965 .ok_or_else(|| anyhow::anyhow!("compensating instance has no frozen plan"))?;
3966 let registry = self.registry_snapshot();
3967 let definition = registry
3968 .get_for_tenant(
3969 &instance.tenant,
3970 &instance.workflow,
3971 instance.definition_version,
3972 )
3973 .ok_or_else(|| anyhow::anyhow!("instance definition is not registered"))?;
3974 let steps = self.backend.store().load_steps(saga_id).await?;
3975 let journal: Vec<StepJournalEntry> = steps
3976 .iter()
3977 .map(|row| StepJournalEntry::new(row.step.clone(), row.forward_status))
3978 .collect();
3979 let rebuilt = freeze_compensation_plan(definition, &journal)
3980 .map_err(|violation| anyhow::anyhow!("{}", violation.code()))?;
3981 if rebuilt.version() != frozen_version {
3982 anyhow::bail!("frozen compensation plan no longer matches durable forward facts");
3983 }
3984 for entry in rebuilt.entries() {
3985 let row = steps
3986 .iter()
3987 .find(|row| row.step == *entry.step())
3988 .ok_or_else(|| anyhow::anyhow!("frozen compensation member is missing"))?;
3989 if row.compensation_plan_version.as_deref() != Some(frozen_version)
3990 || row.compensation_order != Some(entry.order())
3991 || row.compensation_status != StepCompensationStatus::Succeeded
3992 {
3993 anyhow::bail!("frozen compensation member is not durably succeeded");
3994 }
3995 }
3996 let marked_count = steps
3997 .iter()
3998 .filter(|row| row.compensation_plan_version.as_deref() == Some(frozen_version))
3999 .count();
4000 if marked_count != rebuilt.entries().len() {
4001 anyhow::bail!("durable compensation plan contains unexpected members");
4002 }
4003 Ok(instance.definition_version)
4004 }
4005
4006 #[allow(clippy::too_many_arguments)]
4019 async fn advance(
4020 &self,
4021 instance: &SagaInstanceRow,
4022 to: SagaStatus,
4023 direction: Direction,
4024 current_step: Option<&StepName>,
4025 trigger_kind: TriggerKind,
4026 trigger_id: &str,
4027 failure_code: Option<&str>,
4028 plan_version: Option<&str>,
4029 ) -> anyhow::Result<u64> {
4030 self.advance_from(
4031 instance,
4032 instance.status,
4033 to,
4034 direction,
4035 current_step,
4036 trigger_kind,
4037 trigger_id,
4038 failure_code,
4039 plan_version,
4040 )
4041 .await
4042 }
4043
4044 #[allow(clippy::too_many_arguments)]
4051 async fn advance_from(
4052 &self,
4053 instance: &SagaInstanceRow,
4054 from: SagaStatus,
4055 to: SagaStatus,
4056 direction: Direction,
4057 current_step: Option<&StepName>,
4058 trigger_kind: TriggerKind,
4059 trigger_id: &str,
4060 failure_code: Option<&str>,
4061 plan_version: Option<&str>,
4062 ) -> anyhow::Result<u64> {
4063 let outcome = self
4064 .backend
4065 .store()
4066 .advance(
4067 &instance.saga_id,
4068 instance.version,
4069 from,
4070 &TransitionSpec {
4071 to_status: to,
4072 direction,
4073 current_step,
4074 trigger_kind,
4075 trigger_id,
4076 definition_version: instance.definition_version,
4077 failure_code,
4078 compensation_plan_version: plan_version,
4079 },
4080 )
4081 .await?;
4082 ensure_applied(outcome)
4083 }
4084
4085 #[allow(clippy::too_many_arguments)]
4099 async fn issue_command(
4100 &self,
4101 instance: &SagaInstanceRow,
4102 step_def: &StepDefinition,
4103 phase: StepPhase,
4104 attempt: AttemptNo,
4105 payload: Option<CommandInput>,
4106 recovery_operation_id: Option<&str>,
4107 now_ms: i64,
4108 expected_version: u64,
4109 ) -> anyhow::Result<()> {
4110 let step = step_def.name();
4111 let effect = EffectId::derive(&instance.saga_id, instance.definition_version, step, phase);
4112 let command = CommandId::derive(&effect, attempt);
4113 self.backend
4115 .store()
4116 .record_attempt_started(&instance.saga_id, step, phase, attempt, &effect, &command)
4117 .await?;
4118 let (payload, raw_payload) = match payload {
4120 Some(CommandInput::Json(value)) => (Some(value), None),
4121 Some(CommandInput::Raw(value)) => (None, Some(value)),
4122 None => (None, None),
4123 };
4124 let envelope = SagaCommandEnvelope {
4125 saga_id: instance.saga_id.as_str().to_string(),
4126 tenant_id: instance.tenant.as_str().to_string(),
4127 workflow: instance.workflow.as_str().to_string(),
4128 definition_version: instance.definition_version.get(),
4129 definition_digest: instance.definition_digest.clone(),
4130 step: step.as_str().to_string(),
4131 phase: phase.as_str().to_string(),
4132 attempt: attempt.get(),
4133 effect_id: effect.to_string(),
4134 command_id: command.to_string(),
4135 recovery_operation_id: recovery_operation_id.map(str::to_owned),
4136 payload,
4137 raw_payload,
4138 };
4139 let event = match instance
4143 .traceparent
4144 .as_deref()
4145 .and_then(TraceContext::parse_traceparent)
4146 {
4147 Some(base) => envelope
4148 .to_outbox_event()?
4149 .with_traceparent(base.child(natelemetry::random_span_id()).to_traceparent()),
4150 None => envelope.to_outbox_event()?,
4151 };
4152 let write_context = naoutbox_core::OutboxWriteContext::new(instance.tenant.as_str())
4156 .map_err(|reason| anyhow::anyhow!("outbox write context rejected: {reason}"))?;
4157 self.backend
4158 .outbox()
4159 .append_transactional_with_context(&write_context, &event)
4160 .await?;
4161 let timeout_ms = duration_millis(step_def.timeout())?;
4163 self.schedule_timer(
4164 &instance.saga_id,
4165 TimerScope::Step(step),
4166 phase_timeout_kind(phase),
4167 attempt,
4168 checked_deadline(now_ms, timeout_ms)?,
4169 expected_version,
4170 )
4171 .await?;
4172 Ok(())
4173 }
4174
4175 async fn schedule_timer(
4187 &self,
4188 saga_id: &SagaId,
4189 scope: TimerScope<'_>,
4190 kind: &str,
4191 attempt: AttemptNo,
4192 due_at_ms: i64,
4193 expected_version: u64,
4194 ) -> anyhow::Result<()> {
4195 let timer_id = derive_timer_id(saga_id, &scope, kind, attempt);
4196 self.backend
4197 .store()
4198 .schedule_timer(&TimerSpec {
4199 timer_id: &timer_id,
4200 saga_id,
4201 scope,
4202 kind,
4203 due_at_ms,
4204 attempt,
4205 expected_saga_version: expected_version,
4206 })
4207 .await?;
4208 Ok(())
4209 }
4210
4211 async fn cancel_resolution_timers(
4219 &self,
4220 saga_id: &SagaId,
4221 step: &StepName,
4222 ) -> anyhow::Result<()> {
4223 for kind in [
4226 KIND_RESOLUTION_BUDGET,
4227 KIND_FORWARD_RESOLUTION_BUDGET,
4228 KIND_COMPENSATION_RESOLUTION_BUDGET,
4229 ] {
4230 self.backend
4231 .store()
4232 .cancel_scope_timers(saga_id, TimerScope::Step(step), Some(kind))
4233 .await?;
4234 }
4235 self.backend
4236 .store()
4237 .cancel_scope_timers(saga_id, TimerScope::Step(step), Some(KIND_RESOLVE_TIMEOUT))
4238 .await?;
4239 Ok(())
4240 }
4241
4242 async fn patch_step(
4251 &self,
4252 saga_id: &SagaId,
4253 step: &StepName,
4254 patch: StepJournalPatch<'_>,
4255 ) -> anyhow::Result<()> {
4256 self.backend
4257 .store()
4258 .update_step_journal(saga_id, step, &patch)
4259 .await?;
4260 Ok(())
4261 }
4262
4263 async fn next_attempt(
4272 &self,
4273 saga_id: &SagaId,
4274 step: &StepName,
4275 phase: StepPhase,
4276 ) -> anyhow::Result<AttemptNo> {
4277 let attempts = self.backend.store().load_attempts(saga_id, step).await?;
4278 let max = attempts
4279 .iter()
4280 .filter(|row| row.phase == phase)
4281 .map(|row| row.attempt.get())
4282 .max();
4283 match max {
4284 None => Ok(AttemptNo::FIRST),
4285 Some(value) => AttemptNo::new(value)
4286 .and_then(|attempt| attempt.next())
4287 .map_err(|violation| anyhow::anyhow!("attempt overflow: {}", violation.code())),
4288 }
4289 }
4290}
4291
4292fn verify_instance_contract(
4303 instance: &SagaInstanceRow,
4304 identity: &VerifiedIdentity,
4305) -> anyhow::Result<()> {
4306 if instance.tenant != identity.tenant {
4307 anyhow::bail!("envelope tenant does not match the instance tenant");
4308 }
4309 if instance.workflow != identity.workflow
4310 || instance.definition_version != identity.definition_version
4311 {
4312 anyhow::bail!("envelope workflow/version does not match the instance");
4313 }
4314 if instance.definition_digest != identity.definition_digest {
4315 anyhow::bail!("envelope definition digest does not match the instance");
4316 }
4317 Ok(())
4318}
4319
4320fn conflicting_projection_status(
4332 row: &SagaStepRow,
4333 context: &ResultContext<'_>,
4334) -> Option<StepAttemptStatus> {
4335 let forward = || match row.forward_status {
4336 StepForwardStatus::Succeeded => Some(StepAttemptStatus::Succeeded),
4337 StepForwardStatus::Rejected => Some(StepAttemptStatus::Rejected),
4338 StepForwardStatus::Cancelled => Some(StepAttemptStatus::CancelConfirmed),
4339 StepForwardStatus::Halted => Some(StepAttemptStatus::Halted),
4340 StepForwardStatus::Pending | StepForwardStatus::Unknown | StepForwardStatus::TimedOut => {
4341 None
4342 }
4343 };
4344 let cancellation = || match row.cancel_status {
4345 StepCancelStatus::Confirmed => Some(StepAttemptStatus::CancelConfirmed),
4346 StepCancelStatus::AlreadyTerminal => Some(StepAttemptStatus::AlreadyTerminal),
4347 StepCancelStatus::None | StepCancelStatus::ResolutionPending => None,
4348 };
4349 let compensation = || match row.compensation_status {
4350 StepCompensationStatus::Succeeded => Some(StepAttemptStatus::Succeeded),
4351 StepCompensationStatus::Halted => Some(StepAttemptStatus::Halted),
4352 StepCompensationStatus::None
4353 | StepCompensationStatus::Pending
4354 | StepCompensationStatus::Unknown => None,
4355 };
4356 let resolution = || match row.resolution_status {
4357 StepResolutionStatus::Succeeded => Some(StepAttemptStatus::Succeeded),
4358 StepResolutionStatus::Rejected => Some(StepAttemptStatus::Rejected),
4359 StepResolutionStatus::Halted => Some(StepAttemptStatus::Halted),
4360 StepResolutionStatus::None | StepResolutionStatus::Pending => None,
4361 };
4362
4363 match context.identity.phase {
4364 StepPhase::Execute => forward().or_else(cancellation),
4365 StepPhase::Cancel => cancellation().or_else(forward),
4366 StepPhase::Compensate => compensation().or_else(resolution),
4367 StepPhase::Resolve => resolution().or_else(|| {
4368 if context.instance.direction == Direction::Compensating {
4369 compensation()
4370 } else {
4371 forward()
4372 }
4373 }),
4374 }
4375}
4376
4377fn merge_deferred_projection(
4389 row: &SagaStepRow,
4390 patch: &mut StepJournalPatch<'_>,
4391) -> Result<(), ()> {
4392 if let Some(next) = patch.forward_status {
4393 patch.forward_status = if row.forward_status == next {
4394 None
4395 } else {
4396 match (row.forward_status, next) {
4397 (
4398 StepForwardStatus::Pending
4399 | StepForwardStatus::Unknown
4400 | StepForwardStatus::TimedOut,
4401 _,
4402 ) => Some(next),
4403 (_, StepForwardStatus::Unknown | StepForwardStatus::TimedOut) => None,
4405 _ => return Err(()),
4406 }
4407 };
4408 }
4409
4410 if let Some(next) = patch.cancel_status {
4411 patch.cancel_status = if row.cancel_status == next {
4412 None
4413 } else {
4414 match (row.cancel_status, next) {
4415 (StepCancelStatus::None | StepCancelStatus::ResolutionPending, _) => Some(next),
4416 (_, StepCancelStatus::ResolutionPending) => None,
4418 _ => return Err(()),
4419 }
4420 };
4421 }
4422
4423 if let Some(next) = patch.compensation_status {
4424 patch.compensation_status = if row.compensation_status == next {
4425 None
4426 } else {
4427 match (row.compensation_status, next) {
4428 (
4429 StepCompensationStatus::None
4430 | StepCompensationStatus::Pending
4431 | StepCompensationStatus::Unknown,
4432 _,
4433 ) => Some(next),
4434 (_, StepCompensationStatus::Unknown | StepCompensationStatus::Pending) => None,
4436 _ => return Err(()),
4437 }
4438 };
4439 }
4440
4441 if let Some(next) = patch.resolution_status {
4442 patch.resolution_status = if row.resolution_status == next {
4443 None
4444 } else {
4445 match (row.resolution_status, next) {
4446 (StepResolutionStatus::None | StepResolutionStatus::Pending, _) => Some(next),
4447 (_, StepResolutionStatus::Pending) => None,
4449 _ => return Err(()),
4450 }
4451 };
4452 }
4453 Ok(())
4454}
4455
4456fn ensure_applied(outcome: CasOutcome) -> anyhow::Result<u64> {
4466 match outcome {
4467 CasOutcome::Applied { new_version } => Ok(new_version),
4468 CasOutcome::Conflict => Err(SagaConcurrencyError::StaleSnapshot.into()),
4469 CasOutcome::DuplicateTrigger => Err(SagaConcurrencyError::DuplicateTrigger.into()),
4470 }
4471}
4472
4473fn next_pending_forward(
4482 definition: &WorkflowDefinition,
4483 steps: &[SagaStepRow],
4484) -> anyhow::Result<Option<StepName>> {
4485 if steps.len() != definition.steps().len() {
4486 anyhow::bail!("step journal does not completely cover the workflow definition");
4487 }
4488 let mut next = None;
4489 for (index, step_def) in definition.steps().iter().enumerate() {
4490 let row = &steps[index];
4491 if row.step != *step_def.name() || row.ordinal != index as u32 + 1 {
4492 anyhow::bail!("step journal order does not match the workflow definition");
4493 }
4494 match (next.is_some(), row.forward_status) {
4495 (false, StepForwardStatus::Succeeded) => {}
4496 (false, StepForwardStatus::Pending) => next = Some(row.step.clone()),
4497 (true, StepForwardStatus::Pending) => {}
4498 _ => anyhow::bail!("forward journal is not a succeeded-prefix/pending-suffix"),
4501 }
4502 }
4503 Ok(next)
4504}
4505
4506fn next_pending_compensation(
4518 instance: &SagaInstanceRow,
4519 definition: &WorkflowDefinition,
4520 steps: &[SagaStepRow],
4521 allow_halted_recovery: bool,
4522) -> anyhow::Result<Option<StepName>> {
4523 let frozen_version = instance
4524 .compensation_plan_version
4525 .as_deref()
4526 .ok_or_else(|| anyhow::anyhow!("compensating instance has no frozen plan"))?;
4527 let journal: Vec<StepJournalEntry> = steps
4528 .iter()
4529 .map(|row| StepJournalEntry::new(row.step.clone(), row.forward_status))
4530 .collect();
4531 let rebuilt = freeze_compensation_plan(definition, &journal)
4532 .map_err(|violation| anyhow::anyhow!("{}", violation.code()))?;
4533 if rebuilt.version() != frozen_version {
4534 anyhow::bail!("frozen compensation plan digest drifted");
4535 }
4536
4537 let mut next = None;
4538 for entry in rebuilt.entries() {
4539 let row = steps
4540 .iter()
4541 .find(|row| row.step == *entry.step())
4542 .ok_or_else(|| anyhow::anyhow!("frozen compensation member is missing"))?;
4543 if row.compensation_plan_version.as_deref() != Some(frozen_version)
4544 || row.compensation_order != Some(entry.order())
4545 {
4546 anyhow::bail!("durable compensation member metadata drifted");
4547 }
4548 match (next.is_some(), row.compensation_status) {
4549 (false, StepCompensationStatus::Succeeded) => {}
4550 (false, StepCompensationStatus::Pending) => next = Some(row.step.clone()),
4551 (false, StepCompensationStatus::Halted) if allow_halted_recovery => {
4552 next = Some(row.step.clone())
4553 }
4554 (true, StepCompensationStatus::Pending) => {}
4555 _ => anyhow::bail!("compensation journal is not a succeeded-prefix/pending-suffix"),
4556 }
4557 }
4558 let marked_count = steps
4559 .iter()
4560 .filter(|row| row.compensation_plan_version.as_deref() == Some(frozen_version))
4561 .count();
4562 if marked_count != rebuilt.entries().len() {
4563 anyhow::bail!("durable compensation plan contains unexpected members");
4564 }
4565 Ok(next)
4566}