1use std::{
17 sync::{Arc, RwLock},
18 time::{Duration, Instant},
19};
20
21use nainbox_core::{InboxClaim, InboxStore};
22use naoutbox_core::DurableOutboxAppend;
23use nasaga_backend::{
24 AttemptConflictFact, AttemptOutcomeRecord, CasOutcome, ControlCasOutcome,
25 ControlTransitionSpec, ManagementAuditOutcome, NewSagaInstance, SagaAuditEventCursor,
26 SagaAuditEventRecord, SagaAuditStore, SagaBackend, SagaBackendErrorKind, SagaBackendFactory,
27 SagaConflictKind, SagaCreation, SagaGovernanceStore, SagaInstanceQuery, SagaInstanceRow,
28 SagaInstanceStore, SagaInstanceSummary, SagaJournalStore, SagaStepRow, SagaTimerRow,
29 SagaTimerStore, StepJournalPatch, TimerFencing, TimerFencingToken, TimerFencingTokenIssuer,
30 TimerScope, TimerSpec, TransitionSpec,
31};
32use nasaga_core::{
33 freeze_compensation_plan, AttemptNo, BusinessKey, CancelMode, CommandId, DefinitionVersion,
34 Direction, EffectId, ResolutionMode, SagaId, SagaStatus, ServiceIdentity, StepAttemptStatus,
35 StepCancelStatus, StepCompensationStatus, StepDefinition, StepForwardStatus, StepJournalEntry,
36 StepName, StepPhase, StepResolutionStatus, TenantId, TimeoutPolicy, TriggerKind,
37 WorkflowDefinition, WorkflowName,
38};
39use natelemetry::TraceContext;
40use sha2::{Digest as _, Sha256};
41use uuid::Uuid;
42
43use crate::envelope::{canonical_bytes, SagaCommandEnvelope, SagaResultEnvelope, VerifiedIdentity};
44use crate::management::{
45 SagaAuditPage, SagaAuditPageCursor, SagaAuditRecord, SagaAuditTrail, SagaManagementContext,
46 SagaManagementError, SagaManagementExpectation, SagaManagementPermission,
47};
48use crate::observability::{
49 record_action_rate_rejection, record_quota_rejection, SagaOperationalMetrics,
50};
51use crate::registry::DefinitionRegistry;
52use crate::timers::{
53 derive_timer_id, parse_step_scope, phase_timeout_kind, resolution_budget_kind,
54 KIND_CANCEL_TIMEOUT, KIND_COMPENSATE_TIMEOUT, KIND_COMPENSATION_RESOLUTION_BUDGET,
55 KIND_FORWARD_RESOLUTION_BUDGET, KIND_INSTANCE_DEADLINE, KIND_RESOLUTION_BUDGET,
56 KIND_RESOLVE_TIMEOUT, KIND_STEP_TIMEOUT,
57};
58
59const CONVERGE_TRIGGER_NAMESPACE: Uuid = Uuid::from_bytes(*b"nasasaga-v1-cvns");
64
65#[derive(Debug, Clone)]
67pub struct OrchestratorConfig {
68 pub inbox_consumer: String,
70 pub cancel_max_attempts: u32,
72 pub compensate_max_attempts: u32,
74 pub resolve_max_attempts: u32,
76 pub timer_claim_limit: u32,
78 pub timer_lease_ms: i64,
80 pub pause_backoff_ms: i64,
82 pub startup_scan_limit: u32,
84 pub tenant_quotas: std::collections::BTreeMap<String, u64>,
90 pub tenant_action_rates: std::collections::BTreeMap<String, TenantActionRate>,
98 pub enable_manual_close: bool,
104}
105
106impl Default for OrchestratorConfig {
107 fn default() -> Self {
113 Self {
114 inbox_consumer: "nasaga-orchestrator".to_string(),
115 cancel_max_attempts: 5,
116 compensate_max_attempts: 5,
117 resolve_max_attempts: 10,
118 timer_claim_limit: 32,
119 timer_lease_ms: 60_000,
120 pause_backoff_ms: 30_000,
121 startup_scan_limit: 1_000,
122 tenant_quotas: std::collections::BTreeMap::new(),
123 tenant_action_rates: std::collections::BTreeMap::new(),
124 enable_manual_close: false,
126 }
127 }
128}
129
130#[derive(Debug, Clone, Copy, PartialEq, Eq)]
135pub struct TenantActionRate {
136 pub max_actions: u64,
138 pub window_ms: i64,
140}
141
142#[derive(Debug, Clone, PartialEq, Eq)]
144pub enum StartOutcome {
145 Started(SagaInstanceRow),
147 AlreadyExists(SagaInstanceRow),
149}
150
151#[derive(Debug, Clone, Copy, PartialEq, Eq)]
153#[non_exhaustive]
154pub enum StartSagaError {
155 DefinitionInactive,
157 InvalidPayload,
159 DefinitionEmpty,
161 RequestConflict,
163 TenantQuotaExceeded,
165}
166
167impl StartSagaError {
168 pub fn from_error(error: &anyhow::Error) -> Option<Self> {
174 error
175 .chain()
176 .find_map(|cause| cause.downcast_ref::<Self>().copied())
177 }
178}
179
180impl std::fmt::Display for StartSagaError {
181 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
187 formatter.write_str(match self {
188 Self::InvalidPayload => "Saga payload contract is invalid",
189 Self::DefinitionInactive => "Saga workflow definition is not active",
190 Self::DefinitionEmpty => "Saga workflow definition has no steps",
191 Self::RequestConflict => "Saga identity is already bound to a different start request",
192 Self::TenantQuotaExceeded => "saga_tenant_quota_exceeded",
193 })
194 }
195}
196
197impl std::error::Error for StartSagaError {}
198
199#[derive(Debug, Clone, Copy, PartialEq, Eq)]
201#[non_exhaustive]
202pub enum SagaConcurrencyError {
203 StaleSnapshot,
205 DuplicateTrigger,
207}
208
209impl SagaConcurrencyError {
210 pub fn from_error(error: &anyhow::Error) -> Option<Self> {
216 error
217 .chain()
218 .find_map(|cause| cause.downcast_ref::<Self>().copied())
219 }
220}
221
222impl std::fmt::Display for SagaConcurrencyError {
223 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
229 formatter.write_str(match self {
230 Self::StaleSnapshot => "Saga state snapshot is stale",
231 Self::DuplicateTrigger => "Saga transition trigger was already applied",
232 })
233 }
234}
235
236impl std::error::Error for SagaConcurrencyError {}
237
238#[derive(Debug, Clone, PartialEq, Eq)]
240pub enum HandleOutcome {
241 Applied {
243 status: SagaStatus,
245 },
246 Duplicate,
248}
249
250#[derive(Debug, Clone, PartialEq, Eq)]
252pub enum TimerOutcome {
253 Applied {
255 status: SagaStatus,
257 },
258 SkippedPaused,
260 SkippedFencingLost,
262 SkippedStale,
264}
265
266#[derive(Debug, Clone)]
268pub struct StartSagaRequest<'a> {
269 pub saga_id: &'a SagaId,
271 pub tenant: &'a TenantId,
273 pub workflow: &'a WorkflowName,
275 pub version: DefinitionVersion,
277 pub business_key: &'a BusinessKey,
279 pub deadline_at_ms: Option<i64>,
281 pub trigger_kind: TriggerKind,
283 pub trigger_id: &'a str,
285 pub first_command_payload: Option<serde_json::Value>,
287 pub first_command_raw_payload: Option<nasaga_core::SagaPayload>,
289 pub now_ms: i64,
291}
292
293enum CommandInput {
295 Json(serde_json::Value),
296 Raw(nasaga_core::SagaPayload),
297}
298
299pub struct Orchestrator<B: SagaBackend> {
304 backend: B,
305 registry: RwLock<Arc<DefinitionRegistry>>,
306 config: OrchestratorConfig,
307 fencing_tokens: TimerFencingTokenIssuer,
309}
310
311struct ResultContext<'a> {
313 instance: &'a SagaInstanceRow,
314 definition: &'a WorkflowDefinition,
315 step_def: &'a StepDefinition,
316 identity: &'a VerifiedIdentity,
317 status: StepAttemptStatus,
318 terminal: Option<StepForwardStatus>,
319 reason_code: Option<&'a str>,
320 event_id: &'a str,
321 now_ms: i64,
322}
323
324fn validate_config(config: &OrchestratorConfig) -> anyhow::Result<()> {
331 if config.inbox_consumer.is_empty()
332 || config.inbox_consumer.trim() != config.inbox_consumer
333 || config.inbox_consumer.len() > 128
334 || config.inbox_consumer.contains('\0')
335 || !config.inbox_consumer.bytes().all(|byte| {
336 byte.is_ascii_lowercase() || byte.is_ascii_digit() || matches!(byte, b'_' | b'-' | b'.')
337 })
338 || config.cancel_max_attempts == 0
339 || config.compensate_max_attempts == 0
340 || config.resolve_max_attempts == 0
341 || config.timer_claim_limit == 0
342 || config.timer_lease_ms <= 0
343 || config.pause_backoff_ms <= 0
344 || config.startup_scan_limit == 0
345 {
346 anyhow::bail!("orchestrator config violates the Inbox key contract or positive budgets");
347 }
348 for tenant in config.tenant_quotas.keys() {
351 if TenantId::new(tenant.as_str()).is_err() {
352 anyhow::bail!("tenant quota key is not a valid tenant id");
353 }
354 }
355 for (tenant, rate) in &config.tenant_action_rates {
356 if TenantId::new(tenant.as_str()).is_err() {
357 anyhow::bail!("tenant action rate key is not a valid tenant id");
358 }
359 if rate.window_ms <= 0 {
360 anyhow::bail!("tenant action rate window_ms must be positive");
361 }
362 }
363 Ok(())
364}
365
366fn duration_millis(duration: Duration) -> anyhow::Result<i64> {
373 let millis = i64::try_from(duration.as_millis())
374 .map_err(|_| anyhow::anyhow!("saga duration exceeds the supported millisecond range"))?;
375 if millis <= 0 {
376 anyhow::bail!("saga duration must be at least one millisecond");
377 }
378 Ok(millis)
379}
380
381fn checked_deadline(now_ms: i64, delay_ms: i64) -> anyhow::Result<i64> {
389 if delay_ms <= 0 {
390 anyhow::bail!("saga timer delay must be positive");
391 }
392 now_ms
393 .checked_add(delay_ms)
394 .ok_or_else(|| anyhow::anyhow!("saga timer deadline overflow"))
395}
396
397fn management_precondition(message: &'static str) -> anyhow::Error {
403 anyhow::Error::new(SagaManagementError::PreconditionFailed).context(message)
404}
405
406fn hash_canonical_json(hasher: &mut Sha256, value: &serde_json::Value) {
414 match value {
415 serde_json::Value::Null => hasher.update(canonical_bytes(&[b"null"])),
416 serde_json::Value::Bool(value) => {
417 hasher.update(canonical_bytes(&[b"bool", &[u8::from(*value)]]));
418 }
419 serde_json::Value::Number(value) => {
420 hasher.update(canonical_bytes(&[b"number", value.to_string().as_bytes()]));
421 }
422 serde_json::Value::String(value) => {
423 hasher.update(canonical_bytes(&[b"string", value.as_bytes()]));
424 }
425 serde_json::Value::Array(values) => {
426 hasher.update(canonical_bytes(&[b"array", &values.len().to_be_bytes()]));
427 for value in values {
428 hash_canonical_json(hasher, value);
429 }
430 }
431 serde_json::Value::Object(values) => {
432 let mut keys: Vec<_> = values.keys().collect();
433 keys.sort_unstable();
434 hasher.update(canonical_bytes(&[b"object", &keys.len().to_be_bytes()]));
435 for key in keys {
436 hasher.update(canonical_bytes(&[b"key", key.as_bytes()]));
437 hash_canonical_json(hasher, &values[key]);
438 }
439 }
440 }
441}
442
443fn derive_legacy_start_request_digest(
455 request: &StartSagaRequest<'_>,
456 definition_digest: &str,
457 first_step: &StepName,
458) -> String {
459 let version = request.version.get().to_be_bytes();
460 let deadline = request.deadline_at_ms.unwrap_or_default().to_be_bytes();
461 let deadline_present = [u8::from(request.deadline_at_ms.is_some())];
462 let payload_present = [u8::from(request.first_command_payload.is_some())];
463 let mut hasher = Sha256::new();
464 hasher.update(canonical_bytes(&[
465 request.tenant.as_str().as_bytes(),
466 request.workflow.as_str().as_bytes(),
467 &version,
468 definition_digest.as_bytes(),
469 request.business_key.as_str().as_bytes(),
470 first_step.as_str().as_bytes(),
471 &deadline_present,
472 &deadline,
473 &payload_present,
474 ]));
475 if let Some(payload) = request.first_command_payload.as_ref() {
476 hash_canonical_json(&mut hasher, payload);
477 }
478 hasher
479 .finalize()
480 .iter()
481 .map(|byte| format!("{byte:02x}"))
482 .collect()
483}
484
485fn derive_start_request_digest(
491 request: &StartSagaRequest<'_>,
492 definition_digest: &str,
493 first_step: &StepName,
494 payload: Option<&nasaga_core::SagaPayload>,
495) -> String {
496 let mut empty = request.clone();
497 empty.first_command_payload = None;
498 empty.first_command_raw_payload = None;
499 let domain = derive_legacy_start_request_digest(&empty, definition_digest, first_step);
500 let Some(payload) = payload else {
501 return domain;
502 };
503 let mut hasher = Sha256::new();
504 hasher.update(canonical_bytes(&[
505 b"saga-start-payload",
506 domain.as_bytes(),
507 payload.content_type.as_bytes(),
508 payload.schema_id.as_bytes(),
509 &payload.body,
510 ]));
511 hex::encode(hasher.finalize())
512}
513
514fn derive_converge_trigger_id(cause_id: &str) -> String {
521 Uuid::new_v5(
522 &CONVERGE_TRIGGER_NAMESPACE,
523 &canonical_bytes(&[cause_id.as_bytes(), b"converge"]),
524 )
525 .to_string()
526}
527
528fn elapsed_epoch_ms(base_now_ms: i64, claimed_at: Instant) -> anyhow::Result<i64> {
539 let elapsed_ms = i64::try_from(claimed_at.elapsed().as_millis())
540 .map_err(|_| anyhow::anyhow!("timer lease elapsed duration overflow"))?;
541 base_now_ms
542 .checked_add(elapsed_ms)
543 .ok_or_else(|| anyhow::anyhow!("timer lease clock overflow"))
544}
545
546impl<B> Orchestrator<B>
547where
548 B: SagaBackend + SagaBackendFactory,
549{
550 pub fn new(registry: DefinitionRegistry, config: OrchestratorConfig) -> anyhow::Result<Self> {
558 validate_config(&config)?;
559 Ok(Self {
560 backend: B::default_backend()?,
561 registry: RwLock::new(Arc::new(registry)),
562 config,
563 fencing_tokens: TimerFencingTokenIssuer::new(),
564 })
565 }
566
567 pub fn with_datasource(
576 registry: DefinitionRegistry,
577 config: OrchestratorConfig,
578 datasource: impl AsRef<str>,
579 ) -> anyhow::Result<Self> {
580 validate_config(&config)?;
581 Ok(Self {
582 backend: B::backend_for(datasource.as_ref())?,
583 registry: RwLock::new(Arc::new(registry)),
584 config,
585 fencing_tokens: TimerFencingTokenIssuer::new(),
586 })
587 }
588
589 pub fn datasource_ref(&self) -> &natx_core::DatasourceRef {
595 self.backend.datasource_ref()
596 }
597
598 fn registry_snapshot(&self) -> Arc<DefinitionRegistry> {
604 self.registry
605 .read()
606 .unwrap_or_else(std::sync::PoisonError::into_inner)
607 .clone()
608 }
609
610 pub fn replace_registry(&self, registry: DefinitionRegistry) {
616 *self
617 .registry
618 .write()
619 .unwrap_or_else(std::sync::PoisonError::into_inner) = Arc::new(registry);
620 }
621
622 pub async fn verify_registry_snapshot(
628 &self,
629 registry: &DefinitionRegistry,
630 ) -> anyhow::Result<()> {
631 registry
632 .verify_non_terminal(self.backend.store(), self.config.startup_scan_limit)
633 .await
634 }
635
636 pub async fn verify_startup(&self) -> anyhow::Result<()> {
642 let registry = self.registry_snapshot();
645 crate::verify_descriptors(®istry)?;
646 for tenant in self.config.tenant_quotas.keys() {
650 let tenant = TenantId::new(tenant.as_str())
651 .map_err(|_| anyhow::anyhow!("tenant quota key is not a valid tenant id"))?;
652 if !self
653 .backend
654 .store()
655 .tenant_quota_initialized(&tenant)
656 .await?
657 {
658 anyhow::bail!(
659 "tenant quota ledger for a capped tenant is not initialized; \
660 run reconcile_tenant_quota after every writer runs the accounting binary"
661 );
662 }
663 }
664 registry
665 .verify_non_terminal(self.backend.store(), self.config.startup_scan_limit)
666 .await
667 }
668
669 pub fn definition_digest(
675 &self,
676 workflow: &WorkflowName,
677 version: DefinitionVersion,
678 ) -> Option<String> {
679 self.registry_snapshot()
680 .get(workflow, version)
681 .map(WorkflowDefinition::digest)
682 }
683
684 pub fn definition_digest_for_tenant(
690 &self,
691 tenant: &TenantId,
692 workflow: &WorkflowName,
693 version: DefinitionVersion,
694 ) -> Option<String> {
695 self.registry_snapshot()
696 .get_for_tenant(tenant, workflow, version)
697 .map(WorkflowDefinition::digest)
698 }
699
700 pub async fn load_instance(
709 &self,
710 tenant: &TenantId,
711 saga_id: &SagaId,
712 ) -> anyhow::Result<Option<SagaInstanceRow>> {
713 Ok(self
714 .backend
715 .store()
716 .load_instance(saga_id)
717 .await?
718 .filter(|instance| &instance.tenant == tenant))
719 }
720
721 pub async fn load_audit_trail(
734 &self,
735 management: &SagaManagementContext,
736 tenant: &TenantId,
737 saga_id: &SagaId,
738 after_transition_seq: u64,
739 after_control_seq: u64,
740 limit: u32,
741 ) -> anyhow::Result<SagaAuditTrail> {
742 management.require(SagaManagementPermission::ReadAudit)?;
744 self.backend
745 .store()
746 .load_instance(saga_id)
747 .await?
748 .filter(|instance| &instance.tenant == tenant)
749 .ok_or(SagaManagementError::NotFound)?;
750 Ok(SagaAuditTrail {
751 attempts: self
752 .backend
753 .store()
754 .load_attempt_audit(saga_id, None, limit)
755 .await?
756 .into_iter()
757 .map(|row| row.attempt)
758 .collect(),
759 transitions: self
760 .backend
761 .store()
762 .load_transition_audit(saga_id, after_transition_seq, limit)
763 .await?,
764 controls: self
765 .backend
766 .store()
767 .load_control_audit(saga_id, after_control_seq, limit)
768 .await?,
769 management_operations: self
770 .backend
771 .store()
772 .load_management_audit(saga_id, None, limit)
773 .await?,
774 conflicts: self
775 .backend
776 .store()
777 .load_conflict_audit(saga_id, None, limit)
778 .await?,
779 })
780 }
781
782 pub async fn load_audit_page(
795 &self,
796 management: &SagaManagementContext,
797 tenant: &TenantId,
798 saga_id: &SagaId,
799 cursor: Option<&SagaAuditPageCursor>,
800 limit: u32,
801 ) -> anyhow::Result<SagaAuditPage> {
802 if limit == 0 || limit > 1_000 {
803 return Err(SagaManagementError::InvalidContext.into());
804 }
805 management.require(SagaManagementPermission::ReadAudit)?;
807 self.backend
808 .store()
809 .load_instance(saga_id)
810 .await?
811 .filter(|instance| &instance.tenant == tenant)
812 .ok_or(SagaManagementError::NotFound)?;
813 let after = SagaAuditEventCursor {
814 audit_seq: cursor.map_or(0, |cursor| cursor.audit_seq),
815 };
816 let events = self
817 .backend
818 .store()
819 .load_audit_events(saga_id, after, limit)
820 .await?;
821 let next_cursor = events.last().map(|event| SagaAuditPageCursor {
824 audit_seq: event.audit_seq,
825 });
826 let records = events
827 .into_iter()
828 .map(|event| match event.record {
829 SagaAuditEventRecord::Attempt(row) => SagaAuditRecord::Attempt(row),
830 SagaAuditEventRecord::Transition(row) => SagaAuditRecord::Transition(row),
831 SagaAuditEventRecord::Control(row) => SagaAuditRecord::Control(row),
832 SagaAuditEventRecord::Management(row) => SagaAuditRecord::Management(row),
833 SagaAuditEventRecord::Conflict(row) => SagaAuditRecord::Conflict(row),
834 })
835 .collect();
836 Ok(SagaAuditPage {
837 records,
838 next_cursor,
839 })
840 }
841
842 pub async fn list_instances(
856 &self,
857 management: &SagaManagementContext,
858 query: &SagaInstanceQuery<'_>,
859 ) -> anyhow::Result<Vec<SagaInstanceSummary>> {
860 management.require(SagaManagementPermission::ListInstances)?;
862 Ok(self.backend.store().list_instances(query).await?)
863 }
864
865 pub async fn tenant_quota_usage(
873 &self,
874 management: &SagaManagementContext,
875 tenant: &TenantId,
876 ) -> anyhow::Result<u64> {
877 management.require(SagaManagementPermission::ReadAudit)?;
878 Ok(self.backend.store().tenant_quota_usage(tenant).await?)
879 }
880
881 pub async fn tenant_action_rate_usage(
889 &self,
890 management: &SagaManagementContext,
891 tenant: &TenantId,
892 ) -> anyhow::Result<Option<(u64, u64)>> {
893 management.require(SagaManagementPermission::ReadAudit)?;
894 let Some(rate) = self.config.tenant_action_rates.get(tenant.as_str()) else {
895 return Ok(None);
896 };
897 Ok(Some(
898 self.backend
899 .store()
900 .tenant_action_rate_usage(tenant, rate.window_ms)
901 .await?,
902 ))
903 }
904
905 async fn reserve_action_rate_budget(&self, tenant: &TenantId) -> anyhow::Result<()> {
916 let Some(rate) = self.config.tenant_action_rates.get(tenant.as_str()) else {
917 return Ok(());
918 };
919 if matches!(
920 self.backend
921 .store()
922 .reserve_tenant_action_rate(tenant, rate.max_actions, rate.window_ms)
923 .await?,
924 nasaga_backend::ActionRateReservation::Exceeded
925 ) {
926 record_action_rate_rejection();
927 return Err(SagaManagementError::RateLimitExceeded.into());
929 }
930 Ok(())
931 }
932
933 pub async fn load_operational_metrics(
943 &self,
944 now_ms: i64,
945 ) -> anyhow::Result<SagaOperationalMetrics> {
946 Ok(self
947 .backend
948 .store()
949 .load_operational_metrics(now_ms)
950 .await?
951 .into())
952 }
953
954 pub fn verify_result_producer(
968 &self,
969 envelope: &SagaResultEnvelope,
970 producer: &ServiceIdentity,
971 ) -> anyhow::Result<()> {
972 let identity = envelope.verified_for_delivery()?;
973 let registry = self.registry_snapshot();
974 let definition = registry
975 .get_for_tenant(
976 &identity.tenant,
977 &identity.workflow,
978 identity.definition_version,
979 )
980 .ok_or(crate::SagaResultProcessingError::ContractInvalid)?;
981 if definition.digest() != identity.definition_digest {
982 return Err(crate::SagaResultProcessingError::ContractInvalid.into());
983 }
984 let step = definition
985 .step(&identity.step)
986 .ok_or(crate::SagaResultProcessingError::ContractInvalid)?;
987 if step.owner() != producer {
989 return Err(crate::SagaResultProcessingError::ProducerUnauthorized.into());
990 }
991 Ok(())
992 }
993
994 pub async fn handle_authenticated_result(
1004 &self,
1005 envelope: &SagaResultEnvelope,
1006 producer: &ServiceIdentity,
1007 now_ms: i64,
1008 ) -> anyhow::Result<HandleOutcome> {
1009 self.handle_authenticated_result_traced(envelope, producer, None, now_ms)
1010 .await
1011 }
1012
1013 pub async fn handle_authenticated_result_traced(
1028 &self,
1029 envelope: &SagaResultEnvelope,
1030 producer: &ServiceIdentity,
1031 receipt_trace: Option<&TraceContext>,
1032 now_ms: i64,
1033 ) -> anyhow::Result<HandleOutcome> {
1034 self.verify_result_producer(envelope, producer)?;
1036 let _latency =
1037 crate::latency::LatencyGuard::new(crate::latency::LatencyStage::TransitionTransaction);
1038 self.handle_verified_result(envelope, receipt_trace, now_ms)
1039 .await
1040 }
1041
1042 pub async fn start_saga(&self, request: &StartSagaRequest<'_>) -> anyhow::Result<StartOutcome> {
1054 self.start_saga_traced(request, None).await
1055 }
1056
1057 pub async fn start_saga_traced(
1071 &self,
1072 request: &StartSagaRequest<'_>,
1073 trace: Option<&TraceContext>,
1074 ) -> anyhow::Result<StartOutcome> {
1075 self.start_saga_authorized_traced(request, trace, &|| Ok(()))
1076 .await
1077 }
1078
1079 pub async fn start_saga_authorized_traced(
1087 &self,
1088 request: &StartSagaRequest<'_>,
1089 trace: Option<&TraceContext>,
1090 authorize: &(dyn Fn() -> anyhow::Result<()> + Send + Sync),
1091 ) -> anyhow::Result<StartOutcome> {
1092 authorize()?;
1094 let registry = self.registry_snapshot();
1095 let definition = registry
1097 .get_for_tenant(request.tenant, request.workflow, request.version)
1098 .ok_or(StartSagaError::DefinitionInactive)?;
1099 let digest = definition.digest();
1100 let first_step = definition
1101 .steps()
1102 .first()
1103 .ok_or(StartSagaError::DefinitionEmpty)?;
1104 let _latency =
1105 crate::latency::LatencyGuard::new(crate::latency::LatencyStage::StartTransaction);
1106 if request.first_command_payload.is_some() && request.first_command_raw_payload.is_some() {
1107 return Err(StartSagaError::InvalidPayload.into());
1108 }
1109 let payload = request.first_command_raw_payload.clone().or_else(|| {
1110 request
1111 .first_command_payload
1112 .clone()
1113 .map(nasaga_core::SagaPayload::json)
1114 });
1115 if let Some(payload) = &payload {
1116 payload
1117 .validate()
1118 .map_err(|_| StartSagaError::InvalidPayload)?;
1119 if &payload.contract() != first_step.payload_contract() {
1120 return Err(StartSagaError::InvalidPayload.into());
1121 }
1122 }
1123 let start_request_digest =
1124 derive_start_request_digest(request, &digest, first_step.name(), payload.as_ref());
1125 let legacy_start_request_digest = match payload.as_ref() {
1126 Some(payload) if payload.contract() == nasaga_core::SagaPayloadContract::default() => {
1127 let mut legacy = request.clone();
1128 legacy.first_command_payload = Some(
1129 serde_json::from_slice(&payload.body)
1130 .map_err(|_| StartSagaError::InvalidPayload)?,
1131 );
1132 Some(derive_legacy_start_request_digest(
1133 &legacy,
1134 &digest,
1135 first_step.name(),
1136 ))
1137 }
1138 _ => None,
1139 };
1140 let canonical_trace = trace.map(TraceContext::to_traceparent);
1141
1142 let outcome = crate::transaction::run_authorized_for(&self.backend, authorize, async {
1143 let creation = self
1144 .backend
1145 .store()
1146 .create_instance(&NewSagaInstance {
1147 saga_id: request.saga_id,
1148 tenant: request.tenant,
1149 workflow: request.workflow,
1150 business_key: request.business_key,
1151 definition_version: request.version,
1152 definition_digest: &digest,
1153 start_request_digest: &start_request_digest,
1154 legacy_start_request_digest: legacy_start_request_digest.as_deref(),
1155 deadline_at_ms: request.deadline_at_ms,
1156 current_step: Some(first_step.name()),
1158 trigger_kind: request.trigger_kind,
1159 trigger_id: request.trigger_id,
1160 traceparent: canonical_trace.as_deref(),
1163 })
1164 .await
1165 .map_err(|error| {
1166 if error.kind() == SagaBackendErrorKind::Conflict {
1167 anyhow::Error::new(StartSagaError::RequestConflict)
1168 } else {
1169 error.into()
1170 }
1171 })?;
1172 let instance = match creation {
1173 SagaCreation::Existing(row) => return Ok(StartOutcome::AlreadyExists(row)),
1175 SagaCreation::Created(row) => row,
1176 };
1177 if registry
1180 .get_for_start(request.tenant, request.workflow, request.version)
1181 .is_none()
1182 {
1183 return Err(StartSagaError::DefinitionInactive.into());
1184 }
1185 let cap = self
1188 .config
1189 .tenant_quotas
1190 .get(request.tenant.as_str())
1191 .copied();
1192 if matches!(
1193 self.backend
1194 .store()
1195 .reserve_tenant_quota(request.tenant, cap)
1196 .await?,
1197 nasaga_backend::QuotaReservation::Exceeded
1198 ) {
1199 record_quota_rejection();
1200 return Err(StartSagaError::TenantQuotaExceeded.into());
1202 }
1203 self.backend
1204 .store()
1205 .register_steps(request.saga_id, definition)
1206 .await?;
1207 self.issue_command(
1208 &instance,
1209 first_step,
1210 StepPhase::Execute,
1211 AttemptNo::FIRST,
1212 request
1213 .first_command_raw_payload
1214 .clone()
1215 .map(CommandInput::Raw)
1216 .or_else(|| {
1217 request
1218 .first_command_payload
1219 .clone()
1220 .map(CommandInput::Json)
1221 }),
1222 None,
1223 request.now_ms,
1224 instance.version,
1225 )
1226 .await?;
1227 if let Some(deadline) = request.deadline_at_ms {
1229 self.schedule_timer(
1230 request.saga_id,
1231 TimerScope::Instance,
1232 KIND_INSTANCE_DEADLINE,
1233 AttemptNo::FIRST,
1234 deadline,
1235 instance.version,
1236 )
1237 .await?;
1238 }
1239 Ok(StartOutcome::Started(instance))
1240 })
1241 .await?;
1242 match &outcome {
1243 StartOutcome::Started(instance) => tracing::info!(
1244 saga_id = instance.saga_id.as_str(),
1245 tenant = instance.tenant.as_str(),
1246 workflow = instance.workflow.as_str(),
1247 definition_version = instance.definition_version.get(),
1248 saga_status = instance.status.as_str(),
1249 "Saga 创建事务已提交"
1250 ),
1251 StartOutcome::AlreadyExists(instance) => tracing::info!(
1252 saga_id = instance.saga_id.as_str(),
1253 tenant = instance.tenant.as_str(),
1254 workflow = instance.workflow.as_str(),
1255 saga_status = instance.status.as_str(),
1256 "Saga 启动重放已幂等吸收"
1257 ),
1258 }
1259 Ok(outcome)
1260 }
1261
1262 async fn handle_verified_result(
1276 &self,
1277 envelope: &SagaResultEnvelope,
1278 receipt_trace: Option<&TraceContext>,
1279 now_ms: i64,
1280 ) -> anyhow::Result<HandleOutcome> {
1281 let identity = envelope.verified_for_delivery()?;
1283 let status = envelope
1284 .parsed_status()
1285 .map_err(|_| crate::SagaResultProcessingError::ContractInvalid)?;
1286 let terminal = envelope
1287 .parsed_terminal()
1288 .map_err(|_| crate::SagaResultProcessingError::ContractInvalid)?;
1289 let registry = self.registry_snapshot();
1290
1291 let outcome = crate::transaction::run_for(&self.backend, async {
1292 if matches!(
1293 self.backend
1294 .inbox()
1295 .claim(&self.config.inbox_consumer, &envelope.event_id)
1296 .await?,
1297 InboxClaim::Duplicate
1298 ) {
1299 return Ok(HandleOutcome::Duplicate);
1300 }
1301 let instance = self
1302 .backend
1303 .store()
1304 .load_instance(&identity.saga_id)
1305 .await?
1306 .ok_or(crate::SagaResultProcessingError::ContractInvalid)?;
1307 verify_instance_contract(&instance, &identity)
1308 .map_err(|_| crate::SagaResultProcessingError::ContractInvalid)?;
1309 if !instance.control_state.allows_automatic_actions() {
1310 return Err(crate::SagaResultProcessingError::Paused.into());
1313 }
1314 let definition = registry
1315 .get_for_tenant(
1316 &instance.tenant,
1317 &instance.workflow,
1318 instance.definition_version,
1319 )
1320 .ok_or_else(|| anyhow::anyhow!("instance definition is not registered"))?;
1321 let step_def = definition.step(&identity.step).ok_or_else(|| {
1322 anyhow::Error::from(crate::SagaResultProcessingError::ContractInvalid)
1323 })?;
1324
1325 let recorded = self
1328 .backend
1329 .store()
1330 .record_attempt_outcome(
1331 &identity.saga_id,
1332 &identity.step,
1333 identity.phase,
1334 identity.attempt,
1335 status,
1336 Some(&envelope.event_id),
1337 )
1338 .await?;
1339 match recorded {
1340 AttemptOutcomeRecord::AlreadyRecorded => {
1344 return Ok(HandleOutcome::Applied {
1345 status: instance.status,
1346 });
1347 }
1348 AttemptOutcomeRecord::Conflicting { existing } => {
1349 self.backend
1352 .store()
1353 .record_attempt_conflict(&AttemptConflictFact {
1354 saga_id: &identity.saga_id,
1355 step: &identity.step,
1356 phase: identity.phase,
1357 attempt: identity.attempt,
1358 existing_status: existing,
1359 incoming_status: status,
1360 incoming_event_id: &envelope.event_id,
1361 conflict_kind: SagaConflictKind::AttemptTerminal,
1362 })
1363 .await?;
1364 let status = if instance.status.is_terminal()
1365 || instance.status == SagaStatus::ManualIntervention
1366 {
1367 instance.status
1370 } else {
1371 self.escalate_manual(
1372 &instance,
1373 TriggerKind::Event,
1374 &envelope.event_id,
1375 "attempt_outcome_conflict",
1376 )
1377 .await?
1378 };
1379 return Ok(HandleOutcome::Applied { status });
1380 }
1381 AttemptOutcomeRecord::Recorded => {}
1382 }
1383
1384 let instance = match receipt_trace {
1388 Some(trace) => {
1389 let canonical = trace.to_traceparent();
1390 self.backend
1391 .store()
1392 .update_trace_context(&identity.saga_id, &canonical)
1393 .await?;
1394 SagaInstanceRow {
1395 traceparent: Some(canonical),
1396 ..instance
1397 }
1398 }
1399 None => instance,
1400 };
1401
1402 let context = ResultContext {
1403 instance: &instance,
1404 definition,
1405 step_def,
1406 identity: &identity,
1407 status,
1408 terminal,
1409 reason_code: envelope.reason_code.as_deref(),
1410 event_id: &envelope.event_id,
1411 now_ms,
1412 };
1413 let new_status = match identity.phase {
1417 StepPhase::Execute => Box::pin(self.on_execute_result(&context)).await?,
1418 StepPhase::Cancel => Box::pin(self.on_cancel_result(&context)).await?,
1419 StepPhase::Compensate => Box::pin(self.on_compensate_result(&context)).await?,
1420 StepPhase::Resolve => Box::pin(self.on_resolve_result(&context)).await?,
1421 };
1422 Ok(HandleOutcome::Applied { status: new_status })
1423 })
1424 .await?;
1425 match &outcome {
1426 HandleOutcome::Applied { status } => tracing::info!(
1427 saga_id = identity.saga_id.as_str(),
1428 workflow = identity.workflow.as_str(),
1429 step = identity.step.as_str(),
1430 phase = identity.phase.as_str(),
1431 attempt = identity.attempt.get(),
1432 saga_status = status.as_str(),
1433 "Saga 结果事务已提交"
1434 ),
1435 HandleOutcome::Duplicate => tracing::info!(
1436 saga_id = identity.saga_id.as_str(),
1437 workflow = identity.workflow.as_str(),
1438 step = identity.step.as_str(),
1439 phase = identity.phase.as_str(),
1440 attempt = identity.attempt.get(),
1441 "Saga 重复结果已由 Inbox 吸收"
1442 ),
1443 }
1444 Ok(outcome)
1445 }
1446
1447 pub async fn run_due_timers(&self, owner: &str, now_ms: i64) -> anyhow::Result<u32> {
1460 let claimed_at = Instant::now();
1463 let token = self.fencing_tokens.issue(owner, now_ms);
1466 let claimed = self
1467 .backend
1468 .store()
1469 .claim_due_timers(
1470 owner,
1471 token,
1472 now_ms,
1473 self.config.timer_lease_ms,
1474 self.config.timer_claim_limit,
1475 )
1476 .await?;
1477 let mut applied = 0u32;
1478 for timer in claimed.timers() {
1479 match self
1480 .fire_timer_inner(timer, claimed.token(), now_ms, Some(claimed_at))
1481 .await?
1482 {
1483 TimerOutcome::Applied { status } => {
1484 applied += 1;
1485 tracing::info!(
1486 saga_id = timer.saga_id.as_str(),
1487 timer_kind = %timer.kind,
1488 timer_scope = %timer.scope_kind,
1489 attempt = timer.attempt.get(),
1490 saga_status = status.as_str(),
1491 "Saga durable timer 裁决事务已提交"
1492 );
1493 }
1494 TimerOutcome::SkippedPaused => {
1495 let release_now = elapsed_epoch_ms(now_ms, claimed_at)?;
1499 let _ = self
1500 .backend
1501 .store()
1502 .release_timer(
1503 &timer.timer_id,
1504 claimed.token(),
1505 release_now,
1506 checked_deadline(release_now, self.config.pause_backoff_ms)?,
1507 )
1508 .await?;
1509 }
1510 TimerOutcome::SkippedFencingLost => tracing::warn!(
1511 saga_id = timer.saga_id.as_str(),
1512 timer_kind = %timer.kind,
1513 "Saga timer 已失去 fencing 权威,旧 owner 停止动作"
1514 ),
1515 TimerOutcome::SkippedStale => tracing::debug!(
1516 saga_id = timer.saga_id.as_str(),
1517 timer_kind = %timer.kind,
1518 "Saga 过期 timer 已幂等吸收"
1519 ),
1520 }
1521 }
1522 Ok(applied)
1523 }
1524
1525 pub async fn fire_timer(
1535 &self,
1536 timer: &SagaTimerRow,
1537 token: &TimerFencingToken,
1538 now_ms: i64,
1539 ) -> anyhow::Result<TimerOutcome> {
1540 self.fire_timer_inner(timer, token, now_ms, None).await
1541 }
1542
1543 async fn fire_timer_inner(
1554 &self,
1555 timer: &SagaTimerRow,
1556 token: &TimerFencingToken,
1557 base_now_ms: i64,
1558 claimed_at: Option<Instant>,
1559 ) -> anyhow::Result<TimerOutcome> {
1560 let _latency =
1561 crate::latency::LatencyGuard::new(crate::latency::LatencyStage::TransitionTransaction);
1562 let step_scope = parse_step_scope(&timer.scope_kind, &timer.scope_key)?;
1563 let registry = self.registry_snapshot();
1564 crate::transaction::run_for(&self.backend, async {
1565 let Some(instance) = self.backend.store().load_instance(&timer.saga_id).await? else {
1566 let fencing_now = match claimed_at {
1568 Some(started) => elapsed_epoch_ms(base_now_ms, started)?,
1569 None => base_now_ms,
1570 };
1571 return Ok(
1572 match self
1573 .backend
1574 .store()
1575 .complete_timer(&timer.timer_id, token, fencing_now)
1576 .await?
1577 {
1578 TimerFencing::Applied => TimerOutcome::SkippedStale,
1579 TimerFencing::Lost => TimerOutcome::SkippedFencingLost,
1580 },
1581 );
1582 };
1583 if !instance.control_state.allows_automatic_actions() {
1584 return Ok(TimerOutcome::SkippedPaused);
1585 }
1586 let now_ms = match claimed_at {
1587 Some(started) => elapsed_epoch_ms(base_now_ms, started)?,
1588 None => base_now_ms,
1589 };
1590 if matches!(
1592 self.backend
1593 .store()
1594 .complete_timer(&timer.timer_id, token, now_ms)
1595 .await?,
1596 TimerFencing::Lost
1597 ) {
1598 return Ok(TimerOutcome::SkippedFencingLost);
1599 }
1600 crate::latency::observe(
1601 crate::latency::LatencyStage::TimerLateness,
1602 Duration::from_millis(now_ms.saturating_sub(timer.due_at_ms).max(0) as u64),
1603 );
1604 if matches!(
1608 timer.kind.as_str(),
1609 KIND_STEP_TIMEOUT
1610 | KIND_CANCEL_TIMEOUT
1611 | KIND_COMPENSATE_TIMEOUT
1612 | KIND_RESOLVE_TIMEOUT
1613 ) && timer.expected_saga_version != instance.version
1614 {
1615 return Ok(TimerOutcome::SkippedStale);
1616 }
1617 let definition = registry
1618 .get_for_tenant(
1619 &instance.tenant,
1620 &instance.workflow,
1621 instance.definition_version,
1622 )
1623 .ok_or_else(|| anyhow::anyhow!("instance definition is not registered"))?;
1624
1625 let status = match timer.kind.as_str() {
1627 KIND_STEP_TIMEOUT => {
1628 Box::pin(self.on_step_timeout(
1629 &instance,
1630 definition,
1631 step_scope.as_ref(),
1632 timer,
1633 now_ms,
1634 ))
1635 .await?
1636 }
1637 KIND_CANCEL_TIMEOUT => {
1638 Box::pin(self.on_phase_retry_timeout(
1639 &instance,
1640 definition,
1641 step_scope.as_ref(),
1642 timer,
1643 StepPhase::Cancel,
1644 SagaStatus::Cancelling,
1645 self.config.cancel_max_attempts,
1646 "cancel_attempts_exhausted",
1647 now_ms,
1648 ))
1649 .await?
1650 }
1651 KIND_COMPENSATE_TIMEOUT => {
1652 Box::pin(self.on_phase_retry_timeout(
1653 &instance,
1654 definition,
1655 step_scope.as_ref(),
1656 timer,
1657 StepPhase::Compensate,
1658 SagaStatus::Compensating,
1659 self.config.compensate_max_attempts,
1660 "compensation_attempts_exhausted",
1661 now_ms,
1662 ))
1663 .await?
1664 }
1665 KIND_RESOLVE_TIMEOUT => {
1666 Box::pin(self.on_phase_retry_timeout(
1667 &instance,
1668 definition,
1669 step_scope.as_ref(),
1670 timer,
1671 StepPhase::Resolve,
1672 SagaStatus::WaitingResolution,
1673 self.config.resolve_max_attempts,
1674 "resolve_attempts_exhausted",
1675 now_ms,
1676 ))
1677 .await?
1678 }
1679 KIND_RESOLUTION_BUDGET
1680 | KIND_FORWARD_RESOLUTION_BUDGET
1681 | KIND_COMPENSATION_RESOLUTION_BUDGET => {
1682 if instance.status == SagaStatus::WaitingResolution {
1685 Some(
1686 self.escalate_manual(
1687 &instance,
1688 TriggerKind::Timer,
1689 &timer.timer_id,
1690 "resolution_budget_exhausted",
1691 )
1692 .await?,
1693 )
1694 } else {
1695 None
1696 }
1697 }
1698 KIND_INSTANCE_DEADLINE => {
1699 Box::pin(self.on_instance_deadline(&instance, definition, timer, now_ms))
1700 .await?
1701 }
1702 _ => anyhow::bail!("unknown timer kind"),
1703 };
1704 Ok(match status {
1705 Some(status) => TimerOutcome::Applied { status },
1706 None => TimerOutcome::SkippedStale,
1708 })
1709 })
1710 .await
1711 }
1712
1713 pub async fn pause(
1723 &self,
1724 management: &SagaManagementContext,
1725 tenant: &TenantId,
1726 saga_id: &SagaId,
1727 operation_id: &str,
1728 ) -> anyhow::Result<()> {
1729 self.pause_with_expectation(
1730 management,
1731 tenant,
1732 saga_id,
1733 operation_id,
1734 SagaManagementExpectation::default(),
1735 )
1736 .await
1737 }
1738
1739 pub async fn pause_with_expectation(
1746 &self,
1747 management: &SagaManagementContext,
1748 tenant: &TenantId,
1749 saga_id: &SagaId,
1750 operation_id: &str,
1751 expectation: SagaManagementExpectation,
1752 ) -> anyhow::Result<()> {
1753 management.require(SagaManagementPermission::Pause)?;
1755 crate::transaction::run_for(&self.backend, async {
1756 let instance = self
1757 .backend
1758 .store()
1759 .load_instance(saga_id)
1760 .await?
1761 .filter(|instance| &instance.tenant == tenant)
1762 .ok_or(SagaManagementError::NotFound)?;
1763 match self
1766 .backend
1767 .store()
1768 .set_control_state(&ControlTransitionSpec {
1769 saga_id,
1770 expected_version: instance.version,
1771 expected_control_version: instance.control_version,
1772 from: nasaga_core::ControlState::Active,
1773 to: nasaga_core::ControlState::Paused,
1774 operation_id,
1775 actor: management.actor(),
1776 reason: management.reason(),
1777 })
1778 .await?
1779 {
1780 ControlCasOutcome::Applied => {
1781 expectation.verify(instance.version, instance.control_version)?;
1782 self.reserve_action_rate_budget(tenant).await?;
1785 Ok(())
1786 }
1787 ControlCasOutcome::AlreadyApplied => Ok(()),
1788 ControlCasOutcome::Conflict => {
1789 expectation.verify(instance.version, instance.control_version)?;
1790 if instance.control_state == nasaga_core::ControlState::Active {
1791 Err(anyhow::Error::new(SagaConcurrencyError::StaleSnapshot)
1794 .context("pause lost the control-state race; reload and retry"))
1795 } else {
1796 Err(management_precondition(
1797 "pause requires an active control state",
1798 ))
1799 }
1800 }
1801 }
1802 })
1803 .await?;
1804 tracing::info!(
1805 saga_id = saga_id.as_str(),
1806 tenant = tenant.as_str(),
1807 actor = management.actor(),
1808 operation_id,
1809 control_state = "PAUSED",
1810 "Saga 暂停操作已提交"
1811 );
1812 Ok(())
1813 }
1814
1815 pub async fn resume(
1826 &self,
1827 management: &SagaManagementContext,
1828 tenant: &TenantId,
1829 saga_id: &SagaId,
1830 operation_id: &str,
1831 ) -> anyhow::Result<()> {
1832 self.resume_with_expectation(
1833 management,
1834 tenant,
1835 saga_id,
1836 operation_id,
1837 SagaManagementExpectation::default(),
1838 )
1839 .await
1840 }
1841
1842 pub async fn resume_with_expectation(
1849 &self,
1850 management: &SagaManagementContext,
1851 tenant: &TenantId,
1852 saga_id: &SagaId,
1853 operation_id: &str,
1854 expectation: SagaManagementExpectation,
1855 ) -> anyhow::Result<()> {
1856 management.require(SagaManagementPermission::Resume)?;
1858 crate::transaction::run_for(&self.backend, async {
1859 let instance = self
1860 .backend
1861 .store()
1862 .load_instance(saga_id)
1863 .await?
1864 .filter(|instance| &instance.tenant == tenant)
1865 .ok_or(SagaManagementError::NotFound)?;
1866 match self
1869 .backend
1870 .store()
1871 .set_control_state(&ControlTransitionSpec {
1872 saga_id,
1873 expected_version: instance.version,
1874 expected_control_version: instance.control_version,
1875 from: nasaga_core::ControlState::Paused,
1876 to: nasaga_core::ControlState::Active,
1877 operation_id,
1878 actor: management.actor(),
1879 reason: management.reason(),
1880 })
1881 .await?
1882 {
1883 ControlCasOutcome::Applied => {
1884 expectation.verify(instance.version, instance.control_version)?;
1885 self.reserve_action_rate_budget(tenant).await?;
1888 self.backend.store().wake_saga_timers(saga_id).await?;
1891 Ok(())
1892 }
1893 ControlCasOutcome::AlreadyApplied => Ok(()),
1894 ControlCasOutcome::Conflict => {
1895 expectation.verify(instance.version, instance.control_version)?;
1896 if instance.control_state == nasaga_core::ControlState::Paused {
1897 Err(anyhow::Error::new(SagaConcurrencyError::StaleSnapshot)
1900 .context("resume lost the control-state race; reload and retry"))
1901 } else {
1902 Err(management_precondition(
1903 "resume requires a paused control state",
1904 ))
1905 }
1906 }
1907 }
1908 })
1909 .await?;
1910 tracing::info!(
1911 saga_id = saga_id.as_str(),
1912 tenant = tenant.as_str(),
1913 actor = management.actor(),
1914 operation_id,
1915 control_state = "ACTIVE",
1916 "Saga 恢复操作已提交"
1917 );
1918 Ok(())
1919 }
1920
1921 pub async fn retry_compensation(
1936 &self,
1937 management: &SagaManagementContext,
1938 tenant: &TenantId,
1939 saga_id: &SagaId,
1940 operation_id: &str,
1941 now_ms: i64,
1942 ) -> anyhow::Result<SagaStatus> {
1943 self.retry_compensation_with_expectation(
1944 management,
1945 tenant,
1946 saga_id,
1947 operation_id,
1948 now_ms,
1949 SagaManagementExpectation::default(),
1950 )
1951 .await
1952 }
1953
1954 pub async fn retry_compensation_with_expectation(
1960 &self,
1961 management: &SagaManagementContext,
1962 tenant: &TenantId,
1963 saga_id: &SagaId,
1964 operation_id: &str,
1965 now_ms: i64,
1966 expectation: SagaManagementExpectation,
1967 ) -> anyhow::Result<SagaStatus> {
1968 management.require(SagaManagementPermission::RetryCompensation)?;
1970 let registry = self.registry_snapshot();
1971 let status = crate::transaction::run_for(&self.backend, async {
1972 let instance = self
1973 .backend
1974 .store()
1975 .load_instance(saga_id)
1976 .await?
1977 .filter(|instance| &instance.tenant == tenant)
1978 .ok_or(SagaManagementError::NotFound)?;
1979 match self
1980 .backend
1981 .store()
1982 .record_management_operation(
1983 saga_id,
1984 operation_id,
1985 "retry_frozen_compensation",
1986 management.actor(),
1987 management.reason(),
1988 )
1989 .await?
1990 {
1991 ManagementAuditOutcome::AlreadyRecorded => return Ok(instance.status),
1994 ManagementAuditOutcome::Recorded => {}
1995 }
1996 self.reserve_action_rate_budget(tenant).await?;
1999 expectation.verify(instance.version, instance.control_version)?;
2002 if !instance.control_state.allows_automatic_actions() {
2003 return Err(management_precondition(
2004 "retry_compensation requires ACTIVE control state; resume first",
2005 ));
2006 }
2007 if instance.status != SagaStatus::ManualIntervention {
2008 return Err(management_precondition(
2009 "retry_compensation requires MANUAL_INTERVENTION",
2010 ));
2011 }
2012 if instance.compensation_plan_version.is_none() {
2013 return Err(management_precondition(
2014 "no frozen compensation plan to retry",
2015 ));
2016 }
2017 let definition = registry
2018 .get_for_tenant(
2019 &instance.tenant,
2020 &instance.workflow,
2021 instance.definition_version,
2022 )
2023 .ok_or_else(|| management_precondition("instance definition is not registered"))?;
2024 let steps = self.backend.store().load_steps(saga_id).await?;
2025 match next_pending_compensation(&instance, definition, &steps, true)? {
2026 Some(step) => {
2027 let recovering_halted = steps.iter().any(|row| {
2028 row.step == step
2029 && row.compensation_status == StepCompensationStatus::Halted
2030 });
2031 if recovering_halted {
2032 self.backend
2035 .store()
2036 .reopen_halted_compensation(saga_id, &step, operation_id)
2037 .await?;
2038 }
2039 let version = self
2040 .advance(
2041 &instance,
2042 SagaStatus::Compensating,
2043 Direction::Compensating,
2044 Some(&step),
2045 TriggerKind::Admin,
2046 operation_id,
2047 None,
2048 None,
2049 )
2050 .await?;
2051 let step_def = definition.step(&step).ok_or_else(|| {
2052 management_precondition("plan step absent from definition")
2053 })?;
2054 let attempt = self
2055 .next_attempt(saga_id, &step, StepPhase::Compensate)
2056 .await?;
2057 self.issue_command(
2058 &instance,
2059 step_def,
2060 StepPhase::Compensate,
2061 attempt,
2062 None,
2063 Some(operation_id),
2064 now_ms,
2065 version,
2066 )
2067 .await?;
2068 Ok(SagaStatus::Compensating)
2069 }
2070 None => {
2071 let version = self
2074 .advance(
2075 &instance,
2076 SagaStatus::Compensating,
2077 Direction::Compensating,
2078 None,
2079 TriggerKind::Admin,
2080 operation_id,
2081 None,
2082 None,
2083 )
2084 .await?;
2085 self.converge_compensated(&instance, version, TriggerKind::Admin, operation_id)
2086 .await?;
2087 Ok(SagaStatus::Compensated)
2088 }
2089 }
2090 })
2091 .await?;
2092 tracing::info!(
2093 saga_id = saga_id.as_str(),
2094 tenant = tenant.as_str(),
2095 actor = management.actor(),
2096 operation_id,
2097 saga_status = status.as_str(),
2098 "Saga 人工补偿恢复操作已提交"
2099 );
2100 Ok(status)
2101 }
2102
2103 pub async fn retry_resolution(
2118 &self,
2119 management: &SagaManagementContext,
2120 tenant: &TenantId,
2121 saga_id: &SagaId,
2122 operation_id: &str,
2123 now_ms: i64,
2124 ) -> anyhow::Result<SagaStatus> {
2125 self.retry_resolution_with_expectation(
2126 management,
2127 tenant,
2128 saga_id,
2129 operation_id,
2130 now_ms,
2131 SagaManagementExpectation::default(),
2132 )
2133 .await
2134 }
2135
2136 pub async fn retry_resolution_with_expectation(
2142 &self,
2143 management: &SagaManagementContext,
2144 tenant: &TenantId,
2145 saga_id: &SagaId,
2146 operation_id: &str,
2147 now_ms: i64,
2148 expectation: SagaManagementExpectation,
2149 ) -> anyhow::Result<SagaStatus> {
2150 management.require(SagaManagementPermission::RetryResolution)?;
2151 let registry = self.registry_snapshot();
2152 let status = crate::transaction::run_for(&self.backend, async {
2153 let instance = self
2154 .backend
2155 .store()
2156 .load_instance(saga_id)
2157 .await?
2158 .filter(|instance| &instance.tenant == tenant)
2159 .ok_or(SagaManagementError::NotFound)?;
2160 match self
2161 .backend
2162 .store()
2163 .record_management_operation(
2164 saga_id,
2165 operation_id,
2166 "retry_resolution",
2167 management.actor(),
2168 management.reason(),
2169 )
2170 .await?
2171 {
2172 ManagementAuditOutcome::AlreadyRecorded => return Ok(instance.status),
2173 ManagementAuditOutcome::Recorded => {}
2174 }
2175 self.reserve_action_rate_budget(tenant).await?;
2178 expectation.verify(instance.version, instance.control_version)?;
2181 if !instance.control_state.allows_automatic_actions() {
2182 return Err(management_precondition(
2183 "retry_resolution requires ACTIVE control state; resume first",
2184 ));
2185 }
2186 if instance.status != SagaStatus::ManualIntervention {
2187 return Err(management_precondition(
2188 "retry_resolution requires MANUAL_INTERVENTION",
2189 ));
2190 }
2191 let step = instance
2192 .current_step
2193 .as_ref()
2194 .ok_or_else(|| management_precondition("manual resolution has no current step"))?;
2195 let definition = registry
2196 .get_for_tenant(
2197 &instance.tenant,
2198 &instance.workflow,
2199 instance.definition_version,
2200 )
2201 .ok_or_else(|| management_precondition("instance definition is not registered"))?;
2202 let step_def = definition
2203 .step(step)
2204 .ok_or_else(|| management_precondition("resolution step absent from definition"))?;
2205 if step_def.resolution().mode() != Some(ResolutionMode::Poll) {
2206 return Err(management_precondition(
2207 "retry_resolution requires a Poll resolver",
2208 ));
2209 }
2210 let row = self
2211 .backend
2212 .store()
2213 .load_steps(saga_id)
2214 .await?
2215 .into_iter()
2216 .find(|row| row.step == *step)
2217 .ok_or_else(|| management_precondition("resolution step journal is missing"))?;
2218 let has_unknown_target = match instance.direction {
2219 Direction::Forward => row.forward_status == StepForwardStatus::Unknown,
2220 Direction::Compensating => {
2221 row.compensation_status == StepCompensationStatus::Unknown
2222 }
2223 };
2224 if !has_unknown_target
2225 || !matches!(
2226 row.resolution_status,
2227 StepResolutionStatus::Pending | StepResolutionStatus::Halted
2228 )
2229 {
2230 return Err(management_precondition(
2231 "retry_resolution requires an unresolved Unknown effect",
2232 ));
2233 }
2234
2235 let attempt = self.next_attempt(saga_id, step, StepPhase::Resolve).await?;
2236 if row.resolution_status == StepResolutionStatus::Halted {
2237 self.backend
2240 .store()
2241 .reopen_halted_resolution(saga_id, step, operation_id)
2242 .await?;
2243 }
2244 let version = self
2245 .advance(
2246 &instance,
2247 SagaStatus::WaitingResolution,
2248 instance.direction,
2249 Some(step),
2250 TriggerKind::Admin,
2251 operation_id,
2252 None,
2253 None,
2254 )
2255 .await?;
2256 self.schedule_timer(
2257 saga_id,
2258 TimerScope::Step(step),
2259 resolution_budget_kind(instance.direction),
2260 attempt,
2261 checked_deadline(now_ms, duration_millis(step_def.resolution().budget())?)?,
2262 version,
2263 )
2264 .await?;
2265 self.issue_command(
2266 &instance,
2267 step_def,
2268 StepPhase::Resolve,
2269 attempt,
2270 None,
2271 Some(operation_id),
2272 now_ms,
2273 version,
2274 )
2275 .await?;
2276 Ok(SagaStatus::WaitingResolution)
2277 })
2278 .await?;
2279 tracing::info!(
2280 saga_id = saga_id.as_str(),
2281 tenant = tenant.as_str(),
2282 actor = management.actor(),
2283 operation_id,
2284 saga_status = status.as_str(),
2285 "Saga 人工解决查询恢复操作已提交"
2286 );
2287 Ok(status)
2288 }
2289
2290 pub async fn manual_close(
2307 &self,
2308 management: &SagaManagementContext,
2309 tenant: &TenantId,
2310 saga_id: &SagaId,
2311 operation_id: &str,
2312 ) -> anyhow::Result<SagaStatus> {
2313 self.manual_close_with_expectation(
2314 management,
2315 tenant,
2316 saga_id,
2317 operation_id,
2318 SagaManagementExpectation::default(),
2319 )
2320 .await
2321 }
2322
2323 pub async fn manual_close_with_expectation(
2329 &self,
2330 management: &SagaManagementContext,
2331 tenant: &TenantId,
2332 saga_id: &SagaId,
2333 operation_id: &str,
2334 expectation: SagaManagementExpectation,
2335 ) -> anyhow::Result<SagaStatus> {
2336 management.require(SagaManagementPermission::ManualClose)?;
2337 if !self.config.enable_manual_close {
2340 return Err(management_precondition(
2341 "manual close is disabled: deploy MANUALLY_CLOSED-aware readers to every replica, then enable it explicitly",
2342 ));
2343 }
2344 let status = crate::transaction::run_for(&self.backend, async {
2345 let instance = self
2346 .backend
2347 .store()
2348 .load_instance(saga_id)
2349 .await?
2350 .filter(|instance| &instance.tenant == tenant)
2351 .ok_or(SagaManagementError::NotFound)?;
2352 match self
2355 .backend
2356 .store()
2357 .record_management_operation(
2358 saga_id,
2359 operation_id,
2360 nasaga_backend::MANUAL_CLOSE_ACTION,
2361 management.actor(),
2362 management.reason(),
2363 )
2364 .await?
2365 {
2366 ManagementAuditOutcome::AlreadyRecorded => return Ok(instance.status),
2368 ManagementAuditOutcome::Recorded => {}
2369 }
2370 self.reserve_action_rate_budget(tenant).await?;
2373 expectation.verify(instance.version, instance.control_version)?;
2376 if !instance.control_state.allows_automatic_actions() {
2377 return Err(management_precondition(
2378 "manual_close requires ACTIVE control state; resume first",
2379 ));
2380 }
2381 if instance.status != SagaStatus::ManualIntervention {
2382 return Err(management_precondition(
2383 "manual_close requires MANUAL_INTERVENTION",
2384 ));
2385 }
2386 self.advance(
2387 &instance,
2388 SagaStatus::ManuallyClosed,
2389 instance.direction,
2390 None,
2391 TriggerKind::Admin,
2392 operation_id,
2393 None,
2394 None,
2395 )
2396 .await?;
2397 self.backend
2400 .store()
2401 .cancel_scope_timers(saga_id, TimerScope::Instance, None)
2402 .await?;
2403 Ok(SagaStatus::ManuallyClosed)
2404 })
2405 .await?;
2406 tracing::info!(
2407 saga_id = saga_id.as_str(),
2408 tenant = tenant.as_str(),
2409 actor = management.actor(),
2410 operation_id,
2411 saga_status = status.as_str(),
2412 "Saga 人工关闭操作已提交"
2413 );
2414 Ok(status)
2415 }
2416
2417 async fn project_deferred_result(
2432 &self,
2433 context: &ResultContext<'_>,
2434 ) -> anyhow::Result<SagaStatus> {
2435 let saga_id = &context.identity.saga_id;
2436 let step = &context.identity.step;
2437 let row = self
2438 .backend
2439 .store()
2440 .load_steps(saga_id)
2441 .await?
2442 .into_iter()
2443 .find(|row| row.step == *step)
2444 .ok_or_else(|| anyhow::anyhow!("step journal row is missing"))?;
2445 let current_attempt = match context.identity.phase {
2446 StepPhase::Execute => row.execute_attempt,
2447 StepPhase::Cancel => row.cancel_attempt,
2448 StepPhase::Compensate => row.compensate_attempt,
2449 StepPhase::Resolve => row.resolve_attempt,
2450 };
2451 if current_attempt != Some(context.identity.attempt.get()) {
2452 return Ok(context.instance.status);
2453 }
2454
2455 let mut patch = StepJournalPatch {
2456 last_error_code: context.reason_code,
2457 ..Default::default()
2458 };
2459 let timeout_kind = match context.identity.phase {
2460 StepPhase::Execute => {
2461 patch.forward_status = Some(match context.status {
2462 StepAttemptStatus::Succeeded => StepForwardStatus::Succeeded,
2463 StepAttemptStatus::Rejected => StepForwardStatus::Rejected,
2464 StepAttemptStatus::Unknown => StepForwardStatus::Unknown,
2465 StepAttemptStatus::Halted => StepForwardStatus::Halted,
2466 _ => anyhow::bail!("invalid deferred execute status"),
2467 });
2468 patch.mark_finished = !matches!(context.status, StepAttemptStatus::Unknown);
2469 KIND_STEP_TIMEOUT
2470 }
2471 StepPhase::Cancel => {
2472 match context.status {
2473 StepAttemptStatus::CancelConfirmed => {
2474 patch.forward_status = Some(StepForwardStatus::Cancelled);
2475 patch.cancel_status = Some(StepCancelStatus::Confirmed);
2476 patch.mark_finished = true;
2477 }
2478 StepAttemptStatus::AlreadyTerminal => {
2479 patch.forward_status = context.terminal;
2480 patch.cancel_status = Some(StepCancelStatus::AlreadyTerminal);
2481 patch.mark_finished = true;
2482 }
2483 StepAttemptStatus::ResolutionPending => {
2484 patch.cancel_status = Some(StepCancelStatus::ResolutionPending);
2485 }
2486 _ => anyhow::bail!("invalid deferred cancel status"),
2487 }
2488 KIND_CANCEL_TIMEOUT
2489 }
2490 StepPhase::Compensate => {
2491 patch.compensation_status = Some(match context.status {
2492 StepAttemptStatus::Succeeded => StepCompensationStatus::Succeeded,
2493 StepAttemptStatus::Unknown => StepCompensationStatus::Unknown,
2494 StepAttemptStatus::Halted => StepCompensationStatus::Halted,
2495 _ => anyhow::bail!("invalid deferred compensation status"),
2496 });
2497 KIND_COMPENSATE_TIMEOUT
2498 }
2499 StepPhase::Resolve => {
2500 patch.resolution_status = Some(match context.status {
2501 StepAttemptStatus::Succeeded => StepResolutionStatus::Succeeded,
2502 StepAttemptStatus::Rejected => StepResolutionStatus::Rejected,
2503 StepAttemptStatus::Unknown => StepResolutionStatus::Pending,
2504 StepAttemptStatus::Halted => StepResolutionStatus::Halted,
2505 _ => anyhow::bail!("invalid deferred resolution status"),
2506 });
2507 if context.instance.direction == Direction::Forward {
2508 patch.forward_status = match context.status {
2509 StepAttemptStatus::Succeeded => Some(StepForwardStatus::Succeeded),
2510 StepAttemptStatus::Rejected => Some(StepForwardStatus::Rejected),
2511 StepAttemptStatus::Halted => Some(StepForwardStatus::Halted),
2512 StepAttemptStatus::Unknown => Some(StepForwardStatus::Unknown),
2513 _ => None,
2514 };
2515 } else {
2516 patch.compensation_status = match context.status {
2517 StepAttemptStatus::Succeeded => Some(StepCompensationStatus::Succeeded),
2518 StepAttemptStatus::Rejected => Some(StepCompensationStatus::Pending),
2519 StepAttemptStatus::Halted => Some(StepCompensationStatus::Halted),
2520 StepAttemptStatus::Unknown => Some(StepCompensationStatus::Unknown),
2521 _ => None,
2522 };
2523 }
2524 KIND_RESOLVE_TIMEOUT
2525 }
2526 };
2527
2528 let resolve_after_compensation_success = context.identity.phase == StepPhase::Resolve
2532 && context.instance.direction == Direction::Compensating
2533 && context.status == StepAttemptStatus::Rejected
2534 && row.compensation_status == StepCompensationStatus::Succeeded
2535 && patch.compensation_status == Some(StepCompensationStatus::Pending);
2536 if resolve_after_compensation_success
2537 || merge_deferred_projection(&row, &mut patch).is_err()
2538 {
2539 let existing = conflicting_projection_status(&row, context)
2543 .ok_or_else(|| anyhow::anyhow!("conflicting projection has no settled fact"))?;
2544 self.backend
2545 .store()
2546 .record_attempt_conflict(&AttemptConflictFact {
2547 saga_id,
2548 step,
2549 phase: context.identity.phase,
2550 attempt: context.identity.attempt,
2551 existing_status: existing,
2552 incoming_status: context.status,
2553 incoming_event_id: context.event_id,
2554 conflict_kind: SagaConflictKind::CrossPhaseFact,
2555 })
2556 .await?;
2557 if !context.instance.status.is_terminal()
2558 && context.instance.status != SagaStatus::ManualIntervention
2559 {
2560 return self
2561 .escalate_manual(
2562 context.instance,
2563 TriggerKind::Event,
2564 context.event_id,
2565 "cross_phase_fact_conflict",
2566 )
2567 .await;
2568 }
2569 return Ok(context.instance.status);
2570 }
2571 self.backend
2572 .store()
2573 .cancel_scope_timers(saga_id, TimerScope::Step(step), Some(timeout_kind))
2574 .await?;
2575 self.patch_step(saga_id, step, patch).await?;
2576 Ok(context.instance.status)
2577 }
2578
2579 async fn apply_monotonic_projection(
2591 &self,
2592 context: &ResultContext<'_>,
2593 mut patch: StepJournalPatch<'_>,
2594 ) -> anyhow::Result<Option<SagaStatus>> {
2595 let saga_id = &context.identity.saga_id;
2596 let step = &context.identity.step;
2597 let row = self
2598 .backend
2599 .store()
2600 .load_steps(saga_id)
2601 .await?
2602 .into_iter()
2603 .find(|row| row.step == *step)
2604 .ok_or_else(|| anyhow::anyhow!("step journal row is missing"))?;
2605
2606 let resolve_after_compensation_success = context.identity.phase == StepPhase::Resolve
2609 && context.instance.direction == Direction::Compensating
2610 && context.status == StepAttemptStatus::Rejected
2611 && row.compensation_status == StepCompensationStatus::Succeeded
2612 && patch.compensation_status == Some(StepCompensationStatus::Pending);
2613 if resolve_after_compensation_success
2614 || merge_deferred_projection(&row, &mut patch).is_err()
2615 {
2616 let existing = conflicting_projection_status(&row, context)
2617 .ok_or_else(|| anyhow::anyhow!("conflicting projection has no settled fact"))?;
2618 self.backend
2619 .store()
2620 .record_attempt_conflict(&AttemptConflictFact {
2621 saga_id,
2622 step,
2623 phase: context.identity.phase,
2624 attempt: context.identity.attempt,
2625 existing_status: existing,
2626 incoming_status: context.status,
2627 incoming_event_id: context.event_id,
2628 conflict_kind: SagaConflictKind::CrossPhaseFact,
2629 })
2630 .await?;
2631 if !context.instance.status.is_terminal()
2632 && context.instance.status != SagaStatus::ManualIntervention
2633 {
2634 let status = self
2635 .escalate_manual(
2636 context.instance,
2637 TriggerKind::Event,
2638 context.event_id,
2639 "cross_phase_fact_conflict",
2640 )
2641 .await?;
2642 return Ok(Some(status));
2643 }
2644 return Ok(Some(context.instance.status));
2645 }
2646 self.patch_step(saga_id, step, patch).await?;
2647 Ok(None)
2648 }
2649
2650 async fn on_execute_result(&self, context: &ResultContext<'_>) -> anyhow::Result<SagaStatus> {
2657 if context.instance.status != SagaStatus::Running
2660 || context.instance.current_step.as_ref() != Some(&context.identity.step)
2661 {
2662 return self.project_deferred_result(context).await;
2663 }
2664 let saga_id = &context.identity.saga_id;
2665 let step = &context.identity.step;
2666 match context.status {
2667 StepAttemptStatus::Succeeded => {
2668 self.patch_step(
2669 saga_id,
2670 step,
2671 StepJournalPatch {
2672 forward_status: Some(StepForwardStatus::Succeeded),
2673 mark_finished: true,
2674 ..Default::default()
2675 },
2676 )
2677 .await?;
2678 self.backend
2679 .store()
2680 .cancel_scope_timers(saga_id, TimerScope::Step(step), Some(KIND_STEP_TIMEOUT))
2681 .await?;
2682 self.proceed_forward(context).await
2683 }
2684 StepAttemptStatus::Rejected => {
2685 self.patch_step(
2686 saga_id,
2687 step,
2688 StepJournalPatch {
2689 forward_status: Some(StepForwardStatus::Rejected),
2690 last_error_code: context.reason_code,
2691 mark_finished: true,
2692 ..Default::default()
2693 },
2694 )
2695 .await?;
2696 self.backend
2697 .store()
2698 .cancel_scope_timers(saga_id, TimerScope::Step(step), Some(KIND_STEP_TIMEOUT))
2699 .await?;
2700 self.enter_compensation(context).await
2702 }
2703 StepAttemptStatus::Unknown => {
2704 self.enter_waiting_resolution(context, Direction::Forward)
2705 .await
2706 }
2707 StepAttemptStatus::Halted => {
2708 self.patch_step(
2709 saga_id,
2710 step,
2711 StepJournalPatch {
2712 forward_status: Some(StepForwardStatus::Halted),
2713 last_error_code: context.reason_code,
2714 mark_finished: true,
2715 ..Default::default()
2716 },
2717 )
2718 .await?;
2719 self.backend
2720 .store()
2721 .cancel_scope_timers(saga_id, TimerScope::Step(step), None)
2722 .await?;
2723 self.escalate_manual(
2724 context.instance,
2725 TriggerKind::Event,
2726 context.event_id,
2727 "step_halted",
2728 )
2729 .await
2730 }
2731 _ => anyhow::bail!("cancel-vocabulary status on an execute result"),
2732 }
2733 }
2734
2735 async fn on_cancel_result(&self, context: &ResultContext<'_>) -> anyhow::Result<SagaStatus> {
2742 if context.instance.status != SagaStatus::Cancelling
2743 || context.instance.current_step.as_ref() != Some(&context.identity.step)
2744 {
2745 return self.project_deferred_result(context).await;
2746 }
2747 let saga_id = &context.identity.saga_id;
2748 let step = &context.identity.step;
2749 match context.status {
2750 StepAttemptStatus::CancelConfirmed => {
2751 if let Some(status) = self
2753 .apply_monotonic_projection(
2754 context,
2755 StepJournalPatch {
2756 forward_status: Some(StepForwardStatus::Cancelled),
2757 cancel_status: Some(StepCancelStatus::Confirmed),
2758 mark_finished: true,
2759 ..Default::default()
2760 },
2761 )
2762 .await?
2763 {
2764 return Ok(status);
2765 }
2766 self.backend
2767 .store()
2768 .cancel_scope_timers(saga_id, TimerScope::Step(step), Some(KIND_CANCEL_TIMEOUT))
2769 .await?;
2770 self.enter_compensation(context).await
2771 }
2772 StepAttemptStatus::AlreadyTerminal => {
2773 let terminal = context.terminal.ok_or_else(|| {
2774 anyhow::anyhow!("ALREADY_TERMINAL requires a terminal status")
2775 })?;
2776 if let Some(status) = self
2777 .apply_monotonic_projection(
2778 context,
2779 StepJournalPatch {
2780 forward_status: Some(terminal),
2781 cancel_status: Some(StepCancelStatus::AlreadyTerminal),
2782 mark_finished: true,
2783 ..Default::default()
2784 },
2785 )
2786 .await?
2787 {
2788 return Ok(status);
2789 }
2790 self.backend
2791 .store()
2792 .cancel_scope_timers(saga_id, TimerScope::Step(step), Some(KIND_CANCEL_TIMEOUT))
2793 .await?;
2794 match terminal {
2795 StepForwardStatus::Succeeded => match context.step_def.timeout_policy() {
2796 TimeoutPolicy::AcceptLateSuccess => self.proceed_forward(context).await,
2798 TimeoutPolicy::CompensateLateSuccess => {
2800 self.enter_compensation(context).await
2801 }
2802 },
2803 StepForwardStatus::Rejected => self.enter_compensation(context).await,
2804 StepForwardStatus::Halted => {
2805 self.escalate_manual(
2806 context.instance,
2807 TriggerKind::Event,
2808 context.event_id,
2809 "step_halted",
2810 )
2811 .await
2812 }
2813 _ => anyhow::bail!("ALREADY_TERMINAL carried a non-terminal forward status"),
2814 }
2815 }
2816 StepAttemptStatus::ResolutionPending => {
2817 if let Some(status) = self
2819 .apply_monotonic_projection(
2820 context,
2821 StepJournalPatch {
2822 cancel_status: Some(StepCancelStatus::ResolutionPending),
2823 ..Default::default()
2824 },
2825 )
2826 .await?
2827 {
2828 return Ok(status);
2829 }
2830 self.backend
2831 .store()
2832 .cancel_scope_timers(saga_id, TimerScope::Step(step), Some(KIND_CANCEL_TIMEOUT))
2833 .await?;
2834 self.enter_waiting_resolution(context, Direction::Forward)
2835 .await
2836 }
2837 _ => anyhow::bail!("non-cancel status on a cancel result"),
2838 }
2839 }
2840
2841 async fn on_compensate_result(
2848 &self,
2849 context: &ResultContext<'_>,
2850 ) -> anyhow::Result<SagaStatus> {
2851 if context.instance.status != SagaStatus::Compensating
2852 || context.instance.current_step.as_ref() != Some(&context.identity.step)
2853 {
2854 return self.project_deferred_result(context).await;
2855 }
2856 let saga_id = &context.identity.saga_id;
2857 let step = &context.identity.step;
2858 self.backend
2859 .store()
2860 .cancel_scope_timers(
2861 saga_id,
2862 TimerScope::Step(step),
2863 Some(KIND_COMPENSATE_TIMEOUT),
2864 )
2865 .await?;
2866 match context.status {
2867 StepAttemptStatus::Succeeded => {
2868 self.patch_step(
2869 saga_id,
2870 step,
2871 StepJournalPatch {
2872 compensation_status: Some(StepCompensationStatus::Succeeded),
2873 ..Default::default()
2874 },
2875 )
2876 .await?;
2877 self.continue_compensation(context).await
2878 }
2879 StepAttemptStatus::Unknown => {
2880 self.patch_step(
2881 saga_id,
2882 step,
2883 StepJournalPatch {
2884 compensation_status: Some(StepCompensationStatus::Unknown),
2885 ..Default::default()
2886 },
2887 )
2888 .await?;
2889 self.enter_waiting_resolution(context, Direction::Compensating)
2890 .await
2891 }
2892 StepAttemptStatus::Halted => {
2893 self.patch_step(
2894 saga_id,
2895 step,
2896 StepJournalPatch {
2897 compensation_status: Some(StepCompensationStatus::Halted),
2898 last_error_code: context.reason_code,
2899 ..Default::default()
2900 },
2901 )
2902 .await?;
2903 self.escalate_manual(
2904 context.instance,
2905 TriggerKind::Event,
2906 context.event_id,
2907 "compensation_halted",
2908 )
2909 .await
2910 }
2911 _ => anyhow::bail!("invalid status on a compensate result"),
2912 }
2913 }
2914
2915 async fn settle_resolution_from_committed_success(
2927 &self,
2928 context: &ResultContext<'_>,
2929 ) -> anyhow::Result<Option<SagaStatus>> {
2930 let saga_id = &context.identity.saga_id;
2931 let step = &context.identity.step;
2932 let row = self
2933 .backend
2934 .store()
2935 .load_steps(saga_id)
2936 .await?
2937 .into_iter()
2938 .find(|row| row.step == *step)
2939 .ok_or_else(|| anyhow::anyhow!("step journal row is missing"))?;
2940 let committed_success = match context.instance.direction {
2941 Direction::Forward => row.forward_status == StepForwardStatus::Succeeded,
2942 Direction::Compensating => row.compensation_status == StepCompensationStatus::Succeeded,
2943 };
2944 if !committed_success {
2945 return Ok(None);
2946 }
2947
2948 let mut patch = StepJournalPatch {
2949 resolution_status: Some(StepResolutionStatus::Succeeded),
2950 ..Default::default()
2951 };
2952 match context.instance.direction {
2953 Direction::Forward => {
2954 patch.forward_status = Some(StepForwardStatus::Succeeded);
2955 patch.mark_finished = true;
2956 }
2957 Direction::Compensating => {
2958 patch.compensation_status = Some(StepCompensationStatus::Succeeded);
2959 }
2960 }
2961 if let Some(status) = self.apply_monotonic_projection(context, patch).await? {
2962 return Ok(Some(status));
2963 }
2964
2965 self.cancel_resolution_timers(saga_id, step).await?;
2968 let status = match context.instance.direction {
2969 Direction::Forward => match context.step_def.timeout_policy() {
2970 TimeoutPolicy::AcceptLateSuccess => self.proceed_forward(context).await?,
2971 TimeoutPolicy::CompensateLateSuccess => self.enter_compensation(context).await?,
2972 },
2973 Direction::Compensating => self.continue_compensation(context).await?,
2974 };
2975 Ok(Some(status))
2976 }
2977
2978 async fn on_resolve_result(&self, context: &ResultContext<'_>) -> anyhow::Result<SagaStatus> {
2985 if context.instance.status != SagaStatus::WaitingResolution
2986 || context.instance.current_step.as_ref() != Some(&context.identity.step)
2987 {
2988 return self.project_deferred_result(context).await;
2989 }
2990 if let Some(status) = self
2991 .settle_resolution_from_committed_success(context)
2992 .await?
2993 {
2994 return Ok(status);
2995 }
2996 let saga_id = &context.identity.saga_id;
2997 let step = &context.identity.step;
2998 match context.instance.direction {
2999 Direction::Forward => match context.status {
3000 StepAttemptStatus::Succeeded => {
3001 if let Some(status) = self
3002 .apply_monotonic_projection(
3003 context,
3004 StepJournalPatch {
3005 forward_status: Some(StepForwardStatus::Succeeded),
3006 resolution_status: Some(StepResolutionStatus::Succeeded),
3007 mark_finished: true,
3008 ..Default::default()
3009 },
3010 )
3011 .await?
3012 {
3013 return Ok(status);
3014 }
3015 self.cancel_resolution_timers(saga_id, step).await?;
3016 self.proceed_forward(context).await
3017 }
3018 StepAttemptStatus::Rejected => {
3019 if let Some(status) = self
3022 .apply_monotonic_projection(
3023 context,
3024 StepJournalPatch {
3025 forward_status: Some(StepForwardStatus::Rejected),
3026 resolution_status: Some(StepResolutionStatus::Rejected),
3027 last_error_code: context.reason_code,
3028 mark_finished: true,
3029 ..Default::default()
3030 },
3031 )
3032 .await?
3033 {
3034 return Ok(status);
3035 }
3036 self.cancel_resolution_timers(saga_id, step).await?;
3037 self.enter_compensation(context).await
3038 }
3039 StepAttemptStatus::Unknown => self.reissue_resolve(context).await,
3040 StepAttemptStatus::Halted => {
3041 if let Some(status) = self
3042 .apply_monotonic_projection(
3043 context,
3044 StepJournalPatch {
3045 resolution_status: Some(StepResolutionStatus::Halted),
3046 last_error_code: context.reason_code,
3047 ..Default::default()
3048 },
3049 )
3050 .await?
3051 {
3052 return Ok(status);
3053 }
3054 self.escalate_manual(
3055 context.instance,
3056 TriggerKind::Event,
3057 context.event_id,
3058 "resolution_halted",
3059 )
3060 .await
3061 }
3062 _ => anyhow::bail!("invalid status on a resolve result"),
3063 },
3064 Direction::Compensating => match context.status {
3065 StepAttemptStatus::Succeeded => {
3066 if let Some(status) = self
3068 .apply_monotonic_projection(
3069 context,
3070 StepJournalPatch {
3071 compensation_status: Some(StepCompensationStatus::Succeeded),
3072 resolution_status: Some(StepResolutionStatus::Succeeded),
3073 ..Default::default()
3074 },
3075 )
3076 .await?
3077 {
3078 return Ok(status);
3079 }
3080 self.cancel_resolution_timers(saga_id, step).await?;
3081 self.continue_compensation(context).await
3082 }
3083 StepAttemptStatus::Rejected => {
3084 if let Some(status) = self
3086 .apply_monotonic_projection(
3087 context,
3088 StepJournalPatch {
3089 compensation_status: Some(StepCompensationStatus::Pending),
3090 resolution_status: Some(StepResolutionStatus::Rejected),
3091 ..Default::default()
3092 },
3093 )
3094 .await?
3095 {
3096 return Ok(status);
3097 }
3098 self.cancel_resolution_timers(saga_id, step).await?;
3099 let attempt = self
3100 .next_attempt(saga_id, step, StepPhase::Compensate)
3101 .await?;
3102 if attempt.get() > self.config.compensate_max_attempts {
3103 return self
3104 .escalate_manual(
3105 context.instance,
3106 TriggerKind::Event,
3107 context.event_id,
3108 "compensation_attempts_exhausted",
3109 )
3110 .await;
3111 }
3112 let version = self
3113 .advance(
3114 context.instance,
3115 SagaStatus::Compensating,
3116 Direction::Compensating,
3117 Some(step),
3118 TriggerKind::Event,
3119 context.event_id,
3120 None,
3121 None,
3122 )
3123 .await?;
3124 self.issue_command(
3125 context.instance,
3126 context.step_def,
3127 StepPhase::Compensate,
3128 attempt,
3129 None,
3130 None,
3131 context.now_ms,
3132 version,
3133 )
3134 .await?;
3135 Ok(SagaStatus::Compensating)
3136 }
3137 StepAttemptStatus::Unknown => self.reissue_resolve(context).await,
3138 StepAttemptStatus::Halted => {
3139 if let Some(status) = self
3140 .apply_monotonic_projection(
3141 context,
3142 StepJournalPatch {
3143 resolution_status: Some(StepResolutionStatus::Halted),
3144 last_error_code: context.reason_code,
3145 ..Default::default()
3146 },
3147 )
3148 .await?
3149 {
3150 return Ok(status);
3151 }
3152 self.escalate_manual(
3153 context.instance,
3154 TriggerKind::Event,
3155 context.event_id,
3156 "resolution_halted",
3157 )
3158 .await
3159 }
3160 _ => anyhow::bail!("invalid status on a resolve result"),
3161 },
3162 }
3163 }
3164
3165 async fn on_step_timeout(
3178 &self,
3179 instance: &SagaInstanceRow,
3180 definition: &WorkflowDefinition,
3181 step_scope: Option<&StepName>,
3182 timer: &SagaTimerRow,
3183 now_ms: i64,
3184 ) -> anyhow::Result<Option<SagaStatus>> {
3185 let Some(step) = step_scope else {
3186 anyhow::bail!("step-timeout timer without a step scope");
3187 };
3188 if instance.status != SagaStatus::Running || instance.current_step.as_ref() != Some(step) {
3190 return Ok(None);
3191 }
3192 let step_def = definition
3193 .step(step)
3194 .ok_or_else(|| anyhow::anyhow!("timer step absent from definition"))?;
3195 if step_def.cancel_mode() == CancelMode::ResolveOnly {
3196 self.patch_step(
3199 &instance.saga_id,
3200 step,
3201 StepJournalPatch {
3202 forward_status: Some(StepForwardStatus::Unknown),
3203 resolution_status: Some(StepResolutionStatus::Pending),
3204 ..Default::default()
3205 },
3206 )
3207 .await?;
3208 let version = self
3209 .advance(
3210 instance,
3211 SagaStatus::WaitingResolution,
3212 Direction::Forward,
3213 Some(step),
3214 TriggerKind::Timer,
3215 &timer.timer_id,
3216 None,
3217 None,
3218 )
3219 .await?;
3220 let resolution = step_def.resolution();
3221 self.schedule_timer(
3222 &instance.saga_id,
3223 TimerScope::Step(step),
3224 resolution_budget_kind(Direction::Forward),
3225 timer.attempt,
3226 checked_deadline(now_ms, duration_millis(resolution.budget())?)?,
3227 version,
3228 )
3229 .await?;
3230 if resolution.mode() == Some(ResolutionMode::Poll) {
3231 let attempt = self
3232 .next_attempt(&instance.saga_id, step, StepPhase::Resolve)
3233 .await?;
3234 self.issue_command(
3235 instance,
3236 step_def,
3237 StepPhase::Resolve,
3238 attempt,
3239 None,
3240 None,
3241 now_ms,
3242 version,
3243 )
3244 .await?;
3245 }
3246 return Ok(Some(SagaStatus::WaitingResolution));
3247 }
3248
3249 self.patch_step(
3252 &instance.saga_id,
3253 step,
3254 StepJournalPatch {
3255 forward_status: Some(StepForwardStatus::TimedOut),
3256 ..Default::default()
3257 },
3258 )
3259 .await?;
3260 let version = self
3261 .advance(
3262 instance,
3263 SagaStatus::Cancelling,
3264 Direction::Forward,
3265 Some(step),
3266 TriggerKind::Timer,
3267 &timer.timer_id,
3268 None,
3269 None,
3270 )
3271 .await?;
3272 let attempt = self
3273 .next_attempt(&instance.saga_id, step, StepPhase::Cancel)
3274 .await?;
3275 self.issue_command(
3276 instance,
3277 step_def,
3278 StepPhase::Cancel,
3279 attempt,
3280 None,
3281 None,
3282 now_ms,
3283 version,
3284 )
3285 .await?;
3286 Ok(Some(SagaStatus::Cancelling))
3287 }
3288
3289 #[allow(clippy::too_many_arguments)]
3304 async fn on_phase_retry_timeout(
3305 &self,
3306 instance: &SagaInstanceRow,
3307 definition: &WorkflowDefinition,
3308 step_scope: Option<&StepName>,
3309 timer: &SagaTimerRow,
3310 phase: StepPhase,
3311 required_status: SagaStatus,
3312 max_attempts: u32,
3313 exhausted_code: &str,
3314 now_ms: i64,
3315 ) -> anyhow::Result<Option<SagaStatus>> {
3316 let Some(step) = step_scope else {
3317 anyhow::bail!("phase retry timer without a step scope");
3318 };
3319 if instance.status != required_status || instance.current_step.as_ref() != Some(step) {
3320 return Ok(None);
3321 }
3322 let step_def = definition
3323 .step(step)
3324 .ok_or_else(|| anyhow::anyhow!("timer step absent from definition"))?;
3325 let exhausted = if phase == StepPhase::Resolve {
3326 self.backend
3329 .store()
3330 .count_resolution_attempts(&instance.saga_id, step, instance.direction)
3331 .await?
3332 >= max_attempts
3333 } else {
3334 self.next_attempt(&instance.saga_id, step, phase)
3335 .await?
3336 .get()
3337 > max_attempts
3338 };
3339 if exhausted {
3342 return Ok(Some(
3343 self.escalate_manual(
3344 instance,
3345 TriggerKind::Timer,
3346 &timer.timer_id,
3347 exhausted_code,
3348 )
3349 .await?,
3350 ));
3351 }
3352 let attempt = self.next_attempt(&instance.saga_id, step, phase).await?;
3354 let version = self
3355 .advance(
3356 instance,
3357 required_status,
3358 instance.direction,
3359 Some(step),
3360 TriggerKind::Timer,
3361 &timer.timer_id,
3362 None,
3363 None,
3364 )
3365 .await?;
3366 self.issue_command(
3367 instance, step_def, phase, attempt, None, None, now_ms, version,
3368 )
3369 .await?;
3370 Ok(Some(required_status))
3371 }
3372
3373 async fn on_instance_deadline(
3383 &self,
3384 instance: &SagaInstanceRow,
3385 definition: &WorkflowDefinition,
3386 timer: &SagaTimerRow,
3387 now_ms: i64,
3388 ) -> anyhow::Result<Option<SagaStatus>> {
3389 match instance.status {
3390 SagaStatus::Running => {
3392 let Some(step) = instance.current_step.clone() else {
3393 return Ok(None);
3394 };
3395 self.on_step_timeout(instance, definition, Some(&step), timer, now_ms)
3396 .await
3397 }
3398 SagaStatus::Cancelling | SagaStatus::WaitingResolution | SagaStatus::Compensating => {
3401 Ok(Some(
3402 self.escalate_manual(
3403 instance,
3404 TriggerKind::Timer,
3405 &timer.timer_id,
3406 "deadline_exceeded",
3407 )
3408 .await?,
3409 ))
3410 }
3411 _ => Ok(None),
3412 }
3413 }
3414
3415 async fn proceed_forward(&self, context: &ResultContext<'_>) -> anyhow::Result<SagaStatus> {
3427 let saga_id = &context.identity.saga_id;
3428 let steps = self.backend.store().load_steps(saga_id).await?;
3429 match next_pending_forward(context.definition, &steps)? {
3430 Some(next) => {
3431 let version = self
3432 .advance(
3433 context.instance,
3434 SagaStatus::Running,
3435 Direction::Forward,
3436 Some(&next),
3437 TriggerKind::Event,
3438 context.event_id,
3439 None,
3440 None,
3441 )
3442 .await?;
3443 let next_def = context
3444 .definition
3445 .step(&next)
3446 .ok_or_else(|| anyhow::anyhow!("next step absent from definition"))?;
3447 let attempt = self
3448 .next_attempt(saga_id, &next, StepPhase::Execute)
3449 .await?;
3450 self.issue_command(
3452 context.instance,
3453 next_def,
3454 StepPhase::Execute,
3455 attempt,
3456 None,
3457 None,
3458 context.now_ms,
3459 version,
3460 )
3461 .await?;
3462 Ok(SagaStatus::Running)
3463 }
3464 None => {
3465 self.advance(
3466 context.instance,
3467 SagaStatus::Completed,
3468 Direction::Forward,
3469 None,
3470 TriggerKind::Event,
3471 context.event_id,
3472 None,
3473 None,
3474 )
3475 .await?;
3476 self.backend
3478 .store()
3479 .cancel_scope_timers(saga_id, TimerScope::Instance, None)
3480 .await?;
3481 Ok(SagaStatus::Completed)
3482 }
3483 }
3484 }
3485
3486 async fn enter_compensation(&self, context: &ResultContext<'_>) -> anyhow::Result<SagaStatus> {
3497 let saga_id = &context.identity.saga_id;
3498 let steps = self.backend.store().load_steps(saga_id).await?;
3499 let journal: Vec<StepJournalEntry> = steps
3500 .iter()
3501 .map(|row| StepJournalEntry::new(row.step.clone(), row.forward_status))
3502 .collect();
3503 let plan = match freeze_compensation_plan(context.definition, &journal) {
3506 Ok(plan) => plan,
3507 Err(violation) => {
3508 return self
3509 .escalate_manual(
3510 context.instance,
3511 TriggerKind::Event,
3512 context.event_id,
3513 violation.code(),
3514 )
3515 .await;
3516 }
3517 };
3518 let first = plan.entries().first().map(|entry| entry.step().clone());
3519 let version = self
3520 .advance(
3521 context.instance,
3522 SagaStatus::Compensating,
3523 Direction::Compensating,
3524 first.as_ref(),
3525 TriggerKind::Event,
3526 context.event_id,
3527 None,
3528 Some(plan.version()),
3529 )
3530 .await?;
3531 self.backend
3532 .store()
3533 .mark_compensation_plan(saga_id, &plan)
3534 .await?;
3535 match first {
3536 Some(step) => {
3537 let step_def = context
3538 .definition
3539 .step(&step)
3540 .ok_or_else(|| anyhow::anyhow!("plan step absent from definition"))?;
3541 let attempt = self
3542 .next_attempt(saga_id, &step, StepPhase::Compensate)
3543 .await?;
3544 self.issue_command(
3545 context.instance,
3546 step_def,
3547 StepPhase::Compensate,
3548 attempt,
3549 None,
3550 None,
3551 context.now_ms,
3552 version,
3553 )
3554 .await?;
3555 Ok(SagaStatus::Compensating)
3556 }
3557 None => {
3558 self.converge_compensated(
3560 context.instance,
3561 version,
3562 TriggerKind::Event,
3563 context.event_id,
3564 )
3565 .await?;
3566 Ok(SagaStatus::Compensated)
3567 }
3568 }
3569 }
3570
3571 async fn continue_compensation(
3584 &self,
3585 context: &ResultContext<'_>,
3586 ) -> anyhow::Result<SagaStatus> {
3587 let saga_id = &context.identity.saga_id;
3588 let steps = self.backend.store().load_steps(saga_id).await?;
3589 let next =
3590 match next_pending_compensation(context.instance, context.definition, &steps, false) {
3591 Ok(next) => next,
3592 Err(_) => {
3593 return self
3596 .escalate_manual(
3597 context.instance,
3598 TriggerKind::Event,
3599 context.event_id,
3600 "compensation_plan_drift",
3601 )
3602 .await;
3603 }
3604 };
3605 let version = self
3606 .advance(
3607 context.instance,
3608 SagaStatus::Compensating,
3609 Direction::Compensating,
3610 next.as_ref(),
3611 TriggerKind::Event,
3612 context.event_id,
3613 None,
3614 None,
3615 )
3616 .await?;
3617 match next {
3618 Some(step) => {
3619 let step_def = context
3620 .definition
3621 .step(&step)
3622 .ok_or_else(|| anyhow::anyhow!("plan step absent from definition"))?;
3623 let attempt = self
3624 .next_attempt(saga_id, &step, StepPhase::Compensate)
3625 .await?;
3626 self.issue_command(
3627 context.instance,
3628 step_def,
3629 StepPhase::Compensate,
3630 attempt,
3631 None,
3632 None,
3633 context.now_ms,
3634 version,
3635 )
3636 .await?;
3637 Ok(SagaStatus::Compensating)
3638 }
3639 None => {
3640 self.converge_compensated_at(
3641 saga_id,
3642 version,
3643 TriggerKind::Event,
3644 context.event_id,
3645 )
3646 .await?;
3647 Ok(SagaStatus::Compensated)
3648 }
3649 }
3650 }
3651
3652 async fn enter_waiting_resolution(
3660 &self,
3661 context: &ResultContext<'_>,
3662 direction: Direction,
3663 ) -> anyhow::Result<SagaStatus> {
3664 let saga_id = &context.identity.saga_id;
3665 let step = &context.identity.step;
3666 let resolution = context.step_def.resolution();
3667 if !resolution.allow_unknown() {
3670 if context.identity.phase == StepPhase::Execute {
3671 self.patch_step(
3672 saga_id,
3673 step,
3674 StepJournalPatch {
3675 forward_status: Some(StepForwardStatus::Unknown),
3676 ..Default::default()
3677 },
3678 )
3679 .await?;
3680 }
3681 return self
3684 .escalate_manual(
3685 context.instance,
3686 TriggerKind::Event,
3687 context.event_id,
3688 "unknown_not_allowed",
3689 )
3690 .await;
3691 }
3692 if context.identity.phase == StepPhase::Execute {
3693 self.patch_step(
3694 saga_id,
3695 step,
3696 StepJournalPatch {
3697 forward_status: Some(StepForwardStatus::Unknown),
3698 resolution_status: Some(StepResolutionStatus::Pending),
3699 ..Default::default()
3700 },
3701 )
3702 .await?;
3703 } else {
3704 self.patch_step(
3705 saga_id,
3706 step,
3707 StepJournalPatch {
3708 resolution_status: Some(StepResolutionStatus::Pending),
3709 ..Default::default()
3710 },
3711 )
3712 .await?;
3713 }
3714 self.backend
3715 .store()
3716 .cancel_scope_timers(saga_id, TimerScope::Step(step), Some(KIND_STEP_TIMEOUT))
3717 .await?;
3718 let version = self
3719 .advance(
3720 context.instance,
3721 SagaStatus::WaitingResolution,
3722 direction,
3723 Some(step),
3724 TriggerKind::Event,
3725 context.event_id,
3726 None,
3727 None,
3728 )
3729 .await?;
3730 self.schedule_timer(
3732 saga_id,
3733 TimerScope::Step(step),
3734 resolution_budget_kind(direction),
3735 context.identity.attempt,
3736 checked_deadline(context.now_ms, duration_millis(resolution.budget())?)?,
3737 version,
3738 )
3739 .await?;
3740 if resolution.mode() == Some(ResolutionMode::Poll) {
3741 let attempt = self.next_attempt(saga_id, step, StepPhase::Resolve).await?;
3742 self.issue_command(
3743 context.instance,
3744 context.step_def,
3745 StepPhase::Resolve,
3746 attempt,
3747 None,
3748 None,
3749 context.now_ms,
3750 version,
3751 )
3752 .await?;
3753 }
3754 Ok(SagaStatus::WaitingResolution)
3755 }
3756
3757 async fn reissue_resolve(&self, context: &ResultContext<'_>) -> anyhow::Result<SagaStatus> {
3764 let saga_id = &context.identity.saga_id;
3765 let step = &context.identity.step;
3766 let used = self
3767 .backend
3768 .store()
3769 .count_resolution_attempts(saga_id, step, context.instance.direction)
3770 .await?;
3771 if used >= self.config.resolve_max_attempts {
3772 return self
3773 .escalate_manual(
3774 context.instance,
3775 TriggerKind::Event,
3776 context.event_id,
3777 "resolve_attempts_exhausted",
3778 )
3779 .await;
3780 }
3781 let attempt = self.next_attempt(saga_id, step, StepPhase::Resolve).await?;
3784 let version = self
3785 .advance(
3786 context.instance,
3787 SagaStatus::WaitingResolution,
3788 context.instance.direction,
3789 Some(step),
3790 TriggerKind::Event,
3791 context.event_id,
3792 None,
3793 None,
3794 )
3795 .await?;
3796 self.issue_command(
3797 context.instance,
3798 context.step_def,
3799 StepPhase::Resolve,
3800 attempt,
3801 None,
3802 None,
3803 context.now_ms,
3804 version,
3805 )
3806 .await?;
3807 Ok(SagaStatus::WaitingResolution)
3808 }
3809
3810 async fn escalate_manual(
3820 &self,
3821 instance: &SagaInstanceRow,
3822 trigger_kind: TriggerKind,
3823 trigger_id: &str,
3824 failure_code: &str,
3825 ) -> anyhow::Result<SagaStatus> {
3826 self.advance(
3827 instance,
3828 SagaStatus::ManualIntervention,
3829 instance.direction,
3830 instance.current_step.as_ref(),
3831 trigger_kind,
3832 trigger_id,
3833 Some(failure_code),
3834 None,
3835 )
3836 .await?;
3837 Ok(SagaStatus::ManualIntervention)
3838 }
3839
3840 async fn converge_compensated(
3849 &self,
3850 instance: &SagaInstanceRow,
3851 version: u64,
3852 trigger_kind: TriggerKind,
3853 cause_id: &str,
3854 ) -> anyhow::Result<()> {
3855 self.converge_compensated_at(&instance.saga_id, version, trigger_kind, cause_id)
3856 .await
3857 }
3858
3859 async fn converge_compensated_at(
3868 &self,
3869 saga_id: &SagaId,
3870 version: u64,
3871 trigger_kind: TriggerKind,
3872 cause_id: &str,
3873 ) -> anyhow::Result<()> {
3874 let definition_version = self
3877 .verify_compensation_completion(saga_id, version)
3878 .await?;
3879 let converge_trigger = derive_converge_trigger_id(cause_id);
3882 let outcome = self
3883 .backend
3884 .store()
3885 .advance(
3886 saga_id,
3887 version,
3888 SagaStatus::Compensating,
3889 &TransitionSpec {
3890 to_status: SagaStatus::Compensated,
3891 direction: Direction::Compensating,
3892 current_step: None,
3893 trigger_kind,
3894 trigger_id: &converge_trigger,
3895 definition_version,
3896 failure_code: None,
3897 compensation_plan_version: None,
3898 },
3899 )
3900 .await?;
3901 let _ = ensure_applied(outcome)?;
3902 self.backend
3903 .store()
3904 .cancel_scope_timers(saga_id, TimerScope::Instance, None)
3905 .await?;
3906 Ok(())
3907 }
3908
3909 async fn verify_compensation_completion(
3918 &self,
3919 saga_id: &SagaId,
3920 expected_version: u64,
3921 ) -> anyhow::Result<DefinitionVersion> {
3922 let instance = self
3923 .backend
3924 .store()
3925 .load_instance(saga_id)
3926 .await?
3927 .ok_or_else(|| anyhow::anyhow!("saga instance not found while converging"))?;
3928 if instance.status != SagaStatus::Compensating || instance.version != expected_version {
3929 anyhow::bail!("compensation completion proof used a stale instance snapshot");
3930 }
3931 let frozen_version = instance
3932 .compensation_plan_version
3933 .as_deref()
3934 .ok_or_else(|| anyhow::anyhow!("compensating instance has no frozen plan"))?;
3935 let registry = self.registry_snapshot();
3936 let definition = registry
3937 .get_for_tenant(
3938 &instance.tenant,
3939 &instance.workflow,
3940 instance.definition_version,
3941 )
3942 .ok_or_else(|| anyhow::anyhow!("instance definition is not registered"))?;
3943 let steps = self.backend.store().load_steps(saga_id).await?;
3944 let journal: Vec<StepJournalEntry> = steps
3945 .iter()
3946 .map(|row| StepJournalEntry::new(row.step.clone(), row.forward_status))
3947 .collect();
3948 let rebuilt = freeze_compensation_plan(definition, &journal)
3949 .map_err(|violation| anyhow::anyhow!("{}", violation.code()))?;
3950 if rebuilt.version() != frozen_version {
3951 anyhow::bail!("frozen compensation plan no longer matches durable forward facts");
3952 }
3953 for entry in rebuilt.entries() {
3954 let row = steps
3955 .iter()
3956 .find(|row| row.step == *entry.step())
3957 .ok_or_else(|| anyhow::anyhow!("frozen compensation member is missing"))?;
3958 if row.compensation_plan_version.as_deref() != Some(frozen_version)
3959 || row.compensation_order != Some(entry.order())
3960 || row.compensation_status != StepCompensationStatus::Succeeded
3961 {
3962 anyhow::bail!("frozen compensation member is not durably succeeded");
3963 }
3964 }
3965 let marked_count = steps
3966 .iter()
3967 .filter(|row| row.compensation_plan_version.as_deref() == Some(frozen_version))
3968 .count();
3969 if marked_count != rebuilt.entries().len() {
3970 anyhow::bail!("durable compensation plan contains unexpected members");
3971 }
3972 Ok(instance.definition_version)
3973 }
3974
3975 #[allow(clippy::too_many_arguments)]
3988 async fn advance(
3989 &self,
3990 instance: &SagaInstanceRow,
3991 to: SagaStatus,
3992 direction: Direction,
3993 current_step: Option<&StepName>,
3994 trigger_kind: TriggerKind,
3995 trigger_id: &str,
3996 failure_code: Option<&str>,
3997 plan_version: Option<&str>,
3998 ) -> anyhow::Result<u64> {
3999 self.advance_from(
4000 instance,
4001 instance.status,
4002 to,
4003 direction,
4004 current_step,
4005 trigger_kind,
4006 trigger_id,
4007 failure_code,
4008 plan_version,
4009 )
4010 .await
4011 }
4012
4013 #[allow(clippy::too_many_arguments)]
4020 async fn advance_from(
4021 &self,
4022 instance: &SagaInstanceRow,
4023 from: SagaStatus,
4024 to: SagaStatus,
4025 direction: Direction,
4026 current_step: Option<&StepName>,
4027 trigger_kind: TriggerKind,
4028 trigger_id: &str,
4029 failure_code: Option<&str>,
4030 plan_version: Option<&str>,
4031 ) -> anyhow::Result<u64> {
4032 let outcome = self
4033 .backend
4034 .store()
4035 .advance(
4036 &instance.saga_id,
4037 instance.version,
4038 from,
4039 &TransitionSpec {
4040 to_status: to,
4041 direction,
4042 current_step,
4043 trigger_kind,
4044 trigger_id,
4045 definition_version: instance.definition_version,
4046 failure_code,
4047 compensation_plan_version: plan_version,
4048 },
4049 )
4050 .await?;
4051 ensure_applied(outcome)
4052 }
4053
4054 #[allow(clippy::too_many_arguments)]
4068 async fn issue_command(
4069 &self,
4070 instance: &SagaInstanceRow,
4071 step_def: &StepDefinition,
4072 phase: StepPhase,
4073 attempt: AttemptNo,
4074 payload: Option<CommandInput>,
4075 recovery_operation_id: Option<&str>,
4076 now_ms: i64,
4077 expected_version: u64,
4078 ) -> anyhow::Result<()> {
4079 let step = step_def.name();
4080 let effect = EffectId::derive(&instance.saga_id, instance.definition_version, step, phase);
4081 let command = CommandId::derive(&effect, attempt);
4082 self.backend
4084 .store()
4085 .record_attempt_started(&instance.saga_id, step, phase, attempt, &effect, &command)
4086 .await?;
4087 let (payload, raw_payload) = match payload {
4089 Some(CommandInput::Json(value)) => (Some(value), None),
4090 Some(CommandInput::Raw(value)) => (None, Some(value)),
4091 None => (None, None),
4092 };
4093 let envelope = SagaCommandEnvelope {
4094 saga_id: instance.saga_id.as_str().to_string(),
4095 tenant_id: instance.tenant.as_str().to_string(),
4096 workflow: instance.workflow.as_str().to_string(),
4097 definition_version: instance.definition_version.get(),
4098 definition_digest: instance.definition_digest.clone(),
4099 step: step.as_str().to_string(),
4100 phase: phase.as_str().to_string(),
4101 attempt: attempt.get(),
4102 effect_id: effect.to_string(),
4103 command_id: command.to_string(),
4104 recovery_operation_id: recovery_operation_id.map(str::to_owned),
4105 payload,
4106 raw_payload,
4107 };
4108 let event = match instance
4112 .traceparent
4113 .as_deref()
4114 .and_then(TraceContext::parse_traceparent)
4115 {
4116 Some(base) => envelope
4117 .to_outbox_event()?
4118 .with_traceparent(base.child(natelemetry::random_span_id()).to_traceparent()),
4119 None => envelope.to_outbox_event()?,
4120 };
4121 let write_context = naoutbox_core::OutboxWriteContext::new(instance.tenant.as_str())
4125 .map_err(|reason| anyhow::anyhow!("outbox write context rejected: {reason}"))?;
4126 self.backend
4127 .outbox()
4128 .append_transactional_with_context(&write_context, &event)
4129 .await?;
4130 let timeout_ms = duration_millis(step_def.timeout())?;
4132 self.schedule_timer(
4133 &instance.saga_id,
4134 TimerScope::Step(step),
4135 phase_timeout_kind(phase),
4136 attempt,
4137 checked_deadline(now_ms, timeout_ms)?,
4138 expected_version,
4139 )
4140 .await?;
4141 Ok(())
4142 }
4143
4144 async fn schedule_timer(
4156 &self,
4157 saga_id: &SagaId,
4158 scope: TimerScope<'_>,
4159 kind: &str,
4160 attempt: AttemptNo,
4161 due_at_ms: i64,
4162 expected_version: u64,
4163 ) -> anyhow::Result<()> {
4164 let timer_id = derive_timer_id(saga_id, &scope, kind, attempt);
4165 self.backend
4166 .store()
4167 .schedule_timer(&TimerSpec {
4168 timer_id: &timer_id,
4169 saga_id,
4170 scope,
4171 kind,
4172 due_at_ms,
4173 attempt,
4174 expected_saga_version: expected_version,
4175 })
4176 .await?;
4177 Ok(())
4178 }
4179
4180 async fn cancel_resolution_timers(
4188 &self,
4189 saga_id: &SagaId,
4190 step: &StepName,
4191 ) -> anyhow::Result<()> {
4192 for kind in [
4195 KIND_RESOLUTION_BUDGET,
4196 KIND_FORWARD_RESOLUTION_BUDGET,
4197 KIND_COMPENSATION_RESOLUTION_BUDGET,
4198 ] {
4199 self.backend
4200 .store()
4201 .cancel_scope_timers(saga_id, TimerScope::Step(step), Some(kind))
4202 .await?;
4203 }
4204 self.backend
4205 .store()
4206 .cancel_scope_timers(saga_id, TimerScope::Step(step), Some(KIND_RESOLVE_TIMEOUT))
4207 .await?;
4208 Ok(())
4209 }
4210
4211 async fn patch_step(
4220 &self,
4221 saga_id: &SagaId,
4222 step: &StepName,
4223 patch: StepJournalPatch<'_>,
4224 ) -> anyhow::Result<()> {
4225 self.backend
4226 .store()
4227 .update_step_journal(saga_id, step, &patch)
4228 .await?;
4229 Ok(())
4230 }
4231
4232 async fn next_attempt(
4241 &self,
4242 saga_id: &SagaId,
4243 step: &StepName,
4244 phase: StepPhase,
4245 ) -> anyhow::Result<AttemptNo> {
4246 let attempts = self.backend.store().load_attempts(saga_id, step).await?;
4247 let max = attempts
4248 .iter()
4249 .filter(|row| row.phase == phase)
4250 .map(|row| row.attempt.get())
4251 .max();
4252 match max {
4253 None => Ok(AttemptNo::FIRST),
4254 Some(value) => AttemptNo::new(value)
4255 .and_then(|attempt| attempt.next())
4256 .map_err(|violation| anyhow::anyhow!("attempt overflow: {}", violation.code())),
4257 }
4258 }
4259}
4260
4261fn verify_instance_contract(
4272 instance: &SagaInstanceRow,
4273 identity: &VerifiedIdentity,
4274) -> anyhow::Result<()> {
4275 if instance.tenant != identity.tenant {
4276 anyhow::bail!("envelope tenant does not match the instance tenant");
4277 }
4278 if instance.workflow != identity.workflow
4279 || instance.definition_version != identity.definition_version
4280 {
4281 anyhow::bail!("envelope workflow/version does not match the instance");
4282 }
4283 if instance.definition_digest != identity.definition_digest {
4284 anyhow::bail!("envelope definition digest does not match the instance");
4285 }
4286 Ok(())
4287}
4288
4289fn conflicting_projection_status(
4301 row: &SagaStepRow,
4302 context: &ResultContext<'_>,
4303) -> Option<StepAttemptStatus> {
4304 let forward = || match row.forward_status {
4305 StepForwardStatus::Succeeded => Some(StepAttemptStatus::Succeeded),
4306 StepForwardStatus::Rejected => Some(StepAttemptStatus::Rejected),
4307 StepForwardStatus::Cancelled => Some(StepAttemptStatus::CancelConfirmed),
4308 StepForwardStatus::Halted => Some(StepAttemptStatus::Halted),
4309 StepForwardStatus::Pending | StepForwardStatus::Unknown | StepForwardStatus::TimedOut => {
4310 None
4311 }
4312 };
4313 let cancellation = || match row.cancel_status {
4314 StepCancelStatus::Confirmed => Some(StepAttemptStatus::CancelConfirmed),
4315 StepCancelStatus::AlreadyTerminal => Some(StepAttemptStatus::AlreadyTerminal),
4316 StepCancelStatus::None | StepCancelStatus::ResolutionPending => None,
4317 };
4318 let compensation = || match row.compensation_status {
4319 StepCompensationStatus::Succeeded => Some(StepAttemptStatus::Succeeded),
4320 StepCompensationStatus::Halted => Some(StepAttemptStatus::Halted),
4321 StepCompensationStatus::None
4322 | StepCompensationStatus::Pending
4323 | StepCompensationStatus::Unknown => None,
4324 };
4325 let resolution = || match row.resolution_status {
4326 StepResolutionStatus::Succeeded => Some(StepAttemptStatus::Succeeded),
4327 StepResolutionStatus::Rejected => Some(StepAttemptStatus::Rejected),
4328 StepResolutionStatus::Halted => Some(StepAttemptStatus::Halted),
4329 StepResolutionStatus::None | StepResolutionStatus::Pending => None,
4330 };
4331
4332 match context.identity.phase {
4333 StepPhase::Execute => forward().or_else(cancellation),
4334 StepPhase::Cancel => cancellation().or_else(forward),
4335 StepPhase::Compensate => compensation().or_else(resolution),
4336 StepPhase::Resolve => resolution().or_else(|| {
4337 if context.instance.direction == Direction::Compensating {
4338 compensation()
4339 } else {
4340 forward()
4341 }
4342 }),
4343 }
4344}
4345
4346fn merge_deferred_projection(
4358 row: &SagaStepRow,
4359 patch: &mut StepJournalPatch<'_>,
4360) -> Result<(), ()> {
4361 if let Some(next) = patch.forward_status {
4362 patch.forward_status = if row.forward_status == next {
4363 None
4364 } else {
4365 match (row.forward_status, next) {
4366 (
4367 StepForwardStatus::Pending
4368 | StepForwardStatus::Unknown
4369 | StepForwardStatus::TimedOut,
4370 _,
4371 ) => Some(next),
4372 (_, StepForwardStatus::Unknown | StepForwardStatus::TimedOut) => None,
4374 _ => return Err(()),
4375 }
4376 };
4377 }
4378
4379 if let Some(next) = patch.cancel_status {
4380 patch.cancel_status = if row.cancel_status == next {
4381 None
4382 } else {
4383 match (row.cancel_status, next) {
4384 (StepCancelStatus::None | StepCancelStatus::ResolutionPending, _) => Some(next),
4385 (_, StepCancelStatus::ResolutionPending) => None,
4387 _ => return Err(()),
4388 }
4389 };
4390 }
4391
4392 if let Some(next) = patch.compensation_status {
4393 patch.compensation_status = if row.compensation_status == next {
4394 None
4395 } else {
4396 match (row.compensation_status, next) {
4397 (
4398 StepCompensationStatus::None
4399 | StepCompensationStatus::Pending
4400 | StepCompensationStatus::Unknown,
4401 _,
4402 ) => Some(next),
4403 (_, StepCompensationStatus::Unknown | StepCompensationStatus::Pending) => None,
4405 _ => return Err(()),
4406 }
4407 };
4408 }
4409
4410 if let Some(next) = patch.resolution_status {
4411 patch.resolution_status = if row.resolution_status == next {
4412 None
4413 } else {
4414 match (row.resolution_status, next) {
4415 (StepResolutionStatus::None | StepResolutionStatus::Pending, _) => Some(next),
4416 (_, StepResolutionStatus::Pending) => None,
4418 _ => return Err(()),
4419 }
4420 };
4421 }
4422 Ok(())
4423}
4424
4425fn ensure_applied(outcome: CasOutcome) -> anyhow::Result<u64> {
4435 match outcome {
4436 CasOutcome::Applied { new_version } => Ok(new_version),
4437 CasOutcome::Conflict => Err(SagaConcurrencyError::StaleSnapshot.into()),
4438 CasOutcome::DuplicateTrigger => Err(SagaConcurrencyError::DuplicateTrigger.into()),
4439 }
4440}
4441
4442fn next_pending_forward(
4451 definition: &WorkflowDefinition,
4452 steps: &[SagaStepRow],
4453) -> anyhow::Result<Option<StepName>> {
4454 if steps.len() != definition.steps().len() {
4455 anyhow::bail!("step journal does not completely cover the workflow definition");
4456 }
4457 let mut next = None;
4458 for (index, step_def) in definition.steps().iter().enumerate() {
4459 let row = &steps[index];
4460 if row.step != *step_def.name() || row.ordinal != index as u32 + 1 {
4461 anyhow::bail!("step journal order does not match the workflow definition");
4462 }
4463 match (next.is_some(), row.forward_status) {
4464 (false, StepForwardStatus::Succeeded) => {}
4465 (false, StepForwardStatus::Pending) => next = Some(row.step.clone()),
4466 (true, StepForwardStatus::Pending) => {}
4467 _ => anyhow::bail!("forward journal is not a succeeded-prefix/pending-suffix"),
4470 }
4471 }
4472 Ok(next)
4473}
4474
4475fn next_pending_compensation(
4487 instance: &SagaInstanceRow,
4488 definition: &WorkflowDefinition,
4489 steps: &[SagaStepRow],
4490 allow_halted_recovery: bool,
4491) -> anyhow::Result<Option<StepName>> {
4492 let frozen_version = instance
4493 .compensation_plan_version
4494 .as_deref()
4495 .ok_or_else(|| anyhow::anyhow!("compensating instance has no frozen plan"))?;
4496 let journal: Vec<StepJournalEntry> = steps
4497 .iter()
4498 .map(|row| StepJournalEntry::new(row.step.clone(), row.forward_status))
4499 .collect();
4500 let rebuilt = freeze_compensation_plan(definition, &journal)
4501 .map_err(|violation| anyhow::anyhow!("{}", violation.code()))?;
4502 if rebuilt.version() != frozen_version {
4503 anyhow::bail!("frozen compensation plan digest drifted");
4504 }
4505
4506 let mut next = None;
4507 for entry in rebuilt.entries() {
4508 let row = steps
4509 .iter()
4510 .find(|row| row.step == *entry.step())
4511 .ok_or_else(|| anyhow::anyhow!("frozen compensation member is missing"))?;
4512 if row.compensation_plan_version.as_deref() != Some(frozen_version)
4513 || row.compensation_order != Some(entry.order())
4514 {
4515 anyhow::bail!("durable compensation member metadata drifted");
4516 }
4517 match (next.is_some(), row.compensation_status) {
4518 (false, StepCompensationStatus::Succeeded) => {}
4519 (false, StepCompensationStatus::Pending) => next = Some(row.step.clone()),
4520 (false, StepCompensationStatus::Halted) if allow_halted_recovery => {
4521 next = Some(row.step.clone())
4522 }
4523 (true, StepCompensationStatus::Pending) => {}
4524 _ => anyhow::bail!("compensation journal is not a succeeded-prefix/pending-suffix"),
4525 }
4526 }
4527 let marked_count = steps
4528 .iter()
4529 .filter(|row| row.compensation_plan_version.as_deref() == Some(frozen_version))
4530 .count();
4531 if marked_count != rebuilt.entries().len() {
4532 anyhow::bail!("durable compensation plan contains unexpected members");
4533 }
4534 Ok(next)
4535}