1use std::{fmt, sync::Arc, time::Duration};
7
8use serde_json::{Value, json};
9use stateknot_core::{
10 AgentArtifacts, AgentDescriptor, AgentRequest, AgentResult, AgentResultError,
11 AgentResultProvenance, AgentResultValidationError, BoundedJson, BoxFuture, BudgetUsage,
12 CheckpointHead, CheckpointId, Digest, DurableWaitError, EventId, Failure, FailureCategory,
13 FailureCode, FailureId, FailureMessage, FailureOrigin, GraphBarrierDisposition, GraphReference,
14 GraphSchemaValidationError, JournalAppend, JournalEventIntent, JournalEventKind,
15 JournalExpectation, JournalHead, JournalPayload, ResolvedBudget, RetryAdvice, RunFailure,
16 RunFailureError, RunFence, RunLease, RunRevision, RunStatus, RunTransition, SchemaReference,
17 Superstep, Timestamp,
18};
19use stateknot_store_postgres::{
20 AppendOutcome, BarrierCommitOutcome, LeaseReleaseOutcome, PostgresStore, RunProjection,
21 StoreError, StoredRun, WaitCheckpointCommitOutcome,
22};
23use thiserror::Error;
24
25use crate::{
26 ExecutableGraphRegistry, GraphBlockedHandoff, GraphCancellationHandoff, GraphDriveBlockers,
27 GraphLifecycleBarrierHandoff, StandardAgentCancellationSchemaError,
28 StandardGraphLifecycleSchemaError, standard_agent_cancellation_event_schema,
29 standard_graph_lifecycle_event_schema,
30};
31
32const MAX_MUTATION_RETRY_DELAY: Duration = Duration::from_secs(1);
33
34#[derive(Clone, Copy, Debug, Eq, PartialEq)]
36pub struct DurableGraphLifecycleOptions {
37 maximum_mutation_attempts: u8,
38 mutation_retry_initial_delay: Duration,
39}
40
41impl DurableGraphLifecycleOptions {
42 pub const HARD_MAXIMUM_MUTATION_ATTEMPTS: u8 = 10;
44
45 pub fn new(
52 maximum_mutation_attempts: u8,
53 mutation_retry_initial_delay: Duration,
54 ) -> Result<Self, DurableGraphLifecycleOptionsError> {
55 if maximum_mutation_attempts == 0
56 || maximum_mutation_attempts > Self::HARD_MAXIMUM_MUTATION_ATTEMPTS
57 {
58 return Err(DurableGraphLifecycleOptionsError::InvalidMutationAttempts);
59 }
60 if mutation_retry_initial_delay.is_zero()
61 || mutation_retry_initial_delay > MAX_MUTATION_RETRY_DELAY
62 {
63 return Err(DurableGraphLifecycleOptionsError::InvalidMutationRetryDelay);
64 }
65 Ok(Self {
66 maximum_mutation_attempts,
67 mutation_retry_initial_delay,
68 })
69 }
70
71 #[must_use]
73 pub const fn maximum_mutation_attempts(self) -> u8 {
74 self.maximum_mutation_attempts
75 }
76
77 #[must_use]
79 pub const fn mutation_retry_initial_delay(self) -> Duration {
80 self.mutation_retry_initial_delay
81 }
82}
83
84impl Default for DurableGraphLifecycleOptions {
85 fn default() -> Self {
86 Self {
87 maximum_mutation_attempts: 3,
88 mutation_retry_initial_delay: Duration::from_millis(25),
89 }
90 }
91}
92
93#[derive(Clone, Copy, Debug, Eq, Error, PartialEq)]
95#[non_exhaustive]
96pub enum DurableGraphLifecycleOptionsError {
97 #[error("graph lifecycle mutation attempt count is invalid")]
99 InvalidMutationAttempts,
100 #[error("graph lifecycle mutation retry delay is invalid")]
102 InvalidMutationRetryDelay,
103}
104
105#[derive(Clone, Debug)]
107pub struct GraphTerminalEvidenceContext {
108 provenance: AgentResultProvenance,
109 graph: GraphReference,
110 checkpoint: CheckpointHead,
111 successor_checkpoint_id: CheckpointId,
112 successor_superstep: Superstep,
113 output_schema: SchemaReference,
114 output_digest: Digest,
115 expected_revision: RunRevision,
116}
117
118impl GraphTerminalEvidenceContext {
119 #[must_use]
121 pub const fn provenance(&self) -> &AgentResultProvenance {
122 &self.provenance
123 }
124
125 #[must_use]
127 pub const fn graph(&self) -> &GraphReference {
128 &self.graph
129 }
130
131 #[must_use]
133 pub const fn checkpoint(&self) -> &CheckpointHead {
134 &self.checkpoint
135 }
136
137 #[must_use]
139 pub const fn successor_checkpoint_id(&self) -> CheckpointId {
140 self.successor_checkpoint_id
141 }
142
143 #[must_use]
145 pub const fn successor_superstep(&self) -> Superstep {
146 self.successor_superstep
147 }
148
149 #[must_use]
151 pub const fn output_schema(&self) -> &SchemaReference {
152 &self.output_schema
153 }
154
155 #[must_use]
157 pub const fn output_digest(&self) -> Digest {
158 self.output_digest
159 }
160
161 #[must_use]
163 pub const fn expected_revision(&self) -> RunRevision {
164 self.expected_revision
165 }
166}
167
168#[derive(Clone)]
170pub struct GraphTerminalEvidence {
171 descriptor: AgentDescriptor,
172 request: AgentRequest,
173 budget: ResolvedBudget,
174 artifacts: AgentArtifacts,
175 usage: BudgetUsage,
176}
177
178impl GraphTerminalEvidence {
179 #[must_use]
181 pub const fn new(
182 descriptor: AgentDescriptor,
183 request: AgentRequest,
184 budget: ResolvedBudget,
185 artifacts: AgentArtifacts,
186 usage: BudgetUsage,
187 ) -> Self {
188 Self {
189 descriptor,
190 request,
191 budget,
192 artifacts,
193 usage,
194 }
195 }
196
197 #[must_use]
199 pub const fn descriptor(&self) -> &AgentDescriptor {
200 &self.descriptor
201 }
202
203 #[must_use]
205 pub const fn request(&self) -> &AgentRequest {
206 &self.request
207 }
208
209 #[must_use]
211 pub const fn budget(&self) -> &ResolvedBudget {
212 &self.budget
213 }
214
215 #[must_use]
217 pub const fn artifacts(&self) -> &AgentArtifacts {
218 &self.artifacts
219 }
220
221 #[must_use]
223 pub const fn usage(&self) -> &BudgetUsage {
224 &self.usage
225 }
226
227 fn into_parts(
228 self,
229 ) -> (
230 AgentDescriptor,
231 AgentRequest,
232 ResolvedBudget,
233 AgentArtifacts,
234 BudgetUsage,
235 ) {
236 (
237 self.descriptor,
238 self.request,
239 self.budget,
240 self.artifacts,
241 self.usage,
242 )
243 }
244}
245
246impl fmt::Debug for GraphTerminalEvidence {
247 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
248 formatter
249 .debug_struct("GraphTerminalEvidence")
250 .field("agent", self.descriptor.metadata().identity())
251 .field("input_schema", self.request.input_schema())
252 .field("output_schema", self.descriptor.output_schema())
253 .field("artifact_count", &self.artifacts.len())
254 .field("usage_recorded", &true)
255 .finish_non_exhaustive()
256 }
257}
258
259#[derive(Clone, Debug)]
261pub struct GraphFailureEvidenceContext {
262 provenance: AgentResultProvenance,
263 graph: GraphReference,
264 checkpoint: CheckpointHead,
265 observed_at: Timestamp,
266 expected_revision: RunRevision,
267 blockers: GraphDriveBlockers,
268}
269
270impl GraphFailureEvidenceContext {
271 #[must_use]
273 pub const fn provenance(&self) -> &AgentResultProvenance {
274 &self.provenance
275 }
276
277 #[must_use]
279 pub const fn graph(&self) -> &GraphReference {
280 &self.graph
281 }
282
283 #[must_use]
285 pub const fn checkpoint(&self) -> &CheckpointHead {
286 &self.checkpoint
287 }
288
289 #[must_use]
291 pub const fn observed_at(&self) -> Timestamp {
292 self.observed_at
293 }
294
295 #[must_use]
297 pub const fn expected_revision(&self) -> RunRevision {
298 self.expected_revision
299 }
300
301 #[must_use]
303 pub const fn blockers(&self) -> GraphDriveBlockers {
304 self.blockers
305 }
306}
307
308#[derive(Clone, Debug)]
310pub struct GraphFailureEvidence {
311 failure: Failure,
312 usage: BudgetUsage,
313}
314
315#[derive(Clone, Debug)]
317pub struct GraphCancellationEvidenceContext {
318 provenance: AgentResultProvenance,
319 graph: GraphReference,
320 checkpoint: CheckpointHead,
321 expected_revision: RunRevision,
322 cancellation_failure_id: FailureId,
323}
324
325impl GraphCancellationEvidenceContext {
326 #[must_use]
328 pub const fn provenance(&self) -> &AgentResultProvenance {
329 &self.provenance
330 }
331
332 #[must_use]
334 pub const fn graph(&self) -> &GraphReference {
335 &self.graph
336 }
337
338 #[must_use]
340 pub const fn checkpoint(&self) -> &CheckpointHead {
341 &self.checkpoint
342 }
343
344 #[must_use]
346 pub const fn expected_revision(&self) -> RunRevision {
347 self.expected_revision
348 }
349
350 #[must_use]
352 pub const fn cancellation_failure_id(&self) -> FailureId {
353 self.cancellation_failure_id
354 }
355}
356
357#[derive(Clone, Debug)]
359pub struct GraphCancellationEvidence {
360 usage: BudgetUsage,
361}
362
363impl GraphCancellationEvidence {
364 #[must_use]
366 pub const fn new(usage: BudgetUsage) -> Self {
367 Self { usage }
368 }
369
370 #[must_use]
372 pub const fn usage(&self) -> &BudgetUsage {
373 &self.usage
374 }
375
376 fn into_usage(self) -> BudgetUsage {
377 self.usage
378 }
379}
380
381impl GraphFailureEvidence {
382 #[must_use]
384 pub const fn new(failure: Failure, usage: BudgetUsage) -> Self {
385 Self { failure, usage }
386 }
387
388 #[must_use]
390 pub const fn failure(&self) -> &Failure {
391 &self.failure
392 }
393
394 #[must_use]
396 pub const fn usage(&self) -> &BudgetUsage {
397 &self.usage
398 }
399
400 fn into_parts(self) -> (Failure, BudgetUsage) {
401 (self.failure, self.usage)
402 }
403}
404
405#[derive(Clone, Copy, Debug, Eq, Error, PartialEq)]
407#[non_exhaustive]
408pub enum GraphLifecycleEvidenceError {
409 #[error("graph lifecycle evidence is temporarily unavailable")]
411 TemporarilyUnavailable,
412 #[error("graph lifecycle evidence is unavailable")]
414 Unavailable,
415 #[error("graph lifecycle evidence is corrupt")]
417 Corrupt,
418}
419
420pub trait GraphLifecycleEvidenceProvider: Send + Sync + 'static {
432 fn terminal_evidence(
434 &self,
435 context: GraphTerminalEvidenceContext,
436 ) -> BoxFuture<'_, Result<GraphTerminalEvidence, GraphLifecycleEvidenceError>>;
437
438 fn failure_evidence(
440 &self,
441 context: GraphFailureEvidenceContext,
442 ) -> BoxFuture<'_, Result<GraphFailureEvidence, GraphLifecycleEvidenceError>>;
443
444 fn cancellation_evidence(
446 &self,
447 context: GraphCancellationEvidenceContext,
448 ) -> BoxFuture<'_, Result<GraphCancellationEvidence, GraphLifecycleEvidenceError>>;
449}
450
451#[derive(Clone, Debug)]
453#[non_exhaustive]
454pub enum GraphBarrierLifecycleOutcome {
455 FailureClosing(Box<stateknot_store_postgres::RunFailureCloseOutcome>),
457 Cancelled(AppendOutcome),
459 Waiting(WaitCheckpointCommitOutcome),
462 Succeeded(BarrierCommitOutcome),
465 Failed(AppendOutcome),
468 Released(LeaseReleaseOutcome),
470}
471
472#[derive(Clone)]
474pub struct DurableGraphLifecycle {
475 store: PostgresStore,
476 registry: ExecutableGraphRegistry,
477 evidence: Arc<dyn GraphLifecycleEvidenceProvider>,
478 journal_schema: SchemaReference,
479 cancellation_schema: SchemaReference,
480 options: DurableGraphLifecycleOptions,
481}
482
483enum GraphLifecycleHandoffSnapshot {
484 Fresh(StoredRun),
485 Committed(StoredRun),
486}
487
488impl DurableGraphLifecycle {
489 pub fn new(
497 store: PostgresStore,
498 registry: ExecutableGraphRegistry,
499 evidence: Arc<dyn GraphLifecycleEvidenceProvider>,
500 options: DurableGraphLifecycleOptions,
501 ) -> Result<Self, DurableGraphLifecycleBuildError> {
502 let (journal_schema, _) = standard_graph_lifecycle_event_schema()?;
503 if !registry.schemas().contains(&journal_schema) {
504 return Err(DurableGraphLifecycleBuildError::JournalSchemaUnavailable);
505 }
506 let (cancellation_schema, _) = standard_agent_cancellation_event_schema()?;
507 if !registry.schemas().contains(&cancellation_schema) {
508 return Err(DurableGraphLifecycleBuildError::CancellationSchemaUnavailable);
509 }
510 if registry.has_child_graphs() {
511 let (schema, _) = crate::standard_run_failure_close_event_schema()?;
512 if !registry.schemas().contains(&schema) {
513 return Err(crate::RunFailureCloseBuildError::SchemaUnavailable.into());
514 }
515 }
516 Ok(Self {
517 store,
518 registry,
519 evidence,
520 journal_schema,
521 cancellation_schema,
522 options,
523 })
524 }
525
526 #[must_use]
528 pub const fn options(&self) -> DurableGraphLifecycleOptions {
529 self.options
530 }
531
532 pub fn confirm_cancellation(
540 &self,
541 handoff: GraphCancellationHandoff,
542 ) -> BoxFuture<'_, Result<GraphBarrierLifecycleOutcome, GraphLifecycleError>> {
543 Box::pin(self.confirm_cancellation_inner(handoff))
544 }
545
546 async fn confirm_cancellation_inner(
547 &self,
548 handoff: GraphCancellationHandoff,
549 ) -> Result<GraphBarrierLifecycleOutcome, GraphLifecycleError> {
550 let (checkpoint, journal_head, event_id, expected_revision, lease, failure_id) =
551 handoff.into_parts();
552 let fence = lease.fence().clone();
553 validate_cancellation_scope(&fence, &journal_head, &checkpoint)?;
554 let run = self
555 .store
556 .load_run(fence.tenant_id(), fence.run_id())
557 .await?;
558 validate_cancellation_checkpoint(&run, &checkpoint)?;
559
560 let fresh = run.lifecycle().revision() == expected_revision
561 && run.lifecycle().status() == RunStatus::CancellationRequested
562 && run.journal_head() == Some(&journal_head)
563 && run.lease().map(RunLease::fence) == Some(&fence)
564 && run
565 .lifecycle()
566 .cancellation_request()
567 .is_some_and(|request| request.failure().id() == failure_id);
568 let (completed_at, usage) = if fresh {
569 let observation = self.store.observe_live_lease(&fence).await?;
570 if observation.lease().fence() != &fence {
571 return Err(GraphLifecycleError::StaleHandoff);
572 }
573 let context = GraphCancellationEvidenceContext {
574 provenance: run.lifecycle().provenance().clone(),
575 graph: checkpoint.graph().clone(),
576 checkpoint: checkpoint.clone(),
577 expected_revision,
578 cancellation_failure_id: failure_id,
579 };
580 let evidence = self
581 .evidence
582 .cancellation_evidence(context)
583 .await
584 .map_err(GraphLifecycleError::Evidence)?;
585 let usage = self
586 .store
587 .include_child_usage(fence.tenant_id(), fence.run_id(), evidence.into_usage())
588 .await?;
589 (observation.observed_at(), usage)
590 } else {
591 let committed_revision = expected_revision.get().checked_add(1);
592 let committed = committed_revision
593 .is_some_and(|revision| run.lifecycle().revision() == RunRevision::new(revision))
594 && run.lifecycle().status() == RunStatus::Cancelled
595 && run
596 .journal_head()
597 .is_some_and(|head| head.event_id() == event_id)
598 && run.lease().is_none()
599 && run
600 .lifecycle()
601 .cancellation_request()
602 .is_some_and(|request| request.failure().id() == failure_id);
603 if !committed {
604 return Err(GraphLifecycleError::StaleHandoff);
605 }
606 let usage = run.lifecycle().terminal_usage().cloned().ok_or(
607 GraphLifecycleError::InvalidHandoff {
608 operation: "recover committed cancellation accounting",
609 },
610 )?;
611 (run.lifecycle().changed_at(), usage)
612 };
613
614 let payload = self.cancelled_payload(&checkpoint, failure_id)?;
615 let append = worker_append(&fence, journal_head, event_id, payload)?;
616 let projection = RunProjection::transition(
617 expected_revision,
618 RunTransition::ConfirmCancellation {
619 completed_at,
620 usage,
621 },
622 );
623 let outcome = self.commit_failure_with_retry(append, projection).await?;
624 Ok(GraphBarrierLifecycleOutcome::Cancelled(outcome))
625 }
626
627 pub fn commit_barrier(
634 &self,
635 handoff: GraphLifecycleBarrierHandoff,
636 ) -> BoxFuture<'_, Result<GraphBarrierLifecycleOutcome, GraphLifecycleError>> {
637 Box::pin(self.commit_barrier_inner(handoff))
638 }
639
640 async fn commit_barrier_inner(
641 &self,
642 handoff: GraphLifecycleBarrierHandoff,
643 ) -> Result<GraphBarrierLifecycleOutcome, GraphLifecycleError> {
644 let (plan, journal_head, event_id, expected_revision, lease) = handoff.into_parts();
645 let fence = lease.fence().clone();
646 let (barrier, disposition) = plan.into_parts();
647 let base = barrier.base_checkpoint().clone();
648 validate_barrier_scope(&fence, &journal_head, &base, barrier.successor().graph())?;
649
650 match disposition {
651 GraphBarrierDisposition::Wait { waits } => {
652 let payload = self.waiting_payload(
658 barrier.successor().graph(),
659 &base,
660 barrier.successor().checkpoint_id(),
661 barrier.successor().superstep(),
662 waits.len(),
663 )?;
664 let append = worker_append(&fence, journal_head, event_id, payload)?;
665 let registrations = waits
666 .registration_intents(fence.tenant_id(), fence.run_id(), event_id)
667 .map_err(GraphLifecycleError::wait_registration)?;
668 let outcome = self
669 .commit_wait_with_retry(append, expected_revision, barrier, registrations)
670 .await?;
671 Ok(GraphBarrierLifecycleOutcome::Waiting(outcome))
672 }
673 GraphBarrierDisposition::Terminal { output } => {
674 let snapshot = self
675 .validate_handoff_snapshot(
676 &fence,
677 &journal_head,
678 event_id,
679 expected_revision,
680 RunStatus::Succeeded,
681 )
682 .await?;
683 let result = match snapshot {
684 GraphLifecycleHandoffSnapshot::Fresh(run) => {
685 let context = GraphTerminalEvidenceContext {
686 provenance: run.lifecycle().provenance().clone(),
687 graph: barrier.successor().graph().clone(),
688 checkpoint: base.clone(),
689 successor_checkpoint_id: barrier.successor().checkpoint_id(),
690 successor_superstep: barrier.successor().superstep(),
691 output_schema: output.schema().clone(),
692 output_digest: output.digest(),
693 expected_revision,
694 };
695 let mut evidence = self
696 .evidence
697 .terminal_evidence(context)
698 .await
699 .map_err(GraphLifecycleError::Evidence)?;
700 evidence.usage = self
701 .store
702 .include_child_usage(fence.tenant_id(), fence.run_id(), evidence.usage)
703 .await?;
704 self.validate_terminal_result(
705 run.lifecycle().provenance(),
706 journal_head.recorded_at(),
707 &output,
708 evidence,
709 )?
710 }
711 GraphLifecycleHandoffSnapshot::Committed(run) => {
712 run.lifecycle().result().cloned().ok_or(
713 GraphLifecycleError::InvalidHandoff {
714 operation: "recover the committed successful result",
715 },
716 )?
717 }
718 };
719 let payload = self.succeeded_payload(
720 barrier.successor().graph(),
721 &base,
722 barrier.successor().checkpoint_id(),
723 barrier.successor().superstep(),
724 output.digest(),
725 )?;
726 let append = worker_append(&fence, journal_head, event_id, payload)?;
727 let projection =
728 RunProjection::transition(expected_revision, RunTransition::Succeed { result });
729 let outcome = self
730 .commit_success_with_retry(append, projection, barrier)
731 .await?;
732 Ok(GraphBarrierLifecycleOutcome::Succeeded(outcome))
733 }
734 GraphBarrierDisposition::Continue => Err(GraphLifecycleError::InvalidHandoff {
735 operation: "commit a Continue disposition through lifecycle coordination",
736 }),
737 _ => Err(GraphLifecycleError::InvalidHandoff {
738 operation: "commit an unsupported graph barrier disposition",
739 }),
740 }
741 }
742
743 pub fn resolve_blocked(
749 &self,
750 handoff: GraphBlockedHandoff,
751 ) -> BoxFuture<'_, Result<GraphBarrierLifecycleOutcome, GraphLifecycleError>> {
752 Box::pin(self.resolve_blocked_inner(handoff))
753 }
754
755 #[allow(clippy::too_many_lines)]
756 async fn resolve_blocked_inner(
757 &self,
758 handoff: GraphBlockedHandoff,
759 ) -> Result<GraphBarrierLifecycleOutcome, GraphLifecycleError> {
760 let (plan, lease, event_id, expected_revision, blockers) = handoff.into_parts();
761 let fence = lease.fence().clone();
762 if plan.fence() != &fence {
763 return Err(GraphLifecycleError::InvalidHandoff {
764 operation: "bind blocked recovery plan to its retained lease",
765 });
766 }
767 if blockers.in_flight() > 0 {
768 let release = self.release_with_retry(&fence).await?;
769 return Ok(GraphBarrierLifecycleOutcome::Released(release));
770 }
771 if blockers.failed() == 0
772 && blockers.exhausted() == 0
773 && blockers.unsupported() == 0
774 && !blockers.superstep_limit_reached()
775 {
776 return Err(GraphLifecycleError::InvalidHandoff {
777 operation: "supervise a blocked plan without terminal blockers",
778 });
779 }
780
781 let journal_head = plan.journal_head().clone();
782 let child_graph = self
783 .registry
784 .resolve(plan.checkpoint().graph())
785 .is_some_and(|entry| entry.graph().child_runs().is_some());
786 if child_graph {
787 if let Some(record) = self
788 .store
789 .load_run_failure_close(fence.tenant_id(), fence.run_id())
790 .await?
791 {
792 if record.checkpoint() != &plan.checkpoint().head()
793 || record.registration().source().worker_fence() != Some(&fence)
794 || record.lifecycle().revision() != expected_revision
795 {
796 return Err(GraphLifecycleError::StaleHandoff);
797 }
798 return Ok(GraphBarrierLifecycleOutcome::FailureClosing(Box::new(
799 stateknot_store_postgres::RunFailureCloseOutcome::Existing(record),
800 )));
801 }
802 }
803 let snapshot = self
804 .validate_handoff_snapshot(
805 &fence,
806 &journal_head,
807 event_id,
808 expected_revision,
809 RunStatus::Failed,
810 )
811 .await?;
812 let checkpoint = plan.checkpoint().head();
813 validate_blocked_checkpoint_scope(&fence, &journal_head, &checkpoint)?;
814 let failure = match snapshot {
815 GraphLifecycleHandoffSnapshot::Fresh(run) => {
816 let context = GraphFailureEvidenceContext {
817 provenance: run.lifecycle().provenance().clone(),
818 graph: checkpoint.graph().clone(),
819 checkpoint: checkpoint.clone(),
820 observed_at: plan.observed_at(),
821 expected_revision,
822 blockers,
823 };
824 let evidence = self
825 .evidence
826 .failure_evidence(context)
827 .await
828 .map_err(GraphLifecycleError::Evidence)?;
829 let (failure, usage) = evidence.into_parts();
830 let failure = if blockers.superstep_limit_reached() {
831 superstep_limit_failure()
832 } else {
833 failure
834 };
835 if child_graph {
836 return self
837 .request_failure_close(
838 &fence,
839 &checkpoint,
840 journal_head,
841 event_id,
842 failure,
843 usage,
844 )
845 .await;
846 }
847 let usage = self
848 .store
849 .include_child_usage(fence.tenant_id(), fence.run_id(), usage)
850 .await?;
851 RunFailure::new(failure, plan.observed_at(), usage)
852 .map_err(GraphLifecycleError::run_failure)?
853 }
854 GraphLifecycleHandoffSnapshot::Committed(run) => {
855 let lifecycle = run.lifecycle();
856 let failure = lifecycle.terminal_failure().cloned().ok_or(
857 GraphLifecycleError::InvalidHandoff {
858 operation: "recover the committed terminal failure",
859 },
860 )?;
861 let usage = lifecycle.terminal_usage().cloned().ok_or(
862 GraphLifecycleError::InvalidHandoff {
863 operation: "recover committed terminal failure accounting",
864 },
865 )?;
866 RunFailure::new(failure, lifecycle.changed_at(), usage)
867 .map_err(GraphLifecycleError::run_failure)?
868 }
869 };
870 let failure_id = failure.failure().id();
871 let payload = self.failed_payload(checkpoint.graph(), &checkpoint, failure_id, blockers)?;
872 let append = worker_append(&fence, journal_head, event_id, payload)?;
873 let projection =
874 RunProjection::transition(expected_revision, RunTransition::Fail { failure });
875 let outcome = self.commit_failure_with_retry(append, projection).await?;
876 Ok(GraphBarrierLifecycleOutcome::Failed(outcome))
877 }
878
879 async fn request_failure_close(
880 &self,
881 fence: &RunFence,
882 checkpoint: &CheckpointHead,
883 mut head: JournalHead,
884 event: EventId,
885 failure: Failure,
886 usage: BudgetUsage,
887 ) -> Result<GraphBarrierLifecycleOutcome, GraphLifecycleError> {
888 let (schema, _) = crate::standard_run_failure_close_event_schema().map_err(|_| {
889 GraphLifecycleError::InvalidHandoff {
890 operation: "load failure-close schema",
891 }
892 })?;
893 let payload =
894 crate::failure_close::payload(self.registry.schemas(), &schema, failure.id(), false)?;
895 for attempt in 1..=self.options.maximum_mutation_attempts() {
896 let append = worker_append(fence, head.clone(), event, payload.clone())?;
897 match self
898 .store
899 .request_run_failure_close(checkpoint, failure.clone(), usage.clone(), append)
900 .await
901 {
902 Ok(outcome) => {
903 return Ok(GraphBarrierLifecycleOutcome::FailureClosing(Box::new(
904 outcome,
905 )));
906 }
907 Err(error)
908 if attempt < self.options.maximum_mutation_attempts()
909 && (error.is_retryable()
910 || matches!(error, StoreError::StaleJournalHead)) =>
911 {
912 tokio::time::sleep(
913 self.options
914 .mutation_retry_initial_delay()
915 .saturating_mul(1_u32 << (attempt - 1))
916 .min(Duration::from_secs(1)),
917 )
918 .await;
919 let run = self
920 .store
921 .load_run(fence.tenant_id(), fence.run_id())
922 .await?;
923 head = run
924 .journal_head()
925 .cloned()
926 .ok_or(GraphLifecycleError::StaleHandoff)?;
927 }
928 Err(error) => return Err(error.into()),
929 }
930 }
931 unreachable!("validated positive attempt count")
932 }
933
934 async fn validate_handoff_snapshot(
935 &self,
936 fence: &RunFence,
937 journal_head: &JournalHead,
938 event_id: EventId,
939 expected_revision: RunRevision,
940 committed_status: RunStatus,
941 ) -> Result<GraphLifecycleHandoffSnapshot, GraphLifecycleError> {
942 let run = self
943 .store
944 .load_run(fence.tenant_id(), fence.run_id())
945 .await?;
946 let is_fresh = run.lifecycle().revision() == expected_revision
947 && run.lifecycle().status() == RunStatus::Active
948 && run.journal_head() == Some(journal_head)
949 && run.lease().map(RunLease::fence) == Some(fence);
950 if is_fresh {
951 let observation = self.store.observe_live_lease(fence).await?;
952 if observation.lease().fence() != fence {
953 return Err(GraphLifecycleError::StaleHandoff);
954 }
955 return Ok(GraphLifecycleHandoffSnapshot::Fresh(run));
956 }
957
958 let committed_revision = expected_revision.get().checked_add(1);
959 let is_committed_retry = committed_revision
960 .is_some_and(|revision| run.lifecycle().revision() == RunRevision::new(revision))
961 && run.lifecycle().status() == committed_status
962 && run
963 .journal_head()
964 .is_some_and(|head| head.event_id() == event_id)
965 && run.lease().is_none();
966 if is_committed_retry {
967 return Ok(GraphLifecycleHandoffSnapshot::Committed(run));
968 }
969 Err(GraphLifecycleError::StaleHandoff)
970 }
971
972 fn validate_terminal_result(
973 &self,
974 provenance: &AgentResultProvenance,
975 completed_at: Timestamp,
976 output: &stateknot_core::NodeTerminalOutput,
977 evidence: GraphTerminalEvidence,
978 ) -> Result<AgentResult, GraphLifecycleError> {
979 let (descriptor, request, budget, artifacts, usage) = evidence.into_parts();
980 self.registry
981 .schemas()
982 .validate_bounded(request.input_schema(), request.input())
983 .map_err(GraphLifecycleError::input_schema)?;
984 self.registry
985 .schemas()
986 .validate_bounded(output.schema(), output.data())
987 .map_err(GraphLifecycleError::output_schema)?;
988 let result = AgentResult::new(
989 provenance.clone(),
990 completed_at,
991 output.schema().clone(),
992 output.data().clone(),
993 artifacts,
994 usage,
995 )
996 .map_err(GraphLifecycleError::agent_result)?;
997 result
998 .validate_for(provenance, &request, &descriptor, &budget)
999 .map_err(GraphLifecycleError::agent_result_validation)?;
1000 Ok(result)
1001 }
1002
1003 async fn commit_wait_with_retry(
1004 &self,
1005 append: JournalAppend,
1006 expected_revision: RunRevision,
1007 barrier: stateknot_core::CheckpointBarrier,
1008 registrations: Vec<stateknot_core::WaitRegistrationIntent>,
1009 ) -> Result<WaitCheckpointCommitOutcome, GraphLifecycleError> {
1010 let mut attempt = 1_u8;
1011 loop {
1012 match self
1013 .store
1014 .append_worker_wait_barrier(
1015 append.clone(),
1016 expected_revision,
1017 barrier.clone(),
1018 registrations.clone(),
1019 )
1020 .await
1021 {
1022 Ok(outcome) => return Ok(outcome),
1023 Err(error) if self.can_retry_mutation(&error, attempt) => {
1024 self.mutation_backoff(attempt).await;
1025 attempt = attempt.saturating_add(1);
1026 }
1027 Err(error) => return Err(error.into()),
1028 }
1029 }
1030 }
1031
1032 async fn commit_success_with_retry(
1033 &self,
1034 append: JournalAppend,
1035 projection: RunProjection,
1036 barrier: stateknot_core::CheckpointBarrier,
1037 ) -> Result<BarrierCommitOutcome, GraphLifecycleError> {
1038 let mut attempt = 1_u8;
1039 loop {
1040 match self
1041 .store
1042 .append_worker_barrier(append.clone(), projection.clone(), barrier.clone())
1043 .await
1044 {
1045 Ok(outcome) => return Ok(outcome),
1046 Err(error) if self.can_retry_mutation(&error, attempt) => {
1047 self.mutation_backoff(attempt).await;
1048 attempt = attempt.saturating_add(1);
1049 }
1050 Err(error) => return Err(error.into()),
1051 }
1052 }
1053 }
1054
1055 async fn commit_failure_with_retry(
1056 &self,
1057 append: JournalAppend,
1058 projection: RunProjection,
1059 ) -> Result<AppendOutcome, GraphLifecycleError> {
1060 let mut attempt = 1_u8;
1061 loop {
1062 match self
1063 .store
1064 .append_worker(append.clone(), projection.clone())
1065 .await
1066 {
1067 Ok(outcome) => return Ok(outcome),
1068 Err(error) if self.can_retry_mutation(&error, attempt) => {
1069 self.mutation_backoff(attempt).await;
1070 attempt = attempt.saturating_add(1);
1071 }
1072 Err(error) => return Err(error.into()),
1073 }
1074 }
1075 }
1076
1077 async fn release_with_retry(
1078 &self,
1079 fence: &RunFence,
1080 ) -> Result<LeaseReleaseOutcome, GraphLifecycleError> {
1081 let mut attempt = 1_u8;
1082 loop {
1083 match self.store.release_lease(fence).await {
1084 Ok(outcome) => return Ok(outcome),
1085 Err(error) if self.can_retry_mutation(&error, attempt) => {
1086 self.mutation_backoff(attempt).await;
1087 attempt = attempt.saturating_add(1);
1088 }
1089 Err(error) => return Err(error.into()),
1090 }
1091 }
1092 }
1093
1094 fn can_retry_mutation(&self, error: &StoreError, attempt: u8) -> bool {
1095 attempt < self.options.maximum_mutation_attempts && error.is_retryable()
1096 }
1097
1098 async fn mutation_backoff(&self, attempt: u8) {
1099 let multiplier = 1_u32
1100 .checked_shl(u32::from(attempt.saturating_sub(1)))
1101 .unwrap_or(u32::MAX);
1102 let delay = self
1103 .options
1104 .mutation_retry_initial_delay
1105 .checked_mul(multiplier)
1106 .unwrap_or(MAX_MUTATION_RETRY_DELAY)
1107 .min(MAX_MUTATION_RETRY_DELAY);
1108 tokio::time::sleep(delay).await;
1109 }
1110
1111 fn waiting_payload(
1112 &self,
1113 graph: &GraphReference,
1114 checkpoint: &CheckpointHead,
1115 successor_id: CheckpointId,
1116 successor_superstep: Superstep,
1117 wait_count: usize,
1118 ) -> Result<JournalPayload, GraphLifecycleError> {
1119 self.event_payload(
1120 "graph-barrier-waiting",
1121 json!({
1122 "operation": "graph_barrier_waiting",
1123 "graph_digest": digest_hex(graph.definition_digest()),
1124 "checkpoint_id": checkpoint.checkpoint_id().to_string(),
1125 "superstep": checkpoint.superstep().get().to_string(),
1126 "successor_checkpoint_id": successor_id.to_string(),
1127 "successor_superstep": successor_superstep.get().to_string(),
1128 "disposition": "wait",
1129 "wait_count": wait_count
1130 }),
1131 )
1132 }
1133
1134 fn succeeded_payload(
1135 &self,
1136 graph: &GraphReference,
1137 checkpoint: &CheckpointHead,
1138 successor_id: CheckpointId,
1139 successor_superstep: Superstep,
1140 output_digest: Digest,
1141 ) -> Result<JournalPayload, GraphLifecycleError> {
1142 self.event_payload(
1143 "graph-barrier-succeeded",
1144 json!({
1145 "operation": "graph_barrier_succeeded",
1146 "graph_digest": digest_hex(graph.definition_digest()),
1147 "checkpoint_id": checkpoint.checkpoint_id().to_string(),
1148 "superstep": checkpoint.superstep().get().to_string(),
1149 "successor_checkpoint_id": successor_id.to_string(),
1150 "successor_superstep": successor_superstep.get().to_string(),
1151 "disposition": "succeeded",
1152 "output_digest": digest_hex(output_digest)
1153 }),
1154 )
1155 }
1156
1157 fn failed_payload(
1158 &self,
1159 graph: &GraphReference,
1160 checkpoint: &CheckpointHead,
1161 failure_id: stateknot_core::FailureId,
1162 blockers: GraphDriveBlockers,
1163 ) -> Result<JournalPayload, GraphLifecycleError> {
1164 self.event_payload(
1165 "graph-run-failed",
1166 json!({
1167 "operation": "graph_run_failed",
1168 "graph_digest": digest_hex(graph.definition_digest()),
1169 "checkpoint_id": checkpoint.checkpoint_id().to_string(),
1170 "superstep": checkpoint.superstep().get().to_string(),
1171 "disposition": "failed",
1172 "failure_id": failure_id.to_string(),
1173 "in_flight": blockers.in_flight(),
1174 "failed": blockers.failed(),
1175 "exhausted": blockers.exhausted(),
1176 "unsupported": blockers.unsupported()
1177 }),
1178 )
1179 }
1180
1181 fn cancelled_payload(
1182 &self,
1183 checkpoint: &CheckpointHead,
1184 failure_id: FailureId,
1185 ) -> Result<JournalPayload, GraphLifecycleError> {
1186 self.event_payload_with_schema(
1187 &self.cancellation_schema,
1188 "agent-cancellation-confirmed",
1189 json!({
1190 "operation": "agent_cancellation_confirmed",
1191 "graph_digest": digest_hex(checkpoint.graph().definition_digest()),
1192 "checkpoint_id": checkpoint.checkpoint_id().to_string(),
1193 "superstep": checkpoint.superstep().get().to_string(),
1194 "failure_id": failure_id.to_string()
1195 }),
1196 )
1197 }
1198
1199 fn event_payload(
1200 &self,
1201 kind: &'static str,
1202 data: Value,
1203 ) -> Result<JournalPayload, GraphLifecycleError> {
1204 self.event_payload_with_schema(&self.journal_schema, kind, data)
1205 }
1206
1207 fn event_payload_with_schema(
1208 &self,
1209 schema: &SchemaReference,
1210 kind: &'static str,
1211 data: Value,
1212 ) -> Result<JournalPayload, GraphLifecycleError> {
1213 let data = BoundedJson::try_from_value(data)
1214 .map_err(|_| GraphLifecycleError::EventPayloadInvalid)?;
1215 self.registry
1216 .schemas()
1217 .validate_bounded(schema, &data)
1218 .map_err(|_| GraphLifecycleError::EventPayloadInvalid)?;
1219 let kind =
1220 JournalEventKind::new(kind).map_err(|_| GraphLifecycleError::EventPayloadInvalid)?;
1221 JournalPayload::new(schema.clone(), kind, data)
1222 .map_err(|_| GraphLifecycleError::EventPayloadInvalid)
1223 }
1224}
1225
1226pub(crate) fn superstep_limit_failure() -> Failure {
1227 Failure::new(
1228 FailureId::generate(),
1229 FailureCategory::RateLimited,
1230 FailureCode::new("runtime.graph.superstep_limit_reached").expect("static failure code"),
1231 FailureOrigin::new("stateknot.runtime.graph").expect("static failure origin"),
1232 FailureMessage::new("The graph exhausted its configured superstep limit.")
1233 .expect("static failure message"),
1234 RetryAdvice::Never,
1235 )
1236 .expect("static superstep failure semantics")
1237}
1238
1239impl fmt::Debug for DurableGraphLifecycle {
1240 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
1241 formatter
1242 .debug_struct("DurableGraphLifecycle")
1243 .field("registry", &self.registry)
1244 .field("journal_schema", &self.journal_schema)
1245 .field("cancellation_schema", &self.cancellation_schema)
1246 .field("options", &self.options)
1247 .finish_non_exhaustive()
1248 }
1249}
1250
1251#[derive(Clone, Debug, Eq, Error, PartialEq)]
1253#[non_exhaustive]
1254pub enum DurableGraphLifecycleBuildError {
1255 #[error(transparent)]
1257 FailureClose(#[from] crate::RunFailureCloseBuildError),
1258 #[error(transparent)]
1260 StandardSchema(#[from] StandardGraphLifecycleSchemaError),
1261 #[error(transparent)]
1263 CancellationSchema(#[from] StandardAgentCancellationSchemaError),
1264 #[error("standard graph lifecycle event schema is unavailable")]
1266 JournalSchemaUnavailable,
1267 #[error("standard agent cancellation event schema is unavailable")]
1269 CancellationSchemaUnavailable,
1270}
1271
1272#[derive(Debug, Error)]
1274#[non_exhaustive]
1275pub enum GraphLifecycleError {
1276 #[error(transparent)]
1278 Store {
1279 source: Box<StoreError>,
1281 },
1282 #[error("graph lifecycle handoff is stale")]
1285 StaleHandoff,
1286 #[error("graph lifecycle handoff is invalid while attempting to {operation}")]
1288 InvalidHandoff {
1289 operation: &'static str,
1291 },
1292 #[error("graph lifecycle evidence recovery failed: {0}")]
1294 Evidence(GraphLifecycleEvidenceError),
1295 #[error("graph lifecycle wait registration is invalid: {source}")]
1297 WaitRegistration {
1298 #[source]
1300 source: Box<DurableWaitError>,
1301 },
1302 #[error("graph lifecycle input failed its pinned schema: {source}")]
1304 InputSchema {
1305 #[source]
1307 source: GraphSchemaValidationError,
1308 },
1309 #[error("graph lifecycle output failed its pinned schema: {source}")]
1311 OutputSchema {
1312 #[source]
1314 source: GraphSchemaValidationError,
1315 },
1316 #[error("graph lifecycle successful result is invalid: {source}")]
1318 AgentResult {
1319 #[source]
1321 source: Box<AgentResultError>,
1322 },
1323 #[error("graph lifecycle successful result binding is invalid: {source}")]
1325 AgentResultValidation {
1326 #[source]
1328 source: Box<AgentResultValidationError>,
1329 },
1330 #[error("graph lifecycle terminal failure is invalid: {source}")]
1332 RunFailure {
1333 #[source]
1335 source: RunFailureError,
1336 },
1337 #[error("graph lifecycle event payload failed its pinned schema")]
1339 EventPayloadInvalid,
1340 #[error("graph lifecycle worker journal append is invalid")]
1342 JournalAppendInvalid,
1343}
1344
1345impl GraphLifecycleError {
1346 fn wait_registration(source: DurableWaitError) -> Self {
1347 Self::WaitRegistration {
1348 source: Box::new(source),
1349 }
1350 }
1351
1352 const fn input_schema(source: GraphSchemaValidationError) -> Self {
1353 Self::InputSchema { source }
1354 }
1355
1356 const fn output_schema(source: GraphSchemaValidationError) -> Self {
1357 Self::OutputSchema { source }
1358 }
1359
1360 fn agent_result(source: AgentResultError) -> Self {
1361 Self::AgentResult {
1362 source: Box::new(source),
1363 }
1364 }
1365
1366 fn agent_result_validation(source: AgentResultValidationError) -> Self {
1367 Self::AgentResultValidation {
1368 source: Box::new(source),
1369 }
1370 }
1371
1372 const fn run_failure(source: RunFailureError) -> Self {
1373 Self::RunFailure { source }
1374 }
1375}
1376
1377impl From<StoreError> for GraphLifecycleError {
1378 fn from(source: StoreError) -> Self {
1379 Self::Store {
1380 source: Box::new(source),
1381 }
1382 }
1383}
1384
1385fn validate_barrier_scope(
1386 fence: &RunFence,
1387 journal_head: &JournalHead,
1388 checkpoint: &CheckpointHead,
1389 graph: &GraphReference,
1390) -> Result<(), GraphLifecycleError> {
1391 if journal_head.tenant_id() != fence.tenant_id()
1392 || journal_head.run_id() != fence.run_id()
1393 || checkpoint.tenant_id() != fence.tenant_id()
1394 || checkpoint.run_id() != fence.run_id()
1395 || checkpoint.graph() != graph
1396 {
1397 return Err(GraphLifecycleError::InvalidHandoff {
1398 operation: "bind graph barrier scope to its journal and fence",
1399 });
1400 }
1401 Ok(())
1402}
1403
1404fn validate_cancellation_scope(
1405 fence: &RunFence,
1406 journal_head: &JournalHead,
1407 checkpoint: &CheckpointHead,
1408) -> Result<(), GraphLifecycleError> {
1409 if journal_head.tenant_id() != fence.tenant_id()
1410 || journal_head.run_id() != fence.run_id()
1411 || checkpoint.tenant_id() != fence.tenant_id()
1412 || checkpoint.run_id() != fence.run_id()
1413 {
1414 return Err(GraphLifecycleError::InvalidHandoff {
1415 operation: "bind cancellation confirmation to one run scope",
1416 });
1417 }
1418 Ok(())
1419}
1420
1421fn validate_cancellation_checkpoint(
1422 run: &StoredRun,
1423 checkpoint: &CheckpointHead,
1424) -> Result<(), GraphLifecycleError> {
1425 let pointer = run
1426 .checkpoint()
1427 .ok_or(GraphLifecycleError::InvalidHandoff {
1428 operation: "bind cancellation confirmation to a checkpoint",
1429 })?;
1430 if pointer.checkpoint_id() != checkpoint.checkpoint_id()
1431 || pointer.superstep() != checkpoint.superstep()
1432 || pointer.digest() != checkpoint.digest()
1433 {
1434 return Err(GraphLifecycleError::StaleHandoff);
1435 }
1436 Ok(())
1437}
1438
1439fn validate_blocked_checkpoint_scope(
1440 fence: &RunFence,
1441 journal_head: &JournalHead,
1442 checkpoint: &CheckpointHead,
1443) -> Result<(), GraphLifecycleError> {
1444 let checkpoint_journal = checkpoint.journal_head();
1445 let journal_position_is_valid = checkpoint_journal == journal_head
1446 || (checkpoint_journal.sequence() < journal_head.sequence()
1447 && checkpoint_journal.recorded_at() <= journal_head.recorded_at());
1448 if journal_head.tenant_id() != fence.tenant_id()
1449 || journal_head.run_id() != fence.run_id()
1450 || checkpoint.tenant_id() != fence.tenant_id()
1451 || checkpoint.run_id() != fence.run_id()
1452 || checkpoint_journal.tenant_id() != fence.tenant_id()
1453 || checkpoint_journal.run_id() != fence.run_id()
1454 || !journal_position_is_valid
1455 {
1456 return Err(GraphLifecycleError::InvalidHandoff {
1457 operation: "bind blocked checkpoint to its journal and fence",
1458 });
1459 }
1460 Ok(())
1461}
1462
1463fn worker_append(
1464 fence: &RunFence,
1465 head: JournalHead,
1466 event_id: EventId,
1467 payload: JournalPayload,
1468) -> Result<JournalAppend, GraphLifecycleError> {
1469 let intent = JournalEventIntent::worker(
1470 fence.tenant_id().clone(),
1471 fence.run_id(),
1472 event_id,
1473 fence.clone(),
1474 payload,
1475 )
1476 .map_err(|_| GraphLifecycleError::JournalAppendInvalid)?;
1477 JournalAppend::new(JournalExpectation::exact(head), intent)
1478 .map_err(|_| GraphLifecycleError::JournalAppendInvalid)
1479}
1480
1481fn digest_hex(digest: Digest) -> String {
1482 const HEX: &[u8; 16] = b"0123456789abcdef";
1483 let mut value = String::with_capacity(Digest::SHA256_LEN * 2);
1484 for byte in digest.as_bytes() {
1485 value.push(char::from(HEX[usize::from(byte >> 4)]));
1486 value.push(char::from(HEX[usize::from(byte & 0x0f)]));
1487 }
1488 value
1489}
1490
1491#[cfg(test)]
1492mod tests {
1493 use super::*;
1494
1495 #[test]
1496 fn lifecycle_options_enforce_production_retry_bounds() {
1497 assert_eq!(
1498 DurableGraphLifecycleOptions::new(0, Duration::from_millis(1)),
1499 Err(DurableGraphLifecycleOptionsError::InvalidMutationAttempts)
1500 );
1501 assert_eq!(
1502 DurableGraphLifecycleOptions::new(1, Duration::ZERO),
1503 Err(DurableGraphLifecycleOptionsError::InvalidMutationRetryDelay)
1504 );
1505 let options = DurableGraphLifecycleOptions::new(4, Duration::from_millis(50)).unwrap();
1506 assert_eq!(options.maximum_mutation_attempts(), 4);
1507 assert_eq!(
1508 options.mutation_retry_initial_delay(),
1509 Duration::from_millis(50)
1510 );
1511 }
1512}