Skip to main content

stateknot_runtime/
lifecycle.rs

1// Copyright 2026 StateKnot contributors
2// SPDX-License-Identifier: Apache-2.0
3
4//! Fenced graph-to-agent lifecycle commits.
5
6use std::{fmt, sync::Arc, time::Duration};
7
8use serde_json::{Value, json};
9use stateknot_core::{
10    AgentArtifacts, AgentDescriptor, AgentRequest, AgentResult, AgentResultError,
11    AgentResultProvenance, AgentResultValidationError, BoundedJson, BoxFuture, BudgetUsage,
12    CheckpointHead, CheckpointId, Digest, DurableWaitError, EventId, Failure, FailureCategory,
13    FailureCode, FailureId, FailureMessage, FailureOrigin, GraphBarrierDisposition, GraphReference,
14    GraphSchemaValidationError, JournalAppend, JournalEventIntent, JournalEventKind,
15    JournalExpectation, JournalHead, JournalPayload, ResolvedBudget, RetryAdvice, RunFailure,
16    RunFailureError, RunFence, RunLease, RunRevision, RunStatus, RunTransition, SchemaReference,
17    Superstep, Timestamp,
18};
19use stateknot_store_postgres::{
20    AppendOutcome, BarrierCommitOutcome, LeaseReleaseOutcome, PostgresStore, RunProjection,
21    StoreError, StoredRun, WaitCheckpointCommitOutcome,
22};
23use thiserror::Error;
24
25use crate::{
26    ExecutableGraphRegistry, GraphBlockedHandoff, GraphCancellationHandoff, GraphDriveBlockers,
27    GraphLifecycleBarrierHandoff, StandardAgentCancellationSchemaError,
28    StandardGraphLifecycleSchemaError, standard_agent_cancellation_event_schema,
29    standard_graph_lifecycle_event_schema,
30};
31
32const MAX_MUTATION_RETRY_DELAY: Duration = Duration::from_secs(1);
33
34/// Retry policy for atomic graph lifecycle mutations.
35#[derive(Clone, Copy, Debug, Eq, PartialEq)]
36pub struct DurableGraphLifecycleOptions {
37    maximum_mutation_attempts: u8,
38    mutation_retry_initial_delay: Duration,
39}
40
41impl DurableGraphLifecycleOptions {
42    /// Absolute number of identical durable mutation attempts.
43    pub const HARD_MAXIMUM_MUTATION_ATTEMPTS: u8 = 10;
44
45    /// Constructs an explicit bounded retry policy.
46    ///
47    /// # Errors
48    ///
49    /// Rejects zero/excessive attempts or a zero/greater-than-one-second
50    /// initial delay.
51    pub fn new(
52        maximum_mutation_attempts: u8,
53        mutation_retry_initial_delay: Duration,
54    ) -> Result<Self, DurableGraphLifecycleOptionsError> {
55        if maximum_mutation_attempts == 0
56            || maximum_mutation_attempts > Self::HARD_MAXIMUM_MUTATION_ATTEMPTS
57        {
58            return Err(DurableGraphLifecycleOptionsError::InvalidMutationAttempts);
59        }
60        if mutation_retry_initial_delay.is_zero()
61            || mutation_retry_initial_delay > MAX_MUTATION_RETRY_DELAY
62        {
63            return Err(DurableGraphLifecycleOptionsError::InvalidMutationRetryDelay);
64        }
65        Ok(Self {
66            maximum_mutation_attempts,
67            mutation_retry_initial_delay,
68        })
69    }
70
71    /// Returns the maximum identical attempts for one mutation.
72    #[must_use]
73    pub const fn maximum_mutation_attempts(self) -> u8 {
74        self.maximum_mutation_attempts
75    }
76
77    /// Returns the first exponential retry delay.
78    #[must_use]
79    pub const fn mutation_retry_initial_delay(self) -> Duration {
80        self.mutation_retry_initial_delay
81    }
82}
83
84impl Default for DurableGraphLifecycleOptions {
85    fn default() -> Self {
86        Self {
87            maximum_mutation_attempts: 3,
88            mutation_retry_initial_delay: Duration::from_millis(25),
89        }
90    }
91}
92
93/// Invalid lifecycle retry policy.
94#[derive(Clone, Copy, Debug, Eq, Error, PartialEq)]
95#[non_exhaustive]
96pub enum DurableGraphLifecycleOptionsError {
97    /// Mutation attempts were zero or above the hard ceiling.
98    #[error("graph lifecycle mutation attempt count is invalid")]
99    InvalidMutationAttempts,
100    /// Initial mutation backoff was zero or above one second.
101    #[error("graph lifecycle mutation retry delay is invalid")]
102    InvalidMutationRetryDelay,
103}
104
105/// Payload-free trusted context for terminal evidence recovery.
106#[derive(Clone, Debug)]
107pub struct GraphTerminalEvidenceContext {
108    provenance: AgentResultProvenance,
109    graph: GraphReference,
110    checkpoint: CheckpointHead,
111    successor_checkpoint_id: CheckpointId,
112    successor_superstep: Superstep,
113    output_schema: SchemaReference,
114    output_digest: Digest,
115    expected_revision: RunRevision,
116}
117
118impl GraphTerminalEvidenceContext {
119    /// Returns trusted run provenance from the current lifecycle snapshot.
120    #[must_use]
121    pub const fn provenance(&self) -> &AgentResultProvenance {
122        &self.provenance
123    }
124
125    /// Returns the exact pinned executable graph.
126    #[must_use]
127    pub const fn graph(&self) -> &GraphReference {
128        &self.graph
129    }
130
131    /// Returns the barrier's exact base checkpoint.
132    #[must_use]
133    pub const fn checkpoint(&self) -> &CheckpointHead {
134        &self.checkpoint
135    }
136
137    /// Returns the pending terminal successor identity.
138    #[must_use]
139    pub const fn successor_checkpoint_id(&self) -> CheckpointId {
140        self.successor_checkpoint_id
141    }
142
143    /// Returns the pending terminal successor position.
144    #[must_use]
145    pub const fn successor_superstep(&self) -> Superstep {
146        self.successor_superstep
147    }
148
149    /// Returns the exact terminal output schema.
150    #[must_use]
151    pub const fn output_schema(&self) -> &SchemaReference {
152        &self.output_schema
153    }
154
155    /// Returns the terminal output's integrity digest without its data.
156    #[must_use]
157    pub const fn output_digest(&self) -> Digest {
158        self.output_digest
159    }
160
161    /// Returns the lifecycle revision that the final transaction must consume.
162    #[must_use]
163    pub const fn expected_revision(&self) -> RunRevision {
164        self.expected_revision
165    }
166}
167
168/// Trusted admission and cumulative accounting required for success.
169#[derive(Clone)]
170pub struct GraphTerminalEvidence {
171    descriptor: AgentDescriptor,
172    request: AgentRequest,
173    budget: ResolvedBudget,
174    artifacts: AgentArtifacts,
175    usage: BudgetUsage,
176}
177
178impl GraphTerminalEvidence {
179    /// Bundles immutable snapshots recovered from trusted durable sources.
180    #[must_use]
181    pub const fn new(
182        descriptor: AgentDescriptor,
183        request: AgentRequest,
184        budget: ResolvedBudget,
185        artifacts: AgentArtifacts,
186        usage: BudgetUsage,
187    ) -> Self {
188        Self {
189            descriptor,
190            request,
191            budget,
192            artifacts,
193            usage,
194        }
195    }
196
197    /// Returns the immutable admitted agent descriptor.
198    #[must_use]
199    pub const fn descriptor(&self) -> &AgentDescriptor {
200        &self.descriptor
201    }
202
203    /// Returns the immutable admitted request.
204    #[must_use]
205    pub const fn request(&self) -> &AgentRequest {
206        &self.request
207    }
208
209    /// Returns the exact resolved finite budget snapshot.
210    #[must_use]
211    pub const fn budget(&self) -> &ResolvedBudget {
212        &self.budget
213    }
214
215    /// Returns final durable artifact references.
216    #[must_use]
217    pub const fn artifacts(&self) -> &AgentArtifacts {
218        &self.artifacts
219    }
220
221    /// Returns complete cumulative run usage.
222    #[must_use]
223    pub const fn usage(&self) -> &BudgetUsage {
224        &self.usage
225    }
226
227    fn into_parts(
228        self,
229    ) -> (
230        AgentDescriptor,
231        AgentRequest,
232        ResolvedBudget,
233        AgentArtifacts,
234        BudgetUsage,
235    ) {
236        (
237            self.descriptor,
238            self.request,
239            self.budget,
240            self.artifacts,
241            self.usage,
242        )
243    }
244}
245
246impl fmt::Debug for GraphTerminalEvidence {
247    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
248        formatter
249            .debug_struct("GraphTerminalEvidence")
250            .field("agent", self.descriptor.metadata().identity())
251            .field("input_schema", self.request.input_schema())
252            .field("output_schema", self.descriptor.output_schema())
253            .field("artifact_count", &self.artifacts.len())
254            .field("usage_recorded", &true)
255            .finish_non_exhaustive()
256    }
257}
258
259/// Payload-free trusted context for blocked-run failure evidence recovery.
260#[derive(Clone, Debug)]
261pub struct GraphFailureEvidenceContext {
262    provenance: AgentResultProvenance,
263    graph: GraphReference,
264    checkpoint: CheckpointHead,
265    observed_at: Timestamp,
266    expected_revision: RunRevision,
267    blockers: GraphDriveBlockers,
268}
269
270impl GraphFailureEvidenceContext {
271    /// Returns trusted run provenance from the current lifecycle snapshot.
272    #[must_use]
273    pub const fn provenance(&self) -> &AgentResultProvenance {
274        &self.provenance
275    }
276
277    /// Returns the exact pinned graph.
278    #[must_use]
279    pub const fn graph(&self) -> &GraphReference {
280        &self.graph
281    }
282
283    /// Returns the blocked checkpoint.
284    #[must_use]
285    pub const fn checkpoint(&self) -> &CheckpointHead {
286        &self.checkpoint
287    }
288
289    /// Returns the database observation used by recovery classification.
290    #[must_use]
291    pub const fn observed_at(&self) -> Timestamp {
292        self.observed_at
293    }
294
295    /// Returns the lifecycle revision that the failure must consume.
296    #[must_use]
297    pub const fn expected_revision(&self) -> RunRevision {
298        self.expected_revision
299    }
300
301    /// Returns aggregate blocked-node classifications.
302    #[must_use]
303    pub const fn blockers(&self) -> GraphDriveBlockers {
304        self.blockers
305    }
306}
307
308/// Public failure and cumulative accounting selected by durable supervision.
309#[derive(Clone, Debug)]
310pub struct GraphFailureEvidence {
311    failure: Failure,
312    usage: BudgetUsage,
313}
314
315/// Payload-free trusted context for terminal cancellation accounting recovery.
316#[derive(Clone, Debug)]
317pub struct GraphCancellationEvidenceContext {
318    provenance: AgentResultProvenance,
319    graph: GraphReference,
320    checkpoint: CheckpointHead,
321    expected_revision: RunRevision,
322    cancellation_failure_id: FailureId,
323}
324
325impl GraphCancellationEvidenceContext {
326    /// Returns trusted run provenance from the current lifecycle snapshot.
327    #[must_use]
328    pub const fn provenance(&self) -> &AgentResultProvenance {
329        &self.provenance
330    }
331
332    /// Returns the exact pinned graph.
333    #[must_use]
334    pub const fn graph(&self) -> &GraphReference {
335        &self.graph
336    }
337
338    /// Returns the checkpoint at which cancellation stopped graph progress.
339    #[must_use]
340    pub const fn checkpoint(&self) -> &CheckpointHead {
341        &self.checkpoint
342    }
343
344    /// Returns the cancellation-request lifecycle revision to consume.
345    #[must_use]
346    pub const fn expected_revision(&self) -> RunRevision {
347        self.expected_revision
348    }
349
350    /// Returns the immutable cancellation occurrence selected by the request.
351    #[must_use]
352    pub const fn cancellation_failure_id(&self) -> FailureId {
353        self.cancellation_failure_id
354    }
355}
356
357/// Complete cumulative accounting recovered before cancellation acknowledgement.
358#[derive(Clone, Debug)]
359pub struct GraphCancellationEvidence {
360    usage: BudgetUsage,
361}
362
363impl GraphCancellationEvidence {
364    /// Wraps exact cumulative usage reconstructed from trusted durable ledgers.
365    #[must_use]
366    pub const fn new(usage: BudgetUsage) -> Self {
367        Self { usage }
368    }
369
370    /// Returns complete cumulative usage at cancellation acknowledgement.
371    #[must_use]
372    pub const fn usage(&self) -> &BudgetUsage {
373        &self.usage
374    }
375
376    fn into_usage(self) -> BudgetUsage {
377        self.usage
378    }
379}
380
381impl GraphFailureEvidence {
382    /// Bundles a public-safe occurrence and complete cumulative run usage.
383    #[must_use]
384    pub const fn new(failure: Failure, usage: BudgetUsage) -> Self {
385        Self { failure, usage }
386    }
387
388    /// Returns the terminal failure occurrence.
389    #[must_use]
390    pub const fn failure(&self) -> &Failure {
391        &self.failure
392    }
393
394    /// Returns complete cumulative run usage.
395    #[must_use]
396    pub const fn usage(&self) -> &BudgetUsage {
397        &self.usage
398    }
399
400    fn into_parts(self) -> (Failure, BudgetUsage) {
401        (self.failure, self.usage)
402    }
403}
404
405/// Closed, payload-redacted evidence-provider failure.
406#[derive(Clone, Copy, Debug, Eq, Error, PartialEq)]
407#[non_exhaustive]
408pub enum GraphLifecycleEvidenceError {
409    /// A trusted durable dependency could not be reached now.
410    #[error("graph lifecycle evidence is temporarily unavailable")]
411    TemporarilyUnavailable,
412    /// Required evidence was permanently absent for this admitted run.
413    #[error("graph lifecycle evidence is unavailable")]
414    Unavailable,
415    /// Durable evidence failed integrity or relationship validation.
416    #[error("graph lifecycle evidence is corrupt")]
417    Corrupt,
418}
419
420/// Trusted recovery boundary for admission snapshots and cumulative accounting.
421///
422/// Implementations must be read-only, deterministic for one context, bounded,
423/// and free of external side effects. They must recover already-durable facts;
424/// they must never infer missing usage, fabricate a request, or re-run model or
425/// tool work. Provider diagnostics belong in protected telemetry and are mapped
426/// to the closed public-safe error above.
427///
428/// Fresh success/failure/cancellation evidence contains DIRECT-only Run usage.
429/// The coordinator adds verified settled child-subtree charges exactly once;
430/// providers must exclude those charges, and unresolved children block closure.
431pub trait GraphLifecycleEvidenceProvider: Send + Sync + 'static {
432    /// Recovers success evidence for an exact terminal barrier.
433    fn terminal_evidence(
434        &self,
435        context: GraphTerminalEvidenceContext,
436    ) -> BoxFuture<'_, Result<GraphTerminalEvidence, GraphLifecycleEvidenceError>>;
437
438    /// Recovers terminal failure evidence for an exact blocked checkpoint.
439    fn failure_evidence(
440        &self,
441        context: GraphFailureEvidenceContext,
442    ) -> BoxFuture<'_, Result<GraphFailureEvidence, GraphLifecycleEvidenceError>>;
443
444    /// Recovers complete cumulative usage for an exact cancellation request.
445    fn cancellation_evidence(
446        &self,
447        context: GraphCancellationEvidenceContext,
448    ) -> BoxFuture<'_, Result<GraphCancellationEvidence, GraphLifecycleEvidenceError>>;
449}
450
451/// Converged result of one lifecycle handoff.
452#[derive(Clone, Debug)]
453#[non_exhaustive]
454pub enum GraphBarrierLifecycleOutcome {
455    /// Original failure sealed, parent lease released, children draining.
456    FailureClosing(Box<stateknot_store_postgres::RunFailureCloseOutcome>),
457    /// Cancellation acknowledgement, usage, event, and lease release committed atomically.
458    Cancelled(AppendOutcome),
459    /// Barrier, successor checkpoint, wait transition, registrations, and lease
460    /// release committed atomically.
461    Waiting(WaitCheckpointCommitOutcome),
462    /// Barrier, terminal checkpoint, successful result, and lease release
463    /// committed atomically.
464    Succeeded(BarrierCommitOutcome),
465    /// Terminal failure event, lifecycle projection, and lease release committed
466    /// atomically.
467    Failed(AppendOutcome),
468    /// An unfinished same-fence attempt requires a successor fence to recover.
469    Released(LeaseReleaseOutcome),
470}
471
472/// Fenced graph-to-agent lifecycle coordinator.
473#[derive(Clone)]
474pub struct DurableGraphLifecycle {
475    store: PostgresStore,
476    registry: ExecutableGraphRegistry,
477    evidence: Arc<dyn GraphLifecycleEvidenceProvider>,
478    journal_schema: SchemaReference,
479    cancellation_schema: SchemaReference,
480    options: DurableGraphLifecycleOptions,
481}
482
483enum GraphLifecycleHandoffSnapshot {
484    Fresh(StoredRun),
485    Committed(StoredRun),
486}
487
488impl DurableGraphLifecycle {
489    /// Binds one provider pool, frozen executable registry, and trusted evidence
490    /// source.
491    ///
492    /// # Errors
493    ///
494    /// Rejects a malformed embedded release schema or a registry that omitted
495    /// it before freezing.
496    pub fn new(
497        store: PostgresStore,
498        registry: ExecutableGraphRegistry,
499        evidence: Arc<dyn GraphLifecycleEvidenceProvider>,
500        options: DurableGraphLifecycleOptions,
501    ) -> Result<Self, DurableGraphLifecycleBuildError> {
502        let (journal_schema, _) = standard_graph_lifecycle_event_schema()?;
503        if !registry.schemas().contains(&journal_schema) {
504            return Err(DurableGraphLifecycleBuildError::JournalSchemaUnavailable);
505        }
506        let (cancellation_schema, _) = standard_agent_cancellation_event_schema()?;
507        if !registry.schemas().contains(&cancellation_schema) {
508            return Err(DurableGraphLifecycleBuildError::CancellationSchemaUnavailable);
509        }
510        if registry.has_child_graphs() {
511            let (schema, _) = crate::standard_run_failure_close_event_schema()?;
512            if !registry.schemas().contains(&schema) {
513                return Err(crate::RunFailureCloseBuildError::SchemaUnavailable.into());
514            }
515        }
516        Ok(Self {
517            store,
518            registry,
519            evidence,
520            journal_schema,
521            cancellation_schema,
522            options,
523        })
524    }
525
526    /// Returns the immutable mutation retry policy.
527    #[must_use]
528    pub const fn options(&self) -> DurableGraphLifecycleOptions {
529        self.options
530    }
531
532    /// Acknowledges one exact durable cancellation request after cumulative
533    /// usage has been recovered from trusted ledgers.
534    ///
535    /// A fresh commit uses a database clock observed with the exact live fence.
536    /// An identical retry reconstructs that timestamp and usage from the
537    /// terminal lifecycle, producing the same projection digest even after the
538    /// original lease was atomically released.
539    pub fn confirm_cancellation(
540        &self,
541        handoff: GraphCancellationHandoff,
542    ) -> BoxFuture<'_, Result<GraphBarrierLifecycleOutcome, GraphLifecycleError>> {
543        Box::pin(self.confirm_cancellation_inner(handoff))
544    }
545
546    async fn confirm_cancellation_inner(
547        &self,
548        handoff: GraphCancellationHandoff,
549    ) -> Result<GraphBarrierLifecycleOutcome, GraphLifecycleError> {
550        let (checkpoint, journal_head, event_id, expected_revision, lease, failure_id) =
551            handoff.into_parts();
552        let fence = lease.fence().clone();
553        validate_cancellation_scope(&fence, &journal_head, &checkpoint)?;
554        let run = self
555            .store
556            .load_run(fence.tenant_id(), fence.run_id())
557            .await?;
558        validate_cancellation_checkpoint(&run, &checkpoint)?;
559
560        let fresh = run.lifecycle().revision() == expected_revision
561            && run.lifecycle().status() == RunStatus::CancellationRequested
562            && run.journal_head() == Some(&journal_head)
563            && run.lease().map(RunLease::fence) == Some(&fence)
564            && run
565                .lifecycle()
566                .cancellation_request()
567                .is_some_and(|request| request.failure().id() == failure_id);
568        let (completed_at, usage) = if fresh {
569            let observation = self.store.observe_live_lease(&fence).await?;
570            if observation.lease().fence() != &fence {
571                return Err(GraphLifecycleError::StaleHandoff);
572            }
573            let context = GraphCancellationEvidenceContext {
574                provenance: run.lifecycle().provenance().clone(),
575                graph: checkpoint.graph().clone(),
576                checkpoint: checkpoint.clone(),
577                expected_revision,
578                cancellation_failure_id: failure_id,
579            };
580            let evidence = self
581                .evidence
582                .cancellation_evidence(context)
583                .await
584                .map_err(GraphLifecycleError::Evidence)?;
585            let usage = self
586                .store
587                .include_child_usage(fence.tenant_id(), fence.run_id(), evidence.into_usage())
588                .await?;
589            (observation.observed_at(), usage)
590        } else {
591            let committed_revision = expected_revision.get().checked_add(1);
592            let committed = committed_revision
593                .is_some_and(|revision| run.lifecycle().revision() == RunRevision::new(revision))
594                && run.lifecycle().status() == RunStatus::Cancelled
595                && run
596                    .journal_head()
597                    .is_some_and(|head| head.event_id() == event_id)
598                && run.lease().is_none()
599                && run
600                    .lifecycle()
601                    .cancellation_request()
602                    .is_some_and(|request| request.failure().id() == failure_id);
603            if !committed {
604                return Err(GraphLifecycleError::StaleHandoff);
605            }
606            let usage = run.lifecycle().terminal_usage().cloned().ok_or(
607                GraphLifecycleError::InvalidHandoff {
608                    operation: "recover committed cancellation accounting",
609                },
610            )?;
611            (run.lifecycle().changed_at(), usage)
612        };
613
614        let payload = self.cancelled_payload(&checkpoint, failure_id)?;
615        let append = worker_append(&fence, journal_head, event_id, payload)?;
616        let projection = RunProjection::transition(
617            expected_revision,
618            RunTransition::ConfirmCancellation {
619                completed_at,
620                usage,
621            },
622        );
623        let outcome = self.commit_failure_with_retry(append, projection).await?;
624        Ok(GraphBarrierLifecycleOutcome::Cancelled(outcome))
625    }
626
627    /// Commits an exact Wait or successful-Terminal graph barrier.
628    ///
629    /// The terminal path recovers trusted admission/accounting evidence, then
630    /// revalidates provenance, request, budget, input/output schemas, artifacts,
631    /// and usage before the fenced transaction. Wait registration timestamps
632    /// come only from the database transaction.
633    pub fn commit_barrier(
634        &self,
635        handoff: GraphLifecycleBarrierHandoff,
636    ) -> BoxFuture<'_, Result<GraphBarrierLifecycleOutcome, GraphLifecycleError>> {
637        Box::pin(self.commit_barrier_inner(handoff))
638    }
639
640    async fn commit_barrier_inner(
641        &self,
642        handoff: GraphLifecycleBarrierHandoff,
643    ) -> Result<GraphBarrierLifecycleOutcome, GraphLifecycleError> {
644        let (plan, journal_head, event_id, expected_revision, lease) = handoff.into_parts();
645        let fence = lease.fence().clone();
646        let (barrier, disposition) = plan.into_parts();
647        let base = barrier.base_checkpoint().clone();
648        validate_barrier_scope(&fence, &journal_head, &base, barrier.successor().graph())?;
649
650        match disposition {
651            GraphBarrierDisposition::Wait { waits } => {
652                // The provider performs the authoritative fresh-fence checks
653                // and looks up the stable event before inspecting current run
654                // status. Calling it directly also preserves exact lost-ACK
655                // convergence if a resolver has already advanced the run past
656                // Waiting before this caller receives the original commit.
657                let payload = self.waiting_payload(
658                    barrier.successor().graph(),
659                    &base,
660                    barrier.successor().checkpoint_id(),
661                    barrier.successor().superstep(),
662                    waits.len(),
663                )?;
664                let append = worker_append(&fence, journal_head, event_id, payload)?;
665                let registrations = waits
666                    .registration_intents(fence.tenant_id(), fence.run_id(), event_id)
667                    .map_err(GraphLifecycleError::wait_registration)?;
668                let outcome = self
669                    .commit_wait_with_retry(append, expected_revision, barrier, registrations)
670                    .await?;
671                Ok(GraphBarrierLifecycleOutcome::Waiting(outcome))
672            }
673            GraphBarrierDisposition::Terminal { output } => {
674                let snapshot = self
675                    .validate_handoff_snapshot(
676                        &fence,
677                        &journal_head,
678                        event_id,
679                        expected_revision,
680                        RunStatus::Succeeded,
681                    )
682                    .await?;
683                let result = match snapshot {
684                    GraphLifecycleHandoffSnapshot::Fresh(run) => {
685                        let context = GraphTerminalEvidenceContext {
686                            provenance: run.lifecycle().provenance().clone(),
687                            graph: barrier.successor().graph().clone(),
688                            checkpoint: base.clone(),
689                            successor_checkpoint_id: barrier.successor().checkpoint_id(),
690                            successor_superstep: barrier.successor().superstep(),
691                            output_schema: output.schema().clone(),
692                            output_digest: output.digest(),
693                            expected_revision,
694                        };
695                        let mut evidence = self
696                            .evidence
697                            .terminal_evidence(context)
698                            .await
699                            .map_err(GraphLifecycleError::Evidence)?;
700                        evidence.usage = self
701                            .store
702                            .include_child_usage(fence.tenant_id(), fence.run_id(), evidence.usage)
703                            .await?;
704                        self.validate_terminal_result(
705                            run.lifecycle().provenance(),
706                            journal_head.recorded_at(),
707                            &output,
708                            evidence,
709                        )?
710                    }
711                    GraphLifecycleHandoffSnapshot::Committed(run) => {
712                        run.lifecycle().result().cloned().ok_or(
713                            GraphLifecycleError::InvalidHandoff {
714                                operation: "recover the committed successful result",
715                            },
716                        )?
717                    }
718                };
719                let payload = self.succeeded_payload(
720                    barrier.successor().graph(),
721                    &base,
722                    barrier.successor().checkpoint_id(),
723                    barrier.successor().superstep(),
724                    output.digest(),
725                )?;
726                let append = worker_append(&fence, journal_head, event_id, payload)?;
727                let projection =
728                    RunProjection::transition(expected_revision, RunTransition::Succeed { result });
729                let outcome = self
730                    .commit_success_with_retry(append, projection, barrier)
731                    .await?;
732                Ok(GraphBarrierLifecycleOutcome::Succeeded(outcome))
733            }
734            GraphBarrierDisposition::Continue => Err(GraphLifecycleError::InvalidHandoff {
735                operation: "commit a Continue disposition through lifecycle coordination",
736            }),
737            _ => Err(GraphLifecycleError::InvalidHandoff {
738                operation: "commit an unsupported graph barrier disposition",
739            }),
740        }
741    }
742
743    /// Resolves a blocked driver handoff without guessing node outcomes.
744    ///
745    /// Any same-fence in-flight start releases ownership so a higher fence can
746    /// apply the provider's crash-recovery rules. Only a plan with no in-flight
747    /// work may enter terminal failure supervision.
748    pub fn resolve_blocked(
749        &self,
750        handoff: GraphBlockedHandoff,
751    ) -> BoxFuture<'_, Result<GraphBarrierLifecycleOutcome, GraphLifecycleError>> {
752        Box::pin(self.resolve_blocked_inner(handoff))
753    }
754
755    #[allow(clippy::too_many_lines)]
756    async fn resolve_blocked_inner(
757        &self,
758        handoff: GraphBlockedHandoff,
759    ) -> Result<GraphBarrierLifecycleOutcome, GraphLifecycleError> {
760        let (plan, lease, event_id, expected_revision, blockers) = handoff.into_parts();
761        let fence = lease.fence().clone();
762        if plan.fence() != &fence {
763            return Err(GraphLifecycleError::InvalidHandoff {
764                operation: "bind blocked recovery plan to its retained lease",
765            });
766        }
767        if blockers.in_flight() > 0 {
768            let release = self.release_with_retry(&fence).await?;
769            return Ok(GraphBarrierLifecycleOutcome::Released(release));
770        }
771        if blockers.failed() == 0
772            && blockers.exhausted() == 0
773            && blockers.unsupported() == 0
774            && !blockers.superstep_limit_reached()
775        {
776            return Err(GraphLifecycleError::InvalidHandoff {
777                operation: "supervise a blocked plan without terminal blockers",
778            });
779        }
780
781        let journal_head = plan.journal_head().clone();
782        let child_graph = self
783            .registry
784            .resolve(plan.checkpoint().graph())
785            .is_some_and(|entry| entry.graph().child_runs().is_some());
786        if child_graph {
787            if let Some(record) = self
788                .store
789                .load_run_failure_close(fence.tenant_id(), fence.run_id())
790                .await?
791            {
792                if record.checkpoint() != &plan.checkpoint().head()
793                    || record.registration().source().worker_fence() != Some(&fence)
794                    || record.lifecycle().revision() != expected_revision
795                {
796                    return Err(GraphLifecycleError::StaleHandoff);
797                }
798                return Ok(GraphBarrierLifecycleOutcome::FailureClosing(Box::new(
799                    stateknot_store_postgres::RunFailureCloseOutcome::Existing(record),
800                )));
801            }
802        }
803        let snapshot = self
804            .validate_handoff_snapshot(
805                &fence,
806                &journal_head,
807                event_id,
808                expected_revision,
809                RunStatus::Failed,
810            )
811            .await?;
812        let checkpoint = plan.checkpoint().head();
813        validate_blocked_checkpoint_scope(&fence, &journal_head, &checkpoint)?;
814        let failure = match snapshot {
815            GraphLifecycleHandoffSnapshot::Fresh(run) => {
816                let context = GraphFailureEvidenceContext {
817                    provenance: run.lifecycle().provenance().clone(),
818                    graph: checkpoint.graph().clone(),
819                    checkpoint: checkpoint.clone(),
820                    observed_at: plan.observed_at(),
821                    expected_revision,
822                    blockers,
823                };
824                let evidence = self
825                    .evidence
826                    .failure_evidence(context)
827                    .await
828                    .map_err(GraphLifecycleError::Evidence)?;
829                let (failure, usage) = evidence.into_parts();
830                let failure = if blockers.superstep_limit_reached() {
831                    superstep_limit_failure()
832                } else {
833                    failure
834                };
835                if child_graph {
836                    return self
837                        .request_failure_close(
838                            &fence,
839                            &checkpoint,
840                            journal_head,
841                            event_id,
842                            failure,
843                            usage,
844                        )
845                        .await;
846                }
847                let usage = self
848                    .store
849                    .include_child_usage(fence.tenant_id(), fence.run_id(), usage)
850                    .await?;
851                RunFailure::new(failure, plan.observed_at(), usage)
852                    .map_err(GraphLifecycleError::run_failure)?
853            }
854            GraphLifecycleHandoffSnapshot::Committed(run) => {
855                let lifecycle = run.lifecycle();
856                let failure = lifecycle.terminal_failure().cloned().ok_or(
857                    GraphLifecycleError::InvalidHandoff {
858                        operation: "recover the committed terminal failure",
859                    },
860                )?;
861                let usage = lifecycle.terminal_usage().cloned().ok_or(
862                    GraphLifecycleError::InvalidHandoff {
863                        operation: "recover committed terminal failure accounting",
864                    },
865                )?;
866                RunFailure::new(failure, lifecycle.changed_at(), usage)
867                    .map_err(GraphLifecycleError::run_failure)?
868            }
869        };
870        let failure_id = failure.failure().id();
871        let payload = self.failed_payload(checkpoint.graph(), &checkpoint, failure_id, blockers)?;
872        let append = worker_append(&fence, journal_head, event_id, payload)?;
873        let projection =
874            RunProjection::transition(expected_revision, RunTransition::Fail { failure });
875        let outcome = self.commit_failure_with_retry(append, projection).await?;
876        Ok(GraphBarrierLifecycleOutcome::Failed(outcome))
877    }
878
879    async fn request_failure_close(
880        &self,
881        fence: &RunFence,
882        checkpoint: &CheckpointHead,
883        mut head: JournalHead,
884        event: EventId,
885        failure: Failure,
886        usage: BudgetUsage,
887    ) -> Result<GraphBarrierLifecycleOutcome, GraphLifecycleError> {
888        let (schema, _) = crate::standard_run_failure_close_event_schema().map_err(|_| {
889            GraphLifecycleError::InvalidHandoff {
890                operation: "load failure-close schema",
891            }
892        })?;
893        let payload =
894            crate::failure_close::payload(self.registry.schemas(), &schema, failure.id(), false)?;
895        for attempt in 1..=self.options.maximum_mutation_attempts() {
896            let append = worker_append(fence, head.clone(), event, payload.clone())?;
897            match self
898                .store
899                .request_run_failure_close(checkpoint, failure.clone(), usage.clone(), append)
900                .await
901            {
902                Ok(outcome) => {
903                    return Ok(GraphBarrierLifecycleOutcome::FailureClosing(Box::new(
904                        outcome,
905                    )));
906                }
907                Err(error)
908                    if attempt < self.options.maximum_mutation_attempts()
909                        && (error.is_retryable()
910                            || matches!(error, StoreError::StaleJournalHead)) =>
911                {
912                    tokio::time::sleep(
913                        self.options
914                            .mutation_retry_initial_delay()
915                            .saturating_mul(1_u32 << (attempt - 1))
916                            .min(Duration::from_secs(1)),
917                    )
918                    .await;
919                    let run = self
920                        .store
921                        .load_run(fence.tenant_id(), fence.run_id())
922                        .await?;
923                    head = run
924                        .journal_head()
925                        .cloned()
926                        .ok_or(GraphLifecycleError::StaleHandoff)?;
927                }
928                Err(error) => return Err(error.into()),
929            }
930        }
931        unreachable!("validated positive attempt count")
932    }
933
934    async fn validate_handoff_snapshot(
935        &self,
936        fence: &RunFence,
937        journal_head: &JournalHead,
938        event_id: EventId,
939        expected_revision: RunRevision,
940        committed_status: RunStatus,
941    ) -> Result<GraphLifecycleHandoffSnapshot, GraphLifecycleError> {
942        let run = self
943            .store
944            .load_run(fence.tenant_id(), fence.run_id())
945            .await?;
946        let is_fresh = run.lifecycle().revision() == expected_revision
947            && run.lifecycle().status() == RunStatus::Active
948            && run.journal_head() == Some(journal_head)
949            && run.lease().map(RunLease::fence) == Some(fence);
950        if is_fresh {
951            let observation = self.store.observe_live_lease(fence).await?;
952            if observation.lease().fence() != fence {
953                return Err(GraphLifecycleError::StaleHandoff);
954            }
955            return Ok(GraphLifecycleHandoffSnapshot::Fresh(run));
956        }
957
958        let committed_revision = expected_revision.get().checked_add(1);
959        let is_committed_retry = committed_revision
960            .is_some_and(|revision| run.lifecycle().revision() == RunRevision::new(revision))
961            && run.lifecycle().status() == committed_status
962            && run
963                .journal_head()
964                .is_some_and(|head| head.event_id() == event_id)
965            && run.lease().is_none();
966        if is_committed_retry {
967            return Ok(GraphLifecycleHandoffSnapshot::Committed(run));
968        }
969        Err(GraphLifecycleError::StaleHandoff)
970    }
971
972    fn validate_terminal_result(
973        &self,
974        provenance: &AgentResultProvenance,
975        completed_at: Timestamp,
976        output: &stateknot_core::NodeTerminalOutput,
977        evidence: GraphTerminalEvidence,
978    ) -> Result<AgentResult, GraphLifecycleError> {
979        let (descriptor, request, budget, artifacts, usage) = evidence.into_parts();
980        self.registry
981            .schemas()
982            .validate_bounded(request.input_schema(), request.input())
983            .map_err(GraphLifecycleError::input_schema)?;
984        self.registry
985            .schemas()
986            .validate_bounded(output.schema(), output.data())
987            .map_err(GraphLifecycleError::output_schema)?;
988        let result = AgentResult::new(
989            provenance.clone(),
990            completed_at,
991            output.schema().clone(),
992            output.data().clone(),
993            artifacts,
994            usage,
995        )
996        .map_err(GraphLifecycleError::agent_result)?;
997        result
998            .validate_for(provenance, &request, &descriptor, &budget)
999            .map_err(GraphLifecycleError::agent_result_validation)?;
1000        Ok(result)
1001    }
1002
1003    async fn commit_wait_with_retry(
1004        &self,
1005        append: JournalAppend,
1006        expected_revision: RunRevision,
1007        barrier: stateknot_core::CheckpointBarrier,
1008        registrations: Vec<stateknot_core::WaitRegistrationIntent>,
1009    ) -> Result<WaitCheckpointCommitOutcome, GraphLifecycleError> {
1010        let mut attempt = 1_u8;
1011        loop {
1012            match self
1013                .store
1014                .append_worker_wait_barrier(
1015                    append.clone(),
1016                    expected_revision,
1017                    barrier.clone(),
1018                    registrations.clone(),
1019                )
1020                .await
1021            {
1022                Ok(outcome) => return Ok(outcome),
1023                Err(error) if self.can_retry_mutation(&error, attempt) => {
1024                    self.mutation_backoff(attempt).await;
1025                    attempt = attempt.saturating_add(1);
1026                }
1027                Err(error) => return Err(error.into()),
1028            }
1029        }
1030    }
1031
1032    async fn commit_success_with_retry(
1033        &self,
1034        append: JournalAppend,
1035        projection: RunProjection,
1036        barrier: stateknot_core::CheckpointBarrier,
1037    ) -> Result<BarrierCommitOutcome, GraphLifecycleError> {
1038        let mut attempt = 1_u8;
1039        loop {
1040            match self
1041                .store
1042                .append_worker_barrier(append.clone(), projection.clone(), barrier.clone())
1043                .await
1044            {
1045                Ok(outcome) => return Ok(outcome),
1046                Err(error) if self.can_retry_mutation(&error, attempt) => {
1047                    self.mutation_backoff(attempt).await;
1048                    attempt = attempt.saturating_add(1);
1049                }
1050                Err(error) => return Err(error.into()),
1051            }
1052        }
1053    }
1054
1055    async fn commit_failure_with_retry(
1056        &self,
1057        append: JournalAppend,
1058        projection: RunProjection,
1059    ) -> Result<AppendOutcome, GraphLifecycleError> {
1060        let mut attempt = 1_u8;
1061        loop {
1062            match self
1063                .store
1064                .append_worker(append.clone(), projection.clone())
1065                .await
1066            {
1067                Ok(outcome) => return Ok(outcome),
1068                Err(error) if self.can_retry_mutation(&error, attempt) => {
1069                    self.mutation_backoff(attempt).await;
1070                    attempt = attempt.saturating_add(1);
1071                }
1072                Err(error) => return Err(error.into()),
1073            }
1074        }
1075    }
1076
1077    async fn release_with_retry(
1078        &self,
1079        fence: &RunFence,
1080    ) -> Result<LeaseReleaseOutcome, GraphLifecycleError> {
1081        let mut attempt = 1_u8;
1082        loop {
1083            match self.store.release_lease(fence).await {
1084                Ok(outcome) => return Ok(outcome),
1085                Err(error) if self.can_retry_mutation(&error, attempt) => {
1086                    self.mutation_backoff(attempt).await;
1087                    attempt = attempt.saturating_add(1);
1088                }
1089                Err(error) => return Err(error.into()),
1090            }
1091        }
1092    }
1093
1094    fn can_retry_mutation(&self, error: &StoreError, attempt: u8) -> bool {
1095        attempt < self.options.maximum_mutation_attempts && error.is_retryable()
1096    }
1097
1098    async fn mutation_backoff(&self, attempt: u8) {
1099        let multiplier = 1_u32
1100            .checked_shl(u32::from(attempt.saturating_sub(1)))
1101            .unwrap_or(u32::MAX);
1102        let delay = self
1103            .options
1104            .mutation_retry_initial_delay
1105            .checked_mul(multiplier)
1106            .unwrap_or(MAX_MUTATION_RETRY_DELAY)
1107            .min(MAX_MUTATION_RETRY_DELAY);
1108        tokio::time::sleep(delay).await;
1109    }
1110
1111    fn waiting_payload(
1112        &self,
1113        graph: &GraphReference,
1114        checkpoint: &CheckpointHead,
1115        successor_id: CheckpointId,
1116        successor_superstep: Superstep,
1117        wait_count: usize,
1118    ) -> Result<JournalPayload, GraphLifecycleError> {
1119        self.event_payload(
1120            "graph-barrier-waiting",
1121            json!({
1122                "operation": "graph_barrier_waiting",
1123                "graph_digest": digest_hex(graph.definition_digest()),
1124                "checkpoint_id": checkpoint.checkpoint_id().to_string(),
1125                "superstep": checkpoint.superstep().get().to_string(),
1126                "successor_checkpoint_id": successor_id.to_string(),
1127                "successor_superstep": successor_superstep.get().to_string(),
1128                "disposition": "wait",
1129                "wait_count": wait_count
1130            }),
1131        )
1132    }
1133
1134    fn succeeded_payload(
1135        &self,
1136        graph: &GraphReference,
1137        checkpoint: &CheckpointHead,
1138        successor_id: CheckpointId,
1139        successor_superstep: Superstep,
1140        output_digest: Digest,
1141    ) -> Result<JournalPayload, GraphLifecycleError> {
1142        self.event_payload(
1143            "graph-barrier-succeeded",
1144            json!({
1145                "operation": "graph_barrier_succeeded",
1146                "graph_digest": digest_hex(graph.definition_digest()),
1147                "checkpoint_id": checkpoint.checkpoint_id().to_string(),
1148                "superstep": checkpoint.superstep().get().to_string(),
1149                "successor_checkpoint_id": successor_id.to_string(),
1150                "successor_superstep": successor_superstep.get().to_string(),
1151                "disposition": "succeeded",
1152                "output_digest": digest_hex(output_digest)
1153            }),
1154        )
1155    }
1156
1157    fn failed_payload(
1158        &self,
1159        graph: &GraphReference,
1160        checkpoint: &CheckpointHead,
1161        failure_id: stateknot_core::FailureId,
1162        blockers: GraphDriveBlockers,
1163    ) -> Result<JournalPayload, GraphLifecycleError> {
1164        self.event_payload(
1165            "graph-run-failed",
1166            json!({
1167                "operation": "graph_run_failed",
1168                "graph_digest": digest_hex(graph.definition_digest()),
1169                "checkpoint_id": checkpoint.checkpoint_id().to_string(),
1170                "superstep": checkpoint.superstep().get().to_string(),
1171                "disposition": "failed",
1172                "failure_id": failure_id.to_string(),
1173                "in_flight": blockers.in_flight(),
1174                "failed": blockers.failed(),
1175                "exhausted": blockers.exhausted(),
1176                "unsupported": blockers.unsupported()
1177            }),
1178        )
1179    }
1180
1181    fn cancelled_payload(
1182        &self,
1183        checkpoint: &CheckpointHead,
1184        failure_id: FailureId,
1185    ) -> Result<JournalPayload, GraphLifecycleError> {
1186        self.event_payload_with_schema(
1187            &self.cancellation_schema,
1188            "agent-cancellation-confirmed",
1189            json!({
1190                "operation": "agent_cancellation_confirmed",
1191                "graph_digest": digest_hex(checkpoint.graph().definition_digest()),
1192                "checkpoint_id": checkpoint.checkpoint_id().to_string(),
1193                "superstep": checkpoint.superstep().get().to_string(),
1194                "failure_id": failure_id.to_string()
1195            }),
1196        )
1197    }
1198
1199    fn event_payload(
1200        &self,
1201        kind: &'static str,
1202        data: Value,
1203    ) -> Result<JournalPayload, GraphLifecycleError> {
1204        self.event_payload_with_schema(&self.journal_schema, kind, data)
1205    }
1206
1207    fn event_payload_with_schema(
1208        &self,
1209        schema: &SchemaReference,
1210        kind: &'static str,
1211        data: Value,
1212    ) -> Result<JournalPayload, GraphLifecycleError> {
1213        let data = BoundedJson::try_from_value(data)
1214            .map_err(|_| GraphLifecycleError::EventPayloadInvalid)?;
1215        self.registry
1216            .schemas()
1217            .validate_bounded(schema, &data)
1218            .map_err(|_| GraphLifecycleError::EventPayloadInvalid)?;
1219        let kind =
1220            JournalEventKind::new(kind).map_err(|_| GraphLifecycleError::EventPayloadInvalid)?;
1221        JournalPayload::new(schema.clone(), kind, data)
1222            .map_err(|_| GraphLifecycleError::EventPayloadInvalid)
1223    }
1224}
1225
1226pub(crate) fn superstep_limit_failure() -> Failure {
1227    Failure::new(
1228        FailureId::generate(),
1229        FailureCategory::RateLimited,
1230        FailureCode::new("runtime.graph.superstep_limit_reached").expect("static failure code"),
1231        FailureOrigin::new("stateknot.runtime.graph").expect("static failure origin"),
1232        FailureMessage::new("The graph exhausted its configured superstep limit.")
1233            .expect("static failure message"),
1234        RetryAdvice::Never,
1235    )
1236    .expect("static superstep failure semantics")
1237}
1238
1239impl fmt::Debug for DurableGraphLifecycle {
1240    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
1241        formatter
1242            .debug_struct("DurableGraphLifecycle")
1243            .field("registry", &self.registry)
1244            .field("journal_schema", &self.journal_schema)
1245            .field("cancellation_schema", &self.cancellation_schema)
1246            .field("options", &self.options)
1247            .finish_non_exhaustive()
1248    }
1249}
1250
1251/// Startup failure while binding graph lifecycle coordination.
1252#[derive(Clone, Debug, Eq, Error, PartialEq)]
1253#[non_exhaustive]
1254pub enum DurableGraphLifecycleBuildError {
1255    /// Child-enabled lifecycles require the exact failure-close release schema.
1256    #[error(transparent)]
1257    FailureClose(#[from] crate::RunFailureCloseBuildError),
1258    /// The embedded lifecycle schema release artifact was malformed.
1259    #[error(transparent)]
1260    StandardSchema(#[from] StandardGraphLifecycleSchemaError),
1261    /// The embedded cancellation schema release artifact was malformed.
1262    #[error(transparent)]
1263    CancellationSchema(#[from] StandardAgentCancellationSchemaError),
1264    /// The executable registry omitted the required standard lifecycle schema.
1265    #[error("standard graph lifecycle event schema is unavailable")]
1266    JournalSchemaUnavailable,
1267    /// The executable registry omitted the required cancellation schema.
1268    #[error("standard agent cancellation event schema is unavailable")]
1269    CancellationSchemaUnavailable,
1270}
1271
1272/// Payload-redacted graph lifecycle failure.
1273#[derive(Debug, Error)]
1274#[non_exhaustive]
1275pub enum GraphLifecycleError {
1276    /// `PostgreSQL` rejected or could not complete an operation.
1277    #[error(transparent)]
1278    Store {
1279        /// Exact payload-redacted provider failure.
1280        source: Box<StoreError>,
1281    },
1282    /// The handoff no longer matches the current run, journal, revision, lease,
1283    /// or runnable lifecycle.
1284    #[error("graph lifecycle handoff is stale")]
1285    StaleHandoff,
1286    /// A supposedly closed driver/lifecycle relationship failed.
1287    #[error("graph lifecycle handoff is invalid while attempting to {operation}")]
1288    InvalidHandoff {
1289        /// Stable payload-free operation label.
1290        operation: &'static str,
1291    },
1292    /// Trusted evidence could not be recovered.
1293    #[error("graph lifecycle evidence recovery failed: {0}")]
1294    Evidence(GraphLifecycleEvidenceError),
1295    /// A graph wait could not become an event-bound registration intent.
1296    #[error("graph lifecycle wait registration is invalid: {source}")]
1297    WaitRegistration {
1298        /// Exact public-safe wait validation failure.
1299        #[source]
1300        source: Box<DurableWaitError>,
1301    },
1302    /// The recovered request failed its pinned local input schema.
1303    #[error("graph lifecycle input failed its pinned schema: {source}")]
1304    InputSchema {
1305        /// Closed validation classification.
1306        #[source]
1307        source: GraphSchemaValidationError,
1308    },
1309    /// The graph output failed its pinned local output schema.
1310    #[error("graph lifecycle output failed its pinned schema: {source}")]
1311    OutputSchema {
1312        /// Closed validation classification.
1313        #[source]
1314        source: GraphSchemaValidationError,
1315    },
1316    /// Terminal output, artifacts, or accounting were intrinsically invalid.
1317    #[error("graph lifecycle successful result is invalid: {source}")]
1318    AgentResult {
1319        /// Exact public-safe intrinsic validation failure.
1320        #[source]
1321        source: Box<AgentResultError>,
1322    },
1323    /// Terminal success did not match trusted admission and budget snapshots.
1324    #[error("graph lifecycle successful result binding is invalid: {source}")]
1325    AgentResultValidation {
1326        /// Exact public-safe relationship failure.
1327        #[source]
1328        source: Box<AgentResultValidationError>,
1329    },
1330    /// Supervision attempted an invalid ordinary failure transition.
1331    #[error("graph lifecycle terminal failure is invalid: {source}")]
1332    RunFailure {
1333        /// Exact public-safe terminal failure validation error.
1334        #[source]
1335        source: RunFailureError,
1336    },
1337    /// Standard audit event data failed its pinned local schema or bounds.
1338    #[error("graph lifecycle event payload failed its pinned schema")]
1339    EventPayloadInvalid,
1340    /// A worker journal append could not be constructed.
1341    #[error("graph lifecycle worker journal append is invalid")]
1342    JournalAppendInvalid,
1343}
1344
1345impl GraphLifecycleError {
1346    fn wait_registration(source: DurableWaitError) -> Self {
1347        Self::WaitRegistration {
1348            source: Box::new(source),
1349        }
1350    }
1351
1352    const fn input_schema(source: GraphSchemaValidationError) -> Self {
1353        Self::InputSchema { source }
1354    }
1355
1356    const fn output_schema(source: GraphSchemaValidationError) -> Self {
1357        Self::OutputSchema { source }
1358    }
1359
1360    fn agent_result(source: AgentResultError) -> Self {
1361        Self::AgentResult {
1362            source: Box::new(source),
1363        }
1364    }
1365
1366    fn agent_result_validation(source: AgentResultValidationError) -> Self {
1367        Self::AgentResultValidation {
1368            source: Box::new(source),
1369        }
1370    }
1371
1372    const fn run_failure(source: RunFailureError) -> Self {
1373        Self::RunFailure { source }
1374    }
1375}
1376
1377impl From<StoreError> for GraphLifecycleError {
1378    fn from(source: StoreError) -> Self {
1379        Self::Store {
1380            source: Box::new(source),
1381        }
1382    }
1383}
1384
1385fn validate_barrier_scope(
1386    fence: &RunFence,
1387    journal_head: &JournalHead,
1388    checkpoint: &CheckpointHead,
1389    graph: &GraphReference,
1390) -> Result<(), GraphLifecycleError> {
1391    if journal_head.tenant_id() != fence.tenant_id()
1392        || journal_head.run_id() != fence.run_id()
1393        || checkpoint.tenant_id() != fence.tenant_id()
1394        || checkpoint.run_id() != fence.run_id()
1395        || checkpoint.graph() != graph
1396    {
1397        return Err(GraphLifecycleError::InvalidHandoff {
1398            operation: "bind graph barrier scope to its journal and fence",
1399        });
1400    }
1401    Ok(())
1402}
1403
1404fn validate_cancellation_scope(
1405    fence: &RunFence,
1406    journal_head: &JournalHead,
1407    checkpoint: &CheckpointHead,
1408) -> Result<(), GraphLifecycleError> {
1409    if journal_head.tenant_id() != fence.tenant_id()
1410        || journal_head.run_id() != fence.run_id()
1411        || checkpoint.tenant_id() != fence.tenant_id()
1412        || checkpoint.run_id() != fence.run_id()
1413    {
1414        return Err(GraphLifecycleError::InvalidHandoff {
1415            operation: "bind cancellation confirmation to one run scope",
1416        });
1417    }
1418    Ok(())
1419}
1420
1421fn validate_cancellation_checkpoint(
1422    run: &StoredRun,
1423    checkpoint: &CheckpointHead,
1424) -> Result<(), GraphLifecycleError> {
1425    let pointer = run
1426        .checkpoint()
1427        .ok_or(GraphLifecycleError::InvalidHandoff {
1428            operation: "bind cancellation confirmation to a checkpoint",
1429        })?;
1430    if pointer.checkpoint_id() != checkpoint.checkpoint_id()
1431        || pointer.superstep() != checkpoint.superstep()
1432        || pointer.digest() != checkpoint.digest()
1433    {
1434        return Err(GraphLifecycleError::StaleHandoff);
1435    }
1436    Ok(())
1437}
1438
1439fn validate_blocked_checkpoint_scope(
1440    fence: &RunFence,
1441    journal_head: &JournalHead,
1442    checkpoint: &CheckpointHead,
1443) -> Result<(), GraphLifecycleError> {
1444    let checkpoint_journal = checkpoint.journal_head();
1445    let journal_position_is_valid = checkpoint_journal == journal_head
1446        || (checkpoint_journal.sequence() < journal_head.sequence()
1447            && checkpoint_journal.recorded_at() <= journal_head.recorded_at());
1448    if journal_head.tenant_id() != fence.tenant_id()
1449        || journal_head.run_id() != fence.run_id()
1450        || checkpoint.tenant_id() != fence.tenant_id()
1451        || checkpoint.run_id() != fence.run_id()
1452        || checkpoint_journal.tenant_id() != fence.tenant_id()
1453        || checkpoint_journal.run_id() != fence.run_id()
1454        || !journal_position_is_valid
1455    {
1456        return Err(GraphLifecycleError::InvalidHandoff {
1457            operation: "bind blocked checkpoint to its journal and fence",
1458        });
1459    }
1460    Ok(())
1461}
1462
1463fn worker_append(
1464    fence: &RunFence,
1465    head: JournalHead,
1466    event_id: EventId,
1467    payload: JournalPayload,
1468) -> Result<JournalAppend, GraphLifecycleError> {
1469    let intent = JournalEventIntent::worker(
1470        fence.tenant_id().clone(),
1471        fence.run_id(),
1472        event_id,
1473        fence.clone(),
1474        payload,
1475    )
1476    .map_err(|_| GraphLifecycleError::JournalAppendInvalid)?;
1477    JournalAppend::new(JournalExpectation::exact(head), intent)
1478        .map_err(|_| GraphLifecycleError::JournalAppendInvalid)
1479}
1480
1481fn digest_hex(digest: Digest) -> String {
1482    const HEX: &[u8; 16] = b"0123456789abcdef";
1483    let mut value = String::with_capacity(Digest::SHA256_LEN * 2);
1484    for byte in digest.as_bytes() {
1485        value.push(char::from(HEX[usize::from(byte >> 4)]));
1486        value.push(char::from(HEX[usize::from(byte & 0x0f)]));
1487    }
1488    value
1489}
1490
1491#[cfg(test)]
1492mod tests {
1493    use super::*;
1494
1495    #[test]
1496    fn lifecycle_options_enforce_production_retry_bounds() {
1497        assert_eq!(
1498            DurableGraphLifecycleOptions::new(0, Duration::from_millis(1)),
1499            Err(DurableGraphLifecycleOptionsError::InvalidMutationAttempts)
1500        );
1501        assert_eq!(
1502            DurableGraphLifecycleOptions::new(1, Duration::ZERO),
1503            Err(DurableGraphLifecycleOptionsError::InvalidMutationRetryDelay)
1504        );
1505        let options = DurableGraphLifecycleOptions::new(4, Duration::from_millis(50)).unwrap();
1506        assert_eq!(options.maximum_mutation_attempts(), 4);
1507        assert_eq!(
1508            options.mutation_retry_initial_delay(),
1509            Duration::from_millis(50)
1510        );
1511    }
1512}