use std::{fmt, sync::Arc, time::Duration};
use serde_json::{Value, json};
use stateknot_core::{
AgentArtifacts, AgentDescriptor, AgentRequest, AgentResult, AgentResultError,
AgentResultProvenance, AgentResultValidationError, BoundedJson, BoxFuture, BudgetUsage,
CheckpointHead, CheckpointId, Digest, DurableWaitError, EventId, Failure, FailureCategory,
FailureCode, FailureId, FailureMessage, FailureOrigin, GraphBarrierDisposition, GraphReference,
GraphSchemaValidationError, JournalAppend, JournalEventIntent, JournalEventKind,
JournalExpectation, JournalHead, JournalPayload, ResolvedBudget, RetryAdvice, RunFailure,
RunFailureError, RunFence, RunLease, RunRevision, RunStatus, RunTransition, SchemaReference,
Superstep, Timestamp,
};
use stateknot_store_postgres::{
AppendOutcome, BarrierCommitOutcome, LeaseReleaseOutcome, PostgresStore, RunProjection,
StoreError, StoredRun, WaitCheckpointCommitOutcome,
};
use thiserror::Error;
use crate::{
ExecutableGraphRegistry, GraphBlockedHandoff, GraphCancellationHandoff, GraphDriveBlockers,
GraphLifecycleBarrierHandoff, StandardAgentCancellationSchemaError,
StandardGraphLifecycleSchemaError, standard_agent_cancellation_event_schema,
standard_graph_lifecycle_event_schema,
};
const MAX_MUTATION_RETRY_DELAY: Duration = Duration::from_secs(1);
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct DurableGraphLifecycleOptions {
maximum_mutation_attempts: u8,
mutation_retry_initial_delay: Duration,
}
impl DurableGraphLifecycleOptions {
pub const HARD_MAXIMUM_MUTATION_ATTEMPTS: u8 = 10;
pub fn new(
maximum_mutation_attempts: u8,
mutation_retry_initial_delay: Duration,
) -> Result<Self, DurableGraphLifecycleOptionsError> {
if maximum_mutation_attempts == 0
|| maximum_mutation_attempts > Self::HARD_MAXIMUM_MUTATION_ATTEMPTS
{
return Err(DurableGraphLifecycleOptionsError::InvalidMutationAttempts);
}
if mutation_retry_initial_delay.is_zero()
|| mutation_retry_initial_delay > MAX_MUTATION_RETRY_DELAY
{
return Err(DurableGraphLifecycleOptionsError::InvalidMutationRetryDelay);
}
Ok(Self {
maximum_mutation_attempts,
mutation_retry_initial_delay,
})
}
#[must_use]
pub const fn maximum_mutation_attempts(self) -> u8 {
self.maximum_mutation_attempts
}
#[must_use]
pub const fn mutation_retry_initial_delay(self) -> Duration {
self.mutation_retry_initial_delay
}
}
impl Default for DurableGraphLifecycleOptions {
fn default() -> Self {
Self {
maximum_mutation_attempts: 3,
mutation_retry_initial_delay: Duration::from_millis(25),
}
}
}
#[derive(Clone, Copy, Debug, Eq, Error, PartialEq)]
#[non_exhaustive]
pub enum DurableGraphLifecycleOptionsError {
#[error("graph lifecycle mutation attempt count is invalid")]
InvalidMutationAttempts,
#[error("graph lifecycle mutation retry delay is invalid")]
InvalidMutationRetryDelay,
}
#[derive(Clone, Debug)]
pub struct GraphTerminalEvidenceContext {
provenance: AgentResultProvenance,
graph: GraphReference,
checkpoint: CheckpointHead,
successor_checkpoint_id: CheckpointId,
successor_superstep: Superstep,
output_schema: SchemaReference,
output_digest: Digest,
expected_revision: RunRevision,
}
impl GraphTerminalEvidenceContext {
#[must_use]
pub const fn provenance(&self) -> &AgentResultProvenance {
&self.provenance
}
#[must_use]
pub const fn graph(&self) -> &GraphReference {
&self.graph
}
#[must_use]
pub const fn checkpoint(&self) -> &CheckpointHead {
&self.checkpoint
}
#[must_use]
pub const fn successor_checkpoint_id(&self) -> CheckpointId {
self.successor_checkpoint_id
}
#[must_use]
pub const fn successor_superstep(&self) -> Superstep {
self.successor_superstep
}
#[must_use]
pub const fn output_schema(&self) -> &SchemaReference {
&self.output_schema
}
#[must_use]
pub const fn output_digest(&self) -> Digest {
self.output_digest
}
#[must_use]
pub const fn expected_revision(&self) -> RunRevision {
self.expected_revision
}
}
#[derive(Clone)]
pub struct GraphTerminalEvidence {
descriptor: AgentDescriptor,
request: AgentRequest,
budget: ResolvedBudget,
artifacts: AgentArtifacts,
usage: BudgetUsage,
}
impl GraphTerminalEvidence {
#[must_use]
pub const fn new(
descriptor: AgentDescriptor,
request: AgentRequest,
budget: ResolvedBudget,
artifacts: AgentArtifacts,
usage: BudgetUsage,
) -> Self {
Self {
descriptor,
request,
budget,
artifacts,
usage,
}
}
#[must_use]
pub const fn descriptor(&self) -> &AgentDescriptor {
&self.descriptor
}
#[must_use]
pub const fn request(&self) -> &AgentRequest {
&self.request
}
#[must_use]
pub const fn budget(&self) -> &ResolvedBudget {
&self.budget
}
#[must_use]
pub const fn artifacts(&self) -> &AgentArtifacts {
&self.artifacts
}
#[must_use]
pub const fn usage(&self) -> &BudgetUsage {
&self.usage
}
fn into_parts(
self,
) -> (
AgentDescriptor,
AgentRequest,
ResolvedBudget,
AgentArtifacts,
BudgetUsage,
) {
(
self.descriptor,
self.request,
self.budget,
self.artifacts,
self.usage,
)
}
}
impl fmt::Debug for GraphTerminalEvidence {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("GraphTerminalEvidence")
.field("agent", self.descriptor.metadata().identity())
.field("input_schema", self.request.input_schema())
.field("output_schema", self.descriptor.output_schema())
.field("artifact_count", &self.artifacts.len())
.field("usage_recorded", &true)
.finish_non_exhaustive()
}
}
#[derive(Clone, Debug)]
pub struct GraphFailureEvidenceContext {
provenance: AgentResultProvenance,
graph: GraphReference,
checkpoint: CheckpointHead,
observed_at: Timestamp,
expected_revision: RunRevision,
blockers: GraphDriveBlockers,
}
impl GraphFailureEvidenceContext {
#[must_use]
pub const fn provenance(&self) -> &AgentResultProvenance {
&self.provenance
}
#[must_use]
pub const fn graph(&self) -> &GraphReference {
&self.graph
}
#[must_use]
pub const fn checkpoint(&self) -> &CheckpointHead {
&self.checkpoint
}
#[must_use]
pub const fn observed_at(&self) -> Timestamp {
self.observed_at
}
#[must_use]
pub const fn expected_revision(&self) -> RunRevision {
self.expected_revision
}
#[must_use]
pub const fn blockers(&self) -> GraphDriveBlockers {
self.blockers
}
}
#[derive(Clone, Debug)]
pub struct GraphFailureEvidence {
failure: Failure,
usage: BudgetUsage,
}
#[derive(Clone, Debug)]
pub struct GraphCancellationEvidenceContext {
provenance: AgentResultProvenance,
graph: GraphReference,
checkpoint: CheckpointHead,
expected_revision: RunRevision,
cancellation_failure_id: FailureId,
}
impl GraphCancellationEvidenceContext {
#[must_use]
pub const fn provenance(&self) -> &AgentResultProvenance {
&self.provenance
}
#[must_use]
pub const fn graph(&self) -> &GraphReference {
&self.graph
}
#[must_use]
pub const fn checkpoint(&self) -> &CheckpointHead {
&self.checkpoint
}
#[must_use]
pub const fn expected_revision(&self) -> RunRevision {
self.expected_revision
}
#[must_use]
pub const fn cancellation_failure_id(&self) -> FailureId {
self.cancellation_failure_id
}
}
#[derive(Clone, Debug)]
pub struct GraphCancellationEvidence {
usage: BudgetUsage,
}
impl GraphCancellationEvidence {
#[must_use]
pub const fn new(usage: BudgetUsage) -> Self {
Self { usage }
}
#[must_use]
pub const fn usage(&self) -> &BudgetUsage {
&self.usage
}
fn into_usage(self) -> BudgetUsage {
self.usage
}
}
impl GraphFailureEvidence {
#[must_use]
pub const fn new(failure: Failure, usage: BudgetUsage) -> Self {
Self { failure, usage }
}
#[must_use]
pub const fn failure(&self) -> &Failure {
&self.failure
}
#[must_use]
pub const fn usage(&self) -> &BudgetUsage {
&self.usage
}
fn into_parts(self) -> (Failure, BudgetUsage) {
(self.failure, self.usage)
}
}
#[derive(Clone, Copy, Debug, Eq, Error, PartialEq)]
#[non_exhaustive]
pub enum GraphLifecycleEvidenceError {
#[error("graph lifecycle evidence is temporarily unavailable")]
TemporarilyUnavailable,
#[error("graph lifecycle evidence is unavailable")]
Unavailable,
#[error("graph lifecycle evidence is corrupt")]
Corrupt,
}
pub trait GraphLifecycleEvidenceProvider: Send + Sync + 'static {
fn terminal_evidence(
&self,
context: GraphTerminalEvidenceContext,
) -> BoxFuture<'_, Result<GraphTerminalEvidence, GraphLifecycleEvidenceError>>;
fn failure_evidence(
&self,
context: GraphFailureEvidenceContext,
) -> BoxFuture<'_, Result<GraphFailureEvidence, GraphLifecycleEvidenceError>>;
fn cancellation_evidence(
&self,
context: GraphCancellationEvidenceContext,
) -> BoxFuture<'_, Result<GraphCancellationEvidence, GraphLifecycleEvidenceError>>;
}
#[derive(Clone, Debug)]
#[non_exhaustive]
pub enum GraphBarrierLifecycleOutcome {
FailureClosing(Box<stateknot_store_postgres::RunFailureCloseOutcome>),
Cancelled(AppendOutcome),
Waiting(WaitCheckpointCommitOutcome),
Succeeded(BarrierCommitOutcome),
Failed(AppendOutcome),
Released(LeaseReleaseOutcome),
}
#[derive(Clone)]
pub struct DurableGraphLifecycle {
store: PostgresStore,
registry: ExecutableGraphRegistry,
evidence: Arc<dyn GraphLifecycleEvidenceProvider>,
journal_schema: SchemaReference,
cancellation_schema: SchemaReference,
options: DurableGraphLifecycleOptions,
}
enum GraphLifecycleHandoffSnapshot {
Fresh(StoredRun),
Committed(StoredRun),
}
impl DurableGraphLifecycle {
pub fn new(
store: PostgresStore,
registry: ExecutableGraphRegistry,
evidence: Arc<dyn GraphLifecycleEvidenceProvider>,
options: DurableGraphLifecycleOptions,
) -> Result<Self, DurableGraphLifecycleBuildError> {
let (journal_schema, _) = standard_graph_lifecycle_event_schema()?;
if !registry.schemas().contains(&journal_schema) {
return Err(DurableGraphLifecycleBuildError::JournalSchemaUnavailable);
}
let (cancellation_schema, _) = standard_agent_cancellation_event_schema()?;
if !registry.schemas().contains(&cancellation_schema) {
return Err(DurableGraphLifecycleBuildError::CancellationSchemaUnavailable);
}
if registry.has_child_graphs() {
let (schema, _) = crate::standard_run_failure_close_event_schema()?;
if !registry.schemas().contains(&schema) {
return Err(crate::RunFailureCloseBuildError::SchemaUnavailable.into());
}
}
Ok(Self {
store,
registry,
evidence,
journal_schema,
cancellation_schema,
options,
})
}
#[must_use]
pub const fn options(&self) -> DurableGraphLifecycleOptions {
self.options
}
pub fn confirm_cancellation(
&self,
handoff: GraphCancellationHandoff,
) -> BoxFuture<'_, Result<GraphBarrierLifecycleOutcome, GraphLifecycleError>> {
Box::pin(self.confirm_cancellation_inner(handoff))
}
async fn confirm_cancellation_inner(
&self,
handoff: GraphCancellationHandoff,
) -> Result<GraphBarrierLifecycleOutcome, GraphLifecycleError> {
let (checkpoint, journal_head, event_id, expected_revision, lease, failure_id) =
handoff.into_parts();
let fence = lease.fence().clone();
validate_cancellation_scope(&fence, &journal_head, &checkpoint)?;
let run = self
.store
.load_run(fence.tenant_id(), fence.run_id())
.await?;
validate_cancellation_checkpoint(&run, &checkpoint)?;
let fresh = run.lifecycle().revision() == expected_revision
&& run.lifecycle().status() == RunStatus::CancellationRequested
&& run.journal_head() == Some(&journal_head)
&& run.lease().map(RunLease::fence) == Some(&fence)
&& run
.lifecycle()
.cancellation_request()
.is_some_and(|request| request.failure().id() == failure_id);
let (completed_at, usage) = if fresh {
let observation = self.store.observe_live_lease(&fence).await?;
if observation.lease().fence() != &fence {
return Err(GraphLifecycleError::StaleHandoff);
}
let context = GraphCancellationEvidenceContext {
provenance: run.lifecycle().provenance().clone(),
graph: checkpoint.graph().clone(),
checkpoint: checkpoint.clone(),
expected_revision,
cancellation_failure_id: failure_id,
};
let evidence = self
.evidence
.cancellation_evidence(context)
.await
.map_err(GraphLifecycleError::Evidence)?;
let usage = self
.store
.include_child_usage(fence.tenant_id(), fence.run_id(), evidence.into_usage())
.await?;
(observation.observed_at(), usage)
} else {
let committed_revision = expected_revision.get().checked_add(1);
let committed = committed_revision
.is_some_and(|revision| run.lifecycle().revision() == RunRevision::new(revision))
&& run.lifecycle().status() == RunStatus::Cancelled
&& run
.journal_head()
.is_some_and(|head| head.event_id() == event_id)
&& run.lease().is_none()
&& run
.lifecycle()
.cancellation_request()
.is_some_and(|request| request.failure().id() == failure_id);
if !committed {
return Err(GraphLifecycleError::StaleHandoff);
}
let usage = run.lifecycle().terminal_usage().cloned().ok_or(
GraphLifecycleError::InvalidHandoff {
operation: "recover committed cancellation accounting",
},
)?;
(run.lifecycle().changed_at(), usage)
};
let payload = self.cancelled_payload(&checkpoint, failure_id)?;
let append = worker_append(&fence, journal_head, event_id, payload)?;
let projection = RunProjection::transition(
expected_revision,
RunTransition::ConfirmCancellation {
completed_at,
usage,
},
);
let outcome = self.commit_failure_with_retry(append, projection).await?;
Ok(GraphBarrierLifecycleOutcome::Cancelled(outcome))
}
pub fn commit_barrier(
&self,
handoff: GraphLifecycleBarrierHandoff,
) -> BoxFuture<'_, Result<GraphBarrierLifecycleOutcome, GraphLifecycleError>> {
Box::pin(self.commit_barrier_inner(handoff))
}
async fn commit_barrier_inner(
&self,
handoff: GraphLifecycleBarrierHandoff,
) -> Result<GraphBarrierLifecycleOutcome, GraphLifecycleError> {
let (plan, journal_head, event_id, expected_revision, lease) = handoff.into_parts();
let fence = lease.fence().clone();
let (barrier, disposition) = plan.into_parts();
let base = barrier.base_checkpoint().clone();
validate_barrier_scope(&fence, &journal_head, &base, barrier.successor().graph())?;
match disposition {
GraphBarrierDisposition::Wait { waits } => {
let payload = self.waiting_payload(
barrier.successor().graph(),
&base,
barrier.successor().checkpoint_id(),
barrier.successor().superstep(),
waits.len(),
)?;
let append = worker_append(&fence, journal_head, event_id, payload)?;
let registrations = waits
.registration_intents(fence.tenant_id(), fence.run_id(), event_id)
.map_err(GraphLifecycleError::wait_registration)?;
let outcome = self
.commit_wait_with_retry(append, expected_revision, barrier, registrations)
.await?;
Ok(GraphBarrierLifecycleOutcome::Waiting(outcome))
}
GraphBarrierDisposition::Terminal { output } => {
let snapshot = self
.validate_handoff_snapshot(
&fence,
&journal_head,
event_id,
expected_revision,
RunStatus::Succeeded,
)
.await?;
let result = match snapshot {
GraphLifecycleHandoffSnapshot::Fresh(run) => {
let context = GraphTerminalEvidenceContext {
provenance: run.lifecycle().provenance().clone(),
graph: barrier.successor().graph().clone(),
checkpoint: base.clone(),
successor_checkpoint_id: barrier.successor().checkpoint_id(),
successor_superstep: barrier.successor().superstep(),
output_schema: output.schema().clone(),
output_digest: output.digest(),
expected_revision,
};
let mut evidence = self
.evidence
.terminal_evidence(context)
.await
.map_err(GraphLifecycleError::Evidence)?;
evidence.usage = self
.store
.include_child_usage(fence.tenant_id(), fence.run_id(), evidence.usage)
.await?;
self.validate_terminal_result(
run.lifecycle().provenance(),
journal_head.recorded_at(),
&output,
evidence,
)?
}
GraphLifecycleHandoffSnapshot::Committed(run) => {
run.lifecycle().result().cloned().ok_or(
GraphLifecycleError::InvalidHandoff {
operation: "recover the committed successful result",
},
)?
}
};
let payload = self.succeeded_payload(
barrier.successor().graph(),
&base,
barrier.successor().checkpoint_id(),
barrier.successor().superstep(),
output.digest(),
)?;
let append = worker_append(&fence, journal_head, event_id, payload)?;
let projection =
RunProjection::transition(expected_revision, RunTransition::Succeed { result });
let outcome = self
.commit_success_with_retry(append, projection, barrier)
.await?;
Ok(GraphBarrierLifecycleOutcome::Succeeded(outcome))
}
GraphBarrierDisposition::Continue => Err(GraphLifecycleError::InvalidHandoff {
operation: "commit a Continue disposition through lifecycle coordination",
}),
_ => Err(GraphLifecycleError::InvalidHandoff {
operation: "commit an unsupported graph barrier disposition",
}),
}
}
pub fn resolve_blocked(
&self,
handoff: GraphBlockedHandoff,
) -> BoxFuture<'_, Result<GraphBarrierLifecycleOutcome, GraphLifecycleError>> {
Box::pin(self.resolve_blocked_inner(handoff))
}
#[allow(clippy::too_many_lines)]
async fn resolve_blocked_inner(
&self,
handoff: GraphBlockedHandoff,
) -> Result<GraphBarrierLifecycleOutcome, GraphLifecycleError> {
let (plan, lease, event_id, expected_revision, blockers) = handoff.into_parts();
let fence = lease.fence().clone();
if plan.fence() != &fence {
return Err(GraphLifecycleError::InvalidHandoff {
operation: "bind blocked recovery plan to its retained lease",
});
}
if blockers.in_flight() > 0 {
let release = self.release_with_retry(&fence).await?;
return Ok(GraphBarrierLifecycleOutcome::Released(release));
}
if blockers.failed() == 0
&& blockers.exhausted() == 0
&& blockers.unsupported() == 0
&& !blockers.superstep_limit_reached()
{
return Err(GraphLifecycleError::InvalidHandoff {
operation: "supervise a blocked plan without terminal blockers",
});
}
let journal_head = plan.journal_head().clone();
let child_graph = self
.registry
.resolve(plan.checkpoint().graph())
.is_some_and(|entry| entry.graph().child_runs().is_some());
if child_graph {
if let Some(record) = self
.store
.load_run_failure_close(fence.tenant_id(), fence.run_id())
.await?
{
if record.checkpoint() != &plan.checkpoint().head()
|| record.registration().source().worker_fence() != Some(&fence)
|| record.lifecycle().revision() != expected_revision
{
return Err(GraphLifecycleError::StaleHandoff);
}
return Ok(GraphBarrierLifecycleOutcome::FailureClosing(Box::new(
stateknot_store_postgres::RunFailureCloseOutcome::Existing(record),
)));
}
}
let snapshot = self
.validate_handoff_snapshot(
&fence,
&journal_head,
event_id,
expected_revision,
RunStatus::Failed,
)
.await?;
let checkpoint = plan.checkpoint().head();
validate_blocked_checkpoint_scope(&fence, &journal_head, &checkpoint)?;
let failure = match snapshot {
GraphLifecycleHandoffSnapshot::Fresh(run) => {
let context = GraphFailureEvidenceContext {
provenance: run.lifecycle().provenance().clone(),
graph: checkpoint.graph().clone(),
checkpoint: checkpoint.clone(),
observed_at: plan.observed_at(),
expected_revision,
blockers,
};
let evidence = self
.evidence
.failure_evidence(context)
.await
.map_err(GraphLifecycleError::Evidence)?;
let (failure, usage) = evidence.into_parts();
let failure = if blockers.superstep_limit_reached() {
superstep_limit_failure()
} else {
failure
};
if child_graph {
return self
.request_failure_close(
&fence,
&checkpoint,
journal_head,
event_id,
failure,
usage,
)
.await;
}
let usage = self
.store
.include_child_usage(fence.tenant_id(), fence.run_id(), usage)
.await?;
RunFailure::new(failure, plan.observed_at(), usage)
.map_err(GraphLifecycleError::run_failure)?
}
GraphLifecycleHandoffSnapshot::Committed(run) => {
let lifecycle = run.lifecycle();
let failure = lifecycle.terminal_failure().cloned().ok_or(
GraphLifecycleError::InvalidHandoff {
operation: "recover the committed terminal failure",
},
)?;
let usage = lifecycle.terminal_usage().cloned().ok_or(
GraphLifecycleError::InvalidHandoff {
operation: "recover committed terminal failure accounting",
},
)?;
RunFailure::new(failure, lifecycle.changed_at(), usage)
.map_err(GraphLifecycleError::run_failure)?
}
};
let failure_id = failure.failure().id();
let payload = self.failed_payload(checkpoint.graph(), &checkpoint, failure_id, blockers)?;
let append = worker_append(&fence, journal_head, event_id, payload)?;
let projection =
RunProjection::transition(expected_revision, RunTransition::Fail { failure });
let outcome = self.commit_failure_with_retry(append, projection).await?;
Ok(GraphBarrierLifecycleOutcome::Failed(outcome))
}
async fn request_failure_close(
&self,
fence: &RunFence,
checkpoint: &CheckpointHead,
mut head: JournalHead,
event: EventId,
failure: Failure,
usage: BudgetUsage,
) -> Result<GraphBarrierLifecycleOutcome, GraphLifecycleError> {
let (schema, _) = crate::standard_run_failure_close_event_schema().map_err(|_| {
GraphLifecycleError::InvalidHandoff {
operation: "load failure-close schema",
}
})?;
let payload =
crate::failure_close::payload(self.registry.schemas(), &schema, failure.id(), false)?;
for attempt in 1..=self.options.maximum_mutation_attempts() {
let append = worker_append(fence, head.clone(), event, payload.clone())?;
match self
.store
.request_run_failure_close(checkpoint, failure.clone(), usage.clone(), append)
.await
{
Ok(outcome) => {
return Ok(GraphBarrierLifecycleOutcome::FailureClosing(Box::new(
outcome,
)));
}
Err(error)
if attempt < self.options.maximum_mutation_attempts()
&& (error.is_retryable()
|| matches!(error, StoreError::StaleJournalHead)) =>
{
tokio::time::sleep(
self.options
.mutation_retry_initial_delay()
.saturating_mul(1_u32 << (attempt - 1))
.min(Duration::from_secs(1)),
)
.await;
let run = self
.store
.load_run(fence.tenant_id(), fence.run_id())
.await?;
head = run
.journal_head()
.cloned()
.ok_or(GraphLifecycleError::StaleHandoff)?;
}
Err(error) => return Err(error.into()),
}
}
unreachable!("validated positive attempt count")
}
async fn validate_handoff_snapshot(
&self,
fence: &RunFence,
journal_head: &JournalHead,
event_id: EventId,
expected_revision: RunRevision,
committed_status: RunStatus,
) -> Result<GraphLifecycleHandoffSnapshot, GraphLifecycleError> {
let run = self
.store
.load_run(fence.tenant_id(), fence.run_id())
.await?;
let is_fresh = run.lifecycle().revision() == expected_revision
&& run.lifecycle().status() == RunStatus::Active
&& run.journal_head() == Some(journal_head)
&& run.lease().map(RunLease::fence) == Some(fence);
if is_fresh {
let observation = self.store.observe_live_lease(fence).await?;
if observation.lease().fence() != fence {
return Err(GraphLifecycleError::StaleHandoff);
}
return Ok(GraphLifecycleHandoffSnapshot::Fresh(run));
}
let committed_revision = expected_revision.get().checked_add(1);
let is_committed_retry = committed_revision
.is_some_and(|revision| run.lifecycle().revision() == RunRevision::new(revision))
&& run.lifecycle().status() == committed_status
&& run
.journal_head()
.is_some_and(|head| head.event_id() == event_id)
&& run.lease().is_none();
if is_committed_retry {
return Ok(GraphLifecycleHandoffSnapshot::Committed(run));
}
Err(GraphLifecycleError::StaleHandoff)
}
fn validate_terminal_result(
&self,
provenance: &AgentResultProvenance,
completed_at: Timestamp,
output: &stateknot_core::NodeTerminalOutput,
evidence: GraphTerminalEvidence,
) -> Result<AgentResult, GraphLifecycleError> {
let (descriptor, request, budget, artifacts, usage) = evidence.into_parts();
self.registry
.schemas()
.validate_bounded(request.input_schema(), request.input())
.map_err(GraphLifecycleError::input_schema)?;
self.registry
.schemas()
.validate_bounded(output.schema(), output.data())
.map_err(GraphLifecycleError::output_schema)?;
let result = AgentResult::new(
provenance.clone(),
completed_at,
output.schema().clone(),
output.data().clone(),
artifacts,
usage,
)
.map_err(GraphLifecycleError::agent_result)?;
result
.validate_for(provenance, &request, &descriptor, &budget)
.map_err(GraphLifecycleError::agent_result_validation)?;
Ok(result)
}
async fn commit_wait_with_retry(
&self,
append: JournalAppend,
expected_revision: RunRevision,
barrier: stateknot_core::CheckpointBarrier,
registrations: Vec<stateknot_core::WaitRegistrationIntent>,
) -> Result<WaitCheckpointCommitOutcome, GraphLifecycleError> {
let mut attempt = 1_u8;
loop {
match self
.store
.append_worker_wait_barrier(
append.clone(),
expected_revision,
barrier.clone(),
registrations.clone(),
)
.await
{
Ok(outcome) => return Ok(outcome),
Err(error) if self.can_retry_mutation(&error, attempt) => {
self.mutation_backoff(attempt).await;
attempt = attempt.saturating_add(1);
}
Err(error) => return Err(error.into()),
}
}
}
async fn commit_success_with_retry(
&self,
append: JournalAppend,
projection: RunProjection,
barrier: stateknot_core::CheckpointBarrier,
) -> Result<BarrierCommitOutcome, GraphLifecycleError> {
let mut attempt = 1_u8;
loop {
match self
.store
.append_worker_barrier(append.clone(), projection.clone(), barrier.clone())
.await
{
Ok(outcome) => return Ok(outcome),
Err(error) if self.can_retry_mutation(&error, attempt) => {
self.mutation_backoff(attempt).await;
attempt = attempt.saturating_add(1);
}
Err(error) => return Err(error.into()),
}
}
}
async fn commit_failure_with_retry(
&self,
append: JournalAppend,
projection: RunProjection,
) -> Result<AppendOutcome, GraphLifecycleError> {
let mut attempt = 1_u8;
loop {
match self
.store
.append_worker(append.clone(), projection.clone())
.await
{
Ok(outcome) => return Ok(outcome),
Err(error) if self.can_retry_mutation(&error, attempt) => {
self.mutation_backoff(attempt).await;
attempt = attempt.saturating_add(1);
}
Err(error) => return Err(error.into()),
}
}
}
async fn release_with_retry(
&self,
fence: &RunFence,
) -> Result<LeaseReleaseOutcome, GraphLifecycleError> {
let mut attempt = 1_u8;
loop {
match self.store.release_lease(fence).await {
Ok(outcome) => return Ok(outcome),
Err(error) if self.can_retry_mutation(&error, attempt) => {
self.mutation_backoff(attempt).await;
attempt = attempt.saturating_add(1);
}
Err(error) => return Err(error.into()),
}
}
}
fn can_retry_mutation(&self, error: &StoreError, attempt: u8) -> bool {
attempt < self.options.maximum_mutation_attempts && error.is_retryable()
}
async fn mutation_backoff(&self, attempt: u8) {
let multiplier = 1_u32
.checked_shl(u32::from(attempt.saturating_sub(1)))
.unwrap_or(u32::MAX);
let delay = self
.options
.mutation_retry_initial_delay
.checked_mul(multiplier)
.unwrap_or(MAX_MUTATION_RETRY_DELAY)
.min(MAX_MUTATION_RETRY_DELAY);
tokio::time::sleep(delay).await;
}
fn waiting_payload(
&self,
graph: &GraphReference,
checkpoint: &CheckpointHead,
successor_id: CheckpointId,
successor_superstep: Superstep,
wait_count: usize,
) -> Result<JournalPayload, GraphLifecycleError> {
self.event_payload(
"graph-barrier-waiting",
json!({
"operation": "graph_barrier_waiting",
"graph_digest": digest_hex(graph.definition_digest()),
"checkpoint_id": checkpoint.checkpoint_id().to_string(),
"superstep": checkpoint.superstep().get().to_string(),
"successor_checkpoint_id": successor_id.to_string(),
"successor_superstep": successor_superstep.get().to_string(),
"disposition": "wait",
"wait_count": wait_count
}),
)
}
fn succeeded_payload(
&self,
graph: &GraphReference,
checkpoint: &CheckpointHead,
successor_id: CheckpointId,
successor_superstep: Superstep,
output_digest: Digest,
) -> Result<JournalPayload, GraphLifecycleError> {
self.event_payload(
"graph-barrier-succeeded",
json!({
"operation": "graph_barrier_succeeded",
"graph_digest": digest_hex(graph.definition_digest()),
"checkpoint_id": checkpoint.checkpoint_id().to_string(),
"superstep": checkpoint.superstep().get().to_string(),
"successor_checkpoint_id": successor_id.to_string(),
"successor_superstep": successor_superstep.get().to_string(),
"disposition": "succeeded",
"output_digest": digest_hex(output_digest)
}),
)
}
fn failed_payload(
&self,
graph: &GraphReference,
checkpoint: &CheckpointHead,
failure_id: stateknot_core::FailureId,
blockers: GraphDriveBlockers,
) -> Result<JournalPayload, GraphLifecycleError> {
self.event_payload(
"graph-run-failed",
json!({
"operation": "graph_run_failed",
"graph_digest": digest_hex(graph.definition_digest()),
"checkpoint_id": checkpoint.checkpoint_id().to_string(),
"superstep": checkpoint.superstep().get().to_string(),
"disposition": "failed",
"failure_id": failure_id.to_string(),
"in_flight": blockers.in_flight(),
"failed": blockers.failed(),
"exhausted": blockers.exhausted(),
"unsupported": blockers.unsupported()
}),
)
}
fn cancelled_payload(
&self,
checkpoint: &CheckpointHead,
failure_id: FailureId,
) -> Result<JournalPayload, GraphLifecycleError> {
self.event_payload_with_schema(
&self.cancellation_schema,
"agent-cancellation-confirmed",
json!({
"operation": "agent_cancellation_confirmed",
"graph_digest": digest_hex(checkpoint.graph().definition_digest()),
"checkpoint_id": checkpoint.checkpoint_id().to_string(),
"superstep": checkpoint.superstep().get().to_string(),
"failure_id": failure_id.to_string()
}),
)
}
fn event_payload(
&self,
kind: &'static str,
data: Value,
) -> Result<JournalPayload, GraphLifecycleError> {
self.event_payload_with_schema(&self.journal_schema, kind, data)
}
fn event_payload_with_schema(
&self,
schema: &SchemaReference,
kind: &'static str,
data: Value,
) -> Result<JournalPayload, GraphLifecycleError> {
let data = BoundedJson::try_from_value(data)
.map_err(|_| GraphLifecycleError::EventPayloadInvalid)?;
self.registry
.schemas()
.validate_bounded(schema, &data)
.map_err(|_| GraphLifecycleError::EventPayloadInvalid)?;
let kind =
JournalEventKind::new(kind).map_err(|_| GraphLifecycleError::EventPayloadInvalid)?;
JournalPayload::new(schema.clone(), kind, data)
.map_err(|_| GraphLifecycleError::EventPayloadInvalid)
}
}
pub(crate) fn superstep_limit_failure() -> Failure {
Failure::new(
FailureId::generate(),
FailureCategory::RateLimited,
FailureCode::new("runtime.graph.superstep_limit_reached").expect("static failure code"),
FailureOrigin::new("stateknot.runtime.graph").expect("static failure origin"),
FailureMessage::new("The graph exhausted its configured superstep limit.")
.expect("static failure message"),
RetryAdvice::Never,
)
.expect("static superstep failure semantics")
}
impl fmt::Debug for DurableGraphLifecycle {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("DurableGraphLifecycle")
.field("registry", &self.registry)
.field("journal_schema", &self.journal_schema)
.field("cancellation_schema", &self.cancellation_schema)
.field("options", &self.options)
.finish_non_exhaustive()
}
}
#[derive(Clone, Debug, Eq, Error, PartialEq)]
#[non_exhaustive]
pub enum DurableGraphLifecycleBuildError {
#[error(transparent)]
FailureClose(#[from] crate::RunFailureCloseBuildError),
#[error(transparent)]
StandardSchema(#[from] StandardGraphLifecycleSchemaError),
#[error(transparent)]
CancellationSchema(#[from] StandardAgentCancellationSchemaError),
#[error("standard graph lifecycle event schema is unavailable")]
JournalSchemaUnavailable,
#[error("standard agent cancellation event schema is unavailable")]
CancellationSchemaUnavailable,
}
#[derive(Debug, Error)]
#[non_exhaustive]
pub enum GraphLifecycleError {
#[error(transparent)]
Store {
source: Box<StoreError>,
},
#[error("graph lifecycle handoff is stale")]
StaleHandoff,
#[error("graph lifecycle handoff is invalid while attempting to {operation}")]
InvalidHandoff {
operation: &'static str,
},
#[error("graph lifecycle evidence recovery failed: {0}")]
Evidence(GraphLifecycleEvidenceError),
#[error("graph lifecycle wait registration is invalid: {source}")]
WaitRegistration {
#[source]
source: Box<DurableWaitError>,
},
#[error("graph lifecycle input failed its pinned schema: {source}")]
InputSchema {
#[source]
source: GraphSchemaValidationError,
},
#[error("graph lifecycle output failed its pinned schema: {source}")]
OutputSchema {
#[source]
source: GraphSchemaValidationError,
},
#[error("graph lifecycle successful result is invalid: {source}")]
AgentResult {
#[source]
source: Box<AgentResultError>,
},
#[error("graph lifecycle successful result binding is invalid: {source}")]
AgentResultValidation {
#[source]
source: Box<AgentResultValidationError>,
},
#[error("graph lifecycle terminal failure is invalid: {source}")]
RunFailure {
#[source]
source: RunFailureError,
},
#[error("graph lifecycle event payload failed its pinned schema")]
EventPayloadInvalid,
#[error("graph lifecycle worker journal append is invalid")]
JournalAppendInvalid,
}
impl GraphLifecycleError {
fn wait_registration(source: DurableWaitError) -> Self {
Self::WaitRegistration {
source: Box::new(source),
}
}
const fn input_schema(source: GraphSchemaValidationError) -> Self {
Self::InputSchema { source }
}
const fn output_schema(source: GraphSchemaValidationError) -> Self {
Self::OutputSchema { source }
}
fn agent_result(source: AgentResultError) -> Self {
Self::AgentResult {
source: Box::new(source),
}
}
fn agent_result_validation(source: AgentResultValidationError) -> Self {
Self::AgentResultValidation {
source: Box::new(source),
}
}
const fn run_failure(source: RunFailureError) -> Self {
Self::RunFailure { source }
}
}
impl From<StoreError> for GraphLifecycleError {
fn from(source: StoreError) -> Self {
Self::Store {
source: Box::new(source),
}
}
}
fn validate_barrier_scope(
fence: &RunFence,
journal_head: &JournalHead,
checkpoint: &CheckpointHead,
graph: &GraphReference,
) -> Result<(), GraphLifecycleError> {
if journal_head.tenant_id() != fence.tenant_id()
|| journal_head.run_id() != fence.run_id()
|| checkpoint.tenant_id() != fence.tenant_id()
|| checkpoint.run_id() != fence.run_id()
|| checkpoint.graph() != graph
{
return Err(GraphLifecycleError::InvalidHandoff {
operation: "bind graph barrier scope to its journal and fence",
});
}
Ok(())
}
fn validate_cancellation_scope(
fence: &RunFence,
journal_head: &JournalHead,
checkpoint: &CheckpointHead,
) -> Result<(), GraphLifecycleError> {
if journal_head.tenant_id() != fence.tenant_id()
|| journal_head.run_id() != fence.run_id()
|| checkpoint.tenant_id() != fence.tenant_id()
|| checkpoint.run_id() != fence.run_id()
{
return Err(GraphLifecycleError::InvalidHandoff {
operation: "bind cancellation confirmation to one run scope",
});
}
Ok(())
}
fn validate_cancellation_checkpoint(
run: &StoredRun,
checkpoint: &CheckpointHead,
) -> Result<(), GraphLifecycleError> {
let pointer = run
.checkpoint()
.ok_or(GraphLifecycleError::InvalidHandoff {
operation: "bind cancellation confirmation to a checkpoint",
})?;
if pointer.checkpoint_id() != checkpoint.checkpoint_id()
|| pointer.superstep() != checkpoint.superstep()
|| pointer.digest() != checkpoint.digest()
{
return Err(GraphLifecycleError::StaleHandoff);
}
Ok(())
}
fn validate_blocked_checkpoint_scope(
fence: &RunFence,
journal_head: &JournalHead,
checkpoint: &CheckpointHead,
) -> Result<(), GraphLifecycleError> {
let checkpoint_journal = checkpoint.journal_head();
let journal_position_is_valid = checkpoint_journal == journal_head
|| (checkpoint_journal.sequence() < journal_head.sequence()
&& checkpoint_journal.recorded_at() <= journal_head.recorded_at());
if journal_head.tenant_id() != fence.tenant_id()
|| journal_head.run_id() != fence.run_id()
|| checkpoint.tenant_id() != fence.tenant_id()
|| checkpoint.run_id() != fence.run_id()
|| checkpoint_journal.tenant_id() != fence.tenant_id()
|| checkpoint_journal.run_id() != fence.run_id()
|| !journal_position_is_valid
{
return Err(GraphLifecycleError::InvalidHandoff {
operation: "bind blocked checkpoint to its journal and fence",
});
}
Ok(())
}
fn worker_append(
fence: &RunFence,
head: JournalHead,
event_id: EventId,
payload: JournalPayload,
) -> Result<JournalAppend, GraphLifecycleError> {
let intent = JournalEventIntent::worker(
fence.tenant_id().clone(),
fence.run_id(),
event_id,
fence.clone(),
payload,
)
.map_err(|_| GraphLifecycleError::JournalAppendInvalid)?;
JournalAppend::new(JournalExpectation::exact(head), intent)
.map_err(|_| GraphLifecycleError::JournalAppendInvalid)
}
fn digest_hex(digest: Digest) -> String {
const HEX: &[u8; 16] = b"0123456789abcdef";
let mut value = String::with_capacity(Digest::SHA256_LEN * 2);
for byte in digest.as_bytes() {
value.push(char::from(HEX[usize::from(byte >> 4)]));
value.push(char::from(HEX[usize::from(byte & 0x0f)]));
}
value
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn lifecycle_options_enforce_production_retry_bounds() {
assert_eq!(
DurableGraphLifecycleOptions::new(0, Duration::from_millis(1)),
Err(DurableGraphLifecycleOptionsError::InvalidMutationAttempts)
);
assert_eq!(
DurableGraphLifecycleOptions::new(1, Duration::ZERO),
Err(DurableGraphLifecycleOptionsError::InvalidMutationRetryDelay)
);
let options = DurableGraphLifecycleOptions::new(4, Duration::from_millis(50)).unwrap();
assert_eq!(options.maximum_mutation_attempts(), 4);
assert_eq!(
options.mutation_retry_initial_delay(),
Duration::from_millis(50)
);
}
}