Skip to main content

stateknot_runtime/
driver.rs

1// Copyright 2026 StateKnot contributors
2// SPDX-License-Identifier: Apache-2.0
3
4//! Fenced, crash-recoverable execution of registered graph nodes.
5
6use std::{
7    fmt,
8    panic::AssertUnwindSafe,
9    sync::{
10        Arc,
11        atomic::{AtomicBool, Ordering},
12    },
13    time::Duration,
14};
15
16use futures_util::FutureExt;
17use serde_json::{Value, json};
18use stateknot_core::{
19    AttemptId, BoundedJson, BoxFuture, BudgetUsage, CancellationObserver, CancellationSignal,
20    Checkpoint, CheckpointId, Digest, EventId, Failure, FailureCategory, FailureCode, FailureId,
21    FailureMessage, FailureOrigin, GraphBarrierDisposition, GraphBarrierPlan,
22    GraphBarrierPlanError, GraphReference, JournalAppend, JournalEventIntent, JournalEventKind,
23    JournalExpectation, JournalHead, JournalPayload, NodeAttemptStartHead, PendingNodeResult,
24    PendingNodeResultIntent, QuarantineId, ReadyNodeRecoveryPlan, RecoveryNodeKind, RetryAdvice,
25    RunFence, RunLease, RunRevision, RunStatus, Timestamp,
26};
27use stateknot_store_postgres::{
28    BarrierCommitOutcome, ClaimedRunRecovery, CorruptionQuarantineContext,
29    DelayedRetryScheduleOutcome, GraphReplayLimits, GraphReplayReport, LeaseReleaseOutcome,
30    LeaseRenewalOutcome, NodeAttemptCommitOutcome, PendingNodeResultPageSize, PostgresStore,
31    RunProjection, StoreError, StoredRun,
32};
33use thiserror::Error;
34use tokio::{
35    sync::Notify,
36    task::{JoinError, JoinSet},
37    time::Instant,
38};
39
40use crate::{
41    ExecutableGraph, ExecutableGraphRegistry, GraphNodeContext, GraphNodeContextError,
42    GraphNodeExecution, GraphNodeExecutionError, GraphNodeScheduling,
43    StandardGraphDriverSchemaError, standard_graph_driver_event_schema,
44};
45
46const MAX_NODE_TIMEOUT: Duration = Duration::from_secs(7 * 24 * 60 * 60);
47const MAX_MUTATION_RETRY_DELAY: Duration = Duration::from_secs(1);
48const RECOVERY_EVIDENCE_DOMAIN: &[u8] = b"stateknot/runtime/graph-driver/recovery/v1\0";
49
50/// Resource, lease, deadline, and retry policy for one durable driver.
51#[derive(Clone, Copy, Debug, Eq, PartialEq)]
52pub struct DurableGraphDriverOptions {
53    replay_limits: GraphReplayLimits,
54    maximum_durable_events: u32,
55    lease_renewal_interval: Duration,
56    node_execution_timeout: Duration,
57    cancellation_poll_interval: Duration,
58    cancellation_grace_period: Duration,
59    maximum_parallel_nodes: u16,
60    maximum_mutation_attempts: u8,
61    mutation_retry_initial_delay: Duration,
62}
63
64impl DurableGraphDriverOptions {
65    /// Absolute work quantum accepted for one call to [`DurableGraphDriver::drive`].
66    pub const HARD_MAXIMUM_DURABLE_EVENTS: u32 = 65_536;
67    /// Absolute number of identical durable mutation attempts.
68    pub const HARD_MAXIMUM_MUTATION_ATTEMPTS: u8 = 10;
69    /// Fastest accepted database polling cadence for durable cancellation intent.
70    pub const HARD_MINIMUM_CANCELLATION_POLL_INTERVAL: Duration = Duration::from_millis(10);
71    /// Slowest accepted observation cadence for durable cancellation intent.
72    pub const HARD_MAXIMUM_CANCELLATION_POLL_INTERVAL: Duration = Duration::from_secs(60);
73    /// Longest cooperative cleanup window after durable cancellation wins.
74    pub const HARD_MAXIMUM_CANCELLATION_GRACE_PERIOD: Duration = Duration::from_secs(5 * 60);
75    /// Absolute process-local concurrency ceiling for one run.
76    pub const HARD_MAXIMUM_PARALLEL_NODES: u16 = 64;
77
78    /// Constructs a fully explicit driver policy.
79    ///
80    /// # Errors
81    ///
82    /// Rejects a work quantum below one complete node start/completion pair,
83    /// invalid renewal precision, a zero or excessive node deadline, and unsafe
84    /// mutation retry parameters.
85    pub fn new(
86        replay_limits: GraphReplayLimits,
87        maximum_durable_events: u32,
88        lease_renewal_interval: Duration,
89        node_execution_timeout: Duration,
90        maximum_mutation_attempts: u8,
91        mutation_retry_initial_delay: Duration,
92    ) -> Result<Self, DurableGraphDriverOptionsError> {
93        if !(2..=Self::HARD_MAXIMUM_DURABLE_EVENTS).contains(&maximum_durable_events) {
94            return Err(DurableGraphDriverOptionsError::InvalidWorkQuantum);
95        }
96        if lease_renewal_interval.is_zero()
97            || lease_renewal_interval.subsec_nanos() % 1_000 != 0
98            || lease_renewal_interval.as_micros() > i64::MAX as u128
99        {
100            return Err(DurableGraphDriverOptionsError::InvalidLeaseRenewalInterval);
101        }
102        if node_execution_timeout.is_zero() || node_execution_timeout > MAX_NODE_TIMEOUT {
103            return Err(DurableGraphDriverOptionsError::InvalidNodeExecutionTimeout);
104        }
105        if !(1..=Self::HARD_MAXIMUM_MUTATION_ATTEMPTS).contains(&maximum_mutation_attempts) {
106            return Err(DurableGraphDriverOptionsError::InvalidMutationAttempts);
107        }
108        if mutation_retry_initial_delay.is_zero()
109            || mutation_retry_initial_delay > MAX_MUTATION_RETRY_DELAY
110        {
111            return Err(DurableGraphDriverOptionsError::InvalidMutationRetryDelay);
112        }
113        Ok(Self {
114            replay_limits,
115            maximum_durable_events,
116            lease_renewal_interval,
117            node_execution_timeout,
118            cancellation_poll_interval: Duration::from_millis(250),
119            cancellation_grace_period: Duration::from_secs(5),
120            maximum_parallel_nodes: 1,
121            maximum_mutation_attempts,
122            mutation_retry_initial_delay,
123        })
124    }
125
126    /// Overrides the database cancellation-observation cadence and cooperative
127    /// node cleanup window.
128    ///
129    /// # Errors
130    ///
131    /// Rejects non-microsecond-aligned or production-unbounded values. The poll
132    /// interval is also bounded below so one bad configuration cannot turn each
133    /// active node into an unbounded database query loop.
134    pub fn with_cancellation_timing(
135        mut self,
136        poll_interval: Duration,
137        grace_period: Duration,
138    ) -> Result<Self, DurableGraphDriverOptionsError> {
139        if poll_interval < Self::HARD_MINIMUM_CANCELLATION_POLL_INTERVAL
140            || poll_interval.subsec_nanos() % 1_000 != 0
141            || poll_interval > Self::HARD_MAXIMUM_CANCELLATION_POLL_INTERVAL
142        {
143            return Err(DurableGraphDriverOptionsError::InvalidCancellationPollInterval);
144        }
145        if grace_period.is_zero()
146            || grace_period.subsec_nanos() % 1_000 != 0
147            || grace_period > Self::HARD_MAXIMUM_CANCELLATION_GRACE_PERIOD
148        {
149            return Err(DurableGraphDriverOptionsError::InvalidCancellationGracePeriod);
150        }
151        self.cancellation_poll_interval = poll_interval;
152        self.cancellation_grace_period = grace_period;
153        Ok(self)
154    }
155
156    /// Enables bounded overlap for executors that explicitly declare the
157    /// journal-isolated scheduling contract.
158    ///
159    /// The effective batch width is the minimum of this value, the compiled
160    /// graph limit, the ready set, and the remaining durable-event quantum.
161    /// Exclusive executors always run alone.
162    ///
163    /// # Errors
164    ///
165    /// Rejects zero or process-unbounded concurrency.
166    pub fn with_parallel_node_limit(
167        mut self,
168        maximum_parallel_nodes: u16,
169    ) -> Result<Self, DurableGraphDriverOptionsError> {
170        if maximum_parallel_nodes == 0 || maximum_parallel_nodes > Self::HARD_MAXIMUM_PARALLEL_NODES
171        {
172            return Err(DurableGraphDriverOptionsError::InvalidParallelNodeLimit);
173        }
174        self.maximum_parallel_nodes = maximum_parallel_nodes;
175        Ok(self)
176    }
177
178    /// Returns the memory ceiling shared by historical replay and current
179    /// barrier materialization.
180    #[must_use]
181    pub const fn replay_limits(self) -> GraphReplayLimits {
182        self.replay_limits
183    }
184
185    /// Returns the maximum driver-owned durable events in one drive call.
186    #[must_use]
187    pub const fn maximum_durable_events(self) -> u32 {
188        self.maximum_durable_events
189    }
190
191    /// Returns the monotonic interval between renewal attempts.
192    #[must_use]
193    pub const fn lease_renewal_interval(self) -> Duration {
194        self.lease_renewal_interval
195    }
196
197    /// Returns the hard wall-clock deadline for one node task.
198    #[must_use]
199    pub const fn node_execution_timeout(self) -> Duration {
200        self.node_execution_timeout
201    }
202
203    /// Returns the bounded database polling cadence for cancellation intent.
204    #[must_use]
205    pub const fn cancellation_poll_interval(self) -> Duration {
206        self.cancellation_poll_interval
207    }
208
209    /// Returns the cooperative cleanup window after cancellation is observed.
210    #[must_use]
211    pub const fn cancellation_grace_period(self) -> Duration {
212        self.cancellation_grace_period
213    }
214
215    /// Returns the process-local ready-node concurrency ceiling.
216    #[must_use]
217    pub const fn maximum_parallel_nodes(self) -> u16 {
218        self.maximum_parallel_nodes
219    }
220
221    /// Returns the maximum identical attempts for one idempotent mutation.
222    #[must_use]
223    pub const fn maximum_mutation_attempts(self) -> u8 {
224        self.maximum_mutation_attempts
225    }
226
227    /// Returns the first exponential mutation retry delay.
228    #[must_use]
229    pub const fn mutation_retry_initial_delay(self) -> Duration {
230        self.mutation_retry_initial_delay
231    }
232}
233
234impl Default for DurableGraphDriverOptions {
235    fn default() -> Self {
236        Self {
237            replay_limits: GraphReplayLimits::default(),
238            maximum_durable_events: 1_024,
239            lease_renewal_interval: Duration::from_secs(10),
240            node_execution_timeout: Duration::from_secs(15 * 60),
241            cancellation_poll_interval: Duration::from_millis(250),
242            cancellation_grace_period: Duration::from_secs(5),
243            maximum_parallel_nodes: 1,
244            maximum_mutation_attempts: 3,
245            mutation_retry_initial_delay: Duration::from_millis(25),
246        }
247    }
248}
249
250/// Invalid durable driver options.
251#[derive(Clone, Copy, Debug, Eq, Error, PartialEq)]
252#[non_exhaustive]
253pub enum DurableGraphDriverOptionsError {
254    /// A call could not durably start and finish one node, or was unbounded.
255    #[error("graph driver durable-event quantum is outside its production bounds")]
256    InvalidWorkQuantum,
257    /// Renewal timing was zero, not microsecond aligned, or unrepresentable.
258    #[error("graph driver lease renewal interval is invalid")]
259    InvalidLeaseRenewalInterval,
260    /// Node execution had no deadline or exceeded seven days.
261    #[error("graph driver node execution timeout is invalid")]
262    InvalidNodeExecutionTimeout,
263    /// Cancellation polling was imprecise, faster than 10 ms, or slower than one minute.
264    #[error("graph driver cancellation poll interval is invalid")]
265    InvalidCancellationPollInterval,
266    /// Cooperative cancellation cleanup was zero, imprecise, or above five minutes.
267    #[error("graph driver cancellation grace period is invalid")]
268    InvalidCancellationGracePeriod,
269    /// Ready-node concurrency was zero or above the process safety ceiling.
270    #[error("graph driver parallel node limit is invalid")]
271    InvalidParallelNodeLimit,
272    /// Mutation attempts were zero or above the hard ceiling.
273    #[error("graph driver mutation attempt count is invalid")]
274    InvalidMutationAttempts,
275    /// Initial mutation backoff was zero or above one second.
276    #[error("graph driver mutation retry delay is invalid")]
277    InvalidMutationRetryDelay,
278}
279
280/// Fenced durable graph execution service over one `PostgreSQL` provider pool.
281#[derive(Clone)]
282pub struct DurableGraphDriver {
283    activity: crate::GraphExecutionActivity,
284    store: PostgresStore,
285    registry: ExecutableGraphRegistry,
286    journal_schema: stateknot_core::SchemaReference,
287    options: DurableGraphDriverOptions,
288}
289
290impl DurableGraphDriver {
291    /// Binds one provider pool and frozen executable registry.
292    ///
293    /// The standard event schema must have been installed before the schema
294    /// registry was frozen. Renewal cadence must fit at least three times in a
295    /// newly acquired lease, preserving time for two delayed renewal cycles.
296    ///
297    /// # Errors
298    ///
299    /// Returns [`DurableGraphDriverBuildError`] for an invalid release schema,
300    /// absent journal schema, or unsafe lease cadence.
301    pub fn new(
302        store: PostgresStore,
303        registry: ExecutableGraphRegistry,
304        options: DurableGraphDriverOptions,
305    ) -> Result<Self, DurableGraphDriverBuildError> {
306        let (journal_schema, _) = standard_graph_driver_event_schema()?;
307        if !registry.schemas().contains(&journal_schema) {
308            return Err(DurableGraphDriverBuildError::JournalSchemaUnavailable);
309        }
310        let lease_duration = store.options().lease_duration();
311        let renewal_budget = options
312            .lease_renewal_interval
313            .checked_mul(3)
314            .ok_or(DurableGraphDriverBuildError::UnsafeLeaseRenewalCadence)?;
315        if renewal_budget > lease_duration
316            || options.lease_renewal_interval > store.options().maximum_lease_horizon()
317        {
318            return Err(DurableGraphDriverBuildError::UnsafeLeaseRenewalCadence);
319        }
320        Ok(Self {
321            activity: crate::GraphExecutionActivity::default(),
322            store,
323            registry,
324            journal_schema,
325            options,
326        })
327    }
328
329    /// Returns shared node-future completion accounting for this driver and clones.
330    pub fn execution_activity(&self) -> crate::GraphExecutionActivity {
331        self.activity.clone()
332    }
333
334    /// Returns the immutable execution policy.
335    #[must_use]
336    pub const fn options(&self) -> DurableGraphDriverOptions {
337        self.options
338    }
339
340    /// Replays and drives one already-claimed exact worker fence.
341    ///
342    /// Every node start commits before node code is spawned. An idempotently
343    /// observed start is never launched. The node task holds no database
344    /// transaction, receives a monotonic cancellation signal, and is aborted
345    /// if renewal, shutdown, or its deadline wins. Successful and failed node
346    /// completions use the latest exact journal head so invocation-ledger facts
347    /// committed by the node remain ordered before its result.
348    ///
349    /// Continue barriers commit automatically. Wait and successful-terminal
350    /// barriers, plus node failure/exhaustion, return a lease-bound handoff:
351    /// their complete registration, admitted result, or cumulative failure
352    /// metadata belongs to the lifecycle integration layer and is never guessed
353    /// by this driver.
354    ///
355    /// # Errors
356    ///
357    /// Returns explicit storage, fencing, executable-deployment, schema,
358    /// planning, resource, or runtime invariant failures. Durable corruption is
359    /// quarantined by the claimed recovery session before it reaches this API.
360    pub fn drive(
361        &self,
362        fence: RunFence,
363        shutdown: CancellationSignal,
364    ) -> BoxFuture<'_, Result<GraphDriveResult, GraphDriverError>> {
365        Box::pin(self.drive_inner(fence, shutdown))
366    }
367
368    #[allow(clippy::too_many_lines)]
369    async fn drive_inner(
370        &self,
371        fence: RunFence,
372        shutdown: CancellationSignal,
373    ) -> Result<GraphDriveResult, GraphDriverError> {
374        // Recovery, graph loading, and replay are bounded but may still consume
375        // a meaningful fraction of a short lease. Refresh below half-life
376        // before those reads so a valid near-expiry claim is not lost merely
377        // because integrity verification ran before the first durable start.
378        let (startup_renewals, startup_retries) = self.refresh_near_expiry_lease(&fence).await?;
379        let initial_recovery = self.begin_recovery(&fence).await?;
380        let stored_graph = initial_recovery.load_pinned_graph().await?;
381        let graph_reference = stored_graph.graph().reference();
382        let executable = self
383            .registry
384            .resolve(&graph_reference)
385            .cloned()
386            .ok_or_else(|| GraphDriverError::ExecutableGraphUnavailable {
387                graph: Box::new(graph_reference),
388            })?;
389        let replay = initial_recovery
390            .validate_noninitial_replay(
391                executable.schemas(),
392                executable.reducer(),
393                self.options.replay_limits,
394            )
395            .await?;
396        drop(initial_recovery);
397
398        let mut report = GraphDriveReport::new(replay);
399        report.lease_renewals = startup_renewals;
400        report.mutation_retries = startup_retries;
401        loop {
402            if shutdown.is_cancelled() {
403                let release = self.release_with_retry(&fence, &mut report).await?;
404                return Ok(GraphDriveResult::new(
405                    GraphDriveOutcome::Cancelled { release },
406                    report,
407                ));
408            }
409            if report.durable_events >= self.options.maximum_durable_events {
410                let release = self.release_with_retry(&fence, &mut report).await?;
411                return Ok(GraphDriveResult::new(
412                    GraphDriveOutcome::Yielded { release },
413                    report,
414                ));
415            }
416
417            let recovery = self.begin_recovery(&fence).await?;
418            let plan = recovery.plan_ready_nodes().await?;
419            let live = recovery.revalidate().await?;
420            if live.lifecycle().status() == RunStatus::CancellationRequested {
421                let lease = exact_live_lease(&live, &fence)?.clone();
422                let request = live.lifecycle().cancellation_request().ok_or(
423                    GraphDriverError::RuntimeInvariant {
424                        operation: "recover durable cancellation intent",
425                    },
426                )?;
427                let checkpoint = plan.checkpoint().head();
428                let pointer = live
429                    .checkpoint()
430                    .ok_or(GraphDriverError::RuntimeInvariant {
431                        operation: "bind cancellation to the current checkpoint",
432                    })?;
433                if pointer.checkpoint_id() != checkpoint.checkpoint_id()
434                    || pointer.superstep() != checkpoint.superstep()
435                    || pointer.digest() != checkpoint.digest()
436                    || live.journal_head() != Some(plan.journal_head())
437                {
438                    return Err(GraphDriverError::StaleBarrierSnapshot);
439                }
440                return Ok(GraphDriveResult::new(
441                    GraphDriveOutcome::CancellationRequested(Box::new(GraphCancellationHandoff {
442                        checkpoint,
443                        journal_head: plan.journal_head().clone(),
444                        event_id: EventId::generate(),
445                        expected_revision: live.lifecycle().revision(),
446                        lease,
447                        failure_id: request.failure().id(),
448                    })),
449                    report,
450                ));
451            }
452            let mut blockers = GraphDriveBlockers::from_plan(&plan);
453            // Stop before durable attempt starts or external dispatch, including
454            // after recovery at the last permitted barrier. A finite graph
455            // budget is a lifecycle failure, not corruption or a retry loop.
456            blockers.superstep_limit_reached = plan.checkpoint().superstep().get()
457                >= executable.graph().limits().maximum_supersteps().get();
458            if !blockers.is_empty() {
459                let lease = exact_live_lease(&live, &fence)?.clone();
460                return Ok(GraphDriveResult::new(
461                    GraphDriveOutcome::Blocked(Box::new(GraphBlockedHandoff {
462                        plan,
463                        lease,
464                        event_id: EventId::generate(),
465                        expected_revision: live.lifecycle().revision(),
466                        blockers,
467                    })),
468                    report,
469                ));
470            }
471
472            if plan.is_barrier_ready() {
473                if report.durable_events == self.options.maximum_durable_events {
474                    drop(recovery);
475                    let release = self.release_with_retry(&fence, &mut report).await?;
476                    return Ok(GraphDriveResult::new(
477                        GraphDriveOutcome::Yielded { release },
478                        report,
479                    ));
480                }
481                let results = self.load_barrier_results(&recovery, &plan).await?;
482                let barrier_plan = executable
483                    .graph()
484                    .plan_barrier(
485                        plan.checkpoint(),
486                        &results,
487                        CheckpointId::generate(),
488                        executable.schemas(),
489                        executable.reducer(),
490                    )
491                    .map_err(GraphDriverError::graph_plan)?;
492                let lease = exact_live_lease(&live, &fence)?.clone();
493                match barrier_plan.disposition() {
494                    GraphBarrierDisposition::Continue => {
495                        drop(recovery);
496                        self.commit_continue_barrier(&fence, &plan, barrier_plan, &mut report)
497                            .await?;
498                    }
499                    GraphBarrierDisposition::Wait { .. }
500                    | GraphBarrierDisposition::Terminal { .. } => {
501                        return Ok(GraphDriveResult::new(
502                            GraphDriveOutcome::LifecycleBarrierReady(Box::new(
503                                GraphLifecycleBarrierHandoff {
504                                    plan: barrier_plan,
505                                    journal_head: plan.journal_head().clone(),
506                                    event_id: EventId::generate(),
507                                    expected_revision: live.lifecycle().revision(),
508                                    lease,
509                                },
510                            )),
511                            report,
512                        ));
513                    }
514                    _ => {
515                        return Err(GraphDriverError::RuntimeInvariant {
516                            operation: "handle unsupported graph barrier disposition",
517                        });
518                    }
519                }
520                continue;
521            }
522
523            if let Some(not_before) = plan.earliest_deferred_at() {
524                if plan.nodes().iter().all(|node| {
525                    matches!(
526                        node.kind(),
527                        RecoveryNodeKind::Completed | RecoveryNodeKind::Deferred
528                    )
529                }) {
530                    let outcome = self
531                        .schedule_deferred_with_retry(&plan, &mut report)
532                        .await?;
533                    if matches!(outcome, DelayedRetryScheduleOutcome::Due { .. }) {
534                        continue;
535                    }
536                    return Ok(GraphDriveResult::new(
537                        GraphDriveOutcome::Deferred {
538                            not_before,
539                            schedule: outcome,
540                        },
541                        report,
542                    ));
543                }
544            }
545
546            if report.durable_events.saturating_add(2) > self.options.maximum_durable_events {
547                drop(recovery);
548                let release = self.release_with_retry(&fence, &mut report).await?;
549                return Ok(GraphDriveResult::new(
550                    GraphDriveOutcome::Yielded { release },
551                    report,
552                ));
553            }
554            let first_node = plan
555                .nodes()
556                .iter()
557                .find(|node| node.kind() == RecoveryNodeKind::Dispatchable)
558                .ok_or(GraphDriverError::RuntimeInvariant {
559                    operation: "select dispatchable graph node",
560                })?;
561            let first_node_id = first_node.activation().node_id().clone();
562            let first_executor = Self::resolve_node_executor(&executable, &first_node_id)?;
563            let batch_limit = usize::from(self.options.maximum_parallel_nodes)
564                .min(usize::from(
565                    executable.graph().limits().maximum_parallelism(),
566                ))
567                .min(
568                    usize::try_from(
569                        (self
570                            .options
571                            .maximum_durable_events
572                            .saturating_sub(report.durable_events))
573                            / 2,
574                    )
575                    .unwrap_or(usize::MAX),
576                );
577            let first_scheduling = first_executor.scheduling();
578            let mut selected = Vec::with_capacity(batch_limit);
579            selected.push((first_node_id, first_executor));
580            if first_scheduling == GraphNodeScheduling::JournalIsolated && batch_limit > 1 {
581                for node in plan
582                    .nodes()
583                    .iter()
584                    .filter(|node| node.kind() == RecoveryNodeKind::Dispatchable)
585                    .skip(1)
586                    .take(batch_limit.saturating_sub(1))
587                {
588                    let node_id = node.activation().node_id().clone();
589                    let executor = Self::resolve_node_executor(&executable, &node_id)?;
590                    if executor.scheduling() != GraphNodeScheduling::JournalIsolated {
591                        break;
592                    }
593                    selected.push((node_id, executor));
594                }
595            }
596
597            // Start facts are serialized in canonical NodeId order before any
598            // journal-isolated executor is spawned. This makes both launch
599            // authority and the later result-commit order independent of task
600            // completion timing.
601            let mut journal_head = plan.journal_head().clone();
602            let mut started = Vec::with_capacity(selected.len());
603            for (node_id, executor) in selected {
604                let attempt_id = AttemptId::generate();
605                let start_event_id = EventId::generate();
606                let payload = self.node_started_payload(
607                    &executable.graph().reference(),
608                    plan.checkpoint(),
609                    &node_id,
610                    attempt_id,
611                )?;
612                let append = worker_append(&fence, journal_head.clone(), start_event_id, payload)?;
613                let start = self
614                    .start_node_with_retry(append, &plan, &node_id, attempt_id, &mut report)
615                    .await?;
616                match start {
617                    NodeAttemptCommitOutcome::Committed { event, attempt } => {
618                        report.durable_events = report.durable_events.saturating_add(1);
619                        report.node_attempts_started =
620                            report.node_attempts_started.saturating_add(1);
621                        journal_head = event.head();
622                        started.push(StartedNode {
623                            executor,
624                            start: attempt.start().head(),
625                        });
626                    }
627                    NodeAttemptCommitOutcome::Idempotent { .. } => {
628                        // Lost acknowledgement convergence never grants launch
629                        // authority. Any starts committed earlier in this batch
630                        // still execute and settle normally.
631                        report.durable_events = report.durable_events.saturating_add(1);
632                        break;
633                    }
634                    _ => {
635                        return Err(GraphDriverError::RuntimeInvariant {
636                            operation: "handle unsupported node attempt start outcome",
637                        });
638                    }
639                }
640            }
641            if started.is_empty() {
642                continue;
643            }
644            let checkpoint = Arc::new(plan.checkpoint().clone());
645            drop(recovery);
646
647            // A recovery/dispatch pass may have consumed most of the lease that
648            // was originally acquired. Re-observe the database clock only after
649            // the durable start commits, then restore a full configured lease
650            // before node code can perform external work. If this preflight
651            // fails, the committed start remains available for fenced takeover
652            // and the executor is never launched under an unsafe lease margin.
653            let current_lease = self.prepare_execution_lease(&fence, &mut report).await?;
654
655            report.node_batches_started = report.node_batches_started.saturating_add(1);
656            report.maximum_nodes_in_batch = report
657                .maximum_nodes_in_batch
658                .max(u16::try_from(started.len()).unwrap_or(u16::MAX));
659            let execution = self
660                .execute_started_nodes(
661                    &started,
662                    checkpoint,
663                    current_lease,
664                    shutdown.clone(),
665                    &mut report,
666                )
667                .await?;
668            match execution {
669                StartedNodesExecution::Cancelled => {
670                    let release = self.release_with_retry(&fence, &mut report).await?;
671                    return Ok(GraphDriveResult::new(
672                        GraphDriveOutcome::Cancelled { release },
673                        report,
674                    ));
675                }
676                StartedNodesExecution::RunCancellationObserved => {}
677                StartedNodesExecution::Finished(results) => {
678                    for (started, result) in started.iter().zip(results) {
679                        match self
680                            .commit_node_execution(
681                                &fence,
682                                &executable,
683                                started,
684                                result,
685                                &mut report,
686                            )
687                            .await?
688                        {
689                            NodeExecutionCommit::RunCancellationObserved => break,
690                            NodeExecutionCommit::Committed => {}
691                            NodeExecutionCommit::ChildJoin(request) => {
692                                return Ok(GraphDriveResult::new(
693                                    GraphDriveOutcome::ChildJoin(request),
694                                    report,
695                                ));
696                            }
697                        }
698                    }
699                }
700            }
701        }
702    }
703
704    async fn begin_recovery<'driver>(
705        &'driver self,
706        fence: &RunFence,
707    ) -> Result<ClaimedRunRecovery<'driver>, GraphDriverError> {
708        let run = self
709            .store
710            .load_run(fence.tenant_id(), fence.run_id())
711            .await?;
712        exact_live_lease(&run, fence)?;
713        let head = run
714            .journal_head()
715            .cloned()
716            .ok_or(GraphDriverError::MissingJournalHead)?;
717        let evidence_digest = recovery_evidence_digest(&run, fence)?;
718        let context = CorruptionQuarantineContext::new(
719            fence.tenant_id().clone(),
720            fence.run_id(),
721            QuarantineId::generate(),
722            JournalExpectation::exact(head),
723            evidence_digest,
724        )?;
725        self.store
726            .begin_claimed_run_recovery(fence.clone(), context)
727            .await
728            .map_err(GraphDriverError::from)
729    }
730
731    async fn refresh_near_expiry_lease(
732        &self,
733        fence: &RunFence,
734    ) -> Result<(u32, u32), GraphDriverError> {
735        let observation = self.store.observe_live_lease(fence).await?;
736        let remaining_micros = observation
737            .lease()
738            .expires_at()
739            .unix_micros()
740            .checked_sub(observation.observed_at().unix_micros())
741            .and_then(|value| u64::try_from(value).ok())
742            .ok_or(StoreError::LeaseExpired)?;
743        let remaining = Duration::from_micros(remaining_micros);
744        if remaining > self.store.options().lease_duration() / 2 {
745            return Ok((0, 0));
746        }
747
748        let desired = extend_timestamp(
749            observation.observed_at(),
750            self.store.options().lease_duration(),
751        )?;
752        let mut attempt = 1_u8;
753        let mut retries = 0_u32;
754        loop {
755            match self.store.renew_lease(fence, desired).await {
756                Ok(LeaseRenewalOutcome::Renewed(_) | LeaseRenewalOutcome::Idempotent(_)) => {
757                    return Ok((1, retries));
758                }
759                Ok(_) => {
760                    return Err(GraphDriverError::RuntimeInvariant {
761                        operation: "refresh a near-expiry drive lease",
762                    });
763                }
764                Err(error) if self.can_retry_mutation(&error, attempt) => {
765                    retries = retries.saturating_add(1);
766                    self.mutation_backoff(attempt).await;
767                    attempt = attempt.saturating_add(1);
768                }
769                Err(error) => return Err(error.into()),
770            }
771        }
772    }
773
774    async fn load_barrier_results(
775        &self,
776        recovery: &ClaimedRunRecovery<'_>,
777        plan: &ReadyNodeRecoveryPlan,
778    ) -> Result<Vec<PendingNodeResult>, GraphDriverError> {
779        let page_size = PendingNodeResultPageSize::new(PendingNodeResultPageSize::MAX)?;
780        let mut cursor = None;
781        let mut results = Vec::with_capacity(plan.nodes().len());
782        let mut compact_bytes = 0_usize;
783        loop {
784            let page = recovery
785                .load_unconsumed_pending_node_result_page(
786                    &plan.checkpoint().head(),
787                    cursor.as_ref(),
788                    page_size,
789                )
790                .await?;
791            if page.snapshot_journal_head() != plan.journal_head() {
792                return Err(GraphDriverError::StaleBarrierSnapshot);
793            }
794            let next = if page.has_more() {
795                Some(
796                    page.next_cursor()
797                        .ok_or(GraphDriverError::RuntimeInvariant {
798                            operation: "advance graph barrier result cursor",
799                        })?,
800                )
801            } else {
802                None
803            };
804            for result in page.records() {
805                let mut counter = CompactByteCounter::default();
806                serde_json::to_writer(&mut counter, result)
807                    .map_err(|_| GraphDriverError::BarrierResultEncoding)?;
808                compact_bytes = compact_bytes
809                    .checked_add(counter.bytes)
810                    .ok_or(GraphDriverError::BarrierResultResourceLimit)?;
811                if compact_bytes > self.options.replay_limits.maximum_barrier_result_bytes() {
812                    return Err(GraphDriverError::BarrierResultResourceLimit);
813                }
814            }
815            results.extend(page.into_records());
816            let Some(next) = next else {
817                break;
818            };
819            cursor = Some(next);
820        }
821        if results.len() != plan.nodes().len() {
822            return Err(GraphDriverError::RuntimeInvariant {
823                operation: "materialize complete graph barrier result set",
824            });
825        }
826        Ok(results)
827    }
828
829    async fn commit_continue_barrier(
830        &self,
831        fence: &RunFence,
832        ready_plan: &ReadyNodeRecoveryPlan,
833        barrier_plan: GraphBarrierPlan,
834        report: &mut GraphDriveReport,
835    ) -> Result<(), GraphDriverError> {
836        if !matches!(
837            barrier_plan.disposition(),
838            GraphBarrierDisposition::Continue
839        ) {
840            return Err(GraphDriverError::RuntimeInvariant {
841                operation: "commit non-continue graph barrier",
842            });
843        }
844        let payload = self.barrier_continued_payload(
845            ready_plan.checkpoint().graph(),
846            ready_plan.checkpoint(),
847            barrier_plan.barrier().successor().checkpoint_id(),
848            barrier_plan.barrier().successor().superstep(),
849        )?;
850        let append = worker_append(
851            fence,
852            ready_plan.journal_head().clone(),
853            EventId::generate(),
854            payload,
855        )?;
856        let barrier = barrier_plan.barrier().clone();
857        let mut attempt = 1_u8;
858        loop {
859            let result = self
860                .store
861                .append_worker_barrier(append.clone(), RunProjection::unchanged(), barrier.clone())
862                .await;
863            match result {
864                Ok(
865                    BarrierCommitOutcome::Committed { .. }
866                    | BarrierCommitOutcome::Idempotent { .. },
867                ) => {
868                    report.durable_events = report.durable_events.saturating_add(1);
869                    report.barriers_committed = report.barriers_committed.saturating_add(1);
870                    return Ok(());
871                }
872                Ok(_) => {
873                    return Err(GraphDriverError::RuntimeInvariant {
874                        operation: "handle unsupported graph barrier commit outcome",
875                    });
876                }
877                Err(error) if self.can_retry_mutation(&error, attempt) => {
878                    report.mutation_retries = report.mutation_retries.saturating_add(1);
879                    self.mutation_backoff(attempt).await;
880                    attempt = attempt.saturating_add(1);
881                }
882                Err(error) => return Err(error.into()),
883            }
884        }
885    }
886
887    fn resolve_node_executor(
888        executable: &ExecutableGraph,
889        node_id: &stateknot_core::NodeId,
890    ) -> Result<Arc<dyn crate::GraphNodeExecutor>, GraphDriverError> {
891        executable.node_executor(node_id).ok_or_else(|| {
892            GraphDriverError::ExecutableNodeUnavailable {
893                graph: Box::new(executable.graph().reference()),
894                node_id: node_id.clone(),
895            }
896        })
897    }
898
899    #[allow(clippy::too_many_lines)]
900    async fn execute_started_nodes(
901        &self,
902        started: &[StartedNode],
903        checkpoint: Arc<Checkpoint>,
904        lease: GuardedRunLease,
905        shutdown: CancellationSignal,
906        report: &mut GraphDriveReport,
907    ) -> Result<StartedNodesExecution, GraphDriverError> {
908        if shutdown.is_cancelled() {
909            return Ok(StartedNodesExecution::Cancelled);
910        }
911        if started.is_empty() {
912            return Err(GraphDriverError::RuntimeInvariant {
913                operation: "execute an empty graph node batch",
914            });
915        }
916
917        let mut tasks = JoinSet::new();
918        let mut cancellations = Vec::with_capacity(started.len());
919        for (index, started_node) in started.iter().enumerate() {
920            let cancellation = DriverCancellation::new();
921            let context = GraphNodeContext::new(
922                started_node.start.clone(),
923                Arc::clone(&checkpoint),
924                cancellation.signal(),
925            )?;
926            let executor = Arc::clone(&started_node.executor);
927            let store = self.store.clone();
928            let registry = self.registry.clone();
929            let has_children = registry
930                .resolve(executor.graph())
931                .is_some_and(|graph| graph.graph().child_runs().is_some());
932            let task_cancellation = cancellation.clone();
933            let timeout = self.options.node_execution_timeout;
934            cancellations.push(cancellation);
935            tasks.spawn(self.activity.track(async move {
936                let executing = AtomicBool::new(false);
937                let executing_marker = &executing;
938                // Wrap both the synchronous `execute` call and the returned
939                // future. Object-safe executors are third-party code and may
940                // panic before they construct their future.
941                let execution = AssertUnwindSafe(async move {
942                    let mut context = context;
943                    if has_children {
944                        if let Some(record) = store.load_child_join(context.attempt().activation()).await? {
945                            if !executor.supports_child_join() {
946                                return Err(StoreError::ChildJoinRejected.into());
947                            }
948                            context = context.with_child_join(crate::GraphChildJoin::prepare(store, registry, &record).await?);
949                        }
950                    }
951                    let child_join = context.child_join().map(|join| Box::new(join.head().clone()));
952                    executing_marker.store(true, Ordering::Release);
953                    let result = executor.execute(context).await;
954                    Ok(FinishedNodeExecution { result, child_join })
955                }).catch_unwind();
956                tokio::pin!(execution);
957                let deadline = tokio::time::sleep(timeout);
958                tokio::pin!(deadline);
959                let outcome = tokio::select! {
960                    result = &mut execution => match result {
961                        Ok(result) => result,
962                        Err(_) if !executing.load(Ordering::Acquire) => Err(GraphDriverError::ChildJoinPreparationIncomplete),
963                        Err(_) => DurableGraphDriver::node_panic_failure().map(FinishedNodeExecution::without_join),
964                    },
965                    () = &mut deadline => {
966                        task_cancellation.cancel();
967                        if executing.load(Ordering::Acquire) {
968                            DurableGraphDriver::node_timeout_failure().map(|error| FinishedNodeExecution::without_join(Err(error)))
969                        } else {
970                            Err(GraphDriverError::ChildJoinPreparationIncomplete)
971                        }
972                    }
973                };
974                (index, outcome)
975            }));
976        }
977
978        let fence = lease.lease.fence().clone();
979        let lease_deadline = lease.deadline;
980        let maintenance = self.maintain_execution_lease(lease, report);
981        tokio::pin!(maintenance);
982        let durable_cancellation = self.observe_run_cancellation(&fence);
983        tokio::pin!(durable_cancellation);
984        let mut results = std::iter::repeat_with(|| None)
985            .take(started.len())
986            .collect::<Vec<_>>();
987        let mut completed = 0_usize;
988
989        loop {
990            tokio::select! {
991                joined = tasks.join_next(), if completed < started.len() => {
992                    let (index, result) = joined
993                        .ok_or(GraphDriverError::RuntimeInvariant {
994                            operation: "join a complete graph node batch",
995                        })?
996                        .map_err(|_| GraphDriverError::RuntimeInvariant {
997                            operation: "join an unexpectedly cancelled graph node task",
998                        })?;
999                    let result = result?;
1000                    let slot = results.get_mut(index).ok_or(GraphDriverError::RuntimeInvariant {
1001                        operation: "place a graph node batch result",
1002                    })?;
1003                    if slot.replace(result).is_some() {
1004                        return Err(GraphDriverError::RuntimeInvariant {
1005                            operation: "deduplicate a graph node batch result",
1006                        });
1007                    }
1008                    completed = completed.saturating_add(1);
1009                    if completed == started.len() {
1010                        let ordered = results
1011                            .into_iter()
1012                            .collect::<Option<Vec<_>>>()
1013                            .ok_or(GraphDriverError::RuntimeInvariant {
1014                                operation: "materialize ordered graph node batch results",
1015                            })?;
1016                        return Ok(StartedNodesExecution::Finished(ordered));
1017                    }
1018                }
1019                result = &mut maintenance => {
1020                    cancel_node_batch(&cancellations);
1021                    abort_node_batch(&mut tasks).await;
1022                    return match result {
1023                        Ok(()) => Err(GraphDriverError::RuntimeInvariant {
1024                            operation: "maintain a live node execution lease",
1025                        }),
1026                        Err(error) => Err(error),
1027                    };
1028                }
1029                () = shutdown.cancelled() => {
1030                    cancel_node_batch(&cancellations);
1031                    abort_node_batch(&mut tasks).await;
1032                    return Ok(StartedNodesExecution::Cancelled);
1033                }
1034                result = &mut durable_cancellation => {
1035                    cancel_node_batch(&cancellations);
1036                    if let Err(error) = result {
1037                        abort_node_batch(&mut tasks).await;
1038                        return Err(error);
1039                    }
1040                    let grace_deadline = Instant::now()
1041                        .checked_add(self.options.cancellation_grace_period)
1042                        .unwrap_or(lease_deadline)
1043                        .min(lease_deadline);
1044                    drain_node_batch_until(&mut tasks, grace_deadline).await;
1045                    return Ok(StartedNodesExecution::RunCancellationObserved);
1046                }
1047            }
1048        }
1049    }
1050
1051    async fn observe_run_cancellation(&self, fence: &RunFence) -> Result<(), GraphDriverError> {
1052        loop {
1053            let run = self
1054                .store
1055                .load_run(fence.tenant_id(), fence.run_id())
1056                .await?;
1057            exact_live_lease(&run, fence)?;
1058            match run.lifecycle().status() {
1059                RunStatus::CancellationRequested => return Ok(()),
1060                RunStatus::Pending | RunStatus::Active => {}
1061                _ => return Err(StoreError::RunNotRunnable.into()),
1062            }
1063            tokio::time::sleep(self.options.cancellation_poll_interval).await;
1064        }
1065    }
1066
1067    async fn prepare_execution_lease(
1068        &self,
1069        fence: &RunFence,
1070        report: &mut GraphDriveReport,
1071    ) -> Result<GuardedRunLease, GraphDriverError> {
1072        let observation_started = Instant::now();
1073        let recovery = self.begin_recovery(fence).await?;
1074        let lease = exact_live_lease(recovery.initial_run(), fence)?.clone();
1075        let observed_at = recovery.initial_observed_at();
1076        let desired = extend_timestamp(observed_at, self.store.options().lease_duration())?;
1077        drop(recovery);
1078
1079        if desired <= lease.expires_at() {
1080            return guarded_run_lease(lease, observed_at, observation_started);
1081        }
1082        let lease = self.renew_with_retry(fence, desired, report).await?;
1083        report.lease_renewals = report.lease_renewals.saturating_add(1);
1084        Ok(lease)
1085    }
1086
1087    async fn maintain_execution_lease(
1088        &self,
1089        mut lease: GuardedRunLease,
1090        report: &mut GraphDriveReport,
1091    ) -> Result<(), GraphDriverError> {
1092        loop {
1093            let renewal_due = Instant::now()
1094                .checked_add(self.options.lease_renewal_interval)
1095                .ok_or(GraphDriverError::LeaseTimestampOverflow)?;
1096            tokio::select! {
1097                () = tokio::time::sleep_until(renewal_due) => {}
1098                () = tokio::time::sleep_until(lease.deadline) => {
1099                    return Err(StoreError::LeaseExpired.into());
1100                }
1101            }
1102
1103            let desired = extend_timestamp(
1104                lease.lease.expires_at(),
1105                self.options.lease_renewal_interval,
1106            )?;
1107            let fence = lease.lease.fence().clone();
1108            let deadline = lease.deadline;
1109            let renewed = {
1110                let renewal = self.renew_with_retry(&fence, desired, report);
1111                tokio::pin!(renewal);
1112                tokio::select! {
1113                    result = &mut renewal => result?,
1114                    () = tokio::time::sleep_until(deadline) => {
1115                        return Err(StoreError::LeaseExpired.into());
1116                    }
1117                }
1118            };
1119            lease = renewed;
1120            report.lease_renewals = report.lease_renewals.saturating_add(1);
1121        }
1122    }
1123
1124    async fn commit_node_execution(
1125        &self,
1126        fence: &RunFence,
1127        executable: &ExecutableGraph,
1128        started: &StartedNode,
1129        execution: FinishedNodeExecution,
1130        report: &mut GraphDriveReport,
1131    ) -> Result<NodeExecutionCommit, GraphDriverError> {
1132        let start = &started.start;
1133        let run = self
1134            .store
1135            .load_run(fence.tenant_id(), fence.run_id())
1136            .await?;
1137        exact_live_lease(&run, fence)?;
1138        if run.lifecycle().status() == RunStatus::CancellationRequested {
1139            return Ok(NodeExecutionCommit::RunCancellationObserved);
1140        }
1141        let head = run
1142            .journal_head()
1143            .cloned()
1144            .ok_or(GraphDriverError::MissingJournalHead)?;
1145        match execution.result {
1146            Ok(GraphNodeExecution::ChildJoin(request)) => {
1147                if !started.executor.supports_child_join()
1148                    || started.executor.scheduling() != GraphNodeScheduling::Exclusive
1149                    || execution.child_join.is_some()
1150                    || request.activation() != start.activation()
1151                {
1152                    return Err(StoreError::ChildJoinRejected.into());
1153                }
1154                Box::pin(self.register_child_join_with_retry(fence, start, &request, report))
1155                    .await?;
1156                report.durable_events = report.durable_events.saturating_add(1);
1157                // Registration already released ownership atomically. Never run
1158                // another node, complete this attempt, or release a newer fence.
1159                return Ok(NodeExecutionCommit::ChildJoin(request));
1160            }
1161            Ok(GraphNodeExecution::Completed {
1162                state_change,
1163                control,
1164                bindings,
1165                usage,
1166            }) => {
1167                let mut intent = PendingNodeResultIntent::new(
1168                    start.activation().clone(),
1169                    state_change,
1170                    control,
1171                    bindings,
1172                )
1173                .map_err(|_| GraphDriverError::InvalidNodeResult)?;
1174                if let Some(head) = execution.child_join {
1175                    intent = intent
1176                        .with_child_join(*head)
1177                        .map_err(|_| GraphDriverError::InvalidNodeResult)?;
1178                }
1179                let payload = self.node_succeeded_payload(
1180                    &executable.graph().reference(),
1181                    start,
1182                    intent.intent_digest(),
1183                )?;
1184                let append = worker_append(fence, head, EventId::generate(), payload)?;
1185                if let Err(error) = self
1186                    .succeed_node_with_retry(append, start, intent, usage, report)
1187                    .await
1188                {
1189                    if graph_error_is_run_not_runnable(&error)
1190                        && self.run_cancellation_requested(fence).await?
1191                    {
1192                        return Ok(NodeExecutionCommit::RunCancellationObserved);
1193                    }
1194                    return Err(error);
1195                }
1196            }
1197            Err(execution) => {
1198                let (failure, usage) = execution.into_parts();
1199                let event_id = EventId::generate();
1200                let payload =
1201                    self.node_failed_payload(&executable.graph().reference(), start, failure.id())?;
1202                let append = worker_append(fence, head, event_id, payload)?;
1203                let failure = failure.with_caused_by_event(event_id);
1204                if let Err(error) = self
1205                    .fail_node_with_retry(append, start, failure, usage, report)
1206                    .await
1207                {
1208                    if graph_error_is_run_not_runnable(&error)
1209                        && self.run_cancellation_requested(fence).await?
1210                    {
1211                        return Ok(NodeExecutionCommit::RunCancellationObserved);
1212                    }
1213                    return Err(error);
1214                }
1215            }
1216        }
1217        report.durable_events = report.durable_events.saturating_add(1);
1218        report.node_attempts_completed = report.node_attempts_completed.saturating_add(1);
1219        Ok(NodeExecutionCommit::Committed)
1220    }
1221
1222    async fn register_child_join_with_retry(
1223        &self,
1224        fence: &RunFence,
1225        start: &NodeAttemptStartHead,
1226        request: &stateknot_core::ChildRunJoinRequest,
1227        report: &mut GraphDriveReport,
1228    ) -> Result<(), GraphDriverError> {
1229        let id = EventId::generate();
1230        let payload = crate::child_join::payload(self.registry.schemas(), request, false)?;
1231        let mut attempt = 1_u8;
1232        loop {
1233            // Child settlement may append to the parent while its exclusive
1234            // executor finishes spawning. Refresh CAS, retaining logical identity.
1235            let result = async {
1236                let run = self
1237                    .store
1238                    .load_run(fence.tenant_id(), fence.run_id())
1239                    .await?;
1240                let head = run
1241                    .journal_head()
1242                    .cloned()
1243                    .ok_or(GraphDriverError::MissingJournalHead)?;
1244                let append = worker_append(fence, head, id, payload.clone())?;
1245                self.store
1246                    .register_child_join(request.clone(), start, append)
1247                    .await
1248                    .map_err(GraphDriverError::from)
1249            }
1250            .await;
1251            match result {
1252                Ok(_) => return Ok(()),
1253                Err(GraphDriverError::Store { source })
1254                    if attempt < self.options.maximum_mutation_attempts
1255                        && (source.is_retryable()
1256                            || matches!(*source, StoreError::StaleJournalHead)) =>
1257                {
1258                    report.mutation_retries = report.mutation_retries.saturating_add(1);
1259                    self.mutation_backoff(attempt).await;
1260                    attempt = attempt.saturating_add(1);
1261                }
1262                Err(error) => return Err(error),
1263            }
1264        }
1265    }
1266
1267    async fn run_cancellation_requested(&self, fence: &RunFence) -> Result<bool, GraphDriverError> {
1268        let run = self
1269            .store
1270            .load_run(fence.tenant_id(), fence.run_id())
1271            .await?;
1272        exact_live_lease(&run, fence)?;
1273        Ok(run.lifecycle().status() == RunStatus::CancellationRequested)
1274    }
1275
1276    async fn start_node_with_retry(
1277        &self,
1278        append: JournalAppend,
1279        plan: &ReadyNodeRecoveryPlan,
1280        node_id: &stateknot_core::NodeId,
1281        attempt_id: AttemptId,
1282        report: &mut GraphDriveReport,
1283    ) -> Result<NodeAttemptCommitOutcome, GraphDriverError> {
1284        let mut attempt = 1_u8;
1285        loop {
1286            match self
1287                .store
1288                .start_recovered_node_attempt(append.clone(), plan, node_id, attempt_id)
1289                .await
1290            {
1291                Ok(outcome) => return Ok(outcome),
1292                Err(error) if self.can_retry_mutation(&error, attempt) => {
1293                    report.mutation_retries = report.mutation_retries.saturating_add(1);
1294                    self.mutation_backoff(attempt).await;
1295                    attempt = attempt.saturating_add(1);
1296                }
1297                Err(error) => return Err(error.into()),
1298            }
1299        }
1300    }
1301
1302    async fn succeed_node_with_retry(
1303        &self,
1304        append: JournalAppend,
1305        start: &NodeAttemptStartHead,
1306        intent: PendingNodeResultIntent,
1307        usage: BudgetUsage,
1308        report: &mut GraphDriveReport,
1309    ) -> Result<NodeAttemptCommitOutcome, GraphDriverError> {
1310        let mut attempt = 1_u8;
1311        loop {
1312            match self
1313                .store
1314                .succeed_node_attempt(append.clone(), start, intent.clone(), usage.clone())
1315                .await
1316            {
1317                Ok(outcome) => return Ok(outcome),
1318                Err(error) if self.can_retry_mutation(&error, attempt) => {
1319                    report.mutation_retries = report.mutation_retries.saturating_add(1);
1320                    self.mutation_backoff(attempt).await;
1321                    attempt = attempt.saturating_add(1);
1322                }
1323                Err(error) => return Err(error.into()),
1324            }
1325        }
1326    }
1327
1328    async fn fail_node_with_retry(
1329        &self,
1330        append: JournalAppend,
1331        start: &NodeAttemptStartHead,
1332        failure: Failure,
1333        usage: BudgetUsage,
1334        report: &mut GraphDriveReport,
1335    ) -> Result<NodeAttemptCommitOutcome, GraphDriverError> {
1336        let mut attempt = 1_u8;
1337        loop {
1338            match self
1339                .store
1340                .fail_node_attempt(append.clone(), start, failure.clone(), usage.clone())
1341                .await
1342            {
1343                Ok(outcome) => return Ok(outcome),
1344                Err(error) if self.can_retry_mutation(&error, attempt) => {
1345                    report.mutation_retries = report.mutation_retries.saturating_add(1);
1346                    self.mutation_backoff(attempt).await;
1347                    attempt = attempt.saturating_add(1);
1348                }
1349                Err(error) => return Err(error.into()),
1350            }
1351        }
1352    }
1353
1354    async fn renew_with_retry(
1355        &self,
1356        fence: &RunFence,
1357        desired: Timestamp,
1358        report: &mut GraphDriveReport,
1359    ) -> Result<GuardedRunLease, GraphDriverError> {
1360        let renewal_started = Instant::now();
1361        let mut attempt = 1_u8;
1362        loop {
1363            match self.store.renew_lease(fence, desired).await {
1364                Ok(LeaseRenewalOutcome::Renewed(lease)) => {
1365                    let renewed_at = lease.renewed_at();
1366                    return guarded_run_lease(lease, renewed_at, renewal_started);
1367                }
1368                Ok(LeaseRenewalOutcome::Idempotent(_)) => {
1369                    let observation_started = Instant::now();
1370                    let observation = self.store.observe_live_lease(fence).await?;
1371                    let (lease, observed_at) = observation.into_parts();
1372                    return guarded_run_lease(lease, observed_at, observation_started);
1373                }
1374                Ok(_) => {
1375                    return Err(GraphDriverError::RuntimeInvariant {
1376                        operation: "handle a lease renewal outcome",
1377                    });
1378                }
1379                Err(error) if self.can_retry_mutation(&error, attempt) => {
1380                    report.mutation_retries = report.mutation_retries.saturating_add(1);
1381                    self.mutation_backoff(attempt).await;
1382                    attempt = attempt.saturating_add(1);
1383                }
1384                Err(error) => return Err(error.into()),
1385            }
1386        }
1387    }
1388
1389    async fn schedule_deferred_with_retry(
1390        &self,
1391        plan: &ReadyNodeRecoveryPlan,
1392        report: &mut GraphDriveReport,
1393    ) -> Result<DelayedRetryScheduleOutcome, GraphDriverError> {
1394        let mut attempt = 1_u8;
1395        loop {
1396            match self.store.schedule_delayed_retry_wakeup(plan).await {
1397                Ok(outcome) => return Ok(outcome),
1398                Err(error) if self.can_retry_mutation(&error, attempt) => {
1399                    report.mutation_retries = report.mutation_retries.saturating_add(1);
1400                    self.mutation_backoff(attempt).await;
1401                    attempt = attempt.saturating_add(1);
1402                }
1403                Err(error) => return Err(error.into()),
1404            }
1405        }
1406    }
1407
1408    async fn release_with_retry(
1409        &self,
1410        fence: &RunFence,
1411        report: &mut GraphDriveReport,
1412    ) -> Result<LeaseReleaseOutcome, GraphDriverError> {
1413        let mut attempt = 1_u8;
1414        loop {
1415            match self.store.release_lease(fence).await {
1416                Ok(outcome) => return Ok(outcome),
1417                Err(error) if self.can_retry_mutation(&error, attempt) => {
1418                    report.mutation_retries = report.mutation_retries.saturating_add(1);
1419                    self.mutation_backoff(attempt).await;
1420                    attempt = attempt.saturating_add(1);
1421                }
1422                Err(error) => return Err(error.into()),
1423            }
1424        }
1425    }
1426
1427    fn can_retry_mutation(&self, error: &StoreError, attempt: u8) -> bool {
1428        attempt < self.options.maximum_mutation_attempts && error.is_retryable()
1429    }
1430
1431    async fn mutation_backoff(&self, attempt: u8) {
1432        let multiplier = 1_u32
1433            .checked_shl(u32::from(attempt.saturating_sub(1)))
1434            .unwrap_or(u32::MAX);
1435        let delay = self
1436            .options
1437            .mutation_retry_initial_delay
1438            .checked_mul(multiplier)
1439            .unwrap_or(MAX_MUTATION_RETRY_DELAY)
1440            .min(MAX_MUTATION_RETRY_DELAY);
1441        tokio::time::sleep(delay).await;
1442    }
1443
1444    fn node_started_payload(
1445        &self,
1446        graph: &GraphReference,
1447        checkpoint: &Checkpoint,
1448        node_id: &stateknot_core::NodeId,
1449        attempt_id: AttemptId,
1450    ) -> Result<JournalPayload, GraphDriverError> {
1451        self.event_payload(
1452            "graph-node-attempt-started",
1453            json!({
1454                "operation": "node_attempt_started",
1455                "graph_digest": digest_hex(graph.definition_digest()),
1456                "checkpoint_id": checkpoint.checkpoint_id().to_string(),
1457                "superstep": checkpoint.superstep().get().to_string(),
1458                "node_id": node_id.as_str(),
1459                "attempt_id": attempt_id.to_string()
1460            }),
1461        )
1462    }
1463
1464    fn node_succeeded_payload(
1465        &self,
1466        graph: &GraphReference,
1467        start: &NodeAttemptStartHead,
1468        result_digest: Digest,
1469    ) -> Result<JournalPayload, GraphDriverError> {
1470        let base = start.activation().base_checkpoint();
1471        self.event_payload(
1472            "graph-node-attempt-succeeded",
1473            json!({
1474                "operation": "node_attempt_succeeded",
1475                "graph_digest": digest_hex(graph.definition_digest()),
1476                "checkpoint_id": base.checkpoint_id().to_string(),
1477                "superstep": base.superstep().get().to_string(),
1478                "node_id": start.activation().node_id().as_str(),
1479                "attempt_id": start.attempt_id().to_string(),
1480                "result_digest": digest_hex(result_digest)
1481            }),
1482        )
1483    }
1484
1485    fn node_failed_payload(
1486        &self,
1487        graph: &GraphReference,
1488        start: &NodeAttemptStartHead,
1489        failure_id: FailureId,
1490    ) -> Result<JournalPayload, GraphDriverError> {
1491        let base = start.activation().base_checkpoint();
1492        self.event_payload(
1493            "graph-node-attempt-failed",
1494            json!({
1495                "operation": "node_attempt_failed",
1496                "graph_digest": digest_hex(graph.definition_digest()),
1497                "checkpoint_id": base.checkpoint_id().to_string(),
1498                "superstep": base.superstep().get().to_string(),
1499                "node_id": start.activation().node_id().as_str(),
1500                "attempt_id": start.attempt_id().to_string(),
1501                "failure_id": failure_id.to_string()
1502            }),
1503        )
1504    }
1505
1506    fn barrier_continued_payload(
1507        &self,
1508        graph: &GraphReference,
1509        checkpoint: &Checkpoint,
1510        successor_id: CheckpointId,
1511        successor_superstep: stateknot_core::Superstep,
1512    ) -> Result<JournalPayload, GraphDriverError> {
1513        self.event_payload(
1514            "graph-barrier-continued",
1515            json!({
1516                "operation": "graph_barrier_continued",
1517                "graph_digest": digest_hex(graph.definition_digest()),
1518                "checkpoint_id": checkpoint.checkpoint_id().to_string(),
1519                "superstep": checkpoint.superstep().get().to_string(),
1520                "successor_checkpoint_id": successor_id.to_string(),
1521                "successor_superstep": successor_superstep.get().to_string(),
1522                "disposition": "continue"
1523            }),
1524        )
1525    }
1526
1527    fn event_payload(
1528        &self,
1529        kind: &'static str,
1530        data: Value,
1531    ) -> Result<JournalPayload, GraphDriverError> {
1532        let data =
1533            BoundedJson::try_from_value(data).map_err(|_| GraphDriverError::EventPayloadInvalid)?;
1534        self.registry
1535            .schemas()
1536            .validate_bounded(&self.journal_schema, &data)
1537            .map_err(|_| GraphDriverError::EventPayloadInvalid)?;
1538        let kind =
1539            JournalEventKind::new(kind).map_err(|_| GraphDriverError::EventPayloadInvalid)?;
1540        JournalPayload::new(self.journal_schema.clone(), kind, data)
1541            .map_err(|_| GraphDriverError::EventPayloadInvalid)
1542    }
1543
1544    fn node_timeout_failure() -> Result<GraphNodeExecutionError, GraphDriverError> {
1545        let failure = runtime_failure(
1546            FailureCategory::DeadlineExceeded,
1547            "runtime.node_timeout",
1548            "graph node execution exceeded its configured deadline",
1549            None::<JoinError>,
1550        )?;
1551        GraphNodeExecutionError::new(failure, BudgetUsage::zero())
1552            .map_err(|_| GraphDriverError::RuntimeFailureInvalid)
1553    }
1554
1555    fn node_panic_failure()
1556    -> Result<Result<GraphNodeExecution, GraphNodeExecutionError>, GraphDriverError> {
1557        let failure = runtime_failure(
1558            FailureCategory::Internal,
1559            "runtime.node_task_panicked",
1560            "graph node execution task panicked",
1561            None::<JoinError>,
1562        )?;
1563        let execution = GraphNodeExecutionError::new(failure, BudgetUsage::zero())
1564            .map_err(|_| GraphDriverError::RuntimeFailureInvalid)?;
1565        Ok(Err(execution))
1566    }
1567}
1568
1569impl fmt::Debug for DurableGraphDriver {
1570    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
1571        formatter
1572            .debug_struct("DurableGraphDriver")
1573            .field("registry", &self.registry)
1574            .field("journal_schema", &self.journal_schema)
1575            .field("options", &self.options)
1576            .finish_non_exhaustive()
1577    }
1578}
1579
1580/// Startup failure while binding the driver to a provider and registry.
1581#[derive(Clone, Debug, Eq, Error, PartialEq)]
1582#[non_exhaustive]
1583pub enum DurableGraphDriverBuildError {
1584    /// The release's embedded standard schema was malformed.
1585    #[error(transparent)]
1586    StandardSchema(#[from] StandardGraphDriverSchemaError),
1587    /// The standard event schema was not installed before registry freeze.
1588    #[error("standard graph-driver journal schema is not installed")]
1589    JournalSchemaUnavailable,
1590    /// Fewer than three renewal intervals fit inside the provider lease.
1591    #[error("graph driver renewal cadence is unsafe for the provider lease timing")]
1592    UnsafeLeaseRenewalCadence,
1593}
1594
1595/// Observable work and recovery evidence from one drive call.
1596#[derive(Clone, Copy, Debug, Eq, PartialEq)]
1597pub struct GraphDriveReport {
1598    replay: GraphReplayReport,
1599    durable_events: u32,
1600    node_attempts_started: u32,
1601    node_attempts_completed: u32,
1602    node_batches_started: u32,
1603    maximum_nodes_in_batch: u16,
1604    barriers_committed: u32,
1605    lease_renewals: u32,
1606    mutation_retries: u32,
1607}
1608
1609impl GraphDriveReport {
1610    const fn new(replay: GraphReplayReport) -> Self {
1611        Self {
1612            replay,
1613            durable_events: 0,
1614            node_attempts_started: 0,
1615            node_attempts_completed: 0,
1616            node_batches_started: 0,
1617            maximum_nodes_in_batch: 0,
1618            barriers_committed: 0,
1619            lease_renewals: 0,
1620            mutation_retries: 0,
1621        }
1622    }
1623
1624    /// Returns the complete noninitial replay evidence.
1625    #[must_use]
1626    pub const fn replay(self) -> GraphReplayReport {
1627        self.replay
1628    }
1629
1630    /// Returns driver-owned journal events observed or committed this call.
1631    #[must_use]
1632    pub const fn durable_events(self) -> u32 {
1633        self.durable_events
1634    }
1635
1636    /// Returns fresh attempt starts authorized for execution.
1637    #[must_use]
1638    pub const fn node_attempts_started(self) -> u32 {
1639        self.node_attempts_started
1640    }
1641
1642    /// Returns attempt completions converged by this call.
1643    #[must_use]
1644    pub const fn node_attempts_completed(self) -> u32 {
1645        self.node_attempts_completed
1646    }
1647
1648    /// Returns the number of executor batches launched by this drive call.
1649    #[must_use]
1650    pub const fn node_batches_started(self) -> u32 {
1651        self.node_batches_started
1652    }
1653
1654    /// Returns the largest executor batch launched by this drive call.
1655    #[must_use]
1656    pub const fn maximum_nodes_in_batch(self) -> u16 {
1657        self.maximum_nodes_in_batch
1658    }
1659
1660    /// Returns Continue barriers converged by this call.
1661    #[must_use]
1662    pub const fn barriers_committed(self) -> u32 {
1663        self.barriers_committed
1664    }
1665
1666    /// Returns successful or idempotent lease renewals during node work.
1667    #[must_use]
1668    pub const fn lease_renewals(self) -> u32 {
1669        self.lease_renewals
1670    }
1671
1672    /// Returns extra identical attempts after retryable storage failures.
1673    #[must_use]
1674    pub const fn mutation_retries(self) -> u32 {
1675        self.mutation_retries
1676    }
1677}
1678
1679/// Complete result of one bounded drive call.
1680#[derive(Debug)]
1681pub struct GraphDriveResult {
1682    outcome: GraphDriveOutcome,
1683    report: GraphDriveReport,
1684}
1685
1686impl GraphDriveResult {
1687    const fn new(outcome: GraphDriveOutcome, report: GraphDriveReport) -> Self {
1688        Self { outcome, report }
1689    }
1690
1691    /// Returns why the driver stopped or handed execution off.
1692    #[must_use]
1693    pub const fn outcome(&self) -> &GraphDriveOutcome {
1694        &self.outcome
1695    }
1696
1697    /// Returns replay, execution, renewal, and retry counters.
1698    #[must_use]
1699    pub const fn report(&self) -> GraphDriveReport {
1700        self.report
1701    }
1702
1703    /// Consumes the result into its outcome and report.
1704    #[must_use]
1705    pub fn into_parts(self) -> (GraphDriveOutcome, GraphDriveReport) {
1706        (self.outcome, self.report)
1707    }
1708}
1709
1710/// Closed reason one bounded drive call stopped.
1711#[derive(Debug)]
1712#[non_exhaustive]
1713pub enum GraphDriveOutcome {
1714    /// Children were sealed and the lease released in one transaction. The
1715    /// node remains unfinished until publication and a higher-fence recovery.
1716    ChildJoin(Box<stateknot_core::ChildRunJoinRequest>),
1717    /// Durable cancellation intent requires exact evidence and terminal acknowledgement.
1718    CancellationRequested(Box<GraphCancellationHandoff>),
1719    /// A Wait or successful Terminal barrier requires lifecycle metadata.
1720    LifecycleBarrierReady(Box<GraphLifecycleBarrierHandoff>),
1721    /// In-flight, terminally failed, or exhausted nodes require supervision.
1722    Blocked(Box<GraphBlockedHandoff>),
1723    /// Deferred-only work was durably scheduled and its lease released.
1724    Deferred {
1725        /// Inclusive database retry boundary.
1726        not_before: Timestamp,
1727        /// Exact scheduling convergence result.
1728        schedule: DelayedRetryScheduleOutcome,
1729    },
1730    /// The bounded work quantum was exhausted between durable operations.
1731    Yielded {
1732        /// Exact-fence lease release convergence.
1733        release: LeaseReleaseOutcome,
1734    },
1735    /// Cooperative shutdown won between or during node execution.
1736    Cancelled {
1737        /// Exact-fence lease release convergence.
1738        release: LeaseReleaseOutcome,
1739    },
1740}
1741
1742/// Exact lease-bound input for terminal cancellation acknowledgement.
1743#[derive(Clone, Debug)]
1744pub struct GraphCancellationHandoff {
1745    checkpoint: stateknot_core::CheckpointHead,
1746    journal_head: JournalHead,
1747    event_id: EventId,
1748    expected_revision: RunRevision,
1749    lease: RunLease,
1750    failure_id: FailureId,
1751}
1752
1753impl GraphCancellationHandoff {
1754    /// Returns the exact checkpoint at which cancellation stopped graph progress.
1755    #[must_use]
1756    pub const fn checkpoint(&self) -> &stateknot_core::CheckpointHead {
1757        &self.checkpoint
1758    }
1759
1760    /// Returns the cancellation-request event as the exact journal predecessor.
1761    #[must_use]
1762    pub const fn journal_head(&self) -> &JournalHead {
1763        &self.journal_head
1764    }
1765
1766    /// Returns the stable lost-acknowledgement identity for confirmation.
1767    #[must_use]
1768    pub const fn event_id(&self) -> EventId {
1769        self.event_id
1770    }
1771
1772    /// Returns the cancellation-request lifecycle revision to consume.
1773    #[must_use]
1774    pub const fn expected_revision(&self) -> RunRevision {
1775        self.expected_revision
1776    }
1777
1778    /// Returns the exact live lease retained for confirmation.
1779    #[must_use]
1780    pub const fn lease(&self) -> &RunLease {
1781        &self.lease
1782    }
1783
1784    /// Returns the immutable cancellation occurrence selected by the request.
1785    #[must_use]
1786    pub const fn failure_id(&self) -> FailureId {
1787        self.failure_id
1788    }
1789
1790    /// Consumes the handoff into storage-ready parts.
1791    #[must_use]
1792    pub fn into_parts(
1793        self,
1794    ) -> (
1795        stateknot_core::CheckpointHead,
1796        JournalHead,
1797        EventId,
1798        RunRevision,
1799        RunLease,
1800        FailureId,
1801    ) {
1802        (
1803            self.checkpoint,
1804            self.journal_head,
1805            self.event_id,
1806            self.expected_revision,
1807            self.lease,
1808            self.failure_id,
1809        )
1810    }
1811}
1812
1813/// Exact lease-bound Wait/Terminal commit input for a lifecycle integrator.
1814#[derive(Clone, Debug)]
1815pub struct GraphLifecycleBarrierHandoff {
1816    plan: GraphBarrierPlan,
1817    journal_head: JournalHead,
1818    event_id: EventId,
1819    expected_revision: RunRevision,
1820    lease: RunLease,
1821}
1822
1823impl GraphLifecycleBarrierHandoff {
1824    /// Returns the complete deterministic barrier and disposition.
1825    #[must_use]
1826    pub const fn plan(&self) -> &GraphBarrierPlan {
1827        &self.plan
1828    }
1829
1830    /// Returns the exact current journal head for the lifecycle append.
1831    #[must_use]
1832    pub const fn journal_head(&self) -> &JournalHead {
1833        &self.journal_head
1834    }
1835
1836    /// Returns the stable lost-acknowledgement identity for the lifecycle event.
1837    #[must_use]
1838    pub const fn event_id(&self) -> EventId {
1839        self.event_id
1840    }
1841
1842    /// Returns the lifecycle revision that must still match atomically.
1843    #[must_use]
1844    pub const fn expected_revision(&self) -> RunRevision {
1845        self.expected_revision
1846    }
1847
1848    /// Returns the exact lease that must remain unexpired at commit.
1849    #[must_use]
1850    pub const fn lease(&self) -> &RunLease {
1851        &self.lease
1852    }
1853
1854    /// Consumes the handoff into storage-ready parts.
1855    #[must_use]
1856    pub fn into_parts(
1857        self,
1858    ) -> (
1859        GraphBarrierPlan,
1860        JournalHead,
1861        EventId,
1862        RunRevision,
1863        RunLease,
1864    ) {
1865        (
1866            self.plan,
1867            self.journal_head,
1868            self.event_id,
1869            self.expected_revision,
1870            self.lease,
1871        )
1872    }
1873}
1874
1875/// Aggregate classifications that prevent further node dispatch.
1876#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
1877pub struct GraphDriveBlockers {
1878    in_flight: u16,
1879    failed: u16,
1880    exhausted: u16,
1881    unsupported: u16,
1882    superstep_limit_reached: bool,
1883}
1884
1885impl GraphDriveBlockers {
1886    fn from_plan(plan: &ReadyNodeRecoveryPlan) -> Self {
1887        let mut blockers = Self::default();
1888        for node in plan.nodes() {
1889            match node.kind() {
1890                RecoveryNodeKind::InFlight => {
1891                    blockers.in_flight = blockers.in_flight.saturating_add(1);
1892                }
1893                RecoveryNodeKind::Failed => {
1894                    blockers.failed = blockers.failed.saturating_add(1);
1895                }
1896                RecoveryNodeKind::Exhausted => {
1897                    blockers.exhausted = blockers.exhausted.saturating_add(1);
1898                }
1899                RecoveryNodeKind::Completed
1900                | RecoveryNodeKind::Dispatchable
1901                | RecoveryNodeKind::Deferred => {}
1902                _ => {
1903                    blockers.unsupported = blockers.unsupported.saturating_add(1);
1904                }
1905            }
1906        }
1907        blockers
1908    }
1909
1910    /// Returns whether any same-fence unfinished attempt exists.
1911    #[must_use]
1912    pub const fn in_flight(self) -> u16 {
1913        self.in_flight
1914    }
1915
1916    /// Returns terminal node failures without retry authority.
1917    #[must_use]
1918    pub const fn failed(self) -> u16 {
1919        self.failed
1920    }
1921
1922    /// Returns activations at the hard physical-attempt ceiling.
1923    #[must_use]
1924    pub const fn exhausted(self) -> u16 {
1925        self.exhausted
1926    }
1927
1928    /// Returns future scheduler classifications this driver cannot execute.
1929    #[must_use]
1930    pub const fn unsupported(self) -> u16 {
1931        self.unsupported
1932    }
1933
1934    /// Returns whether the graph exhausted its pinned global barrier budget.
1935    ///
1936    /// This is independent of physical node-attempt counts. Failure evidence
1937    /// must still recover exact cumulative usage, including earlier iterations.
1938    #[must_use]
1939    pub const fn superstep_limit_reached(self) -> bool {
1940        self.superstep_limit_reached
1941    }
1942
1943    /// Returns whether no blocking classification exists.
1944    #[must_use]
1945    pub const fn is_empty(self) -> bool {
1946        self.in_flight == 0
1947            && self.failed == 0
1948            && self.exhausted == 0
1949            && self.unsupported == 0
1950            && !self.superstep_limit_reached
1951    }
1952}
1953
1954/// Exact recovery plan and lease retained for supervision.
1955#[derive(Clone, Debug)]
1956pub struct GraphBlockedHandoff {
1957    plan: ReadyNodeRecoveryPlan,
1958    lease: RunLease,
1959    event_id: EventId,
1960    expected_revision: RunRevision,
1961    blockers: GraphDriveBlockers,
1962}
1963
1964impl GraphBlockedHandoff {
1965    /// Returns every deterministic ready-node classification and evidence.
1966    #[must_use]
1967    pub const fn plan(&self) -> &ReadyNodeRecoveryPlan {
1968        &self.plan
1969    }
1970
1971    /// Returns the exact lease retained while supervision decides the edge.
1972    #[must_use]
1973    pub const fn lease(&self) -> &RunLease {
1974        &self.lease
1975    }
1976
1977    /// Returns the stable lost-acknowledgement identity for failure supervision.
1978    #[must_use]
1979    pub const fn event_id(&self) -> EventId {
1980        self.event_id
1981    }
1982
1983    /// Returns the lifecycle revision observed with the retained live lease.
1984    #[must_use]
1985    pub const fn expected_revision(&self) -> RunRevision {
1986        self.expected_revision
1987    }
1988
1989    /// Returns aggregate blocking counts.
1990    #[must_use]
1991    pub const fn blockers(&self) -> GraphDriveBlockers {
1992        self.blockers
1993    }
1994
1995    /// Consumes the handoff into supervision-ready parts.
1996    #[must_use]
1997    pub fn into_parts(
1998        self,
1999    ) -> (
2000        ReadyNodeRecoveryPlan,
2001        RunLease,
2002        EventId,
2003        RunRevision,
2004        GraphDriveBlockers,
2005    ) {
2006        (
2007            self.plan,
2008            self.lease,
2009            self.event_id,
2010            self.expected_revision,
2011            self.blockers,
2012        )
2013    }
2014}
2015
2016/// Durable driver failure with payload-redacted diagnostics.
2017#[derive(Debug, Error)]
2018#[non_exhaustive]
2019pub enum GraphDriverError {
2020    /// Join evidence preparation timed out or panicked before node dispatch;
2021    /// no node completion, failure or usage was synthesized.
2022    #[error("child Join preparation did not complete before node dispatch")]
2023    ChildJoinPreparationIncomplete,
2024    /// `PostgreSQL` rejected or could not complete an operation.
2025    #[error(transparent)]
2026    Store {
2027        /// Exact payload-redacted provider failure.
2028        source: Box<StoreError>,
2029    },
2030    /// A claimed checkpoint had no journal anchor.
2031    #[error("graph driver requires a non-empty run journal")]
2032    MissingJournalHead,
2033    /// The exact pinned graph code was absent from this deployment.
2034    #[error("pinned graph has no executable deployment binding")]
2035    ExecutableGraphUnavailable {
2036        /// Complete unavailable graph reference.
2037        graph: Box<GraphReference>,
2038    },
2039    /// A graph registry invariant lost one node executor.
2040    #[error("registered graph node executor is unavailable")]
2041    ExecutableNodeUnavailable {
2042        /// Complete graph reference.
2043        graph: Box<GraphReference>,
2044        /// Missing compiled node identity.
2045        node_id: stateknot_core::NodeId,
2046    },
2047    /// A durable attempt could not bind to its immutable checkpoint.
2048    #[error(transparent)]
2049    NodeContext(#[from] GraphNodeContextError),
2050    /// A node returned a crossed or noncanonical semantic result.
2051    #[error("graph node returned an invalid durable result")]
2052    InvalidNodeResult,
2053    /// Pure barrier planning rejected current result semantics.
2054    #[error("graph barrier planning failed: {source}")]
2055    GraphPlanning {
2056        /// Exact public-safe planner failure.
2057        #[source]
2058        source: Box<GraphBarrierPlanError>,
2059    },
2060    /// Barrier result serialization failed closed.
2061    #[error("graph barrier results could not be measured")]
2062    BarrierResultEncoding,
2063    /// Current barrier results exceeded the configured memory ceiling.
2064    #[error("graph barrier results exceed the configured memory limit")]
2065    BarrierResultResourceLimit,
2066    /// Result pagination did not preserve the recovery journal observation.
2067    #[error("graph barrier result snapshot became stale")]
2068    StaleBarrierSnapshot,
2069    /// Standard audit event data failed its pinned local schema.
2070    #[error("graph driver event payload failed its pinned schema")]
2071    EventPayloadInvalid,
2072    /// A worker journal append could not be constructed.
2073    #[error("graph driver worker journal append is invalid")]
2074    JournalAppendInvalid,
2075    /// Lease extension overflowed canonical timestamp bounds.
2076    #[error("graph driver lease extension exceeds canonical time bounds")]
2077    LeaseTimestampOverflow,
2078    /// The driver's own public-safe failure could not be constructed.
2079    #[error("graph driver could not construct runtime failure evidence")]
2080    RuntimeFailureInvalid,
2081    /// A supposedly closed registry/recovery invariant failed.
2082    #[error("graph driver invariant failed while attempting to {operation}")]
2083    RuntimeInvariant {
2084        /// Stable payload-free operation label.
2085        operation: &'static str,
2086    },
2087}
2088
2089impl GraphDriverError {
2090    fn graph_plan(source: GraphBarrierPlanError) -> Self {
2091        Self::GraphPlanning {
2092            source: Box::new(source),
2093        }
2094    }
2095}
2096
2097impl From<StoreError> for GraphDriverError {
2098    fn from(source: StoreError) -> Self {
2099        Self::Store {
2100            source: Box::new(source),
2101        }
2102    }
2103}
2104
2105struct StartedNode {
2106    executor: Arc<dyn crate::GraphNodeExecutor>,
2107    start: NodeAttemptStartHead,
2108}
2109
2110#[allow(clippy::large_enum_variant)]
2111enum StartedNodesExecution {
2112    Finished(Vec<FinishedNodeExecution>),
2113    Cancelled,
2114    RunCancellationObserved,
2115}
2116
2117enum NodeExecutionCommit {
2118    Committed,
2119    RunCancellationObserved,
2120    ChildJoin(Box<stateknot_core::ChildRunJoinRequest>),
2121}
2122
2123struct FinishedNodeExecution {
2124    result: Result<GraphNodeExecution, GraphNodeExecutionError>,
2125    child_join: Option<Box<stateknot_core::ChildRunJoinHead>>,
2126}
2127
2128impl FinishedNodeExecution {
2129    fn without_join(result: Result<GraphNodeExecution, GraphNodeExecutionError>) -> Self {
2130        Self {
2131            result,
2132            child_join: None,
2133        }
2134    }
2135}
2136
2137struct GuardedRunLease {
2138    lease: RunLease,
2139    deadline: Instant,
2140}
2141
2142fn guarded_run_lease(
2143    lease: RunLease,
2144    observed_at: Timestamp,
2145    observation_started: Instant,
2146) -> Result<GuardedRunLease, GraphDriverError> {
2147    let remaining_micros = lease
2148        .expires_at()
2149        .unix_micros()
2150        .checked_sub(observed_at.unix_micros())
2151        .and_then(|value| u64::try_from(value).ok())
2152        .filter(|value| *value > 0)
2153        .ok_or(StoreError::LeaseExpired)?;
2154    let deadline = observation_started
2155        .checked_add(Duration::from_micros(remaining_micros))
2156        .ok_or(GraphDriverError::LeaseTimestampOverflow)?;
2157    if deadline <= Instant::now() {
2158        return Err(StoreError::LeaseExpired.into());
2159    }
2160    Ok(GuardedRunLease { lease, deadline })
2161}
2162
2163fn exact_live_lease<'run>(
2164    run: &'run StoredRun,
2165    fence: &RunFence,
2166) -> Result<&'run RunLease, GraphDriverError> {
2167    if run.is_quarantined() {
2168        return Err(StoreError::RunQuarantined.into());
2169    }
2170    let lease = run.lease().ok_or(StoreError::NoActiveLease)?;
2171    if lease.fence() != fence {
2172        return Err(StoreError::StaleFence.into());
2173    }
2174    Ok(lease)
2175}
2176
2177fn graph_error_is_run_not_runnable(error: &GraphDriverError) -> bool {
2178    matches!(
2179        error,
2180        GraphDriverError::Store { source } if matches!(source.as_ref(), StoreError::RunNotRunnable)
2181    )
2182}
2183
2184fn recovery_evidence_digest(run: &StoredRun, fence: &RunFence) -> Result<Digest, GraphDriverError> {
2185    let head = run
2186        .journal_head()
2187        .ok_or(GraphDriverError::MissingJournalHead)?;
2188    let checkpoint = run
2189        .checkpoint()
2190        .ok_or(StoreError::ReadyNodeRecoveryCheckpointMissing)?;
2191    let mut evidence =
2192        Vec::with_capacity(RECOVERY_EVIDENCE_DOMAIN.len() + Digest::SHA256_LEN * 2 + 16 + 8);
2193    evidence.extend_from_slice(RECOVERY_EVIDENCE_DOMAIN);
2194    evidence.extend_from_slice(head.digest().as_bytes());
2195    evidence.extend_from_slice(checkpoint.digest().as_bytes());
2196    evidence.extend_from_slice(fence.attempt_id().as_uuid().as_bytes());
2197    evidence.extend_from_slice(&fence.epoch().get().to_be_bytes());
2198    Ok(Digest::sha256(evidence))
2199}
2200
2201fn worker_append(
2202    fence: &RunFence,
2203    head: JournalHead,
2204    event_id: EventId,
2205    payload: JournalPayload,
2206) -> Result<JournalAppend, GraphDriverError> {
2207    let intent = JournalEventIntent::worker(
2208        fence.tenant_id().clone(),
2209        fence.run_id(),
2210        event_id,
2211        fence.clone(),
2212        payload,
2213    )
2214    .map_err(|_| GraphDriverError::JournalAppendInvalid)?;
2215    JournalAppend::new(JournalExpectation::exact(head), intent)
2216        .map_err(|_| GraphDriverError::JournalAppendInvalid)
2217}
2218
2219fn extend_timestamp(
2220    timestamp: Timestamp,
2221    duration: Duration,
2222) -> Result<Timestamp, GraphDriverError> {
2223    let micros = i64::try_from(duration.as_micros())
2224        .map_err(|_| GraphDriverError::LeaseTimestampOverflow)?;
2225    let value = timestamp
2226        .unix_micros()
2227        .checked_add(micros)
2228        .ok_or(GraphDriverError::LeaseTimestampOverflow)?;
2229    Timestamp::from_unix_micros(value).map_err(|_| GraphDriverError::LeaseTimestampOverflow)
2230}
2231
2232fn digest_hex(digest: Digest) -> String {
2233    const HEX: &[u8; 16] = b"0123456789abcdef";
2234    let mut value = String::with_capacity(Digest::SHA256_LEN * 2);
2235    for byte in digest.as_bytes() {
2236        value.push(char::from(HEX[usize::from(byte >> 4)]));
2237        value.push(char::from(HEX[usize::from(byte & 0x0f)]));
2238    }
2239    value
2240}
2241
2242fn runtime_failure<E>(
2243    category: FailureCategory,
2244    code: &'static str,
2245    message: &'static str,
2246    source: Option<E>,
2247) -> Result<Failure, GraphDriverError>
2248where
2249    E: std::error::Error + Send + Sync + 'static,
2250{
2251    let code = FailureCode::new(code).map_err(|_| GraphDriverError::RuntimeFailureInvalid)?;
2252    let origin = FailureOrigin::new("stateknot.runtime.graph-driver")
2253        .map_err(|_| GraphDriverError::RuntimeFailureInvalid)?;
2254    let message =
2255        FailureMessage::new(message).map_err(|_| GraphDriverError::RuntimeFailureInvalid)?;
2256    let mut failure = Failure::new(
2257        FailureId::generate(),
2258        category,
2259        code,
2260        origin,
2261        message,
2262        RetryAdvice::Never,
2263    )
2264    .map_err(|_| GraphDriverError::RuntimeFailureInvalid)?;
2265    if let Some(source) = source {
2266        failure = failure.with_private_source(source);
2267    }
2268    Ok(failure)
2269}
2270
2271fn cancel_node_batch(cancellations: &[DriverCancellation]) {
2272    for cancellation in cancellations {
2273        cancellation.cancel();
2274    }
2275}
2276
2277async fn abort_node_batch<T>(tasks: &mut JoinSet<T>)
2278where
2279    T: Send + 'static,
2280{
2281    tasks.abort_all();
2282    while tasks.join_next().await.is_some() {}
2283}
2284
2285async fn drain_node_batch_until<T>(tasks: &mut JoinSet<T>, deadline: Instant)
2286where
2287    T: Send + 'static,
2288{
2289    while !tasks.is_empty() && deadline > Instant::now() {
2290        tokio::select! {
2291            _ = tasks.join_next() => {}
2292            () = tokio::time::sleep_until(deadline) => break,
2293        }
2294    }
2295    if !tasks.is_empty() {
2296        abort_node_batch(tasks).await;
2297    }
2298}
2299
2300#[derive(Default)]
2301struct CompactByteCounter {
2302    bytes: usize,
2303}
2304
2305impl std::io::Write for CompactByteCounter {
2306    fn write(&mut self, buffer: &[u8]) -> std::io::Result<usize> {
2307        self.bytes = self.bytes.saturating_add(buffer.len());
2308        Ok(buffer.len())
2309    }
2310
2311    fn flush(&mut self) -> std::io::Result<()> {
2312        Ok(())
2313    }
2314}
2315
2316#[derive(Clone)]
2317struct DriverCancellation {
2318    state: Arc<DriverCancellationState>,
2319}
2320
2321struct DriverCancellationState {
2322    cancelled: AtomicBool,
2323    notify: Notify,
2324}
2325
2326impl DriverCancellation {
2327    fn new() -> Self {
2328        Self {
2329            state: Arc::new(DriverCancellationState {
2330                cancelled: AtomicBool::new(false),
2331                notify: Notify::new(),
2332            }),
2333        }
2334    }
2335
2336    fn signal(&self) -> CancellationSignal {
2337        CancellationSignal::new(self.clone())
2338    }
2339
2340    fn cancel(&self) {
2341        if !self.state.cancelled.swap(true, Ordering::AcqRel) {
2342            self.state.notify.notify_waiters();
2343        }
2344    }
2345}
2346
2347impl CancellationObserver for DriverCancellation {
2348    fn is_cancelled(&self) -> bool {
2349        self.state.cancelled.load(Ordering::Acquire)
2350    }
2351
2352    fn cancelled(&self) -> BoxFuture<'_, ()> {
2353        Box::pin(async move {
2354            loop {
2355                let notified = self.state.notify.notified();
2356                if self.is_cancelled() {
2357                    return;
2358                }
2359                notified.await;
2360            }
2361        })
2362    }
2363}
2364
2365#[cfg(test)]
2366mod tests {
2367    use super::*;
2368
2369    #[test]
2370    fn options_reject_unsafe_local_bounds() {
2371        let limits = GraphReplayLimits::default();
2372        assert!(
2373            DurableGraphDriverOptions::new(
2374                limits,
2375                1,
2376                Duration::from_secs(10),
2377                Duration::from_secs(60),
2378                3,
2379                Duration::from_millis(25),
2380            )
2381            .is_err()
2382        );
2383        assert!(
2384            DurableGraphDriverOptions::new(
2385                limits,
2386                2,
2387                Duration::from_nanos(1),
2388                Duration::from_secs(60),
2389                3,
2390                Duration::from_millis(25),
2391            )
2392            .is_err()
2393        );
2394        assert!(
2395            DurableGraphDriverOptions::new(
2396                limits,
2397                2,
2398                Duration::from_secs(1),
2399                Duration::ZERO,
2400                3,
2401                Duration::from_millis(25),
2402            )
2403            .is_err()
2404        );
2405        let options = DurableGraphDriverOptions::default();
2406        assert!(matches!(
2407            options.with_cancellation_timing(Duration::ZERO, Duration::from_secs(1)),
2408            Err(DurableGraphDriverOptionsError::InvalidCancellationPollInterval)
2409        ));
2410        assert!(matches!(
2411            options.with_cancellation_timing(Duration::from_millis(9), Duration::from_secs(1),),
2412            Err(DurableGraphDriverOptionsError::InvalidCancellationPollInterval)
2413        ));
2414        assert!(matches!(
2415            options.with_cancellation_timing(
2416                DurableGraphDriverOptions::HARD_MINIMUM_CANCELLATION_POLL_INTERVAL,
2417                Duration::from_secs(5 * 60 + 1),
2418            ),
2419            Err(DurableGraphDriverOptionsError::InvalidCancellationGracePeriod)
2420        ));
2421        assert!(matches!(
2422            options.with_parallel_node_limit(0),
2423            Err(DurableGraphDriverOptionsError::InvalidParallelNodeLimit)
2424        ));
2425        assert!(matches!(
2426            options.with_parallel_node_limit(
2427                DurableGraphDriverOptions::HARD_MAXIMUM_PARALLEL_NODES + 1
2428            ),
2429            Err(DurableGraphDriverOptionsError::InvalidParallelNodeLimit)
2430        ));
2431        assert_eq!(
2432            options
2433                .with_parallel_node_limit(4)
2434                .unwrap()
2435                .maximum_parallel_nodes(),
2436            4
2437        );
2438    }
2439
2440    #[tokio::test]
2441    async fn cancellation_is_monotonic_and_race_safe() {
2442        let cancellation = DriverCancellation::new();
2443        let signal = cancellation.signal();
2444        assert!(!signal.is_cancelled());
2445        let waiter = tokio::spawn({
2446            let signal = signal.clone();
2447            async move { signal.cancelled().await }
2448        });
2449        cancellation.cancel();
2450        waiter.await.unwrap();
2451        cancellation.cancel();
2452        assert!(signal.is_cancelled());
2453        signal.cancelled().await;
2454    }
2455
2456    #[test]
2457    fn digest_data_uses_schema_hex_without_algorithm_prefix() {
2458        let value = digest_hex(Digest::sha256(b"driver"));
2459        assert_eq!(value.len(), 64);
2460        assert!(value.bytes().all(|byte| byte.is_ascii_hexdigit()));
2461        assert!(!value.contains(':'));
2462    }
2463}