1use std::{
7 fmt,
8 panic::AssertUnwindSafe,
9 sync::{
10 Arc,
11 atomic::{AtomicBool, Ordering},
12 },
13 time::Duration,
14};
15
16use futures_util::FutureExt;
17use serde_json::{Value, json};
18use stateknot_core::{
19 AttemptId, BoundedJson, BoxFuture, BudgetUsage, CancellationObserver, CancellationSignal,
20 Checkpoint, CheckpointId, Digest, EventId, Failure, FailureCategory, FailureCode, FailureId,
21 FailureMessage, FailureOrigin, GraphBarrierDisposition, GraphBarrierPlan,
22 GraphBarrierPlanError, GraphReference, JournalAppend, JournalEventIntent, JournalEventKind,
23 JournalExpectation, JournalHead, JournalPayload, NodeAttemptStartHead, PendingNodeResult,
24 PendingNodeResultIntent, QuarantineId, ReadyNodeRecoveryPlan, RecoveryNodeKind, RetryAdvice,
25 RunFence, RunLease, RunRevision, RunStatus, Timestamp,
26};
27use stateknot_store_postgres::{
28 BarrierCommitOutcome, ClaimedRunRecovery, CorruptionQuarantineContext,
29 DelayedRetryScheduleOutcome, GraphReplayLimits, GraphReplayReport, LeaseReleaseOutcome,
30 LeaseRenewalOutcome, NodeAttemptCommitOutcome, PendingNodeResultPageSize, PostgresStore,
31 RunProjection, StoreError, StoredRun,
32};
33use thiserror::Error;
34use tokio::{
35 sync::Notify,
36 task::{JoinError, JoinSet},
37 time::Instant,
38};
39
40use crate::{
41 ExecutableGraph, ExecutableGraphRegistry, GraphNodeContext, GraphNodeContextError,
42 GraphNodeExecution, GraphNodeExecutionError, GraphNodeScheduling,
43 StandardGraphDriverSchemaError, standard_graph_driver_event_schema,
44};
45
46const MAX_NODE_TIMEOUT: Duration = Duration::from_secs(7 * 24 * 60 * 60);
47const MAX_MUTATION_RETRY_DELAY: Duration = Duration::from_secs(1);
48const RECOVERY_EVIDENCE_DOMAIN: &[u8] = b"stateknot/runtime/graph-driver/recovery/v1\0";
49
50#[derive(Clone, Copy, Debug, Eq, PartialEq)]
52pub struct DurableGraphDriverOptions {
53 replay_limits: GraphReplayLimits,
54 maximum_durable_events: u32,
55 lease_renewal_interval: Duration,
56 node_execution_timeout: Duration,
57 cancellation_poll_interval: Duration,
58 cancellation_grace_period: Duration,
59 maximum_parallel_nodes: u16,
60 maximum_mutation_attempts: u8,
61 mutation_retry_initial_delay: Duration,
62}
63
64impl DurableGraphDriverOptions {
65 pub const HARD_MAXIMUM_DURABLE_EVENTS: u32 = 65_536;
67 pub const HARD_MAXIMUM_MUTATION_ATTEMPTS: u8 = 10;
69 pub const HARD_MINIMUM_CANCELLATION_POLL_INTERVAL: Duration = Duration::from_millis(10);
71 pub const HARD_MAXIMUM_CANCELLATION_POLL_INTERVAL: Duration = Duration::from_secs(60);
73 pub const HARD_MAXIMUM_CANCELLATION_GRACE_PERIOD: Duration = Duration::from_secs(5 * 60);
75 pub const HARD_MAXIMUM_PARALLEL_NODES: u16 = 64;
77
78 pub fn new(
86 replay_limits: GraphReplayLimits,
87 maximum_durable_events: u32,
88 lease_renewal_interval: Duration,
89 node_execution_timeout: Duration,
90 maximum_mutation_attempts: u8,
91 mutation_retry_initial_delay: Duration,
92 ) -> Result<Self, DurableGraphDriverOptionsError> {
93 if !(2..=Self::HARD_MAXIMUM_DURABLE_EVENTS).contains(&maximum_durable_events) {
94 return Err(DurableGraphDriverOptionsError::InvalidWorkQuantum);
95 }
96 if lease_renewal_interval.is_zero()
97 || lease_renewal_interval.subsec_nanos() % 1_000 != 0
98 || lease_renewal_interval.as_micros() > i64::MAX as u128
99 {
100 return Err(DurableGraphDriverOptionsError::InvalidLeaseRenewalInterval);
101 }
102 if node_execution_timeout.is_zero() || node_execution_timeout > MAX_NODE_TIMEOUT {
103 return Err(DurableGraphDriverOptionsError::InvalidNodeExecutionTimeout);
104 }
105 if !(1..=Self::HARD_MAXIMUM_MUTATION_ATTEMPTS).contains(&maximum_mutation_attempts) {
106 return Err(DurableGraphDriverOptionsError::InvalidMutationAttempts);
107 }
108 if mutation_retry_initial_delay.is_zero()
109 || mutation_retry_initial_delay > MAX_MUTATION_RETRY_DELAY
110 {
111 return Err(DurableGraphDriverOptionsError::InvalidMutationRetryDelay);
112 }
113 Ok(Self {
114 replay_limits,
115 maximum_durable_events,
116 lease_renewal_interval,
117 node_execution_timeout,
118 cancellation_poll_interval: Duration::from_millis(250),
119 cancellation_grace_period: Duration::from_secs(5),
120 maximum_parallel_nodes: 1,
121 maximum_mutation_attempts,
122 mutation_retry_initial_delay,
123 })
124 }
125
126 pub fn with_cancellation_timing(
135 mut self,
136 poll_interval: Duration,
137 grace_period: Duration,
138 ) -> Result<Self, DurableGraphDriverOptionsError> {
139 if poll_interval < Self::HARD_MINIMUM_CANCELLATION_POLL_INTERVAL
140 || poll_interval.subsec_nanos() % 1_000 != 0
141 || poll_interval > Self::HARD_MAXIMUM_CANCELLATION_POLL_INTERVAL
142 {
143 return Err(DurableGraphDriverOptionsError::InvalidCancellationPollInterval);
144 }
145 if grace_period.is_zero()
146 || grace_period.subsec_nanos() % 1_000 != 0
147 || grace_period > Self::HARD_MAXIMUM_CANCELLATION_GRACE_PERIOD
148 {
149 return Err(DurableGraphDriverOptionsError::InvalidCancellationGracePeriod);
150 }
151 self.cancellation_poll_interval = poll_interval;
152 self.cancellation_grace_period = grace_period;
153 Ok(self)
154 }
155
156 pub fn with_parallel_node_limit(
167 mut self,
168 maximum_parallel_nodes: u16,
169 ) -> Result<Self, DurableGraphDriverOptionsError> {
170 if maximum_parallel_nodes == 0 || maximum_parallel_nodes > Self::HARD_MAXIMUM_PARALLEL_NODES
171 {
172 return Err(DurableGraphDriverOptionsError::InvalidParallelNodeLimit);
173 }
174 self.maximum_parallel_nodes = maximum_parallel_nodes;
175 Ok(self)
176 }
177
178 #[must_use]
181 pub const fn replay_limits(self) -> GraphReplayLimits {
182 self.replay_limits
183 }
184
185 #[must_use]
187 pub const fn maximum_durable_events(self) -> u32 {
188 self.maximum_durable_events
189 }
190
191 #[must_use]
193 pub const fn lease_renewal_interval(self) -> Duration {
194 self.lease_renewal_interval
195 }
196
197 #[must_use]
199 pub const fn node_execution_timeout(self) -> Duration {
200 self.node_execution_timeout
201 }
202
203 #[must_use]
205 pub const fn cancellation_poll_interval(self) -> Duration {
206 self.cancellation_poll_interval
207 }
208
209 #[must_use]
211 pub const fn cancellation_grace_period(self) -> Duration {
212 self.cancellation_grace_period
213 }
214
215 #[must_use]
217 pub const fn maximum_parallel_nodes(self) -> u16 {
218 self.maximum_parallel_nodes
219 }
220
221 #[must_use]
223 pub const fn maximum_mutation_attempts(self) -> u8 {
224 self.maximum_mutation_attempts
225 }
226
227 #[must_use]
229 pub const fn mutation_retry_initial_delay(self) -> Duration {
230 self.mutation_retry_initial_delay
231 }
232}
233
234impl Default for DurableGraphDriverOptions {
235 fn default() -> Self {
236 Self {
237 replay_limits: GraphReplayLimits::default(),
238 maximum_durable_events: 1_024,
239 lease_renewal_interval: Duration::from_secs(10),
240 node_execution_timeout: Duration::from_secs(15 * 60),
241 cancellation_poll_interval: Duration::from_millis(250),
242 cancellation_grace_period: Duration::from_secs(5),
243 maximum_parallel_nodes: 1,
244 maximum_mutation_attempts: 3,
245 mutation_retry_initial_delay: Duration::from_millis(25),
246 }
247 }
248}
249
250#[derive(Clone, Copy, Debug, Eq, Error, PartialEq)]
252#[non_exhaustive]
253pub enum DurableGraphDriverOptionsError {
254 #[error("graph driver durable-event quantum is outside its production bounds")]
256 InvalidWorkQuantum,
257 #[error("graph driver lease renewal interval is invalid")]
259 InvalidLeaseRenewalInterval,
260 #[error("graph driver node execution timeout is invalid")]
262 InvalidNodeExecutionTimeout,
263 #[error("graph driver cancellation poll interval is invalid")]
265 InvalidCancellationPollInterval,
266 #[error("graph driver cancellation grace period is invalid")]
268 InvalidCancellationGracePeriod,
269 #[error("graph driver parallel node limit is invalid")]
271 InvalidParallelNodeLimit,
272 #[error("graph driver mutation attempt count is invalid")]
274 InvalidMutationAttempts,
275 #[error("graph driver mutation retry delay is invalid")]
277 InvalidMutationRetryDelay,
278}
279
280#[derive(Clone)]
282pub struct DurableGraphDriver {
283 activity: crate::GraphExecutionActivity,
284 store: PostgresStore,
285 registry: ExecutableGraphRegistry,
286 journal_schema: stateknot_core::SchemaReference,
287 options: DurableGraphDriverOptions,
288}
289
290impl DurableGraphDriver {
291 pub fn new(
302 store: PostgresStore,
303 registry: ExecutableGraphRegistry,
304 options: DurableGraphDriverOptions,
305 ) -> Result<Self, DurableGraphDriverBuildError> {
306 let (journal_schema, _) = standard_graph_driver_event_schema()?;
307 if !registry.schemas().contains(&journal_schema) {
308 return Err(DurableGraphDriverBuildError::JournalSchemaUnavailable);
309 }
310 let lease_duration = store.options().lease_duration();
311 let renewal_budget = options
312 .lease_renewal_interval
313 .checked_mul(3)
314 .ok_or(DurableGraphDriverBuildError::UnsafeLeaseRenewalCadence)?;
315 if renewal_budget > lease_duration
316 || options.lease_renewal_interval > store.options().maximum_lease_horizon()
317 {
318 return Err(DurableGraphDriverBuildError::UnsafeLeaseRenewalCadence);
319 }
320 Ok(Self {
321 activity: crate::GraphExecutionActivity::default(),
322 store,
323 registry,
324 journal_schema,
325 options,
326 })
327 }
328
329 pub fn execution_activity(&self) -> crate::GraphExecutionActivity {
331 self.activity.clone()
332 }
333
334 #[must_use]
336 pub const fn options(&self) -> DurableGraphDriverOptions {
337 self.options
338 }
339
340 pub fn drive(
361 &self,
362 fence: RunFence,
363 shutdown: CancellationSignal,
364 ) -> BoxFuture<'_, Result<GraphDriveResult, GraphDriverError>> {
365 Box::pin(self.drive_inner(fence, shutdown))
366 }
367
368 #[allow(clippy::too_many_lines)]
369 async fn drive_inner(
370 &self,
371 fence: RunFence,
372 shutdown: CancellationSignal,
373 ) -> Result<GraphDriveResult, GraphDriverError> {
374 let (startup_renewals, startup_retries) = self.refresh_near_expiry_lease(&fence).await?;
379 let initial_recovery = self.begin_recovery(&fence).await?;
380 let stored_graph = initial_recovery.load_pinned_graph().await?;
381 let graph_reference = stored_graph.graph().reference();
382 let executable = self
383 .registry
384 .resolve(&graph_reference)
385 .cloned()
386 .ok_or_else(|| GraphDriverError::ExecutableGraphUnavailable {
387 graph: Box::new(graph_reference),
388 })?;
389 let replay = initial_recovery
390 .validate_noninitial_replay(
391 executable.schemas(),
392 executable.reducer(),
393 self.options.replay_limits,
394 )
395 .await?;
396 drop(initial_recovery);
397
398 let mut report = GraphDriveReport::new(replay);
399 report.lease_renewals = startup_renewals;
400 report.mutation_retries = startup_retries;
401 loop {
402 if shutdown.is_cancelled() {
403 let release = self.release_with_retry(&fence, &mut report).await?;
404 return Ok(GraphDriveResult::new(
405 GraphDriveOutcome::Cancelled { release },
406 report,
407 ));
408 }
409 if report.durable_events >= self.options.maximum_durable_events {
410 let release = self.release_with_retry(&fence, &mut report).await?;
411 return Ok(GraphDriveResult::new(
412 GraphDriveOutcome::Yielded { release },
413 report,
414 ));
415 }
416
417 let recovery = self.begin_recovery(&fence).await?;
418 let plan = recovery.plan_ready_nodes().await?;
419 let live = recovery.revalidate().await?;
420 if live.lifecycle().status() == RunStatus::CancellationRequested {
421 let lease = exact_live_lease(&live, &fence)?.clone();
422 let request = live.lifecycle().cancellation_request().ok_or(
423 GraphDriverError::RuntimeInvariant {
424 operation: "recover durable cancellation intent",
425 },
426 )?;
427 let checkpoint = plan.checkpoint().head();
428 let pointer = live
429 .checkpoint()
430 .ok_or(GraphDriverError::RuntimeInvariant {
431 operation: "bind cancellation to the current checkpoint",
432 })?;
433 if pointer.checkpoint_id() != checkpoint.checkpoint_id()
434 || pointer.superstep() != checkpoint.superstep()
435 || pointer.digest() != checkpoint.digest()
436 || live.journal_head() != Some(plan.journal_head())
437 {
438 return Err(GraphDriverError::StaleBarrierSnapshot);
439 }
440 return Ok(GraphDriveResult::new(
441 GraphDriveOutcome::CancellationRequested(Box::new(GraphCancellationHandoff {
442 checkpoint,
443 journal_head: plan.journal_head().clone(),
444 event_id: EventId::generate(),
445 expected_revision: live.lifecycle().revision(),
446 lease,
447 failure_id: request.failure().id(),
448 })),
449 report,
450 ));
451 }
452 let mut blockers = GraphDriveBlockers::from_plan(&plan);
453 blockers.superstep_limit_reached = plan.checkpoint().superstep().get()
457 >= executable.graph().limits().maximum_supersteps().get();
458 if !blockers.is_empty() {
459 let lease = exact_live_lease(&live, &fence)?.clone();
460 return Ok(GraphDriveResult::new(
461 GraphDriveOutcome::Blocked(Box::new(GraphBlockedHandoff {
462 plan,
463 lease,
464 event_id: EventId::generate(),
465 expected_revision: live.lifecycle().revision(),
466 blockers,
467 })),
468 report,
469 ));
470 }
471
472 if plan.is_barrier_ready() {
473 if report.durable_events == self.options.maximum_durable_events {
474 drop(recovery);
475 let release = self.release_with_retry(&fence, &mut report).await?;
476 return Ok(GraphDriveResult::new(
477 GraphDriveOutcome::Yielded { release },
478 report,
479 ));
480 }
481 let results = self.load_barrier_results(&recovery, &plan).await?;
482 let barrier_plan = executable
483 .graph()
484 .plan_barrier(
485 plan.checkpoint(),
486 &results,
487 CheckpointId::generate(),
488 executable.schemas(),
489 executable.reducer(),
490 )
491 .map_err(GraphDriverError::graph_plan)?;
492 let lease = exact_live_lease(&live, &fence)?.clone();
493 match barrier_plan.disposition() {
494 GraphBarrierDisposition::Continue => {
495 drop(recovery);
496 self.commit_continue_barrier(&fence, &plan, barrier_plan, &mut report)
497 .await?;
498 }
499 GraphBarrierDisposition::Wait { .. }
500 | GraphBarrierDisposition::Terminal { .. } => {
501 return Ok(GraphDriveResult::new(
502 GraphDriveOutcome::LifecycleBarrierReady(Box::new(
503 GraphLifecycleBarrierHandoff {
504 plan: barrier_plan,
505 journal_head: plan.journal_head().clone(),
506 event_id: EventId::generate(),
507 expected_revision: live.lifecycle().revision(),
508 lease,
509 },
510 )),
511 report,
512 ));
513 }
514 _ => {
515 return Err(GraphDriverError::RuntimeInvariant {
516 operation: "handle unsupported graph barrier disposition",
517 });
518 }
519 }
520 continue;
521 }
522
523 if let Some(not_before) = plan.earliest_deferred_at() {
524 if plan.nodes().iter().all(|node| {
525 matches!(
526 node.kind(),
527 RecoveryNodeKind::Completed | RecoveryNodeKind::Deferred
528 )
529 }) {
530 let outcome = self
531 .schedule_deferred_with_retry(&plan, &mut report)
532 .await?;
533 if matches!(outcome, DelayedRetryScheduleOutcome::Due { .. }) {
534 continue;
535 }
536 return Ok(GraphDriveResult::new(
537 GraphDriveOutcome::Deferred {
538 not_before,
539 schedule: outcome,
540 },
541 report,
542 ));
543 }
544 }
545
546 if report.durable_events.saturating_add(2) > self.options.maximum_durable_events {
547 drop(recovery);
548 let release = self.release_with_retry(&fence, &mut report).await?;
549 return Ok(GraphDriveResult::new(
550 GraphDriveOutcome::Yielded { release },
551 report,
552 ));
553 }
554 let first_node = plan
555 .nodes()
556 .iter()
557 .find(|node| node.kind() == RecoveryNodeKind::Dispatchable)
558 .ok_or(GraphDriverError::RuntimeInvariant {
559 operation: "select dispatchable graph node",
560 })?;
561 let first_node_id = first_node.activation().node_id().clone();
562 let first_executor = Self::resolve_node_executor(&executable, &first_node_id)?;
563 let batch_limit = usize::from(self.options.maximum_parallel_nodes)
564 .min(usize::from(
565 executable.graph().limits().maximum_parallelism(),
566 ))
567 .min(
568 usize::try_from(
569 (self
570 .options
571 .maximum_durable_events
572 .saturating_sub(report.durable_events))
573 / 2,
574 )
575 .unwrap_or(usize::MAX),
576 );
577 let first_scheduling = first_executor.scheduling();
578 let mut selected = Vec::with_capacity(batch_limit);
579 selected.push((first_node_id, first_executor));
580 if first_scheduling == GraphNodeScheduling::JournalIsolated && batch_limit > 1 {
581 for node in plan
582 .nodes()
583 .iter()
584 .filter(|node| node.kind() == RecoveryNodeKind::Dispatchable)
585 .skip(1)
586 .take(batch_limit.saturating_sub(1))
587 {
588 let node_id = node.activation().node_id().clone();
589 let executor = Self::resolve_node_executor(&executable, &node_id)?;
590 if executor.scheduling() != GraphNodeScheduling::JournalIsolated {
591 break;
592 }
593 selected.push((node_id, executor));
594 }
595 }
596
597 let mut journal_head = plan.journal_head().clone();
602 let mut started = Vec::with_capacity(selected.len());
603 for (node_id, executor) in selected {
604 let attempt_id = AttemptId::generate();
605 let start_event_id = EventId::generate();
606 let payload = self.node_started_payload(
607 &executable.graph().reference(),
608 plan.checkpoint(),
609 &node_id,
610 attempt_id,
611 )?;
612 let append = worker_append(&fence, journal_head.clone(), start_event_id, payload)?;
613 let start = self
614 .start_node_with_retry(append, &plan, &node_id, attempt_id, &mut report)
615 .await?;
616 match start {
617 NodeAttemptCommitOutcome::Committed { event, attempt } => {
618 report.durable_events = report.durable_events.saturating_add(1);
619 report.node_attempts_started =
620 report.node_attempts_started.saturating_add(1);
621 journal_head = event.head();
622 started.push(StartedNode {
623 executor,
624 start: attempt.start().head(),
625 });
626 }
627 NodeAttemptCommitOutcome::Idempotent { .. } => {
628 report.durable_events = report.durable_events.saturating_add(1);
632 break;
633 }
634 _ => {
635 return Err(GraphDriverError::RuntimeInvariant {
636 operation: "handle unsupported node attempt start outcome",
637 });
638 }
639 }
640 }
641 if started.is_empty() {
642 continue;
643 }
644 let checkpoint = Arc::new(plan.checkpoint().clone());
645 drop(recovery);
646
647 let current_lease = self.prepare_execution_lease(&fence, &mut report).await?;
654
655 report.node_batches_started = report.node_batches_started.saturating_add(1);
656 report.maximum_nodes_in_batch = report
657 .maximum_nodes_in_batch
658 .max(u16::try_from(started.len()).unwrap_or(u16::MAX));
659 let execution = self
660 .execute_started_nodes(
661 &started,
662 checkpoint,
663 current_lease,
664 shutdown.clone(),
665 &mut report,
666 )
667 .await?;
668 match execution {
669 StartedNodesExecution::Cancelled => {
670 let release = self.release_with_retry(&fence, &mut report).await?;
671 return Ok(GraphDriveResult::new(
672 GraphDriveOutcome::Cancelled { release },
673 report,
674 ));
675 }
676 StartedNodesExecution::RunCancellationObserved => {}
677 StartedNodesExecution::Finished(results) => {
678 for (started, result) in started.iter().zip(results) {
679 match self
680 .commit_node_execution(
681 &fence,
682 &executable,
683 started,
684 result,
685 &mut report,
686 )
687 .await?
688 {
689 NodeExecutionCommit::RunCancellationObserved => break,
690 NodeExecutionCommit::Committed => {}
691 NodeExecutionCommit::ChildJoin(request) => {
692 return Ok(GraphDriveResult::new(
693 GraphDriveOutcome::ChildJoin(request),
694 report,
695 ));
696 }
697 }
698 }
699 }
700 }
701 }
702 }
703
704 async fn begin_recovery<'driver>(
705 &'driver self,
706 fence: &RunFence,
707 ) -> Result<ClaimedRunRecovery<'driver>, GraphDriverError> {
708 let run = self
709 .store
710 .load_run(fence.tenant_id(), fence.run_id())
711 .await?;
712 exact_live_lease(&run, fence)?;
713 let head = run
714 .journal_head()
715 .cloned()
716 .ok_or(GraphDriverError::MissingJournalHead)?;
717 let evidence_digest = recovery_evidence_digest(&run, fence)?;
718 let context = CorruptionQuarantineContext::new(
719 fence.tenant_id().clone(),
720 fence.run_id(),
721 QuarantineId::generate(),
722 JournalExpectation::exact(head),
723 evidence_digest,
724 )?;
725 self.store
726 .begin_claimed_run_recovery(fence.clone(), context)
727 .await
728 .map_err(GraphDriverError::from)
729 }
730
731 async fn refresh_near_expiry_lease(
732 &self,
733 fence: &RunFence,
734 ) -> Result<(u32, u32), GraphDriverError> {
735 let observation = self.store.observe_live_lease(fence).await?;
736 let remaining_micros = observation
737 .lease()
738 .expires_at()
739 .unix_micros()
740 .checked_sub(observation.observed_at().unix_micros())
741 .and_then(|value| u64::try_from(value).ok())
742 .ok_or(StoreError::LeaseExpired)?;
743 let remaining = Duration::from_micros(remaining_micros);
744 if remaining > self.store.options().lease_duration() / 2 {
745 return Ok((0, 0));
746 }
747
748 let desired = extend_timestamp(
749 observation.observed_at(),
750 self.store.options().lease_duration(),
751 )?;
752 let mut attempt = 1_u8;
753 let mut retries = 0_u32;
754 loop {
755 match self.store.renew_lease(fence, desired).await {
756 Ok(LeaseRenewalOutcome::Renewed(_) | LeaseRenewalOutcome::Idempotent(_)) => {
757 return Ok((1, retries));
758 }
759 Ok(_) => {
760 return Err(GraphDriverError::RuntimeInvariant {
761 operation: "refresh a near-expiry drive lease",
762 });
763 }
764 Err(error) if self.can_retry_mutation(&error, attempt) => {
765 retries = retries.saturating_add(1);
766 self.mutation_backoff(attempt).await;
767 attempt = attempt.saturating_add(1);
768 }
769 Err(error) => return Err(error.into()),
770 }
771 }
772 }
773
774 async fn load_barrier_results(
775 &self,
776 recovery: &ClaimedRunRecovery<'_>,
777 plan: &ReadyNodeRecoveryPlan,
778 ) -> Result<Vec<PendingNodeResult>, GraphDriverError> {
779 let page_size = PendingNodeResultPageSize::new(PendingNodeResultPageSize::MAX)?;
780 let mut cursor = None;
781 let mut results = Vec::with_capacity(plan.nodes().len());
782 let mut compact_bytes = 0_usize;
783 loop {
784 let page = recovery
785 .load_unconsumed_pending_node_result_page(
786 &plan.checkpoint().head(),
787 cursor.as_ref(),
788 page_size,
789 )
790 .await?;
791 if page.snapshot_journal_head() != plan.journal_head() {
792 return Err(GraphDriverError::StaleBarrierSnapshot);
793 }
794 let next = if page.has_more() {
795 Some(
796 page.next_cursor()
797 .ok_or(GraphDriverError::RuntimeInvariant {
798 operation: "advance graph barrier result cursor",
799 })?,
800 )
801 } else {
802 None
803 };
804 for result in page.records() {
805 let mut counter = CompactByteCounter::default();
806 serde_json::to_writer(&mut counter, result)
807 .map_err(|_| GraphDriverError::BarrierResultEncoding)?;
808 compact_bytes = compact_bytes
809 .checked_add(counter.bytes)
810 .ok_or(GraphDriverError::BarrierResultResourceLimit)?;
811 if compact_bytes > self.options.replay_limits.maximum_barrier_result_bytes() {
812 return Err(GraphDriverError::BarrierResultResourceLimit);
813 }
814 }
815 results.extend(page.into_records());
816 let Some(next) = next else {
817 break;
818 };
819 cursor = Some(next);
820 }
821 if results.len() != plan.nodes().len() {
822 return Err(GraphDriverError::RuntimeInvariant {
823 operation: "materialize complete graph barrier result set",
824 });
825 }
826 Ok(results)
827 }
828
829 async fn commit_continue_barrier(
830 &self,
831 fence: &RunFence,
832 ready_plan: &ReadyNodeRecoveryPlan,
833 barrier_plan: GraphBarrierPlan,
834 report: &mut GraphDriveReport,
835 ) -> Result<(), GraphDriverError> {
836 if !matches!(
837 barrier_plan.disposition(),
838 GraphBarrierDisposition::Continue
839 ) {
840 return Err(GraphDriverError::RuntimeInvariant {
841 operation: "commit non-continue graph barrier",
842 });
843 }
844 let payload = self.barrier_continued_payload(
845 ready_plan.checkpoint().graph(),
846 ready_plan.checkpoint(),
847 barrier_plan.barrier().successor().checkpoint_id(),
848 barrier_plan.barrier().successor().superstep(),
849 )?;
850 let append = worker_append(
851 fence,
852 ready_plan.journal_head().clone(),
853 EventId::generate(),
854 payload,
855 )?;
856 let barrier = barrier_plan.barrier().clone();
857 let mut attempt = 1_u8;
858 loop {
859 let result = self
860 .store
861 .append_worker_barrier(append.clone(), RunProjection::unchanged(), barrier.clone())
862 .await;
863 match result {
864 Ok(
865 BarrierCommitOutcome::Committed { .. }
866 | BarrierCommitOutcome::Idempotent { .. },
867 ) => {
868 report.durable_events = report.durable_events.saturating_add(1);
869 report.barriers_committed = report.barriers_committed.saturating_add(1);
870 return Ok(());
871 }
872 Ok(_) => {
873 return Err(GraphDriverError::RuntimeInvariant {
874 operation: "handle unsupported graph barrier commit outcome",
875 });
876 }
877 Err(error) if self.can_retry_mutation(&error, attempt) => {
878 report.mutation_retries = report.mutation_retries.saturating_add(1);
879 self.mutation_backoff(attempt).await;
880 attempt = attempt.saturating_add(1);
881 }
882 Err(error) => return Err(error.into()),
883 }
884 }
885 }
886
887 fn resolve_node_executor(
888 executable: &ExecutableGraph,
889 node_id: &stateknot_core::NodeId,
890 ) -> Result<Arc<dyn crate::GraphNodeExecutor>, GraphDriverError> {
891 executable.node_executor(node_id).ok_or_else(|| {
892 GraphDriverError::ExecutableNodeUnavailable {
893 graph: Box::new(executable.graph().reference()),
894 node_id: node_id.clone(),
895 }
896 })
897 }
898
899 #[allow(clippy::too_many_lines)]
900 async fn execute_started_nodes(
901 &self,
902 started: &[StartedNode],
903 checkpoint: Arc<Checkpoint>,
904 lease: GuardedRunLease,
905 shutdown: CancellationSignal,
906 report: &mut GraphDriveReport,
907 ) -> Result<StartedNodesExecution, GraphDriverError> {
908 if shutdown.is_cancelled() {
909 return Ok(StartedNodesExecution::Cancelled);
910 }
911 if started.is_empty() {
912 return Err(GraphDriverError::RuntimeInvariant {
913 operation: "execute an empty graph node batch",
914 });
915 }
916
917 let mut tasks = JoinSet::new();
918 let mut cancellations = Vec::with_capacity(started.len());
919 for (index, started_node) in started.iter().enumerate() {
920 let cancellation = DriverCancellation::new();
921 let context = GraphNodeContext::new(
922 started_node.start.clone(),
923 Arc::clone(&checkpoint),
924 cancellation.signal(),
925 )?;
926 let executor = Arc::clone(&started_node.executor);
927 let store = self.store.clone();
928 let registry = self.registry.clone();
929 let has_children = registry
930 .resolve(executor.graph())
931 .is_some_and(|graph| graph.graph().child_runs().is_some());
932 let task_cancellation = cancellation.clone();
933 let timeout = self.options.node_execution_timeout;
934 cancellations.push(cancellation);
935 tasks.spawn(self.activity.track(async move {
936 let executing = AtomicBool::new(false);
937 let executing_marker = &executing;
938 let execution = AssertUnwindSafe(async move {
942 let mut context = context;
943 if has_children {
944 if let Some(record) = store.load_child_join(context.attempt().activation()).await? {
945 if !executor.supports_child_join() {
946 return Err(StoreError::ChildJoinRejected.into());
947 }
948 context = context.with_child_join(crate::GraphChildJoin::prepare(store, registry, &record).await?);
949 }
950 }
951 let child_join = context.child_join().map(|join| Box::new(join.head().clone()));
952 executing_marker.store(true, Ordering::Release);
953 let result = executor.execute(context).await;
954 Ok(FinishedNodeExecution { result, child_join })
955 }).catch_unwind();
956 tokio::pin!(execution);
957 let deadline = tokio::time::sleep(timeout);
958 tokio::pin!(deadline);
959 let outcome = tokio::select! {
960 result = &mut execution => match result {
961 Ok(result) => result,
962 Err(_) if !executing.load(Ordering::Acquire) => Err(GraphDriverError::ChildJoinPreparationIncomplete),
963 Err(_) => DurableGraphDriver::node_panic_failure().map(FinishedNodeExecution::without_join),
964 },
965 () = &mut deadline => {
966 task_cancellation.cancel();
967 if executing.load(Ordering::Acquire) {
968 DurableGraphDriver::node_timeout_failure().map(|error| FinishedNodeExecution::without_join(Err(error)))
969 } else {
970 Err(GraphDriverError::ChildJoinPreparationIncomplete)
971 }
972 }
973 };
974 (index, outcome)
975 }));
976 }
977
978 let fence = lease.lease.fence().clone();
979 let lease_deadline = lease.deadline;
980 let maintenance = self.maintain_execution_lease(lease, report);
981 tokio::pin!(maintenance);
982 let durable_cancellation = self.observe_run_cancellation(&fence);
983 tokio::pin!(durable_cancellation);
984 let mut results = std::iter::repeat_with(|| None)
985 .take(started.len())
986 .collect::<Vec<_>>();
987 let mut completed = 0_usize;
988
989 loop {
990 tokio::select! {
991 joined = tasks.join_next(), if completed < started.len() => {
992 let (index, result) = joined
993 .ok_or(GraphDriverError::RuntimeInvariant {
994 operation: "join a complete graph node batch",
995 })?
996 .map_err(|_| GraphDriverError::RuntimeInvariant {
997 operation: "join an unexpectedly cancelled graph node task",
998 })?;
999 let result = result?;
1000 let slot = results.get_mut(index).ok_or(GraphDriverError::RuntimeInvariant {
1001 operation: "place a graph node batch result",
1002 })?;
1003 if slot.replace(result).is_some() {
1004 return Err(GraphDriverError::RuntimeInvariant {
1005 operation: "deduplicate a graph node batch result",
1006 });
1007 }
1008 completed = completed.saturating_add(1);
1009 if completed == started.len() {
1010 let ordered = results
1011 .into_iter()
1012 .collect::<Option<Vec<_>>>()
1013 .ok_or(GraphDriverError::RuntimeInvariant {
1014 operation: "materialize ordered graph node batch results",
1015 })?;
1016 return Ok(StartedNodesExecution::Finished(ordered));
1017 }
1018 }
1019 result = &mut maintenance => {
1020 cancel_node_batch(&cancellations);
1021 abort_node_batch(&mut tasks).await;
1022 return match result {
1023 Ok(()) => Err(GraphDriverError::RuntimeInvariant {
1024 operation: "maintain a live node execution lease",
1025 }),
1026 Err(error) => Err(error),
1027 };
1028 }
1029 () = shutdown.cancelled() => {
1030 cancel_node_batch(&cancellations);
1031 abort_node_batch(&mut tasks).await;
1032 return Ok(StartedNodesExecution::Cancelled);
1033 }
1034 result = &mut durable_cancellation => {
1035 cancel_node_batch(&cancellations);
1036 if let Err(error) = result {
1037 abort_node_batch(&mut tasks).await;
1038 return Err(error);
1039 }
1040 let grace_deadline = Instant::now()
1041 .checked_add(self.options.cancellation_grace_period)
1042 .unwrap_or(lease_deadline)
1043 .min(lease_deadline);
1044 drain_node_batch_until(&mut tasks, grace_deadline).await;
1045 return Ok(StartedNodesExecution::RunCancellationObserved);
1046 }
1047 }
1048 }
1049 }
1050
1051 async fn observe_run_cancellation(&self, fence: &RunFence) -> Result<(), GraphDriverError> {
1052 loop {
1053 let run = self
1054 .store
1055 .load_run(fence.tenant_id(), fence.run_id())
1056 .await?;
1057 exact_live_lease(&run, fence)?;
1058 match run.lifecycle().status() {
1059 RunStatus::CancellationRequested => return Ok(()),
1060 RunStatus::Pending | RunStatus::Active => {}
1061 _ => return Err(StoreError::RunNotRunnable.into()),
1062 }
1063 tokio::time::sleep(self.options.cancellation_poll_interval).await;
1064 }
1065 }
1066
1067 async fn prepare_execution_lease(
1068 &self,
1069 fence: &RunFence,
1070 report: &mut GraphDriveReport,
1071 ) -> Result<GuardedRunLease, GraphDriverError> {
1072 let observation_started = Instant::now();
1073 let recovery = self.begin_recovery(fence).await?;
1074 let lease = exact_live_lease(recovery.initial_run(), fence)?.clone();
1075 let observed_at = recovery.initial_observed_at();
1076 let desired = extend_timestamp(observed_at, self.store.options().lease_duration())?;
1077 drop(recovery);
1078
1079 if desired <= lease.expires_at() {
1080 return guarded_run_lease(lease, observed_at, observation_started);
1081 }
1082 let lease = self.renew_with_retry(fence, desired, report).await?;
1083 report.lease_renewals = report.lease_renewals.saturating_add(1);
1084 Ok(lease)
1085 }
1086
1087 async fn maintain_execution_lease(
1088 &self,
1089 mut lease: GuardedRunLease,
1090 report: &mut GraphDriveReport,
1091 ) -> Result<(), GraphDriverError> {
1092 loop {
1093 let renewal_due = Instant::now()
1094 .checked_add(self.options.lease_renewal_interval)
1095 .ok_or(GraphDriverError::LeaseTimestampOverflow)?;
1096 tokio::select! {
1097 () = tokio::time::sleep_until(renewal_due) => {}
1098 () = tokio::time::sleep_until(lease.deadline) => {
1099 return Err(StoreError::LeaseExpired.into());
1100 }
1101 }
1102
1103 let desired = extend_timestamp(
1104 lease.lease.expires_at(),
1105 self.options.lease_renewal_interval,
1106 )?;
1107 let fence = lease.lease.fence().clone();
1108 let deadline = lease.deadline;
1109 let renewed = {
1110 let renewal = self.renew_with_retry(&fence, desired, report);
1111 tokio::pin!(renewal);
1112 tokio::select! {
1113 result = &mut renewal => result?,
1114 () = tokio::time::sleep_until(deadline) => {
1115 return Err(StoreError::LeaseExpired.into());
1116 }
1117 }
1118 };
1119 lease = renewed;
1120 report.lease_renewals = report.lease_renewals.saturating_add(1);
1121 }
1122 }
1123
1124 async fn commit_node_execution(
1125 &self,
1126 fence: &RunFence,
1127 executable: &ExecutableGraph,
1128 started: &StartedNode,
1129 execution: FinishedNodeExecution,
1130 report: &mut GraphDriveReport,
1131 ) -> Result<NodeExecutionCommit, GraphDriverError> {
1132 let start = &started.start;
1133 let run = self
1134 .store
1135 .load_run(fence.tenant_id(), fence.run_id())
1136 .await?;
1137 exact_live_lease(&run, fence)?;
1138 if run.lifecycle().status() == RunStatus::CancellationRequested {
1139 return Ok(NodeExecutionCommit::RunCancellationObserved);
1140 }
1141 let head = run
1142 .journal_head()
1143 .cloned()
1144 .ok_or(GraphDriverError::MissingJournalHead)?;
1145 match execution.result {
1146 Ok(GraphNodeExecution::ChildJoin(request)) => {
1147 if !started.executor.supports_child_join()
1148 || started.executor.scheduling() != GraphNodeScheduling::Exclusive
1149 || execution.child_join.is_some()
1150 || request.activation() != start.activation()
1151 {
1152 return Err(StoreError::ChildJoinRejected.into());
1153 }
1154 Box::pin(self.register_child_join_with_retry(fence, start, &request, report))
1155 .await?;
1156 report.durable_events = report.durable_events.saturating_add(1);
1157 return Ok(NodeExecutionCommit::ChildJoin(request));
1160 }
1161 Ok(GraphNodeExecution::Completed {
1162 state_change,
1163 control,
1164 bindings,
1165 usage,
1166 }) => {
1167 let mut intent = PendingNodeResultIntent::new(
1168 start.activation().clone(),
1169 state_change,
1170 control,
1171 bindings,
1172 )
1173 .map_err(|_| GraphDriverError::InvalidNodeResult)?;
1174 if let Some(head) = execution.child_join {
1175 intent = intent
1176 .with_child_join(*head)
1177 .map_err(|_| GraphDriverError::InvalidNodeResult)?;
1178 }
1179 let payload = self.node_succeeded_payload(
1180 &executable.graph().reference(),
1181 start,
1182 intent.intent_digest(),
1183 )?;
1184 let append = worker_append(fence, head, EventId::generate(), payload)?;
1185 if let Err(error) = self
1186 .succeed_node_with_retry(append, start, intent, usage, report)
1187 .await
1188 {
1189 if graph_error_is_run_not_runnable(&error)
1190 && self.run_cancellation_requested(fence).await?
1191 {
1192 return Ok(NodeExecutionCommit::RunCancellationObserved);
1193 }
1194 return Err(error);
1195 }
1196 }
1197 Err(execution) => {
1198 let (failure, usage) = execution.into_parts();
1199 let event_id = EventId::generate();
1200 let payload =
1201 self.node_failed_payload(&executable.graph().reference(), start, failure.id())?;
1202 let append = worker_append(fence, head, event_id, payload)?;
1203 let failure = failure.with_caused_by_event(event_id);
1204 if let Err(error) = self
1205 .fail_node_with_retry(append, start, failure, usage, report)
1206 .await
1207 {
1208 if graph_error_is_run_not_runnable(&error)
1209 && self.run_cancellation_requested(fence).await?
1210 {
1211 return Ok(NodeExecutionCommit::RunCancellationObserved);
1212 }
1213 return Err(error);
1214 }
1215 }
1216 }
1217 report.durable_events = report.durable_events.saturating_add(1);
1218 report.node_attempts_completed = report.node_attempts_completed.saturating_add(1);
1219 Ok(NodeExecutionCommit::Committed)
1220 }
1221
1222 async fn register_child_join_with_retry(
1223 &self,
1224 fence: &RunFence,
1225 start: &NodeAttemptStartHead,
1226 request: &stateknot_core::ChildRunJoinRequest,
1227 report: &mut GraphDriveReport,
1228 ) -> Result<(), GraphDriverError> {
1229 let id = EventId::generate();
1230 let payload = crate::child_join::payload(self.registry.schemas(), request, false)?;
1231 let mut attempt = 1_u8;
1232 loop {
1233 let result = async {
1236 let run = self
1237 .store
1238 .load_run(fence.tenant_id(), fence.run_id())
1239 .await?;
1240 let head = run
1241 .journal_head()
1242 .cloned()
1243 .ok_or(GraphDriverError::MissingJournalHead)?;
1244 let append = worker_append(fence, head, id, payload.clone())?;
1245 self.store
1246 .register_child_join(request.clone(), start, append)
1247 .await
1248 .map_err(GraphDriverError::from)
1249 }
1250 .await;
1251 match result {
1252 Ok(_) => return Ok(()),
1253 Err(GraphDriverError::Store { source })
1254 if attempt < self.options.maximum_mutation_attempts
1255 && (source.is_retryable()
1256 || matches!(*source, StoreError::StaleJournalHead)) =>
1257 {
1258 report.mutation_retries = report.mutation_retries.saturating_add(1);
1259 self.mutation_backoff(attempt).await;
1260 attempt = attempt.saturating_add(1);
1261 }
1262 Err(error) => return Err(error),
1263 }
1264 }
1265 }
1266
1267 async fn run_cancellation_requested(&self, fence: &RunFence) -> Result<bool, GraphDriverError> {
1268 let run = self
1269 .store
1270 .load_run(fence.tenant_id(), fence.run_id())
1271 .await?;
1272 exact_live_lease(&run, fence)?;
1273 Ok(run.lifecycle().status() == RunStatus::CancellationRequested)
1274 }
1275
1276 async fn start_node_with_retry(
1277 &self,
1278 append: JournalAppend,
1279 plan: &ReadyNodeRecoveryPlan,
1280 node_id: &stateknot_core::NodeId,
1281 attempt_id: AttemptId,
1282 report: &mut GraphDriveReport,
1283 ) -> Result<NodeAttemptCommitOutcome, GraphDriverError> {
1284 let mut attempt = 1_u8;
1285 loop {
1286 match self
1287 .store
1288 .start_recovered_node_attempt(append.clone(), plan, node_id, attempt_id)
1289 .await
1290 {
1291 Ok(outcome) => return Ok(outcome),
1292 Err(error) if self.can_retry_mutation(&error, attempt) => {
1293 report.mutation_retries = report.mutation_retries.saturating_add(1);
1294 self.mutation_backoff(attempt).await;
1295 attempt = attempt.saturating_add(1);
1296 }
1297 Err(error) => return Err(error.into()),
1298 }
1299 }
1300 }
1301
1302 async fn succeed_node_with_retry(
1303 &self,
1304 append: JournalAppend,
1305 start: &NodeAttemptStartHead,
1306 intent: PendingNodeResultIntent,
1307 usage: BudgetUsage,
1308 report: &mut GraphDriveReport,
1309 ) -> Result<NodeAttemptCommitOutcome, GraphDriverError> {
1310 let mut attempt = 1_u8;
1311 loop {
1312 match self
1313 .store
1314 .succeed_node_attempt(append.clone(), start, intent.clone(), usage.clone())
1315 .await
1316 {
1317 Ok(outcome) => return Ok(outcome),
1318 Err(error) if self.can_retry_mutation(&error, attempt) => {
1319 report.mutation_retries = report.mutation_retries.saturating_add(1);
1320 self.mutation_backoff(attempt).await;
1321 attempt = attempt.saturating_add(1);
1322 }
1323 Err(error) => return Err(error.into()),
1324 }
1325 }
1326 }
1327
1328 async fn fail_node_with_retry(
1329 &self,
1330 append: JournalAppend,
1331 start: &NodeAttemptStartHead,
1332 failure: Failure,
1333 usage: BudgetUsage,
1334 report: &mut GraphDriveReport,
1335 ) -> Result<NodeAttemptCommitOutcome, GraphDriverError> {
1336 let mut attempt = 1_u8;
1337 loop {
1338 match self
1339 .store
1340 .fail_node_attempt(append.clone(), start, failure.clone(), usage.clone())
1341 .await
1342 {
1343 Ok(outcome) => return Ok(outcome),
1344 Err(error) if self.can_retry_mutation(&error, attempt) => {
1345 report.mutation_retries = report.mutation_retries.saturating_add(1);
1346 self.mutation_backoff(attempt).await;
1347 attempt = attempt.saturating_add(1);
1348 }
1349 Err(error) => return Err(error.into()),
1350 }
1351 }
1352 }
1353
1354 async fn renew_with_retry(
1355 &self,
1356 fence: &RunFence,
1357 desired: Timestamp,
1358 report: &mut GraphDriveReport,
1359 ) -> Result<GuardedRunLease, GraphDriverError> {
1360 let renewal_started = Instant::now();
1361 let mut attempt = 1_u8;
1362 loop {
1363 match self.store.renew_lease(fence, desired).await {
1364 Ok(LeaseRenewalOutcome::Renewed(lease)) => {
1365 let renewed_at = lease.renewed_at();
1366 return guarded_run_lease(lease, renewed_at, renewal_started);
1367 }
1368 Ok(LeaseRenewalOutcome::Idempotent(_)) => {
1369 let observation_started = Instant::now();
1370 let observation = self.store.observe_live_lease(fence).await?;
1371 let (lease, observed_at) = observation.into_parts();
1372 return guarded_run_lease(lease, observed_at, observation_started);
1373 }
1374 Ok(_) => {
1375 return Err(GraphDriverError::RuntimeInvariant {
1376 operation: "handle a lease renewal outcome",
1377 });
1378 }
1379 Err(error) if self.can_retry_mutation(&error, attempt) => {
1380 report.mutation_retries = report.mutation_retries.saturating_add(1);
1381 self.mutation_backoff(attempt).await;
1382 attempt = attempt.saturating_add(1);
1383 }
1384 Err(error) => return Err(error.into()),
1385 }
1386 }
1387 }
1388
1389 async fn schedule_deferred_with_retry(
1390 &self,
1391 plan: &ReadyNodeRecoveryPlan,
1392 report: &mut GraphDriveReport,
1393 ) -> Result<DelayedRetryScheduleOutcome, GraphDriverError> {
1394 let mut attempt = 1_u8;
1395 loop {
1396 match self.store.schedule_delayed_retry_wakeup(plan).await {
1397 Ok(outcome) => return Ok(outcome),
1398 Err(error) if self.can_retry_mutation(&error, attempt) => {
1399 report.mutation_retries = report.mutation_retries.saturating_add(1);
1400 self.mutation_backoff(attempt).await;
1401 attempt = attempt.saturating_add(1);
1402 }
1403 Err(error) => return Err(error.into()),
1404 }
1405 }
1406 }
1407
1408 async fn release_with_retry(
1409 &self,
1410 fence: &RunFence,
1411 report: &mut GraphDriveReport,
1412 ) -> Result<LeaseReleaseOutcome, GraphDriverError> {
1413 let mut attempt = 1_u8;
1414 loop {
1415 match self.store.release_lease(fence).await {
1416 Ok(outcome) => return Ok(outcome),
1417 Err(error) if self.can_retry_mutation(&error, attempt) => {
1418 report.mutation_retries = report.mutation_retries.saturating_add(1);
1419 self.mutation_backoff(attempt).await;
1420 attempt = attempt.saturating_add(1);
1421 }
1422 Err(error) => return Err(error.into()),
1423 }
1424 }
1425 }
1426
1427 fn can_retry_mutation(&self, error: &StoreError, attempt: u8) -> bool {
1428 attempt < self.options.maximum_mutation_attempts && error.is_retryable()
1429 }
1430
1431 async fn mutation_backoff(&self, attempt: u8) {
1432 let multiplier = 1_u32
1433 .checked_shl(u32::from(attempt.saturating_sub(1)))
1434 .unwrap_or(u32::MAX);
1435 let delay = self
1436 .options
1437 .mutation_retry_initial_delay
1438 .checked_mul(multiplier)
1439 .unwrap_or(MAX_MUTATION_RETRY_DELAY)
1440 .min(MAX_MUTATION_RETRY_DELAY);
1441 tokio::time::sleep(delay).await;
1442 }
1443
1444 fn node_started_payload(
1445 &self,
1446 graph: &GraphReference,
1447 checkpoint: &Checkpoint,
1448 node_id: &stateknot_core::NodeId,
1449 attempt_id: AttemptId,
1450 ) -> Result<JournalPayload, GraphDriverError> {
1451 self.event_payload(
1452 "graph-node-attempt-started",
1453 json!({
1454 "operation": "node_attempt_started",
1455 "graph_digest": digest_hex(graph.definition_digest()),
1456 "checkpoint_id": checkpoint.checkpoint_id().to_string(),
1457 "superstep": checkpoint.superstep().get().to_string(),
1458 "node_id": node_id.as_str(),
1459 "attempt_id": attempt_id.to_string()
1460 }),
1461 )
1462 }
1463
1464 fn node_succeeded_payload(
1465 &self,
1466 graph: &GraphReference,
1467 start: &NodeAttemptStartHead,
1468 result_digest: Digest,
1469 ) -> Result<JournalPayload, GraphDriverError> {
1470 let base = start.activation().base_checkpoint();
1471 self.event_payload(
1472 "graph-node-attempt-succeeded",
1473 json!({
1474 "operation": "node_attempt_succeeded",
1475 "graph_digest": digest_hex(graph.definition_digest()),
1476 "checkpoint_id": base.checkpoint_id().to_string(),
1477 "superstep": base.superstep().get().to_string(),
1478 "node_id": start.activation().node_id().as_str(),
1479 "attempt_id": start.attempt_id().to_string(),
1480 "result_digest": digest_hex(result_digest)
1481 }),
1482 )
1483 }
1484
1485 fn node_failed_payload(
1486 &self,
1487 graph: &GraphReference,
1488 start: &NodeAttemptStartHead,
1489 failure_id: FailureId,
1490 ) -> Result<JournalPayload, GraphDriverError> {
1491 let base = start.activation().base_checkpoint();
1492 self.event_payload(
1493 "graph-node-attempt-failed",
1494 json!({
1495 "operation": "node_attempt_failed",
1496 "graph_digest": digest_hex(graph.definition_digest()),
1497 "checkpoint_id": base.checkpoint_id().to_string(),
1498 "superstep": base.superstep().get().to_string(),
1499 "node_id": start.activation().node_id().as_str(),
1500 "attempt_id": start.attempt_id().to_string(),
1501 "failure_id": failure_id.to_string()
1502 }),
1503 )
1504 }
1505
1506 fn barrier_continued_payload(
1507 &self,
1508 graph: &GraphReference,
1509 checkpoint: &Checkpoint,
1510 successor_id: CheckpointId,
1511 successor_superstep: stateknot_core::Superstep,
1512 ) -> Result<JournalPayload, GraphDriverError> {
1513 self.event_payload(
1514 "graph-barrier-continued",
1515 json!({
1516 "operation": "graph_barrier_continued",
1517 "graph_digest": digest_hex(graph.definition_digest()),
1518 "checkpoint_id": checkpoint.checkpoint_id().to_string(),
1519 "superstep": checkpoint.superstep().get().to_string(),
1520 "successor_checkpoint_id": successor_id.to_string(),
1521 "successor_superstep": successor_superstep.get().to_string(),
1522 "disposition": "continue"
1523 }),
1524 )
1525 }
1526
1527 fn event_payload(
1528 &self,
1529 kind: &'static str,
1530 data: Value,
1531 ) -> Result<JournalPayload, GraphDriverError> {
1532 let data =
1533 BoundedJson::try_from_value(data).map_err(|_| GraphDriverError::EventPayloadInvalid)?;
1534 self.registry
1535 .schemas()
1536 .validate_bounded(&self.journal_schema, &data)
1537 .map_err(|_| GraphDriverError::EventPayloadInvalid)?;
1538 let kind =
1539 JournalEventKind::new(kind).map_err(|_| GraphDriverError::EventPayloadInvalid)?;
1540 JournalPayload::new(self.journal_schema.clone(), kind, data)
1541 .map_err(|_| GraphDriverError::EventPayloadInvalid)
1542 }
1543
1544 fn node_timeout_failure() -> Result<GraphNodeExecutionError, GraphDriverError> {
1545 let failure = runtime_failure(
1546 FailureCategory::DeadlineExceeded,
1547 "runtime.node_timeout",
1548 "graph node execution exceeded its configured deadline",
1549 None::<JoinError>,
1550 )?;
1551 GraphNodeExecutionError::new(failure, BudgetUsage::zero())
1552 .map_err(|_| GraphDriverError::RuntimeFailureInvalid)
1553 }
1554
1555 fn node_panic_failure()
1556 -> Result<Result<GraphNodeExecution, GraphNodeExecutionError>, GraphDriverError> {
1557 let failure = runtime_failure(
1558 FailureCategory::Internal,
1559 "runtime.node_task_panicked",
1560 "graph node execution task panicked",
1561 None::<JoinError>,
1562 )?;
1563 let execution = GraphNodeExecutionError::new(failure, BudgetUsage::zero())
1564 .map_err(|_| GraphDriverError::RuntimeFailureInvalid)?;
1565 Ok(Err(execution))
1566 }
1567}
1568
1569impl fmt::Debug for DurableGraphDriver {
1570 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
1571 formatter
1572 .debug_struct("DurableGraphDriver")
1573 .field("registry", &self.registry)
1574 .field("journal_schema", &self.journal_schema)
1575 .field("options", &self.options)
1576 .finish_non_exhaustive()
1577 }
1578}
1579
1580#[derive(Clone, Debug, Eq, Error, PartialEq)]
1582#[non_exhaustive]
1583pub enum DurableGraphDriverBuildError {
1584 #[error(transparent)]
1586 StandardSchema(#[from] StandardGraphDriverSchemaError),
1587 #[error("standard graph-driver journal schema is not installed")]
1589 JournalSchemaUnavailable,
1590 #[error("graph driver renewal cadence is unsafe for the provider lease timing")]
1592 UnsafeLeaseRenewalCadence,
1593}
1594
1595#[derive(Clone, Copy, Debug, Eq, PartialEq)]
1597pub struct GraphDriveReport {
1598 replay: GraphReplayReport,
1599 durable_events: u32,
1600 node_attempts_started: u32,
1601 node_attempts_completed: u32,
1602 node_batches_started: u32,
1603 maximum_nodes_in_batch: u16,
1604 barriers_committed: u32,
1605 lease_renewals: u32,
1606 mutation_retries: u32,
1607}
1608
1609impl GraphDriveReport {
1610 const fn new(replay: GraphReplayReport) -> Self {
1611 Self {
1612 replay,
1613 durable_events: 0,
1614 node_attempts_started: 0,
1615 node_attempts_completed: 0,
1616 node_batches_started: 0,
1617 maximum_nodes_in_batch: 0,
1618 barriers_committed: 0,
1619 lease_renewals: 0,
1620 mutation_retries: 0,
1621 }
1622 }
1623
1624 #[must_use]
1626 pub const fn replay(self) -> GraphReplayReport {
1627 self.replay
1628 }
1629
1630 #[must_use]
1632 pub const fn durable_events(self) -> u32 {
1633 self.durable_events
1634 }
1635
1636 #[must_use]
1638 pub const fn node_attempts_started(self) -> u32 {
1639 self.node_attempts_started
1640 }
1641
1642 #[must_use]
1644 pub const fn node_attempts_completed(self) -> u32 {
1645 self.node_attempts_completed
1646 }
1647
1648 #[must_use]
1650 pub const fn node_batches_started(self) -> u32 {
1651 self.node_batches_started
1652 }
1653
1654 #[must_use]
1656 pub const fn maximum_nodes_in_batch(self) -> u16 {
1657 self.maximum_nodes_in_batch
1658 }
1659
1660 #[must_use]
1662 pub const fn barriers_committed(self) -> u32 {
1663 self.barriers_committed
1664 }
1665
1666 #[must_use]
1668 pub const fn lease_renewals(self) -> u32 {
1669 self.lease_renewals
1670 }
1671
1672 #[must_use]
1674 pub const fn mutation_retries(self) -> u32 {
1675 self.mutation_retries
1676 }
1677}
1678
1679#[derive(Debug)]
1681pub struct GraphDriveResult {
1682 outcome: GraphDriveOutcome,
1683 report: GraphDriveReport,
1684}
1685
1686impl GraphDriveResult {
1687 const fn new(outcome: GraphDriveOutcome, report: GraphDriveReport) -> Self {
1688 Self { outcome, report }
1689 }
1690
1691 #[must_use]
1693 pub const fn outcome(&self) -> &GraphDriveOutcome {
1694 &self.outcome
1695 }
1696
1697 #[must_use]
1699 pub const fn report(&self) -> GraphDriveReport {
1700 self.report
1701 }
1702
1703 #[must_use]
1705 pub fn into_parts(self) -> (GraphDriveOutcome, GraphDriveReport) {
1706 (self.outcome, self.report)
1707 }
1708}
1709
1710#[derive(Debug)]
1712#[non_exhaustive]
1713pub enum GraphDriveOutcome {
1714 ChildJoin(Box<stateknot_core::ChildRunJoinRequest>),
1717 CancellationRequested(Box<GraphCancellationHandoff>),
1719 LifecycleBarrierReady(Box<GraphLifecycleBarrierHandoff>),
1721 Blocked(Box<GraphBlockedHandoff>),
1723 Deferred {
1725 not_before: Timestamp,
1727 schedule: DelayedRetryScheduleOutcome,
1729 },
1730 Yielded {
1732 release: LeaseReleaseOutcome,
1734 },
1735 Cancelled {
1737 release: LeaseReleaseOutcome,
1739 },
1740}
1741
1742#[derive(Clone, Debug)]
1744pub struct GraphCancellationHandoff {
1745 checkpoint: stateknot_core::CheckpointHead,
1746 journal_head: JournalHead,
1747 event_id: EventId,
1748 expected_revision: RunRevision,
1749 lease: RunLease,
1750 failure_id: FailureId,
1751}
1752
1753impl GraphCancellationHandoff {
1754 #[must_use]
1756 pub const fn checkpoint(&self) -> &stateknot_core::CheckpointHead {
1757 &self.checkpoint
1758 }
1759
1760 #[must_use]
1762 pub const fn journal_head(&self) -> &JournalHead {
1763 &self.journal_head
1764 }
1765
1766 #[must_use]
1768 pub const fn event_id(&self) -> EventId {
1769 self.event_id
1770 }
1771
1772 #[must_use]
1774 pub const fn expected_revision(&self) -> RunRevision {
1775 self.expected_revision
1776 }
1777
1778 #[must_use]
1780 pub const fn lease(&self) -> &RunLease {
1781 &self.lease
1782 }
1783
1784 #[must_use]
1786 pub const fn failure_id(&self) -> FailureId {
1787 self.failure_id
1788 }
1789
1790 #[must_use]
1792 pub fn into_parts(
1793 self,
1794 ) -> (
1795 stateknot_core::CheckpointHead,
1796 JournalHead,
1797 EventId,
1798 RunRevision,
1799 RunLease,
1800 FailureId,
1801 ) {
1802 (
1803 self.checkpoint,
1804 self.journal_head,
1805 self.event_id,
1806 self.expected_revision,
1807 self.lease,
1808 self.failure_id,
1809 )
1810 }
1811}
1812
1813#[derive(Clone, Debug)]
1815pub struct GraphLifecycleBarrierHandoff {
1816 plan: GraphBarrierPlan,
1817 journal_head: JournalHead,
1818 event_id: EventId,
1819 expected_revision: RunRevision,
1820 lease: RunLease,
1821}
1822
1823impl GraphLifecycleBarrierHandoff {
1824 #[must_use]
1826 pub const fn plan(&self) -> &GraphBarrierPlan {
1827 &self.plan
1828 }
1829
1830 #[must_use]
1832 pub const fn journal_head(&self) -> &JournalHead {
1833 &self.journal_head
1834 }
1835
1836 #[must_use]
1838 pub const fn event_id(&self) -> EventId {
1839 self.event_id
1840 }
1841
1842 #[must_use]
1844 pub const fn expected_revision(&self) -> RunRevision {
1845 self.expected_revision
1846 }
1847
1848 #[must_use]
1850 pub const fn lease(&self) -> &RunLease {
1851 &self.lease
1852 }
1853
1854 #[must_use]
1856 pub fn into_parts(
1857 self,
1858 ) -> (
1859 GraphBarrierPlan,
1860 JournalHead,
1861 EventId,
1862 RunRevision,
1863 RunLease,
1864 ) {
1865 (
1866 self.plan,
1867 self.journal_head,
1868 self.event_id,
1869 self.expected_revision,
1870 self.lease,
1871 )
1872 }
1873}
1874
1875#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
1877pub struct GraphDriveBlockers {
1878 in_flight: u16,
1879 failed: u16,
1880 exhausted: u16,
1881 unsupported: u16,
1882 superstep_limit_reached: bool,
1883}
1884
1885impl GraphDriveBlockers {
1886 fn from_plan(plan: &ReadyNodeRecoveryPlan) -> Self {
1887 let mut blockers = Self::default();
1888 for node in plan.nodes() {
1889 match node.kind() {
1890 RecoveryNodeKind::InFlight => {
1891 blockers.in_flight = blockers.in_flight.saturating_add(1);
1892 }
1893 RecoveryNodeKind::Failed => {
1894 blockers.failed = blockers.failed.saturating_add(1);
1895 }
1896 RecoveryNodeKind::Exhausted => {
1897 blockers.exhausted = blockers.exhausted.saturating_add(1);
1898 }
1899 RecoveryNodeKind::Completed
1900 | RecoveryNodeKind::Dispatchable
1901 | RecoveryNodeKind::Deferred => {}
1902 _ => {
1903 blockers.unsupported = blockers.unsupported.saturating_add(1);
1904 }
1905 }
1906 }
1907 blockers
1908 }
1909
1910 #[must_use]
1912 pub const fn in_flight(self) -> u16 {
1913 self.in_flight
1914 }
1915
1916 #[must_use]
1918 pub const fn failed(self) -> u16 {
1919 self.failed
1920 }
1921
1922 #[must_use]
1924 pub const fn exhausted(self) -> u16 {
1925 self.exhausted
1926 }
1927
1928 #[must_use]
1930 pub const fn unsupported(self) -> u16 {
1931 self.unsupported
1932 }
1933
1934 #[must_use]
1939 pub const fn superstep_limit_reached(self) -> bool {
1940 self.superstep_limit_reached
1941 }
1942
1943 #[must_use]
1945 pub const fn is_empty(self) -> bool {
1946 self.in_flight == 0
1947 && self.failed == 0
1948 && self.exhausted == 0
1949 && self.unsupported == 0
1950 && !self.superstep_limit_reached
1951 }
1952}
1953
1954#[derive(Clone, Debug)]
1956pub struct GraphBlockedHandoff {
1957 plan: ReadyNodeRecoveryPlan,
1958 lease: RunLease,
1959 event_id: EventId,
1960 expected_revision: RunRevision,
1961 blockers: GraphDriveBlockers,
1962}
1963
1964impl GraphBlockedHandoff {
1965 #[must_use]
1967 pub const fn plan(&self) -> &ReadyNodeRecoveryPlan {
1968 &self.plan
1969 }
1970
1971 #[must_use]
1973 pub const fn lease(&self) -> &RunLease {
1974 &self.lease
1975 }
1976
1977 #[must_use]
1979 pub const fn event_id(&self) -> EventId {
1980 self.event_id
1981 }
1982
1983 #[must_use]
1985 pub const fn expected_revision(&self) -> RunRevision {
1986 self.expected_revision
1987 }
1988
1989 #[must_use]
1991 pub const fn blockers(&self) -> GraphDriveBlockers {
1992 self.blockers
1993 }
1994
1995 #[must_use]
1997 pub fn into_parts(
1998 self,
1999 ) -> (
2000 ReadyNodeRecoveryPlan,
2001 RunLease,
2002 EventId,
2003 RunRevision,
2004 GraphDriveBlockers,
2005 ) {
2006 (
2007 self.plan,
2008 self.lease,
2009 self.event_id,
2010 self.expected_revision,
2011 self.blockers,
2012 )
2013 }
2014}
2015
2016#[derive(Debug, Error)]
2018#[non_exhaustive]
2019pub enum GraphDriverError {
2020 #[error("child Join preparation did not complete before node dispatch")]
2023 ChildJoinPreparationIncomplete,
2024 #[error(transparent)]
2026 Store {
2027 source: Box<StoreError>,
2029 },
2030 #[error("graph driver requires a non-empty run journal")]
2032 MissingJournalHead,
2033 #[error("pinned graph has no executable deployment binding")]
2035 ExecutableGraphUnavailable {
2036 graph: Box<GraphReference>,
2038 },
2039 #[error("registered graph node executor is unavailable")]
2041 ExecutableNodeUnavailable {
2042 graph: Box<GraphReference>,
2044 node_id: stateknot_core::NodeId,
2046 },
2047 #[error(transparent)]
2049 NodeContext(#[from] GraphNodeContextError),
2050 #[error("graph node returned an invalid durable result")]
2052 InvalidNodeResult,
2053 #[error("graph barrier planning failed: {source}")]
2055 GraphPlanning {
2056 #[source]
2058 source: Box<GraphBarrierPlanError>,
2059 },
2060 #[error("graph barrier results could not be measured")]
2062 BarrierResultEncoding,
2063 #[error("graph barrier results exceed the configured memory limit")]
2065 BarrierResultResourceLimit,
2066 #[error("graph barrier result snapshot became stale")]
2068 StaleBarrierSnapshot,
2069 #[error("graph driver event payload failed its pinned schema")]
2071 EventPayloadInvalid,
2072 #[error("graph driver worker journal append is invalid")]
2074 JournalAppendInvalid,
2075 #[error("graph driver lease extension exceeds canonical time bounds")]
2077 LeaseTimestampOverflow,
2078 #[error("graph driver could not construct runtime failure evidence")]
2080 RuntimeFailureInvalid,
2081 #[error("graph driver invariant failed while attempting to {operation}")]
2083 RuntimeInvariant {
2084 operation: &'static str,
2086 },
2087}
2088
2089impl GraphDriverError {
2090 fn graph_plan(source: GraphBarrierPlanError) -> Self {
2091 Self::GraphPlanning {
2092 source: Box::new(source),
2093 }
2094 }
2095}
2096
2097impl From<StoreError> for GraphDriverError {
2098 fn from(source: StoreError) -> Self {
2099 Self::Store {
2100 source: Box::new(source),
2101 }
2102 }
2103}
2104
2105struct StartedNode {
2106 executor: Arc<dyn crate::GraphNodeExecutor>,
2107 start: NodeAttemptStartHead,
2108}
2109
2110#[allow(clippy::large_enum_variant)]
2111enum StartedNodesExecution {
2112 Finished(Vec<FinishedNodeExecution>),
2113 Cancelled,
2114 RunCancellationObserved,
2115}
2116
2117enum NodeExecutionCommit {
2118 Committed,
2119 RunCancellationObserved,
2120 ChildJoin(Box<stateknot_core::ChildRunJoinRequest>),
2121}
2122
2123struct FinishedNodeExecution {
2124 result: Result<GraphNodeExecution, GraphNodeExecutionError>,
2125 child_join: Option<Box<stateknot_core::ChildRunJoinHead>>,
2126}
2127
2128impl FinishedNodeExecution {
2129 fn without_join(result: Result<GraphNodeExecution, GraphNodeExecutionError>) -> Self {
2130 Self {
2131 result,
2132 child_join: None,
2133 }
2134 }
2135}
2136
2137struct GuardedRunLease {
2138 lease: RunLease,
2139 deadline: Instant,
2140}
2141
2142fn guarded_run_lease(
2143 lease: RunLease,
2144 observed_at: Timestamp,
2145 observation_started: Instant,
2146) -> Result<GuardedRunLease, GraphDriverError> {
2147 let remaining_micros = lease
2148 .expires_at()
2149 .unix_micros()
2150 .checked_sub(observed_at.unix_micros())
2151 .and_then(|value| u64::try_from(value).ok())
2152 .filter(|value| *value > 0)
2153 .ok_or(StoreError::LeaseExpired)?;
2154 let deadline = observation_started
2155 .checked_add(Duration::from_micros(remaining_micros))
2156 .ok_or(GraphDriverError::LeaseTimestampOverflow)?;
2157 if deadline <= Instant::now() {
2158 return Err(StoreError::LeaseExpired.into());
2159 }
2160 Ok(GuardedRunLease { lease, deadline })
2161}
2162
2163fn exact_live_lease<'run>(
2164 run: &'run StoredRun,
2165 fence: &RunFence,
2166) -> Result<&'run RunLease, GraphDriverError> {
2167 if run.is_quarantined() {
2168 return Err(StoreError::RunQuarantined.into());
2169 }
2170 let lease = run.lease().ok_or(StoreError::NoActiveLease)?;
2171 if lease.fence() != fence {
2172 return Err(StoreError::StaleFence.into());
2173 }
2174 Ok(lease)
2175}
2176
2177fn graph_error_is_run_not_runnable(error: &GraphDriverError) -> bool {
2178 matches!(
2179 error,
2180 GraphDriverError::Store { source } if matches!(source.as_ref(), StoreError::RunNotRunnable)
2181 )
2182}
2183
2184fn recovery_evidence_digest(run: &StoredRun, fence: &RunFence) -> Result<Digest, GraphDriverError> {
2185 let head = run
2186 .journal_head()
2187 .ok_or(GraphDriverError::MissingJournalHead)?;
2188 let checkpoint = run
2189 .checkpoint()
2190 .ok_or(StoreError::ReadyNodeRecoveryCheckpointMissing)?;
2191 let mut evidence =
2192 Vec::with_capacity(RECOVERY_EVIDENCE_DOMAIN.len() + Digest::SHA256_LEN * 2 + 16 + 8);
2193 evidence.extend_from_slice(RECOVERY_EVIDENCE_DOMAIN);
2194 evidence.extend_from_slice(head.digest().as_bytes());
2195 evidence.extend_from_slice(checkpoint.digest().as_bytes());
2196 evidence.extend_from_slice(fence.attempt_id().as_uuid().as_bytes());
2197 evidence.extend_from_slice(&fence.epoch().get().to_be_bytes());
2198 Ok(Digest::sha256(evidence))
2199}
2200
2201fn worker_append(
2202 fence: &RunFence,
2203 head: JournalHead,
2204 event_id: EventId,
2205 payload: JournalPayload,
2206) -> Result<JournalAppend, GraphDriverError> {
2207 let intent = JournalEventIntent::worker(
2208 fence.tenant_id().clone(),
2209 fence.run_id(),
2210 event_id,
2211 fence.clone(),
2212 payload,
2213 )
2214 .map_err(|_| GraphDriverError::JournalAppendInvalid)?;
2215 JournalAppend::new(JournalExpectation::exact(head), intent)
2216 .map_err(|_| GraphDriverError::JournalAppendInvalid)
2217}
2218
2219fn extend_timestamp(
2220 timestamp: Timestamp,
2221 duration: Duration,
2222) -> Result<Timestamp, GraphDriverError> {
2223 let micros = i64::try_from(duration.as_micros())
2224 .map_err(|_| GraphDriverError::LeaseTimestampOverflow)?;
2225 let value = timestamp
2226 .unix_micros()
2227 .checked_add(micros)
2228 .ok_or(GraphDriverError::LeaseTimestampOverflow)?;
2229 Timestamp::from_unix_micros(value).map_err(|_| GraphDriverError::LeaseTimestampOverflow)
2230}
2231
2232fn digest_hex(digest: Digest) -> String {
2233 const HEX: &[u8; 16] = b"0123456789abcdef";
2234 let mut value = String::with_capacity(Digest::SHA256_LEN * 2);
2235 for byte in digest.as_bytes() {
2236 value.push(char::from(HEX[usize::from(byte >> 4)]));
2237 value.push(char::from(HEX[usize::from(byte & 0x0f)]));
2238 }
2239 value
2240}
2241
2242fn runtime_failure<E>(
2243 category: FailureCategory,
2244 code: &'static str,
2245 message: &'static str,
2246 source: Option<E>,
2247) -> Result<Failure, GraphDriverError>
2248where
2249 E: std::error::Error + Send + Sync + 'static,
2250{
2251 let code = FailureCode::new(code).map_err(|_| GraphDriverError::RuntimeFailureInvalid)?;
2252 let origin = FailureOrigin::new("stateknot.runtime.graph-driver")
2253 .map_err(|_| GraphDriverError::RuntimeFailureInvalid)?;
2254 let message =
2255 FailureMessage::new(message).map_err(|_| GraphDriverError::RuntimeFailureInvalid)?;
2256 let mut failure = Failure::new(
2257 FailureId::generate(),
2258 category,
2259 code,
2260 origin,
2261 message,
2262 RetryAdvice::Never,
2263 )
2264 .map_err(|_| GraphDriverError::RuntimeFailureInvalid)?;
2265 if let Some(source) = source {
2266 failure = failure.with_private_source(source);
2267 }
2268 Ok(failure)
2269}
2270
2271fn cancel_node_batch(cancellations: &[DriverCancellation]) {
2272 for cancellation in cancellations {
2273 cancellation.cancel();
2274 }
2275}
2276
2277async fn abort_node_batch<T>(tasks: &mut JoinSet<T>)
2278where
2279 T: Send + 'static,
2280{
2281 tasks.abort_all();
2282 while tasks.join_next().await.is_some() {}
2283}
2284
2285async fn drain_node_batch_until<T>(tasks: &mut JoinSet<T>, deadline: Instant)
2286where
2287 T: Send + 'static,
2288{
2289 while !tasks.is_empty() && deadline > Instant::now() {
2290 tokio::select! {
2291 _ = tasks.join_next() => {}
2292 () = tokio::time::sleep_until(deadline) => break,
2293 }
2294 }
2295 if !tasks.is_empty() {
2296 abort_node_batch(tasks).await;
2297 }
2298}
2299
2300#[derive(Default)]
2301struct CompactByteCounter {
2302 bytes: usize,
2303}
2304
2305impl std::io::Write for CompactByteCounter {
2306 fn write(&mut self, buffer: &[u8]) -> std::io::Result<usize> {
2307 self.bytes = self.bytes.saturating_add(buffer.len());
2308 Ok(buffer.len())
2309 }
2310
2311 fn flush(&mut self) -> std::io::Result<()> {
2312 Ok(())
2313 }
2314}
2315
2316#[derive(Clone)]
2317struct DriverCancellation {
2318 state: Arc<DriverCancellationState>,
2319}
2320
2321struct DriverCancellationState {
2322 cancelled: AtomicBool,
2323 notify: Notify,
2324}
2325
2326impl DriverCancellation {
2327 fn new() -> Self {
2328 Self {
2329 state: Arc::new(DriverCancellationState {
2330 cancelled: AtomicBool::new(false),
2331 notify: Notify::new(),
2332 }),
2333 }
2334 }
2335
2336 fn signal(&self) -> CancellationSignal {
2337 CancellationSignal::new(self.clone())
2338 }
2339
2340 fn cancel(&self) {
2341 if !self.state.cancelled.swap(true, Ordering::AcqRel) {
2342 self.state.notify.notify_waiters();
2343 }
2344 }
2345}
2346
2347impl CancellationObserver for DriverCancellation {
2348 fn is_cancelled(&self) -> bool {
2349 self.state.cancelled.load(Ordering::Acquire)
2350 }
2351
2352 fn cancelled(&self) -> BoxFuture<'_, ()> {
2353 Box::pin(async move {
2354 loop {
2355 let notified = self.state.notify.notified();
2356 if self.is_cancelled() {
2357 return;
2358 }
2359 notified.await;
2360 }
2361 })
2362 }
2363}
2364
2365#[cfg(test)]
2366mod tests {
2367 use super::*;
2368
2369 #[test]
2370 fn options_reject_unsafe_local_bounds() {
2371 let limits = GraphReplayLimits::default();
2372 assert!(
2373 DurableGraphDriverOptions::new(
2374 limits,
2375 1,
2376 Duration::from_secs(10),
2377 Duration::from_secs(60),
2378 3,
2379 Duration::from_millis(25),
2380 )
2381 .is_err()
2382 );
2383 assert!(
2384 DurableGraphDriverOptions::new(
2385 limits,
2386 2,
2387 Duration::from_nanos(1),
2388 Duration::from_secs(60),
2389 3,
2390 Duration::from_millis(25),
2391 )
2392 .is_err()
2393 );
2394 assert!(
2395 DurableGraphDriverOptions::new(
2396 limits,
2397 2,
2398 Duration::from_secs(1),
2399 Duration::ZERO,
2400 3,
2401 Duration::from_millis(25),
2402 )
2403 .is_err()
2404 );
2405 let options = DurableGraphDriverOptions::default();
2406 assert!(matches!(
2407 options.with_cancellation_timing(Duration::ZERO, Duration::from_secs(1)),
2408 Err(DurableGraphDriverOptionsError::InvalidCancellationPollInterval)
2409 ));
2410 assert!(matches!(
2411 options.with_cancellation_timing(Duration::from_millis(9), Duration::from_secs(1),),
2412 Err(DurableGraphDriverOptionsError::InvalidCancellationPollInterval)
2413 ));
2414 assert!(matches!(
2415 options.with_cancellation_timing(
2416 DurableGraphDriverOptions::HARD_MINIMUM_CANCELLATION_POLL_INTERVAL,
2417 Duration::from_secs(5 * 60 + 1),
2418 ),
2419 Err(DurableGraphDriverOptionsError::InvalidCancellationGracePeriod)
2420 ));
2421 assert!(matches!(
2422 options.with_parallel_node_limit(0),
2423 Err(DurableGraphDriverOptionsError::InvalidParallelNodeLimit)
2424 ));
2425 assert!(matches!(
2426 options.with_parallel_node_limit(
2427 DurableGraphDriverOptions::HARD_MAXIMUM_PARALLEL_NODES + 1
2428 ),
2429 Err(DurableGraphDriverOptionsError::InvalidParallelNodeLimit)
2430 ));
2431 assert_eq!(
2432 options
2433 .with_parallel_node_limit(4)
2434 .unwrap()
2435 .maximum_parallel_nodes(),
2436 4
2437 );
2438 }
2439
2440 #[tokio::test]
2441 async fn cancellation_is_monotonic_and_race_safe() {
2442 let cancellation = DriverCancellation::new();
2443 let signal = cancellation.signal();
2444 assert!(!signal.is_cancelled());
2445 let waiter = tokio::spawn({
2446 let signal = signal.clone();
2447 async move { signal.cancelled().await }
2448 });
2449 cancellation.cancel();
2450 waiter.await.unwrap();
2451 cancellation.cancel();
2452 assert!(signal.is_cancelled());
2453 signal.cancelled().await;
2454 }
2455
2456 #[test]
2457 fn digest_data_uses_schema_hex_without_algorithm_prefix() {
2458 let value = digest_hex(Digest::sha256(b"driver"));
2459 assert_eq!(value.len(), 64);
2460 assert!(value.bytes().all(|byte| byte.is_ascii_hexdigit()));
2461 assert!(!value.contains(':'));
2462 }
2463}