Skip to main content

stateknot_core/
node_attempt.rs

1// Copyright 2026 StateKnot contributors
2// SPDX-License-Identifier: Apache-2.0
3
4//! Durable physical node-attempt starts, completions, and recovery history.
5//!
6//! A worker must commit [`NodeAttemptStart`] before invoking user node code.
7//! The start owns a fresh physical [`AttemptId`] while [`RunFence`] proves the
8//! worker lease that authorized it. Completion is append-only: success binds
9//! the exact immutable pending result committed by the same worker event, and
10//! failure binds public-safe evidence plus explicit retry advice. A crash may
11//! leave a start without a completion; only a higher fencing epoch may recover
12//! that in-flight attempt.
13
14use std::{collections::BTreeSet, fmt};
15
16use schemars::JsonSchema;
17use serde::{Deserialize, Deserializer, Serialize, de};
18use thiserror::Error;
19
20use crate::{
21    AttemptId, BudgetUsage, Digest, Failure, JournalHead, NodeActivation, PendingNodeResultHead,
22    RetryAdvice, RunFence, Timestamp,
23};
24
25const ACTIVATION_DIGEST_DOMAIN: &[u8] = b"stateknot-node-activation-v1\0";
26const START_DIGEST_DOMAIN: &[u8] = b"stateknot-node-attempt-start-v1\0";
27const COMPLETION_DIGEST_DOMAIN: &[u8] = b"stateknot-node-attempt-completion-v1\0";
28
29/// Durable lifecycle state of one physical node attempt.
30#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, JsonSchema, PartialEq, Serialize)]
31#[serde(rename_all = "snake_case")]
32#[non_exhaustive]
33pub enum NodeAttemptStatus {
34    /// The start committed, but no terminal completion exists.
35    Executing,
36    /// The exact pending node result committed atomically with completion.
37    Succeeded,
38    /// Public-safe failure evidence committed without a logical node result.
39    Failed,
40}
41
42/// Immutable terminal outcome of one physical node attempt.
43#[allow(clippy::large_enum_variant)]
44#[derive(Clone, JsonSchema, Serialize)]
45#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
46#[non_exhaustive]
47pub enum NodeAttemptOutcome {
48    /// The attempt produced one exact immutable pending result.
49    Succeeded {
50        /// Result committed in the same journal transaction as completion.
51        result: Box<PendingNodeResultHead>,
52    },
53    /// The attempt stopped without a logical node result.
54    Failed {
55        /// Public-safe failure evidence with explicit recovery advice.
56        failure: Failure,
57    },
58}
59
60impl NodeAttemptOutcome {
61    /// Returns the terminal lifecycle discriminator.
62    #[must_use]
63    pub const fn status(&self) -> NodeAttemptStatus {
64        match self {
65            Self::Succeeded { .. } => NodeAttemptStatus::Succeeded,
66            Self::Failed { .. } => NodeAttemptStatus::Failed,
67        }
68    }
69
70    /// Returns the successful pending-result head, if present.
71    #[must_use]
72    pub fn result(&self) -> Option<&PendingNodeResultHead> {
73        match self {
74            Self::Succeeded { result } => Some(result.as_ref()),
75            Self::Failed { .. } => None,
76        }
77    }
78
79    /// Returns the public-safe failure, if present.
80    #[must_use]
81    pub const fn failure(&self) -> Option<&Failure> {
82        match self {
83            Self::Succeeded { .. } => None,
84            Self::Failed { failure } => Some(failure),
85        }
86    }
87}
88
89impl fmt::Debug for NodeAttemptOutcome {
90    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
91        match self {
92            Self::Succeeded { result } => formatter
93                .debug_struct("Succeeded")
94                .field("result", result)
95                .finish(),
96            Self::Failed { failure } => formatter
97                .debug_struct("Failed")
98                .field("failure_id", &failure.id())
99                .field("category", &failure.category())
100                .field("code", failure.code())
101                .field("origin", failure.origin())
102                .field("retry_advice", &failure.retry_advice())
103                .finish_non_exhaustive(),
104        }
105    }
106}
107
108impl<'de> Deserialize<'de> for NodeAttemptOutcome {
109    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
110    where
111        D: Deserializer<'de>,
112    {
113        #[allow(clippy::large_enum_variant)]
114        #[derive(Deserialize)]
115        #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
116        enum Wire {
117            Succeeded { result: Box<PendingNodeResultHead> },
118            Failed { failure: Failure },
119        }
120
121        Ok(match Wire::deserialize(deserializer)? {
122            Wire::Succeeded { result } => Self::Succeeded { result },
123            Wire::Failed { failure } => Self::Failed { failure },
124        })
125    }
126}
127
128/// Immutable proof that one physical node execution was durably admitted.
129///
130/// `attempt_id` identifies this node execution and is distinct from the
131/// worker-run attempt carried by `fence`. A production store must claim it in
132/// the run-wide physical-attempt namespace and repeat the exact live fence on
133/// the event and start-row inserts.
134#[derive(Clone, Eq, JsonSchema, PartialEq, Serialize)]
135#[serde(deny_unknown_fields)]
136pub struct NodeAttemptStart {
137    activation: NodeActivation,
138    activation_digest: Digest,
139    attempt_id: AttemptId,
140    fence: RunFence,
141    journal_head: JournalHead,
142    digest: Digest,
143}
144
145impl NodeAttemptStart {
146    /// Constructs an integrity-bound durable start after its event commits.
147    ///
148    /// # Errors
149    ///
150    /// Returns [`NodeAttemptError`] for crossed scope, invalid ordering,
151    /// worker/node attempt identity reuse, or integrity failure.
152    pub fn new(
153        activation: NodeActivation,
154        attempt_id: AttemptId,
155        fence: RunFence,
156        journal_head: JournalHead,
157    ) -> Result<Self, NodeAttemptError> {
158        validate_start_scope(&activation, attempt_id, &fence, &journal_head)?;
159        let activation_digest = compute_activation_digest(&activation)?;
160        let digest = compute_start_digest(&NodeAttemptStartDigestWire {
161            activation_digest,
162            attempt_id,
163            fence: &fence,
164            journal_head: &journal_head,
165        })?;
166        Ok(Self {
167            activation,
168            activation_digest,
169            attempt_id,
170            fence,
171            journal_head,
172            digest,
173        })
174    }
175
176    /// Restores and verifies a durable start record.
177    ///
178    /// # Errors
179    ///
180    /// Returns [`NodeAttemptError`] when any invariant or checksum differs.
181    #[allow(clippy::too_many_arguments)]
182    pub fn restore(
183        activation: NodeActivation,
184        activation_digest: Digest,
185        attempt_id: AttemptId,
186        fence: RunFence,
187        journal_head: JournalHead,
188        digest: Digest,
189    ) -> Result<Self, NodeAttemptError> {
190        let restored = Self::new(activation, attempt_id, fence, journal_head)?;
191        if restored.activation_digest != activation_digest {
192            return Err(NodeAttemptError::ActivationDigestMismatch);
193        }
194        if restored.digest != digest {
195            return Err(NodeAttemptError::StartDigestMismatch);
196        }
197        Ok(restored)
198    }
199
200    /// Returns the exact logical activation.
201    #[must_use]
202    pub const fn activation(&self) -> &NodeActivation {
203        &self.activation
204    }
205
206    /// Returns the stable logical-activation fingerprint.
207    #[must_use]
208    pub const fn activation_digest(&self) -> Digest {
209        self.activation_digest
210    }
211
212    /// Returns the unique physical node-attempt identity.
213    #[must_use]
214    pub const fn attempt_id(&self) -> AttemptId {
215        self.attempt_id
216    }
217
218    /// Returns the worker lease that admitted this attempt.
219    #[must_use]
220    pub const fn fence(&self) -> &RunFence {
221        &self.fence
222    }
223
224    /// Returns the exact start journal anchor.
225    #[must_use]
226    pub const fn journal_head(&self) -> &JournalHead {
227        &self.journal_head
228    }
229
230    /// Returns the complete start-record checksum.
231    #[must_use]
232    pub const fn digest(&self) -> Digest {
233        self.digest
234    }
235
236    /// Returns a compact exact reference used by completion records.
237    #[must_use]
238    pub fn head(&self) -> NodeAttemptStartHead {
239        NodeAttemptStartHead {
240            activation: self.activation.clone(),
241            attempt_id: self.attempt_id,
242            fence: self.fence.clone(),
243            journal_head: self.journal_head.clone(),
244            digest: self.digest,
245        }
246    }
247}
248
249impl fmt::Debug for NodeAttemptStart {
250    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
251        formatter
252            .debug_struct("NodeAttemptStart")
253            .field("activation", &self.activation)
254            .field("activation_digest", &self.activation_digest)
255            .field("attempt_id", &self.attempt_id)
256            .field("fence", &self.fence)
257            .field("journal_head", &self.journal_head)
258            .field("digest", &self.digest)
259            .finish_non_exhaustive()
260    }
261}
262
263impl<'de> Deserialize<'de> for NodeAttemptStart {
264    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
265    where
266        D: Deserializer<'de>,
267    {
268        #[derive(Deserialize)]
269        #[serde(deny_unknown_fields)]
270        struct Wire {
271            activation: NodeActivation,
272            activation_digest: Digest,
273            attempt_id: AttemptId,
274            fence: RunFence,
275            journal_head: JournalHead,
276            digest: Digest,
277        }
278
279        let wire = Wire::deserialize(deserializer)?;
280        Self::restore(
281            wire.activation,
282            wire.activation_digest,
283            wire.attempt_id,
284            wire.fence,
285            wire.journal_head,
286            wire.digest,
287        )
288        .map_err(de::Error::custom)
289    }
290}
291
292/// Exact compact identity of one durable node-attempt start.
293#[derive(Clone, Debug, Eq, JsonSchema, PartialEq, Serialize)]
294#[serde(deny_unknown_fields)]
295pub struct NodeAttemptStartHead {
296    activation: NodeActivation,
297    attempt_id: AttemptId,
298    fence: RunFence,
299    journal_head: JournalHead,
300    digest: Digest,
301}
302
303impl NodeAttemptStartHead {
304    /// Restores a compact start identity and verifies its complete checksum.
305    ///
306    /// # Errors
307    ///
308    /// Returns [`NodeAttemptError`] for invalid scope, ordering, or checksum.
309    pub fn restore(
310        activation: NodeActivation,
311        attempt_id: AttemptId,
312        fence: RunFence,
313        journal_head: JournalHead,
314        digest: Digest,
315    ) -> Result<Self, NodeAttemptError> {
316        let activation_digest = compute_activation_digest(&activation)?;
317        Ok(NodeAttemptStart::restore(
318            activation,
319            activation_digest,
320            attempt_id,
321            fence,
322            journal_head,
323            digest,
324        )?
325        .head())
326    }
327
328    /// Returns the exact logical activation.
329    #[must_use]
330    pub const fn activation(&self) -> &NodeActivation {
331        &self.activation
332    }
333
334    /// Returns the physical node-attempt identity.
335    #[must_use]
336    pub const fn attempt_id(&self) -> AttemptId {
337        self.attempt_id
338    }
339
340    /// Returns the worker lease that admitted the attempt.
341    #[must_use]
342    pub const fn fence(&self) -> &RunFence {
343        &self.fence
344    }
345
346    /// Returns the exact start journal anchor.
347    #[must_use]
348    pub const fn journal_head(&self) -> &JournalHead {
349        &self.journal_head
350    }
351
352    /// Returns the exact start-record checksum.
353    #[must_use]
354    pub const fn digest(&self) -> Digest {
355        self.digest
356    }
357}
358
359impl<'de> Deserialize<'de> for NodeAttemptStartHead {
360    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
361    where
362        D: Deserializer<'de>,
363    {
364        #[derive(Deserialize)]
365        #[serde(deny_unknown_fields)]
366        struct Wire {
367            activation: NodeActivation,
368            attempt_id: AttemptId,
369            fence: RunFence,
370            journal_head: JournalHead,
371            digest: Digest,
372        }
373
374        let wire = Wire::deserialize(deserializer)?;
375        validate_start_scope(
376            &wire.activation,
377            wire.attempt_id,
378            &wire.fence,
379            &wire.journal_head,
380        )
381        .map_err(de::Error::custom)?;
382        let activation_digest =
383            compute_activation_digest(&wire.activation).map_err(de::Error::custom)?;
384        let expected = compute_start_digest(&NodeAttemptStartDigestWire {
385            activation_digest,
386            attempt_id: wire.attempt_id,
387            fence: &wire.fence,
388            journal_head: &wire.journal_head,
389        })
390        .map_err(de::Error::custom)?;
391        if expected != wire.digest {
392            return Err(de::Error::custom(NodeAttemptError::StartDigestMismatch));
393        }
394        Ok(Self {
395            activation: wire.activation,
396            attempt_id: wire.attempt_id,
397            fence: wire.fence,
398            journal_head: wire.journal_head,
399            digest: wire.digest,
400        })
401    }
402}
403
404/// Immutable terminal record for one already-started physical node attempt.
405#[derive(Clone, JsonSchema, Serialize)]
406#[serde(deny_unknown_fields)]
407pub struct NodeAttemptCompletion {
408    start: NodeAttemptStartHead,
409    outcome: NodeAttemptOutcome,
410    usage: BudgetUsage,
411    journal_head: JournalHead,
412    digest: Digest,
413}
414
415impl NodeAttemptCompletion {
416    /// Commits success against the exact pending result produced by the start.
417    ///
418    /// The completion and pending result deliberately share one journal head;
419    /// the storage adapter must insert both in the same transaction.
420    ///
421    /// # Errors
422    ///
423    /// Returns [`NodeAttemptError`] for substituted activation, fence, journal,
424    /// or integrity material.
425    pub fn succeed(
426        start: &NodeAttemptStart,
427        result: PendingNodeResultHead,
428        usage: BudgetUsage,
429    ) -> Result<Self, NodeAttemptError> {
430        let journal_head = result.journal_head().clone();
431        Self::materialize(
432            start.head(),
433            NodeAttemptOutcome::Succeeded {
434                result: Box::new(result),
435            },
436            usage,
437            journal_head,
438        )
439    }
440
441    /// Commits a failed attempt with explicit public recovery evidence.
442    ///
443    /// The failure must name `journal_head.event_id()` as its durable cause.
444    /// Reconcile-first failures are rejected because node code may perform
445    /// external effects only through the separate invocation ledgers.
446    ///
447    /// # Errors
448    ///
449    /// Returns [`NodeAttemptError`] for invalid causation, retry semantics,
450    /// scope, ordering, or integrity material.
451    pub fn fail(
452        start: &NodeAttemptStart,
453        failure: Failure,
454        usage: BudgetUsage,
455        journal_head: JournalHead,
456    ) -> Result<Self, NodeAttemptError> {
457        Self::materialize(
458            start.head(),
459            NodeAttemptOutcome::Failed { failure },
460            usage,
461            journal_head,
462        )
463    }
464
465    /// Restores and verifies a durable terminal record.
466    ///
467    /// # Errors
468    ///
469    /// Returns [`NodeAttemptError`] when any local invariant or checksum fails.
470    pub fn restore(
471        start: NodeAttemptStartHead,
472        outcome: NodeAttemptOutcome,
473        usage: BudgetUsage,
474        journal_head: JournalHead,
475        digest: Digest,
476    ) -> Result<Self, NodeAttemptError> {
477        let restored = Self::materialize(start, outcome, usage, journal_head)?;
478        if restored.digest != digest {
479            return Err(NodeAttemptError::CompletionDigestMismatch);
480        }
481        Ok(restored)
482    }
483
484    fn materialize(
485        start: NodeAttemptStartHead,
486        outcome: NodeAttemptOutcome,
487        usage: BudgetUsage,
488        journal_head: JournalHead,
489    ) -> Result<Self, NodeAttemptError> {
490        validate_completion_shape(&start, &outcome, &journal_head)?;
491        let digest = compute_completion_digest(&NodeAttemptCompletionDigestWire {
492            start: &start,
493            outcome: &outcome,
494            usage: &usage,
495            journal_head: &journal_head,
496        })?;
497        Ok(Self {
498            start,
499            outcome,
500            usage,
501            journal_head,
502            digest,
503        })
504    }
505
506    /// Returns the exact start completed by this record.
507    #[must_use]
508    pub const fn start(&self) -> &NodeAttemptStartHead {
509        &self.start
510    }
511
512    /// Returns the terminal outcome.
513    #[must_use]
514    pub const fn outcome(&self) -> &NodeAttemptOutcome {
515        &self.outcome
516    }
517
518    /// Returns the terminal status.
519    #[must_use]
520    pub const fn status(&self) -> NodeAttemptStatus {
521        self.outcome.status()
522    }
523
524    /// Returns the exact usage attributed to this physical attempt.
525    #[must_use]
526    pub const fn usage(&self) -> &BudgetUsage {
527        &self.usage
528    }
529
530    /// Returns the exact terminal journal anchor.
531    #[must_use]
532    pub const fn journal_head(&self) -> &JournalHead {
533        &self.journal_head
534    }
535
536    /// Returns the complete completion-record checksum.
537    #[must_use]
538    pub const fn digest(&self) -> Digest {
539        self.digest
540    }
541}
542
543impl fmt::Debug for NodeAttemptCompletion {
544    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
545        formatter
546            .debug_struct("NodeAttemptCompletion")
547            .field("start", &self.start)
548            .field("outcome", &self.outcome)
549            .field("usage_recorded", &true)
550            .field("journal_head", &self.journal_head)
551            .field("digest", &self.digest)
552            .finish_non_exhaustive()
553    }
554}
555
556impl<'de> Deserialize<'de> for NodeAttemptCompletion {
557    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
558    where
559        D: Deserializer<'de>,
560    {
561        #[derive(Deserialize)]
562        #[serde(deny_unknown_fields)]
563        struct Wire {
564            start: NodeAttemptStartHead,
565            outcome: NodeAttemptOutcome,
566            usage: BudgetUsage,
567            journal_head: JournalHead,
568            digest: Digest,
569        }
570
571        let wire = Wire::deserialize(deserializer)?;
572        Self::restore(
573            wire.start,
574            wire.outcome,
575            wire.usage,
576            wire.journal_head,
577            wire.digest,
578        )
579        .map_err(de::Error::custom)
580    }
581}
582
583/// Fully restored physical node attempt, executing or terminal.
584#[derive(Clone, JsonSchema, Serialize)]
585#[serde(deny_unknown_fields)]
586pub struct NodeAttempt {
587    start: NodeAttemptStart,
588    #[serde(skip_serializing_if = "Option::is_none")]
589    completion: Option<NodeAttemptCompletion>,
590}
591
592impl NodeAttempt {
593    /// Constructs an executing attempt from its durable start.
594    #[must_use]
595    pub const fn executing(start: NodeAttemptStart) -> Self {
596        Self {
597            start,
598            completion: None,
599        }
600    }
601
602    /// Restores an attempt and verifies the exact start/completion join.
603    ///
604    /// # Errors
605    ///
606    /// Returns [`NodeAttemptError::CompletionStartMismatch`] when completion
607    /// belongs to another physical attempt.
608    pub fn restore(
609        start: NodeAttemptStart,
610        completion: Option<NodeAttemptCompletion>,
611    ) -> Result<Self, NodeAttemptError> {
612        if completion
613            .as_ref()
614            .is_some_and(|completion| completion.start() != &start.head())
615        {
616            return Err(NodeAttemptError::CompletionStartMismatch);
617        }
618        Ok(Self { start, completion })
619    }
620
621    /// Returns the immutable durable start.
622    #[must_use]
623    pub const fn start(&self) -> &NodeAttemptStart {
624        &self.start
625    }
626
627    /// Returns the optional terminal completion.
628    #[must_use]
629    pub const fn completion(&self) -> Option<&NodeAttemptCompletion> {
630        self.completion.as_ref()
631    }
632
633    /// Returns the current lifecycle state.
634    #[must_use]
635    pub const fn status(&self) -> NodeAttemptStatus {
636        match &self.completion {
637            Some(completion) => completion.status(),
638            None => NodeAttemptStatus::Executing,
639        }
640    }
641}
642
643impl fmt::Debug for NodeAttempt {
644    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
645        formatter
646            .debug_struct("NodeAttempt")
647            .field("start", &self.start)
648            .field("completion", &self.completion)
649            .finish_non_exhaustive()
650    }
651}
652
653impl<'de> Deserialize<'de> for NodeAttempt {
654    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
655    where
656        D: Deserializer<'de>,
657    {
658        #[derive(Deserialize)]
659        #[serde(deny_unknown_fields)]
660        struct Wire {
661            start: NodeAttemptStart,
662            #[serde(default)]
663            completion: Option<NodeAttemptCompletion>,
664        }
665
666        let wire = Wire::deserialize(deserializer)?;
667        Self::restore(wire.start, wire.completion).map_err(de::Error::custom)
668    }
669}
670
671/// Streaming validator for one logical activation's physical attempts.
672///
673/// Histories are ordered by start journal position. A failed attempt may be
674/// followed only after its explicit safe-after delay; an unfinished attempt
675/// may be replaced only by a higher worker fencing epoch. Rejections never
676/// advance verifier state.
677#[derive(Clone, Debug, Default)]
678pub struct NodeAttemptHistoryVerifier {
679    last: Option<NodeAttempt>,
680    attempt_ids: BTreeSet<AttemptId>,
681}
682
683impl NodeAttemptHistoryVerifier {
684    /// Constructs an empty history verifier.
685    #[must_use]
686    pub const fn new() -> Self {
687        Self {
688            last: None,
689            attempt_ids: BTreeSet::new(),
690        }
691    }
692
693    /// Continues after one trusted full record used as a page cursor.
694    ///
695    /// The durable run-wide attempt registry must still prove that attempts
696    /// preceding the cursor did not reuse the next ID.
697    #[must_use]
698    pub fn after(record: NodeAttempt) -> Self {
699        let attempt_id = record.start.attempt_id;
700        Self {
701            last: Some(record),
702            attempt_ids: BTreeSet::from([attempt_id]),
703        }
704    }
705
706    /// Returns whether at least one attempt has been verified.
707    #[must_use]
708    pub const fn has_records(&self) -> bool {
709        self.last.is_some()
710    }
711
712    /// Returns the last verified full attempt, if present.
713    #[must_use]
714    pub const fn last(&self) -> Option<&NodeAttempt> {
715        self.last.as_ref()
716    }
717
718    /// Verifies and advances to the next physical attempt.
719    ///
720    /// # Errors
721    ///
722    /// Returns [`NodeAttemptHistoryError`] for activation substitution, reused
723    /// physical identity, regressing ownership/journal order, unsafe retry, or
724    /// retry before the durable safe-after delay.
725    pub fn verify_next(&mut self, attempt: &NodeAttempt) -> Result<(), NodeAttemptHistoryError> {
726        let attempt_id = attempt.start.attempt_id;
727        if self.attempt_ids.contains(&attempt_id) {
728            return Err(NodeAttemptHistoryError::AttemptIdReused { attempt_id });
729        }
730
731        if let Some(previous) = self.last.as_ref() {
732            validate_history_successor(previous, attempt)?;
733        }
734
735        self.attempt_ids.insert(attempt_id);
736        self.last = Some(attempt.clone());
737        Ok(())
738    }
739}
740
741/// Invalid physical-attempt history for one logical node activation.
742#[derive(Clone, Debug, Eq, Error, PartialEq)]
743#[non_exhaustive]
744pub enum NodeAttemptHistoryError {
745    /// A physical attempt identity was used twice in the supplied history.
746    #[error("node attempt history reused physical attempt {attempt_id}")]
747    AttemptIdReused {
748        /// Reused identity.
749        attempt_id: AttemptId,
750    },
751    /// A successor changed the logical node activation.
752    #[error("node attempt history changed its logical activation")]
753    ActivationMismatch,
754    /// A successor start did not strictly follow the preceding durable record.
755    #[error("node attempt history journal position did not advance")]
756    JournalNotAfterPrevious,
757    /// A successor durable clock preceded the preceding record.
758    #[error("node attempt history durable clock regressed")]
759    ClockRegression,
760    /// Worker ownership epoch moved backwards.
761    #[error("node attempt history worker fencing epoch regressed")]
762    FenceEpochRegression,
763    /// One epoch named two different worker attempts.
764    #[error("node attempt history changed worker attempt without advancing the epoch")]
765    WorkerChangedWithinEpoch,
766    /// A higher epoch improperly reused an earlier worker attempt identity.
767    #[error("node attempt history reused a worker attempt across fencing epochs")]
768    WorkerReusedAcrossEpoch,
769    /// An unfinished attempt was retried without a higher fencing epoch.
770    #[error("an unfinished node attempt may be recovered only under a higher fencing epoch")]
771    UnfinishedAttemptNotSuperseded,
772    /// A successful logical activation was executed again.
773    #[error("a successful node activation cannot start another physical attempt")]
774    PreviousAttemptSucceeded,
775    /// Failure evidence did not authorize an automatic retry.
776    #[error("node failure recovery advice does not authorize automatic retry")]
777    RetryNotAuthorized {
778        /// Explicit advice that blocked retry.
779        advice: RetryAdvice,
780    },
781    /// A successor start preceded the explicit safe-after delay.
782    #[error("node attempt retry started before its explicit safe-after delay elapsed")]
783    RetryDelayNotElapsed {
784        /// Required delay.
785        delay_millis: i64,
786        /// Failure commit time.
787        failed_at: Timestamp,
788        /// Rejected successor start time.
789        started_at: Timestamp,
790    },
791}
792
793/// Invalid node-attempt scope, causation, lifecycle, or integrity material.
794#[derive(Clone, Debug, Eq, Error, PartialEq)]
795#[non_exhaustive]
796pub enum NodeAttemptError {
797    /// Node physical attempt reused the worker-run attempt identity.
798    #[error("node physical attempt must differ from the worker-run attempt")]
799    WorkerAttemptReused,
800    /// Worker fence crossed the activation tenant boundary.
801    #[error("node attempt fence crosses the activation tenant boundary")]
802    FenceTenantMismatch,
803    /// Worker fence named another run.
804    #[error("node attempt fence does not belong to the activation run")]
805    FenceRunMismatch,
806    /// Journal head crossed the activation tenant boundary.
807    #[error("node attempt journal crosses the activation tenant boundary")]
808    JournalTenantMismatch,
809    /// Journal head named another run.
810    #[error("node attempt journal does not belong to the activation run")]
811    JournalRunMismatch,
812    /// Start journal did not strictly follow the base checkpoint.
813    #[error("node attempt start journal does not follow its base checkpoint")]
814    StartJournalNotAfterBase,
815    /// Start durable time preceded the base checkpoint.
816    #[error("node attempt start clock precedes its base checkpoint")]
817    StartClockRegression,
818    /// Completion journal did not strictly follow its start.
819    #[error("node attempt completion journal does not follow its start")]
820    CompletionJournalNotAfterStart,
821    /// Completion durable time preceded its start.
822    #[error("node attempt completion clock precedes its start")]
823    CompletionClockRegression,
824    /// Successful result belonged to another logical activation.
825    #[error("node attempt success result belongs to another activation")]
826    ResultActivationMismatch,
827    /// Successful result was committed by another worker fence.
828    #[error("node attempt success result uses another worker fence")]
829    ResultFenceMismatch,
830    /// Successful result and attempt completion did not share one journal event.
831    #[error("node attempt success result and completion journal differ")]
832    ResultJournalMismatch,
833    /// Failed attempt did not name its completion event as the direct cause.
834    #[error("node attempt failure must name its exact completion event as cause")]
835    FailureEventMismatch,
836    /// Pure node execution cannot own an ambiguous external side effect.
837    #[error("node attempt failure cannot require external reconciliation")]
838    ReconciliationUnsupported,
839    /// Completion joined to another start record.
840    #[error("node attempt completion does not belong to its start")]
841    CompletionStartMismatch,
842    /// Persisted activation checksum did not match the activation.
843    #[error("node activation digest does not match its fields")]
844    ActivationDigestMismatch,
845    /// Persisted start checksum did not match the start fields.
846    #[error("node attempt start digest does not match its fields")]
847    StartDigestMismatch,
848    /// Persisted completion checksum did not match completion fields.
849    #[error("node attempt completion digest does not match its fields")]
850    CompletionDigestMismatch,
851    /// Canonical integrity material could not be serialized.
852    #[error("node attempt integrity calculation failed: {source}")]
853    Integrity {
854        /// Exact integrity failure.
855        #[source]
856        source: NodeAttemptIntegrityError,
857    },
858}
859
860impl From<NodeAttemptIntegrityError> for NodeAttemptError {
861    fn from(source: NodeAttemptIntegrityError) -> Self {
862        Self::Integrity { source }
863    }
864}
865
866/// Failure to canonicalize node-attempt integrity material.
867#[derive(Clone, Debug, Eq, Error, PartialEq)]
868#[non_exhaustive]
869pub enum NodeAttemptIntegrityError {
870    /// A closed typed checksum preimage could not be canonicalized.
871    #[error("node attempt checksum preimage serialization failed")]
872    CanonicalSerialization,
873}
874
875#[derive(Serialize)]
876struct NodeAttemptStartDigestWire<'a> {
877    activation_digest: Digest,
878    attempt_id: AttemptId,
879    fence: &'a RunFence,
880    journal_head: &'a JournalHead,
881}
882
883#[derive(Serialize)]
884struct NodeAttemptCompletionDigestWire<'a> {
885    start: &'a NodeAttemptStartHead,
886    outcome: &'a NodeAttemptOutcome,
887    usage: &'a BudgetUsage,
888    journal_head: &'a JournalHead,
889}
890
891pub(crate) fn compute_activation_digest(
892    activation: &NodeActivation,
893) -> Result<Digest, NodeAttemptIntegrityError> {
894    domain_separated_digest(ACTIVATION_DIGEST_DOMAIN, activation)
895}
896
897fn compute_start_digest(
898    value: &NodeAttemptStartDigestWire<'_>,
899) -> Result<Digest, NodeAttemptIntegrityError> {
900    domain_separated_digest(START_DIGEST_DOMAIN, value)
901}
902
903fn compute_completion_digest(
904    value: &NodeAttemptCompletionDigestWire<'_>,
905) -> Result<Digest, NodeAttemptIntegrityError> {
906    domain_separated_digest(COMPLETION_DIGEST_DOMAIN, value)
907}
908
909fn domain_separated_digest<T: Serialize>(
910    domain: &[u8],
911    value: &T,
912) -> Result<Digest, NodeAttemptIntegrityError> {
913    let canonical = serde_json_canonicalizer::to_vec(value)
914        .map_err(|_| NodeAttemptIntegrityError::CanonicalSerialization)?;
915    let mut preimage = Vec::with_capacity(domain.len() + canonical.len());
916    preimage.extend_from_slice(domain);
917    preimage.extend_from_slice(&canonical);
918    Ok(Digest::sha256(preimage))
919}
920
921fn validate_start_scope(
922    activation: &NodeActivation,
923    attempt_id: AttemptId,
924    fence: &RunFence,
925    journal_head: &JournalHead,
926) -> Result<(), NodeAttemptError> {
927    if attempt_id == fence.attempt_id() {
928        return Err(NodeAttemptError::WorkerAttemptReused);
929    }
930    validate_scope(activation, fence, journal_head)?;
931    let base = activation.base_checkpoint().journal_head();
932    if journal_head.sequence() <= base.sequence() {
933        return Err(NodeAttemptError::StartJournalNotAfterBase);
934    }
935    if journal_head.recorded_at() < base.recorded_at() {
936        return Err(NodeAttemptError::StartClockRegression);
937    }
938    Ok(())
939}
940
941fn validate_scope(
942    activation: &NodeActivation,
943    fence: &RunFence,
944    journal_head: &JournalHead,
945) -> Result<(), NodeAttemptError> {
946    if fence.tenant_id() != activation.tenant_id() {
947        return Err(NodeAttemptError::FenceTenantMismatch);
948    }
949    if fence.run_id() != activation.run_id() {
950        return Err(NodeAttemptError::FenceRunMismatch);
951    }
952    if journal_head.tenant_id() != activation.tenant_id() {
953        return Err(NodeAttemptError::JournalTenantMismatch);
954    }
955    if journal_head.run_id() != activation.run_id() {
956        return Err(NodeAttemptError::JournalRunMismatch);
957    }
958    Ok(())
959}
960
961fn validate_completion_shape(
962    start: &NodeAttemptStartHead,
963    outcome: &NodeAttemptOutcome,
964    journal_head: &JournalHead,
965) -> Result<(), NodeAttemptError> {
966    validate_scope(start.activation(), start.fence(), journal_head)?;
967    if journal_head.sequence() <= start.journal_head().sequence() {
968        return Err(NodeAttemptError::CompletionJournalNotAfterStart);
969    }
970    if journal_head.recorded_at() < start.journal_head().recorded_at() {
971        return Err(NodeAttemptError::CompletionClockRegression);
972    }
973
974    match outcome {
975        NodeAttemptOutcome::Succeeded { result } => {
976            if result.activation() != start.activation() {
977                return Err(NodeAttemptError::ResultActivationMismatch);
978            }
979            if result.fence() != start.fence() {
980                return Err(NodeAttemptError::ResultFenceMismatch);
981            }
982            if result.journal_head() != journal_head {
983                return Err(NodeAttemptError::ResultJournalMismatch);
984            }
985        }
986        NodeAttemptOutcome::Failed { failure } => {
987            if failure.retry_advice().requires_reconciliation() {
988                return Err(NodeAttemptError::ReconciliationUnsupported);
989            }
990            if failure.caused_by_event_id() != Some(journal_head.event_id()) {
991                return Err(NodeAttemptError::FailureEventMismatch);
992            }
993        }
994    }
995    Ok(())
996}
997
998fn validate_history_successor(
999    previous: &NodeAttempt,
1000    next: &NodeAttempt,
1001) -> Result<(), NodeAttemptHistoryError> {
1002    if next.start.activation != previous.start.activation {
1003        return Err(NodeAttemptHistoryError::ActivationMismatch);
1004    }
1005
1006    let previous_anchor = previous.completion.as_ref().map_or_else(
1007        || previous.start.journal_head(),
1008        |value| value.journal_head(),
1009    );
1010    let next_anchor = next.start.journal_head();
1011    if next_anchor.sequence() <= previous_anchor.sequence() {
1012        return Err(NodeAttemptHistoryError::JournalNotAfterPrevious);
1013    }
1014    if next_anchor.recorded_at() < previous_anchor.recorded_at() {
1015        return Err(NodeAttemptHistoryError::ClockRegression);
1016    }
1017
1018    let previous_fence = previous.start.fence();
1019    let next_fence = next.start.fence();
1020    if next_fence.epoch() < previous_fence.epoch() {
1021        return Err(NodeAttemptHistoryError::FenceEpochRegression);
1022    }
1023    if next_fence.epoch() == previous_fence.epoch()
1024        && next_fence.attempt_id() != previous_fence.attempt_id()
1025    {
1026        return Err(NodeAttemptHistoryError::WorkerChangedWithinEpoch);
1027    }
1028    if next_fence.epoch() > previous_fence.epoch()
1029        && next_fence.attempt_id() == previous_fence.attempt_id()
1030    {
1031        return Err(NodeAttemptHistoryError::WorkerReusedAcrossEpoch);
1032    }
1033
1034    let Some(completion) = previous.completion.as_ref() else {
1035        if next_fence.epoch() <= previous_fence.epoch() {
1036            return Err(NodeAttemptHistoryError::UnfinishedAttemptNotSuperseded);
1037        }
1038        return Ok(());
1039    };
1040
1041    match completion.outcome() {
1042        NodeAttemptOutcome::Succeeded { .. } => {
1043            Err(NodeAttemptHistoryError::PreviousAttemptSucceeded)
1044        }
1045        NodeAttemptOutcome::Failed { failure } => match failure.retry_advice() {
1046            RetryAdvice::SafeAfter { delay } => {
1047                let failed_at = completion.journal_head().recorded_at();
1048                let started_at = next.start.journal_head().recorded_at();
1049                let eligible_at =
1050                    i128::from(failed_at.unix_micros()) + i128::from(delay.as_i64()) * 1_000;
1051                if i128::from(started_at.unix_micros()) < eligible_at {
1052                    Err(NodeAttemptHistoryError::RetryDelayNotElapsed {
1053                        delay_millis: delay.as_i64(),
1054                        failed_at,
1055                        started_at,
1056                    })
1057                } else {
1058                    Ok(())
1059                }
1060            }
1061            advice @ (RetryAdvice::Never | RetryAdvice::ReconcileFirst) => {
1062                Err(NodeAttemptHistoryError::RetryNotAuthorized { advice })
1063            }
1064        },
1065    }
1066}
1067
1068#[cfg(test)]
1069mod tests {
1070    use super::*;
1071    use crate::{
1072        DurationMillis, EventId, FailureCategory, FailureCode, FailureId, FailureMessage,
1073        FailureOrigin, FencingEpoch, GraphNamespace, JournalSequence, NodeControl,
1074        NodeInvocationBindings, NodeStateChange, PendingNodeResult, PendingNodeResultIntent,
1075    };
1076    use serde_json::{Value, from_value, json, to_value};
1077
1078    fn checkpoint() -> crate::Checkpoint {
1079        let fixture: Value =
1080            serde_json::from_str(include_str!("../tests/fixtures/core-checkpoint-v1.json"))
1081                .unwrap();
1082        from_value(fixture["checkpoints"][0].clone()).unwrap()
1083    }
1084
1085    fn activation(node: &str) -> NodeActivation {
1086        NodeActivation::new(
1087            checkpoint().head(),
1088            GraphNamespace::root(),
1089            crate::NodeId::new(node).unwrap(),
1090            Digest::sha256(format!("{node}-input")),
1091        )
1092    }
1093
1094    fn id<T: std::str::FromStr>(suffix: u8) -> T
1095    where
1096        T::Err: fmt::Debug,
1097    {
1098        format!("01912345-6789-7abc-8def-0123456789{suffix:02x}")
1099            .parse()
1100            .unwrap()
1101    }
1102
1103    fn fence(activation: &NodeActivation, worker_suffix: u8, epoch: u64) -> RunFence {
1104        RunFence::new(
1105            activation.tenant_id().clone(),
1106            activation.run_id(),
1107            id(worker_suffix),
1108            FencingEpoch::new(epoch).unwrap(),
1109        )
1110    }
1111
1112    fn journal(activation: &NodeActivation, sequence: u64) -> JournalHead {
1113        let base = activation.base_checkpoint().journal_head();
1114        JournalHead::new(
1115            activation.tenant_id().clone(),
1116            activation.run_id(),
1117            JournalSequence::new(sequence).unwrap(),
1118            id::<EventId>(u8::try_from(0xd0 + sequence).unwrap()),
1119            Timestamp::from_unix_micros(
1120                base.recorded_at().unix_micros()
1121                    + i64::try_from(sequence - base.sequence().get()).unwrap() * 1_000_000,
1122            )
1123            .unwrap(),
1124            Digest::sha256(sequence.to_be_bytes()),
1125        )
1126    }
1127
1128    fn start(
1129        activation: &NodeActivation,
1130        node_suffix: u8,
1131        worker_suffix: u8,
1132        epoch: u64,
1133        sequence: u64,
1134    ) -> NodeAttemptStart {
1135        NodeAttemptStart::new(
1136            activation.clone(),
1137            id(node_suffix),
1138            fence(activation, worker_suffix, epoch),
1139            journal(activation, sequence),
1140        )
1141        .unwrap()
1142    }
1143
1144    fn pending_result(start: &NodeAttemptStart, sequence: u64) -> PendingNodeResultHead {
1145        let intent = PendingNodeResultIntent::new(
1146            start.activation().clone(),
1147            NodeStateChange::Unchanged,
1148            NodeControl::Continue,
1149            NodeInvocationBindings::empty(),
1150        )
1151        .unwrap();
1152        PendingNodeResult::commit(
1153            intent,
1154            start.fence().clone(),
1155            journal(start.activation(), sequence),
1156        )
1157        .unwrap()
1158        .head()
1159    }
1160
1161    fn failure(event_id: EventId, suffix: u8, retry_advice: RetryAdvice) -> Failure {
1162        Failure::new(
1163            id::<FailureId>(suffix),
1164            FailureCategory::Internal,
1165            FailureCode::new("node.failed").unwrap(),
1166            FailureOrigin::new("graph.node").unwrap(),
1167            FailureMessage::new("Node execution failed safely").unwrap(),
1168            retry_advice,
1169        )
1170        .unwrap()
1171        .with_caused_by_event(event_id)
1172    }
1173
1174    #[test]
1175    fn starts_and_successes_are_exact_closed_and_integrity_bound() {
1176        let activation = activation("authorize");
1177        let start = start(&activation, 0xa1, 0xb1, 1, 2);
1178        let result = pending_result(&start, 3);
1179        let completion =
1180            NodeAttemptCompletion::succeed(&start, result, BudgetUsage::zero()).unwrap();
1181        let attempt = NodeAttempt::restore(start.clone(), Some(completion.clone())).unwrap();
1182
1183        assert_eq!(attempt.status(), NodeAttemptStatus::Succeeded);
1184        assert_eq!(
1185            completion.outcome().result().unwrap().activation(),
1186            &activation
1187        );
1188        assert_eq!(
1189            from_value::<NodeAttempt>(to_value(&attempt).unwrap())
1190                .unwrap()
1191                .status(),
1192            NodeAttemptStatus::Succeeded
1193        );
1194        assert_eq!(
1195            from_value::<NodeAttemptStartHead>(to_value(start.head()).unwrap()).unwrap(),
1196            start.head()
1197        );
1198        assert_eq!(
1199            completion.digest().to_string(),
1200            "sha256:f6a5ea9ec8026144396c17a583ee350ac5097d28998e1613893501515b438ac7"
1201        );
1202
1203        let mut changed = to_value(&start).unwrap();
1204        changed["attempt_id"] = json!(id::<AttemptId>(0xa2));
1205        assert!(from_value::<NodeAttemptStart>(changed).is_err());
1206
1207        let mut extra = to_value(&completion).unwrap();
1208        extra["unsafe_extension"] = json!(true);
1209        assert!(from_value::<NodeAttemptCompletion>(extra).is_err());
1210    }
1211
1212    #[test]
1213    fn completion_rejects_substitution_and_ambiguous_node_effects() {
1214        let activation = activation("authorize");
1215        let first_start = start(&activation, 0xa1, 0xb1, 1, 2);
1216        let another_start = start(&activation, 0xa2, 0xb2, 2, 3);
1217        let another_result = pending_result(&another_start, 4);
1218        assert!(matches!(
1219            NodeAttemptCompletion::succeed(&first_start, another_result, BudgetUsage::zero()),
1220            Err(NodeAttemptError::ResultFenceMismatch)
1221        ));
1222
1223        let head = journal(&activation, 3);
1224        let ambiguous = Failure::new(
1225            id(0xf1),
1226            FailureCategory::AmbiguousExternalOutcome,
1227            FailureCode::new("node.external_unknown").unwrap(),
1228            FailureOrigin::new("graph.node").unwrap(),
1229            FailureMessage::new("External outcome requires reconciliation").unwrap(),
1230            RetryAdvice::ReconcileFirst,
1231        )
1232        .unwrap()
1233        .with_caused_by_event(head.event_id());
1234        assert!(matches!(
1235            NodeAttemptCompletion::fail(&first_start, ambiguous, BudgetUsage::zero(), head),
1236            Err(NodeAttemptError::ReconciliationUnsupported)
1237        ));
1238
1239        let wrong_cause = failure(id(0xee), 0xf2, RetryAdvice::Never);
1240        assert!(matches!(
1241            NodeAttemptCompletion::fail(
1242                &first_start,
1243                wrong_cause,
1244                BudgetUsage::zero(),
1245                journal(&activation, 3),
1246            ),
1247            Err(NodeAttemptError::FailureEventMismatch)
1248        ));
1249    }
1250
1251    #[test]
1252    fn history_enforces_safe_retry_and_fenced_crash_recovery() {
1253        let activation = activation("authorize");
1254        let first_start = start(&activation, 0xa1, 0xb1, 1, 2);
1255        let failed_head = journal(&activation, 3);
1256        let failed = NodeAttempt::restore(
1257            first_start.clone(),
1258            Some(
1259                NodeAttemptCompletion::fail(
1260                    &first_start,
1261                    failure(
1262                        failed_head.event_id(),
1263                        0xf1,
1264                        RetryAdvice::SafeAfter {
1265                            delay: DurationMillis::new(2_000).unwrap(),
1266                        },
1267                    ),
1268                    BudgetUsage::zero(),
1269                    failed_head,
1270                )
1271                .unwrap(),
1272            ),
1273        )
1274        .unwrap();
1275        let too_early = NodeAttempt::executing(start(&activation, 0xa2, 0xb1, 1, 4));
1276        let allowed = NodeAttempt::executing(start(&activation, 0xa3, 0xb1, 1, 5));
1277
1278        let mut verifier = NodeAttemptHistoryVerifier::new();
1279        verifier.verify_next(&failed).unwrap();
1280        assert!(matches!(
1281            verifier.verify_next(&too_early),
1282            Err(NodeAttemptHistoryError::RetryDelayNotElapsed { .. })
1283        ));
1284        verifier.verify_next(&allowed).unwrap();
1285
1286        let crashed = NodeAttempt::executing(start(&activation, 0xa4, 0xb2, 2, 6));
1287        let same_fence_retry = NodeAttempt::executing(start(&activation, 0xa5, 0xb2, 2, 7));
1288        let takeover = NodeAttempt::executing(start(&activation, 0xa6, 0xb3, 3, 7));
1289        let mut recovery = NodeAttemptHistoryVerifier::new();
1290        recovery.verify_next(&crashed).unwrap();
1291        assert_eq!(
1292            recovery.verify_next(&same_fence_retry),
1293            Err(NodeAttemptHistoryError::UnfinishedAttemptNotSuperseded)
1294        );
1295        recovery.verify_next(&takeover).unwrap();
1296    }
1297
1298    #[test]
1299    fn success_and_non_retryable_failure_are_absorbing() {
1300        let activation = activation("authorize");
1301        let success_start = start(&activation, 0xa1, 0xb1, 1, 2);
1302        let success = NodeAttempt::restore(
1303            success_start.clone(),
1304            Some(
1305                NodeAttemptCompletion::succeed(
1306                    &success_start,
1307                    pending_result(&success_start, 3),
1308                    BudgetUsage::zero(),
1309                )
1310                .unwrap(),
1311            ),
1312        )
1313        .unwrap();
1314        let successor = NodeAttempt::executing(start(&activation, 0xa2, 0xb2, 2, 4));
1315        let mut verifier = NodeAttemptHistoryVerifier::new();
1316        verifier.verify_next(&success).unwrap();
1317        assert_eq!(
1318            verifier.verify_next(&successor),
1319            Err(NodeAttemptHistoryError::PreviousAttemptSucceeded)
1320        );
1321
1322        let failed_start = start(&activation, 0xa3, 0xb3, 3, 5);
1323        let failed_head = journal(&activation, 6);
1324        let failed = NodeAttempt::restore(
1325            failed_start.clone(),
1326            Some(
1327                NodeAttemptCompletion::fail(
1328                    &failed_start,
1329                    failure(failed_head.event_id(), 0xf2, RetryAdvice::Never),
1330                    BudgetUsage::zero(),
1331                    failed_head,
1332                )
1333                .unwrap(),
1334            ),
1335        )
1336        .unwrap();
1337        let mut verifier = NodeAttemptHistoryVerifier::new();
1338        verifier.verify_next(&failed).unwrap();
1339        assert_eq!(
1340            verifier.verify_next(&NodeAttempt::executing(start(
1341                &activation,
1342                0xa4,
1343                0xb4,
1344                4,
1345                7,
1346            ))),
1347            Err(NodeAttemptHistoryError::RetryNotAuthorized {
1348                advice: RetryAdvice::Never,
1349            })
1350        );
1351    }
1352
1353    #[test]
1354    fn diagnostics_redact_failure_messages() {
1355        let activation = activation("authorize");
1356        let start = start(&activation, 0xa1, 0xb1, 1, 2);
1357        let head = journal(&activation, 3);
1358        let completion = NodeAttemptCompletion::fail(
1359            &start,
1360            failure(head.event_id(), 0xf1, RetryAdvice::Never),
1361            BudgetUsage::zero(),
1362            head,
1363        )
1364        .unwrap();
1365        let debug = format!("{completion:?}");
1366        assert!(!debug.contains("Node execution failed safely"));
1367        assert!(debug.contains("node.failed"));
1368    }
1369}