1use std::{collections::BTreeSet, fmt};
15
16use schemars::JsonSchema;
17use serde::{Deserialize, Deserializer, Serialize, de};
18use thiserror::Error;
19
20use crate::{
21 AttemptId, BudgetUsage, Digest, Failure, JournalHead, NodeActivation, PendingNodeResultHead,
22 RetryAdvice, RunFence, Timestamp,
23};
24
25const ACTIVATION_DIGEST_DOMAIN: &[u8] = b"stateknot-node-activation-v1\0";
26const START_DIGEST_DOMAIN: &[u8] = b"stateknot-node-attempt-start-v1\0";
27const COMPLETION_DIGEST_DOMAIN: &[u8] = b"stateknot-node-attempt-completion-v1\0";
28
29#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, JsonSchema, PartialEq, Serialize)]
31#[serde(rename_all = "snake_case")]
32#[non_exhaustive]
33pub enum NodeAttemptStatus {
34 Executing,
36 Succeeded,
38 Failed,
40}
41
42#[allow(clippy::large_enum_variant)]
44#[derive(Clone, JsonSchema, Serialize)]
45#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
46#[non_exhaustive]
47pub enum NodeAttemptOutcome {
48 Succeeded {
50 result: Box<PendingNodeResultHead>,
52 },
53 Failed {
55 failure: Failure,
57 },
58}
59
60impl NodeAttemptOutcome {
61 #[must_use]
63 pub const fn status(&self) -> NodeAttemptStatus {
64 match self {
65 Self::Succeeded { .. } => NodeAttemptStatus::Succeeded,
66 Self::Failed { .. } => NodeAttemptStatus::Failed,
67 }
68 }
69
70 #[must_use]
72 pub fn result(&self) -> Option<&PendingNodeResultHead> {
73 match self {
74 Self::Succeeded { result } => Some(result.as_ref()),
75 Self::Failed { .. } => None,
76 }
77 }
78
79 #[must_use]
81 pub const fn failure(&self) -> Option<&Failure> {
82 match self {
83 Self::Succeeded { .. } => None,
84 Self::Failed { failure } => Some(failure),
85 }
86 }
87}
88
89impl fmt::Debug for NodeAttemptOutcome {
90 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
91 match self {
92 Self::Succeeded { result } => formatter
93 .debug_struct("Succeeded")
94 .field("result", result)
95 .finish(),
96 Self::Failed { failure } => formatter
97 .debug_struct("Failed")
98 .field("failure_id", &failure.id())
99 .field("category", &failure.category())
100 .field("code", failure.code())
101 .field("origin", failure.origin())
102 .field("retry_advice", &failure.retry_advice())
103 .finish_non_exhaustive(),
104 }
105 }
106}
107
108impl<'de> Deserialize<'de> for NodeAttemptOutcome {
109 fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
110 where
111 D: Deserializer<'de>,
112 {
113 #[allow(clippy::large_enum_variant)]
114 #[derive(Deserialize)]
115 #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
116 enum Wire {
117 Succeeded { result: Box<PendingNodeResultHead> },
118 Failed { failure: Failure },
119 }
120
121 Ok(match Wire::deserialize(deserializer)? {
122 Wire::Succeeded { result } => Self::Succeeded { result },
123 Wire::Failed { failure } => Self::Failed { failure },
124 })
125 }
126}
127
128#[derive(Clone, Eq, JsonSchema, PartialEq, Serialize)]
135#[serde(deny_unknown_fields)]
136pub struct NodeAttemptStart {
137 activation: NodeActivation,
138 activation_digest: Digest,
139 attempt_id: AttemptId,
140 fence: RunFence,
141 journal_head: JournalHead,
142 digest: Digest,
143}
144
145impl NodeAttemptStart {
146 pub fn new(
153 activation: NodeActivation,
154 attempt_id: AttemptId,
155 fence: RunFence,
156 journal_head: JournalHead,
157 ) -> Result<Self, NodeAttemptError> {
158 validate_start_scope(&activation, attempt_id, &fence, &journal_head)?;
159 let activation_digest = compute_activation_digest(&activation)?;
160 let digest = compute_start_digest(&NodeAttemptStartDigestWire {
161 activation_digest,
162 attempt_id,
163 fence: &fence,
164 journal_head: &journal_head,
165 })?;
166 Ok(Self {
167 activation,
168 activation_digest,
169 attempt_id,
170 fence,
171 journal_head,
172 digest,
173 })
174 }
175
176 #[allow(clippy::too_many_arguments)]
182 pub fn restore(
183 activation: NodeActivation,
184 activation_digest: Digest,
185 attempt_id: AttemptId,
186 fence: RunFence,
187 journal_head: JournalHead,
188 digest: Digest,
189 ) -> Result<Self, NodeAttemptError> {
190 let restored = Self::new(activation, attempt_id, fence, journal_head)?;
191 if restored.activation_digest != activation_digest {
192 return Err(NodeAttemptError::ActivationDigestMismatch);
193 }
194 if restored.digest != digest {
195 return Err(NodeAttemptError::StartDigestMismatch);
196 }
197 Ok(restored)
198 }
199
200 #[must_use]
202 pub const fn activation(&self) -> &NodeActivation {
203 &self.activation
204 }
205
206 #[must_use]
208 pub const fn activation_digest(&self) -> Digest {
209 self.activation_digest
210 }
211
212 #[must_use]
214 pub const fn attempt_id(&self) -> AttemptId {
215 self.attempt_id
216 }
217
218 #[must_use]
220 pub const fn fence(&self) -> &RunFence {
221 &self.fence
222 }
223
224 #[must_use]
226 pub const fn journal_head(&self) -> &JournalHead {
227 &self.journal_head
228 }
229
230 #[must_use]
232 pub const fn digest(&self) -> Digest {
233 self.digest
234 }
235
236 #[must_use]
238 pub fn head(&self) -> NodeAttemptStartHead {
239 NodeAttemptStartHead {
240 activation: self.activation.clone(),
241 attempt_id: self.attempt_id,
242 fence: self.fence.clone(),
243 journal_head: self.journal_head.clone(),
244 digest: self.digest,
245 }
246 }
247}
248
249impl fmt::Debug for NodeAttemptStart {
250 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
251 formatter
252 .debug_struct("NodeAttemptStart")
253 .field("activation", &self.activation)
254 .field("activation_digest", &self.activation_digest)
255 .field("attempt_id", &self.attempt_id)
256 .field("fence", &self.fence)
257 .field("journal_head", &self.journal_head)
258 .field("digest", &self.digest)
259 .finish_non_exhaustive()
260 }
261}
262
263impl<'de> Deserialize<'de> for NodeAttemptStart {
264 fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
265 where
266 D: Deserializer<'de>,
267 {
268 #[derive(Deserialize)]
269 #[serde(deny_unknown_fields)]
270 struct Wire {
271 activation: NodeActivation,
272 activation_digest: Digest,
273 attempt_id: AttemptId,
274 fence: RunFence,
275 journal_head: JournalHead,
276 digest: Digest,
277 }
278
279 let wire = Wire::deserialize(deserializer)?;
280 Self::restore(
281 wire.activation,
282 wire.activation_digest,
283 wire.attempt_id,
284 wire.fence,
285 wire.journal_head,
286 wire.digest,
287 )
288 .map_err(de::Error::custom)
289 }
290}
291
292#[derive(Clone, Debug, Eq, JsonSchema, PartialEq, Serialize)]
294#[serde(deny_unknown_fields)]
295pub struct NodeAttemptStartHead {
296 activation: NodeActivation,
297 attempt_id: AttemptId,
298 fence: RunFence,
299 journal_head: JournalHead,
300 digest: Digest,
301}
302
303impl NodeAttemptStartHead {
304 pub fn restore(
310 activation: NodeActivation,
311 attempt_id: AttemptId,
312 fence: RunFence,
313 journal_head: JournalHead,
314 digest: Digest,
315 ) -> Result<Self, NodeAttemptError> {
316 let activation_digest = compute_activation_digest(&activation)?;
317 Ok(NodeAttemptStart::restore(
318 activation,
319 activation_digest,
320 attempt_id,
321 fence,
322 journal_head,
323 digest,
324 )?
325 .head())
326 }
327
328 #[must_use]
330 pub const fn activation(&self) -> &NodeActivation {
331 &self.activation
332 }
333
334 #[must_use]
336 pub const fn attempt_id(&self) -> AttemptId {
337 self.attempt_id
338 }
339
340 #[must_use]
342 pub const fn fence(&self) -> &RunFence {
343 &self.fence
344 }
345
346 #[must_use]
348 pub const fn journal_head(&self) -> &JournalHead {
349 &self.journal_head
350 }
351
352 #[must_use]
354 pub const fn digest(&self) -> Digest {
355 self.digest
356 }
357}
358
359impl<'de> Deserialize<'de> for NodeAttemptStartHead {
360 fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
361 where
362 D: Deserializer<'de>,
363 {
364 #[derive(Deserialize)]
365 #[serde(deny_unknown_fields)]
366 struct Wire {
367 activation: NodeActivation,
368 attempt_id: AttemptId,
369 fence: RunFence,
370 journal_head: JournalHead,
371 digest: Digest,
372 }
373
374 let wire = Wire::deserialize(deserializer)?;
375 validate_start_scope(
376 &wire.activation,
377 wire.attempt_id,
378 &wire.fence,
379 &wire.journal_head,
380 )
381 .map_err(de::Error::custom)?;
382 let activation_digest =
383 compute_activation_digest(&wire.activation).map_err(de::Error::custom)?;
384 let expected = compute_start_digest(&NodeAttemptStartDigestWire {
385 activation_digest,
386 attempt_id: wire.attempt_id,
387 fence: &wire.fence,
388 journal_head: &wire.journal_head,
389 })
390 .map_err(de::Error::custom)?;
391 if expected != wire.digest {
392 return Err(de::Error::custom(NodeAttemptError::StartDigestMismatch));
393 }
394 Ok(Self {
395 activation: wire.activation,
396 attempt_id: wire.attempt_id,
397 fence: wire.fence,
398 journal_head: wire.journal_head,
399 digest: wire.digest,
400 })
401 }
402}
403
404#[derive(Clone, JsonSchema, Serialize)]
406#[serde(deny_unknown_fields)]
407pub struct NodeAttemptCompletion {
408 start: NodeAttemptStartHead,
409 outcome: NodeAttemptOutcome,
410 usage: BudgetUsage,
411 journal_head: JournalHead,
412 digest: Digest,
413}
414
415impl NodeAttemptCompletion {
416 pub fn succeed(
426 start: &NodeAttemptStart,
427 result: PendingNodeResultHead,
428 usage: BudgetUsage,
429 ) -> Result<Self, NodeAttemptError> {
430 let journal_head = result.journal_head().clone();
431 Self::materialize(
432 start.head(),
433 NodeAttemptOutcome::Succeeded {
434 result: Box::new(result),
435 },
436 usage,
437 journal_head,
438 )
439 }
440
441 pub fn fail(
452 start: &NodeAttemptStart,
453 failure: Failure,
454 usage: BudgetUsage,
455 journal_head: JournalHead,
456 ) -> Result<Self, NodeAttemptError> {
457 Self::materialize(
458 start.head(),
459 NodeAttemptOutcome::Failed { failure },
460 usage,
461 journal_head,
462 )
463 }
464
465 pub fn restore(
471 start: NodeAttemptStartHead,
472 outcome: NodeAttemptOutcome,
473 usage: BudgetUsage,
474 journal_head: JournalHead,
475 digest: Digest,
476 ) -> Result<Self, NodeAttemptError> {
477 let restored = Self::materialize(start, outcome, usage, journal_head)?;
478 if restored.digest != digest {
479 return Err(NodeAttemptError::CompletionDigestMismatch);
480 }
481 Ok(restored)
482 }
483
484 fn materialize(
485 start: NodeAttemptStartHead,
486 outcome: NodeAttemptOutcome,
487 usage: BudgetUsage,
488 journal_head: JournalHead,
489 ) -> Result<Self, NodeAttemptError> {
490 validate_completion_shape(&start, &outcome, &journal_head)?;
491 let digest = compute_completion_digest(&NodeAttemptCompletionDigestWire {
492 start: &start,
493 outcome: &outcome,
494 usage: &usage,
495 journal_head: &journal_head,
496 })?;
497 Ok(Self {
498 start,
499 outcome,
500 usage,
501 journal_head,
502 digest,
503 })
504 }
505
506 #[must_use]
508 pub const fn start(&self) -> &NodeAttemptStartHead {
509 &self.start
510 }
511
512 #[must_use]
514 pub const fn outcome(&self) -> &NodeAttemptOutcome {
515 &self.outcome
516 }
517
518 #[must_use]
520 pub const fn status(&self) -> NodeAttemptStatus {
521 self.outcome.status()
522 }
523
524 #[must_use]
526 pub const fn usage(&self) -> &BudgetUsage {
527 &self.usage
528 }
529
530 #[must_use]
532 pub const fn journal_head(&self) -> &JournalHead {
533 &self.journal_head
534 }
535
536 #[must_use]
538 pub const fn digest(&self) -> Digest {
539 self.digest
540 }
541}
542
543impl fmt::Debug for NodeAttemptCompletion {
544 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
545 formatter
546 .debug_struct("NodeAttemptCompletion")
547 .field("start", &self.start)
548 .field("outcome", &self.outcome)
549 .field("usage_recorded", &true)
550 .field("journal_head", &self.journal_head)
551 .field("digest", &self.digest)
552 .finish_non_exhaustive()
553 }
554}
555
556impl<'de> Deserialize<'de> for NodeAttemptCompletion {
557 fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
558 where
559 D: Deserializer<'de>,
560 {
561 #[derive(Deserialize)]
562 #[serde(deny_unknown_fields)]
563 struct Wire {
564 start: NodeAttemptStartHead,
565 outcome: NodeAttemptOutcome,
566 usage: BudgetUsage,
567 journal_head: JournalHead,
568 digest: Digest,
569 }
570
571 let wire = Wire::deserialize(deserializer)?;
572 Self::restore(
573 wire.start,
574 wire.outcome,
575 wire.usage,
576 wire.journal_head,
577 wire.digest,
578 )
579 .map_err(de::Error::custom)
580 }
581}
582
583#[derive(Clone, JsonSchema, Serialize)]
585#[serde(deny_unknown_fields)]
586pub struct NodeAttempt {
587 start: NodeAttemptStart,
588 #[serde(skip_serializing_if = "Option::is_none")]
589 completion: Option<NodeAttemptCompletion>,
590}
591
592impl NodeAttempt {
593 #[must_use]
595 pub const fn executing(start: NodeAttemptStart) -> Self {
596 Self {
597 start,
598 completion: None,
599 }
600 }
601
602 pub fn restore(
609 start: NodeAttemptStart,
610 completion: Option<NodeAttemptCompletion>,
611 ) -> Result<Self, NodeAttemptError> {
612 if completion
613 .as_ref()
614 .is_some_and(|completion| completion.start() != &start.head())
615 {
616 return Err(NodeAttemptError::CompletionStartMismatch);
617 }
618 Ok(Self { start, completion })
619 }
620
621 #[must_use]
623 pub const fn start(&self) -> &NodeAttemptStart {
624 &self.start
625 }
626
627 #[must_use]
629 pub const fn completion(&self) -> Option<&NodeAttemptCompletion> {
630 self.completion.as_ref()
631 }
632
633 #[must_use]
635 pub const fn status(&self) -> NodeAttemptStatus {
636 match &self.completion {
637 Some(completion) => completion.status(),
638 None => NodeAttemptStatus::Executing,
639 }
640 }
641}
642
643impl fmt::Debug for NodeAttempt {
644 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
645 formatter
646 .debug_struct("NodeAttempt")
647 .field("start", &self.start)
648 .field("completion", &self.completion)
649 .finish_non_exhaustive()
650 }
651}
652
653impl<'de> Deserialize<'de> for NodeAttempt {
654 fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
655 where
656 D: Deserializer<'de>,
657 {
658 #[derive(Deserialize)]
659 #[serde(deny_unknown_fields)]
660 struct Wire {
661 start: NodeAttemptStart,
662 #[serde(default)]
663 completion: Option<NodeAttemptCompletion>,
664 }
665
666 let wire = Wire::deserialize(deserializer)?;
667 Self::restore(wire.start, wire.completion).map_err(de::Error::custom)
668 }
669}
670
671#[derive(Clone, Debug, Default)]
678pub struct NodeAttemptHistoryVerifier {
679 last: Option<NodeAttempt>,
680 attempt_ids: BTreeSet<AttemptId>,
681}
682
683impl NodeAttemptHistoryVerifier {
684 #[must_use]
686 pub const fn new() -> Self {
687 Self {
688 last: None,
689 attempt_ids: BTreeSet::new(),
690 }
691 }
692
693 #[must_use]
698 pub fn after(record: NodeAttempt) -> Self {
699 let attempt_id = record.start.attempt_id;
700 Self {
701 last: Some(record),
702 attempt_ids: BTreeSet::from([attempt_id]),
703 }
704 }
705
706 #[must_use]
708 pub const fn has_records(&self) -> bool {
709 self.last.is_some()
710 }
711
712 #[must_use]
714 pub const fn last(&self) -> Option<&NodeAttempt> {
715 self.last.as_ref()
716 }
717
718 pub fn verify_next(&mut self, attempt: &NodeAttempt) -> Result<(), NodeAttemptHistoryError> {
726 let attempt_id = attempt.start.attempt_id;
727 if self.attempt_ids.contains(&attempt_id) {
728 return Err(NodeAttemptHistoryError::AttemptIdReused { attempt_id });
729 }
730
731 if let Some(previous) = self.last.as_ref() {
732 validate_history_successor(previous, attempt)?;
733 }
734
735 self.attempt_ids.insert(attempt_id);
736 self.last = Some(attempt.clone());
737 Ok(())
738 }
739}
740
741#[derive(Clone, Debug, Eq, Error, PartialEq)]
743#[non_exhaustive]
744pub enum NodeAttemptHistoryError {
745 #[error("node attempt history reused physical attempt {attempt_id}")]
747 AttemptIdReused {
748 attempt_id: AttemptId,
750 },
751 #[error("node attempt history changed its logical activation")]
753 ActivationMismatch,
754 #[error("node attempt history journal position did not advance")]
756 JournalNotAfterPrevious,
757 #[error("node attempt history durable clock regressed")]
759 ClockRegression,
760 #[error("node attempt history worker fencing epoch regressed")]
762 FenceEpochRegression,
763 #[error("node attempt history changed worker attempt without advancing the epoch")]
765 WorkerChangedWithinEpoch,
766 #[error("node attempt history reused a worker attempt across fencing epochs")]
768 WorkerReusedAcrossEpoch,
769 #[error("an unfinished node attempt may be recovered only under a higher fencing epoch")]
771 UnfinishedAttemptNotSuperseded,
772 #[error("a successful node activation cannot start another physical attempt")]
774 PreviousAttemptSucceeded,
775 #[error("node failure recovery advice does not authorize automatic retry")]
777 RetryNotAuthorized {
778 advice: RetryAdvice,
780 },
781 #[error("node attempt retry started before its explicit safe-after delay elapsed")]
783 RetryDelayNotElapsed {
784 delay_millis: i64,
786 failed_at: Timestamp,
788 started_at: Timestamp,
790 },
791}
792
793#[derive(Clone, Debug, Eq, Error, PartialEq)]
795#[non_exhaustive]
796pub enum NodeAttemptError {
797 #[error("node physical attempt must differ from the worker-run attempt")]
799 WorkerAttemptReused,
800 #[error("node attempt fence crosses the activation tenant boundary")]
802 FenceTenantMismatch,
803 #[error("node attempt fence does not belong to the activation run")]
805 FenceRunMismatch,
806 #[error("node attempt journal crosses the activation tenant boundary")]
808 JournalTenantMismatch,
809 #[error("node attempt journal does not belong to the activation run")]
811 JournalRunMismatch,
812 #[error("node attempt start journal does not follow its base checkpoint")]
814 StartJournalNotAfterBase,
815 #[error("node attempt start clock precedes its base checkpoint")]
817 StartClockRegression,
818 #[error("node attempt completion journal does not follow its start")]
820 CompletionJournalNotAfterStart,
821 #[error("node attempt completion clock precedes its start")]
823 CompletionClockRegression,
824 #[error("node attempt success result belongs to another activation")]
826 ResultActivationMismatch,
827 #[error("node attempt success result uses another worker fence")]
829 ResultFenceMismatch,
830 #[error("node attempt success result and completion journal differ")]
832 ResultJournalMismatch,
833 #[error("node attempt failure must name its exact completion event as cause")]
835 FailureEventMismatch,
836 #[error("node attempt failure cannot require external reconciliation")]
838 ReconciliationUnsupported,
839 #[error("node attempt completion does not belong to its start")]
841 CompletionStartMismatch,
842 #[error("node activation digest does not match its fields")]
844 ActivationDigestMismatch,
845 #[error("node attempt start digest does not match its fields")]
847 StartDigestMismatch,
848 #[error("node attempt completion digest does not match its fields")]
850 CompletionDigestMismatch,
851 #[error("node attempt integrity calculation failed: {source}")]
853 Integrity {
854 #[source]
856 source: NodeAttemptIntegrityError,
857 },
858}
859
860impl From<NodeAttemptIntegrityError> for NodeAttemptError {
861 fn from(source: NodeAttemptIntegrityError) -> Self {
862 Self::Integrity { source }
863 }
864}
865
866#[derive(Clone, Debug, Eq, Error, PartialEq)]
868#[non_exhaustive]
869pub enum NodeAttemptIntegrityError {
870 #[error("node attempt checksum preimage serialization failed")]
872 CanonicalSerialization,
873}
874
875#[derive(Serialize)]
876struct NodeAttemptStartDigestWire<'a> {
877 activation_digest: Digest,
878 attempt_id: AttemptId,
879 fence: &'a RunFence,
880 journal_head: &'a JournalHead,
881}
882
883#[derive(Serialize)]
884struct NodeAttemptCompletionDigestWire<'a> {
885 start: &'a NodeAttemptStartHead,
886 outcome: &'a NodeAttemptOutcome,
887 usage: &'a BudgetUsage,
888 journal_head: &'a JournalHead,
889}
890
891pub(crate) fn compute_activation_digest(
892 activation: &NodeActivation,
893) -> Result<Digest, NodeAttemptIntegrityError> {
894 domain_separated_digest(ACTIVATION_DIGEST_DOMAIN, activation)
895}
896
897fn compute_start_digest(
898 value: &NodeAttemptStartDigestWire<'_>,
899) -> Result<Digest, NodeAttemptIntegrityError> {
900 domain_separated_digest(START_DIGEST_DOMAIN, value)
901}
902
903fn compute_completion_digest(
904 value: &NodeAttemptCompletionDigestWire<'_>,
905) -> Result<Digest, NodeAttemptIntegrityError> {
906 domain_separated_digest(COMPLETION_DIGEST_DOMAIN, value)
907}
908
909fn domain_separated_digest<T: Serialize>(
910 domain: &[u8],
911 value: &T,
912) -> Result<Digest, NodeAttemptIntegrityError> {
913 let canonical = serde_json_canonicalizer::to_vec(value)
914 .map_err(|_| NodeAttemptIntegrityError::CanonicalSerialization)?;
915 let mut preimage = Vec::with_capacity(domain.len() + canonical.len());
916 preimage.extend_from_slice(domain);
917 preimage.extend_from_slice(&canonical);
918 Ok(Digest::sha256(preimage))
919}
920
921fn validate_start_scope(
922 activation: &NodeActivation,
923 attempt_id: AttemptId,
924 fence: &RunFence,
925 journal_head: &JournalHead,
926) -> Result<(), NodeAttemptError> {
927 if attempt_id == fence.attempt_id() {
928 return Err(NodeAttemptError::WorkerAttemptReused);
929 }
930 validate_scope(activation, fence, journal_head)?;
931 let base = activation.base_checkpoint().journal_head();
932 if journal_head.sequence() <= base.sequence() {
933 return Err(NodeAttemptError::StartJournalNotAfterBase);
934 }
935 if journal_head.recorded_at() < base.recorded_at() {
936 return Err(NodeAttemptError::StartClockRegression);
937 }
938 Ok(())
939}
940
941fn validate_scope(
942 activation: &NodeActivation,
943 fence: &RunFence,
944 journal_head: &JournalHead,
945) -> Result<(), NodeAttemptError> {
946 if fence.tenant_id() != activation.tenant_id() {
947 return Err(NodeAttemptError::FenceTenantMismatch);
948 }
949 if fence.run_id() != activation.run_id() {
950 return Err(NodeAttemptError::FenceRunMismatch);
951 }
952 if journal_head.tenant_id() != activation.tenant_id() {
953 return Err(NodeAttemptError::JournalTenantMismatch);
954 }
955 if journal_head.run_id() != activation.run_id() {
956 return Err(NodeAttemptError::JournalRunMismatch);
957 }
958 Ok(())
959}
960
961fn validate_completion_shape(
962 start: &NodeAttemptStartHead,
963 outcome: &NodeAttemptOutcome,
964 journal_head: &JournalHead,
965) -> Result<(), NodeAttemptError> {
966 validate_scope(start.activation(), start.fence(), journal_head)?;
967 if journal_head.sequence() <= start.journal_head().sequence() {
968 return Err(NodeAttemptError::CompletionJournalNotAfterStart);
969 }
970 if journal_head.recorded_at() < start.journal_head().recorded_at() {
971 return Err(NodeAttemptError::CompletionClockRegression);
972 }
973
974 match outcome {
975 NodeAttemptOutcome::Succeeded { result } => {
976 if result.activation() != start.activation() {
977 return Err(NodeAttemptError::ResultActivationMismatch);
978 }
979 if result.fence() != start.fence() {
980 return Err(NodeAttemptError::ResultFenceMismatch);
981 }
982 if result.journal_head() != journal_head {
983 return Err(NodeAttemptError::ResultJournalMismatch);
984 }
985 }
986 NodeAttemptOutcome::Failed { failure } => {
987 if failure.retry_advice().requires_reconciliation() {
988 return Err(NodeAttemptError::ReconciliationUnsupported);
989 }
990 if failure.caused_by_event_id() != Some(journal_head.event_id()) {
991 return Err(NodeAttemptError::FailureEventMismatch);
992 }
993 }
994 }
995 Ok(())
996}
997
998fn validate_history_successor(
999 previous: &NodeAttempt,
1000 next: &NodeAttempt,
1001) -> Result<(), NodeAttemptHistoryError> {
1002 if next.start.activation != previous.start.activation {
1003 return Err(NodeAttemptHistoryError::ActivationMismatch);
1004 }
1005
1006 let previous_anchor = previous.completion.as_ref().map_or_else(
1007 || previous.start.journal_head(),
1008 |value| value.journal_head(),
1009 );
1010 let next_anchor = next.start.journal_head();
1011 if next_anchor.sequence() <= previous_anchor.sequence() {
1012 return Err(NodeAttemptHistoryError::JournalNotAfterPrevious);
1013 }
1014 if next_anchor.recorded_at() < previous_anchor.recorded_at() {
1015 return Err(NodeAttemptHistoryError::ClockRegression);
1016 }
1017
1018 let previous_fence = previous.start.fence();
1019 let next_fence = next.start.fence();
1020 if next_fence.epoch() < previous_fence.epoch() {
1021 return Err(NodeAttemptHistoryError::FenceEpochRegression);
1022 }
1023 if next_fence.epoch() == previous_fence.epoch()
1024 && next_fence.attempt_id() != previous_fence.attempt_id()
1025 {
1026 return Err(NodeAttemptHistoryError::WorkerChangedWithinEpoch);
1027 }
1028 if next_fence.epoch() > previous_fence.epoch()
1029 && next_fence.attempt_id() == previous_fence.attempt_id()
1030 {
1031 return Err(NodeAttemptHistoryError::WorkerReusedAcrossEpoch);
1032 }
1033
1034 let Some(completion) = previous.completion.as_ref() else {
1035 if next_fence.epoch() <= previous_fence.epoch() {
1036 return Err(NodeAttemptHistoryError::UnfinishedAttemptNotSuperseded);
1037 }
1038 return Ok(());
1039 };
1040
1041 match completion.outcome() {
1042 NodeAttemptOutcome::Succeeded { .. } => {
1043 Err(NodeAttemptHistoryError::PreviousAttemptSucceeded)
1044 }
1045 NodeAttemptOutcome::Failed { failure } => match failure.retry_advice() {
1046 RetryAdvice::SafeAfter { delay } => {
1047 let failed_at = completion.journal_head().recorded_at();
1048 let started_at = next.start.journal_head().recorded_at();
1049 let eligible_at =
1050 i128::from(failed_at.unix_micros()) + i128::from(delay.as_i64()) * 1_000;
1051 if i128::from(started_at.unix_micros()) < eligible_at {
1052 Err(NodeAttemptHistoryError::RetryDelayNotElapsed {
1053 delay_millis: delay.as_i64(),
1054 failed_at,
1055 started_at,
1056 })
1057 } else {
1058 Ok(())
1059 }
1060 }
1061 advice @ (RetryAdvice::Never | RetryAdvice::ReconcileFirst) => {
1062 Err(NodeAttemptHistoryError::RetryNotAuthorized { advice })
1063 }
1064 },
1065 }
1066}
1067
1068#[cfg(test)]
1069mod tests {
1070 use super::*;
1071 use crate::{
1072 DurationMillis, EventId, FailureCategory, FailureCode, FailureId, FailureMessage,
1073 FailureOrigin, FencingEpoch, GraphNamespace, JournalSequence, NodeControl,
1074 NodeInvocationBindings, NodeStateChange, PendingNodeResult, PendingNodeResultIntent,
1075 };
1076 use serde_json::{Value, from_value, json, to_value};
1077
1078 fn checkpoint() -> crate::Checkpoint {
1079 let fixture: Value =
1080 serde_json::from_str(include_str!("../tests/fixtures/core-checkpoint-v1.json"))
1081 .unwrap();
1082 from_value(fixture["checkpoints"][0].clone()).unwrap()
1083 }
1084
1085 fn activation(node: &str) -> NodeActivation {
1086 NodeActivation::new(
1087 checkpoint().head(),
1088 GraphNamespace::root(),
1089 crate::NodeId::new(node).unwrap(),
1090 Digest::sha256(format!("{node}-input")),
1091 )
1092 }
1093
1094 fn id<T: std::str::FromStr>(suffix: u8) -> T
1095 where
1096 T::Err: fmt::Debug,
1097 {
1098 format!("01912345-6789-7abc-8def-0123456789{suffix:02x}")
1099 .parse()
1100 .unwrap()
1101 }
1102
1103 fn fence(activation: &NodeActivation, worker_suffix: u8, epoch: u64) -> RunFence {
1104 RunFence::new(
1105 activation.tenant_id().clone(),
1106 activation.run_id(),
1107 id(worker_suffix),
1108 FencingEpoch::new(epoch).unwrap(),
1109 )
1110 }
1111
1112 fn journal(activation: &NodeActivation, sequence: u64) -> JournalHead {
1113 let base = activation.base_checkpoint().journal_head();
1114 JournalHead::new(
1115 activation.tenant_id().clone(),
1116 activation.run_id(),
1117 JournalSequence::new(sequence).unwrap(),
1118 id::<EventId>(u8::try_from(0xd0 + sequence).unwrap()),
1119 Timestamp::from_unix_micros(
1120 base.recorded_at().unix_micros()
1121 + i64::try_from(sequence - base.sequence().get()).unwrap() * 1_000_000,
1122 )
1123 .unwrap(),
1124 Digest::sha256(sequence.to_be_bytes()),
1125 )
1126 }
1127
1128 fn start(
1129 activation: &NodeActivation,
1130 node_suffix: u8,
1131 worker_suffix: u8,
1132 epoch: u64,
1133 sequence: u64,
1134 ) -> NodeAttemptStart {
1135 NodeAttemptStart::new(
1136 activation.clone(),
1137 id(node_suffix),
1138 fence(activation, worker_suffix, epoch),
1139 journal(activation, sequence),
1140 )
1141 .unwrap()
1142 }
1143
1144 fn pending_result(start: &NodeAttemptStart, sequence: u64) -> PendingNodeResultHead {
1145 let intent = PendingNodeResultIntent::new(
1146 start.activation().clone(),
1147 NodeStateChange::Unchanged,
1148 NodeControl::Continue,
1149 NodeInvocationBindings::empty(),
1150 )
1151 .unwrap();
1152 PendingNodeResult::commit(
1153 intent,
1154 start.fence().clone(),
1155 journal(start.activation(), sequence),
1156 )
1157 .unwrap()
1158 .head()
1159 }
1160
1161 fn failure(event_id: EventId, suffix: u8, retry_advice: RetryAdvice) -> Failure {
1162 Failure::new(
1163 id::<FailureId>(suffix),
1164 FailureCategory::Internal,
1165 FailureCode::new("node.failed").unwrap(),
1166 FailureOrigin::new("graph.node").unwrap(),
1167 FailureMessage::new("Node execution failed safely").unwrap(),
1168 retry_advice,
1169 )
1170 .unwrap()
1171 .with_caused_by_event(event_id)
1172 }
1173
1174 #[test]
1175 fn starts_and_successes_are_exact_closed_and_integrity_bound() {
1176 let activation = activation("authorize");
1177 let start = start(&activation, 0xa1, 0xb1, 1, 2);
1178 let result = pending_result(&start, 3);
1179 let completion =
1180 NodeAttemptCompletion::succeed(&start, result, BudgetUsage::zero()).unwrap();
1181 let attempt = NodeAttempt::restore(start.clone(), Some(completion.clone())).unwrap();
1182
1183 assert_eq!(attempt.status(), NodeAttemptStatus::Succeeded);
1184 assert_eq!(
1185 completion.outcome().result().unwrap().activation(),
1186 &activation
1187 );
1188 assert_eq!(
1189 from_value::<NodeAttempt>(to_value(&attempt).unwrap())
1190 .unwrap()
1191 .status(),
1192 NodeAttemptStatus::Succeeded
1193 );
1194 assert_eq!(
1195 from_value::<NodeAttemptStartHead>(to_value(start.head()).unwrap()).unwrap(),
1196 start.head()
1197 );
1198 assert_eq!(
1199 completion.digest().to_string(),
1200 "sha256:f6a5ea9ec8026144396c17a583ee350ac5097d28998e1613893501515b438ac7"
1201 );
1202
1203 let mut changed = to_value(&start).unwrap();
1204 changed["attempt_id"] = json!(id::<AttemptId>(0xa2));
1205 assert!(from_value::<NodeAttemptStart>(changed).is_err());
1206
1207 let mut extra = to_value(&completion).unwrap();
1208 extra["unsafe_extension"] = json!(true);
1209 assert!(from_value::<NodeAttemptCompletion>(extra).is_err());
1210 }
1211
1212 #[test]
1213 fn completion_rejects_substitution_and_ambiguous_node_effects() {
1214 let activation = activation("authorize");
1215 let first_start = start(&activation, 0xa1, 0xb1, 1, 2);
1216 let another_start = start(&activation, 0xa2, 0xb2, 2, 3);
1217 let another_result = pending_result(&another_start, 4);
1218 assert!(matches!(
1219 NodeAttemptCompletion::succeed(&first_start, another_result, BudgetUsage::zero()),
1220 Err(NodeAttemptError::ResultFenceMismatch)
1221 ));
1222
1223 let head = journal(&activation, 3);
1224 let ambiguous = Failure::new(
1225 id(0xf1),
1226 FailureCategory::AmbiguousExternalOutcome,
1227 FailureCode::new("node.external_unknown").unwrap(),
1228 FailureOrigin::new("graph.node").unwrap(),
1229 FailureMessage::new("External outcome requires reconciliation").unwrap(),
1230 RetryAdvice::ReconcileFirst,
1231 )
1232 .unwrap()
1233 .with_caused_by_event(head.event_id());
1234 assert!(matches!(
1235 NodeAttemptCompletion::fail(&first_start, ambiguous, BudgetUsage::zero(), head),
1236 Err(NodeAttemptError::ReconciliationUnsupported)
1237 ));
1238
1239 let wrong_cause = failure(id(0xee), 0xf2, RetryAdvice::Never);
1240 assert!(matches!(
1241 NodeAttemptCompletion::fail(
1242 &first_start,
1243 wrong_cause,
1244 BudgetUsage::zero(),
1245 journal(&activation, 3),
1246 ),
1247 Err(NodeAttemptError::FailureEventMismatch)
1248 ));
1249 }
1250
1251 #[test]
1252 fn history_enforces_safe_retry_and_fenced_crash_recovery() {
1253 let activation = activation("authorize");
1254 let first_start = start(&activation, 0xa1, 0xb1, 1, 2);
1255 let failed_head = journal(&activation, 3);
1256 let failed = NodeAttempt::restore(
1257 first_start.clone(),
1258 Some(
1259 NodeAttemptCompletion::fail(
1260 &first_start,
1261 failure(
1262 failed_head.event_id(),
1263 0xf1,
1264 RetryAdvice::SafeAfter {
1265 delay: DurationMillis::new(2_000).unwrap(),
1266 },
1267 ),
1268 BudgetUsage::zero(),
1269 failed_head,
1270 )
1271 .unwrap(),
1272 ),
1273 )
1274 .unwrap();
1275 let too_early = NodeAttempt::executing(start(&activation, 0xa2, 0xb1, 1, 4));
1276 let allowed = NodeAttempt::executing(start(&activation, 0xa3, 0xb1, 1, 5));
1277
1278 let mut verifier = NodeAttemptHistoryVerifier::new();
1279 verifier.verify_next(&failed).unwrap();
1280 assert!(matches!(
1281 verifier.verify_next(&too_early),
1282 Err(NodeAttemptHistoryError::RetryDelayNotElapsed { .. })
1283 ));
1284 verifier.verify_next(&allowed).unwrap();
1285
1286 let crashed = NodeAttempt::executing(start(&activation, 0xa4, 0xb2, 2, 6));
1287 let same_fence_retry = NodeAttempt::executing(start(&activation, 0xa5, 0xb2, 2, 7));
1288 let takeover = NodeAttempt::executing(start(&activation, 0xa6, 0xb3, 3, 7));
1289 let mut recovery = NodeAttemptHistoryVerifier::new();
1290 recovery.verify_next(&crashed).unwrap();
1291 assert_eq!(
1292 recovery.verify_next(&same_fence_retry),
1293 Err(NodeAttemptHistoryError::UnfinishedAttemptNotSuperseded)
1294 );
1295 recovery.verify_next(&takeover).unwrap();
1296 }
1297
1298 #[test]
1299 fn success_and_non_retryable_failure_are_absorbing() {
1300 let activation = activation("authorize");
1301 let success_start = start(&activation, 0xa1, 0xb1, 1, 2);
1302 let success = NodeAttempt::restore(
1303 success_start.clone(),
1304 Some(
1305 NodeAttemptCompletion::succeed(
1306 &success_start,
1307 pending_result(&success_start, 3),
1308 BudgetUsage::zero(),
1309 )
1310 .unwrap(),
1311 ),
1312 )
1313 .unwrap();
1314 let successor = NodeAttempt::executing(start(&activation, 0xa2, 0xb2, 2, 4));
1315 let mut verifier = NodeAttemptHistoryVerifier::new();
1316 verifier.verify_next(&success).unwrap();
1317 assert_eq!(
1318 verifier.verify_next(&successor),
1319 Err(NodeAttemptHistoryError::PreviousAttemptSucceeded)
1320 );
1321
1322 let failed_start = start(&activation, 0xa3, 0xb3, 3, 5);
1323 let failed_head = journal(&activation, 6);
1324 let failed = NodeAttempt::restore(
1325 failed_start.clone(),
1326 Some(
1327 NodeAttemptCompletion::fail(
1328 &failed_start,
1329 failure(failed_head.event_id(), 0xf2, RetryAdvice::Never),
1330 BudgetUsage::zero(),
1331 failed_head,
1332 )
1333 .unwrap(),
1334 ),
1335 )
1336 .unwrap();
1337 let mut verifier = NodeAttemptHistoryVerifier::new();
1338 verifier.verify_next(&failed).unwrap();
1339 assert_eq!(
1340 verifier.verify_next(&NodeAttempt::executing(start(
1341 &activation,
1342 0xa4,
1343 0xb4,
1344 4,
1345 7,
1346 ))),
1347 Err(NodeAttemptHistoryError::RetryNotAuthorized {
1348 advice: RetryAdvice::Never,
1349 })
1350 );
1351 }
1352
1353 #[test]
1354 fn diagnostics_redact_failure_messages() {
1355 let activation = activation("authorize");
1356 let start = start(&activation, 0xa1, 0xb1, 1, 2);
1357 let head = journal(&activation, 3);
1358 let completion = NodeAttemptCompletion::fail(
1359 &start,
1360 failure(head.event_id(), 0xf1, RetryAdvice::Never),
1361 BudgetUsage::zero(),
1362 head,
1363 )
1364 .unwrap();
1365 let debug = format!("{completion:?}");
1366 assert!(!debug.contains("Node execution failed safely"));
1367 assert!(debug.contains("node.failed"));
1368 }
1369}