1use std::collections::{BTreeMap, VecDeque};
35use std::fmt;
36
37use serde::Serialize;
38
39use super::checkpoint::{
40 AcceptedCancellationState, AcceptedInputState, CanonicalInput, CheckpointCandidate,
41 CheckpointDraft, KernelCheckpoint, LaunchTokenState, LogicalKernelState,
42 LogicalStateProjection, ResolvedEffectState, TransitionState,
43};
44use super::command::{CancelCommand, HostCommand};
45use super::config::{ConfigDefaults, ResolvedOperationConfig, TailBounds};
46use super::effect::{Digest, EffectKind, EffectKindTag, EffectOutcome, KernelEffect, LaunchToken};
47use super::envelope::{OperationLifecycle, WireEnvelope, WireRejection, WireRejectionKind};
48use super::fault::{
49 KernelFault, KernelFaultCode, KernelPreparation, PrepareToken, PreparedTransition,
50 RejectedTransition, ReplayedTransition,
51};
52use super::record::{
53 ChainAnchor, KernelRecord, NormalizedInput, NormalizedPayload, RecordError, RecordPreparation,
54 canonical_bytes, canonical_digest, verify_record_chain,
55};
56use super::root::{ExecutionFocus, RootKind};
57use super::scalar::{EffectId, InputId, OperationId, WireU64};
58use super::terminal::{KernelTerminal, StepDisposition, TerminalSlot};
59
60pub trait TransitionStep: Serialize + Clone {
71 fn disposition(&self) -> &StepDisposition;
72
73 fn effects(&self) -> &[KernelEffect] {
74 self.disposition().effects()
75 }
76
77 fn terminal(&self) -> Option<&KernelTerminal> {
78 self.disposition().terminal()
79 }
80}
81
82pub trait RecordIndex {
93 fn record_for_input(
95 &self,
96 operation_id: &OperationId,
97 input_id: &InputId,
98 ) -> Option<KernelRecord>;
99
100 fn note_committed(&mut self, record: &KernelRecord) {
106 let _ = record;
107 }
108}
109
110#[derive(Debug, Clone, Default, PartialEq)]
112pub struct InMemoryRecordIndex {
113 records: BTreeMap<(OperationId, InputId), KernelRecord>,
114}
115
116impl InMemoryRecordIndex {
117 pub fn new() -> Self {
118 Self::default()
119 }
120
121 pub fn from_records(records: &[KernelRecord]) -> Self {
123 let mut index = Self::new();
124 for record in records {
125 index.note_committed(record);
126 }
127 index
128 }
129
130 pub fn len(&self) -> usize {
131 self.records.len()
132 }
133
134 pub fn is_empty(&self) -> bool {
135 self.records.is_empty()
136 }
137}
138
139impl RecordIndex for InMemoryRecordIndex {
140 fn record_for_input(
141 &self,
142 operation_id: &OperationId,
143 input_id: &InputId,
144 ) -> Option<KernelRecord> {
145 self.records
146 .get(&(operation_id.clone(), input_id.clone()))
147 .cloned()
148 }
149
150 fn note_committed(&mut self, record: &KernelRecord) {
151 self.records.insert(
152 (record.operation_id().clone(), record.input_id().clone()),
153 record.clone(),
154 );
155 }
156}
157
158#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
164pub struct TailUsage {
165 pub records: u64,
166 pub bytes: u64,
167}
168
169#[derive(Debug, Clone, Copy, PartialEq, Eq)]
172pub enum TailPressure {
173 Nominal,
174 Watermark,
176 Full,
178}
179
180#[derive(Debug, Clone, PartialEq)]
187struct TailEntry {
188 step_seq: WireU64,
189 record_digest: Digest,
190 bytes: u64,
191 input: NormalizedInput,
192}
193
194#[derive(Debug, Clone, PartialEq, Eq)]
200pub struct DurableHead {
201 pub digest: Digest,
202 pub step_seq: WireU64,
203}
204
205#[derive(Debug, Clone, PartialEq, Eq)]
211pub struct CheckpointBoundary {
212 pub through_step_seq: WireU64,
213 pub covered_head: Digest,
214}
215
216#[derive(Debug)]
220pub struct PlanContext<'a> {
221 pub input: &'a NormalizedInput,
222 pub step_seq: WireU64,
223 pub previous_head: Option<&'a Digest>,
224 pub config: &'a ResolvedOperationConfig,
225 pub resolving: Option<&'a KernelEffect>,
233}
234
235#[derive(Debug, Clone, PartialEq)]
237pub struct CommittedTransition<Step> {
238 pub record: KernelRecord,
239 pub step: Step,
240 pub step_seq: WireU64,
241 pub checkpoint_advice: Option<CheckpointAdvice>,
251}
252
253#[derive(Debug, Clone, Copy, PartialEq, Eq)]
260pub struct CheckpointAdvice {
261 pub through_step_seq: WireU64,
262 pub usage: TailUsage,
263 pub bounds: TailBounds,
264}
265
266impl<Step: TransitionStep> CommittedTransition<Step> {
267 pub fn published_effects(&self) -> &[KernelEffect] {
271 self.step.effects()
272 }
273
274 pub fn terminal(&self) -> Option<&KernelTerminal> {
275 self.step.terminal()
276 }
277}
278
279#[derive(Debug, Clone, PartialEq)]
280struct Candidate<Step> {
281 token: PrepareToken,
282 record: KernelRecord,
283 step: Step,
284 input: NormalizedInput,
285 record_bytes: u64,
286}
287
288struct InputFacts {
294 operation_id: OperationId,
295 input_id: InputId,
296 observed_at_ms: WireU64,
297 admissible_lifecycles: &'static [OperationLifecycle],
298}
299
300#[derive(Debug, Clone, PartialEq)]
301struct ResolvedEffectRecord {
302 outcome_digest: Digest,
303 input_id: InputId,
304 step_seq: WireU64,
305}
306
307#[derive(Debug, Clone, PartialEq)]
318pub struct KernelTransaction<Step, Index> {
319 defaults: ConfigDefaults,
320 bounds: TailBounds,
321 index: Index,
322 operation_id: Option<OperationId>,
323 genesis_digest: Option<Digest>,
326 config: Option<ResolvedOperationConfig>,
327 head: Option<ChainAnchor>,
333 lifecycle: OperationLifecycle,
334 terminal: TerminalSlot,
335 candidate: Option<Candidate<Step>>,
336 prepare_epoch: u64,
337 last_observed_at_ms: WireU64,
338 steps: BTreeMap<InputId, Step>,
350 accepted: BTreeMap<InputId, AcceptedInputState>,
358 replay_floor: Option<WireU64>,
364 pending_effects: BTreeMap<EffectId, KernelEffect>,
365 resolved_effects: BTreeMap<EffectId, ResolvedEffectRecord>,
366 launch_tokens: BTreeMap<LaunchToken, WireU64>,
367 accepted_cancellation: Option<AcceptedCancellation>,
375 tail: VecDeque<TailEntry>,
376 poison: Option<KernelFault>,
377}
378
379#[derive(Debug, Clone, PartialEq)]
381struct AcceptedCancellation {
382 command_digest: Digest,
385 input_id: InputId,
386 step_seq: WireU64,
387}
388
389impl<Step, Index> KernelTransaction<Step, Index>
390where
391 Step: TransitionStep,
392 Index: RecordIndex,
393{
394 pub fn new(defaults: ConfigDefaults, index: Index) -> Self {
401 let bounds = defaults.baseline.recovery_policy.tail_bounds;
402 Self {
403 defaults,
404 bounds,
405 index,
406 operation_id: None,
407 genesis_digest: None,
408 config: None,
409 head: None,
410 lifecycle: OperationLifecycle::Created,
411 terminal: TerminalSlot::empty(),
412 candidate: None,
413 prepare_epoch: 0,
414 last_observed_at_ms: WireU64::ZERO,
415 steps: BTreeMap::new(),
416 accepted: BTreeMap::new(),
417 replay_floor: None,
418 pending_effects: BTreeMap::new(),
419 resolved_effects: BTreeMap::new(),
420 launch_tokens: BTreeMap::new(),
421 accepted_cancellation: None,
422 tail: VecDeque::new(),
423 poison: None,
424 }
425 }
426
427 pub fn prepare<F>(&mut self, envelope: &WireEnvelope, plan: F) -> RecordPreparation<Step>
438 where
439 F: FnOnce(&PlanContext<'_>) -> Result<Step, KernelFault>,
440 {
441 match self.prepare_inner(envelope, plan) {
442 Ok(preparation) => preparation,
443 Err(fault) => KernelPreparation::Rejected(RejectedTransition { fault }),
444 }
445 }
446
447 fn prepare_inner<F>(
448 &mut self,
449 envelope: &WireEnvelope,
450 plan: F,
451 ) -> Result<RecordPreparation<Step>, KernelFault>
452 where
453 F: FnOnce(&PlanContext<'_>) -> Result<Step, KernelFault>,
454 {
455 self.check_preparable(&envelope.operation_id)?;
456 let input =
457 NormalizedInput::normalize(envelope, &self.defaults).map_err(rejection_fault)?;
458 self.prepare_normalized_inner(input, plan)
459 }
460
461 fn check_preparable(&self, operation_id: &OperationId) -> Result<(), KernelFault> {
463 if let Some(fault) = &self.poison {
464 return Err(fault.clone());
465 }
466
467 if let Some(candidate) = &self.candidate {
471 return Err(KernelFault::new(
472 KernelFaultCode::TransactionConflict,
473 format!(
474 "transaction candidate {} is still outstanding at step {}; commit or abort it \
475 before preparing another input",
476 candidate.token,
477 candidate.record.step_seq()
478 ),
479 ));
480 }
481
482 if let Some(bound) = &self.operation_id
483 && bound != operation_id
484 {
485 return Err(KernelFault::new(
486 KernelFaultCode::OperationMismatch,
487 format!(
488 "input belongs to operation {operation_id}, but this runtime is bound to \
489 {bound}"
490 ),
491 ));
492 }
493 Ok(())
494 }
495
496 fn prepare_normalized_inner<F>(
497 &mut self,
498 input: NormalizedInput,
499 plan: F,
500 ) -> Result<RecordPreparation<Step>, KernelFault>
501 where
502 F: FnOnce(&PlanContext<'_>) -> Result<Step, KernelFault>,
503 {
504 self.check_preparable(&input.operation_id)?;
505 let envelope = InputFacts {
506 operation_id: input.operation_id.clone(),
507 input_id: input.input_id.clone(),
508 observed_at_ms: input.observed_at_ms,
509 admissible_lifecycles: input.input.admissible_lifecycles(),
510 };
511 let envelope = &envelope;
512 let canonical_input = canonical_bytes(&input).map_err(record_fault)?;
513
514 if let Some(existing) = self
517 .index
518 .record_for_input(&envelope.operation_id, &envelope.input_id)
519 {
520 if existing.canonical_input() != &canonical_input {
521 return Err(KernelFault::new(
522 KernelFaultCode::DuplicateInputConflict,
523 format!(
524 "input {} was already accepted at step {} with a different payload",
525 envelope.input_id,
526 existing.step_seq()
527 ),
528 ));
529 }
530 return self.replay_of(existing);
531 }
532
533 if let Some(entry) = self.accepted.get(&envelope.input_id)
539 && self.below_replay_floor(entry.step_seq)
540 {
541 return Ok(self.replay_by_reference(entry));
542 }
543
544 if let Some(config) = &self.config
545 && canonical_input.len() > config.kernel_limits.max_input_bytes as usize
546 {
547 return Err(KernelFault::new(
548 KernelFaultCode::ResourceLimitExceeded,
549 format!(
550 "canonical input carries {} bytes; the operation limit is {}",
551 canonical_input.len(),
552 config.kernel_limits.max_input_bytes
553 ),
554 ));
555 }
556
557 if envelope.observed_at_ms.get() < self.last_observed_at_ms.get() {
558 return Err(KernelFault::new(
559 KernelFaultCode::ClockRegression,
560 format!(
561 "input observed at {} precedes the last accepted input at {}",
562 envelope.observed_at_ms, self.last_observed_at_ms
563 ),
564 ));
565 }
566
567 if let NormalizedPayload::HostControl(control) = &input.input
571 && let HostCommand::Cancel(cancel) = &control.command
572 && let Some(preparation) = self.cancel_guard(cancel)?
573 {
574 return Ok(preparation);
575 }
576
577 if !envelope.admissible_lifecycles.contains(&self.lifecycle) {
578 return Err(KernelFault::new(
579 KernelFaultCode::InvalidLifecycle,
580 format!(
581 "a {} input is not admissible while the operation is {:?}",
582 input.input.kind(),
583 self.lifecycle
584 ),
585 ));
586 }
587
588 if let NormalizedPayload::ResolveEffect(resolve) = &input.input
590 && let Some(preparation) = self.resolve_effect_guard(resolve)?
591 {
592 return Ok(preparation);
593 }
594
595 if self.tail_records() + 1 > self.bounds.hard_records.get() {
596 return Err(self.checkpoint_required(format!(
597 "the journal tail already holds {} records; its hard limit is {}",
598 self.tail_records(),
599 self.bounds.hard_records
600 )));
601 }
602
603 let step_seq = self.next_step_seq()?;
604 let genesis_config = input.resolved_config().cloned();
605 let config = match (&genesis_config, &self.config) {
606 (Some(config), _) => config,
607 (None, Some(config)) => config,
608 (None, None) => {
609 return Err(KernelFault::new(
610 KernelFaultCode::InvalidLifecycle,
611 "the operation has no genesis record, so it has no configuration to plan \
612 against"
613 .to_string(),
614 ));
615 }
616 };
617
618 let settled = match &input.input {
619 NormalizedPayload::ResolveEffect(resolve) => Some(&resolve.effect_id),
620 _ => None,
621 };
622 let resolving = settled.and_then(|effect_id| self.pending_effects.get(effect_id));
623 let step = plan(&PlanContext {
624 input: &input,
625 step_seq,
626 previous_head: self.head.as_ref().map(|anchor| &anchor.record_digest),
627 config,
628 resolving,
629 })?;
630
631 self.screen_planned_effects(&step, config, settled)?;
632
633 let record =
634 KernelRecord::chain_after(self.head.as_ref(), &input, &step).map_err(record_fault)?;
635 let record_bytes = record.record_bytes().len() as u64;
636 if self.tail_bytes() + record_bytes > self.bounds.hard_bytes.get() {
637 return Err(self.checkpoint_required(format!(
638 "the journal tail holds {} bytes and this record adds {record_bytes}; \
639 the hard limit is {}",
640 self.tail_bytes(),
641 self.bounds.hard_bytes
642 )));
643 }
644
645 self.prepare_epoch += 1;
647 let token = PrepareToken::new(format!(
648 "{}:prepare:{step_seq}:{}",
649 envelope.operation_id, self.prepare_epoch
650 ))
651 .expect("an operation-scoped prepare token is always a legal branded ref");
652 self.candidate = Some(Candidate {
653 token: token.clone(),
654 record: record.clone(),
655 step: step.clone(),
656 input,
657 record_bytes,
658 });
659
660 Ok(KernelPreparation::Prepared(PreparedTransition {
661 token,
662 record,
663 planned_step: step,
664 }))
665 }
666
667 pub fn commit(
678 &mut self,
679 token: &PrepareToken,
680 appended_head: &Digest,
681 ) -> Result<CommittedTransition<Step>, KernelFault> {
682 if let Some(fault) = &self.poison {
683 return Err(fault.clone());
684 }
685
686 let Some(candidate) = self.candidate.take() else {
687 return Err(self.poison_with(KernelFault::new(
688 KernelFaultCode::TransactionConflict,
689 format!(
690 "commit reports a durable append for token {token}, but no candidate is \
691 outstanding; a committed record is never re-committed and never aborted"
692 ),
693 )));
694 };
695
696 if &candidate.token != token {
697 let outstanding = candidate.token.clone();
698 return Err(self.poison_with(KernelFault::new(
699 KernelFaultCode::TransactionConflict,
700 format!(
701 "commit names token {token}, but the outstanding candidate is {outstanding}; \
702 the runtime no longer describes what the journal holds"
703 ),
704 )));
705 }
706
707 if appended_head != candidate.record.record_digest() {
708 let expected = candidate.record.record_digest().clone();
709 return Err(self.poison_with(KernelFault::new(
710 KernelFaultCode::TransactionConflict,
711 format!(
712 "the journal head after the append is {appended_head}, but this candidate is \
713 {expected}; the append did not place this record, so the runtime must be \
714 rebuilt from the journal"
715 ),
716 )));
717 }
718
719 let Candidate {
720 record,
721 step,
722 input,
723 record_bytes,
724 ..
725 } = candidate;
726 match self.integrate(record, step, &input, record_bytes) {
727 Ok(committed) => Ok(committed),
728 Err(fault) => Err(self.poison_with(fault)),
729 }
730 }
731
732 pub fn abort(&mut self, token: &PrepareToken) -> Result<KernelRecord, KernelFault> {
740 if let Some(fault) = &self.poison {
741 return Err(fault.clone());
742 }
743
744 let Some(candidate) = &self.candidate else {
745 return Err(KernelFault::new(
746 KernelFaultCode::TransactionConflict,
747 format!(
748 "no candidate is outstanding for token {token}; a record that reached the \
749 journal is never abortable"
750 ),
751 ));
752 };
753 if &candidate.token != token {
754 return Err(KernelFault::new(
755 KernelFaultCode::TransactionConflict,
756 format!(
757 "token {token} does not name the outstanding candidate {}",
758 candidate.token
759 ),
760 ));
761 }
762
763 let candidate = self.candidate.take().expect("checked just above");
764 Ok(candidate.record)
765 }
766
767 pub fn note_append_conflict(
774 &mut self,
775 token: &PrepareToken,
776 observed_head: Option<&Digest>,
777 ) -> KernelFault {
778 let expected = self
779 .candidate
780 .as_ref()
781 .and_then(|candidate| candidate.record.expected_head().cloned());
782 self.candidate = None;
783 let observed =
784 observed_head.map_or_else(|| "an empty journal".to_string(), Digest::to_string);
785 let expected =
786 expected.map_or_else(|| "an empty journal".to_string(), |head| head.to_string());
787 let fault = KernelFault::new(
788 KernelFaultCode::TransactionConflict,
789 format!(
790 "the CAS append for token {token} expected head {expected} but the journal holds \
791 {observed}; the candidate is discarded and this runtime must be rebuilt from the \
792 journal before the input is replayed"
793 ),
794 );
795 self.poison_with(fault)
796 }
797
798 pub fn rebuild_from_records<F>(
808 records: &[KernelRecord],
809 defaults: ConfigDefaults,
810 index: Index,
811 mut plan: F,
812 ) -> Result<Self, KernelFault>
813 where
814 F: FnMut(&PlanContext<'_>) -> Result<Step, KernelFault>,
815 {
816 let mut transaction = Self::new(defaults, index);
817 if records.is_empty() {
818 return Ok(transaction);
819 }
820 verify_record_chain(records).map_err(corrupt_chain_fault)?;
821
822 for record in records {
823 let input = record.normalized_input().map_err(record_fault)?;
824 transaction.replay_committed(&input, record.record_digest(), &mut plan)?;
825 }
826 Ok(transaction)
827 }
828
829 pub fn replay_committed<F>(
843 &mut self,
844 input: &NormalizedInput,
845 expected_record_digest: &Digest,
846 plan: &mut F,
847 ) -> Result<KernelRecord, KernelFault>
848 where
849 F: FnMut(&PlanContext<'_>) -> Result<Step, KernelFault>,
850 {
851 if let Some(fault) = &self.poison {
852 return Err(fault.clone());
853 }
854 let step_seq = self.next_step_seq()?;
855 let genesis_config = input.resolved_config().cloned();
856 let config = match (&genesis_config, &self.config) {
857 (Some(config), _) => config,
858 (None, Some(config)) => config,
859 (None, None) => {
860 return Err(KernelFault::new(
861 KernelFaultCode::RecordCorrupted,
862 format!(
863 "the transition at step {step_seq} has no genesis configuration before it"
864 ),
865 ));
866 }
867 };
868
869 let resolving = match &input.input {
873 NormalizedPayload::ResolveEffect(resolve) => {
874 self.pending_effects.get(&resolve.effect_id)
875 }
876 _ => None,
877 };
878 let step = plan(&PlanContext {
879 input,
880 step_seq,
881 previous_head: self.head.as_ref().map(|anchor| &anchor.record_digest),
882 config,
883 resolving,
884 })?;
885
886 let rebuilt = KernelRecord::chain_after(self.head.as_ref(), input, &step)
887 .map_err(corrupt_chain_fault)?;
888 if rebuilt.record_digest() != expected_record_digest {
889 return Err(KernelFault::new(
890 KernelFaultCode::RecordCorrupted,
891 format!(
892 "replaying the transition at step {step_seq} produced record digest {} against \
893 the durable {expected_record_digest}; this binary does not reproduce the \
894 history it is resuming",
895 rebuilt.record_digest(),
896 ),
897 ));
898 }
899
900 let bytes = rebuilt.record_bytes().len() as u64;
901 self.integrate(rebuilt.clone(), step, input, bytes)?;
902 Ok(rebuilt)
903 }
904
905 pub fn restore_from_checkpoint(
920 checkpoint: &KernelCheckpoint,
921 defaults: ConfigDefaults,
922 index: Index,
923 ) -> Result<Self, KernelFault> {
924 let state = checkpoint.logical_state();
925 let transition = &state.transition;
926 let mut transaction = Self::new(defaults, index);
927
928 transaction.operation_id = Some(checkpoint.operation_id().clone());
929 transaction.genesis_digest = Some(checkpoint.genesis_digest().clone());
930 transaction.bounds = transition.resolved_config.recovery_policy.tail_bounds;
931 transaction.config = Some(transition.resolved_config.clone());
932 transaction.head = Some(ChainAnchor {
933 operation_id: checkpoint.operation_id().clone(),
934 step_seq: checkpoint.base_step_seq(),
935 record_digest: checkpoint.base_record_digest().clone(),
936 });
937 transaction.lifecycle = transition.lifecycle;
938 transaction.last_observed_at_ms = transition.last_observed_at_ms;
939 transaction.replay_floor = Some(checkpoint.base_step_seq());
940
941 if let Some(terminal) = &transition.terminal {
942 transaction
943 .terminal
944 .commit(terminal.clone())
945 .map_err(|error| {
946 KernelFault::new(KernelFaultCode::CheckpointCorrupted, error.to_string())
947 })?;
948 }
949 transaction.pending_effects = transition
950 .pending_effects
951 .iter()
952 .map(|effect| (effect.effect_id.clone(), effect.clone()))
953 .collect();
954 transaction.resolved_effects = transition
955 .resolved_effects
956 .iter()
957 .map(|resolved| {
958 (
959 resolved.effect_id.clone(),
960 ResolvedEffectRecord {
961 outcome_digest: resolved.outcome_digest.clone(),
962 input_id: resolved.input_id.clone(),
963 step_seq: resolved.step_seq,
964 },
965 )
966 })
967 .collect();
968 transaction.launch_tokens = transition
969 .launch_tokens
970 .iter()
971 .map(|token| (token.launch_token.clone(), token.step_seq))
972 .collect();
973 transaction.accepted = transition
974 .accepted_inputs
975 .iter()
976 .map(|entry| (entry.input_id.clone(), entry.clone()))
977 .collect();
978 transaction.accepted_cancellation =
979 transition
980 .accepted_cancellation
981 .as_ref()
982 .map(|cancellation| AcceptedCancellation {
983 command_digest: cancellation.command_digest.clone(),
984 input_id: cancellation.input_id.clone(),
985 step_seq: cancellation.step_seq,
986 });
987 Ok(transaction)
988 }
989
990 pub fn checkpoint_boundary(&self) -> Option<CheckpointBoundary> {
997 self.head.as_ref().map(|head| CheckpointBoundary {
998 through_step_seq: head.step_seq,
999 covered_head: head.record_digest.clone(),
1000 })
1001 }
1002
1003 pub fn tail_inputs(&self) -> Vec<CanonicalInput> {
1009 self.tail
1010 .iter()
1011 .map(|entry| CanonicalInput {
1012 step_seq: entry.step_seq,
1013 record_digest: entry.record_digest.clone(),
1014 input: entry.input.clone(),
1015 })
1016 .collect()
1017 }
1018
1019 pub fn checkpoint_candidate(
1030 &self,
1031 projection: LogicalStateProjection,
1032 ) -> Result<CheckpointCandidate, KernelFault> {
1033 let head = self.require_head()?.clone();
1034 self.assemble_checkpoint(
1035 projection,
1036 head.step_seq,
1037 head.record_digest.clone(),
1038 head.step_seq,
1039 head.record_digest,
1040 Vec::new(),
1041 )
1042 }
1043
1044 pub fn checkpoint_rebase(
1058 &self,
1059 base: &CheckpointBoundary,
1060 base_state: LogicalKernelState,
1061 ) -> Result<CheckpointCandidate, KernelFault> {
1062 let head = self.require_head()?.clone();
1063 if base.through_step_seq > head.step_seq {
1064 return Err(KernelFault::new(
1065 KernelFaultCode::CheckpointIncompatible,
1066 format!(
1067 "a rebase bases at step {} but this journal's head is step {}",
1068 base.through_step_seq, head.step_seq
1069 ),
1070 ));
1071 }
1072 let tail: Vec<CanonicalInput> = self
1073 .tail_inputs()
1074 .into_iter()
1075 .filter(|entry| entry.step_seq.get() > base.through_step_seq.get())
1076 .collect();
1077 if tail.len() as u64 != head.step_seq.get() - base.through_step_seq.get() {
1078 return Err(KernelFault::new(
1079 KernelFaultCode::CheckpointIncompatible,
1080 format!(
1081 "a rebase over ({}, {}] needs {} tail inputs, but this runtime's tail holds \
1082 {} of them — the prefix it would rebase onto was already reclaimed",
1083 base.through_step_seq,
1084 head.step_seq,
1085 head.step_seq.get() - base.through_step_seq.get(),
1086 tail.len()
1087 ),
1088 ));
1089 }
1090
1091 let operation_id = self.require_operation()?;
1092 let genesis_digest = self.require_genesis()?;
1093 let checkpoint = KernelCheckpoint::assemble(CheckpointDraft {
1094 operation_id: operation_id.clone(),
1095 genesis_digest: genesis_digest.clone(),
1096 base_step_seq: base.through_step_seq,
1097 base_record_digest: base.covered_head.clone(),
1098 through_step_seq: head.step_seq,
1099 covered_transaction_head_digest: head.record_digest,
1100 logical_state: base_state,
1101 tail_inputs: tail,
1102 })
1103 .map_err(|error| error.fault())?;
1104 Ok(checkpoint.into_candidate())
1105 }
1106
1107 #[allow(clippy::too_many_arguments)]
1108 fn assemble_checkpoint(
1109 &self,
1110 projection: LogicalStateProjection,
1111 base_step_seq: WireU64,
1112 base_record_digest: Digest,
1113 through_step_seq: WireU64,
1114 covered_transaction_head_digest: Digest,
1115 tail_inputs: Vec<CanonicalInput>,
1116 ) -> Result<CheckpointCandidate, KernelFault> {
1117 let operation_id = self.require_operation()?.clone();
1118 let genesis_digest = self.require_genesis()?.clone();
1119 let LogicalStateProjection {
1120 root_kind,
1121 focus,
1122 syscall,
1123 scheduler,
1124 context_vm,
1125 } = projection;
1126 let logical_state = LogicalKernelState {
1127 transition: self.transition_state(root_kind, focus)?,
1128 syscall,
1129 scheduler,
1130 context_vm,
1131 };
1132 let checkpoint = KernelCheckpoint::assemble(CheckpointDraft {
1133 operation_id,
1134 genesis_digest,
1135 base_step_seq,
1136 base_record_digest,
1137 through_step_seq,
1138 covered_transaction_head_digest,
1139 logical_state,
1140 tail_inputs,
1141 })
1142 .map_err(|error| error.fault())?;
1143 Ok(checkpoint.into_candidate())
1144 }
1145
1146 fn require_head(&self) -> Result<&ChainAnchor, KernelFault> {
1147 if let Some(fault) = &self.poison {
1148 return Err(fault.clone());
1149 }
1150 self.head.as_ref().ok_or_else(|| {
1151 KernelFault::new(
1152 KernelFaultCode::InvalidLifecycle,
1153 "an operation with no genesis record has no logical state to checkpoint"
1154 .to_string(),
1155 )
1156 })
1157 }
1158
1159 fn require_operation(&self) -> Result<&OperationId, KernelFault> {
1160 self.operation_id.as_ref().ok_or_else(|| {
1161 KernelFault::new(
1162 KernelFaultCode::InvalidLifecycle,
1163 "an unbound operation has no logical state to checkpoint".to_string(),
1164 )
1165 })
1166 }
1167
1168 fn require_genesis(&self) -> Result<&Digest, KernelFault> {
1169 self.genesis_digest.as_ref().ok_or_else(|| {
1170 KernelFault::new(
1171 KernelFaultCode::InvalidLifecycle,
1172 "an operation with no genesis record has no identity to bind a checkpoint to"
1173 .to_string(),
1174 )
1175 })
1176 }
1177
1178 pub fn transition_state_for_restore(
1184 &self,
1185 root_kind: Option<RootKind>,
1186 focus: Option<ExecutionFocus>,
1187 ) -> Result<TransitionState, KernelFault> {
1188 self.transition_state(root_kind, focus)
1189 }
1190
1191 fn transition_state(
1197 &self,
1198 root_kind: Option<RootKind>,
1199 focus: Option<ExecutionFocus>,
1200 ) -> Result<TransitionState, KernelFault> {
1201 let config = self.config.as_ref().ok_or_else(|| {
1202 KernelFault::new(
1203 KernelFaultCode::InvalidLifecycle,
1204 "an operation with no genesis record has no resolved configuration to checkpoint"
1205 .to_string(),
1206 )
1207 })?;
1208
1209 Ok(TransitionState {
1210 lifecycle: self.lifecycle,
1211 resolved_config: config.clone(),
1212 root_kind,
1213 focus,
1214 last_observed_at_ms: self.last_observed_at_ms,
1215 pending_effects: self.pending_effects.values().cloned().collect(),
1216 resolved_effects: self
1217 .resolved_effects
1218 .iter()
1219 .map(|(effect_id, resolved)| ResolvedEffectState {
1220 effect_id: effect_id.clone(),
1221 outcome_digest: resolved.outcome_digest.clone(),
1222 input_id: resolved.input_id.clone(),
1223 step_seq: resolved.step_seq,
1224 })
1225 .collect(),
1226 launch_tokens: self
1227 .launch_tokens
1228 .iter()
1229 .map(|(launch_token, step_seq)| LaunchTokenState {
1230 launch_token: launch_token.clone(),
1231 step_seq: *step_seq,
1232 })
1233 .collect(),
1234 accepted_inputs: self.accepted.values().cloned().collect(),
1235 accepted_cancellation: self.accepted_cancellation.as_ref().map(|cancellation| {
1236 AcceptedCancellationState {
1237 command_digest: cancellation.command_digest.clone(),
1238 input_id: cancellation.input_id.clone(),
1239 step_seq: cancellation.step_seq,
1240 }
1241 }),
1242 terminal: self.terminal.get().cloned(),
1243 })
1244 }
1245
1246 pub fn note_checkpoint_acked(
1256 &mut self,
1257 boundary: &CheckpointBoundary,
1258 ) -> Result<TailUsage, KernelFault> {
1259 if let Some(fault) = &self.poison {
1260 return Err(fault.clone());
1261 }
1262 let matches = self.tail.iter().any(|entry| {
1263 entry.step_seq == boundary.through_step_seq
1264 && entry.record_digest == boundary.covered_head
1265 });
1266 if !matches {
1267 return Err(KernelFault::new(
1268 KernelFaultCode::CheckpointIncompatible,
1269 format!(
1270 "no tail record at step {} has digest {}; this checkpoint does not cover a \
1271 prefix of this journal",
1272 boundary.through_step_seq, boundary.covered_head
1273 ),
1274 ));
1275 }
1276 while let Some(entry) = self.tail.front() {
1277 if entry.step_seq.get() <= boundary.through_step_seq.get() {
1278 self.tail.pop_front();
1279 } else {
1280 break;
1281 }
1282 }
1283 Ok(self.tail_usage())
1284 }
1285
1286 pub fn operation_id(&self) -> Option<&OperationId> {
1289 self.operation_id.as_ref()
1290 }
1291
1292 pub fn config(&self) -> Option<&ResolvedOperationConfig> {
1293 self.config.as_ref()
1294 }
1295
1296 pub fn head(&self) -> Option<DurableHead> {
1297 self.head.as_ref().map(|anchor| DurableHead {
1298 digest: anchor.record_digest.clone(),
1299 step_seq: anchor.step_seq,
1300 })
1301 }
1302
1303 pub fn lifecycle(&self) -> OperationLifecycle {
1304 self.lifecycle
1305 }
1306
1307 pub fn terminal(&self) -> Option<&KernelTerminal> {
1308 self.terminal.get()
1309 }
1310
1311 pub fn pending_effects(&self) -> impl Iterator<Item = &KernelEffect> {
1314 self.pending_effects.values()
1315 }
1316
1317 pub fn is_effect_resolved(&self, effect_id: &EffectId) -> bool {
1318 self.resolved_effects.contains_key(effect_id)
1319 }
1320
1321 pub fn knows_launch_token(&self, token: &LaunchToken) -> bool {
1322 self.launch_tokens.contains_key(token)
1323 }
1324
1325 pub fn committed_step(&self, input_id: &InputId) -> Option<&Step> {
1326 self.steps.get(input_id)
1327 }
1328
1329 pub fn outstanding_token(&self) -> Option<&PrepareToken> {
1330 self.candidate.as_ref().map(|candidate| &candidate.token)
1331 }
1332
1333 pub fn has_candidate(&self) -> bool {
1334 self.candidate.is_some()
1335 }
1336
1337 pub fn poison(&self) -> Option<&KernelFault> {
1340 self.poison.as_ref()
1341 }
1342
1343 pub fn is_poisoned(&self) -> bool {
1344 self.poison.is_some()
1345 }
1346
1347 pub fn bounds(&self) -> TailBounds {
1348 self.bounds
1349 }
1350
1351 pub fn index(&self) -> &Index {
1352 &self.index
1353 }
1354
1355 pub fn tail_usage(&self) -> TailUsage {
1356 TailUsage {
1357 records: self.tail_records(),
1358 bytes: self.tail_bytes(),
1359 }
1360 }
1361
1362 pub fn tail_pressure(&self) -> TailPressure {
1363 let usage = self.tail_usage();
1364 if usage.records >= self.bounds.hard_records.get()
1365 || usage.bytes >= self.bounds.hard_bytes.get()
1366 {
1367 TailPressure::Full
1368 } else if usage.records >= self.bounds.soft_records.get()
1369 || usage.bytes >= self.bounds.soft_bytes.get()
1370 {
1371 TailPressure::Watermark
1372 } else {
1373 TailPressure::Nominal
1374 }
1375 }
1376
1377 fn tail_records(&self) -> u64 {
1380 self.tail.len() as u64
1381 }
1382
1383 fn tail_bytes(&self) -> u64 {
1384 self.tail.iter().map(|entry| entry.bytes).sum()
1385 }
1386
1387 fn next_step_seq(&self) -> Result<WireU64, KernelFault> {
1388 match &self.head {
1389 None => Ok(WireU64::ZERO),
1390 Some(head) => head
1391 .step_seq
1392 .get()
1393 .checked_add(1)
1394 .map(WireU64::new)
1395 .ok_or_else(|| {
1396 KernelFault::new(
1397 KernelFaultCode::ResourceLimitExceeded,
1398 "step sequence overflowed u64".to_string(),
1399 )
1400 }),
1401 }
1402 }
1403
1404 fn checkpoint_required(&self, detail: String) -> KernelFault {
1405 KernelFault::new(
1406 KernelFaultCode::CheckpointRequired,
1407 format!(
1408 "{detail}; take a checkpoint candidate, install and ack it, then retry this input \
1409 unchanged — it was never accepted"
1410 ),
1411 )
1412 }
1413
1414 fn poison_with(&mut self, fault: KernelFault) -> KernelFault {
1415 self.candidate = None;
1416 self.poison.get_or_insert(fault).clone()
1417 }
1418
1419 fn replay_of(&self, existing: KernelRecord) -> Result<RecordPreparation<Step>, KernelFault> {
1427 let step_seq = existing.step_seq();
1428 let record_digest = existing.record_digest().clone();
1429 match self.steps.get(existing.input_id()) {
1430 Some(step) => {
1431 existing.verify_step(step).map_err(record_fault)?;
1432 Ok(KernelPreparation::Replayed(ReplayedTransition {
1433 record: Some(existing),
1434 record_digest,
1435 committed_step: Some(step.clone()),
1436 step_seq,
1437 }))
1438 }
1439 None if self.below_replay_floor(step_seq) => {
1440 Ok(KernelPreparation::Replayed(ReplayedTransition {
1441 record: Some(existing),
1442 record_digest,
1443 committed_step: None,
1444 step_seq,
1445 }))
1446 }
1447 None => Err(KernelFault::new(
1448 KernelFaultCode::RecordCorrupted,
1449 format!(
1450 "the journal holds record {record_digest} at step {step_seq} for input {}, but \
1451 this runtime has not replayed it and cannot reproduce its step; rebuild from \
1452 the journal first",
1453 existing.input_id()
1454 ),
1455 )),
1456 }
1457 }
1458
1459 fn replay_by_reference(&self, entry: &AcceptedInputState) -> RecordPreparation<Step> {
1462 KernelPreparation::Replayed(ReplayedTransition {
1463 record: None,
1464 record_digest: entry.record_digest.clone(),
1465 committed_step: None,
1466 step_seq: entry.step_seq,
1467 })
1468 }
1469
1470 fn below_replay_floor(&self, step_seq: WireU64) -> bool {
1471 self.replay_floor
1472 .is_some_and(|floor| step_seq.get() <= floor.get())
1473 }
1474
1475 fn resolve_effect_guard(
1480 &self,
1481 resolve: &super::envelope::ResolveEffect,
1482 ) -> Result<Option<RecordPreparation<Step>>, KernelFault> {
1484 let outcome_digest = outcome_digest(&resolve.outcome)?;
1485 if let Some(resolved) = self.resolved_effects.get(&resolve.effect_id) {
1486 if resolved.outcome_digest != outcome_digest {
1487 return Err(KernelFault::new(
1488 KernelFaultCode::UnexpectedEffectOutcome,
1489 format!(
1490 "effect {} was already resolved at step {} with a different outcome",
1491 resolve.effect_id, resolved.step_seq
1492 ),
1493 ));
1494 }
1495 let Some(existing) = self.index.record_for_input(
1498 self.operation_id.as_ref().expect("bound by the genesis"),
1499 &resolved.input_id,
1500 ) else {
1501 return Err(KernelFault::new(
1502 KernelFaultCode::RecordCorrupted,
1503 format!(
1504 "effect {} is resolved by input {} at step {}, but the journal has no such \
1505 record",
1506 resolve.effect_id, resolved.input_id, resolved.step_seq
1507 ),
1508 ));
1509 };
1510 return self.replay_of(existing).map(Some);
1511 }
1512
1513 let Some(pending) = self.pending_effects.get(&resolve.effect_id) else {
1514 return Err(KernelFault::new(
1515 KernelFaultCode::UnexpectedEffectOutcome,
1516 format!(
1517 "effect {} is not pending; the kernel is not waiting on it",
1518 resolve.effect_id
1519 ),
1520 ));
1521 };
1522 pending.accept_outcome(&resolve.outcome).map_err(|error| {
1523 KernelFault::new(KernelFaultCode::UnexpectedEffectOutcome, error.to_string())
1524 })?;
1525 Ok(None)
1526 }
1527
1528 fn cancel_guard(
1536 &self,
1537 cancel: &CancelCommand,
1538 ) -> Result<Option<RecordPreparation<Step>>, KernelFault> {
1539 let Some(accepted) = &self.accepted_cancellation else {
1540 return Ok(None);
1541 };
1542 if accepted.command_digest != cancel_digest(cancel)? {
1543 return Err(KernelFault::new(
1544 KernelFaultCode::DuplicateInputConflict,
1545 format!(
1546 "this operation committed a different cancellation at step {}; a cancel is not \
1547 re-decided once the terminal it produced exists",
1548 accepted.step_seq
1549 ),
1550 ));
1551 }
1552 let Some(existing) = self.index.record_for_input(
1553 self.operation_id.as_ref().expect("bound by the genesis"),
1554 &accepted.input_id,
1555 ) else {
1556 return Err(KernelFault::new(
1557 KernelFaultCode::RecordCorrupted,
1558 format!(
1559 "this operation was cancelled by input {} at step {}, but the journal has no \
1560 such record",
1561 accepted.input_id, accepted.step_seq
1562 ),
1563 ));
1564 };
1565 self.replay_of(existing).map(Some)
1566 }
1567
1568 fn screen_planned_effects(
1571 &self,
1572 step: &Step,
1573 config: &ResolvedOperationConfig,
1574 settled: Option<&EffectId>,
1575 ) -> Result<(), KernelFault> {
1576 let mut kinds_in_step: Vec<EffectKindTag> = Vec::new();
1577 let mut tokens_in_step: Vec<&LaunchToken> = Vec::new();
1578 let still_pending = |effect_id: &EffectId| {
1582 self.pending_effects.contains_key(effect_id) && Some(effect_id) != settled
1583 };
1584
1585 for effect in step.effects() {
1586 let tag = effect.tag();
1587
1588 if !config.host_effect_support.supports(tag) {
1590 return Err(KernelFault::new(
1591 KernelFaultCode::UnsupportedEffect,
1592 format!(
1593 "this operation's host does not declare support for {tag} effects, so \
1594 effect {} is refused before emission",
1595 effect.effect_id
1596 ),
1597 ));
1598 }
1599
1600 if self.pending_effects.contains_key(&effect.effect_id)
1603 || self.resolved_effects.contains_key(&effect.effect_id)
1604 || Some(&effect.effect_id) == settled
1605 {
1606 return Err(KernelFault::new(
1607 KernelFaultCode::TransactionConflict,
1608 format!(
1609 "effect id {} was already published; effect identity is minted once",
1610 effect.effect_id
1611 ),
1612 ));
1613 }
1614
1615 if kinds_in_step.contains(&tag)
1617 || self
1618 .pending_effects
1619 .iter()
1620 .any(|(id, pending)| pending.tag() == tag && still_pending(id))
1621 {
1622 return Err(KernelFault::new(
1623 KernelFaultCode::ResourceLimitExceeded,
1624 format!(
1625 "a {tag} effect is already pending; resolve it before emitting another \
1626 (§15.3 admits at most one pending effect per kind)"
1627 ),
1628 ));
1629 }
1630 kinds_in_step.push(tag);
1631
1632 if let EffectKind::SpawnTasks(spawn) = &effect.effect {
1633 for launch in &spawn.tasks {
1634 if self.launch_tokens.contains_key(&launch.launch_token)
1635 || tokens_in_step.contains(&&launch.launch_token)
1636 {
1637 return Err(KernelFault::new(
1638 KernelFaultCode::TransactionConflict,
1639 format!(
1640 "launch token {} was already published; a re-launch reuses the \
1641 committed token so the host's launch dedup stays exact",
1642 launch.launch_token
1643 ),
1644 ));
1645 }
1646 tokens_in_step.push(&launch.launch_token);
1647 }
1648 }
1649 }
1650 Ok(())
1651 }
1652
1653 fn integrate(
1656 &mut self,
1657 record: KernelRecord,
1658 step: Step,
1659 input: &NormalizedInput,
1660 record_bytes: u64,
1661 ) -> Result<CommittedTransition<Step>, KernelFault> {
1662 let step_seq = record.step_seq();
1663 let was_nominal = self.tail_pressure() == TailPressure::Nominal;
1664
1665 if self.operation_id.is_none() {
1666 self.operation_id = Some(record.operation_id().clone());
1667 }
1668 if record.is_genesis() {
1669 self.genesis_digest = Some(record.record_digest().clone());
1670 }
1671 if let Some(config) = input.resolved_config() {
1672 self.bounds = config.recovery_policy.tail_bounds;
1677 self.config = Some(config.clone());
1678 self.lifecycle = OperationLifecycle::Configured;
1679 } else if matches!(input.input, NormalizedPayload::StartOperation(_)) {
1680 self.lifecycle = OperationLifecycle::Running;
1681 }
1682
1683 if let NormalizedPayload::HostControl(control) = &input.input
1684 && let HostCommand::Cancel(cancel) = &control.command
1685 {
1686 self.accepted_cancellation = Some(AcceptedCancellation {
1687 command_digest: cancel_digest(cancel)?,
1688 input_id: record.input_id().clone(),
1689 step_seq,
1690 });
1691 }
1692
1693 if let NormalizedPayload::ResolveEffect(resolve) = &input.input {
1694 self.pending_effects.remove(&resolve.effect_id);
1695 self.resolved_effects.insert(
1696 resolve.effect_id.clone(),
1697 ResolvedEffectRecord {
1698 outcome_digest: outcome_digest(&resolve.outcome)?,
1699 input_id: record.input_id().clone(),
1700 step_seq,
1701 },
1702 );
1703 }
1704
1705 for effect in step.effects() {
1706 self.pending_effects
1707 .insert(effect.effect_id.clone(), effect.clone());
1708 if let EffectKind::SpawnTasks(spawn) = &effect.effect {
1709 for launch in &spawn.tasks {
1710 self.launch_tokens
1711 .insert(launch.launch_token.clone(), step_seq);
1712 }
1713 }
1714 }
1715
1716 if let Some(terminal) = step.terminal() {
1717 self.terminal.commit(terminal.clone()).map_err(|error| {
1718 KernelFault::new(KernelFaultCode::InvalidLifecycle, error.to_string())
1719 })?;
1720 self.lifecycle = terminal_lifecycle(terminal);
1721 self.pending_effects.clear();
1723 }
1724
1725 self.last_observed_at_ms = input.observed_at_ms;
1726 self.steps.insert(record.input_id().clone(), step.clone());
1727 self.accepted.insert(
1728 record.input_id().clone(),
1729 AcceptedInputState {
1730 input_id: record.input_id().clone(),
1731 step_seq,
1732 record_digest: record.record_digest().clone(),
1733 },
1734 );
1735 self.tail.push_back(TailEntry {
1736 step_seq,
1737 record_digest: record.record_digest().clone(),
1738 bytes: record_bytes,
1739 input: input.clone(),
1740 });
1741 self.index.note_committed(&record);
1742 self.head = Some(record.anchor());
1743
1744 let checkpoint_advice = (was_nominal && self.tail_pressure() != TailPressure::Nominal)
1747 .then(|| CheckpointAdvice {
1748 through_step_seq: step_seq,
1749 usage: self.tail_usage(),
1750 bounds: self.bounds,
1751 });
1752
1753 Ok(CommittedTransition {
1754 record,
1755 step,
1756 step_seq,
1757 checkpoint_advice,
1758 })
1759 }
1760}
1761
1762fn outcome_digest(outcome: &EffectOutcome) -> Result<Digest, KernelFault> {
1767 canonical_bytes(outcome)
1768 .map(|bytes| canonical_digest(bytes.as_slice()))
1769 .map_err(record_fault)
1770}
1771
1772fn cancel_digest(cancel: &CancelCommand) -> Result<Digest, KernelFault> {
1773 canonical_bytes(cancel)
1774 .map(|bytes| canonical_digest(bytes.as_slice()))
1775 .map_err(record_fault)
1776}
1777
1778fn record_fault(error: RecordError) -> KernelFault {
1779 KernelFault::new(error.code(), error.message().to_string())
1780}
1781
1782fn corrupt_chain_fault(error: RecordError) -> KernelFault {
1785 match error {
1786 RecordError::ChainBroken(message) => {
1787 KernelFault::new(KernelFaultCode::RecordCorrupted, message)
1788 }
1789 other => record_fault(other),
1790 }
1791}
1792
1793fn rejection_fault(rejection: WireRejection) -> KernelFault {
1794 let code = match rejection.kind {
1795 WireRejectionKind::PolicyViolation => KernelFaultCode::InvalidConfig,
1796 _ => KernelFaultCode::MalformedEnvelope,
1797 };
1798 KernelFault::new(code, rejection.message)
1799}
1800
1801fn terminal_lifecycle(terminal: &KernelTerminal) -> OperationLifecycle {
1802 match terminal {
1803 KernelTerminal::Agent(_) | KernelTerminal::Workflow(_) => OperationLifecycle::Completed,
1804 KernelTerminal::Cancelled(_) => OperationLifecycle::Cancelled,
1805 KernelTerminal::Failed(_) => OperationLifecycle::Failed,
1806 }
1807}
1808
1809impl fmt::Display for CheckpointBoundary {
1810 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1811 write!(
1812 f,
1813 "through step {} at head {}",
1814 self.through_step_seq, self.covered_head
1815 )
1816 }
1817}
1818
1819#[cfg(test)]
1820mod tests {
1821 use serde::Serialize;
1822
1823 use super::super::*;
1824
1825 const OPERATION: &str = "op-tx-1";
1830
1831 fn operation() -> OperationId {
1832 OperationId::new(OPERATION).unwrap()
1833 }
1834
1835 fn input_id(name: &str) -> InputId {
1836 InputId::new(name).unwrap()
1837 }
1838
1839 fn boot_config(supported: impl IntoIterator<Item = EffectKindTag>) -> OperationConfig {
1840 OperationConfig {
1841 execution_policy: Some(ExecutionPolicy {
1842 max_turns: Some(12),
1843 ..ExecutionPolicy::default()
1844 }),
1845 host_effect_support: HostEffectSupport::new(supported),
1846 ..OperationConfig::default()
1847 }
1848 }
1849
1850 fn envelope(id: &str, observed_at_ms: u64, input: KernelInput) -> WireEnvelope {
1851 WireEnvelope::new(
1852 operation(),
1853 input_id(id),
1854 WireU64::new(observed_at_ms),
1855 input,
1856 )
1857 }
1858
1859 fn configure_at(id: &str, supported: impl IntoIterator<Item = EffectKindTag>) -> WireEnvelope {
1860 envelope(
1861 id,
1862 1_700_000_000_000,
1863 KernelInput::ConfigureOperation(ConfigureOperation {
1864 config: boot_config(supported),
1865 }),
1866 )
1867 }
1868
1869 fn configure() -> WireEnvelope {
1870 configure_at(
1871 "in-configure",
1872 [EffectKindTag::CallProvider, EffectKindTag::SpawnTasks],
1873 )
1874 }
1875
1876 fn start_at(id: &str, observed_at_ms: u64) -> WireEnvelope {
1877 envelope(
1878 id,
1879 observed_at_ms,
1880 KernelInput::StartOperation(StartOperation {
1881 entry: RootEntry::Agent(RootAgentEntry {
1882 task: LogicalTask::new("write the brief"),
1883 run_spec: None,
1884 }),
1885 initial_context: InitialContext::default(),
1886 }),
1887 )
1888 }
1889
1890 fn start() -> WireEnvelope {
1891 start_at("in-start", 1_700_000_001_000)
1892 }
1893
1894 fn provider_outcome() -> EffectOutcome {
1895 EffectOutcome::Succeeded(EffectSucceeded {
1896 result: EffectSuccess::Provider(ProviderSuccess {
1897 outcome: ProviderOutcome::ContextOverflow(ProviderContextOverflow::default()),
1898 }),
1899 })
1900 }
1901
1902 fn failure_outcome() -> EffectOutcome {
1903 EffectOutcome::Failed(EffectFailed {
1904 failure: HostEffectFailure {
1905 kind: HostEffectFailureKind::TransportExhausted,
1906 message: "the vendor gave up".to_string(),
1907 retryable: Some(false),
1908 },
1909 })
1910 }
1911
1912 fn resolve_at(
1913 id: &str,
1914 observed_at_ms: u64,
1915 effect_id: &EffectId,
1916 outcome: EffectOutcome,
1917 ) -> WireEnvelope {
1918 envelope(
1919 id,
1920 observed_at_ms,
1921 KernelInput::ResolveEffect(ResolveEffect {
1922 effect_id: effect_id.clone(),
1923 outcome,
1924 }),
1925 )
1926 }
1927
1928 fn cancel_at(id: &str, observed_at_ms: u64) -> WireEnvelope {
1929 envelope(
1930 id,
1931 observed_at_ms,
1932 KernelInput::HostControl(HostControl {
1933 command: HostCommand::Cancel(CancelCommand {
1934 reason: CancellationReason::User,
1935 pending_call_ids: vec![],
1936 }),
1937 }),
1938 )
1939 }
1940
1941 fn signal_at(id: &str, observed_at_ms: u64) -> WireEnvelope {
1942 use super::super::event::{DeliverSignal, ExternalEvent, LogicalSignal};
1943 use super::super::scalar::{DeliveryId, SignalId};
1944
1945 envelope(
1946 id,
1947 observed_at_ms,
1948 KernelInput::DeliverExternalEvent(DeliverExternalEvent {
1949 event: ExternalEvent::DeliverSignal(DeliverSignal {
1950 delivery_id: DeliveryId::new(format!("delivery-{id}")).unwrap(),
1951 attempt: 1,
1952 signal: LogicalSignal::new(SignalId::new("sig-late").unwrap()),
1953 }),
1954 }),
1955 )
1956 }
1957
1958 #[derive(Debug, Clone, PartialEq, Serialize)]
1961 struct TestStep {
1962 plan: String,
1963 disposition: StepDisposition,
1964 }
1965
1966 impl TransitionStep for TestStep {
1967 fn disposition(&self) -> &StepDisposition {
1968 &self.disposition
1969 }
1970 }
1971
1972 fn nothing(plan: &str) -> TestStep {
1973 TestStep {
1974 plan: plan.to_string(),
1975 disposition: StepDisposition::Effects(EffectsDisposition::default()),
1976 }
1977 }
1978
1979 fn publishing(plan: &str, effects: Vec<KernelEffect>) -> TestStep {
1980 TestStep {
1981 plan: plan.to_string(),
1982 disposition: StepDisposition::Effects(EffectsDisposition { effects }),
1983 }
1984 }
1985
1986 fn effect(id: &str, causation: &InputId, kind: EffectKind) -> KernelEffect {
1987 KernelEffect {
1988 effect_id: EffectId::new(id).unwrap(),
1989 causation_input_id: causation.clone(),
1990 effect: kind,
1991 }
1992 }
1993
1994 fn provider_effect_id(step_seq: WireU64) -> EffectId {
1995 EffectId::new(format!("{OPERATION}:step:{step_seq}:effect:0")).unwrap()
1996 }
1997
1998 fn plan(context: &PlanContext<'_>) -> Result<TestStep, KernelFault> {
2004 let label = format!("{}@{}", context.input.input.kind(), context.step_seq);
2005 Ok(match &context.input.input {
2006 NormalizedPayload::StartOperation(_) => publishing(
2007 &label,
2008 vec![effect(
2009 provider_effect_id(context.step_seq).as_str(),
2010 &context.input.input_id,
2011 EffectKind::CallProvider(CallProviderEffect::default()),
2012 )],
2013 ),
2014 NormalizedPayload::HostControl(_) => TestStep {
2015 plan: label,
2016 disposition: StepDisposition::Terminal(TerminalDisposition {
2017 terminal: KernelTerminal::Cancelled(CancelledTerminal {
2018 reason: CancellationReason::User,
2019 usage: UsageReport::default(),
2020 }),
2021 }),
2022 },
2023 _ => nothing(&label),
2024 })
2025 }
2026
2027 type Tx = KernelTransaction<TestStep, InMemoryRecordIndex>;
2028
2029 fn transaction() -> Tx {
2030 KernelTransaction::new(ConfigDefaults::default(), InMemoryRecordIndex::new())
2031 }
2032
2033 fn bounded(bounds: TailBounds) -> Tx {
2041 let mut defaults = ConfigDefaults::default();
2042 defaults.baseline.recovery_policy.tail_bounds = bounds;
2043 KernelTransaction::new(defaults, InMemoryRecordIndex::new())
2044 }
2045
2046 fn run(tx: &mut Tx, envelope: &WireEnvelope) -> CommittedTransition<TestStep> {
2048 run_with(tx, envelope, plan)
2049 }
2050
2051 fn run_with<F>(
2052 tx: &mut Tx,
2053 envelope: &WireEnvelope,
2054 planner: F,
2055 ) -> CommittedTransition<TestStep>
2056 where
2057 F: FnOnce(&PlanContext<'_>) -> Result<TestStep, KernelFault>,
2058 {
2059 let preparation = tx.prepare(envelope, planner);
2060 let token = preparation
2061 .token()
2062 .unwrap_or_else(|| {
2063 panic!(
2064 "expected a prepared transition, got {:?}",
2065 preparation.fault()
2066 )
2067 })
2068 .clone();
2069 let head = preparation.record().unwrap().record_digest().clone();
2070 tx.commit(&token, &head).expect("the commit must succeed")
2071 }
2072
2073 fn fault_of(preparation: &RecordPreparation<TestStep>) -> KernelFaultCode {
2074 preparation
2075 .fault()
2076 .unwrap_or_else(|| panic!("expected a rejection, got a success"))
2077 .code
2078 }
2079
2080 fn observable(
2083 tx: &Tx,
2084 ) -> (
2085 Option<DurableHead>,
2086 OperationLifecycle,
2087 Vec<String>,
2088 TailUsage,
2089 bool,
2090 ) {
2091 (
2092 tx.head(),
2093 tx.lifecycle(),
2094 tx.pending_effects()
2095 .map(|effect| effect.effect_id.to_string())
2096 .collect(),
2097 tx.tail_usage(),
2098 tx.has_candidate(),
2099 )
2100 }
2101
2102 fn started() -> (Tx, Vec<KernelRecord>, EffectId) {
2103 let mut tx = transaction();
2104 let genesis = run(&mut tx, &configure());
2105 let started = run(&mut tx, &start());
2106 let effect_id = provider_effect_id(started.step_seq);
2107 (tx, vec![genesis.record, started.record], effect_id)
2108 }
2109
2110 #[test]
2116 fn matrix_row1_before_a_prepare_nothing_exists() {
2117 let tx = transaction();
2118 assert!(!tx.has_candidate());
2119 assert_eq!(tx.head(), None);
2120 assert_eq!(tx.lifecycle(), OperationLifecycle::Created);
2121 assert_eq!(tx.pending_effects().count(), 0);
2122 assert_eq!(tx.tail_usage(), TailUsage::default());
2123 assert!(tx.index().is_empty(), "no record exists yet");
2124 assert_eq!(tx.checkpoint_boundary(), None);
2125 assert!(tx.outstanding_token().is_none());
2126 }
2127
2128 #[test]
2133 fn matrix_row2_a_rejected_prepare_hands_out_no_token_and_moves_nothing() {
2134 let (mut tx, _, effect_id) = started();
2135
2136 let rejections: Vec<(&str, RecordPreparation<TestStep>)> = vec![
2137 (
2139 "second configure",
2140 tx.prepare(
2141 &configure_at("in-again", [EffectKindTag::CallProvider]),
2142 plan,
2143 ),
2144 ),
2145 (
2147 "unknown effect",
2148 tx.prepare(
2149 &resolve_at(
2150 "in-unknown",
2151 1_700_000_002_000,
2152 &EffectId::new("op-tx-1:step:9:effect:0").unwrap(),
2153 failure_outcome(),
2154 ),
2155 plan,
2156 ),
2157 ),
2158 (
2160 "planner fault",
2161 tx.prepare(
2162 &resolve_at(
2163 "in-planned",
2164 1_700_000_002_000,
2165 &effect_id,
2166 failure_outcome(),
2167 ),
2168 |_| {
2169 Err(KernelFault::new(
2170 KernelFaultCode::ResourceLimitExceeded,
2171 "the planner refused",
2172 ))
2173 },
2174 ),
2175 ),
2176 ];
2177
2178 let before = tx.clone();
2179 for (label, preparation) in rejections {
2180 assert!(preparation.is_zero_mutation(), "{label}");
2181 assert!(preparation.token().is_none(), "{label}");
2182 assert!(preparation.record().is_none(), "{label}");
2183 assert!(preparation.step().is_none(), "{label}");
2184 assert!(preparation.step_seq().is_none(), "{label}");
2185 }
2186 assert_eq!(tx, before, "a rejected prepare must not move one byte");
2187 }
2188
2189 #[test]
2191 fn matrix_row3_a_crash_before_the_append_is_undone_by_abort() {
2192 let (mut tx, records, _) = started();
2193 let before = observable(&tx);
2194
2195 let preparation = tx.prepare(&cancel_at("in-cancel", 1_700_000_003_000), plan);
2196 let token = preparation.token().expect("prepared").clone();
2197 assert!(tx.has_candidate());
2198 assert_eq!(
2199 tx.head().map(|head| head.digest),
2200 Some(records[1].record_digest().clone()),
2201 "a candidate does not move the durable head"
2202 );
2203
2204 let discarded = tx.abort(&token).expect("the candidate was never appended");
2205 assert_eq!(discarded.step_seq(), WireU64::new(2));
2206 assert_eq!(
2207 observable(&tx),
2208 before,
2209 "an aborted candidate leaves no trace"
2210 );
2211
2212 assert_eq!(
2214 tx.abort(&token).unwrap_err().code,
2215 KernelFaultCode::TransactionConflict
2216 );
2217 assert!(
2218 !tx.is_poisoned(),
2219 "an abort is a normal, non-poisoning path"
2220 );
2221 }
2222
2223 #[test]
2225 fn matrix_row4_a_cas_conflict_discards_the_candidate_and_demands_a_rebuild() {
2226 let (mut tx, records, _) = started();
2227 let preparation = tx.prepare(&cancel_at("in-cancel", 1_700_000_003_000), plan);
2228 let token = preparation.token().expect("prepared").clone();
2229
2230 let forked = canonical_digest(b"another writer's record");
2232 let fault = tx.note_append_conflict(&token, Some(&forked));
2233
2234 assert_eq!(fault.code, KernelFaultCode::TransactionConflict);
2235 assert!(!fault.is_retryable(), "a conflict is not a bare retry");
2236 assert!(
2237 !tx.has_candidate(),
2238 "the candidate is discarded, not appended"
2239 );
2240 assert!(tx.is_poisoned(), "this layer never rebuilds itself");
2241
2242 assert_eq!(
2244 fault_of(&tx.prepare(&cancel_at("in-cancel-2", 1_700_000_004_000), plan)),
2245 KernelFaultCode::TransactionConflict
2246 );
2247 assert_eq!(
2248 tx.abort(&token).unwrap_err().code,
2249 KernelFaultCode::TransactionConflict
2250 );
2251 assert_eq!(
2252 tx.commit(&token, &forked).unwrap_err().code,
2253 KernelFaultCode::TransactionConflict
2254 );
2255
2256 let rebuilt = Tx::rebuild_from_records(
2258 &records,
2259 ConfigDefaults::default(),
2260 InMemoryRecordIndex::from_records(&records),
2261 plan,
2262 )
2263 .expect("the journal still verifies");
2264 assert_eq!(
2265 rebuilt.head().map(|head| head.digest),
2266 Some(records[1].record_digest().clone())
2267 );
2268 assert!(!rebuilt.is_poisoned());
2269 }
2270
2271 #[test]
2274 fn matrix_row5_a_crash_between_append_and_commit_rebuilds_and_publishes_once() {
2275 let mut tx = transaction();
2276 let genesis = run(&mut tx, &configure());
2277 let mut journal = vec![genesis.record];
2278
2279 let preparation = tx.prepare(&start(), plan);
2280 let appended = preparation.record().expect("prepared").clone();
2281 journal.push(appended.clone());
2283 assert_eq!(
2285 tx.pending_effects().count(),
2286 0,
2287 "a record that has not been committed publishes no effect (§15.2)"
2288 );
2289 drop(tx);
2290
2291 let rebuilt = Tx::rebuild_from_records(
2292 &journal,
2293 ConfigDefaults::default(),
2294 InMemoryRecordIndex::from_records(&journal),
2295 plan,
2296 )
2297 .expect("the journal rebuilds");
2298
2299 assert_eq!(
2300 rebuilt.head().map(|head| head.digest),
2301 Some(appended.record_digest().clone())
2302 );
2303 assert_eq!(rebuilt.lifecycle(), OperationLifecycle::Running);
2304 let pending: Vec<&EffectId> = rebuilt
2305 .pending_effects()
2306 .map(|effect| &effect.effect_id)
2307 .collect();
2308 assert_eq!(
2309 pending,
2310 vec![&provider_effect_id(WireU64::new(1))],
2311 "the rebuilt runtime re-exposes the one pending effect, with the same identity"
2312 );
2313 }
2314
2315 #[test]
2318 fn matrix_row6_a_failed_commit_never_becomes_an_abort() {
2319 let mut tx = transaction();
2320 let genesis = run(&mut tx, &configure());
2321 let journal = vec![genesis.record];
2322
2323 let preparation = tx.prepare(&start(), plan);
2324 let token = preparation.token().expect("prepared").clone();
2325 let wrong_head = canonical_digest(b"some other record");
2328 let fault = tx
2329 .commit(&token, &wrong_head)
2330 .expect_err("a commit that cannot be attributed must fail");
2331 assert_eq!(fault.code, KernelFaultCode::TransactionConflict);
2332
2333 assert!(tx.is_poisoned());
2336 assert!(!tx.has_candidate());
2337 assert_eq!(
2338 tx.abort(&token).unwrap_err().code,
2339 KernelFaultCode::TransactionConflict,
2340 "append-then-abort is not expressible"
2341 );
2342
2343 let rebuilt = Tx::rebuild_from_records(
2345 &journal,
2346 ConfigDefaults::default(),
2347 InMemoryRecordIndex::from_records(&journal),
2348 plan,
2349 )
2350 .expect("the durable record survives a failed commit");
2351 assert_eq!(rebuilt.lifecycle(), OperationLifecycle::Configured);
2352 }
2353
2354 #[test]
2356 fn matrix_row7_a_lost_commit_response_replays_the_same_record_and_step() {
2357 let mut tx = transaction();
2358 run(&mut tx, &configure());
2359 let committed = run(&mut tx, &start());
2360 let before = observable(&tx);
2361
2362 let replay = tx.prepare(&start(), plan);
2364
2365 assert!(replay.token().is_none(), "a replay has nothing to commit");
2366 assert_eq!(replay.step_seq(), Some(committed.step_seq));
2367 assert_eq!(replay.record(), Some(&committed.record));
2368 assert_eq!(replay.step(), Some(&committed.step));
2369 assert_eq!(
2370 observable(&tx),
2371 before,
2372 "a replay creates no record and re-publishes no effect"
2373 );
2374 assert_eq!(tx.index().len(), 2, "no second record was minted");
2375 }
2376
2377 #[test]
2380 fn matrix_row8_a_lost_resolution_response_replays_idempotently() {
2381 let (mut tx, _, effect_id) = started();
2382 let resolved = run(
2383 &mut tx,
2384 &resolve_at(
2385 "in-resolve",
2386 1_700_000_002_000,
2387 &effect_id,
2388 provider_outcome(),
2389 ),
2390 );
2391 assert_eq!(tx.pending_effects().count(), 0, "the effect is settled");
2392 let before = observable(&tx);
2393
2394 let same_input = tx.prepare(
2396 &resolve_at(
2397 "in-resolve",
2398 1_700_000_002_000,
2399 &effect_id,
2400 provider_outcome(),
2401 ),
2402 plan,
2403 );
2404 assert_eq!(same_input.step_seq(), Some(resolved.step_seq));
2405 assert_eq!(same_input.record(), Some(&resolved.record));
2406
2407 assert_eq!(
2411 fault_of(&tx.prepare(
2412 &resolve_at(
2413 "in-resolve",
2414 1_700_000_002_500,
2415 &effect_id,
2416 provider_outcome()
2417 ),
2418 plan
2419 )),
2420 KernelFaultCode::DuplicateInputConflict
2421 );
2422
2423 let new_input = tx.prepare(
2426 &resolve_at(
2427 "in-resolve-again",
2428 1_700_000_003_000,
2429 &effect_id,
2430 provider_outcome(),
2431 ),
2432 plan,
2433 );
2434 assert_eq!(
2435 new_input.step_seq(),
2436 Some(resolved.step_seq),
2437 "effect-level dedup points at the existing record's step_seq"
2438 );
2439 assert_eq!(new_input.record(), Some(&resolved.record));
2440 assert!(
2441 new_input.token().is_none(),
2442 "reporting this as Prepared with an old step_seq is the dead end this replaces"
2443 );
2444 assert_eq!(observable(&tx), before);
2445 assert_eq!(tx.index().len(), 3, "still three records");
2446 }
2447
2448 #[test]
2453 fn a_committed_record_can_never_be_aborted() {
2454 let mut tx = transaction();
2455 let preparation = tx.prepare(&configure(), plan);
2456 let token = preparation.token().expect("prepared").clone();
2457 let head = preparation.record().unwrap().record_digest().clone();
2458 tx.commit(&token, &head).expect("commit");
2459
2460 let error = tx
2461 .abort(&token)
2462 .expect_err("a committed record has no candidate to abort");
2463 assert_eq!(error.code, KernelFaultCode::TransactionConflict);
2464 assert!(!tx.is_poisoned(), "asking is not itself a corruption");
2465 assert_eq!(
2466 tx.head().map(|head| head.step_seq),
2467 Some(WireU64::ZERO),
2468 "the committed record stands"
2469 );
2470 }
2471
2472 #[test]
2473 fn a_second_prepare_is_refused_while_a_candidate_is_outstanding() {
2474 let mut tx = transaction();
2475 let first = tx.prepare(&configure(), plan);
2476 let token = first.token().expect("prepared").clone();
2477
2478 let second = tx.prepare(&start(), plan);
2479 assert_eq!(fault_of(&second), KernelFaultCode::TransactionConflict);
2480 assert_eq!(
2481 tx.outstanding_token(),
2482 Some(&token),
2483 "the first candidate is not displaced by the second attempt"
2484 );
2485
2486 let head = first.record().unwrap().record_digest().clone();
2488 tx.commit(&token, &head).expect("the first candidate wins");
2489 }
2490
2491 #[test]
2492 fn effects_become_visible_only_when_their_record_is_durable() {
2493 let mut tx = transaction();
2494 run(&mut tx, &configure());
2495
2496 let preparation = tx.prepare(&start(), plan);
2497 assert_eq!(
2498 preparation.step().unwrap().effects().len(),
2499 1,
2500 "the planned step carries the effect"
2501 );
2502 assert_eq!(
2503 tx.pending_effects().count(),
2504 0,
2505 "but nothing is pending until the record is durable"
2506 );
2507
2508 let token = preparation.token().unwrap().clone();
2509 let head = preparation.record().unwrap().record_digest().clone();
2510 let committed = tx.commit(&token, &head).unwrap();
2511 assert_eq!(committed.published_effects().len(), 1);
2512 assert_eq!(tx.pending_effects().count(), 1);
2513 }
2514
2515 #[test]
2516 fn an_aborted_candidate_publishes_nothing() {
2517 let mut tx = transaction();
2518 run(&mut tx, &configure());
2519 let preparation = tx.prepare(&start(), plan);
2520 let token = preparation.token().unwrap().clone();
2521 tx.abort(&token).unwrap();
2522 assert_eq!(tx.pending_effects().count(), 0);
2523 assert_eq!(tx.lifecycle(), OperationLifecycle::Configured);
2524 }
2525
2526 #[test]
2527 fn the_operation_id_is_bound_by_the_genesis_record() {
2528 let mut tx = transaction();
2529 run(&mut tx, &configure());
2530 assert_eq!(tx.operation_id(), Some(&operation()));
2531
2532 let foreign = WireEnvelope::new(
2533 OperationId::new("op-other").unwrap(),
2534 input_id("in-foreign"),
2535 WireU64::new(1_700_000_002_000),
2536 KernelInput::HostControl(HostControl {
2537 command: HostCommand::Cancel(CancelCommand {
2538 reason: CancellationReason::User,
2539 pending_call_ids: vec![],
2540 }),
2541 }),
2542 );
2543 assert_eq!(
2544 fault_of(&tx.prepare(&foreign, plan)),
2545 KernelFaultCode::OperationMismatch
2546 );
2547 }
2548
2549 #[test]
2550 fn a_duplicate_input_id_with_a_different_payload_is_a_conflict() {
2551 let mut tx = transaction();
2552 run(&mut tx, &configure());
2553 run(&mut tx, &start());
2554
2555 let mut divergent = start();
2557 divergent.input = KernelInput::StartOperation(StartOperation {
2558 entry: RootEntry::Agent(RootAgentEntry {
2559 task: LogicalTask::new("write something else"),
2560 run_spec: None,
2561 }),
2562 initial_context: InitialContext::default(),
2563 });
2564 assert_eq!(
2565 fault_of(&tx.prepare(&divergent, plan)),
2566 KernelFaultCode::DuplicateInputConflict
2567 );
2568 }
2569
2570 #[test]
2571 fn an_exact_replay_is_answered_before_the_clock_check() {
2572 let mut tx = transaction();
2573 run(&mut tx, &configure());
2574 let started = run(&mut tx, &start_at("in-start", 1_700_000_005_000));
2575
2576 let replay = tx.prepare(&start_at("in-start", 1_700_000_005_000), plan);
2578 assert_eq!(replay.step_seq(), Some(started.step_seq));
2579
2580 assert_eq!(
2582 fault_of(&tx.prepare(&cancel_at("in-past", 1_700_000_004_000), plan)),
2583 KernelFaultCode::ClockRegression
2584 );
2585 }
2586
2587 #[test]
2588 fn a_terminal_closes_the_operation_to_every_later_input() {
2589 let (mut tx, _, effect_id) = started();
2590 assert_eq!(tx.pending_effects().count(), 1);
2591
2592 let terminal = run(&mut tx, &cancel_at("in-cancel", 1_700_000_003_000));
2593 assert!(matches!(
2594 terminal.terminal(),
2595 Some(KernelTerminal::Cancelled(_))
2596 ));
2597 assert_eq!(tx.lifecycle(), OperationLifecycle::Cancelled);
2598 assert!(tx.lifecycle().is_terminal());
2599 assert_eq!(
2600 tx.pending_effects().count(),
2601 0,
2602 "a terminal leaves nothing waiting on the host"
2603 );
2604
2605 let before = tx.clone();
2608 for envelope in [
2609 resolve_at("in-late", 1_700_000_004_000, &effect_id, provider_outcome()),
2610 start_at("in-restart", 1_700_000_004_000),
2611 signal_at("in-late-signal", 1_700_000_004_000),
2612 ] {
2613 assert_eq!(
2614 fault_of(&tx.prepare(&envelope, plan)),
2615 KernelFaultCode::InvalidLifecycle,
2616 "{} must be refused after a terminal",
2617 envelope.input_id
2618 );
2619 }
2620 assert_eq!(tx, before, "a refused input leaves the transaction alone");
2621 }
2622
2623 #[test]
2625 fn a_re_issued_cancellation_replays_the_terminal_it_already_committed() {
2626 let (mut tx, _, _) = started();
2627 let cancelled = run(&mut tx, &cancel_at("in-cancel", 1_700_000_003_000));
2628 let before = tx.clone();
2629
2630 let replay = tx.prepare(&cancel_at("in-cancel-again", 1_700_000_004_000), plan);
2633 assert!(
2634 matches!(replay, KernelPreparation::Replayed(_)),
2635 "a re-issued cancellation must replay, not be refused by the latch it created"
2636 );
2637 assert_eq!(replay.step_seq(), Some(cancelled.step_seq));
2638 assert_eq!(
2639 replay.record().unwrap().record_digest(),
2640 cancelled.record.record_digest(),
2641 "the replay points at the existing record, so no second record exists"
2642 );
2643 assert_eq!(tx, before, "a replay moves nothing");
2644
2645 let exact = tx.prepare(&cancel_at("in-cancel", 1_700_000_003_000), plan);
2647 assert_eq!(exact.step_seq(), Some(cancelled.step_seq));
2648 assert_eq!(tx, before);
2649 }
2650
2651 #[test]
2652 fn configured_input_byte_limit_rejects_later_inputs_without_mutation() {
2653 let mut tx = transaction();
2654 let mut config = boot_config([EffectKindTag::CallProvider]);
2655 config.kernel_limits = Some(KernelLimits {
2656 max_input_bytes: Some(1_024),
2657 ..KernelLimits::default()
2658 });
2659 let configure = envelope(
2660 "in-configure-limited",
2661 1_700_000_000_000,
2662 KernelInput::ConfigureOperation(ConfigureOperation { config }),
2663 );
2664 run(&mut tx, &configure);
2665
2666 let oversized = envelope(
2667 "in-oversized",
2668 1_700_000_001_000,
2669 KernelInput::StartOperation(StartOperation {
2670 entry: RootEntry::Agent(RootAgentEntry {
2671 task: LogicalTask::new("x".repeat(4_096)),
2672 run_spec: None,
2673 }),
2674 initial_context: InitialContext::default(),
2675 }),
2676 );
2677 let before = tx.clone();
2678 let rejected = tx.prepare(&oversized, plan);
2679
2680 assert_eq!(fault_of(&rejected), KernelFaultCode::ResourceLimitExceeded);
2681 assert!(rejected.is_zero_mutation());
2682 assert_eq!(tx, before);
2683 }
2684
2685 #[test]
2686 fn a_differing_cancellation_after_the_terminal_is_a_conflict_not_an_overwrite() {
2687 let (mut tx, _, _) = started();
2688 run(&mut tx, &cancel_at("in-cancel", 1_700_000_003_000));
2689 let before = tx.clone();
2690
2691 let divergent = envelope(
2692 "in-cancel-other",
2693 1_700_000_004_000,
2694 KernelInput::HostControl(HostControl {
2695 command: HostCommand::Cancel(CancelCommand {
2696 reason: CancellationReason::HostShutdown,
2697 pending_call_ids: vec![],
2698 }),
2699 }),
2700 );
2701 assert_eq!(
2702 fault_of(&tx.prepare(&divergent, plan)),
2703 KernelFaultCode::DuplicateInputConflict,
2704 "the operation already ended for the first reason; the second cannot re-decide it"
2705 );
2706 assert_eq!(tx, before);
2707 }
2708
2709 #[test]
2714 fn a_conflicting_resolution_of_a_settled_effect_fails_closed() {
2715 let (mut tx, _, effect_id) = started();
2716 run(
2717 &mut tx,
2718 &resolve_at(
2719 "in-resolve",
2720 1_700_000_002_000,
2721 &effect_id,
2722 provider_outcome(),
2723 ),
2724 );
2725 let before = tx.clone();
2726
2727 let conflicting = tx.prepare(
2728 &resolve_at(
2729 "in-resolve-conflict",
2730 1_700_000_003_000,
2731 &effect_id,
2732 failure_outcome(),
2733 ),
2734 plan,
2735 );
2736 assert_eq!(
2737 fault_of(&conflicting),
2738 KernelFaultCode::UnexpectedEffectOutcome
2739 );
2740 assert_eq!(tx, before);
2741 }
2742
2743 #[test]
2744 fn an_unknown_effect_id_fails_closed() {
2745 let (mut tx, _, _) = started();
2746 let unknown = EffectId::new("op-tx-1:step:99:effect:0").unwrap();
2747 assert_eq!(
2748 fault_of(&tx.prepare(
2749 &resolve_at(
2750 "in-unknown",
2751 1_700_000_002_000,
2752 &unknown,
2753 provider_outcome()
2754 ),
2755 plan
2756 )),
2757 KernelFaultCode::UnexpectedEffectOutcome
2758 );
2759 }
2760
2761 #[test]
2762 fn a_resolution_carrying_another_effect_kinds_payload_fails_closed() {
2763 let (mut tx, _, effect_id) = started();
2764 let wrong_shape = EffectOutcome::Succeeded(EffectSucceeded {
2765 result: EffectSuccess::Tools(ToolsSuccess::default()),
2766 });
2767 assert_eq!(
2768 fault_of(&tx.prepare(
2769 &resolve_at("in-wrong", 1_700_000_002_000, &effect_id, wrong_shape),
2770 plan
2771 )),
2772 KernelFaultCode::UnexpectedEffectOutcome
2773 );
2774 }
2775
2776 #[test]
2777 fn at_most_one_pending_effect_per_kind() {
2778 let (mut tx, _, _) = started();
2779 let before = tx.clone();
2780
2781 let greedy = tx.prepare(&cancel_at("in-second", 1_700_000_002_000), |context| {
2783 Ok(publishing(
2784 "greedy",
2785 vec![effect(
2786 "op-tx-1:step:2:effect:0",
2787 &context.input.input_id,
2788 EffectKind::CallProvider(CallProviderEffect::default()),
2789 )],
2790 ))
2791 });
2792 assert_eq!(fault_of(&greedy), KernelFaultCode::ResourceLimitExceeded);
2793 assert_eq!(tx, before, "the refusal is zero mutation");
2794
2795 let doubled = tx.prepare(&cancel_at("in-double", 1_700_000_002_000), |context| {
2797 Ok(publishing(
2798 "doubled",
2799 vec![
2800 effect(
2801 "op-tx-1:step:2:effect:0",
2802 &context.input.input_id,
2803 EffectKind::SpawnTasks(SpawnTasksEffect::default()),
2804 ),
2805 effect(
2806 "op-tx-1:step:2:effect:1",
2807 &context.input.input_id,
2808 EffectKind::SpawnTasks(SpawnTasksEffect::default()),
2809 ),
2810 ],
2811 ))
2812 });
2813 assert_eq!(fault_of(&doubled), KernelFaultCode::ResourceLimitExceeded);
2814 }
2815
2816 #[test]
2817 fn an_effect_kind_the_host_did_not_declare_is_refused_before_emission() {
2818 let mut tx = transaction();
2819 run(
2821 &mut tx,
2822 &configure_at("in-configure", [EffectKindTag::CallProvider]),
2823 );
2824 let before = tx.clone();
2825
2826 let refused = tx.prepare(&start(), |context| {
2827 Ok(publishing(
2828 "tools",
2829 vec![effect(
2830 "op-tx-1:step:1:effect:0",
2831 &context.input.input_id,
2832 EffectKind::ExecuteTools(ExecuteToolsEffect::default()),
2833 )],
2834 ))
2835 });
2836 assert_eq!(fault_of(&refused), KernelFaultCode::UnsupportedEffect);
2837 assert_eq!(tx, before, "no record, no effect, no state change");
2838 }
2839
2840 #[test]
2841 fn a_launch_token_is_never_minted_twice() {
2842 let (mut tx, _, effect_id) = started();
2843 let spawn = |id: &str, token: &str| {
2844 let effect_id = id.to_string();
2845 let launch_token = token.to_string();
2846 move |context: &PlanContext<'_>| {
2847 Ok(publishing(
2848 "spawn",
2849 vec![effect(
2850 &effect_id,
2851 &context.input.input_id,
2852 EffectKind::SpawnTasks(SpawnTasksEffect {
2853 tasks: vec![TaskLaunch {
2854 task_id: TaskId::new("task-1").unwrap(),
2855 attempt_id: AttemptId::new("task-1:attempt:1").unwrap(),
2856 launch_token: LaunchToken::new(launch_token.clone()).unwrap(),
2857 node_id: NodeId::new("node-1").unwrap(),
2858 spec: LogicalAgentSpec::new("do the thing"),
2859 }],
2860 budget: None,
2861 }),
2862 )],
2863 ))
2864 }
2865 };
2866
2867 let spawn_effect_id = EffectId::new("op-tx-1:step:2:effect:0").unwrap();
2868 run_with(
2869 &mut tx,
2870 &resolve_at(
2871 "in-resolve",
2872 1_700_000_002_000,
2873 &effect_id,
2874 provider_outcome(),
2875 ),
2876 spawn(spawn_effect_id.as_str(), "op-tx-1:launch:1"),
2877 );
2878 assert!(tx.knows_launch_token(&LaunchToken::new("op-tx-1:launch:1").unwrap()));
2879
2880 run(
2882 &mut tx,
2883 &resolve_at(
2884 "in-spawned",
2885 1_700_000_003_000,
2886 &spawn_effect_id,
2887 EffectOutcome::Succeeded(EffectSucceeded {
2888 result: EffectSuccess::TasksSpawned(TasksSpawnedSuccess::default()),
2889 }),
2890 ),
2891 );
2892
2893 let reused = tx.prepare(
2896 &cancel_at("in-relaunch", 1_700_000_004_000),
2897 spawn("op-tx-1:step:4:effect:0", "op-tx-1:launch:1"),
2898 );
2899 assert_eq!(fault_of(&reused), KernelFaultCode::TransactionConflict);
2900
2901 let fresh = tx.prepare(
2903 &cancel_at("in-relaunch", 1_700_000_004_000),
2904 spawn("op-tx-1:step:4:effect:0", "op-tx-1:launch:2"),
2905 );
2906 assert!(fresh.token().is_some(), "{:?}", fresh.fault());
2907 }
2908
2909 #[test]
2910 fn an_effect_id_is_never_minted_twice() {
2911 let (mut tx, _, effect_id) = started();
2912 let collision = tx.prepare(
2913 &resolve_at(
2914 "in-resolve",
2915 1_700_000_002_000,
2916 &effect_id,
2917 provider_outcome(),
2918 ),
2919 move |context| {
2920 Ok(publishing(
2921 "collide",
2922 vec![effect(
2923 "op-tx-1:step:1:effect:0",
2925 &context.input.input_id,
2926 EffectKind::SpawnTasks(SpawnTasksEffect::default()),
2927 )],
2928 ))
2929 },
2930 );
2931 assert_eq!(fault_of(&collision), KernelFaultCode::TransactionConflict);
2932 }
2933
2934 #[test]
2939 fn a_full_tail_asks_for_a_checkpoint_and_the_retry_is_a_fresh_prepare() {
2940 let mut tx = bounded(TailBounds::new(1, 2, 1024, 1024 * 1024).unwrap());
2941 run(&mut tx, &configure());
2942 run(&mut tx, &start());
2943 assert_eq!(tx.tail_pressure(), TailPressure::Full);
2944 let before = tx.clone();
2945
2946 let retry_envelope = cancel_at("in-cancel", 1_700_000_003_000);
2947 let refused = tx.prepare(&retry_envelope, plan);
2948 let fault = refused.fault().expect("rejected").clone();
2949 assert_eq!(fault.code, KernelFaultCode::CheckpointRequired);
2950 assert!(fault.is_retryable(), "the one retryable code (GAP-2)");
2951 assert!(refused.is_zero_mutation());
2952 assert_eq!(tx, before, "the input was never accepted");
2953
2954 let boundary = tx.checkpoint_boundary().expect("a head exists");
2956 assert_eq!(boundary.through_step_seq, WireU64::new(1));
2957 let usage = tx.note_checkpoint_acked(&boundary).expect("ack");
2958 assert_eq!(
2959 usage,
2960 TailUsage::default(),
2961 "the covered prefix is reclaimed"
2962 );
2963 assert_eq!(tx.tail_pressure(), TailPressure::Nominal, "not a latch");
2964
2965 let retried = tx.prepare(&retry_envelope, plan);
2967 assert!(retried.token().is_some(), "{:?}", retried.fault());
2968 assert_eq!(retried.record().unwrap().step_seq(), WireU64::new(2));
2969 }
2970
2971 #[test]
2972 fn the_tail_bounds_the_byte_axis_too() {
2973 let mut tx = bounded(TailBounds::new(64, 128, 128, 512).unwrap());
2974 let refused = tx.prepare(&configure(), plan);
2976 assert_eq!(fault_of(&refused), KernelFaultCode::CheckpointRequired);
2977 assert!(refused.fault().unwrap().is_retryable());
2978 }
2979
2980 #[test]
2981 fn tail_bounds_refuse_an_incoherent_watermark() {
2982 assert_eq!(
2983 TailBounds::new(10, 4, 100, 100).unwrap_err().code,
2984 KernelFaultCode::InvalidConfig
2985 );
2986 assert_eq!(
2987 TailBounds::new(1, 4, 0, 0).unwrap_err().code,
2988 KernelFaultCode::InvalidConfig
2989 );
2990 assert_eq!(TailBounds::default(), TailBounds::DEFAULT);
2991 }
2992
2993 #[test]
2994 fn the_tail_reports_its_soft_watermark_before_its_hard_limit() {
2995 let mut tx = bounded(TailBounds::new(2, 8, 1024 * 1024, 4 * 1024 * 1024).unwrap());
2996 assert_eq!(tx.tail_pressure(), TailPressure::Nominal);
2997 run(&mut tx, &configure());
2998 assert_eq!(tx.tail_pressure(), TailPressure::Nominal);
2999 run(&mut tx, &start());
3000 assert_eq!(tx.tail_pressure(), TailPressure::Watermark);
3001 }
3002
3003 #[test]
3006 fn a_checkpoint_boundary_neither_blocks_nor_is_blocked_by_a_candidate() {
3007 let (mut tx, records, _) = started();
3008
3009 let preparation = tx.prepare(&cancel_at("in-cancel", 1_700_000_003_000), plan);
3010 let token = preparation.token().expect("prepared").clone();
3011 let candidate_digest = preparation.record().unwrap().record_digest().clone();
3012
3013 let boundary = tx.checkpoint_boundary().expect("a head exists");
3014 assert_eq!(
3015 boundary.covered_head,
3016 *records[1].record_digest(),
3017 "the boundary follows the durable head, not the outstanding candidate"
3018 );
3019 assert_ne!(boundary.covered_head, candidate_digest);
3020
3021 tx.note_checkpoint_acked(&boundary).expect("ack");
3023 assert_eq!(tx.tail_usage(), TailUsage::default());
3024 assert!(tx.has_candidate(), "the candidate survived the checkpoint");
3025
3026 let committed = tx.commit(&token, &candidate_digest).expect("commit");
3028 assert_eq!(committed.step_seq, WireU64::new(2));
3029 assert_eq!(
3030 tx.tail_usage().records,
3031 1,
3032 "records after the candidate are tail"
3033 );
3034 }
3035
3036 #[test]
3037 fn a_checkpoint_that_covers_no_prefix_of_this_journal_is_refused() {
3038 let (mut tx, _, _) = started();
3039 let bogus = CheckpointBoundary {
3040 through_step_seq: WireU64::new(1),
3041 covered_head: canonical_digest(b"another operation's head"),
3042 };
3043 assert_eq!(
3044 tx.note_checkpoint_acked(&bogus).unwrap_err().code,
3045 KernelFaultCode::CheckpointIncompatible
3046 );
3047 assert_eq!(tx.tail_usage().records, 2, "nothing was reclaimed");
3048 }
3049
3050 #[test]
3055 fn a_rebuild_reproduces_every_step_digest_and_the_next_transition() {
3056 let mut live = transaction();
3057 let mut journal = Vec::new();
3058 journal.push(run(&mut live, &configure()).record);
3059 journal.push(run(&mut live, &start()).record);
3060 let effect_id = provider_effect_id(WireU64::new(1));
3061 journal.push(
3062 run(
3063 &mut live,
3064 &resolve_at(
3065 "in-resolve",
3066 1_700_000_002_000,
3067 &effect_id,
3068 provider_outcome(),
3069 ),
3070 )
3071 .record,
3072 );
3073
3074 let mut rebuilt = Tx::rebuild_from_records(
3075 &journal,
3076 ConfigDefaults::default(),
3077 InMemoryRecordIndex::from_records(&journal),
3078 plan,
3079 )
3080 .expect("the journal rebuilds");
3081
3082 assert_eq!(rebuilt.head(), live.head());
3083 assert_eq!(rebuilt.lifecycle(), live.lifecycle());
3084 assert_eq!(rebuilt.config(), live.config());
3085 assert_eq!(rebuilt.tail_usage(), live.tail_usage());
3086 for record in &journal {
3087 let step = rebuilt
3088 .committed_step(record.input_id())
3089 .expect("every replayed step is recoverable");
3090 record
3091 .verify_step(step)
3092 .expect("the rebuilt step matches the frozen digest");
3093 assert_eq!(Some(step), live.committed_step(record.input_id()));
3094 }
3095
3096 let next = cancel_at("in-cancel", 1_700_000_003_000);
3098 let uninterrupted = live.prepare(&next, plan);
3099 let after_rebuild = rebuilt.prepare(&next, plan);
3100 assert_eq!(
3101 after_rebuild.record().unwrap().step_digest(),
3102 uninterrupted.record().unwrap().step_digest()
3103 );
3104 assert_eq!(
3105 after_rebuild.record().unwrap().record_digest(),
3106 uninterrupted.record().unwrap().record_digest()
3107 );
3108 }
3109
3110 #[test]
3111 fn a_rebuild_refuses_a_broken_chain() {
3112 let mut live = transaction();
3113 let genesis = run(&mut live, &configure()).record;
3114 let started = run(&mut live, &start()).record;
3115 let resolved = run(
3116 &mut live,
3117 &resolve_at(
3118 "in-resolve",
3119 1_700_000_002_000,
3120 &provider_effect_id(WireU64::new(1)),
3121 provider_outcome(),
3122 ),
3123 )
3124 .record;
3125
3126 let gapped = vec![genesis.clone(), resolved.clone()];
3128 let error = Tx::rebuild_from_records(
3129 &gapped,
3130 ConfigDefaults::default(),
3131 InMemoryRecordIndex::from_records(&gapped),
3132 plan,
3133 )
3134 .expect_err("a chain with a hole is not a journal");
3135 assert_eq!(error.code, KernelFaultCode::RecordCorrupted);
3136
3137 let intact = vec![genesis, started, resolved];
3139 let error = Tx::rebuild_from_records(
3140 &intact,
3141 ConfigDefaults::default(),
3142 InMemoryRecordIndex::from_records(&intact),
3143 |context| {
3144 let mut step = plan(context)?;
3145 step.plan.push_str(" (drifted)");
3146 Ok(step)
3147 },
3148 )
3149 .expect_err("a drifted planner must not silently resume");
3150 assert_eq!(error.code, KernelFaultCode::RecordCorrupted);
3151 assert!(
3152 error.message.contains("step"),
3153 "the fault names the digest that disagreed: {}",
3154 error.message
3155 );
3156 }
3157
3158 #[test]
3159 fn a_rebuild_of_an_empty_journal_is_a_fresh_operation() {
3160 let rebuilt = Tx::rebuild_from_records(
3161 &[],
3162 ConfigDefaults::default(),
3163 InMemoryRecordIndex::new(),
3164 plan,
3165 )
3166 .expect("an empty journal is a legal starting point");
3167 assert_eq!(rebuilt.lifecycle(), OperationLifecycle::Created);
3168 assert_eq!(rebuilt.head(), None);
3169 }
3170
3171 #[test]
3174 fn a_replay_of_a_record_this_runtime_never_saw_demands_a_rebuild() {
3175 let mut source = transaction();
3176 let genesis = run(&mut source, &configure()).record;
3177
3178 let mut cold = KernelTransaction::<TestStep, _>::new(
3179 ConfigDefaults::default(),
3180 InMemoryRecordIndex::from_records(&[genesis]),
3181 );
3182 assert_eq!(
3183 fault_of(&cold.prepare(&configure(), plan)),
3184 KernelFaultCode::RecordCorrupted
3185 );
3186 }
3187
3188 #[test]
3189 fn a_poisoned_transaction_refuses_every_call() {
3190 let (mut tx, _, _) = started();
3191 let preparation = tx.prepare(&cancel_at("in-cancel", 1_700_000_003_000), plan);
3192 let token = preparation.token().unwrap().clone();
3193 tx.note_append_conflict(&token, None);
3194
3195 assert!(tx.is_poisoned());
3196 assert_eq!(
3197 tx.poison().map(|fault| fault.code),
3198 Some(KernelFaultCode::TransactionConflict)
3199 );
3200 assert_eq!(
3201 fault_of(&tx.prepare(&start_at("in-any", 1_700_000_009_000), plan)),
3202 KernelFaultCode::TransactionConflict
3203 );
3204 let boundary = CheckpointBoundary {
3205 through_step_seq: WireU64::new(1),
3206 covered_head: canonical_digest(b"whatever"),
3207 };
3208 assert_eq!(
3209 tx.note_checkpoint_acked(&boundary).unwrap_err().code,
3210 KernelFaultCode::TransactionConflict
3211 );
3212 }
3213}