1use std::collections::{BTreeMap, VecDeque};
35use std::fmt;
36
37use serde::Serialize;
38
39use super::checkpoint::{
40 AcceptedCancellationState, AcceptedInputState, CanonicalInput, CheckpointCandidate,
41 CheckpointDraft, KernelCheckpoint, LaunchTokenState, LogicalKernelState,
42 LogicalStateProjection, ResolvedEffectState, TransitionStateV1,
43};
44use super::command::{CancelCommand, HostCommand};
45use super::config::{ConfigDefaults, ResolvedOperationConfig, TailBounds};
46use super::effect::{Digest, EffectKind, EffectKindTag, EffectOutcome, KernelEffect, LaunchToken};
47use super::envelope::{OperationLifecycle, WireEnvelope, WireRejection, WireRejectionKind};
48use super::fault::{
49 KernelFault, KernelFaultCode, KernelPreparation, PrepareToken, PreparedTransition,
50 RejectedTransition, ReplayedTransition,
51};
52use super::record::{
53 ChainAnchor, KernelRecord, NormalizedInput, NormalizedPayload, RecordError, RecordPreparation,
54 canonical_bytes, canonical_digest, verify_record_chain,
55};
56use super::root::{ExecutionFocus, RootKind};
57use super::scalar::{EffectId, InputId, OperationId, WireU64};
58use super::terminal::{KernelTerminal, StepDisposition, TerminalSlot};
59
60pub trait TransitionStep: Serialize + Clone {
71 fn disposition(&self) -> &StepDisposition;
72
73 fn effects(&self) -> &[KernelEffect] {
74 self.disposition().effects()
75 }
76
77 fn terminal(&self) -> Option<&KernelTerminal> {
78 self.disposition().terminal()
79 }
80}
81
82pub trait RecordIndex {
93 fn record_for_input(
95 &self,
96 operation_id: &OperationId,
97 input_id: &InputId,
98 ) -> Option<KernelRecord>;
99
100 fn note_committed(&mut self, record: &KernelRecord) {
106 let _ = record;
107 }
108}
109
110#[derive(Debug, Clone, Default, PartialEq)]
112pub struct InMemoryRecordIndex {
113 records: BTreeMap<(OperationId, InputId), KernelRecord>,
114}
115
116impl InMemoryRecordIndex {
117 pub fn new() -> Self {
118 Self::default()
119 }
120
121 pub fn from_records(records: &[KernelRecord]) -> Self {
123 let mut index = Self::new();
124 for record in records {
125 index.note_committed(record);
126 }
127 index
128 }
129
130 pub fn len(&self) -> usize {
131 self.records.len()
132 }
133
134 pub fn is_empty(&self) -> bool {
135 self.records.is_empty()
136 }
137}
138
139impl RecordIndex for InMemoryRecordIndex {
140 fn record_for_input(
141 &self,
142 operation_id: &OperationId,
143 input_id: &InputId,
144 ) -> Option<KernelRecord> {
145 self.records
146 .get(&(operation_id.clone(), input_id.clone()))
147 .cloned()
148 }
149
150 fn note_committed(&mut self, record: &KernelRecord) {
151 self.records.insert(
152 (record.operation_id().clone(), record.input_id().clone()),
153 record.clone(),
154 );
155 }
156}
157
158#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
164pub struct TailUsage {
165 pub records: u64,
166 pub bytes: u64,
167}
168
169#[derive(Debug, Clone, Copy, PartialEq, Eq)]
172pub enum TailPressure {
173 Nominal,
174 Watermark,
176 Full,
178}
179
180#[derive(Debug, Clone, PartialEq)]
187struct TailEntry {
188 step_seq: WireU64,
189 record_digest: Digest,
190 bytes: u64,
191 input: NormalizedInput,
192}
193
194#[derive(Debug, Clone, PartialEq, Eq)]
200pub struct DurableHead {
201 pub digest: Digest,
202 pub step_seq: WireU64,
203}
204
205#[derive(Debug, Clone, PartialEq, Eq)]
211pub struct CheckpointBoundary {
212 pub through_step_seq: WireU64,
213 pub covered_head: Digest,
214}
215
216#[derive(Debug)]
220pub struct PlanContext<'a> {
221 pub input: &'a NormalizedInput,
222 pub step_seq: WireU64,
223 pub previous_head: Option<&'a Digest>,
224 pub config: &'a ResolvedOperationConfig,
225 pub resolving: Option<&'a KernelEffect>,
233}
234
235#[derive(Debug, Clone, PartialEq)]
237pub struct CommittedTransition<Step> {
238 pub record: KernelRecord,
239 pub step: Step,
240 pub step_seq: WireU64,
241 pub checkpoint_advice: Option<CheckpointAdvice>,
251}
252
253#[derive(Debug, Clone, Copy, PartialEq, Eq)]
260pub struct CheckpointAdvice {
261 pub through_step_seq: WireU64,
262 pub usage: TailUsage,
263 pub bounds: TailBounds,
264}
265
266impl<Step: TransitionStep> CommittedTransition<Step> {
267 pub fn published_effects(&self) -> &[KernelEffect] {
271 self.step.effects()
272 }
273
274 pub fn terminal(&self) -> Option<&KernelTerminal> {
275 self.step.terminal()
276 }
277}
278
279#[derive(Debug, Clone, PartialEq)]
280struct Candidate<Step> {
281 token: PrepareToken,
282 record: KernelRecord,
283 step: Step,
284 input: NormalizedInput,
285 record_bytes: u64,
286}
287
288struct InputFacts {
294 operation_id: OperationId,
295 input_id: InputId,
296 observed_at_ms: WireU64,
297 admissible_lifecycles: &'static [OperationLifecycle],
298}
299
300#[derive(Debug, Clone, PartialEq)]
301struct ResolvedEffectRecord {
302 outcome_digest: Digest,
303 input_id: InputId,
304 step_seq: WireU64,
305}
306
307#[derive(Debug, Clone, PartialEq)]
318pub struct KernelTransaction<Step, Index> {
319 defaults: ConfigDefaults,
320 bounds: TailBounds,
321 index: Index,
322 operation_id: Option<OperationId>,
323 genesis_digest: Option<Digest>,
326 config: Option<ResolvedOperationConfig>,
327 head: Option<ChainAnchor>,
333 lifecycle: OperationLifecycle,
334 terminal: TerminalSlot,
335 candidate: Option<Candidate<Step>>,
336 prepare_epoch: u64,
337 last_observed_at_ms: WireU64,
338 steps: BTreeMap<InputId, Step>,
350 accepted: BTreeMap<InputId, AcceptedInputState>,
358 replay_floor: Option<WireU64>,
364 pending_effects: BTreeMap<EffectId, KernelEffect>,
365 resolved_effects: BTreeMap<EffectId, ResolvedEffectRecord>,
366 launch_tokens: BTreeMap<LaunchToken, WireU64>,
367 accepted_cancellation: Option<AcceptedCancellation>,
375 tail: VecDeque<TailEntry>,
376 poison: Option<KernelFault>,
377}
378
379#[derive(Debug, Clone, PartialEq)]
381struct AcceptedCancellation {
382 command_digest: Digest,
385 input_id: InputId,
386 step_seq: WireU64,
387}
388
389impl<Step, Index> KernelTransaction<Step, Index>
390where
391 Step: TransitionStep,
392 Index: RecordIndex,
393{
394 pub fn new(defaults: ConfigDefaults, index: Index) -> Self {
401 let bounds = defaults.baseline.recovery_policy.tail_bounds;
402 Self {
403 defaults,
404 bounds,
405 index,
406 operation_id: None,
407 genesis_digest: None,
408 config: None,
409 head: None,
410 lifecycle: OperationLifecycle::Created,
411 terminal: TerminalSlot::empty(),
412 candidate: None,
413 prepare_epoch: 0,
414 last_observed_at_ms: WireU64::ZERO,
415 steps: BTreeMap::new(),
416 accepted: BTreeMap::new(),
417 replay_floor: None,
418 pending_effects: BTreeMap::new(),
419 resolved_effects: BTreeMap::new(),
420 launch_tokens: BTreeMap::new(),
421 accepted_cancellation: None,
422 tail: VecDeque::new(),
423 poison: None,
424 }
425 }
426
427 pub fn prepare<F>(&mut self, envelope: &WireEnvelope, plan: F) -> RecordPreparation<Step>
438 where
439 F: FnOnce(&PlanContext<'_>) -> Result<Step, KernelFault>,
440 {
441 match self.prepare_inner(envelope, plan) {
442 Ok(preparation) => preparation,
443 Err(fault) => KernelPreparation::Rejected(RejectedTransition { fault }),
444 }
445 }
446
447 fn prepare_inner<F>(
448 &mut self,
449 envelope: &WireEnvelope,
450 plan: F,
451 ) -> Result<RecordPreparation<Step>, KernelFault>
452 where
453 F: FnOnce(&PlanContext<'_>) -> Result<Step, KernelFault>,
454 {
455 self.check_preparable(&envelope.operation_id)?;
456 let input =
457 NormalizedInput::normalize(envelope, &self.defaults).map_err(rejection_fault)?;
458 self.prepare_normalized_inner(input, plan)
459 }
460
461 fn check_preparable(&self, operation_id: &OperationId) -> Result<(), KernelFault> {
463 if let Some(fault) = &self.poison {
464 return Err(fault.clone());
465 }
466
467 if let Some(candidate) = &self.candidate {
471 return Err(KernelFault::new(
472 KernelFaultCode::TransactionConflict,
473 format!(
474 "transaction candidate {} is still outstanding at step {}; commit or abort it \
475 before preparing another input",
476 candidate.token,
477 candidate.record.step_seq()
478 ),
479 ));
480 }
481
482 if let Some(bound) = &self.operation_id
483 && bound != operation_id
484 {
485 return Err(KernelFault::new(
486 KernelFaultCode::OperationMismatch,
487 format!(
488 "input belongs to operation {operation_id}, but this runtime is bound to \
489 {bound}"
490 ),
491 ));
492 }
493 Ok(())
494 }
495
496 fn prepare_normalized_inner<F>(
497 &mut self,
498 input: NormalizedInput,
499 plan: F,
500 ) -> Result<RecordPreparation<Step>, KernelFault>
501 where
502 F: FnOnce(&PlanContext<'_>) -> Result<Step, KernelFault>,
503 {
504 self.check_preparable(&input.operation_id)?;
505 let envelope = InputFacts {
506 operation_id: input.operation_id.clone(),
507 input_id: input.input_id.clone(),
508 observed_at_ms: input.observed_at_ms,
509 admissible_lifecycles: input.input.admissible_lifecycles(),
510 };
511 let envelope = &envelope;
512 let canonical_input = canonical_bytes(&input).map_err(record_fault)?;
513
514 if let Some(existing) = self
517 .index
518 .record_for_input(&envelope.operation_id, &envelope.input_id)
519 {
520 if existing.canonical_input() != &canonical_input {
521 return Err(KernelFault::new(
522 KernelFaultCode::DuplicateInputConflict,
523 format!(
524 "input {} was already accepted at step {} with a different payload",
525 envelope.input_id,
526 existing.step_seq()
527 ),
528 ));
529 }
530 return self.replay_of(existing);
531 }
532
533 if let Some(entry) = self.accepted.get(&envelope.input_id)
539 && self.below_replay_floor(entry.step_seq)
540 {
541 return Ok(self.replay_by_reference(entry));
542 }
543
544 if let Some(config) = &self.config
545 && canonical_input.len() > config.kernel_limits.max_input_bytes as usize
546 {
547 return Err(KernelFault::new(
548 KernelFaultCode::ResourceLimitExceeded,
549 format!(
550 "canonical input carries {} bytes; the operation limit is {}",
551 canonical_input.len(),
552 config.kernel_limits.max_input_bytes
553 ),
554 ));
555 }
556
557 if envelope.observed_at_ms.get() < self.last_observed_at_ms.get() {
558 return Err(KernelFault::new(
559 KernelFaultCode::ClockRegression,
560 format!(
561 "input observed at {} precedes the last accepted input at {}",
562 envelope.observed_at_ms, self.last_observed_at_ms
563 ),
564 ));
565 }
566
567 if let NormalizedPayload::HostControl(control) = &input.input
571 && let HostCommand::Cancel(cancel) = &control.command
572 && let Some(preparation) = self.cancel_guard(cancel)?
573 {
574 return Ok(preparation);
575 }
576
577 if !envelope.admissible_lifecycles.contains(&self.lifecycle) {
578 return Err(KernelFault::new(
579 KernelFaultCode::InvalidLifecycle,
580 format!(
581 "a {} input is not admissible while the operation is {:?}",
582 input.input.kind(),
583 self.lifecycle
584 ),
585 ));
586 }
587
588 if let NormalizedPayload::ResolveEffect(resolve) = &input.input
590 && let Some(preparation) = self.resolve_effect_guard(resolve)?
591 {
592 return Ok(preparation);
593 }
594
595 if self.tail_records() + 1 > self.bounds.hard_records.get() {
596 return Err(self.checkpoint_required(format!(
597 "the journal tail already holds {} records; its hard limit is {}",
598 self.tail_records(),
599 self.bounds.hard_records
600 )));
601 }
602
603 let step_seq = self.next_step_seq()?;
604 let genesis_config = input.resolved_config().cloned();
605 let config = match (&genesis_config, &self.config) {
606 (Some(config), _) => config,
607 (None, Some(config)) => config,
608 (None, None) => {
609 return Err(KernelFault::new(
610 KernelFaultCode::InvalidLifecycle,
611 "the operation has no genesis record, so it has no configuration to plan \
612 against"
613 .to_string(),
614 ));
615 }
616 };
617
618 let settled = match &input.input {
619 NormalizedPayload::ResolveEffect(resolve) => Some(&resolve.effect_id),
620 _ => None,
621 };
622 let resolving = settled.and_then(|effect_id| self.pending_effects.get(effect_id));
623 let step = plan(&PlanContext {
624 input: &input,
625 step_seq,
626 previous_head: self.head.as_ref().map(|anchor| &anchor.record_digest),
627 config,
628 resolving,
629 })?;
630
631 self.screen_planned_effects(&step, config, settled)?;
632
633 let record =
634 KernelRecord::chain_after(self.head.as_ref(), &input, &step).map_err(record_fault)?;
635 let record_bytes = record.record_bytes().len() as u64;
636 if self.tail_bytes() + record_bytes > self.bounds.hard_bytes.get() {
637 return Err(self.checkpoint_required(format!(
638 "the journal tail holds {} bytes and this record adds {record_bytes}; \
639 the hard limit is {}",
640 self.tail_bytes(),
641 self.bounds.hard_bytes
642 )));
643 }
644
645 self.prepare_epoch += 1;
647 let token = PrepareToken::new(format!(
648 "{}:prepare:{step_seq}:{}",
649 envelope.operation_id, self.prepare_epoch
650 ))
651 .expect("an operation-scoped prepare token is always a legal branded ref");
652 self.candidate = Some(Candidate {
653 token: token.clone(),
654 record: record.clone(),
655 step: step.clone(),
656 input,
657 record_bytes,
658 });
659
660 Ok(KernelPreparation::Prepared(PreparedTransition {
661 token,
662 record,
663 planned_step: step,
664 }))
665 }
666
667 pub fn commit(
678 &mut self,
679 token: &PrepareToken,
680 appended_head: &Digest,
681 ) -> Result<CommittedTransition<Step>, KernelFault> {
682 if let Some(fault) = &self.poison {
683 return Err(fault.clone());
684 }
685
686 let Some(candidate) = self.candidate.take() else {
687 return Err(self.poison_with(KernelFault::new(
688 KernelFaultCode::TransactionConflict,
689 format!(
690 "commit reports a durable append for token {token}, but no candidate is \
691 outstanding; a committed record is never re-committed and never aborted"
692 ),
693 )));
694 };
695
696 if &candidate.token != token {
697 let outstanding = candidate.token.clone();
698 return Err(self.poison_with(KernelFault::new(
699 KernelFaultCode::TransactionConflict,
700 format!(
701 "commit names token {token}, but the outstanding candidate is {outstanding}; \
702 the runtime no longer describes what the journal holds"
703 ),
704 )));
705 }
706
707 if appended_head != candidate.record.record_digest() {
708 let expected = candidate.record.record_digest().clone();
709 return Err(self.poison_with(KernelFault::new(
710 KernelFaultCode::TransactionConflict,
711 format!(
712 "the journal head after the append is {appended_head}, but this candidate is \
713 {expected}; the append did not place this record, so the runtime must be \
714 rebuilt from the journal"
715 ),
716 )));
717 }
718
719 let Candidate {
720 record,
721 step,
722 input,
723 record_bytes,
724 ..
725 } = candidate;
726 match self.integrate(record, step, &input, record_bytes) {
727 Ok(committed) => Ok(committed),
728 Err(fault) => Err(self.poison_with(fault)),
729 }
730 }
731
732 pub fn abort(&mut self, token: &PrepareToken) -> Result<KernelRecord, KernelFault> {
740 if let Some(fault) = &self.poison {
741 return Err(fault.clone());
742 }
743
744 let Some(candidate) = &self.candidate else {
745 return Err(KernelFault::new(
746 KernelFaultCode::TransactionConflict,
747 format!(
748 "no candidate is outstanding for token {token}; a record that reached the \
749 journal is never abortable"
750 ),
751 ));
752 };
753 if &candidate.token != token {
754 return Err(KernelFault::new(
755 KernelFaultCode::TransactionConflict,
756 format!(
757 "token {token} does not name the outstanding candidate {}",
758 candidate.token
759 ),
760 ));
761 }
762
763 let candidate = self.candidate.take().expect("checked just above");
764 Ok(candidate.record)
765 }
766
767 pub fn note_append_conflict(
774 &mut self,
775 token: &PrepareToken,
776 observed_head: Option<&Digest>,
777 ) -> KernelFault {
778 let expected = self
779 .candidate
780 .as_ref()
781 .and_then(|candidate| candidate.record.expected_head().cloned());
782 self.candidate = None;
783 let observed =
784 observed_head.map_or_else(|| "an empty journal".to_string(), Digest::to_string);
785 let expected =
786 expected.map_or_else(|| "an empty journal".to_string(), |head| head.to_string());
787 let fault = KernelFault::new(
788 KernelFaultCode::TransactionConflict,
789 format!(
790 "the CAS append for token {token} expected head {expected} but the journal holds \
791 {observed}; the candidate is discarded and this runtime must be rebuilt from the \
792 journal before the input is replayed"
793 ),
794 );
795 self.poison_with(fault)
796 }
797
798 pub fn rebuild_from_records<F>(
808 records: &[KernelRecord],
809 defaults: ConfigDefaults,
810 index: Index,
811 mut plan: F,
812 ) -> Result<Self, KernelFault>
813 where
814 F: FnMut(&PlanContext<'_>) -> Result<Step, KernelFault>,
815 {
816 let mut transaction = Self::new(defaults, index);
817 if records.is_empty() {
818 return Ok(transaction);
819 }
820 verify_record_chain(records).map_err(corrupt_chain_fault)?;
821
822 for record in records {
823 let input = record.normalized_input().map_err(record_fault)?;
824 transaction.replay_committed(&input, record.record_digest(), &mut plan)?;
825 }
826 Ok(transaction)
827 }
828
829 pub fn replay_committed<F>(
843 &mut self,
844 input: &NormalizedInput,
845 expected_record_digest: &Digest,
846 plan: &mut F,
847 ) -> Result<KernelRecord, KernelFault>
848 where
849 F: FnMut(&PlanContext<'_>) -> Result<Step, KernelFault>,
850 {
851 if let Some(fault) = &self.poison {
852 return Err(fault.clone());
853 }
854 let step_seq = self.next_step_seq()?;
855 let genesis_config = input.resolved_config().cloned();
856 let config = match (&genesis_config, &self.config) {
857 (Some(config), _) => config,
858 (None, Some(config)) => config,
859 (None, None) => {
860 return Err(KernelFault::new(
861 KernelFaultCode::RecordCorrupted,
862 format!(
863 "the transition at step {step_seq} has no genesis configuration before it"
864 ),
865 ));
866 }
867 };
868
869 let resolving = match &input.input {
873 NormalizedPayload::ResolveEffect(resolve) => {
874 self.pending_effects.get(&resolve.effect_id)
875 }
876 _ => None,
877 };
878 let step = plan(&PlanContext {
879 input,
880 step_seq,
881 previous_head: self.head.as_ref().map(|anchor| &anchor.record_digest),
882 config,
883 resolving,
884 })?;
885
886 let rebuilt = KernelRecord::chain_after(self.head.as_ref(), input, &step)
887 .map_err(corrupt_chain_fault)?;
888 if rebuilt.record_digest() != expected_record_digest {
889 return Err(KernelFault::new(
890 KernelFaultCode::RecordCorrupted,
891 format!(
892 "replaying the transition at step {step_seq} produced record digest {} against \
893 the durable {expected_record_digest}; this binary does not reproduce the \
894 history it is resuming",
895 rebuilt.record_digest(),
896 ),
897 ));
898 }
899
900 let bytes = rebuilt.record_bytes().len() as u64;
901 self.integrate(rebuilt.clone(), step, input, bytes)?;
902 Ok(rebuilt)
903 }
904
905 pub fn restore_from_checkpoint(
920 checkpoint: &KernelCheckpoint,
921 defaults: ConfigDefaults,
922 index: Index,
923 ) -> Result<Self, KernelFault> {
924 let state = checkpoint.logical_state();
925 let transition = &state.transition;
926 let mut transaction = Self::new(defaults, index);
927
928 transaction.operation_id = Some(checkpoint.operation_id().clone());
929 transaction.genesis_digest = Some(checkpoint.genesis_digest().clone());
930 transaction.bounds = transition.resolved_config.recovery_policy.tail_bounds;
931 transaction.config = Some(transition.resolved_config.clone());
932 transaction.head = Some(ChainAnchor {
933 operation_id: checkpoint.operation_id().clone(),
934 step_seq: checkpoint.base_step_seq(),
935 record_digest: checkpoint.base_record_digest().clone(),
936 });
937 transaction.lifecycle = transition.lifecycle;
938 transaction.last_observed_at_ms = transition.last_observed_at_ms;
939 transaction.replay_floor = Some(checkpoint.base_step_seq());
940
941 if let Some(terminal) = &transition.terminal {
942 transaction
943 .terminal
944 .commit(terminal.clone())
945 .map_err(|error| {
946 KernelFault::new(KernelFaultCode::CheckpointCorrupted, error.to_string())
947 })?;
948 }
949 transaction.pending_effects = transition
950 .pending_effects
951 .iter()
952 .map(|effect| (effect.effect_id.clone(), effect.clone()))
953 .collect();
954 transaction.resolved_effects = transition
955 .resolved_effects
956 .iter()
957 .map(|resolved| {
958 (
959 resolved.effect_id.clone(),
960 ResolvedEffectRecord {
961 outcome_digest: resolved.outcome_digest.clone(),
962 input_id: resolved.input_id.clone(),
963 step_seq: resolved.step_seq,
964 },
965 )
966 })
967 .collect();
968 transaction.launch_tokens = transition
969 .launch_tokens
970 .iter()
971 .map(|token| (token.launch_token.clone(), token.step_seq))
972 .collect();
973 transaction.accepted = transition
974 .accepted_inputs
975 .iter()
976 .map(|entry| (entry.input_id.clone(), entry.clone()))
977 .collect();
978 transaction.accepted_cancellation =
979 transition
980 .accepted_cancellation
981 .as_ref()
982 .map(|cancellation| AcceptedCancellation {
983 command_digest: cancellation.command_digest.clone(),
984 input_id: cancellation.input_id.clone(),
985 step_seq: cancellation.step_seq,
986 });
987 Ok(transaction)
988 }
989
990 pub fn checkpoint_boundary(&self) -> Option<CheckpointBoundary> {
997 self.head.as_ref().map(|head| CheckpointBoundary {
998 through_step_seq: head.step_seq,
999 covered_head: head.record_digest.clone(),
1000 })
1001 }
1002
1003 pub fn tail_inputs(&self) -> Vec<CanonicalInput> {
1009 self.tail
1010 .iter()
1011 .map(|entry| CanonicalInput {
1012 step_seq: entry.step_seq,
1013 record_digest: entry.record_digest.clone(),
1014 input: entry.input.clone(),
1015 })
1016 .collect()
1017 }
1018
1019 pub fn checkpoint_candidate(
1030 &self,
1031 projection: LogicalStateProjection,
1032 ) -> Result<CheckpointCandidate, KernelFault> {
1033 let head = self.require_head()?.clone();
1034 self.assemble_checkpoint(
1035 projection,
1036 head.step_seq,
1037 head.record_digest.clone(),
1038 head.step_seq,
1039 head.record_digest,
1040 Vec::new(),
1041 )
1042 }
1043
1044 pub fn checkpoint_rebase(
1058 &self,
1059 base: &CheckpointBoundary,
1060 base_state: LogicalKernelState,
1061 ) -> Result<CheckpointCandidate, KernelFault> {
1062 let head = self.require_head()?.clone();
1063 if base.through_step_seq > head.step_seq {
1064 return Err(KernelFault::new(
1065 KernelFaultCode::CheckpointIncompatible,
1066 format!(
1067 "a rebase bases at step {} but this journal's head is step {}",
1068 base.through_step_seq, head.step_seq
1069 ),
1070 ));
1071 }
1072 let tail: Vec<CanonicalInput> = self
1073 .tail_inputs()
1074 .into_iter()
1075 .filter(|entry| entry.step_seq.get() > base.through_step_seq.get())
1076 .collect();
1077 if tail.len() as u64 != head.step_seq.get() - base.through_step_seq.get() {
1078 return Err(KernelFault::new(
1079 KernelFaultCode::CheckpointIncompatible,
1080 format!(
1081 "a rebase over ({}, {}] needs {} tail inputs, but this runtime's tail holds \
1082 {} of them — the prefix it would rebase onto was already reclaimed",
1083 base.through_step_seq,
1084 head.step_seq,
1085 head.step_seq.get() - base.through_step_seq.get(),
1086 tail.len()
1087 ),
1088 ));
1089 }
1090
1091 let operation_id = self.require_operation()?;
1092 let genesis_digest = self.require_genesis()?;
1093 let checkpoint = KernelCheckpoint::assemble(CheckpointDraft {
1094 operation_id: operation_id.clone(),
1095 genesis_digest: genesis_digest.clone(),
1096 base_step_seq: base.through_step_seq,
1097 base_record_digest: base.covered_head.clone(),
1098 through_step_seq: head.step_seq,
1099 covered_transaction_head_digest: head.record_digest,
1100 logical_state: base_state,
1101 tail_inputs: tail,
1102 })
1103 .map_err(|error| error.fault())?;
1104 Ok(checkpoint.into_candidate())
1105 }
1106
1107 #[allow(clippy::too_many_arguments)]
1108 fn assemble_checkpoint(
1109 &self,
1110 projection: LogicalStateProjection,
1111 base_step_seq: WireU64,
1112 base_record_digest: Digest,
1113 through_step_seq: WireU64,
1114 covered_transaction_head_digest: Digest,
1115 tail_inputs: Vec<CanonicalInput>,
1116 ) -> Result<CheckpointCandidate, KernelFault> {
1117 let operation_id = self.require_operation()?.clone();
1118 let genesis_digest = self.require_genesis()?.clone();
1119 let LogicalStateProjection {
1120 root_kind,
1121 focus,
1122 syscall,
1123 scheduler,
1124 context_vm,
1125 } = projection;
1126 let logical_state = LogicalKernelState {
1127 transition: self.transition_state(root_kind, focus)?,
1128 syscall,
1129 scheduler,
1130 context_vm,
1131 };
1132 let checkpoint = KernelCheckpoint::assemble(CheckpointDraft {
1133 operation_id,
1134 genesis_digest,
1135 base_step_seq,
1136 base_record_digest,
1137 through_step_seq,
1138 covered_transaction_head_digest,
1139 logical_state,
1140 tail_inputs,
1141 })
1142 .map_err(|error| error.fault())?;
1143 Ok(checkpoint.into_candidate())
1144 }
1145
1146 fn require_head(&self) -> Result<&ChainAnchor, KernelFault> {
1147 if let Some(fault) = &self.poison {
1148 return Err(fault.clone());
1149 }
1150 self.head.as_ref().ok_or_else(|| {
1151 KernelFault::new(
1152 KernelFaultCode::InvalidLifecycle,
1153 "an operation with no genesis record has no logical state to checkpoint"
1154 .to_string(),
1155 )
1156 })
1157 }
1158
1159 fn require_operation(&self) -> Result<&OperationId, KernelFault> {
1160 self.operation_id.as_ref().ok_or_else(|| {
1161 KernelFault::new(
1162 KernelFaultCode::InvalidLifecycle,
1163 "an unbound operation has no logical state to checkpoint".to_string(),
1164 )
1165 })
1166 }
1167
1168 fn require_genesis(&self) -> Result<&Digest, KernelFault> {
1169 self.genesis_digest.as_ref().ok_or_else(|| {
1170 KernelFault::new(
1171 KernelFaultCode::InvalidLifecycle,
1172 "an operation with no genesis record has no identity to bind a checkpoint to"
1173 .to_string(),
1174 )
1175 })
1176 }
1177
1178 pub fn transition_state_for_restore(
1184 &self,
1185 root_kind: Option<RootKind>,
1186 focus: Option<ExecutionFocus>,
1187 ) -> Result<TransitionStateV1, KernelFault> {
1188 self.transition_state(root_kind, focus)
1189 }
1190
1191 fn transition_state(
1197 &self,
1198 root_kind: Option<RootKind>,
1199 focus: Option<ExecutionFocus>,
1200 ) -> Result<TransitionStateV1, KernelFault> {
1201 let config = self.config.as_ref().ok_or_else(|| {
1202 KernelFault::new(
1203 KernelFaultCode::InvalidLifecycle,
1204 "an operation with no genesis record has no resolved configuration to checkpoint"
1205 .to_string(),
1206 )
1207 })?;
1208
1209 Ok(TransitionStateV1 {
1210 lifecycle: self.lifecycle,
1211 resolved_config: config.clone(),
1212 root_kind,
1213 focus,
1214 last_observed_at_ms: self.last_observed_at_ms,
1215 pending_effects: self.pending_effects.values().cloned().collect(),
1216 resolved_effects: self
1217 .resolved_effects
1218 .iter()
1219 .map(|(effect_id, resolved)| ResolvedEffectState {
1220 effect_id: effect_id.clone(),
1221 outcome_digest: resolved.outcome_digest.clone(),
1222 input_id: resolved.input_id.clone(),
1223 step_seq: resolved.step_seq,
1224 })
1225 .collect(),
1226 launch_tokens: self
1227 .launch_tokens
1228 .iter()
1229 .map(|(launch_token, step_seq)| LaunchTokenState {
1230 launch_token: launch_token.clone(),
1231 step_seq: *step_seq,
1232 })
1233 .collect(),
1234 accepted_inputs: self.accepted.values().cloned().collect(),
1235 accepted_cancellation: self.accepted_cancellation.as_ref().map(|cancellation| {
1236 AcceptedCancellationState {
1237 command_digest: cancellation.command_digest.clone(),
1238 input_id: cancellation.input_id.clone(),
1239 step_seq: cancellation.step_seq,
1240 }
1241 }),
1242 terminal: self.terminal.get().cloned(),
1243 })
1244 }
1245
1246 pub fn note_checkpoint_acked(
1256 &mut self,
1257 boundary: &CheckpointBoundary,
1258 ) -> Result<TailUsage, KernelFault> {
1259 if let Some(fault) = &self.poison {
1260 return Err(fault.clone());
1261 }
1262 let matches = self.tail.iter().any(|entry| {
1263 entry.step_seq == boundary.through_step_seq
1264 && entry.record_digest == boundary.covered_head
1265 });
1266 if !matches {
1267 return Err(KernelFault::new(
1268 KernelFaultCode::CheckpointIncompatible,
1269 format!(
1270 "no tail record at step {} has digest {}; this checkpoint does not cover a \
1271 prefix of this journal",
1272 boundary.through_step_seq, boundary.covered_head
1273 ),
1274 ));
1275 }
1276 while let Some(entry) = self.tail.front() {
1277 if entry.step_seq.get() <= boundary.through_step_seq.get() {
1278 self.tail.pop_front();
1279 } else {
1280 break;
1281 }
1282 }
1283 Ok(self.tail_usage())
1284 }
1285
1286 pub fn operation_id(&self) -> Option<&OperationId> {
1289 self.operation_id.as_ref()
1290 }
1291
1292 pub fn config(&self) -> Option<&ResolvedOperationConfig> {
1293 self.config.as_ref()
1294 }
1295
1296 pub fn head(&self) -> Option<DurableHead> {
1297 self.head.as_ref().map(|anchor| DurableHead {
1298 digest: anchor.record_digest.clone(),
1299 step_seq: anchor.step_seq,
1300 })
1301 }
1302
1303 pub fn lifecycle(&self) -> OperationLifecycle {
1304 self.lifecycle
1305 }
1306
1307 pub fn terminal(&self) -> Option<&KernelTerminal> {
1308 self.terminal.get()
1309 }
1310
1311 pub fn pending_effects(&self) -> impl Iterator<Item = &KernelEffect> {
1314 self.pending_effects.values()
1315 }
1316
1317 pub fn is_effect_resolved(&self, effect_id: &EffectId) -> bool {
1318 self.resolved_effects.contains_key(effect_id)
1319 }
1320
1321 pub fn knows_launch_token(&self, token: &LaunchToken) -> bool {
1322 self.launch_tokens.contains_key(token)
1323 }
1324
1325 pub fn committed_step(&self, input_id: &InputId) -> Option<&Step> {
1326 self.steps.get(input_id)
1327 }
1328
1329 pub fn outstanding_token(&self) -> Option<&PrepareToken> {
1330 self.candidate.as_ref().map(|candidate| &candidate.token)
1331 }
1332
1333 pub fn has_candidate(&self) -> bool {
1334 self.candidate.is_some()
1335 }
1336
1337 pub fn poison(&self) -> Option<&KernelFault> {
1340 self.poison.as_ref()
1341 }
1342
1343 pub fn is_poisoned(&self) -> bool {
1344 self.poison.is_some()
1345 }
1346
1347 pub fn bounds(&self) -> TailBounds {
1348 self.bounds
1349 }
1350
1351 pub fn index(&self) -> &Index {
1352 &self.index
1353 }
1354
1355 pub fn tail_usage(&self) -> TailUsage {
1356 TailUsage {
1357 records: self.tail_records(),
1358 bytes: self.tail_bytes(),
1359 }
1360 }
1361
1362 pub fn tail_pressure(&self) -> TailPressure {
1363 let usage = self.tail_usage();
1364 if usage.records >= self.bounds.hard_records.get()
1365 || usage.bytes >= self.bounds.hard_bytes.get()
1366 {
1367 TailPressure::Full
1368 } else if usage.records >= self.bounds.soft_records.get()
1369 || usage.bytes >= self.bounds.soft_bytes.get()
1370 {
1371 TailPressure::Watermark
1372 } else {
1373 TailPressure::Nominal
1374 }
1375 }
1376
1377 fn tail_records(&self) -> u64 {
1380 self.tail.len() as u64
1381 }
1382
1383 fn tail_bytes(&self) -> u64 {
1384 self.tail.iter().map(|entry| entry.bytes).sum()
1385 }
1386
1387 fn next_step_seq(&self) -> Result<WireU64, KernelFault> {
1388 match &self.head {
1389 None => Ok(WireU64::ZERO),
1390 Some(head) => head
1391 .step_seq
1392 .get()
1393 .checked_add(1)
1394 .map(WireU64::new)
1395 .ok_or_else(|| {
1396 KernelFault::new(
1397 KernelFaultCode::ResourceLimitExceeded,
1398 "step sequence overflowed u64".to_string(),
1399 )
1400 }),
1401 }
1402 }
1403
1404 fn checkpoint_required(&self, detail: String) -> KernelFault {
1405 KernelFault::new(
1406 KernelFaultCode::CheckpointRequired,
1407 format!(
1408 "{detail}; take a checkpoint candidate, install and ack it, then retry this input \
1409 unchanged — it was never accepted"
1410 ),
1411 )
1412 }
1413
1414 fn poison_with(&mut self, fault: KernelFault) -> KernelFault {
1415 self.candidate = None;
1416 self.poison.get_or_insert(fault).clone()
1417 }
1418
1419 fn replay_of(&self, existing: KernelRecord) -> Result<RecordPreparation<Step>, KernelFault> {
1427 let step_seq = existing.step_seq();
1428 let record_digest = existing.record_digest().clone();
1429 match self.steps.get(existing.input_id()) {
1430 Some(step) => {
1431 existing.verify_step(step).map_err(record_fault)?;
1432 Ok(KernelPreparation::Replayed(ReplayedTransition {
1433 record: Some(existing),
1434 record_digest,
1435 committed_step: Some(step.clone()),
1436 step_seq,
1437 }))
1438 }
1439 None if self.below_replay_floor(step_seq) => {
1440 Ok(KernelPreparation::Replayed(ReplayedTransition {
1441 record: Some(existing),
1442 record_digest,
1443 committed_step: None,
1444 step_seq,
1445 }))
1446 }
1447 None => Err(KernelFault::new(
1448 KernelFaultCode::RecordCorrupted,
1449 format!(
1450 "the journal holds record {record_digest} at step {step_seq} for input {}, but \
1451 this runtime has not replayed it and cannot reproduce its step; rebuild from \
1452 the journal first",
1453 existing.input_id()
1454 ),
1455 )),
1456 }
1457 }
1458
1459 fn replay_by_reference(&self, entry: &AcceptedInputState) -> RecordPreparation<Step> {
1462 KernelPreparation::Replayed(ReplayedTransition {
1463 record: None,
1464 record_digest: entry.record_digest.clone(),
1465 committed_step: None,
1466 step_seq: entry.step_seq,
1467 })
1468 }
1469
1470 fn below_replay_floor(&self, step_seq: WireU64) -> bool {
1471 self.replay_floor
1472 .is_some_and(|floor| step_seq.get() <= floor.get())
1473 }
1474
1475 fn resolve_effect_guard(
1480 &self,
1481 resolve: &super::envelope::ResolveEffect,
1482 ) -> Result<Option<RecordPreparation<Step>>, KernelFault> {
1484 let outcome_digest = outcome_digest(&resolve.outcome)?;
1485 if let Some(resolved) = self.resolved_effects.get(&resolve.effect_id) {
1486 if resolved.outcome_digest != outcome_digest {
1487 return Err(KernelFault::new(
1488 KernelFaultCode::UnexpectedEffectOutcome,
1489 format!(
1490 "effect {} was already resolved at step {} with a different outcome",
1491 resolve.effect_id, resolved.step_seq
1492 ),
1493 ));
1494 }
1495 let Some(existing) = self.index.record_for_input(
1498 self.operation_id.as_ref().expect("bound by the genesis"),
1499 &resolved.input_id,
1500 ) else {
1501 return Err(KernelFault::new(
1502 KernelFaultCode::RecordCorrupted,
1503 format!(
1504 "effect {} is resolved by input {} at step {}, but the journal has no such \
1505 record",
1506 resolve.effect_id, resolved.input_id, resolved.step_seq
1507 ),
1508 ));
1509 };
1510 return self.replay_of(existing).map(Some);
1511 }
1512
1513 let Some(pending) = self.pending_effects.get(&resolve.effect_id) else {
1514 return Err(KernelFault::new(
1515 KernelFaultCode::UnexpectedEffectOutcome,
1516 format!(
1517 "effect {} is not pending; the kernel is not waiting on it",
1518 resolve.effect_id
1519 ),
1520 ));
1521 };
1522 pending.accept_outcome(&resolve.outcome).map_err(|error| {
1523 KernelFault::new(KernelFaultCode::UnexpectedEffectOutcome, error.to_string())
1524 })?;
1525 Ok(None)
1526 }
1527
1528 fn cancel_guard(
1536 &self,
1537 cancel: &CancelCommand,
1538 ) -> Result<Option<RecordPreparation<Step>>, KernelFault> {
1539 let Some(accepted) = &self.accepted_cancellation else {
1540 return Ok(None);
1541 };
1542 if accepted.command_digest != cancel_digest(cancel)? {
1543 return Err(KernelFault::new(
1544 KernelFaultCode::DuplicateInputConflict,
1545 format!(
1546 "this operation committed a different cancellation at step {}; a cancel is not \
1547 re-decided once the terminal it produced exists",
1548 accepted.step_seq
1549 ),
1550 ));
1551 }
1552 let Some(existing) = self.index.record_for_input(
1553 self.operation_id.as_ref().expect("bound by the genesis"),
1554 &accepted.input_id,
1555 ) else {
1556 return Err(KernelFault::new(
1557 KernelFaultCode::RecordCorrupted,
1558 format!(
1559 "this operation was cancelled by input {} at step {}, but the journal has no \
1560 such record",
1561 accepted.input_id, accepted.step_seq
1562 ),
1563 ));
1564 };
1565 self.replay_of(existing).map(Some)
1566 }
1567
1568 fn screen_planned_effects(
1571 &self,
1572 step: &Step,
1573 config: &ResolvedOperationConfig,
1574 settled: Option<&EffectId>,
1575 ) -> Result<(), KernelFault> {
1576 let mut kinds_in_step: Vec<EffectKindTag> = Vec::new();
1577 let mut tokens_in_step: Vec<&LaunchToken> = Vec::new();
1578 let still_pending = |effect_id: &EffectId| {
1582 self.pending_effects.contains_key(effect_id) && Some(effect_id) != settled
1583 };
1584
1585 for effect in step.effects() {
1586 let tag = effect.tag();
1587
1588 if !config.host_effect_support.supports(tag) {
1590 return Err(KernelFault::new(
1591 KernelFaultCode::UnsupportedEffect,
1592 format!(
1593 "this operation's host does not declare support for {tag} effects, so \
1594 effect {} is refused before emission",
1595 effect.effect_id
1596 ),
1597 ));
1598 }
1599
1600 if self.pending_effects.contains_key(&effect.effect_id)
1603 || self.resolved_effects.contains_key(&effect.effect_id)
1604 || Some(&effect.effect_id) == settled
1605 {
1606 return Err(KernelFault::new(
1607 KernelFaultCode::TransactionConflict,
1608 format!(
1609 "effect id {} was already published; effect identity is minted once",
1610 effect.effect_id
1611 ),
1612 ));
1613 }
1614
1615 if kinds_in_step.contains(&tag)
1617 || self
1618 .pending_effects
1619 .iter()
1620 .any(|(id, pending)| pending.tag() == tag && still_pending(id))
1621 {
1622 return Err(KernelFault::new(
1623 KernelFaultCode::ResourceLimitExceeded,
1624 format!(
1625 "a {tag} effect is already pending; resolve it before emitting another \
1626 (§15.3 admits at most one pending effect per kind)"
1627 ),
1628 ));
1629 }
1630 kinds_in_step.push(tag);
1631
1632 if let EffectKind::SpawnTasks(spawn) = &effect.effect {
1633 for launch in &spawn.tasks {
1634 if self.launch_tokens.contains_key(&launch.launch_token)
1635 || tokens_in_step.contains(&&launch.launch_token)
1636 {
1637 return Err(KernelFault::new(
1638 KernelFaultCode::TransactionConflict,
1639 format!(
1640 "launch token {} was already published; a re-launch reuses the \
1641 committed token so the host's launch dedup stays exact",
1642 launch.launch_token
1643 ),
1644 ));
1645 }
1646 tokens_in_step.push(&launch.launch_token);
1647 }
1648 }
1649 }
1650 Ok(())
1651 }
1652
1653 fn integrate(
1656 &mut self,
1657 record: KernelRecord,
1658 step: Step,
1659 input: &NormalizedInput,
1660 record_bytes: u64,
1661 ) -> Result<CommittedTransition<Step>, KernelFault> {
1662 let step_seq = record.step_seq();
1663 let was_nominal = self.tail_pressure() == TailPressure::Nominal;
1664
1665 if self.operation_id.is_none() {
1666 self.operation_id = Some(record.operation_id().clone());
1667 }
1668 if record.is_genesis() {
1669 self.genesis_digest = Some(record.record_digest().clone());
1670 }
1671 if let Some(config) = input.resolved_config() {
1672 self.bounds = config.recovery_policy.tail_bounds;
1677 self.config = Some(config.clone());
1678 self.lifecycle = OperationLifecycle::Configured;
1679 } else if matches!(input.input, NormalizedPayload::StartOperation(_)) {
1680 self.lifecycle = OperationLifecycle::Running;
1681 }
1682
1683 if let NormalizedPayload::HostControl(control) = &input.input
1684 && let HostCommand::Cancel(cancel) = &control.command
1685 {
1686 self.accepted_cancellation = Some(AcceptedCancellation {
1687 command_digest: cancel_digest(cancel)?,
1688 input_id: record.input_id().clone(),
1689 step_seq,
1690 });
1691 }
1692
1693 if let NormalizedPayload::ResolveEffect(resolve) = &input.input {
1694 self.pending_effects.remove(&resolve.effect_id);
1695 self.resolved_effects.insert(
1696 resolve.effect_id.clone(),
1697 ResolvedEffectRecord {
1698 outcome_digest: outcome_digest(&resolve.outcome)?,
1699 input_id: record.input_id().clone(),
1700 step_seq,
1701 },
1702 );
1703 }
1704
1705 for effect in step.effects() {
1706 self.pending_effects
1707 .insert(effect.effect_id.clone(), effect.clone());
1708 if let EffectKind::SpawnTasks(spawn) = &effect.effect {
1709 for launch in &spawn.tasks {
1710 self.launch_tokens
1711 .insert(launch.launch_token.clone(), step_seq);
1712 }
1713 }
1714 }
1715
1716 if let Some(terminal) = step.terminal() {
1717 self.terminal.commit(terminal.clone()).map_err(|error| {
1718 KernelFault::new(KernelFaultCode::InvalidLifecycle, error.to_string())
1719 })?;
1720 self.lifecycle = terminal_lifecycle(terminal);
1721 self.pending_effects.clear();
1723 }
1724
1725 self.last_observed_at_ms = input.observed_at_ms;
1726 self.steps.insert(record.input_id().clone(), step.clone());
1727 self.accepted.insert(
1728 record.input_id().clone(),
1729 AcceptedInputState {
1730 input_id: record.input_id().clone(),
1731 step_seq,
1732 record_digest: record.record_digest().clone(),
1733 },
1734 );
1735 self.tail.push_back(TailEntry {
1736 step_seq,
1737 record_digest: record.record_digest().clone(),
1738 bytes: record_bytes,
1739 input: input.clone(),
1740 });
1741 self.index.note_committed(&record);
1742 self.head = Some(record.anchor());
1743
1744 let checkpoint_advice = (was_nominal && self.tail_pressure() != TailPressure::Nominal)
1747 .then(|| CheckpointAdvice {
1748 through_step_seq: step_seq,
1749 usage: self.tail_usage(),
1750 bounds: self.bounds,
1751 });
1752
1753 Ok(CommittedTransition {
1754 record,
1755 step,
1756 step_seq,
1757 checkpoint_advice,
1758 })
1759 }
1760}
1761
1762fn outcome_digest(outcome: &EffectOutcome) -> Result<Digest, KernelFault> {
1767 canonical_bytes(outcome)
1768 .map(|bytes| canonical_digest(bytes.as_slice()))
1769 .map_err(record_fault)
1770}
1771
1772fn cancel_digest(cancel: &CancelCommand) -> Result<Digest, KernelFault> {
1773 canonical_bytes(cancel)
1774 .map(|bytes| canonical_digest(bytes.as_slice()))
1775 .map_err(record_fault)
1776}
1777
1778fn record_fault(error: RecordError) -> KernelFault {
1779 KernelFault::new(error.code(), error.message().to_string())
1780}
1781
1782fn corrupt_chain_fault(error: RecordError) -> KernelFault {
1785 match error {
1786 RecordError::ChainBroken(message) => {
1787 KernelFault::new(KernelFaultCode::RecordCorrupted, message)
1788 }
1789 other => record_fault(other),
1790 }
1791}
1792
1793fn rejection_fault(rejection: WireRejection) -> KernelFault {
1794 let code = match rejection.kind {
1795 WireRejectionKind::PolicyViolation => KernelFaultCode::InvalidConfig,
1796 WireRejectionKind::VersionMismatch => KernelFaultCode::VersionMismatch,
1797 _ => KernelFaultCode::MalformedEnvelope,
1798 };
1799 KernelFault::new(code, rejection.message)
1800}
1801
1802fn terminal_lifecycle(terminal: &KernelTerminal) -> OperationLifecycle {
1803 match terminal {
1804 KernelTerminal::Agent(_) | KernelTerminal::Workflow(_) => OperationLifecycle::Completed,
1805 KernelTerminal::Cancelled(_) => OperationLifecycle::Cancelled,
1806 KernelTerminal::Failed(_) => OperationLifecycle::Failed,
1807 }
1808}
1809
1810impl fmt::Display for CheckpointBoundary {
1811 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1812 write!(
1813 f,
1814 "through step {} at head {}",
1815 self.through_step_seq, self.covered_head
1816 )
1817 }
1818}
1819
1820#[cfg(test)]
1821mod tests {
1822 use serde::Serialize;
1823
1824 use super::super::*;
1825
1826 const OPERATION: &str = "op-tx-1";
1831
1832 fn operation() -> OperationId {
1833 OperationId::new(OPERATION).unwrap()
1834 }
1835
1836 fn input_id(name: &str) -> InputId {
1837 InputId::new(name).unwrap()
1838 }
1839
1840 fn boot_config(supported: impl IntoIterator<Item = EffectKindTag>) -> OperationConfig {
1841 OperationConfig {
1842 execution_policy: Some(ExecutionPolicy {
1843 max_turns: Some(12),
1844 ..ExecutionPolicy::default()
1845 }),
1846 host_effect_support: HostEffectSupport::new(supported),
1847 ..OperationConfig::default()
1848 }
1849 }
1850
1851 fn envelope(id: &str, observed_at_ms: u64, input: KernelInput) -> WireEnvelope {
1852 WireEnvelope::new(
1853 operation(),
1854 input_id(id),
1855 WireU64::new(observed_at_ms),
1856 input,
1857 )
1858 }
1859
1860 fn configure_at(id: &str, supported: impl IntoIterator<Item = EffectKindTag>) -> WireEnvelope {
1861 envelope(
1862 id,
1863 1_700_000_000_000,
1864 KernelInput::ConfigureOperation(ConfigureOperation {
1865 config: boot_config(supported),
1866 }),
1867 )
1868 }
1869
1870 fn configure() -> WireEnvelope {
1871 configure_at(
1872 "in-configure",
1873 [EffectKindTag::CallProvider, EffectKindTag::SpawnTasks],
1874 )
1875 }
1876
1877 fn start_at(id: &str, observed_at_ms: u64) -> WireEnvelope {
1878 envelope(
1879 id,
1880 observed_at_ms,
1881 KernelInput::StartOperation(StartOperation {
1882 entry: RootEntry::Agent(RootAgentEntry {
1883 task: LogicalTask::new("write the brief"),
1884 run_spec: None,
1885 }),
1886 initial_context: InitialContext::default(),
1887 }),
1888 )
1889 }
1890
1891 fn start() -> WireEnvelope {
1892 start_at("in-start", 1_700_000_001_000)
1893 }
1894
1895 fn provider_outcome() -> EffectOutcome {
1896 EffectOutcome::Succeeded(EffectSucceeded {
1897 result: EffectSuccess::Provider(ProviderSuccess {
1898 outcome: ProviderOutcome::ContextOverflow(ProviderContextOverflow::default()),
1899 }),
1900 })
1901 }
1902
1903 fn failure_outcome() -> EffectOutcome {
1904 EffectOutcome::Failed(EffectFailed {
1905 failure: HostEffectFailure {
1906 kind: HostEffectFailureKind::TransportExhausted,
1907 message: "the vendor gave up".to_string(),
1908 retryable: Some(false),
1909 },
1910 })
1911 }
1912
1913 fn resolve_at(
1914 id: &str,
1915 observed_at_ms: u64,
1916 effect_id: &EffectId,
1917 outcome: EffectOutcome,
1918 ) -> WireEnvelope {
1919 envelope(
1920 id,
1921 observed_at_ms,
1922 KernelInput::ResolveEffect(ResolveEffect {
1923 effect_id: effect_id.clone(),
1924 outcome,
1925 }),
1926 )
1927 }
1928
1929 fn cancel_at(id: &str, observed_at_ms: u64) -> WireEnvelope {
1930 envelope(
1931 id,
1932 observed_at_ms,
1933 KernelInput::HostControl(HostControl {
1934 command: HostCommand::Cancel(CancelCommand {
1935 reason: CancellationReason::User,
1936 pending_call_ids: vec![],
1937 }),
1938 }),
1939 )
1940 }
1941
1942 fn signal_at(id: &str, observed_at_ms: u64) -> WireEnvelope {
1943 use super::super::event::{DeliverSignal, ExternalEvent, LogicalSignal};
1944 use super::super::scalar::{DeliveryId, SignalId};
1945
1946 envelope(
1947 id,
1948 observed_at_ms,
1949 KernelInput::DeliverExternalEvent(DeliverExternalEvent {
1950 event: ExternalEvent::DeliverSignal(DeliverSignal {
1951 delivery_id: DeliveryId::new(format!("delivery-{id}")).unwrap(),
1952 attempt: 1,
1953 signal: LogicalSignal::new(SignalId::new("sig-late").unwrap()),
1954 }),
1955 }),
1956 )
1957 }
1958
1959 #[derive(Debug, Clone, PartialEq, Serialize)]
1962 struct TestStep {
1963 plan: String,
1964 disposition: StepDisposition,
1965 }
1966
1967 impl TransitionStep for TestStep {
1968 fn disposition(&self) -> &StepDisposition {
1969 &self.disposition
1970 }
1971 }
1972
1973 fn nothing(plan: &str) -> TestStep {
1974 TestStep {
1975 plan: plan.to_string(),
1976 disposition: StepDisposition::Effects(EffectsDisposition::default()),
1977 }
1978 }
1979
1980 fn publishing(plan: &str, effects: Vec<KernelEffect>) -> TestStep {
1981 TestStep {
1982 plan: plan.to_string(),
1983 disposition: StepDisposition::Effects(EffectsDisposition { effects }),
1984 }
1985 }
1986
1987 fn effect(id: &str, causation: &InputId, kind: EffectKind) -> KernelEffect {
1988 KernelEffect {
1989 effect_id: EffectId::new(id).unwrap(),
1990 causation_input_id: causation.clone(),
1991 effect: kind,
1992 }
1993 }
1994
1995 fn provider_effect_id(step_seq: WireU64) -> EffectId {
1996 EffectId::new(format!("{OPERATION}:step:{step_seq}:effect:0")).unwrap()
1997 }
1998
1999 fn plan(context: &PlanContext<'_>) -> Result<TestStep, KernelFault> {
2005 let label = format!("{}@{}", context.input.input.kind(), context.step_seq);
2006 Ok(match &context.input.input {
2007 NormalizedPayload::StartOperation(_) => publishing(
2008 &label,
2009 vec![effect(
2010 provider_effect_id(context.step_seq).as_str(),
2011 &context.input.input_id,
2012 EffectKind::CallProvider(CallProviderEffect::default()),
2013 )],
2014 ),
2015 NormalizedPayload::HostControl(_) => TestStep {
2016 plan: label,
2017 disposition: StepDisposition::Terminal(TerminalDisposition {
2018 terminal: KernelTerminal::Cancelled(CancelledTerminal {
2019 reason: CancellationReason::User,
2020 usage: UsageReport::default(),
2021 }),
2022 }),
2023 },
2024 _ => nothing(&label),
2025 })
2026 }
2027
2028 type Tx = KernelTransaction<TestStep, InMemoryRecordIndex>;
2029
2030 fn transaction() -> Tx {
2031 KernelTransaction::new(ConfigDefaults::default(), InMemoryRecordIndex::new())
2032 }
2033
2034 fn bounded(bounds: TailBounds) -> Tx {
2042 let mut defaults = ConfigDefaults::default();
2043 defaults.baseline.recovery_policy.tail_bounds = bounds;
2044 KernelTransaction::new(defaults, InMemoryRecordIndex::new())
2045 }
2046
2047 fn run(tx: &mut Tx, envelope: &WireEnvelope) -> CommittedTransition<TestStep> {
2049 run_with(tx, envelope, plan)
2050 }
2051
2052 fn run_with<F>(
2053 tx: &mut Tx,
2054 envelope: &WireEnvelope,
2055 planner: F,
2056 ) -> CommittedTransition<TestStep>
2057 where
2058 F: FnOnce(&PlanContext<'_>) -> Result<TestStep, KernelFault>,
2059 {
2060 let preparation = tx.prepare(envelope, planner);
2061 let token = preparation
2062 .token()
2063 .unwrap_or_else(|| {
2064 panic!(
2065 "expected a prepared transition, got {:?}",
2066 preparation.fault()
2067 )
2068 })
2069 .clone();
2070 let head = preparation.record().unwrap().record_digest().clone();
2071 tx.commit(&token, &head).expect("the commit must succeed")
2072 }
2073
2074 fn fault_of(preparation: &RecordPreparation<TestStep>) -> KernelFaultCode {
2075 preparation
2076 .fault()
2077 .unwrap_or_else(|| panic!("expected a rejection, got a success"))
2078 .code
2079 }
2080
2081 fn observable(
2084 tx: &Tx,
2085 ) -> (
2086 Option<DurableHead>,
2087 OperationLifecycle,
2088 Vec<String>,
2089 TailUsage,
2090 bool,
2091 ) {
2092 (
2093 tx.head(),
2094 tx.lifecycle(),
2095 tx.pending_effects()
2096 .map(|effect| effect.effect_id.to_string())
2097 .collect(),
2098 tx.tail_usage(),
2099 tx.has_candidate(),
2100 )
2101 }
2102
2103 fn started() -> (Tx, Vec<KernelRecord>, EffectId) {
2104 let mut tx = transaction();
2105 let genesis = run(&mut tx, &configure());
2106 let started = run(&mut tx, &start());
2107 let effect_id = provider_effect_id(started.step_seq);
2108 (tx, vec![genesis.record, started.record], effect_id)
2109 }
2110
2111 #[test]
2117 fn matrix_row1_before_a_prepare_nothing_exists() {
2118 let tx = transaction();
2119 assert!(!tx.has_candidate());
2120 assert_eq!(tx.head(), None);
2121 assert_eq!(tx.lifecycle(), OperationLifecycle::Created);
2122 assert_eq!(tx.pending_effects().count(), 0);
2123 assert_eq!(tx.tail_usage(), TailUsage::default());
2124 assert!(tx.index().is_empty(), "no record exists yet");
2125 assert_eq!(tx.checkpoint_boundary(), None);
2126 assert!(tx.outstanding_token().is_none());
2127 }
2128
2129 #[test]
2134 fn matrix_row2_a_rejected_prepare_hands_out_no_token_and_moves_nothing() {
2135 let (mut tx, _, effect_id) = started();
2136
2137 let rejections: Vec<(&str, RecordPreparation<TestStep>)> = vec![
2138 (
2140 "second configure",
2141 tx.prepare(
2142 &configure_at("in-again", [EffectKindTag::CallProvider]),
2143 plan,
2144 ),
2145 ),
2146 (
2148 "unknown effect",
2149 tx.prepare(
2150 &resolve_at(
2151 "in-unknown",
2152 1_700_000_002_000,
2153 &EffectId::new("op-tx-1:step:9:effect:0").unwrap(),
2154 failure_outcome(),
2155 ),
2156 plan,
2157 ),
2158 ),
2159 (
2161 "planner fault",
2162 tx.prepare(
2163 &resolve_at(
2164 "in-planned",
2165 1_700_000_002_000,
2166 &effect_id,
2167 failure_outcome(),
2168 ),
2169 |_| {
2170 Err(KernelFault::new(
2171 KernelFaultCode::ResourceLimitExceeded,
2172 "the planner refused",
2173 ))
2174 },
2175 ),
2176 ),
2177 ];
2178
2179 let before = tx.clone();
2180 for (label, preparation) in rejections {
2181 assert!(preparation.is_zero_mutation(), "{label}");
2182 assert!(preparation.token().is_none(), "{label}");
2183 assert!(preparation.record().is_none(), "{label}");
2184 assert!(preparation.step().is_none(), "{label}");
2185 assert!(preparation.step_seq().is_none(), "{label}");
2186 }
2187 assert_eq!(tx, before, "a rejected prepare must not move one byte");
2188 }
2189
2190 #[test]
2192 fn matrix_row3_a_crash_before_the_append_is_undone_by_abort() {
2193 let (mut tx, records, _) = started();
2194 let before = observable(&tx);
2195
2196 let preparation = tx.prepare(&cancel_at("in-cancel", 1_700_000_003_000), plan);
2197 let token = preparation.token().expect("prepared").clone();
2198 assert!(tx.has_candidate());
2199 assert_eq!(
2200 tx.head().map(|head| head.digest),
2201 Some(records[1].record_digest().clone()),
2202 "a candidate does not move the durable head"
2203 );
2204
2205 let discarded = tx.abort(&token).expect("the candidate was never appended");
2206 assert_eq!(discarded.step_seq(), WireU64::new(2));
2207 assert_eq!(
2208 observable(&tx),
2209 before,
2210 "an aborted candidate leaves no trace"
2211 );
2212
2213 assert_eq!(
2215 tx.abort(&token).unwrap_err().code,
2216 KernelFaultCode::TransactionConflict
2217 );
2218 assert!(
2219 !tx.is_poisoned(),
2220 "an abort is a normal, non-poisoning path"
2221 );
2222 }
2223
2224 #[test]
2226 fn matrix_row4_a_cas_conflict_discards_the_candidate_and_demands_a_rebuild() {
2227 let (mut tx, records, _) = started();
2228 let preparation = tx.prepare(&cancel_at("in-cancel", 1_700_000_003_000), plan);
2229 let token = preparation.token().expect("prepared").clone();
2230
2231 let forked = canonical_digest(b"another writer's record");
2233 let fault = tx.note_append_conflict(&token, Some(&forked));
2234
2235 assert_eq!(fault.code, KernelFaultCode::TransactionConflict);
2236 assert!(!fault.is_retryable(), "a conflict is not a bare retry");
2237 assert!(
2238 !tx.has_candidate(),
2239 "the candidate is discarded, not appended"
2240 );
2241 assert!(tx.is_poisoned(), "this layer never rebuilds itself");
2242
2243 assert_eq!(
2245 fault_of(&tx.prepare(&cancel_at("in-cancel-2", 1_700_000_004_000), plan)),
2246 KernelFaultCode::TransactionConflict
2247 );
2248 assert_eq!(
2249 tx.abort(&token).unwrap_err().code,
2250 KernelFaultCode::TransactionConflict
2251 );
2252 assert_eq!(
2253 tx.commit(&token, &forked).unwrap_err().code,
2254 KernelFaultCode::TransactionConflict
2255 );
2256
2257 let rebuilt = Tx::rebuild_from_records(
2259 &records,
2260 ConfigDefaults::default(),
2261 InMemoryRecordIndex::from_records(&records),
2262 plan,
2263 )
2264 .expect("the journal still verifies");
2265 assert_eq!(
2266 rebuilt.head().map(|head| head.digest),
2267 Some(records[1].record_digest().clone())
2268 );
2269 assert!(!rebuilt.is_poisoned());
2270 }
2271
2272 #[test]
2275 fn matrix_row5_a_crash_between_append_and_commit_rebuilds_and_publishes_once() {
2276 let mut tx = transaction();
2277 let genesis = run(&mut tx, &configure());
2278 let mut journal = vec![genesis.record];
2279
2280 let preparation = tx.prepare(&start(), plan);
2281 let appended = preparation.record().expect("prepared").clone();
2282 journal.push(appended.clone());
2284 assert_eq!(
2286 tx.pending_effects().count(),
2287 0,
2288 "a record that has not been committed publishes no effect (§15.2)"
2289 );
2290 drop(tx);
2291
2292 let rebuilt = Tx::rebuild_from_records(
2293 &journal,
2294 ConfigDefaults::default(),
2295 InMemoryRecordIndex::from_records(&journal),
2296 plan,
2297 )
2298 .expect("the journal rebuilds");
2299
2300 assert_eq!(
2301 rebuilt.head().map(|head| head.digest),
2302 Some(appended.record_digest().clone())
2303 );
2304 assert_eq!(rebuilt.lifecycle(), OperationLifecycle::Running);
2305 let pending: Vec<&EffectId> = rebuilt
2306 .pending_effects()
2307 .map(|effect| &effect.effect_id)
2308 .collect();
2309 assert_eq!(
2310 pending,
2311 vec![&provider_effect_id(WireU64::new(1))],
2312 "the rebuilt runtime re-exposes the one pending effect, with the same identity"
2313 );
2314 }
2315
2316 #[test]
2319 fn matrix_row6_a_failed_commit_never_becomes_an_abort() {
2320 let mut tx = transaction();
2321 let genesis = run(&mut tx, &configure());
2322 let journal = vec![genesis.record];
2323
2324 let preparation = tx.prepare(&start(), plan);
2325 let token = preparation.token().expect("prepared").clone();
2326 let wrong_head = canonical_digest(b"some other record");
2329 let fault = tx
2330 .commit(&token, &wrong_head)
2331 .expect_err("a commit that cannot be attributed must fail");
2332 assert_eq!(fault.code, KernelFaultCode::TransactionConflict);
2333
2334 assert!(tx.is_poisoned());
2337 assert!(!tx.has_candidate());
2338 assert_eq!(
2339 tx.abort(&token).unwrap_err().code,
2340 KernelFaultCode::TransactionConflict,
2341 "append-then-abort is not expressible"
2342 );
2343
2344 let rebuilt = Tx::rebuild_from_records(
2346 &journal,
2347 ConfigDefaults::default(),
2348 InMemoryRecordIndex::from_records(&journal),
2349 plan,
2350 )
2351 .expect("the durable record survives a failed commit");
2352 assert_eq!(rebuilt.lifecycle(), OperationLifecycle::Configured);
2353 }
2354
2355 #[test]
2357 fn matrix_row7_a_lost_commit_response_replays_the_same_record_and_step() {
2358 let mut tx = transaction();
2359 run(&mut tx, &configure());
2360 let committed = run(&mut tx, &start());
2361 let before = observable(&tx);
2362
2363 let replay = tx.prepare(&start(), plan);
2365
2366 assert!(replay.token().is_none(), "a replay has nothing to commit");
2367 assert_eq!(replay.step_seq(), Some(committed.step_seq));
2368 assert_eq!(replay.record(), Some(&committed.record));
2369 assert_eq!(replay.step(), Some(&committed.step));
2370 assert_eq!(
2371 observable(&tx),
2372 before,
2373 "a replay creates no record and re-publishes no effect"
2374 );
2375 assert_eq!(tx.index().len(), 2, "no second record was minted");
2376 }
2377
2378 #[test]
2381 fn matrix_row8_a_lost_resolution_response_replays_idempotently() {
2382 let (mut tx, _, effect_id) = started();
2383 let resolved = run(
2384 &mut tx,
2385 &resolve_at(
2386 "in-resolve",
2387 1_700_000_002_000,
2388 &effect_id,
2389 provider_outcome(),
2390 ),
2391 );
2392 assert_eq!(tx.pending_effects().count(), 0, "the effect is settled");
2393 let before = observable(&tx);
2394
2395 let same_input = tx.prepare(
2397 &resolve_at(
2398 "in-resolve",
2399 1_700_000_002_000,
2400 &effect_id,
2401 provider_outcome(),
2402 ),
2403 plan,
2404 );
2405 assert_eq!(same_input.step_seq(), Some(resolved.step_seq));
2406 assert_eq!(same_input.record(), Some(&resolved.record));
2407
2408 assert_eq!(
2412 fault_of(&tx.prepare(
2413 &resolve_at(
2414 "in-resolve",
2415 1_700_000_002_500,
2416 &effect_id,
2417 provider_outcome()
2418 ),
2419 plan
2420 )),
2421 KernelFaultCode::DuplicateInputConflict
2422 );
2423
2424 let new_input = tx.prepare(
2427 &resolve_at(
2428 "in-resolve-again",
2429 1_700_000_003_000,
2430 &effect_id,
2431 provider_outcome(),
2432 ),
2433 plan,
2434 );
2435 assert_eq!(
2436 new_input.step_seq(),
2437 Some(resolved.step_seq),
2438 "effect-level dedup points at the existing record's step_seq"
2439 );
2440 assert_eq!(new_input.record(), Some(&resolved.record));
2441 assert!(
2442 new_input.token().is_none(),
2443 "reporting this as Prepared with an old step_seq is the dead end this replaces"
2444 );
2445 assert_eq!(observable(&tx), before);
2446 assert_eq!(tx.index().len(), 3, "still three records");
2447 }
2448
2449 #[test]
2454 fn a_committed_record_can_never_be_aborted() {
2455 let mut tx = transaction();
2456 let preparation = tx.prepare(&configure(), plan);
2457 let token = preparation.token().expect("prepared").clone();
2458 let head = preparation.record().unwrap().record_digest().clone();
2459 tx.commit(&token, &head).expect("commit");
2460
2461 let error = tx
2462 .abort(&token)
2463 .expect_err("a committed record has no candidate to abort");
2464 assert_eq!(error.code, KernelFaultCode::TransactionConflict);
2465 assert!(!tx.is_poisoned(), "asking is not itself a corruption");
2466 assert_eq!(
2467 tx.head().map(|head| head.step_seq),
2468 Some(WireU64::ZERO),
2469 "the committed record stands"
2470 );
2471 }
2472
2473 #[test]
2474 fn a_second_prepare_is_refused_while_a_candidate_is_outstanding() {
2475 let mut tx = transaction();
2476 let first = tx.prepare(&configure(), plan);
2477 let token = first.token().expect("prepared").clone();
2478
2479 let second = tx.prepare(&start(), plan);
2480 assert_eq!(fault_of(&second), KernelFaultCode::TransactionConflict);
2481 assert_eq!(
2482 tx.outstanding_token(),
2483 Some(&token),
2484 "the first candidate is not displaced by the second attempt"
2485 );
2486
2487 let head = first.record().unwrap().record_digest().clone();
2489 tx.commit(&token, &head).expect("the first candidate wins");
2490 }
2491
2492 #[test]
2493 fn effects_become_visible_only_when_their_record_is_durable() {
2494 let mut tx = transaction();
2495 run(&mut tx, &configure());
2496
2497 let preparation = tx.prepare(&start(), plan);
2498 assert_eq!(
2499 preparation.step().unwrap().effects().len(),
2500 1,
2501 "the planned step carries the effect"
2502 );
2503 assert_eq!(
2504 tx.pending_effects().count(),
2505 0,
2506 "but nothing is pending until the record is durable"
2507 );
2508
2509 let token = preparation.token().unwrap().clone();
2510 let head = preparation.record().unwrap().record_digest().clone();
2511 let committed = tx.commit(&token, &head).unwrap();
2512 assert_eq!(committed.published_effects().len(), 1);
2513 assert_eq!(tx.pending_effects().count(), 1);
2514 }
2515
2516 #[test]
2517 fn an_aborted_candidate_publishes_nothing() {
2518 let mut tx = transaction();
2519 run(&mut tx, &configure());
2520 let preparation = tx.prepare(&start(), plan);
2521 let token = preparation.token().unwrap().clone();
2522 tx.abort(&token).unwrap();
2523 assert_eq!(tx.pending_effects().count(), 0);
2524 assert_eq!(tx.lifecycle(), OperationLifecycle::Configured);
2525 }
2526
2527 #[test]
2528 fn the_operation_id_is_bound_by_the_genesis_record() {
2529 let mut tx = transaction();
2530 run(&mut tx, &configure());
2531 assert_eq!(tx.operation_id(), Some(&operation()));
2532
2533 let foreign = WireEnvelope::new(
2534 OperationId::new("op-other").unwrap(),
2535 input_id("in-foreign"),
2536 WireU64::new(1_700_000_002_000),
2537 KernelInput::HostControl(HostControl {
2538 command: HostCommand::Cancel(CancelCommand {
2539 reason: CancellationReason::User,
2540 pending_call_ids: vec![],
2541 }),
2542 }),
2543 );
2544 assert_eq!(
2545 fault_of(&tx.prepare(&foreign, plan)),
2546 KernelFaultCode::OperationMismatch
2547 );
2548 }
2549
2550 #[test]
2551 fn a_duplicate_input_id_with_a_different_payload_is_a_conflict() {
2552 let mut tx = transaction();
2553 run(&mut tx, &configure());
2554 run(&mut tx, &start());
2555
2556 let mut divergent = start();
2558 divergent.input = KernelInput::StartOperation(StartOperation {
2559 entry: RootEntry::Agent(RootAgentEntry {
2560 task: LogicalTask::new("write something else"),
2561 run_spec: None,
2562 }),
2563 initial_context: InitialContext::default(),
2564 });
2565 assert_eq!(
2566 fault_of(&tx.prepare(&divergent, plan)),
2567 KernelFaultCode::DuplicateInputConflict
2568 );
2569 }
2570
2571 #[test]
2572 fn an_exact_replay_is_answered_before_the_clock_check() {
2573 let mut tx = transaction();
2574 run(&mut tx, &configure());
2575 let started = run(&mut tx, &start_at("in-start", 1_700_000_005_000));
2576
2577 let replay = tx.prepare(&start_at("in-start", 1_700_000_005_000), plan);
2579 assert_eq!(replay.step_seq(), Some(started.step_seq));
2580
2581 assert_eq!(
2583 fault_of(&tx.prepare(&cancel_at("in-past", 1_700_000_004_000), plan)),
2584 KernelFaultCode::ClockRegression
2585 );
2586 }
2587
2588 #[test]
2589 fn a_terminal_closes_the_operation_to_every_later_input() {
2590 let (mut tx, _, effect_id) = started();
2591 assert_eq!(tx.pending_effects().count(), 1);
2592
2593 let terminal = run(&mut tx, &cancel_at("in-cancel", 1_700_000_003_000));
2594 assert!(matches!(
2595 terminal.terminal(),
2596 Some(KernelTerminal::Cancelled(_))
2597 ));
2598 assert_eq!(tx.lifecycle(), OperationLifecycle::Cancelled);
2599 assert!(tx.lifecycle().is_terminal());
2600 assert_eq!(
2601 tx.pending_effects().count(),
2602 0,
2603 "a terminal leaves nothing waiting on the host"
2604 );
2605
2606 let before = tx.clone();
2609 for envelope in [
2610 resolve_at("in-late", 1_700_000_004_000, &effect_id, provider_outcome()),
2611 start_at("in-restart", 1_700_000_004_000),
2612 signal_at("in-late-signal", 1_700_000_004_000),
2613 ] {
2614 assert_eq!(
2615 fault_of(&tx.prepare(&envelope, plan)),
2616 KernelFaultCode::InvalidLifecycle,
2617 "{} must be refused after a terminal",
2618 envelope.input_id
2619 );
2620 }
2621 assert_eq!(tx, before, "a refused input leaves the transaction alone");
2622 }
2623
2624 #[test]
2626 fn a_re_issued_cancellation_replays_the_terminal_it_already_committed() {
2627 let (mut tx, _, _) = started();
2628 let cancelled = run(&mut tx, &cancel_at("in-cancel", 1_700_000_003_000));
2629 let before = tx.clone();
2630
2631 let replay = tx.prepare(&cancel_at("in-cancel-again", 1_700_000_004_000), plan);
2634 assert!(
2635 matches!(replay, KernelPreparation::Replayed(_)),
2636 "a re-issued cancellation must replay, not be refused by the latch it created"
2637 );
2638 assert_eq!(replay.step_seq(), Some(cancelled.step_seq));
2639 assert_eq!(
2640 replay.record().unwrap().record_digest(),
2641 cancelled.record.record_digest(),
2642 "the replay points at the existing record, so no second record exists"
2643 );
2644 assert_eq!(tx, before, "a replay moves nothing");
2645
2646 let exact = tx.prepare(&cancel_at("in-cancel", 1_700_000_003_000), plan);
2648 assert_eq!(exact.step_seq(), Some(cancelled.step_seq));
2649 assert_eq!(tx, before);
2650 }
2651
2652 #[test]
2653 fn configured_input_byte_limit_rejects_later_inputs_without_mutation() {
2654 let mut tx = transaction();
2655 let mut config = boot_config([EffectKindTag::CallProvider]);
2656 config.kernel_limits = Some(KernelLimits {
2657 max_input_bytes: Some(1_024),
2658 ..KernelLimits::default()
2659 });
2660 let configure = envelope(
2661 "in-configure-limited",
2662 1_700_000_000_000,
2663 KernelInput::ConfigureOperation(ConfigureOperation { config }),
2664 );
2665 run(&mut tx, &configure);
2666
2667 let oversized = envelope(
2668 "in-oversized",
2669 1_700_000_001_000,
2670 KernelInput::StartOperation(StartOperation {
2671 entry: RootEntry::Agent(RootAgentEntry {
2672 task: LogicalTask::new("x".repeat(4_096)),
2673 run_spec: None,
2674 }),
2675 initial_context: InitialContext::default(),
2676 }),
2677 );
2678 let before = tx.clone();
2679 let rejected = tx.prepare(&oversized, plan);
2680
2681 assert_eq!(fault_of(&rejected), KernelFaultCode::ResourceLimitExceeded);
2682 assert!(rejected.is_zero_mutation());
2683 assert_eq!(tx, before);
2684 }
2685
2686 #[test]
2687 fn a_differing_cancellation_after_the_terminal_is_a_conflict_not_an_overwrite() {
2688 let (mut tx, _, _) = started();
2689 run(&mut tx, &cancel_at("in-cancel", 1_700_000_003_000));
2690 let before = tx.clone();
2691
2692 let divergent = envelope(
2693 "in-cancel-other",
2694 1_700_000_004_000,
2695 KernelInput::HostControl(HostControl {
2696 command: HostCommand::Cancel(CancelCommand {
2697 reason: CancellationReason::HostShutdown,
2698 pending_call_ids: vec![],
2699 }),
2700 }),
2701 );
2702 assert_eq!(
2703 fault_of(&tx.prepare(&divergent, plan)),
2704 KernelFaultCode::DuplicateInputConflict,
2705 "the operation already ended for the first reason; the second cannot re-decide it"
2706 );
2707 assert_eq!(tx, before);
2708 }
2709
2710 #[test]
2715 fn a_conflicting_resolution_of_a_settled_effect_fails_closed() {
2716 let (mut tx, _, effect_id) = started();
2717 run(
2718 &mut tx,
2719 &resolve_at(
2720 "in-resolve",
2721 1_700_000_002_000,
2722 &effect_id,
2723 provider_outcome(),
2724 ),
2725 );
2726 let before = tx.clone();
2727
2728 let conflicting = tx.prepare(
2729 &resolve_at(
2730 "in-resolve-conflict",
2731 1_700_000_003_000,
2732 &effect_id,
2733 failure_outcome(),
2734 ),
2735 plan,
2736 );
2737 assert_eq!(
2738 fault_of(&conflicting),
2739 KernelFaultCode::UnexpectedEffectOutcome
2740 );
2741 assert_eq!(tx, before);
2742 }
2743
2744 #[test]
2745 fn an_unknown_effect_id_fails_closed() {
2746 let (mut tx, _, _) = started();
2747 let unknown = EffectId::new("op-tx-1:step:99:effect:0").unwrap();
2748 assert_eq!(
2749 fault_of(&tx.prepare(
2750 &resolve_at(
2751 "in-unknown",
2752 1_700_000_002_000,
2753 &unknown,
2754 provider_outcome()
2755 ),
2756 plan
2757 )),
2758 KernelFaultCode::UnexpectedEffectOutcome
2759 );
2760 }
2761
2762 #[test]
2763 fn a_resolution_carrying_another_effect_kinds_payload_fails_closed() {
2764 let (mut tx, _, effect_id) = started();
2765 let wrong_shape = EffectOutcome::Succeeded(EffectSucceeded {
2766 result: EffectSuccess::Tools(ToolsSuccess::default()),
2767 });
2768 assert_eq!(
2769 fault_of(&tx.prepare(
2770 &resolve_at("in-wrong", 1_700_000_002_000, &effect_id, wrong_shape),
2771 plan
2772 )),
2773 KernelFaultCode::UnexpectedEffectOutcome
2774 );
2775 }
2776
2777 #[test]
2778 fn at_most_one_pending_effect_per_kind() {
2779 let (mut tx, _, _) = started();
2780 let before = tx.clone();
2781
2782 let greedy = tx.prepare(&cancel_at("in-second", 1_700_000_002_000), |context| {
2784 Ok(publishing(
2785 "greedy",
2786 vec![effect(
2787 "op-tx-1:step:2:effect:0",
2788 &context.input.input_id,
2789 EffectKind::CallProvider(CallProviderEffect::default()),
2790 )],
2791 ))
2792 });
2793 assert_eq!(fault_of(&greedy), KernelFaultCode::ResourceLimitExceeded);
2794 assert_eq!(tx, before, "the refusal is zero mutation");
2795
2796 let doubled = tx.prepare(&cancel_at("in-double", 1_700_000_002_000), |context| {
2798 Ok(publishing(
2799 "doubled",
2800 vec![
2801 effect(
2802 "op-tx-1:step:2:effect:0",
2803 &context.input.input_id,
2804 EffectKind::SpawnTasks(SpawnTasksEffect::default()),
2805 ),
2806 effect(
2807 "op-tx-1:step:2:effect:1",
2808 &context.input.input_id,
2809 EffectKind::SpawnTasks(SpawnTasksEffect::default()),
2810 ),
2811 ],
2812 ))
2813 });
2814 assert_eq!(fault_of(&doubled), KernelFaultCode::ResourceLimitExceeded);
2815 }
2816
2817 #[test]
2818 fn an_effect_kind_the_host_did_not_declare_is_refused_before_emission() {
2819 let mut tx = transaction();
2820 run(
2822 &mut tx,
2823 &configure_at("in-configure", [EffectKindTag::CallProvider]),
2824 );
2825 let before = tx.clone();
2826
2827 let refused = tx.prepare(&start(), |context| {
2828 Ok(publishing(
2829 "tools",
2830 vec![effect(
2831 "op-tx-1:step:1:effect:0",
2832 &context.input.input_id,
2833 EffectKind::ExecuteTools(ExecuteToolsEffect::default()),
2834 )],
2835 ))
2836 });
2837 assert_eq!(fault_of(&refused), KernelFaultCode::UnsupportedEffect);
2838 assert_eq!(tx, before, "no record, no effect, no state change");
2839 }
2840
2841 #[test]
2842 fn a_launch_token_is_never_minted_twice() {
2843 let (mut tx, _, effect_id) = started();
2844 let spawn = |id: &str, token: &str| {
2845 let effect_id = id.to_string();
2846 let launch_token = token.to_string();
2847 move |context: &PlanContext<'_>| {
2848 Ok(publishing(
2849 "spawn",
2850 vec![effect(
2851 &effect_id,
2852 &context.input.input_id,
2853 EffectKind::SpawnTasks(SpawnTasksEffect {
2854 tasks: vec![TaskLaunch {
2855 task_id: TaskId::new("task-1").unwrap(),
2856 attempt_id: AttemptId::new("task-1:attempt:1").unwrap(),
2857 launch_token: LaunchToken::new(launch_token.clone()).unwrap(),
2858 node_id: NodeId::new("node-1").unwrap(),
2859 spec: LogicalAgentSpec::new("do the thing"),
2860 }],
2861 budget: None,
2862 }),
2863 )],
2864 ))
2865 }
2866 };
2867
2868 let spawn_effect_id = EffectId::new("op-tx-1:step:2:effect:0").unwrap();
2869 run_with(
2870 &mut tx,
2871 &resolve_at(
2872 "in-resolve",
2873 1_700_000_002_000,
2874 &effect_id,
2875 provider_outcome(),
2876 ),
2877 spawn(spawn_effect_id.as_str(), "op-tx-1:launch:1"),
2878 );
2879 assert!(tx.knows_launch_token(&LaunchToken::new("op-tx-1:launch:1").unwrap()));
2880
2881 run(
2883 &mut tx,
2884 &resolve_at(
2885 "in-spawned",
2886 1_700_000_003_000,
2887 &spawn_effect_id,
2888 EffectOutcome::Succeeded(EffectSucceeded {
2889 result: EffectSuccess::TasksSpawned(TasksSpawnedSuccess::default()),
2890 }),
2891 ),
2892 );
2893
2894 let reused = tx.prepare(
2897 &cancel_at("in-relaunch", 1_700_000_004_000),
2898 spawn("op-tx-1:step:4:effect:0", "op-tx-1:launch:1"),
2899 );
2900 assert_eq!(fault_of(&reused), KernelFaultCode::TransactionConflict);
2901
2902 let fresh = tx.prepare(
2904 &cancel_at("in-relaunch", 1_700_000_004_000),
2905 spawn("op-tx-1:step:4:effect:0", "op-tx-1:launch:2"),
2906 );
2907 assert!(fresh.token().is_some(), "{:?}", fresh.fault());
2908 }
2909
2910 #[test]
2911 fn an_effect_id_is_never_minted_twice() {
2912 let (mut tx, _, effect_id) = started();
2913 let collision = tx.prepare(
2914 &resolve_at(
2915 "in-resolve",
2916 1_700_000_002_000,
2917 &effect_id,
2918 provider_outcome(),
2919 ),
2920 move |context| {
2921 Ok(publishing(
2922 "collide",
2923 vec![effect(
2924 "op-tx-1:step:1:effect:0",
2926 &context.input.input_id,
2927 EffectKind::SpawnTasks(SpawnTasksEffect::default()),
2928 )],
2929 ))
2930 },
2931 );
2932 assert_eq!(fault_of(&collision), KernelFaultCode::TransactionConflict);
2933 }
2934
2935 #[test]
2940 fn a_full_tail_asks_for_a_checkpoint_and_the_retry_is_a_fresh_prepare() {
2941 let mut tx = bounded(TailBounds::new(1, 2, 1024, 1024 * 1024).unwrap());
2942 run(&mut tx, &configure());
2943 run(&mut tx, &start());
2944 assert_eq!(tx.tail_pressure(), TailPressure::Full);
2945 let before = tx.clone();
2946
2947 let retry_envelope = cancel_at("in-cancel", 1_700_000_003_000);
2948 let refused = tx.prepare(&retry_envelope, plan);
2949 let fault = refused.fault().expect("rejected").clone();
2950 assert_eq!(fault.code, KernelFaultCode::CheckpointRequired);
2951 assert!(fault.is_retryable(), "the one retryable code (GAP-2)");
2952 assert!(refused.is_zero_mutation());
2953 assert_eq!(tx, before, "the input was never accepted");
2954
2955 let boundary = tx.checkpoint_boundary().expect("a head exists");
2957 assert_eq!(boundary.through_step_seq, WireU64::new(1));
2958 let usage = tx.note_checkpoint_acked(&boundary).expect("ack");
2959 assert_eq!(
2960 usage,
2961 TailUsage::default(),
2962 "the covered prefix is reclaimed"
2963 );
2964 assert_eq!(tx.tail_pressure(), TailPressure::Nominal, "not a latch");
2965
2966 let retried = tx.prepare(&retry_envelope, plan);
2968 assert!(retried.token().is_some(), "{:?}", retried.fault());
2969 assert_eq!(retried.record().unwrap().step_seq(), WireU64::new(2));
2970 }
2971
2972 #[test]
2973 fn the_tail_bounds_the_byte_axis_too() {
2974 let mut tx = bounded(TailBounds::new(64, 128, 128, 512).unwrap());
2975 let refused = tx.prepare(&configure(), plan);
2977 assert_eq!(fault_of(&refused), KernelFaultCode::CheckpointRequired);
2978 assert!(refused.fault().unwrap().is_retryable());
2979 }
2980
2981 #[test]
2982 fn tail_bounds_refuse_an_incoherent_watermark() {
2983 assert_eq!(
2984 TailBounds::new(10, 4, 100, 100).unwrap_err().code,
2985 KernelFaultCode::InvalidConfig
2986 );
2987 assert_eq!(
2988 TailBounds::new(1, 4, 0, 0).unwrap_err().code,
2989 KernelFaultCode::InvalidConfig
2990 );
2991 assert_eq!(TailBounds::default(), TailBounds::DEFAULT);
2992 }
2993
2994 #[test]
2995 fn the_tail_reports_its_soft_watermark_before_its_hard_limit() {
2996 let mut tx = bounded(TailBounds::new(2, 8, 1024 * 1024, 4 * 1024 * 1024).unwrap());
2997 assert_eq!(tx.tail_pressure(), TailPressure::Nominal);
2998 run(&mut tx, &configure());
2999 assert_eq!(tx.tail_pressure(), TailPressure::Nominal);
3000 run(&mut tx, &start());
3001 assert_eq!(tx.tail_pressure(), TailPressure::Watermark);
3002 }
3003
3004 #[test]
3007 fn a_checkpoint_boundary_neither_blocks_nor_is_blocked_by_a_candidate() {
3008 let (mut tx, records, _) = started();
3009
3010 let preparation = tx.prepare(&cancel_at("in-cancel", 1_700_000_003_000), plan);
3011 let token = preparation.token().expect("prepared").clone();
3012 let candidate_digest = preparation.record().unwrap().record_digest().clone();
3013
3014 let boundary = tx.checkpoint_boundary().expect("a head exists");
3015 assert_eq!(
3016 boundary.covered_head,
3017 *records[1].record_digest(),
3018 "the boundary follows the durable head, not the outstanding candidate"
3019 );
3020 assert_ne!(boundary.covered_head, candidate_digest);
3021
3022 tx.note_checkpoint_acked(&boundary).expect("ack");
3024 assert_eq!(tx.tail_usage(), TailUsage::default());
3025 assert!(tx.has_candidate(), "the candidate survived the checkpoint");
3026
3027 let committed = tx.commit(&token, &candidate_digest).expect("commit");
3029 assert_eq!(committed.step_seq, WireU64::new(2));
3030 assert_eq!(
3031 tx.tail_usage().records,
3032 1,
3033 "records after the candidate are tail"
3034 );
3035 }
3036
3037 #[test]
3038 fn a_checkpoint_that_covers_no_prefix_of_this_journal_is_refused() {
3039 let (mut tx, _, _) = started();
3040 let bogus = CheckpointBoundary {
3041 through_step_seq: WireU64::new(1),
3042 covered_head: canonical_digest(b"another operation's head"),
3043 };
3044 assert_eq!(
3045 tx.note_checkpoint_acked(&bogus).unwrap_err().code,
3046 KernelFaultCode::CheckpointIncompatible
3047 );
3048 assert_eq!(tx.tail_usage().records, 2, "nothing was reclaimed");
3049 }
3050
3051 #[test]
3056 fn a_rebuild_reproduces_every_step_digest_and_the_next_transition() {
3057 let mut live = transaction();
3058 let mut journal = Vec::new();
3059 journal.push(run(&mut live, &configure()).record);
3060 journal.push(run(&mut live, &start()).record);
3061 let effect_id = provider_effect_id(WireU64::new(1));
3062 journal.push(
3063 run(
3064 &mut live,
3065 &resolve_at(
3066 "in-resolve",
3067 1_700_000_002_000,
3068 &effect_id,
3069 provider_outcome(),
3070 ),
3071 )
3072 .record,
3073 );
3074
3075 let mut rebuilt = Tx::rebuild_from_records(
3076 &journal,
3077 ConfigDefaults::default(),
3078 InMemoryRecordIndex::from_records(&journal),
3079 plan,
3080 )
3081 .expect("the journal rebuilds");
3082
3083 assert_eq!(rebuilt.head(), live.head());
3084 assert_eq!(rebuilt.lifecycle(), live.lifecycle());
3085 assert_eq!(rebuilt.config(), live.config());
3086 assert_eq!(rebuilt.tail_usage(), live.tail_usage());
3087 for record in &journal {
3088 let step = rebuilt
3089 .committed_step(record.input_id())
3090 .expect("every replayed step is recoverable");
3091 record
3092 .verify_step(step)
3093 .expect("the rebuilt step matches the frozen digest");
3094 assert_eq!(Some(step), live.committed_step(record.input_id()));
3095 }
3096
3097 let next = cancel_at("in-cancel", 1_700_000_003_000);
3099 let uninterrupted = live.prepare(&next, plan);
3100 let after_rebuild = rebuilt.prepare(&next, plan);
3101 assert_eq!(
3102 after_rebuild.record().unwrap().step_digest(),
3103 uninterrupted.record().unwrap().step_digest()
3104 );
3105 assert_eq!(
3106 after_rebuild.record().unwrap().record_digest(),
3107 uninterrupted.record().unwrap().record_digest()
3108 );
3109 }
3110
3111 #[test]
3112 fn a_rebuild_refuses_a_broken_chain() {
3113 let mut live = transaction();
3114 let genesis = run(&mut live, &configure()).record;
3115 let started = run(&mut live, &start()).record;
3116 let resolved = run(
3117 &mut live,
3118 &resolve_at(
3119 "in-resolve",
3120 1_700_000_002_000,
3121 &provider_effect_id(WireU64::new(1)),
3122 provider_outcome(),
3123 ),
3124 )
3125 .record;
3126
3127 let gapped = vec![genesis.clone(), resolved.clone()];
3129 let error = Tx::rebuild_from_records(
3130 &gapped,
3131 ConfigDefaults::default(),
3132 InMemoryRecordIndex::from_records(&gapped),
3133 plan,
3134 )
3135 .expect_err("a chain with a hole is not a journal");
3136 assert_eq!(error.code, KernelFaultCode::RecordCorrupted);
3137
3138 let intact = vec![genesis, started, resolved];
3140 let error = Tx::rebuild_from_records(
3141 &intact,
3142 ConfigDefaults::default(),
3143 InMemoryRecordIndex::from_records(&intact),
3144 |context| {
3145 let mut step = plan(context)?;
3146 step.plan.push_str(" (drifted)");
3147 Ok(step)
3148 },
3149 )
3150 .expect_err("a drifted planner must not silently resume");
3151 assert_eq!(error.code, KernelFaultCode::RecordCorrupted);
3152 assert!(
3153 error.message.contains("step"),
3154 "the fault names the digest that disagreed: {}",
3155 error.message
3156 );
3157 }
3158
3159 #[test]
3160 fn a_rebuild_of_an_empty_journal_is_a_fresh_operation() {
3161 let rebuilt = Tx::rebuild_from_records(
3162 &[],
3163 ConfigDefaults::default(),
3164 InMemoryRecordIndex::new(),
3165 plan,
3166 )
3167 .expect("an empty journal is a legal starting point");
3168 assert_eq!(rebuilt.lifecycle(), OperationLifecycle::Created);
3169 assert_eq!(rebuilt.head(), None);
3170 }
3171
3172 #[test]
3175 fn a_replay_of_a_record_this_runtime_never_saw_demands_a_rebuild() {
3176 let mut source = transaction();
3177 let genesis = run(&mut source, &configure()).record;
3178
3179 let mut cold = KernelTransaction::<TestStep, _>::new(
3180 ConfigDefaults::default(),
3181 InMemoryRecordIndex::from_records(&[genesis]),
3182 );
3183 assert_eq!(
3184 fault_of(&cold.prepare(&configure(), plan)),
3185 KernelFaultCode::RecordCorrupted
3186 );
3187 }
3188
3189 #[test]
3190 fn a_poisoned_transaction_refuses_every_call() {
3191 let (mut tx, _, _) = started();
3192 let preparation = tx.prepare(&cancel_at("in-cancel", 1_700_000_003_000), plan);
3193 let token = preparation.token().unwrap().clone();
3194 tx.note_append_conflict(&token, None);
3195
3196 assert!(tx.is_poisoned());
3197 assert_eq!(
3198 tx.poison().map(|fault| fault.code),
3199 Some(KernelFaultCode::TransactionConflict)
3200 );
3201 assert_eq!(
3202 fault_of(&tx.prepare(&start_at("in-any", 1_700_000_009_000), plan)),
3203 KernelFaultCode::TransactionConflict
3204 );
3205 let boundary = CheckpointBoundary {
3206 through_step_seq: WireU64::new(1),
3207 covered_head: canonical_digest(b"whatever"),
3208 };
3209 assert_eq!(
3210 tx.note_checkpoint_acked(&boundary).unwrap_err().code,
3211 KernelFaultCode::TransactionConflict
3212 );
3213 }
3214}