1use std::collections::{BTreeMap, BTreeSet};
44use std::sync::Arc;
45
46use crate::identifiers::LogicalRuntimeId;
47use crate::input_state::{InputLifecycleState, InputStatePersistenceRecord, StoredInputState};
48use crate::meerkat_machine::dsl as mm_dsl;
49use crate::runtime_state::RuntimeState;
50use crate::store::{
51 CommittedRecoveryBoundary, CommittedWholeBlobProvisionalTail, CommittedWholeBlobSnapshot,
52 MachineLifecycleBindingFacts, MachineLifecycleCommit, MachineLifecycleExpectedVersion,
53 MachineLifecycleObservation, MachineLifecycleRunFacts, PreparedDurableTailRecoverySource,
54 PreparedRecoveryEvidence, PreparedRecoveryReceiptDigestEnrichment,
55 PreparedRecoveryReceiptSource, PreparedRuntimeSessionCommit, RecoveryInputSetRevision,
56 RuntimeSessionPersistenceProfile, RuntimeStore, RuntimeStoreError, SupervisorAuthoritySnapshot,
57};
58use meerkat_core::lifecycle::InputId;
59use meerkat_core::lifecycle::core_executor::BoundSessionCommit;
60use meerkat_core::lifecycle::run_primitive::RunApplyBoundary;
61use meerkat_core::lifecycle::run_receipt::RunBoundaryReceipt;
62use meerkat_core::session_document::{
63 DurableHeadRelation, DurableTailRecoveryClass as ClassifiedRecoveryClass,
64 DurableTailStopReason, RunIdCardinality, SessionDocumentEffect, SessionDocumentKey,
65 SessionDocumentMachineAuthority,
66};
67use meerkat_core::session_store::PreparedHeadCanonicalMutation;
68use meerkat_core::types::SessionId;
69use meerkat_core::{Message, RunId, Session, StopReason, SystemNoticeKind, SystemNoticeMessage};
70use sha2::{Digest, Sha256};
71
72pub use mm_dsl::{DurableTailRecoveryClass, DurableTailRecoveryDisposition};
73
74const NO_OBSERVED_RUN_IDENTITY: &str = "current:no-observed-run-identity";
79
80#[derive(Debug)]
84struct DurableTailObservation {
85 tail_run_id: Option<RunId>,
86 run_id_cardinality: RunIdCardinality,
87 terminal_stop_reason: DurableTailStopReason,
88 dangling_tool_use_ids: Vec<String>,
89 orphan_tool_result_count: u64,
90 messages_after_terminal: bool,
91}
92
93#[derive(Debug)]
100enum PreparedRecoveryStoreTransition {
101 WholeBlob {
102 base_store_revision: u64,
103 base_blob_sha256: String,
104 provisional_candidate_blob_sha256: String,
105 provisional_candidate_sequence: u64,
106 recovered_blob_sha256: String,
107 },
108 HeadCanonical {
109 committed_store_revision: u64,
110 committed_head_token: String,
111 physical_store_revision: u64,
112 physical_head_cas_token: String,
113 recovered_head_token: String,
114 },
115}
116
117#[derive(Debug)]
118struct PreparedRecoveryCandidate {
119 session_id: SessionId,
120 candidate_id: String,
121 candidate_run_id: RunId,
122 class: DurableTailRecoveryClass,
123 store_transition: PreparedRecoveryStoreTransition,
124 recovered: Arc<Session>,
125 document: Option<BoundSessionCommit>,
126 conversation_digest: String,
127 message_count: usize,
128}
129
130enum RecoveryCandidatePreparation {
131 Prepared(PreparedRecoveryCandidate),
132 AlreadyAligned(Arc<Session>),
133 IncompleteHeadCanonicalIntent {
134 provisional: crate::store::HeadCanonicalProvisionalTailAuthority,
135 },
136 Held,
137}
138
139fn observe_durable_tail(authority_len: usize, head: &Session) -> DurableTailObservation {
140 let tail = &head.messages()[authority_len.min(head.messages().len())..];
141 let mut run_ids = BTreeSet::<String>::new();
142 let mut assistant_without_run_id = false;
143 let mut tail_run_id = None;
144 let mut open_call_ids = Vec::<String>::new();
148 let mut orphan_tool_result_count = 0_u64;
149 let mut last_assistant_stop: Option<DurableTailStopReason> = None;
150 let mut terminal_seen = false;
151 let mut messages_after_terminal = false;
152 for message in tail {
153 if terminal_seen {
159 messages_after_terminal = true;
160 }
161 match message {
162 Message::BlockAssistant(assistant) => {
163 match assistant.identity.run_id.as_ref() {
164 Some(run_id) => {
165 run_ids.insert(run_id.to_string());
166 tail_run_id = Some(run_id.clone());
167 }
168 None => assistant_without_run_id = true,
169 }
170 open_call_ids.extend(assistant.tool_calls().map(|call| call.id.to_string()));
171 let effective_stop = match assistant.stop_reason {
172 StopReason::EndTurn => DurableTailStopReason::EndTurn,
173 StopReason::ToolUse if assistant.has_tool_calls() => {
174 DurableTailStopReason::ToolUse
175 }
176 StopReason::ToolUse => DurableTailStopReason::EndTurn,
181 _ => DurableTailStopReason::Other,
182 };
183 last_assistant_stop = Some(effective_stop);
184 terminal_seen = effective_stop != DurableTailStopReason::ToolUse;
190 }
191 Message::ToolResults { results, .. } => {
192 for result in results {
193 if let Some(position) = open_call_ids
194 .iter()
195 .position(|call_id| *call_id == result.tool_use_id)
196 {
197 open_call_ids.remove(position);
198 } else {
199 orphan_tool_result_count += 1;
200 }
201 }
202 }
203 _ => {}
204 }
205 }
206 let terminal_stop_reason = last_assistant_stop.unwrap_or(DurableTailStopReason::Absent);
207 let run_id_cardinality = match (run_ids.len(), assistant_without_run_id) {
208 (0, _) => RunIdCardinality::NoRunId,
209 (1, false) => RunIdCardinality::SingleRunId,
210 _ => RunIdCardinality::MultipleRunIds,
211 };
212 DurableTailObservation {
213 tail_run_id,
214 run_id_cardinality,
215 terminal_stop_reason,
216 dangling_tool_use_ids: open_call_ids,
217 orphan_tool_result_count,
218 messages_after_terminal,
219 }
220}
221
222fn hash_part(hasher: &mut Sha256, label: &str, value: &[u8]) {
223 hasher.update((label.len() as u64).to_be_bytes());
224 hasher.update(label.as_bytes());
225 hasher.update((value.len() as u64).to_be_bytes());
226 hasher.update(value);
227}
228
229fn run_id_cardinality_name(cardinality: RunIdCardinality) -> &'static str {
230 match cardinality {
231 RunIdCardinality::NoRunId => "no_run_id",
232 RunIdCardinality::SingleRunId => "single_run_id",
233 RunIdCardinality::MultipleRunIds => "multiple_run_ids",
234 }
235}
236
237fn durable_tail_stop_reason_name(reason: DurableTailStopReason) -> &'static str {
238 match reason {
239 DurableTailStopReason::Absent => "absent",
240 DurableTailStopReason::EndTurn => "end_turn",
241 DurableTailStopReason::ToolUse => "tool_use",
242 DurableTailStopReason::Other => "other",
243 }
244}
245
246fn exact_candidate_id(
247 session_id: &SessionId,
248 committed_store_revision: u64,
249 committed_head_token: &str,
250 physical_store_revision: u64,
251 physical_head_cas_token: &str,
252 provisional_run_id: &RunId,
253 observation: &DurableTailObservation,
254) -> String {
255 let observed_run = observation
256 .tail_run_id
257 .as_ref()
258 .map(ToString::to_string)
259 .unwrap_or_else(|| NO_OBSERVED_RUN_IDENTITY.to_string());
260 let mut hasher = Sha256::new();
261 hash_part(
262 &mut hasher,
263 "domain",
264 b"meerkat:durable-tail-recovery-candidate:v5",
265 );
266 hash_part(&mut hasher, "session", session_id.to_string().as_bytes());
267 hash_part(
268 &mut hasher,
269 "committed_store_revision",
270 &committed_store_revision.to_be_bytes(),
271 );
272 hash_part(
273 &mut hasher,
274 "committed_head_token",
275 committed_head_token.as_bytes(),
276 );
277 hash_part(
278 &mut hasher,
279 "physical_store_revision",
280 &physical_store_revision.to_be_bytes(),
281 );
282 hash_part(
283 &mut hasher,
284 "physical_head_token",
285 physical_head_cas_token.as_bytes(),
286 );
287 hash_part(
288 &mut hasher,
289 "provisional_run",
290 provisional_run_id.to_string().as_bytes(),
291 );
292 hash_part(&mut hasher, "observed_run", observed_run.as_bytes());
293 hash_part(
294 &mut hasher,
295 "run_cardinality",
296 run_id_cardinality_name(observation.run_id_cardinality).as_bytes(),
297 );
298 hash_part(
299 &mut hasher,
300 "stop_reason",
301 durable_tail_stop_reason_name(observation.terminal_stop_reason).as_bytes(),
302 );
303 hash_part(
304 &mut hasher,
305 "dangling_call_count",
306 observation
307 .dangling_tool_use_ids
308 .len()
309 .to_string()
310 .as_bytes(),
311 );
312 for call_id in &observation.dangling_tool_use_ids {
313 hash_part(&mut hasher, "dangling_call", call_id.as_bytes());
314 }
315 hash_part(
316 &mut hasher,
317 "orphan_results",
318 observation.orphan_tool_result_count.to_string().as_bytes(),
319 );
320 hash_part(
321 &mut hasher,
322 "after_terminal",
323 if observation.messages_after_terminal {
324 b"true"
325 } else {
326 b"false"
327 },
328 );
329 format!("sha256:{:x}", hasher.finalize())
330}
331
332fn exact_whole_blob_candidate_id(
333 session_id: &SessionId,
334 base_store_revision: u64,
335 base_blob_sha256: &str,
336 candidate_blob_sha256: &str,
337 candidate_sequence: u64,
338 provisional_run_id: &RunId,
339 observation: &DurableTailObservation,
340) -> String {
341 let mut hasher = Sha256::new();
342 hash_part(
343 &mut hasher,
344 "domain",
345 b"meerkat:whole-blob-durable-tail-recovery-candidate:v2",
346 );
347 hash_part(&mut hasher, "session", session_id.to_string().as_bytes());
348 hash_part(
349 &mut hasher,
350 "base_store_revision",
351 &base_store_revision.to_be_bytes(),
352 );
353 hash_part(&mut hasher, "base_blob_sha256", base_blob_sha256.as_bytes());
354 hash_part(
355 &mut hasher,
356 "candidate_blob_sha256",
357 candidate_blob_sha256.as_bytes(),
358 );
359 hash_part(
360 &mut hasher,
361 "candidate_sequence",
362 &candidate_sequence.to_be_bytes(),
363 );
364 hash_part(
365 &mut hasher,
366 "provisional_run",
367 provisional_run_id.to_string().as_bytes(),
368 );
369 hash_part(
370 &mut hasher,
371 "run_cardinality",
372 run_id_cardinality_name(observation.run_id_cardinality).as_bytes(),
373 );
374 hash_part(
375 &mut hasher,
376 "stop_reason",
377 durable_tail_stop_reason_name(observation.terminal_stop_reason).as_bytes(),
378 );
379 for call_id in &observation.dangling_tool_use_ids {
380 hash_part(&mut hasher, "dangling_call", call_id.as_bytes());
381 }
382 hash_part(
383 &mut hasher,
384 "orphan_results",
385 &observation.orphan_tool_result_count.to_be_bytes(),
386 );
387 hash_part(
388 &mut hasher,
389 "after_terminal",
390 if observation.messages_after_terminal {
391 b"true"
392 } else {
393 b"false"
394 },
395 );
396 format!("sha256:{:x}", hasher.finalize())
397}
398
399fn generated_recovery_classification(
400 session_id: &SessionId,
401 candidate_id: &str,
402 observation: &DurableTailObservation,
403) -> Result<ClassifiedRecoveryClass, DurableTailRecoveryError> {
404 let mut classifier = SessionDocumentMachineAuthority::new();
408 let effects = classifier
409 .classify_durable_tail(
410 SessionDocumentKey::new(session_id.to_string()),
411 candidate_id.to_string(),
412 DurableHeadRelation::VerifiedStrictDescendant,
413 observation.run_id_cardinality,
414 observation.terminal_stop_reason,
415 observation.dangling_tool_use_ids.len() as u64,
416 observation.orphan_tool_result_count,
417 observation.messages_after_terminal,
418 )
419 .map_err(|error| {
420 DurableTailRecoveryError::Authority(format!(
421 "durable-tail classification rejected: {error}"
422 ))
423 })?;
424 let mut matching = effects.iter().filter_map(|effect| match effect {
425 SessionDocumentEffect::DurableTailClassified {
426 candidate_id: emitted_candidate,
427 class,
428 } if emitted_candidate == candidate_id => Some(*class),
429 _ => None,
430 });
431 let Some(class) = matching.next() else {
432 return Err(DurableTailRecoveryError::Authority(
433 "classifier emitted no verdict for the exact evidence-bound candidate".to_string(),
434 ));
435 };
436 if matching.next().is_some() {
437 return Err(DurableTailRecoveryError::Authority(
438 "classifier emitted more than one verdict for the same candidate".to_string(),
439 ));
440 }
441 Ok(class)
442}
443
444fn runtime_recovery_class(
445 class: ClassifiedRecoveryClass,
446) -> Result<DurableTailRecoveryClass, DurableTailRecoveryError> {
447 Ok(match class {
448 ClassifiedRecoveryClass::CompletedCandidate => DurableTailRecoveryClass::CompletedCandidate,
449 ClassifiedRecoveryClass::InterruptedRepairableCandidate => {
450 DurableTailRecoveryClass::InterruptedRepairableCandidate
451 }
452 ClassifiedRecoveryClass::Ambiguous => DurableTailRecoveryClass::Ambiguous,
453 #[allow(unreachable_patterns)]
457 _ => {
458 return Err(DurableTailRecoveryError::Authority(
459 "session-document classifier emitted unsupported recovery vocabulary".to_string(),
460 ));
461 }
462 })
463}
464
465fn message_timestamp(message: &Message) -> meerkat_core::types::MessageTimestamp {
466 match message {
467 Message::System(message) => message.created_at,
468 Message::SystemNotice(message) => message.created_at,
469 Message::User(message) => message.created_at,
470 Message::BlockAssistant(message) => message.created_at,
471 Message::ToolResults { created_at, .. } => *created_at,
472 }
473}
474
475fn repair_interrupted_tail(
476 recovered: &mut Session,
477 dangling_tool_use_ids: &[String],
478 durable_tail_timestamp: meerkat_core::types::MessageTimestamp,
479) -> Result<(), DurableTailRecoveryError> {
480 if !dangling_tool_use_ids.is_empty() {
484 return Err(DurableTailRecoveryError::InvalidEvidence(format!(
485 "interrupted-tail repair was classified despite {} dangling tool call(s): {}",
486 dangling_tool_use_ids.len(),
487 dangling_tool_use_ids.join(", ")
488 )));
489 }
490 recovered.push(Message::SystemNotice(SystemNoticeMessage {
491 kind: SystemNoticeKind::Generic,
492 body: Some(
493 "A previous run was interrupted before its boundary committed. Recovery preserved \
494 every durable message, closed the run as InterruptedByRecovery, and did not requeue \
495 its input. Continue from a new turn."
496 .to_string(),
497 ),
498 blocks: Vec::new(),
499 created_at: durable_tail_timestamp,
503 }));
504 Ok(())
505}
506
507fn prepare_head_canonical_recovery_candidate(
508 source: &PreparedDurableTailRecoverySource,
509) -> Result<RecoveryCandidatePreparation, DurableTailRecoveryError> {
510 let runtime_authority = source.runtime_authority();
511 let committed_authority = source.committed_session().as_ref();
512 let observed_physical_head = source.physical_head();
513 let physical_head = source.physical_session().as_ref();
514 let session_id = committed_authority.id().clone();
515 if runtime_authority.profile() != RuntimeSessionPersistenceProfile::HeadCanonicalV1 {
516 return Err(DurableTailRecoveryError::InvalidEvidence(format!(
517 "runtime persistence profile {} cannot atomically recover an external physical head",
518 runtime_authority.profile()
519 )));
520 }
521 let committed_store_authority = runtime_authority.head_canonical().ok_or_else(|| {
522 DurableTailRecoveryError::InvalidEvidence(
523 "HeadCanonical recovery received a different store authority profile".to_string(),
524 )
525 })?;
526 if runtime_authority.session_id() != &session_id {
527 return Err(DurableTailRecoveryError::InvalidEvidence(format!(
528 "runtime authority belongs to session {}, not committed document {session_id}",
529 runtime_authority.session_id()
530 )));
531 }
532 if observed_physical_head.id != session_id {
533 return Err(DurableTailRecoveryError::InvalidEvidence(format!(
534 "observed physical head belongs to session {}, not committed authority {session_id}",
535 observed_physical_head.id
536 )));
537 }
538 if physical_head.id() != &session_id {
539 return Err(DurableTailRecoveryError::InvalidEvidence(format!(
540 "physical head belongs to session {}, not committed authority {session_id}",
541 physical_head.id()
542 )));
543 }
544 if committed_authority.version() != physical_head.version()
545 || committed_authority.created_at() != physical_head.created_at()
546 {
547 return Err(DurableTailRecoveryError::InvalidEvidence(
548 "physical head changes immutable session envelope identity".to_string(),
549 ));
550 }
551 let boundary_head = committed_store_authority.boundary_head();
552 let authority_head_cas_token = committed_store_authority.committed_head_token();
553
554 if let Some(provisional) = source.provisional_authority()
555 && !source.provisional_target_applied()
556 {
557 return Ok(
558 RecoveryCandidatePreparation::IncompleteHeadCanonicalIntent {
559 provisional: provisional.clone(),
560 },
561 );
562 }
563
564 if observed_physical_head == boundary_head {
569 if source.physical_head_cas_token() != authority_head_cas_token {
570 return Err(DurableTailRecoveryError::InvalidEvidence(
571 "equal runtime and physical heads carry contradictory store authority".to_string(),
572 ));
573 }
574 return Ok(RecoveryCandidatePreparation::AlreadyAligned(Arc::clone(
575 source.physical_session(),
576 )));
577 }
578 let provisional = source.provisional_authority().ok_or_else(|| {
579 DurableTailRecoveryError::InvalidEvidence(
580 "newer physical head has no store-issued provisional authority".to_string(),
581 )
582 })?;
583 if provisional.session_id() != &session_id
584 || provisional.base_store_revision() != committed_store_authority.store_revision()
585 || provisional.base_committed_head_token() != authority_head_cas_token
586 || provisional.physical_head_token() != source.physical_head_cas_token()
587 {
588 return Err(DurableTailRecoveryError::InvalidEvidence(
589 "provisional tail does not name the exact committed parent and physical head"
590 .to_string(),
591 ));
592 }
593 if physical_head.messages().len() <= committed_authority.messages().len()
594 || !physical_head
595 .messages()
596 .starts_with(committed_authority.messages())
597 {
598 return Err(DurableTailRecoveryError::InvalidEvidence(
599 "physical recovery head is not an exact strict transcript continuation".to_string(),
600 ));
601 }
602
603 let derived_physical_head_token = source.physical_head_cas_token().to_string();
604
605 let observation = observe_durable_tail(committed_authority.messages().len(), physical_head);
606 if observation.tail_run_id.as_ref() != Some(provisional.run_id())
607 || observation.run_id_cardinality != RunIdCardinality::SingleRunId
608 {
609 return Err(DurableTailRecoveryError::InvalidEvidence(
610 "physical tail transcript contradicts its store-issued provisional run identity"
611 .to_string(),
612 ));
613 }
614 let candidate_id = exact_candidate_id(
615 &session_id,
616 committed_store_authority.store_revision(),
617 authority_head_cas_token,
618 provisional.physical_store_revision(),
619 source.physical_head_cas_token(),
620 provisional.run_id(),
621 &observation,
622 );
623 let classified = generated_recovery_classification(&session_id, &candidate_id, &observation)?;
624 let class = runtime_recovery_class(classified)?;
625 if class == DurableTailRecoveryClass::Ambiguous {
626 return Ok(RecoveryCandidatePreparation::Held);
627 }
628 let candidate_run_id = provisional.run_id().clone();
629
630 if physical_head
640 .validated_transcript_history_state()
641 .map_err(|error| {
642 DurableTailRecoveryError::InvalidEvidence(format!(
643 "physical-head transcript history is malformed: {error}"
644 ))
645 })?
646 .is_some()
647 {
648 return Err(DurableTailRecoveryError::InvalidEvidence(
649 "head-canonical recovery source unexpectedly contains inline transcript history"
650 .to_string(),
651 ));
652 }
653 let mut recovered = physical_head.clone();
654 match class {
655 DurableTailRecoveryClass::CompletedCandidate => {
656 }
658 DurableTailRecoveryClass::InterruptedRepairableCandidate => {
659 let durable_tail_timestamp = physical_head
660 .messages()
661 .last()
662 .map(message_timestamp)
663 .ok_or_else(|| {
664 DurableTailRecoveryError::InvalidEvidence(
665 "interrupted recovery has no durable tail timestamp".to_string(),
666 )
667 })?;
668 repair_interrupted_tail(
669 &mut recovered,
670 &observation.dangling_tool_use_ids,
671 durable_tail_timestamp,
672 )?;
673 }
674 DurableTailRecoveryClass::Ambiguous => {
675 return Err(DurableTailRecoveryError::Authority(
676 "ambiguous recovery candidate reached document preparation after hold".to_string(),
677 ));
678 }
679 }
680 recovered
686 .adopt_recovered_head_state(physical_head)
687 .map_err(DurableTailRecoveryError::InvalidEvidence)?;
688 if recovered.messages().len() < physical_head.messages().len() {
689 return Err(DurableTailRecoveryError::InvalidEvidence(format!(
690 "internally recovered document lost durable content: {} < {} messages",
691 recovered.messages().len(),
692 physical_head.messages().len()
693 )));
694 }
695 let recovered = Arc::new(recovered);
696 let recovered_mutation = PreparedHeadCanonicalMutation::prepare(
697 recovered.as_ref(),
698 Some(observed_physical_head.clone()),
699 )
700 .map_err(|error| {
701 DurableTailRecoveryError::InvalidEvidence(format!(
702 "recovered HeadCanonical mutation preparation failed: {error}"
703 ))
704 })?;
705 if recovered_mutation.predecessor_head_token() != Some(derived_physical_head_token.as_str()) {
706 return Err(DurableTailRecoveryError::InvalidEvidence(
707 "recovered mutation changed the provisional physical-head token".to_string(),
708 ));
709 }
710 let recovered_head_token = meerkat_core::session_head_cas_token(
711 recovered_mutation.successor_head(),
712 )
713 .map_err(|error| {
714 DurableTailRecoveryError::InvalidEvidence(format!(
715 "recovered successor head token is invalid: {error}"
716 ))
717 })?;
718 let document = BoundSessionCommit::sealed(Arc::clone(&recovered))
719 .map_err(|error| {
720 DurableTailRecoveryError::InvalidEvidence(format!(
721 "failed to seal recovered HeadCanonical document: {error}"
722 ))
723 })?
724 .with_head_canonical_mutation(recovered_mutation)
725 .map_err(|error| {
726 DurableTailRecoveryError::InvalidEvidence(format!(
727 "failed to bind recovered HeadCanonical mutation: {error}"
728 ))
729 })?;
730 let conversation_digest = recovered.transcript_content_digest().map_err(|error| {
731 DurableTailRecoveryError::InvalidEvidence(format!(
732 "recovered transcript digest failed: {error}"
733 ))
734 })?;
735 let message_count = recovered.messages().len();
736
737 Ok(RecoveryCandidatePreparation::Prepared(
738 PreparedRecoveryCandidate {
739 session_id,
740 candidate_id,
741 candidate_run_id,
742 class,
743 store_transition: PreparedRecoveryStoreTransition::HeadCanonical {
744 committed_store_revision: committed_store_authority.store_revision(),
745 committed_head_token: authority_head_cas_token.to_string(),
746 physical_store_revision: provisional.physical_store_revision(),
747 physical_head_cas_token: derived_physical_head_token,
748 recovered_head_token,
749 },
750 recovered,
751 document: Some(document),
752 conversation_digest,
753 message_count,
754 },
755 ))
756}
757
758fn prepare_whole_blob_recovery_candidate(
759 committed: CommittedWholeBlobSnapshot,
760 provisional: Option<CommittedWholeBlobProvisionalTail>,
761) -> Result<RecoveryCandidatePreparation, DurableTailRecoveryError> {
762 let (committed_session, _committed_bytes, committed_authority) = committed.into_parts();
763 if committed_session.id() != committed_authority.session_id() {
764 return Err(DurableTailRecoveryError::InvalidEvidence(
765 "committed WholeBlob payload identity differs from store authority".to_string(),
766 ));
767 }
768 let Some(provisional) = provisional else {
769 return Ok(RecoveryCandidatePreparation::AlreadyAligned(
770 committed_session,
771 ));
772 };
773 let provisional_authority = provisional.authority();
774 if provisional_authority.session_id() != committed_authority.session_id()
775 || provisional_authority.base_store_revision() != committed_authority.store_revision()
776 || provisional_authority.base_blob_sha256() != committed_authority.blob_sha256()
777 {
778 return Err(DurableTailRecoveryError::InvalidEvidence(
779 "WholeBlob provisional tail does not name the exact committed base".to_string(),
780 ));
781 }
782 let candidate_session =
783 Session::from_persisted_bytes(provisional.candidate_bytes()).map_err(|error| {
784 DurableTailRecoveryError::InvalidEvidence(format!(
785 "provisional WholeBlob payload is invalid: {error}"
786 ))
787 })?;
788 if candidate_session.id() != committed_session.id()
789 || candidate_session.version() != committed_session.version()
790 || candidate_session.created_at() != committed_session.created_at()
791 {
792 return Err(DurableTailRecoveryError::InvalidEvidence(
793 "WholeBlob provisional tail changes immutable session identity".to_string(),
794 ));
795 }
796 if candidate_session.messages().len() <= committed_session.messages().len()
797 || !candidate_session
798 .messages()
799 .starts_with(committed_session.messages())
800 {
801 return Err(DurableTailRecoveryError::InvalidEvidence(
802 "WholeBlob provisional tail is not an exact strict transcript continuation".to_string(),
803 ));
804 }
805 let observation = observe_durable_tail(committed_session.messages().len(), &candidate_session);
806 if observation.tail_run_id.as_ref() != Some(provisional_authority.run_id())
807 || observation.run_id_cardinality != RunIdCardinality::SingleRunId
808 {
809 return Err(DurableTailRecoveryError::InvalidEvidence(
810 "WholeBlob provisional transcript contradicts its store-issued run identity"
811 .to_string(),
812 ));
813 }
814 let candidate_id = exact_whole_blob_candidate_id(
815 committed_authority.session_id(),
816 committed_authority.store_revision(),
817 committed_authority.blob_sha256(),
818 provisional_authority.candidate_blob_sha256(),
819 provisional_authority.candidate_sequence(),
820 provisional_authority.run_id(),
821 &observation,
822 );
823 let class = runtime_recovery_class(generated_recovery_classification(
824 committed_authority.session_id(),
825 &candidate_id,
826 &observation,
827 )?)?;
828 if class == DurableTailRecoveryClass::Ambiguous {
829 return Ok(RecoveryCandidatePreparation::Held);
830 }
831 let physical_envelope = candidate_session.clone();
832 let mut recovered = candidate_session;
833 match class {
834 DurableTailRecoveryClass::CompletedCandidate => {}
835 DurableTailRecoveryClass::InterruptedRepairableCandidate => {
836 let durable_tail_timestamp = recovered
837 .messages()
838 .last()
839 .map(message_timestamp)
840 .ok_or_else(|| {
841 DurableTailRecoveryError::InvalidEvidence(
842 "interrupted WholeBlob recovery has no durable tail timestamp".to_string(),
843 )
844 })?;
845 repair_interrupted_tail(
846 &mut recovered,
847 &observation.dangling_tool_use_ids,
848 durable_tail_timestamp,
849 )?;
850 }
851 DurableTailRecoveryClass::Ambiguous => {
852 return Err(DurableTailRecoveryError::Authority(
853 "ambiguous WholeBlob recovery reached document preparation".to_string(),
854 ));
855 }
856 }
857 recovered
858 .adopt_recovered_head_state(&physical_envelope)
859 .map_err(DurableTailRecoveryError::InvalidEvidence)?;
860 let recovered = Arc::new(recovered);
861 let (document, recovered_blob_sha256) = match class {
862 DurableTailRecoveryClass::CompletedCandidate => (
863 None,
864 provisional_authority.candidate_blob_sha256().to_string(),
865 ),
866 DurableTailRecoveryClass::InterruptedRepairableCandidate => {
867 let document = BoundSessionCommit::sealed(Arc::clone(&recovered)).map_err(|error| {
868 DurableTailRecoveryError::InvalidEvidence(format!(
869 "failed to seal repaired WholeBlob document: {error}"
870 ))
871 })?;
872 let recovered_blob_sha256 = document
873 .whole_blob_artifact()
874 .map_err(|error| {
875 DurableTailRecoveryError::InvalidEvidence(format!(
876 "failed to serialize repaired WholeBlob document: {error}"
877 ))
878 })?
879 .row_sha256_token()
880 .to_string();
881 (Some(document), recovered_blob_sha256)
882 }
883 DurableTailRecoveryClass::Ambiguous => {
884 return Err(DurableTailRecoveryError::Authority(
885 "ambiguous WholeBlob recovery reached artifact preparation".to_string(),
886 ));
887 }
888 };
889 let conversation_digest = recovered.transcript_content_digest().map_err(|error| {
890 DurableTailRecoveryError::InvalidEvidence(format!(
891 "recovered WholeBlob transcript digest failed: {error}"
892 ))
893 })?;
894 let message_count = recovered.messages().len();
895 Ok(RecoveryCandidatePreparation::Prepared(
896 PreparedRecoveryCandidate {
897 session_id: committed_authority.session_id().clone(),
898 candidate_id,
899 candidate_run_id: provisional_authority.run_id().clone(),
900 class,
901 store_transition: PreparedRecoveryStoreTransition::WholeBlob {
902 base_store_revision: committed_authority.store_revision(),
903 base_blob_sha256: committed_authority.blob_sha256().to_string(),
904 provisional_candidate_blob_sha256: provisional_authority
905 .candidate_blob_sha256()
906 .to_string(),
907 provisional_candidate_sequence: provisional_authority.candidate_sequence(),
908 recovered_blob_sha256,
909 },
910 recovered,
911 document,
912 conversation_digest,
913 message_count,
914 },
915 ))
916}
917
918fn bind_recovered_store_document(
919 candidate: &PreparedRecoveryCandidate,
920) -> Result<Option<BoundSessionCommit>, DurableTailRecoveryError> {
921 let document = candidate.document.clone();
922 match &candidate.store_transition {
923 PreparedRecoveryStoreTransition::WholeBlob {
924 provisional_candidate_blob_sha256,
925 recovered_blob_sha256,
926 ..
927 } => {
928 if recovered_blob_sha256 == provisional_candidate_blob_sha256 {
929 if candidate.class != DurableTailRecoveryClass::CompletedCandidate
930 || document.is_some()
931 {
932 return Err(DurableTailRecoveryError::InvalidEvidence(
933 "metadata-only WholeBlob promotion carries a repaired document or \
934 non-completed class"
935 .to_string(),
936 ));
937 }
938 return Ok(None);
939 }
940 let document = document.ok_or_else(|| {
941 DurableTailRecoveryError::InvalidEvidence(
942 "repaired WholeBlob recovery lost its sealed successor artifact".to_string(),
943 )
944 })?;
945 let artifact = document.whole_blob_artifact().map_err(|error| {
946 DurableTailRecoveryError::InvalidEvidence(format!(
947 "failed to materialize recovered WholeBlob artifact: {error}"
948 ))
949 })?;
950 if artifact.row_sha256_token() != recovered_blob_sha256 {
951 return Err(DurableTailRecoveryError::InvalidEvidence(
952 "sealed WholeBlob recovery document differs from its successor digest"
953 .to_string(),
954 ));
955 }
956 Ok(Some(document))
957 }
958 PreparedRecoveryStoreTransition::HeadCanonical {
959 physical_head_cas_token,
960 recovered_head_token,
961 ..
962 } => {
963 let document = document.ok_or_else(|| {
964 DurableTailRecoveryError::InvalidEvidence(
965 "HeadCanonical recovery lost its prepared mutation".to_string(),
966 )
967 })?;
968 let mutation = document
969 .head_canonical()
970 .ok_or_else(|| {
971 DurableTailRecoveryError::InvalidEvidence(
972 "sealed HeadCanonical recovery lost its prepared mutation".to_string(),
973 )
974 })?
975 .mutation();
976 if mutation.predecessor_head_token() != Some(physical_head_cas_token.as_str()) {
977 return Err(DurableTailRecoveryError::InvalidEvidence(
978 "prepared recovery mutation changed the observed physical-head CAS token"
979 .to_string(),
980 ));
981 }
982 let successor_head_token = meerkat_core::session_head_cas_token(
983 mutation.successor_head(),
984 )
985 .map_err(|error| {
986 DurableTailRecoveryError::InvalidEvidence(format!(
987 "prepared recovery successor head token is invalid: {error}"
988 ))
989 })?;
990 if successor_head_token != *recovered_head_token {
991 return Err(DurableTailRecoveryError::InvalidEvidence(
992 "prepared recovery mutation changed the recovered store head".to_string(),
993 ));
994 }
995 Ok(Some(document))
996 }
997 }
998}
999
1000#[allow(clippy::too_many_arguments)]
1001fn seal_recovery_evidence(
1002 candidate: &PreparedRecoveryCandidate,
1003 document: Option<&BoundSessionCommit>,
1004 disposition: DurableTailRecoveryDisposition,
1005 receipt_digest_enrichments: Vec<PreparedRecoveryReceiptDigestEnrichment>,
1006 predecessor_nonterminal_input_set_revision: RecoveryInputSetRevision,
1007 predecessor_nonterminal_input_set_token: String,
1008 input_updates: Vec<InputStatePersistenceRecord>,
1009 receipt: &RunBoundaryReceipt,
1010 lifecycle: &MachineLifecycleCommit,
1011) -> Result<PreparedRecoveryEvidence, RuntimeStoreError> {
1012 match &candidate.store_transition {
1013 PreparedRecoveryStoreTransition::WholeBlob {
1014 base_store_revision,
1015 base_blob_sha256,
1016 provisional_candidate_blob_sha256,
1017 provisional_candidate_sequence,
1018 recovered_blob_sha256,
1019 } => PreparedRecoveryEvidence::seal_whole_blob(
1020 candidate.recovered.as_ref(),
1021 document,
1022 candidate.session_id.clone(),
1023 candidate.candidate_id.clone(),
1024 candidate.candidate_run_id.clone(),
1025 candidate.class,
1026 disposition,
1027 *base_store_revision,
1028 base_blob_sha256.clone(),
1029 provisional_candidate_blob_sha256.clone(),
1030 *provisional_candidate_sequence,
1031 recovered_blob_sha256.clone(),
1032 receipt_digest_enrichments,
1033 predecessor_nonterminal_input_set_revision,
1034 predecessor_nonterminal_input_set_token,
1035 input_updates,
1036 receipt,
1037 lifecycle,
1038 ),
1039 PreparedRecoveryStoreTransition::HeadCanonical {
1040 committed_store_revision,
1041 committed_head_token,
1042 physical_store_revision,
1043 physical_head_cas_token,
1044 recovered_head_token,
1045 ..
1046 } => {
1047 let document =
1048 document.ok_or_else(|| RuntimeStoreError::SessionPersistenceAuthorityConflict {
1049 runtime_id: candidate.session_id.to_string(),
1050 detail: "HeadCanonical recovery evidence lost its prepared mutation"
1051 .to_string(),
1052 })?;
1053 PreparedRecoveryEvidence::seal_head_canonical(
1054 candidate.recovered.as_ref(),
1055 document,
1056 candidate.session_id.clone(),
1057 candidate.candidate_id.clone(),
1058 candidate.candidate_run_id.clone(),
1059 candidate.class,
1060 disposition,
1061 *committed_store_revision,
1062 committed_head_token.clone(),
1063 *physical_store_revision,
1064 physical_head_cas_token.clone(),
1065 recovered_head_token.clone(),
1066 receipt_digest_enrichments,
1067 predecessor_nonterminal_input_set_revision,
1068 predecessor_nonterminal_input_set_token,
1069 input_updates,
1070 receipt,
1071 lifecycle,
1072 )
1073 }
1074 }
1075}
1076
1077fn verify_exact_committed_recovery(
1078 candidate: &PreparedRecoveryCandidate,
1079 committed: &CommittedRecoveryBoundary,
1080 lifecycle: &MachineLifecycleCommit,
1081) -> Result<(DurableTailRecoveryDisposition, u64), DurableTailRecoveryError> {
1082 let receipt = committed.receipt();
1083 if receipt.boundary != RunApplyBoundary::Immediate {
1084 return Err(DurableTailRecoveryError::InvalidEvidence(
1085 "committed recovery witness carries a non-immediate boundary".to_string(),
1086 ));
1087 }
1088 let document = bind_recovered_store_document(candidate)?;
1089 let expected_evidence = seal_recovery_evidence(
1090 candidate,
1091 document.as_ref(),
1092 committed.evidence().disposition(),
1093 committed.evidence().receipt_digest_enrichments().to_vec(),
1094 committed
1095 .evidence()
1096 .predecessor_nonterminal_input_set_revision(),
1097 committed
1098 .evidence()
1099 .predecessor_nonterminal_input_set_token()
1100 .to_owned(),
1101 committed.evidence().cloned_input_updates(),
1102 receipt,
1103 lifecycle,
1104 )?;
1105 if &expected_evidence != committed.evidence() {
1106 return Err(DurableTailRecoveryError::InvalidEvidence(
1107 "committed recovery candidate id exists with a different exact source, recovered \
1108 store authority, receipt, lifecycle target, migration witness, or evidence witness"
1109 .to_string(),
1110 ));
1111 }
1112 Ok((committed.evidence().disposition(), receipt.sequence))
1113}
1114
1115async fn load_exact_committed_recovery(
1116 store: &dyn RuntimeStore,
1117 runtime_id: &LogicalRuntimeId,
1118 candidate: &PreparedRecoveryCandidate,
1119 lifecycle: &MachineLifecycleCommit,
1120) -> Result<Option<(DurableTailRecoveryDisposition, u64)>, DurableTailRecoveryError> {
1121 let committed = store
1122 .load_committed_recovery_boundary(runtime_id, &candidate.candidate_id)
1123 .await?;
1124 committed
1125 .as_ref()
1126 .map(|committed| verify_exact_committed_recovery(candidate, committed, lifecycle))
1127 .transpose()
1128}
1129
1130fn committed_recovery_outcome(
1131 candidate: PreparedRecoveryCandidate,
1132 disposition: DurableTailRecoveryDisposition,
1133 boundary_sequence: u64,
1134) -> DurableTailRecoveryOutcome {
1135 let PreparedRecoveryCandidate {
1136 recovered,
1137 document,
1138 ..
1139 } = candidate;
1140 drop(document);
1145 let recovered = Arc::try_unwrap(recovered).unwrap_or_else(|shared| shared.as_ref().clone());
1146 DurableTailRecoveryOutcome::Committed {
1147 disposition,
1148 boundary_sequence,
1149 recovered: Box::new(recovered),
1150 }
1151}
1152
1153#[derive(Debug)]
1156pub enum DurableTailRecoveryOutcome {
1157 Committed {
1162 disposition: DurableTailRecoveryDisposition,
1163 boundary_sequence: u64,
1164 recovered: Box<Session>,
1165 },
1166 AlreadyAligned { recovered: Box<Session> },
1172 Held,
1176 Refused,
1180}
1181
1182#[derive(Debug, thiserror::Error)]
1185pub enum DurableTailRecoveryError {
1186 #[error("recovery authorization could not be driven: {0}")]
1187 Authority(String),
1188 #[error("recovery evidence is invalid: {0}")]
1191 InvalidEvidence(String),
1192 #[error("recovery commit failed: {0}")]
1193 Store(#[from] RuntimeStoreError),
1194}
1195
1196struct ObservedPersistedLifecycle {
1200 lifecycle: mm_dsl::DurableRecoveryObservedLifecycle,
1201 current_run: mm_dsl::DurableRecoveryObservedRun,
1202 expected_version: MachineLifecycleExpectedVersion,
1203 reassert_state: RuntimeState,
1208 binding: MachineLifecycleBindingFacts,
1209 supervisor_authority: SupervisorAuthoritySnapshot,
1210 unregister_progress: Option<crate::store::MachineUnregisterProgressSnapshot>,
1211}
1212
1213fn observe_persisted_lifecycle(
1214 observation: MachineLifecycleObservation,
1215 candidate_run_id: &meerkat_core::RunId,
1216) -> ObservedPersistedLifecycle {
1217 match observation {
1218 MachineLifecycleObservation::Missing => ObservedPersistedLifecycle {
1219 lifecycle: mm_dsl::DurableRecoveryObservedLifecycle::MissingRow,
1220 current_run: mm_dsl::DurableRecoveryObservedRun::NoRun,
1221 expected_version: MachineLifecycleExpectedVersion::Missing,
1222 reassert_state: RuntimeState::Idle,
1223 binding: MachineLifecycleBindingFacts::default(),
1224 supervisor_authority: SupervisorAuthoritySnapshot::UnboundNoReceipt,
1225 unregister_progress: None,
1226 },
1227 MachineLifecycleObservation::Decoded { record, version } => {
1228 let (lifecycle, reassert_state) = match record.runtime_state() {
1229 Some(RuntimeState::Idle) => (
1230 mm_dsl::DurableRecoveryObservedLifecycle::Idle,
1231 RuntimeState::Idle,
1232 ),
1233 Some(RuntimeState::Retired) => (
1234 mm_dsl::DurableRecoveryObservedLifecycle::Retired,
1235 RuntimeState::Retired,
1236 ),
1237 Some(_) => (
1238 mm_dsl::DurableRecoveryObservedLifecycle::NonQuiescent,
1239 RuntimeState::Idle,
1240 ),
1241 None => (
1244 mm_dsl::DurableRecoveryObservedLifecycle::Undecodable,
1245 RuntimeState::Idle,
1246 ),
1247 };
1248 let current_run = match record.run().current_run_id() {
1249 None => mm_dsl::DurableRecoveryObservedRun::NoRun,
1250 Some(run_id) if run_id == candidate_run_id => {
1251 mm_dsl::DurableRecoveryObservedRun::CandidateRun
1252 }
1253 Some(_) => mm_dsl::DurableRecoveryObservedRun::OtherRun,
1254 };
1255 ObservedPersistedLifecycle {
1256 lifecycle,
1257 current_run,
1258 expected_version: MachineLifecycleExpectedVersion::Version(version),
1259 reassert_state,
1260 binding: record.binding().clone(),
1261 supervisor_authority: record.supervisor_authority().clone(),
1262 unregister_progress: record.unregister_progress().cloned(),
1263 }
1264 }
1265 MachineLifecycleObservation::Unsupported { version, .. }
1266 | MachineLifecycleObservation::Malformed { version, .. } => ObservedPersistedLifecycle {
1267 lifecycle: mm_dsl::DurableRecoveryObservedLifecycle::Undecodable,
1268 current_run: mm_dsl::DurableRecoveryObservedRun::OtherRun,
1269 expected_version: MachineLifecycleExpectedVersion::Version(version),
1270 reassert_state: RuntimeState::Idle,
1271 binding: MachineLifecycleBindingFacts::default(),
1272 supervisor_authority: SupervisorAuthoritySnapshot::UnboundNoReceipt,
1273 unregister_progress: None,
1274 },
1275 }
1276}
1277
1278struct PriorCommitObservation {
1290 last_committed_sequence: u64,
1291 classification: mm_dsl::DurableRecoveryPriorCommit,
1292 receipt_bound_inputs: BTreeSet<String>,
1293}
1294
1295fn materialize_verified_receipts(
1301 sources: &[PreparedRecoveryReceiptSource],
1302 enrichments: &[PreparedRecoveryReceiptDigestEnrichment],
1303) -> Result<Vec<RunBoundaryReceipt>, DurableTailRecoveryError> {
1304 let mut by_sequence = BTreeMap::new();
1305 for enrichment in enrichments {
1306 if by_sequence
1307 .insert(enrichment.original_receipt().sequence, enrichment)
1308 .is_some()
1309 {
1310 return Err(DurableTailRecoveryError::InvalidEvidence(
1311 "store-owned receipt migration contains duplicate sequences".to_string(),
1312 ));
1313 }
1314 }
1315
1316 let mut materialized = Vec::with_capacity(sources.len());
1317 let mut consumed_enrichments = 0_usize;
1318 for source in sources {
1319 let receipt = source.receipt();
1320 if receipt.conversation_digest.is_some() {
1321 if by_sequence.contains_key(&receipt.sequence) {
1322 return Err(DurableTailRecoveryError::InvalidEvidence(format!(
1323 "receipt sequence {} is already digest-bound but also carries a migration",
1324 receipt.sequence
1325 )));
1326 }
1327 materialized.push(receipt.clone());
1328 continue;
1329 }
1330 let enrichment = by_sequence.get(&receipt.sequence).ok_or_else(|| {
1331 DurableTailRecoveryError::InvalidEvidence(format!(
1332 "digestless receipt sequence {} has no sealed store migration",
1333 receipt.sequence
1334 ))
1335 })?;
1336 if enrichment.original_receipt() != receipt
1337 || enrichment.original_exact_row_token() != source.exact_row_token()
1338 {
1339 return Err(DurableTailRecoveryError::InvalidEvidence(format!(
1340 "receipt migration sequence {} does not bind the exact original row",
1341 receipt.sequence
1342 )));
1343 }
1344 materialized.push(enrichment.enriched_receipt());
1345 consumed_enrichments += 1;
1346 }
1347 if consumed_enrichments != enrichments.len() {
1348 return Err(DurableTailRecoveryError::InvalidEvidence(
1349 "store-owned receipt migration contains a row absent from the exact receipt read"
1350 .to_string(),
1351 ));
1352 }
1353 Ok(materialized)
1354}
1355
1356fn prepare_receipt_digest_enrichments(
1357 candidate: &PreparedRecoveryCandidate,
1358 receipts: &[PreparedRecoveryReceiptSource],
1359) -> Result<Vec<PreparedRecoveryReceiptDigestEnrichment>, DurableTailRecoveryError> {
1360 let mut expected_sequence = 1_u64;
1361 let mut previous_message_count = 0_usize;
1362 let mut enrichments = Vec::new();
1363 for source in receipts {
1364 let receipt = source.receipt();
1365 if receipt.run_id != candidate.candidate_run_id {
1366 return Err(DurableTailRecoveryError::InvalidEvidence(
1367 "recovery receipt source contains a different run identity".to_string(),
1368 ));
1369 }
1370 if receipt.sequence != expected_sequence {
1371 return Err(DurableTailRecoveryError::InvalidEvidence(format!(
1372 "recovery receipt sequence {} is not the required dense sequence {expected_sequence}",
1373 receipt.sequence
1374 )));
1375 }
1376 if receipt.message_count < previous_message_count
1377 || receipt.message_count > candidate.recovered.messages().len()
1378 {
1379 return Err(DurableTailRecoveryError::InvalidEvidence(
1380 "recovery receipt message counts are not monotonic within the store-bound transcript"
1381 .to_string(),
1382 ));
1383 }
1384 let derived_digest = candidate
1385 .recovered
1386 .transcript_prefix_digest(receipt.message_count)
1387 .map_err(|error| {
1388 DurableTailRecoveryError::InvalidEvidence(format!(
1389 "failed to derive recovery receipt transcript prefix: {error}"
1390 ))
1391 })?;
1392 match receipt.conversation_digest.as_deref() {
1393 Some(existing) if existing != derived_digest => {
1394 return Err(DurableTailRecoveryError::InvalidEvidence(format!(
1395 "receipt sequence {} digest differs from the store-bound transcript prefix",
1396 receipt.sequence
1397 )));
1398 }
1399 Some(_) => {}
1400 None => enrichments.push(
1401 PreparedRecoveryReceiptDigestEnrichment::new(source, derived_digest)
1402 .map_err(DurableTailRecoveryError::Store)?,
1403 ),
1404 }
1405 expected_sequence = expected_sequence.checked_add(1).ok_or_else(|| {
1406 DurableTailRecoveryError::InvalidEvidence(
1407 "recovery receipt sequence overflow".to_string(),
1408 )
1409 })?;
1410 previous_message_count = receipt.message_count;
1411 }
1412 Ok(enrichments)
1413}
1414
1415fn observe_prior_commits(
1421 committed: &[RunBoundaryReceipt],
1422 candidate: &Session,
1423 candidate_run_id: &RunId,
1424) -> PriorCommitObservation {
1425 let last_committed_sequence = committed
1426 .iter()
1427 .map(|receipt| receipt.sequence)
1428 .max()
1429 .unwrap_or(0);
1430 if committed.is_empty() {
1431 return PriorCommitObservation {
1432 last_committed_sequence,
1433 classification: mm_dsl::DurableRecoveryPriorCommit::NoPriorCommit,
1434 receipt_bound_inputs: BTreeSet::new(),
1435 };
1436 }
1437
1438 let mut ordered = committed.iter().collect::<Vec<_>>();
1439 ordered.sort_by_key(|receipt| receipt.sequence);
1440 let mut previous: Option<&RunBoundaryReceipt> = None;
1441 let mut receipt_bound_inputs = BTreeSet::new();
1442 for receipt in &ordered {
1443 let expected_sequence = previous
1444 .and_then(|prior| prior.sequence.checked_add(1))
1445 .unwrap_or(1);
1446 let structurally_monotonic = receipt.sequence == expected_sequence
1447 && previous.is_none_or(|prior| receipt.message_count >= prior.message_count);
1448 let prefix_matches = receipt.run_id == *candidate_run_id
1449 && receipt.message_count <= candidate.messages().len()
1450 && receipt
1451 .conversation_digest
1452 .as_deref()
1453 .is_some_and(|digest| {
1454 candidate
1455 .transcript_prefix_digest(receipt.message_count)
1456 .is_ok_and(|prefix| prefix == digest)
1457 });
1458 if !structurally_monotonic || !prefix_matches {
1459 return PriorCommitObservation {
1460 last_committed_sequence,
1461 classification: mm_dsl::DurableRecoveryPriorCommit::DivergesFromCandidate,
1462 receipt_bound_inputs: BTreeSet::new(),
1466 };
1467 }
1468 receipt_bound_inputs.extend(
1469 receipt
1470 .contributing_input_ids
1471 .iter()
1472 .map(ToString::to_string),
1473 );
1474 previous = Some(receipt);
1475 }
1476
1477 PriorCommitObservation {
1478 last_committed_sequence,
1479 classification: ordered.last().map_or(
1480 mm_dsl::DurableRecoveryPriorCommit::NoPriorCommit,
1481 |highest| {
1482 if highest.message_count == candidate.messages().len() {
1483 mm_dsl::DurableRecoveryPriorCommit::MatchesCandidate
1484 } else {
1485 mm_dsl::DurableRecoveryPriorCommit::PrecedesCandidate
1486 }
1487 },
1488 ),
1489 receipt_bound_inputs,
1490 }
1491}
1492
1493async fn load_recovery_preparation(
1494 store: &dyn RuntimeStore,
1495 runtime_id: &LogicalRuntimeId,
1496 session_id: &SessionId,
1497) -> Result<RecoveryCandidatePreparation, DurableTailRecoveryError> {
1498 match store.session_persistence_profile() {
1499 RuntimeSessionPersistenceProfile::HeadCanonicalV1 => {
1500 let source = match store.load_durable_tail_recovery_source(runtime_id).await {
1501 Ok(Some(source)) => source,
1502 Ok(None) => return Ok(RecoveryCandidatePreparation::Held),
1503 Err(RuntimeStoreError::PreparedRecoveryRequiresAtomicPhysicalHeadCas {
1504 ..
1505 }) => {
1506 return Ok(RecoveryCandidatePreparation::Held);
1507 }
1508 Err(error) => return Err(error.into()),
1509 };
1510 if source.runtime_authority().session_id() != session_id {
1511 return Err(DurableTailRecoveryError::InvalidEvidence(format!(
1512 "store-owned recovery source belongs to session {}, not requested {session_id}",
1513 source.runtime_authority().session_id()
1514 )));
1515 }
1516 prepare_head_canonical_recovery_candidate(&source)
1517 }
1518 RuntimeSessionPersistenceProfile::WholeBlobV1 => {
1519 let committed = store
1520 .load_committed_whole_blob_snapshot(runtime_id)
1521 .await?
1522 .ok_or_else(|| {
1523 DurableTailRecoveryError::InvalidEvidence(
1524 "WholeBlob recovery has no committed base snapshot".to_string(),
1525 )
1526 })?;
1527 if committed.authority().session_id() != session_id {
1528 return Err(DurableTailRecoveryError::InvalidEvidence(format!(
1529 "WholeBlob committed authority belongs to {}, not requested {session_id}",
1530 committed.authority().session_id()
1531 )));
1532 }
1533 let provisional = store.load_whole_blob_provisional_tail(runtime_id).await?;
1534 prepare_whole_blob_recovery_candidate(committed, provisional)
1535 }
1536 }
1537}
1538
1539pub async fn recover_durable_tail(
1555 store: &dyn RuntimeStore,
1556 session_id: &SessionId,
1557) -> Result<DurableTailRecoveryOutcome, DurableTailRecoveryError> {
1558 let runtime_id = LogicalRuntimeId::for_session(session_id);
1559 let mut preparation = load_recovery_preparation(store, &runtime_id, session_id).await?;
1560 let mut rolled_back_incomplete_intent = false;
1561 let candidate = loop {
1562 match preparation {
1563 RecoveryCandidatePreparation::Prepared(candidate) => break candidate,
1564 RecoveryCandidatePreparation::AlreadyAligned(recovered) => {
1565 tracing::info!(
1566 %session_id,
1567 "durable-tail recovery source is already exactly aligned"
1568 );
1569 let recovered =
1570 Arc::try_unwrap(recovered).unwrap_or_else(|shared| shared.as_ref().clone());
1571 return Ok(DurableTailRecoveryOutcome::AlreadyAligned {
1572 recovered: Box::new(recovered),
1573 });
1574 }
1575 RecoveryCandidatePreparation::IncompleteHeadCanonicalIntent { provisional } => {
1576 if rolled_back_incomplete_intent {
1577 return Err(DurableTailRecoveryError::InvalidEvidence(
1578 "HeadCanonical provisional rollback did not expose a stable physical authority"
1579 .to_string(),
1580 ));
1581 }
1582 if !store
1583 .discard_head_canonical_provisional_tail(&runtime_id, &provisional)
1584 .await?
1585 {
1586 tracing::warn!(
1587 %session_id,
1588 "durable-tail recovery held: incomplete HeadCanonical intent changed before exact rollback"
1589 );
1590 return Ok(DurableTailRecoveryOutcome::Held);
1591 }
1592 tracing::info!(
1593 %session_id,
1594 physical_revision = provisional.physical_store_revision(),
1595 "rolled back incomplete HeadCanonical provisional intent"
1596 );
1597 rolled_back_incomplete_intent = true;
1598 preparation = load_recovery_preparation(store, &runtime_id, session_id).await?;
1599 }
1600 RecoveryCandidatePreparation::Held => return Ok(DurableTailRecoveryOutcome::Held),
1601 }
1602 };
1603
1604 let observed = match store.observe_machine_lifecycle(&runtime_id).await {
1605 Ok(observation) => observe_persisted_lifecycle(observation, &candidate.candidate_run_id),
1606 Err(RuntimeStoreError::Unsupported(_)) => ObservedPersistedLifecycle {
1609 lifecycle: mm_dsl::DurableRecoveryObservedLifecycle::Undecodable,
1610 current_run: mm_dsl::DurableRecoveryObservedRun::OtherRun,
1611 expected_version: MachineLifecycleExpectedVersion::Missing,
1612 reassert_state: RuntimeState::Idle,
1613 binding: MachineLifecycleBindingFacts::default(),
1614 supervisor_authority: SupervisorAuthoritySnapshot::UnboundNoReceipt,
1615 unregister_progress: None,
1616 },
1617 Err(error) => return Err(error.into()),
1618 };
1619 let lifecycle = MachineLifecycleCommit::new_with_binding_run_and_unregister_progress(
1624 observed.reassert_state,
1625 observed.binding.clone(),
1626 MachineLifecycleRunFacts::default(),
1627 observed.supervisor_authority.clone(),
1628 observed.unregister_progress.clone(),
1629 )
1630 .with_expected_version(observed.expected_version.clone());
1631
1632 let prior_recovery = match load_exact_committed_recovery(
1633 store,
1634 &runtime_id,
1635 &candidate,
1636 &lifecycle,
1637 )
1638 .await
1639 {
1640 Ok(boundary) => boundary,
1641 Err(DurableTailRecoveryError::Store(
1642 RuntimeStoreError::PreparedRecoveryRequiresAtomicPhysicalHeadCas { profile },
1643 )) => {
1644 tracing::warn!(
1645 session_id = %candidate.session_id,
1646 %profile,
1647 "durable-tail recovery held: store cannot prove an exact atomic recovery boundary"
1648 );
1649 return Ok(DurableTailRecoveryOutcome::Held);
1650 }
1651 Err(error) => return Err(error),
1652 };
1653 if let Some((disposition, boundary_sequence)) = prior_recovery {
1654 tracing::info!(
1655 session_id = %candidate.session_id,
1656 candidate_run_id = %candidate.candidate_run_id,
1657 ?disposition,
1658 boundary_sequence,
1659 "durable-tail recovery converged on an exact committed witness"
1660 );
1661 return Ok(committed_recovery_outcome(
1662 candidate,
1663 disposition,
1664 boundary_sequence,
1665 ));
1666 }
1667
1668 let committed_receipt_sources = match store
1674 .load_durable_tail_recovery_receipts(&runtime_id, &candidate.candidate_run_id)
1675 .await
1676 {
1677 Ok(receipts) => receipts,
1678 Err(RuntimeStoreError::PreparedRecoveryRequiresAtomicPhysicalHeadCas { profile }) => {
1679 tracing::warn!(
1680 session_id = %candidate.session_id,
1681 %profile,
1682 "durable-tail recovery held: store cannot load exact receipt rows"
1683 );
1684 return Ok(DurableTailRecoveryOutcome::Held);
1685 }
1686 Err(error) => return Err(error.into()),
1687 };
1688 let receipt_digest_enrichments =
1689 prepare_receipt_digest_enrichments(&candidate, &committed_receipt_sources)?;
1690 let committed_receipts =
1691 materialize_verified_receipts(&committed_receipt_sources, &receipt_digest_enrichments)?;
1692 let prior = observe_prior_commits(
1693 &committed_receipts,
1694 candidate.recovered.as_ref(),
1695 &candidate.candidate_run_id,
1696 );
1697
1698 let inputs = observe_candidate_run_inputs(
1699 store,
1700 &runtime_id,
1701 &candidate.candidate_run_id,
1702 &prior.receipt_bound_inputs,
1703 )
1704 .await?;
1705
1706 let mut authority =
1707 crate::meerkat_machine::dsl_authority::new_registered_authority(&candidate.session_id)
1708 .map_err(|error| DurableTailRecoveryError::Authority(error.to_string()))?;
1709 let transition = mm_dsl::MeerkatMachineMutator::apply(
1710 &mut authority,
1711 mm_dsl::MeerkatMachineInput::AuthorizeDurableTailRecovery {
1712 session_id: mm_dsl::SessionId::from_domain(&candidate.session_id),
1713 candidate_id: candidate.candidate_id.clone(),
1714 candidate_run_id: mm_dsl::RunId(candidate.candidate_run_id.to_string()),
1715 class: candidate.class,
1716 observed_lifecycle: observed.lifecycle,
1717 observed_current_run: observed.current_run,
1718 last_committed_sequence: prior.last_committed_sequence,
1719 prior_commit: prior.classification,
1720 input_evidence: inputs.evidence,
1721 },
1722 )
1723 .map_err(|error| DurableTailRecoveryError::Authority(error.to_string()))?;
1724
1725 let mut commit_verdict: Option<(DurableTailRecoveryDisposition, u64)> = None;
1726 let mut non_commit_verdict: Option<DurableTailRecoveryDisposition> = None;
1727 for effect in transition.effects() {
1728 match effect {
1729 mm_dsl::MeerkatMachineEffect::DurableTailRecoveryCommitAuthorized {
1730 candidate_id,
1731 disposition,
1732 boundary_sequence,
1733 } if *candidate_id == candidate.candidate_id => {
1734 commit_verdict = Some((*disposition, *boundary_sequence));
1735 }
1736 mm_dsl::MeerkatMachineEffect::DurableTailRecoveryAuthorized {
1737 candidate_id,
1738 disposition,
1739 } if *candidate_id == candidate.candidate_id => {
1740 non_commit_verdict = Some(*disposition);
1741 }
1742 _ => {}
1743 }
1744 }
1745
1746 let (disposition, boundary_sequence) = match (commit_verdict, non_commit_verdict) {
1747 (Some((disposition, sequence)), _) => (disposition, sequence),
1748 (
1749 None,
1750 Some(
1751 non_commit @ (DurableTailRecoveryDisposition::HoldIntact
1752 | DurableTailRecoveryDisposition::RefuseRecovery),
1753 ),
1754 ) => {
1755 if let Some((disposition, boundary_sequence)) =
1761 load_exact_committed_recovery(store, &runtime_id, &candidate, &lifecycle).await?
1762 {
1763 tracing::info!(
1764 session_id = %candidate.session_id,
1765 candidate_run_id = %candidate.candidate_run_id,
1766 ?disposition,
1767 boundary_sequence,
1768 "durable-tail recovery converged after a competing exact commit"
1769 );
1770 return Ok(committed_recovery_outcome(
1771 candidate,
1772 disposition,
1773 boundary_sequence,
1774 ));
1775 }
1776 match non_commit {
1777 DurableTailRecoveryDisposition::HoldIntact => {
1778 tracing::warn!(
1779 session_id = %candidate.session_id,
1780 candidate_run_id = %candidate.candidate_run_id,
1781 class = ?candidate.class,
1782 prior_commit = ?prior.classification,
1783 input_evidence = ?inputs.evidence,
1784 "durable-tail recovery held intact by machine verdict"
1785 );
1786 return Ok(DurableTailRecoveryOutcome::Held);
1787 }
1788 DurableTailRecoveryDisposition::RefuseRecovery => {
1789 return Ok(DurableTailRecoveryOutcome::Refused);
1790 }
1791 other => {
1792 return Err(DurableTailRecoveryError::Authority(format!(
1793 "generated machine emitted unexpected non-commit disposition {other:?}"
1794 )));
1795 }
1796 }
1797 }
1798 (None, Some(other)) => {
1799 return Err(DurableTailRecoveryError::Authority(format!(
1800 "generated machine emitted commit disposition {other:?} without a commit \
1801 authorization effect"
1802 )));
1803 }
1804 (None, None) => {
1805 return Err(DurableTailRecoveryError::Authority(
1806 "generated machine returned no recovery disposition for the exact candidate"
1807 .to_string(),
1808 ));
1809 }
1810 };
1811
1812 let CandidateInputObservation {
1826 evidence: _,
1827 predecessor_nonterminal_input_set_revision,
1828 predecessor_nonterminal_input_set_token,
1829 attributed,
1830 } = inputs;
1831 let predecessor_nonterminal_input_set_revision = predecessor_nonterminal_input_set_revision
1832 .ok_or_else(|| {
1833 DurableTailRecoveryError::Authority(
1834 "machine authorized recovery without an exact input-set revision".to_string(),
1835 )
1836 })?;
1837 let predecessor_nonterminal_input_set_token = predecessor_nonterminal_input_set_token
1838 .ok_or_else(|| {
1839 DurableTailRecoveryError::Authority(
1840 "machine authorized recovery without an exact nonterminal input-set witness"
1841 .to_string(),
1842 )
1843 })?;
1844 let mut input_updates =
1845 terminalize_attributed_inputs(attributed, &candidate.candidate_run_id, boundary_sequence)?;
1846 let contributing_input_ids: Vec<InputId> = input_updates
1847 .iter()
1848 .map(|record| record.as_stored().state.input_id.clone())
1849 .collect();
1850 input_updates.sort_by_key(|record| record.as_stored().state.input_id.to_string());
1856
1857 let receipt = RunBoundaryReceipt {
1858 run_id: candidate.candidate_run_id.clone(),
1859 boundary: RunApplyBoundary::Immediate,
1862 contributing_input_ids,
1863 conversation_digest: Some(candidate.conversation_digest.clone()),
1864 message_count: candidate.message_count,
1865 sequence: boundary_sequence,
1870 };
1871 let document = bind_recovered_store_document(&candidate)?;
1872 let evidence = seal_recovery_evidence(
1873 &candidate,
1874 document.as_ref(),
1875 disposition,
1876 receipt_digest_enrichments,
1877 predecessor_nonterminal_input_set_revision,
1878 predecessor_nonterminal_input_set_token,
1879 input_updates,
1880 &receipt,
1881 &lifecycle,
1882 )?;
1883 let request = match &candidate.store_transition {
1884 PreparedRecoveryStoreTransition::WholeBlob { .. } => {
1885 PreparedRuntimeSessionCommit::machine_terminal_whole_blob_recovery(
1886 document,
1887 evidence,
1888 receipt,
1889 lifecycle,
1890 candidate.session_id.clone(),
1891 )
1892 }
1893 PreparedRecoveryStoreTransition::HeadCanonical { .. } => {
1894 let document = document.ok_or_else(|| {
1895 DurableTailRecoveryError::Authority(
1896 "HeadCanonical recovery lost its prepared mutation before commit".to_string(),
1897 )
1898 })?;
1899 PreparedRuntimeSessionCommit::machine_terminal_recovery(
1900 document,
1901 evidence,
1902 receipt,
1903 lifecycle,
1904 candidate.session_id.clone(),
1905 )
1906 }
1907 }?;
1908 let committed = match store
1909 .commit_prepared_session_boundary(&runtime_id, request)
1910 .await
1911 {
1912 Ok(result) => result,
1913 Err(RuntimeStoreError::PreparedRecoveryRequiresAtomicPhysicalHeadCas { profile }) => {
1914 tracing::warn!(
1915 session_id = %candidate.session_id,
1916 %profile,
1917 "durable-tail recovery held: store lacks atomic physical-head CAS"
1918 );
1919 return Ok(DurableTailRecoveryOutcome::Held);
1920 }
1921 Err(error) => return Err(error.into()),
1922 };
1923 let status = committed.recovery_status().ok_or_else(|| {
1924 DurableTailRecoveryError::Authority(
1925 "prepared recovery commit returned no exact recovery status".to_string(),
1926 )
1927 })?;
1928 let committed_authority = committed.authority().ok_or_else(|| {
1929 DurableTailRecoveryError::Authority(
1930 "prepared recovery commit returned no successor session authority".to_string(),
1931 )
1932 })?;
1933 let successor_matches = match &candidate.store_transition {
1934 PreparedRecoveryStoreTransition::WholeBlob {
1935 base_store_revision,
1936 recovered_blob_sha256,
1937 ..
1938 } => committed_authority.whole_blob().is_some_and(|authority| {
1939 authority.session_id() == &candidate.session_id
1940 && base_store_revision
1941 .checked_add(1)
1942 .is_some_and(|expected| authority.store_revision() == expected)
1943 && authority.blob_sha256() == recovered_blob_sha256
1944 }),
1945 PreparedRecoveryStoreTransition::HeadCanonical {
1946 physical_store_revision,
1947 recovered_head_token,
1948 ..
1949 } => committed_authority
1950 .head_canonical()
1951 .is_some_and(|authority| {
1952 authority.session_id() == &candidate.session_id
1953 && physical_store_revision
1954 .checked_add(1)
1955 .is_some_and(|expected| authority.store_revision() == expected)
1956 && authority.committed_head_token() == recovered_head_token
1957 }),
1958 };
1959 if !successor_matches {
1960 return Err(DurableTailRecoveryError::Authority(
1961 "prepared recovery commit returned a different successor authority".to_string(),
1962 ));
1963 }
1964 if committed.downstream_projection_required() {
1965 return Err(DurableTailRecoveryError::Authority(
1966 "store-owned recovery unexpectedly requested a SessionStore projection".to_string(),
1967 ));
1968 }
1969 tracing::info!(
1970 session_id = %candidate.session_id,
1971 candidate_run_id = %candidate.candidate_run_id,
1972 ?disposition,
1973 ?status,
1974 boundary_sequence,
1975 message_count = candidate.message_count,
1976 "durable-tail recovery committed as a recovered runtime boundary"
1977 );
1978 Ok(committed_recovery_outcome(
1979 candidate,
1980 disposition,
1981 boundary_sequence,
1982 ))
1983}
1984
1985struct CandidateInputObservation {
1989 evidence: mm_dsl::DurableRecoveryInputEvidence,
1990 predecessor_nonterminal_input_set_revision: Option<RecoveryInputSetRevision>,
1994 predecessor_nonterminal_input_set_token: Option<String>,
1998 attributed: Vec<(StoredInputState, String)>,
2004}
2005
2006fn is_terminal(phase: InputLifecycleState) -> bool {
2007 matches!(
2008 phase,
2009 InputLifecycleState::Consumed
2010 | InputLifecycleState::Superseded
2011 | InputLifecycleState::Coalesced
2012 | InputLifecycleState::Abandoned
2013 )
2014}
2015
2016fn carries_redeliverable_content(input: Option<&crate::input::Input>) -> bool {
2021 matches!(
2022 input,
2023 Some(
2024 crate::input::Input::Prompt(_)
2025 | crate::input::Input::FlowStep(_)
2026 | crate::input::Input::Peer(_)
2027 )
2028 )
2029}
2030
2031async fn observe_candidate_run_inputs(
2047 store: &dyn RuntimeStore,
2048 runtime_id: &LogicalRuntimeId,
2049 candidate_run_id: &meerkat_core::RunId,
2050 receipt_bound_inputs: &BTreeSet<String>,
2051) -> Result<CandidateInputObservation, DurableTailRecoveryError> {
2052 let snapshot = match store.load_input_states_with_versions(runtime_id).await {
2053 Ok(snapshot) => snapshot,
2054 Err(RuntimeStoreError::Unsupported(_)) => {
2060 return Ok(CandidateInputObservation {
2061 evidence: mm_dsl::DurableRecoveryInputEvidence::Unfenceable,
2062 predecessor_nonterminal_input_set_revision: None,
2063 predecessor_nonterminal_input_set_token: None,
2064 attributed: Vec::new(),
2065 });
2066 }
2067 Err(error) => return Err(error.into()),
2068 };
2069 if snapshot.runtime_id() != runtime_id {
2070 return Err(DurableTailRecoveryError::Authority(format!(
2071 "store returned a recovery input snapshot for {}, not {runtime_id}",
2072 snapshot.runtime_id()
2073 )));
2074 }
2075 let (rows, predecessor_nonterminal_input_set_revision, predecessor_nonterminal_input_set_token) =
2076 snapshot.into_parts();
2077
2078 let mut attributed = Vec::new();
2090 let mut unbound_content_input = false;
2091 for (bundle, row_digest) in rows {
2092 if is_terminal(bundle.seed.phase) {
2093 continue;
2094 }
2095 let bound_to_candidate = bundle.seed.last_run_id.as_ref() == Some(candidate_run_id)
2096 || receipt_bound_inputs.contains(&bundle.state.input_id.to_string());
2097 if !bound_to_candidate {
2098 if carries_redeliverable_content(bundle.state.persisted_input.as_ref()) {
2099 unbound_content_input = true;
2100 }
2101 continue;
2102 }
2103 attributed.push((bundle, row_digest));
2104 }
2105 Ok(CandidateInputObservation {
2106 evidence: if unbound_content_input {
2107 mm_dsl::DurableRecoveryInputEvidence::UnboundContentInput
2108 } else {
2109 mm_dsl::DurableRecoveryInputEvidence::AllBoundOrInert
2110 },
2111 predecessor_nonterminal_input_set_revision: Some(
2112 predecessor_nonterminal_input_set_revision,
2113 ),
2114 predecessor_nonterminal_input_set_token: Some(predecessor_nonterminal_input_set_token),
2115 attributed,
2116 })
2117}
2118
2119fn terminalize_attributed_inputs(
2123 attributed: Vec<(StoredInputState, String)>,
2124 candidate_run_id: &meerkat_core::RunId,
2125 boundary_sequence: u64,
2126) -> Result<Vec<InputStatePersistenceRecord>, DurableTailRecoveryError> {
2127 let mut updates = Vec::with_capacity(attributed.len());
2128 for (mut bundle, row_digest) in attributed {
2129 bundle.seed.phase = InputLifecycleState::Consumed;
2130 bundle.seed.terminal_outcome = Some(crate::input_state::InputTerminalOutcome::Consumed);
2131 bundle.seed.recovery_lane = None;
2134 bundle.seed.last_run_id = Some(candidate_run_id.clone());
2135 bundle.seed.last_boundary_sequence = Some(boundary_sequence);
2136 if crate::store::input_state_payload_is_retirable(&bundle) {
2142 bundle.state.persisted_input = None;
2143 }
2144 let record = InputStatePersistenceRecord::from_machine_snapshot(bundle)
2145 .map_err(DurableTailRecoveryError::Authority)?
2146 .with_expected_row_digest(row_digest);
2147 updates.push(record);
2148 }
2149 Ok(updates)
2150}
2151
2152#[cfg(test)]
2153mod store_authority_tests {
2154 use super::*;
2155 use crate::store::{
2156 CommittedWholeBlobProvisionalTail, CommittedWholeBlobSnapshot, WholeBlobStoreAuthority,
2157 };
2158 use meerkat_core::WholeBlobProvisionalTailAuthority;
2159 use meerkat_core::types::{
2160 AssistantBlock, BlockAssistantMessage, TranscriptMessageIdentity, UserMessage,
2161 message_timestamp_now,
2162 };
2163
2164 fn observation(run_id: &RunId) -> DurableTailObservation {
2165 DurableTailObservation {
2166 tail_run_id: Some(run_id.clone()),
2167 run_id_cardinality: RunIdCardinality::SingleRunId,
2168 terminal_stop_reason: DurableTailStopReason::EndTurn,
2169 dangling_tool_use_ids: Vec::new(),
2170 orphan_tool_result_count: 0,
2171 messages_after_terminal: false,
2172 }
2173 }
2174
2175 #[test]
2176 fn whole_blob_candidate_identity_binds_exact_store_base_and_run() {
2177 let session_id = SessionId::new();
2178 let first_run = RunId::new();
2179 let second_run = RunId::new();
2180 let first = exact_whole_blob_candidate_id(
2181 &session_id,
2182 7,
2183 "row-sha256:base",
2184 "row-sha256:candidate",
2185 1,
2186 &first_run,
2187 &observation(&first_run),
2188 );
2189 let different_run = exact_whole_blob_candidate_id(
2190 &session_id,
2191 7,
2192 "row-sha256:base",
2193 "row-sha256:candidate",
2194 1,
2195 &second_run,
2196 &observation(&second_run),
2197 );
2198 let different_base = exact_whole_blob_candidate_id(
2199 &session_id,
2200 8,
2201 "row-sha256:other-base",
2202 "row-sha256:candidate",
2203 1,
2204 &first_run,
2205 &observation(&first_run),
2206 );
2207 let different_sequence = exact_whole_blob_candidate_id(
2208 &session_id,
2209 7,
2210 "row-sha256:base",
2211 "row-sha256:candidate",
2212 2,
2213 &first_run,
2214 &observation(&first_run),
2215 );
2216 assert_ne!(first, different_run);
2217 assert_ne!(first, different_base);
2218 assert_ne!(first, different_sequence);
2219 }
2220
2221 fn whole_blob_recovery_fixture(
2222 authority_run: RunId,
2223 transcript_run: RunId,
2224 ) -> (
2225 CommittedWholeBlobSnapshot,
2226 CommittedWholeBlobProvisionalTail,
2227 ) {
2228 let mut committed = Session::new();
2229 committed.push(Message::User(UserMessage::text("committed input")));
2230 let session_id = committed.id().clone();
2231 let committed_bytes = Arc::new(committed.to_persisted_bytes().unwrap());
2232 let committed_sha = format!("row-sha256:{:x}", Sha256::digest(committed_bytes.as_ref()));
2233 let committed_authority =
2234 WholeBlobStoreAuthority::issued(session_id.clone(), 7, committed_sha.clone()).unwrap();
2235
2236 let mut candidate = committed;
2237 candidate.push(Message::BlockAssistant(BlockAssistantMessage {
2238 blocks: vec![AssistantBlock::Text {
2239 text: "durable reply".to_string(),
2240 meta: None,
2241 }],
2242 stop_reason: StopReason::EndTurn,
2243 identity: TranscriptMessageIdentity::default().with_run_id(transcript_run),
2244 created_at: message_timestamp_now(),
2245 }));
2246 let candidate_bytes = Arc::new(candidate.to_persisted_bytes().unwrap());
2247 let candidate_sha = format!("row-sha256:{:x}", Sha256::digest(candidate_bytes.as_ref()));
2248 let provisional_authority = WholeBlobProvisionalTailAuthority::issued(
2249 session_id,
2250 7,
2251 committed_sha,
2252 authority_run,
2253 candidate_sha,
2254 1,
2255 )
2256 .unwrap();
2257 (
2258 CommittedWholeBlobSnapshot::new(committed_bytes, committed_authority).unwrap(),
2259 CommittedWholeBlobProvisionalTail::new(provisional_authority, candidate_bytes),
2260 )
2261 }
2262
2263 #[test]
2264 fn whole_blob_provisional_recovery_uses_store_authority_without_session_stamp() {
2265 let run_id = RunId::new();
2266 let (committed, provisional) = whole_blob_recovery_fixture(run_id.clone(), run_id.clone());
2267
2268 let RecoveryCandidatePreparation::Prepared(candidate) =
2269 prepare_whole_blob_recovery_candidate(committed, Some(provisional))
2270 .expect("store-issued WholeBlob tail prepares")
2271 else {
2272 panic!("valid WholeBlob provisional tail must prepare");
2273 };
2274 assert_eq!(candidate.candidate_run_id, run_id);
2275 assert_eq!(
2276 candidate.class,
2277 DurableTailRecoveryClass::CompletedCandidate
2278 );
2279 assert!(
2280 candidate.document.is_none(),
2281 "completed WholeBlob recovery is metadata-only and must not materialize a successor"
2282 );
2283 assert!(matches!(
2284 candidate.store_transition,
2285 PreparedRecoveryStoreTransition::WholeBlob {
2286 base_store_revision: 7,
2287 ..
2288 }
2289 ));
2290 }
2291
2292 #[test]
2293 fn whole_blob_provisional_recovery_refuses_wrong_run_authority() {
2294 let (committed, provisional) = whole_blob_recovery_fixture(RunId::new(), RunId::new());
2295
2296 assert!(matches!(
2297 prepare_whole_blob_recovery_candidate(committed, Some(provisional)),
2298 Err(DurableTailRecoveryError::InvalidEvidence(detail))
2299 if detail.contains("store-issued run identity")
2300 ));
2301 }
2302}