Skip to main content

meerkat_runtime/
recovery.rs

1//! Machine-authorized durable-tail recovery.
2//!
3//! When a durable store head is a verified strict descendant of the committed
4//! runtime snapshot but carries intra-turn provenance, its tail is real turn
5//! content whose boundary commit lost a race with shutdown. The recovery rule:
6//! every verified durable descendant is preserved — recovery either commits it
7//! as completed, closes it as interrupted, or holds it intact. It never rolls
8//! back and never falsely marks an incomplete turn completed.
9//!
10//! Ownership split (see the recovery spec):
11//! - this module proves the exact committed authority, exact physical store
12//!   head, strict descendant relation, and tail structure, then drives
13//!   `SessionDocumentMachine` itself. Generated effects never cross the public
14//!   API as caller-assembled authority.
15//! - `MeerkatMachine` AUTHORIZES recovery — here, by driving the production
16//!   generated authority with `AuthorizeDurableTailRecovery`, whose guards
17//!   judge typed projections of the PERSISTED lifecycle row, the durably
18//!   committed receipts, and the input-lifecycle rows, and whose commit arms
19//!   mint the recovery boundary sequence.
20//! - one opaque prepared-session boundary REALIZES the recovered document,
21//!   exact receipt witness, quiescent lifecycle re-commit, physical
22//!   `SessionStore` head CAS, runtime predecessor CAS, and fenced input
23//!   terminalization in one atomic commit.
24//! - No shell promotes, discards, or downgrades the tail: every disposition
25//!   here is mirrored from an emitted machine verdict.
26//!
27//! Input identity is durable evidence only: a record is terminalized when the
28//! persisted machine facts bound it to the candidate run, or when a durably
29//! committed receipt for that run names it. Content matching is NOT identity —
30//! two identical prompts are indistinguishable by text — so an unbound,
31//! non-terminal, content-carrying input is reported to the machine as
32//! unattributable evidence, and the machine holds the recovery intact.
33//!
34//! The one machine-owned exception is the retain-inputs commit: for a clean
35//! COMPLETED candidate with an unbound content input the machine commits the
36//! proved transcript and RETAINS the unbound row in its own lifecycle for
37//! ordinary redelivery, terminalizing only rows the observation proved bound
38//! to the recovered run. At the supported 0.8.10 state floor, execution is
39//! fenced by a durable run binding first, so an unbound content row never
40//! started and redelivering it is correct. Content matching must never be used
41//! to manufacture consumption identity.
42
43use std::collections::{BTreeMap, BTreeSet};
44use std::sync::Arc;
45
46use crate::identifiers::LogicalRuntimeId;
47use crate::input_state::{InputLifecycleState, InputStatePersistenceRecord, StoredInputState};
48use crate::meerkat_machine::dsl as mm_dsl;
49use crate::runtime_state::RuntimeState;
50use crate::store::{
51    CommittedRecoveryBoundary, CommittedWholeBlobProvisionalTail, CommittedWholeBlobSnapshot,
52    MachineLifecycleBindingFacts, MachineLifecycleCommit, MachineLifecycleExpectedVersion,
53    MachineLifecycleObservation, MachineLifecycleRunFacts, PreparedDurableTailRecoverySource,
54    PreparedRecoveryEvidence, PreparedRecoveryReceiptDigestEnrichment,
55    PreparedRecoveryReceiptSource, PreparedRuntimeSessionCommit, RecoveryInputSetRevision,
56    RuntimeSessionPersistenceProfile, RuntimeStore, RuntimeStoreError, SupervisorAuthoritySnapshot,
57};
58use meerkat_core::lifecycle::InputId;
59use meerkat_core::lifecycle::core_executor::BoundSessionCommit;
60use meerkat_core::lifecycle::run_primitive::RunApplyBoundary;
61use meerkat_core::lifecycle::run_receipt::RunBoundaryReceipt;
62use meerkat_core::session_document::{
63    DurableHeadRelation, DurableTailRecoveryClass as ClassifiedRecoveryClass,
64    DurableTailStopReason, RunIdCardinality, SessionDocumentEffect, SessionDocumentKey,
65    SessionDocumentMachineAuthority,
66};
67use meerkat_core::session_store::PreparedHeadCanonicalMutation;
68use meerkat_core::types::SessionId;
69use meerkat_core::{Message, RunId, Session, StopReason, SystemNoticeKind, SystemNoticeMessage};
70use sha2::{Digest, Sha256};
71
72pub use mm_dsl::{DurableTailRecoveryClass, DurableTailRecoveryDisposition};
73
74/// Candidate hash sentinel for a current tail that carries no observed run
75/// identity. It authorizes nothing: the generated classifier sees
76/// `RunIdCardinality::NoRunId` and emits the Ambiguous/Hold verdict. The
77/// sentinel only binds that exact absence into the candidate identity.
78const NO_OBSERVED_RUN_IDENTITY: &str = "current:no-observed-run-identity";
79
80/// Structural observation of exactly the messages after committed authority.
81/// This is private because callers must not be able to assert the shape a
82/// generated classifier judges.
83#[derive(Debug)]
84struct DurableTailObservation {
85    tail_run_id: Option<RunId>,
86    run_id_cardinality: RunIdCardinality,
87    terminal_stop_reason: DurableTailStopReason,
88    dangling_tool_use_ids: Vec<String>,
89    orphan_tool_result_count: u64,
90    messages_after_terminal: bool,
91}
92
93/// Fully evidence-bound recovery candidate.
94///
95/// Every field is derived in one private constructor from the exact committed
96/// authority and exact observed physical head. In particular, no public seam
97/// accepts a generated effect, class, run id, recovered document, store token,
98/// digest, or receipt fact independently.
99#[derive(Debug)]
100enum PreparedRecoveryStoreTransition {
101    WholeBlob {
102        base_store_revision: u64,
103        base_blob_sha256: String,
104        provisional_candidate_blob_sha256: String,
105        provisional_candidate_sequence: u64,
106        recovered_blob_sha256: String,
107    },
108    HeadCanonical {
109        committed_store_revision: u64,
110        committed_head_token: String,
111        physical_store_revision: u64,
112        physical_head_cas_token: String,
113        recovered_head_token: String,
114    },
115}
116
117#[derive(Debug)]
118struct PreparedRecoveryCandidate {
119    session_id: SessionId,
120    candidate_id: String,
121    candidate_run_id: RunId,
122    class: DurableTailRecoveryClass,
123    store_transition: PreparedRecoveryStoreTransition,
124    recovered: Arc<Session>,
125    document: Option<BoundSessionCommit>,
126    conversation_digest: String,
127    message_count: usize,
128}
129
130enum RecoveryCandidatePreparation {
131    Prepared(PreparedRecoveryCandidate),
132    AlreadyAligned(Arc<Session>),
133    IncompleteHeadCanonicalIntent {
134        provisional: crate::store::HeadCanonicalProvisionalTailAuthority,
135    },
136    Held,
137}
138
139fn observe_durable_tail(authority_len: usize, head: &Session) -> DurableTailObservation {
140    let tail = &head.messages()[authority_len.min(head.messages().len())..];
141    let mut run_ids = BTreeSet::<String>::new();
142    let mut assistant_without_run_id = false;
143    let mut tail_run_id = None;
144    // Ordered multiset pairing. A result consumes one earlier call with the
145    // same id; a result before its call is an orphan and duplicated ids retain
146    // distinct obligations.
147    let mut open_call_ids = Vec::<String>::new();
148    let mut orphan_tool_result_count = 0_u64;
149    let mut last_assistant_stop: Option<DurableTailStopReason> = None;
150    let mut terminal_seen = false;
151    let mut messages_after_terminal = false;
152    for message in tail {
153        // No typed transcript fact distinguishes a generated structured-output
154        // continuation prompt from an ordinary User message. Fail closed:
155        // once a non-ToolUse assistant has ended, every later message makes
156        // the candidate ambiguous. A future extraction-phase identity can
157        // narrow this without inferring authority from prompt text.
158        if terminal_seen {
159            messages_after_terminal = true;
160        }
161        match message {
162            Message::BlockAssistant(assistant) => {
163                match assistant.identity.run_id.as_ref() {
164                    Some(run_id) => {
165                        run_ids.insert(run_id.to_string());
166                        tail_run_id = Some(run_id.clone());
167                    }
168                    None => assistant_without_run_id = true,
169                }
170                open_call_ids.extend(assistant.tool_calls().map(|call| call.id.to_string()));
171                let effective_stop = match assistant.stop_reason {
172                    StopReason::EndTurn => DurableTailStopReason::EndTurn,
173                    StopReason::ToolUse if assistant.has_tool_calls() => {
174                        DurableTailStopReason::ToolUse
175                    }
176                    // The live agent decides the tool phase from actual call
177                    // blocks, not the provider's stop label. A ToolUse label
178                    // with no calls is operationally terminal and recovery
179                    // must mirror that exact decision.
180                    StopReason::ToolUse => DurableTailStopReason::EndTurn,
181                    _ => DurableTailStopReason::Other,
182                };
183                last_assistant_stop = Some(effective_stop);
184                // ToolUse is an intermediate provider boundary: durable tool
185                // results and a later assistant continuation can belong to
186                // the same run. Every other stop reason closes that run, so
187                // any later message makes the candidate ambiguous even when
188                // a later assistant reuses the same run id and ends cleanly.
189                terminal_seen = effective_stop != DurableTailStopReason::ToolUse;
190            }
191            Message::ToolResults { results, .. } => {
192                for result in results {
193                    if let Some(position) = open_call_ids
194                        .iter()
195                        .position(|call_id| *call_id == result.tool_use_id)
196                    {
197                        open_call_ids.remove(position);
198                    } else {
199                        orphan_tool_result_count += 1;
200                    }
201                }
202            }
203            _ => {}
204        }
205    }
206    let terminal_stop_reason = last_assistant_stop.unwrap_or(DurableTailStopReason::Absent);
207    let run_id_cardinality = match (run_ids.len(), assistant_without_run_id) {
208        (0, _) => RunIdCardinality::NoRunId,
209        (1, false) => RunIdCardinality::SingleRunId,
210        _ => RunIdCardinality::MultipleRunIds,
211    };
212    DurableTailObservation {
213        tail_run_id,
214        run_id_cardinality,
215        terminal_stop_reason,
216        dangling_tool_use_ids: open_call_ids,
217        orphan_tool_result_count,
218        messages_after_terminal,
219    }
220}
221
222fn hash_part(hasher: &mut Sha256, label: &str, value: &[u8]) {
223    hasher.update((label.len() as u64).to_be_bytes());
224    hasher.update(label.as_bytes());
225    hasher.update((value.len() as u64).to_be_bytes());
226    hasher.update(value);
227}
228
229fn run_id_cardinality_name(cardinality: RunIdCardinality) -> &'static str {
230    match cardinality {
231        RunIdCardinality::NoRunId => "no_run_id",
232        RunIdCardinality::SingleRunId => "single_run_id",
233        RunIdCardinality::MultipleRunIds => "multiple_run_ids",
234    }
235}
236
237fn durable_tail_stop_reason_name(reason: DurableTailStopReason) -> &'static str {
238    match reason {
239        DurableTailStopReason::Absent => "absent",
240        DurableTailStopReason::EndTurn => "end_turn",
241        DurableTailStopReason::ToolUse => "tool_use",
242        DurableTailStopReason::Other => "other",
243    }
244}
245
246fn exact_candidate_id(
247    session_id: &SessionId,
248    committed_store_revision: u64,
249    committed_head_token: &str,
250    physical_store_revision: u64,
251    physical_head_cas_token: &str,
252    provisional_run_id: &RunId,
253    observation: &DurableTailObservation,
254) -> String {
255    let observed_run = observation
256        .tail_run_id
257        .as_ref()
258        .map(ToString::to_string)
259        .unwrap_or_else(|| NO_OBSERVED_RUN_IDENTITY.to_string());
260    let mut hasher = Sha256::new();
261    hash_part(
262        &mut hasher,
263        "domain",
264        b"meerkat:durable-tail-recovery-candidate:v5",
265    );
266    hash_part(&mut hasher, "session", session_id.to_string().as_bytes());
267    hash_part(
268        &mut hasher,
269        "committed_store_revision",
270        &committed_store_revision.to_be_bytes(),
271    );
272    hash_part(
273        &mut hasher,
274        "committed_head_token",
275        committed_head_token.as_bytes(),
276    );
277    hash_part(
278        &mut hasher,
279        "physical_store_revision",
280        &physical_store_revision.to_be_bytes(),
281    );
282    hash_part(
283        &mut hasher,
284        "physical_head_token",
285        physical_head_cas_token.as_bytes(),
286    );
287    hash_part(
288        &mut hasher,
289        "provisional_run",
290        provisional_run_id.to_string().as_bytes(),
291    );
292    hash_part(&mut hasher, "observed_run", observed_run.as_bytes());
293    hash_part(
294        &mut hasher,
295        "run_cardinality",
296        run_id_cardinality_name(observation.run_id_cardinality).as_bytes(),
297    );
298    hash_part(
299        &mut hasher,
300        "stop_reason",
301        durable_tail_stop_reason_name(observation.terminal_stop_reason).as_bytes(),
302    );
303    hash_part(
304        &mut hasher,
305        "dangling_call_count",
306        observation
307            .dangling_tool_use_ids
308            .len()
309            .to_string()
310            .as_bytes(),
311    );
312    for call_id in &observation.dangling_tool_use_ids {
313        hash_part(&mut hasher, "dangling_call", call_id.as_bytes());
314    }
315    hash_part(
316        &mut hasher,
317        "orphan_results",
318        observation.orphan_tool_result_count.to_string().as_bytes(),
319    );
320    hash_part(
321        &mut hasher,
322        "after_terminal",
323        if observation.messages_after_terminal {
324            b"true"
325        } else {
326            b"false"
327        },
328    );
329    format!("sha256:{:x}", hasher.finalize())
330}
331
332fn exact_whole_blob_candidate_id(
333    session_id: &SessionId,
334    base_store_revision: u64,
335    base_blob_sha256: &str,
336    candidate_blob_sha256: &str,
337    candidate_sequence: u64,
338    provisional_run_id: &RunId,
339    observation: &DurableTailObservation,
340) -> String {
341    let mut hasher = Sha256::new();
342    hash_part(
343        &mut hasher,
344        "domain",
345        b"meerkat:whole-blob-durable-tail-recovery-candidate:v2",
346    );
347    hash_part(&mut hasher, "session", session_id.to_string().as_bytes());
348    hash_part(
349        &mut hasher,
350        "base_store_revision",
351        &base_store_revision.to_be_bytes(),
352    );
353    hash_part(&mut hasher, "base_blob_sha256", base_blob_sha256.as_bytes());
354    hash_part(
355        &mut hasher,
356        "candidate_blob_sha256",
357        candidate_blob_sha256.as_bytes(),
358    );
359    hash_part(
360        &mut hasher,
361        "candidate_sequence",
362        &candidate_sequence.to_be_bytes(),
363    );
364    hash_part(
365        &mut hasher,
366        "provisional_run",
367        provisional_run_id.to_string().as_bytes(),
368    );
369    hash_part(
370        &mut hasher,
371        "run_cardinality",
372        run_id_cardinality_name(observation.run_id_cardinality).as_bytes(),
373    );
374    hash_part(
375        &mut hasher,
376        "stop_reason",
377        durable_tail_stop_reason_name(observation.terminal_stop_reason).as_bytes(),
378    );
379    for call_id in &observation.dangling_tool_use_ids {
380        hash_part(&mut hasher, "dangling_call", call_id.as_bytes());
381    }
382    hash_part(
383        &mut hasher,
384        "orphan_results",
385        &observation.orphan_tool_result_count.to_be_bytes(),
386    );
387    hash_part(
388        &mut hasher,
389        "after_terminal",
390        if observation.messages_after_terminal {
391            b"true"
392        } else {
393            b"false"
394        },
395    );
396    format!("sha256:{:x}", hasher.finalize())
397}
398
399fn generated_recovery_classification(
400    session_id: &SessionId,
401    candidate_id: &str,
402    observation: &DurableTailObservation,
403) -> Result<ClassifiedRecoveryClass, DurableTailRecoveryError> {
404    // The generated machine is invoked and consumed inside this function.
405    // There is intentionally no parameter through which a caller can inject a
406    // `SessionDocumentEffect` or any of the shape projections below.
407    let mut classifier = SessionDocumentMachineAuthority::new();
408    let effects = classifier
409        .classify_durable_tail(
410            SessionDocumentKey::new(session_id.to_string()),
411            candidate_id.to_string(),
412            DurableHeadRelation::VerifiedStrictDescendant,
413            observation.run_id_cardinality,
414            observation.terminal_stop_reason,
415            observation.dangling_tool_use_ids.len() as u64,
416            observation.orphan_tool_result_count,
417            observation.messages_after_terminal,
418        )
419        .map_err(|error| {
420            DurableTailRecoveryError::Authority(format!(
421                "durable-tail classification rejected: {error}"
422            ))
423        })?;
424    let mut matching = effects.iter().filter_map(|effect| match effect {
425        SessionDocumentEffect::DurableTailClassified {
426            candidate_id: emitted_candidate,
427            class,
428        } if emitted_candidate == candidate_id => Some(*class),
429        _ => None,
430    });
431    let Some(class) = matching.next() else {
432        return Err(DurableTailRecoveryError::Authority(
433            "classifier emitted no verdict for the exact evidence-bound candidate".to_string(),
434        ));
435    };
436    if matching.next().is_some() {
437        return Err(DurableTailRecoveryError::Authority(
438            "classifier emitted more than one verdict for the same candidate".to_string(),
439        ));
440    }
441    Ok(class)
442}
443
444fn runtime_recovery_class(
445    class: ClassifiedRecoveryClass,
446) -> Result<DurableTailRecoveryClass, DurableTailRecoveryError> {
447    Ok(match class {
448        ClassifiedRecoveryClass::CompletedCandidate => DurableTailRecoveryClass::CompletedCandidate,
449        ClassifiedRecoveryClass::InterruptedRepairableCandidate => {
450            DurableTailRecoveryClass::InterruptedRepairableCandidate
451        }
452        ClassifiedRecoveryClass::Ambiguous => DurableTailRecoveryClass::Ambiguous,
453        // Fail closed against a stale generated artifact while the 0.8.10
454        // floor removal propagates through generated code. This is not a
455        // mapping or adoption path; the variant disappears at regeneration.
456        #[allow(unreachable_patterns)]
457        _ => {
458            return Err(DurableTailRecoveryError::Authority(
459                "session-document classifier emitted unsupported recovery vocabulary".to_string(),
460            ));
461        }
462    })
463}
464
465fn message_timestamp(message: &Message) -> meerkat_core::types::MessageTimestamp {
466    match message {
467        Message::System(message) => message.created_at,
468        Message::SystemNotice(message) => message.created_at,
469        Message::User(message) => message.created_at,
470        Message::BlockAssistant(message) => message.created_at,
471        Message::ToolResults { created_at, .. } => *created_at,
472    }
473}
474
475fn repair_interrupted_tail(
476    recovered: &mut Session,
477    dangling_tool_use_ids: &[String],
478    durable_tail_timestamp: meerkat_core::types::MessageTimestamp,
479) -> Result<(), DurableTailRecoveryError> {
480    // The classifier admits repair only with zero dangling calls. Recheck in
481    // release code: manufacturing tool results would invent external
482    // execution truth for a call whose side effect may already have fired.
483    if !dangling_tool_use_ids.is_empty() {
484        return Err(DurableTailRecoveryError::InvalidEvidence(format!(
485            "interrupted-tail repair was classified despite {} dangling tool call(s): {}",
486            dangling_tool_use_ids.len(),
487            dangling_tool_use_ids.join(", ")
488        )));
489    }
490    recovered.push(Message::SystemNotice(SystemNoticeMessage {
491        kind: SystemNoticeKind::Generic,
492        body: Some(
493            "A previous run was interrupted before its boundary committed. Recovery preserved \
494             every durable message, closed the run as InterruptedByRecovery, and did not requeue \
495             its input. Continue from a new turn."
496                .to_string(),
497        ),
498        blocks: Vec::new(),
499        // Recovery of the same exact durable head must be byte-identical in
500        // every process. Wall-clock `now` would make the recovered boundary
501        // and idempotency witness race-dependent.
502        created_at: durable_tail_timestamp,
503    }));
504    Ok(())
505}
506
507fn prepare_head_canonical_recovery_candidate(
508    source: &PreparedDurableTailRecoverySource,
509) -> Result<RecoveryCandidatePreparation, DurableTailRecoveryError> {
510    let runtime_authority = source.runtime_authority();
511    let committed_authority = source.committed_session().as_ref();
512    let observed_physical_head = source.physical_head();
513    let physical_head = source.physical_session().as_ref();
514    let session_id = committed_authority.id().clone();
515    if runtime_authority.profile() != RuntimeSessionPersistenceProfile::HeadCanonicalV1 {
516        return Err(DurableTailRecoveryError::InvalidEvidence(format!(
517            "runtime persistence profile {} cannot atomically recover an external physical head",
518            runtime_authority.profile()
519        )));
520    }
521    let committed_store_authority = runtime_authority.head_canonical().ok_or_else(|| {
522        DurableTailRecoveryError::InvalidEvidence(
523            "HeadCanonical recovery received a different store authority profile".to_string(),
524        )
525    })?;
526    if runtime_authority.session_id() != &session_id {
527        return Err(DurableTailRecoveryError::InvalidEvidence(format!(
528            "runtime authority belongs to session {}, not committed document {session_id}",
529            runtime_authority.session_id()
530        )));
531    }
532    if observed_physical_head.id != session_id {
533        return Err(DurableTailRecoveryError::InvalidEvidence(format!(
534            "observed physical head belongs to session {}, not committed authority {session_id}",
535            observed_physical_head.id
536        )));
537    }
538    if physical_head.id() != &session_id {
539        return Err(DurableTailRecoveryError::InvalidEvidence(format!(
540            "physical head belongs to session {}, not committed authority {session_id}",
541            physical_head.id()
542        )));
543    }
544    if committed_authority.version() != physical_head.version()
545        || committed_authority.created_at() != physical_head.created_at()
546    {
547        return Err(DurableTailRecoveryError::InvalidEvidence(
548            "physical head changes immutable session envelope identity".to_string(),
549        ));
550    }
551    let boundary_head = committed_store_authority.boundary_head();
552    let authority_head_cas_token = committed_store_authority.committed_head_token();
553
554    if let Some(provisional) = source.provisional_authority()
555        && !source.provisional_target_applied()
556    {
557        return Ok(
558            RecoveryCandidatePreparation::IncompleteHeadCanonicalIntent {
559                provisional: provisional.clone(),
560            },
561        );
562    }
563
564    // Benign convergence: another process may have committed recovery after
565    // the caller observed A/H but before this store-owned snapshot. Equality
566    // is not a hold and not a recovery candidate. Return the exact
567    // committed document the source paired with the authoritative head.
568    if observed_physical_head == boundary_head {
569        if source.physical_head_cas_token() != authority_head_cas_token {
570            return Err(DurableTailRecoveryError::InvalidEvidence(
571                "equal runtime and physical heads carry contradictory store authority".to_string(),
572            ));
573        }
574        return Ok(RecoveryCandidatePreparation::AlreadyAligned(Arc::clone(
575            source.physical_session(),
576        )));
577    }
578    let provisional = source.provisional_authority().ok_or_else(|| {
579        DurableTailRecoveryError::InvalidEvidence(
580            "newer physical head has no store-issued provisional authority".to_string(),
581        )
582    })?;
583    if provisional.session_id() != &session_id
584        || provisional.base_store_revision() != committed_store_authority.store_revision()
585        || provisional.base_committed_head_token() != authority_head_cas_token
586        || provisional.physical_head_token() != source.physical_head_cas_token()
587    {
588        return Err(DurableTailRecoveryError::InvalidEvidence(
589            "provisional tail does not name the exact committed parent and physical head"
590                .to_string(),
591        ));
592    }
593    if physical_head.messages().len() <= committed_authority.messages().len()
594        || !physical_head
595            .messages()
596            .starts_with(committed_authority.messages())
597    {
598        return Err(DurableTailRecoveryError::InvalidEvidence(
599            "physical recovery head is not an exact strict transcript continuation".to_string(),
600        ));
601    }
602
603    let derived_physical_head_token = source.physical_head_cas_token().to_string();
604
605    let observation = observe_durable_tail(committed_authority.messages().len(), physical_head);
606    if observation.tail_run_id.as_ref() != Some(provisional.run_id())
607        || observation.run_id_cardinality != RunIdCardinality::SingleRunId
608    {
609        return Err(DurableTailRecoveryError::InvalidEvidence(
610            "physical tail transcript contradicts its store-issued provisional run identity"
611                .to_string(),
612        ));
613    }
614    let candidate_id = exact_candidate_id(
615        &session_id,
616        committed_store_authority.store_revision(),
617        authority_head_cas_token,
618        provisional.physical_store_revision(),
619        source.physical_head_cas_token(),
620        provisional.run_id(),
621        &observation,
622    );
623    let classified = generated_recovery_classification(&session_id, &candidate_id, &observation)?;
624    let class = runtime_recovery_class(classified)?;
625    if class == DurableTailRecoveryClass::Ambiguous {
626        return Ok(RecoveryCandidatePreparation::Held);
627    }
628    let candidate_run_id = provisional.run_id().clone();
629
630    // The canonical physical materialization is already the exact durable
631    // successor content. Starting from it is both simpler and deterministic:
632    // replaying the suffix onto an inline transcript graph manufactures fresh
633    // revision-body timestamps in `Session::push`, so two processes would
634    // mint different recovered boundaries for the same durable rows.
635    //
636    // Head-canonical materialization must be slim. Retained rewrite bodies are
637    // an out-of-line store concern and the carried rewrite-prefix/history
638    // witness in SessionHead is the recovery document's authority.
639    if physical_head
640        .validated_transcript_history_state()
641        .map_err(|error| {
642            DurableTailRecoveryError::InvalidEvidence(format!(
643                "physical-head transcript history is malformed: {error}"
644            ))
645        })?
646        .is_some()
647    {
648        return Err(DurableTailRecoveryError::InvalidEvidence(
649            "head-canonical recovery source unexpectedly contains inline transcript history"
650                .to_string(),
651        ));
652    }
653    let mut recovered = physical_head.clone();
654    match class {
655        DurableTailRecoveryClass::CompletedCandidate => {
656            // The exact physical transcript is already the completed boundary.
657        }
658        DurableTailRecoveryClass::InterruptedRepairableCandidate => {
659            let durable_tail_timestamp = physical_head
660                .messages()
661                .last()
662                .map(message_timestamp)
663                .ok_or_else(|| {
664                    DurableTailRecoveryError::InvalidEvidence(
665                        "interrupted recovery has no durable tail timestamp".to_string(),
666                    )
667                })?;
668            repair_interrupted_tail(
669                &mut recovered,
670                &observation.dangling_tool_use_ids,
671                durable_tail_timestamp,
672            )?;
673        }
674        DurableTailRecoveryClass::Ambiguous => {
675            return Err(DurableTailRecoveryError::Authority(
676                "ambiguous recovery candidate reached document preparation after hold".to_string(),
677            ));
678        }
679    }
680    // Synthetic repair necessarily advances the outer updated_at with
681    // wall-clock time. Re-adopting the exact physical envelope restores the
682    // durable timestamp and all non-recovery-owned state. Completed recovery
683    // is already byte-for-byte the physical materialization; applying this
684    // uniformly keeps the field-ownership rule in one place.
685    recovered
686        .adopt_recovered_head_state(physical_head)
687        .map_err(DurableTailRecoveryError::InvalidEvidence)?;
688    if recovered.messages().len() < physical_head.messages().len() {
689        return Err(DurableTailRecoveryError::InvalidEvidence(format!(
690            "internally recovered document lost durable content: {} < {} messages",
691            recovered.messages().len(),
692            physical_head.messages().len()
693        )));
694    }
695    let recovered = Arc::new(recovered);
696    let recovered_mutation = PreparedHeadCanonicalMutation::prepare(
697        recovered.as_ref(),
698        Some(observed_physical_head.clone()),
699    )
700    .map_err(|error| {
701        DurableTailRecoveryError::InvalidEvidence(format!(
702            "recovered HeadCanonical mutation preparation failed: {error}"
703        ))
704    })?;
705    if recovered_mutation.predecessor_head_token() != Some(derived_physical_head_token.as_str()) {
706        return Err(DurableTailRecoveryError::InvalidEvidence(
707            "recovered mutation changed the provisional physical-head token".to_string(),
708        ));
709    }
710    let recovered_head_token = meerkat_core::session_head_cas_token(
711        recovered_mutation.successor_head(),
712    )
713    .map_err(|error| {
714        DurableTailRecoveryError::InvalidEvidence(format!(
715            "recovered successor head token is invalid: {error}"
716        ))
717    })?;
718    let document = BoundSessionCommit::sealed(Arc::clone(&recovered))
719        .map_err(|error| {
720            DurableTailRecoveryError::InvalidEvidence(format!(
721                "failed to seal recovered HeadCanonical document: {error}"
722            ))
723        })?
724        .with_head_canonical_mutation(recovered_mutation)
725        .map_err(|error| {
726            DurableTailRecoveryError::InvalidEvidence(format!(
727                "failed to bind recovered HeadCanonical mutation: {error}"
728            ))
729        })?;
730    let conversation_digest = recovered.transcript_content_digest().map_err(|error| {
731        DurableTailRecoveryError::InvalidEvidence(format!(
732            "recovered transcript digest failed: {error}"
733        ))
734    })?;
735    let message_count = recovered.messages().len();
736
737    Ok(RecoveryCandidatePreparation::Prepared(
738        PreparedRecoveryCandidate {
739            session_id,
740            candidate_id,
741            candidate_run_id,
742            class,
743            store_transition: PreparedRecoveryStoreTransition::HeadCanonical {
744                committed_store_revision: committed_store_authority.store_revision(),
745                committed_head_token: authority_head_cas_token.to_string(),
746                physical_store_revision: provisional.physical_store_revision(),
747                physical_head_cas_token: derived_physical_head_token,
748                recovered_head_token,
749            },
750            recovered,
751            document: Some(document),
752            conversation_digest,
753            message_count,
754        },
755    ))
756}
757
758fn prepare_whole_blob_recovery_candidate(
759    committed: CommittedWholeBlobSnapshot,
760    provisional: Option<CommittedWholeBlobProvisionalTail>,
761) -> Result<RecoveryCandidatePreparation, DurableTailRecoveryError> {
762    let (committed_session, _committed_bytes, committed_authority) = committed.into_parts();
763    if committed_session.id() != committed_authority.session_id() {
764        return Err(DurableTailRecoveryError::InvalidEvidence(
765            "committed WholeBlob payload identity differs from store authority".to_string(),
766        ));
767    }
768    let Some(provisional) = provisional else {
769        return Ok(RecoveryCandidatePreparation::AlreadyAligned(
770            committed_session,
771        ));
772    };
773    let provisional_authority = provisional.authority();
774    if provisional_authority.session_id() != committed_authority.session_id()
775        || provisional_authority.base_store_revision() != committed_authority.store_revision()
776        || provisional_authority.base_blob_sha256() != committed_authority.blob_sha256()
777    {
778        return Err(DurableTailRecoveryError::InvalidEvidence(
779            "WholeBlob provisional tail does not name the exact committed base".to_string(),
780        ));
781    }
782    let candidate_session =
783        Session::from_persisted_bytes(provisional.candidate_bytes()).map_err(|error| {
784            DurableTailRecoveryError::InvalidEvidence(format!(
785                "provisional WholeBlob payload is invalid: {error}"
786            ))
787        })?;
788    if candidate_session.id() != committed_session.id()
789        || candidate_session.version() != committed_session.version()
790        || candidate_session.created_at() != committed_session.created_at()
791    {
792        return Err(DurableTailRecoveryError::InvalidEvidence(
793            "WholeBlob provisional tail changes immutable session identity".to_string(),
794        ));
795    }
796    if candidate_session.messages().len() <= committed_session.messages().len()
797        || !candidate_session
798            .messages()
799            .starts_with(committed_session.messages())
800    {
801        return Err(DurableTailRecoveryError::InvalidEvidence(
802            "WholeBlob provisional tail is not an exact strict transcript continuation".to_string(),
803        ));
804    }
805    let observation = observe_durable_tail(committed_session.messages().len(), &candidate_session);
806    if observation.tail_run_id.as_ref() != Some(provisional_authority.run_id())
807        || observation.run_id_cardinality != RunIdCardinality::SingleRunId
808    {
809        return Err(DurableTailRecoveryError::InvalidEvidence(
810            "WholeBlob provisional transcript contradicts its store-issued run identity"
811                .to_string(),
812        ));
813    }
814    let candidate_id = exact_whole_blob_candidate_id(
815        committed_authority.session_id(),
816        committed_authority.store_revision(),
817        committed_authority.blob_sha256(),
818        provisional_authority.candidate_blob_sha256(),
819        provisional_authority.candidate_sequence(),
820        provisional_authority.run_id(),
821        &observation,
822    );
823    let class = runtime_recovery_class(generated_recovery_classification(
824        committed_authority.session_id(),
825        &candidate_id,
826        &observation,
827    )?)?;
828    if class == DurableTailRecoveryClass::Ambiguous {
829        return Ok(RecoveryCandidatePreparation::Held);
830    }
831    let physical_envelope = candidate_session.clone();
832    let mut recovered = candidate_session;
833    match class {
834        DurableTailRecoveryClass::CompletedCandidate => {}
835        DurableTailRecoveryClass::InterruptedRepairableCandidate => {
836            let durable_tail_timestamp = recovered
837                .messages()
838                .last()
839                .map(message_timestamp)
840                .ok_or_else(|| {
841                    DurableTailRecoveryError::InvalidEvidence(
842                        "interrupted WholeBlob recovery has no durable tail timestamp".to_string(),
843                    )
844                })?;
845            repair_interrupted_tail(
846                &mut recovered,
847                &observation.dangling_tool_use_ids,
848                durable_tail_timestamp,
849            )?;
850        }
851        DurableTailRecoveryClass::Ambiguous => {
852            return Err(DurableTailRecoveryError::Authority(
853                "ambiguous WholeBlob recovery reached document preparation".to_string(),
854            ));
855        }
856    }
857    recovered
858        .adopt_recovered_head_state(&physical_envelope)
859        .map_err(DurableTailRecoveryError::InvalidEvidence)?;
860    let recovered = Arc::new(recovered);
861    let (document, recovered_blob_sha256) = match class {
862        DurableTailRecoveryClass::CompletedCandidate => (
863            None,
864            provisional_authority.candidate_blob_sha256().to_string(),
865        ),
866        DurableTailRecoveryClass::InterruptedRepairableCandidate => {
867            let document = BoundSessionCommit::sealed(Arc::clone(&recovered)).map_err(|error| {
868                DurableTailRecoveryError::InvalidEvidence(format!(
869                    "failed to seal repaired WholeBlob document: {error}"
870                ))
871            })?;
872            let recovered_blob_sha256 = document
873                .whole_blob_artifact()
874                .map_err(|error| {
875                    DurableTailRecoveryError::InvalidEvidence(format!(
876                        "failed to serialize repaired WholeBlob document: {error}"
877                    ))
878                })?
879                .row_sha256_token()
880                .to_string();
881            (Some(document), recovered_blob_sha256)
882        }
883        DurableTailRecoveryClass::Ambiguous => {
884            return Err(DurableTailRecoveryError::Authority(
885                "ambiguous WholeBlob recovery reached artifact preparation".to_string(),
886            ));
887        }
888    };
889    let conversation_digest = recovered.transcript_content_digest().map_err(|error| {
890        DurableTailRecoveryError::InvalidEvidence(format!(
891            "recovered WholeBlob transcript digest failed: {error}"
892        ))
893    })?;
894    let message_count = recovered.messages().len();
895    Ok(RecoveryCandidatePreparation::Prepared(
896        PreparedRecoveryCandidate {
897            session_id: committed_authority.session_id().clone(),
898            candidate_id,
899            candidate_run_id: provisional_authority.run_id().clone(),
900            class,
901            store_transition: PreparedRecoveryStoreTransition::WholeBlob {
902                base_store_revision: committed_authority.store_revision(),
903                base_blob_sha256: committed_authority.blob_sha256().to_string(),
904                provisional_candidate_blob_sha256: provisional_authority
905                    .candidate_blob_sha256()
906                    .to_string(),
907                provisional_candidate_sequence: provisional_authority.candidate_sequence(),
908                recovered_blob_sha256,
909            },
910            recovered,
911            document,
912            conversation_digest,
913            message_count,
914        },
915    ))
916}
917
918fn bind_recovered_store_document(
919    candidate: &PreparedRecoveryCandidate,
920) -> Result<Option<BoundSessionCommit>, DurableTailRecoveryError> {
921    let document = candidate.document.clone();
922    match &candidate.store_transition {
923        PreparedRecoveryStoreTransition::WholeBlob {
924            provisional_candidate_blob_sha256,
925            recovered_blob_sha256,
926            ..
927        } => {
928            if recovered_blob_sha256 == provisional_candidate_blob_sha256 {
929                if candidate.class != DurableTailRecoveryClass::CompletedCandidate
930                    || document.is_some()
931                {
932                    return Err(DurableTailRecoveryError::InvalidEvidence(
933                        "metadata-only WholeBlob promotion carries a repaired document or \
934                         non-completed class"
935                            .to_string(),
936                    ));
937                }
938                return Ok(None);
939            }
940            let document = document.ok_or_else(|| {
941                DurableTailRecoveryError::InvalidEvidence(
942                    "repaired WholeBlob recovery lost its sealed successor artifact".to_string(),
943                )
944            })?;
945            let artifact = document.whole_blob_artifact().map_err(|error| {
946                DurableTailRecoveryError::InvalidEvidence(format!(
947                    "failed to materialize recovered WholeBlob artifact: {error}"
948                ))
949            })?;
950            if artifact.row_sha256_token() != recovered_blob_sha256 {
951                return Err(DurableTailRecoveryError::InvalidEvidence(
952                    "sealed WholeBlob recovery document differs from its successor digest"
953                        .to_string(),
954                ));
955            }
956            Ok(Some(document))
957        }
958        PreparedRecoveryStoreTransition::HeadCanonical {
959            physical_head_cas_token,
960            recovered_head_token,
961            ..
962        } => {
963            let document = document.ok_or_else(|| {
964                DurableTailRecoveryError::InvalidEvidence(
965                    "HeadCanonical recovery lost its prepared mutation".to_string(),
966                )
967            })?;
968            let mutation = document
969                .head_canonical()
970                .ok_or_else(|| {
971                    DurableTailRecoveryError::InvalidEvidence(
972                        "sealed HeadCanonical recovery lost its prepared mutation".to_string(),
973                    )
974                })?
975                .mutation();
976            if mutation.predecessor_head_token() != Some(physical_head_cas_token.as_str()) {
977                return Err(DurableTailRecoveryError::InvalidEvidence(
978                    "prepared recovery mutation changed the observed physical-head CAS token"
979                        .to_string(),
980                ));
981            }
982            let successor_head_token = meerkat_core::session_head_cas_token(
983                mutation.successor_head(),
984            )
985            .map_err(|error| {
986                DurableTailRecoveryError::InvalidEvidence(format!(
987                    "prepared recovery successor head token is invalid: {error}"
988                ))
989            })?;
990            if successor_head_token != *recovered_head_token {
991                return Err(DurableTailRecoveryError::InvalidEvidence(
992                    "prepared recovery mutation changed the recovered store head".to_string(),
993                ));
994            }
995            Ok(Some(document))
996        }
997    }
998}
999
1000#[allow(clippy::too_many_arguments)]
1001fn seal_recovery_evidence(
1002    candidate: &PreparedRecoveryCandidate,
1003    document: Option<&BoundSessionCommit>,
1004    disposition: DurableTailRecoveryDisposition,
1005    receipt_digest_enrichments: Vec<PreparedRecoveryReceiptDigestEnrichment>,
1006    predecessor_nonterminal_input_set_revision: RecoveryInputSetRevision,
1007    predecessor_nonterminal_input_set_token: String,
1008    input_updates: Vec<InputStatePersistenceRecord>,
1009    receipt: &RunBoundaryReceipt,
1010    lifecycle: &MachineLifecycleCommit,
1011) -> Result<PreparedRecoveryEvidence, RuntimeStoreError> {
1012    match &candidate.store_transition {
1013        PreparedRecoveryStoreTransition::WholeBlob {
1014            base_store_revision,
1015            base_blob_sha256,
1016            provisional_candidate_blob_sha256,
1017            provisional_candidate_sequence,
1018            recovered_blob_sha256,
1019        } => PreparedRecoveryEvidence::seal_whole_blob(
1020            candidate.recovered.as_ref(),
1021            document,
1022            candidate.session_id.clone(),
1023            candidate.candidate_id.clone(),
1024            candidate.candidate_run_id.clone(),
1025            candidate.class,
1026            disposition,
1027            *base_store_revision,
1028            base_blob_sha256.clone(),
1029            provisional_candidate_blob_sha256.clone(),
1030            *provisional_candidate_sequence,
1031            recovered_blob_sha256.clone(),
1032            receipt_digest_enrichments,
1033            predecessor_nonterminal_input_set_revision,
1034            predecessor_nonterminal_input_set_token,
1035            input_updates,
1036            receipt,
1037            lifecycle,
1038        ),
1039        PreparedRecoveryStoreTransition::HeadCanonical {
1040            committed_store_revision,
1041            committed_head_token,
1042            physical_store_revision,
1043            physical_head_cas_token,
1044            recovered_head_token,
1045            ..
1046        } => {
1047            let document =
1048                document.ok_or_else(|| RuntimeStoreError::SessionPersistenceAuthorityConflict {
1049                    runtime_id: candidate.session_id.to_string(),
1050                    detail: "HeadCanonical recovery evidence lost its prepared mutation"
1051                        .to_string(),
1052                })?;
1053            PreparedRecoveryEvidence::seal_head_canonical(
1054                candidate.recovered.as_ref(),
1055                document,
1056                candidate.session_id.clone(),
1057                candidate.candidate_id.clone(),
1058                candidate.candidate_run_id.clone(),
1059                candidate.class,
1060                disposition,
1061                *committed_store_revision,
1062                committed_head_token.clone(),
1063                *physical_store_revision,
1064                physical_head_cas_token.clone(),
1065                recovered_head_token.clone(),
1066                receipt_digest_enrichments,
1067                predecessor_nonterminal_input_set_revision,
1068                predecessor_nonterminal_input_set_token,
1069                input_updates,
1070                receipt,
1071                lifecycle,
1072            )
1073        }
1074    }
1075}
1076
1077fn verify_exact_committed_recovery(
1078    candidate: &PreparedRecoveryCandidate,
1079    committed: &CommittedRecoveryBoundary,
1080    lifecycle: &MachineLifecycleCommit,
1081) -> Result<(DurableTailRecoveryDisposition, u64), DurableTailRecoveryError> {
1082    let receipt = committed.receipt();
1083    if receipt.boundary != RunApplyBoundary::Immediate {
1084        return Err(DurableTailRecoveryError::InvalidEvidence(
1085            "committed recovery witness carries a non-immediate boundary".to_string(),
1086        ));
1087    }
1088    let document = bind_recovered_store_document(candidate)?;
1089    let expected_evidence = seal_recovery_evidence(
1090        candidate,
1091        document.as_ref(),
1092        committed.evidence().disposition(),
1093        committed.evidence().receipt_digest_enrichments().to_vec(),
1094        committed
1095            .evidence()
1096            .predecessor_nonterminal_input_set_revision(),
1097        committed
1098            .evidence()
1099            .predecessor_nonterminal_input_set_token()
1100            .to_owned(),
1101        committed.evidence().cloned_input_updates(),
1102        receipt,
1103        lifecycle,
1104    )?;
1105    if &expected_evidence != committed.evidence() {
1106        return Err(DurableTailRecoveryError::InvalidEvidence(
1107            "committed recovery candidate id exists with a different exact source, recovered \
1108             store authority, receipt, lifecycle target, migration witness, or evidence witness"
1109                .to_string(),
1110        ));
1111    }
1112    Ok((committed.evidence().disposition(), receipt.sequence))
1113}
1114
1115async fn load_exact_committed_recovery(
1116    store: &dyn RuntimeStore,
1117    runtime_id: &LogicalRuntimeId,
1118    candidate: &PreparedRecoveryCandidate,
1119    lifecycle: &MachineLifecycleCommit,
1120) -> Result<Option<(DurableTailRecoveryDisposition, u64)>, DurableTailRecoveryError> {
1121    let committed = store
1122        .load_committed_recovery_boundary(runtime_id, &candidate.candidate_id)
1123        .await?;
1124    committed
1125        .as_ref()
1126        .map(|committed| verify_exact_committed_recovery(candidate, committed, lifecycle))
1127        .transpose()
1128}
1129
1130fn committed_recovery_outcome(
1131    candidate: PreparedRecoveryCandidate,
1132    disposition: DurableTailRecoveryDisposition,
1133    boundary_sequence: u64,
1134) -> DurableTailRecoveryOutcome {
1135    let PreparedRecoveryCandidate {
1136        recovered,
1137        document,
1138        ..
1139    } = candidate;
1140    // Drop the sealed carrier before unwrapping its shared typed Session.
1141    // Otherwise every successful recovery deep-clones the accumulated
1142    // document solely because its own completed commit carrier still holds an
1143    // Arc at outcome construction.
1144    drop(document);
1145    let recovered = Arc::try_unwrap(recovered).unwrap_or_else(|shared| shared.as_ref().clone());
1146    DurableTailRecoveryOutcome::Committed {
1147        disposition,
1148        boundary_sequence,
1149        recovered: Box::new(recovered),
1150    }
1151}
1152
1153/// Outcome of one authorization + commit attempt. Refusal and hold both
1154/// retain the durable tail intact; nothing here deletes anything.
1155#[derive(Debug)]
1156pub enum DurableTailRecoveryOutcome {
1157    /// The machine authorized the commit and the atomic boundary succeeded,
1158    /// with the machine-minted boundary sequence. The returned document is the
1159    /// exact internally built successor bound to the store authority the
1160    /// commit returned; callers do not rebuild it.
1161    Committed {
1162        disposition: DurableTailRecoveryDisposition,
1163        boundary_sequence: u64,
1164        recovered: Box<Session>,
1165    },
1166    /// The store-owned snapshot proved runtime authority and the physical
1167    /// canonical head were already byte-exactly aligned. This is benign
1168    /// convergence (typically another process won recovery before source
1169    /// loading), and the returned store-bound committed document is safe to
1170    /// serve directly.
1171    AlreadyAligned { recovered: Box<Session> },
1172    /// The candidate is held intact: ambiguous tail evidence, or input
1173    /// records whose identity cannot be proven durably. Autonomy stays
1174    /// blocked; the tail clears only through reconciliation.
1175    Held,
1176    /// The machine refused (non-quiescent persisted or in-process runtime,
1177    /// conflicting run facts, or durable receipts that already cover — or
1178    /// contradict — this candidate).
1179    Refused,
1180}
1181
1182/// Typed error: authorization/commit mechanics failed (as opposed to the
1183/// machine refusing, which is an [`DurableTailRecoveryOutcome`]).
1184#[derive(Debug, thiserror::Error)]
1185pub enum DurableTailRecoveryError {
1186    #[error("recovery authorization could not be driven: {0}")]
1187    Authority(String),
1188    /// Exact store authorities, physical rows, tail shape, or the internally
1189    /// built recovered successor failed validation.
1190    #[error("recovery evidence is invalid: {0}")]
1191    InvalidEvidence(String),
1192    #[error("recovery commit failed: {0}")]
1193    Store(#[from] RuntimeStoreError),
1194}
1195
1196/// Typed projections of the persisted machine-lifecycle row that recovery
1197/// observed, carried alongside the exact row version so the eventual commit
1198/// can fence on precisely the evidence the machine judged.
1199struct ObservedPersistedLifecycle {
1200    lifecycle: mm_dsl::DurableRecoveryObservedLifecycle,
1201    current_run: mm_dsl::DurableRecoveryObservedRun,
1202    expected_version: MachineLifecycleExpectedVersion,
1203    /// The lifecycle phase to re-assert on commit. Recovery never invents a
1204    /// phase: Missing rows create quiescent Idle; Idle stays Idle; Retired
1205    /// stays Retired (a retired runtime is not resurrected by recovering its
1206    /// session document).
1207    reassert_state: RuntimeState,
1208    binding: MachineLifecycleBindingFacts,
1209    supervisor_authority: SupervisorAuthoritySnapshot,
1210    unregister_progress: Option<crate::store::MachineUnregisterProgressSnapshot>,
1211}
1212
1213fn observe_persisted_lifecycle(
1214    observation: MachineLifecycleObservation,
1215    candidate_run_id: &meerkat_core::RunId,
1216) -> ObservedPersistedLifecycle {
1217    match observation {
1218        MachineLifecycleObservation::Missing => ObservedPersistedLifecycle {
1219            lifecycle: mm_dsl::DurableRecoveryObservedLifecycle::MissingRow,
1220            current_run: mm_dsl::DurableRecoveryObservedRun::NoRun,
1221            expected_version: MachineLifecycleExpectedVersion::Missing,
1222            reassert_state: RuntimeState::Idle,
1223            binding: MachineLifecycleBindingFacts::default(),
1224            supervisor_authority: SupervisorAuthoritySnapshot::UnboundNoReceipt,
1225            unregister_progress: None,
1226        },
1227        MachineLifecycleObservation::Decoded { record, version } => {
1228            let (lifecycle, reassert_state) = match record.runtime_state() {
1229                Some(RuntimeState::Idle) => (
1230                    mm_dsl::DurableRecoveryObservedLifecycle::Idle,
1231                    RuntimeState::Idle,
1232                ),
1233                Some(RuntimeState::Retired) => (
1234                    mm_dsl::DurableRecoveryObservedLifecycle::Retired,
1235                    RuntimeState::Retired,
1236                ),
1237                Some(_) => (
1238                    mm_dsl::DurableRecoveryObservedLifecycle::NonQuiescent,
1239                    RuntimeState::Idle,
1240                ),
1241                // A decoded row without a lifecycle phase is a torn shape;
1242                // fail closed as undecodable evidence.
1243                None => (
1244                    mm_dsl::DurableRecoveryObservedLifecycle::Undecodable,
1245                    RuntimeState::Idle,
1246                ),
1247            };
1248            let current_run = match record.run().current_run_id() {
1249                None => mm_dsl::DurableRecoveryObservedRun::NoRun,
1250                Some(run_id) if run_id == candidate_run_id => {
1251                    mm_dsl::DurableRecoveryObservedRun::CandidateRun
1252                }
1253                Some(_) => mm_dsl::DurableRecoveryObservedRun::OtherRun,
1254            };
1255            ObservedPersistedLifecycle {
1256                lifecycle,
1257                current_run,
1258                expected_version: MachineLifecycleExpectedVersion::Version(version),
1259                reassert_state,
1260                binding: record.binding().clone(),
1261                supervisor_authority: record.supervisor_authority().clone(),
1262                unregister_progress: record.unregister_progress().cloned(),
1263            }
1264        }
1265        MachineLifecycleObservation::Unsupported { version, .. }
1266        | MachineLifecycleObservation::Malformed { version, .. } => ObservedPersistedLifecycle {
1267            lifecycle: mm_dsl::DurableRecoveryObservedLifecycle::Undecodable,
1268            current_run: mm_dsl::DurableRecoveryObservedRun::OtherRun,
1269            expected_version: MachineLifecycleExpectedVersion::Version(version),
1270            reassert_state: RuntimeState::Idle,
1271            binding: MachineLifecycleBindingFacts::default(),
1272            supervisor_authority: SupervisorAuthoritySnapshot::UnboundNoReceipt,
1273            unregister_progress: None,
1274        },
1275    }
1276}
1277
1278/// Classify the highest durably committed boundary receipt for the candidate
1279/// run against the candidate transcript itself.
1280///
1281/// This is the only observation that can see a PRIOR SUCCESS of this same
1282/// recovery. The in-process `turn_terminal_run_id` is vacuous on cold recovery
1283/// (a freshly registered authority is driven), and receipt-key uniqueness
1284/// `(runtime_id, run_id, sequence)` fences only a SAME-sequence race: a second
1285/// process that observes the first recovery's receipt mints one past it and
1286/// would commit a phantom recovered boundary.
1287///
1288/// The message count carries the safety property on its own: a boundary
1289struct PriorCommitObservation {
1290    last_committed_sequence: u64,
1291    classification: mm_dsl::DurableRecoveryPriorCommit,
1292    receipt_bound_inputs: BTreeSet<String>,
1293}
1294
1295/// Materialize the exact receipts already proved by the store-owned source,
1296/// applying only the sealed one-time digest enrichments for 0.8.10 rows.
1297///
1298/// Pairing is by both sequence and exact original row token. A migration for
1299/// another row can never donate transcript ancestry to this candidate.
1300fn materialize_verified_receipts(
1301    sources: &[PreparedRecoveryReceiptSource],
1302    enrichments: &[PreparedRecoveryReceiptDigestEnrichment],
1303) -> Result<Vec<RunBoundaryReceipt>, DurableTailRecoveryError> {
1304    let mut by_sequence = BTreeMap::new();
1305    for enrichment in enrichments {
1306        if by_sequence
1307            .insert(enrichment.original_receipt().sequence, enrichment)
1308            .is_some()
1309        {
1310            return Err(DurableTailRecoveryError::InvalidEvidence(
1311                "store-owned receipt migration contains duplicate sequences".to_string(),
1312            ));
1313        }
1314    }
1315
1316    let mut materialized = Vec::with_capacity(sources.len());
1317    let mut consumed_enrichments = 0_usize;
1318    for source in sources {
1319        let receipt = source.receipt();
1320        if receipt.conversation_digest.is_some() {
1321            if by_sequence.contains_key(&receipt.sequence) {
1322                return Err(DurableTailRecoveryError::InvalidEvidence(format!(
1323                    "receipt sequence {} is already digest-bound but also carries a migration",
1324                    receipt.sequence
1325                )));
1326            }
1327            materialized.push(receipt.clone());
1328            continue;
1329        }
1330        let enrichment = by_sequence.get(&receipt.sequence).ok_or_else(|| {
1331            DurableTailRecoveryError::InvalidEvidence(format!(
1332                "digestless receipt sequence {} has no sealed store migration",
1333                receipt.sequence
1334            ))
1335        })?;
1336        if enrichment.original_receipt() != receipt
1337            || enrichment.original_exact_row_token() != source.exact_row_token()
1338        {
1339            return Err(DurableTailRecoveryError::InvalidEvidence(format!(
1340                "receipt migration sequence {} does not bind the exact original row",
1341                receipt.sequence
1342            )));
1343        }
1344        materialized.push(enrichment.enriched_receipt());
1345        consumed_enrichments += 1;
1346    }
1347    if consumed_enrichments != enrichments.len() {
1348        return Err(DurableTailRecoveryError::InvalidEvidence(
1349            "store-owned receipt migration contains a row absent from the exact receipt read"
1350                .to_string(),
1351        ));
1352    }
1353    Ok(materialized)
1354}
1355
1356fn prepare_receipt_digest_enrichments(
1357    candidate: &PreparedRecoveryCandidate,
1358    receipts: &[PreparedRecoveryReceiptSource],
1359) -> Result<Vec<PreparedRecoveryReceiptDigestEnrichment>, DurableTailRecoveryError> {
1360    let mut expected_sequence = 1_u64;
1361    let mut previous_message_count = 0_usize;
1362    let mut enrichments = Vec::new();
1363    for source in receipts {
1364        let receipt = source.receipt();
1365        if receipt.run_id != candidate.candidate_run_id {
1366            return Err(DurableTailRecoveryError::InvalidEvidence(
1367                "recovery receipt source contains a different run identity".to_string(),
1368            ));
1369        }
1370        if receipt.sequence != expected_sequence {
1371            return Err(DurableTailRecoveryError::InvalidEvidence(format!(
1372                "recovery receipt sequence {} is not the required dense sequence {expected_sequence}",
1373                receipt.sequence
1374            )));
1375        }
1376        if receipt.message_count < previous_message_count
1377            || receipt.message_count > candidate.recovered.messages().len()
1378        {
1379            return Err(DurableTailRecoveryError::InvalidEvidence(
1380                "recovery receipt message counts are not monotonic within the store-bound transcript"
1381                    .to_string(),
1382            ));
1383        }
1384        let derived_digest = candidate
1385            .recovered
1386            .transcript_prefix_digest(receipt.message_count)
1387            .map_err(|error| {
1388                DurableTailRecoveryError::InvalidEvidence(format!(
1389                    "failed to derive recovery receipt transcript prefix: {error}"
1390                ))
1391            })?;
1392        match receipt.conversation_digest.as_deref() {
1393            Some(existing) if existing != derived_digest => {
1394                return Err(DurableTailRecoveryError::InvalidEvidence(format!(
1395                    "receipt sequence {} digest differs from the store-bound transcript prefix",
1396                    receipt.sequence
1397                )));
1398            }
1399            Some(_) => {}
1400            None => enrichments.push(
1401                PreparedRecoveryReceiptDigestEnrichment::new(source, derived_digest)
1402                    .map_err(DurableTailRecoveryError::Store)?,
1403            ),
1404        }
1405        expected_sequence = expected_sequence.checked_add(1).ok_or_else(|| {
1406            DurableTailRecoveryError::InvalidEvidence(
1407                "recovery receipt sequence overflow".to_string(),
1408            )
1409        })?;
1410        previous_message_count = receipt.message_count;
1411    }
1412    Ok(enrichments)
1413}
1414
1415/// Verify every same-run receipt against the exact candidate transcript prefix
1416/// it claims. Message count alone is not ancestry, and a digest of some other
1417/// shorter conversation is not "precedes". Supported-floor digestless rows
1418/// reach this function only after [`materialize_verified_receipts`] has paired
1419/// them with a sealed store-owned enrichment.
1420fn observe_prior_commits(
1421    committed: &[RunBoundaryReceipt],
1422    candidate: &Session,
1423    candidate_run_id: &RunId,
1424) -> PriorCommitObservation {
1425    let last_committed_sequence = committed
1426        .iter()
1427        .map(|receipt| receipt.sequence)
1428        .max()
1429        .unwrap_or(0);
1430    if committed.is_empty() {
1431        return PriorCommitObservation {
1432            last_committed_sequence,
1433            classification: mm_dsl::DurableRecoveryPriorCommit::NoPriorCommit,
1434            receipt_bound_inputs: BTreeSet::new(),
1435        };
1436    }
1437
1438    let mut ordered = committed.iter().collect::<Vec<_>>();
1439    ordered.sort_by_key(|receipt| receipt.sequence);
1440    let mut previous: Option<&RunBoundaryReceipt> = None;
1441    let mut receipt_bound_inputs = BTreeSet::new();
1442    for receipt in &ordered {
1443        let expected_sequence = previous
1444            .and_then(|prior| prior.sequence.checked_add(1))
1445            .unwrap_or(1);
1446        let structurally_monotonic = receipt.sequence == expected_sequence
1447            && previous.is_none_or(|prior| receipt.message_count >= prior.message_count);
1448        let prefix_matches = receipt.run_id == *candidate_run_id
1449            && receipt.message_count <= candidate.messages().len()
1450            && receipt
1451                .conversation_digest
1452                .as_deref()
1453                .is_some_and(|digest| {
1454                    candidate
1455                        .transcript_prefix_digest(receipt.message_count)
1456                        .is_ok_and(|prefix| prefix == digest)
1457                });
1458        if !structurally_monotonic || !prefix_matches {
1459            return PriorCommitObservation {
1460                last_committed_sequence,
1461                classification: mm_dsl::DurableRecoveryPriorCommit::DivergesFromCandidate,
1462                // A divergent receipt cannot contribute input identity to
1463                // this candidate. The refusal path must never terminalize its
1464                // named rows even if later code is rearranged.
1465                receipt_bound_inputs: BTreeSet::new(),
1466            };
1467        }
1468        receipt_bound_inputs.extend(
1469            receipt
1470                .contributing_input_ids
1471                .iter()
1472                .map(ToString::to_string),
1473        );
1474        previous = Some(receipt);
1475    }
1476
1477    PriorCommitObservation {
1478        last_committed_sequence,
1479        classification: ordered.last().map_or(
1480            mm_dsl::DurableRecoveryPriorCommit::NoPriorCommit,
1481            |highest| {
1482                if highest.message_count == candidate.messages().len() {
1483                    mm_dsl::DurableRecoveryPriorCommit::MatchesCandidate
1484                } else {
1485                    mm_dsl::DurableRecoveryPriorCommit::PrecedesCandidate
1486                }
1487            },
1488        ),
1489        receipt_bound_inputs,
1490    }
1491}
1492
1493async fn load_recovery_preparation(
1494    store: &dyn RuntimeStore,
1495    runtime_id: &LogicalRuntimeId,
1496    session_id: &SessionId,
1497) -> Result<RecoveryCandidatePreparation, DurableTailRecoveryError> {
1498    match store.session_persistence_profile() {
1499        RuntimeSessionPersistenceProfile::HeadCanonicalV1 => {
1500            let source = match store.load_durable_tail_recovery_source(runtime_id).await {
1501                Ok(Some(source)) => source,
1502                Ok(None) => return Ok(RecoveryCandidatePreparation::Held),
1503                Err(RuntimeStoreError::PreparedRecoveryRequiresAtomicPhysicalHeadCas {
1504                    ..
1505                }) => {
1506                    return Ok(RecoveryCandidatePreparation::Held);
1507                }
1508                Err(error) => return Err(error.into()),
1509            };
1510            if source.runtime_authority().session_id() != session_id {
1511                return Err(DurableTailRecoveryError::InvalidEvidence(format!(
1512                    "store-owned recovery source belongs to session {}, not requested {session_id}",
1513                    source.runtime_authority().session_id()
1514                )));
1515            }
1516            prepare_head_canonical_recovery_candidate(&source)
1517        }
1518        RuntimeSessionPersistenceProfile::WholeBlobV1 => {
1519            let committed = store
1520                .load_committed_whole_blob_snapshot(runtime_id)
1521                .await?
1522                .ok_or_else(|| {
1523                    DurableTailRecoveryError::InvalidEvidence(
1524                        "WholeBlob recovery has no committed base snapshot".to_string(),
1525                    )
1526                })?;
1527            if committed.authority().session_id() != session_id {
1528                return Err(DurableTailRecoveryError::InvalidEvidence(format!(
1529                    "WholeBlob committed authority belongs to {}, not requested {session_id}",
1530                    committed.authority().session_id()
1531                )));
1532            }
1533            let provisional = store.load_whole_blob_provisional_tail(runtime_id).await?;
1534            prepare_whole_blob_recovery_candidate(committed, provisional)
1535        }
1536    }
1537}
1538
1539/// Prove, classify, authorize, and atomically commit one durable-tail recovery.
1540///
1541/// This is the only public preparation seam. The caller supplies only the
1542/// store and stable session identity. A capable store loads one opaque,
1543/// store-bound source from its own runtime-authority and physical-session
1544/// rows. The caller cannot supply or mix a committed document, physical head,
1545/// materialization, class, candidate/run identity, recovered snapshot,
1546/// store authority, receipt fact, or CAS token. Those are derived and sealed
1547/// internally before the generated machines are driven. Generated classifier
1548/// DTOs remain private observations, never public recovery capabilities.
1549///
1550/// Machine authorization judges persisted lifecycle, receipts, and
1551/// input-lifecycle rows. A freshly registered in-process authority alone
1552/// would be vacuously quiescent, so every durable observation is taken before
1553/// the drive; afterwards this module only realizes the emitted verdict.
1554pub async fn recover_durable_tail(
1555    store: &dyn RuntimeStore,
1556    session_id: &SessionId,
1557) -> Result<DurableTailRecoveryOutcome, DurableTailRecoveryError> {
1558    let runtime_id = LogicalRuntimeId::for_session(session_id);
1559    let mut preparation = load_recovery_preparation(store, &runtime_id, session_id).await?;
1560    let mut rolled_back_incomplete_intent = false;
1561    let candidate = loop {
1562        match preparation {
1563            RecoveryCandidatePreparation::Prepared(candidate) => break candidate,
1564            RecoveryCandidatePreparation::AlreadyAligned(recovered) => {
1565                tracing::info!(
1566                    %session_id,
1567                    "durable-tail recovery source is already exactly aligned"
1568                );
1569                let recovered =
1570                    Arc::try_unwrap(recovered).unwrap_or_else(|shared| shared.as_ref().clone());
1571                return Ok(DurableTailRecoveryOutcome::AlreadyAligned {
1572                    recovered: Box::new(recovered),
1573                });
1574            }
1575            RecoveryCandidatePreparation::IncompleteHeadCanonicalIntent { provisional } => {
1576                if rolled_back_incomplete_intent {
1577                    return Err(DurableTailRecoveryError::InvalidEvidence(
1578                        "HeadCanonical provisional rollback did not expose a stable physical authority"
1579                            .to_string(),
1580                    ));
1581                }
1582                if !store
1583                    .discard_head_canonical_provisional_tail(&runtime_id, &provisional)
1584                    .await?
1585                {
1586                    tracing::warn!(
1587                        %session_id,
1588                        "durable-tail recovery held: incomplete HeadCanonical intent changed before exact rollback"
1589                    );
1590                    return Ok(DurableTailRecoveryOutcome::Held);
1591                }
1592                tracing::info!(
1593                    %session_id,
1594                    physical_revision = provisional.physical_store_revision(),
1595                    "rolled back incomplete HeadCanonical provisional intent"
1596                );
1597                rolled_back_incomplete_intent = true;
1598                preparation = load_recovery_preparation(store, &runtime_id, session_id).await?;
1599            }
1600            RecoveryCandidatePreparation::Held => return Ok(DurableTailRecoveryOutcome::Held),
1601        }
1602    };
1603
1604    let observed = match store.observe_machine_lifecycle(&runtime_id).await {
1605        Ok(observation) => observe_persisted_lifecycle(observation, &candidate.candidate_run_id),
1606        // A store that cannot observe its lifecycle row cannot prove
1607        // quiescence; the machine refuses undecodable evidence.
1608        Err(RuntimeStoreError::Unsupported(_)) => ObservedPersistedLifecycle {
1609            lifecycle: mm_dsl::DurableRecoveryObservedLifecycle::Undecodable,
1610            current_run: mm_dsl::DurableRecoveryObservedRun::OtherRun,
1611            expected_version: MachineLifecycleExpectedVersion::Missing,
1612            reassert_state: RuntimeState::Idle,
1613            binding: MachineLifecycleBindingFacts::default(),
1614            supervisor_authority: SupervisorAuthoritySnapshot::UnboundNoReceipt,
1615            unregister_progress: None,
1616        },
1617        Err(error) => return Err(error.into()),
1618    };
1619    // Re-assert the observed quiescent lifecycle (never a new phase), fenced
1620    // on the exact row version the machine judged. The expected version is a
1621    // first-apply CAS fence; the target record itself is the durable outcome
1622    // identity used by exact retry convergence.
1623    let lifecycle = MachineLifecycleCommit::new_with_binding_run_and_unregister_progress(
1624        observed.reassert_state,
1625        observed.binding.clone(),
1626        MachineLifecycleRunFacts::default(),
1627        observed.supervisor_authority.clone(),
1628        observed.unregister_progress.clone(),
1629    )
1630    .with_expected_version(observed.expected_version.clone());
1631
1632    let prior_recovery = match load_exact_committed_recovery(
1633        store,
1634        &runtime_id,
1635        &candidate,
1636        &lifecycle,
1637    )
1638    .await
1639    {
1640        Ok(boundary) => boundary,
1641        Err(DurableTailRecoveryError::Store(
1642            RuntimeStoreError::PreparedRecoveryRequiresAtomicPhysicalHeadCas { profile },
1643        )) => {
1644            tracing::warn!(
1645                session_id = %candidate.session_id,
1646                %profile,
1647                "durable-tail recovery held: store cannot prove an exact atomic recovery boundary"
1648            );
1649            return Ok(DurableTailRecoveryOutcome::Held);
1650        }
1651        Err(error) => return Err(error),
1652    };
1653    if let Some((disposition, boundary_sequence)) = prior_recovery {
1654        tracing::info!(
1655            session_id = %candidate.session_id,
1656            candidate_run_id = %candidate.candidate_run_id,
1657            ?disposition,
1658            boundary_sequence,
1659            "durable-tail recovery converged on an exact committed witness"
1660        );
1661        return Ok(committed_recovery_outcome(
1662            candidate,
1663            disposition,
1664            boundary_sequence,
1665        ));
1666    }
1667
1668    // Durably committed receipts for the candidate run: an interrupted tool
1669    // loop can have committed BoundaryContinue receipts before losing only
1670    // its final boundary. They carry (a) the last committed sequence the
1671    // machine mints past, (b) exact contributing input identities, and (c)
1672    // whether this exact recovery already landed.
1673    let committed_receipt_sources = match store
1674        .load_durable_tail_recovery_receipts(&runtime_id, &candidate.candidate_run_id)
1675        .await
1676    {
1677        Ok(receipts) => receipts,
1678        Err(RuntimeStoreError::PreparedRecoveryRequiresAtomicPhysicalHeadCas { profile }) => {
1679            tracing::warn!(
1680                session_id = %candidate.session_id,
1681                %profile,
1682                "durable-tail recovery held: store cannot load exact receipt rows"
1683            );
1684            return Ok(DurableTailRecoveryOutcome::Held);
1685        }
1686        Err(error) => return Err(error.into()),
1687    };
1688    let receipt_digest_enrichments =
1689        prepare_receipt_digest_enrichments(&candidate, &committed_receipt_sources)?;
1690    let committed_receipts =
1691        materialize_verified_receipts(&committed_receipt_sources, &receipt_digest_enrichments)?;
1692    let prior = observe_prior_commits(
1693        &committed_receipts,
1694        candidate.recovered.as_ref(),
1695        &candidate.candidate_run_id,
1696    );
1697
1698    let inputs = observe_candidate_run_inputs(
1699        store,
1700        &runtime_id,
1701        &candidate.candidate_run_id,
1702        &prior.receipt_bound_inputs,
1703    )
1704    .await?;
1705
1706    let mut authority =
1707        crate::meerkat_machine::dsl_authority::new_registered_authority(&candidate.session_id)
1708            .map_err(|error| DurableTailRecoveryError::Authority(error.to_string()))?;
1709    let transition = mm_dsl::MeerkatMachineMutator::apply(
1710        &mut authority,
1711        mm_dsl::MeerkatMachineInput::AuthorizeDurableTailRecovery {
1712            session_id: mm_dsl::SessionId::from_domain(&candidate.session_id),
1713            candidate_id: candidate.candidate_id.clone(),
1714            candidate_run_id: mm_dsl::RunId(candidate.candidate_run_id.to_string()),
1715            class: candidate.class,
1716            observed_lifecycle: observed.lifecycle,
1717            observed_current_run: observed.current_run,
1718            last_committed_sequence: prior.last_committed_sequence,
1719            prior_commit: prior.classification,
1720            input_evidence: inputs.evidence,
1721        },
1722    )
1723    .map_err(|error| DurableTailRecoveryError::Authority(error.to_string()))?;
1724
1725    let mut commit_verdict: Option<(DurableTailRecoveryDisposition, u64)> = None;
1726    let mut non_commit_verdict: Option<DurableTailRecoveryDisposition> = None;
1727    for effect in transition.effects() {
1728        match effect {
1729            mm_dsl::MeerkatMachineEffect::DurableTailRecoveryCommitAuthorized {
1730                candidate_id,
1731                disposition,
1732                boundary_sequence,
1733            } if *candidate_id == candidate.candidate_id => {
1734                commit_verdict = Some((*disposition, *boundary_sequence));
1735            }
1736            mm_dsl::MeerkatMachineEffect::DurableTailRecoveryAuthorized {
1737                candidate_id,
1738                disposition,
1739            } if *candidate_id == candidate.candidate_id => {
1740                non_commit_verdict = Some(*disposition);
1741            }
1742            _ => {}
1743        }
1744    }
1745
1746    let (disposition, boundary_sequence) = match (commit_verdict, non_commit_verdict) {
1747        (Some((disposition, sequence)), _) => (disposition, sequence),
1748        (
1749            None,
1750            Some(
1751                non_commit @ (DurableTailRecoveryDisposition::HoldIntact
1752                | DurableTailRecoveryDisposition::RefuseRecovery),
1753            ),
1754        ) => {
1755            // A competing process can commit after our initial witness read
1756            // but before receipt/input observation. Never collapse that race
1757            // into a hold/refusal: re-read the durable recovery boundary and
1758            // converge only after reconstructing its exact source, recovered
1759            // store authority, receipt, lifecycle target, and migration witness.
1760            if let Some((disposition, boundary_sequence)) =
1761                load_exact_committed_recovery(store, &runtime_id, &candidate, &lifecycle).await?
1762            {
1763                tracing::info!(
1764                    session_id = %candidate.session_id,
1765                    candidate_run_id = %candidate.candidate_run_id,
1766                    ?disposition,
1767                    boundary_sequence,
1768                    "durable-tail recovery converged after a competing exact commit"
1769                );
1770                return Ok(committed_recovery_outcome(
1771                    candidate,
1772                    disposition,
1773                    boundary_sequence,
1774                ));
1775            }
1776            match non_commit {
1777                DurableTailRecoveryDisposition::HoldIntact => {
1778                    tracing::warn!(
1779                        session_id = %candidate.session_id,
1780                        candidate_run_id = %candidate.candidate_run_id,
1781                        class = ?candidate.class,
1782                        prior_commit = ?prior.classification,
1783                        input_evidence = ?inputs.evidence,
1784                        "durable-tail recovery held intact by machine verdict"
1785                    );
1786                    return Ok(DurableTailRecoveryOutcome::Held);
1787                }
1788                DurableTailRecoveryDisposition::RefuseRecovery => {
1789                    return Ok(DurableTailRecoveryOutcome::Refused);
1790                }
1791                other => {
1792                    return Err(DurableTailRecoveryError::Authority(format!(
1793                        "generated machine emitted unexpected non-commit disposition {other:?}"
1794                    )));
1795                }
1796            }
1797        }
1798        (None, Some(other)) => {
1799            return Err(DurableTailRecoveryError::Authority(format!(
1800                "generated machine emitted commit disposition {other:?} without a commit \
1801                 authorization effect"
1802            )));
1803        }
1804        (None, None) => {
1805            return Err(DurableTailRecoveryError::Authority(
1806                "generated machine returned no recovery disposition for the exact candidate"
1807                    .to_string(),
1808            ));
1809        }
1810    };
1811
1812    // Realize-only pass: terminalize exactly the rows the observation PROVED
1813    // bound to the candidate run — durable staging bindings or a committed
1814    // boundary receipt naming them. Never more.
1815    //
1816    // This holds for the retain-inputs disposition too, and that is the point
1817    // of the word "retain": the unbound rows that set its evidence class are
1818    // retained for ordinary redelivery, while rows the same scan proved
1819    // consumed by the adopted tail are closed out. Clearing the attribution
1820    // wholesale here (as this once did) does not make the pass safer — it
1821    // strands proven-consumed rows non-terminal, and redelivery then
1822    // re-executes a turn the boundary just committed. Attribution is the
1823    // safety property; it is enforced by the observation, which only ever
1824    // attributes a row on durable run-binding evidence.
1825    let CandidateInputObservation {
1826        evidence: _,
1827        predecessor_nonterminal_input_set_revision,
1828        predecessor_nonterminal_input_set_token,
1829        attributed,
1830    } = inputs;
1831    let predecessor_nonterminal_input_set_revision = predecessor_nonterminal_input_set_revision
1832        .ok_or_else(|| {
1833            DurableTailRecoveryError::Authority(
1834                "machine authorized recovery without an exact input-set revision".to_string(),
1835            )
1836        })?;
1837    let predecessor_nonterminal_input_set_token = predecessor_nonterminal_input_set_token
1838        .ok_or_else(|| {
1839            DurableTailRecoveryError::Authority(
1840                "machine authorized recovery without an exact nonterminal input-set witness"
1841                    .to_string(),
1842            )
1843        })?;
1844    let mut input_updates =
1845        terminalize_attributed_inputs(attributed, &candidate.candidate_run_id, boundary_sequence)?;
1846    let contributing_input_ids: Vec<InputId> = input_updates
1847        .iter()
1848        .map(|record| record.as_stored().state.input_id.clone())
1849        .collect();
1850    // Retained rows are deliberately not rewritten. The store-owned input-set
1851    // revision fences every mutation since observation, while the exact set
1852    // token records the classified identities and bytes in durable evidence.
1853    // Re-upserting every retained row would turn one recovered delta back into
1854    // O(current outstanding work) writes without strengthening the proof.
1855    input_updates.sort_by_key(|record| record.as_stored().state.input_id.to_string());
1856
1857    let receipt = RunBoundaryReceipt {
1858        run_id: candidate.candidate_run_id.clone(),
1859        // The recovered boundary applies at commit time; no live run exists
1860        // to carry a checkpoint position.
1861        boundary: RunApplyBoundary::Immediate,
1862        contributing_input_ids,
1863        conversation_digest: Some(candidate.conversation_digest.clone()),
1864        message_count: candidate.message_count,
1865        // Machine-minted: one past the last durably committed receipt for
1866        // this run. The (runtime_id, run_id, sequence) key fences only a
1867        // SAME-sequence race; a recovery that already landed is fenced by the
1868        // machine's prior-commit guard, not by this key.
1869        sequence: boundary_sequence,
1870    };
1871    let document = bind_recovered_store_document(&candidate)?;
1872    let evidence = seal_recovery_evidence(
1873        &candidate,
1874        document.as_ref(),
1875        disposition,
1876        receipt_digest_enrichments,
1877        predecessor_nonterminal_input_set_revision,
1878        predecessor_nonterminal_input_set_token,
1879        input_updates,
1880        &receipt,
1881        &lifecycle,
1882    )?;
1883    let request = match &candidate.store_transition {
1884        PreparedRecoveryStoreTransition::WholeBlob { .. } => {
1885            PreparedRuntimeSessionCommit::machine_terminal_whole_blob_recovery(
1886                document,
1887                evidence,
1888                receipt,
1889                lifecycle,
1890                candidate.session_id.clone(),
1891            )
1892        }
1893        PreparedRecoveryStoreTransition::HeadCanonical { .. } => {
1894            let document = document.ok_or_else(|| {
1895                DurableTailRecoveryError::Authority(
1896                    "HeadCanonical recovery lost its prepared mutation before commit".to_string(),
1897                )
1898            })?;
1899            PreparedRuntimeSessionCommit::machine_terminal_recovery(
1900                document,
1901                evidence,
1902                receipt,
1903                lifecycle,
1904                candidate.session_id.clone(),
1905            )
1906        }
1907    }?;
1908    let committed = match store
1909        .commit_prepared_session_boundary(&runtime_id, request)
1910        .await
1911    {
1912        Ok(result) => result,
1913        Err(RuntimeStoreError::PreparedRecoveryRequiresAtomicPhysicalHeadCas { profile }) => {
1914            tracing::warn!(
1915                session_id = %candidate.session_id,
1916                %profile,
1917                "durable-tail recovery held: store lacks atomic physical-head CAS"
1918            );
1919            return Ok(DurableTailRecoveryOutcome::Held);
1920        }
1921        Err(error) => return Err(error.into()),
1922    };
1923    let status = committed.recovery_status().ok_or_else(|| {
1924        DurableTailRecoveryError::Authority(
1925            "prepared recovery commit returned no exact recovery status".to_string(),
1926        )
1927    })?;
1928    let committed_authority = committed.authority().ok_or_else(|| {
1929        DurableTailRecoveryError::Authority(
1930            "prepared recovery commit returned no successor session authority".to_string(),
1931        )
1932    })?;
1933    let successor_matches = match &candidate.store_transition {
1934        PreparedRecoveryStoreTransition::WholeBlob {
1935            base_store_revision,
1936            recovered_blob_sha256,
1937            ..
1938        } => committed_authority.whole_blob().is_some_and(|authority| {
1939            authority.session_id() == &candidate.session_id
1940                && base_store_revision
1941                    .checked_add(1)
1942                    .is_some_and(|expected| authority.store_revision() == expected)
1943                && authority.blob_sha256() == recovered_blob_sha256
1944        }),
1945        PreparedRecoveryStoreTransition::HeadCanonical {
1946            physical_store_revision,
1947            recovered_head_token,
1948            ..
1949        } => committed_authority
1950            .head_canonical()
1951            .is_some_and(|authority| {
1952                authority.session_id() == &candidate.session_id
1953                    && physical_store_revision
1954                        .checked_add(1)
1955                        .is_some_and(|expected| authority.store_revision() == expected)
1956                    && authority.committed_head_token() == recovered_head_token
1957            }),
1958    };
1959    if !successor_matches {
1960        return Err(DurableTailRecoveryError::Authority(
1961            "prepared recovery commit returned a different successor authority".to_string(),
1962        ));
1963    }
1964    if committed.downstream_projection_required() {
1965        return Err(DurableTailRecoveryError::Authority(
1966            "store-owned recovery unexpectedly requested a SessionStore projection".to_string(),
1967        ));
1968    }
1969    tracing::info!(
1970        session_id = %candidate.session_id,
1971        candidate_run_id = %candidate.candidate_run_id,
1972        ?disposition,
1973        ?status,
1974        boundary_sequence,
1975        message_count = candidate.message_count,
1976        "durable-tail recovery committed as a recovered runtime boundary"
1977    );
1978    Ok(committed_recovery_outcome(
1979        candidate,
1980        disposition,
1981        boundary_sequence,
1982    ))
1983}
1984
1985/// What the input-lifecycle rows say about the candidate run, plus the rows a
1986/// commit would terminalize. The evidence class is the machine's input; the
1987/// attributed rows are realized only after a commit verdict.
1988struct CandidateInputObservation {
1989    evidence: mm_dsl::DurableRecoveryInputEvidence,
1990    /// Store-owned revision observed in the same backend snapshot as the
1991    /// classified rows. First apply compares it in O(1) inside the write
1992    /// transaction, including for the empty-set absence case.
1993    predecessor_nonterminal_input_set_revision: Option<RecoveryInputSetRevision>,
1994    /// Exact store-owned set/absence token over every nonterminal row that
1995    /// could affect this classification. `None` means the store could not
1996    /// provide an atomically enforceable complete-set witness.
1997    predecessor_nonterminal_input_set_token: Option<String>,
1998    /// Non-terminal rows durable identity attributed to the candidate run,
1999    /// each paired with the row digest the commit fences on. Populated
2000    /// whenever the rows could be scanned — including under
2001    /// `UnboundContentInput`, where the retain-inputs commit still
2002    /// terminalizes exactly these proven-bound rows.
2003    attributed: Vec<(StoredInputState, String)>,
2004}
2005
2006fn is_terminal(phase: InputLifecycleState) -> bool {
2007    matches!(
2008        phase,
2009        InputLifecycleState::Consumed
2010            | InputLifecycleState::Superseded
2011            | InputLifecycleState::Coalesced
2012            | InputLifecycleState::Abandoned
2013    )
2014}
2015
2016/// Does this input carry redeliverable content? A content input the delivery
2017/// layer would re-run duplicates the recovered turn if it was actually the
2018/// tail's input; a non-content input (operation, continuation, external
2019/// event) does not re-execute turn content.
2020fn carries_redeliverable_content(input: Option<&crate::input::Input>) -> bool {
2021    matches!(
2022        input,
2023        Some(
2024            crate::input::Input::Prompt(_)
2025                | crate::input::Input::FlowStep(_)
2026                | crate::input::Input::Peer(_)
2027        )
2028    )
2029}
2030
2031/// Observation pass: classify the input-lifecycle evidence the machine judges,
2032/// on durable identity only.
2033///
2034/// Attribution classes:
2035/// 1. Records the persisted machine facts already bound to the candidate run
2036///    (`seed.last_run_id`).
2037/// 2. Records a durably committed receipt for the candidate run names in its
2038///    `contributing_input_ids`.
2039///
2040/// Anything else that is non-terminal and carries redeliverable content might
2041/// be the tail's own input whose binding never became durable — text equality
2042/// is content evidence, never identity — and is reported as
2043/// `UnboundContentInput`. A store that cannot version input rows while
2044/// blocking rows exist is reported as `Unfenceable`. The DISPOSITION for both
2045/// belongs to the machine.
2046async fn observe_candidate_run_inputs(
2047    store: &dyn RuntimeStore,
2048    runtime_id: &LogicalRuntimeId,
2049    candidate_run_id: &meerkat_core::RunId,
2050    receipt_bound_inputs: &BTreeSet<String>,
2051) -> Result<CandidateInputObservation, DurableTailRecoveryError> {
2052    let snapshot = match store.load_input_states_with_versions(runtime_id).await {
2053        Ok(snapshot) => snapshot,
2054        // An unversioned row scan cannot be carried into the atomic recovery
2055        // commit: even an empty result has no range/absence fence and can race
2056        // a durable input insertion. Unsupported therefore proves no
2057        // observation capability, never "this store has no input state".
2058        // A future no-input store needs an explicit sealed capability.
2059        Err(RuntimeStoreError::Unsupported(_)) => {
2060            return Ok(CandidateInputObservation {
2061                evidence: mm_dsl::DurableRecoveryInputEvidence::Unfenceable,
2062                predecessor_nonterminal_input_set_revision: None,
2063                predecessor_nonterminal_input_set_token: None,
2064                attributed: Vec::new(),
2065            });
2066        }
2067        Err(error) => return Err(error.into()),
2068    };
2069    if snapshot.runtime_id() != runtime_id {
2070        return Err(DurableTailRecoveryError::Authority(format!(
2071            "store returned a recovery input snapshot for {}, not {runtime_id}",
2072            snapshot.runtime_id()
2073        )));
2074    }
2075    let (rows, predecessor_nonterminal_input_set_revision, predecessor_nonterminal_input_set_token) =
2076        snapshot.into_parts();
2077
2078    // Scan EVERY row before deciding. An unbound content row sets the
2079    // evidence class, but it must not erase the rows this same scan proved
2080    // bound to the candidate run: those were consumed by the very tail being
2081    // adopted (durable staging bindings, or a committed boundary receipt
2082    // naming them). Returning early with an empty attribution — as this did
2083    // — strands them non-terminal, and the input lifecycle then rolls
2084    // Staged back to Queued and re-admits them, re-executing an
2085    // already-committed turn: a duplicate provider call with re-fired tool
2086    // side effects. Proven-bound rows are terminalized; only genuinely
2087    // unbound rows are retained for redelivery.
2088    //
2089    let mut attributed = Vec::new();
2090    let mut unbound_content_input = false;
2091    for (bundle, row_digest) in rows {
2092        if is_terminal(bundle.seed.phase) {
2093            continue;
2094        }
2095        let bound_to_candidate = bundle.seed.last_run_id.as_ref() == Some(candidate_run_id)
2096            || receipt_bound_inputs.contains(&bundle.state.input_id.to_string());
2097        if !bound_to_candidate {
2098            if carries_redeliverable_content(bundle.state.persisted_input.as_ref()) {
2099                unbound_content_input = true;
2100            }
2101            continue;
2102        }
2103        attributed.push((bundle, row_digest));
2104    }
2105    Ok(CandidateInputObservation {
2106        evidence: if unbound_content_input {
2107            mm_dsl::DurableRecoveryInputEvidence::UnboundContentInput
2108        } else {
2109            mm_dsl::DurableRecoveryInputEvidence::AllBoundOrInert
2110        },
2111        predecessor_nonterminal_input_set_revision: Some(
2112            predecessor_nonterminal_input_set_revision,
2113        ),
2114        predecessor_nonterminal_input_set_token: Some(predecessor_nonterminal_input_set_token),
2115        attributed,
2116    })
2117}
2118
2119/// Realize pass: terminalize the observed rows the recovered boundary
2120/// consumed, fenced on the exact row bytes the observation read. Called only
2121/// after a commit verdict, and only over rows the observation attributed.
2122fn terminalize_attributed_inputs(
2123    attributed: Vec<(StoredInputState, String)>,
2124    candidate_run_id: &meerkat_core::RunId,
2125    boundary_sequence: u64,
2126) -> Result<Vec<InputStatePersistenceRecord>, DurableTailRecoveryError> {
2127    let mut updates = Vec::with_capacity(attributed.len());
2128    for (mut bundle, row_digest) in attributed {
2129        bundle.seed.phase = InputLifecycleState::Consumed;
2130        bundle.seed.terminal_outcome = Some(crate::input_state::InputTerminalOutcome::Consumed);
2131        // Terminal seeds carry no recovery lane — the generated authority
2132        // refuses a Consumed seed that still claims one.
2133        bundle.seed.recovery_lane = None;
2134        bundle.seed.last_run_id = Some(candidate_run_id.clone());
2135        bundle.seed.last_boundary_sequence = Some(boundary_sequence);
2136        // The same fenced recovery transaction installs the recovered
2137        // boundary receipt and this terminal row. Once attributed, ordinary
2138        // content is neither redeliverable nor needed to identify a future
2139        // durable tail. A directed input without its terminal outbox is kept
2140        // fail-closed because its payload still owns the Interaction identity.
2141        if crate::store::input_state_payload_is_retirable(&bundle) {
2142            bundle.state.persisted_input = None;
2143        }
2144        let record = InputStatePersistenceRecord::from_machine_snapshot(bundle)
2145            .map_err(DurableTailRecoveryError::Authority)?
2146            .with_expected_row_digest(row_digest);
2147        updates.push(record);
2148    }
2149    Ok(updates)
2150}
2151
2152#[cfg(test)]
2153mod store_authority_tests {
2154    use super::*;
2155    use crate::store::{
2156        CommittedWholeBlobProvisionalTail, CommittedWholeBlobSnapshot, WholeBlobStoreAuthority,
2157    };
2158    use meerkat_core::WholeBlobProvisionalTailAuthority;
2159    use meerkat_core::types::{
2160        AssistantBlock, BlockAssistantMessage, TranscriptMessageIdentity, UserMessage,
2161        message_timestamp_now,
2162    };
2163
2164    fn observation(run_id: &RunId) -> DurableTailObservation {
2165        DurableTailObservation {
2166            tail_run_id: Some(run_id.clone()),
2167            run_id_cardinality: RunIdCardinality::SingleRunId,
2168            terminal_stop_reason: DurableTailStopReason::EndTurn,
2169            dangling_tool_use_ids: Vec::new(),
2170            orphan_tool_result_count: 0,
2171            messages_after_terminal: false,
2172        }
2173    }
2174
2175    #[test]
2176    fn whole_blob_candidate_identity_binds_exact_store_base_and_run() {
2177        let session_id = SessionId::new();
2178        let first_run = RunId::new();
2179        let second_run = RunId::new();
2180        let first = exact_whole_blob_candidate_id(
2181            &session_id,
2182            7,
2183            "row-sha256:base",
2184            "row-sha256:candidate",
2185            1,
2186            &first_run,
2187            &observation(&first_run),
2188        );
2189        let different_run = exact_whole_blob_candidate_id(
2190            &session_id,
2191            7,
2192            "row-sha256:base",
2193            "row-sha256:candidate",
2194            1,
2195            &second_run,
2196            &observation(&second_run),
2197        );
2198        let different_base = exact_whole_blob_candidate_id(
2199            &session_id,
2200            8,
2201            "row-sha256:other-base",
2202            "row-sha256:candidate",
2203            1,
2204            &first_run,
2205            &observation(&first_run),
2206        );
2207        let different_sequence = exact_whole_blob_candidate_id(
2208            &session_id,
2209            7,
2210            "row-sha256:base",
2211            "row-sha256:candidate",
2212            2,
2213            &first_run,
2214            &observation(&first_run),
2215        );
2216        assert_ne!(first, different_run);
2217        assert_ne!(first, different_base);
2218        assert_ne!(first, different_sequence);
2219    }
2220
2221    fn whole_blob_recovery_fixture(
2222        authority_run: RunId,
2223        transcript_run: RunId,
2224    ) -> (
2225        CommittedWholeBlobSnapshot,
2226        CommittedWholeBlobProvisionalTail,
2227    ) {
2228        let mut committed = Session::new();
2229        committed.push(Message::User(UserMessage::text("committed input")));
2230        let session_id = committed.id().clone();
2231        let committed_bytes = Arc::new(committed.to_persisted_bytes().unwrap());
2232        let committed_sha = format!("row-sha256:{:x}", Sha256::digest(committed_bytes.as_ref()));
2233        let committed_authority =
2234            WholeBlobStoreAuthority::issued(session_id.clone(), 7, committed_sha.clone()).unwrap();
2235
2236        let mut candidate = committed;
2237        candidate.push(Message::BlockAssistant(BlockAssistantMessage {
2238            blocks: vec![AssistantBlock::Text {
2239                text: "durable reply".to_string(),
2240                meta: None,
2241            }],
2242            stop_reason: StopReason::EndTurn,
2243            identity: TranscriptMessageIdentity::default().with_run_id(transcript_run),
2244            created_at: message_timestamp_now(),
2245        }));
2246        let candidate_bytes = Arc::new(candidate.to_persisted_bytes().unwrap());
2247        let candidate_sha = format!("row-sha256:{:x}", Sha256::digest(candidate_bytes.as_ref()));
2248        let provisional_authority = WholeBlobProvisionalTailAuthority::issued(
2249            session_id,
2250            7,
2251            committed_sha,
2252            authority_run,
2253            candidate_sha,
2254            1,
2255        )
2256        .unwrap();
2257        (
2258            CommittedWholeBlobSnapshot::new(committed_bytes, committed_authority).unwrap(),
2259            CommittedWholeBlobProvisionalTail::new(provisional_authority, candidate_bytes),
2260        )
2261    }
2262
2263    #[test]
2264    fn whole_blob_provisional_recovery_uses_store_authority_without_session_stamp() {
2265        let run_id = RunId::new();
2266        let (committed, provisional) = whole_blob_recovery_fixture(run_id.clone(), run_id.clone());
2267
2268        let RecoveryCandidatePreparation::Prepared(candidate) =
2269            prepare_whole_blob_recovery_candidate(committed, Some(provisional))
2270                .expect("store-issued WholeBlob tail prepares")
2271        else {
2272            panic!("valid WholeBlob provisional tail must prepare");
2273        };
2274        assert_eq!(candidate.candidate_run_id, run_id);
2275        assert_eq!(
2276            candidate.class,
2277            DurableTailRecoveryClass::CompletedCandidate
2278        );
2279        assert!(
2280            candidate.document.is_none(),
2281            "completed WholeBlob recovery is metadata-only and must not materialize a successor"
2282        );
2283        assert!(matches!(
2284            candidate.store_transition,
2285            PreparedRecoveryStoreTransition::WholeBlob {
2286                base_store_revision: 7,
2287                ..
2288            }
2289        ));
2290    }
2291
2292    #[test]
2293    fn whole_blob_provisional_recovery_refuses_wrong_run_authority() {
2294        let (committed, provisional) = whole_blob_recovery_fixture(RunId::new(), RunId::new());
2295
2296        assert!(matches!(
2297            prepare_whole_blob_recovery_candidate(committed, Some(provisional)),
2298            Err(DurableTailRecoveryError::InvalidEvidence(detail))
2299                if detail.contains("store-issued run identity")
2300        ));
2301    }
2302}