use std::collections::BTreeSet;
use crate::identifiers::LogicalRuntimeId;
use crate::input_state::{InputLifecycleState, InputStatePersistenceRecord, StoredInputState};
use crate::meerkat_machine::dsl as mm_dsl;
use crate::runtime_state::RuntimeState;
use crate::store::{
MachineLifecycleBindingFacts, MachineLifecycleCommit, MachineLifecycleExpectedVersion,
MachineLifecycleObservation, MachineLifecycleRunFacts, RuntimeStore, RuntimeStoreError,
SessionDelta, SupervisorAuthoritySnapshot,
};
use meerkat_core::lifecycle::InputId;
use meerkat_core::lifecycle::run_primitive::RunApplyBoundary;
use meerkat_core::lifecycle::run_receipt::RunBoundaryReceipt;
use meerkat_core::types::SessionId;
pub use mm_dsl::{DurableTailRecoveryClass, DurableTailRecoveryDisposition};
#[derive(Debug)]
pub struct DurableTailRecoveryRequest {
session_id: SessionId,
candidate_id: String,
candidate_run_id: meerkat_core::RunId,
class: DurableTailRecoveryClass,
recovered_snapshot: Vec<u8>,
conversation_digest: String,
message_count: usize,
}
impl DurableTailRecoveryRequest {
pub fn from_classification(
verdict: &meerkat_core::session_document::SessionDocumentEffect,
session_id: SessionId,
candidate_run_id: meerkat_core::RunId,
recovered_snapshot: Vec<u8>,
conversation_digest: String,
message_count: usize,
) -> Result<Self, DurableTailRecoveryError> {
use meerkat_core::session_document::{
DurableTailRecoveryClass as ClassifiedClass, SessionDocumentEffect,
};
let SessionDocumentEffect::DurableTailClassified {
candidate_id,
class,
} = verdict
else {
return Err(DurableTailRecoveryError::Authority(
"durable-tail recovery requires the SessionDocumentMachine's \
DurableTailClassified verdict"
.to_string(),
));
};
let class = match class {
ClassifiedClass::CompletedCandidate => DurableTailRecoveryClass::CompletedCandidate,
ClassifiedClass::InterruptedRepairableCandidate => {
DurableTailRecoveryClass::InterruptedRepairableCandidate
}
ClassifiedClass::Ambiguous => DurableTailRecoveryClass::Ambiguous,
};
Ok(Self {
session_id,
candidate_id: candidate_id.clone(),
candidate_run_id,
class,
recovered_snapshot,
conversation_digest,
message_count,
})
}
pub fn session_id(&self) -> &SessionId {
&self.session_id
}
pub fn candidate_id(&self) -> &str {
&self.candidate_id
}
pub fn candidate_run_id(&self) -> &meerkat_core::RunId {
&self.candidate_run_id
}
pub fn class(&self) -> DurableTailRecoveryClass {
self.class
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum DurableTailRecoveryOutcome {
Committed {
disposition: DurableTailRecoveryDisposition,
boundary_sequence: u64,
},
Held,
Refused,
}
#[derive(Debug, thiserror::Error)]
pub enum DurableTailRecoveryError {
#[error("recovery authorization could not be driven: {0}")]
Authority(String),
#[error("recovery commit failed: {0}")]
Store(#[from] RuntimeStoreError),
}
struct ObservedPersistedLifecycle {
lifecycle: mm_dsl::DurableRecoveryObservedLifecycle,
current_run: mm_dsl::DurableRecoveryObservedRun,
expected_version: MachineLifecycleExpectedVersion,
reassert_state: RuntimeState,
binding: MachineLifecycleBindingFacts,
supervisor_authority: SupervisorAuthoritySnapshot,
unregister_progress: Option<crate::store::MachineUnregisterProgressSnapshot>,
}
fn observe_persisted_lifecycle(
observation: MachineLifecycleObservation,
candidate_run_id: &meerkat_core::RunId,
) -> ObservedPersistedLifecycle {
match observation {
MachineLifecycleObservation::Missing => ObservedPersistedLifecycle {
lifecycle: mm_dsl::DurableRecoveryObservedLifecycle::MissingRow,
current_run: mm_dsl::DurableRecoveryObservedRun::NoRun,
expected_version: MachineLifecycleExpectedVersion::Missing,
reassert_state: RuntimeState::Idle,
binding: MachineLifecycleBindingFacts::default(),
supervisor_authority: SupervisorAuthoritySnapshot::UnboundNoReceipt,
unregister_progress: None,
},
MachineLifecycleObservation::Decoded { record, version } => {
let (lifecycle, reassert_state) = match record.runtime_state() {
Some(RuntimeState::Idle) => (
mm_dsl::DurableRecoveryObservedLifecycle::Idle,
RuntimeState::Idle,
),
Some(RuntimeState::Retired) => (
mm_dsl::DurableRecoveryObservedLifecycle::Retired,
RuntimeState::Retired,
),
Some(_) => (
mm_dsl::DurableRecoveryObservedLifecycle::NonQuiescent,
RuntimeState::Idle,
),
None => (
mm_dsl::DurableRecoveryObservedLifecycle::Undecodable,
RuntimeState::Idle,
),
};
let current_run = match record.run().current_run_id() {
None => mm_dsl::DurableRecoveryObservedRun::NoRun,
Some(run_id) if run_id == candidate_run_id => {
mm_dsl::DurableRecoveryObservedRun::CandidateRun
}
Some(_) => mm_dsl::DurableRecoveryObservedRun::OtherRun,
};
ObservedPersistedLifecycle {
lifecycle,
current_run,
expected_version: MachineLifecycleExpectedVersion::Version(version),
reassert_state,
binding: record.binding().clone(),
supervisor_authority: record.supervisor_authority().clone(),
unregister_progress: record.unregister_progress().cloned(),
}
}
MachineLifecycleObservation::Unsupported { version, .. }
| MachineLifecycleObservation::Malformed { version, .. } => ObservedPersistedLifecycle {
lifecycle: mm_dsl::DurableRecoveryObservedLifecycle::Undecodable,
current_run: mm_dsl::DurableRecoveryObservedRun::OtherRun,
expected_version: MachineLifecycleExpectedVersion::Version(version),
reassert_state: RuntimeState::Idle,
binding: MachineLifecycleBindingFacts::default(),
supervisor_authority: SupervisorAuthoritySnapshot::UnboundNoReceipt,
unregister_progress: None,
},
}
}
fn classify_prior_commit(
highest_committed: Option<&RunBoundaryReceipt>,
conversation_digest: &str,
message_count: usize,
) -> mm_dsl::DurableRecoveryPriorCommit {
let Some(highest) = highest_committed else {
return mm_dsl::DurableRecoveryPriorCommit::NoPriorCommit;
};
match highest.conversation_digest.as_deref() {
Some(digest) if digest == conversation_digest && highest.message_count == message_count => {
mm_dsl::DurableRecoveryPriorCommit::MatchesCandidate
}
Some(digest) if digest != conversation_digest && highest.message_count < message_count => {
mm_dsl::DurableRecoveryPriorCommit::PrecedesCandidate
}
Some(_) => mm_dsl::DurableRecoveryPriorCommit::DivergesFromCandidate,
None if highest.message_count < message_count => {
mm_dsl::DurableRecoveryPriorCommit::PrecedesCandidate
}
None if highest.message_count == message_count => {
mm_dsl::DurableRecoveryPriorCommit::MatchesCandidate
}
None => mm_dsl::DurableRecoveryPriorCommit::DivergesFromCandidate,
}
}
pub async fn authorize_and_commit_durable_tail_recovery(
store: &dyn RuntimeStore,
request: DurableTailRecoveryRequest,
) -> Result<DurableTailRecoveryOutcome, DurableTailRecoveryError> {
let runtime_id = LogicalRuntimeId::for_session(&request.session_id);
let observed = match store.observe_machine_lifecycle(&runtime_id).await {
Ok(observation) => observe_persisted_lifecycle(observation, &request.candidate_run_id),
Err(RuntimeStoreError::Unsupported(_)) => ObservedPersistedLifecycle {
lifecycle: mm_dsl::DurableRecoveryObservedLifecycle::Undecodable,
current_run: mm_dsl::DurableRecoveryObservedRun::OtherRun,
expected_version: MachineLifecycleExpectedVersion::Missing,
reassert_state: RuntimeState::Idle,
binding: MachineLifecycleBindingFacts::default(),
supervisor_authority: SupervisorAuthoritySnapshot::UnboundNoReceipt,
unregister_progress: None,
},
Err(error) => return Err(error.into()),
};
let committed_receipts = store
.load_committed_boundary_receipts(&runtime_id, &request.candidate_run_id)
.await?;
let highest_committed = committed_receipts
.iter()
.max_by_key(|receipt| receipt.sequence);
let last_committed_sequence = highest_committed
.map(|receipt| receipt.sequence)
.unwrap_or(0);
let prior_commit = classify_prior_commit(
highest_committed,
&request.conversation_digest,
request.message_count,
);
let receipt_bound_inputs: BTreeSet<String> = committed_receipts
.iter()
.flat_map(|receipt| {
receipt
.contributing_input_ids
.iter()
.map(|input_id| input_id.to_string())
})
.collect();
let inputs = observe_candidate_run_inputs(
store,
&runtime_id,
&request.candidate_run_id,
&receipt_bound_inputs,
)
.await?;
let mut authority =
crate::meerkat_machine::dsl_authority::new_registered_authority(&request.session_id)
.map_err(|error| DurableTailRecoveryError::Authority(error.to_string()))?;
let transition = mm_dsl::MeerkatMachineMutator::apply(
&mut authority,
mm_dsl::MeerkatMachineInput::AuthorizeDurableTailRecovery {
session_id: mm_dsl::SessionId::from_domain(&request.session_id),
candidate_id: request.candidate_id.clone(),
candidate_run_id: mm_dsl::RunId(request.candidate_run_id.to_string()),
class: request.class,
observed_lifecycle: observed.lifecycle,
observed_current_run: observed.current_run,
last_committed_sequence,
prior_commit,
input_evidence: inputs.evidence,
},
)
.map_err(|error| DurableTailRecoveryError::Authority(error.to_string()))?;
let mut commit_verdict: Option<(DurableTailRecoveryDisposition, u64)> = None;
let mut non_commit_verdict: Option<DurableTailRecoveryDisposition> = None;
for effect in transition.effects() {
match effect {
mm_dsl::MeerkatMachineEffect::DurableTailRecoveryCommitAuthorized {
candidate_id,
disposition,
boundary_sequence,
} if *candidate_id == request.candidate_id => {
commit_verdict = Some((*disposition, *boundary_sequence));
}
mm_dsl::MeerkatMachineEffect::DurableTailRecoveryAuthorized {
candidate_id,
disposition,
} if *candidate_id == request.candidate_id => {
non_commit_verdict = Some(*disposition);
}
_ => {}
}
}
let (disposition, boundary_sequence) = match (commit_verdict, non_commit_verdict) {
(Some((disposition, sequence)), _) => (disposition, sequence),
(None, Some(DurableTailRecoveryDisposition::HoldIntact)) => {
tracing::warn!(
session_id = %request.session_id,
candidate_run_id = %request.candidate_run_id,
class = ?request.class,
?prior_commit,
input_evidence = ?inputs.evidence,
"durable-tail recovery held intact by machine verdict"
);
return Ok(DurableTailRecoveryOutcome::Held);
}
(None, Some(DurableTailRecoveryDisposition::RefuseRecovery)) => {
return Ok(DurableTailRecoveryOutcome::Refused);
}
(None, Some(other)) => {
return Err(DurableTailRecoveryError::Authority(format!(
"generated machine emitted commit disposition {other:?} without a commit \
authorization effect"
)));
}
(None, None) => {
return Err(DurableTailRecoveryError::Authority(
"generated machine returned no recovery disposition for the exact candidate"
.to_string(),
));
}
};
let input_updates = terminalize_attributed_inputs(
inputs.attributed,
&request.candidate_run_id,
boundary_sequence,
)?;
let contributing_input_ids: Vec<InputId> = input_updates
.iter()
.map(|record| record.as_stored().state.input_id.clone())
.collect();
let receipt = RunBoundaryReceipt {
run_id: request.candidate_run_id.clone(),
boundary: RunApplyBoundary::Immediate,
contributing_input_ids,
conversation_digest: Some(request.conversation_digest.clone()),
message_count: request.message_count,
sequence: boundary_sequence,
};
let lifecycle = MachineLifecycleCommit::new_with_binding_run_and_unregister_progress(
observed.reassert_state,
observed.binding,
MachineLifecycleRunFacts::default(),
observed.supervisor_authority,
observed.unregister_progress,
)
.with_expected_version(observed.expected_version);
store
.atomic_apply_with_machine_lifecycle(
&runtime_id,
SessionDelta {
session_snapshot: request.recovered_snapshot,
},
receipt,
lifecycle,
input_updates,
request.session_id.clone(),
)
.await?;
tracing::info!(
session_id = %request.session_id,
candidate_run_id = %request.candidate_run_id,
?disposition,
boundary_sequence,
message_count = request.message_count,
"durable-tail recovery committed as a recovered runtime boundary"
);
Ok(DurableTailRecoveryOutcome::Committed {
disposition,
boundary_sequence,
})
}
struct CandidateInputObservation {
evidence: mm_dsl::DurableRecoveryInputEvidence,
attributed: Vec<(StoredInputState, String)>,
}
fn is_terminal(phase: InputLifecycleState) -> bool {
matches!(
phase,
InputLifecycleState::Consumed
| InputLifecycleState::Superseded
| InputLifecycleState::Coalesced
| InputLifecycleState::Abandoned
)
}
fn carries_redeliverable_content(input: Option<&crate::input::Input>) -> bool {
matches!(
input,
Some(
crate::input::Input::Prompt(_)
| crate::input::Input::FlowStep(_)
| crate::input::Input::Peer(_)
)
)
}
async fn observe_candidate_run_inputs(
store: &dyn RuntimeStore,
runtime_id: &LogicalRuntimeId,
candidate_run_id: &meerkat_core::RunId,
receipt_bound_inputs: &BTreeSet<String>,
) -> Result<CandidateInputObservation, DurableTailRecoveryError> {
let rows = match store.load_input_states_with_versions(runtime_id).await {
Ok(rows) => rows,
Err(RuntimeStoreError::Unsupported(_)) => {
return match store.load_input_states_strict(runtime_id).await {
Ok(rows) => {
let blocking = rows.iter().any(|bundle| {
!is_terminal(bundle.seed.phase)
&& (bundle.seed.last_run_id.as_ref() == Some(candidate_run_id)
|| carries_redeliverable_content(
bundle.state.persisted_input.as_ref(),
))
});
Ok(CandidateInputObservation {
evidence: if blocking {
mm_dsl::DurableRecoveryInputEvidence::Unfenceable
} else {
mm_dsl::DurableRecoveryInputEvidence::AllBoundOrInert
},
attributed: Vec::new(),
})
}
Err(RuntimeStoreError::Unsupported(_)) => Ok(CandidateInputObservation {
evidence: mm_dsl::DurableRecoveryInputEvidence::AllBoundOrInert,
attributed: Vec::new(),
}),
Err(error) => Err(error.into()),
};
}
Err(error) => return Err(error.into()),
};
let mut attributed = Vec::new();
for (bundle, row_digest) in rows {
if is_terminal(bundle.seed.phase) {
continue;
}
let bound_to_candidate = bundle.seed.last_run_id.as_ref() == Some(candidate_run_id)
|| receipt_bound_inputs.contains(&bundle.state.input_id.to_string());
if !bound_to_candidate {
if carries_redeliverable_content(bundle.state.persisted_input.as_ref()) {
return Ok(CandidateInputObservation {
evidence: mm_dsl::DurableRecoveryInputEvidence::UnboundContentInput,
attributed: Vec::new(),
});
}
continue;
}
attributed.push((bundle, row_digest));
}
Ok(CandidateInputObservation {
evidence: mm_dsl::DurableRecoveryInputEvidence::AllBoundOrInert,
attributed,
})
}
fn terminalize_attributed_inputs(
attributed: Vec<(StoredInputState, String)>,
candidate_run_id: &meerkat_core::RunId,
boundary_sequence: u64,
) -> Result<Vec<InputStatePersistenceRecord>, DurableTailRecoveryError> {
let mut updates = Vec::with_capacity(attributed.len());
for (mut bundle, row_digest) in attributed {
bundle.seed.phase = InputLifecycleState::Consumed;
bundle.seed.terminal_outcome = Some(crate::input_state::InputTerminalOutcome::Consumed);
bundle.seed.recovery_lane = None;
bundle.seed.last_run_id = Some(candidate_run_id.clone());
bundle.seed.last_boundary_sequence = Some(boundary_sequence);
let record = InputStatePersistenceRecord::from_machine_snapshot(bundle)
.map_err(DurableTailRecoveryError::Authority)?
.with_expected_row_digest(row_digest);
updates.push(record);
}
Ok(updates)
}