#![cfg_attr(test, allow(dead_code))]
use async_trait::async_trait;
use futures::StreamExt;
use meerkat_core::BlobStore;
#[allow(unused_imports)] use meerkat_core::Session;
use meerkat_core::error::AgentError;
use meerkat_core::image_content::{
MAX_REALTIME_USER_IMAGE_PROJECTION_BYTES, externalize_deferred_turn_state,
externalize_messages_from,
};
use meerkat_core::lifecycle::core_executor::{
BoundSessionCommit, CoreApplyOutput, CoreApplyTerminal, CoreBoundaryStageOutput,
CoreExecutorError, CoreExecutorPublicationHandle, CoreInteractionTerminalPublicationReceipt,
};
use meerkat_core::lifecycle::run_primitive::{RunApplyBoundary, TurnRequestContext};
use meerkat_core::lifecycle::run_receipt::RunBoundaryReceiptDraft;
use meerkat_core::service::{
AppendSystemContextRequest, AppendSystemContextResult, CreateSessionRequest, InitialTurnPolicy,
MobToolAuthorityContext, SessionControlError, SessionError, SessionForkAtRequest,
SessionForkReplaceRequest, SessionForkResult, SessionHistoryPage, SessionHistoryQuery,
SessionInfo, SessionQuery, SessionService, SessionServiceCommsExt, SessionServiceControlExt,
SessionServiceHistoryExt, SessionServiceTranscriptEditExt, SessionSummary,
SessionTranscriptRestoreRevisionRequest, SessionTranscriptRevisionList,
SessionTranscriptRevisionListEntry, SessionTranscriptRevisionListQuery,
SessionTranscriptRevisionPage, SessionTranscriptRevisionQuery, SessionTranscriptRewriteRequest,
SessionTranscriptRewriteResult, SessionUsage, SessionView, StageToolResultsDisposition,
StageToolResultsRequest, StageToolResultsResult, StartTurnRequest,
};
use meerkat_core::session_document::{
LiveSessionAuthorityKind, LiveSessionAuthorityReason, SessionArchiveDisposition,
SessionArchiveRuntimeObservation, SessionDocumentEffect, SessionDocumentKey,
SessionDocumentMachineAuthority, TranscriptEditKind,
};
use meerkat_core::session_store::{IncrementalSessionStore, SessionHead, session_head_cas_token};
use meerkat_core::types::{RunResult, SessionId, ToolResult};
use meerkat_core::{DeferredFirstTurnPhase, SessionDeferredTurnState, SessionLifecycleTerminal};
use meerkat_core::{InputId, RunCheckpointAuthority, RunCheckpointReceipt, RunId};
use meerkat_runtime::identifiers::LogicalRuntimeId;
#[cfg(test)]
use meerkat_runtime::input_state::{
InputLifecycleState, InputStatePersistenceRecord, InputTerminalOutcome, StoredInputState,
};
use meerkat_runtime::store::{
CommittedWholeBlobSnapshot, PreparedWholeBlobProvisionalTail, SerializedSessionSnapshot,
};
use meerkat_runtime::{
ArchivedSessionActorMaterializationAuthorization, HeadCanonicalProvisionalTailAuthority,
HeadCanonicalStoreAuthority, MachineSessionControlAuthority, MeerkatMachine,
PreparedArchivedResumeCommitLease, PreparedHeadCanonicalProvisionalTail,
PreparedWholeBlobRewriteBoundary, RuntimeSessionAuthority, RuntimeSessionPersistenceProfile,
RuntimeState, RuntimeStore, VerifiedCommittedWholeBlobPayload, WholeBlobStoreAuthority,
};
use meerkat_store::{SessionFilter, SessionStore, SessionStoreError};
use std::collections::{BTreeMap, HashMap, HashSet};
use std::sync::Arc;
use tokio::sync::{Mutex, OwnedMutexGuard, mpsc, watch};
use crate::SESSION_LABELS_KEY;
#[cfg(test)]
use crate::ephemeral::SessionTranscriptAuthoritySnapshot;
use crate::ephemeral::{
EphemeralSessionService, HeadCanonicalRuntimeBoundaryAuthority, LiveSessionActorWitness,
LiveSessionActorWitnessSlot, LiveSessionTranscriptAuthoritySnapshot, SessionAgentBuilder,
};
use crate::event_store::{
EventStore, TranscriptRewriteAuditExpectation, TranscriptRewriteAuditRead,
TranscriptRewritePrefixReceipt,
};
use crate::projector::SessionProjector;
fn runtime_driver_error_to_session_error(err: meerkat_runtime::RuntimeDriverError) -> SessionError {
SessionError::Agent(AgentError::InternalError(err.to_string()))
}
fn durable_session_restore_error(
id: &SessionId,
context: &str,
error: impl std::fmt::Display,
) -> SessionError {
SessionError::Store(Box::new(SessionStoreError::Serialization(format!(
"{context} for session {id}: {error}"
))))
}
fn session_id_from_event(event: &meerkat_core::event::AgentEvent) -> Option<SessionId> {
match event {
meerkat_core::event::AgentEvent::RunStarted { session_id, .. }
| meerkat_core::event::AgentEvent::RunCompleted { session_id, .. }
| meerkat_core::event::AgentEvent::ExtractionSucceeded { session_id, .. }
| meerkat_core::event::AgentEvent::ExtractionFailed { session_id, .. }
| meerkat_core::event::AgentEvent::RunFailed { session_id, .. }
| meerkat_core::event::AgentEvent::TranscriptRewriteCommitted { session_id, .. }
| meerkat_core::event::AgentEvent::TranscriptRewriteAuditReceiptCommitted {
session_id,
..
} => Some(session_id.clone()),
_ => None,
}
}
type EventProjectionFaultRegistry = Arc<Mutex<HashMap<SessionId, Arc<SessionError>>>>;
type EventProjectionGateRegistry = Arc<Mutex<HashMap<SessionId, Arc<Mutex<()>>>>>;
#[derive(Debug, Clone, PartialEq, Eq)]
enum EventProjectionDrainState {
Running,
Drained,
Faulted(String),
}
type EventProjectionDrainRegistry =
Arc<Mutex<HashMap<SessionId, Vec<watch::Receiver<EventProjectionDrainState>>>>>;
async fn register_event_projection_drain(
registry: &EventProjectionDrainRegistry,
session_id: &SessionId,
drain: watch::Receiver<EventProjectionDrainState>,
) {
let mut registry = registry.lock().await;
let drains = registry.entry(session_id.clone()).or_default();
drains.retain(|existing| !matches!(&*existing.borrow(), EventProjectionDrainState::Drained));
if !drains.iter().any(|existing| existing.same_channel(&drain)) {
drains.push(drain);
}
}
struct CreateTimeEventProjection {
session_tx: watch::Sender<Option<SessionId>>,
drain_rx: watch::Receiver<EventProjectionDrainState>,
}
struct CreateTimeEventProjectionLifecycle {
projection_faults: EventProjectionFaultRegistry,
projection_gates: EventProjectionGateRegistry,
drain_registration: Option<(
EventProjectionDrainRegistry,
watch::Receiver<EventProjectionDrainState>,
)>,
}
#[derive(Debug, thiserror::Error)]
#[error("event projection halted for session {session_id}: durable event append failed")]
pub struct EventProjectionHalted {
session_id: SessionId,
#[source]
cause: Arc<SessionError>,
}
#[derive(Debug, thiserror::Error)]
#[error("durable event projection halt marker for session {session_id}: {reason}")]
pub struct DurableEventProjectionHaltMarker {
session_id: SessionId,
reason: String,
}
#[derive(Debug, thiserror::Error)]
#[error(
"session event projection lagged for session {session_id}: {dropped} canonical events were dropped"
)]
pub struct SessionEventProjectionLagged {
session_id: SessionId,
dropped: u64,
}
fn event_projection_halted_error(session_id: &SessionId, cause: Arc<SessionError>) -> SessionError {
SessionError::Store(Box::new(EventProjectionHalted {
session_id: session_id.clone(),
cause,
}))
}
async fn latch_event_projection_fault(
faults: &EventProjectionFaultRegistry,
session_id: &SessionId,
error: SessionError,
) -> Arc<SessionError> {
let mut faults = faults.lock().await;
Arc::clone(
faults
.entry(session_id.clone())
.or_insert_with(|| Arc::new(error)),
)
}
async fn event_projection_gate(
gates: &EventProjectionGateRegistry,
session_id: &SessionId,
) -> Arc<Mutex<()>> {
Arc::clone(
gates
.lock()
.await
.entry(session_id.clone())
.or_insert_with(|| Arc::new(Mutex::new(()))),
)
}
async fn event_projection_admission_fault(
faults: &EventProjectionFaultRegistry,
gates: &EventProjectionGateRegistry,
event_store: &Arc<dyn EventStore>,
session_id: &SessionId,
) -> Option<Arc<SessionError>> {
if let Some(cause) = faults.lock().await.get(session_id).cloned() {
return Some(cause);
}
let gate = event_projection_gate(gates, session_id).await;
let _guard = gate.lock().await;
if let Some(cause) = faults.lock().await.get(session_id).cloned() {
return Some(cause);
}
match event_store.projection_halt(session_id).await {
Ok(Some(marker)) => {
let cause = SessionError::Store(Box::new(DurableEventProjectionHaltMarker {
session_id: marker.session_id,
reason: marker.reason,
}));
Some(latch_event_projection_fault(faults, session_id, cause).await)
}
Ok(None) => faults.lock().await.get(session_id).cloned(),
Err(error) => {
tracing::error!(
session_id = %session_id,
error = %error,
"event projection admission failed closed: durable halt marker unreadable"
);
Some(
latch_event_projection_fault(
faults,
session_id,
SessionError::Store(Box::new(error)),
)
.await,
)
}
}
}
async fn record_event_projection_fault_under_gate(
faults: &EventProjectionFaultRegistry,
event_store: &Arc<dyn EventStore>,
session_id: &SessionId,
error: SessionError,
) -> Arc<SessionError> {
let cause = latch_event_projection_fault(faults, session_id, error).await;
if let Err(marker_error) = event_store
.record_projection_halt(session_id, &cause.to_string())
.await
{
tracing::error!(
session_id = %session_id,
error = %marker_error,
"failed to persist durable event projection halt marker"
);
}
cause
}
async fn record_event_projection_fault(
faults: &EventProjectionFaultRegistry,
gates: &EventProjectionGateRegistry,
event_store: &Arc<dyn EventStore>,
session_id: &SessionId,
error: SessionError,
) {
let gate = event_projection_gate(gates, session_id).await;
let _guard = gate.lock().await;
record_event_projection_fault_under_gate(faults, event_store, session_id, error).await;
}
async fn append_and_project_event(
event_store: &Arc<dyn EventStore>,
projector: &Arc<SessionProjector>,
session_id: &SessionId,
envelope: meerkat_core::event::EventEnvelope<meerkat_core::event::AgentEvent>,
projection_faults: &EventProjectionFaultRegistry,
projection_gates: &EventProjectionGateRegistry,
) -> Result<u64, SessionError> {
let envelopes = [envelope];
let seq = {
let gate = event_projection_gate(projection_gates, session_id).await;
let _guard = gate.lock().await;
if let Some(cause) = projection_faults.lock().await.get(session_id).cloned() {
return Err(event_projection_halted_error(session_id, cause));
}
match event_store.append_envelopes(session_id, &envelopes).await {
Ok(seq) => seq,
Err(error) => {
let cause = record_event_projection_fault_under_gate(
projection_faults,
event_store,
session_id,
SessionError::Store(Box::new(error)),
)
.await;
return Err(event_projection_halted_error(session_id, cause));
}
}
};
if let Err(error) = projector
.project(event_store.as_ref(), session_id, seq)
.await
{
tracing::warn!(
session_id = %session_id,
degraded_projection = true,
error = %error,
"derived .rkat/ projection degraded; canonical event log is intact and replay will regenerate it"
);
}
Ok(seq)
}
async fn project_session_event_stream(
event_store: Arc<dyn EventStore>,
projector: Arc<SessionProjector>,
session_id: SessionId,
mut stream: meerkat_core::comms::EventStream,
projection_faults: EventProjectionFaultRegistry,
projection_gates: EventProjectionGateRegistry,
) {
if let Some(cause) = event_projection_admission_fault(
&projection_faults,
&projection_gates,
&event_store,
&session_id,
)
.await
{
tracing::error!(
session_id = %session_id,
error = %cause,
"event projection rejected before subscription drain: session is terminally halted"
);
return;
}
while let Some(envelope) = stream.next().await {
if let Some(cause) = projection_faults.lock().await.get(&session_id).cloned() {
tracing::error!(
session_id = %session_id,
error = %cause,
"event projection stopped before append: session acquired a terminal halt"
);
break;
}
if let meerkat_core::event::AgentEvent::StreamTruncated {
reason: meerkat_core::event::StreamTruncationReason::StreamLagged { dropped },
} = &envelope.payload
{
let error = SessionError::Store(Box::new(SessionEventProjectionLagged {
session_id: session_id.clone(),
dropped: *dropped,
}));
tracing::error!(
session_id = %session_id,
dropped = *dropped,
"event projection halted: canonical live event subscription lagged"
);
record_event_projection_fault(
&projection_faults,
&projection_gates,
&event_store,
&session_id,
error,
)
.await;
break;
}
if let Err(error) = append_and_project_event(
&event_store,
&projector,
&session_id,
envelope,
&projection_faults,
&projection_gates,
)
.await
{
tracing::error!(
session_id = %session_id,
error = %error,
"event projection halted: durable event append failed"
);
break;
}
}
}
async fn flush_projected_events(
event_store: &Arc<dyn EventStore>,
projector: &Arc<SessionProjector>,
session_id: &SessionId,
pending: &mut Vec<meerkat_core::event::EventEnvelope<meerkat_core::event::AgentEvent>>,
projection_faults: &EventProjectionFaultRegistry,
projection_gates: &EventProjectionGateRegistry,
) -> Result<(), SessionError> {
for envelope in pending.drain(..) {
append_and_project_event(
event_store,
projector,
session_id,
envelope,
projection_faults,
projection_gates,
)
.await?;
}
Ok(())
}
async fn project_create_time_events(
event_store: Arc<dyn EventStore>,
projector: Arc<SessionProjector>,
mut projection_rx: mpsc::Receiver<
meerkat_core::event::EventEnvelope<meerkat_core::event::AgentEvent>,
>,
mut session_rx: watch::Receiver<Option<SessionId>>,
caller_event_tx: Option<
mpsc::Sender<meerkat_core::event::EventEnvelope<meerkat_core::event::AgentEvent>>,
>,
lifecycle: CreateTimeEventProjectionLifecycle,
) {
let CreateTimeEventProjectionLifecycle {
projection_faults,
projection_gates,
drain_registration,
} = lifecycle;
let mut session_id = session_rx.borrow().clone();
let mut projection_admitted = false;
let mut pending = Vec::new();
if let (Some(session_id), Some((registry, drain))) =
(session_id.as_ref(), drain_registration.as_ref())
{
register_event_projection_drain(registry, session_id, drain.clone()).await;
}
loop {
tokio::select! {
envelope = projection_rx.recv() => {
let Some(envelope) = envelope else {
break;
};
if session_id.is_none() {
session_id = session_id_from_event(&envelope.payload);
}
if let (Some(session_id), Some((registry, drain))) =
(session_id.as_ref(), drain_registration.as_ref())
{
register_event_projection_drain(registry, session_id, drain.clone()).await;
}
if let Some(session_id) = session_id.as_ref()
&& !projection_admitted
{
if let Some(cause) = event_projection_admission_fault(
&projection_faults,
&projection_gates,
&event_store,
session_id,
)
.await
{
tracing::error!(
session_id = %session_id,
error = %cause,
"create-time event projection rejected before first append: session is terminally halted"
);
return;
}
projection_admitted = true;
}
let durable_envelope = envelope.clone();
if let Some(tx) = caller_event_tx.as_ref()
&& tx.send(envelope).await.is_err()
{
tracing::warn!("session event stream receiver dropped; continuing event projection");
}
if let Some(session_id) = session_id.as_ref() {
if let Some(cause) = projection_faults.lock().await.get(session_id).cloned() {
tracing::error!(
session_id = %session_id,
error = %cause,
"create-time event projection stopped before append: session acquired a terminal halt"
);
return;
}
pending.push(durable_envelope);
if let Err(error) = flush_projected_events(
&event_store,
&projector,
session_id,
&mut pending,
&projection_faults,
&projection_gates,
)
.await
{
tracing::error!(
session_id = %session_id,
error = %error,
"create-time event projection halted: durable event append failed"
);
return;
}
} else {
pending.push(durable_envelope);
}
}
changed = session_rx.changed(), if session_id.is_none() => {
if changed.is_err() {
break;
}
session_id = session_rx.borrow().clone();
if let Some(session_id) = session_id.as_ref() {
if let Some((registry, drain)) = drain_registration.as_ref() {
register_event_projection_drain(registry, session_id, drain.clone()).await;
}
if !projection_admitted {
if let Some(cause) = event_projection_admission_fault(
&projection_faults,
&projection_gates,
&event_store,
session_id,
)
.await
{
tracing::error!(
session_id = %session_id,
error = %cause,
"create-time event projection rejected before pending flush: session is terminally halted"
);
return;
}
projection_admitted = true;
}
if let Some(cause) = projection_faults.lock().await.get(session_id).cloned() {
tracing::error!(
session_id = %session_id,
error = %cause,
"create-time event projection stopped before pending flush: session acquired a terminal halt"
);
return;
}
if let Err(error) = flush_projected_events(
&event_store,
&projector,
session_id,
&mut pending,
&projection_faults,
&projection_gates,
)
.await
{
tracing::error!(
session_id = %session_id,
error = %error,
"create-time event projection halted: durable event append failed"
);
return;
}
}
}
}
}
if let Some(session_id) = session_id.as_ref()
&& let Some(cause) = projection_faults.lock().await.get(session_id).cloned()
{
tracing::error!(
session_id = %session_id,
error = %cause,
"create-time event projection skipped final flush: session acquired a terminal halt"
);
return;
}
if let Some(session_id) = session_id.as_ref()
&& projection_admitted
&& let Err(error) = flush_projected_events(
&event_store,
&projector,
session_id,
&mut pending,
&projection_faults,
&projection_gates,
)
.await
{
tracing::error!(
session_id = %session_id,
error = %error,
"final create-time event projection flush failed: durable event append failed"
);
}
}
fn write_deferred_turn_state(
session: &mut Session,
state: SessionDeferredTurnState,
) -> Result<(), SessionControlError> {
session.set_deferred_turn_state(state).map_err(|err| {
SessionControlError::Session(SessionError::Agent(
meerkat_core::error::AgentError::InternalError(format!(
"failed to serialize deferred-turn state: {err}"
)),
))
})
}
#[cfg(test)]
fn rollback_tool_visibility_state_snapshot(
session: &Session,
) -> Result<Option<meerkat_core::SessionToolVisibilityState>, SessionError> {
session.try_tool_visibility_state().map_err(|err| {
SessionError::Agent(AgentError::InternalError(format!(
"invalid canonical tool visibility state: {err}"
)))
})
}
fn is_durable_session_sync_unsupported(err: &SessionError) -> bool {
matches!(
err,
SessionError::Agent(AgentError::DurableSnapshotSyncUnsupported)
)
}
fn validate_tool_result_video(results: &[ToolResult]) -> Result<(), SessionError> {
if results.iter().any(ToolResult::has_video) {
return Err(SessionError::Agent(AgentError::ConfigError(
"video blocks are not supported in tool results".to_string(),
)));
}
Ok(())
}
struct CheckpointerGate {
cancelled: Mutex<bool>,
}
struct StoreCheckpointer {
runtime_store: Arc<dyn RuntimeStore>,
incremental: Option<Arc<dyn IncrementalSessionStore>>,
blob_store: Arc<dyn BlobStore>,
gate: Arc<CheckpointerGate>,
whole_blob_base_authority: std::sync::Mutex<Option<WholeBlobStoreAuthority>>,
latest_run_checkpoint_receipt: Mutex<Option<RunCheckpointReceipt>>,
}
struct HeadCanonicalProvisionalIntent<'a> {
committed: &'a HeadCanonicalStoreAuthority,
run_id: &'a RunId,
previous: Option<&'a meerkat_core::HeadCanonicalProvisionalTailAuthority>,
candidate_session: &'a Session,
successor_head: &'a SessionHead,
successor_head_token: &'a str,
}
async fn load_committed_whole_blob_session(
runtime_store: &dyn RuntimeStore,
session_id: &SessionId,
role: &str,
) -> Result<Option<(Session, CommittedWholeBlobSnapshot)>, SessionError> {
let snapshot = runtime_store
.load_committed_whole_blob_snapshot(&LogicalRuntimeId::for_session(session_id))
.await
.map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"failed to load {role} for session {session_id}: {error}"
)))
})?;
snapshot
.map(|snapshot| {
if snapshot.authority().session_id() != session_id
|| snapshot.session().id() != session_id
{
return Err(SessionError::Agent(AgentError::InternalError(format!(
"{role} WholeBlob snapshot identifies session {}/{}, not {session_id}",
snapshot.authority().session_id(),
snapshot.session().id()
))));
}
Ok((snapshot.session().clone(), snapshot))
})
.transpose()
}
struct BoundaryPersistencePlan {
commits: Vec<meerkat_core::TranscriptRewriteCommit>,
}
fn serialize_session_document(
session: &Session,
role: &str,
) -> Result<meerkat_core::SerializedSessionArtifact, SessionError> {
session.to_persisted_artifact().map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"failed to serialize {role} for session {}: {error}",
session.id()
)))
})
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
pub enum CommittedBoundaryRecovery {
AlreadyCommitted,
Recovered {
message_count: usize,
},
Unprovable {
reason: String,
},
}
fn rewrite_materialize_trace_enabled() -> bool {
static ENABLED: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
*ENABLED.get_or_init(|| std::env::var_os("MEERKAT_TRACE_REWRITE_MATERIALIZE").is_some())
}
struct TranscriptRewriteReplay {
session: Option<Session>,
materialized_from_replay: bool,
audit: Option<TranscriptRewriteAuditCoverage>,
}
enum TranscriptRewriteAuditCoverage {
AuthorizedTail,
FullReconciliation {
logged_commits: Vec<meerkat_core::TranscriptRewriteCommit>,
},
}
impl TranscriptRewriteReplay {
fn absent() -> Self {
Self {
session: None,
materialized_from_replay: false,
audit: None,
}
}
fn unchanged(session: Session, audit: Option<TranscriptRewriteAuditCoverage>) -> Self {
Self {
session: Some(session),
materialized_from_replay: false,
audit,
}
}
}
struct PreparedTranscriptRewriteAudit {
scope: PreparedTranscriptRewriteAuditScope,
logged_commits: Vec<meerkat_core::TranscriptRewriteCommit>,
records: Vec<meerkat_core::TranscriptRewriteRecord>,
receipt: Option<TranscriptRewritePrefixReceipt>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum PreparedTranscriptRewriteAuditScope {
AuthorizedTail,
FullReconciliation,
}
#[cfg(test)]
fn transcript_rewrite_record_for_session(
session: &Session,
commit: &meerkat_core::TranscriptRewriteCommit,
) -> Result<meerkat_core::TranscriptRewriteRecord, SessionError> {
let history = session
.validated_transcript_history_state()
.map_err(|err| {
SessionError::Agent(meerkat_core::error::AgentError::InternalError(format!(
"failed to read compact transcript graph for legacy test record: {err}"
)))
})?
.ok_or_else(|| {
SessionError::Agent(meerkat_core::error::AgentError::InternalError(format!(
"missing compact transcript graph for rewrite occurrence {}",
commit.rewrite_generation
)))
})?;
let parent_body = history.materialize_rewrite_parent(commit).map_err(|err| {
SessionError::Agent(meerkat_core::error::AgentError::InternalError(format!(
"failed to materialize exact rewrite parent occurrence: {err}"
)))
})?;
let revision_body = history.materialize_rewrite_child(commit).map_err(|err| {
SessionError::Agent(meerkat_core::error::AgentError::InternalError(format!(
"failed to materialize exact rewrite child occurrence: {err}"
)))
})?;
meerkat_core::TranscriptRewriteRecord::new(commit.clone(), parent_body, revision_body).map_err(
|err| {
SessionError::Agent(meerkat_core::error::AgentError::InternalError(format!(
"transcript rewrite audit record validation failed: {err}"
)))
},
)
}
fn transcript_rewrite_audit_receipt_for_commits(
session: &Session,
commits: &[meerkat_core::TranscriptRewriteCommit],
) -> Result<meerkat_core::TranscriptRewriteAuditReceiptBatch, SessionError> {
let Some(first) = commits.first() else {
return Err(SessionError::Agent(AgentError::InternalError(
"cannot prepare an empty transcript rewrite receipt".to_string(),
)));
};
let history = session
.validated_transcript_history_state()
.map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"transcript rewrite receipt graph is invalid: {error}"
)))
})?
.ok_or_else(|| {
SessionError::Agent(AgentError::InternalError(
"transcript rewrite receipt session has no sealed graph".to_string(),
))
})?;
let suffix = history
.prove_commit_suffix_starting_with(first)
.map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"transcript rewrite receipt suffix is invalid: {error}"
)))
})?;
let proved = suffix.commits();
if proved.len() != commits.len()
|| !proved
.zip(commits)
.all(|(proved, supplied)| proved == supplied)
{
return Err(SessionError::Agent(AgentError::InternalError(
"transcript rewrite receipt commits are not the exact sealed graph suffix".to_string(),
)));
}
meerkat_core::TranscriptRewriteAuditReceiptBatch::new(
suffix.start_prefix().clone(),
commits.to_vec(),
suffix.end_prefix().clone(),
)
.map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"failed to seal transcript rewrite receipt: {error}"
)))
})
}
async fn append_prepared_transcript_rewrite_receipt(
event_store: Option<&Arc<dyn EventStore>>,
projector: Option<&Arc<SessionProjector>>,
session_id: &SessionId,
receipt: &meerkat_core::TranscriptRewriteAuditReceiptBatch,
final_assistant_text: Option<&str>,
) -> Result<(), SessionError> {
let Some(event_store) = event_store else {
return Ok(());
};
let stored = match event_store
.append_transcript_rewrite_receipt_exact(session_id, receipt, final_assistant_text)
.await
{
Ok(result) => result.stored_event().clone(),
Err(error) => {
tracing::error!(
session_id = %session_id,
revisions = ?receipt.commits().iter().map(|commit| commit.revision.as_str()).collect::<Vec<_>>(),
error = %error,
"failed to publish exact transcript rewrite audit receipt after authority commit"
);
return Err(SessionError::Store(Box::new(error)));
}
};
if let Some(projector) = projector
&& let Err(error) = projector.resume(event_store.as_ref(), session_id).await
{
tracing::warn!(
session_id = %session_id,
seq = stored.seq,
error = %error,
"failed to project transcript rewrite receipt"
);
}
Ok(())
}
async fn append_transcript_rewrite_receipt_for_commits(
event_store: Option<&Arc<dyn EventStore>>,
projector: Option<&Arc<SessionProjector>>,
session: &Session,
commits: &[meerkat_core::TranscriptRewriteCommit],
) -> Result<(), SessionError> {
if event_store.is_none() || commits.is_empty() {
return Ok(());
}
let receipt = transcript_rewrite_audit_receipt_for_commits(session, commits)?;
let final_assistant_text = session.last_assistant_text();
append_prepared_transcript_rewrite_receipt(
event_store,
projector,
session.id(),
&receipt,
final_assistant_text.as_deref(),
)
.await
}
impl StoreCheckpointer {
async fn run_checkpoint_receipt(
&self,
session_id: &SessionId,
expected_run_id: &RunId,
) -> Result<Option<RunCheckpointReceipt>, AgentError> {
let _gate = self.gate.cancelled.lock().await;
let receipt_slot = self.latest_run_checkpoint_receipt.lock().await;
let Some(receipt) = receipt_slot.as_ref() else {
return Ok(None);
};
if receipt.session_id() != session_id {
return Err(AgentError::InternalError(format!(
"provisional receipt belongs to session {}, not terminal session {session_id}",
receipt.session_id()
)));
}
if receipt.run_id() != expected_run_id {
return Ok(None);
}
Ok(Some(receipt.clone()))
}
async fn acknowledge_promoted_checkpoint(
&self,
session_id: &SessionId,
committed_store_revision: u64,
committed_authority_token: &str,
) -> Result<(), AgentError> {
let _gate = self.gate.cancelled.lock().await;
let mut receipt_slot = self.latest_run_checkpoint_receipt.lock().await;
let receipt = receipt_slot.as_ref().ok_or_else(|| {
AgentError::InternalError(format!(
"promoted provisional boundary has no actor-carried checkpoint receipt for session {session_id}"
))
})?;
if receipt.session_id() != session_id {
return Err(AgentError::InternalError(format!(
"promoted provisional boundary receipt belongs to session {}, not {session_id}",
receipt.session_id()
)));
}
let exact_match = match receipt.authority() {
RunCheckpointAuthority::WholeBlob(authority) => {
authority.base_store_revision().checked_add(1) == Some(committed_store_revision)
&& authority.candidate_blob_sha256() == committed_authority_token
}
RunCheckpointAuthority::HeadCanonical(authority) => {
authority.physical_store_revision().checked_add(1) == Some(committed_store_revision)
&& authority.physical_head_token() == committed_authority_token
}
};
if !exact_match {
return Err(AgentError::InternalError(format!(
"promoted provisional authority does not match the actor-carried checkpoint receipt for session {session_id}"
)));
}
*receipt_slot = None;
Ok(())
}
async fn checkpoint_whole_blob(
&self,
session: &mut Session,
run_id: &RunId,
previous: Option<&RunCheckpointReceipt>,
) -> Result<Option<RunCheckpointReceipt>, AgentError> {
let previous = previous
.map(|receipt| {
receipt.whole_blob().ok_or_else(|| {
AgentError::InternalError(format!(
"WholeBlob checkpoint for session {} received a HeadCanonical provisional receipt",
session.id()
))
})
})
.transpose()?;
if let Some(previous) = previous
&& (previous.session_id() != session.id() || previous.run_id() != run_id)
{
return Err(AgentError::InternalError(format!(
"WholeBlob checkpoint for session {} received a provisional receipt for another session/run",
session.id()
)));
}
let candidate_sequence = previous
.map(|authority| {
authority
.candidate_sequence()
.checked_add(1)
.ok_or_else(|| {
AgentError::InternalError(format!(
"WholeBlob provisional candidate sequence exhausted for session {}",
session.id()
))
})
})
.transpose()?
.unwrap_or(1);
let committed_authority = self
.whole_blob_base_authority
.lock()
.map_err(|_| {
AgentError::InternalError(format!(
"WholeBlob actor base authority lock is poisoned for session {}",
session.id()
))
})?
.clone()
.ok_or_else(|| {
AgentError::InternalError(format!(
"WholeBlob actor for session {} has no exact materialization authority; live actor reload required",
session.id()
))
})?;
if committed_authority.session_id() != session.id() {
return Err(AgentError::InternalError(format!(
"WholeBlob actor base authority identifies session {}, not {}",
committed_authority.session_id(),
session.id()
)));
}
if let Some(previous) = previous
&& (previous.base_store_revision() != committed_authority.store_revision()
|| previous.base_blob_sha256() != committed_authority.blob_sha256())
{
return Err(AgentError::InternalError(format!(
"WholeBlob prior provisional receipt does not descend from this actor's committed base for session {}",
session.id()
)));
}
let runtime_id = LogicalRuntimeId::for_session(session.id());
let observed_authority = self
.runtime_store
.load_whole_blob_store_authority(&runtime_id)
.await
.map_err(|error| {
AgentError::InternalError(format!(
"failed to revalidate committed WholeBlob authority for provisional checkpoint of session {}: {error}",
session.id()
))
})?
.ok_or_else(|| {
AgentError::InternalError(format!(
"committed WholeBlob authority disappeared before provisional checkpoint of session {}; live actor reload required",
session.id()
))
})?;
if observed_authority != committed_authority {
return Err(AgentError::InternalError(format!(
"WholeBlob committed authority advanced beyond the live actor base for session {}; live actor reload required",
session.id()
)));
}
session
.externalize_media(self.blob_store.as_ref(), 0)
.await
.map_err(|error| {
AgentError::InternalError(format!(
"WholeBlob provisional checkpoint media externalization failed for session {}: {error}",
session.id()
))
})?;
if let Some(mut state) = session.deferred_turn_state() {
externalize_deferred_turn_state(self.blob_store.as_ref(), &mut state)
.await
.map_err(|error| {
AgentError::InternalError(format!(
"WholeBlob provisional checkpoint deferred-turn externalization failed for session {}: {error}",
session.id()
))
})?;
session.set_deferred_turn_state(state).map_err(|error| {
AgentError::InternalError(format!(
"WholeBlob provisional checkpoint deferred-turn projection failed for session {}: {error}",
session.id()
))
})?;
}
let prepared = PreparedWholeBlobProvisionalTail::prepare_from_session(
committed_authority,
run_id.clone(),
candidate_sequence,
session,
)
.map_err(|error| {
AgentError::InternalError(format!(
"failed to prepare WholeBlob provisional checkpoint for session {}: {error}",
session.id()
))
})?;
let conversation_digest = prepared.conversation_digest().to_string();
let message_count = prepared.message_count();
let expected_authority = prepared.authority().clone();
let stored_authority = self
.runtime_store
.write_prepared_whole_blob_provisional_tail(&runtime_id, prepared)
.await
.map_err(|error| {
AgentError::InternalError(format!(
"failed to write WholeBlob provisional checkpoint for session {}: {error}",
session.id()
))
})?;
if stored_authority != expected_authority {
return Err(AgentError::InternalError(format!(
"WholeBlob provisional checkpoint store acknowledged a different candidate for session {}",
session.id()
)));
}
let receipt = RunCheckpointReceipt::issued(
RunCheckpointAuthority::WholeBlob(stored_authority),
conversation_digest,
message_count,
)
.map_err(|error| {
AgentError::InternalError(format!(
"failed to bind WholeBlob provisional receipt facts for session {}: {error}",
session.id()
))
})?;
Ok(Some(receipt))
}
async fn write_head_canonical_provisional_intent(
&self,
runtime_id: &LogicalRuntimeId,
intent: HeadCanonicalProvisionalIntent<'_>,
) -> Result<HeadCanonicalProvisionalTailAuthority, AgentError> {
let HeadCanonicalProvisionalIntent {
committed,
run_id,
previous,
candidate_session,
successor_head,
successor_head_token,
} = intent;
let candidate_sequence =
previous
.map(|authority| {
authority.candidate_sequence().checked_add(1).ok_or_else(|| {
AgentError::InternalError(format!(
"HeadCanonical provisional candidate sequence exhausted for session {}",
committed.session_id()
))
})
})
.transpose()?
.unwrap_or(1);
let prepared = PreparedHeadCanonicalProvisionalTail::prepare(
committed.clone(),
run_id.clone(),
successor_head,
successor_head_token,
candidate_session,
)
.map_err(|error| {
AgentError::InternalError(format!(
"failed to prepare HeadCanonical provisional intent for session {}: {error}",
committed.session_id()
))
})?;
let authority = self
.runtime_store
.write_prepared_head_canonical_provisional_tail(runtime_id, prepared)
.await
.map_err(|error| {
AgentError::InternalError(format!(
"failed to write HeadCanonical provisional intent for session {}: {error}",
committed.session_id()
))
})?;
let expected_physical_revision = committed
.store_revision()
.checked_add(candidate_sequence)
.ok_or_else(|| {
AgentError::InternalError(format!(
"HeadCanonical provisional revision exhausted for session {}",
committed.session_id()
))
})?;
if authority.session_id() != committed.session_id()
|| authority.base_store_revision() != committed.store_revision()
|| authority.base_committed_head_token() != committed.committed_head_token()
|| authority.physical_store_revision() != expected_physical_revision
|| authority.physical_head_token() != successor_head_token
|| authority.run_id() != run_id
|| authority.candidate_sequence() != candidate_sequence
{
return Err(AgentError::InternalError(format!(
"HeadCanonical store acknowledged a different provisional intent for session {}",
committed.session_id()
)));
}
Ok(authority)
}
async fn discard_head_canonical_provisional_intent_after_cas_conflict(
&self,
runtime_id: &LogicalRuntimeId,
authority: &HeadCanonicalProvisionalTailAuthority,
) -> Result<(), AgentError> {
let discarded = self
.runtime_store
.discard_head_canonical_provisional_tail(runtime_id, authority)
.await
.map_err(|error| {
AgentError::InternalError(format!(
"failed to discard rejected HeadCanonical provisional intent for session {}: {error}",
authority.session_id()
))
})?;
if !discarded {
return Err(AgentError::InternalError(format!(
"rejected HeadCanonical provisional intent changed before exact discard for session {}",
authority.session_id()
)));
}
Ok(())
}
async fn checkpoint_head_canonical(
&self,
session: &mut Session,
run_id: &RunId,
previous: Option<&RunCheckpointReceipt>,
) -> Result<Option<RunCheckpointReceipt>, AgentError> {
let id = session.id().clone();
let previous = previous
.map(|receipt| {
receipt.head_canonical().ok_or_else(|| {
AgentError::InternalError(format!(
"HeadCanonical checkpoint for session {id} received a WholeBlob provisional receipt"
))
})
})
.transpose()?;
if let Some(previous) = previous
&& (previous.session_id() != &id || previous.run_id() != run_id)
{
return Err(AgentError::InternalError(format!(
"HeadCanonical checkpoint for session {id} received a provisional receipt for another session/run"
)));
}
let incremental = self.incremental.as_ref().ok_or_else(|| {
AgentError::InternalError(format!(
"head-canonical intra-turn checkpoint for session {id} has no incremental SessionStore capability; live actor reload required"
))
})?;
let runtime_id = LogicalRuntimeId::for_session(&id);
let authority = self
.runtime_store
.load_session_boundary_authority(&runtime_id)
.await
.map_err(|error| {
AgentError::InternalError(format!(
"failed to load head-canonical runtime authority for intra-turn checkpoint of session {id}: {error}; live actor reload required"
))
})?;
let Some(authority) = authority else {
let observed_head = incremental.load_head(&id).await.map_err(|error| {
AgentError::InternalError(format!(
"failed to prove generation-zero physical absence before checkpointing session {id}: {error}; live actor reload required"
))
})?;
if observed_head.is_none() {
if previous.is_some() {
return Err(AgentError::InternalError(format!(
"HeadCanonical committed authority disappeared while session {id} retained a provisional receipt; live actor reload required"
)));
}
return Ok(None);
}
return Err(AgentError::InternalError(format!(
"head-canonical runtime authority is absent while a physical head exists for session {id}; live actor reload required"
)));
};
let committed_authority = authority.head_canonical().ok_or_else(|| {
AgentError::InternalError(format!(
"head-canonical intra-turn checkpoint loaded WholeBlob authority for session {id}; live actor reload required"
))
})?;
if committed_authority.session_id() != &id {
return Err(AgentError::InternalError(format!(
"head-canonical intra-turn checkpoint loaded mismatched runtime authority for session {id}; live actor reload required"
)));
}
if let Some(previous) = previous
&& (previous.base_store_revision() != committed_authority.store_revision()
|| previous.base_committed_head_token()
!= committed_authority.committed_head_token())
{
return Err(AgentError::InternalError(format!(
"HeadCanonical prior provisional receipt does not descend from the committed base for session {id}; live actor reload required"
)));
}
let boundary_head = committed_authority.boundary_head();
let boundary_token = committed_authority.committed_head_token();
let derived_boundary_token = session_head_cas_token(boundary_head).map_err(|error| {
AgentError::InternalError(format!(
"head-canonical runtime boundary for session {id} has no valid CAS token: {error}; live actor reload required"
))
})?;
if derived_boundary_token != boundary_token {
return Err(AgentError::InternalError(format!(
"head-canonical runtime boundary token disagrees with its exact head for session {id}; live actor reload required"
)));
}
let observed_head = incremental.load_head(&id).await.map_err(|error| {
AgentError::InternalError(format!(
"failed to load physical head for intra-turn checkpoint of session {id}: {error}; live actor reload required"
))
})?;
let observed_head = observed_head.ok_or_else(|| {
AgentError::InternalError(format!(
"head-canonical runtime authority exists without a physical head for session {id}; live actor reload required"
))
})?;
let observed_token = session_head_cas_token(&observed_head).map_err(|error| {
AgentError::InternalError(format!(
"physical head for session {id} has no valid CAS token: {error}; live actor reload required"
))
})?;
match previous {
Some(previous) => {
if observed_token != previous.physical_head_token() {
return Err(AgentError::InternalError(format!(
"HeadCanonical physical head is not the exact prior provisional successor for session {id}; live actor reload required"
)));
}
}
None => {
if &observed_head != boundary_head || observed_token != boundary_token {
return Err(AgentError::InternalError(format!(
"HeadCanonical physical head is ahead of committed authority without an actor-carried provisional receipt for session {id}; live actor reload required"
)));
}
}
}
if &observed_head == boundary_head && observed_token != boundary_token {
return Err(AgentError::InternalError(format!(
"physical and runtime boundary tokens disagree for session {id}; live actor reload required"
)));
}
let rewrite_preflight =
meerkat_core::session_store::PreparedHeadCanonicalRewritePreflight::prepare(
session,
&observed_head,
)
.map_err(|error| {
AgentError::InternalError(format!(
"failed to preflight the head-canonical intra-turn rewrite route for session {id}: {error}; live actor reload required"
))
})?;
let suffix_start = usize::try_from(observed_head.message_count).map_err(|_| {
AgentError::InternalError(format!(
"physical head message count exceeds the host index range for session {id}; live actor reload required"
))
})?;
let rewrite_preflight = match rewrite_preflight {
Some(preflight) => {
tracing::trace!(
session_id = %id,
live_tail_base = preflight.live_tail_base(),
"externalizing the preflight-authorized head-canonical rewrite tail"
);
Some(
preflight
.externalize_live_tail(session, self.blob_store.as_ref())
.await
.map_err(|error| {
AgentError::InternalError(format!(
"failed to externalize the preflight-authorized head-canonical rewrite tail for session {id}: {error}; live actor reload required"
))
})?,
)
}
None => {
session
.externalize_media(self.blob_store.as_ref(), suffix_start)
.await
.map_err(|error| {
AgentError::InternalError(format!(
"failed to externalize head-canonical intra-turn suffix for session {id}: {error}; live actor reload required"
))
})?;
None
}
};
if let Some(mut state) = session.deferred_turn_state() {
externalize_deferred_turn_state(self.blob_store.as_ref(), &mut state)
.await
.map_err(|error| {
AgentError::InternalError(format!(
"failed to externalize deferred state for head-canonical intra-turn checkpoint of session {id}: {error}; live actor reload required"
))
})?;
session.set_deferred_turn_state(state).map_err(|error| {
AgentError::InternalError(format!(
"failed to project deferred state for head-canonical intra-turn checkpoint of session {id}: {error}; live actor reload required"
))
})?;
}
let route =
meerkat_core::session_store::PreparedHeadCanonicalMutationRoute::prepare_intra_turn_after_preflight(
session,
&observed_head,
observed_head.clone(),
rewrite_preflight,
)
.map_err(|error| {
AgentError::InternalError(format!(
"failed to prepare the preflighted head-canonical intra-turn checkpoint route for session {id}: {error}; live actor reload required"
))
})?;
match route {
meerkat_core::session_store::PreparedHeadCanonicalMutationRoute::Rewrite(mutation) => {
let provisional = self
.write_head_canonical_provisional_intent(
&runtime_id,
HeadCanonicalProvisionalIntent {
committed: committed_authority,
run_id,
previous,
candidate_session: session,
successor_head: mutation.successor_head(),
successor_head_token: mutation.successor_head_token(),
},
)
.await?;
let committed_head_token = match incremental
.apply_prepared_head_canonical_rewrite_mutation(&mutation)
.await
{
Ok(committed_head_token) => committed_head_token,
Err(error) => {
if matches!(error, SessionStoreError::TranscriptRevisionConflict { .. }) {
self.discard_head_canonical_provisional_intent_after_cas_conflict(
&runtime_id,
&provisional,
)
.await?;
}
return Err(AgentError::InternalError(format!(
"head-canonical intra-turn rewrite store outcome is not safely reusable for session {id}: {error}; live actor reload required"
)));
}
};
if committed_head_token != mutation.successor_head_token() {
return Err(AgentError::InternalError(format!(
"head-canonical intra-turn rewrite store acknowledged a different successor for session {id}; live actor reload required"
)));
}
mutation
.acknowledge_physical_projection(session, &committed_head_token)
.map_err(|error| {
AgentError::InternalError(format!(
"durable head-canonical intra-turn rewrite could not be acknowledged on the live session {id}: {error}; live actor reload required"
))
})?;
let receipt = RunCheckpointReceipt::issued(
RunCheckpointAuthority::HeadCanonical(provisional),
mutation.successor_head().head_revision.clone(),
mutation.successor_head().message_count,
)
.map_err(|error| {
AgentError::InternalError(format!(
"failed to bind HeadCanonical provisional receipt facts for session {id}: {error}"
))
})?;
Ok(Some(receipt))
}
meerkat_core::session_store::PreparedHeadCanonicalMutationRoute::Ordinary(mutation) => {
let provisional = self
.write_head_canonical_provisional_intent(
&runtime_id,
HeadCanonicalProvisionalIntent {
committed: committed_authority,
run_id,
previous,
candidate_session: session,
successor_head: mutation.successor_head(),
successor_head_token: mutation.successor_head_token(),
},
)
.await?;
let committed_head_token = match incremental
.apply_prepared_head_canonical_mutation(&mutation)
.await
{
Ok(committed_head_token) => committed_head_token,
Err(error) => {
if matches!(error, SessionStoreError::TranscriptRevisionConflict { .. }) {
self.discard_head_canonical_provisional_intent_after_cas_conflict(
&runtime_id,
&provisional,
)
.await?;
}
return Err(AgentError::InternalError(format!(
"head-canonical intra-turn store outcome is not safely reusable for session {id}: {error}; live actor reload required"
)));
}
};
if committed_head_token != mutation.successor_head_token() {
return Err(AgentError::InternalError(format!(
"head-canonical intra-turn store acknowledged a different successor for session {id}; live actor reload required"
)));
}
mutation
.acknowledge_physical_projection(session, &committed_head_token)
.map_err(|error| {
AgentError::InternalError(format!(
"durable head-canonical intra-turn checkpoint could not be acknowledged on the live session {id}: {error}; live actor reload required"
))
})?;
let receipt = RunCheckpointReceipt::issued(
RunCheckpointAuthority::HeadCanonical(provisional),
mutation.successor_head().head_revision.clone(),
mutation.successor_head().message_count,
)
.map_err(|error| {
AgentError::InternalError(format!(
"failed to bind HeadCanonical provisional receipt facts for session {id}: {error}"
))
})?;
Ok(Some(receipt))
}
}
}
}
#[async_trait]
impl meerkat_core::SessionCheckpointer for StoreCheckpointer {
async fn acknowledge_control_commit(
&self,
receipt: &meerkat_core::SessionControlCommitReceipt,
) -> Result<(), AgentError> {
let guard = self.gate.cancelled.lock().await;
if *guard {
return Err(AgentError::InternalError(format!(
"checkpoint gate is cancelled for session {}; control commit acknowledgement requires actor reload",
receipt.session_id()
)));
}
if self.runtime_store.session_persistence_profile()
!= RuntimeSessionPersistenceProfile::WholeBlobV1
{
return Err(AgentError::InternalError(format!(
"WholeBlob control commit receipt cannot rebind a non-WholeBlob actor for session {}",
receipt.session_id()
)));
}
if self.latest_run_checkpoint_receipt.lock().await.is_some() {
return Err(AgentError::InternalError(format!(
"control commit for session {} cannot bypass an actor-carried provisional tail",
receipt.session_id()
)));
}
let authority = self
.runtime_store
.load_whole_blob_store_authority(&LogicalRuntimeId::for_session(receipt.session_id()))
.await
.map_err(|error| {
AgentError::InternalError(format!(
"failed to confirm control commit authority for session {}: {error}",
receipt.session_id()
))
})?
.ok_or_else(|| {
AgentError::InternalError(format!(
"control commit authority disappeared for session {}",
receipt.session_id()
))
})?;
if authority.session_id() != receipt.session_id()
|| authority.store_revision() != receipt.store_revision()
|| authority.blob_sha256() != receipt.authority_token()
{
return Err(AgentError::InternalError(format!(
"control commit receipt no longer matches store authority for session {}",
receipt.session_id()
)));
}
*self.whole_blob_base_authority.lock().map_err(|_| {
AgentError::InternalError(format!(
"WholeBlob actor base authority lock is poisoned for session {}",
receipt.session_id()
))
})? = Some(authority);
Ok(())
}
async fn checkpoint_run(
&self,
session: &mut Session,
run_id: &RunId,
previous: Option<&RunCheckpointReceipt>,
) -> Result<Option<RunCheckpointReceipt>, AgentError> {
let guard = self.gate.cancelled.lock().await;
if *guard {
return match self.runtime_store.session_persistence_profile() {
RuntimeSessionPersistenceProfile::WholeBlobV1
| RuntimeSessionPersistenceProfile::HeadCanonicalV1 => {
Err(AgentError::InternalError(format!(
"checkpoint gate is cancelled for session {}; the required provisional \
run tail was not attempted; live actor reload required",
session.id()
)))
}
profile => Err(AgentError::InternalError(format!(
"checkpoint gate is cancelled for session {} under unsupported runtime \
persistence profile {profile}; live actor reload required",
session.id()
))),
};
}
let result = match self.runtime_store.session_persistence_profile() {
RuntimeSessionPersistenceProfile::WholeBlobV1 => {
self.checkpoint_whole_blob(session, run_id, previous).await
}
RuntimeSessionPersistenceProfile::HeadCanonicalV1 => {
self.checkpoint_head_canonical(session, run_id, previous)
.await
}
profile => Err(AgentError::InternalError(format!(
"unsupported runtime session persistence profile {profile} for checkpoint of session {}; live actor reload required",
session.id()
))),
};
if let Ok(receipt) = result.as_ref() {
*self.latest_run_checkpoint_receipt.lock().await = receipt.clone();
}
drop(guard);
result
}
}
pub struct LiveSessionActorTurnBoundaryLease {
witness: LiveSessionActorWitness,
_turn_finalization_guard: OwnedMutexGuard<()>,
}
impl LiveSessionActorTurnBoundaryLease {
pub fn witness(&self) -> &LiveSessionActorWitness {
&self.witness
}
pub fn session_id(&self) -> &SessionId {
self.witness.session_id()
}
}
pub struct PersistentSessionService<B: SessionAgentBuilder> {
inner: EphemeralSessionService<B>,
incremental: Option<Arc<dyn IncrementalSessionStore>>,
runtime_store: Arc<dyn RuntimeStore>,
blob_store: Arc<dyn BlobStore>,
event_store: Option<Arc<dyn EventStore>>,
projector: Option<Arc<SessionProjector>>,
checkpointer_gates: Mutex<HashMap<SessionId, Arc<CheckpointerGate>>>,
live_checkpointers: Mutex<HashMap<SessionId, std::sync::Weak<StoreCheckpointer>>>,
recovery_gates: Mutex<HashMap<SessionId, Arc<Mutex<()>>>>,
rejected_run_compaction_aborted: std::sync::Mutex<HashSet<SessionId>>,
turn_finalization_gates: Mutex<HashMap<SessionId, std::sync::Weak<Mutex<()>>>>,
event_projection_faults: EventProjectionFaultRegistry,
event_projection_gates: EventProjectionGateRegistry,
event_projection_drains: EventProjectionDrainRegistry,
}
struct SessionMutationGuard {
_turn_finalization_guard: tokio::sync::OwnedMutexGuard<()>,
_recovery_guard: tokio::sync::OwnedMutexGuard<()>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct PreparedRuntimeBoundaryIdentity {
profile: RuntimeSessionPersistenceProfile,
session_id: SessionId,
authority_token: String,
}
impl PreparedRuntimeBoundaryIdentity {
fn whole_blob(session_id: SessionId, document_token: &str) -> Self {
Self {
profile: RuntimeSessionPersistenceProfile::WholeBlobV1,
session_id,
authority_token: document_token.to_string(),
}
}
fn from_runtime_authority(
authority: &RuntimeSessionAuthority,
id: &SessionId,
) -> Result<Self, SessionError> {
if authority.session_id() != id {
return Err(SessionError::Agent(AgentError::InternalError(format!(
"runtime boundary authority for session {} cannot identify session {id}",
authority.session_id()
))));
}
let authority_token = match authority {
RuntimeSessionAuthority::WholeBlob(authority) => authority.blob_sha256().to_string(),
RuntimeSessionAuthority::HeadCanonical(authority) => {
authority.committed_head_token().to_string()
}
};
Ok(Self {
profile: authority.profile(),
session_id: id.clone(),
authority_token,
})
}
}
struct PreparedRuntimeBoundary {
committed: BoundSessionCommit,
identity: PreparedRuntimeBoundaryIdentity,
conversation_digest: String,
message_count: usize,
}
#[derive(Debug, Clone)]
struct PreparedRealtimeUserContentBlob {
pending: meerkat_core::PendingRealtimeUserContentBlob,
inline_data: Option<String>,
}
fn realtime_user_content_blob_error(
context: &'static str,
error: impl std::fmt::Display,
) -> SessionError {
SessionError::Agent(AgentError::InternalError(format!("{context}: {error}")))
}
fn user_content_outcome(
user_content: meerkat_core::RealtimeUserContentApplyOutcome,
) -> meerkat_core::RealtimeTranscriptApplyOutcome {
meerkat_core::RealtimeTranscriptApplyOutcome {
user_content: Some(user_content),
..meerkat_core::RealtimeTranscriptApplyOutcome::default()
}
}
fn pending_blob_is_definitively_invalid(error: &meerkat_core::ImageBlobIntegrityError) -> bool {
match error {
meerkat_core::ImageBlobIntegrityError::UnsupportedMediaType { .. }
| meerkat_core::ImageBlobIntegrityError::EncodedTooLarge { .. }
| meerkat_core::ImageBlobIntegrityError::InvalidBase64 { .. }
| meerkat_core::ImageBlobIntegrityError::DecodedTooLarge { .. }
| meerkat_core::ImageBlobIntegrityError::SignatureMismatch { .. }
| meerkat_core::ImageBlobIntegrityError::MediaTypeMismatch { .. }
| meerkat_core::ImageBlobIntegrityError::BlobIdentityMismatch { .. } => true,
meerkat_core::ImageBlobIntegrityError::Store(error) => matches!(
error,
meerkat_core::BlobStoreError::InvalidId(_)
| meerkat_core::BlobStoreError::NotFound(_)
| meerkat_core::BlobStoreError::ReadLimitExceeded { .. }
| meerkat_core::BlobStoreError::WriteLimitExceeded { .. }
| meerkat_core::BlobStoreError::Corrupt { .. }
),
}
}
fn extract_labels_from_metadata(
metadata: &serde_json::Map<String, serde_json::Value>,
) -> BTreeMap<String, String> {
match metadata.get(SESSION_LABELS_KEY) {
Some(v) => match serde_json::from_value::<BTreeMap<String, String>>(v.clone()) {
Ok(labels) => labels,
Err(e) => {
tracing::warn!(
key = SESSION_LABELS_KEY,
error = %e,
"failed to deserialize session labels from metadata"
);
BTreeMap::new()
}
},
None => BTreeMap::new(),
}
}
fn view_from_authoritative_session(session: &Session) -> SessionView {
let metadata = session.session_metadata();
SessionView {
state: SessionInfo {
session_id: session.id().clone(),
created_at: session.created_at(),
updated_at: session.updated_at(),
message_count: session.messages().len(),
is_active: false,
model: metadata
.as_ref()
.map(|metadata| metadata.model.clone())
.unwrap_or_default(),
provider: metadata
.as_ref()
.map(|metadata| metadata.provider)
.unwrap_or(meerkat_core::Provider::Other),
last_assistant_text: session.last_assistant_text(),
labels: extract_labels_from_metadata(session.metadata()),
},
billing: SessionUsage {
total_tokens: session.total_tokens(),
usage: session.total_usage(),
},
}
}
enum LiveSessionAuthority {
NoLive,
LiveAuthoritative {
snapshot: LiveSessionTranscriptAuthoritySnapshot,
},
DurableAuthoritative {
session: Box<Session>,
reason: LiveSessionAuthorityReason,
},
}
#[derive(Debug, Clone, Copy)]
enum LiveSessionSyncCause {
Authority(LiveSessionAuthorityReason),
TranscriptRewriteCommitted,
}
impl LiveSessionSyncCause {
fn trace_label(self) -> &'static str {
match self {
LiveSessionSyncCause::Authority(LiveSessionAuthorityReason::StoredArchived) => {
"stored_archived"
}
LiveSessionSyncCause::Authority(
LiveSessionAuthorityReason::LiveUncommittedTranscript,
) => "live_uncommitted_transcript",
LiveSessionSyncCause::Authority(
LiveSessionAuthorityReason::StoredTranscriptRevisionDiverged,
) => "stored_transcript_revision_diverged",
LiveSessionSyncCause::TranscriptRewriteCommitted => "transcript_rewrite_committed",
}
}
}
enum ArchivedResumeAuthorization {
RejectArchived,
MachinePendingPromotion {
authorization: ArchivedSessionActorMaterializationAuthorization,
},
}
#[derive(Clone, Copy)]
enum ActorSessionSeedAuthority {
FreshGenerationZero,
DurableCommitted,
}
impl ArchivedResumeAuthorization {
fn allows_archived_resume(&self) -> bool {
matches!(
self,
ArchivedResumeAuthorization::MachinePendingPromotion { .. }
)
}
}
#[derive(Clone, Copy)]
pub struct MachineServiceTurnCommitProtocol<'a> {
runtime_adapter: &'a MeerkatMachine,
_authority: MachineSessionControlAuthority,
}
impl<'a> MachineServiceTurnCommitProtocol<'a> {
#[must_use]
pub fn from_machine(runtime_adapter: &'a MeerkatMachine) -> Self {
Self {
runtime_adapter,
_authority: runtime_adapter.session_control_authority(),
}
}
}
#[derive(Clone, Copy)]
pub struct MachineSessionArchiveProtocol<'a> {
runtime_adapter: &'a MeerkatMachine,
_authority: MachineSessionControlAuthority,
}
struct StoredOnlyArchiveTerminalPublisher<'a, B: SessionAgentBuilder> {
service: &'a PersistentSessionService<B>,
session_id: &'a SessionId,
}
#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
impl<B: SessionAgentBuilder + 'static> CoreExecutorPublicationHandle
for StoredOnlyArchiveTerminalPublisher<'_, B>
{
async fn publish_interaction_terminals(
&self,
events: &[meerkat_core::event::AgentEvent],
) -> Result<Vec<CoreInteractionTerminalPublicationReceipt>, CoreExecutorError> {
self.service
.publish_interaction_terminals_stored_only_exact_batch(self.session_id, events)
.await
.map_err(CoreExecutorError::apply_failed_from_session_error)
}
}
impl<'a> MachineSessionArchiveProtocol<'a> {
#[must_use]
pub fn from_machine(runtime_adapter: &'a MeerkatMachine) -> Self {
Self {
runtime_adapter,
_authority: runtime_adapter.session_control_authority(),
}
}
async fn prepare_archive_lease(
&self,
id: &SessionId,
) -> Result<Option<meerkat_runtime::MachineSessionArchiveLease>, SessionError> {
self.runtime_adapter
.prepare_session_archive_lease(id)
.await
.map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"machine archive lease preparation failed: {error}"
)))
})
}
async fn runtime_archive_authority_present(
&self,
id: &SessionId,
_runtime_store: &Arc<dyn RuntimeStore>,
) -> Result<bool, SessionError> {
let present = self
.runtime_adapter
.archive_runtime_residue_present(id)
.await
.map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"machine archive runtime lifecycle revalidation failed: {error}"
)))
})?;
Ok(present)
}
async fn release_quiescent_archive_lease(
&self,
lease: meerkat_runtime::MachineSessionArchiveLease,
) -> Result<(), SessionError> {
self.runtime_adapter
.release_quiescent_session_archive_lease(lease)
.await
.map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"machine quiescent archive lease cleanup failed: {error}"
)))
})
}
async fn drain_archive_lease_terminals(
&self,
lease: Option<&meerkat_runtime::MachineSessionArchiveLease>,
stored_only_publication_handle: Option<&dyn CoreExecutorPublicationHandle>,
) -> Result<(), SessionError> {
let Some(lease) = lease else {
return Ok(());
};
self.runtime_adapter
.drain_session_archive_lease_terminals(lease, stored_only_publication_handle)
.await
.map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"machine archive pending-terminal drain failed: {error}"
)))
})
}
fn require_shared_runtime_store(
&self,
id: &SessionId,
runtime_store: &Arc<dyn RuntimeStore>,
) -> Result<(), SessionError> {
if self
.runtime_adapter
.shares_runtime_store_authority(runtime_store)
{
return Ok(());
}
Err(SessionError::Unsupported(format!(
"archive for session {id} requires MachineSessionArchiveProtocol backed by the same durable runtime/session machine store"
)))
}
async fn retire_session(
&self,
id: &SessionId,
lease: Option<meerkat_runtime::MachineSessionArchiveLease>,
stored_only_publication_handle: Option<&dyn CoreExecutorPublicationHandle>,
) -> Result<(), SessionError> {
let lease = lease.ok_or_else(|| {
SessionError::Agent(AgentError::InternalError(format!(
"machine archive authorized runtime retirement for session {id} without an exact archive lease"
)))
})?;
let retire = match stored_only_publication_handle {
Some(publication_handle) => {
self.runtime_adapter
.retire_session_with_archive_lease_and_publication_handle(
lease,
publication_handle,
)
.await
}
None => {
self.runtime_adapter
.retire_session_with_archive_lease(lease)
.await
}
};
retire.map(|_| ()).map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"machine archive retire with lease failed: {error}"
)))
})
}
}
impl<B: SessionAgentBuilder + 'static> PersistentSessionService<B> {
fn archived_not_found(id: &SessionId) -> SessionControlError {
SessionControlError::Session(SessionError::NotFound { id: id.clone() })
}
async fn reject_if_archived_session(
&self,
id: &SessionId,
session: &Session,
) -> Result<(), SessionControlError> {
if self
.session_archived_by_authority(id, session)
.await
.map_err(SessionControlError::Session)?
{
return Err(Self::archived_not_found(id));
}
Ok(())
}
pub async fn session_archived_by_authority(
&self,
id: &SessionId,
_session: &Session,
) -> Result<bool, SessionError> {
self.session_archived_by_runtime_store_authority(id).await
}
pub async fn session_archived_by_authority_with_terminal(
&self,
id: &SessionId,
_lifecycle_terminal: Option<&SessionLifecycleTerminal>,
) -> Result<bool, SessionError> {
self.session_archived_by_runtime_store_authority(id).await
}
async fn session_archived_by_runtime_store_authority(
&self,
id: &SessionId,
) -> Result<bool, SessionError> {
let runtime_id = Self::runtime_id_for_session(id);
let catalog_archived = self
.runtime_store
.load_runtime_session_catalog_entry(&runtime_id)
.await
.map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"failed to load RuntimeStore archive catalog authority for session {id}: {error}"
)))
})?
.is_some_and(|entry| {
entry.lifecycle_terminal()
== Some(meerkat_core::SessionLifecycleTerminal::Archived)
});
if catalog_archived {
return Ok(true);
}
Ok(matches!(
Self::load_runtime_state_for_session(&self.runtime_store, id).await?,
Some(RuntimeState::Retired | RuntimeState::Destroyed)
))
}
fn runtime_id_for_session(id: &SessionId) -> LogicalRuntimeId {
LogicalRuntimeId::for_session(id)
}
#[cfg(test)]
async fn runtime_input_updates(
&self,
id: &SessionId,
run_id: &RunId,
sequence: u64,
contributing_input_ids: &[InputId],
) -> Result<Vec<StoredInputState>, SessionError> {
let runtime_id = Self::runtime_id_for_session(id);
let stored_states = self
.runtime_store
.load_input_states_strict(&runtime_id)
.await
.map_err(|err| {
SessionError::Agent(meerkat_core::error::AgentError::InternalError(format!(
"failed to load runtime input states: {err}"
)))
})?;
Ok(contributing_input_ids
.iter()
.filter_map(|input_id| {
let mut bundle = stored_states
.iter()
.find(|candidate| &candidate.state.input_id == input_id)?
.clone();
bundle.seed.last_run_id = Some(run_id.clone());
bundle.seed.last_boundary_sequence = Some(sequence);
bundle.seed.phase = InputLifecycleState::Consumed;
bundle.seed.terminal_outcome = Some(InputTerminalOutcome::Consumed);
Some(bundle)
})
.collect())
}
pub async fn live_session_actor_registered(&self, id: &SessionId) -> bool {
self.inner.live_session_actor_registered(id).await
}
async fn export_session_with_labels(&self, id: &SessionId) -> Result<Session, SessionError> {
self.inner.export_session(id).await
}
async fn export_session_with_labels_if_transcript_authority(
&self,
id: &SessionId,
expected: LiveSessionTranscriptAuthoritySnapshot,
) -> Result<Option<Session>, SessionError> {
self.inner
.export_session_if_transcript_authority(id, expected)
.await
}
pub async fn apply_runtime_session_llm_identity_under_runtime_turn_boundary(
&self,
id: &SessionId,
client: std::sync::Arc<dyn meerkat_core::AgentLlmClient>,
identity: meerkat_core::SessionLlmIdentity,
request_policy: meerkat_core::SessionLlmRequestPolicy,
) -> Result<(), SessionError> {
let _recovery_guard = self.recovery_gate_for_session(id).await.lock_owned().await;
let _ = self.discard_stale_live_session_if_needed(id).await?;
if let Some(session) = self.load_authoritative_session_base(id).await? {
self.reject_if_archived_session(id, &session)
.await
.map_err(crate::control_error_into_session_error)?;
}
self.inner
.hot_swap_session_llm_identity(id, client, identity, request_policy)
.await
}
async fn load_authoritative_session_base(
&self,
id: &SessionId,
) -> Result<Option<Session>, SessionError> {
let replay = self
.load_authoritative_session_base_with_replay_info(id)
.await?;
if let Some(session) = replay.session.as_ref() {
self.verify_transcript_rewrite_audit_events(session, &replay)
.await?;
}
Ok(replay.session)
}
async fn load_authoritative_session_base_with_replay_info(
&self,
id: &SessionId,
) -> Result<TranscriptRewriteReplay, SessionError> {
let session = self
.load_committed_runtime_session_for_body(id, "authoritative session read")
.await?;
self.apply_transcript_rewrite_replay(id, session).await
}
async fn transcript_rewrite_full_fallback(
&self,
id: &SessionId,
proved: Option<&meerkat_core::ValidatedTranscriptHistory>,
) -> Result<PreparedTranscriptRewriteAudit, SessionError> {
let events = self.event_log_read_from(id, 1).await?.unwrap_or_default();
let observed_through_log_seq = events.iter().map(|stored| stored.seq).max().unwrap_or(0);
let records = events
.into_iter()
.filter_map(|stored| match stored.event {
meerkat_core::event::AgentEvent::TranscriptRewriteCommitted {
session_id,
record,
} if session_id == *id => Some((stored.seq, record)),
_ => None,
})
.collect::<Vec<_>>();
let records = Self::prepare_full_reconciliation_records(
id,
records,
proved,
observed_through_log_seq,
)?;
Ok(PreparedTranscriptRewriteAudit {
scope: PreparedTranscriptRewriteAuditScope::FullReconciliation,
logged_commits: records.iter().map(|record| record.commit.clone()).collect(),
records,
receipt: None,
})
}
async fn read_prepared_transcript_rewrite_audit(
&self,
session: &Session,
proved: Option<&meerkat_core::ValidatedTranscriptHistory>,
) -> Result<Option<PreparedTranscriptRewriteAudit>, SessionError> {
let Some(event_store) = self.event_store.as_ref() else {
return Ok(None);
};
let empty_prefix = meerkat_core::TranscriptRewritePrefixAccumulator::empty();
let expected_prefix = proved
.map(|proved| proved.rewrite_prefix())
.unwrap_or(&empty_prefix);
let expected_last_commit = proved.and_then(|proved| proved.last_commit());
let authority_key = meerkat_core::SESSION_TRANSCRIPT_REWRITE_PREFIX_AUTHORITY_KEY;
let bound_prefix_present = session.metadata().contains_key(authority_key);
if bound_prefix_present {
let bound_prefix = session
.transcript_rewrite_prefix_authority()
.ok_or_else(|| {
durable_session_restore_error(
session.id(),
"failed to decode checkpoint-bound transcript rewrite prefix",
"metadata value is malformed",
)
})?;
if &bound_prefix != expected_prefix {
return Err(durable_session_restore_error(
session.id(),
"failed to bind transcript rewrite audit expectation",
"checkpoint-bound prefix disagrees with the validated graph",
));
}
}
let proved_commits = proved
.map(|proved| proved.commits().cloned().collect::<Vec<_>>())
.unwrap_or_default();
let expectation = match proved {
Some(_) if !bound_prefix_present => {
TranscriptRewriteAuditExpectation::LegacyGenerationZero {
expected_prefix,
ordered_commits: &proved_commits,
}
}
_ => TranscriptRewriteAuditExpectation::Current(expected_prefix),
};
let read = {
if let Some(cause) = event_projection_admission_fault(
&self.event_projection_faults,
&self.event_projection_gates,
event_store,
session.id(),
)
.await
{
return Err(event_projection_halted_error(session.id(), cause));
}
let gate = event_projection_gate(&self.event_projection_gates, session.id()).await;
let _guard = gate.lock().await;
if let Some(cause) = self
.event_projection_faults
.lock()
.await
.get(session.id())
.cloned()
{
return Err(event_projection_halted_error(session.id(), cause));
}
event_store
.read_transcript_rewrite_audit(session.id(), expectation)
.await
.map_err(|error| {
durable_session_restore_error(
session.id(),
"failed to read transcript rewrite audit",
error,
)
})?
};
let Some(read) = read else {
return self
.transcript_rewrite_full_fallback(session.id(), proved)
.await
.map(Some);
};
read.verify_authorized_tail(expected_prefix, expected_last_commit)
.map_err(|error| {
durable_session_restore_error(
session.id(),
"failed to authorize transcript rewrite tail",
error,
)
})?;
let (scope, rows) = match read {
TranscriptRewriteAuditRead::AuthorizedTail(rows) => {
(PreparedTranscriptRewriteAuditScope::AuthorizedTail, rows)
}
TranscriptRewriteAuditRead::FullReconciliation(rows) => (
PreparedTranscriptRewriteAuditScope::FullReconciliation,
rows,
),
};
if rows.session_id() != session.id() {
return Err(durable_session_restore_error(
session.id(),
"failed to bind transcript rewrite audit result",
format!("result belongs to session {}", rows.session_id()),
));
}
let observed_through_log_seq = rows.observed_through_log_seq();
let receipt = rows.receipt().cloned();
if let Some(receipt) = receipt.as_ref() {
Self::validate_rewrite_receipt_binding(
session.id(),
observed_through_log_seq,
receipt,
)?;
}
let raw_rows = rows.into_rewrite_rows();
let (records, raw_logged_commits) =
Self::materialize_rewrite_audit_rows(session.id(), observed_through_log_seq, raw_rows)?;
let records = match scope {
PreparedTranscriptRewriteAuditScope::AuthorizedTail => {
Self::prepare_authorized_tail_records(
session.id(),
expected_prefix,
expected_last_commit,
records,
receipt.as_ref().ok_or_else(|| {
durable_session_restore_error(
session.id(),
"failed to authorize transcript rewrite tail",
"authorized tail has no end receipt",
)
})?,
)?
}
PreparedTranscriptRewriteAuditScope::FullReconciliation => {
Self::prepare_full_reconciliation_records(
session.id(),
records,
proved,
observed_through_log_seq,
)?
}
};
let mut commits_by_generation =
BTreeMap::<u64, meerkat_core::TranscriptRewriteCommit>::new();
for commit in records.iter().map(|record| record.commit.clone()).chain(
raw_logged_commits
.into_iter()
.filter(|commit| commit.rewrite_generation != 0),
) {
match commits_by_generation.get(&commit.rewrite_generation) {
Some(existing) if existing == &commit => {}
Some(_) => {
return Err(durable_session_restore_error(
session.id(),
"failed to validate transcript rewrite audit coverage",
format!(
"conflicting rows for occurrence generation {}",
commit.rewrite_generation
),
));
}
None => {
commits_by_generation.insert(commit.rewrite_generation, commit);
}
}
}
let logged_commits = commits_by_generation.into_values().collect::<Vec<_>>();
if scope == PreparedTranscriptRewriteAuditScope::FullReconciliation
&& let Some(receipt) = receipt.as_ref()
{
let rebuilt =
meerkat_core::TranscriptRewritePrefixAccumulator::from_commits(&logged_commits)
.map_err(|error| {
durable_session_restore_error(
session.id(),
"failed to rebuild full transcript rewrite audit receipt",
error,
)
})?;
if receipt.accumulator() != &rebuilt || receipt.last_commit() != logged_commits.last() {
return Err(durable_session_restore_error(
session.id(),
"failed to verify full transcript rewrite audit receipt",
"receipt does not bind the exact validated occurrence set",
));
}
}
Ok(Some(PreparedTranscriptRewriteAudit {
scope,
logged_commits,
records,
receipt,
}))
}
async fn finalize_prepared_transcript_rewrite_audit(
&self,
id: &SessionId,
receipt: &TranscriptRewritePrefixReceipt,
) -> Result<(), SessionError> {
let event_store = self.event_store.as_ref().ok_or_else(|| {
durable_session_restore_error(
id,
"failed to finalize transcript rewrite audit",
"audit receipt exists without an EventStore",
)
})?;
if let Some(cause) = event_projection_admission_fault(
&self.event_projection_faults,
&self.event_projection_gates,
event_store,
id,
)
.await
{
return Err(event_projection_halted_error(id, cause));
}
let gate = event_projection_gate(&self.event_projection_gates, id).await;
let _guard = gate.lock().await;
if let Some(cause) = self.event_projection_faults.lock().await.get(id).cloned() {
return Err(event_projection_halted_error(id, cause));
}
event_store
.finalize_transcript_rewrite_audit(receipt)
.await
.map_err(|error| {
durable_session_restore_error(
id,
"failed to finalize transcript rewrite audit",
error,
)
})
}
fn validate_rewrite_receipt_binding(
id: &SessionId,
observed_through_log_seq: u64,
receipt: &TranscriptRewritePrefixReceipt,
) -> Result<(), SessionError> {
if receipt.session_id() != id || receipt.through_log_seq() != observed_through_log_seq {
return Err(durable_session_restore_error(
id,
"failed to bind transcript rewrite receipt",
"receipt session or high-water disagrees with the stable audit read",
));
}
match (
receipt.accumulator().occurrence_count(),
receipt.last_commit(),
) {
(0, None) => Ok(()),
(count, Some(commit)) if commit.rewrite_generation == count => Ok(()),
_ => Err(durable_session_restore_error(
id,
"failed to bind transcript rewrite receipt",
"receipt occurrence count and last commit disagree",
)),
}
}
fn materialize_rewrite_audit_rows(
id: &SessionId,
observed_through_log_seq: u64,
rows: Vec<crate::event_store::RawTranscriptRewriteEvent>,
) -> Result<
(
Vec<(u64, meerkat_core::TranscriptRewriteRecord)>,
Vec<meerkat_core::TranscriptRewriteCommit>,
),
SessionError,
> {
let mut previous_seq = 0_u64;
let mut records = Vec::with_capacity(rows.len());
let mut commits = Vec::new();
for row in rows {
let seq = row.seq();
if seq == 0 || seq <= previous_seq || seq > observed_through_log_seq {
return Err(durable_session_restore_error(
id,
"failed to validate transcript rewrite audit row order",
format!(
"row sequence {seq} is not strictly after {previous_seq} and at or below high-water {observed_through_log_seq}"
),
));
}
previous_seq = seq;
let Some((commit_session_id, row_commits)) =
meerkat_core::event::transcript_rewrite_commits_from_payload(row.event()).map_err(
|error| {
durable_session_restore_error(
id,
"failed to read transcript rewrite audit commit",
error,
)
},
)?
else {
return Err(durable_session_restore_error(
id,
"failed to read transcript rewrite audit commit",
"raw rewrite wrapper contains a different event variant",
));
};
if commit_session_id != *id {
return Err(durable_session_restore_error(
id,
"failed to bind transcript rewrite audit commit",
format!("row belongs to session {commit_session_id}"),
));
}
commits.extend(row_commits);
let event: meerkat_core::event::AgentEvent = serde_json::from_str(row.event().get())
.map_err(|error| {
durable_session_restore_error(
id,
"failed to materialize transcript rewrite audit row",
error,
)
})?;
match event {
meerkat_core::event::AgentEvent::TranscriptRewriteCommitted {
session_id,
record,
} => {
if session_id != *id {
return Err(durable_session_restore_error(
id,
"failed to materialize transcript rewrite audit row",
format!("row belongs to session {session_id}"),
));
}
records.push((seq, record));
}
meerkat_core::event::AgentEvent::TranscriptRewriteAuditReceiptCommitted {
session_id,
..
} if session_id == *id => {}
_ => {
return Err(durable_session_restore_error(
id,
"failed to materialize transcript rewrite audit row",
"raw rewrite wrapper contains a different event variant",
));
}
}
}
Ok((records, commits))
}
fn validate_current_rewrite_record(
id: &SessionId,
record: meerkat_core::TranscriptRewriteRecord,
) -> Result<meerkat_core::TranscriptRewriteRecord, SessionError> {
meerkat_core::TranscriptRewriteRecord::new(
record.commit,
record.parent_body,
record.revision_body,
)
.map_err(|error| {
durable_session_restore_error(
id,
"failed to validate transcript rewrite audit record",
error,
)
})
}
fn prepare_authorized_tail_records(
id: &SessionId,
expected_prefix: &meerkat_core::TranscriptRewritePrefixAccumulator,
expected_last_commit: Option<&meerkat_core::TranscriptRewriteCommit>,
records: Vec<(u64, meerkat_core::TranscriptRewriteRecord)>,
receipt: &TranscriptRewritePrefixReceipt,
) -> Result<Vec<meerkat_core::TranscriptRewriteRecord>, SessionError> {
let mut accumulator = expected_prefix.clone();
let mut last_commit = expected_last_commit.cloned();
let mut unique = BTreeMap::<u64, meerkat_core::TranscriptRewriteRecord>::new();
for (_, record) in records {
let record = Self::validate_current_rewrite_record(id, record)?;
let generation = record.commit.rewrite_generation;
let current = accumulator.occurrence_count();
if generation == current {
if last_commit.as_ref() != Some(&record.commit) {
return Err(durable_session_restore_error(
id,
"failed to validate transcript rewrite audit tail",
format!("conflicting retry at occurrence generation {generation}"),
));
}
continue;
}
if current.checked_add(1) != Some(generation) {
return Err(durable_session_restore_error(
id,
"failed to validate transcript rewrite audit tail",
format!("tail jumps from generation {current} to {generation}"),
));
}
accumulator = accumulator.extend(&record.commit).map_err(|error| {
durable_session_restore_error(
id,
"failed to extend transcript rewrite audit tail",
error,
)
})?;
last_commit = Some(record.commit.clone());
unique.insert(generation, record);
}
if receipt.accumulator() != &accumulator || receipt.last_commit() != last_commit.as_ref() {
return Err(durable_session_restore_error(
id,
"failed to validate transcript rewrite audit tail",
"materialized records do not fold to the exact end receipt",
));
}
Ok(unique.into_values().collect())
}
fn prepare_full_reconciliation_records(
id: &SessionId,
records: Vec<(u64, meerkat_core::TranscriptRewriteRecord)>,
proved: Option<&meerkat_core::ValidatedTranscriptHistory>,
_observed_through_log_seq: u64,
) -> Result<Vec<meerkat_core::TranscriptRewriteRecord>, SessionError> {
let expected = proved
.map(|proved| proved.commits().cloned().collect::<Vec<_>>())
.unwrap_or_default();
let mut current = BTreeMap::<u64, meerkat_core::TranscriptRewriteRecord>::new();
let mut legacy = Vec::new();
for (_, record) in records {
if record.commit.rewrite_generation == 0 {
legacy.push(record);
continue;
}
let record = Self::validate_current_rewrite_record(id, record)?;
let generation = record.commit.rewrite_generation;
if let Some(expected_commit) = generation
.checked_sub(1)
.and_then(|index| usize::try_from(index).ok())
.and_then(|index| expected.get(index))
&& expected_commit != &record.commit
{
return Err(durable_session_restore_error(
id,
"failed to reconcile transcript rewrite audit",
format!(
"logged occurrence generation {generation} conflicts with the checkpoint-proved graph"
),
));
}
match current.entry(generation) {
std::collections::btree_map::Entry::Vacant(slot) => {
slot.insert(record);
}
std::collections::btree_map::Entry::Occupied(slot)
if slot.get().commit == record.commit => {}
std::collections::btree_map::Entry::Occupied(_) => {
return Err(durable_session_restore_error(
id,
"failed to reconcile transcript rewrite audit",
format!("conflicting rows claim occurrence generation {generation}"),
));
}
}
}
if !legacy.is_empty() && expected.is_empty() {
return Err(durable_session_restore_error(
id,
"failed to reconcile generation-zero transcript rewrite audit",
"no checkpoint-proved graph order is available",
));
}
let mut assigned_legacy = HashSet::<u64>::new();
for mut record in legacy {
let matching = expected
.iter()
.filter(|expected_commit| {
let mut generation_zero = (**expected_commit).clone();
generation_zero.rewrite_generation = 0;
generation_zero == record.commit
})
.collect::<Vec<_>>();
let Some(first_match) = matching.first().copied() else {
return Err(durable_session_restore_error(
id,
"failed to reconcile generation-zero transcript rewrite audit",
"legacy row is foreign to the checkpoint-proved graph",
));
};
let assigned = matching
.iter()
.copied()
.find(|candidate| {
!current.contains_key(&candidate.rewrite_generation)
&& !assigned_legacy.contains(&candidate.rewrite_generation)
})
.unwrap_or(first_match);
record.commit.rewrite_generation = assigned.rewrite_generation;
let record = Self::validate_current_rewrite_record(id, record)?;
assigned_legacy.insert(assigned.rewrite_generation);
current.entry(assigned.rewrite_generation).or_insert(record);
}
Ok(current.into_values().collect())
}
fn commits_missing_from_full_audit(
id: &SessionId,
state: &meerkat_core::TranscriptHistoryState,
logged: &[meerkat_core::TranscriptRewriteCommit],
) -> Result<Vec<meerkat_core::TranscriptRewriteCommit>, SessionError> {
let logged = logged
.iter()
.map(|commit| (commit.rewrite_generation, commit))
.collect::<BTreeMap<_, _>>();
let mut missing = Vec::new();
for commit in state.commits() {
match logged.get(&commit.rewrite_generation) {
Some(logged) if *logged == commit => {}
Some(_) => {
return Err(durable_session_restore_error(
id,
"failed to verify transcript rewrite audit coverage",
format!(
"logged occurrence generation {} conflicts with the graph",
commit.rewrite_generation
),
));
}
None => missing.push(commit.clone()),
}
}
if let Some(foreign) = logged.values().copied().find(|commit| {
state.commit(
usize::try_from(commit.rewrite_generation.saturating_sub(1)).unwrap_or(usize::MAX),
) != Some(*commit)
}) {
return Err(durable_session_restore_error(
id,
"failed to verify transcript rewrite audit coverage",
format!(
"logged occurrence generation {} is not represented in the materialized graph",
foreign.rewrite_generation
),
));
}
Ok(missing)
}
async fn verify_transcript_rewrite_audit_events(
&self,
session: &Session,
replay: &TranscriptRewriteReplay,
) -> Result<(), SessionError> {
self.verify_transcript_rewrite_audit_events_locked(session, replay)
.await
}
async fn verify_transcript_rewrite_audit_events_locked(
&self,
session: &Session,
replay: &TranscriptRewriteReplay,
) -> Result<(), SessionError> {
session
.validate_transcript_history_state()
.map_err(|error| {
durable_session_restore_error(
session.id(),
"failed to validate transcript history",
error,
)
})?;
let Some(audit) = replay.audit.as_ref() else {
return Ok(());
};
let Some(state) = session
.already_validated_transcript_history_state()
.map_err(|error| {
durable_session_restore_error(
session.id(),
"failed to read transcript history",
error,
)
})?
else {
return Ok(());
};
let missing = match audit {
TranscriptRewriteAuditCoverage::AuthorizedTail => return Ok(()),
TranscriptRewriteAuditCoverage::FullReconciliation { logged_commits } => {
Self::commits_missing_from_full_audit(session.id(), &state, logged_commits)?
}
};
if missing.is_empty() {
return Ok(());
}
append_transcript_rewrite_receipt_for_commits(
self.event_store.as_ref(),
self.projector.as_ref(),
session,
&missing,
)
.await
}
async fn apply_transcript_rewrite_replay(
&self,
id: &SessionId,
session: Option<Session>,
) -> Result<TranscriptRewriteReplay, SessionError> {
let Some(mut session) = session else {
return Ok(TranscriptRewriteReplay::absent());
};
let proved = session
.already_validated_transcript_history_state()
.map_err(|error| {
durable_session_restore_error(
id,
"failed to read validated transcript history",
error,
)
})?;
let prepared = self
.read_prepared_transcript_rewrite_audit(&session, proved.as_ref())
.await?;
let Some(prepared) = prepared else {
return Ok(TranscriptRewriteReplay::unchanged(session, None));
};
let PreparedTranscriptRewriteAudit {
scope,
logged_commits,
records,
receipt,
} = prepared;
let has_logged_commits = !logged_commits.is_empty();
let audit = match scope {
PreparedTranscriptRewriteAuditScope::AuthorizedTail => {
TranscriptRewriteAuditCoverage::AuthorizedTail
}
PreparedTranscriptRewriteAuditScope::FullReconciliation => {
TranscriptRewriteAuditCoverage::FullReconciliation { logged_commits }
}
};
let proved_count = proved
.as_ref()
.map(|proved| proved.rewrite_prefix().occurrence_count())
.unwrap_or(0);
let tail_records = records
.into_iter()
.filter(|record| record.commit.rewrite_generation > proved_count)
.collect::<Vec<_>>();
let prior_graph_prefix = proved
.as_ref()
.map(|history| history.graph_prefix().clone());
let prior_prefix = session.transcript_rewrite_prefix_authority();
let needs_canonical_graph_install = proved.as_ref().is_some_and(|proved| {
session.transcript_rewrite_prefix_authority().as_ref() != Some(proved.rewrite_prefix())
});
let validated_to_apply = if tail_records.is_empty() {
proved
.as_ref()
.and_then(|proved| needs_canonical_graph_install.then(|| proved.clone()))
} else {
meerkat_core::ValidatedTranscriptHistory::from_rewrite_records_with_proved(
tail_records,
proved.as_ref(),
)
.map_err(|error| {
durable_session_restore_error(
id,
"failed to rebuild transcript history from audit delta",
error,
)
})?
};
if let Some(validated) = validated_to_apply {
session
.install_validated_audited_transcript_history_preserving_live(validated)
.map_err(|error| {
durable_session_restore_error(
id,
"failed to install transcript rewrite audit",
error,
)
})?;
} else if has_logged_commits && proved.is_none() {
return Err(durable_session_restore_error(
id,
"failed to materialize transcript rewrite audit",
"audit contains rewrite occurrences but produced no graph",
));
}
if scope == PreparedTranscriptRewriteAuditScope::AuthorizedTail {
let receipt = receipt.as_ref().ok_or_else(|| {
durable_session_restore_error(
id,
"failed to verify transcript rewrite audit tail",
"authorized tail lost its end receipt",
)
})?;
let final_state = session
.validated_transcript_history_state()
.map_err(|error| {
durable_session_restore_error(
id,
"failed to read materialized transcript history",
error,
)
})?;
let final_prefix = final_state
.as_ref()
.map(|state| state.rewrite_prefix().clone())
.unwrap_or_else(meerkat_core::TranscriptRewritePrefixAccumulator::empty);
let final_last = final_state.as_ref().and_then(|state| state.last_commit());
if receipt.accumulator() != &final_prefix || receipt.last_commit() != final_last {
return Err(durable_session_restore_error(
id,
"failed to verify transcript rewrite audit tail",
"materialized graph does not end at the store-owned receipt",
));
}
}
let final_history = session
.already_validated_transcript_history_state()
.map_err(|error| {
durable_session_restore_error(
id,
"failed to read final transcript history after audit replay",
error,
)
})?;
let final_graph_prefix = final_history
.as_ref()
.map(|history| history.graph_prefix().clone());
let materialized_from_replay = prior_graph_prefix != final_graph_prefix
|| prior_prefix != session.transcript_rewrite_prefix_authority();
if let Some(receipt) = receipt.as_ref() {
self.finalize_prepared_transcript_rewrite_audit(id, receipt)
.await?;
}
Ok(TranscriptRewriteReplay {
session: Some(session),
materialized_from_replay,
audit: Some(audit),
})
}
#[cfg(test)]
async fn load_runtime_session_snapshot(
runtime_store: &Arc<dyn RuntimeStore>,
runtime_id: &LogicalRuntimeId,
) -> Result<Option<Session>, SessionError> {
runtime_store
.load_session_snapshot(runtime_id)
.await
.map_err(|err| {
SessionError::Agent(meerkat_core::error::AgentError::InternalError(format!(
"failed to load runtime session snapshot: {err}"
)))
})?
.map(|bytes| {
Session::from_persisted_bytes(&bytes).map_err(|err| {
SessionError::Store(Box::new(SessionStoreError::Serialization(format!(
"failed to deserialize runtime session snapshot for {runtime_id}: {err}"
))))
})
})
.transpose()
}
#[cfg(test)]
async fn load_runtime_session_snapshot_for_session(
runtime_store: &Arc<dyn RuntimeStore>,
id: &SessionId,
) -> Result<Option<Session>, SessionError> {
let runtime_id = Self::runtime_id_for_session(id);
Self::load_runtime_session_snapshot(runtime_store, &runtime_id).await
}
async fn load_committed_runtime_session_for_body(
&self,
id: &SessionId,
role: &str,
) -> Result<Option<Session>, SessionError> {
match self.runtime_store.session_persistence_profile() {
RuntimeSessionPersistenceProfile::WholeBlobV1 => {
load_committed_whole_blob_session(self.runtime_store.as_ref(), id, role)
.await
.map(|loaded| loaded.map(|(session, _snapshot)| session))
}
RuntimeSessionPersistenceProfile::HeadCanonicalV1 => {
let authority = self
.runtime_store
.load_session_boundary_authority(&Self::runtime_id_for_session(id))
.await
.map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"failed to load {role} HeadCanonical authority for session {id}: {error}"
)))
})?;
let Some(authority) = authority else {
return Ok(None);
};
let authority = authority.head_canonical().ok_or_else(|| {
SessionError::Agent(AgentError::InternalError(format!(
"{role} loaded non-HeadCanonical authority for session {id}"
)))
})?;
if authority.session_id() != id {
return Err(SessionError::Agent(AgentError::InternalError(format!(
"{role} HeadCanonical authority identifies session {}, not {id}",
authority.session_id()
))));
}
let incremental = self.incremental.as_ref().ok_or_else(|| {
SessionError::Agent(AgentError::InternalError(format!(
"{role} cannot materialize HeadCanonical session {id} without incremental store capability"
)))
})?;
incremental
.materialize_head(authority.boundary_head())
.await
.map(|materialized| Some(materialized.session().as_ref().clone()))
.map_err(|error| SessionError::Store(Box::new(error)))
}
profile => Err(SessionError::Agent(AgentError::InternalError(format!(
"unsupported runtime session persistence profile {profile} while loading {role} for session {id}"
)))),
}
}
async fn load_runtime_state_for_session(
runtime_store: &Arc<dyn RuntimeStore>,
id: &SessionId,
) -> Result<Option<RuntimeState>, SessionError> {
let runtime_id = Self::runtime_id_for_session(id);
meerkat_runtime::store::load_runtime_state(runtime_store.as_ref(), &runtime_id)
.await
.map_err(|err| {
SessionError::Agent(AgentError::InternalError(format!(
"failed to load runtime state: {err}"
)))
})
}
async fn load_runtime_authority_session_for_control(
&self,
id: &SessionId,
operation: &str,
) -> Result<Option<Session>, SessionError> {
if let Some(runtime) = self
.load_committed_runtime_session_for_body(id, operation)
.await?
{
return Ok(Some(runtime));
}
if matches!(
Self::load_runtime_state_for_session(&self.runtime_store, id).await?,
Some(RuntimeState::Retired)
) {
return Err(SessionError::NotFound { id: id.clone() });
}
Ok(None)
}
async fn load_persisted_session_for_control(
&self,
id: &SessionId,
operation: &str,
) -> Result<Option<Session>, SessionError> {
self.load_runtime_authority_session_for_control(id, operation)
.await
}
async fn load_persisted_session_for_archive(
&self,
id: &SessionId,
) -> Result<Option<Session>, SessionError> {
self.load_committed_runtime_session_for_body(id, "session archive")
.await
}
async fn live_session_authority(
&self,
id: &SessionId,
) -> Result<LiveSessionAuthority, SessionError> {
loop {
let live_authority = match self.inner.observe_session_transcript_authority(id).await {
Ok(authority) => authority,
Err(SessionError::NotFound { .. }) => {
return Ok(LiveSessionAuthority::NoLive);
}
Err(error) => return Err(error),
};
if live_authority.session_id() != id {
return Err(SessionError::Agent(AgentError::InternalError(format!(
"live transcript authority identifies session {}, not {id}",
live_authority.session_id()
))));
}
if self.runtime_store.session_persistence_profile()
== RuntimeSessionPersistenceProfile::HeadCanonicalV1
{
let runtime_state =
Self::load_runtime_state_for_session(&self.runtime_store, id).await?;
let active_lifecycle = matches!(
runtime_state,
Some(RuntimeState::Idle | RuntimeState::Attached | RuntimeState::Running)
);
let store_authority = self.observe_persisted_session_authority(id).await?;
if active_lifecycle
&& let Some(RuntimeSessionAuthority::HeadCanonical(store_authority)) =
store_authority
&& store_authority.session_id() == id
&& usize::try_from(store_authority.boundary_head().message_count).ok()
== Some(live_authority.message_count())
&& store_authority.boundary_head().head_revision.as_str()
== live_authority.transcript_revision()
{
let (kind, _) = Self::classify_live_session_authority_observations(
id, false, false, false,
)?;
return match kind {
LiveSessionAuthorityKind::LiveAuthoritative => {
Ok(LiveSessionAuthority::LiveAuthoritative {
snapshot: live_authority,
})
}
LiveSessionAuthorityKind::DurableAuthoritative => {
Err(SessionError::Agent(AgentError::InternalError(format!(
"generated session document authority rejected an exact active live/store boundary for session {id}"
))))
}
};
}
}
let live = match self
.export_session_with_labels_if_transcript_authority(id, live_authority.clone())
.await
{
Ok(Some(session)) => session,
Ok(None) => continue,
Err(SessionError::NotFound { .. }) => {
return Ok(LiveSessionAuthority::NoLive);
}
Err(error) => return Err(error),
};
let Some(stored) = self
.load_committed_runtime_session_for_body(
id,
"live session authority classification",
)
.await?
else {
return Ok(LiveSessionAuthority::LiveAuthoritative {
snapshot: live_authority,
});
};
let stored_revision = stored.transcript_revision().map_err(|err| {
SessionError::Agent(AgentError::InternalError(format!(
"failed to read durable transcript revision for session {id}: {err}"
)))
})?;
let live_revision = live.transcript_revision().map_err(|err| {
SessionError::Agent(AgentError::InternalError(format!(
"failed to read live transcript revision for session {id}: {err}"
)))
})?;
let stored_transcript_diverged = stored_revision != live_revision;
let live_has_uncommitted_transcript = live.messages().len() > stored.messages().len();
let stored_is_archived = self.session_archived_by_authority(id, &stored).await?;
let (kind, reason) = Self::classify_live_session_authority_observations(
id,
stored_transcript_diverged,
live_has_uncommitted_transcript,
stored_is_archived,
)?;
return match kind {
LiveSessionAuthorityKind::LiveAuthoritative => {
Ok(LiveSessionAuthority::LiveAuthoritative {
snapshot: live_authority,
})
}
LiveSessionAuthorityKind::DurableAuthoritative => {
Ok(LiveSessionAuthority::DurableAuthoritative {
session: Box::new(stored),
reason,
})
}
};
}
}
fn classify_live_session_authority_observations(
id: &SessionId,
stored_transcript_diverged: bool,
live_has_uncommitted_transcript: bool,
stored_is_archived: bool,
) -> Result<(LiveSessionAuthorityKind, LiveSessionAuthorityReason), SessionError> {
let mut authority = SessionDocumentMachineAuthority::new();
let effects = authority
.classify_live_session_authority(
stored_transcript_diverged,
live_has_uncommitted_transcript,
stored_is_archived,
)
.map_err(|err| {
SessionError::Agent(AgentError::InternalError(format!(
"generated session document authority rejected live-session authority \
classification for session {id}: {err}"
)))
})?;
let (kind, reason) = effects
.iter()
.find_map(|effect| match effect {
SessionDocumentEffect::LiveSessionAuthorityClassified { authority, reason } => {
Some((*authority, *reason))
}
_ => None,
})
.ok_or_else(|| {
SessionError::Agent(AgentError::InternalError(format!(
"generated session document authority returned no live-session authority \
verdict for session {id}"
)))
})?;
Ok((kind, reason))
}
async fn store_owned_durable_tail_recovery_outcome(
&self,
id: &SessionId,
) -> Result<meerkat_runtime::recovery::DurableTailRecoveryOutcome, SessionError> {
meerkat_runtime::recovery::recover_durable_tail(self.runtime_store.as_ref(), id)
.await
.map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"durable-tail recovery for session {id}: {error}"
)))
})
}
async fn discard_stale_live_session_if_needed(
&self,
id: &SessionId,
) -> Result<bool, SessionError> {
let LiveSessionAuthority::DurableAuthoritative { session, reason } =
self.live_session_authority(id).await?
else {
return Ok(false);
};
if self
.synchronize_runtime_backed_live_from_durable_authority(id, session.as_ref(), reason)
.await?
{
return Ok(false);
}
let live = match self.export_session_with_labels(id).await {
Ok(session) => session,
Err(SessionError::NotFound { .. }) => return Ok(false),
Err(err) => return Err(err),
};
tracing::debug!(
session_id = %id,
live_updated_at = ?live.updated_at(),
stored_updated_at = ?session.updated_at(),
live_message_count = live.messages().len(),
stored_message_count = session.messages().len(),
reason = ?reason,
"discarding stale live session in favor of newer RuntimeStore authority"
);
self.discard_live_session_unfenced(id).await?;
Ok(true)
}
async fn synchronize_live_session_from_durable(
&self,
id: &SessionId,
durable: &Session,
reason: LiveSessionSyncCause,
) -> Result<(), SessionError> {
let (durable, whole_blob_authority) = if self.runtime_store.session_persistence_profile()
== RuntimeSessionPersistenceProfile::WholeBlobV1
{
let (session, snapshot) = load_committed_whole_blob_session(
self.runtime_store.as_ref(),
id,
"live session synchronization",
)
.await?
.ok_or_else(|| SessionError::NotFound { id: id.clone() })?;
(session, Some(snapshot.authority().clone()))
} else {
(durable.clone(), None)
};
self.inner
.sync_session_from_durable_snapshot(id, durable)
.await?;
if let Some(authority) = whole_blob_authority
&& let Some(checkpointer) = self
.live_checkpointers
.lock()
.await
.get(id)
.cloned()
.and_then(|checkpointer| checkpointer.upgrade())
{
*checkpointer.latest_run_checkpoint_receipt.lock().await = None;
*checkpointer.whole_blob_base_authority.lock().map_err(|_| {
SessionError::Agent(AgentError::InternalError(format!(
"WholeBlob actor base authority lock is poisoned for session {id}"
)))
})? = Some(authority);
}
tracing::debug!(
session_id = %id,
reason = reason.trace_label(),
"synchronized live session snapshot from durable realtime authority"
);
Ok(())
}
async fn synchronize_runtime_backed_live_from_durable_authority(
&self,
id: &SessionId,
durable: &Session,
reason: LiveSessionAuthorityReason,
) -> Result<bool, SessionError> {
if self.session_archived_by_authority(id, durable).await? {
return Ok(false);
}
match self
.synchronize_live_session_from_durable(
id,
durable,
LiveSessionSyncCause::Authority(reason),
)
.await
{
Ok(()) => {}
Err(error) if is_durable_session_sync_unsupported(&error) => return Ok(false),
Err(error) => return Err(error),
}
Ok(true)
}
pub async fn synchronize_live_session_from_durable_authority_if_needed(
&self,
id: &SessionId,
) -> Result<bool, SessionError> {
let recovery_gate = self.recovery_gate_for_session(id).await;
let _recovery_guard = recovery_gate.lock().await;
match self.live_session_authority(id).await? {
LiveSessionAuthority::DurableAuthoritative { session, reason } => {
self.synchronize_runtime_backed_live_from_durable_authority(
id,
session.as_ref(),
reason,
)
.await
}
LiveSessionAuthority::NoLive | LiveSessionAuthority::LiveAuthoritative { .. } => {
Ok(false)
}
}
}
async fn recover_live_session_from_store_if_needed_locked(
&self,
id: &SessionId,
) -> Result<bool, SessionError> {
if self.inner.has_live_session(id).await? {
return Ok(false);
}
let Some(stored) = self.load_authoritative_session_base(id).await? else {
return Ok(false);
};
self.reject_if_archived_session(id, &stored)
.await
.map_err(crate::control_error_into_session_error)?;
let _ = stored;
Err(SessionError::Agent(AgentError::InternalError(
"stored-session recovery via non-canonical runtime-binding providers has been deleted; callers must materialize sessions through the canonical runtime-binding seam".to_string(),
)))
}
pub async fn export_live_session(&self, id: &SessionId) -> Result<Session, SessionError> {
loop {
match self.live_session_authority(id).await? {
LiveSessionAuthority::NoLive
| LiveSessionAuthority::DurableAuthoritative { .. } => {
return Err(SessionError::NotFound { id: id.clone() });
}
LiveSessionAuthority::LiveAuthoritative { snapshot } => {
if let Some(session) = self
.export_session_with_labels_if_transcript_authority(id, snapshot)
.await?
{
return Ok(session);
}
}
}
}
}
async fn source_session_for_transcript_edit(
&self,
id: &SessionId,
) -> Result<Session, SessionError> {
let _mutation_guard = self.transcript_edit_mutation_guard(id).await?;
self.source_session_for_transcript_edit_locked(id).await
}
async fn source_session_for_transcript_edit_locked(
&self,
id: &SessionId,
) -> Result<Session, SessionError> {
let view = self.read(id).await?;
if view.state.is_active {
return Err(SessionError::Busy { id: id.clone() });
}
let mut replay = self
.load_authoritative_session_base_with_replay_info(id)
.await?;
let session = match replay.session.take() {
Some(session) => {
if replay.materialized_from_replay {
self.persist_replayed_transcript_projection_for_mutation(&session)
.await?;
}
self.verify_transcript_rewrite_audit_events_locked(&session, &replay)
.await?;
session
}
None => match self.export_session_with_labels(id).await {
Ok(session) => session,
Err(SessionError::NotFound { .. }) => {
return Err(SessionError::NotFound { id: id.clone() });
}
Err(err) => return Err(err),
},
};
self.reject_if_archived_session(id, &session)
.await
.map_err(crate::control_error_into_session_error)?;
Ok(session)
}
async fn persist_replayed_transcript_projection_for_mutation(
&self,
session: &Session,
) -> Result<(), SessionError> {
let session = self
.normalized_session_for_persistence(session.clone())
.await?;
let runtime_session_snapshot =
serialize_session_document(&session, "replayed transcript projection")?;
self.runtime_store
.commit_session_snapshot(
&Self::runtime_id_for_session(session.id()),
SerializedSessionSnapshot {
session_snapshot: runtime_session_snapshot.bytes_arc(),
},
)
.await
.map_err(|err| {
SessionError::Agent(meerkat_core::error::AgentError::InternalError(format!(
"runtime replay projection persistence failed: {err}"
)))
})?;
Ok(())
}
async fn transcript_edit_mutation_guard(
&self,
id: &SessionId,
) -> Result<SessionMutationGuard, SessionError> {
match self.inner.join_active_runtime_context_admission(id).await {
Ok(Some(active_admission)) => {
drop(active_admission);
return Err(SessionError::Busy { id: id.clone() });
}
Ok(None) | Err(SessionError::NotFound { .. }) => {}
Err(error) => return Err(error),
}
let turn_finalization_guard = self.acquire_runtime_turn_finalization_guard(id).await;
let recovery_gate = self.recovery_gate_for_session(id).await;
let recovery_guard = recovery_gate.lock_owned().await;
match self.inner.join_active_runtime_context_admission(id).await {
Ok(Some(active_admission)) => {
drop(active_admission);
return Err(SessionError::Busy { id: id.clone() });
}
Ok(None) | Err(SessionError::NotFound { .. }) => {}
Err(error) => return Err(error),
}
let _ = self.discard_stale_live_session_if_needed(id).await?;
Ok(SessionMutationGuard {
_turn_finalization_guard: turn_finalization_guard,
_recovery_guard: recovery_guard,
})
}
fn authorize_transcript_edit(
&self,
id: &SessionId,
directive: TranscriptEditKind,
) -> Result<(), SessionError> {
let mut authority = SessionDocumentMachineAuthority::new();
let effects = authority
.transcript_edit(SessionDocumentKey::new(id.to_string()), directive)
.map_err(|err| {
SessionError::Agent(AgentError::InternalError(format!(
"generated session document authority rejected transcript edit for \
session {id}: {err}"
)))
})?;
let (kind, success) = effects
.iter()
.find_map(|effect| match effect {
SessionDocumentEffect::TranscriptRewriteCommitted { kind, success } => {
Some((*kind, *success))
}
_ => None,
})
.ok_or_else(|| {
SessionError::Agent(AgentError::InternalError(format!(
"generated session document authority returned no transcript edit commit \
verdict for session {id}"
)))
})?;
if kind != directive {
return Err(SessionError::Agent(AgentError::InternalError(format!(
"generated session document authority echoed transcript edit directive {kind:?} \
for requested {directive:?} on session {id}"
))));
}
if !success {
return Err(SessionError::Agent(AgentError::InternalError(format!(
"generated session document authority rejected transcript edit commit for \
session {id}"
))));
}
Ok(())
}
async fn persist_transcript_fork(
&self,
source_session_id: SessionId,
mut forked: Session,
source_metadata: Option<meerkat_core::SessionMetadata>,
requested_tool_access_policy: Option<meerkat_core::ops::ToolAccessPolicy>,
) -> Result<SessionForkResult, SessionError> {
if let Some(mut metadata) = source_metadata {
let effective_policy = match requested_tool_access_policy {
Some(meerkat_core::ops::ToolAccessPolicy::Inherit) | None => {
metadata.tooling.tool_access_policy.clone()
}
Some(policy) => Some(policy),
};
metadata.comms_name = None;
metadata.peer_meta = None;
metadata.mob_member_binding = None;
metadata.keep_alive = false;
metadata.tooling.tool_access_policy = effective_policy;
forked.set_session_metadata(metadata).map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"failed to persist fork tool authorization for session {source_session_id}: {error}"
)))
})?;
} else if requested_tool_access_policy
.is_some_and(|policy| !matches!(policy, meerkat_core::ops::ToolAccessPolicy::Inherit))
{
return Err(SessionError::Agent(AgentError::InternalError(format!(
"session {source_session_id} has no canonical session metadata for fork tool authorization"
))));
}
let saved = match self.runtime_store.session_persistence_profile() {
RuntimeSessionPersistenceProfile::WholeBlobV1 => {
self.save_normalized_session(forked).await?
}
RuntimeSessionPersistenceProfile::HeadCanonicalV1 => {
self.persist_detached_head_canonical_session(forked, "transcript fork root")
.await?
}
profile => {
return Err(SessionError::Agent(AgentError::InternalError(format!(
"unsupported runtime session persistence profile {profile} while persisting transcript fork"
))));
}
};
Ok(SessionForkResult {
source_session_id,
session_id: saved.id().clone(),
message_count: saved.messages().len(),
session_ref: None,
})
}
async fn persist_transcript_rewrite(
&self,
session: Session,
commit: &meerkat_core::TranscriptRewriteCommit,
) -> Result<Session, SessionError> {
let session = self.normalized_session_for_persistence(session).await?;
self.persist_normalized_transcript_rewrite(session, commit, true)
.await
}
async fn persist_normalized_transcript_rewrite(
&self,
session: Session,
commit: &meerkat_core::TranscriptRewriteCommit,
converge_live: bool,
) -> Result<Session, SessionError> {
self.persist_normalized_transcript_rewrite_chain(
session,
std::slice::from_ref(commit),
converge_live,
)
.await
}
async fn persist_normalized_transcript_rewrite_chain(
&self,
session: Session,
commits: &[meerkat_core::TranscriptRewriteCommit],
converge_live: bool,
) -> Result<Session, SessionError> {
if commits.is_empty() {
return Ok(session);
}
if self.runtime_store.session_persistence_profile()
== RuntimeSessionPersistenceProfile::HeadCanonicalV1
{
let persisted = self
.persist_detached_head_canonical_session(session, "same-session transcript rewrite")
.await?;
append_transcript_rewrite_receipt_for_commits(
self.event_store.as_ref(),
self.projector.as_ref(),
&persisted,
commits,
)
.await?;
if converge_live {
self.converge_live_session_after_transcript_rewrite(&persisted)
.await?;
}
return Ok(persisted);
}
if rewrite_materialize_trace_enabled() {
tracing::info!(
session_id = %session.id(),
commits = commits.len(),
converge_live,
"rewrite-chain: preparing one exact occurrence suffix boundary"
);
}
let runtime_id = Self::runtime_id_for_session(session.id());
let committed_snapshot = self
.runtime_store
.load_committed_whole_blob_snapshot(&runtime_id)
.await
.map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"failed to load pre-rewrite WholeBlob authority for session {}: {error}",
session.id()
)))
})?
.ok_or_else(|| {
SessionError::Agent(AgentError::InternalError(format!(
"cannot persist transcript rewrite for session {} without WholeBlob runtime authority",
session.id()
)))
})?;
let expected_runtime =
VerifiedCommittedWholeBlobPayload::from_committed(session.id(), committed_snapshot)
.map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"failed to verify pre-rewrite WholeBlob authority for session {}: {error}",
session.id()
)))
})?;
let boundary =
PreparedWholeBlobRewriteBoundary::prepare(expected_runtime, session, commits).map_err(
|error| {
SessionError::Agent(AgentError::InternalError(format!(
"failed to prepare one-CAS WholeBlob rewrite boundary: {error}"
)))
},
)?;
let committed_authority = self
.runtime_store
.commit_prepared_whole_blob_rewrite_boundary(&runtime_id, boundary.store_parts())
.await
.map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"WholeBlob transcript rewrite CAS failed: {error}"
)))
})?;
if !boundary.accepts_committed_authority(&committed_authority) {
return Err(SessionError::Agent(AgentError::InternalError(format!(
"WholeBlob store returned authority other than the prepared exact successor for session {}",
boundary.successor().id()
))));
}
let successor = boundary.successor();
let final_assistant_text = successor.last_assistant_text();
if let Err(error) = append_prepared_transcript_rewrite_receipt(
self.event_store.as_ref(),
self.projector.as_ref(),
successor.id(),
boundary.audit_receipt(),
final_assistant_text.as_deref(),
)
.await
{
tracing::error!(
session_id = %successor.id(),
error = %error,
"WholeBlob rewrite authority committed; audit receipt remains pending repair"
);
match self.discard_live_session_unfenced(successor.id()).await {
Ok(()) | Err(SessionError::NotFound { .. }) => {}
Err(discard_error) => {
tracing::warn!(
session_id = %successor.id(),
error = %discard_error,
"failed to discard live session after committed rewrite receipt deferral"
);
}
}
return Err(error);
}
let session = boundary.into_successor().map_err(|shared| {
SessionError::Agent(AgentError::InternalError(format!(
"prepared WholeBlob successor retained {} unexpected strong references",
Arc::strong_count(&shared)
)))
})?;
if converge_live {
self.converge_live_session_after_transcript_rewrite(&session)
.await?;
}
Ok(session)
}
async fn converge_live_session_after_transcript_rewrite(
&self,
session: &Session,
) -> Result<(), SessionError> {
if !self.inner.has_live_session(session.id()).await? {
return Ok(());
}
match self
.synchronize_live_session_from_durable(
session.id(),
session,
LiveSessionSyncCause::TranscriptRewriteCommitted,
)
.await
{
Ok(()) => Ok(()),
Err(error) if is_durable_session_sync_unsupported(&error) => {
self.discard_live_session_unfenced(session.id()).await
}
Err(error) => Err(error),
}
}
async fn normalized_transcript_rewrite_replacement(
&self,
mut replacement: Vec<meerkat_core::Message>,
) -> Result<Vec<meerkat_core::Message>, SessionError> {
externalize_messages_from(self.blob_store.as_ref(), &mut replacement, 0)
.await
.map_err(|err| {
SessionError::Agent(meerkat_core::error::AgentError::InternalError(format!(
"failed to externalize transcript rewrite replacement media: {err}"
)))
})?;
Ok(replacement)
}
pub async fn export_realtime_open_session_snapshot(
&self,
id: &SessionId,
) -> Result<Session, SessionError> {
self.export_realtime_open_session_snapshot_with_image_usage(id)
.await
.map(|(session, _)| session)
}
pub async fn export_realtime_open_session_snapshot_with_image_usage(
&self,
id: &SessionId,
) -> Result<(Session, usize), SessionError> {
self.export_realtime_open_session_snapshot_with_image_budget(
id,
MAX_REALTIME_USER_IMAGE_PROJECTION_BYTES,
)
.await
}
async fn export_realtime_open_session_snapshot_with_image_budget(
&self,
id: &SessionId,
max_image_bytes: usize,
) -> Result<(Session, usize), SessionError> {
let _mutation_guard = self.realtime_transcript_mutation_guard(id).await?;
let mut session = self.export_realtime_session_authority_snapshot(id).await?;
if session.pending_realtime_user_content_blob().is_some() {
let outcome = self
.reconcile_pending_realtime_user_content_blob(id, &mut session, None)
.await?;
if outcome.is_some() {
return Err(realtime_user_content_blob_error(
"realtime-open pending image recovery returned a request-scoped receipt",
"open recovery must reconcile before provider projection without a caller receipt",
));
}
}
let canonical_user_image_decoded_bytes = session
.hydrate_realtime_user_images_with_usage(self.blob_store.as_ref(), max_image_bytes)
.await
.map_err(|err| {
SessionError::Agent(meerkat_core::error::AgentError::InternalError(format!(
"failed to hydrate realtime history images before provider projection: {err}"
)))
})?;
Ok((session, canonical_user_image_decoded_bytes))
}
pub async fn export_realtime_refresh_session_snapshot(
&self,
id: &SessionId,
) -> Result<Session, SessionError> {
let _mutation_guard = self.realtime_transcript_mutation_guard(id).await?;
self.export_realtime_session_authority_snapshot(id).await
}
async fn export_realtime_session_authority_snapshot(
&self,
id: &SessionId,
) -> Result<Session, SessionError> {
loop {
match self.live_session_authority(id).await? {
LiveSessionAuthority::DurableAuthoritative { session, reason } => {
if self
.session_archived_by_authority(id, session.as_ref())
.await?
{
return Err(SessionError::NotFound { id: id.clone() });
}
self.synchronize_live_session_from_durable(
id,
session.as_ref(),
LiveSessionSyncCause::Authority(reason),
)
.await?;
tracing::debug!(
session_id = %id,
reason = ?reason,
"using durable session authority for realtime open snapshot"
);
return Ok(*session);
}
LiveSessionAuthority::NoLive => {
return Err(SessionError::NotFound { id: id.clone() });
}
LiveSessionAuthority::LiveAuthoritative { snapshot } => {
if let Some(session) = self
.export_session_with_labels_if_transcript_authority(id, snapshot)
.await?
{
return Ok(session);
}
}
}
}
}
pub async fn wait_for_session_mutation_after(
&self,
id: &SessionId,
after: std::time::SystemTime,
) -> Result<std::time::SystemTime, meerkat_core::comms::StreamError> {
self.inner.wait_for_session_mutation_after(id, after).await
}
pub async fn execution_snapshot(
&self,
id: &SessionId,
) -> Result<Option<meerkat_core::AgentExecutionSnapshot>, SessionError> {
self.inner.execution_snapshot(id).await
}
pub async fn tool_scope_snapshot(
&self,
id: &SessionId,
) -> Result<Option<meerkat_core::ToolScopeSnapshot>, SessionError> {
self.inner.tool_scope_snapshot(id).await
}
pub async fn live_visible_tool_defs(
&self,
id: &SessionId,
) -> Result<Vec<meerkat_core::ToolDef>, SessionError> {
self.inner.live_visible_tool_defs(id).await
}
pub async fn external_tool_surface_snapshot(
&self,
id: &SessionId,
) -> Result<Option<meerkat_core::ExternalToolSurfaceSnapshot>, SessionError> {
self.inner.external_tool_surface_snapshot(id).await
}
pub async fn live_session_llm_identity(
&self,
id: &SessionId,
) -> Result<meerkat_core::SessionLlmIdentity, SessionError> {
self.inner.live_session_llm_identity(id).await
}
pub async fn has_live_session_under_runtime_turn_boundary(
&self,
id: &SessionId,
) -> Result<bool, SessionError> {
match self.live_session_authority(id).await? {
LiveSessionAuthority::NoLive => Ok(false),
LiveSessionAuthority::LiveAuthoritative { .. } => Ok(true),
LiveSessionAuthority::DurableAuthoritative { session, .. } => {
if self
.session_archived_by_authority(id, session.as_ref())
.await?
{
self.discard_live_session_under_runtime_turn_boundary(id)
.await?;
Ok(false)
} else {
Ok(true)
}
}
}
}
pub async fn discard_live_session(&self, id: &SessionId) -> Result<(), SessionError> {
self.discard_live_session_under_runtime_turn_boundary(id)
.await
}
pub async fn discard_live_session_after_runtime_stop_terminalized(
&self,
id: &SessionId,
) -> Result<(), SessionError> {
let _turn_finalization_guard = self.acquire_runtime_turn_finalization_guard(id).await;
self.discard_live_session_after_runtime_stop_terminalized_under_runtime_turn_boundary(id)
.await
}
pub async fn discard_live_session_after_runtime_stop_terminalized_under_runtime_turn_boundary(
&self,
id: &SessionId,
) -> Result<(), SessionError> {
let recovery_gate = self.recovery_gate_for_session(id).await;
let _guard = recovery_gate.lock_owned().await;
self.discard_live_session_unfenced(id).await
}
async fn discard_live_session_unfenced(&self, id: &SessionId) -> Result<(), SessionError> {
let result = self.inner.discard_live_session(id).await;
self.checkpointer_gates.lock().await.remove(id);
self.live_checkpointers.lock().await.remove(id);
result
}
pub async fn persist_live_session_now(&self, id: &SessionId) -> Result<usize, SessionError> {
let _mutation_guard = self.live_persist_mutation_guard(id).await?;
self.persist_full_session(id)
.await
.map(|(message_count, _revision)| message_count)
}
pub async fn persist_live_session_now_under_runtime_turn_boundary(
&self,
id: &SessionId,
) -> Result<usize, SessionError> {
let _recovery_guard = self.recovery_gate_for_session(id).await.lock_owned().await;
let _ = self.discard_stale_live_session_if_needed(id).await?;
if let Some(session) = self.load_authoritative_session_base(id).await? {
self.reject_if_archived_session(id, &session)
.await
.map_err(crate::control_error_into_session_error)?;
}
self.persist_full_session(id)
.await
.map(|(message_count, _revision)| message_count)
}
pub async fn apply_runtime_session_tool_visibility_state_under_runtime_turn_boundary(
&self,
id: &SessionId,
state: Option<meerkat_core::SessionToolVisibilityState>,
) -> Result<(), SessionError> {
let _recovery_guard = self.recovery_gate_for_session(id).await.lock_owned().await;
let _ = self.discard_stale_live_session_if_needed(id).await?;
if let Some(session) = self.load_authoritative_session_base(id).await? {
self.reject_if_archived_session(id, &session)
.await
.map_err(crate::control_error_into_session_error)?;
}
self.inner
.set_session_tool_visibility_state(id, state)
.await
}
pub async fn discard_live_session_under_runtime_turn_boundary(
&self,
id: &SessionId,
) -> Result<(), SessionError> {
let _recovery_guard = self.recovery_gate_for_session(id).await.lock_owned().await;
self.discard_live_session_unfenced(id).await
}
pub async fn dispatch_external_tool_call(
&self,
id: &SessionId,
call: meerkat_core::ToolCall,
) -> Result<meerkat_core::ops::ToolDispatchOutcome, SessionError> {
self.dispatch_external_tool_call_with_timeout_policy(
id,
call,
meerkat_core::ToolDispatchTimeoutPolicy::Disabled,
)
.await
}
pub async fn dispatch_external_tool_call_with_timeout_policy(
&self,
id: &SessionId,
call: meerkat_core::ToolCall,
timeout_policy: meerkat_core::ToolDispatchTimeoutPolicy,
) -> Result<meerkat_core::ops::ToolDispatchOutcome, SessionError> {
let _mutation_guard = self.live_persist_mutation_guard(id).await?;
let outcome = self
.inner
.dispatch_external_tool_call_with_timeout_policy(id, call, timeout_policy)
.await?;
if let Err(error) = self.persist_full_session(id).await {
let _ = self.discard_live_session_unfenced(id).await;
return Err(error);
}
Ok(outcome)
}
pub async fn append_external_user_content(
&self,
id: &SessionId,
content: meerkat_core::types::ContentInput,
) -> Result<(), SessionError> {
let _mutation_guard = self.live_persist_mutation_guard(id).await?;
self.inner.append_external_user_content(id, content).await?;
if let Err(error) = self.persist_full_session(id).await {
let _ = self.discard_live_session_unfenced(id).await;
return Err(error);
}
Ok(())
}
pub async fn append_external_assistant_output(
&self,
id: &SessionId,
blocks: Vec<meerkat_core::types::AssistantBlock>,
stop_reason: meerkat_core::types::StopReason,
usage: meerkat_core::types::Usage,
) -> Result<(), SessionError> {
let _mutation_guard = self.live_persist_mutation_guard(id).await?;
self.inner
.append_external_assistant_output(id, blocks, stop_reason, usage)
.await?;
if let Err(error) = self.persist_full_session(id).await {
let _ = self.discard_live_session_unfenced(id).await;
return Err(error);
}
Ok(())
}
pub async fn append_realtime_transcript_event(
&self,
id: &SessionId,
event: meerkat_core::RealtimeTranscriptEvent,
) -> Result<meerkat_core::RealtimeTranscriptApplyOutcome, SessionError> {
let _mutation_guard = self.realtime_transcript_mutation_guard(id).await?;
let mut current = self.inner.export_session(id).await?;
if let Some(user_content) = current.preflight_realtime_user_content_event(&event) {
if matches!(
user_content,
meerkat_core::RealtimeUserContentApplyOutcome::AlreadyCommitted(_)
) {
let prepared = self
.prepare_realtime_user_content_blob(&event)
.await?
.ok_or_else(|| {
realtime_user_content_blob_error(
"already-committed realtime user content was not a canonical image",
"missing image ingress",
)
})?;
self.ensure_prepared_realtime_user_content_blob(&prepared)
.await?;
}
return Ok(user_content_outcome(user_content));
}
let prepared = self.prepare_realtime_user_content_blob(&event).await?;
if let Some(outcome) = self
.reconcile_pending_realtime_user_content_blob(id, &mut current, prepared.as_ref())
.await?
{
return Ok(outcome);
}
if let Some(user_content) = current.preflight_realtime_user_content_event(&event) {
if matches!(
user_content,
meerkat_core::RealtimeUserContentApplyOutcome::AlreadyCommitted(_)
) {
let prepared = prepared.as_ref().ok_or_else(|| {
realtime_user_content_blob_error(
"reconciled realtime user content was not a canonical image",
"missing image ingress",
)
})?;
self.ensure_prepared_realtime_user_content_blob(prepared)
.await?;
}
return Ok(user_content_outcome(user_content));
}
let Some(prepared) = prepared else {
let outcome = self
.inner
.append_realtime_transcript_event(id, event)
.await?;
if let Err(error) = self.persist_full_session(id).await {
let _ = self.discard_live_session_unfenced(id).await;
return Err(error);
}
return Ok(outcome);
};
use meerkat_core::generated::session_document::RealtimeUserContentBlobStageDisposition;
let stage = current
.stage_pending_realtime_user_content_blob(prepared.pending.clone())
.map_err(|error| {
realtime_user_content_blob_error(
"failed to authorize realtime user-content blob staging",
error,
)
})?;
if stage != RealtimeUserContentBlobStageDisposition::StageNew {
return Err(realtime_user_content_blob_error(
"realtime user-content blob staging did not acquire an empty slot",
format!("unexpected stage disposition {stage:?}"),
));
}
self.persist_realtime_transcript_snapshot(id, ¤t)
.await?;
self.ensure_prepared_realtime_user_content_blob(&prepared)
.await?;
let outcome = self
.commit_pending_realtime_user_content_blob(id, &mut current, &prepared)
.await?;
self.ensure_prepared_realtime_user_content_blob(&prepared)
.await?;
Ok(outcome)
}
async fn prepare_realtime_user_content_blob(
&self,
event: &meerkat_core::RealtimeTranscriptEvent,
) -> Result<Option<PreparedRealtimeUserContentBlob>, SessionError> {
let meerkat_core::RealtimeTranscriptEvent::UserContentFinal {
idempotency_key,
item_id,
previous_item_id,
content_index,
content,
} = event
else {
return Ok(None);
};
let [meerkat_core::types::ContentBlock::Image { media_type, data }] = content.as_slice()
else {
return Err(realtime_user_content_blob_error(
"realtime user-content blob ingress must contain exactly one image",
"invalid content shape",
));
};
let (verified, inline_data) = match data {
meerkat_core::types::ImageData::Inline { data } => (
meerkat_core::validate_image_blob_payload(
media_type,
data,
meerkat_core::live_adapter::MAX_LIVE_IMAGE_BYTES,
)
.map_err(|error| {
realtime_user_content_blob_error(
"realtime inline image failed bounded integrity validation",
error,
)
})?,
Some(data.clone()),
),
meerkat_core::types::ImageData::Blob { blob_id } => (
meerkat_core::verify_stored_image_blob(
self.blob_store.as_ref(),
blob_id,
media_type,
meerkat_core::live_adapter::MAX_LIVE_IMAGE_BYTES,
)
.await
.map_err(|error| {
realtime_user_content_blob_error(
"realtime blob reference failed bounded integrity validation",
error,
)
})?,
None,
),
};
Ok(Some(PreparedRealtimeUserContentBlob {
pending: meerkat_core::PendingRealtimeUserContentBlob {
idempotency_key: idempotency_key.clone(),
item_id: item_id.clone(),
previous_item_id: previous_item_id.clone(),
content_index: *content_index,
blob_id: verified.blob_ref.blob_id,
media_type: verified.blob_ref.media_type,
},
inline_data,
}))
}
async fn ensure_prepared_realtime_user_content_blob(
&self,
prepared: &PreparedRealtimeUserContentBlob,
) -> Result<(), SessionError> {
let verified = if let Some(data) = prepared.inline_data.as_deref() {
meerkat_core::ensure_stored_image_blob(
self.blob_store.as_ref(),
&prepared.pending.media_type,
data,
meerkat_core::live_adapter::MAX_LIVE_IMAGE_BYTES,
)
.await
} else {
meerkat_core::verify_stored_image_blob(
self.blob_store.as_ref(),
&prepared.pending.blob_id,
&prepared.pending.media_type,
meerkat_core::live_adapter::MAX_LIVE_IMAGE_BYTES,
)
.await
}
.map_err(|error| {
realtime_user_content_blob_error(
"realtime user-content blob was not durably verified",
error,
)
})?;
if verified.blob_ref.blob_id != prepared.pending.blob_id
|| verified.blob_ref.media_type != prepared.pending.media_type
{
return Err(realtime_user_content_blob_error(
"realtime user-content blob verification changed staged identity",
format!(
"expected {} {}, found {} {}",
prepared.pending.blob_id,
prepared.pending.media_type,
verified.blob_ref.blob_id,
verified.blob_ref.media_type
),
));
}
Ok(())
}
async fn reconcile_pending_realtime_user_content_blob(
&self,
id: &SessionId,
current: &mut Session,
request: Option<&PreparedRealtimeUserContentBlob>,
) -> Result<Option<meerkat_core::RealtimeTranscriptApplyOutcome>, SessionError> {
let Some(pending) = current.pending_realtime_user_content_blob() else {
return Ok(None);
};
let pending_blob_valid = match meerkat_core::verify_stored_image_blob(
self.blob_store.as_ref(),
&pending.blob_id,
&pending.media_type,
meerkat_core::live_adapter::MAX_LIVE_IMAGE_BYTES,
)
.await
{
Ok(_) => true,
Err(error) if pending_blob_is_definitively_invalid(&error) => false,
Err(error) => {
return Err(realtime_user_content_blob_error(
"pending realtime user-content blob verification was inconclusive",
error,
));
}
};
let request_anchor = request.map(|prepared| &prepared.pending);
let disposition = current
.resolve_pending_realtime_user_content_blob_recovery(request_anchor, pending_blob_valid)
.map_err(|error| {
realtime_user_content_blob_error(
"failed to resolve pending realtime user-content blob recovery",
error,
)
})?;
use meerkat_core::generated::session_document::RealtimeUserContentBlobRecoveryDisposition;
match disposition {
RealtimeUserContentBlobRecoveryDisposition::NoPending => {
Err(realtime_user_content_blob_error(
"pending realtime user-content blob disappeared during recovery",
"machine observed no pending anchor",
))
}
RealtimeUserContentBlobRecoveryDisposition::RetryExact => {
let request = request.ok_or_else(|| {
realtime_user_content_blob_error(
"exact pending realtime user-content retry lacked caller bytes/reference",
"missing exact request",
)
})?;
self.ensure_prepared_realtime_user_content_blob(request)
.await?;
let outcome = self
.commit_pending_realtime_user_content_blob(id, current, request)
.await?;
self.ensure_prepared_realtime_user_content_blob(request)
.await?;
Ok(Some(outcome))
}
RealtimeUserContentBlobRecoveryDisposition::CommitVerifiedBeforeCurrent => {
let recovered = PreparedRealtimeUserContentBlob {
pending,
inline_data: None,
};
self.ensure_prepared_realtime_user_content_blob(&recovered)
.await?;
let _ = self
.commit_pending_realtime_user_content_blob(id, current, &recovered)
.await?;
self.ensure_prepared_realtime_user_content_blob(&recovered)
.await?;
Ok(None)
}
RealtimeUserContentBlobRecoveryDisposition::ClearInvalidBeforeCurrent => {
current
.clear_invalid_pending_realtime_user_content_blob(request_anchor)
.map_err(|error| {
realtime_user_content_blob_error(
"failed to clear invalid pending realtime user-content blob",
error,
)
})?;
self.persist_realtime_transcript_snapshot(id, current)
.await?;
Ok(None)
}
}
}
async fn commit_pending_realtime_user_content_blob(
&self,
id: &SessionId,
current: &mut Session,
prepared: &PreparedRealtimeUserContentBlob,
) -> Result<meerkat_core::RealtimeTranscriptApplyOutcome, SessionError> {
let outcome = current.append_realtime_transcript_event(prepared.pending.canonical_event());
let committed_matches = matches!(
outcome.user_content.as_ref(),
Some(meerkat_core::RealtimeUserContentApplyOutcome::Committed(identity))
if prepared.pending.matches_identity(identity)
);
if !committed_matches || current.pending_realtime_user_content_blob().is_some() {
return Err(realtime_user_content_blob_error(
"pending realtime user-content blob reducer commit failed closed",
format!("unexpected reducer outcome {outcome:?}"),
));
}
self.persist_realtime_transcript_snapshot(id, current)
.await?;
Ok(outcome)
}
async fn persist_realtime_transcript_snapshot(
&self,
id: &SessionId,
session: &Session,
) -> Result<(), SessionError> {
self.inner
.sync_session_from_durable_snapshot(id, session.clone())
.await?;
if let Err(error) = self.persist_full_session(id).await {
let _ = self.discard_live_session_unfenced(id).await;
return Err(error);
}
Ok(())
}
pub fn new(
builder: B,
max_sessions: usize,
store: Arc<dyn SessionStore>,
runtime_store: Arc<dyn RuntimeStore>,
blob_store: Arc<dyn BlobStore>,
) -> Self {
Self::new_with_capacities(builder, max_sessions, 0, store, runtime_store, blob_store)
}
pub fn new_with_archived_history_capacity(
builder: B,
max_sessions: usize,
_archived_history_capacity: usize,
store: Arc<dyn SessionStore>,
runtime_store: Arc<dyn RuntimeStore>,
blob_store: Arc<dyn BlobStore>,
) -> Self {
Self::new_with_capacities(builder, max_sessions, 0, store, runtime_store, blob_store)
}
pub fn new_with_capacities(
builder: B,
active_session_capacity: usize,
_archived_history_capacity: usize,
store: Arc<dyn SessionStore>,
runtime_store: Arc<dyn RuntimeStore>,
blob_store: Arc<dyn BlobStore>,
) -> Self {
let incremental = store.as_incremental();
Self {
inner: EphemeralSessionService::new(builder, active_session_capacity),
incremental,
runtime_store,
blob_store,
event_store: None,
projector: None,
checkpointer_gates: Mutex::new(HashMap::new()),
live_checkpointers: Mutex::new(HashMap::new()),
recovery_gates: Mutex::new(HashMap::new()),
rejected_run_compaction_aborted: std::sync::Mutex::new(HashSet::new()),
turn_finalization_gates: Mutex::new(HashMap::new()),
event_projection_faults: Arc::new(Mutex::new(HashMap::new())),
event_projection_gates: Arc::new(Mutex::new(HashMap::new())),
event_projection_drains: Arc::new(Mutex::new(HashMap::new())),
}
}
pub fn with_event_projection(
mut self,
event_store: Arc<dyn EventStore>,
projector: Arc<SessionProjector>,
) -> Self {
self.event_store = Some(event_store);
self.projector = Some(projector);
self
}
pub fn runtime_store(&self) -> Arc<dyn RuntimeStore> {
self.runtime_store.clone()
}
pub async fn observe_persisted_session_authority(
&self,
id: &SessionId,
) -> Result<Option<RuntimeSessionAuthority>, SessionError> {
let authority = self
.runtime_store
.load_session_boundary_authority(&Self::runtime_id_for_session(id))
.await
.map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"failed to observe persisted runtime session authority for session {id}: {error}"
)))
})?;
if let Some(authority) = authority.as_ref() {
let _ = PreparedRuntimeBoundaryIdentity::from_runtime_authority(authority, id)?;
}
Ok(authority)
}
pub async fn persisted_runtime_state(
&self,
id: &SessionId,
) -> Result<Option<RuntimeState>, SessionError> {
Self::load_runtime_state_for_session(&self.runtime_store, id).await
}
pub fn has_event_projection(&self) -> bool {
self.event_store.is_some() && self.projector.is_some()
}
pub fn ensure_active_capacity_available(&self) -> Result<(), SessionError> {
self.inner.ensure_active_capacity_available()
}
pub async fn reserve_create_session_admission(
&self,
) -> Result<crate::ephemeral::RuntimeContextAdmissionGuard, SessionError> {
self.inner.acquire_runtime_capacity_admission().await
}
pub async fn reserve_runtime_turn_admission(
&self,
id: &SessionId,
) -> Result<crate::ephemeral::RuntimeContextAdmissionGuard, SessionError> {
match self.inner.join_active_runtime_context_admission(id).await {
Ok(Some(admission)) => return Ok(admission),
Ok(None) | Err(SessionError::NotFound { .. }) => {}
Err(error) => return Err(error),
}
let recovery_gate = self.recovery_gate_for_session(id).await;
let _turn_guard = recovery_gate.lock().await;
let _ = self.discard_stale_live_session_if_needed(id).await?;
if let Some(session) = self.load_authoritative_session_base(id).await? {
self.reject_if_archived_session(id, &session)
.await
.map_err(crate::control_error_into_session_error)?;
}
match self.inner.acquire_runtime_context_admission(id).await {
Ok(admission) => Ok(admission),
Err(SessionError::NotFound { .. }) => {
if self.load_authoritative_session_base(id).await?.is_some() {
self.inner.acquire_runtime_capacity_admission().await
} else {
Err(SessionError::NotFound { id: id.clone() })
}
}
Err(error) => Err(error),
}
}
pub async fn create_session_with_reserved_admission(
&self,
req: CreateSessionRequest,
admission: crate::ephemeral::RuntimeContextAdmissionGuard,
) -> Result<RunResult, SessionError> {
let actor_seed_authority = self.resolve_actor_session_seed_authority(&req).await?;
self.create_session_with_admission(
req,
Some(admission),
actor_seed_authority,
ArchivedResumeAuthorization::RejectArchived,
false,
None,
)
.await
}
async fn resolve_actor_session_seed_authority(
&self,
req: &CreateSessionRequest,
) -> Result<ActorSessionSeedAuthority, SessionError> {
let Some(session_id) = req
.build
.as_ref()
.and_then(|build| build.resume_session.as_ref())
.map(Session::id)
else {
return Ok(ActorSessionSeedAuthority::FreshGenerationZero);
};
let runtime_id = Self::runtime_id_for_session(session_id);
let durable_authority_exists = match self.runtime_store.session_persistence_profile() {
RuntimeSessionPersistenceProfile::WholeBlobV1 => self
.runtime_store
.load_whole_blob_store_authority(&runtime_id)
.await
.map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"failed to resolve WholeBlob actor seed authority for session {session_id}: {error}"
)))
})?
.is_some(),
RuntimeSessionPersistenceProfile::HeadCanonicalV1 => self
.runtime_store
.load_session_boundary_authority(&runtime_id)
.await
.map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"failed to resolve HeadCanonical actor seed authority for session {session_id}: {error}"
)))
})?
.is_some(),
profile => {
return Err(SessionError::Agent(AgentError::InternalError(format!(
"unsupported runtime session persistence profile {profile} while resolving actor seed authority for session {session_id}"
))));
}
};
Ok(if durable_authority_exists {
ActorSessionSeedAuthority::DurableCommitted
} else {
ActorSessionSeedAuthority::FreshGenerationZero
})
}
#[doc(hidden)]
pub async fn create_fresh_session_with_reserved_admission(
&self,
req: CreateSessionRequest,
admission: crate::ephemeral::RuntimeContextAdmissionGuard,
) -> Result<RunResult, SessionError> {
let actor_seed_authority = self.resolve_actor_session_seed_authority(&req).await?;
self.create_session_with_admission(
req,
Some(admission),
actor_seed_authority,
ArchivedResumeAuthorization::RejectArchived,
false,
None,
)
.await
}
pub async fn create_session_with_reserved_admission_and_actor_witness(
&self,
req: CreateSessionRequest,
admission: crate::ephemeral::RuntimeContextAdmissionGuard,
actor_witness_slot: &LiveSessionActorWitnessSlot,
) -> Result<RunResult, SessionError> {
let actor_seed_authority = self.resolve_actor_session_seed_authority(&req).await?;
self.create_session_with_admission(
req,
Some(admission),
actor_seed_authority,
ArchivedResumeAuthorization::RejectArchived,
false,
Some(actor_witness_slot),
)
.await
}
#[doc(hidden)]
pub async fn create_session_with_reserved_admission_and_actor_witness_under_runtime_turn_boundary(
&self,
req: CreateSessionRequest,
admission: crate::ephemeral::RuntimeContextAdmissionGuard,
actor_witness_slot: &LiveSessionActorWitnessSlot,
) -> Result<RunResult, SessionError> {
let actor_seed_authority = self.resolve_actor_session_seed_authority(&req).await?;
self.create_session_with_admission(
req,
Some(admission),
actor_seed_authority,
ArchivedResumeAuthorization::RejectArchived,
true,
Some(actor_witness_slot),
)
.await
}
pub async fn create_session_with_reserved_admission_under_runtime_turn_boundary(
&self,
req: CreateSessionRequest,
admission: crate::ephemeral::RuntimeContextAdmissionGuard,
) -> Result<RunResult, SessionError> {
let actor_seed_authority = self.resolve_actor_session_seed_authority(&req).await?;
self.create_session_with_admission(
req,
Some(admission),
actor_seed_authority,
ArchivedResumeAuthorization::RejectArchived,
true,
None,
)
.await
}
pub async fn create_session_with_reserved_machine_archived_resume_admission(
&self,
req: CreateSessionRequest,
admission: crate::ephemeral::RuntimeContextAdmissionGuard,
authorization: ArchivedSessionActorMaterializationAuthorization,
) -> Result<RunResult, SessionError> {
self.create_session_with_admission(
req,
Some(admission),
ActorSessionSeedAuthority::DurableCommitted,
ArchivedResumeAuthorization::MachinePendingPromotion { authorization },
false,
None,
)
.await
}
pub async fn create_session_with_reserved_machine_archived_resume_admission_under_runtime_turn_boundary(
&self,
req: CreateSessionRequest,
admission: crate::ephemeral::RuntimeContextAdmissionGuard,
authorization: ArchivedSessionActorMaterializationAuthorization,
) -> Result<RunResult, SessionError> {
self.create_session_with_admission(
req,
Some(admission),
ActorSessionSeedAuthority::DurableCommitted,
ArchivedResumeAuthorization::MachinePendingPromotion { authorization },
true,
None,
)
.await
}
#[doc(hidden)]
pub async fn create_session_with_reserved_machine_archived_resume_admission_and_actor_witness_under_runtime_turn_boundary(
&self,
req: CreateSessionRequest,
admission: crate::ephemeral::RuntimeContextAdmissionGuard,
authorization: ArchivedSessionActorMaterializationAuthorization,
actor_witness_slot: &LiveSessionActorWitnessSlot,
) -> Result<RunResult, SessionError> {
self.create_session_with_admission(
req,
Some(admission),
ActorSessionSeedAuthority::DurableCommitted,
ArchivedResumeAuthorization::MachinePendingPromotion { authorization },
true,
Some(actor_witness_slot),
)
.await
}
pub async fn run_machine_committed_live_turn(
&self,
protocol: MachineServiceTurnCommitProtocol<'_>,
id: &SessionId,
req: StartTurnRequest,
admission: crate::ephemeral::RuntimeContextAdmissionGuard,
) -> Result<
RunResult,
(
SessionError,
Option<crate::ephemeral::RuntimeContextAdmissionGuard>,
),
> {
self.run_machine_committed_live_turn_with_boundary_mode(protocol, id, req, admission, false)
.await
}
pub async fn run_machine_committed_live_turn_under_runtime_turn_boundary(
&self,
protocol: MachineServiceTurnCommitProtocol<'_>,
id: &SessionId,
req: StartTurnRequest,
admission: crate::ephemeral::RuntimeContextAdmissionGuard,
) -> Result<
RunResult,
(
SessionError,
Option<crate::ephemeral::RuntimeContextAdmissionGuard>,
),
> {
self.run_machine_committed_live_turn_with_boundary_mode(protocol, id, req, admission, true)
.await
}
async fn run_machine_committed_live_turn_with_boundary_mode(
&self,
protocol: MachineServiceTurnCommitProtocol<'_>,
id: &SessionId,
req: StartTurnRequest,
admission: crate::ephemeral::RuntimeContextAdmissionGuard,
turn_boundary_already_held: bool,
) -> Result<
RunResult,
(
SessionError,
Option<crate::ephemeral::RuntimeContextAdmissionGuard>,
),
> {
if let Err(error) = self.validate_machine_service_turn_protocol(protocol) {
return Err((error, Some(admission)));
}
let _turn_finalization_guard = if turn_boundary_already_held {
None
} else {
Some(self.acquire_runtime_turn_finalization_guard(id).await)
};
let turn_identity = match protocol
.runtime_adapter
.capture_service_turn_identity(id)
.await
{
Ok(identity) => identity,
Err(error) => {
return Err((
runtime_driver_error_to_session_error(error),
Some(admission),
));
}
};
let result = self
.start_turn_inner_with_admission(id, req, Some(admission), protocol, &turn_identity)
.await;
match result {
Ok(result) => Ok(result),
Err((error, admission))
if self
.service_turn_error_requires_machine_terminal_receipt(id, &error)
.await =>
{
Err((error, admission))
}
Err((error, admission)) => {
if !matches!(error, SessionError::NotFound { .. }) {
let _ = self
.discard_live_session_under_runtime_turn_boundary(id)
.await;
}
Err((error, admission))
}
}
}
fn validate_machine_service_turn_protocol(
&self,
protocol: MachineServiceTurnCommitProtocol<'_>,
) -> Result<(), SessionError> {
if protocol
.runtime_adapter
.shares_runtime_store_authority(&self.runtime_store)
{
return Ok(());
}
Err(SessionError::Unsupported(
"machine service-turn commit protocol runtime authority does not match the session service runtime store"
.to_string(),
))
}
pub async fn service_turn_error_requires_machine_terminal_receipt(
&self,
_id: &SessionId,
error: &SessionError,
) -> bool {
Self::callback_pending_terminal(error).is_some()
|| matches!(
error,
SessionError::Agent(meerkat_core::error::AgentError::TerminalFailure { .. })
)
}
async fn validate_runtime_terminal_failure_witness(
&self,
id: &SessionId,
witness: &meerkat_core::TurnErrorMetadata,
) -> Result<(), SessionError> {
let snapshot = self.execution_snapshot(id).await?.ok_or_else(|| {
SessionError::Agent(AgentError::InternalError(
"runtime terminal witness had no live generated execution snapshot".to_string(),
))
})?;
let witness_outcome = witness.outcome.ok_or_else(|| {
SessionError::Agent(AgentError::InternalError(
"runtime terminal witness omitted terminal outcome".to_string(),
))
})?;
if !matches!(
meerkat_core::generated::terminal_surface_mapping::classify_terminal(
&snapshot.terminal_outcome,
snapshot.terminal_cause_kind,
),
meerkat_core::generated::terminal_surface_mapping::SurfaceResultClass::HardFailure
) || !snapshot.turn_terminal
|| snapshot.terminal_outcome != witness_outcome
|| snapshot.terminal_cause_kind != Some(witness.kind)
|| !witness.terminal
|| !witness.kind.is_specific_failure_cause()
{
return Err(SessionError::Agent(AgentError::InternalError(format!(
"exact runtime terminal witness disagreed with generated terminal truth: witness=({witness_outcome:?}, {:?}, terminal={}), generated=({:?}, {:?}, terminal={})",
witness.kind,
witness.terminal,
snapshot.terminal_outcome,
snapshot.terminal_cause_kind,
snapshot.turn_terminal,
))));
}
Ok(())
}
async fn validated_runtime_execution_after_live_mutation(
&self,
id: &SessionId,
execution: crate::ephemeral::SessionTurnExecutionOutcome,
) -> Result<
(
Result<RunResult, meerkat_core::error::AgentError>,
Option<meerkat_core::TurnErrorMetadata>,
),
SessionError,
> {
let parts = execution.into_runtime_parts().map_err(SessionError::Agent);
let (result, witness) = match parts {
Ok(parts) => parts,
Err(error) => {
if let Err(discard_error) = self.discard_live_session_unfenced(id).await {
tracing::warn!(
session_id = %id,
error = %discard_error,
"failed to discard mutated live session after terminal-witness extraction failure"
);
}
return Err(SessionError::runtime_executor_stopped(format!(
"runtime terminal-witness extraction failed after live mutation: {error}"
)));
}
};
if let Some(witness) = witness.as_ref()
&& let Err(error) = self
.validate_runtime_terminal_failure_witness(id, witness)
.await
{
if let Err(discard_error) = self.discard_live_session_unfenced(id).await {
tracing::warn!(
session_id = %id,
error = %discard_error,
"failed to discard mutated live session after terminal-witness validation failure"
);
}
return Err(SessionError::runtime_executor_stopped(format!(
"runtime terminal-witness validation failed after live mutation: {error}"
)));
}
Ok((result, witness))
}
fn terminal_surface_error(
error: SessionError,
witness: &meerkat_core::TurnErrorMetadata,
) -> SessionError {
let outcome = witness
.outcome
.unwrap_or(meerkat_core::TurnTerminalOutcome::Failed);
match error {
error @ SessionError::Agent(AgentError::TerminalFailure { .. }) => error,
error => SessionError::Agent(AgentError::TerminalFailure {
outcome,
cause_kind: witness.kind,
message: error.to_string(),
}),
}
}
fn reject_runtime_backed_eager_create_session(
&self,
req: &CreateSessionRequest,
) -> Result<(), SessionError> {
if req.initial_turn == InitialTurnPolicy::RunImmediately {
return Err(SessionError::Unsupported(
"runtime-backed eager create_session must route through the MeerkatMachine service-turn commit protocol"
.to_string(),
));
}
Ok(())
}
async fn commit_machine_service_turn_snapshot(
&self,
id: &SessionId,
protocol: MachineServiceTurnCommitProtocol<'_>,
turn_identity: &meerkat_runtime::MachineServiceTurnIdentity,
) -> Result<(), SessionError> {
let mut commit_lease = protocol
.runtime_adapter
.prepare_service_turn_commit_lease(turn_identity)
.await
.map_err(runtime_driver_error_to_session_error)?;
let recovery_gate = self.recovery_gate_for_session(id).await;
let _turn_guard = recovery_gate.lock().await;
let PreparedRuntimeBoundary {
committed,
identity: staged_boundary,
conversation_digest: _,
message_count: _,
} = self
.prepare_runtime_boundary(
id,
"machine service-turn session boundary",
Some(commit_lease.run_id()),
)
.await?;
let promotes_provisional_tail = committed.provisional_promotion_receipt().is_some();
let commit_result = protocol
.runtime_adapter
.commit_service_turn_terminal_receipt_with_lease(&mut commit_lease, committed.clone())
.await
.map_err(runtime_driver_error_to_session_error)?;
let committed_authority = commit_result
.as_ref()
.and_then(|result| result.authority())
.ok_or_else(|| {
SessionError::Agent(AgentError::InternalError(format!(
"machine service-turn boundary committed without session authority for session {id}"
)))
})?;
let durable_boundary =
PreparedRuntimeBoundaryIdentity::from_runtime_authority(committed_authority, id)?;
if durable_boundary != staged_boundary {
return Err(SessionError::Agent(AgentError::InternalError(format!(
"machine service-turn committed authority differs from the prepared boundary for session {id}"
))));
}
if promotes_provisional_tail {
let (committed_store_revision, committed_authority_token) = match committed_authority {
RuntimeSessionAuthority::WholeBlob(authority) => (
authority.store_revision(),
authority.blob_sha256().to_string(),
),
RuntimeSessionAuthority::HeadCanonical(authority) => (
authority.store_revision(),
authority.committed_head_token().to_string(),
),
};
return self
.acknowledge_provisional_runtime_boundary_after_commit(
id,
committed_store_revision,
&committed_authority_token,
)
.await;
}
match staged_boundary.profile {
RuntimeSessionPersistenceProfile::WholeBlobV1 => {
let authority = committed_authority.whole_blob().ok_or_else(|| {
SessionError::Agent(AgentError::InternalError(format!(
"WholeBlob machine service-turn returned non-WholeBlob authority for session {id}"
)))
})?;
self.acknowledge_whole_blob_runtime_boundary_after_commit(
id,
authority.store_revision(),
authority.blob_sha256(),
)
.await
}
RuntimeSessionPersistenceProfile::HeadCanonicalV1 => {
self.acknowledge_head_canonical_runtime_boundary_after_commit(id, &durable_boundary)
.await
}
profile => Err(SessionError::Agent(AgentError::InternalError(format!(
"unsupported committed machine service-turn persistence profile {profile} for session {id}"
)))),
}
}
async fn start_turn_inner_with_admission(
&self,
id: &SessionId,
req: StartTurnRequest,
mut admission: Option<crate::ephemeral::RuntimeContextAdmissionGuard>,
protocol: MachineServiceTurnCommitProtocol<'_>,
turn_identity: &meerkat_runtime::MachineServiceTurnIdentity,
) -> Result<
RunResult,
(
SessionError,
Option<crate::ephemeral::RuntimeContextAdmissionGuard>,
),
> {
let recovery_gate = self.recovery_gate_for_session(id).await;
let turn_guard = recovery_gate.lock().await;
let _ = self
.discard_stale_live_session_if_needed(id)
.await
.map_err(|error| (error, admission.take()))?;
let _ = self
.recover_live_session_from_store_if_needed_locked(id)
.await
.map_err(|error| (error, admission.take()))?;
let execution = match admission.take() {
Some(admission) => {
self.inner
.start_runtime_turn_execution_with_admission_recovering_not_found(
id, req, admission,
)
.await
}
None => self
.inner
.start_runtime_turn_execution(id, req)
.await
.map_err(|error| (error, None)),
}?;
let (result, machine_terminal_failure) = self
.validated_runtime_execution_after_live_mutation(id, execution)
.await
.map_err(|error| (error, None))?;
let result = match result.map_err(SessionError::Agent) {
Ok(result) => result,
Err(error) => {
let terminal_error = match machine_terminal_failure.as_ref() {
Some(witness) => Self::terminal_surface_error(error, witness),
None => error,
};
if machine_terminal_failure.is_some()
|| Self::callback_pending_terminal(&terminal_error).is_some()
{
drop(turn_guard);
if let Err(commit_error) = self
.commit_machine_service_turn_snapshot(id, protocol, turn_identity)
.await
{
let _ = self
.discard_live_session_under_runtime_turn_boundary(id)
.await;
return Err((commit_error, None));
}
}
return Err((terminal_error, None));
}
};
drop(turn_guard);
if let Err(error) = self
.commit_machine_service_turn_snapshot(id, protocol, turn_identity)
.await
{
let _ = self
.discard_live_session_under_runtime_turn_boundary(id)
.await;
return Err((error, None));
}
Ok(result)
}
pub async fn event_log_latest_seq(&self, id: &SessionId) -> Result<Option<u64>, SessionError> {
let Some(event_store) = self.event_store.as_ref() else {
return Ok(None);
};
if let Some(cause) = event_projection_admission_fault(
&self.event_projection_faults,
&self.event_projection_gates,
event_store,
id,
)
.await
{
return Err(event_projection_halted_error(id, cause));
}
let gate = event_projection_gate(&self.event_projection_gates, id).await;
let _guard = gate.lock().await;
if let Some(cause) = self.event_projection_faults.lock().await.get(id).cloned() {
return Err(event_projection_halted_error(id, cause));
}
event_store
.last_seq(id)
.await
.map(Some)
.map_err(|err| SessionError::Store(Box::new(err)))
}
pub async fn publish_interaction_terminal_exact(
&self,
id: &SessionId,
interaction_id: meerkat_core::interaction::InteractionId,
event: meerkat_core::event::AgentEvent,
) -> Result<
meerkat_core::lifecycle::core_executor::CoreInteractionTerminalPublicationReceipt,
SessionError,
> {
let Some(event_store) = self.event_store.clone() else {
return Err(SessionError::Unsupported(
"exact interaction terminal publication requires an EventStore".to_string(),
));
};
if let Some(marker) = event_store
.projection_halt(id)
.await
.map_err(|error| SessionError::Store(Box::new(error)))?
{
return Err(event_projection_halted_error(
id,
Arc::new(SessionError::Store(Box::new(
DurableEventProjectionHaltMarker {
session_id: marker.session_id,
reason: marker.reason,
},
))),
));
}
if let Some(cause) = self.event_projection_faults.lock().await.get(id) {
return Err(event_projection_halted_error(id, Arc::clone(cause)));
}
self.inner
.publish_interaction_terminal_exact(id, interaction_id, event, event_store)
.await
}
pub async fn publish_interaction_terminals_exact_batch(
&self,
id: &SessionId,
events: &[meerkat_core::event::AgentEvent],
) -> Result<
Vec<meerkat_core::lifecycle::core_executor::CoreInteractionTerminalPublicationReceipt>,
SessionError,
> {
if events.is_empty() {
return Ok(Vec::new());
}
let Some(event_store) = self.event_store.clone() else {
return Err(SessionError::Unsupported(
"exact interaction terminal publication requires an EventStore".to_string(),
));
};
if let Some(marker) = event_store
.projection_halt(id)
.await
.map_err(|error| SessionError::Store(Box::new(error)))?
{
return Err(event_projection_halted_error(
id,
Arc::new(SessionError::Store(Box::new(
DurableEventProjectionHaltMarker {
session_id: marker.session_id,
reason: marker.reason,
},
))),
));
}
if let Some(cause) = self.event_projection_faults.lock().await.get(id) {
return Err(event_projection_halted_error(id, Arc::clone(cause)));
}
self.inner
.publish_interaction_terminals_exact_batch(id, events.to_vec(), event_store)
.await
}
async fn publish_interaction_terminals_stored_only_exact_batch(
&self,
id: &SessionId,
events: &[meerkat_core::event::AgentEvent],
) -> Result<Vec<CoreInteractionTerminalPublicationReceipt>, SessionError> {
use crate::event_store::{EventStoreError, ExactInteractionAppend};
use meerkat_core::event::{EventEnvelope, EventSourceIdentity};
if events.is_empty() {
return Ok(Vec::new());
}
if self.inner.has_live_session(id).await? {
return Err(SessionError::Unsupported(format!(
"stored-only terminal publisher refused live session {id}"
)));
}
if events.len() > crate::event_store::MAX_EXACT_INTERACTION_TERMINAL_BATCH {
return Err(SessionError::Store(Box::new(
EventStoreError::InvalidExactInteractionTerminalBatch {
reason: format!(
"batch contains {} terminals, exceeding the maximum of {}",
events.len(),
crate::event_store::MAX_EXACT_INTERACTION_TERMINAL_BATCH
),
},
)));
}
let Some(event_store) = self.event_store.clone() else {
return Err(SessionError::Unsupported(
"exact interaction terminal publication requires an EventStore".to_string(),
));
};
if let Some(marker) = event_store
.projection_halt(id)
.await
.map_err(|error| SessionError::Store(Box::new(error)))?
{
return Err(event_projection_halted_error(
id,
Arc::new(SessionError::Store(Box::new(
DurableEventProjectionHaltMarker {
session_id: marker.session_id,
reason: marker.reason,
},
))),
));
}
if let Some(cause) = self.event_projection_faults.lock().await.get(id) {
return Err(event_projection_halted_error(id, Arc::clone(cause)));
}
let mut terminals = Vec::with_capacity(events.len());
for event in events {
let interaction_id = match event {
meerkat_core::event::AgentEvent::InteractionComplete { interaction_id, .. }
| meerkat_core::event::AgentEvent::InteractionCallbackPending {
interaction_id,
..
}
| meerkat_core::event::AgentEvent::InteractionFailed { interaction_id, .. } => {
*interaction_id
}
_ => {
return Err(SessionError::Agent(AgentError::InternalError(
"stored-only terminal publisher received a non-interaction event"
.to_string(),
)));
}
};
terminals.push((
interaction_id,
EventEnvelope::new_with_source(
EventSourceIdentity::interaction(interaction_id),
0,
None,
event.clone(),
),
));
}
crate::event_store::validate_exact_interaction_terminal_batch(&terminals)
.map_err(|error| SessionError::Store(Box::new(error)))?;
let stream_seq_floor = event_store
.read_from(id, 1)
.await
.map_err(|error| SessionError::Store(Box::new(error)))?
.into_iter()
.map(|stored| stored.stream_seq)
.max()
.unwrap_or(0);
let appends = event_store
.append_interaction_terminals_exact_batch(id, stream_seq_floor, &terminals)
.await
.map_err(|error| SessionError::Store(Box::new(error)))?;
if appends.len() != terminals.len() {
return Err(SessionError::Agent(AgentError::InternalError(format!(
"stored-only exact batch returned {} receipts for {} terminals",
appends.len(),
terminals.len()
))));
}
let mut receipts = Vec::with_capacity(terminals.len());
let mut saw_inserted = false;
let mut replay_tail = None;
let mut inserted_tail = None;
for ((interaction_id, requested), append) in terminals.iter().zip(appends) {
let (inserted, stored) = match append {
ExactInteractionAppend::Inserted(stored) => (true, stored),
ExactInteractionAppend::Replayed(stored) => (false, stored),
};
let envelope = stored.to_envelope();
crate::event_store::validate_exact_interaction_terminal(*interaction_id, &envelope)
.map_err(|error| SessionError::Store(Box::new(error)))?;
if stored.mob_id != requested.mob_id
|| !crate::event_store::interaction_terminal_events_semantically_equal(
&stored.event,
&requested.payload,
)
{
return Err(SessionError::Agent(AgentError::InternalError(format!(
"stored-only exact batch returned a non-canonical row for {interaction_id}"
))));
}
if inserted {
saw_inserted = true;
let expected = inserted_tail
.or(replay_tail)
.unwrap_or(stream_seq_floor)
.checked_add(1)
.ok_or_else(|| {
SessionError::Store(Box::new(
EventStoreError::InvalidExactInteractionTerminalBatch {
reason: "stored-only session stream sequence overflow".to_string(),
},
))
})?;
if stored.stream_seq != expected {
return Err(SessionError::Agent(AgentError::InternalError(format!(
"stored-only exact batch inserted stream seq {}, expected {expected}",
stored.stream_seq
))));
}
inserted_tail = Some(stored.stream_seq);
} else {
if saw_inserted {
return Err(SessionError::Agent(AgentError::InternalError(
"stored-only exact batch replayed after an inserted suffix".to_string(),
)));
}
if let Some(previous) = replay_tail {
let expected = previous.checked_add(1).ok_or_else(|| {
SessionError::Agent(AgentError::InternalError(
"stored-only exact replay sequence overflow".to_string(),
))
})?;
if stored.stream_seq != expected {
return Err(SessionError::Agent(AgentError::InternalError(format!(
"stored-only exact replay stream seq {}, expected {expected}",
stored.stream_seq
))));
}
}
replay_tail = Some(stored.stream_seq);
}
receipts.push(
CoreInteractionTerminalPublicationReceipt::try_new(&stored.event, stored.seq)
.map_err(|error| {
SessionError::Agent(AgentError::InternalError(error.to_string()))
})?,
);
}
Ok(receipts)
}
pub async fn event_log_await_projection_drain(
&self,
id: &SessionId,
) -> Result<bool, SessionError> {
let mut observed_projection = false;
loop {
let drains = self
.event_projection_drains
.lock()
.await
.get(id)
.cloned()
.unwrap_or_default();
if drains.is_empty() {
if let Some(cause) = self.event_projection_faults.lock().await.get(id) {
return Err(event_projection_halted_error(id, Arc::clone(cause)));
}
return Ok(observed_projection);
}
observed_projection = true;
for mut drain in drains {
loop {
let state = drain.borrow().clone();
match state {
EventProjectionDrainState::Drained => break,
EventProjectionDrainState::Faulted(detail) => {
if let Some(cause) = self.event_projection_faults.lock().await.get(id) {
return Err(event_projection_halted_error(id, Arc::clone(cause)));
}
return Err(SessionError::Agent(AgentError::InternalError(format!(
"event projection faulted while draining session {id}: {detail}"
))));
}
EventProjectionDrainState::Running => {}
}
drain.changed().await.map_err(|_| {
SessionError::Agent(AgentError::InternalError(format!(
"event projection drain witness closed before reaching a terminal state for session {id}"
)))
})?;
}
}
if let Some(cause) = self.event_projection_faults.lock().await.get(id) {
return Err(event_projection_halted_error(id, Arc::clone(cause)));
}
let mut registry = self.event_projection_drains.lock().await;
let remove_entry = if let Some(current) = registry.get_mut(id) {
current.retain(|drain| {
!matches!(&*drain.borrow(), EventProjectionDrainState::Drained)
});
current.is_empty()
} else {
false
};
if remove_entry {
registry.remove(id);
}
}
}
pub async fn event_log_read_from(
&self,
id: &SessionId,
from_seq: u64,
) -> Result<Option<Vec<crate::event_store::StoredEvent>>, SessionError> {
let Some(event_store) = self.event_store.as_ref() else {
return Ok(None);
};
if let Some(cause) = event_projection_admission_fault(
&self.event_projection_faults,
&self.event_projection_gates,
event_store,
id,
)
.await
{
return Err(event_projection_halted_error(id, cause));
}
let gate = event_projection_gate(&self.event_projection_gates, id).await;
let _guard = gate.lock().await;
if let Some(cause) = self.event_projection_faults.lock().await.get(id).cloned() {
return Err(event_projection_halted_error(id, cause));
}
event_store
.read_from(id, from_seq)
.await
.map(Some)
.map_err(|err| SessionError::Store(Box::new(err)))
}
pub async fn event_log_read_from_bounded(
&self,
id: &SessionId,
from_seq: u64,
max_rows: usize,
) -> Result<Option<Vec<crate::event_store::StoredEvent>>, SessionError> {
let Some(event_store) = self.event_store.as_ref() else {
return Ok(None);
};
if let Some(marker) = event_store
.projection_halt(id)
.await
.map_err(|err| SessionError::Store(Box::new(err)))?
{
return Err(event_projection_halted_error(
id,
Arc::new(SessionError::Store(Box::new(
DurableEventProjectionHaltMarker {
session_id: marker.session_id,
reason: marker.reason,
},
))),
));
}
if let Some(cause) = self.event_projection_faults.lock().await.get(id) {
return Err(event_projection_halted_error(id, Arc::clone(cause)));
}
event_store
.read_from_bounded(id, from_seq, max_rows)
.await
.map(Some)
.map_err(|err| SessionError::Store(Box::new(err)))
}
pub async fn event_log_read_page(
&self,
id: &SessionId,
from_seq: u64,
limit: usize,
) -> Result<Option<Vec<crate::event_store::StoredEvent>>, SessionError> {
let Some(event_store) = self.event_store.as_ref() else {
return Ok(None);
};
if let Some(cause) = event_projection_admission_fault(
&self.event_projection_faults,
&self.event_projection_gates,
event_store,
id,
)
.await
{
return Err(event_projection_halted_error(id, cause));
}
let gate = event_projection_gate(&self.event_projection_gates, id).await;
let _guard = gate.lock().await;
if let Some(cause) = self.event_projection_faults.lock().await.get(id).cloned() {
return Err(event_projection_halted_error(id, cause));
}
event_store
.read_page(id, from_seq, limit)
.await
.map(Some)
.map_err(|err| SessionError::Store(Box::new(err)))
}
pub fn blob_store(&self) -> Arc<dyn BlobStore> {
self.blob_store.clone()
}
async fn install_create_time_event_projection(
&self,
req: &mut CreateSessionRequest,
known_session_id: Option<&SessionId>,
) -> Option<CreateTimeEventProjection> {
let (Some(event_store), Some(projector)) =
(self.event_store.clone(), self.projector.clone())
else {
return None;
};
let caller_event_tx = req.event_tx.take();
let (projection_tx, projection_rx) = mpsc::channel(128);
let (session_tx, session_rx) = watch::channel(None);
let (drain_tx, drain_rx) = watch::channel(EventProjectionDrainState::Running);
req.event_tx = Some(projection_tx);
if let Some(session_id) = known_session_id {
register_event_projection_drain(
&self.event_projection_drains,
session_id,
drain_rx.clone(),
)
.await;
}
let drain_registration =
Some((Arc::clone(&self.event_projection_drains), drain_rx.clone()));
let projection_faults = Arc::clone(&self.event_projection_faults);
let projection_gates = Arc::clone(&self.event_projection_gates);
tokio::spawn(async move {
project_create_time_events(
event_store,
projector,
projection_rx,
session_rx,
caller_event_tx,
CreateTimeEventProjectionLifecycle {
projection_faults,
projection_gates,
drain_registration,
},
)
.await;
drain_tx.send_replace(EventProjectionDrainState::Drained);
});
Some(CreateTimeEventProjection {
session_tx,
drain_rx,
})
}
async fn ensure_event_projection_admitted(&self, id: &SessionId) -> Result<(), SessionError> {
let Some(event_store) = self.event_store.as_ref() else {
return Ok(());
};
if let Some(cause) = event_projection_admission_fault(
&self.event_projection_faults,
&self.event_projection_gates,
event_store,
id,
)
.await
{
return Err(event_projection_halted_error(id, cause));
}
Ok(())
}
async fn spawn_event_projection_task(&self, id: &SessionId) -> Result<(), SessionError> {
let (Some(event_store), Some(projector)) =
(self.event_store.clone(), self.projector.clone())
else {
return Ok(());
};
self.ensure_event_projection_admitted(id).await?;
let session_id = id.clone();
let stream = self.inner.subscribe_session_events(&session_id).await;
let Ok(stream) = stream else {
return Ok(());
};
let (drain_tx, drain_rx) = watch::channel(EventProjectionDrainState::Running);
register_event_projection_drain(&self.event_projection_drains, &session_id, drain_rx).await;
let projection_faults = Arc::clone(&self.event_projection_faults);
let projection_gates = Arc::clone(&self.event_projection_gates);
tokio::spawn(async move {
project_session_event_stream(
event_store,
projector,
session_id.clone(),
stream,
Arc::clone(&projection_faults),
projection_gates,
)
.await;
let state = projection_faults
.lock()
.await
.get(&session_id)
.map(|cause| EventProjectionDrainState::Faulted(cause.to_string()))
.unwrap_or(EventProjectionDrainState::Drained);
drain_tx.send_replace(state);
});
Ok(())
}
async fn normalized_session_for_persistence(
&self,
mut session: Session,
) -> Result<Session, SessionError> {
session
.externalize_media(self.blob_store.as_ref(), 0)
.await
.map_err(|err| {
SessionError::Agent(meerkat_core::error::AgentError::InternalError(format!(
"failed to externalize session images for persistence: {err}"
)))
})?;
if let Some(mut state) = session.deferred_turn_state() {
externalize_deferred_turn_state(self.blob_store.as_ref(), &mut state)
.await
.map_err(|err| {
SessionError::Agent(meerkat_core::error::AgentError::InternalError(format!(
"failed to externalize deferred-turn images for persistence: {err}"
)))
})?;
session.set_deferred_turn_state(state).map_err(|err| {
SessionError::Agent(meerkat_core::error::AgentError::InternalError(format!(
"failed to serialize deferred-turn state for persistence: {err}"
)))
})?;
}
Ok(session)
}
async fn save_normalized_session(&self, session: Session) -> Result<Session, SessionError> {
let profile = self.runtime_store.session_persistence_profile();
if profile != RuntimeSessionPersistenceProfile::WholeBlobV1 {
return Err(SessionError::Agent(AgentError::InternalError(format!(
"whole-document save helper refused runtime persistence profile {profile}; head-canonical callers must prepare a typed suffix/head mutation before encoding"
))));
}
let session = self.normalized_session_for_persistence(session).await?;
let plan = self
.transcript_rewrite_commit_chain_for_persistence(&session)
.await?;
if !plan.commits.is_empty() {
return self
.persist_normalized_transcript_rewrite_chain(session, &plan.commits, false)
.await;
}
let session_id = session.id().clone();
let session = Arc::new(session);
let committed = BoundSessionCommit::sealed(Arc::clone(&session)).map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"failed to seal session control snapshot for session {session_id}: {error}"
)))
})?;
let expected_blob_sha256 = committed
.whole_blob_artifact()
.map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"failed to encode session control snapshot for session {session_id}: {error}"
)))
})?
.row_sha256_token()
.to_string();
let result = self
.runtime_store
.commit_prepared_session_boundary(
&Self::runtime_id_for_session(&session_id),
meerkat_runtime::PreparedRuntimeSessionCommit::snapshot_only(committed),
)
.await
.map_err(|err| {
SessionError::Agent(meerkat_core::error::AgentError::InternalError(format!(
"runtime session control snapshot persistence failed: {err}"
)))
})?;
let authority = result
.authority()
.and_then(meerkat_runtime::store::RuntimeSessionAuthority::whole_blob)
.ok_or_else(|| {
SessionError::Agent(AgentError::InternalError(format!(
"runtime session control snapshot committed no WholeBlob authority for session {session_id}"
)))
})?;
if authority.session_id() != &session_id || authority.blob_sha256() != expected_blob_sha256
{
return Err(SessionError::Agent(AgentError::InternalError(format!(
"runtime session control snapshot returned different WholeBlob authority for session {session_id}"
))));
}
let committed_authority = meerkat_core::CommittedSessionBoundaryAuthority::WholeBlob {
session_id: session_id.clone(),
committed_store_revision: authority.store_revision(),
committed_blob_sha256: authority.blob_sha256().to_string(),
};
self.acknowledge_committed_runtime_session_boundary_with_recovery_gate(
&session_id,
&committed_authority,
)
.await?;
Arc::try_unwrap(session).map_err(|_| {
SessionError::Agent(AgentError::InternalError(format!(
"runtime store retained the typed control snapshot for session {session_id}"
)))
})
}
async fn persist_detached_head_canonical_session(
&self,
mut session: Session,
role: &str,
) -> Result<Session, SessionError> {
let profile = self.runtime_store.session_persistence_profile();
if profile != RuntimeSessionPersistenceProfile::HeadCanonicalV1 {
return Err(SessionError::Agent(AgentError::InternalError(format!(
"head-canonical detached mutation helper refused runtime persistence profile {profile}"
))));
}
let id = session.id().clone();
let runtime_id = Self::runtime_id_for_session(&id);
let runtime_authority = self
.runtime_store
.load_session_boundary_authority(&runtime_id)
.await
.map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"failed to load head-canonical runtime authority while preparing {role} for session {id}: {error}"
)))
})?;
if let Some(authority) = runtime_authority.as_ref()
&& (authority.profile() != RuntimeSessionPersistenceProfile::HeadCanonicalV1
|| authority.session_id() != &id)
{
return Err(SessionError::Agent(AgentError::InternalError(format!(
"head-canonical runtime authority while preparing {role} does not match session {id}"
))));
}
let incremental = self.incremental.as_ref().ok_or_else(|| {
SessionError::Agent(AgentError::InternalError(format!(
"head-canonical runtime session {id} has no incremental SessionStore capability"
)))
})?;
let observed_head = incremental
.load_head(&id)
.await
.map_err(|error| SessionError::Store(Box::new(error)))?;
let suffix_start_u64 = observed_head.as_ref().map_or(0, |head| head.message_count);
let suffix_start = usize::try_from(suffix_start_u64).map_err(|_| {
SessionError::Agent(AgentError::InternalError(format!(
"head-canonical predecessor message count exceeds the host index range while preparing {role} for session {id}"
)))
})?;
session
.externalize_media(self.blob_store.as_ref(), suffix_start)
.await
.map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"failed to externalize head-canonical suffix while preparing {role} for session {id}: {error}"
)))
})?;
if let Some(mut state) = session.deferred_turn_state() {
externalize_deferred_turn_state(self.blob_store.as_ref(), &mut state)
.await
.map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"failed to externalize deferred-turn state while preparing {role} for session {id}: {error}"
)))
})?;
session.set_deferred_turn_state(state).map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"failed to project deferred-turn state while preparing {role} for session {id}: {error}"
)))
})?;
}
let mutation = match (runtime_authority.as_ref(), observed_head) {
(None, None) => {
meerkat_core::session_store::PreparedHeadCanonicalMutation::prepare_root(&session)
.map(
meerkat_core::lifecycle::core_executor::PreparedHeadCanonicalPhysicalMutation::from,
)
}
(Some(authority), Some(observed_head)) => {
let authority = authority.head_canonical().ok_or_else(|| {
SessionError::Agent(AgentError::InternalError(format!(
"cannot prepare {role} for session {id}: runtime authority is not head-canonical"
)))
})?;
meerkat_core::session_store::PreparedHeadCanonicalMutationRoute::prepare_successor(
&session,
authority.boundary_head(),
observed_head,
)
.map(|route| match route {
meerkat_core::session_store::PreparedHeadCanonicalMutationRoute::Ordinary(
mutation,
) => mutation.into(),
meerkat_core::session_store::PreparedHeadCanonicalMutationRoute::Rewrite(
mutation,
) => mutation.into(),
})
}
(None, Some(_)) => {
return Err(SessionError::Agent(AgentError::InternalError(format!(
"cannot prepare {role} for session {id}: physical head exists without runtime authority"
))));
}
(Some(_), None) => {
return Err(SessionError::Agent(AgentError::InternalError(format!(
"cannot prepare {role} for session {id}: runtime authority exists without a physical head"
))));
}
};
let mutation = mutation.map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"failed to prepare {role} head-canonical mutation for session {id}: {error}"
)))
})?;
let committed = BoundSessionCommit::head_canonical_physical_from_session(
&session, mutation,
)
.map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"failed to seal {role} head-canonical boundary for session {id}: {error}"
)))
})?;
let acknowledgement = committed.clone();
let expected_boundary = {
let boundary = acknowledgement.head_canonical().ok_or_else(|| {
SessionError::Agent(AgentError::InternalError(format!(
"prepared {role} boundary for session {id} is not head-canonical"
)))
})?;
PreparedRuntimeBoundaryIdentity {
profile: RuntimeSessionPersistenceProfile::HeadCanonicalV1,
session_id: id.clone(),
authority_token: boundary.mutation().successor_head_token().to_string(),
}
};
let result = self
.runtime_store
.commit_prepared_session_boundary(
&runtime_id,
meerkat_runtime::PreparedRuntimeSessionCommit::snapshot_only(committed),
)
.await
.map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"head-canonical {role} persistence failed for session {id}: {error}"
)))
})?;
let authority = result.authority().ok_or_else(|| {
SessionError::Agent(AgentError::InternalError(format!(
"head-canonical {role} persistence committed no session authority for session {id}"
)))
})?;
let committed_boundary =
PreparedRuntimeBoundaryIdentity::from_runtime_authority(authority, &id)?;
if committed_boundary != expected_boundary {
return Err(SessionError::Agent(AgentError::InternalError(format!(
"head-canonical {role} persistence returned different authority for session {id}"
))));
}
let boundary = acknowledgement.head_canonical().ok_or_else(|| {
SessionError::Agent(AgentError::InternalError(format!(
"head-canonical {role} acknowledgement carrier disappeared for session {id}"
)))
})?;
boundary
.mutation()
.acknowledge_session(&mut session, &committed_boundary.authority_token)
.map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"failed to acknowledge head-canonical {role} for session {id}: {error}"
)))
})?;
Ok(session)
}
async fn transcript_rewrite_commit_chain_for_persistence(
&self,
session: &Session,
) -> Result<BoundaryPersistencePlan, SessionError> {
let previous = self
.load_committed_runtime_session_for_body(
session.id(),
"transcript rewrite persistence planning",
)
.await?;
let Some(previous) = previous else {
return Ok(BoundaryPersistencePlan {
commits: Vec::new(),
});
};
let previous_prefix = previous
.transcript_rewrite_prefix_authority()
.unwrap_or_else(meerkat_core::TranscriptRewritePrefixAccumulator::empty);
let incoming_prefix = session
.transcript_rewrite_prefix_authority()
.unwrap_or_else(meerkat_core::TranscriptRewritePrefixAccumulator::empty);
if incoming_prefix == previous_prefix {
return Ok(BoundaryPersistencePlan {
commits: Vec::new(),
});
}
let incoming_history = session
.validated_transcript_history_state()
.map_err(|err| {
SessionError::Agent(meerkat_core::error::AgentError::InternalError(format!(
"failed to read incoming transcript history for persistence: {err}"
)))
})?;
let commits: Vec<meerkat_core::TranscriptRewriteCommit> = match incoming_history.as_ref() {
Some(history) => history
.prove_commit_suffix_after(&previous_prefix)
.map_err(|error| {
SessionError::Store(Box::new(SessionStoreError::InvalidTranscriptRewrite {
id: session.id().clone(),
reason: format!(
"incoming rewrite occurrences do not extend exact previous prefix: {error}"
),
}))
})?
.commits()
.cloned()
.collect(),
None => {
return Err(SessionError::Store(Box::new(
SessionStoreError::InvalidTranscriptRewrite {
id: session.id().clone(),
reason:
"incoming rewrite-prefix authority changed without a sealed transcript graph"
.to_string(),
},
)));
}
};
if commits.is_empty() {
return Err(SessionError::Store(Box::new(
SessionStoreError::InvalidTranscriptRewrite {
id: session.id().clone(),
reason: "changed rewrite-prefix authority produced an empty occurrence suffix"
.to_string(),
},
)));
}
Ok(BoundaryPersistencePlan { commits })
}
pub async fn checkpoint_committed_runtime_session_snapshot(
&self,
id: &SessionId,
session_snapshot: Arc<Vec<u8>>,
) -> Result<(), SessionError> {
let _turn_finalization_guard = self.acquire_runtime_turn_finalization_guard(id).await;
self.checkpoint_committed_runtime_session_snapshot_under_runtime_turn_boundary(
id,
session_snapshot,
)
.await
}
pub async fn checkpoint_committed_runtime_session_snapshot_under_runtime_turn_boundary(
&self,
id: &SessionId,
session_snapshot: Arc<Vec<u8>>,
) -> Result<(), SessionError> {
let _projection_guard = self.recovery_gate_for_session(id).await.lock_owned().await;
self.checkpoint_committed_runtime_session_snapshot_with_recovery_gate(id, session_snapshot)
.await
}
pub async fn acknowledge_committed_runtime_session_boundary_under_runtime_turn_boundary(
&self,
id: &SessionId,
authority: &meerkat_core::CommittedSessionBoundaryAuthority,
) -> Result<(), SessionError> {
let _projection_guard = self.recovery_gate_for_session(id).await.lock_owned().await;
self.acknowledge_committed_runtime_session_boundary_with_recovery_gate(id, authority)
.await
}
async fn acknowledge_committed_runtime_session_boundary_with_recovery_gate(
&self,
id: &SessionId,
authority: &meerkat_core::CommittedSessionBoundaryAuthority,
) -> Result<(), SessionError> {
let authority_session_id = match authority {
meerkat_core::CommittedSessionBoundaryAuthority::WholeBlob { session_id, .. }
| meerkat_core::CommittedSessionBoundaryAuthority::HeadCanonical {
session_id, ..
}
| meerkat_core::CommittedSessionBoundaryAuthority::Provisional { session_id, .. } => {
session_id
}
};
if authority_session_id != id {
return Err(SessionError::Agent(AgentError::InternalError(format!(
"committed session boundary authority for {authority_session_id} cannot acknowledge actor {id}"
))));
}
match authority {
meerkat_core::CommittedSessionBoundaryAuthority::WholeBlob {
committed_store_revision,
committed_blob_sha256,
..
} => {
self.acknowledge_whole_blob_runtime_boundary_after_commit(
id,
*committed_store_revision,
committed_blob_sha256,
)
.await
}
meerkat_core::CommittedSessionBoundaryAuthority::HeadCanonical {
committed_head_token,
..
} => {
if self.runtime_store.session_persistence_profile()
!= RuntimeSessionPersistenceProfile::HeadCanonicalV1
{
return Err(SessionError::Agent(AgentError::InternalError(format!(
"runtime attempted head-canonical acknowledgement for non-head-canonical session {id}"
))));
}
let authority = self
.runtime_store
.load_session_boundary_authority(&Self::runtime_id_for_session(id))
.await
.map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"failed to verify committed head-canonical authority for session {id}: {error}"
)))
})?
.ok_or_else(|| {
SessionError::Agent(AgentError::InternalError(format!(
"committed head-canonical authority is absent for session {id}"
)))
})?;
let durable_boundary =
PreparedRuntimeBoundaryIdentity::from_runtime_authority(&authority, id)?;
let expected_boundary = PreparedRuntimeBoundaryIdentity {
profile: RuntimeSessionPersistenceProfile::HeadCanonicalV1,
session_id: id.clone(),
authority_token: committed_head_token.clone(),
};
if durable_boundary != expected_boundary {
return Err(SessionError::Agent(AgentError::InternalError(format!(
"committed head-canonical checkpoint/token for session {id} do not match RuntimeStore authority"
))));
}
self.acknowledge_head_canonical_runtime_boundary_after_commit(id, &durable_boundary)
.await
}
meerkat_core::CommittedSessionBoundaryAuthority::Provisional {
committed_store_revision,
committed_authority_token,
..
} => {
self.acknowledge_provisional_runtime_boundary_after_commit(
id,
*committed_store_revision,
committed_authority_token,
)
.await
}
}
}
#[cfg(test)]
async fn acknowledge_whole_blob_runtime_session_boundary_under_runtime_turn_boundary(
&self,
id: &SessionId,
committed_store_revision: u64,
committed_blob_sha256: &str,
) -> Result<(), SessionError> {
let _projection_guard = self.recovery_gate_for_session(id).await.lock_owned().await;
self.acknowledge_whole_blob_runtime_boundary_after_commit(
id,
committed_store_revision,
committed_blob_sha256,
)
.await
}
#[cfg(test)]
async fn acknowledge_head_canonical_runtime_session_boundary_under_runtime_turn_boundary(
&self,
id: &SessionId,
boundary_head_cas_token: &str,
) -> Result<(), SessionError> {
let _projection_guard = self.recovery_gate_for_session(id).await.lock_owned().await;
if self.runtime_store.session_persistence_profile()
!= RuntimeSessionPersistenceProfile::HeadCanonicalV1
{
return Err(SessionError::Agent(AgentError::InternalError(format!(
"runtime attempted head-canonical acknowledgement for non-head-canonical session {id}"
))));
}
let authority = self
.runtime_store
.load_session_boundary_authority(&Self::runtime_id_for_session(id))
.await
.map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"failed to verify committed head-canonical authority for session {id}: {error}"
)))
})?
.ok_or_else(|| {
SessionError::Agent(AgentError::InternalError(format!(
"committed head-canonical authority is absent for session {id}"
)))
})?;
let durable_boundary =
PreparedRuntimeBoundaryIdentity::from_runtime_authority(&authority, id)?;
let expected_boundary = PreparedRuntimeBoundaryIdentity {
profile: RuntimeSessionPersistenceProfile::HeadCanonicalV1,
session_id: id.clone(),
authority_token: boundary_head_cas_token.to_string(),
};
if durable_boundary != expected_boundary {
return Err(SessionError::Agent(AgentError::InternalError(format!(
"committed head-canonical checkpoint/token for session {id} do not match RuntimeStore authority"
))));
}
self.acknowledge_head_canonical_runtime_boundary_after_commit(id, &durable_boundary)
.await
}
#[cfg(test)]
async fn acknowledge_provisional_runtime_session_boundary_under_runtime_turn_boundary(
&self,
id: &SessionId,
committed_store_revision: u64,
committed_authority_token: &str,
) -> Result<(), SessionError> {
let _projection_guard = self.recovery_gate_for_session(id).await.lock_owned().await;
self.acknowledge_provisional_runtime_boundary_after_commit(
id,
committed_store_revision,
committed_authority_token,
)
.await
}
async fn checkpoint_committed_runtime_session_snapshot_with_recovery_gate(
&self,
id: &SessionId,
session_snapshot: Arc<Vec<u8>>,
) -> Result<(), SessionError> {
let runtime_id = Self::runtime_id_for_session(id);
let durable_snapshot = self
.runtime_store
.load_committed_whole_blob_snapshot(&runtime_id)
.await
.map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"failed to verify committed runtime checkpoint for session {id}: {error}"
)))
})?;
let Some(durable_snapshot) = durable_snapshot else {
return Err(SessionError::Agent(AgentError::InternalError(format!(
"committed runtime checkpoint bytes for session {id} do not match RuntimeStore authority"
))));
};
if durable_snapshot.bytes() != session_snapshot.as_slice() {
return Err(SessionError::Agent(AgentError::InternalError(format!(
"committed runtime checkpoint bytes for session {id} do not match RuntimeStore authority"
))));
}
if durable_snapshot.session().id() != id {
return Err(SessionError::Agent(AgentError::InternalError(format!(
"committed runtime checkpoint body identifies session {}, not {id}",
durable_snapshot.session().id()
))));
}
if let Some(checkpointer) = self
.live_checkpointers
.lock()
.await
.get(id)
.cloned()
.and_then(|checkpointer| checkpointer.upgrade())
{
*checkpointer.whole_blob_base_authority.lock().map_err(|_| {
SessionError::Agent(AgentError::InternalError(format!(
"WholeBlob actor base authority lock is poisoned for session {id}"
)))
})? = Some(durable_snapshot.authority().clone());
}
Ok(())
}
pub async fn prepare_live_transient_turn_context_boundary(
&self,
id: &SessionId,
expected_run_id: &RunId,
contexts: Vec<TurnRequestContext>,
) -> Result<CoreBoundaryStageOutput, meerkat_core::CoreBoundaryStageError> {
let prepared = self
.inner
.prepare_transient_turn_context_for_active_turn(id, expected_run_id, contexts)
.await?;
tracing::debug!(
session_id = %id,
run_id = %prepared.expected_run_id(),
boundary_generation = prepared.boundary_generation(),
"prepared exact parked transient turn context"
);
Ok(prepared.into_stage_output(None))
}
#[cfg(test)]
async fn fail_closed_runtime_projection_update(
&self,
id: &SessionId,
error: SessionStoreError,
_rejected_snapshot: Option<&[u8]>,
) -> SessionError {
tracing::error!(
session_id = %id,
error = %error,
"session-store projection update failed after runtime authority commit; failing closed"
);
match self.discard_live_session_unfenced(id).await {
Ok(()) | Err(SessionError::NotFound { .. }) => {}
Err(discard_error) => {
tracing::warn!(
session_id = %id,
error = %discard_error,
"failed to discard live session after runtime-backed projection update failure"
);
}
}
SessionError::Store(Box::new(error))
}
async fn gate_for_session(&self, id: &SessionId) -> Arc<CheckpointerGate> {
let mut gates = self.checkpointer_gates.lock().await;
Arc::clone(gates.entry(id.clone()).or_insert_with(|| {
Arc::new(CheckpointerGate {
cancelled: Mutex::new(false),
})
}))
}
async fn existing_gate_for_session(&self, id: &SessionId) -> Option<Arc<CheckpointerGate>> {
let gates = self.checkpointer_gates.lock().await;
gates.get(id).cloned()
}
async fn recovery_gate_for_session(&self, id: &SessionId) -> Arc<Mutex<()>> {
let mut gates = self.recovery_gates.lock().await;
Arc::clone(
gates
.entry(id.clone())
.or_insert_with(|| Arc::new(Mutex::new(()))),
)
}
fn rejected_run_compaction_aborted_under_recovery_gate(&self, id: &SessionId) -> bool {
self.rejected_run_compaction_aborted
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.contains(id)
}
fn mark_rejected_run_compaction_aborted_under_recovery_gate(&self, id: &SessionId) {
self.rejected_run_compaction_aborted
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.insert(id.clone());
}
fn clear_rejected_run_compaction_aborted_under_recovery_gate(&self, id: &SessionId) {
self.rejected_run_compaction_aborted
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.remove(id);
}
async fn converge_rejected_run_cleanup_before_materialization_under_recovery_gate(
&self,
id: &SessionId,
) -> Result<(), SessionError> {
if !self.rejected_run_compaction_aborted_under_recovery_gate(id) {
return Ok(());
}
let result = self.discard_live_session_unfenced(id).await;
if matches!(&result, Ok(()) | Err(SessionError::NotFound { .. })) {
self.clear_rejected_run_compaction_aborted_under_recovery_gate(id);
}
match result {
Ok(()) | Err(SessionError::NotFound { .. }) => Ok(()),
Err(error) => Err(error),
}
}
async fn turn_finalization_gate_for_session(&self, id: &SessionId) -> Arc<Mutex<()>> {
let mut gates = self.turn_finalization_gates.lock().await;
gates.retain(|_, gate| gate.strong_count() != 0);
if let Some(gate) = gates.get(id).and_then(std::sync::Weak::upgrade) {
return gate;
}
let gate = Arc::new(Mutex::new(()));
gates.insert(id.clone(), Arc::downgrade(&gate));
gate
}
pub async fn acquire_runtime_turn_finalization_guard(
&self,
id: &SessionId,
) -> tokio::sync::OwnedMutexGuard<()> {
self.turn_finalization_gate_for_session(id)
.await
.lock_owned()
.await
}
pub async fn acquire_live_session_actor_turn_boundary_lease(
&self,
id: &SessionId,
) -> Result<LiveSessionActorTurnBoundaryLease, SessionError> {
let turn_finalization_guard = self.acquire_runtime_turn_finalization_guard(id).await;
let witness = self
.inner
.live_session_actor_witness(id)
.await
.ok_or_else(|| SessionError::NotFound { id: id.clone() })?;
Ok(LiveSessionActorTurnBoundaryLease {
witness,
_turn_finalization_guard: turn_finalization_guard,
})
}
pub async fn acquire_live_session_actor_turn_boundary_lease_exact(
&self,
witness: &LiveSessionActorWitness,
) -> Result<Option<LiveSessionActorTurnBoundaryLease>, SessionError> {
let turn_finalization_guard = self
.acquire_runtime_turn_finalization_guard(witness.session_id())
.await;
let Some(current) = self
.inner
.live_session_actor_witness(witness.session_id())
.await
else {
return Ok(None);
};
if ¤t != witness {
return Ok(None);
}
Ok(Some(LiveSessionActorTurnBoundaryLease {
witness: current,
_turn_finalization_guard: turn_finalization_guard,
}))
}
pub async fn reserve_runtime_turn_admission_for_actor(
&self,
lease: &LiveSessionActorTurnBoundaryLease,
) -> Result<crate::ephemeral::RuntimeContextAdmissionGuard, SessionError> {
self.inner
.acquire_runtime_context_admission_for_actor(lease.witness())
.await
}
pub async fn discard_live_session_actor(
&self,
lease: &LiveSessionActorTurnBoundaryLease,
) -> Result<bool, SessionError> {
self.inner.discard_live_session_actor(lease.witness()).await
}
#[doc(hidden)]
pub async fn discard_live_session_actor_under_runtime_turn_boundary(
&self,
witness: &LiveSessionActorWitness,
) -> Result<bool, SessionError> {
self.inner.discard_live_session_actor(witness).await
}
async fn live_persist_mutation_guard(
&self,
id: &SessionId,
) -> Result<SessionMutationGuard, SessionError> {
let turn_finalization_guard = self.acquire_runtime_turn_finalization_guard(id).await;
let recovery_gate = self.recovery_gate_for_session(id).await;
let recovery_guard = recovery_gate.lock_owned().await;
let _ = self.discard_stale_live_session_if_needed(id).await?;
if let Some(session) = self.load_authoritative_session_base(id).await? {
self.reject_if_archived_session(id, &session)
.await
.map_err(crate::control_error_into_session_error)?;
}
Ok(SessionMutationGuard {
_turn_finalization_guard: turn_finalization_guard,
_recovery_guard: recovery_guard,
})
}
async fn realtime_transcript_mutation_guard(
&self,
id: &SessionId,
) -> Result<SessionMutationGuard, SessionError> {
let turn_finalization_guard = self.acquire_runtime_turn_finalization_guard(id).await;
let recovery_gate = self.recovery_gate_for_session(id).await;
let recovery_guard = recovery_gate.lock_owned().await;
match self.live_session_authority(id).await? {
LiveSessionAuthority::DurableAuthoritative { session, reason } => {
if self
.session_archived_by_authority(id, session.as_ref())
.await?
{
return Err(SessionError::NotFound { id: id.clone() });
}
self.synchronize_live_session_from_durable(
id,
session.as_ref(),
LiveSessionSyncCause::Authority(reason),
)
.await?;
}
LiveSessionAuthority::NoLive => {
tracing::debug!(
session_id = %id,
"realtime transcript append found no live session before mutation"
);
}
LiveSessionAuthority::LiveAuthoritative { .. } => {}
}
if let Some(session) = self.load_authoritative_session_base(id).await? {
self.reject_if_archived_session(id, &session)
.await
.map_err(crate::control_error_into_session_error)?;
}
Ok(SessionMutationGuard {
_turn_finalization_guard: turn_finalization_guard,
_recovery_guard: recovery_guard,
})
}
fn build_runtime_receipt(
run_id: RunId,
boundary: RunApplyBoundary,
contributing_input_ids: Vec<InputId>,
conversation_digest: String,
message_count: usize,
) -> RunBoundaryReceiptDraft {
RunBoundaryReceiptDraft {
run_id,
boundary,
contributing_input_ids,
conversation_digest: Some(conversation_digest),
message_count,
}
}
async fn prepare_runtime_boundary(
&self,
id: &SessionId,
role: &str,
expected_run_id: Option<&RunId>,
) -> Result<PreparedRuntimeBoundary, SessionError> {
self.prepare_runtime_boundary_with_control_overrides(id, role, expected_run_id, None)
.await
}
async fn prepare_runtime_boundary_with_deferred_override(
&self,
id: &SessionId,
role: &str,
deferred_turn_state_override: Option<SessionDeferredTurnState>,
) -> Result<PreparedRuntimeBoundary, SessionError> {
self.prepare_runtime_boundary_with_control_overrides(
id,
role,
None,
deferred_turn_state_override,
)
.await
}
async fn prepare_runtime_boundary_with_control_overrides(
&self,
id: &SessionId,
role: &str,
expected_run_id: Option<&RunId>,
deferred_turn_state_override: Option<SessionDeferredTurnState>,
) -> Result<PreparedRuntimeBoundary, SessionError> {
if let Some(expected_run_id) = expected_run_id {
if deferred_turn_state_override.is_some() {
return Err(SessionError::Agent(AgentError::InternalError(format!(
"run-bound provisional promotion for session {id} cannot consume control overrides"
))));
}
let checkpointer = self
.live_checkpointers
.lock()
.await
.get(id)
.cloned()
.and_then(|checkpointer| checkpointer.upgrade());
if let Some(receipt) = match checkpointer {
Some(checkpointer) => checkpointer
.run_checkpoint_receipt(id, expected_run_id)
.await
.map_err(SessionError::Agent)?,
None => None,
} {
let profile = self.runtime_store.session_persistence_profile();
let authority_token = match (profile, receipt.authority()) {
(
RuntimeSessionPersistenceProfile::WholeBlobV1,
RunCheckpointAuthority::WholeBlob(authority),
) => authority.candidate_blob_sha256().to_string(),
(
RuntimeSessionPersistenceProfile::HeadCanonicalV1,
RunCheckpointAuthority::HeadCanonical(authority),
) => authority.physical_head_token().to_string(),
(profile, authority) => {
return Err(SessionError::Agent(AgentError::InternalError(format!(
"run checkpoint authority {authority:?} cannot promote through runtime persistence profile {profile} for session {id}"
))));
}
};
let message_count = usize::try_from(receipt.message_count()).map_err(|_| {
SessionError::Agent(AgentError::InternalError(format!(
"run checkpoint receipt message count exceeds host range for session {id}"
)))
})?;
let conversation_digest = receipt.conversation_digest().to_string();
let identity = PreparedRuntimeBoundaryIdentity {
profile,
session_id: id.clone(),
authority_token,
};
return Ok(PreparedRuntimeBoundary {
committed: BoundSessionCommit::provisional_promotion(receipt),
identity,
conversation_digest,
message_count,
});
}
}
match self.runtime_store.session_persistence_profile() {
RuntimeSessionPersistenceProfile::WholeBlobV1 => {
if deferred_turn_state_override.is_some() {
return Err(SessionError::Agent(AgentError::InternalError(format!(
"whole-blob boundary preparation for session {id} cannot consume head-canonical control candidates"
))));
}
let session = self.export_session_with_labels(id).await?;
let prepared_session = self.normalized_session_for_persistence(session).await?;
let conversation_digest =
prepared_session
.transcript_content_digest()
.map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"failed to digest {role} transcript for session {id}: {error}"
)))
})?;
let message_count = prepared_session.messages().len();
let committed =
BoundSessionCommit::sealed(Arc::new(prepared_session)).map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"failed to seal {role} for session {id}: {error}"
)))
})?;
let identity = PreparedRuntimeBoundaryIdentity::whole_blob(
id.clone(),
committed
.whole_blob_artifact()
.map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"failed to encode {role} for session {id}: {error}"
)))
})?
.row_sha256_token(),
);
Ok(PreparedRuntimeBoundary {
committed,
identity,
conversation_digest,
message_count,
})
}
RuntimeSessionPersistenceProfile::HeadCanonicalV1 => {
let runtime_id = Self::runtime_id_for_session(id);
let runtime_authority = self
.runtime_store
.load_session_boundary_authority(&runtime_id)
.await
.map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"failed to load head-canonical runtime authority while preparing {role} for session {id}: {error}"
)))
})?;
if runtime_authority.as_ref().is_some_and(|authority| {
authority.profile() != RuntimeSessionPersistenceProfile::HeadCanonicalV1
|| authority.session_id() != id
}) {
return Err(SessionError::Agent(AgentError::InternalError(format!(
"head-canonical runtime authority while preparing {role} does not match session {id}"
))));
}
let incremental = self.incremental.as_ref().ok_or_else(|| {
SessionError::Agent(AgentError::InternalError(format!(
"head-canonical runtime session {id} has no incremental SessionStore capability"
)))
})?;
let observed_head = incremental
.load_head(id)
.await
.map_err(|error| SessionError::Store(Box::new(error)))?;
let authority = match (runtime_authority.as_ref(), observed_head.as_ref()) {
(None, None) => HeadCanonicalRuntimeBoundaryAuthority::root(),
(Some(runtime_authority), Some(_)) => {
let runtime_authority =
runtime_authority.head_canonical().ok_or_else(|| {
SessionError::Agent(AgentError::InternalError(format!(
"head-canonical boundary preparation loaded WholeBlob authority for session {id}"
)))
})?;
HeadCanonicalRuntimeBoundaryAuthority::successor(
runtime_authority.store_revision(),
runtime_authority.boundary_head().clone(),
runtime_authority.committed_head_token().to_string(),
)
.map_err(|error| SessionError::Store(Box::new(error)))?
}
(None, Some(_)) => {
return Err(SessionError::Agent(AgentError::InternalError(format!(
"cannot prepare {role} for session {id}: physical head exists without runtime authority"
))));
}
(Some(_), None) => {
return Err(SessionError::Agent(AgentError::InternalError(format!(
"cannot prepare {role} for session {id}: runtime authority exists without a physical head"
))));
}
};
let prepared = self
.inner
.prepare_head_canonical_runtime_boundary(
id,
authority,
observed_head,
Arc::clone(&self.blob_store),
role,
deferred_turn_state_override,
)
.await?;
let identity = PreparedRuntimeBoundaryIdentity {
profile: RuntimeSessionPersistenceProfile::HeadCanonicalV1,
session_id: id.clone(),
authority_token: prepared.successor_head_token().to_string(),
};
let conversation_digest = prepared.conversation_digest().to_string();
let message_count = prepared.message_count();
let committed = prepared.into_committed();
Ok(PreparedRuntimeBoundary {
committed,
identity,
conversation_digest,
message_count,
})
}
profile => Err(SessionError::Agent(AgentError::InternalError(format!(
"unsupported runtime session persistence profile {profile} while preparing {role} for session {id}"
)))),
}
}
async fn commit_prepared_head_canonical_actor_boundary(
&self,
id: &SessionId,
role: &str,
prepared: PreparedRuntimeBoundary,
) -> Result<(usize, String), SessionError> {
if prepared.identity.profile != RuntimeSessionPersistenceProfile::HeadCanonicalV1 {
return Err(SessionError::Agent(AgentError::InternalError(format!(
"{role} for session {id} produced a non-head-canonical actor boundary"
))));
}
let result = self
.runtime_store
.commit_prepared_session_boundary(
&Self::runtime_id_for_session(id),
meerkat_runtime::PreparedRuntimeSessionCommit::snapshot_only(prepared.committed),
)
.await
.map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"head-canonical {role} failed for session {id}: {error}"
)))
})?;
let authority = result.authority().ok_or_else(|| {
SessionError::Agent(AgentError::InternalError(format!(
"head-canonical {role} committed no session authority for session {id}"
)))
})?;
let committed_boundary =
PreparedRuntimeBoundaryIdentity::from_runtime_authority(authority, id)?;
if committed_boundary != prepared.identity {
return Err(SessionError::Agent(AgentError::InternalError(format!(
"head-canonical {role} returned different authority for session {id}"
))));
}
self.acknowledge_head_canonical_runtime_boundary_after_commit(id, &committed_boundary)
.await?;
Ok((prepared.message_count, prepared.conversation_digest))
}
fn callback_pending_terminal(error: &SessionError) -> Option<CoreApplyTerminal> {
match error {
SessionError::Agent(AgentError::CallbackPending {
tool_use_id,
tool_name,
args,
}) => Some(CoreApplyTerminal::CallbackPending {
tool_use_id: tool_use_id.clone(),
tool_name: tool_name.clone(),
args: args.clone(),
}),
SessionError::Agent(AgentError::CallbackBatchPending { pending_tool_calls }) => {
Some(CoreApplyTerminal::CallbackBatchPending {
pending_tool_calls: pending_tool_calls.clone(),
})
}
_ => None,
}
}
async fn acknowledge_whole_blob_runtime_boundary_after_commit(
&self,
id: &SessionId,
committed_store_revision: u64,
committed_blob_sha256: &str,
) -> Result<(), SessionError> {
if self.runtime_store.session_persistence_profile()
!= RuntimeSessionPersistenceProfile::WholeBlobV1
{
return Err(SessionError::Agent(AgentError::InternalError(format!(
"runtime attempted WholeBlob acknowledgement for non-WholeBlob session {id}"
))));
}
let authority = self
.runtime_store
.load_whole_blob_store_authority(&Self::runtime_id_for_session(id))
.await
.map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"failed to confirm committed WholeBlob authority for session {id}: {error}"
)))
})?
.ok_or_else(|| {
SessionError::Agent(AgentError::InternalError(format!(
"committed WholeBlob authority is absent for session {id}"
)))
})?;
if authority.session_id() != id
|| authority.store_revision() != committed_store_revision
|| authority.blob_sha256() != committed_blob_sha256
{
return Err(SessionError::Agent(AgentError::InternalError(format!(
"committed WholeBlob authority changed before executor acknowledgement for session {id}"
))));
}
if let Some(checkpointer) = self
.live_checkpointers
.lock()
.await
.get(id)
.cloned()
.and_then(|checkpointer| checkpointer.upgrade())
{
*checkpointer.whole_blob_base_authority.lock().map_err(|_| {
SessionError::Agent(AgentError::InternalError(format!(
"WholeBlob actor base authority lock is poisoned for session {id}"
)))
})? = Some(authority);
}
Ok(())
}
async fn acknowledge_provisional_runtime_boundary_after_commit(
&self,
id: &SessionId,
committed_store_revision: u64,
committed_authority_token: &str,
) -> Result<(), SessionError> {
let authority = self
.runtime_store
.load_session_boundary_authority(&Self::runtime_id_for_session(id))
.await
.map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"failed to verify promoted provisional authority for session {id}: {error}"
)))
})?
.ok_or_else(|| {
SessionError::Agent(AgentError::InternalError(format!(
"promoted provisional authority is absent for session {id}"
)))
})?;
if authority.session_id() != id
|| authority.profile() != self.runtime_store.session_persistence_profile()
{
return Err(SessionError::Agent(AgentError::InternalError(format!(
"promoted provisional authority has the wrong session/profile for session {id}"
))));
}
let (observed_revision, observed_token) = match &authority {
RuntimeSessionAuthority::WholeBlob(authority) => {
(authority.store_revision(), authority.blob_sha256())
}
RuntimeSessionAuthority::HeadCanonical(authority) => {
(authority.store_revision(), authority.committed_head_token())
}
};
if observed_revision != committed_store_revision
|| observed_token != committed_authority_token
{
return Err(SessionError::Agent(AgentError::InternalError(format!(
"promoted provisional authority changed before executor acknowledgement for session {id}"
))));
}
match authority {
RuntimeSessionAuthority::WholeBlob(authority) => {
let checkpointer = self
.live_checkpointers
.lock()
.await
.get(id)
.cloned()
.and_then(|checkpointer| checkpointer.upgrade())
.ok_or_else(|| {
SessionError::Agent(AgentError::InternalError(format!(
"promoted WholeBlob boundary has no exact live checkpointer for session {id}"
)))
})?;
*checkpointer.whole_blob_base_authority.lock().map_err(|_| {
SessionError::Agent(AgentError::InternalError(format!(
"WholeBlob actor base authority lock is poisoned for session {id}"
)))
})? = Some(authority);
checkpointer
.acknowledge_promoted_checkpoint(
id,
committed_store_revision,
committed_authority_token,
)
.await
.map_err(SessionError::Agent)?;
}
RuntimeSessionAuthority::HeadCanonical(_) => {
let checkpointer = self
.live_checkpointers
.lock()
.await
.get(id)
.cloned()
.and_then(|checkpointer| checkpointer.upgrade())
.ok_or_else(|| {
SessionError::Agent(AgentError::InternalError(format!(
"promoted HeadCanonical boundary has no exact live checkpointer for session {id}"
)))
})?;
checkpointer
.acknowledge_promoted_checkpoint(
id,
committed_store_revision,
committed_authority_token,
)
.await
.map_err(SessionError::Agent)?;
}
}
Ok(())
}
async fn acknowledge_head_canonical_runtime_boundary_after_commit(
&self,
id: &SessionId,
committed_boundary: &PreparedRuntimeBoundaryIdentity,
) -> Result<(), SessionError> {
if committed_boundary.profile != RuntimeSessionPersistenceProfile::HeadCanonicalV1 {
return Err(SessionError::Agent(AgentError::InternalError(format!(
"cannot acknowledge non-head-canonical runtime boundary for session {id}"
))));
}
self.inner
.acknowledge_head_canonical_runtime_boundary(
id,
committed_boundary.authority_token.clone(),
)
.await?;
Ok(())
}
async fn build_runtime_output(
&self,
id: &SessionId,
run_id: RunId,
boundary: RunApplyBoundary,
contributing_input_ids: Vec<InputId>,
terminal: Option<CoreApplyTerminal>,
) -> Result<CoreApplyOutput, SessionError> {
let PreparedRuntimeBoundary {
committed,
identity: _,
conversation_digest,
message_count,
} = self
.prepare_runtime_boundary(id, "runtime output session boundary", Some(&run_id))
.await?;
let receipt = Self::build_runtime_receipt(
run_id,
boundary,
contributing_input_ids,
conversation_digest,
message_count,
);
let output = CoreApplyOutput::new(receipt, terminal).with_bound_session(committed);
Ok(output)
}
async fn build_runtime_output_after_live_mutation(
&self,
id: &SessionId,
run_id: RunId,
boundary: RunApplyBoundary,
contributing_input_ids: Vec<InputId>,
terminal: Option<CoreApplyTerminal>,
) -> Result<CoreApplyOutput, SessionError> {
match self
.build_runtime_output(id, run_id, boundary, contributing_input_ids, terminal)
.await
{
Ok(output) => Ok(output),
Err(error) => {
if let Err(discard_error) = self.discard_live_session_unfenced(id).await {
tracing::warn!(
session_id = %id,
error = %discard_error,
"failed to discard live session after runtime output build failure"
);
}
Err(error)
}
}
}
pub async fn apply_runtime_turn(
&self,
id: &SessionId,
run_id: RunId,
req: StartTurnRequest,
boundary: RunApplyBoundary,
contributing_input_ids: Vec<InputId>,
) -> Result<CoreApplyOutput, SessionError> {
self.apply_runtime_turn_outcome(id, run_id, req, boundary, contributing_input_ids)
.await
}
pub async fn apply_runtime_turn_outcome(
&self,
id: &SessionId,
run_id: RunId,
req: StartTurnRequest,
boundary: RunApplyBoundary,
contributing_input_ids: Vec<InputId>,
) -> Result<CoreApplyOutput, SessionError> {
self.apply_runtime_turn_outcome_with_admission(
id,
run_id,
req,
boundary,
contributing_input_ids,
None,
)
.await
}
pub async fn apply_runtime_turn_with_reserved_admission(
&self,
id: &SessionId,
run_id: RunId,
req: StartTurnRequest,
boundary: RunApplyBoundary,
contributing_input_ids: Vec<InputId>,
admission: crate::ephemeral::RuntimeContextAdmissionGuard,
) -> Result<CoreApplyOutput, SessionError> {
self.apply_runtime_turn_with_recoverable_reserved_admission(
id,
run_id,
req,
boundary,
contributing_input_ids,
admission,
)
.await
.map_err(|(error, _admission)| error)
}
pub async fn apply_runtime_turn_with_recoverable_reserved_admission(
&self,
id: &SessionId,
run_id: RunId,
req: StartTurnRequest,
boundary: RunApplyBoundary,
contributing_input_ids: Vec<InputId>,
admission: crate::ephemeral::RuntimeContextAdmissionGuard,
) -> Result<
CoreApplyOutput,
(
SessionError,
Option<crate::ephemeral::RuntimeContextAdmissionGuard>,
),
> {
Self::require_runtime_execution_kind_stamp(&req).map_err(|error| (error, None))?;
let recovery_gate = self.recovery_gate_for_session(id).await;
let _turn_guard = recovery_gate.lock().await;
let _ = self
.discard_stale_live_session_if_needed(id)
.await
.map_err(|error| (error, None))?;
let execution = self
.inner
.start_runtime_turn_execution_with_admission_recovering_not_found(id, req, admission)
.await?;
let (result, machine_terminal_failure) = self
.validated_runtime_execution_after_live_mutation(id, execution)
.await
.map_err(|error| (error, None))?;
match result.map_err(SessionError::Agent) {
Ok(run_result) => self
.build_runtime_output_after_live_mutation(
id,
run_id,
boundary,
contributing_input_ids,
Some(CoreApplyTerminal::RunResult(Box::new(run_result))),
)
.await
.map_err(|error| (error, None)),
Err(SessionError::Agent(meerkat_core::error::AgentError::NoPendingBoundary)) => self
.build_runtime_output_after_live_mutation(
id,
run_id,
boundary,
contributing_input_ids,
Some(CoreApplyTerminal::NoPendingBoundary),
)
.await
.map_err(|error| (error, None)),
Err(error) => {
if let Some(error) = machine_terminal_failure {
self.build_runtime_output_after_live_mutation(
id,
run_id,
boundary,
contributing_input_ids,
Some(CoreApplyTerminal::MachineTerminalFailure { error }),
)
.await
.map_err(|error| (error, None))
} else if let Some(terminal) = Self::callback_pending_terminal(&error) {
self.build_runtime_output_after_live_mutation(
id,
run_id,
boundary,
contributing_input_ids,
Some(terminal),
)
.await
.map_err(|error| (error, None))
} else if matches!(
error,
SessionError::Agent(meerkat_core::error::AgentError::Cancelled)
) {
Err((error, None))
} else {
tracing::warn!(
session_id = %id,
error = %error,
"runtime turn failed; preserving the live session for the next queued input"
);
Err((error, None))
}
}
}
}
async fn apply_runtime_turn_outcome_with_admission(
&self,
id: &SessionId,
run_id: RunId,
req: StartTurnRequest,
boundary: RunApplyBoundary,
contributing_input_ids: Vec<InputId>,
admission: Option<crate::ephemeral::RuntimeContextAdmissionGuard>,
) -> Result<CoreApplyOutput, SessionError> {
Self::require_runtime_execution_kind_stamp(&req)?;
let recovery_gate = self.recovery_gate_for_session(id).await;
let _turn_guard = recovery_gate.lock().await;
let _ = self.discard_stale_live_session_if_needed(id).await?;
let execution = match admission {
Some(admission) => self
.inner
.start_runtime_turn_execution_with_admission_recovering_not_found(
id, req, admission,
)
.await
.map_err(|(error, _admission)| error),
None => self.inner.start_runtime_turn_execution(id, req).await,
}?;
let (result, machine_terminal_failure) = self
.validated_runtime_execution_after_live_mutation(id, execution)
.await?;
match result.map_err(SessionError::Agent) {
Ok(run_result) => {
self.build_runtime_output_after_live_mutation(
id,
run_id,
boundary,
contributing_input_ids,
Some(CoreApplyTerminal::RunResult(Box::new(run_result))),
)
.await
}
Err(SessionError::Agent(meerkat_core::error::AgentError::NoPendingBoundary)) => {
self.build_runtime_output_after_live_mutation(
id,
run_id,
boundary,
contributing_input_ids,
Some(CoreApplyTerminal::NoPendingBoundary),
)
.await
}
Err(error) => {
if let Some(error) = machine_terminal_failure {
self.build_runtime_output_after_live_mutation(
id,
run_id,
boundary,
contributing_input_ids,
Some(CoreApplyTerminal::MachineTerminalFailure { error }),
)
.await
} else if let Some(terminal) = Self::callback_pending_terminal(&error) {
self.build_runtime_output_after_live_mutation(
id,
run_id,
boundary,
contributing_input_ids,
Some(terminal),
)
.await
} else if matches!(
error,
SessionError::Agent(meerkat_core::error::AgentError::Cancelled)
) {
Err(error)
} else {
tracing::warn!(
session_id = %id,
error = %error,
"runtime turn failed; preserving the live session for the next queued input"
);
Err(error)
}
}
}
}
fn require_runtime_execution_kind_stamp(req: &StartTurnRequest) -> Result<(), SessionError> {
if req
.runtime
.turn_metadata
.as_ref()
.and_then(|metadata| metadata.execution_kind)
.is_some()
{
return Ok(());
}
Err(SessionError::Agent(
meerkat_core::error::AgentError::InternalError(
"runtime_execution_kind not set: runtime-backed turn did not stamp RuntimeTurnMetadata.execution_kind"
.to_string(),
),
))
}
}
impl<B: SessionAgentBuilder + 'static> PersistentSessionService<B> {
pub async fn revive_archived_session_with_prepared_materialization(
&self,
id: &SessionId,
authority: PreparedArchivedResumeCommitLease,
) -> Result<meerkat_runtime::AuthorizedArchivedResumeCommitLease, SessionError> {
if authority.session_id() != id {
return Err(SessionError::Agent(AgentError::InternalError(format!(
"retired-session revival authority for session {} cannot authorize session {id}",
authority.session_id()
))));
}
self.revive_archived_session_lifecycle_inner(id, true)
.await?;
authority
.confirm_runtime_store_authority(&self.runtime_store)
.map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"retired-session authorization for session {id} could not certify shared runtime authority: {error}"
)))
})
}
#[cfg(test)]
pub async fn revive_archived_session_with_machine_authority(
&self,
id: &SessionId,
_authority: MachineSessionControlAuthority,
) -> Result<(), SessionError> {
self.revive_archived_session_lifecycle_inner(id, false)
.await
}
async fn revive_archived_session_lifecycle_inner(
&self,
id: &SessionId,
allow_normalized_idle_retry: bool,
) -> Result<(), SessionError> {
let recovery_gate = self.recovery_gate_for_session(id).await;
let _recovery_guard = recovery_gate.lock().await;
let runtime_id = Self::runtime_id_for_session(id);
let authority = self
.runtime_store
.load_session_boundary_authority(&runtime_id)
.await
.map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"failed to load archived session authority for session {id}: {error}"
)))
})?
.ok_or_else(|| SessionError::NotFound { id: id.clone() })?;
if authority.session_id() != id
|| authority.profile() != self.runtime_store.session_persistence_profile()
{
return Err(SessionError::Agent(AgentError::InternalError(format!(
"archived session authority does not match session {id}"
))));
}
let runtime_state = Self::load_runtime_state_for_session(&self.runtime_store, id).await?;
let runtime_is_retired = runtime_state == Some(RuntimeState::Retired);
let runtime_is_normalized_idle = runtime_state == Some(RuntimeState::Idle);
let quiescent_runtime_is_admissible =
runtime_is_retired || (runtime_is_normalized_idle && allow_normalized_idle_retry);
if !quiescent_runtime_is_admissible {
return Err(SessionError::Agent(AgentError::InternalError(format!(
"session {id} has no retired RuntimeStore lifecycle authority"
))));
}
if runtime_is_normalized_idle {
return Ok(());
}
if let Some(gate) = self.existing_gate_for_session(id).await {
*gate.cancelled.lock().await = false;
}
Ok(())
}
pub async fn reconcile_runtime_compaction_projections(
&self,
id: &SessionId,
intents: Vec<meerkat_core::CompactionProjectionIntent>,
) -> Result<(), SessionError> {
let has_committed_rewrite = !intents.is_empty();
self.inner
.reconcile_runtime_compaction_projections(id, intents)
.await?;
if has_committed_rewrite {
self.load_authoritative_session_base(id)
.await?
.ok_or_else(|| SessionError::NotFound { id: id.clone() })?;
}
Ok(())
}
pub async fn abort_uncommitted_compaction_projections(
&self,
id: &SessionId,
) -> Result<(), SessionError> {
self.inner
.abort_uncommitted_compaction_projections(id)
.await
}
pub async fn abort_rejected_runtime_run_projections(
&self,
id: &SessionId,
) -> Result<(), SessionError> {
let recovery_gate = self.recovery_gate_for_session(id).await;
let _recovery_guard = recovery_gate.lock_owned().await;
if !self.rejected_run_compaction_aborted_under_recovery_gate(id) {
self.abort_uncommitted_compaction_projections(id).await?;
self.mark_rejected_run_compaction_aborted_under_recovery_gate(id);
}
let result = self.discard_live_session_unfenced(id).await;
if matches!(&result, Ok(()) | Err(SessionError::NotFound { .. })) {
self.clear_rejected_run_compaction_aborted_under_recovery_gate(id);
}
match result {
Ok(()) | Err(SessionError::NotFound { .. }) => Ok(()),
Err(error) => Err(error),
}
}
async fn create_session_with_admission(
&self,
mut req: CreateSessionRequest,
reserved_create_admission: Option<crate::ephemeral::RuntimeContextAdmissionGuard>,
actor_seed_authority: ActorSessionSeedAuthority,
archived_resume_authorization: ArchivedResumeAuthorization,
turn_boundary_already_held: bool,
actor_witness_slot: Option<&LiveSessionActorWitnessSlot>,
) -> Result<RunResult, SessionError> {
self.reject_runtime_backed_eager_create_session(&req)?;
let gate = Arc::new(CheckpointerGate {
cancelled: Mutex::new(false),
});
let checkpointer = Arc::new(StoreCheckpointer {
runtime_store: Arc::clone(&self.runtime_store),
incremental: self.incremental.clone(),
blob_store: Arc::clone(&self.blob_store),
gate: Arc::clone(&gate),
whole_blob_base_authority: std::sync::Mutex::new(None),
latest_run_checkpoint_receipt: Mutex::new(None),
});
let requested_resume_session_id = req
.build
.as_ref()
.and_then(|build| build.resume_session.as_ref())
.map(|session| session.id().clone());
if let Some(resume_session_id) = requested_resume_session_id.as_ref() {
if matches!(
actor_seed_authority,
ActorSessionSeedAuthority::DurableCommitted
) {
match self.recover_committed_boundary(resume_session_id).await? {
CommittedBoundaryRecovery::AlreadyCommitted
| CommittedBoundaryRecovery::Recovered { .. } => {}
CommittedBoundaryRecovery::Unprovable { reason } => {
return Err(SessionError::Agent(AgentError::InternalError(reason)));
}
}
}
match (
self.runtime_store.session_persistence_profile(),
actor_seed_authority,
) {
(
RuntimeSessionPersistenceProfile::WholeBlobV1,
ActorSessionSeedAuthority::DurableCommitted,
) => {
let (exact_session, exact_snapshot) = load_committed_whole_blob_session(
self.runtime_store.as_ref(),
resume_session_id,
"live actor materialization",
)
.await?
.ok_or_else(|| SessionError::NotFound {
id: resume_session_id.clone(),
})?;
req.build
.get_or_insert_with(Default::default)
.resume_session = Some(exact_session);
*checkpointer
.whole_blob_base_authority
.lock()
.map_err(|_| {
SessionError::Agent(AgentError::InternalError(format!(
"WholeBlob actor base authority lock is poisoned for session {resume_session_id}"
)))
})? = Some(exact_snapshot.authority().clone());
}
(
RuntimeSessionPersistenceProfile::WholeBlobV1,
ActorSessionSeedAuthority::FreshGenerationZero,
) => {
let runtime_id = Self::runtime_id_for_session(resume_session_id);
if self
.runtime_store
.load_whole_blob_store_authority(&runtime_id)
.await
.map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"failed to prove fresh WholeBlob materialization for session {resume_session_id}: {error}"
)))
})?
.is_some()
{
return Err(SessionError::Agent(AgentError::InternalError(format!(
"fresh WholeBlob materialization conflicts with existing durable authority for session {resume_session_id}"
))));
}
}
(
RuntimeSessionPersistenceProfile::HeadCanonicalV1,
ActorSessionSeedAuthority::DurableCommitted,
) => {
let exact_session = self
.load_committed_runtime_session_for_body(
resume_session_id,
"live actor materialization",
)
.await?
.ok_or_else(|| SessionError::NotFound {
id: resume_session_id.clone(),
})?;
req.build
.get_or_insert_with(Default::default)
.resume_session = Some(exact_session);
}
(
RuntimeSessionPersistenceProfile::HeadCanonicalV1,
ActorSessionSeedAuthority::FreshGenerationZero,
) => {
let runtime_id = Self::runtime_id_for_session(resume_session_id);
if self
.runtime_store
.load_session_boundary_authority(&runtime_id)
.await
.map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"failed to prove fresh HeadCanonical materialization for session {resume_session_id}: {error}"
)))
})?
.is_some()
{
return Err(SessionError::Agent(AgentError::InternalError(format!(
"fresh HeadCanonical materialization conflicts with existing durable authority for session {resume_session_id}"
))));
}
}
(profile, _) => {
return Err(SessionError::Agent(AgentError::InternalError(format!(
"unsupported runtime session persistence profile {profile} while materializing actor for session {resume_session_id}"
))));
}
}
}
let (resume_session_id, resume_session) = {
let build = req.build.get_or_insert_with(Default::default);
let resume_session_id = build
.resume_session
.as_ref()
.map(|session| session.id().clone());
let resume_session = build.resume_session.clone();
build.checkpointer = Some(checkpointer.clone());
build.blob_store_override = Some(Arc::clone(&self.blob_store));
(resume_session_id, resume_session)
};
let runtime_binding_session_id =
req.build
.as_ref()
.and_then(|build| match &build.runtime_build_mode {
meerkat_core::RuntimeBuildMode::SessionOwned(bindings) => {
Some(bindings.session_id().clone())
}
meerkat_core::RuntimeBuildMode::StandaloneEphemeral => None,
});
let archived_resume_allowed = archived_resume_authorization.allows_archived_resume();
match (
archived_resume_allowed,
resume_session_id.as_ref(),
runtime_binding_session_id.as_ref(),
) {
(true, None, _) => {
return Err(SessionError::Agent(AgentError::InternalError(
"machine-authorized archived resume requires an exact resume-session snapshot"
.to_string(),
)));
}
(true, Some(_), None) => {
return Err(SessionError::Agent(AgentError::InternalError(
"machine-authorized archived resume requires exact SessionOwned runtime bindings"
.to_string(),
)));
}
(_, Some(resume_session_id), Some(runtime_binding_session_id))
if resume_session_id != runtime_binding_session_id =>
{
return Err(SessionError::Agent(AgentError::InternalError(format!(
"runtime binding session {runtime_binding_session_id} does not match resumed session {resume_session_id}"
))));
}
_ => {}
}
if let Some(session) = resume_session.as_ref()
&& self
.session_archived_by_authority(session.id(), session)
.await?
&& !archived_resume_authorization.allows_archived_resume()
{
return Err(SessionError::NotFound {
id: session.id().clone(),
});
}
if archived_resume_allowed
&& let Some(session) = req
.build
.as_mut()
.and_then(|build| build.resume_session.as_mut())
&& session.try_lifecycle_terminal().map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"archived resume session {} has malformed lifecycle terminal: {error}",
session.id()
)))
})? == Some(SessionLifecycleTerminal::Archived)
{
let mut authority = SessionDocumentMachineAuthority::new();
let document_key = SessionDocumentKey::new(session.id().to_string());
authority
.recover_session_lifecycle_terminal(
document_key.clone(),
SessionLifecycleTerminal::Archived.into(),
)
.map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"generated session document authority rejected archived lifecycle recovery for session {}: {error}",
session.id()
)))
})?;
let effects = authority
.revive_archived_session_document(document_key)
.map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"generated session document authority rejected revival for session {}: {error}",
session.id()
)))
})?;
if !effects
.iter()
.any(|effect| matches!(effect, SessionDocumentEffect::SessionRevivalResolved))
{
return Err(SessionError::Agent(AgentError::InternalError(format!(
"generated session document authority emitted no revival verdict for session {}",
session.id()
))));
}
session
.set_lifecycle_terminal(SessionLifecycleTerminal::Active)
.map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"failed to realize generated session revival for session {}: {error}",
session.id()
)))
})?;
}
let materialization_session_id = runtime_binding_session_id
.as_ref()
.or(resume_session_id.as_ref());
let _turn_finalization_guard =
match (materialization_session_id, turn_boundary_already_held) {
(Some(materialization_session_id), false) => Some(
self.acquire_runtime_turn_finalization_guard(materialization_session_id)
.await,
),
(Some(_), true) | (None, _) => None,
};
let mut actor_materialization_permit = if let Some(bindings) =
req.build
.as_ref()
.and_then(|build| match &build.runtime_build_mode {
meerkat_core::RuntimeBuildMode::SessionOwned(bindings) => Some(bindings),
meerkat_core::RuntimeBuildMode::StandaloneEphemeral => None,
}) {
let actor_materialization = match archived_resume_authorization {
ArchivedResumeAuthorization::RejectArchived => {
meerkat_runtime::begin_session_runtime_actor_materialization(bindings)
}
ArchivedResumeAuthorization::MachinePendingPromotion { authorization } => {
meerkat_runtime::begin_session_runtime_actor_materialization_for_archived_resume(
bindings,
authorization,
)
.await
}
};
match actor_materialization {
Ok(permit) => Some(permit),
Err(meerkat_runtime::RuntimeActorMaterializationError::RegistrationClosed) => {
return Err(SessionError::NotFound {
id: materialization_session_id
.cloned()
.unwrap_or_else(|| bindings.session_id().clone()),
});
}
Err(meerkat_runtime::RuntimeActorMaterializationError::InvalidAuthority(
reason,
)) => {
return Err(SessionError::Agent(AgentError::InternalError(reason)));
}
}
} else {
None
};
let _resume_recovery_guard =
if let Some(materialization_session_id) = materialization_session_id {
let recovery_gate = self
.recovery_gate_for_session(materialization_session_id)
.await;
let guard = recovery_gate.lock_owned().await;
self.converge_rejected_run_cleanup_before_materialization_under_recovery_gate(
materialization_session_id,
)
.await?;
if let Some(resume_session_id) = resume_session_id.as_ref()
&& let Some(session) = self
.load_authoritative_session_base(resume_session_id)
.await?
&& self
.session_archived_by_authority(resume_session_id, &session)
.await?
&& !archived_resume_allowed
{
self.reject_if_archived_session(resume_session_id, &session)
.await
.map_err(crate::control_error_into_session_error)?;
}
Some(guard)
} else {
None
};
if let Some(resume_session_id) = resume_session_id.as_ref() {
self.ensure_event_projection_admitted(resume_session_id)
.await?;
}
let create_time_projection = self
.install_create_time_event_projection(&mut req, materialization_session_id)
.await;
let callback_session_id = resume_session_id.clone();
let (result, _actor_witness) = match self
.inner
.create_session_with_admission_and_witness(
req,
reserved_create_admission,
actor_witness_slot,
)
.await
{
Ok(result) => result,
Err(error) => {
if Self::callback_pending_terminal(&error).is_some()
&& let Some(session_id) = callback_session_id
{
self.persist_full_session_or_discard_live(&session_id)
.await?;
}
return Err(error);
}
};
if let Some(permit) = actor_materialization_permit.take()
&& let Err(error) = permit.commit()
{
let _ = self.discard_live_session_unfenced(&result.session_id).await;
return Err(SessionError::Agent(AgentError::InternalError(format!(
"runtime actor materialization commit failed for session {}: {error}",
result.session_id
))));
}
{
self.checkpointer_gates
.lock()
.await
.insert(result.session_id.clone(), gate);
self.live_checkpointers
.lock()
.await
.insert(result.session_id.clone(), Arc::downgrade(&checkpointer));
}
if let Some(create_time_projection) = create_time_projection {
register_event_projection_drain(
&self.event_projection_drains,
&result.session_id,
create_time_projection.drain_rx.clone(),
)
.await;
let _ = create_time_projection
.session_tx
.send(Some(result.session_id.clone()));
}
if let Err(error) = self.spawn_event_projection_task(&result.session_id).await {
let _ = self.discard_live_session(&result.session_id).await;
return Err(error);
}
self.persist_full_session_or_discard_live(&result.session_id)
.await?;
if self.runtime_store.session_persistence_profile()
== RuntimeSessionPersistenceProfile::WholeBlobV1
{
let authority = self
.runtime_store
.load_whole_blob_store_authority(&Self::runtime_id_for_session(
&result.session_id,
))
.await
.map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"failed to seed WholeBlob actor base authority for session {}: {error}",
result.session_id
)))
})?
.ok_or_else(|| {
SessionError::Agent(AgentError::InternalError(format!(
"WholeBlob generation-zero commit returned without authority for session {}",
result.session_id
)))
})?;
*checkpointer.whole_blob_base_authority.lock().map_err(|_| {
SessionError::Agent(AgentError::InternalError(format!(
"WholeBlob actor base authority lock is poisoned for session {}",
result.session_id
)))
})? = Some(authority);
}
Ok(result)
}
pub async fn archive_with_machine_protocol(
&self,
id: &SessionId,
machine_archive: MachineSessionArchiveProtocol<'_>,
) -> Result<(), SessionError> {
let _turn_finalization_guard = self.acquire_runtime_turn_finalization_guard(id).await;
self.archive_with_machine_protocol_under_runtime_turn_boundary(id, machine_archive)
.await
}
pub async fn archive_with_machine_protocol_under_runtime_turn_boundary(
&self,
id: &SessionId,
machine_archive: MachineSessionArchiveProtocol<'_>,
) -> Result<(), SessionError> {
machine_archive.require_shared_runtime_store(id, &self.runtime_store)?;
let (mut archive_lease, _recovery_guard) = loop {
let candidate_lease = machine_archive.prepare_archive_lease(id).await?;
let recovery_gate = self.recovery_gate_for_session(id).await;
let recovery_guard = recovery_gate.lock_owned().await;
if candidate_lease.is_none()
&& machine_archive
.runtime_archive_authority_present(id, &self.runtime_store)
.await?
{
drop(recovery_guard);
continue;
}
break (candidate_lease, recovery_guard);
};
let archive_lease_owns_recovered_registration = archive_lease
.as_ref()
.is_some_and(|lease| lease.owns_recovered_registration());
let archive_result = async {
let _ = self.discard_stale_live_session_if_needed(id).await?;
let stored_only_publication_handle = if self.inner.has_live_session(id).await? {
None
} else {
let _ = self.event_log_await_projection_drain(id).await?;
Some(StoredOnlyArchiveTerminalPublisher {
service: self,
session_id: id,
})
};
machine_archive
.drain_archive_lease_terminals(
archive_lease.as_ref(),
stored_only_publication_handle
.as_ref()
.map(|publisher| publisher as &dyn CoreExecutorPublicationHandle),
)
.await?;
let archived_snapshot = match self.export_session_with_labels(id).await {
Ok(session) => Some(session),
Err(SessionError::NotFound { .. }) => {
self.load_persisted_session_for_archive(id).await?
}
Err(err) => return Err(err),
};
let document_lifecycle_seed = if let Some(ref session) = archived_snapshot
&& self.session_archived_by_authority(id, session).await?
{
SessionLifecycleTerminal::Archived
} else {
SessionLifecycleTerminal::Active
};
let runtime_observation = match archive_lease.as_ref() {
Some(lease) => match lease.runtime_state().await {
RuntimeState::Retired | RuntimeState::Destroyed => {
SessionArchiveRuntimeObservation::QuiescentTerminal
}
_ => SessionArchiveRuntimeObservation::RetirementRequired,
},
None => SessionArchiveRuntimeObservation::Absent,
};
let mut authority = SessionDocumentMachineAuthority::new();
let document_key = SessionDocumentKey::new(id.to_string());
authority
.recover_session_lifecycle_terminal(
document_key.clone(),
document_lifecycle_seed.into(),
)
.map_err(|err| {
SessionError::Agent(AgentError::InternalError(format!(
"generated session document authority rejected lifecycle-terminal recovery \
for session {id}: {err}"
)))
})?;
let effects = authority
.archive_session_document(
document_key,
true,
archived_snapshot.is_some(),
runtime_observation,
)
.map_err(|err| {
SessionError::Agent(AgentError::InternalError(format!(
"generated session document authority rejected archive for session {id}: {err}"
)))
})?;
let (disposition, write_document, retire_runtime) = effects
.iter()
.find_map(|effect| match effect {
SessionDocumentEffect::SessionArchiveResolved {
disposition,
write_document,
retire_runtime,
} => Some((*disposition, *write_document, *retire_runtime)),
_ => None,
})
.ok_or_else(|| {
SessionError::Agent(AgentError::InternalError(format!(
"generated session document authority returned no archive verdict for \
session {id}"
)))
})?;
if disposition == SessionArchiveDisposition::AlreadyArchived {
return Err(SessionError::NotFound { id: id.clone() });
}
let gate = self.existing_gate_for_session(id).await;
let mut gate_guard = if let Some(ref gate) = gate {
let mut guard = gate.cancelled.lock().await;
*guard = true;
Some(guard)
} else {
None
};
if retire_runtime
&& let Err(err) = machine_archive
.retire_session(
id,
archive_lease.take(),
stored_only_publication_handle
.as_ref()
.map(|publisher| publisher as &dyn CoreExecutorPublicationHandle),
)
.await
{
drop(gate_guard.take());
if let Err(discard_error) = self.inner.discard_live_session(id).await {
tracing::warn!(
session_id = %id,
error = %discard_error,
"failed to evict live session after RuntimeStore lifecycle retirement failed"
);
}
return Err(err);
}
let live_result = self.inner.archive(id).await;
drop(gate_guard.take());
self.checkpointer_gates.lock().await.remove(id);
self.live_checkpointers.lock().await.remove(id);
match (&live_result, write_document || retire_runtime) {
(Ok(()), _) | (_, true) => Ok(()),
_ => live_result,
}
}
.await;
let cleanup_quiescent_registration = if archive_lease_owns_recovered_registration {
match archive_lease.as_ref() {
Some(lease) => matches!(
lease.runtime_state().await,
RuntimeState::Retired | RuntimeState::Destroyed
),
None => false,
}
} else {
false
};
let cleanup_result = if cleanup_quiescent_registration {
match archive_lease.take() {
Some(lease) => machine_archive.release_quiescent_archive_lease(lease).await,
None => Err(SessionError::Agent(AgentError::InternalError(format!(
"quiescent archive cleanup for session {id} lost its retained mutation lease"
)))),
}
} else {
Ok(())
};
match (archive_result, cleanup_result) {
(result, Ok(())) => result,
(Ok(()), Err(cleanup_error)) => Err(cleanup_error),
(Err(archive_error), Err(cleanup_error)) => {
Err(SessionError::Agent(AgentError::InternalError(format!(
"archive failed for session {id}: {archive_error}; quiescent runtime cleanup also failed: {cleanup_error}"
))))
}
}
}
pub async fn interrupt_with_machine_authority(
&self,
id: &SessionId,
_authority: MachineSessionControlAuthority,
) -> Result<(), SessionError> {
self.inner.interrupt(id).await
}
pub async fn cancel_after_boundary_with_machine_authority(
&self,
id: &SessionId,
expected_run_id: &RunId,
_authority: MachineSessionControlAuthority,
) -> Result<(), SessionError> {
self.inner
.cancel_after_boundary_for_run(id, expected_run_id)
.await
}
pub async fn cancel_current_after_boundary_with_machine_authority(
&self,
id: &SessionId,
_authority: MachineSessionControlAuthority,
) -> Result<(), SessionError> {
self.inner.cancel_after_boundary(id).await
}
}
#[async_trait]
impl<B: SessionAgentBuilder + 'static> SessionService for PersistentSessionService<B> {
async fn create_session(&self, req: CreateSessionRequest) -> Result<RunResult, SessionError> {
let actor_seed_authority = self.resolve_actor_session_seed_authority(&req).await?;
self.create_session_with_admission(
req,
None,
actor_seed_authority,
ArchivedResumeAuthorization::RejectArchived,
false,
None,
)
.await
}
async fn start_turn(
&self,
_id: &SessionId,
_req: StartTurnRequest,
) -> Result<RunResult, SessionError> {
Err(SessionError::Unsupported(
"runtime-backed direct start_turn must route through the MeerkatMachine service-turn commit protocol"
.to_string(),
))
}
async fn reconcile_runtime_compaction_projections(
&self,
id: &SessionId,
intents: Vec<meerkat_core::CompactionProjectionIntent>,
) -> Result<(), SessionError> {
PersistentSessionService::reconcile_runtime_compaction_projections(self, id, intents).await
}
async fn abort_uncommitted_compaction_projections(
&self,
id: &SessionId,
) -> Result<(), SessionError> {
PersistentSessionService::abort_uncommitted_compaction_projections(self, id).await
}
async fn abort_rejected_runtime_run_projections(
&self,
id: &SessionId,
) -> Result<(), SessionError> {
PersistentSessionService::abort_rejected_runtime_run_projections(self, id).await
}
async fn interrupt(&self, id: &SessionId) -> Result<(), SessionError> {
Err(SessionError::Unsupported(format!(
"interrupt for runtime-backed session {id} is machine-owned: callers interrupt through \
MeerkatMachine::hard_cancel_current_run; executor interrupt-handle implementations must \
apply the cancel to the live agent via interrupt_with_machine_authority and must never \
re-enter the machine"
)))
}
async fn cancel_after_boundary(&self, id: &SessionId) -> Result<(), SessionError> {
Err(SessionError::Unsupported(format!(
"cancel_after_boundary for runtime-backed session {id} is machine-owned: callers cancel \
through MeerkatMachine::cancel_after_boundary; executor boundary-handle implementations \
must apply the cancel to the live agent via cancel_after_boundary_with_machine_authority \
and must never re-enter the machine"
)))
}
async fn cancel_after_boundary_for_run(
&self,
id: &SessionId,
_expected_run_id: &RunId,
) -> Result<(), SessionError> {
Err(SessionError::Unsupported(format!(
"cancel_after_boundary_for_run for runtime-backed session {id} is machine-owned: executor boundary handles must use the machine-authority seam"
)))
}
async fn set_session_client(
&self,
_id: &SessionId,
_client: std::sync::Arc<dyn meerkat_core::AgentLlmClient>,
) -> Result<(), SessionError> {
Err(SessionError::Unsupported(
"set_session_client is disabled for runtime-backed sessions; use runtime turn metadata reconfiguration".to_string(),
))
}
async fn hot_swap_session_llm_identity(
&self,
_id: &SessionId,
_client: std::sync::Arc<dyn meerkat_core::AgentLlmClient>,
_identity: meerkat_core::SessionLlmIdentity,
_request_policy: meerkat_core::SessionLlmRequestPolicy,
) -> Result<(), SessionError> {
Err(SessionError::Unsupported(
"hot_swap_session_llm_identity is a bespoke metadata seam that bypasses the canonical RuntimeTurnMetadata carrier; model/provider/provider_params must travel through the single runtime-backed turn seam instead".to_string(),
))
}
async fn set_session_tool_visibility_state(
&self,
id: &SessionId,
state: Option<meerkat_core::SessionToolVisibilityState>,
) -> Result<(), SessionError> {
let _turn_finalization_guard = self.acquire_runtime_turn_finalization_guard(id).await;
self.apply_runtime_session_tool_visibility_state_under_runtime_turn_boundary(id, state)
.await?;
self.persist_live_session_now_under_runtime_turn_boundary(id)
.await
.map(|_| ())
}
async fn set_session_tool_filter(
&self,
id: &SessionId,
filter: meerkat_core::ToolFilter,
) -> Result<(), SessionError> {
let _mutation_guard = self.live_persist_mutation_guard(id).await?;
self.inner.set_session_tool_filter(id, filter).await?;
self.persist_full_session_or_discard_live(id)
.await
.map(|_| ())
}
async fn read(&self, id: &SessionId) -> Result<SessionView, SessionError> {
loop {
match self.live_session_authority(id).await? {
LiveSessionAuthority::DurableAuthoritative { session, .. } => {
self.reject_if_archived_session(id, &session)
.await
.map_err(crate::control_error_into_session_error)?;
return Ok(view_from_authoritative_session(&session));
}
LiveSessionAuthority::LiveAuthoritative { snapshot } => {
let view = match self.inner.read(id).await {
Ok(view) => view,
Err(SessionError::NotFound { .. }) => continue,
Err(error) => return Err(error),
};
match self.inner.observe_session_transcript_authority(id).await {
Ok(current) if current == snapshot => return Ok(view),
Ok(_) | Err(SessionError::NotFound { .. }) => continue,
Err(error) => return Err(error),
}
}
LiveSessionAuthority::NoLive => {
let Some(session) = self.load_authoritative_session_base(id).await? else {
return Err(SessionError::NotFound { id: id.clone() });
};
self.reject_if_archived_session(id, &session)
.await
.map_err(crate::control_error_into_session_error)?;
return Ok(view_from_authoritative_session(&session));
}
}
}
}
async fn list(&self, query: SessionQuery) -> Result<Vec<SessionSummary>, SessionError> {
let catalog = self
.runtime_store
.list_runtime_session_catalog_entries(SessionFilter::default())
.await
.map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"failed to list runtime-owned session catalog: {error}"
)))
})?;
let mut summaries: Vec<SessionSummary> = catalog
.into_iter()
.filter(|entry| {
entry.lifecycle_terminal() != Some(meerkat_core::SessionLifecycleTerminal::Archived)
&& !matches!(
entry.runtime_state(),
Some(RuntimeState::Retired | RuntimeState::Destroyed)
)
})
.map(|entry| SessionSummary {
session_id: entry.session_id().clone(),
created_at: entry.created_at(),
updated_at: entry.updated_at(),
message_count: entry.message_count(),
total_tokens: entry.total_tokens(),
is_active: matches!(entry.runtime_state(), Some(RuntimeState::Running)),
labels: entry.labels().clone(),
})
.collect();
if let Some(ref filter_labels) = query.labels {
summaries.retain(|s| {
filter_labels
.iter()
.all(|(k, v)| s.labels.get(k) == Some(v))
});
}
if let Some(offset) = query.offset {
if offset < summaries.len() {
summaries = summaries.split_off(offset);
} else {
summaries.clear();
}
}
if let Some(limit) = query.limit {
summaries.truncate(limit);
}
Ok(summaries)
}
async fn has_live_session(&self, id: &SessionId) -> Result<bool, SessionError> {
match self.live_session_authority(id).await? {
LiveSessionAuthority::NoLive => Ok(false),
LiveSessionAuthority::LiveAuthoritative { .. } => Ok(true),
LiveSessionAuthority::DurableAuthoritative { session, .. } => {
if self
.session_archived_by_authority(id, session.as_ref())
.await?
{
self.discard_live_session(id).await?;
Ok(false)
} else {
Ok(true)
}
}
}
}
async fn archive(&self, id: &SessionId) -> Result<(), SessionError> {
Err(SessionError::Unsupported(format!(
"runtime-backed archive for session {id} requires MachineSessionArchiveProtocol"
)))
}
async fn update_session_mob_authority_context(
&self,
id: &SessionId,
authority_context: Option<MobToolAuthorityContext>,
) -> Result<(), SessionError> {
let _mutation_guard = self.live_persist_mutation_guard(id).await?;
self.inner
.update_session_mob_authority_context(id, authority_context)
.await?;
self.persist_full_session_or_discard_live(id)
.await
.map(|_| ())
}
async fn subscribe_session_events(
&self,
id: &SessionId,
) -> Result<meerkat_core::comms::EventStream, meerkat_core::comms::StreamError> {
self.inner.subscribe_session_events(id).await
}
async fn record_live_terminal_error(
&self,
id: &SessionId,
cause: meerkat_core::live_adapter::LiveAdapterErrorCode,
) -> Result<(), SessionError> {
self.inner.record_live_terminal_error(id, cause).await
}
async fn record_live_output_audio_degraded(
&self,
id: &SessionId,
dropped: u64,
) -> Result<(), SessionError> {
self.inner
.record_live_output_audio_degraded(id, dropped)
.await
}
}
#[async_trait]
impl<B: SessionAgentBuilder + 'static> SessionServiceHistoryExt for PersistentSessionService<B> {
async fn read_history(
&self,
id: &SessionId,
query: SessionHistoryQuery,
) -> Result<SessionHistoryPage, SessionError> {
let session = self
.load_authoritative_session_base(id)
.await?
.ok_or_else(|| SessionError::NotFound { id: id.clone() })?;
Ok(SessionHistoryPage::from_messages(
session.id().clone(),
session.messages(),
query,
))
}
async fn read_transcript_revision(
&self,
id: &SessionId,
query: SessionTranscriptRevisionQuery,
) -> Result<SessionTranscriptRevisionPage, SessionError> {
let session = self
.load_authoritative_session_base(id)
.await?
.ok_or_else(|| SessionError::NotFound { id: id.clone() })?;
let head_revision = session.transcript_revision().map_err(|err| {
SessionError::Agent(meerkat_core::error::AgentError::InternalError(format!(
"failed to read transcript head revision: {err}"
)))
})?;
let revision = match meerkat_contracts::RevisionSelector::parse(query.revision) {
meerkat_contracts::RevisionSelector::Current => head_revision.clone(),
meerkat_contracts::RevisionSelector::Specific(id) => id.into_string(),
};
let history = session
.validated_transcript_history_state()
.map_err(|err| {
SessionError::Agent(meerkat_core::error::AgentError::InternalError(format!(
"failed to read transcript revision graph: {err}"
)))
})?;
let messages = if revision == head_revision {
session.messages().to_vec()
} else if let Some(history) = history.as_ref()
&& history.contains_revision(&revision)
{
history
.materialize_revision(&revision)
.map_err(|error| {
SessionError::Agent(meerkat_core::error::AgentError::InternalError(format!(
"failed to materialize explicit transcript revision {revision}: {error}"
)))
})?
.messages
} else {
let mut fallback = None;
if let Some(incremental) = self.incremental.as_ref() {
for record in incremental
.load_rewrites(id)
.await
.map_err(|err| SessionError::Store(Box::new(err)))?
{
if record.commit.revision == revision {
fallback = Some(record.revision_body.messages);
break;
}
if record.commit.parent_revision == revision {
fallback = Some(record.parent_body.messages);
break;
}
}
}
match fallback {
Some(messages) => messages,
None => {
return Err(SessionError::Agent(
meerkat_core::error::AgentError::ConfigError(format!(
"transcript revision {revision} not found for session {id}",
)),
));
}
}
};
Ok(SessionTranscriptRevisionPage::from_messages(
session.id().clone(),
revision,
head_revision,
&messages,
query.offset,
query.limit,
))
}
async fn list_transcript_revisions(
&self,
id: &SessionId,
query: SessionTranscriptRevisionListQuery,
) -> Result<SessionTranscriptRevisionList, SessionError> {
let session = self
.load_authoritative_session_base(id)
.await?
.ok_or_else(|| SessionError::NotFound { id: id.clone() })?;
let head_revision = session.transcript_revision().map_err(|err| {
SessionError::Agent(meerkat_core::error::AgentError::InternalError(format!(
"failed to read transcript head revision: {err}"
)))
})?;
let mut commits = session
.validated_transcript_history_state()
.map_err(|err| {
SessionError::Agent(meerkat_core::error::AgentError::InternalError(format!(
"failed to read transcript revision graph: {err}"
)))
})?
.map(|state| state.commits().cloned().collect::<Vec<_>>())
.unwrap_or_default();
if commits.is_empty()
&& let Some(incremental) = self.incremental.as_ref()
{
commits = incremental
.load_rewrites(id)
.await
.map_err(|err| SessionError::Store(Box::new(err)))?
.into_iter()
.map(|record| record.commit)
.collect();
}
let start = query.offset.unwrap_or(0).min(commits.len());
let end = match query.limit {
Some(limit) => start.saturating_add(limit).min(commits.len()),
None => commits.len(),
};
let entries = commits[start..end]
.iter()
.map(|commit| SessionTranscriptRevisionListEntry {
revision: commit.revision.clone(),
parent_revision: commit.parent_revision.clone(),
actor: commit.actor.clone(),
reason: commit.reason.to_string(),
committed_at: commit.committed_at,
})
.collect();
Ok(SessionTranscriptRevisionList {
entries,
head_revision,
})
}
}
#[async_trait]
impl<B: SessionAgentBuilder + 'static> SessionServiceTranscriptEditExt
for PersistentSessionService<B>
{
async fn fork_session_at(
&self,
id: &SessionId,
req: SessionForkAtRequest,
) -> Result<SessionForkResult, SessionError> {
let _ = req.running_behavior;
let source = self.source_session_for_transcript_edit(id).await?;
let source_metadata = source.try_session_metadata().map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"failed to decode source session metadata while forking session {id}: {error}"
)))
})?;
if req.message_index > source.messages().len() {
return Err(meerkat_core::TranscriptEditError::MessageIndexOutOfBounds {
message_index: req.message_index,
message_count: source.messages().len(),
}
.into_session_error());
}
let forked = source.fork_at(req.message_index);
self.authorize_transcript_edit(id, TranscriptEditKind::Fork)?;
self.persist_transcript_fork(id.clone(), forked, source_metadata, req.tool_access_policy)
.await
}
async fn fork_session_replace(
&self,
id: &SessionId,
req: SessionForkReplaceRequest,
) -> Result<SessionForkResult, SessionError> {
let _ = req.running_behavior;
let source = self.source_session_for_transcript_edit(id).await?;
let source_metadata = source.try_session_metadata().map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"failed to decode source session metadata while forking session {id}: {error}"
)))
})?;
let forked = source
.fork_replacing(req.message_index, req.replacement)
.map_err(meerkat_core::TranscriptEditError::into_session_error)?;
self.authorize_transcript_edit(id, TranscriptEditKind::Fork)?;
self.persist_transcript_fork(id.clone(), forked, source_metadata, req.tool_access_policy)
.await
}
async fn rewrite_session_transcript(
&self,
id: &SessionId,
req: SessionTranscriptRewriteRequest,
) -> Result<SessionTranscriptRewriteResult, SessionError> {
let _ = req.running_behavior;
let _mutation_guard = self.transcript_edit_mutation_guard(id).await?;
let mut source = self.source_session_for_transcript_edit_locked(id).await?;
source = self.normalized_session_for_persistence(source).await?;
let replacement = self
.normalized_transcript_rewrite_replacement(req.replacement)
.await?;
let commit = source
.commit_transcript_rewrite(
req.selection,
replacement,
req.reason,
req.actor,
req.expected_parent_revision,
)
.map_err(meerkat_core::TranscriptEditError::into_session_error)?;
self.authorize_transcript_edit(id, TranscriptEditKind::Rewrite)?;
let saved = self.persist_transcript_rewrite(source, &commit).await?;
Ok(SessionTranscriptRewriteResult {
session_id: saved.id().clone(),
parent_revision: commit.parent_revision.clone(),
revision: commit.revision.clone(),
message_count: saved.messages().len(),
commit,
})
}
async fn restore_session_transcript_revision(
&self,
id: &SessionId,
req: SessionTranscriptRestoreRevisionRequest,
) -> Result<SessionTranscriptRewriteResult, SessionError> {
let _ = req.running_behavior;
let _mutation_guard = self.transcript_edit_mutation_guard(id).await?;
let mut source = self.source_session_for_transcript_edit_locked(id).await?;
source = self.normalized_session_for_persistence(source).await?;
let head_revision = source.transcript_revision().map_err(|err| {
SessionError::Agent(meerkat_core::error::AgentError::InternalError(format!(
"failed to read transcript head revision: {err}"
)))
})?;
let revision = match meerkat_contracts::RevisionSelector::parse(req.revision) {
meerkat_contracts::RevisionSelector::Current => head_revision.clone(),
meerkat_contracts::RevisionSelector::Specific(id) => id.into_string(),
};
let history = source.validated_transcript_history_state().map_err(|err| {
SessionError::Agent(meerkat_core::error::AgentError::InternalError(format!(
"failed to read transcript revision graph: {err}"
)))
})?;
let replacement = if revision == head_revision {
source.messages().to_vec()
} else if let Some(history) = history.as_ref()
&& history.contains_revision(&revision)
{
history
.materialize_revision(&revision)
.map_err(|error| {
SessionError::Agent(meerkat_core::error::AgentError::InternalError(format!(
"failed to materialize explicit restore revision {revision}: {error}"
)))
})?
.messages
} else {
return Err(SessionError::Agent(
meerkat_core::error::AgentError::ConfigError(format!(
"transcript revision {revision} not found for session {id}",
)),
));
};
let replacement = self
.normalized_transcript_rewrite_replacement(replacement)
.await?;
let message_count = source.messages().len();
let commit = source
.commit_transcript_rewrite(
meerkat_core::TranscriptRewriteSelection::MessageRange {
start: 0,
end: message_count,
},
replacement,
req.reason,
req.actor,
req.expected_parent_revision,
)
.map_err(meerkat_core::TranscriptEditError::into_session_error)?;
self.authorize_transcript_edit(id, TranscriptEditKind::Rewrite)?;
let saved = self.persist_transcript_rewrite(source, &commit).await?;
Ok(SessionTranscriptRewriteResult {
session_id: saved.id().clone(),
parent_revision: commit.parent_revision.clone(),
revision: commit.revision.clone(),
message_count: saved.messages().len(),
commit,
})
}
}
#[async_trait]
impl<B: SessionAgentBuilder + 'static> SessionServiceCommsExt for PersistentSessionService<B> {
async fn comms_runtime(
&self,
session_id: &SessionId,
) -> Option<std::sync::Arc<dyn meerkat_core::agent::CommsRuntime>> {
self.inner.comms_runtime(session_id).await
}
async fn event_injector(
&self,
session_id: &SessionId,
) -> Option<std::sync::Arc<dyn meerkat_core::EventInjector>> {
self.inner.event_injector(session_id).await
}
async fn interaction_event_injector(
&self,
session_id: &SessionId,
) -> Option<std::sync::Arc<dyn meerkat_core::event_injector::SubscribableInjector>> {
self.inner.interaction_event_injector(session_id).await
}
}
#[async_trait]
impl<B: SessionAgentBuilder + 'static> SessionServiceControlExt for PersistentSessionService<B> {
async fn append_system_context(
&self,
id: &SessionId,
req: AppendSystemContextRequest,
) -> Result<AppendSystemContextResult, SessionControlError> {
let persistence_profile = self.runtime_store.session_persistence_profile();
let _turn_finalization_guard = self.acquire_runtime_turn_finalization_guard(id).await;
let _projection_guard = self.recovery_gate_for_session(id).await.lock_owned().await;
let existing_gate = self.existing_gate_for_session(id).await;
if let Some(gate) = existing_gate.as_ref() {
let gate_guard = gate.cancelled.lock().await;
if *gate_guard {
return Err(SessionControlError::Session(SessionError::NotFound {
id: id.clone(),
}));
}
drop(gate_guard);
}
if self.inner.has_live_session(id).await? {
if self
.session_archived_by_authority_with_terminal(id, None)
.await
.map_err(SessionControlError::Session)?
{
return Err(Self::archived_not_found(id));
}
let status = self
.inner
.append_system_message_control(id, req)
.await
.map_err(SessionControlError::Session)?;
if status == meerkat_core::service::AppendSystemContextStatus::Duplicate {
return Ok(AppendSystemContextResult { status });
}
let persist_result = match persistence_profile {
RuntimeSessionPersistenceProfile::WholeBlobV1 => {
let session = self
.inner
.export_session(id)
.await
.map_err(SessionControlError::Session)?;
self.save_normalized_session(session)
.await
.map(|_| ())
.map_err(SessionControlError::Session)
}
RuntimeSessionPersistenceProfile::HeadCanonicalV1 => {
let prepared = self
.prepare_runtime_boundary(id, "System-message control append", None)
.await
.map_err(SessionControlError::Session)?;
self.commit_prepared_head_canonical_actor_boundary(
id,
"System-message control append",
prepared,
)
.await
.map(|_| ())
.map_err(SessionControlError::Session)
}
profile => Err(SessionControlError::Session(SessionError::Agent(
AgentError::InternalError(format!(
"unsupported runtime session persistence profile {profile} while appending System message for session {id}"
)),
))),
};
if let Err(error) = persist_result {
let _ = self.discard_live_session_unfenced(id).await;
return Err(error);
}
return Ok(AppendSystemContextResult { status });
}
let mut session = match self
.load_persisted_session_for_control(id, "append_system_context")
.await?
{
Some(session) => session,
None => {
self.checkpointer_gates.lock().await.remove(id);
return Err(SessionControlError::Session(SessionError::NotFound {
id: id.clone(),
}));
}
};
self.reject_if_archived_session(id, &session).await?;
let status = session
.append_system_message_idempotent(
req.content.render_text(),
req.source,
req.idempotency_key,
meerkat_core::types::message_timestamp_now(),
)
.map_err(|error| {
SessionControlError::Session(SessionError::Agent(AgentError::ConfigError(
error.to_string(),
)))
})?;
if status == meerkat_core::service::AppendSystemContextStatus::Duplicate {
return Ok(AppendSystemContextResult { status });
}
match persistence_profile {
RuntimeSessionPersistenceProfile::WholeBlobV1 => {
self.save_normalized_session(session)
.await
.map_err(SessionControlError::Session)?;
}
RuntimeSessionPersistenceProfile::HeadCanonicalV1 => {
self.persist_detached_head_canonical_session(
session,
"store-only System-message control append",
)
.await
.map_err(SessionControlError::Session)?;
}
profile => {
return Err(SessionControlError::Session(SessionError::Agent(
AgentError::InternalError(format!(
"unsupported runtime session persistence profile {profile} while persisting system context for session {id}"
)),
)));
}
}
Ok(AppendSystemContextResult { status })
}
async fn stage_tool_results(
&self,
id: &SessionId,
req: StageToolResultsRequest,
) -> Result<StageToolResultsResult, SessionError> {
validate_tool_result_video(&req.results)?;
let persistence_profile = self.runtime_store.session_persistence_profile();
let _turn_finalization_guard = self.acquire_runtime_turn_finalization_guard(id).await;
let _projection_guard = self.recovery_gate_for_session(id).await.lock_owned().await;
let existing_gate = self.existing_gate_for_session(id).await;
if let Some(state_arc) = self.inner.deferred_turn_state(id).await {
let created_gate = existing_gate.is_none();
let gate = match existing_gate {
Some(gate) => gate,
None => self.gate_for_session(id).await,
};
let gate_guard = gate.cancelled.lock().await;
if *gate_guard {
return Err(SessionError::NotFound { id: id.clone() });
}
let accepted_at = meerkat_core::time_compat::SystemTime::now();
let mut attempts = 0usize;
loop {
attempts += 1;
let (accepted, snapshot_state, persisted_state) = {
let guard = match state_arc.lock() {
Ok(guard) => guard,
Err(poisoned) => {
tracing::warn!(
session_id = %id,
"deferred-turn state lock poisoned while snapshotting staged tool results"
);
poisoned.into_inner()
}
};
let snapshot_state = guard.clone();
let mut candidate = snapshot_state.clone();
let accepted = candidate
.try_stage_tool_results(req.results.clone(), accepted_at)
.map_err(|error| {
SessionError::Agent(AgentError::ConfigError(error.to_string()))
})?;
(accepted, snapshot_state, candidate)
};
if accepted == 0 {
let ingress = match persistence_profile {
RuntimeSessionPersistenceProfile::HeadCanonicalV1 => {
self.inner
.classify_callback_result_ingress(id, req.results.clone())
.await?
}
RuntimeSessionPersistenceProfile::WholeBlobV1 => {
let session = self
.load_authoritative_session_base(id)
.await?
.ok_or_else(|| SessionError::NotFound { id: id.clone() })?;
session.classify_callback_result_ingress(&req.results)?
}
profile => {
return Err(SessionError::Agent(AgentError::InternalError(format!(
"unsupported runtime session persistence profile {profile} while classifying staged tool results for session {id}"
))));
}
};
let disposition = if matches!(
ingress,
meerkat_core::session::CallbackResultIngress::AlreadyApplied
) {
StageToolResultsDisposition::AlreadyApplied
} else {
StageToolResultsDisposition::AlreadyStaged
};
drop(gate_guard);
return Ok(StageToolResultsResult {
accepted_result_count: accepted,
disposition,
});
}
match persistence_profile {
RuntimeSessionPersistenceProfile::HeadCanonicalV1 => {
if self
.session_archived_by_authority_with_terminal(id, None)
.await?
{
drop(gate_guard);
return Err(SessionError::NotFound { id: id.clone() });
}
if matches!(
self.inner
.classify_callback_result_ingress(id, req.results.clone())
.await?,
meerkat_core::session::CallbackResultIngress::AlreadyApplied
) {
drop(gate_guard);
return Ok(StageToolResultsResult {
accepted_result_count: 0,
disposition: StageToolResultsDisposition::AlreadyApplied,
});
}
let persist_result = async {
let prepared = self
.prepare_runtime_boundary_with_deferred_override(
id,
"staged callback tool results",
Some(persisted_state),
)
.await?;
self.commit_prepared_head_canonical_actor_boundary(
id,
"staged callback tool results",
prepared,
)
.await?;
Ok::<(), SessionError>(())
}
.await;
if let Err(error) = persist_result {
drop(gate_guard);
let _ = self.discard_live_session_unfenced(id).await;
return Err(error);
}
}
RuntimeSessionPersistenceProfile::WholeBlobV1 => {
let mut session = match self.load_authoritative_session_base(id).await? {
Some(session) => session,
None => {
if created_gate {
drop(gate_guard);
self.checkpointer_gates.lock().await.remove(id);
}
return Err(SessionError::Agent(
meerkat_core::error::AgentError::InternalError(
"runtime-backed live session is missing its last committed snapshot"
.to_string(),
),
));
}
};
self.reject_if_archived_session(id, &session)
.await
.map_err(crate::control_error_into_session_error)?;
if matches!(
session.classify_callback_result_ingress(&req.results)?,
meerkat_core::session::CallbackResultIngress::AlreadyApplied
) {
drop(gate_guard);
return Ok(StageToolResultsResult {
accepted_result_count: 0,
disposition: StageToolResultsDisposition::AlreadyApplied,
});
}
write_deferred_turn_state(&mut session, persisted_state)
.map_err(crate::control_error_into_session_error)?;
self.save_normalized_session(session).await?;
}
profile => {
return Err(SessionError::Agent(AgentError::InternalError(format!(
"unsupported runtime session persistence profile {profile} while staging tool results for session {id}"
))));
}
}
let commit_result = {
let mut guard = match state_arc.lock() {
Ok(guard) => guard,
Err(poisoned) => {
tracing::warn!(
session_id = %id,
"deferred-turn state lock poisoned while committing staged tool results"
);
poisoned.into_inner()
}
};
if *guard == snapshot_state {
Some(
guard
.try_stage_tool_results(req.results.clone(), accepted_at)
.map_err(|error| {
SessionError::Agent(AgentError::ConfigError(error.to_string()))
})?,
)
} else {
None
}
};
if let Some(live_accepted) = commit_result {
debug_assert_eq!(live_accepted, accepted);
drop(gate_guard);
return Ok(StageToolResultsResult {
accepted_result_count: accepted,
disposition: StageToolResultsDisposition::Staged,
});
}
if attempts >= 8 {
tracing::warn!(
session_id = %id,
"deferred-turn state kept changing after durable tool-result staging; discarding live session to force authoritative reload"
);
drop(gate_guard);
let _ = self.discard_live_session_unfenced(id).await;
return Ok(StageToolResultsResult {
accepted_result_count: accepted,
disposition: StageToolResultsDisposition::Staged,
});
}
}
}
if let Some(gate) = existing_gate {
let gate_guard = gate.cancelled.lock().await;
if *gate_guard {
return Err(SessionError::NotFound { id: id.clone() });
}
drop(gate_guard);
}
let mut session = match self
.load_persisted_session_for_control(id, "stage_tool_results")
.await?
{
Some(session) => session,
None => {
self.checkpointer_gates.lock().await.remove(id);
return Err(SessionError::NotFound { id: id.clone() });
}
};
self.reject_if_archived_session(id, &session)
.await
.map_err(crate::control_error_into_session_error)?;
if matches!(
session.classify_callback_result_ingress(&req.results)?,
meerkat_core::session::CallbackResultIngress::AlreadyApplied
) {
return Ok(StageToolResultsResult {
accepted_result_count: 0,
disposition: StageToolResultsDisposition::AlreadyApplied,
});
}
let mut state = session.deferred_turn_state().unwrap_or_default();
let accepted = state
.try_stage_tool_results(req.results, meerkat_core::time_compat::SystemTime::now())
.map_err(|error| SessionError::Agent(AgentError::ConfigError(error.to_string())))?;
write_deferred_turn_state(&mut session, state)
.map_err(crate::control_error_into_session_error)?;
match persistence_profile {
RuntimeSessionPersistenceProfile::WholeBlobV1 => {
self.save_normalized_session(session).await?;
}
RuntimeSessionPersistenceProfile::HeadCanonicalV1 => {
self.persist_detached_head_canonical_session(
session,
"store-only staged callback tool results",
)
.await?;
}
profile => {
return Err(SessionError::Agent(AgentError::InternalError(format!(
"unsupported runtime session persistence profile {profile} while persisting staged tool results for session {id}"
))));
}
}
Ok(StageToolResultsResult {
accepted_result_count: accepted,
disposition: if accepted == 0 {
StageToolResultsDisposition::AlreadyStaged
} else {
StageToolResultsDisposition::Staged
},
})
}
}
impl<B: SessionAgentBuilder + 'static> PersistentSessionService<B> {
pub async fn event_injector(
&self,
session_id: &SessionId,
) -> Option<std::sync::Arc<dyn meerkat_core::EventInjector>> {
self.inner.event_injector(session_id).await
}
#[doc(hidden)]
pub async fn interaction_event_injector(
&self,
session_id: &SessionId,
) -> Option<std::sync::Arc<dyn meerkat_core::event_injector::SubscribableInjector>> {
self.inner.interaction_event_injector(session_id).await
}
pub async fn comms_runtime(
&self,
session_id: &SessionId,
) -> Option<std::sync::Arc<dyn meerkat_core::agent::CommsRuntime>> {
self.inner.comms_runtime(session_id).await
}
pub async fn wait_session_registered(&self) {
self.inner.wait_session_registered().await;
}
pub async fn shutdown(&self) {
self.inner.shutdown().await;
}
pub async fn cancel_all_checkpointers(&self) {
let gates = self.checkpointer_gates.lock().await;
for gate in gates.values() {
let mut cancelled = gate.cancelled.lock().await;
*cancelled = true;
}
}
pub async fn rearm_all_checkpointers(&self) {
let gates = self.checkpointer_gates.lock().await;
for gate in gates.values() {
let mut cancelled = gate.cancelled.lock().await;
*cancelled = false;
}
}
pub async fn subscribe_session_events(
&self,
id: &SessionId,
) -> Result<meerkat_core::comms::EventStream, meerkat_core::comms::StreamError> {
self.inner.subscribe_session_events(id).await
}
pub async fn live_deferred_first_turn_pending(
&self,
id: &SessionId,
) -> Result<bool, SessionError> {
let _turn_finalization_guard = self.acquire_runtime_turn_finalization_guard(id).await;
self.live_deferred_first_turn_pending_under_runtime_turn_boundary(id)
.await
}
pub async fn live_deferred_first_turn_pending_under_runtime_turn_boundary(
&self,
id: &SessionId,
) -> Result<bool, SessionError> {
let _recovery_guard = self.recovery_gate_for_session(id).await.lock_owned().await;
let _ = self.discard_stale_live_session_if_needed(id).await?;
let Some(state) = self.inner.deferred_turn_state(id).await else {
return Ok(false);
};
let state = state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
Ok(matches!(
state.first_turn_phase(),
DeferredFirstTurnPhase::Pending
))
}
pub async fn load_authoritative_session(
&self,
id: &SessionId,
) -> Result<Option<Session>, SessionError> {
self.load_authoritative_session_base(id).await
}
pub async fn observe_authoritative_session_body(
&self,
id: &SessionId,
) -> Result<Option<Session>, SessionError> {
self.load_committed_runtime_session_for_body(id, "authoritative session observation")
.await
}
pub async fn recover_committed_boundary(
&self,
id: &SessionId,
) -> Result<CommittedBoundaryRecovery, SessionError> {
if self.inner.has_live_session(id).await? {
return Err(SessionError::Busy { id: id.clone() });
}
let recovery_gate = self.recovery_gate_for_session(id).await;
let _recovery_guard = recovery_gate.lock_owned().await;
if self.inner.has_live_session(id).await? {
return Err(SessionError::Busy { id: id.clone() });
}
if !self.session_known_to_archive_authority(id).await? {
return Err(SessionError::NotFound { id: id.clone() });
}
match self.store_owned_durable_tail_recovery_outcome(id).await? {
meerkat_runtime::recovery::DurableTailRecoveryOutcome::Committed {
recovered, ..
} => Ok(CommittedBoundaryRecovery::Recovered {
message_count: recovered.messages().len(),
}),
meerkat_runtime::recovery::DurableTailRecoveryOutcome::AlreadyAligned { .. } => {
Ok(CommittedBoundaryRecovery::AlreadyCommitted)
}
meerkat_runtime::recovery::DurableTailRecoveryOutcome::Held => {
Ok(CommittedBoundaryRecovery::Unprovable {
reason: format!(
"the durable tail for session {id} was held intact: the RuntimeStore could not produce and atomically consume exact recovery evidence; every durable copy is retained for reconciliation"
),
})
}
meerkat_runtime::recovery::DurableTailRecoveryOutcome::Refused => {
Ok(CommittedBoundaryRecovery::Unprovable {
reason: format!(
"the durable tail for session {id} was refused by machine authority because persisted runtime facts contradict or already supersede the candidate; every durable copy is retained for operator reconciliation"
),
})
}
}
}
pub async fn session_known_to_archive_authority(
&self,
id: &SessionId,
) -> Result<bool, SessionError> {
let runtime_id = Self::runtime_id_for_session(id);
if self
.runtime_store
.load_runtime_session_catalog_entry(&runtime_id)
.await
.map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"failed to load runtime session catalog for session {id}: {error}"
)))
})?
.is_some()
{
return Ok(true);
}
Ok(self
.runtime_store
.load_session_boundary_authority(&runtime_id)
.await
.map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"failed to load runtime session authority for session {id}: {error}"
)))
})?
.is_some())
}
pub async fn load_authoritative_session_metadata(
&self,
id: &SessionId,
) -> Result<Option<meerkat_core::PersistedSessionMetadataView>, SessionError> {
let corrupt_metadata_error = |err: serde_json::Error| {
durable_session_restore_error(id, "durable metadata failed typed restore", err)
};
let runtime_id = Self::runtime_id_for_session(id);
let Some(authority) = self
.runtime_store
.load_session_boundary_authority(&runtime_id)
.await
.map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"failed to load RuntimeStore metadata authority for session {id}: {error}"
)))
})?
else {
return Ok(None);
};
if authority.session_id() != id {
return Err(SessionError::Agent(AgentError::InternalError(format!(
"RuntimeStore metadata authority identifies session {}, not {id}",
authority.session_id()
))));
}
match authority {
RuntimeSessionAuthority::WholeBlob(_) => {
let snapshot = self
.runtime_store
.load_committed_whole_blob_snapshot(&runtime_id)
.await
.map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"failed to load exact WholeBlob metadata authority for session {id}: {error}"
)))
})?
.ok_or_else(|| {
SessionError::Agent(AgentError::InternalError(format!(
"WholeBlob metadata authority disappeared for session {id}"
)))
})?;
meerkat_core::PersistedSessionMetadataView::try_from_session(snapshot.session())
.map(Some)
.map_err(corrupt_metadata_error)
}
RuntimeSessionAuthority::HeadCanonical(authority) => {
let metadata = authority
.boundary_head()
.materialized_metadata()
.map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"failed to materialize committed HeadCanonical metadata for session {id}: {error}"
)))
})?;
meerkat_core::PersistedSessionMetadataView::try_from_metadata_map(
id.clone(),
&metadata,
)
.map(Some)
.map_err(corrupt_metadata_error)
}
}
}
async fn persist_full_session(&self, id: &SessionId) -> Result<(usize, String), SessionError> {
match self.runtime_store.session_persistence_profile() {
RuntimeSessionPersistenceProfile::WholeBlobV1 => {
let session = self.export_session_with_labels(id).await?;
let persisted = self.save_normalized_session(session).await?;
let message_count = persisted.messages().len();
let revision = meerkat_core::transcript_messages_digest(persisted.messages())
.map_err(|err| {
SessionError::Agent(meerkat_core::error::AgentError::InternalError(
format!(
"failed to digest persisted transcript for checkpoint seed: {err}"
),
))
})?;
Ok((message_count, revision))
}
RuntimeSessionPersistenceProfile::HeadCanonicalV1 => {
let PreparedRuntimeBoundary {
committed,
identity: prepared_boundary,
conversation_digest,
message_count,
} = self
.prepare_runtime_boundary(id, "direct live session persistence", None)
.await?;
let result = self
.runtime_store
.commit_prepared_session_boundary(
&Self::runtime_id_for_session(id),
meerkat_runtime::PreparedRuntimeSessionCommit::snapshot_only(committed),
)
.await
.map_err(|error| {
SessionError::Agent(AgentError::InternalError(format!(
"head-canonical live session persistence failed for session {id}: {error}"
)))
})?;
let committed_authority = result.authority().ok_or_else(|| {
SessionError::Agent(AgentError::InternalError(format!(
"head-canonical live session persistence committed no session authority for session {id}"
)))
})?;
let committed_boundary = PreparedRuntimeBoundaryIdentity::from_runtime_authority(
committed_authority,
id,
)?;
if committed_boundary != prepared_boundary {
return Err(SessionError::Agent(AgentError::InternalError(format!(
"head-canonical live session persistence returned different authority for session {id}"
))));
}
self.acknowledge_head_canonical_runtime_boundary_after_commit(
id,
&committed_boundary,
)
.await?;
Ok((message_count, conversation_digest))
}
profile => Err(SessionError::Agent(AgentError::InternalError(format!(
"unsupported runtime session persistence profile {profile} for session {id}"
)))),
}
}
async fn persist_full_session_or_discard_live(
&self,
id: &SessionId,
) -> Result<String, SessionError> {
match self.persist_full_session(id).await {
Ok((_message_count, revision)) => Ok(revision),
Err(error) => {
match self.discard_live_session_unfenced(id).await {
Ok(()) | Err(SessionError::NotFound { .. }) => {}
Err(discard_error) => {
tracing::warn!(
session_id = %id,
error = %discard_error,
"failed to discard live session after full-session persistence failure"
);
}
}
Err(error)
}
}
}
}
#[cfg(test)]
#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
mod tests {
use super::*;
use crate::ephemeral::{
EphemeralSessionService, ObservedSessionTailKind, SessionAgent, SessionAgentBuilder,
SessionSnapshot,
};
use crate::event_store::{
EVENT_SCHEMA_VERSION, EventStoreError, ExactInteractionAppend, StoredEvent,
};
use meerkat_core::ToolDispatchOutcome;
use meerkat_core::event::AgentEvent;
use meerkat_core::service::{
DeferredPromptPolicy, InitialTurnPolicy, SessionBuildOptions, SessionService,
SessionServiceControlExt, StageToolResultsRequest, TranscriptEditRunningBehavior,
TranscriptRewriteReason, TranscriptRewriteSelection,
};
use meerkat_core::session::SESSION_METADATA_KEY;
use meerkat_core::session_store::{SessionHeadCas, TranscriptStrandId};
use meerkat_core::types::{
ContentBlock, ContentInput, ImageData, Message, StopReason, ToolCall, ToolResult,
UserMessage,
};
use meerkat_core::{RunId, lifecycle::run_primitive::RunApplyBoundary};
use meerkat_runtime::{InMemoryRuntimeStore, RuntimeStore};
use meerkat_store::{FsBlobStore, MemoryBlobStore, MemoryStore, SessionStoreError};
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
struct TestPreparedInterruptHandle;
#[async_trait::async_trait]
impl meerkat_core::lifecycle::CoreExecutorInterruptHandle for TestPreparedInterruptHandle {
async fn hard_cancel_current_run(
&self,
_reason: String,
) -> Result<(), meerkat_core::lifecycle::CoreExecutorError> {
Ok(())
}
}
struct TestPreparedCleanupHandle;
#[async_trait::async_trait]
impl meerkat_core::lifecycle::CoreExecutorPostStopCleanupHandle for TestPreparedCleanupHandle {
async fn cleanup_after_runtime_stop_terminalized(
&self,
) -> Result<(), meerkat_core::lifecycle::CoreExecutorError> {
Ok(())
}
}
fn memory_blob_store() -> Arc<dyn BlobStore> {
Arc::new(MemoryBlobStore::new())
}
#[tokio::test]
async fn whole_blob_control_commit_acknowledgement_rebinds_only_exact_store_authority() {
let runtime_store: Arc<dyn RuntimeStore> = Arc::new(InMemoryRuntimeStore::new());
let session = Session::new();
let session_id = session.id().clone();
let runtime_id = LogicalRuntimeId::for_session(&session_id);
let initial_artifact = session
.to_persisted_artifact()
.expect("encode initial WholeBlob session");
runtime_store
.commit_session_snapshot(
&runtime_id,
SerializedSessionSnapshot {
session_snapshot: initial_artifact.bytes_arc(),
},
)
.await
.expect("commit initial WholeBlob authority");
let initial_authority = runtime_store
.load_whole_blob_store_authority(&runtime_id)
.await
.expect("load initial WholeBlob authority")
.expect("initial WholeBlob authority should exist");
let checkpointer = StoreCheckpointer {
runtime_store: Arc::clone(&runtime_store),
incremental: None,
blob_store: memory_blob_store(),
gate: Arc::new(CheckpointerGate {
cancelled: Mutex::new(false),
}),
whole_blob_base_authority: std::sync::Mutex::new(Some(initial_authority.clone())),
latest_run_checkpoint_receipt: Mutex::new(None),
};
let mut successor = session;
successor.push(Message::User(UserMessage::text(
"control mutation successor".to_string(),
)));
let successor_artifact = successor
.to_persisted_artifact()
.expect("encode successor WholeBlob session");
runtime_store
.commit_session_snapshot(
&runtime_id,
SerializedSessionSnapshot {
session_snapshot: successor_artifact.bytes_arc(),
},
)
.await
.expect("commit successor WholeBlob authority");
let successor_authority = runtime_store
.load_whole_blob_store_authority(&runtime_id)
.await
.expect("load successor WholeBlob authority")
.expect("successor WholeBlob authority should exist");
let stale_receipt = meerkat_core::SessionControlCommitReceipt::new(
session_id.clone(),
initial_authority.store_revision(),
initial_authority.blob_sha256(),
)
.expect("construct stale receipt");
assert!(
meerkat_core::SessionCheckpointer::acknowledge_control_commit(
&checkpointer,
&stale_receipt,
)
.await
.is_err(),
"a stale store authority must fail closed"
);
assert_eq!(
checkpointer
.whole_blob_base_authority
.lock()
.expect("base authority lock")
.as_ref(),
Some(&initial_authority),
"a rejected receipt must not mutate the actor base"
);
let exact_receipt = meerkat_core::SessionControlCommitReceipt::new(
session_id,
successor_authority.store_revision(),
successor_authority.blob_sha256(),
)
.expect("construct exact receipt");
meerkat_core::SessionCheckpointer::acknowledge_control_commit(
&checkpointer,
&exact_receipt,
)
.await
.expect("exact store authority should rebind the actor base");
assert_eq!(
checkpointer
.whole_blob_base_authority
.lock()
.expect("base authority lock")
.as_ref(),
Some(&successor_authority)
);
}
fn session_marks_archived(session: &Session) -> bool {
session
.try_lifecycle_terminal()
.is_ok_and(|terminal| terminal.is_some_and(|terminal| terminal.is_archived()))
}
fn mutate_test_session(
mut session: Session,
_role: &str,
mutate: impl FnOnce(&mut Session),
) -> Session {
mutate(&mut session);
session
}
fn without_session_owned_metadata(session: &Session) -> Session {
let mut document = serde_json::to_value(session).expect("session should serialize");
let metadata = document
.get_mut("metadata")
.and_then(serde_json::Value::as_object_mut)
.expect("session metadata should be an object");
metadata.remove(SESSION_METADATA_KEY);
metadata.remove(meerkat_core::SESSION_BUILD_STATE_KEY);
serde_json::from_value(document).expect("session without owned metadata should deserialize")
}
#[tokio::test]
async fn idle_turn_finalization_gate_keys_are_reaped_without_splitting_overlap() {
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::new(MemoryStore::new()),
Arc::new(InMemoryRuntimeStore::new()),
memory_blob_store(),
);
let first_session_id = SessionId::new();
let first_gate = service
.turn_finalization_gate_for_session(&first_session_id)
.await;
let overlapping_gate = service
.turn_finalization_gate_for_session(&first_session_id)
.await;
assert!(Arc::ptr_eq(&first_gate, &overlapping_gate));
drop(first_gate);
drop(overlapping_gate);
let second_session_id = SessionId::new();
let second_gate = service
.turn_finalization_gate_for_session(&second_session_id)
.await;
let gates = service.turn_finalization_gates.lock().await;
assert!(!gates.contains_key(&first_session_id));
let retained_second = gates
.get(&second_session_id)
.and_then(std::sync::Weak::upgrade)
.expect("live gate should remain upgradeable");
assert!(Arc::ptr_eq(&second_gate, &retained_second));
}
struct FailOneBoundedReadBlobStore {
inner: FsBlobStore,
fail_next_bounded_read: AtomicBool,
}
impl FailOneBoundedReadBlobStore {
fn new(root: std::path::PathBuf) -> Self {
Self {
inner: FsBlobStore::new(root),
fail_next_bounded_read: AtomicBool::new(true),
}
}
}
#[async_trait::async_trait]
impl BlobStore for FailOneBoundedReadBlobStore {
async fn put_image(
&self,
media_type: &str,
data: &str,
) -> Result<meerkat_core::BlobRef, meerkat_core::BlobStoreError> {
self.inner.put_image(media_type, data).await
}
async fn get(
&self,
blob_id: &meerkat_core::BlobId,
) -> Result<meerkat_core::BlobPayload, meerkat_core::BlobStoreError> {
self.inner.get(blob_id).await
}
async fn get_with_encoded_limit(
&self,
blob_id: &meerkat_core::BlobId,
max_encoded_bytes: usize,
) -> Result<meerkat_core::BlobPayload, meerkat_core::BlobStoreError> {
if self.fail_next_bounded_read.swap(false, Ordering::AcqRel) {
return Err(meerkat_core::BlobStoreError::ReadFailed(
"synthetic crash after durable put before readback/reducer".to_string(),
));
}
self.inner
.get_with_encoded_limit(blob_id, max_encoded_bytes)
.await
}
async fn delete(
&self,
blob_id: &meerkat_core::BlobId,
) -> Result<(), meerkat_core::BlobStoreError> {
self.inner.delete(blob_id).await
}
fn is_persistent(&self) -> bool {
true
}
}
#[test]
fn durable_sync_unsupported_matches_typed_variant_not_substring() {
let unsupported = SessionError::Agent(AgentError::DurableSnapshotSyncUnsupported);
assert!(is_durable_session_sync_unsupported(&unsupported));
let config_error = SessionError::Agent(AgentError::ConfigError(
"durable session snapshot synchronization is not supported by this session agent"
.to_string(),
));
assert!(!is_durable_session_sync_unsupported(&config_error));
}
#[tokio::test]
async fn append_and_project_event_propagates_durable_append_failure_with_no_hole()
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let event_store = Arc::new(RecordingEventStore::default());
let event_store_trait: Arc<dyn EventStore> = event_store.clone();
let dir = tempfile::tempdir()?;
let projector = Arc::new(SessionProjector::new(dir.path().join(".rkat")));
let session_id = SessionId::new();
let projection_faults: EventProjectionFaultRegistry = Arc::new(Mutex::new(HashMap::new()));
let projection_gates: EventProjectionGateRegistry = Arc::new(Mutex::new(HashMap::new()));
let started = || {
meerkat_core::event::EventEnvelope::new_session(
session_id.clone(),
0,
None,
AgentEvent::RunStarted {
session_id: session_id.clone(),
input: meerkat_core::types::RunInput::Content {
content: ContentInput::Text("seed".to_string()),
},
},
)
};
event_store.fail_appends();
let err = append_and_project_event(
&event_store_trait,
&projector,
&session_id,
started(),
&projection_faults,
&projection_gates,
)
.await
.expect_err("durable append failure must surface a typed terminal error");
assert!(
matches!(err, SessionError::Store(_)),
"unexpected error: {err}"
);
assert_eq!(event_store.last_seq(&session_id).await?, 0);
event_store.allow_appends();
let second = append_and_project_event(
&event_store_trait,
&projector,
&session_id,
started(),
&projection_faults,
&projection_gates,
)
.await;
assert!(
second.is_err(),
"an append failure is absorbing even if the backing store later recovers"
);
let replayed = event_store.read_from(&session_id, 0).await?;
assert!(replayed.is_empty());
Ok(())
}
#[tokio::test]
async fn halted_event_projection_fails_replay_closed()
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let event_store = Arc::new(RecordingEventStore::default());
let event_store_trait: Arc<dyn EventStore> = event_store.clone();
let dir = tempfile::tempdir()?;
let projector = Arc::new(SessionProjector::new(dir.path().join(".rkat")));
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::new(InMemoryRuntimeStore::new()),
memory_blob_store(),
)
.with_event_projection(event_store_trait.clone(), Arc::clone(&projector));
let session_id = SessionId::new();
event_store.fail_appends();
let (projection_tx, projection_rx) = mpsc::channel(4);
let (session_tx, session_rx) = watch::channel(Some(session_id.clone()));
let task = tokio::spawn(project_create_time_events(
event_store_trait,
projector,
projection_rx,
session_rx,
None,
CreateTimeEventProjectionLifecycle {
projection_faults: Arc::clone(&service.event_projection_faults),
projection_gates: Arc::clone(&service.event_projection_gates),
drain_registration: None,
},
));
projection_tx
.send(meerkat_core::event::EventEnvelope::new_session(
session_id.clone(),
0,
None,
AgentEvent::RunStarted {
session_id: session_id.clone(),
input: meerkat_core::types::RunInput::Content {
content: ContentInput::Text("seed".to_string()),
},
},
))
.await?;
drop(projection_tx);
drop(session_tx);
task.await?;
let latest_err = service
.event_log_latest_seq(&session_id)
.await
.expect_err("latest cursor after a halted projection must fail closed");
assert!(
matches!(&latest_err, SessionError::Store(source)
if source.downcast_ref::<EventProjectionHalted>().is_some()),
"expected typed EventProjectionHalted latest-cursor fault, got: {latest_err}"
);
let err = service
.event_log_read_from(&session_id, 0)
.await
.expect_err("replay after a halted projection must fail closed");
match err {
SessionError::Store(source) => {
assert!(
source.downcast_ref::<EventProjectionHalted>().is_some(),
"expected typed EventProjectionHalted fault, got: {source}"
);
}
other => panic!("expected SessionError::Store, got: {other}"),
}
let other_session = SessionId::new();
assert!(
service
.event_log_read_from(&other_session, 0)
.await?
.is_some_and(|events| events.is_empty())
);
Ok(())
}
#[tokio::test]
async fn generation_cutover_drain_waits_for_create_time_projection()
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let event_store = Arc::new(PausingAppendEventStore::default());
let event_store_trait: Arc<dyn EventStore> = event_store.clone();
let dir = tempfile::tempdir()?;
let service = Arc::new(
PersistentSessionService::new(
DummyBuilder,
4,
store,
Arc::new(InMemoryRuntimeStore::new()),
memory_blob_store(),
)
.with_event_projection(
event_store_trait,
Arc::new(SessionProjector::new(dir.path().join(".rkat"))),
),
);
let session_id = SessionId::new();
let mut request = CreateSessionRequest {
injected_context: Vec::new(),
model: "projection-drain-probe".to_string(),
prompt: ContentInput::Text("seed".to_string()),
system_prompt: meerkat_core::SystemPromptOverride::Inherit,
max_tokens: None,
event_tx: None,
initial_turn: InitialTurnPolicy::Defer,
deferred_prompt_policy: DeferredPromptPolicy::Discard,
build: None,
labels: None,
};
let CreateTimeEventProjection {
session_tx,
drain_rx: _drain_rx,
} = service
.install_create_time_event_projection(&mut request, Some(&session_id))
.await
.expect("persistent event projection is configured");
let projection_tx = request
.event_tx
.take()
.expect("create-time projection sender installed");
assert!(
session_tx.send(Some(session_id.clone())).is_ok(),
"create-time projector still owns the session-id receiver"
);
projection_tx
.send(meerkat_core::event::EventEnvelope::new_session(
session_id.clone(),
1,
None,
AgentEvent::RunStarted {
session_id: session_id.clone(),
input: meerkat_core::types::RunInput::Content {
content: ContentInput::Text("seed".to_string()),
},
},
))
.await?;
drop(projection_tx);
drop(session_tx);
event_store.wait_for_append().await;
let drain_service = Arc::clone(&service);
let drain_session_id = session_id.clone();
let mut drain_task = tokio::spawn(async move {
drain_service
.event_log_await_projection_drain(&drain_session_id)
.await
});
assert!(
tokio::time::timeout(std::time::Duration::from_millis(100), &mut drain_task)
.await
.is_err(),
"generation cutover must remain blocked while a create-time append is in flight"
);
event_store.release_append();
let drained = tokio::time::timeout(std::time::Duration::from_secs(5), &mut drain_task)
.await
.expect("projection drain completed after the append gate opened")
.expect("projection drain task joined")?;
assert!(drained, "the create-time projector was part of the drain");
let final_seq = event_store.last_seq(&session_id).await?;
assert_eq!(final_seq, 1);
tokio::task::yield_now().await;
assert_eq!(
event_store.last_seq(&session_id).await?,
final_seq,
"no old-incarnation event may append after the drain returns"
);
Ok(())
}
#[tokio::test]
async fn lagged_event_projection_persists_halt_before_renumbering_later_events()
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let dir = tempfile::tempdir()?;
let session_id = SessionId::new();
let concrete_store = Arc::new(crate::event_store::FileEventStore::new(
dir.path().join("events"),
));
let event_store: Arc<dyn EventStore> = concrete_store.clone();
let projector = Arc::new(SessionProjector::new(dir.path().join(".rkat")));
let faults: EventProjectionFaultRegistry = Arc::new(Mutex::new(HashMap::new()));
let gates: EventProjectionGateRegistry = Arc::new(Mutex::new(HashMap::new()));
let gap = meerkat_core::event::EventEnvelope::new_session(
session_id.clone(),
0,
None,
AgentEvent::StreamTruncated {
reason: meerkat_core::event::StreamTruncationReason::StreamLagged { dropped: 17 },
},
);
let later = meerkat_core::event::EventEnvelope::new_session(
session_id.clone(),
99,
None,
AgentEvent::StreamTruncated {
reason: meerkat_core::event::StreamTruncationReason::ChannelFull,
},
);
let stream: meerkat_core::comms::EventStream =
Box::pin(futures::stream::iter(vec![gap, later]));
project_session_event_stream(
event_store,
projector,
session_id.clone(),
stream,
Arc::clone(&faults),
gates,
)
.await;
let marker = concrete_store
.projection_halt(&session_id)
.await?
.expect("lag must persist a durable projection halt");
assert!(marker.reason.contains("17 canonical events were dropped"));
assert!(
concrete_store.read_from(&session_id, 0).await?.is_empty(),
"neither the synthetic marker nor later events may be renumbered into the durable log"
);
let faults = faults.lock().await;
let cause = faults.get(&session_id).expect("typed in-process fault");
assert!(matches!(
cause.as_ref(),
SessionError::Store(source)
if source.downcast_ref::<SessionEventProjectionLagged>().is_some()
));
Ok(())
}
#[tokio::test]
async fn durable_event_projection_halt_marker_fails_replay_after_restart()
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let dir = tempfile::tempdir()?;
let event_root = dir.path().join("events");
let session_id = SessionId::new();
let initial_store = crate::event_store::FileEventStore::new(&event_root);
initial_store
.append(
&session_id,
&[AgentEvent::StreamTruncated {
reason: meerkat_core::event::StreamTruncationReason::ChannelFull,
}],
)
.await?;
initial_store
.record_projection_halt(&session_id, "injected durable append failure")
.await?;
assert_eq!(initial_store.last_seq(&session_id).await?, 1);
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let restarted_concrete = Arc::new(crate::event_store::FileEventStore::new(&event_root));
let restarted_event_store: Arc<dyn EventStore> = restarted_concrete.clone();
let projector = Arc::new(SessionProjector::new(dir.path().join(".rkat")));
let stream_faults: EventProjectionFaultRegistry = Arc::new(Mutex::new(HashMap::new()));
let stream_gates: EventProjectionGateRegistry = Arc::new(Mutex::new(HashMap::new()));
let later_stream: meerkat_core::comms::EventStream = Box::pin(futures::stream::iter(vec![
meerkat_core::event::EventEnvelope::new_session(
session_id.clone(),
2,
None,
AgentEvent::StreamTruncated {
reason: meerkat_core::event::StreamTruncationReason::ChannelFull,
},
),
]));
project_session_event_stream(
restarted_event_store.clone(),
projector.clone(),
session_id.clone(),
later_stream,
stream_faults,
stream_gates,
)
.await;
assert_eq!(
restarted_concrete.last_seq(&session_id).await?,
1,
"restart projector appended past a durable terminal marker"
);
let create_faults: EventProjectionFaultRegistry = Arc::new(Mutex::new(HashMap::new()));
let create_gates: EventProjectionGateRegistry = Arc::new(Mutex::new(HashMap::new()));
let (projection_tx, projection_rx) = mpsc::channel(1);
let (_session_tx, session_rx) = watch::channel(Some(session_id.clone()));
let create_task = tokio::spawn(project_create_time_events(
restarted_event_store.clone(),
projector.clone(),
projection_rx,
session_rx,
None,
CreateTimeEventProjectionLifecycle {
projection_faults: create_faults,
projection_gates: create_gates,
drain_registration: None,
},
));
projection_tx
.send(meerkat_core::event::EventEnvelope::new_session(
session_id.clone(),
3,
None,
AgentEvent::StreamTruncated {
reason: meerkat_core::event::StreamTruncationReason::ChannelFull,
},
))
.await?;
drop(projection_tx);
create_task.await?;
assert_eq!(
restarted_concrete.last_seq(&session_id).await?,
1,
"restart create-time projector appended past a durable terminal marker"
);
let restarted = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::new(InMemoryRuntimeStore::new()),
memory_blob_store(),
)
.with_event_projection(restarted_event_store, projector);
let latest_err = restarted
.event_log_latest_seq(&session_id)
.await
.expect_err("durable halt marker must fail latest cursor after restart");
assert!(
matches!(&latest_err, SessionError::Store(source)
if source.downcast_ref::<EventProjectionHalted>().is_some()),
"expected typed EventProjectionHalted latest-cursor fault after restart, got: {latest_err}"
);
let err = restarted
.event_log_read_from(&session_id, 0)
.await
.expect_err("durable halt marker must fail replay after restart");
match err {
SessionError::Store(source) => {
let halted = source
.downcast_ref::<EventProjectionHalted>()
.expect("expected typed EventProjectionHalted replay error");
let cause = halted.cause.as_ref();
assert!(
matches!(cause, SessionError::Store(marker)
if marker.downcast_ref::<DurableEventProjectionHaltMarker>().is_some()),
"expected durable marker source, got: {cause}"
);
}
other => panic!("expected SessionError::Store, got: {other}"),
}
Ok(())
}
#[tokio::test]
async fn projection_latch_precedes_blocking_marker_and_fences_concurrent_append()
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let session_store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let concrete_store = Arc::new(BlockingProjectionHaltStore::default());
let event_store: Arc<dyn EventStore> = concrete_store.clone();
let dir = tempfile::tempdir()?;
let projector = Arc::new(SessionProjector::new(dir.path().join(".rkat")));
let service = Arc::new(
PersistentSessionService::new(
DummyBuilder,
4,
session_store,
Arc::new(InMemoryRuntimeStore::new()),
memory_blob_store(),
)
.with_event_projection(event_store.clone(), projector.clone()),
);
let session_id = SessionId::new();
let record_faults = Arc::clone(&service.event_projection_faults);
let record_gates = Arc::clone(&service.event_projection_gates);
let record_store = event_store.clone();
let record_session_id = session_id.clone();
let record_task = tokio::spawn(async move {
record_event_projection_fault(
&record_faults,
&record_gates,
&record_store,
&record_session_id,
SessionError::Store(Box::new(EventStoreError::Store(
"synthetic projection gap".to_string(),
))),
)
.await;
});
concrete_store.wait_for_marker_write().await;
let replay = tokio::time::timeout(
std::time::Duration::from_millis(250),
service.event_log_read_from(&session_id, 0),
)
.await
.expect("replay waited on the blocked marker write")
.expect_err("replay crossed an already-published terminal latch");
assert!(matches!(replay, SessionError::Store(_)));
let append_store = event_store.clone();
let append_projector = projector.clone();
let append_faults = Arc::clone(&service.event_projection_faults);
let append_gates = Arc::clone(&service.event_projection_gates);
let append_session_id = session_id.clone();
let append_task = tokio::spawn(async move {
append_and_project_event(
&append_store,
&append_projector,
&append_session_id,
meerkat_core::event::EventEnvelope::new_session(
append_session_id.clone(),
1,
None,
AgentEvent::StreamTruncated {
reason: meerkat_core::event::StreamTruncationReason::ChannelFull,
},
),
&append_faults,
&append_gates,
)
.await
});
tokio::task::yield_now().await;
assert!(
!append_task.is_finished(),
"same-session append must wait behind latch+marker publication"
);
concrete_store.release_marker_write();
record_task.await?;
assert!(
append_task.await?.is_err(),
"append admitted after the terminal latch"
);
assert_eq!(concrete_store.last_seq(&session_id).await?, 0);
Ok(())
}
#[tokio::test]
async fn durable_projection_halt_rejects_resume_before_agent_build()
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let dir = tempfile::tempdir()?;
let session_id = SessionId::new();
let event_store = Arc::new(crate::event_store::FileEventStore::new(
dir.path().join("events"),
));
event_store
.record_projection_halt(&session_id, "restart projection gap")
.await?;
let event_store: Arc<dyn EventStore> = event_store;
let builds = Arc::new(AtomicUsize::new(0));
let service = PersistentSessionService::new(
CountingBuilder {
builds: Arc::clone(&builds),
},
4,
Arc::new(MemoryStore::new()),
Arc::new(InMemoryRuntimeStore::new()),
memory_blob_store(),
)
.with_event_projection(
event_store,
Arc::new(SessionProjector::new(dir.path().join(".rkat"))),
);
let error = service
.create_session(resume_request(Session::with_id(session_id)))
.await
.expect_err("resume crossed an absorbing projection halt");
assert!(matches!(error, SessionError::Store(_)));
assert_eq!(
builds.load(Ordering::Acquire),
0,
"resume reached the inner agent builder before projector admission"
);
Ok(())
}
struct RecordingEventStore {
events: Mutex<HashMap<SessionId, Vec<StoredEvent>>>,
projection_halts: Mutex<HashMap<SessionId, crate::event_store::EventProjectionHaltMarker>>,
notify: tokio::sync::Notify,
fail_appends: AtomicBool,
rewrite_append_calls: AtomicUsize,
fail_rewrite_append_call: AtomicUsize,
}
impl Default for RecordingEventStore {
fn default() -> Self {
Self {
events: Mutex::new(HashMap::new()),
projection_halts: Mutex::new(HashMap::new()),
notify: tokio::sync::Notify::new(),
fail_appends: AtomicBool::new(false),
rewrite_append_calls: AtomicUsize::new(0),
fail_rewrite_append_call: AtomicUsize::new(0),
}
}
}
impl RecordingEventStore {
fn fail_appends(&self) {
self.fail_appends.store(true, Ordering::Release);
}
fn allow_appends(&self) {
self.fail_appends.store(false, Ordering::Release);
}
fn fail_rewrite_append_call(&self, call: usize) {
self.fail_rewrite_append_call.store(call, Ordering::Release);
}
async fn wait_for_seq(&self, session_id: &SessionId, target_seq: u64) {
tokio::time::timeout(std::time::Duration::from_secs(10), async {
loop {
if self.last_seq(session_id).await.unwrap() >= target_seq {
return;
}
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
}
})
.await
.expect("event store projection did not reach expected sequence");
}
}
async fn read_projected_events_after(events_path: &std::path::Path, expected: &str) -> String {
tokio::time::timeout(std::time::Duration::from_secs(10), async {
loop {
if let Ok(projected) = tokio::fs::read_to_string(events_path).await
&& projected.contains(expected)
{
return projected;
}
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
}
})
.await
.expect("projected events.jsonl did not contain expected event")
}
#[async_trait::async_trait]
impl EventStore for RecordingEventStore {
async fn append_envelopes(
&self,
session_id: &SessionId,
envelopes: &[meerkat_core::event::EventEnvelope<AgentEvent>],
) -> Result<u64, EventStoreError> {
let receipts: Vec<_> = envelopes
.iter()
.filter_map(|envelope| match &envelope.payload {
AgentEvent::TranscriptRewriteAuditReceiptCommitted {
session_id,
receipt,
final_assistant_text,
} => Some((session_id, receipt, final_assistant_text)),
_ => None,
})
.collect();
if let [(receipt_session_id, receipt, final_assistant_text)] = receipts.as_slice() {
if envelopes.len() != 1 || *receipt_session_id != session_id {
return Err(EventStoreError::Store(
"generic transcript rewrite receipt batches are forbidden".to_string(),
));
}
return self
.append_transcript_rewrite_receipt_exact(
session_id,
receipt,
final_assistant_text.as_deref(),
)
.await
.map(|result| result.stored_event().seq);
}
if !receipts.is_empty() {
return Err(EventStoreError::Store(
"generic transcript rewrite receipt batches are forbidden".to_string(),
));
}
let contains_interaction_terminal = envelopes.iter().any(|envelope| {
matches!(
envelope.source,
meerkat_core::event::EventSourceIdentity::Interaction { .. }
) || matches!(
envelope.payload,
AgentEvent::InteractionComplete { .. }
| AgentEvent::InteractionCallbackPending { .. }
| AgentEvent::InteractionFailed { .. }
)
});
if contains_interaction_terminal {
let [envelope] = envelopes else {
return Err(EventStoreError::Store(
"interaction terminals must use a singleton exact append".to_string(),
));
};
let meerkat_core::event::EventSourceIdentity::Interaction { interaction_id } =
&envelope.source
else {
return Err(EventStoreError::Store(
"interaction terminal payload requires Interaction source".to_string(),
));
};
crate::event_store::validate_exact_interaction_terminal(*interaction_id, envelope)?;
return self
.append_interaction_terminal_exact(session_id, *interaction_id, envelope)
.await
.map(|result| result.stored_event().seq);
}
if self.fail_appends.load(Ordering::Acquire) {
return Err(EventStoreError::Store(
"synthetic transcript rewrite audit append failure".to_string(),
));
}
if envelopes.iter().any(|envelope| {
matches!(
envelope.payload,
AgentEvent::TranscriptRewriteCommitted { .. }
)
}) {
let call = self.rewrite_append_calls.fetch_add(1, Ordering::AcqRel) + 1;
if self.fail_rewrite_append_call.load(Ordering::Acquire) == call {
return Err(EventStoreError::Store(
"synthetic transcript rewrite audit append failure".to_string(),
));
}
}
let mut all_events = self.events.lock().await;
let session_events = all_events.entry(session_id.clone()).or_default();
for envelope in envelopes {
let seq = session_events.len() as u64 + 1;
session_events.push(StoredEvent {
seq,
schema_version: EVENT_SCHEMA_VERSION,
timestamp: meerkat_core::time_compat::SystemTime::now(),
source: envelope.source.clone(),
mob_id: envelope.mob_id.clone(),
stream_seq: envelope.seq,
event: envelope.payload.clone(),
});
}
let last_seq = session_events.len() as u64;
drop(all_events);
self.notify.notify_waiters();
Ok(last_seq)
}
async fn append_transcript_rewrite_receipt_exact(
&self,
session_id: &SessionId,
receipt: &meerkat_core::TranscriptRewriteAuditReceiptBatch,
final_assistant_text: Option<&str>,
) -> Result<crate::event_store::ExactTranscriptRewriteReceiptAppend, EventStoreError>
{
if self.fail_appends.load(Ordering::Acquire) {
return Err(EventStoreError::Store(
"synthetic transcript rewrite audit append failure".to_string(),
));
}
let call = self.rewrite_append_calls.fetch_add(1, Ordering::AcqRel) + 1;
if self.fail_rewrite_append_call.load(Ordering::Acquire) == call {
return Err(EventStoreError::Store(
"synthetic transcript rewrite audit append failure".to_string(),
));
}
let incoming_summary = final_assistant_text.map(ToOwned::to_owned);
let mut all_events = self.events.lock().await;
let session_events = all_events.entry(session_id.clone()).or_default();
let exact: Vec<_> = session_events
.iter()
.filter(|stored| match &stored.event {
AgentEvent::TranscriptRewriteAuditReceiptCommitted {
session_id: stored_session_id,
receipt: stored_receipt,
..
} => stored_session_id == session_id && stored_receipt == receipt,
_ => false,
})
.cloned()
.collect();
match exact.as_slice() {
[] => {
let stored = StoredEvent {
seq: session_events.len() as u64 + 1,
schema_version: EVENT_SCHEMA_VERSION,
timestamp: meerkat_core::time_compat::SystemTime::now(),
source: meerkat_core::event::EventSourceIdentity::session(
session_id.clone(),
),
mob_id: None,
stream_seq: 0,
event: AgentEvent::TranscriptRewriteAuditReceiptCommitted {
session_id: session_id.clone(),
receipt: receipt.clone(),
final_assistant_text: incoming_summary,
},
};
session_events.push(stored.clone());
drop(all_events);
self.notify.notify_waiters();
Ok(crate::event_store::ExactTranscriptRewriteReceiptAppend::Inserted(stored))
}
[stored] => {
let stored_summary = match &stored.event {
AgentEvent::TranscriptRewriteAuditReceiptCommitted {
final_assistant_text,
..
} => final_assistant_text,
_ => unreachable!("exact receipt filter changed event variant"),
};
if stored_summary.as_deref() == incoming_summary.as_deref() {
Ok(
crate::event_store::ExactTranscriptRewriteReceiptAppend::Replayed(
stored.clone(),
),
)
} else {
Err(EventStoreError::ExactTranscriptRewriteReceiptConflict {
session_id: session_id.clone(),
existing_count: 1,
reason: format!(
"row {} carries a different terminal assistant-text projection",
stored.seq
),
})
}
}
duplicates => Err(EventStoreError::ExactTranscriptRewriteReceiptConflict {
session_id: session_id.clone(),
existing_count: duplicates.len(),
reason: "multiple exact receipt rows".to_string(),
}),
}
}
async fn record_projection_halt(
&self,
session_id: &SessionId,
reason: &str,
) -> Result<(), EventStoreError> {
self.projection_halts.lock().await.insert(
session_id.clone(),
crate::event_store::EventProjectionHaltMarker {
session_id: session_id.clone(),
reason: reason.to_string(),
recorded_at: meerkat_core::time_compat::SystemTime::now(),
},
);
Ok(())
}
async fn projection_halt(
&self,
session_id: &SessionId,
) -> Result<Option<crate::event_store::EventProjectionHaltMarker>, EventStoreError>
{
Ok(self.projection_halts.lock().await.get(session_id).cloned())
}
async fn append_interaction_terminal_exact(
&self,
session_id: &SessionId,
interaction_id: meerkat_core::interaction::InteractionId,
envelope: &meerkat_core::event::EventEnvelope<AgentEvent>,
) -> Result<ExactInteractionAppend, EventStoreError> {
crate::event_store::validate_exact_interaction_terminal(interaction_id, envelope)?;
if self.fail_appends.load(Ordering::Acquire) {
return Err(EventStoreError::Store(
"synthetic transcript rewrite audit append failure".to_string(),
));
}
let mut all_events = self.events.lock().await;
let session_events = all_events.entry(session_id.clone()).or_default();
let exact_rows: Vec<_> = session_events
.iter()
.filter(|row| {
matches!(
row.source,
meerkat_core::event::EventSourceIdentity::Interaction {
interaction_id: source_id
} if source_id == interaction_id
)
})
.cloned()
.collect();
match exact_rows.as_slice() {
[] => {
let stored = StoredEvent {
seq: session_events.len() as u64 + 1,
schema_version: EVENT_SCHEMA_VERSION,
timestamp: meerkat_core::time_compat::SystemTime::now(),
source: envelope.source.clone(),
mob_id: envelope.mob_id.clone(),
stream_seq: envelope.seq,
event: envelope.payload.clone(),
};
session_events.push(stored.clone());
drop(all_events);
self.notify.notify_waiters();
Ok(ExactInteractionAppend::Inserted(stored))
}
[existing]
if existing.mob_id == envelope.mob_id
&& crate::event_store::interaction_terminal_events_semantically_equal(
&existing.event,
&envelope.payload,
) =>
{
Ok(ExactInteractionAppend::Replayed(existing.clone()))
}
rows => Err(EventStoreError::ExactInteractionTerminalConflict {
session_id: session_id.clone(),
interaction_id,
existing_count: rows.len(),
reason: "recording store exact interaction terminal conflict".to_string(),
}),
}
}
async fn append_interaction_terminals_exact_batch(
&self,
session_id: &SessionId,
stream_seq_floor: u64,
terminals: &[(
meerkat_core::interaction::InteractionId,
meerkat_core::event::EventEnvelope<AgentEvent>,
)],
) -> Result<Vec<ExactInteractionAppend>, EventStoreError> {
crate::event_store::validate_exact_interaction_terminal_batch(terminals)?;
if terminals.is_empty() {
return Ok(Vec::new());
}
if self.fail_appends.load(Ordering::Acquire) {
return Err(EventStoreError::Store(
"synthetic transcript rewrite audit append failure".to_string(),
));
}
let mut all_events = self.events.lock().await;
let session_events = all_events.entry(session_id.clone()).or_default();
let occupants: Vec<_> = terminals
.iter()
.map(|(interaction_id, _)| {
let exact: Vec<_> = session_events
.iter()
.filter(|row| {
matches!(
row.source,
meerkat_core::event::EventSourceIdentity::Interaction {
interaction_id: source_id
} if source_id == *interaction_id
)
})
.cloned()
.collect();
match exact.as_slice() {
[] => crate::event_store::ExactInteractionOccupancy::Empty,
[stored] => {
crate::event_store::ExactInteractionOccupancy::One(stored.clone())
}
[first, ..] => crate::event_store::ExactInteractionOccupancy::Multiple {
first: first.clone(),
count: exact.len(),
},
}
})
.collect();
let prefix_len = crate::event_store::validate_exact_interaction_terminal_replay_prefix(
session_id, terminals, &occupants,
)?;
let mut results = Vec::with_capacity(terminals.len());
let mut replay_tail = 0_u64;
for occupant in occupants.iter().take(prefix_len) {
let crate::event_store::ExactInteractionOccupancy::One(stored) = occupant else {
return Err(EventStoreError::Store(
"validated replay prefix lost its stored row".to_string(),
));
};
replay_tail = stored.stream_seq;
results.push(ExactInteractionAppend::Replayed(stored.clone()));
}
let mut next_stream_seq = stream_seq_floor.max(replay_tail);
let mut inserted_any = false;
for (_, envelope) in &terminals[prefix_len..] {
next_stream_seq = next_stream_seq.checked_add(1).ok_or_else(|| {
EventStoreError::InvalidExactInteractionTerminalBatch {
reason: "session event stream sequence overflow".to_string(),
}
})?;
let stored = StoredEvent {
seq: session_events.len() as u64 + 1,
schema_version: EVENT_SCHEMA_VERSION,
timestamp: meerkat_core::time_compat::SystemTime::now(),
source: envelope.source.clone(),
mob_id: envelope.mob_id.clone(),
stream_seq: next_stream_seq,
event: envelope.payload.clone(),
};
session_events.push(stored.clone());
results.push(ExactInteractionAppend::Inserted(stored));
inserted_any = true;
}
drop(all_events);
if inserted_any {
self.notify.notify_waiters();
}
Ok(results)
}
async fn read_from(
&self,
session_id: &SessionId,
from_seq: u64,
) -> Result<Vec<StoredEvent>, EventStoreError> {
let all_events = self.events.lock().await;
Ok(all_events
.get(session_id)
.into_iter()
.flat_map(|events| events.iter())
.filter(|event| event.seq >= from_seq)
.cloned()
.collect())
}
async fn last_seq(&self, session_id: &SessionId) -> Result<u64, EventStoreError> {
let all_events = self.events.lock().await;
Ok(all_events
.get(session_id)
.map_or(0, |events| events.len() as u64))
}
}
struct PausingAppendEventStore {
inner: RecordingEventStore,
append_entered: AtomicBool,
release_append: tokio::sync::Semaphore,
}
impl Default for PausingAppendEventStore {
fn default() -> Self {
Self {
inner: RecordingEventStore::default(),
append_entered: AtomicBool::new(false),
release_append: tokio::sync::Semaphore::new(0),
}
}
}
impl PausingAppendEventStore {
async fn wait_for_append(&self) {
tokio::time::timeout(std::time::Duration::from_secs(5), async {
while !self.append_entered.load(Ordering::Acquire) {
tokio::task::yield_now().await;
}
})
.await
.expect("create-time event append did not reach the test gate");
}
fn release_append(&self) {
self.release_append.add_permits(1);
}
}
#[async_trait::async_trait]
impl EventStore for PausingAppendEventStore {
async fn append_envelopes(
&self,
session_id: &SessionId,
envelopes: &[meerkat_core::event::EventEnvelope<AgentEvent>],
) -> Result<u64, EventStoreError> {
self.append_entered.store(true, Ordering::Release);
self.release_append
.acquire()
.await
.map_err(|_| EventStoreError::Store("append gate closed".to_string()))?
.forget();
self.inner.append_envelopes(session_id, envelopes).await
}
async fn append_transcript_rewrite_receipt_exact(
&self,
session_id: &SessionId,
receipt: &meerkat_core::TranscriptRewriteAuditReceiptBatch,
final_assistant_text: Option<&str>,
) -> Result<crate::event_store::ExactTranscriptRewriteReceiptAppend, EventStoreError>
{
self.append_entered.store(true, Ordering::Release);
self.release_append
.acquire()
.await
.map_err(|_| EventStoreError::Store("append gate closed".to_string()))?
.forget();
self.inner
.append_transcript_rewrite_receipt_exact(session_id, receipt, final_assistant_text)
.await
}
async fn record_projection_halt(
&self,
session_id: &SessionId,
reason: &str,
) -> Result<(), EventStoreError> {
self.inner.record_projection_halt(session_id, reason).await
}
async fn projection_halt(
&self,
session_id: &SessionId,
) -> Result<Option<crate::event_store::EventProjectionHaltMarker>, EventStoreError>
{
self.inner.projection_halt(session_id).await
}
async fn read_from(
&self,
session_id: &SessionId,
from_seq: u64,
) -> Result<Vec<StoredEvent>, EventStoreError> {
self.inner.read_from(session_id, from_seq).await
}
async fn last_seq(&self, session_id: &SessionId) -> Result<u64, EventStoreError> {
self.inner.last_seq(session_id).await
}
}
struct BlockingProjectionHaltStore {
inner: RecordingEventStore,
marker_entered: tokio::sync::Notify,
release_marker: tokio::sync::Notify,
}
impl Default for BlockingProjectionHaltStore {
fn default() -> Self {
Self {
inner: RecordingEventStore::default(),
marker_entered: tokio::sync::Notify::new(),
release_marker: tokio::sync::Notify::new(),
}
}
}
impl BlockingProjectionHaltStore {
async fn wait_for_marker_write(&self) {
tokio::time::timeout(
std::time::Duration::from_secs(5),
self.marker_entered.notified(),
)
.await
.expect("projection halt marker write did not block");
}
fn release_marker_write(&self) {
self.release_marker.notify_one();
}
}
#[async_trait::async_trait]
impl EventStore for BlockingProjectionHaltStore {
async fn append_envelopes(
&self,
session_id: &SessionId,
envelopes: &[meerkat_core::event::EventEnvelope<AgentEvent>],
) -> Result<u64, EventStoreError> {
self.inner.append_envelopes(session_id, envelopes).await
}
async fn append_transcript_rewrite_receipt_exact(
&self,
session_id: &SessionId,
receipt: &meerkat_core::TranscriptRewriteAuditReceiptBatch,
final_assistant_text: Option<&str>,
) -> Result<crate::event_store::ExactTranscriptRewriteReceiptAppend, EventStoreError>
{
self.inner
.append_transcript_rewrite_receipt_exact(session_id, receipt, final_assistant_text)
.await
}
async fn record_projection_halt(
&self,
session_id: &SessionId,
reason: &str,
) -> Result<(), EventStoreError> {
self.marker_entered.notify_one();
self.release_marker.notified().await;
self.inner.record_projection_halt(session_id, reason).await
}
async fn projection_halt(
&self,
session_id: &SessionId,
) -> Result<Option<crate::event_store::EventProjectionHaltMarker>, EventStoreError>
{
self.inner.projection_halt(session_id).await
}
async fn read_from(
&self,
session_id: &SessionId,
from_seq: u64,
) -> Result<Vec<StoredEvent>, EventStoreError> {
self.inner.read_from(session_id, from_seq).await
}
async fn last_seq(&self, session_id: &SessionId) -> Result<u64, EventStoreError> {
self.inner.last_seq(session_id).await
}
}
struct PausingTranscriptRewriteStore {
inner: MemoryStore,
pause_rewrite_save: AtomicBool,
entered_rewrite_save: tokio::sync::Notify,
release_rewrite_save: tokio::sync::Notify,
}
impl PausingTranscriptRewriteStore {
fn new() -> Self {
Self {
inner: MemoryStore::new(),
pause_rewrite_save: AtomicBool::new(false),
entered_rewrite_save: tokio::sync::Notify::new(),
release_rewrite_save: tokio::sync::Notify::new(),
}
}
fn pause_rewrite_saves(&self) {
self.pause_rewrite_save.store(true, Ordering::Release);
}
async fn wait_for_rewrite_save(&self) {
tokio::time::timeout(
std::time::Duration::from_secs(10),
self.entered_rewrite_save.notified(),
)
.await
.expect("rewrite save did not pause");
}
fn release_rewrite_save(&self) {
self.release_rewrite_save.notify_waiters();
}
}
#[async_trait::async_trait]
impl SessionStore for PausingTranscriptRewriteStore {
async fn save(&self, session: &Session) -> Result<(), SessionStoreError> {
self.inner.save(session).await
}
async fn save_transcript_rewrite(
&self,
session: &Session,
commit: &meerkat_core::TranscriptRewriteCommit,
) -> Result<(), SessionStoreError> {
if self.pause_rewrite_save.load(Ordering::Acquire) {
self.entered_rewrite_save.notify_waiters();
self.release_rewrite_save.notified().await;
}
self.inner.save_transcript_rewrite(session, commit).await
}
async fn save_authoritative_projection(
&self,
session: &Session,
) -> Result<(), SessionStoreError> {
self.inner.save_authoritative_projection(session).await
}
async fn save_authoritative_projection_if_current_revision(
&self,
session: &Session,
expected_current_revision: Option<String>,
) -> Result<(), SessionStoreError> {
if self.pause_rewrite_save.load(Ordering::Acquire) {
self.entered_rewrite_save.notify_waiters();
self.release_rewrite_save.notified().await;
}
self.inner
.save_authoritative_projection_if_current_revision(
session,
expected_current_revision,
)
.await
}
async fn load(&self, id: &SessionId) -> Result<Option<Session>, SessionStoreError> {
self.inner.load(id).await
}
async fn list(
&self,
filter: meerkat_store::SessionFilter,
) -> Result<Vec<meerkat_core::SessionMeta>, SessionStoreError> {
self.inner.list(filter).await
}
async fn delete(&self, id: &SessionId) -> Result<(), SessionStoreError> {
self.inner.delete(id).await
}
async fn delete_if_current_revision(
&self,
id: &SessionId,
expected_current_revision: &str,
) -> Result<bool, SessionStoreError> {
self.inner
.delete_if_current_revision(id, expected_current_revision)
.await
}
}
struct FailSaveStore {
inner: MemoryStore,
fail_save: AtomicBool,
}
impl FailSaveStore {
fn new() -> Self {
Self {
inner: MemoryStore::new(),
fail_save: AtomicBool::new(false),
}
}
fn set_fail_save(&self, fail: bool) {
self.fail_save.store(fail, Ordering::Release);
}
}
#[async_trait::async_trait]
impl SessionStore for FailSaveStore {
async fn save(&self, session: &Session) -> Result<(), SessionStoreError> {
if self.fail_save.load(Ordering::Acquire) {
return Err(SessionStoreError::Internal(
"forced save failure".to_string(),
));
}
self.inner.save(session).await
}
async fn save_authoritative_projection_if_current_revision(
&self,
session: &Session,
expected_current_revision: Option<String>,
) -> Result<(), SessionStoreError> {
if self.fail_save.load(Ordering::Acquire) {
return Err(SessionStoreError::Internal(
"forced save failure".to_string(),
));
}
self.inner
.save_authoritative_projection_if_current_revision(
session,
expected_current_revision,
)
.await
}
async fn load(&self, id: &SessionId) -> Result<Option<Session>, SessionStoreError> {
self.inner.load(id).await
}
async fn list(
&self,
filter: meerkat_store::SessionFilter,
) -> Result<Vec<meerkat_core::SessionMeta>, SessionStoreError> {
self.inner.list(filter).await
}
async fn delete(&self, id: &SessionId) -> Result<(), SessionStoreError> {
self.inner.delete(id).await
}
async fn delete_if_current_revision(
&self,
id: &SessionId,
expected_current_revision: &str,
) -> Result<bool, SessionStoreError> {
self.inner
.delete_if_current_revision(id, expected_current_revision)
.await
}
}
struct GatedSnapshotRuntimeStore {
inner: InMemoryRuntimeStore,
hidden_snapshot_loads: AtomicUsize,
fail_snapshot_loads: AtomicBool,
fail_snapshot_commits: AtomicBool,
snapshot_commit_count: AtomicUsize,
fail_snapshot_commit_ordinal: AtomicUsize,
fail_machine_lifecycle_commits: AtomicBool,
session_snapshot_overrides: Mutex<HashMap<LogicalRuntimeId, Vec<u8>>>,
replace_snapshot_interlopers: Mutex<HashMap<LogicalRuntimeId, Vec<u8>>>,
input_state_load_errors: Mutex<HashSet<LogicalRuntimeId>>,
boundary_commits: Mutex<Vec<meerkat_core::lifecycle::RunBoundaryReceipt>>,
pause_rewrite_commit: AtomicBool,
entered_rewrite_commit: tokio::sync::Notify,
release_rewrite_commit: tokio::sync::Notify,
pause_machine_lifecycle_commit: AtomicBool,
entered_machine_lifecycle_commit: tokio::sync::Notify,
release_machine_lifecycle_commit: tokio::sync::Notify,
}
impl GatedSnapshotRuntimeStore {
fn new() -> Self {
Self {
inner: InMemoryRuntimeStore::new(),
hidden_snapshot_loads: AtomicUsize::new(0),
fail_snapshot_loads: AtomicBool::new(false),
fail_snapshot_commits: AtomicBool::new(false),
snapshot_commit_count: AtomicUsize::new(0),
fail_snapshot_commit_ordinal: AtomicUsize::new(usize::MAX),
fail_machine_lifecycle_commits: AtomicBool::new(false),
session_snapshot_overrides: Mutex::new(HashMap::new()),
replace_snapshot_interlopers: Mutex::new(HashMap::new()),
input_state_load_errors: Mutex::new(HashSet::new()),
boundary_commits: Mutex::new(Vec::new()),
pause_rewrite_commit: AtomicBool::new(false),
entered_rewrite_commit: tokio::sync::Notify::new(),
release_rewrite_commit: tokio::sync::Notify::new(),
pause_machine_lifecycle_commit: AtomicBool::new(false),
entered_machine_lifecycle_commit: tokio::sync::Notify::new(),
release_machine_lifecycle_commit: tokio::sync::Notify::new(),
}
}
fn set_fail_snapshot_commits(&self, fail: bool) {
self.fail_snapshot_commits.store(fail, Ordering::Release);
}
fn set_fail_snapshot_loads(&self, fail: bool) {
self.fail_snapshot_loads.store(fail, Ordering::Release);
}
fn fail_snapshot_commit_in(&self, commits_from_now: usize) {
assert!(commits_from_now > 0);
let target = self
.snapshot_commit_count
.load(Ordering::Acquire)
.saturating_add(commits_from_now);
self.fail_snapshot_commit_ordinal
.store(target, Ordering::Release);
}
fn set_fail_machine_lifecycle_commits(&self, fail: bool) {
self.fail_machine_lifecycle_commits
.store(fail, Ordering::Release);
}
fn hide_next_session_snapshot_loads(&self, count: usize) {
self.hidden_snapshot_loads.store(count, Ordering::Release);
}
fn should_hide_session_snapshot_load(&self) -> bool {
let mut remaining = self.hidden_snapshot_loads.load(Ordering::Acquire);
while remaining > 0 {
match self.hidden_snapshot_loads.compare_exchange(
remaining,
remaining - 1,
Ordering::AcqRel,
Ordering::Acquire,
) {
Ok(_) => return true,
Err(current) => remaining = current,
}
}
false
}
async fn boundary_commits(&self) -> Vec<meerkat_core::lifecycle::RunBoundaryReceipt> {
self.boundary_commits.lock().await.clone()
}
async fn reset_boundary_commits(&self) {
self.boundary_commits.lock().await.clear();
}
async fn override_session_snapshot(&self, runtime_id: LogicalRuntimeId, snapshot: Vec<u8>) {
self.session_snapshot_overrides
.lock()
.await
.insert(runtime_id, snapshot);
}
async fn interlope_before_snapshot_replace(
&self,
runtime_id: LogicalRuntimeId,
snapshot: Vec<u8>,
) {
self.replace_snapshot_interlopers
.lock()
.await
.insert(runtime_id, snapshot);
}
async fn fail_input_state_load_for(&self, runtime_id: LogicalRuntimeId) {
self.input_state_load_errors.lock().await.insert(runtime_id);
}
fn pause_next_rewrite_commit(&self) {
self.pause_rewrite_commit.store(true, Ordering::Release);
}
async fn wait_for_rewrite_commit(&self) {
self.entered_rewrite_commit.notified().await;
}
fn release_rewrite_commit(&self) {
self.release_rewrite_commit.notify_one();
}
fn pause_next_machine_lifecycle_commit(&self) {
self.pause_machine_lifecycle_commit
.store(true, Ordering::Release);
}
async fn wait_for_machine_lifecycle_commit(&self) {
self.entered_machine_lifecycle_commit.notified().await;
}
fn release_machine_lifecycle_commit(&self) {
self.release_machine_lifecycle_commit.notify_one();
}
async fn maybe_pause_machine_lifecycle_commit(&self) {
if self
.pause_machine_lifecycle_commit
.swap(false, Ordering::AcqRel)
{
self.entered_machine_lifecycle_commit.notify_one();
self.release_machine_lifecycle_commit.notified().await;
}
}
}
#[async_trait::async_trait]
impl RuntimeStore for GatedSnapshotRuntimeStore {
fn session_authority_ops(&self) -> &dyn meerkat_runtime::store::RuntimeSessionAuthorityOps {
self.inner.session_authority_ops()
}
fn session_persistence_profile(
&self,
) -> meerkat_runtime::store::RuntimeSessionPersistenceProfile {
RuntimeStore::session_persistence_profile(&self.inner)
}
fn session_boundary_authority_read_cost(
&self,
) -> meerkat_runtime::store::RuntimeSessionAuthorityReadCost {
self.inner.session_boundary_authority_read_cost()
}
fn supports_compaction_projection_outbox(&self) -> bool {
self.inner.supports_compaction_projection_outbox()
}
fn input_state_batch_cas_implementation_profile(
&self,
) -> meerkat_runtime::store::InputStateBatchCasImplementationProfile {
self.inner.input_state_batch_cas_implementation_profile()
}
async fn load_session_boundary_authority(
&self,
runtime_id: &LogicalRuntimeId,
) -> Result<
Option<meerkat_runtime::store::RuntimeSessionAuthority>,
meerkat_runtime::store::RuntimeStoreError,
> {
self.inner.load_session_boundary_authority(runtime_id).await
}
async fn load_whole_blob_store_authority(
&self,
runtime_id: &LogicalRuntimeId,
) -> Result<Option<WholeBlobStoreAuthority>, meerkat_runtime::store::RuntimeStoreError>
{
self.inner.load_whole_blob_store_authority(runtime_id).await
}
async fn load_committed_whole_blob_snapshot(
&self,
runtime_id: &LogicalRuntimeId,
) -> Result<Option<CommittedWholeBlobSnapshot>, meerkat_runtime::store::RuntimeStoreError>
{
self.inner
.load_committed_whole_blob_snapshot(runtime_id)
.await
}
async fn commit_prepared_whole_blob_snapshot_cas(
&self,
runtime_id: &LogicalRuntimeId,
prepared: meerkat_runtime::store::PreparedWholeBlobSnapshotCas,
) -> Result<
meerkat_runtime::store::WholeBlobSnapshotCasOutcome,
meerkat_runtime::store::RuntimeStoreError,
> {
let ordinal = self
.snapshot_commit_count
.fetch_add(1, Ordering::AcqRel)
.saturating_add(1);
if self.fail_snapshot_commits.load(Ordering::Acquire) {
return Err(meerkat_runtime::store::RuntimeStoreError::WriteFailed(
"synthetic runtime snapshot commit failure".to_string(),
));
}
if self
.fail_snapshot_commit_ordinal
.compare_exchange(ordinal, usize::MAX, Ordering::AcqRel, Ordering::Acquire)
.is_ok()
{
return Err(meerkat_runtime::store::RuntimeStoreError::WriteFailed(
format!("synthetic runtime snapshot commit failure at ordinal {ordinal}"),
));
}
if let Some(interloper) = self
.replace_snapshot_interlopers
.lock()
.await
.remove(runtime_id)
{
self.inner
.commit_session_snapshot(
runtime_id,
SerializedSessionSnapshot {
session_snapshot: interloper.into(),
},
)
.await?;
}
self.inner
.commit_prepared_whole_blob_snapshot_cas(runtime_id, prepared)
.await
}
async fn delete_runtime_session_catalog_entry(
&self,
runtime_id: &LogicalRuntimeId,
) -> Result<(), meerkat_runtime::store::RuntimeStoreError> {
self.inner
.delete_runtime_session_catalog_entry(runtime_id)
.await
}
async fn load_runtime_session_catalog_entry(
&self,
runtime_id: &LogicalRuntimeId,
) -> Result<
Option<meerkat_runtime::store::RuntimeSessionCatalogEntry>,
meerkat_runtime::store::RuntimeStoreError,
> {
self.inner
.load_runtime_session_catalog_entry(runtime_id)
.await
}
async fn list_runtime_session_catalog_entries(
&self,
filter: meerkat_core::SessionFilter,
) -> Result<
Vec<meerkat_runtime::store::RuntimeSessionCatalogEntry>,
meerkat_runtime::store::RuntimeStoreError,
> {
self.inner
.list_runtime_session_catalog_entries(filter)
.await
}
async fn write_prepared_whole_blob_provisional_tail(
&self,
runtime_id: &LogicalRuntimeId,
prepared: PreparedWholeBlobProvisionalTail,
) -> Result<
meerkat_runtime::store::WholeBlobProvisionalTailAuthority,
meerkat_runtime::store::RuntimeStoreError,
> {
self.inner
.write_prepared_whole_blob_provisional_tail(runtime_id, prepared)
.await
}
async fn load_whole_blob_provisional_tail(
&self,
runtime_id: &LogicalRuntimeId,
) -> Result<
Option<meerkat_runtime::store::CommittedWholeBlobProvisionalTail>,
meerkat_runtime::store::RuntimeStoreError,
> {
self.inner
.load_whole_blob_provisional_tail(runtime_id)
.await
}
async fn discard_whole_blob_provisional_tail(
&self,
runtime_id: &LogicalRuntimeId,
expected: &meerkat_runtime::store::WholeBlobProvisionalTailAuthority,
) -> Result<bool, meerkat_runtime::store::RuntimeStoreError> {
self.inner
.discard_whole_blob_provisional_tail(runtime_id, expected)
.await
}
async fn observe_machine_lifecycle(
&self,
runtime_id: &LogicalRuntimeId,
) -> Result<
meerkat_runtime::store::MachineLifecycleObservation,
meerkat_runtime::store::RuntimeStoreError,
> {
self.inner.observe_machine_lifecycle(runtime_id).await
}
async fn compare_and_swap_machine_lifecycle(
&self,
runtime_id: &LogicalRuntimeId,
expected: meerkat_runtime::store::MachineLifecycleExpectedVersion,
replacement: meerkat_runtime::store::MachineLifecycleCommit,
) -> Result<
meerkat_runtime::store::MachineLifecycleCasOutcome,
meerkat_runtime::store::RuntimeStoreError,
> {
self.maybe_pause_machine_lifecycle_commit().await;
if self.fail_machine_lifecycle_commits.load(Ordering::Acquire) {
return Err(meerkat_runtime::store::RuntimeStoreError::WriteFailed(
"synthetic machine lifecycle commit failure".to_string(),
));
}
self.inner
.compare_and_swap_machine_lifecycle(runtime_id, expected, replacement)
.await
}
async fn compare_and_swap_machine_lifecycle_with_fence(
&self,
runtime_id: &LogicalRuntimeId,
expected: meerkat_runtime::store::MachineLifecycleExpectedVersion,
replacement: meerkat_runtime::store::MachineLifecycleCommit,
write_fence: Arc<dyn meerkat_runtime::store::RuntimeStoreWriteFence>,
) -> Result<
meerkat_runtime::store::FencedMachineLifecycleCasOutcome,
meerkat_runtime::store::RuntimeStoreError,
> {
self.maybe_pause_machine_lifecycle_commit().await;
if self.fail_machine_lifecycle_commits.load(Ordering::Acquire) {
return Err(meerkat_runtime::store::RuntimeStoreError::WriteFailed(
"synthetic machine lifecycle commit failure".to_string(),
));
}
self.inner
.compare_and_swap_machine_lifecycle_with_fence(
runtime_id,
expected,
replacement,
write_fence,
)
.await
}
async fn commit_prepared_session_boundary(
&self,
runtime_id: &LogicalRuntimeId,
request: meerkat_runtime::PreparedRuntimeSessionCommit,
) -> Result<
meerkat_runtime::store::PreparedRuntimeSessionCommitResult,
meerkat_runtime::store::RuntimeStoreError,
> {
let ordinal = self
.snapshot_commit_count
.fetch_add(1, Ordering::AcqRel)
.saturating_add(1);
if self.fail_snapshot_commits.load(Ordering::Acquire) {
return Err(meerkat_runtime::store::RuntimeStoreError::WriteFailed(
"synthetic runtime snapshot commit failure".to_string(),
));
}
if self
.fail_snapshot_commit_ordinal
.compare_exchange(ordinal, usize::MAX, Ordering::AcqRel, Ordering::Acquire)
.is_ok()
{
return Err(meerkat_runtime::store::RuntimeStoreError::WriteFailed(
format!("synthetic runtime snapshot commit failure at ordinal {ordinal}"),
));
}
if let Some(interloper) = self
.replace_snapshot_interlopers
.lock()
.await
.remove(runtime_id)
{
self.inner
.commit_session_snapshot(
runtime_id,
SerializedSessionSnapshot {
session_snapshot: interloper.into(),
},
)
.await?;
}
self.inner
.commit_prepared_session_boundary(runtime_id, request)
.await
}
async fn commit_session_snapshot(
&self,
runtime_id: &LogicalRuntimeId,
session_delta: meerkat_runtime::store::SerializedSessionSnapshot,
) -> Result<(), meerkat_runtime::store::RuntimeStoreError> {
let ordinal = self
.snapshot_commit_count
.fetch_add(1, Ordering::AcqRel)
.saturating_add(1);
if self.fail_snapshot_commits.load(Ordering::Acquire) {
return Err(meerkat_runtime::store::RuntimeStoreError::WriteFailed(
"synthetic runtime snapshot commit failure".to_string(),
));
}
if self
.fail_snapshot_commit_ordinal
.compare_exchange(ordinal, usize::MAX, Ordering::AcqRel, Ordering::Acquire)
.is_ok()
{
return Err(meerkat_runtime::store::RuntimeStoreError::WriteFailed(
format!("synthetic runtime snapshot commit failure at ordinal {ordinal}"),
));
}
if let Some(interloper) = self
.replace_snapshot_interlopers
.lock()
.await
.remove(runtime_id)
{
self.inner
.commit_session_snapshot(
runtime_id,
SerializedSessionSnapshot {
session_snapshot: (interloper).into(),
},
)
.await?;
}
self.inner
.commit_session_snapshot(runtime_id, session_delta)
.await
}
async fn commit_prepared_whole_blob_rewrite_boundary(
&self,
runtime_id: &LogicalRuntimeId,
boundary: meerkat_runtime::store::PreparedWholeBlobRewriteStoreParts,
) -> Result<
meerkat_runtime::store::WholeBlobStoreAuthority,
meerkat_runtime::store::RuntimeStoreError,
> {
if self.pause_rewrite_commit.swap(false, Ordering::AcqRel) {
self.entered_rewrite_commit.notify_one();
self.release_rewrite_commit.notified().await;
}
let ordinal = self
.snapshot_commit_count
.fetch_add(1, Ordering::AcqRel)
.saturating_add(1);
if self.fail_snapshot_commits.load(Ordering::Acquire) {
return Err(meerkat_runtime::store::RuntimeStoreError::WriteFailed(
"synthetic runtime snapshot commit failure".to_string(),
));
}
if self
.fail_snapshot_commit_ordinal
.compare_exchange(ordinal, usize::MAX, Ordering::AcqRel, Ordering::Acquire)
.is_ok()
{
return Err(meerkat_runtime::store::RuntimeStoreError::WriteFailed(
format!("synthetic runtime snapshot commit failure at ordinal {ordinal}"),
));
}
if let Some(interloper) = self
.replace_snapshot_interlopers
.lock()
.await
.remove(runtime_id)
{
self.inner
.commit_session_snapshot(
runtime_id,
SerializedSessionSnapshot {
session_snapshot: (interloper).into(),
},
)
.await?;
}
self.inner
.commit_prepared_whole_blob_rewrite_boundary(runtime_id, boundary)
.await
}
async fn atomic_apply(
&self,
runtime_id: &LogicalRuntimeId,
session_delta: Option<meerkat_runtime::store::SerializedSessionSnapshot>,
receipt: meerkat_core::lifecycle::RunBoundaryReceipt,
input_updates: Vec<InputStatePersistenceRecord>,
session_store_key: Option<SessionId>,
) -> Result<(), meerkat_runtime::store::RuntimeStoreError> {
self.inner
.atomic_apply(
runtime_id,
session_delta,
receipt,
input_updates,
session_store_key,
)
.await
}
async fn load_input_states(
&self,
runtime_id: &LogicalRuntimeId,
) -> Result<Vec<meerkat_runtime::InputStateRow>, meerkat_runtime::store::RuntimeStoreError>
{
if self
.input_state_load_errors
.lock()
.await
.contains(runtime_id)
{
return Err(meerkat_runtime::store::RuntimeStoreError::ReadFailed(
"synthetic legacy input-state load failure".to_string(),
));
}
self.inner.load_input_states(runtime_id).await
}
async fn load_input_states_with_versions(
&self,
runtime_id: &LogicalRuntimeId,
) -> Result<
meerkat_runtime::store::PreparedRecoveryInputSnapshot,
meerkat_runtime::store::RuntimeStoreError,
> {
self.inner.load_input_states_with_versions(runtime_id).await
}
async fn load_boundary_receipt(
&self,
runtime_id: &LogicalRuntimeId,
run_id: &RunId,
sequence: u64,
) -> Result<
Option<meerkat_core::lifecycle::RunBoundaryReceipt>,
meerkat_runtime::store::RuntimeStoreError,
> {
self.inner
.load_boundary_receipt(runtime_id, run_id, sequence)
.await
}
async fn load_session_snapshot(
&self,
runtime_id: &LogicalRuntimeId,
) -> Result<Option<Arc<Vec<u8>>>, meerkat_runtime::store::RuntimeStoreError> {
if self.fail_snapshot_loads.load(Ordering::Acquire) {
return Err(meerkat_runtime::store::RuntimeStoreError::ReadFailed(
"synthetic runtime snapshot transport failure".to_string(),
));
}
if self.should_hide_session_snapshot_load() {
return Ok(None);
}
if let Some(snapshot) = self
.session_snapshot_overrides
.lock()
.await
.get(runtime_id)
.cloned()
{
return Ok(Some(snapshot.into()));
}
self.inner.load_session_snapshot(runtime_id).await
}
async fn load_pending_compaction_projections(
&self,
runtime_id: &LogicalRuntimeId,
) -> Result<
Vec<meerkat_core::CompactionProjectionIntent>,
meerkat_runtime::store::RuntimeStoreError,
> {
self.inner
.load_pending_compaction_projections(runtime_id)
.await
}
async fn mark_compaction_projection_finalized(
&self,
runtime_id: &LogicalRuntimeId,
projection: &meerkat_core::CompactionProjectionId,
) -> Result<(), meerkat_runtime::store::RuntimeStoreError> {
self.inner
.mark_compaction_projection_finalized(runtime_id, projection)
.await
}
async fn clear_session_snapshot(
&self,
runtime_id: &LogicalRuntimeId,
) -> Result<(), meerkat_runtime::store::RuntimeStoreError> {
self.session_snapshot_overrides
.lock()
.await
.remove(runtime_id);
self.inner.clear_session_snapshot(runtime_id).await
}
async fn replace_session_snapshot_if_current(
&self,
runtime_id: &LogicalRuntimeId,
expected_current: &[u8],
replacement: Vec<u8>,
) -> Result<bool, meerkat_runtime::store::RuntimeStoreError> {
if self.fail_snapshot_commits.load(Ordering::Acquire) {
return Err(meerkat_runtime::store::RuntimeStoreError::WriteFailed(
"synthetic runtime snapshot commit failure".to_string(),
));
}
if let Some(interloper) = self
.replace_snapshot_interlopers
.lock()
.await
.remove(runtime_id)
{
self.inner
.commit_session_snapshot(
runtime_id,
SerializedSessionSnapshot {
session_snapshot: (interloper).into(),
},
)
.await?;
}
{
let mut overrides = self.session_snapshot_overrides.lock().await;
if let Some(current) = overrides.get_mut(runtime_id) {
if current.as_slice() != expected_current {
return Ok(false);
}
*current = replacement;
return Ok(true);
}
}
self.inner
.replace_session_snapshot_if_current(runtime_id, expected_current, replacement)
.await
}
async fn clear_session_snapshot_if_current(
&self,
runtime_id: &LogicalRuntimeId,
expected_current: &[u8],
) -> Result<bool, meerkat_runtime::store::RuntimeStoreError> {
{
let mut overrides = self.session_snapshot_overrides.lock().await;
if let Some(current) = overrides.get(runtime_id) {
if current.as_slice() != expected_current {
return Ok(false);
}
overrides.remove(runtime_id);
return Ok(true);
}
}
self.inner
.clear_session_snapshot_if_current(runtime_id, expected_current)
.await
}
async fn is_runtime_projection_quarantined(
&self,
runtime_id: &LogicalRuntimeId,
) -> Result<bool, meerkat_runtime::store::RuntimeStoreError> {
self.inner
.is_runtime_projection_quarantined(runtime_id)
.await
}
async fn persist_input_state(
&self,
runtime_id: &LogicalRuntimeId,
state: &InputStatePersistenceRecord,
) -> Result<(), meerkat_runtime::store::RuntimeStoreError> {
self.inner.persist_input_state(runtime_id, state).await
}
async fn persist_input_states_atomically(
&self,
runtime_id: &LogicalRuntimeId,
states: &[InputStatePersistenceRecord],
) -> Result<(), meerkat_runtime::store::RuntimeStoreError> {
self.inner
.persist_input_states_atomically(runtime_id, states)
.await
}
async fn compare_and_swap_input_states_atomically(
&self,
runtime_id: &LogicalRuntimeId,
expected: &[StoredInputState],
replacements: &[InputStatePersistenceRecord],
) -> Result<
meerkat_runtime::store::InputStateBatchCasOutcome,
meerkat_runtime::store::RuntimeStoreError,
> {
self.inner
.compare_and_swap_input_states_atomically(runtime_id, expected, replacements)
.await
}
async fn compare_and_swap_input_states_atomically_with_fence(
&self,
runtime_id: &LogicalRuntimeId,
expected: &[StoredInputState],
replacements: &[InputStatePersistenceRecord],
write_fence: Arc<dyn meerkat_runtime::store::RuntimeStoreWriteFence>,
) -> Result<
meerkat_runtime::store::FencedInputStateBatchCasOutcome,
meerkat_runtime::store::RuntimeStoreError,
> {
self.inner
.compare_and_swap_input_states_atomically_with_fence(
runtime_id,
expected,
replacements,
write_fence,
)
.await
}
async fn compare_and_swap_recovery_input_states_atomically(
&self,
runtime_id: &LogicalRuntimeId,
expected_revision: meerkat_runtime::store::RecoveryInputSetRevision,
mutations: &[meerkat_runtime::store::RecoveryInputStateMutation],
) -> Result<
meerkat_runtime::store::InputStateBatchCasOutcome,
meerkat_runtime::store::RuntimeStoreError,
> {
self.inner
.compare_and_swap_recovery_input_states_atomically(
runtime_id,
expected_revision,
mutations,
)
.await
}
async fn compare_and_swap_recovery_input_states_atomically_with_fence(
&self,
runtime_id: &LogicalRuntimeId,
expected_revision: meerkat_runtime::store::RecoveryInputSetRevision,
mutations: &[meerkat_runtime::store::RecoveryInputStateMutation],
write_fence: Arc<dyn meerkat_runtime::store::RuntimeStoreWriteFence>,
) -> Result<
meerkat_runtime::store::FencedInputStateBatchCasOutcome,
meerkat_runtime::store::RuntimeStoreError,
> {
self.inner
.compare_and_swap_recovery_input_states_atomically_with_fence(
runtime_id,
expected_revision,
mutations,
write_fence,
)
.await
}
async fn load_input_state(
&self,
runtime_id: &LogicalRuntimeId,
input_id: &InputId,
) -> Result<Option<StoredInputState>, meerkat_runtime::store::RuntimeStoreError> {
self.inner.load_input_state(runtime_id, input_id).await
}
async fn load_input_state_by_idempotency_key(
&self,
runtime_id: &LogicalRuntimeId,
key: &meerkat_runtime::IdempotencyKey,
) -> Result<
Option<meerkat_runtime::store::ExactInputStateObservation>,
meerkat_runtime::store::RuntimeStoreError,
> {
self.inner
.load_input_state_by_idempotency_key(runtime_id, key)
.await
}
async fn load_input_states_by_ids(
&self,
runtime_id: &LogicalRuntimeId,
input_ids: &[InputId],
) -> Result<Vec<Option<StoredInputState>>, meerkat_runtime::store::RuntimeStoreError>
{
self.inner
.load_input_states_by_ids(runtime_id, input_ids)
.await
}
async fn load_pending_terminal_owner_ids_page(
&self,
runtime_id: &LogicalRuntimeId,
after: Option<&InputId>,
limit: usize,
) -> Result<Vec<InputId>, meerkat_runtime::store::RuntimeStoreError> {
self.inner
.load_pending_terminal_owner_ids_page(runtime_id, after, limit)
.await
}
async fn load_machine_lifecycle_record(
&self,
runtime_id: &LogicalRuntimeId,
) -> Result<Option<Vec<u8>>, meerkat_runtime::store::RuntimeStoreError> {
self.inner.load_machine_lifecycle_record(runtime_id).await
}
async fn commit_machine_lifecycle(
&self,
runtime_id: &LogicalRuntimeId,
commit: meerkat_runtime::store::MachineLifecycleCommit,
input_states: &[InputStatePersistenceRecord],
) -> Result<(), meerkat_runtime::store::RuntimeStoreError> {
self.maybe_pause_machine_lifecycle_commit().await;
if self.fail_machine_lifecycle_commits.load(Ordering::Acquire) {
return Err(meerkat_runtime::store::RuntimeStoreError::WriteFailed(
"synthetic machine lifecycle commit failure".to_string(),
));
}
self.inner
.commit_machine_lifecycle(runtime_id, commit, input_states)
.await
}
async fn commit_unregister_finalization(
&self,
runtime_id: &LogicalRuntimeId,
finalization: meerkat_runtime::store::UnregisterFinalizationCommit,
) -> Result<(), meerkat_runtime::store::RuntimeStoreError> {
self.maybe_pause_machine_lifecycle_commit().await;
if self.fail_machine_lifecycle_commits.load(Ordering::Acquire) {
return Err(meerkat_runtime::store::RuntimeStoreError::WriteFailed(
"synthetic machine lifecycle commit failure".to_string(),
));
}
self.inner
.commit_unregister_finalization(runtime_id, finalization)
.await
}
async fn persist_ops_lifecycle(
&self,
runtime_id: &LogicalRuntimeId,
snapshot: &meerkat_runtime::ops_lifecycle::PersistedOpsSnapshot,
) -> Result<(), meerkat_runtime::store::RuntimeStoreError> {
self.inner.persist_ops_lifecycle(runtime_id, snapshot).await
}
async fn initialize_ops_lifecycle_if_absent(
&self,
runtime_id: &LogicalRuntimeId,
candidate: &meerkat_runtime::ops_lifecycle::PersistedOpsSnapshot,
) -> Result<
meerkat_runtime::ops_lifecycle::PersistedOpsSnapshot,
meerkat_runtime::store::RuntimeStoreError,
> {
self.inner
.initialize_ops_lifecycle_if_absent(runtime_id, candidate)
.await
}
async fn load_ops_lifecycle(
&self,
runtime_id: &LogicalRuntimeId,
) -> Result<
Option<meerkat_runtime::ops_lifecycle::PersistedOpsSnapshot>,
meerkat_runtime::store::RuntimeStoreError,
> {
self.inner.load_ops_lifecycle(runtime_id).await
}
}
#[tokio::test]
async fn runtime_snapshot_corruption_remains_typed_serialization() {
let gated = Arc::new(GatedSnapshotRuntimeStore::new());
let runtime_id = LogicalRuntimeId::for_session(&SessionId::new());
gated
.override_session_snapshot(runtime_id.clone(), b"{ not a serialized Session".to_vec())
.await;
let runtime_store: Arc<dyn RuntimeStore> = gated;
let error = PersistentSessionService::<DummyBuilder>::load_runtime_session_snapshot(
&runtime_store,
&runtime_id,
)
.await
.expect_err("malformed runtime snapshot bytes must fail decode");
match error {
SessionError::Store(source) => assert!(
matches!(
source.downcast_ref::<SessionStoreError>(),
Some(SessionStoreError::Serialization(_))
),
"malformed runtime snapshot bytes must remain a typed serialization error, got {source:?}"
),
other => panic!("expected typed SessionError::Store, got {other:?}"),
}
}
#[tokio::test]
async fn runtime_snapshot_transport_failure_remains_internal_and_distinct() {
let gated = Arc::new(GatedSnapshotRuntimeStore::new());
gated.set_fail_snapshot_loads(true);
let runtime_store: Arc<dyn RuntimeStore> = gated;
let runtime_id = LogicalRuntimeId::for_session(&SessionId::new());
let error = PersistentSessionService::<DummyBuilder>::load_runtime_session_snapshot(
&runtime_store,
&runtime_id,
)
.await
.expect_err("transport failure must fail runtime snapshot load");
assert!(
matches!(error, SessionError::Agent(AgentError::InternalError(_))),
"RuntimeStore transport failure must remain distinct from persisted serialization corruption, got {error:?}"
);
}
fn test_transient_turn_context_state_handle() -> meerkat_core::TransientTurnContextStateHandle {
meerkat_core::TransientTurnContextStateHandle::new()
}
struct DummyAgent {
session: Arc<std::sync::Mutex<Session>>,
transient_turn_context_state: meerkat_core::TransientTurnContextStateHandle,
run_failure: Option<String>,
flow_overlay_failure: Option<String>,
callback_pending_after_run: bool,
execution_snapshot: Option<meerkat_core::AgentExecutionSnapshot>,
compaction_abort_failure: Option<String>,
}
#[async_trait::async_trait]
impl SessionAgent for DummyAgent {
async fn run_with_events(
&mut self,
prompt: meerkat_core::types::ContentInput,
_event_tx: tokio::sync::mpsc::Sender<meerkat_core::event::AgentEvent>,
) -> Result<RunResult, meerkat_core::error::AgentError> {
if let Some(message) = self.run_failure.clone() {
if self
.execution_snapshot
.as_ref()
.is_some_and(|snapshot| snapshot.turn_terminal)
{
let mut session = self
.session
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
session.push(meerkat_core::types::Message::User(
meerkat_core::types::UserMessage::text(prompt.text_content()),
));
}
return Err(meerkat_core::error::AgentError::InternalError(message));
}
let session_id = self.session_id();
let result = {
let mut session = match self.session.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
session.push(meerkat_core::types::Message::User(
meerkat_core::types::UserMessage::text(prompt.text_content()),
));
session.push(meerkat_core::types::Message::BlockAssistant(
meerkat_core::types::BlockAssistantMessage {
blocks: vec![meerkat_core::types::AssistantBlock::Text {
text: "ok".to_string(),
meta: None,
}],
stop_reason: meerkat_core::types::StopReason::EndTurn,
identity: meerkat_core::types::TranscriptMessageIdentity::default(),
created_at: meerkat_core::types::message_timestamp_now(),
},
));
RunResult {
text: "ok".to_string(),
session_id,
usage: meerkat_core::types::Usage::default(),
turns: 1,
tool_calls: 0,
terminal_cause_kind: None,
structured_output: None,
extraction_error: None,
schema_warnings: None,
skill_diagnostics: None,
}
};
if self.callback_pending_after_run {
return Err(meerkat_core::error::AgentError::CallbackPending {
tool_use_id: "test-call".to_string(),
tool_name: "test_callback".to_string(),
args: serde_json::json!({}),
});
}
Ok(result)
}
async fn abort_uncommitted_compaction_projections(
&mut self,
) -> Result<(), meerkat_core::error::AgentError> {
match self.compaction_abort_failure.take() {
Some(message) => Err(meerkat_core::error::AgentError::InternalError(message)),
None => Ok(()),
}
}
fn take_runtime_terminal_failure_witness(
&mut self,
) -> Result<Option<meerkat_core::TurnErrorMetadata>, meerkat_core::error::AgentError>
{
let Some(snapshot) = self.execution_snapshot.as_ref() else {
return Ok(None);
};
if !snapshot.turn_terminal {
return Ok(None);
}
let Some(cause_kind) = snapshot.terminal_cause_kind else {
return Err(meerkat_core::error::AgentError::InternalError(
"dummy terminal snapshot omitted cause".to_string(),
));
};
Ok(Some(meerkat_core::TurnErrorMetadata::terminal(
cause_kind,
snapshot.terminal_outcome,
self.run_failure
.clone()
.unwrap_or_else(|| "dummy terminal failure".to_string()),
)))
}
fn set_skill_references(&mut self, _refs: Option<Vec<meerkat_core::skills::SkillKey>>) {}
fn set_turn_tool_overlay(
&mut self,
_overlay: Option<meerkat_core::service::TurnToolOverlay>,
) -> Result<(), meerkat_core::error::AgentError> {
if let Some(message) = self.flow_overlay_failure.clone() {
return Err(meerkat_core::error::AgentError::InternalError(message));
}
Ok(())
}
fn cancel(&mut self) {}
fn hot_swap_llm_identity(
&mut self,
_client: Arc<dyn meerkat_core::AgentLlmClient>,
identity: meerkat_core::SessionLlmIdentity,
_request_policy: meerkat_core::SessionLlmRequestPolicy,
) -> Result<(), meerkat_core::error::AgentError> {
let mut session = match self.session.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
let mut metadata =
session
.session_metadata()
.unwrap_or(meerkat_core::SessionMetadata {
schema_version: meerkat_core::SESSION_METADATA_SCHEMA_VERSION,
model: identity.model.clone(),
max_tokens: 0,
structured_output_retries: 2,
provider: identity.provider,
self_hosted_server_id: None,
provider_params: identity.provider_params.clone(),
tooling: meerkat_core::SessionTooling::default(),
keep_alive: false,
comms_name: None,
peer_meta: None,
realm_id: None,
instance_id: None,
backend: None,
config_generation: None,
auth_binding: None,
mob_member_binding: None,
});
metadata.apply_llm_identity(&identity);
session.set_session_metadata(metadata).map_err(|err| {
meerkat_core::error::AgentError::InternalError(format!(
"failed to update dummy session metadata: {err}"
))
})
}
fn stage_external_tool_filter(
&mut self,
filter: meerkat_core::ToolFilter,
) -> Result<(), meerkat_core::error::AgentError> {
let mut state = {
let session = match self.session.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
session
.tool_visibility_state()
.map_err(|err| {
meerkat_core::error::AgentError::InternalError(format!(
"failed to decode dummy visibility state: {err}"
))
})?
.unwrap_or_default()
};
state.staged_filter = filter;
state.staged_revision = state.staged_revision.max(state.active_revision) + 1;
self.set_tool_visibility_state(Some(state))
}
fn set_tool_visibility_state(
&mut self,
state: Option<meerkat_core::SessionToolVisibilityState>,
) -> Result<(), meerkat_core::error::AgentError> {
let _ = state;
Ok(())
}
fn session_id(&self) -> SessionId {
match self.session.lock() {
Ok(guard) => guard.id().clone(),
Err(poisoned) => poisoned.into_inner().id().clone(),
}
}
fn snapshot(&self) -> SessionSnapshot {
let session = match self.session.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
SessionSnapshot {
created_at: session.created_at(),
updated_at: session.updated_at(),
message_count: session.messages().len(),
total_tokens: session.total_tokens(),
usage: session.total_usage(),
last_assistant_text: session.last_assistant_text(),
}
}
fn execution_snapshot(
&self,
) -> Result<
Option<meerkat_core::AgentExecutionSnapshot>,
meerkat_core::SnapshotProjectionError,
> {
Ok(self.execution_snapshot.clone())
}
fn session_clone(&self) -> Result<Session, AgentError> {
match self.session.lock() {
Ok(guard) => Ok(guard.clone()),
Err(poisoned) => Ok(poisoned.into_inner().clone()),
}
}
fn session_transcript_authority(
&self,
) -> Result<SessionTranscriptAuthoritySnapshot, AgentError> {
let session = self.session_clone()?;
SessionTranscriptAuthoritySnapshot::from_session(&session)
}
fn durable_llm_identity(&self) -> Option<meerkat_core::SessionLlmIdentity> {
let session = match self.session.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
Some(test_durable_llm_identity(&session))
}
fn observed_session_tail(&self) -> ObservedSessionTailKind {
let session = match self.session.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
meerkat_core::pending_continuation::observe_session_tail(session.messages())
}
fn update_keep_alive(&mut self, keep_alive: bool) {
let mut session = match self.session.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
if let Some(mut meta) = session.session_metadata() {
meta.keep_alive = keep_alive;
let _ = session.set_session_metadata(meta);
}
}
fn update_mob_tool_authority_context(
&mut self,
authority_context: Option<MobToolAuthorityContext>,
) -> Result<(), meerkat_core::error::AgentError> {
let mut session = match self.session.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
session
.set_mob_tool_authority_context(authority_context)
.map_err(|err| {
meerkat_core::error::AgentError::InternalError(format!(
"failed to update dummy mob authority context: {err}"
))
})
}
fn append_system_messages(
&mut self,
contents: Vec<String>,
) -> Result<(), meerkat_core::error::AgentError> {
let mut session = match self.session.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
for content in contents {
session.append_system_message(content);
}
Ok(())
}
fn append_system_message_control(
&mut self,
req: AppendSystemContextRequest,
) -> Result<meerkat_core::service::AppendSystemContextStatus, AgentError> {
let mut session = self
.session
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
session
.append_system_message_idempotent(
req.content.render_text(),
req.source,
req.idempotency_key,
meerkat_core::types::message_timestamp_now(),
)
.map_err(|error| AgentError::ConfigError(error.to_string()))
}
fn transient_turn_context_state(&self) -> meerkat_core::TransientTurnContextStateHandle {
self.transient_turn_context_state.clone()
}
fn sync_session_from_durable_snapshot(
&mut self,
session: Session,
) -> Result<(), meerkat_core::error::AgentError> {
if session.id() != &self.session_id() {
return Err(meerkat_core::error::AgentError::InternalError(format!(
"durable snapshot session id {} does not match live session {}",
session.id(),
self.session_id()
)));
}
match self.session.lock() {
Ok(mut guard) => {
*guard = session;
}
Err(poisoned) => {
*poisoned.into_inner() = session;
}
}
Ok(())
}
}
struct EventfulDummyAgent {
inner: DummyAgent,
}
#[async_trait::async_trait]
impl SessionAgent for EventfulDummyAgent {
async fn run_with_events(
&mut self,
prompt: meerkat_core::types::ContentInput,
event_tx: tokio::sync::mpsc::Sender<meerkat_core::event::AgentEvent>,
) -> Result<RunResult, meerkat_core::error::AgentError> {
let session_id = self.inner.session_id();
let _ = event_tx
.send(AgentEvent::RunStarted {
session_id: session_id.clone(),
input: meerkat_core::types::RunInput::Content {
content: prompt.clone(),
},
})
.await;
let result = self.inner.run_with_events(prompt, event_tx.clone()).await?;
let _ = event_tx
.send(AgentEvent::RunCompleted {
session_id,
result: result.text.clone(),
structured_output: result.structured_output.clone(),
extraction_required: false,
usage: result.usage.clone(),
terminal_cause_kind: result.terminal_cause_kind,
})
.await;
Ok(result)
}
fn set_skill_references(&mut self, refs: Option<Vec<meerkat_core::skills::SkillKey>>) {
self.inner.set_skill_references(refs);
}
fn set_turn_tool_overlay(
&mut self,
overlay: Option<meerkat_core::service::TurnToolOverlay>,
) -> Result<(), meerkat_core::error::AgentError> {
self.inner.set_turn_tool_overlay(overlay)
}
fn cancel(&mut self) {
self.inner.cancel();
}
fn hot_swap_llm_identity(
&mut self,
client: Arc<dyn meerkat_core::AgentLlmClient>,
identity: meerkat_core::SessionLlmIdentity,
request_policy: meerkat_core::SessionLlmRequestPolicy,
) -> Result<(), meerkat_core::error::AgentError> {
self.inner
.hot_swap_llm_identity(client, identity, request_policy)
}
fn stage_external_tool_filter(
&mut self,
filter: meerkat_core::ToolFilter,
) -> Result<(), meerkat_core::error::AgentError> {
self.inner.stage_external_tool_filter(filter)
}
fn set_tool_visibility_state(
&mut self,
state: Option<meerkat_core::SessionToolVisibilityState>,
) -> Result<(), meerkat_core::error::AgentError> {
self.inner.set_tool_visibility_state(state)
}
fn session_id(&self) -> SessionId {
self.inner.session_id()
}
fn snapshot(&self) -> SessionSnapshot {
self.inner.snapshot()
}
fn execution_snapshot(
&self,
) -> Result<
Option<meerkat_core::AgentExecutionSnapshot>,
meerkat_core::SnapshotProjectionError,
> {
self.inner.execution_snapshot()
}
fn session_clone(&self) -> Result<Session, AgentError> {
self.inner.session_clone()
}
fn session_transcript_authority(
&self,
) -> Result<SessionTranscriptAuthoritySnapshot, AgentError> {
self.inner.session_transcript_authority()
}
fn durable_llm_identity(&self) -> Option<meerkat_core::SessionLlmIdentity> {
self.inner.durable_llm_identity()
}
fn observed_session_tail(&self) -> ObservedSessionTailKind {
self.inner.observed_session_tail()
}
fn update_keep_alive(&mut self, keep_alive: bool) {
self.inner.update_keep_alive(keep_alive);
}
fn update_mob_tool_authority_context(
&mut self,
authority_context: Option<MobToolAuthorityContext>,
) -> Result<(), meerkat_core::error::AgentError> {
self.inner
.update_mob_tool_authority_context(authority_context)
}
fn append_system_messages(
&mut self,
contents: Vec<String>,
) -> Result<(), meerkat_core::error::AgentError> {
self.inner.append_system_messages(contents)
}
fn transient_turn_context_state(&self) -> meerkat_core::TransientTurnContextStateHandle {
self.inner.transient_turn_context_state()
}
}
struct NoopSubscribableInjector;
impl meerkat_core::EventInjector for NoopSubscribableInjector {
fn inject(
&self,
_content: meerkat_core::types::ContentInput,
_source: meerkat_core::PlainEventSource,
_handling_mode: meerkat_core::types::HandlingMode,
_render_metadata: Option<meerkat_core::types::RenderMetadata>,
) -> Result<(), meerkat_core::event_injector::EventInjectorError> {
Ok(())
}
}
impl meerkat_core::event_injector::SubscribableInjector for NoopSubscribableInjector {
fn inject_with_delivery_identity(
&self,
_input_identity: meerkat_core::service::StartTurnInputIdentity,
_objective_id: Option<meerkat_core::interaction::ObjectiveId>,
_content: meerkat_core::types::ContentInput,
_source: meerkat_core::PlainEventSource,
_handling_mode: meerkat_core::types::HandlingMode,
_render_metadata: Option<meerkat_core::types::RenderMetadata>,
) -> Result<(), meerkat_core::event_injector::EventInjectorError> {
Err(meerkat_core::event_injector::EventInjectorError::Closed)
}
fn inject_with_subscription(
&self,
_content: meerkat_core::types::ContentInput,
_source: meerkat_core::PlainEventSource,
_handling_mode: meerkat_core::types::HandlingMode,
_render_metadata: Option<meerkat_core::types::RenderMetadata>,
) -> Result<
meerkat_core::event_injector::InteractionSubscription,
meerkat_core::event_injector::EventInjectorError,
> {
let (_tx, events) = tokio::sync::mpsc::channel(1);
Ok(meerkat_core::event_injector::InteractionSubscription {
id: serde_json::from_str("\"00000000-0000-0000-0000-000000000001\"")
.expect("static interaction id should deserialize"),
events,
})
}
fn inject_with_interaction_id(
&self,
_interaction_id: meerkat_core::InteractionId,
content: meerkat_core::types::ContentInput,
source: meerkat_core::PlainEventSource,
handling_mode: meerkat_core::types::HandlingMode,
render_metadata: Option<meerkat_core::types::RenderMetadata>,
) -> Result<(), meerkat_core::event_injector::EventInjectorError> {
meerkat_core::EventInjector::inject(
self,
content,
source,
handling_mode,
render_metadata,
)
}
}
struct NoopCommsRuntime {
notify: Arc<tokio::sync::Notify>,
injector: Arc<NoopSubscribableInjector>,
}
impl NoopCommsRuntime {
fn new(injector: Arc<NoopSubscribableInjector>) -> Self {
Self {
notify: Arc::new(tokio::sync::Notify::new()),
injector,
}
}
}
#[async_trait::async_trait]
impl meerkat_core::agent::CommsRuntime for NoopCommsRuntime {
async fn drain_messages(&self) -> Vec<String> {
Vec::new()
}
fn inbox_notify(&self) -> Arc<tokio::sync::Notify> {
Arc::clone(&self.notify)
}
fn interaction_event_injector(
&self,
) -> Option<Arc<dyn meerkat_core::event_injector::SubscribableInjector>> {
let injector: Arc<dyn meerkat_core::event_injector::SubscribableInjector> =
self.injector.clone();
Some(injector)
}
}
struct CapabilityAgent {
inner: DummyAgent,
comms: Arc<NoopCommsRuntime>,
injector: Arc<NoopSubscribableInjector>,
}
#[async_trait::async_trait]
impl SessionAgent for CapabilityAgent {
async fn run_with_events(
&mut self,
prompt: meerkat_core::types::ContentInput,
event_tx: tokio::sync::mpsc::Sender<meerkat_core::event::AgentEvent>,
) -> Result<RunResult, meerkat_core::error::AgentError> {
self.inner.run_with_events(prompt, event_tx).await
}
fn set_skill_references(&mut self, refs: Option<Vec<meerkat_core::skills::SkillKey>>) {
self.inner.set_skill_references(refs);
}
fn set_turn_tool_overlay(
&mut self,
overlay: Option<meerkat_core::service::TurnToolOverlay>,
) -> Result<(), meerkat_core::error::AgentError> {
self.inner.set_turn_tool_overlay(overlay)
}
fn cancel(&mut self) {
self.inner.cancel();
}
fn hot_swap_llm_identity(
&mut self,
client: Arc<dyn meerkat_core::AgentLlmClient>,
identity: meerkat_core::SessionLlmIdentity,
request_policy: meerkat_core::SessionLlmRequestPolicy,
) -> Result<(), meerkat_core::error::AgentError> {
self.inner
.hot_swap_llm_identity(client, identity, request_policy)
}
fn stage_external_tool_filter(
&mut self,
filter: meerkat_core::ToolFilter,
) -> Result<(), meerkat_core::error::AgentError> {
self.inner.stage_external_tool_filter(filter)
}
fn set_tool_visibility_state(
&mut self,
state: Option<meerkat_core::SessionToolVisibilityState>,
) -> Result<(), meerkat_core::error::AgentError> {
self.inner.set_tool_visibility_state(state)
}
fn session_id(&self) -> SessionId {
self.inner.session_id()
}
fn snapshot(&self) -> SessionSnapshot {
self.inner.snapshot()
}
fn session_clone(&self) -> Result<Session, AgentError> {
self.inner.session_clone()
}
fn session_transcript_authority(
&self,
) -> Result<SessionTranscriptAuthoritySnapshot, AgentError> {
self.inner.session_transcript_authority()
}
fn durable_llm_identity(&self) -> Option<meerkat_core::SessionLlmIdentity> {
self.inner.durable_llm_identity()
}
fn observed_session_tail(&self) -> ObservedSessionTailKind {
self.inner.observed_session_tail()
}
fn update_keep_alive(&mut self, keep_alive: bool) {
self.inner.update_keep_alive(keep_alive);
}
fn update_mob_tool_authority_context(
&mut self,
authority_context: Option<MobToolAuthorityContext>,
) -> Result<(), meerkat_core::error::AgentError> {
self.inner
.update_mob_tool_authority_context(authority_context)
}
fn append_system_messages(
&mut self,
contents: Vec<String>,
) -> Result<(), meerkat_core::error::AgentError> {
self.inner.append_system_messages(contents)
}
fn transient_turn_context_state(&self) -> meerkat_core::TransientTurnContextStateHandle {
self.inner.transient_turn_context_state()
}
fn sync_session_from_durable_snapshot(
&mut self,
session: Session,
) -> Result<(), meerkat_core::error::AgentError> {
self.inner.sync_session_from_durable_snapshot(session)
}
fn append_realtime_transcript_event(
&mut self,
event: meerkat_core::RealtimeTranscriptEvent,
) -> Result<meerkat_core::RealtimeTranscriptApplyOutcome, meerkat_core::error::AgentError>
{
let mut session = match self.inner.session.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
Ok(session.append_realtime_transcript_event(event))
}
fn interaction_event_injector(
&self,
) -> Option<Arc<dyn meerkat_core::event_injector::SubscribableInjector>> {
let injector: Arc<dyn meerkat_core::event_injector::SubscribableInjector> =
self.injector.clone();
Some(injector)
}
fn comms_runtime(&self) -> Option<Arc<dyn meerkat_core::agent::CommsRuntime>> {
let comms: Arc<dyn meerkat_core::agent::CommsRuntime> = self.comms.clone();
Some(comms)
}
}
struct CapabilityBuilder;
#[async_trait::async_trait]
impl SessionAgentBuilder for CapabilityBuilder {
type Agent = CapabilityAgent;
async fn build_agent(
&self,
req: &CreateSessionRequest,
_event_tx: tokio::sync::mpsc::Sender<meerkat_core::event::AgentEvent>,
) -> Result<Self::Agent, SessionError> {
let session = req
.build
.as_ref()
.and_then(|build| build.resume_session.clone())
.unwrap_or_default();
let injector = Arc::new(NoopSubscribableInjector);
let comms = Arc::new(NoopCommsRuntime::new(Arc::clone(&injector)));
Ok(CapabilityAgent {
inner: DummyAgent {
session: Arc::new(std::sync::Mutex::new(session)),
transient_turn_context_state: test_transient_turn_context_state_handle(),
run_failure: None,
flow_overlay_failure: None,
callback_pending_after_run: false,
execution_snapshot: None,
compaction_abort_failure: None,
},
comms,
injector,
})
}
}
struct DummyBuilder;
#[async_trait::async_trait]
impl SessionAgentBuilder for DummyBuilder {
type Agent = DummyAgent;
async fn build_agent(
&self,
req: &CreateSessionRequest,
_event_tx: tokio::sync::mpsc::Sender<meerkat_core::event::AgentEvent>,
) -> Result<Self::Agent, SessionError> {
let session = req
.build
.as_ref()
.and_then(|build| build.resume_session.clone())
.unwrap_or_default();
Ok(DummyAgent {
session: Arc::new(std::sync::Mutex::new(session)),
transient_turn_context_state: test_transient_turn_context_state_handle(),
run_failure: None,
flow_overlay_failure: None,
callback_pending_after_run: false,
execution_snapshot: None,
compaction_abort_failure: None,
})
}
}
struct FailOnceCompactionAbortBuilder;
#[async_trait::async_trait]
impl SessionAgentBuilder for FailOnceCompactionAbortBuilder {
type Agent = DummyAgent;
async fn build_agent(
&self,
req: &CreateSessionRequest,
_event_tx: tokio::sync::mpsc::Sender<meerkat_core::event::AgentEvent>,
) -> Result<Self::Agent, SessionError> {
let session = req
.build
.as_ref()
.and_then(|build| build.resume_session.clone())
.unwrap_or_default();
Ok(DummyAgent {
session: Arc::new(std::sync::Mutex::new(session)),
transient_turn_context_state: test_transient_turn_context_state_handle(),
run_failure: None,
flow_overlay_failure: None,
callback_pending_after_run: false,
execution_snapshot: None,
compaction_abort_failure: Some(
"synthetic rejected-run compaction abort failure".to_string(),
),
})
}
}
struct CountingBuilder {
builds: Arc<AtomicUsize>,
}
#[async_trait::async_trait]
impl SessionAgentBuilder for CountingBuilder {
type Agent = DummyAgent;
async fn build_agent(
&self,
req: &CreateSessionRequest,
_event_tx: tokio::sync::mpsc::Sender<meerkat_core::event::AgentEvent>,
) -> Result<Self::Agent, SessionError> {
self.builds.fetch_add(1, Ordering::AcqRel);
let session = req
.build
.as_ref()
.and_then(|build| build.resume_session.clone())
.unwrap_or_default();
Ok(DummyAgent {
session: Arc::new(std::sync::Mutex::new(session)),
transient_turn_context_state: test_transient_turn_context_state_handle(),
run_failure: None,
flow_overlay_failure: None,
callback_pending_after_run: false,
execution_snapshot: None,
compaction_abort_failure: None,
})
}
}
struct CallbackPendingBuilder;
#[async_trait::async_trait]
impl SessionAgentBuilder for CallbackPendingBuilder {
type Agent = DummyAgent;
async fn build_agent(
&self,
req: &CreateSessionRequest,
_event_tx: tokio::sync::mpsc::Sender<meerkat_core::event::AgentEvent>,
) -> Result<Self::Agent, SessionError> {
let session = req
.build
.as_ref()
.and_then(|build| build.resume_session.clone())
.unwrap_or_default();
Ok(DummyAgent {
session: Arc::new(std::sync::Mutex::new(session)),
transient_turn_context_state: test_transient_turn_context_state_handle(),
run_failure: None,
flow_overlay_failure: None,
callback_pending_after_run: true,
execution_snapshot: None,
compaction_abort_failure: None,
})
}
}
struct FailingOverlayBuilder;
#[async_trait::async_trait]
impl SessionAgentBuilder for FailingOverlayBuilder {
type Agent = DummyAgent;
async fn build_agent(
&self,
req: &CreateSessionRequest,
_event_tx: tokio::sync::mpsc::Sender<meerkat_core::event::AgentEvent>,
) -> Result<Self::Agent, SessionError> {
let session = req
.build
.as_ref()
.and_then(|build| build.resume_session.clone())
.unwrap_or_default();
Ok(DummyAgent {
session: Arc::new(std::sync::Mutex::new(session)),
transient_turn_context_state: test_transient_turn_context_state_handle(),
run_failure: None,
flow_overlay_failure: Some("synthetic flow overlay failure".to_string()),
callback_pending_after_run: false,
execution_snapshot: None,
compaction_abort_failure: None,
})
}
}
#[derive(Clone)]
struct BlockingBuildBuilder {
entered_builds: Arc<AtomicUsize>,
max_concurrent_builds: Arc<AtomicUsize>,
active_builds: Arc<AtomicUsize>,
entered_notify: Arc<tokio::sync::Notify>,
release_notify: Arc<tokio::sync::Semaphore>,
}
impl BlockingBuildBuilder {
fn new() -> Self {
Self {
entered_builds: Arc::new(AtomicUsize::new(0)),
max_concurrent_builds: Arc::new(AtomicUsize::new(0)),
active_builds: Arc::new(AtomicUsize::new(0)),
entered_notify: Arc::new(tokio::sync::Notify::new()),
release_notify: Arc::new(tokio::sync::Semaphore::new(0)),
}
}
fn record_build_start(&self) {
self.entered_builds.fetch_add(1, Ordering::AcqRel);
let active = self.active_builds.fetch_add(1, Ordering::AcqRel) + 1;
let mut observed = self.max_concurrent_builds.load(Ordering::Acquire);
while active > observed {
match self.max_concurrent_builds.compare_exchange(
observed,
active,
Ordering::AcqRel,
Ordering::Acquire,
) {
Ok(_) => break,
Err(current) => observed = current,
}
}
self.entered_notify.notify_waiters();
}
fn record_build_finish(&self) {
self.active_builds.fetch_sub(1, Ordering::AcqRel);
}
async fn wait_for_entered_builds(&self, expected: usize) {
tokio::time::timeout(std::time::Duration::from_secs(10), async {
loop {
if self.entered_builds.load(Ordering::Acquire) >= expected {
return;
}
self.entered_notify.notified().await;
}
})
.await
.unwrap_or_else(|_| {
panic!(
"blocking build builder did not enter {expected} build(s); observed {}",
self.entered_builds.load(Ordering::Acquire)
)
});
}
}
#[async_trait::async_trait]
impl SessionAgentBuilder for BlockingBuildBuilder {
type Agent = DummyAgent;
async fn build_agent(
&self,
req: &CreateSessionRequest,
_event_tx: tokio::sync::mpsc::Sender<meerkat_core::event::AgentEvent>,
) -> Result<Self::Agent, SessionError> {
self.record_build_start();
self.release_notify
.acquire()
.await
.expect("blocking build release semaphore should stay open")
.forget();
self.record_build_finish();
let session = req
.build
.as_ref()
.and_then(|build| build.resume_session.clone())
.unwrap_or_default();
Ok(DummyAgent {
session: Arc::new(std::sync::Mutex::new(session)),
transient_turn_context_state: test_transient_turn_context_state_handle(),
run_failure: None,
flow_overlay_failure: None,
callback_pending_after_run: false,
execution_snapshot: None,
compaction_abort_failure: None,
})
}
}
#[derive(Clone)]
struct BlockingRunBuilder {
entered_runs: Arc<AtomicUsize>,
entered_notify: Arc<tokio::sync::Notify>,
release_notify: Arc<tokio::sync::Semaphore>,
}
impl BlockingRunBuilder {
fn new() -> Self {
Self {
entered_runs: Arc::new(AtomicUsize::new(0)),
entered_notify: Arc::new(tokio::sync::Notify::new()),
release_notify: Arc::new(tokio::sync::Semaphore::new(0)),
}
}
async fn wait_for_entered_runs(&self, expected: usize) {
tokio::time::timeout(std::time::Duration::from_secs(10), async {
loop {
if self.entered_runs.load(Ordering::Acquire) >= expected {
return;
}
self.entered_notify.notified().await;
}
})
.await
.unwrap_or_else(|_| {
panic!(
"blocking run builder did not enter {expected} run(s); observed {}",
self.entered_runs.load(Ordering::Acquire)
)
});
}
}
struct BlockingRunAgent {
inner: DummyAgent,
entered_runs: Arc<AtomicUsize>,
entered_notify: Arc<tokio::sync::Notify>,
release_notify: Arc<tokio::sync::Semaphore>,
}
#[async_trait::async_trait]
impl SessionAgentBuilder for BlockingRunBuilder {
type Agent = BlockingRunAgent;
async fn build_agent(
&self,
req: &CreateSessionRequest,
_event_tx: tokio::sync::mpsc::Sender<meerkat_core::event::AgentEvent>,
) -> Result<Self::Agent, SessionError> {
let session = req
.build
.as_ref()
.and_then(|build| build.resume_session.clone())
.unwrap_or_default();
Ok(BlockingRunAgent {
inner: DummyAgent {
session: Arc::new(std::sync::Mutex::new(session)),
transient_turn_context_state: test_transient_turn_context_state_handle(),
run_failure: None,
flow_overlay_failure: None,
callback_pending_after_run: false,
execution_snapshot: None,
compaction_abort_failure: None,
},
entered_runs: Arc::clone(&self.entered_runs),
entered_notify: Arc::clone(&self.entered_notify),
release_notify: Arc::clone(&self.release_notify),
})
}
}
#[async_trait::async_trait]
impl SessionAgent for BlockingRunAgent {
async fn run_with_events(
&mut self,
prompt: meerkat_core::types::ContentInput,
event_tx: tokio::sync::mpsc::Sender<meerkat_core::event::AgentEvent>,
) -> Result<RunResult, meerkat_core::error::AgentError> {
self.entered_runs.fetch_add(1, Ordering::AcqRel);
self.entered_notify.notify_waiters();
self.release_notify
.acquire()
.await
.expect("blocking run release semaphore should stay open")
.forget();
self.inner.run_with_events(prompt, event_tx).await
}
fn set_skill_references(&mut self, refs: Option<Vec<meerkat_core::skills::SkillKey>>) {
self.inner.set_skill_references(refs);
}
fn set_turn_tool_overlay(
&mut self,
overlay: Option<meerkat_core::service::TurnToolOverlay>,
) -> Result<(), meerkat_core::error::AgentError> {
self.inner.set_turn_tool_overlay(overlay)
}
fn cancel(&mut self) {
self.inner.cancel();
}
fn hot_swap_llm_identity(
&mut self,
client: Arc<dyn meerkat_core::AgentLlmClient>,
identity: meerkat_core::SessionLlmIdentity,
request_policy: meerkat_core::SessionLlmRequestPolicy,
) -> Result<(), meerkat_core::error::AgentError> {
self.inner
.hot_swap_llm_identity(client, identity, request_policy)
}
fn stage_external_tool_filter(
&mut self,
filter: meerkat_core::ToolFilter,
) -> Result<(), meerkat_core::error::AgentError> {
self.inner.stage_external_tool_filter(filter)
}
fn set_tool_visibility_state(
&mut self,
state: Option<meerkat_core::SessionToolVisibilityState>,
) -> Result<(), meerkat_core::error::AgentError> {
self.inner.set_tool_visibility_state(state)
}
fn session_id(&self) -> SessionId {
self.inner.session_id()
}
fn snapshot(&self) -> SessionSnapshot {
self.inner.snapshot()
}
fn session_clone(&self) -> Result<Session, AgentError> {
self.inner.session_clone()
}
fn session_transcript_authority(
&self,
) -> Result<SessionTranscriptAuthoritySnapshot, AgentError> {
self.inner.session_transcript_authority()
}
fn durable_llm_identity(&self) -> Option<meerkat_core::SessionLlmIdentity> {
self.inner.durable_llm_identity()
}
fn observed_session_tail(&self) -> ObservedSessionTailKind {
self.inner.observed_session_tail()
}
fn update_keep_alive(&mut self, keep_alive: bool) {
self.inner.update_keep_alive(keep_alive);
}
fn update_mob_tool_authority_context(
&mut self,
authority_context: Option<MobToolAuthorityContext>,
) -> Result<(), meerkat_core::error::AgentError> {
self.inner
.update_mob_tool_authority_context(authority_context)
}
fn append_system_messages(
&mut self,
contents: Vec<String>,
) -> Result<(), meerkat_core::error::AgentError> {
self.inner.append_system_messages(contents)
}
fn transient_turn_context_state(&self) -> meerkat_core::TransientTurnContextStateHandle {
self.inner.transient_turn_context_state()
}
}
struct EventfulBuilder;
#[async_trait::async_trait]
impl SessionAgentBuilder for EventfulBuilder {
type Agent = EventfulDummyAgent;
async fn build_agent(
&self,
req: &CreateSessionRequest,
_event_tx: tokio::sync::mpsc::Sender<meerkat_core::event::AgentEvent>,
) -> Result<Self::Agent, SessionError> {
let session = req
.build
.as_ref()
.and_then(|build| build.resume_session.clone())
.unwrap_or_default();
Ok(EventfulDummyAgent {
inner: DummyAgent {
session: Arc::new(std::sync::Mutex::new(session)),
transient_turn_context_state: test_transient_turn_context_state_handle(),
run_failure: None,
flow_overlay_failure: None,
callback_pending_after_run: false,
execution_snapshot: None,
compaction_abort_failure: None,
},
})
}
}
struct FailingRunBuilder;
#[async_trait::async_trait]
impl SessionAgentBuilder for FailingRunBuilder {
type Agent = DummyAgent;
async fn build_agent(
&self,
req: &CreateSessionRequest,
_event_tx: tokio::sync::mpsc::Sender<meerkat_core::event::AgentEvent>,
) -> Result<Self::Agent, SessionError> {
let session = req
.build
.as_ref()
.and_then(|build| build.resume_session.clone())
.unwrap_or_default();
Ok(DummyAgent {
session: Arc::new(std::sync::Mutex::new(session)),
transient_turn_context_state: test_transient_turn_context_state_handle(),
run_failure: Some("synthetic run failure".to_string()),
flow_overlay_failure: None,
callback_pending_after_run: false,
execution_snapshot: None,
compaction_abort_failure: None,
})
}
}
struct MachineTerminalRunBuilder;
struct StaleTerminalRunBuilder;
fn machine_terminal_tool_failure_snapshot() -> meerkat_core::AgentExecutionSnapshot {
meerkat_core::AgentExecutionSnapshot {
loop_state: meerkat_core::state::LoopState::Completed,
turn_phase: meerkat_core::turn_execution_authority::TurnPhase::Failed,
turn_terminal: true,
active_run_id: Some(RunId::new()),
terminal_run_id: None,
primitive_kind:
meerkat_core::turn_execution_authority::TurnPrimitiveKind::ConversationTurn,
admitted_content_shape: Some(
meerkat_core::turn_execution_authority::ContentShape::Conversation,
),
vision_enabled: false,
image_tool_results_enabled: false,
tool_calls_pending: 0,
pending_operation_ids: None,
barrier_operation_ids: Vec::new(),
has_barrier_ops: false,
barrier_satisfied: true,
boundary_count: 0,
cancel_after_boundary: false,
terminal_outcome: meerkat_core::TurnTerminalOutcome::Failed,
terminal_cause_kind: Some(meerkat_core::TurnTerminalCauseKind::ToolFailure),
extraction_attempts: 0,
max_extraction_retries: 0,
applied_cursor: 0,
}
}
#[async_trait::async_trait]
impl SessionAgentBuilder for MachineTerminalRunBuilder {
type Agent = DummyAgent;
async fn build_agent(
&self,
req: &CreateSessionRequest,
_event_tx: tokio::sync::mpsc::Sender<meerkat_core::event::AgentEvent>,
) -> Result<Self::Agent, SessionError> {
let session = req
.build
.as_ref()
.and_then(|build| build.resume_session.clone())
.unwrap_or_default();
Ok(DummyAgent {
session: Arc::new(std::sync::Mutex::new(session)),
transient_turn_context_state: test_transient_turn_context_state_handle(),
run_failure: Some("synthetic member-side access denied".to_string()),
flow_overlay_failure: None,
callback_pending_after_run: false,
execution_snapshot: Some(machine_terminal_tool_failure_snapshot()),
compaction_abort_failure: None,
})
}
}
#[async_trait::async_trait]
impl SessionAgentBuilder for StaleTerminalRunBuilder {
type Agent = EventfulDummyAgent;
async fn build_agent(
&self,
req: &CreateSessionRequest,
_event_tx: tokio::sync::mpsc::Sender<meerkat_core::event::AgentEvent>,
) -> Result<Self::Agent, SessionError> {
let session = req
.build
.as_ref()
.and_then(|build| build.resume_session.clone())
.unwrap_or_default();
Ok(EventfulDummyAgent {
inner: DummyAgent {
session: Arc::new(std::sync::Mutex::new(session)),
transient_turn_context_state: test_transient_turn_context_state_handle(),
run_failure: Some("new pre-dispatch failure".to_string()),
flow_overlay_failure: None,
callback_pending_after_run: false,
execution_snapshot: Some(machine_terminal_tool_failure_snapshot()),
compaction_abort_failure: None,
},
})
}
}
struct CapturingBuildBuilder {
captured_builds: Arc<tokio::sync::Mutex<Vec<SessionBuildOptions>>>,
}
impl CapturingBuildBuilder {
fn new() -> Self {
Self {
captured_builds: Arc::new(tokio::sync::Mutex::new(Vec::new())),
}
}
}
#[async_trait::async_trait]
impl SessionAgentBuilder for CapturingBuildBuilder {
type Agent = DummyAgent;
async fn build_agent(
&self,
req: &CreateSessionRequest,
_event_tx: tokio::sync::mpsc::Sender<meerkat_core::event::AgentEvent>,
) -> Result<Self::Agent, SessionError> {
if let Some(build) = req.build.clone() {
self.captured_builds.lock().await.push(build);
}
let session = req
.build
.as_ref()
.and_then(|build| build.resume_session.clone())
.unwrap_or_default();
Ok(DummyAgent {
session: Arc::new(std::sync::Mutex::new(session)),
transient_turn_context_state: test_transient_turn_context_state_handle(),
run_failure: None,
flow_overlay_failure: None,
callback_pending_after_run: false,
execution_snapshot: None,
compaction_abort_failure: None,
})
}
}
struct ImagePreservingAgent {
session: Arc<std::sync::Mutex<Session>>,
transient_turn_context_state: meerkat_core::TransientTurnContextStateHandle,
}
#[async_trait::async_trait]
impl SessionAgent for ImagePreservingAgent {
async fn run_with_events(
&mut self,
prompt: meerkat_core::types::ContentInput,
_event_tx: tokio::sync::mpsc::Sender<meerkat_core::event::AgentEvent>,
) -> Result<RunResult, meerkat_core::error::AgentError> {
let mut session = match self.session.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
session.push(Message::User(UserMessage::with_blocks(
prompt.into_blocks(),
)));
session.push(Message::BlockAssistant(
meerkat_core::types::BlockAssistantMessage {
blocks: vec![meerkat_core::types::AssistantBlock::Text {
text: "ok".to_string(),
meta: None,
}],
stop_reason: meerkat_core::types::StopReason::EndTurn,
identity: meerkat_core::types::TranscriptMessageIdentity::default(),
created_at: meerkat_core::types::message_timestamp_now(),
},
));
Ok(RunResult {
text: "ok".to_string(),
session_id: session.id().clone(),
usage: meerkat_core::types::Usage::default(),
turns: 1,
tool_calls: 0,
terminal_cause_kind: None,
structured_output: None,
extraction_error: None,
schema_warnings: None,
skill_diagnostics: None,
})
}
fn set_skill_references(&mut self, _refs: Option<Vec<meerkat_core::skills::SkillKey>>) {}
fn set_turn_tool_overlay(
&mut self,
_overlay: Option<meerkat_core::service::TurnToolOverlay>,
) -> Result<(), meerkat_core::error::AgentError> {
Ok(())
}
fn hot_swap_llm_identity(
&mut self,
_client: Arc<dyn meerkat_core::AgentLlmClient>,
identity: meerkat_core::SessionLlmIdentity,
_request_policy: meerkat_core::SessionLlmRequestPolicy,
) -> Result<(), meerkat_core::error::AgentError> {
let mut session = match self.session.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
let mut metadata =
session
.session_metadata()
.unwrap_or(meerkat_core::SessionMetadata {
schema_version: meerkat_core::SESSION_METADATA_SCHEMA_VERSION,
model: identity.model.clone(),
max_tokens: 0,
structured_output_retries: 2,
provider: identity.provider,
self_hosted_server_id: None,
provider_params: identity.provider_params.clone(),
tooling: meerkat_core::SessionTooling::default(),
keep_alive: false,
comms_name: None,
peer_meta: None,
realm_id: None,
instance_id: None,
backend: None,
config_generation: None,
auth_binding: None,
mob_member_binding: None,
});
metadata.apply_llm_identity(&identity);
session.set_session_metadata(metadata).map_err(|err| {
meerkat_core::error::AgentError::InternalError(format!(
"failed to update image-preserving session metadata: {err}"
))
})
}
fn cancel(&mut self) {}
fn snapshot(&self) -> SessionSnapshot {
let session = match self.session.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
SessionSnapshot {
created_at: session.created_at(),
updated_at: session.updated_at(),
message_count: session.messages().len(),
total_tokens: session.total_tokens(),
usage: session.total_usage(),
last_assistant_text: session.last_assistant_text(),
}
}
fn session_clone(&self) -> Result<Session, AgentError> {
match self.session.lock() {
Ok(guard) => Ok(guard.clone()),
Err(poisoned) => Ok(poisoned.into_inner().clone()),
}
}
fn session_transcript_authority(
&self,
) -> Result<SessionTranscriptAuthoritySnapshot, AgentError> {
let session = self.session_clone()?;
SessionTranscriptAuthoritySnapshot::from_session(&session)
}
fn durable_llm_identity(&self) -> Option<meerkat_core::SessionLlmIdentity> {
let session = match self.session.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
Some(test_durable_llm_identity(&session))
}
fn session_id(&self) -> SessionId {
let guard = match self.session.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
guard.id().clone()
}
fn observed_session_tail(&self) -> ObservedSessionTailKind {
let session = match self.session.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
meerkat_core::pending_continuation::observe_session_tail(session.messages())
}
fn transient_turn_context_state(&self) -> meerkat_core::TransientTurnContextStateHandle {
self.transient_turn_context_state.clone()
}
fn sync_session_from_durable_snapshot(
&mut self,
session: Session,
) -> Result<(), meerkat_core::error::AgentError> {
if session.id() != &self.session_id() {
return Err(meerkat_core::error::AgentError::InternalError(format!(
"durable snapshot session id {} does not match live session {}",
session.id(),
self.session_id()
)));
}
match self.session.lock() {
Ok(mut guard) => {
*guard = session;
}
Err(poisoned) => {
*poisoned.into_inner() = session;
}
}
Ok(())
}
}
struct ImagePreservingBuilder;
#[async_trait::async_trait]
impl SessionAgentBuilder for ImagePreservingBuilder {
type Agent = ImagePreservingAgent;
async fn build_agent(
&self,
req: &CreateSessionRequest,
_event_tx: tokio::sync::mpsc::Sender<meerkat_core::event::AgentEvent>,
) -> Result<Self::Agent, SessionError> {
let session = req
.build
.as_ref()
.and_then(|build| build.resume_session.clone())
.unwrap_or_default();
Ok(ImagePreservingAgent {
session: Arc::new(std::sync::Mutex::new(session)),
transient_turn_context_state: test_transient_turn_context_state_handle(),
})
}
}
struct ToolDispatchAgent {
session: Arc<std::sync::Mutex<Session>>,
transient_turn_context_state: meerkat_core::TransientTurnContextStateHandle,
}
fn expected_tool_dispatch_witness(tool_name: &str) -> meerkat_core::ToolVisibilityWitness {
meerkat_core::ToolVisibilityWitness {
last_seen_provenance: Some(meerkat_core::ToolProvenance {
kind: meerkat_core::ToolSourceKind::Callback,
source_id: tool_name.to_string().into(),
}),
}
}
#[async_trait::async_trait]
impl SessionAgent for ToolDispatchAgent {
async fn run_with_events(
&mut self,
prompt: meerkat_core::types::ContentInput,
_event_tx: tokio::sync::mpsc::Sender<meerkat_core::event::AgentEvent>,
) -> Result<RunResult, meerkat_core::error::AgentError> {
let session_id = self.session_id();
let mut session = match self.session.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
session.push(meerkat_core::types::Message::User(
meerkat_core::types::UserMessage::text(prompt.text_content()),
));
session.push(meerkat_core::types::Message::BlockAssistant(
meerkat_core::types::BlockAssistantMessage {
blocks: vec![meerkat_core::types::AssistantBlock::Text {
text: "ok".to_string(),
meta: None,
}],
stop_reason: meerkat_core::types::StopReason::EndTurn,
identity: meerkat_core::types::TranscriptMessageIdentity::default(),
created_at: meerkat_core::types::message_timestamp_now(),
},
));
Ok(RunResult {
text: "ok".to_string(),
session_id,
usage: meerkat_core::types::Usage::default(),
turns: 1,
tool_calls: 0,
terminal_cause_kind: None,
structured_output: None,
extraction_error: None,
schema_warnings: None,
skill_diagnostics: None,
})
}
fn set_skill_references(&mut self, _refs: Option<Vec<meerkat_core::skills::SkillKey>>) {}
fn set_turn_tool_overlay(
&mut self,
_overlay: Option<meerkat_core::service::TurnToolOverlay>,
) -> Result<(), meerkat_core::error::AgentError> {
Ok(())
}
async fn dispatch_external_tool_call(
&mut self,
call: ToolCall,
) -> Result<ToolDispatchOutcome, meerkat_core::error::AgentError> {
let session = match self.session.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
let mut state = session
.tool_visibility_state()
.map_err(|err| {
meerkat_core::error::AgentError::InternalError(format!(
"failed to decode dummy visibility state: {err}"
))
})?
.unwrap_or_default();
let requested_name = meerkat_core::ToolName::from(format!("requested:{}", call.name));
state
.staged_requested_deferred_names
.insert(requested_name.clone());
state
.requested_witnesses
.insert(requested_name, expected_tool_dispatch_witness(&call.name));
let _ = state;
Ok(ToolDispatchOutcome::sync_result(ToolResult::new(
call.id,
format!("handled {}", call.name),
false,
)))
}
fn cancel(&mut self) {}
fn hot_swap_llm_identity(
&mut self,
_client: Arc<dyn meerkat_core::AgentLlmClient>,
identity: meerkat_core::SessionLlmIdentity,
_request_policy: meerkat_core::SessionLlmRequestPolicy,
) -> Result<(), meerkat_core::error::AgentError> {
let mut session = match self.session.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
let mut metadata =
session
.session_metadata()
.unwrap_or(meerkat_core::SessionMetadata {
schema_version: meerkat_core::SESSION_METADATA_SCHEMA_VERSION,
model: identity.model.clone(),
max_tokens: 0,
structured_output_retries: 2,
provider: identity.provider,
self_hosted_server_id: None,
provider_params: identity.provider_params.clone(),
tooling: meerkat_core::SessionTooling::default(),
keep_alive: false,
comms_name: None,
peer_meta: None,
realm_id: None,
instance_id: None,
backend: None,
config_generation: None,
auth_binding: None,
mob_member_binding: None,
});
metadata.apply_llm_identity(&identity);
session.set_session_metadata(metadata).map_err(|err| {
meerkat_core::error::AgentError::InternalError(format!(
"failed to update tool-dispatch session metadata: {err}"
))
})
}
fn session_id(&self) -> SessionId {
match self.session.lock() {
Ok(guard) => guard.id().clone(),
Err(poisoned) => poisoned.into_inner().id().clone(),
}
}
fn snapshot(&self) -> SessionSnapshot {
let session = match self.session.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
SessionSnapshot {
created_at: session.created_at(),
updated_at: session.updated_at(),
message_count: session.messages().len(),
total_tokens: session.total_tokens(),
usage: session.total_usage(),
last_assistant_text: session.last_assistant_text(),
}
}
fn session_clone(&self) -> Result<Session, AgentError> {
match self.session.lock() {
Ok(guard) => Ok(guard.clone()),
Err(poisoned) => Ok(poisoned.into_inner().clone()),
}
}
fn session_transcript_authority(
&self,
) -> Result<SessionTranscriptAuthoritySnapshot, AgentError> {
let session = self.session_clone()?;
SessionTranscriptAuthoritySnapshot::from_session(&session)
}
fn durable_llm_identity(&self) -> Option<meerkat_core::SessionLlmIdentity> {
let session = match self.session.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
Some(test_durable_llm_identity(&session))
}
fn observed_session_tail(&self) -> ObservedSessionTailKind {
let session = match self.session.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
meerkat_core::pending_continuation::observe_session_tail(session.messages())
}
fn transient_turn_context_state(&self) -> meerkat_core::TransientTurnContextStateHandle {
self.transient_turn_context_state.clone()
}
}
struct ToolDispatchBuilder;
#[async_trait::async_trait]
impl SessionAgentBuilder for ToolDispatchBuilder {
type Agent = ToolDispatchAgent;
async fn build_agent(
&self,
req: &CreateSessionRequest,
_event_tx: tokio::sync::mpsc::Sender<meerkat_core::event::AgentEvent>,
) -> Result<Self::Agent, SessionError> {
let session = req
.build
.as_ref()
.and_then(|build| build.resume_session.clone())
.unwrap_or_default();
Ok(ToolDispatchAgent {
session: Arc::new(std::sync::Mutex::new(session)),
transient_turn_context_state: test_transient_turn_context_state_handle(),
})
}
}
fn create_request(prompt: &str, initial_turn: InitialTurnPolicy) -> CreateSessionRequest {
CreateSessionRequest {
injected_context: Vec::new(),
model: "test".to_string(),
prompt: prompt.to_string().into(),
deferred_prompt_policy: DeferredPromptPolicy::Discard,
system_prompt: meerkat_core::SystemPromptOverride::Inherit,
max_tokens: None,
event_tx: None,
initial_turn,
build: None,
labels: None,
}
}
fn test_durable_llm_identity(session: &Session) -> meerkat_core::SessionLlmIdentity {
session
.session_metadata()
.map(|metadata| metadata.llm_identity())
.unwrap_or(meerkat_core::SessionLlmIdentity {
model: "test".to_string(),
provider: meerkat_core::Provider::Anthropic,
self_hosted_server_id: None,
provider_params: None,
auth_binding: None,
})
}
fn resume_request(session: Session) -> CreateSessionRequest {
let mut req = create_request("", InitialTurnPolicy::Defer);
req.build = Some(SessionBuildOptions {
resume_session: Some(session),
..Default::default()
});
req
}
async fn archived_resume_admission_fixture() -> (
PersistentSessionService<DummyBuilder>,
Arc<meerkat_runtime::MeerkatMachine>,
meerkat_runtime::PreparedSessionMaterialization,
Session,
) {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
1,
Arc::clone(&store),
Arc::clone(&runtime_store) as Arc<dyn RuntimeStore>,
memory_blob_store(),
);
let archived = Session::new();
let id = archived.id().clone();
runtime_store
.commit_session_snapshot(
&PersistentSessionService::<DummyBuilder>::runtime_id_for_session(&id),
SerializedSessionSnapshot {
session_snapshot: (serde_json::to_vec(&archived)
.expect("serialize archived-resume content body"))
.into(),
},
)
.await
.expect("seed archived-resume RuntimeStore body");
let machine = Arc::new(meerkat_runtime::MeerkatMachine::persistent(
Arc::clone(&runtime_store) as Arc<dyn RuntimeStore>,
memory_blob_store(),
));
machine
.register_session(id.clone())
.await
.expect("register archived resume fixture");
meerkat_runtime::RuntimeControlPlane::retire(
machine.as_ref(),
&PersistentSessionService::<DummyBuilder>::runtime_id_for_session(&id),
)
.await
.expect("retire archived resume fixture");
let prepared = machine
.prepare_session_materialization(id)
.await
.expect("prepare exact archived resume fixture");
machine
.install_prepared_session_executor_handles(
prepared.bindings(),
Arc::new(TestPreparedInterruptHandle),
Arc::new(TestPreparedCleanupHandle),
)
.await
.expect("install provisional archived-resume cleanup");
(service, machine, prepared, archived)
}
fn realtime_inline_image_event(
idempotency_key: &str,
item_id: &str,
data: &str,
) -> meerkat_core::RealtimeTranscriptEvent {
meerkat_core::RealtimeTranscriptEvent::UserContentFinal {
idempotency_key: idempotency_key.to_string(),
item_id: item_id.to_string(),
previous_item_id: None,
content_index: 0,
content: vec![ContentBlock::Image {
media_type: "image/png".to_string(),
data: ImageData::Inline {
data: data.to_string(),
},
}],
}
}
fn inline_image_block(label: &str) -> ContentBlock {
ContentBlock::Image {
media_type: "image/png".to_string(),
data: ImageData::Inline {
data: format!("base64-{label}"),
},
}
}
fn image_prompt(label: &str) -> ContentInput {
ContentInput::Blocks(vec![
ContentBlock::Text {
text: format!("look at {label}"),
},
inline_image_block(label),
])
}
fn assert_no_inline_images_in_session(session: &Session) {
for message in session.messages() {
match message {
Message::User(user) => {
for block in &user.content {
assert!(
!matches!(
block,
ContentBlock::Image {
data: ImageData::Inline { .. },
..
}
),
"persisted session unexpectedly retained inline image bytes: {session:?}"
);
}
}
Message::ToolResults { results, .. } => {
for result in results {
for block in &result.content {
assert!(
!matches!(
block,
ContentBlock::Image {
data: ImageData::Inline { .. },
..
}
),
"persisted session unexpectedly retained inline image bytes: {session:?}"
);
}
}
}
_ => {}
}
}
}
fn start_turn_request(prompt: &str) -> StartTurnRequest {
StartTurnRequest {
injected_context: Vec::new(),
prompt: prompt.to_string().into(),
system_prompt: None,
event_tx: None,
runtime: meerkat_core::service::StartTurnRuntimeSemantics::default(),
}
}
fn assert_runtime_backed_direct_start_turn_rejected(error: &SessionError) {
match error {
SessionError::Unsupported(message) => assert!(
message.contains(
"runtime-backed direct start_turn must route through the MeerkatMachine service-turn commit protocol"
),
"unexpected unsupported error: {message}"
),
other => panic!("expected runtime-backed direct start_turn rejection, got {other:?}"),
}
}
fn assert_runtime_backed_eager_create_rejected(error: &SessionError) {
match error {
SessionError::Unsupported(message) => assert!(
message.contains(
"runtime-backed eager create_session must route through the MeerkatMachine service-turn commit protocol"
),
"unexpected unsupported error: {message}"
),
other => panic!("expected runtime-backed eager create rejection, got {other:?}"),
}
}
fn runtime_content_turn_request(prompt: &str) -> StartTurnRequest {
let mut req = start_turn_request(prompt);
req.runtime.turn_metadata = Some(
meerkat_core::lifecycle::run_primitive::RuntimeTurnMetadata {
execution_kind: Some(meerkat_core::lifecycle::RuntimeExecutionKind::ContentTurn),
..Default::default()
},
);
req
}
async fn machine_commit_runtime_output<B: SessionAgentBuilder + 'static>(
service: &PersistentSessionService<B>,
runtime_store: &dyn RuntimeStore,
session_id: &SessionId,
output: &CoreApplyOutput,
) -> Result<(), String> {
let runtime_id = PersistentSessionService::<B>::runtime_id_for_session(session_id);
let committed = output.committed().cloned();
let provisional = committed
.as_ref()
.and_then(BoundSessionCommit::provisional_promotion_receipt)
.cloned();
let receipt = output.receipt.clone().into_sequenced(0);
let prepared = match provisional.as_ref() {
Some(checkpoint) => match runtime_store.session_persistence_profile() {
RuntimeSessionPersistenceProfile::WholeBlobV1 => {
let promotion =
meerkat_runtime::store::PreparedWholeBlobProvisionalPromotion::prepare(
checkpoint.clone(),
&receipt.run_id,
)
.expect("prepare WholeBlob provisional promotion");
meerkat_runtime::store::PreparedRuntimeSessionCommit::promote_whole_blob_success(
promotion,
receipt,
Vec::new(),
session_id.clone(),
)
.expect("bind WholeBlob terminal receipt to checkpoint facts")
}
RuntimeSessionPersistenceProfile::HeadCanonicalV1 => {
let promotion =
meerkat_runtime::store::PreparedHeadCanonicalProvisionalPromotion::prepare(
checkpoint.clone(),
&receipt.run_id,
)
.expect("prepare HeadCanonical provisional promotion");
meerkat_runtime::store::PreparedRuntimeSessionCommit::promote_head_canonical_success(
promotion,
receipt,
Vec::new(),
session_id.clone(),
)
.expect("bind HeadCanonical terminal receipt to checkpoint facts")
}
profile => {
return Err(format!(
"test runtime store uses unsupported session persistence profile {profile:?}"
));
}
},
None => meerkat_runtime::store::PreparedRuntimeSessionCommit::success(
committed.clone(),
receipt,
Vec::new(),
Some(session_id.clone()),
),
};
let result = runtime_store
.commit_prepared_session_boundary(&runtime_id, prepared)
.await
.expect("machine-owned runtime output commit should succeed");
if provisional.is_some() {
let authority = result
.authority()
.expect("provisional promotion must return committed authority");
let (revision, token) = match authority {
RuntimeSessionAuthority::WholeBlob(authority) => {
(authority.store_revision(), authority.blob_sha256())
}
RuntimeSessionAuthority::HeadCanonical(authority) => {
(authority.store_revision(), authority.committed_head_token())
}
};
service
.acknowledge_provisional_runtime_session_boundary_under_runtime_turn_boundary(
session_id, revision, token,
)
.await
.expect("acknowledge exact provisional promotion");
} else if let Some(committed) = committed {
match runtime_store.session_persistence_profile() {
RuntimeSessionPersistenceProfile::WholeBlobV1 => {
let authority = result
.authority()
.and_then(RuntimeSessionAuthority::whole_blob)
.expect("whole-blob commit must return authority");
let artifact = committed
.whole_blob_artifact()
.expect("whole-blob runtime output boundary should encode");
assert_eq!(
authority.blob_sha256(),
artifact.row_sha256_token(),
"store authority must bind the exact prepared artifact"
);
service
.acknowledge_whole_blob_runtime_session_boundary_under_runtime_turn_boundary(
session_id,
authority.store_revision(),
authority.blob_sha256(),
)
.await
.expect("acknowledge exact WholeBlob authority");
}
RuntimeSessionPersistenceProfile::HeadCanonicalV1 => {
let authority = result
.authority()
.and_then(RuntimeSessionAuthority::head_canonical)
.expect("head-canonical commit must return authority");
service
.acknowledge_head_canonical_runtime_session_boundary_under_runtime_turn_boundary(
session_id,
authority.committed_head_token(),
)
.await
.expect("acknowledge exact HeadCanonical boundary");
}
profile => {
return Err(format!(
"test runtime store uses unsupported session persistence profile {profile:?}"
));
}
}
}
Ok(())
}
#[tokio::test]
async fn whole_blob_promotion_then_compaction_refreshes_next_run_checkpoint_base() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store: Arc<dyn RuntimeStore> = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
CapturingBuildBuilder::new(),
4,
Arc::clone(&store),
Arc::clone(&runtime_store),
memory_blob_store(),
);
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create WholeBlob live actor");
let session_id = created.session_id;
let runtime_id =
PersistentSessionService::<DummyBuilder>::runtime_id_for_session(&session_id);
let checkpointer = service
.live_checkpointers
.lock()
.await
.get(&session_id)
.cloned()
.and_then(|checkpointer| checkpointer.upgrade())
.expect("live WholeBlob actor must retain its exact checkpointer");
let committed_base = runtime_store
.load_committed_whole_blob_snapshot(&runtime_id)
.await
.expect("load generation-zero authority")
.expect("generation-zero authority must exist");
let mut provisional_session = committed_base.session().clone();
provisional_session.push(Message::User(UserMessage::text(
"verbose context one".to_string(),
)));
provisional_session.push(Message::User(UserMessage::text(
"verbose context two".to_string(),
)));
let (_rewrite, intent) = provisional_session
.stage_validated_compaction_for_test(
vec![Message::User(UserMessage::compaction_summary(
"compacted context",
))],
2,
)
.expect("core should construct an exact typed compaction fixture");
let promotion_run = RunId::new();
let checkpoint = meerkat_core::SessionCheckpointer::checkpoint_run(
checkpointer.as_ref(),
&mut provisional_session,
&promotion_run,
None,
)
.await
.expect("write provisional WholeBlob candidate")
.expect("WholeBlob checkpoint must return a store-issued receipt");
let terminal_receipt = meerkat_core::lifecycle::RunBoundaryReceipt {
run_id: promotion_run.clone(),
boundary: RunApplyBoundary::RunStart,
contributing_input_ids: Vec::new(),
conversation_digest: Some(checkpoint.conversation_digest().to_string()),
message_count: usize::try_from(checkpoint.message_count())
.expect("checkpoint count fits usize"),
sequence: 1,
};
let promotion = meerkat_runtime::store::PreparedWholeBlobProvisionalPromotion::prepare(
checkpoint,
&promotion_run,
)
.expect("prepare exact metadata-only promotion");
let promoted = runtime_store
.commit_prepared_session_boundary(
&runtime_id,
meerkat_runtime::store::PreparedRuntimeSessionCommit::promote_whole_blob_success(
promotion,
terminal_receipt,
Vec::new(),
session_id.clone(),
)
.expect("bind final receipt to provisional facts"),
)
.await
.expect("promote exact provisional candidate");
let promoted_authority = promoted
.authority()
.and_then(RuntimeSessionAuthority::whole_blob)
.expect("promotion must return WholeBlob authority")
.clone();
service
.acknowledge_provisional_runtime_session_boundary_under_runtime_turn_boundary(
&session_id,
promoted_authority.store_revision(),
promoted_authority.blob_sha256(),
)
.await
.expect("advance actor base after metadata-only promotion");
let mut cleaned_session = provisional_session;
assert_eq!(
cleaned_session
.complete_compaction_projection_intent(&intent.projection)
.expect("remove finalized compaction intent"),
Some(intent.clone()),
);
runtime_store
.mark_compaction_projection_finalized(&runtime_id, &intent.projection)
.await
.expect("finalize the store-owned compaction outbox");
let cleaned_snapshot = runtime_store
.load_committed_whole_blob_snapshot(&runtime_id)
.await
.expect("load store-cleaned compaction checkpoint")
.expect("store-cleaned compaction checkpoint must exist");
service
.checkpoint_committed_runtime_session_snapshot(
&session_id,
cleaned_snapshot.bytes_arc(),
)
.await
.expect("compaction postcommit refresh must advance the actor base");
let cleaned_authority = cleaned_snapshot.authority().clone();
assert_eq!(
checkpointer
.whole_blob_base_authority
.lock()
.expect("actor base authority lock")
.as_ref(),
Some(&cleaned_authority),
"compaction checkpoint acknowledgement must install the exact cleaned authority"
);
let mut next_run_session = cleaned_session;
next_run_session.push(Message::User(UserMessage::text(
"next run after compaction".to_string(),
)));
let next_receipt = meerkat_core::SessionCheckpointer::checkpoint_run(
checkpointer.as_ref(),
&mut next_run_session,
&RunId::new(),
None,
)
.await
.expect("next run must accept the refreshed actor base")
.expect("next run must write a provisional successor");
let next_authority = next_receipt
.whole_blob()
.expect("WholeBlob checkpoint must retain its profile");
assert_eq!(
next_authority.base_store_revision(),
cleaned_authority.store_revision()
);
assert_eq!(
next_authority.base_blob_sha256(),
cleaned_authority.blob_sha256()
);
}
fn recovered_input_persistence_record(
runtime_id: &LogicalRuntimeId,
mut bundle: StoredInputState,
) -> InputStatePersistenceRecord {
if bundle.state.persisted_input.is_none() {
let mut input = meerkat_runtime::PromptInput::new("recovered test input", None);
input.header.id = bundle.state.input_id.clone();
let input = meerkat_runtime::Input::Prompt(input);
bundle.state.runtime_semantics.get_or_insert_with(|| {
meerkat_runtime::ingress_types::RuntimeInputSemantics::try_from_generated_admission(
&input, true,
)
.expect("test input should receive generated runtime semantics")
});
bundle.state.persisted_input = Some(input);
}
bundle
.seed
.recovery_lane
.get_or_insert(meerkat_core::types::HandlingMode::Queue);
let mut driver = meerkat_runtime::EphemeralRuntimeDriver::new(runtime_id.clone());
driver
.recover_input_state_persistence_record(bundle)
.expect("test input-state bundle should recover through generated authority")
}
fn start_turn_request_with_system_prompt(
prompt: &str,
system_prompt: Option<&str>,
) -> StartTurnRequest {
StartTurnRequest {
injected_context: Vec::new(),
prompt: prompt.to_string().into(),
system_prompt: system_prompt.map(str::to_string),
event_tx: None,
runtime: meerkat_core::service::StartTurnRuntimeSemantics::default(),
}
}
async fn committed_content_turn<B: SessionAgentBuilder + 'static>(
service: &PersistentSessionService<B>,
runtime_store: &dyn RuntimeStore,
id: &SessionId,
prompt: &str,
) {
committed_turn_with_request(
service,
runtime_store,
id,
runtime_content_turn_request(prompt),
)
.await;
}
async fn committed_turn_with_request<B: SessionAgentBuilder + 'static>(
service: &PersistentSessionService<B>,
runtime_store: &dyn RuntimeStore,
id: &SessionId,
req: StartTurnRequest,
) {
let output = service
.apply_runtime_turn(
id,
RunId::new(),
req,
RunApplyBoundary::RunStart,
vec![InputId::new()],
)
.await
.expect("runtime turn should apply");
machine_commit_runtime_output(service, runtime_store, id, &output)
.await
.expect("machine-owned runtime output should commit");
}
async fn transcript_edit_fixture() -> (
PersistentSessionService<DummyBuilder>,
Arc<dyn SessionStore>,
Arc<dyn RuntimeStore>,
SessionId,
) {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store: Arc<dyn RuntimeStore> = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store),
memory_blob_store(),
);
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create_session should succeed");
let session_id = created.session_id;
committed_content_turn(&service, runtime_store.as_ref(), &session_id, "hello").await;
committed_content_turn(&service, runtime_store.as_ref(), &session_id, "follow up").await;
(service, store, runtime_store, session_id)
}
struct CountingProjectionSessionStore {
inner: MemoryStore,
authoritative_projection_calls: AtomicUsize,
}
impl CountingProjectionSessionStore {
fn new() -> Self {
Self {
inner: MemoryStore::new(),
authoritative_projection_calls: AtomicUsize::new(0),
}
}
fn reset(&self) {
self.authoritative_projection_calls
.store(0, Ordering::Release);
}
}
#[async_trait::async_trait]
impl SessionStore for CountingProjectionSessionStore {
async fn save(&self, session: &Session) -> Result<(), SessionStoreError> {
self.inner.save(session).await
}
async fn save_transcript_rewrite(
&self,
session: &Session,
commit: &meerkat_core::TranscriptRewriteCommit,
) -> Result<(), SessionStoreError> {
self.inner.save_transcript_rewrite(session, commit).await
}
async fn save_authoritative_projection(
&self,
session: &Session,
) -> Result<(), SessionStoreError> {
self.inner.save_authoritative_projection(session).await
}
async fn save_authoritative_projection_if_current_revision(
&self,
session: &Session,
expected_current_revision: Option<String>,
) -> Result<(), SessionStoreError> {
self.authoritative_projection_calls
.fetch_add(1, Ordering::AcqRel);
self.inner
.save_authoritative_projection_if_current_revision(
session,
expected_current_revision,
)
.await
}
async fn load(&self, id: &SessionId) -> Result<Option<Session>, SessionStoreError> {
self.inner.load(id).await
}
async fn list(
&self,
filter: meerkat_store::SessionFilter,
) -> Result<Vec<meerkat_core::SessionMeta>, SessionStoreError> {
self.inner.list(filter).await
}
async fn delete(&self, id: &SessionId) -> Result<(), SessionStoreError> {
self.inner.delete(id).await
}
async fn delete_if_current_revision(
&self,
id: &SessionId,
expected_current_revision: &str,
) -> Result<bool, SessionStoreError> {
self.inner
.delete_if_current_revision(id, expected_current_revision)
.await
}
}
struct CountingRuntimeStore {
inner: InMemoryRuntimeStore,
snapshot_commit_bytes: std::sync::atomic::AtomicU64,
snapshot_commits: AtomicUsize,
rewrite_snapshot_bytes: std::sync::atomic::AtomicU64,
rewrite_successor_encode_bytes: std::sync::atomic::AtomicU64,
rewrite_snapshot_cas_calls: AtomicUsize,
rewrite_snapshot_commits: AtomicUsize,
boundary_apply_bytes: std::sync::atomic::AtomicU64,
boundary_apply_snapshots: AtomicUsize,
}
impl CountingRuntimeStore {
fn new() -> Self {
Self {
inner: InMemoryRuntimeStore::new(),
snapshot_commit_bytes: std::sync::atomic::AtomicU64::new(0),
snapshot_commits: AtomicUsize::new(0),
rewrite_snapshot_bytes: std::sync::atomic::AtomicU64::new(0),
rewrite_successor_encode_bytes: std::sync::atomic::AtomicU64::new(0),
rewrite_snapshot_cas_calls: AtomicUsize::new(0),
rewrite_snapshot_commits: AtomicUsize::new(0),
boundary_apply_bytes: std::sync::atomic::AtomicU64::new(0),
boundary_apply_snapshots: AtomicUsize::new(0),
}
}
fn reset(&self) {
self.snapshot_commit_bytes.store(0, Ordering::Release);
self.snapshot_commits.store(0, Ordering::Release);
self.rewrite_snapshot_bytes.store(0, Ordering::Release);
self.rewrite_successor_encode_bytes
.store(0, Ordering::Release);
self.rewrite_snapshot_cas_calls.store(0, Ordering::Release);
self.rewrite_snapshot_commits.store(0, Ordering::Release);
self.boundary_apply_bytes.store(0, Ordering::Release);
self.boundary_apply_snapshots.store(0, Ordering::Release);
}
}
struct WholeBlobAuthorityReadProbe {
inner: InMemoryRuntimeStore,
bounded_authority_reads: AtomicUsize,
full_snapshot_reads: AtomicUsize,
}
#[async_trait::async_trait]
impl RuntimeStore for WholeBlobAuthorityReadProbe {
fn session_authority_ops(&self) -> &dyn meerkat_runtime::store::RuntimeSessionAuthorityOps {
self.inner.session_authority_ops()
}
fn session_persistence_profile(&self) -> RuntimeSessionPersistenceProfile {
RuntimeSessionPersistenceProfile::WholeBlobV1
}
fn session_boundary_authority_read_cost(
&self,
) -> meerkat_runtime::store::RuntimeSessionAuthorityReadCost {
self.inner.session_boundary_authority_read_cost()
}
async fn commit_prepared_session_boundary(
&self,
runtime_id: &LogicalRuntimeId,
request: meerkat_runtime::PreparedRuntimeSessionCommit,
) -> Result<
meerkat_runtime::store::PreparedRuntimeSessionCommitResult,
meerkat_runtime::store::RuntimeStoreError,
> {
self.inner
.commit_prepared_session_boundary(runtime_id, request)
.await
}
async fn commit_session_snapshot(
&self,
runtime_id: &LogicalRuntimeId,
session_delta: SerializedSessionSnapshot,
) -> Result<(), meerkat_runtime::store::RuntimeStoreError> {
self.inner
.commit_session_snapshot(runtime_id, session_delta)
.await
}
async fn commit_prepared_whole_blob_rewrite_boundary(
&self,
runtime_id: &LogicalRuntimeId,
boundary: meerkat_runtime::store::PreparedWholeBlobRewriteStoreParts,
) -> Result<WholeBlobStoreAuthority, meerkat_runtime::store::RuntimeStoreError> {
self.inner
.commit_prepared_whole_blob_rewrite_boundary(runtime_id, boundary)
.await
}
async fn atomic_apply(
&self,
runtime_id: &LogicalRuntimeId,
session_delta: Option<SerializedSessionSnapshot>,
receipt: meerkat_core::lifecycle::RunBoundaryReceipt,
input_updates: Vec<InputStatePersistenceRecord>,
session_store_key: Option<SessionId>,
) -> Result<(), meerkat_runtime::store::RuntimeStoreError> {
self.inner
.atomic_apply(
runtime_id,
session_delta,
receipt,
input_updates,
session_store_key,
)
.await
}
async fn load_input_states(
&self,
runtime_id: &LogicalRuntimeId,
) -> Result<Vec<meerkat_runtime::InputStateRow>, meerkat_runtime::store::RuntimeStoreError>
{
self.inner.load_input_states(runtime_id).await
}
async fn load_boundary_receipt(
&self,
runtime_id: &LogicalRuntimeId,
run_id: &RunId,
sequence: u64,
) -> Result<
Option<meerkat_core::lifecycle::RunBoundaryReceipt>,
meerkat_runtime::store::RuntimeStoreError,
> {
self.inner
.load_boundary_receipt(runtime_id, run_id, sequence)
.await
}
async fn load_session_snapshot(
&self,
runtime_id: &LogicalRuntimeId,
) -> Result<Option<Arc<Vec<u8>>>, meerkat_runtime::store::RuntimeStoreError> {
self.inner.load_session_snapshot(runtime_id).await
}
async fn clear_session_snapshot(
&self,
runtime_id: &LogicalRuntimeId,
) -> Result<(), meerkat_runtime::store::RuntimeStoreError> {
self.inner.clear_session_snapshot(runtime_id).await
}
async fn replace_session_snapshot_if_current(
&self,
runtime_id: &LogicalRuntimeId,
expected_current: &[u8],
replacement: Vec<u8>,
) -> Result<bool, meerkat_runtime::store::RuntimeStoreError> {
self.inner
.replace_session_snapshot_if_current(runtime_id, expected_current, replacement)
.await
}
async fn clear_session_snapshot_if_current(
&self,
runtime_id: &LogicalRuntimeId,
expected_current: &[u8],
) -> Result<bool, meerkat_runtime::store::RuntimeStoreError> {
self.inner
.clear_session_snapshot_if_current(runtime_id, expected_current)
.await
}
async fn persist_input_state(
&self,
runtime_id: &LogicalRuntimeId,
state: &InputStatePersistenceRecord,
) -> Result<(), meerkat_runtime::store::RuntimeStoreError> {
self.inner.persist_input_state(runtime_id, state).await
}
async fn load_input_state(
&self,
runtime_id: &LogicalRuntimeId,
input_id: &InputId,
) -> Result<Option<StoredInputState>, meerkat_runtime::store::RuntimeStoreError> {
self.inner.load_input_state(runtime_id, input_id).await
}
async fn load_machine_lifecycle_record(
&self,
runtime_id: &LogicalRuntimeId,
) -> Result<Option<Vec<u8>>, meerkat_runtime::store::RuntimeStoreError> {
self.inner.load_machine_lifecycle_record(runtime_id).await
}
async fn commit_machine_lifecycle(
&self,
runtime_id: &LogicalRuntimeId,
commit: meerkat_runtime::store::MachineLifecycleCommit,
input_states: &[InputStatePersistenceRecord],
) -> Result<(), meerkat_runtime::store::RuntimeStoreError> {
self.inner
.commit_machine_lifecycle(runtime_id, commit, input_states)
.await
}
async fn load_whole_blob_store_authority(
&self,
runtime_id: &LogicalRuntimeId,
) -> Result<Option<WholeBlobStoreAuthority>, meerkat_runtime::store::RuntimeStoreError>
{
self.bounded_authority_reads.fetch_add(1, Ordering::SeqCst);
self.inner.load_whole_blob_store_authority(runtime_id).await
}
async fn load_committed_whole_blob_snapshot(
&self,
runtime_id: &LogicalRuntimeId,
) -> Result<Option<CommittedWholeBlobSnapshot>, meerkat_runtime::store::RuntimeStoreError>
{
self.full_snapshot_reads.fetch_add(1, Ordering::SeqCst);
self.inner
.load_committed_whole_blob_snapshot(runtime_id)
.await
}
}
#[tokio::test]
async fn whole_blob_ordinary_acknowledgement_reads_only_bounded_store_authority() {
let session = Session::new();
let session_id = session.id().clone();
let runtime_id = LogicalRuntimeId::for_session(&session_id);
let artifact = session
.to_persisted_artifact()
.expect("encode test WholeBlob once");
let probe = Arc::new(WholeBlobAuthorityReadProbe {
inner: InMemoryRuntimeStore::new(),
bounded_authority_reads: AtomicUsize::new(0),
full_snapshot_reads: AtomicUsize::new(0),
});
probe
.inner
.commit_session_snapshot(
&runtime_id,
SerializedSessionSnapshot {
session_snapshot: artifact.bytes_arc(),
},
)
.await
.expect("seed exact committed WholeBlob row");
let authority = probe
.inner
.load_whole_blob_store_authority(&runtime_id)
.await
.expect("load seeded authority")
.expect("seeded authority exists");
let runtime_store: Arc<dyn RuntimeStore> = Arc::clone(&probe) as Arc<dyn RuntimeStore>;
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::new(MemoryStore::new()),
Arc::clone(&runtime_store),
memory_blob_store(),
);
service
.acknowledge_whole_blob_runtime_session_boundary_under_runtime_turn_boundary(
&session_id,
authority.store_revision(),
authority.blob_sha256(),
)
.await
.expect("exact authority acknowledgement succeeds");
assert_eq!(
probe.bounded_authority_reads.load(Ordering::SeqCst),
1,
"ordinary acknowledgement must make one bounded authority read"
);
assert_eq!(
probe.full_snapshot_reads.load(Ordering::SeqCst),
0,
"ordinary acknowledgement must not reload/hash/decode the WholeBlob body"
);
}
#[async_trait::async_trait]
impl RuntimeStore for CountingRuntimeStore {
fn session_authority_ops(&self) -> &dyn meerkat_runtime::store::RuntimeSessionAuthorityOps {
self.inner.session_authority_ops()
}
fn session_persistence_profile(
&self,
) -> meerkat_runtime::store::RuntimeSessionPersistenceProfile {
RuntimeStore::session_persistence_profile(&self.inner)
}
fn session_boundary_authority_read_cost(
&self,
) -> meerkat_runtime::store::RuntimeSessionAuthorityReadCost {
self.inner.session_boundary_authority_read_cost()
}
fn supports_compaction_projection_outbox(&self) -> bool {
self.inner.supports_compaction_projection_outbox()
}
fn input_state_batch_cas_implementation_profile(
&self,
) -> meerkat_runtime::store::InputStateBatchCasImplementationProfile {
self.inner.input_state_batch_cas_implementation_profile()
}
async fn load_runtime_delivery_authority(
&self,
runtime_id: &LogicalRuntimeId,
) -> Result<
Option<meerkat_runtime::store::RuntimeDeliveryAuthorityRecord>,
meerkat_runtime::store::RuntimeStoreError,
> {
self.inner.load_runtime_delivery_authority(runtime_id).await
}
async fn load_runtime_delivery_record(
&self,
runtime_id: &LogicalRuntimeId,
delivery_id: &str,
) -> Result<
Option<meerkat_runtime::store::RuntimeDeliveryStoreRecord>,
meerkat_runtime::store::RuntimeStoreError,
> {
self.inner
.load_runtime_delivery_record(runtime_id, delivery_id)
.await
}
async fn compare_and_swap_runtime_delivery_authority(
&self,
runtime_id: &LogicalRuntimeId,
expected_revision: Option<u64>,
replacement: meerkat_runtime::store::RuntimeDeliveryAuthorityRecord,
inserted_delivery: Option<meerkat_runtime::store::RuntimeDeliveryStoreRecord>,
) -> Result<
meerkat_runtime::store::RuntimeDeliveryAuthorityCasOutcome,
meerkat_runtime::store::RuntimeStoreError,
> {
self.inner
.compare_and_swap_runtime_delivery_authority(
runtime_id,
expected_revision,
replacement,
inserted_delivery,
)
.await
}
async fn list_runtime_delivery_records(
&self,
runtime_id: &LogicalRuntimeId,
after_sequence: u64,
limit: usize,
) -> Result<
Vec<meerkat_runtime::store::RuntimeDeliveryStoreRecord>,
meerkat_runtime::store::RuntimeStoreError,
> {
self.inner
.list_runtime_delivery_records(runtime_id, after_sequence, limit)
.await
}
fn persist_auth_oauth_flow_snapshot(
&self,
snapshot_json: &[u8],
) -> Result<(), meerkat_runtime::store::RuntimeStoreError> {
self.inner.persist_auth_oauth_flow_snapshot(snapshot_json)
}
fn load_auth_oauth_flow_snapshot(
&self,
) -> Result<Option<Vec<u8>>, meerkat_runtime::store::RuntimeStoreError> {
self.inner.load_auth_oauth_flow_snapshot()
}
fn update_auth_oauth_flow_snapshot(
&self,
update: &mut meerkat_runtime::store::AuthOAuthFlowSnapshotUpdate<'_>,
) -> Result<(), meerkat_runtime::store::RuntimeStoreError> {
self.inner.update_auth_oauth_flow_snapshot(update)
}
async fn load_session_boundary_authority(
&self,
runtime_id: &LogicalRuntimeId,
) -> Result<
Option<meerkat_runtime::store::RuntimeSessionAuthority>,
meerkat_runtime::store::RuntimeStoreError,
> {
self.inner.load_session_boundary_authority(runtime_id).await
}
async fn load_whole_blob_store_authority(
&self,
runtime_id: &LogicalRuntimeId,
) -> Result<Option<WholeBlobStoreAuthority>, meerkat_runtime::store::RuntimeStoreError>
{
self.inner.load_whole_blob_store_authority(runtime_id).await
}
async fn load_committed_whole_blob_snapshot(
&self,
runtime_id: &LogicalRuntimeId,
) -> Result<Option<CommittedWholeBlobSnapshot>, meerkat_runtime::store::RuntimeStoreError>
{
self.inner
.load_committed_whole_blob_snapshot(runtime_id)
.await
}
async fn commit_prepared_whole_blob_snapshot_cas(
&self,
runtime_id: &LogicalRuntimeId,
prepared: meerkat_runtime::store::PreparedWholeBlobSnapshotCas,
) -> Result<
meerkat_runtime::store::WholeBlobSnapshotCasOutcome,
meerkat_runtime::store::RuntimeStoreError,
> {
self.inner
.commit_prepared_whole_blob_snapshot_cas(runtime_id, prepared)
.await
}
async fn delete_runtime_session_catalog_entry(
&self,
runtime_id: &LogicalRuntimeId,
) -> Result<(), meerkat_runtime::store::RuntimeStoreError> {
self.inner
.delete_runtime_session_catalog_entry(runtime_id)
.await
}
async fn load_runtime_session_catalog_entry(
&self,
runtime_id: &LogicalRuntimeId,
) -> Result<
Option<meerkat_runtime::store::RuntimeSessionCatalogEntry>,
meerkat_runtime::store::RuntimeStoreError,
> {
self.inner
.load_runtime_session_catalog_entry(runtime_id)
.await
}
async fn list_runtime_session_catalog_entries(
&self,
filter: meerkat_core::SessionFilter,
) -> Result<
Vec<meerkat_runtime::store::RuntimeSessionCatalogEntry>,
meerkat_runtime::store::RuntimeStoreError,
> {
self.inner
.list_runtime_session_catalog_entries(filter)
.await
}
async fn write_prepared_whole_blob_provisional_tail(
&self,
runtime_id: &LogicalRuntimeId,
prepared: PreparedWholeBlobProvisionalTail,
) -> Result<
meerkat_runtime::store::WholeBlobProvisionalTailAuthority,
meerkat_runtime::store::RuntimeStoreError,
> {
self.inner
.write_prepared_whole_blob_provisional_tail(runtime_id, prepared)
.await
}
async fn load_whole_blob_provisional_tail(
&self,
runtime_id: &LogicalRuntimeId,
) -> Result<
Option<meerkat_runtime::store::CommittedWholeBlobProvisionalTail>,
meerkat_runtime::store::RuntimeStoreError,
> {
self.inner
.load_whole_blob_provisional_tail(runtime_id)
.await
}
async fn discard_whole_blob_provisional_tail(
&self,
runtime_id: &LogicalRuntimeId,
expected: &meerkat_runtime::store::WholeBlobProvisionalTailAuthority,
) -> Result<bool, meerkat_runtime::store::RuntimeStoreError> {
self.inner
.discard_whole_blob_provisional_tail(runtime_id, expected)
.await
}
async fn observe_machine_lifecycle(
&self,
runtime_id: &LogicalRuntimeId,
) -> Result<
meerkat_runtime::store::MachineLifecycleObservation,
meerkat_runtime::store::RuntimeStoreError,
> {
self.inner.observe_machine_lifecycle(runtime_id).await
}
async fn compare_and_swap_machine_lifecycle(
&self,
runtime_id: &LogicalRuntimeId,
expected: meerkat_runtime::store::MachineLifecycleExpectedVersion,
replacement: meerkat_runtime::store::MachineLifecycleCommit,
) -> Result<
meerkat_runtime::store::MachineLifecycleCasOutcome,
meerkat_runtime::store::RuntimeStoreError,
> {
self.inner
.compare_and_swap_machine_lifecycle(runtime_id, expected, replacement)
.await
}
async fn compare_and_swap_machine_lifecycle_with_fence(
&self,
runtime_id: &LogicalRuntimeId,
expected: meerkat_runtime::store::MachineLifecycleExpectedVersion,
replacement: meerkat_runtime::store::MachineLifecycleCommit,
write_fence: Arc<dyn meerkat_runtime::store::RuntimeStoreWriteFence>,
) -> Result<
meerkat_runtime::store::FencedMachineLifecycleCasOutcome,
meerkat_runtime::store::RuntimeStoreError,
> {
self.inner
.compare_and_swap_machine_lifecycle_with_fence(
runtime_id,
expected,
replacement,
write_fence,
)
.await
}
async fn commit_prepared_session_boundary(
&self,
runtime_id: &LogicalRuntimeId,
request: meerkat_runtime::PreparedRuntimeSessionCommit,
) -> Result<
meerkat_runtime::store::PreparedRuntimeSessionCommitResult,
meerkat_runtime::store::RuntimeStoreError,
> {
if let Some(session) = request.session()
&& let Ok(bytes) = session.whole_blob_bytes()
{
if request.receipt().is_some() {
self.boundary_apply_snapshots.fetch_add(1, Ordering::AcqRel);
self.boundary_apply_bytes
.fetch_add(bytes.len() as u64, Ordering::AcqRel);
} else {
self.snapshot_commits.fetch_add(1, Ordering::AcqRel);
self.snapshot_commit_bytes
.fetch_add(bytes.len() as u64, Ordering::AcqRel);
}
}
self.inner
.commit_prepared_session_boundary(runtime_id, request)
.await
}
async fn commit_session_snapshot(
&self,
runtime_id: &LogicalRuntimeId,
session_delta: meerkat_runtime::store::SerializedSessionSnapshot,
) -> Result<(), meerkat_runtime::store::RuntimeStoreError> {
self.snapshot_commits.fetch_add(1, Ordering::AcqRel);
self.snapshot_commit_bytes.fetch_add(
session_delta.session_snapshot.len() as u64,
Ordering::AcqRel,
);
self.inner
.commit_session_snapshot(runtime_id, session_delta)
.await
}
async fn commit_prepared_whole_blob_rewrite_boundary(
&self,
runtime_id: &LogicalRuntimeId,
boundary: meerkat_runtime::store::PreparedWholeBlobRewriteStoreParts,
) -> Result<
meerkat_runtime::store::WholeBlobStoreAuthority,
meerkat_runtime::store::RuntimeStoreError,
> {
self.rewrite_snapshot_cas_calls
.fetch_add(1, Ordering::AcqRel);
self.rewrite_successor_encode_bytes
.fetch_add(boundary.successor_encode_bytes(), Ordering::AcqRel);
if boundary.expected_authority().blob_sha256() != boundary.successor_blob_sha256() {
self.rewrite_snapshot_commits.fetch_add(1, Ordering::AcqRel);
self.rewrite_snapshot_bytes
.fetch_add(boundary.successor_bytes().len() as u64, Ordering::AcqRel);
}
self.inner
.commit_prepared_whole_blob_rewrite_boundary(runtime_id, boundary)
.await
}
async fn atomic_apply(
&self,
runtime_id: &LogicalRuntimeId,
session_delta: Option<meerkat_runtime::store::SerializedSessionSnapshot>,
receipt: meerkat_core::lifecycle::RunBoundaryReceipt,
input_updates: Vec<InputStatePersistenceRecord>,
session_store_key: Option<SessionId>,
) -> Result<(), meerkat_runtime::store::RuntimeStoreError> {
if let Some(delta) = session_delta.as_ref() {
self.boundary_apply_snapshots.fetch_add(1, Ordering::AcqRel);
self.boundary_apply_bytes
.fetch_add(delta.session_snapshot.len() as u64, Ordering::AcqRel);
}
self.inner
.atomic_apply(
runtime_id,
session_delta,
receipt,
input_updates,
session_store_key,
)
.await
}
async fn atomic_apply_with_machine_lifecycle(
&self,
runtime_id: &LogicalRuntimeId,
session_delta: meerkat_runtime::store::SerializedSessionSnapshot,
receipt: meerkat_core::lifecycle::RunBoundaryReceipt,
machine_lifecycle: meerkat_runtime::store::MachineLifecycleCommit,
input_updates: Vec<InputStatePersistenceRecord>,
session_store_key: SessionId,
) -> Result<(), meerkat_runtime::store::RuntimeStoreError> {
self.boundary_apply_snapshots.fetch_add(1, Ordering::AcqRel);
self.boundary_apply_bytes.fetch_add(
session_delta.session_snapshot.len() as u64,
Ordering::AcqRel,
);
self.inner
.atomic_apply_with_machine_lifecycle(
runtime_id,
session_delta,
receipt,
machine_lifecycle,
input_updates,
session_store_key,
)
.await
}
async fn load_committed_boundary_receipts(
&self,
runtime_id: &LogicalRuntimeId,
run_id: &RunId,
) -> Result<
Vec<meerkat_core::lifecycle::RunBoundaryReceipt>,
meerkat_runtime::store::RuntimeStoreError,
> {
self.inner
.load_committed_boundary_receipts(runtime_id, run_id)
.await
}
async fn load_durable_tail_recovery_receipts(
&self,
runtime_id: &LogicalRuntimeId,
run_id: &RunId,
) -> Result<
Vec<meerkat_runtime::store::PreparedRecoveryReceiptSource>,
meerkat_runtime::store::RuntimeStoreError,
> {
self.inner
.load_durable_tail_recovery_receipts(runtime_id, run_id)
.await
}
async fn load_committed_recovery_boundary(
&self,
runtime_id: &LogicalRuntimeId,
candidate_id: &str,
) -> Result<
Option<meerkat_runtime::store::CommittedRecoveryBoundary>,
meerkat_runtime::store::RuntimeStoreError,
> {
self.inner
.load_committed_recovery_boundary(runtime_id, candidate_id)
.await
}
async fn load_input_states(
&self,
runtime_id: &LogicalRuntimeId,
) -> Result<Vec<meerkat_runtime::InputStateRow>, meerkat_runtime::store::RuntimeStoreError>
{
self.inner.load_input_states(runtime_id).await
}
async fn load_input_states_with_versions(
&self,
runtime_id: &LogicalRuntimeId,
) -> Result<
meerkat_runtime::store::PreparedRecoveryInputSnapshot,
meerkat_runtime::store::RuntimeStoreError,
> {
self.inner.load_input_states_with_versions(runtime_id).await
}
async fn load_boundary_receipt(
&self,
runtime_id: &LogicalRuntimeId,
run_id: &RunId,
sequence: u64,
) -> Result<
Option<meerkat_core::lifecycle::RunBoundaryReceipt>,
meerkat_runtime::store::RuntimeStoreError,
> {
self.inner
.load_boundary_receipt(runtime_id, run_id, sequence)
.await
}
async fn load_session_snapshot(
&self,
runtime_id: &LogicalRuntimeId,
) -> Result<Option<Arc<Vec<u8>>>, meerkat_runtime::store::RuntimeStoreError> {
self.inner.load_session_snapshot(runtime_id).await
}
async fn load_pending_compaction_projections(
&self,
runtime_id: &LogicalRuntimeId,
) -> Result<
Vec<meerkat_core::CompactionProjectionIntent>,
meerkat_runtime::store::RuntimeStoreError,
> {
self.inner
.load_pending_compaction_projections(runtime_id)
.await
}
async fn mark_compaction_projection_finalized(
&self,
runtime_id: &LogicalRuntimeId,
projection: &meerkat_core::CompactionProjectionId,
) -> Result<(), meerkat_runtime::store::RuntimeStoreError> {
self.inner
.mark_compaction_projection_finalized(runtime_id, projection)
.await
}
async fn clear_session_snapshot(
&self,
runtime_id: &LogicalRuntimeId,
) -> Result<(), meerkat_runtime::store::RuntimeStoreError> {
self.inner.clear_session_snapshot(runtime_id).await
}
async fn replace_session_snapshot_if_current(
&self,
runtime_id: &LogicalRuntimeId,
expected_current: &[u8],
replacement: Vec<u8>,
) -> Result<bool, meerkat_runtime::store::RuntimeStoreError> {
self.inner
.replace_session_snapshot_if_current(runtime_id, expected_current, replacement)
.await
}
async fn clear_session_snapshot_if_current(
&self,
runtime_id: &LogicalRuntimeId,
expected_current: &[u8],
) -> Result<bool, meerkat_runtime::store::RuntimeStoreError> {
self.inner
.clear_session_snapshot_if_current(runtime_id, expected_current)
.await
}
async fn is_runtime_projection_quarantined(
&self,
runtime_id: &LogicalRuntimeId,
) -> Result<bool, meerkat_runtime::store::RuntimeStoreError> {
self.inner
.is_runtime_projection_quarantined(runtime_id)
.await
}
async fn persist_input_state(
&self,
runtime_id: &LogicalRuntimeId,
state: &InputStatePersistenceRecord,
) -> Result<(), meerkat_runtime::store::RuntimeStoreError> {
self.inner.persist_input_state(runtime_id, state).await
}
async fn persist_input_states_atomically(
&self,
runtime_id: &LogicalRuntimeId,
states: &[InputStatePersistenceRecord],
) -> Result<(), meerkat_runtime::store::RuntimeStoreError> {
self.inner
.persist_input_states_atomically(runtime_id, states)
.await
}
async fn compare_and_swap_input_states_atomically(
&self,
runtime_id: &LogicalRuntimeId,
expected: &[StoredInputState],
replacements: &[InputStatePersistenceRecord],
) -> Result<
meerkat_runtime::store::InputStateBatchCasOutcome,
meerkat_runtime::store::RuntimeStoreError,
> {
self.inner
.compare_and_swap_input_states_atomically(runtime_id, expected, replacements)
.await
}
async fn compare_and_swap_input_states_atomically_with_fence(
&self,
runtime_id: &LogicalRuntimeId,
expected: &[StoredInputState],
replacements: &[InputStatePersistenceRecord],
write_fence: Arc<dyn meerkat_runtime::store::RuntimeStoreWriteFence>,
) -> Result<
meerkat_runtime::store::FencedInputStateBatchCasOutcome,
meerkat_runtime::store::RuntimeStoreError,
> {
self.inner
.compare_and_swap_input_states_atomically_with_fence(
runtime_id,
expected,
replacements,
write_fence,
)
.await
}
async fn compare_and_swap_recovery_input_states_atomically(
&self,
runtime_id: &LogicalRuntimeId,
expected_revision: meerkat_runtime::store::RecoveryInputSetRevision,
mutations: &[meerkat_runtime::store::RecoveryInputStateMutation],
) -> Result<
meerkat_runtime::store::InputStateBatchCasOutcome,
meerkat_runtime::store::RuntimeStoreError,
> {
self.inner
.compare_and_swap_recovery_input_states_atomically(
runtime_id,
expected_revision,
mutations,
)
.await
}
async fn compare_and_swap_recovery_input_states_atomically_with_fence(
&self,
runtime_id: &LogicalRuntimeId,
expected_revision: meerkat_runtime::store::RecoveryInputSetRevision,
mutations: &[meerkat_runtime::store::RecoveryInputStateMutation],
write_fence: Arc<dyn meerkat_runtime::store::RuntimeStoreWriteFence>,
) -> Result<
meerkat_runtime::store::FencedInputStateBatchCasOutcome,
meerkat_runtime::store::RuntimeStoreError,
> {
self.inner
.compare_and_swap_recovery_input_states_atomically_with_fence(
runtime_id,
expected_revision,
mutations,
write_fence,
)
.await
}
async fn load_input_state(
&self,
runtime_id: &LogicalRuntimeId,
input_id: &InputId,
) -> Result<Option<StoredInputState>, meerkat_runtime::store::RuntimeStoreError> {
self.inner.load_input_state(runtime_id, input_id).await
}
async fn load_input_state_by_idempotency_key(
&self,
runtime_id: &LogicalRuntimeId,
key: &meerkat_runtime::IdempotencyKey,
) -> Result<
Option<meerkat_runtime::store::ExactInputStateObservation>,
meerkat_runtime::store::RuntimeStoreError,
> {
self.inner
.load_input_state_by_idempotency_key(runtime_id, key)
.await
}
async fn load_input_states_by_ids(
&self,
runtime_id: &LogicalRuntimeId,
input_ids: &[InputId],
) -> Result<Vec<Option<StoredInputState>>, meerkat_runtime::store::RuntimeStoreError>
{
self.inner
.load_input_states_by_ids(runtime_id, input_ids)
.await
}
async fn load_pending_terminal_owner_ids_page(
&self,
runtime_id: &LogicalRuntimeId,
after: Option<&InputId>,
limit: usize,
) -> Result<Vec<InputId>, meerkat_runtime::store::RuntimeStoreError> {
self.inner
.load_pending_terminal_owner_ids_page(runtime_id, after, limit)
.await
}
async fn load_machine_lifecycle_record(
&self,
runtime_id: &LogicalRuntimeId,
) -> Result<Option<Vec<u8>>, meerkat_runtime::store::RuntimeStoreError> {
self.inner.load_machine_lifecycle_record(runtime_id).await
}
async fn commit_machine_lifecycle(
&self,
runtime_id: &LogicalRuntimeId,
commit: meerkat_runtime::store::MachineLifecycleCommit,
input_states: &[InputStatePersistenceRecord],
) -> Result<(), meerkat_runtime::store::RuntimeStoreError> {
self.inner
.commit_machine_lifecycle(runtime_id, commit, input_states)
.await
}
async fn commit_unregister_finalization(
&self,
runtime_id: &LogicalRuntimeId,
finalization: meerkat_runtime::store::UnregisterFinalizationCommit,
) -> Result<(), meerkat_runtime::store::RuntimeStoreError> {
self.inner
.commit_unregister_finalization(runtime_id, finalization)
.await
}
async fn persist_ops_lifecycle(
&self,
runtime_id: &LogicalRuntimeId,
snapshot: &meerkat_runtime::ops_lifecycle::PersistedOpsSnapshot,
) -> Result<(), meerkat_runtime::store::RuntimeStoreError> {
self.inner.persist_ops_lifecycle(runtime_id, snapshot).await
}
async fn initialize_ops_lifecycle_if_absent(
&self,
runtime_id: &LogicalRuntimeId,
candidate: &meerkat_runtime::ops_lifecycle::PersistedOpsSnapshot,
) -> Result<
meerkat_runtime::ops_lifecycle::PersistedOpsSnapshot,
meerkat_runtime::store::RuntimeStoreError,
> {
self.inner
.initialize_ops_lifecycle_if_absent(runtime_id, candidate)
.await
}
async fn load_ops_lifecycle(
&self,
runtime_id: &LogicalRuntimeId,
) -> Result<
Option<meerkat_runtime::ops_lifecycle::PersistedOpsSnapshot>,
meerkat_runtime::store::RuntimeStoreError,
> {
self.inner.load_ops_lifecycle(runtime_id).await
}
async fn delete_ops_lifecycle(
&self,
runtime_id: &LogicalRuntimeId,
) -> Result<(), meerkat_runtime::store::RuntimeStoreError> {
self.inner.delete_ops_lifecycle(runtime_id).await
}
}
#[tokio::test]
async fn boundary_and_rewrite_snapshot_writes_are_single_document_sized() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let counting = Arc::new(CountingRuntimeStore::new());
let runtime_store: Arc<dyn RuntimeStore> = Arc::clone(&counting) as Arc<dyn RuntimeStore>;
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store),
memory_blob_store(),
);
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create_session should succeed");
let session_id = created.session_id;
let runtime_id =
PersistentSessionService::<DummyBuilder>::runtime_id_for_session(&session_id);
committed_content_turn(&service, runtime_store.as_ref(), &session_id, "hello").await;
counting.reset();
let encode_before = meerkat_core::global_session_encode_bytes();
committed_content_turn(&service, runtime_store.as_ref(), &session_id, "follow up").await;
let boundary_doc = runtime_store
.load_session_snapshot(&runtime_id)
.await
.expect("boundary snapshot should load")
.expect("boundary snapshot should exist");
assert_eq!(
counting.boundary_apply_snapshots.load(Ordering::Acquire),
1,
"one committed turn must carry exactly one boundary snapshot"
);
assert_eq!(
counting.boundary_apply_bytes.load(Ordering::Acquire),
boundary_doc.len() as u64,
"the boundary snapshot payload must be exactly one whole document"
);
assert_eq!(
counting.snapshot_commits.load(Ordering::Acquire),
0,
"a committed turn must not add side-channel whole-document snapshot commits"
);
assert_eq!(
counting.rewrite_snapshot_commits.load(Ordering::Acquire),
0,
"a committed turn must not write rewrite snapshots"
);
println!(
"boundary save: {} bytes to the runtime store, {} bytes encoded",
boundary_doc.len(),
meerkat_core::global_session_encode_bytes() - encode_before,
);
let parent_revision = service
.load_authoritative_session_base(&session_id)
.await
.expect("load runtime authority before rewrite")
.expect("session exists")
.transcript_revision()
.expect("parent revision");
counting.reset();
let encode_before = meerkat_core::global_session_encode_bytes();
let result = service
.rewrite_session_transcript(
&session_id,
meerkat_core::SessionTranscriptRewriteRequest {
selection: TranscriptRewriteSelection::MessageRange { start: 1, end: 4 },
replacement: vec![Message::BlockAssistant(
meerkat_core::BlockAssistantMessage::new(
vec![meerkat_core::AssistantBlock::Text {
text: "compacted assistant trace".to_string(),
meta: None,
}],
StopReason::EndTurn,
),
)],
reason: TranscriptRewriteReason::new("compaction"),
actor: Some("test".to_string()),
expected_parent_revision: Some(parent_revision),
running_behavior: TranscriptEditRunningBehavior::Reject,
},
)
.await
.expect("rewrite should commit");
assert_ne!(result.revision, result.parent_revision);
let rewrite_doc = runtime_store
.load_session_snapshot(&runtime_id)
.await
.expect("rewrite snapshot should load")
.expect("rewrite snapshot should exist");
assert_eq!(
counting.rewrite_snapshot_commits.load(Ordering::Acquire),
1,
"one persisted rewrite commit must write exactly one rewrite snapshot"
);
assert_eq!(
counting.rewrite_snapshot_bytes.load(Ordering::Acquire),
rewrite_doc.len() as u64,
"the rewrite snapshot payload must be exactly one whole document"
);
assert_eq!(
counting.snapshot_commits.load(Ordering::Acquire),
0,
"an anchored single-commit rewrite must not write bridge or post-rewrite snapshots"
);
assert_eq!(
counting.boundary_apply_snapshots.load(Ordering::Acquire),
0,
"a rewrite must not masquerade as a boundary apply"
);
println!(
"rewrite-chain save: {} bytes to the runtime store, {} bytes encoded",
rewrite_doc.len(),
meerkat_core::global_session_encode_bytes() - encode_before,
);
}
#[tokio::test]
async fn whole_blob_k32_rewrite_is_one_encode_cas_and_receipt() {
const REWRITE_COUNT: usize = 32;
let projection_store = Arc::new(CountingProjectionSessionStore::new());
let store: Arc<dyn SessionStore> = Arc::clone(&projection_store) as Arc<dyn SessionStore>;
let counting = Arc::new(CountingRuntimeStore::new());
let runtime_store: Arc<dyn RuntimeStore> = Arc::clone(&counting) as Arc<dyn RuntimeStore>;
let recording_events = Arc::new(RecordingEventStore::default());
let event_store: Arc<dyn EventStore> = Arc::clone(&recording_events) as Arc<dyn EventStore>;
let projector_root = tempfile::tempdir().expect("projector tempdir");
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store),
memory_blob_store(),
)
.with_event_projection(
event_store,
Arc::new(SessionProjector::new(projector_root.path().join(".rkat"))),
);
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create_session should succeed");
let session_id = created.session_id;
let runtime_id =
PersistentSessionService::<DummyBuilder>::runtime_id_for_session(&session_id);
committed_content_turn(&service, runtime_store.as_ref(), &session_id, "hello").await;
let mut rewritten = service
.load_authoritative_session_base(&session_id)
.await
.expect("load runtime authority before rewrite chain")
.expect("session exists");
let mut commits = Vec::with_capacity(REWRITE_COUNT);
for index in 0..REWRITE_COUNT {
let parent_revision = rewritten
.transcript_revision()
.expect("rewrite parent revision");
let commit = rewritten
.commit_transcript_rewrite(
TranscriptRewriteSelection::MessageRange { start: 1, end: 2 },
vec![Message::User(UserMessage::text(format!(
"compacted rewrite {index}"
)))],
TranscriptRewriteReason::new("k32 whole-blob acceptance"),
Some("test".to_string()),
Some(parent_revision),
)
.expect("rewrite occurrence should commit");
commits.push(commit);
}
let rewritten = service
.normalized_session_for_persistence(rewritten)
.await
.expect("rewrite chain should normalize");
let (receipt_rows_before, body_rows_before) = {
let events = recording_events.events.lock().await;
let session_events = events.get(&session_id).map(Vec::as_slice).unwrap_or(&[]);
(
session_events
.iter()
.filter(|stored| {
matches!(
stored.event,
AgentEvent::TranscriptRewriteAuditReceiptCommitted { .. }
)
})
.count(),
session_events
.iter()
.filter(|stored| {
matches!(stored.event, AgentEvent::TranscriptRewriteCommitted { .. })
})
.count(),
)
};
counting.reset();
projection_store.reset();
let receipt_append_calls_before = recording_events
.rewrite_append_calls
.load(Ordering::Acquire);
let persisted = service
.persist_normalized_transcript_rewrite_chain(rewritten, &commits, false)
.await
.expect("k=32 rewrite chain should persist");
let rewrite_doc = runtime_store
.load_session_snapshot(&runtime_id)
.await
.expect("rewrite snapshot should load")
.expect("rewrite snapshot should exist");
let (receipt_rows_after, body_rows_after, receipt_commit_count) = {
let events = recording_events.events.lock().await;
let session_events = events
.get(&session_id)
.expect("rewrite receipt should create a session event row");
let receipt_commit_count = session_events
.iter()
.find_map(|stored| match &stored.event {
AgentEvent::TranscriptRewriteAuditReceiptCommitted { receipt, .. } => {
Some(receipt.commits().len())
}
_ => None,
})
.expect("rewrite receipt should be present");
(
session_events
.iter()
.filter(|stored| {
matches!(
stored.event,
AgentEvent::TranscriptRewriteAuditReceiptCommitted { .. }
)
})
.count(),
session_events
.iter()
.filter(|stored| {
matches!(stored.event, AgentEvent::TranscriptRewriteCommitted { .. })
})
.count(),
receipt_commit_count,
)
};
assert_eq!(
counting.rewrite_snapshot_cas_calls.load(Ordering::Acquire),
1,
"one logical rewrite suffix must cross one RuntimeStore CAS"
);
assert_eq!(
counting.rewrite_snapshot_commits.load(Ordering::Acquire),
1,
"one logical rewrite suffix must write one final RuntimeStore document"
);
assert_eq!(
counting.rewrite_snapshot_bytes.load(Ordering::Acquire),
rewrite_doc.len() as u64,
"the only RuntimeStore rewrite payload must be the final document"
);
assert_eq!(
projection_store
.authoritative_projection_calls
.load(Ordering::Acquire),
0,
"one logical rewrite suffix must not duplicate RuntimeStore authority into SessionStore"
);
assert_eq!(
counting
.rewrite_successor_encode_bytes
.load(Ordering::Acquire),
rewrite_doc.len() as u64,
"the successor must be whole-document encoded exactly once"
);
assert_eq!(receipt_rows_after - receipt_rows_before, 1);
assert_eq!(receipt_commit_count, REWRITE_COUNT);
assert_eq!(
recording_events
.rewrite_append_calls
.load(Ordering::Acquire)
- receipt_append_calls_before,
1,
"one logical rewrite suffix must make one exact receipt append"
);
assert_eq!(
body_rows_after - body_rows_before,
0,
"current writers must emit no full-body transcript rewrite rows"
);
counting.reset();
projection_store.reset();
let retry_receipt_rows_before = receipt_rows_after;
let retry_receipt_calls_before = recording_events
.rewrite_append_calls
.load(Ordering::Acquire);
let retried = service
.persist_normalized_transcript_rewrite_chain(persisted, &commits, false)
.await
.expect("exact k=32 retry should heal idempotently");
let retry_receipt_rows_after = recording_events
.events
.lock()
.await
.get(&session_id)
.into_iter()
.flatten()
.filter(|stored| {
matches!(
stored.event,
AgentEvent::TranscriptRewriteAuditReceiptCommitted { .. }
)
})
.count();
assert_eq!(retried.id(), &session_id);
assert_eq!(
counting.rewrite_snapshot_cas_calls.load(Ordering::Acquire),
1,
"exact healing still performs one authority check"
);
assert_eq!(
counting.rewrite_snapshot_commits.load(Ordering::Acquire),
0,
"exact healing must not rewrite the RuntimeStore document"
);
assert_eq!(
counting.rewrite_snapshot_bytes.load(Ordering::Acquire),
0,
"exact healing must cross no RuntimeStore document bytes"
);
assert_eq!(
counting
.rewrite_successor_encode_bytes
.load(Ordering::Acquire),
0,
"exact healing must reuse the already-landed document bytes"
);
assert_eq!(
projection_store
.authoritative_projection_calls
.load(Ordering::Acquire),
0,
"healing must not consult or rewrite a non-authoritative SessionStore projection"
);
assert_eq!(
recording_events
.rewrite_append_calls
.load(Ordering::Acquire)
- retry_receipt_calls_before,
1,
"exact healing must perform one receipt identity check"
);
assert_eq!(
retry_receipt_rows_after, retry_receipt_rows_before,
"exact receipt publication must replay rather than append"
);
}
#[tokio::test]
async fn test_persistent_fork_at_creates_new_idle_session_without_shrinking_parent() {
let (service, _store, _runtime_store, parent_id) = transcript_edit_fixture().await;
let forked = service
.fork_session_at(
&parent_id,
SessionForkAtRequest {
message_index: 2,
running_behavior: TranscriptEditRunningBehavior::Reject,
tool_access_policy: None,
},
)
.await
.expect("fork_at should create a branch session");
assert_ne!(forked.session_id, parent_id);
assert_eq!(forked.message_count, 2);
let parent_history = service
.read_history(&parent_id, SessionHistoryQuery::default())
.await
.expect("parent history should remain readable");
assert_eq!(parent_history.message_count, 4);
let fork_history = service
.read_history(&forked.session_id, SessionHistoryQuery::default())
.await
.expect("fork history should be persisted");
assert_eq!(fork_history.message_count, 2);
assert_eq!(fork_history.messages.len(), 2);
}
#[tokio::test]
async fn transcript_fork_persists_call_scoped_tool_policy_on_branch_only() {
let (service, _store, runtime_store, parent_id) = transcript_edit_fixture().await;
let source_policy = meerkat_core::ops::ToolAccessPolicy::DenyList(
["shell".to_string()].into_iter().collect(),
);
let parent = service
.load_authoritative_session_base(&parent_id)
.await
.expect("load runtime-authoritative parent")
.expect("parent exists");
let parent = mutate_test_session(parent, "run boundary", |parent| {
parent
.set_session_metadata(meerkat_core::SessionMetadata {
schema_version: meerkat_core::SESSION_METADATA_SCHEMA_VERSION,
model: "test".to_string(),
max_tokens: 1024,
structured_output_retries: 2,
provider: meerkat_core::Provider::Anthropic,
self_hosted_server_id: None,
provider_params: None,
tooling: meerkat_core::SessionTooling {
tool_access_policy: Some(source_policy.clone()),
..meerkat_core::SessionTooling::default()
},
keep_alive: false,
comms_name: None,
peer_meta: None,
realm_id: None,
instance_id: None,
backend: None,
config_generation: None,
auth_binding: None,
mob_member_binding: None,
})
.expect("seed canonical parent metadata");
});
runtime_store
.commit_session_snapshot(
&PersistentSessionService::<DummyBuilder>::runtime_id_for_session(&parent_id),
meerkat_runtime::store::SerializedSessionSnapshot {
session_snapshot: (serde_json::to_vec(&parent)
.expect("serialize parent metadata snapshot"))
.into(),
},
)
.await
.expect("commit parent metadata snapshot");
service
.discard_live_session(&parent_id)
.await
.expect("force fork source through the durable runtime snapshot");
let policy = meerkat_core::ops::ToolAccessPolicy::AllowList(
["memory_search".to_string()].into_iter().collect(),
);
let forked = service
.fork_session_at(
&parent_id,
SessionForkAtRequest {
message_index: 2,
running_behavior: TranscriptEditRunningBehavior::Reject,
tool_access_policy: Some(policy.clone()),
},
)
.await
.expect("fork policy should persist");
let branch = service
.load_authoritative_session_base(&forked.session_id)
.await
.expect("load runtime-authoritative branch")
.expect("branch exists");
assert_eq!(
branch
.try_session_metadata()
.expect("decode branch metadata")
.expect("branch metadata")
.tooling
.tool_access_policy,
Some(policy)
);
assert_eq!(
service
.load_authoritative_session_base(&parent_id)
.await
.expect("load runtime-authoritative parent")
.expect("parent exists")
.try_session_metadata()
.expect("decode parent metadata")
.expect("parent metadata")
.tooling
.tool_access_policy,
Some(source_policy.clone()),
"call-scoped fork authorization must not mutate the source"
);
for requested_policy in [None, Some(meerkat_core::ops::ToolAccessPolicy::Inherit)] {
let inherited = service
.fork_session_at(
&parent_id,
SessionForkAtRequest {
message_index: 2,
running_behavior: TranscriptEditRunningBehavior::Reject,
tool_access_policy: requested_policy,
},
)
.await
.expect("default/Inherit fork should retain source authorization");
let inherited = service
.load_authoritative_session_base(&inherited.session_id)
.await
.expect("load runtime-authoritative inherited branch")
.expect("inherited branch exists");
assert_eq!(
inherited
.try_session_metadata()
.expect("decode inherited branch metadata")
.expect("inherited branch metadata")
.tooling
.tool_access_policy,
Some(source_policy.clone()),
"None/Inherit must preserve the source session's effective policy"
);
}
}
#[tokio::test]
async fn test_persistent_fork_replace_message_creates_changed_branch() {
let (service, _store, _runtime_store, parent_id) = transcript_edit_fixture().await;
let forked = service
.fork_session_replace(
&parent_id,
SessionForkReplaceRequest {
message_index: 2,
replacement: meerkat_core::TranscriptReplacement::Message {
message: Message::User(UserMessage::text("edited follow up")),
},
running_behavior: TranscriptEditRunningBehavior::Reject,
tool_access_policy: None,
},
)
.await
.expect("fork_replace should create a branch session");
assert_ne!(forked.session_id, parent_id);
assert_eq!(forked.message_count, 3);
let fork_history = service
.read_history(&forked.session_id, SessionHistoryQuery::default())
.await
.expect("fork history should be persisted");
assert_eq!(fork_history.message_count, 3);
assert!(matches!(
&fork_history.messages[2],
Message::User(user) if user.text_content() == "edited follow up"
));
let parent_history = service
.read_history(&parent_id, SessionHistoryQuery::default())
.await
.expect("parent history should remain unchanged");
assert!(matches!(
&parent_history.messages[2],
Message::User(user) if user.text_content() == "follow up"
));
}
#[tokio::test]
async fn test_persistent_rewrite_transcript_advances_same_session_head() {
let (service, _store, _runtime_store, session_id) = transcript_edit_fixture().await;
let before = service
.read_history(&session_id, SessionHistoryQuery::default())
.await
.expect("history before rewrite");
assert_eq!(before.message_count, 4);
let parent_revision = service
.load_authoritative_session(&session_id)
.await
.expect("load RuntimeStore authority before rewrite")
.expect("session exists")
.transcript_revision()
.expect("parent revision");
let result = service
.rewrite_session_transcript(
&session_id,
meerkat_core::SessionTranscriptRewriteRequest {
selection: TranscriptRewriteSelection::MessageRange { start: 1, end: 4 },
replacement: vec![Message::BlockAssistant(
meerkat_core::BlockAssistantMessage::new(
vec![meerkat_core::AssistantBlock::Text {
text: "compacted assistant trace".to_string(),
meta: None,
}],
StopReason::EndTurn,
),
)],
reason: TranscriptRewriteReason::new("compaction"),
actor: Some("test".to_string()),
expected_parent_revision: Some(parent_revision.clone()),
running_behavior: TranscriptEditRunningBehavior::Reject,
},
)
.await
.expect("rewrite should commit");
assert_eq!(result.session_id, session_id);
assert_eq!(result.parent_revision, parent_revision);
assert_ne!(result.revision, result.parent_revision);
assert_eq!(result.message_count, 2);
let after = service
.read_history(&session_id, SessionHistoryQuery::default())
.await
.expect("history after rewrite");
assert_eq!(after.session_id, session_id);
assert_eq!(after.message_count, 2);
assert!(matches!(
&after.messages[1],
Message::BlockAssistant(assistant)
if assistant.to_string() == "compacted assistant trace"
));
let saved = service
.load_authoritative_session(&session_id)
.await
.expect("load RuntimeStore authority after rewrite")
.expect("session exists");
assert_eq!(
saved.transcript_revision().expect("saved revision"),
result.revision
);
let graph = saved
.validated_transcript_history_state()
.expect("runtime authority graph should validate")
.expect("runtime authority graph should exist");
assert_eq!(graph.commit_count(), 1);
let commit = graph.last_commit().expect("rewrite commit should exist");
assert_eq!(commit.parent_revision, parent_revision);
assert_eq!(commit.revision, result.revision);
let parent_page = service
.read_transcript_revision(
&session_id,
SessionTranscriptRevisionQuery {
revision: parent_revision,
offset: 0,
limit: None,
},
)
.await
.expect("parent revision should be recoverable");
assert_eq!(parent_page.message_count, 4);
assert_eq!(
serde_json::to_value(&parent_page.messages).expect("parent page serializes"),
serde_json::to_value(&before.messages).expect("before history serializes")
);
let restored = service
.restore_session_transcript_revision(
&session_id,
SessionTranscriptRestoreRevisionRequest {
revision: result.parent_revision.clone(),
reason: TranscriptRewriteReason::new("restore"),
actor: Some("test".to_string()),
expected_parent_revision: Some(result.revision.clone()),
running_behavior: TranscriptEditRunningBehavior::Reject,
},
)
.await
.expect("restore should commit");
assert_eq!(restored.session_id, session_id);
assert_eq!(restored.parent_revision, result.revision);
assert_eq!(restored.revision, result.parent_revision);
assert_eq!(restored.message_count, before.message_count);
let restored_history = service
.read_history(&session_id, SessionHistoryQuery::default())
.await
.expect("history after restore");
assert_eq!(
serde_json::to_value(&restored_history.messages).expect("restored history serializes"),
serde_json::to_value(&before.messages).expect("before history serializes")
);
let restored_saved = service
.load_authoritative_session(&session_id)
.await
.expect("load RuntimeStore authority after restore")
.expect("session exists");
assert_eq!(
restored_saved.transcript_revision().expect("revision"),
restored.revision
);
let restored_graph = restored_saved
.validated_transcript_history_state()
.expect("restored runtime authority graph should validate")
.expect("restored runtime authority graph should exist");
assert_eq!(restored_graph.commit_count(), 2);
assert_eq!(
restored_graph
.last_commit()
.expect("restore commit should exist")
.replacement_digest,
meerkat_core::transcript_messages_digest(&before.messages)
.expect("before history digest should compute")
);
}
#[tokio::test]
async fn test_persistent_rewrite_transcript_rejects_stale_parent_revision() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store: Arc<dyn RuntimeStore> = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store),
memory_blob_store(),
);
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create_session should succeed");
let session_id = created.session_id;
committed_content_turn(&service, runtime_store.as_ref(), &session_id, "hello").await;
let err = service
.rewrite_session_transcript(
&session_id,
meerkat_core::SessionTranscriptRewriteRequest {
selection: TranscriptRewriteSelection::MessageRange { start: 1, end: 2 },
replacement: vec![Message::BlockAssistant(
meerkat_core::BlockAssistantMessage::new(
vec![meerkat_core::AssistantBlock::Text {
text: "replacement".to_string(),
meta: None,
}],
StopReason::EndTurn,
),
)],
reason: TranscriptRewriteReason::new("correction"),
actor: None,
expected_parent_revision: Some("sha256:not-the-head".to_string()),
running_behavior: TranscriptEditRunningBehavior::Reject,
},
)
.await
.expect_err("stale parent revision must fail");
assert!(
err.to_string().contains("parent revision mismatch"),
"unexpected error: {err}"
);
}
#[tokio::test]
async fn test_persistent_rewrite_transcript_maps_concurrent_conflict_to_revision_conflict() {
let (service, _store, _runtime_store, session_id) = transcript_edit_fixture().await;
let stale_parent_revision = service
.load_authoritative_session(&session_id)
.await
.expect("authoritative load should succeed")
.expect("session exists")
.transcript_revision()
.expect("parent revision");
service
.rewrite_session_transcript(
&session_id,
meerkat_core::SessionTranscriptRewriteRequest {
selection: TranscriptRewriteSelection::MessageRange { start: 1, end: 2 },
replacement: vec![Message::BlockAssistant(
meerkat_core::BlockAssistantMessage::new(
vec![meerkat_core::AssistantBlock::Text {
text: "first writer wins".to_string(),
meta: None,
}],
StopReason::EndTurn,
),
)],
reason: TranscriptRewriteReason::new("correction"),
actor: None,
expected_parent_revision: Some(stale_parent_revision.clone()),
running_behavior: TranscriptEditRunningBehavior::Reject,
},
)
.await
.expect("first rewrite should commit");
let err = service
.rewrite_session_transcript(
&session_id,
meerkat_core::SessionTranscriptRewriteRequest {
selection: TranscriptRewriteSelection::MessageRange { start: 1, end: 2 },
replacement: vec![Message::BlockAssistant(
meerkat_core::BlockAssistantMessage::new(
vec![meerkat_core::AssistantBlock::Text {
text: "second writer must conflict".to_string(),
meta: None,
}],
StopReason::EndTurn,
),
)],
reason: TranscriptRewriteReason::new("correction"),
actor: None,
expected_parent_revision: Some(stale_parent_revision),
running_behavior: TranscriptEditRunningBehavior::Reject,
},
)
.await
.expect_err("concurrent head advance must fail the stale rewrite");
assert!(
matches!(
err,
SessionError::Agent(meerkat_core::AgentError::ConfigError(_))
),
"concurrent conflict should surface as typed revision conflict, not store/internal: {err:?}"
);
assert!(
err.to_string().contains("parent revision mismatch"),
"unexpected error: {err}"
);
}
#[tokio::test]
async fn test_persistent_list_transcript_revisions_pages_and_reports_head() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store: Arc<dyn RuntimeStore> = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store),
memory_blob_store(),
);
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create_session should succeed");
let session_id = created.session_id;
committed_content_turn(&service, runtime_store.as_ref(), &session_id, "hello").await;
let original_revision = service
.load_authoritative_session(&session_id)
.await
.expect("load RuntimeStore authority before rewrites")
.expect("session exists")
.transcript_revision()
.expect("original revision");
let first = service
.rewrite_session_transcript(
&session_id,
meerkat_core::SessionTranscriptRewriteRequest {
selection: TranscriptRewriteSelection::MessageRange { start: 1, end: 2 },
replacement: vec![Message::BlockAssistant(
meerkat_core::BlockAssistantMessage::new(
vec![meerkat_core::AssistantBlock::Text {
text: "first rewrite".to_string(),
meta: None,
}],
StopReason::EndTurn,
),
)],
reason: TranscriptRewriteReason::new("correction"),
actor: None,
expected_parent_revision: None,
running_behavior: TranscriptEditRunningBehavior::Reject,
},
)
.await
.expect("first rewrite should commit");
let second = service
.rewrite_session_transcript(
&session_id,
meerkat_core::SessionTranscriptRewriteRequest {
selection: TranscriptRewriteSelection::MessageRange { start: 1, end: 2 },
replacement: vec![Message::BlockAssistant(
meerkat_core::BlockAssistantMessage::new(
vec![meerkat_core::AssistantBlock::Text {
text: "second rewrite".to_string(),
meta: None,
}],
StopReason::EndTurn,
),
)],
reason: TranscriptRewriteReason {
kind: "polish".to_string(),
note: Some("fix tone".to_string()),
},
actor: Some("editor".to_string()),
expected_parent_revision: None,
running_behavior: TranscriptEditRunningBehavior::Reject,
},
)
.await
.expect("second rewrite should commit");
let list = service
.list_transcript_revisions(
&session_id,
meerkat_core::SessionTranscriptRevisionListQuery::default(),
)
.await
.expect("list should succeed");
assert_eq!(list.head_revision, second.revision);
assert_eq!(list.entries.len(), 2);
assert_eq!(list.entries[0].revision, first.revision);
assert_eq!(list.entries[0].parent_revision, original_revision);
assert_eq!(list.entries[0].actor, None);
assert_eq!(list.entries[0].reason, "correction");
assert_eq!(list.entries[1].revision, second.revision);
assert_eq!(list.entries[1].parent_revision, first.revision);
assert_eq!(list.entries[1].actor.as_deref(), Some("editor"));
assert_eq!(
list.entries[1].reason, "polish: fix tone",
"reason must be the Display projection of the typed rewrite reason"
);
let first_page = service
.list_transcript_revisions(
&session_id,
meerkat_core::SessionTranscriptRevisionListQuery {
limit: Some(1),
offset: None,
},
)
.await
.expect("limited list should succeed");
assert_eq!(first_page.head_revision, second.revision);
assert_eq!(first_page.entries.len(), 1);
assert_eq!(first_page.entries[0].revision, first.revision);
let second_page = service
.list_transcript_revisions(
&session_id,
meerkat_core::SessionTranscriptRevisionListQuery {
limit: Some(1),
offset: Some(1),
},
)
.await
.expect("offset list should succeed");
assert_eq!(second_page.entries.len(), 1);
assert_eq!(second_page.entries[0].revision, second.revision);
let beyond = service
.list_transcript_revisions(
&session_id,
meerkat_core::SessionTranscriptRevisionListQuery {
limit: None,
offset: Some(5),
},
)
.await
.expect("out-of-range offset should succeed");
assert!(beyond.entries.is_empty());
assert_eq!(
beyond.head_revision, second.revision,
"the head revision is reported even when the requested page is empty"
);
}
#[tokio::test]
async fn test_persistent_list_transcript_revisions_empty_before_first_rewrite() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store: Arc<dyn RuntimeStore> = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store),
memory_blob_store(),
);
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create_session should succeed");
let session_id = created.session_id;
committed_content_turn(&service, runtime_store.as_ref(), &session_id, "hello").await;
let head_revision = service
.load_authoritative_session(&session_id)
.await
.expect("load session")
.expect("session exists")
.transcript_revision()
.expect("head revision");
let list = service
.list_transcript_revisions(
&session_id,
meerkat_core::SessionTranscriptRevisionListQuery::default(),
)
.await
.expect("list should succeed for a pre-revision session");
assert!(
list.entries.is_empty(),
"a session without rewrite commits has a legitimately empty commit log"
);
assert_eq!(
list.head_revision, head_revision,
"the implicit digest head must still be reported"
);
}
#[tokio::test]
async fn test_persistent_restore_current_revision_surfaces_noop_rewrite() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store: Arc<dyn RuntimeStore> = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store),
memory_blob_store(),
);
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create_session should succeed");
let session_id = created.session_id;
committed_content_turn(&service, runtime_store.as_ref(), &session_id, "hello").await;
let rewritten = service
.rewrite_session_transcript(
&session_id,
meerkat_core::SessionTranscriptRewriteRequest {
selection: TranscriptRewriteSelection::MessageRange { start: 1, end: 2 },
replacement: vec![Message::BlockAssistant(
meerkat_core::BlockAssistantMessage::new(
vec![meerkat_core::AssistantBlock::Text {
text: "rewritten".to_string(),
meta: None,
}],
StopReason::EndTurn,
),
)],
reason: TranscriptRewriteReason::new("correction"),
actor: None,
expected_parent_revision: None,
running_behavior: TranscriptEditRunningBehavior::Reject,
},
)
.await
.expect("rewrite should commit");
committed_content_turn(
&service,
runtime_store.as_ref(),
&session_id,
"after rewrite",
)
.await;
let appended = service
.load_authoritative_session(&session_id)
.await
.expect("load appended rewritten session")
.expect("session exists");
let appended_revision = appended
.transcript_revision()
.expect("appended live revision");
let history = appended
.validated_transcript_history_state()
.expect("history state should decode")
.expect("rewritten session should retain audited history");
assert_eq!(history.head(), rewritten.revision);
assert_ne!(history.head(), appended_revision);
assert!(
!history.contains_revision(&appended_revision),
"ordinary append must not manufacture a retained full live body"
);
let current = service
.read_transcript_revision(
&session_id,
SessionTranscriptRevisionQuery {
revision: "current".to_string(),
offset: 0,
limit: None,
},
)
.await
.expect("current revision should resolve from the live appended transcript");
assert_eq!(current.revision, appended_revision);
assert_eq!(current.message_count, appended.messages().len());
let runtime_id =
PersistentSessionService::<DummyBuilder>::runtime_id_for_session(&session_id);
let authority_before = runtime_store
.load_whole_blob_store_authority(&runtime_id)
.await
.expect("load store-owned authority before no-op restore")
.expect("store-owned authority exists before no-op restore");
let err = service
.restore_session_transcript_revision(
&session_id,
SessionTranscriptRestoreRevisionRequest {
revision: "current".to_string(),
reason: TranscriptRewriteReason::new("restore"),
actor: None,
expected_parent_revision: None,
running_behavior: TranscriptEditRunningBehavior::Reject,
},
)
.await
.expect_err(
"restoring an appended current head without a retained live body must surface the typed no-op rewrite",
);
assert!(
err.to_string()
.contains("does not change transcript revision"),
"unexpected error: {err}"
);
let authority_after = runtime_store
.load_whole_blob_store_authority(&runtime_id)
.await
.expect("load store-owned authority after no-op restore")
.expect("store-owned authority exists after no-op restore");
assert_eq!(
authority_after, authority_before,
"a no-op restore must not advance or replace WholeBlob store authority"
);
}
#[tokio::test]
async fn test_persistent_restore_current_on_pre_revision_session_surfaces_noop_rewrite() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store: Arc<dyn RuntimeStore> = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store),
memory_blob_store(),
);
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create_session should succeed");
let session_id = created.session_id;
committed_content_turn(&service, runtime_store.as_ref(), &session_id, "hello").await;
let err = service
.restore_session_transcript_revision(
&session_id,
SessionTranscriptRestoreRevisionRequest {
revision: "current".to_string(),
reason: TranscriptRewriteReason::new("restore"),
actor: None,
expected_parent_revision: None,
running_behavior: TranscriptEditRunningBehavior::Reject,
},
)
.await
.expect_err("restoring the digest head must surface the typed no-op rewrite");
assert!(
err.to_string()
.contains("does not change transcript revision"),
"unexpected error: {err}"
);
}
#[tokio::test]
async fn test_persistent_restore_unknown_revision_fails_closed() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store: Arc<dyn RuntimeStore> = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store),
memory_blob_store(),
);
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create_session should succeed");
let session_id = created.session_id;
committed_content_turn(&service, runtime_store.as_ref(), &session_id, "hello").await;
let err = service
.restore_session_transcript_revision(
&session_id,
SessionTranscriptRestoreRevisionRequest {
revision: "sha256:absent".to_string(),
reason: TranscriptRewriteReason::new("restore"),
actor: None,
expected_parent_revision: None,
running_behavior: TranscriptEditRunningBehavior::Reject,
},
)
.await
.expect_err("an unknown concrete revision id must fail closed");
assert!(
err.to_string().contains("not found"),
"unexpected error: {err}"
);
}
#[tokio::test]
async fn test_read_current_transcript_revision_works_without_rewrite_graph() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store: Arc<dyn RuntimeStore> = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store),
memory_blob_store(),
);
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create_session should succeed");
let session_id = created.session_id;
committed_content_turn(&service, runtime_store.as_ref(), &session_id, "hello").await;
let session = service
.load_authoritative_session(&session_id)
.await
.expect("load saved session")
.expect("session exists");
assert!(
session
.validated_transcript_history_state()
.expect("history state should decode")
.is_none()
);
let head_revision = session.transcript_revision().expect("implicit head digest");
let current = service
.read_transcript_revision(
&session_id,
SessionTranscriptRevisionQuery {
revision: "current".to_string(),
offset: 0,
limit: None,
},
)
.await
.expect("current alias should page over current messages without rewrite metadata");
assert_eq!(current.revision, head_revision);
assert_eq!(current.head_revision, head_revision);
assert_eq!(current.message_count, session.messages().len());
}
#[tokio::test]
async fn test_persistent_rewrite_transcript_excludes_runtime_turn_admission_until_commit_finishes()
{
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let pausing_runtime_store = Arc::new(GatedSnapshotRuntimeStore::new());
let runtime_store: Arc<dyn RuntimeStore> = pausing_runtime_store.clone();
let service = Arc::new(PersistentSessionService::new(
DummyBuilder,
1,
Arc::clone(&store),
Arc::clone(&runtime_store),
memory_blob_store(),
));
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create_session should succeed");
let session_id = created.session_id;
committed_content_turn(
service.as_ref(),
runtime_store.as_ref(),
&session_id,
"hello",
)
.await;
let parent_revision = service
.load_authoritative_session(&session_id)
.await
.expect("load before rewrite")
.expect("session exists")
.transcript_revision()
.expect("parent revision");
pausing_runtime_store.pause_next_rewrite_commit();
let rewrite_service = Arc::clone(&service);
let rewrite_session_id = session_id.clone();
let rewrite = tokio::spawn(async move {
rewrite_service
.rewrite_session_transcript(
&rewrite_session_id,
meerkat_core::SessionTranscriptRewriteRequest {
selection: TranscriptRewriteSelection::MessageRange { start: 1, end: 2 },
replacement: vec![Message::BlockAssistant(
meerkat_core::BlockAssistantMessage {
blocks: vec![meerkat_core::AssistantBlock::Text {
text: "compact answer".to_string(),
meta: None,
}],
stop_reason: StopReason::EndTurn,
identity: meerkat_core::types::TranscriptMessageIdentity::default(),
created_at: meerkat_core::types::message_timestamp_now(),
},
)],
reason: TranscriptRewriteReason::new("compaction"),
actor: Some("test".to_string()),
expected_parent_revision: Some(parent_revision),
running_behavior: TranscriptEditRunningBehavior::Reject,
},
)
.await
});
pausing_runtime_store.wait_for_rewrite_commit().await;
let blocked_admission = tokio::time::timeout(
std::time::Duration::from_millis(100),
service.reserve_runtime_turn_admission(&session_id),
)
.await;
assert!(
blocked_admission.is_err(),
"runtime turn admission must wait while transcript rewrite is committing"
);
pausing_runtime_store.release_rewrite_commit();
rewrite
.await
.expect("rewrite task should join")
.expect("rewrite should commit");
let admission = tokio::time::timeout(
std::time::Duration::from_secs(10),
service.reserve_runtime_turn_admission(&session_id),
)
.await
.expect("runtime turn admission should resume after rewrite")
.expect("runtime turn admission should succeed");
drop(admission);
}
#[tokio::test]
async fn test_persistent_rewrite_transcript_externalizes_media_before_digesting_commit() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store: Arc<dyn RuntimeStore> = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
ImagePreservingBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store),
memory_blob_store(),
);
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create_session should succeed");
let session_id = created.session_id;
committed_content_turn(&service, runtime_store.as_ref(), &session_id, "hello").await;
let parent_revision = service
.load_authoritative_session(&session_id)
.await
.expect("load before rewrite")
.expect("session exists")
.transcript_revision()
.expect("parent revision");
let result = service
.rewrite_session_transcript(
&session_id,
meerkat_core::SessionTranscriptRewriteRequest {
selection: TranscriptRewriteSelection::MessageRange { start: 1, end: 2 },
replacement: vec![Message::User(UserMessage::with_blocks(
image_prompt("rewrite").into_blocks(),
))],
reason: TranscriptRewriteReason::new("media rewrite"),
actor: Some("test".to_string()),
expected_parent_revision: Some(parent_revision),
running_behavior: TranscriptEditRunningBehavior::Reject,
},
)
.await
.expect("rewrite with inline media replacement should commit");
let saved = service
.load_authoritative_session(&session_id)
.await
.expect("load RuntimeStore authority after rewrite")
.expect("session exists");
assert_no_inline_images_in_session(&saved);
let saved_digest = meerkat_core::transcript_messages_digest(saved.messages())
.expect("saved messages should digest");
assert_eq!(
saved.transcript_revision().expect("saved revision"),
saved_digest
);
assert_eq!(result.revision, saved_digest);
}
#[tokio::test]
async fn test_persistent_post_rewrite_media_externalization_refreshes_transcript_head() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store: Arc<dyn RuntimeStore> = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
ImagePreservingBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store),
memory_blob_store(),
);
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create_session should succeed");
let session_id = created.session_id;
committed_content_turn(&service, runtime_store.as_ref(), &session_id, "hello").await;
let parent_revision = service
.load_authoritative_session(&session_id)
.await
.expect("load before rewrite")
.expect("session exists")
.transcript_revision()
.expect("parent revision");
service
.rewrite_session_transcript(
&session_id,
meerkat_core::SessionTranscriptRewriteRequest {
selection: TranscriptRewriteSelection::MessageRange { start: 1, end: 2 },
replacement: vec![Message::BlockAssistant(
meerkat_core::BlockAssistantMessage {
blocks: vec![meerkat_core::AssistantBlock::Text {
text: "compact answer".to_string(),
meta: None,
}],
stop_reason: StopReason::EndTurn,
identity: meerkat_core::types::TranscriptMessageIdentity::default(),
created_at: meerkat_core::types::message_timestamp_now(),
},
)],
reason: TranscriptRewriteReason::new("compaction"),
actor: Some("test".to_string()),
expected_parent_revision: Some(parent_revision),
running_behavior: TranscriptEditRunningBehavior::Reject,
},
)
.await
.expect("rewrite should commit");
let mut media_turn = runtime_content_turn_request("ignored");
media_turn.prompt = image_prompt("post-rewrite");
committed_turn_with_request(&service, runtime_store.as_ref(), &session_id, media_turn)
.await;
let saved = service
.load_authoritative_session(&session_id)
.await
.expect("load RuntimeStore authority after media turn")
.expect("session exists");
assert_no_inline_images_in_session(&saved);
let saved_digest = meerkat_core::transcript_messages_digest(saved.messages())
.expect("saved messages should digest");
assert_eq!(
saved.transcript_revision().expect("saved revision"),
saved_digest
);
}
#[tokio::test]
async fn test_persistent_rewrite_transcript_appends_audit_event() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store: Arc<dyn RuntimeStore> = Arc::new(InMemoryRuntimeStore::new());
let event_store = Arc::new(RecordingEventStore::default());
let event_store_trait: Arc<dyn EventStore> = event_store.clone();
let dir = tempfile::tempdir().expect("tempdir");
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store),
memory_blob_store(),
)
.with_event_projection(
event_store_trait,
Arc::new(SessionProjector::new(dir.path().join(".rkat"))),
);
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create_session should succeed");
let session_id = created.session_id;
committed_content_turn(&service, runtime_store.as_ref(), &session_id, "hello").await;
let parent_revision = service
.load_authoritative_session(&session_id)
.await
.expect("load before rewrite")
.expect("session exists")
.transcript_revision()
.expect("parent revision");
let rewrite = service
.rewrite_session_transcript(
&session_id,
meerkat_core::SessionTranscriptRewriteRequest {
selection: TranscriptRewriteSelection::MessageRange { start: 1, end: 2 },
replacement: vec![Message::BlockAssistant(
meerkat_core::BlockAssistantMessage::new(
vec![meerkat_core::AssistantBlock::Text {
text: "audit compacted trace".to_string(),
meta: None,
}],
StopReason::EndTurn,
),
)],
reason: TranscriptRewriteReason::new("compaction"),
actor: Some("audit-test".to_string()),
expected_parent_revision: Some(parent_revision.clone()),
running_behavior: TranscriptEditRunningBehavior::Reject,
},
)
.await
.expect("rewrite should commit");
let events = service
.event_log_read_from(&session_id, 1)
.await
.expect("event log should read")
.expect("event projection installed");
let audit = events
.iter()
.find_map(|stored| match &stored.event {
AgentEvent::TranscriptRewriteAuditReceiptCommitted {
session_id,
receipt,
..
} => Some((session_id, receipt)),
_ => None,
})
.expect("rewrite audit receipt should be appended");
assert_eq!(audit.0, &session_id);
assert_eq!(audit.1.commits().len(), 1);
assert_eq!(audit.1.commits()[0].parent_revision, parent_revision);
assert_eq!(audit.1.commits()[0].revision, rewrite.revision);
assert_eq!(audit.1.commits()[0].reason.kind, "compaction");
assert!(
events.iter().all(|stored| !matches!(
stored.event,
AgentEvent::TranscriptRewriteCommitted { .. }
)),
"current writers must not emit full-body rewrite events"
);
let durable = service
.load_authoritative_session(&session_id)
.await
.expect("load durable RuntimeStore rewrite")
.expect("rewritten session exists");
let graph = durable
.validated_transcript_history_state()
.expect("durable graph should validate")
.expect("durable graph should exist");
let proved = graph
.prove_commit_suffix_starting_with(&audit.1.commits()[0])
.expect("receipt commit should select a sealed graph suffix");
assert_eq!(proved.start_prefix(), audit.1.start_prefix());
assert_eq!(proved.end_prefix(), audit.1.end_prefix());
assert_eq!(
serde_json::to_value(durable.messages()).expect("durable head serializes"),
serde_json::to_value(
&service
.read_history(&session_id, SessionHistoryQuery::default())
.await
.expect("history after rewrite")
.messages
)
.expect("history serializes")
);
let events_path = dir
.path()
.join(".rkat")
.join("sessions")
.join(session_id.to_string())
.join("events.jsonl");
let projected =
read_projected_events_after(&events_path, "transcript_rewrite_audit_receipt_committed")
.await;
assert!(projected.contains(&rewrite.revision));
}
#[tokio::test]
async fn test_persistent_rewrite_receipt_failure_is_committed_pending() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store: Arc<dyn RuntimeStore> = Arc::new(InMemoryRuntimeStore::new());
let event_store = Arc::new(RecordingEventStore::default());
let event_store_trait: Arc<dyn EventStore> = event_store.clone();
let dir = tempfile::tempdir().expect("tempdir");
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store),
memory_blob_store(),
)
.with_event_projection(
event_store_trait,
Arc::new(SessionProjector::new(dir.path().join(".rkat"))),
);
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create_session should succeed");
let session_id = created.session_id;
committed_content_turn(&service, runtime_store.as_ref(), &session_id, "hello").await;
let parent_revision = service
.load_authoritative_session(&session_id)
.await
.expect("load before rewrite")
.expect("session exists")
.transcript_revision()
.expect("parent revision");
event_store.fail_appends();
let rewrite_err = service
.rewrite_session_transcript(
&session_id,
meerkat_core::SessionTranscriptRewriteRequest {
selection: TranscriptRewriteSelection::MessageRange { start: 1, end: 2 },
replacement: vec![Message::BlockAssistant(
meerkat_core::BlockAssistantMessage {
blocks: vec![meerkat_core::AssistantBlock::Text {
text: "compact answer despite audit outage".to_string(),
meta: None,
}],
stop_reason: StopReason::EndTurn,
identity: meerkat_core::types::TranscriptMessageIdentity::default(),
created_at: meerkat_core::types::message_timestamp_now(),
},
)],
reason: TranscriptRewriteReason::new("compaction"),
actor: Some("audit-failure-test".to_string()),
expected_parent_revision: Some(parent_revision),
running_behavior: TranscriptEditRunningBehavior::Reject,
},
)
.await
.expect_err(
"rewrite must fail closed when the canonical audit event cannot be appended",
);
assert!(
rewrite_err
.to_string()
.contains("synthetic transcript rewrite audit append failure"),
"unexpected error: {rewrite_err}"
);
let load_error = service
.load_authoritative_session(&session_id)
.await
.expect_err("authoritative load must keep the missing audit receipt fail-closed");
assert!(
load_error
.to_string()
.contains("synthetic transcript rewrite audit append failure"),
"unexpected pending-receipt load error: {load_error}"
);
let runtime_id =
PersistentSessionService::<DummyBuilder>::runtime_id_for_session(&session_id);
let runtime_bytes = runtime_store
.load_session_snapshot(&runtime_id)
.await
.expect("RuntimeStore load after failed receipt append")
.expect("committed RuntimeStore authority remains present");
let runtime_session =
Session::from_persisted_bytes(&runtime_bytes).expect("runtime authority should decode");
assert!(matches!(
&runtime_session.messages()[1],
Message::BlockAssistant(assistant)
if assistant.to_string() == "compact answer despite audit outage"
));
assert!(
event_store
.read_from(&session_id, 1)
.await
.unwrap()
.iter()
.all(|stored| !matches!(
stored.event,
AgentEvent::TranscriptRewriteAuditReceiptCommitted { .. }
)),
"the failed receipt must remain pending rather than being reported as durable"
);
}
#[tokio::test]
async fn test_persistent_rewrite_does_not_consult_session_store_projection() {
let projection_store = Arc::new(FailSaveStore::new());
let store: Arc<dyn SessionStore> = Arc::clone(&projection_store) as Arc<dyn SessionStore>;
let runtime_store: Arc<dyn RuntimeStore> = Arc::new(InMemoryRuntimeStore::new());
let event_store = Arc::new(RecordingEventStore::default());
let event_store_trait: Arc<dyn EventStore> = event_store.clone();
let dir = tempfile::tempdir().expect("tempdir");
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store),
memory_blob_store(),
)
.with_event_projection(
event_store_trait,
Arc::new(SessionProjector::new(dir.path().join(".rkat"))),
);
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create_session should succeed");
let session_id = created.session_id;
committed_content_turn(&service, runtime_store.as_ref(), &session_id, "hello").await;
let committed_before = service
.load_authoritative_session(&session_id)
.await
.expect("load runtime authority before rewrite")
.expect("runtime authority exists");
let committed_before_revision = committed_before
.transcript_revision()
.expect("runtime authority revision");
assert!(
store
.load(&session_id)
.await
.expect("SessionStore probe before rewrite")
.is_none(),
"WholeBlob runtime persistence must not mint a second SessionStore authority"
);
projection_store.set_fail_save(true);
let rewrite = service
.rewrite_session_transcript(
&session_id,
meerkat_core::SessionTranscriptRewriteRequest {
selection: TranscriptRewriteSelection::MessageRange { start: 1, end: 2 },
replacement: vec![Message::BlockAssistant(
meerkat_core::BlockAssistantMessage::new(
vec![meerkat_core::AssistantBlock::Text {
text: "committed while projection is unavailable".to_string(),
meta: None,
}],
StopReason::EndTurn,
),
)],
reason: TranscriptRewriteReason::new("compaction"),
actor: Some("projection-failure-test".to_string()),
expected_parent_revision: Some(committed_before_revision.clone()),
running_behavior: TranscriptEditRunningBehavior::Reject,
},
)
.await
.expect("SessionStore save outage must not affect RuntimeStore authority");
let runtime_id =
PersistentSessionService::<DummyBuilder>::runtime_id_for_session(&session_id);
let runtime_bytes = runtime_store
.load_session_snapshot(&runtime_id)
.await
.expect("load committed runtime authority")
.expect("runtime authority exists");
let runtime_session =
Session::from_persisted_bytes(&runtime_bytes).expect("runtime authority decodes");
let committed_revision = runtime_session
.transcript_revision()
.expect("committed rewrite revision");
assert_eq!(committed_revision, rewrite.revision);
assert_ne!(committed_revision, committed_before_revision);
assert!(
store
.load(&session_id)
.await
.expect("SessionStore probe after rewrite")
.is_none(),
"rewrite must not create or repair a second SessionStore authority"
);
assert_eq!(
event_store.rewrite_append_calls.load(Ordering::Acquire),
1,
"receipt publication follows the singular committed RuntimeStore authority"
);
let receipt_rows = event_store
.read_from(&session_id, 1)
.await
.expect("read repaired receipt")
.into_iter()
.filter(|stored| {
matches!(
stored.event,
AgentEvent::TranscriptRewriteAuditReceiptCommitted { .. }
)
})
.count();
assert_eq!(receipt_rows, 1, "rewrite must publish one exact receipt");
}
#[tokio::test]
async fn test_persistent_rewrite_transcript_recovers_missing_audit_event_from_graph() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store: Arc<dyn RuntimeStore> = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store),
memory_blob_store(),
);
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create_session should succeed");
let session_id = created.session_id;
committed_content_turn(&service, runtime_store.as_ref(), &session_id, "hello").await;
let parent_revision = service
.load_authoritative_session(&session_id)
.await
.expect("load before rewrite")
.expect("session exists")
.transcript_revision()
.expect("parent revision");
let rewrite = service
.rewrite_session_transcript(
&session_id,
meerkat_core::SessionTranscriptRewriteRequest {
selection: TranscriptRewriteSelection::MessageRange { start: 1, end: 2 },
replacement: vec![Message::BlockAssistant(
meerkat_core::BlockAssistantMessage {
blocks: vec![meerkat_core::AssistantBlock::Text {
text: "compact answer before audit projection existed".to_string(),
meta: None,
}],
stop_reason: StopReason::EndTurn,
identity: meerkat_core::types::TranscriptMessageIdentity::default(),
created_at: meerkat_core::types::message_timestamp_now(),
},
)],
reason: TranscriptRewriteReason::new("compaction"),
actor: Some("audit-repair-test".to_string()),
expected_parent_revision: Some(parent_revision),
running_behavior: TranscriptEditRunningBehavior::Reject,
},
)
.await
.expect("rewrite should commit without event projection");
let committed_updated_at = service
.load_authoritative_session(&session_id)
.await
.expect("load committed rewrite before audit repair")
.expect("committed rewrite exists")
.updated_at();
let event_store = Arc::new(RecordingEventStore::default());
let event_store_trait: Arc<dyn EventStore> = event_store.clone();
let dir = tempfile::tempdir().expect("tempdir");
let recovery_service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store),
memory_blob_store(),
)
.with_event_projection(
event_store_trait,
Arc::new(SessionProjector::new(dir.path().join(".rkat"))),
);
recovery_service
.read_history(&session_id, SessionHistoryQuery::default())
.await
.expect("missing audit event should be repaired from retained graph");
assert_eq!(event_store.last_seq(&session_id).await.unwrap(), 1);
let events = event_store
.read_from(&session_id, 1)
.await
.expect("audit event should read back");
let AgentEvent::TranscriptRewriteAuditReceiptCommitted { receipt, .. } = &events[0].event
else {
panic!("repaired event should be a transcript rewrite receipt");
};
assert_eq!(receipt.commits().len(), 1);
assert_eq!(receipt.commits()[0].revision, rewrite.revision);
let repaired = recovery_service
.load_authoritative_session(&session_id)
.await
.expect("load repaired RuntimeStore session")
.expect("repaired session exists");
assert_eq!(
repaired.updated_at(),
committed_updated_at,
"installing already-proved audit graph authority must not manufacture a content timestamp"
);
let graph = repaired
.validated_transcript_history_state()
.expect("repaired graph validates")
.expect("repaired graph exists");
let suffix = graph
.prove_commit_suffix_starting_with(&receipt.commits()[0])
.expect("repaired receipt must be an exact sealed-graph suffix");
assert_eq!(suffix.start_prefix(), receipt.start_prefix());
assert_eq!(suffix.end_prefix(), receipt.end_prefix());
}
#[tokio::test]
async fn authoritative_session_observation_never_repairs_missing_rewrite_audit() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store: Arc<dyn RuntimeStore> = Arc::new(InMemoryRuntimeStore::new());
let writer = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store),
memory_blob_store(),
);
let created = writer
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create session");
let session_id = created.session_id;
committed_content_turn(&writer, runtime_store.as_ref(), &session_id, "hello").await;
let parent_revision = writer
.load_authoritative_session(&session_id)
.await
.expect("load before rewrite")
.expect("session exists")
.transcript_revision()
.expect("parent revision");
let rewrite = writer
.rewrite_session_transcript(
&session_id,
meerkat_core::SessionTranscriptRewriteRequest {
selection: TranscriptRewriteSelection::MessageRange { start: 1, end: 2 },
replacement: vec![Message::BlockAssistant(
meerkat_core::BlockAssistantMessage::new(
vec![meerkat_core::AssistantBlock::Text {
text: "committed rewrite without audit".to_string(),
meta: None,
}],
StopReason::EndTurn,
),
)],
reason: TranscriptRewriteReason::new("observation purity"),
actor: Some("observation-purity-test".to_string()),
expected_parent_revision: Some(parent_revision),
running_behavior: TranscriptEditRunningBehavior::Reject,
},
)
.await
.expect("rewrite commits without event projection");
let event_store = Arc::new(RecordingEventStore::default());
event_store.fail_appends();
let event_store_trait: Arc<dyn EventStore> = event_store.clone();
let dir = tempfile::tempdir().expect("tempdir");
let observer = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store),
memory_blob_store(),
)
.with_event_projection(
event_store_trait,
Arc::new(SessionProjector::new(dir.path().join(".rkat"))),
);
let observed = observer
.observe_authoritative_session_body(&session_id)
.await
.expect("read-only observation must not attempt the refused audit append")
.expect("committed body exists");
assert_eq!(
observed
.transcript_revision()
.expect("read observed transcript revision"),
rewrite.revision,
"observation must still return the exact committed body"
);
assert_eq!(
event_store.rewrite_append_calls.load(Ordering::Acquire),
0,
"observation must not append missing rewrite-audit evidence"
);
assert_eq!(
event_store
.last_seq(&session_id)
.await
.expect("event-store observation"),
0,
"observation must leave the EventStore untouched"
);
}
#[tokio::test]
async fn test_persistent_rewrite_receipt_conflict_fails_closed_without_rollback() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store: Arc<dyn RuntimeStore> = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store),
memory_blob_store(),
);
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create_session should succeed");
let session_id = created.session_id;
committed_content_turn(&service, runtime_store.as_ref(), &session_id, "hello").await;
let parent_revision = service
.load_authoritative_session(&session_id)
.await
.expect("load RuntimeStore authority before rewrite")
.expect("session exists")
.transcript_revision()
.expect("parent revision");
let rewrite = service
.rewrite_session_transcript(
&session_id,
meerkat_core::SessionTranscriptRewriteRequest {
selection: TranscriptRewriteSelection::MessageRange { start: 1, end: 2 },
replacement: vec![Message::BlockAssistant(
meerkat_core::BlockAssistantMessage::new(
vec![meerkat_core::AssistantBlock::Text {
text: "receipt conflict target".to_string(),
meta: None,
}],
StopReason::EndTurn,
),
)],
reason: TranscriptRewriteReason::new("compaction"),
actor: Some("audit-conflict-test".to_string()),
expected_parent_revision: Some(parent_revision),
running_behavior: TranscriptEditRunningBehavior::Reject,
},
)
.await
.expect("rewrite should commit without event projection");
let committed = service
.load_authoritative_session(&session_id)
.await
.expect("load committed RuntimeStore rewrite")
.expect("committed rewrite exists");
let commit = committed
.validated_transcript_history_state()
.expect("rewrite graph should decode")
.expect("rewrite graph should exist")
.last_commit()
.expect("rewrite commit should exist")
.clone();
let receipt =
transcript_rewrite_audit_receipt_for_commits(&committed, std::slice::from_ref(&commit))
.expect("sealed graph should prepare the missing receipt");
let event_store = Arc::new(RecordingEventStore::default());
event_store
.append_transcript_rewrite_receipt_exact(
&session_id,
&receipt,
Some("forged conflicting terminal projection"),
)
.await
.expect("conflicting durable occupant should be seeded");
let event_store_trait: Arc<dyn EventStore> = event_store.clone();
let dir = tempfile::tempdir().expect("tempdir");
let recovery_service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store),
memory_blob_store(),
)
.with_event_projection(
event_store_trait,
Arc::new(SessionProjector::new(dir.path().join(".rkat"))),
);
let error = recovery_service
.read_history(&session_id, SessionHistoryQuery::default())
.await
.expect_err("conflicting receipt occupant must fail recovery closed");
assert!(
error
.to_string()
.contains("exact transcript rewrite receipt conflict"),
"unexpected receipt conflict: {error}"
);
let repeated = recovery_service
.load_authoritative_session(&session_id)
.await
.expect_err("the conflicting audit occupant must remain fail-closed");
assert!(
repeated
.to_string()
.contains("exact transcript rewrite receipt conflict"),
"unexpected repeated receipt conflict: {repeated}"
);
let runtime_id =
PersistentSessionService::<DummyBuilder>::runtime_id_for_session(&session_id);
let runtime = runtime_store
.load_session_snapshot(&runtime_id)
.await
.expect("load retained runtime authority")
.expect("runtime authority remains present");
let runtime =
Session::from_persisted_bytes(&runtime).expect("runtime authority should decode");
assert_eq!(runtime.transcript_revision().unwrap(), rewrite.revision);
assert_eq!(
event_store.last_seq(&session_id).await.unwrap(),
1,
"conflict must neither overwrite nor duplicate the durable occupant"
);
}
#[tokio::test]
async fn test_persistent_archived_history_survives_restart_and_cache_eviction() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store: Arc<dyn RuntimeStore> = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new_with_archived_history_capacity(
DummyBuilder,
1,
1,
Arc::clone(&store),
Arc::clone(&runtime_store),
memory_blob_store(),
);
let machine = MeerkatMachine::persistent(Arc::clone(&runtime_store), memory_blob_store());
let first = service
.create_session(create_request("first", InitialTurnPolicy::Defer))
.await
.expect("create first session");
committed_content_turn(&service, runtime_store.as_ref(), &first.session_id, "first").await;
service
.archive_with_machine_protocol(
&first.session_id,
MachineSessionArchiveProtocol::from_machine(&machine),
)
.await
.expect("archive first session");
let second = service
.create_session(create_request("second", InitialTurnPolicy::Defer))
.await
.expect("create second session");
committed_content_turn(
&service,
runtime_store.as_ref(),
&second.session_id,
"second",
)
.await;
service
.archive_with_machine_protocol(
&second.session_id,
MachineSessionArchiveProtocol::from_machine(&machine),
)
.await
.expect("archive second session");
let restarted = PersistentSessionService::new_with_archived_history_capacity(
DummyBuilder,
1,
1,
Arc::clone(&store),
Arc::clone(&runtime_store),
memory_blob_store(),
);
let archived = restarted
.read_history(
&first.session_id,
SessionHistoryQuery {
offset: 0,
limit: None,
},
)
.await
.expect("archived history should survive restart and cache eviction");
assert_eq!(archived.session_id, first.session_id);
assert_eq!(archived.message_count, 2);
assert_eq!(archived.messages.len(), 2);
let listed = restarted
.list(SessionQuery::default())
.await
.expect("list sessions");
assert!(
listed.is_empty(),
"archived sessions should remain hidden from list even when stored durably"
);
}
#[tokio::test]
async fn test_persistent_fork_at_rejects_running_session() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store: Arc<dyn RuntimeStore> = Arc::new(InMemoryRuntimeStore::new());
let builder = BlockingRunBuilder::new();
let service = Arc::new(PersistentSessionService::new(
builder.clone(),
4,
Arc::clone(&store),
Arc::clone(&runtime_store),
memory_blob_store(),
));
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create_session should succeed");
let parent_id = created.session_id;
let admission = service
.reserve_runtime_turn_admission(&parent_id)
.await
.expect("runtime turn admission should be reserved");
let turn_service = Arc::clone(&service);
let turn_id = parent_id.clone();
let active_turn = tokio::spawn(async move {
turn_service
.apply_runtime_turn_with_reserved_admission(
&turn_id,
RunId::new(),
runtime_content_turn_request("slow follow up"),
RunApplyBoundary::RunStart,
vec![InputId::new()],
admission,
)
.await
});
builder.wait_for_entered_runs(1).await;
let rejected = service
.fork_session_at(
&parent_id,
SessionForkAtRequest {
message_index: 1,
running_behavior: TranscriptEditRunningBehavior::Reject,
tool_access_policy: None,
},
)
.await;
assert!(
matches!(rejected, Err(SessionError::Busy { ref id }) if id == &parent_id),
"fork_at should reject a running session: {rejected:?}"
);
builder.release_notify.add_permits(1);
active_turn
.await
.expect("active turn task should join")
.expect("active turn should complete after fork rejection");
}
#[tokio::test]
async fn test_persistent_restore_transcript_revision_rejects_running_session() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store: Arc<dyn RuntimeStore> = Arc::new(InMemoryRuntimeStore::new());
let builder = BlockingRunBuilder::new();
let service = Arc::new(PersistentSessionService::new(
builder.clone(),
4,
Arc::clone(&store),
Arc::clone(&runtime_store),
memory_blob_store(),
));
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create_session should succeed");
let session_id = created.session_id;
let admission = service
.reserve_runtime_turn_admission(&session_id)
.await
.expect("runtime turn admission should be reserved");
let turn_service = Arc::clone(&service);
let turn_id = session_id.clone();
let active_turn = tokio::spawn(async move {
turn_service
.apply_runtime_turn_with_reserved_admission(
&turn_id,
RunId::new(),
runtime_content_turn_request("slow follow up"),
RunApplyBoundary::RunStart,
vec![InputId::new()],
admission,
)
.await
});
builder.wait_for_entered_runs(1).await;
let rejected = service
.restore_session_transcript_revision(
&session_id,
SessionTranscriptRestoreRevisionRequest {
revision: "current".to_string(),
reason: TranscriptRewriteReason::new("restore"),
actor: Some("restore-active-test".to_string()),
expected_parent_revision: None,
running_behavior: TranscriptEditRunningBehavior::Reject,
},
)
.await;
assert!(
matches!(rejected, Err(SessionError::Busy { ref id }) if id == &session_id),
"restore should reject a running session: {rejected:?}"
);
builder.release_notify.add_permits(1);
active_turn
.await
.expect("active turn task should join")
.expect("active turn should complete after restore rejection");
}
fn spawn_machine_archive<B: SessionAgentBuilder + 'static>(
service: &Arc<PersistentSessionService<B>>,
runtime_store: &Arc<dyn RuntimeStore>,
id: &SessionId,
) -> tokio::task::JoinHandle<Result<(), SessionError>> {
let archive_service = Arc::clone(service);
let archive_runtime_store = Arc::clone(runtime_store);
let archive_id = id.clone();
tokio::spawn(async move {
let machine = MeerkatMachine::persistent(archive_runtime_store, memory_blob_store());
archive_service
.archive_with_machine_protocol(
&archive_id,
MachineSessionArchiveProtocol::from_machine(&machine),
)
.await
})
}
#[tokio::test]
async fn test_persistent_runtime_turn_waits_for_archive_gate() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let gated_runtime_store = Arc::new(GatedSnapshotRuntimeStore::new());
let runtime_store: Arc<dyn RuntimeStore> = gated_runtime_store.clone();
let service = Arc::new(PersistentSessionService::new(
DummyBuilder,
2,
Arc::clone(&store),
Arc::clone(&runtime_store),
memory_blob_store(),
));
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create live session");
gated_runtime_store.pause_next_machine_lifecycle_commit();
let archive_task = spawn_machine_archive(&service, &runtime_store, &created.session_id);
tokio::time::timeout(
std::time::Duration::from_secs(2),
gated_runtime_store.wait_for_machine_lifecycle_commit(),
)
.await
.expect("archive should reach the blocked RuntimeStore lifecycle commit");
let turn_service = Arc::clone(&service);
let turn_id = created.session_id.clone();
let mut turn_task = Box::pin(tokio::spawn(async move {
let _runtime_owned_boundary = turn_service
.acquire_runtime_turn_finalization_guard(&turn_id)
.await;
turn_service
.apply_runtime_turn(
&turn_id,
RunId::new(),
runtime_content_turn_request("turn during archive"),
RunApplyBoundary::RunStart,
vec![InputId::new()],
)
.await
}));
assert!(
tokio::time::timeout(std::time::Duration::from_millis(50), &mut turn_task)
.await
.is_err(),
"runtime turn should wait while archive owns the per-session gate"
);
gated_runtime_store.release_machine_lifecycle_commit();
archive_task
.await
.expect("archive task should join")
.expect("archive should succeed");
let turn_result = turn_task.await.expect("turn task should join");
assert!(
matches!(turn_result, Err(SessionError::NotFound { .. })),
"turn that waited behind archive should see archived session as not found: {turn_result:?}"
);
}
#[tokio::test]
async fn test_persistent_external_user_append_waits_for_archive_gate() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let gated_runtime_store = Arc::new(GatedSnapshotRuntimeStore::new());
let runtime_store: Arc<dyn RuntimeStore> = gated_runtime_store.clone();
let service = Arc::new(PersistentSessionService::new(
DummyBuilder,
2,
Arc::clone(&store),
Arc::clone(&runtime_store),
memory_blob_store(),
));
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create live session");
gated_runtime_store.pause_next_machine_lifecycle_commit();
let archive_task = spawn_machine_archive(&service, &runtime_store, &created.session_id);
tokio::time::timeout(
std::time::Duration::from_secs(2),
gated_runtime_store.wait_for_machine_lifecycle_commit(),
)
.await
.expect("archive should reach the blocked RuntimeStore lifecycle commit");
let append_service = Arc::clone(&service);
let append_id = created.session_id.clone();
let mut append_task = Box::pin(tokio::spawn(async move {
append_service
.append_external_user_content(
&append_id,
ContentInput::Text("external user during archive".to_string()),
)
.await
}));
assert!(
tokio::time::timeout(std::time::Duration::from_millis(50), &mut append_task)
.await
.is_err(),
"external user append should wait while archive owns the per-session gate"
);
gated_runtime_store.release_machine_lifecycle_commit();
archive_task
.await
.expect("archive task should join")
.expect("archive should succeed");
let append_result = append_task.await.expect("append task should join");
assert!(
matches!(append_result, Err(SessionError::NotFound { .. })),
"external user append that waited behind archive should see archived session as not found: {append_result:?}"
);
}
#[tokio::test]
async fn test_persistent_external_assistant_append_waits_for_archive_gate() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let gated_runtime_store = Arc::new(GatedSnapshotRuntimeStore::new());
let runtime_store: Arc<dyn RuntimeStore> = gated_runtime_store.clone();
let service = Arc::new(PersistentSessionService::new(
DummyBuilder,
2,
Arc::clone(&store),
Arc::clone(&runtime_store),
memory_blob_store(),
));
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create live session");
gated_runtime_store.pause_next_machine_lifecycle_commit();
let archive_task = spawn_machine_archive(&service, &runtime_store, &created.session_id);
tokio::time::timeout(
std::time::Duration::from_secs(2),
gated_runtime_store.wait_for_machine_lifecycle_commit(),
)
.await
.expect("archive should reach the blocked RuntimeStore lifecycle commit");
let append_service = Arc::clone(&service);
let append_id = created.session_id.clone();
let mut append_task = Box::pin(tokio::spawn(async move {
append_service
.append_external_assistant_output(
&append_id,
vec![meerkat_core::AssistantBlock::Text {
text: "external assistant during archive".to_string(),
meta: None,
}],
StopReason::EndTurn,
meerkat_core::types::Usage::default(),
)
.await
}));
assert!(
tokio::time::timeout(std::time::Duration::from_millis(50), &mut append_task)
.await
.is_err(),
"external assistant append should wait while archive owns the per-session gate"
);
gated_runtime_store.release_machine_lifecycle_commit();
archive_task
.await
.expect("archive task should join")
.expect("archive should succeed");
let append_result = append_task.await.expect("append task should join");
assert!(
matches!(append_result, Err(SessionError::NotFound { .. })),
"external assistant append that waited behind archive should see archived session as not found: {append_result:?}"
);
}
#[tokio::test]
async fn test_persistent_external_tool_dispatch_waits_for_archive_gate() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let gated_runtime_store = Arc::new(GatedSnapshotRuntimeStore::new());
let runtime_store: Arc<dyn RuntimeStore> = gated_runtime_store.clone();
let service = Arc::new(PersistentSessionService::new(
ToolDispatchBuilder,
2,
Arc::clone(&store),
Arc::clone(&runtime_store),
memory_blob_store(),
));
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create live session");
gated_runtime_store.pause_next_machine_lifecycle_commit();
let archive_task = spawn_machine_archive(&service, &runtime_store, &created.session_id);
tokio::time::timeout(
std::time::Duration::from_secs(2),
gated_runtime_store.wait_for_machine_lifecycle_commit(),
)
.await
.expect("archive should reach the blocked RuntimeStore lifecycle commit");
let dispatch_service = Arc::clone(&service);
let dispatch_id = created.session_id.clone();
let mut dispatch_task = Box::pin(tokio::spawn(async move {
dispatch_service
.dispatch_external_tool_call(
&dispatch_id,
ToolCall::new(
"call-during-archive".to_string(),
"tool_catalog_load".to_string(),
serde_json::json!({}),
),
)
.await
}));
assert!(
tokio::time::timeout(std::time::Duration::from_millis(50), &mut dispatch_task)
.await
.is_err(),
"external tool dispatch should wait while archive owns the per-session gate"
);
gated_runtime_store.release_machine_lifecycle_commit();
archive_task
.await
.expect("archive task should join")
.expect("archive should succeed");
let dispatch_result = dispatch_task.await.expect("dispatch task should join");
assert!(
matches!(dispatch_result, Err(SessionError::NotFound { .. })),
"external tool dispatch that waited behind archive should see archived session as not found: {dispatch_result:?}"
);
}
#[tokio::test]
async fn test_persistent_completed_sessions_do_not_consume_active_capacity() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store: Arc<dyn RuntimeStore> = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
1,
Arc::clone(&store),
Arc::clone(&runtime_store),
memory_blob_store(),
);
for index in 0..3 {
let created = service
.create_session(create_request(
&format!("completed {index}"),
InitialTurnPolicy::Defer,
))
.await
.unwrap_or_else(|err| {
panic!("completed session {index} should release active capacity: {err}")
});
committed_content_turn(
&service,
runtime_store.as_ref(),
&created.session_id,
&format!("completed {index}"),
)
.await;
}
let sessions = service
.list(SessionQuery::default())
.await
.expect("list sessions");
assert_eq!(
sessions.len(),
3,
"completed live sessions should remain readable without holding active capacity"
);
}
#[tokio::test]
async fn test_persistent_deferred_sessions_consume_capacity_until_archived() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store: Arc<dyn RuntimeStore> = Arc::new(InMemoryRuntimeStore::new());
let service = Arc::new(PersistentSessionService::new(
DummyBuilder,
1,
Arc::clone(&store),
Arc::clone(&runtime_store),
memory_blob_store(),
));
let staged = service
.create_session(create_request("staged", InitialTurnPolicy::Defer))
.await
.expect("first deferred session should be staged");
let blocked = service
.create_session(create_request("blocked", InitialTurnPolicy::Defer))
.await;
assert!(
blocked
.as_ref()
.err()
.is_some_and(|err| err.to_string().contains("Max sessions")),
"second deferred session should be rejected while first is staged: {blocked:?}"
);
let machine = MeerkatMachine::persistent(Arc::clone(&runtime_store), memory_blob_store());
service
.archive_with_machine_protocol(
&staged.session_id,
MachineSessionArchiveProtocol::from_machine(&machine),
)
.await
.expect("archiving staged session should release capacity");
service
.create_session(create_request("after archive", InitialTurnPolicy::Defer))
.await
.expect("deferred capacity should be reusable after archive");
}
#[tokio::test]
async fn test_persistent_deferred_capacity_releases_after_first_turn() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store: Arc<dyn RuntimeStore> = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
1,
Arc::clone(&store),
Arc::clone(&runtime_store),
memory_blob_store(),
);
let staged = service
.create_session(create_request("staged", InitialTurnPolicy::Defer))
.await
.expect("deferred session should be staged");
committed_content_turn(
&service,
runtime_store.as_ref(),
&staged.session_id,
"materialize",
)
.await;
service
.create_session(create_request("next staged", InitialTurnPolicy::Defer))
.await
.expect("completed first turn should release deferred capacity");
}
#[tokio::test]
async fn test_persistent_archived_resume_session_rejected_without_capacity_leak() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
1,
Arc::clone(&store),
Arc::clone(&runtime_store) as Arc<dyn RuntimeStore>,
memory_blob_store(),
);
let archived = Session::new();
let id = archived.id().clone();
let runtime_id = PersistentSessionService::<DummyBuilder>::runtime_id_for_session(&id);
runtime_store
.commit_session_snapshot(
&runtime_id,
SerializedSessionSnapshot {
session_snapshot: serde_json::to_vec(&archived)
.expect("serialize content-only RuntimeStore body")
.into(),
},
)
.await
.expect("seed RuntimeStore body");
let machine = MeerkatMachine::persistent(
Arc::clone(&runtime_store) as Arc<dyn RuntimeStore>,
memory_blob_store(),
);
machine
.register_session(id)
.await
.expect("register RuntimeStore lifecycle");
meerkat_runtime::RuntimeControlPlane::retire(&machine, &runtime_id)
.await
.expect("seed Retired RuntimeStore lifecycle");
let rejected = service.create_session(resume_request(archived)).await;
assert!(
matches!(rejected, Err(SessionError::NotFound { .. })),
"archived resume should be rejected before reserving capacity: {rejected:?}"
);
service
.create_session(create_request(
"after archived resume",
InitialTurnPolicy::Defer,
))
.await
.expect("rejected archived resume should not leak active capacity");
}
#[tokio::test]
async fn test_runtime_turn_admission_joins_existing_live_session_capacity() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store: Arc<dyn RuntimeStore> = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
1,
Arc::clone(&store),
Arc::clone(&runtime_store),
memory_blob_store(),
);
let created = service
.create_session(create_request("joiner", InitialTurnPolicy::Defer))
.await
.expect("create live session");
committed_content_turn(
&service,
runtime_store.as_ref(),
&created.session_id,
"joiner",
)
.await;
let first = service
.reserve_runtime_turn_admission(&created.session_id)
.await
.expect("first active admission should reserve capacity");
let second = service
.reserve_runtime_turn_admission(&created.session_id)
.await
.expect("second same-session admission should join existing capacity");
let blocker = recoverable_store_row();
let blocker_id = blocker.id().clone();
let blocker_artifact = blocker
.to_persisted_artifact()
.expect("encode blocker RuntimeStore authority");
runtime_store
.commit_session_snapshot(
&PersistentSessionService::<DummyBuilder>::runtime_id_for_session(&blocker_id),
SerializedSessionSnapshot {
session_snapshot: blocker_artifact.bytes_arc(),
},
)
.await
.expect("persist blocker RuntimeStore authority");
let blocked = service.reserve_runtime_turn_admission(&blocker_id).await;
assert!(
blocked
.as_ref()
.err()
.is_some_and(|err| err.to_string().contains("Max sessions")),
"joined same-session admission must still consume only one active slot"
);
drop(first);
let still_blocked = service.reserve_runtime_turn_admission(&blocker_id).await;
assert!(
still_blocked
.as_ref()
.err()
.is_some_and(|err| err.to_string().contains("Max sessions")),
"capacity must remain held until the final same-session lease drops"
);
drop(second);
service
.reserve_runtime_turn_admission(&blocker_id)
.await
.expect("dropping all same-session leases should release capacity");
}
#[tokio::test]
async fn test_reserved_create_admission_cancel_during_create_releases_capacity() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store: Arc<dyn RuntimeStore> = Arc::new(InMemoryRuntimeStore::new());
let builder = BlockingBuildBuilder::new();
let service = Arc::new(PersistentSessionService::new(
builder.clone(),
1,
Arc::clone(&store),
Arc::clone(&runtime_store),
memory_blob_store(),
));
let persisted = recoverable_store_row();
let persisted_id = persisted.id().clone();
runtime_store
.commit_session_snapshot(
&PersistentSessionService::<BlockingBuildBuilder>::runtime_id_for_session(
&persisted_id,
),
SerializedSessionSnapshot {
session_snapshot: persisted
.to_persisted_artifact()
.expect("encode persisted resume source")
.bytes_arc(),
},
)
.await
.expect("persist RuntimeStore resume source");
let admission = service
.reserve_runtime_turn_admission(&persisted_id)
.await
.expect("reserve create admission");
let service_for_task = Arc::clone(&service);
let create_task = tokio::spawn(async move {
service_for_task
.create_session_with_reserved_admission(resume_request(persisted), admission)
.await
});
builder.wait_for_entered_builds(1).await;
create_task.abort();
let aborted = create_task
.await
.expect_err("aborted reserved create task should report cancellation");
assert!(aborted.is_cancelled());
let blocker = recoverable_store_row();
let blocker_id = blocker.id().clone();
runtime_store
.commit_session_snapshot(
&PersistentSessionService::<BlockingBuildBuilder>::runtime_id_for_session(
&blocker_id,
),
SerializedSessionSnapshot {
session_snapshot: blocker
.to_persisted_artifact()
.expect("encode blocker")
.bytes_arc(),
},
)
.await
.expect("persist RuntimeStore blocker");
service
.reserve_runtime_turn_admission(&blocker_id)
.await
.expect("aborted reserved create should release active capacity");
}
#[tokio::test]
async fn test_persistent_discard_during_deferred_first_turn_keeps_capacity_until_turn_stops() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store: Arc<dyn RuntimeStore> = Arc::new(InMemoryRuntimeStore::new());
let builder = BlockingRunBuilder::new();
let service = Arc::new(PersistentSessionService::new(
builder.clone(),
1,
Arc::clone(&store),
Arc::clone(&runtime_store),
memory_blob_store(),
));
let staged = service
.create_session(create_request("staged", InitialTurnPolicy::Defer))
.await
.expect("deferred session should be staged");
let service_for_turn = Arc::clone(&service);
let session_for_turn = staged.session_id.clone();
let first_turn = tokio::spawn(async move {
service_for_turn
.apply_runtime_turn(
&session_for_turn,
RunId::new(),
runtime_content_turn_request("materialize"),
RunApplyBoundary::RunStart,
vec![InputId::new()],
)
.await
});
builder.wait_for_entered_runs(1).await;
let service_for_discard = Arc::clone(&service);
let session_for_discard = staged.session_id.clone();
let discard = tokio::spawn(async move {
service_for_discard
.discard_live_session(&session_for_discard)
.await
});
tokio::task::yield_now().await;
assert!(
!discard.is_finished(),
"discard must wait for the active turn's stable finalization boundary"
);
let blocked = service
.create_session(create_request("blocked", InitialTurnPolicy::Defer))
.await;
assert!(
blocked
.as_ref()
.err()
.is_some_and(|err| err.to_string().contains("Max sessions")),
"discard during a deferred first turn must not release capacity early: {blocked:?}"
);
builder.release_notify.add_permits(1);
let _ = first_turn.await.expect("first turn task should join");
discard
.await
.expect("discard task should join")
.expect("discard should complete after the active turn stops");
service
.create_session(create_request("after turn", InitialTurnPolicy::Defer))
.await
.expect("capacity should release after discarded turn stops");
}
#[tokio::test]
async fn test_set_session_tool_filter_does_not_forge_visibility_state() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store: Arc<dyn RuntimeStore> = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store),
memory_blob_store(),
);
let created = service
.create_session(create_request_with_metadata(
"hello",
InitialTurnPolicy::Defer,
))
.await
.expect("create session");
let id = created.session_id;
let filter =
meerkat_core::ToolFilter::Deny(["view_image".to_string()].into_iter().collect());
service
.set_session_tool_filter(&id, filter.clone())
.await
.expect("set_session_tool_filter should succeed");
let exported = service.export_session_with_labels(&id).await.unwrap();
let exported_state = exported
.tool_visibility_state()
.expect("live visibility state should decode");
assert!(
exported_state.is_none(),
"filter update must not forge generated visibility authority"
);
let persisted = service
.load_authoritative_session(&id)
.await
.unwrap()
.unwrap();
let persisted_state = persisted
.tool_visibility_state()
.expect("persisted visibility state should decode");
assert_eq!(persisted_state, exported_state);
}
#[tokio::test]
async fn test_set_session_tool_filter_rolls_back_on_store_failure() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let gated_runtime_store = Arc::new(GatedSnapshotRuntimeStore::new());
let runtime_store: Arc<dyn RuntimeStore> = gated_runtime_store.clone();
let service = PersistentSessionService::new(
DummyBuilder,
4,
store,
Arc::clone(&runtime_store),
memory_blob_store(),
);
let created = service
.create_session(create_request_with_metadata(
"hello",
InitialTurnPolicy::Defer,
))
.await
.expect("create session");
let id = created.session_id;
committed_content_turn(&service, runtime_store.as_ref(), &id, "seed turn").await;
let baseline = service.export_session_with_labels(&id).await.unwrap();
assert!(
baseline
.tool_visibility_state()
.expect("baseline visibility state should decode")
.is_none(),
"new sessions should not materialize visibility metadata before updates"
);
let filter =
meerkat_core::ToolFilter::Deny(["view_image".to_string()].into_iter().collect());
gated_runtime_store.set_fail_snapshot_commits(true);
let result = service.set_session_tool_filter(&id, filter).await;
assert!(
result.is_err(),
"RuntimeStore authority failure should surface from the filter update"
);
gated_runtime_store.set_fail_snapshot_commits(false);
let live_export = service.export_session_with_labels(&id).await;
assert!(
matches!(live_export, Err(SessionError::NotFound { .. })),
"authority-commit failure must discard the live session: {live_export:?}"
);
let authoritative = service
.load_authoritative_session_base(&id)
.await
.expect("authoritative load should succeed")
.expect("runtime authority should retain the session");
let authoritative_visibility_state = authoritative
.tool_visibility_state()
.expect("authoritative visibility state should decode");
let baseline_visibility_state = baseline
.tool_visibility_state()
.expect("baseline visibility state should decode");
assert_eq!(
authoritative_visibility_state, baseline_visibility_state,
"the failed authority commit must retain the exact durable predecessor"
);
}
#[tokio::test]
async fn test_set_session_tool_filter_rollback_does_not_promote_legacy_metadata() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let gated_runtime_store = Arc::new(GatedSnapshotRuntimeStore::new());
let runtime_store: Arc<dyn RuntimeStore> = gated_runtime_store.clone();
let service = PersistentSessionService::new(
DummyBuilder,
4,
store,
Arc::clone(&runtime_store),
memory_blob_store(),
);
let mut request = create_request_with_metadata("hello", InitialTurnPolicy::Defer);
let session = request
.build
.as_mut()
.and_then(|build| build.resume_session.as_mut())
.expect("request should carry a resumable session");
session.set_metadata(
meerkat_core::EXTERNAL_TOOL_FILTER_METADATA_KEY,
serde_json::to_value(meerkat_core::ToolFilter::Deny(
["secret".to_string()].into_iter().collect(),
))
.unwrap(),
);
session.set_metadata(
meerkat_core::tool_scope::INHERITED_TOOL_FILTER_METADATA_KEY,
serde_json::to_value(meerkat_core::ToolFilter::Allow(
["visible".to_string()].into_iter().collect(),
))
.unwrap(),
);
let created = service
.create_session(request)
.await
.expect("create session");
let id = created.session_id;
committed_content_turn(&service, runtime_store.as_ref(), &id, "seed turn").await;
let baseline = service.export_session_with_labels(&id).await.unwrap();
assert!(
baseline
.tool_visibility_state()
.expect("canonical visibility metadata should parse")
.is_none(),
"legacy-only metadata must not materialize canonical visibility state"
);
assert!(
baseline
.metadata()
.contains_key(meerkat_core::EXTERNAL_TOOL_FILTER_METADATA_KEY),
"fixture should retain the stale external filter metadata key"
);
assert!(
baseline
.metadata()
.contains_key(meerkat_core::tool_scope::INHERITED_TOOL_FILTER_METADATA_KEY),
"fixture should retain the stale inherited filter metadata key"
);
let filter =
meerkat_core::ToolFilter::Deny(["view_image".to_string()].into_iter().collect());
gated_runtime_store.set_fail_snapshot_commits(true);
let result = service.set_session_tool_filter(&id, filter).await;
assert!(
result.is_err(),
"RuntimeStore authority failure should surface from the filter update"
);
gated_runtime_store.set_fail_snapshot_commits(false);
let live_export = service.export_session_with_labels(&id).await;
assert!(
matches!(live_export, Err(SessionError::NotFound { .. })),
"authority-commit failure must discard the live session: {live_export:?}"
);
let authoritative = service
.load_authoritative_session_base(&id)
.await
.expect("authoritative load should succeed")
.expect("runtime authority should retain the session");
let authoritative_state = authoritative
.tool_visibility_state()
.expect("canonical visibility metadata should parse");
if let Some(state) = authoritative_state.as_ref() {
let state_json = serde_json::to_string(&state).expect("visibility state serializes");
assert!(
!state_json.contains("secret"),
"failure path must never promote stale legacy metadata into canonical visibility: {state_json}"
);
}
assert!(
authoritative_state.is_none(),
"durable predecessor should retain no canonical visibility state"
);
}
#[tokio::test]
async fn test_set_session_tool_filter_rollback_rejects_malformed_canonical_visibility_state() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store: Arc<dyn RuntimeStore> = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
store,
Arc::clone(&runtime_store),
memory_blob_store(),
);
let mut request = create_request_with_metadata("hello", InitialTurnPolicy::Defer);
let malformed_visibility_state = serde_json::json!("not-a-visibility-state");
let session = request
.build
.as_mut()
.and_then(|build| build.resume_session.as_mut())
.expect("request should carry a resumable session");
let mut raw_session = serde_json::to_value(&*session).expect("session should serialize");
raw_session
.get_mut("metadata")
.and_then(serde_json::Value::as_object_mut)
.expect("session JSON should carry metadata")
.insert(
meerkat_core::SESSION_TOOL_VISIBILITY_STATE_KEY.to_string(),
malformed_visibility_state.clone(),
);
*session = serde_json::from_value(raw_session).expect("session should deserialize");
let created = service
.create_session(request)
.await
.expect("create session");
let id = created.session_id;
let baseline = service.export_session_with_labels(&id).await.unwrap();
assert!(
baseline.try_tool_visibility_state().is_err(),
"fixture should carry malformed canonical visibility metadata"
);
assert_eq!(
baseline
.metadata()
.get(meerkat_core::SESSION_TOOL_VISIBILITY_STATE_KEY),
Some(&malformed_visibility_state),
"fixture should retain the raw malformed canonical metadata"
);
let filter =
meerkat_core::ToolFilter::Deny(["view_image".to_string()].into_iter().collect());
let result = service.set_session_tool_filter(&id, filter).await;
let err = result
.expect_err("malformed canonical visibility should fail before staging or rollback");
assert!(
matches!(err, SessionError::Agent(AgentError::InternalError(ref message))
if message.contains("failed to decode dummy visibility state")),
"unexpected error: {err}"
);
let exported = service.export_session_with_labels(&id).await.unwrap();
assert_eq!(
exported
.metadata()
.get(meerkat_core::SESSION_TOOL_VISIBILITY_STATE_KEY),
Some(&malformed_visibility_state),
"failed mutation must preserve malformed canonical visibility metadata"
);
assert!(
exported.try_tool_visibility_state().is_err(),
"failed mutation must not replace malformed canonical visibility with default state"
);
let authoritative = service
.load_authoritative_session_base(&id)
.await
.expect("runtime authority should remain readable")
.expect("runtime authority should remain present");
assert_eq!(
authoritative
.metadata()
.get(meerkat_core::SESSION_TOOL_VISIBILITY_STATE_KEY),
Some(&malformed_visibility_state),
"RuntimeStore authority should retain the raw malformed metadata after rejection"
);
}
#[tokio::test]
async fn test_persistent_load_persisted_returns_stored_session() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let session = Session::new();
let id = session.id().clone();
store.save(&session).await.unwrap();
let loaded = store.load(&id).await.unwrap();
assert!(loaded.is_some());
assert_eq!(loaded.unwrap().id(), &id);
}
#[tokio::test]
async fn test_persistent_load_persisted_returns_none_for_unknown() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let unknown = SessionId::new();
let loaded = store.load(&unknown).await.unwrap();
assert!(loaded.is_none());
}
#[tokio::test]
async fn test_persistent_archive_deletes_from_store() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let session = Session::new();
let id = session.id().clone();
store.save(&session).await.unwrap();
assert!(store.load(&id).await.unwrap().is_some());
store.delete(&id).await.unwrap();
assert!(store.load(&id).await.unwrap().is_none());
}
#[tokio::test]
async fn test_projection_continuity_fail_closed_cleanup_preserves_newer_runtime_snapshot()
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store: Arc<dyn RuntimeStore> = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store),
memory_blob_store(),
);
let mut persisted = Session::new();
persisted.push(Message::User(UserMessage::text("persisted".to_string())));
store.save(&persisted).await?;
let mut rejected = persisted.clone();
rejected.push(Message::BlockAssistant(
meerkat_core::BlockAssistantMessage {
blocks: vec![meerkat_core::AssistantBlock::Text {
text: "rejected runtime snapshot".to_string(),
meta: None,
}],
stop_reason: StopReason::EndTurn,
identity: meerkat_core::types::TranscriptMessageIdentity::default(),
created_at: meerkat_core::types::message_timestamp_now(),
},
));
let rejected_snapshot = serde_json::to_vec(&rejected)?;
let mut newer = rejected.clone();
newer.push(Message::User(UserMessage::text(
"newer runtime authority".to_string(),
)));
let newer_snapshot = serde_json::to_vec(&newer)?;
let runtime_id =
PersistentSessionService::<DummyBuilder>::runtime_id_for_session(persisted.id());
runtime_store
.commit_session_snapshot(
&runtime_id,
SerializedSessionSnapshot {
session_snapshot: (newer_snapshot.clone()).into(),
},
)
.await?;
let err = service
.fail_closed_runtime_projection_update(
persisted.id(),
SessionStoreError::Internal("forced projection rejection".to_string()),
Some(rejected_snapshot.as_slice()),
)
.await;
assert!(
err.to_string().contains("forced projection rejection"),
"unexpected error: {err}"
);
let current = runtime_store
.load_session_snapshot(&runtime_id)
.await?
.ok_or_else(|| std::io::Error::other("runtime snapshot should remain present"))?;
assert_eq!(current.as_ref(), &newer_snapshot);
Ok(())
}
#[tokio::test]
async fn test_projection_continuity_fail_closed_quarantine_preserves_newer_runtime_snapshot()
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let gated_runtime_store = Arc::new(GatedSnapshotRuntimeStore::new());
let runtime_store: Arc<dyn RuntimeStore> = gated_runtime_store.clone();
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store),
memory_blob_store(),
);
let mut persisted = Session::new();
persisted.push(Message::User(UserMessage::text("persisted".to_string())));
store.save(&persisted).await?;
let mut rejected = persisted.clone();
rejected.push(Message::BlockAssistant(
meerkat_core::BlockAssistantMessage {
blocks: vec![meerkat_core::AssistantBlock::Text {
text: "rejected runtime snapshot".to_string(),
meta: None,
}],
stop_reason: StopReason::EndTurn,
identity: meerkat_core::types::TranscriptMessageIdentity::default(),
created_at: meerkat_core::types::message_timestamp_now(),
},
));
let rejected_snapshot = serde_json::to_vec(&rejected)?;
let mut newer = rejected.clone();
newer.push(Message::User(UserMessage::text(
"newer runtime authority before quarantine".to_string(),
)));
let newer_snapshot = serde_json::to_vec(&newer)?;
let runtime_id =
PersistentSessionService::<DummyBuilder>::runtime_id_for_session(persisted.id());
runtime_store
.commit_session_snapshot(
&runtime_id,
SerializedSessionSnapshot {
session_snapshot: (newer_snapshot.clone()).into(),
},
)
.await?;
gated_runtime_store.set_fail_snapshot_commits(true);
let err = service
.fail_closed_runtime_projection_update(
persisted.id(),
SessionStoreError::Internal("forced projection rejection".to_string()),
Some(rejected_snapshot.as_slice()),
)
.await;
assert!(
err.to_string().contains("forced projection rejection"),
"unexpected error: {err}"
);
let current = runtime_store
.load_session_snapshot(&runtime_id)
.await?
.ok_or_else(|| std::io::Error::other("newer runtime snapshot should remain present"))?;
assert_eq!(current.as_ref(), &newer_snapshot);
Ok(())
}
#[tokio::test]
async fn test_prepared_rewrite_cas_rejects_interloping_newer_runtime_snapshot()
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let gated_runtime_store = Arc::new(GatedSnapshotRuntimeStore::new());
let runtime_store: Arc<dyn RuntimeStore> = gated_runtime_store.clone();
let event_store = Arc::new(RecordingEventStore::default());
let event_store_trait: Arc<dyn EventStore> = event_store.clone();
let dir = tempfile::tempdir()?;
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store),
memory_blob_store(),
)
.with_event_projection(
event_store_trait,
Arc::new(SessionProjector::new(dir.path().join(".rkat"))),
);
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await?;
let original = service
.load_authoritative_session_base(&created.session_id)
.await?
.ok_or_else(|| std::io::Error::other("created runtime session should exist"))?;
let parent_revision = original.transcript_revision()?;
let mut incoming = original.clone();
let commit = incoming
.commit_transcript_rewrite(
TranscriptRewriteSelection::MessageRange {
start: 0,
end: original.messages().len(),
},
vec![Message::User(UserMessage::text(
"[Context compacted] audit failure".to_string(),
))],
TranscriptRewriteReason::new("compaction"),
Some("meerkat-core".to_string()),
Some(parent_revision),
)
.map_err(|err| std::io::Error::other(format!("rewrite should commit: {err}")))?;
let runtime_id =
PersistentSessionService::<DummyBuilder>::runtime_id_for_session(&created.session_id);
runtime_store
.commit_session_snapshot(
&runtime_id,
SerializedSessionSnapshot {
session_snapshot: (serde_json::to_vec(&original)?).into(),
},
)
.await?;
let mut newer = incoming.clone();
newer.push(Message::BlockAssistant(
meerkat_core::BlockAssistantMessage {
blocks: vec![meerkat_core::AssistantBlock::Text {
text: "newer runtime authority".to_string(),
meta: None,
}],
stop_reason: StopReason::EndTurn,
identity: meerkat_core::types::TranscriptMessageIdentity::default(),
created_at: meerkat_core::types::message_timestamp_now(),
},
));
let newer_snapshot = serde_json::to_vec(&newer)?;
gated_runtime_store
.interlope_before_snapshot_replace(runtime_id.clone(), newer_snapshot.clone())
.await;
event_store.fail_appends();
let save_result = service.save_normalized_session(incoming).await;
let err = save_result.expect_err("stale prepared rewrite predecessor must fail closed");
let rendered = err.to_string();
assert!(
rendered.contains("session persistence authority conflict")
&& rendered.contains(
"prepared WholeBlob predecessor revision/token does not match current authority"
),
"unexpected error: {rendered}"
);
assert_eq!(
event_store.rewrite_append_calls.load(Ordering::Acquire),
0,
"a failed authority CAS must not attempt receipt publication"
);
let current = runtime_store
.load_session_snapshot(&runtime_id)
.await?
.ok_or_else(|| std::io::Error::other("newer runtime snapshot should remain present"))?;
assert_eq!(current.as_ref(), &newer_snapshot);
let current: Session = serde_json::from_slice(¤t)?;
assert_eq!(current.transcript_revision()?, newer.transcript_revision()?);
assert_ne!(current.transcript_revision()?, commit.revision);
Ok(())
}
#[tokio::test]
async fn test_runtime_rewrite_receipt_failure_does_not_quarantine_committed_authority()
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let gated_runtime_store = Arc::new(GatedSnapshotRuntimeStore::new());
let runtime_store: Arc<dyn RuntimeStore> = gated_runtime_store.clone();
let event_store = Arc::new(RecordingEventStore::default());
let event_store_trait: Arc<dyn EventStore> = event_store.clone();
let dir = tempfile::tempdir()?;
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store),
memory_blob_store(),
)
.with_event_projection(
event_store_trait,
Arc::new(SessionProjector::new(dir.path().join(".rkat"))),
);
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await?;
let original = service
.load_authoritative_session_base(&created.session_id)
.await?
.ok_or_else(|| std::io::Error::other("created runtime session should exist"))?;
let parent_revision = original.transcript_revision()?;
let mut incoming = original.clone();
let commit = incoming
.commit_transcript_rewrite(
TranscriptRewriteSelection::MessageRange {
start: 0,
end: original.messages().len(),
},
vec![Message::User(UserMessage::text(
"[Context compacted] audit failure".to_string(),
))],
TranscriptRewriteReason::new("compaction"),
Some("meerkat-core".to_string()),
Some(parent_revision),
)
.map_err(|err| std::io::Error::other(format!("rewrite should commit: {err}")))?;
let runtime_id =
PersistentSessionService::<DummyBuilder>::runtime_id_for_session(&created.session_id);
runtime_store
.commit_session_snapshot(
&runtime_id,
SerializedSessionSnapshot {
session_snapshot: (serde_json::to_vec(&original)?).into(),
},
)
.await?;
event_store.fail_appends();
let save_result = service.save_normalized_session(incoming).await;
let err = save_result.expect_err("committed rewrite must report its pending receipt");
assert!(
err.to_string()
.contains("synthetic transcript rewrite audit append failure"),
"unexpected error: {err}"
);
let current = runtime_store
.load_session_snapshot(&runtime_id)
.await?
.ok_or_else(|| std::io::Error::other("committed runtime snapshot should remain"))?;
let current: Session = serde_json::from_slice(¤t)?;
assert_eq!(current.transcript_revision()?, commit.revision);
Ok(())
}
#[tokio::test]
async fn test_runtime_rewrite_ignores_non_authoritative_session_store_projection()
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store: Arc<dyn RuntimeStore> = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store),
memory_blob_store(),
);
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await?;
let baseline = service
.load_authoritative_session_base(&created.session_id)
.await?
.ok_or_else(|| std::io::Error::other("created runtime session should exist"))?;
let mut newer_projection = baseline.clone();
newer_projection.push(Message::User(UserMessage::text(
"newer persisted turn that runtime rewrite persistence must preserve".to_string(),
)));
let newer_revision = newer_projection.transcript_revision()?;
store
.save_authoritative_projection(&newer_projection)
.await?;
let mut stale_parent = baseline;
stale_parent.append_system_message("stale runtime system projection".to_string());
stale_parent.push(Message::User(UserMessage::text(
"stale runtime branch prompt".to_string(),
)));
stale_parent.push(Message::BlockAssistant(
meerkat_core::BlockAssistantMessage {
blocks: vec![meerkat_core::AssistantBlock::Text {
text: "stale runtime branch answer".to_string(),
meta: None,
}],
stop_reason: StopReason::EndTurn,
identity: meerkat_core::types::TranscriptMessageIdentity::default(),
created_at: meerkat_core::types::message_timestamp_now(),
},
));
let stale_parent_revision = stale_parent.transcript_revision()?;
runtime_store
.commit_session_snapshot(
&PersistentSessionService::<DummyBuilder>::runtime_id_for_session(
&created.session_id,
),
SerializedSessionSnapshot {
session_snapshot: (serde_json::to_vec(&stale_parent)?).into(),
},
)
.await?;
let mut incoming = stale_parent.clone();
incoming
.commit_transcript_rewrite(
TranscriptRewriteSelection::MessageRange {
start: 0,
end: stale_parent.messages().len(),
},
vec![Message::User(UserMessage::text(
"[Context compacted] stale runtime branch".to_string(),
))],
TranscriptRewriteReason::new("compaction"),
Some("meerkat-core".to_string()),
Some(stale_parent_revision.clone()),
)
.map_err(|err| std::io::Error::other(format!("rewrite should commit: {err}")))?;
let incoming_revision = incoming.transcript_revision()?;
let saved_runtime = service
.save_normalized_session(incoming)
.await
.expect("SessionStore projection must not veto RuntimeStore-owned rewrite authority");
let runtime_saved =
PersistentSessionService::<DummyBuilder>::load_runtime_session_snapshot(
&runtime_store,
&PersistentSessionService::<DummyBuilder>::runtime_id_for_session(
&created.session_id,
),
)
.await?
.ok_or_else(|| std::io::Error::other("runtime snapshot should remain present"))?;
assert_eq!(runtime_saved.transcript_revision()?, incoming_revision);
assert_eq!(saved_runtime.transcript_revision()?, incoming_revision);
assert_ne!(runtime_saved.transcript_revision()?, stale_parent_revision);
let saved = store
.load(&created.session_id)
.await?
.ok_or_else(|| std::io::Error::other("persisted session should remain present"))?;
assert_eq!(saved.transcript_revision()?, newer_revision);
assert_ne!(saved.transcript_revision()?, incoming_revision);
Ok(())
}
#[tokio::test]
async fn test_apply_runtime_turn_externalizes_inline_images_in_runtime_snapshot() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
ImagePreservingBuilder,
4,
Arc::clone(&store),
runtime_store.clone(),
memory_blob_store(),
);
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create_session should succeed");
let run_id = RunId::new();
let mut req = runtime_content_turn_request("ignored");
req.prompt = image_prompt("runtime");
service
.apply_runtime_turn(
&created.session_id,
run_id,
req,
RunApplyBoundary::RunStart,
vec![],
)
.await
.expect("apply_runtime_turn should succeed");
let runtime_id =
super::PersistentSessionService::<ImagePreservingBuilder>::runtime_id_for_session(
&created.session_id,
);
let snapshot = runtime_store
.load_session_snapshot(&runtime_id)
.await
.expect("runtime snapshot load should succeed")
.expect("runtime snapshot should exist");
let persisted: Session =
serde_json::from_slice(&snapshot).expect("runtime snapshot should deserialize");
assert_no_inline_images_in_session(&persisted);
}
#[tokio::test]
async fn test_apply_runtime_turn_resume_pending_without_boundary_is_not_run_result() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
runtime_store.clone(),
memory_blob_store(),
);
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create_session should succeed");
let run_id = RunId::new();
let contributing_input_ids = vec![meerkat_core::lifecycle::InputId::new()];
let mut req = start_turn_request("resume");
req.runtime.turn_metadata = Some(
meerkat_core::lifecycle::run_primitive::RuntimeTurnMetadata {
execution_kind: Some(meerkat_core::lifecycle::RuntimeExecutionKind::ResumePending),
..Default::default()
},
);
let output = service
.apply_runtime_turn(
&created.session_id,
run_id.clone(),
req,
RunApplyBoundary::RunStart,
contributing_input_ids.clone(),
)
.await
.expect("runtime apply should commit typed no-pending terminal");
assert_eq!(output.receipt.run_id, run_id);
assert_eq!(
output.receipt.contributing_input_ids,
contributing_input_ids
);
assert!(matches!(
output.terminal,
Some(CoreApplyTerminal::NoPendingBoundary)
));
}
#[tokio::test]
async fn test_apply_runtime_turn_rejects_missing_execution_kind_before_no_pending_terminal() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
runtime_store,
memory_blob_store(),
);
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create_session should succeed");
let error = service
.apply_runtime_turn(
&created.session_id,
RunId::new(),
start_turn_request(""),
RunApplyBoundary::RunStart,
vec![meerkat_core::lifecycle::InputId::new()],
)
.await
.expect_err(
"runtime apply must reject missing execution kind before no-pending commit",
);
assert!(
error.to_string().contains("runtime_execution_kind not set"),
"unexpected error: {error}"
);
}
#[tokio::test]
async fn test_machine_terminal_runtime_apply_returns_failed_but_applied_snapshot() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
MachineTerminalRunBuilder,
4,
Arc::clone(&store),
runtime_store,
memory_blob_store(),
);
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create_session should succeed");
let output = service
.apply_runtime_turn(
&created.session_id,
RunId::new(),
runtime_content_turn_request("denied runtime turn"),
RunApplyBoundary::RunStart,
vec![meerkat_core::lifecycle::InputId::new()],
)
.await
.expect("machine-terminal runtime turn should return an applied terminal carrier");
assert!(matches!(
&output.terminal,
Some(CoreApplyTerminal::MachineTerminalFailure { error })
if error.outcome == Some(meerkat_core::TurnTerminalOutcome::Failed)
&& error.kind == meerkat_core::TurnTerminalCauseKind::ToolFailure
&& error.detail.as_deref().is_some_and(|detail| detail.contains("synthetic member-side access denied"))
));
let snapshot = output
.whole_blob_bytes()
.expect("failed-but-applied whole-blob boundary should encode")
.expect("failed-but-applied carrier must include the mutated session snapshot");
let snapshot: Session = serde_json::from_slice(snapshot)
.expect("failed-but-applied snapshot should deserialize");
assert!(
snapshot
.messages()
.iter()
.any(|message| { format!("{message:?}").contains("denied runtime turn") }),
"failed turn transcript must travel with the atomic commit carrier"
);
assert!(
service
.has_live_session(&created.session_id)
.await
.expect("live-session status should succeed"),
"machine-terminal runtime failure must retain the sequencer used for exact terminal publication"
);
}
#[tokio::test]
async fn test_stale_prior_terminal_snapshot_cannot_classify_new_runtime_failure() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
StaleTerminalRunBuilder,
4,
Arc::clone(&store),
runtime_store,
memory_blob_store(),
);
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create_session should succeed");
let error = service
.apply_runtime_turn(
&created.session_id,
RunId::new(),
runtime_content_turn_request("must not inherit prior terminal"),
RunApplyBoundary::RunStart,
vec![InputId::new()],
)
.await
.expect_err("a new failure without an exact run witness must remain recoverable");
assert!(
error.to_string().contains("new pre-dispatch failure"),
"unexpected unclassified failure: {error}"
);
assert!(
service
.live_session_actor_registered(&created.session_id)
.await,
"an unclassified runtime failure must not remove the live actor while the machine remains registered"
);
}
#[tokio::test]
async fn test_reserved_runtime_admission_returns_same_machine_terminal_carrier() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
MachineTerminalRunBuilder,
4,
Arc::clone(&store),
runtime_store,
memory_blob_store(),
);
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create_session should succeed");
let admission = service
.reserve_runtime_turn_admission(&created.session_id)
.await
.expect("reserved runtime admission should succeed");
let output = match service
.apply_runtime_turn_with_recoverable_reserved_admission(
&created.session_id,
RunId::new(),
runtime_content_turn_request("reserved denied runtime turn"),
RunApplyBoundary::RunStart,
vec![InputId::new()],
admission,
)
.await
{
Ok(output) => output,
Err((error, _admission)) => {
panic!("reserved runtime path should preserve failed-but-applied truth: {error}")
}
};
assert!(
output
.whole_blob_bytes()
.expect("reserved whole-blob runtime boundary should encode")
.is_some()
);
assert!(matches!(
output.terminal,
Some(CoreApplyTerminal::MachineTerminalFailure { error })
if error.kind == meerkat_core::TurnTerminalCauseKind::ToolFailure
&& error.outcome == Some(meerkat_core::TurnTerminalOutcome::Failed)
));
}
#[tokio::test]
async fn test_runtime_apply_without_machine_commit_fails_closed_live_export() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
runtime_store,
memory_blob_store(),
);
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create_session should succeed");
let output = service
.apply_runtime_turn(
&created.session_id,
RunId::new(),
runtime_content_turn_request("uncommitted runtime turn"),
RunApplyBoundary::Immediate,
vec![InputId::new()],
)
.await
.expect("runtime apply should build a machine-owned commit output");
let staged_snapshot: Session = serde_json::from_slice(
output
.whole_blob_bytes()
.expect("staged whole-blob runtime boundary should encode")
.expect("runtime output should carry a staged session snapshot"),
)
.expect("staged session snapshot should deserialize");
assert!(
!staged_snapshot.messages().is_empty(),
"runtime apply should mutate live session state before the machine commit"
);
let error = service
.export_live_session(&created.session_id)
.await
.expect_err("uncommitted live session must not remain externally visible");
assert!(
matches!(error, SessionError::NotFound { .. }),
"unexpected export error after stale-live discard: {error:?}"
);
assert!(
service
.has_live_session(&created.session_id)
.await
.expect("live-session status should succeed"),
"live export must fail closed without discarding the mechanical live handle"
);
let authoritative = service
.load_authoritative_session(&created.session_id)
.await
.expect("authoritative load should succeed")
.expect("created runtime snapshot should remain durable");
assert!(
authoritative.messages().is_empty(),
"without the machine atomic commit, durable runtime authority must not expose the staged turn"
);
}
#[tokio::test]
async fn test_runtime_backed_read_uses_durable_authority_without_discarding_live_handle() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
runtime_store,
memory_blob_store(),
);
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create_session should succeed");
service
.apply_runtime_turn(
&created.session_id,
RunId::new(),
runtime_content_turn_request("uncommitted runtime turn"),
RunApplyBoundary::Immediate,
vec![InputId::new()],
)
.await
.expect("runtime apply should build output without committing authority");
let view = service
.read(&created.session_id)
.await
.expect("read should fail closed to durable authority");
assert_eq!(
view.state.message_count, 0,
"read() must not publish uncommitted live transcript as durable truth"
);
assert!(
service
.has_live_session(&created.session_id)
.await
.expect("live-session status should succeed"),
"read() must not discard the live handle that owns mechanical capabilities"
);
}
#[tokio::test]
async fn test_runtime_backed_list_uses_durable_summary_without_discarding_live_handle() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
runtime_store,
memory_blob_store(),
);
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create_session should succeed");
let output = service
.apply_runtime_turn(
&created.session_id,
RunId::new(),
runtime_content_turn_request("uncommitted runtime turn"),
RunApplyBoundary::Immediate,
vec![InputId::new()],
)
.await
.expect("runtime apply should build output without committing authority");
let staged_snapshot: Session = serde_json::from_slice(
output
.whole_blob_bytes()
.expect("staged whole-blob runtime boundary should encode")
.expect("runtime output should carry a staged session snapshot"),
)
.expect("staged session snapshot should deserialize");
assert!(
!staged_snapshot.messages().is_empty(),
"test must create an uncommitted live transcript before list()"
);
let listed = service
.list(SessionQuery::default())
.await
.expect("list should fail closed to durable authority");
let summary = listed
.iter()
.find(|summary| summary.session_id == created.session_id)
.expect("created session should remain discoverable");
assert_eq!(
summary.message_count, 0,
"list() must not publish uncommitted live transcript as summary truth"
);
assert!(
service
.has_live_session(&created.session_id)
.await
.expect("live-session status should succeed"),
"list() must not discard the live handle that owns mechanical capabilities"
);
}
#[tokio::test]
async fn test_runtime_backed_observations_preserve_live_capability_handles() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
CapabilityBuilder,
4,
Arc::clone(&store),
runtime_store,
memory_blob_store(),
);
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create_session should succeed");
assert!(
service.comms_runtime(&created.session_id).await.is_some(),
"test setup should install a live comms runtime"
);
assert!(
service
.interaction_event_injector(&created.session_id)
.await
.is_some(),
"test setup should install a live interaction injector"
);
service
.apply_runtime_turn(
&created.session_id,
RunId::new(),
runtime_content_turn_request("uncommitted runtime turn"),
RunApplyBoundary::Immediate,
vec![InputId::new()],
)
.await
.expect("runtime apply should build output without committing authority");
let view = service
.read(&created.session_id)
.await
.expect("read should fail closed to durable authority");
assert_eq!(view.state.message_count, 0);
let listed = service
.list(SessionQuery::default())
.await
.expect("list should fail closed to durable authority");
let summary = listed
.iter()
.find(|summary| summary.session_id == created.session_id)
.expect("created session should remain discoverable");
assert_eq!(summary.message_count, 0);
let export = service.export_live_session(&created.session_id).await;
assert!(
matches!(export, Err(SessionError::NotFound { .. })),
"live export must fail closed while durable authority is behind"
);
assert!(
service
.has_live_session(&created.session_id)
.await
.expect("live-session status should succeed"),
"observation paths must not discard the live session"
);
assert!(
service.comms_runtime(&created.session_id).await.is_some(),
"observation paths must not drop the comms runtime"
);
assert!(
service
.interaction_event_injector(&created.session_id)
.await
.is_some(),
"observation paths must not drop the interaction event injector"
);
}
#[tokio::test]
async fn test_realtime_open_snapshot_synchronizes_stale_live_transcript_authority() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store: Arc<dyn RuntimeStore> = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
CapabilityBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store),
memory_blob_store(),
);
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create_session should succeed");
assert!(
service.comms_runtime(&created.session_id).await.is_some(),
"test setup should install a live comms runtime"
);
let durable = service
.export_live_session(&created.session_id)
.await
.expect("initial live session should export");
let durable = mutate_test_session(durable, "run boundary", |durable| {
durable.push(Message::User(UserMessage::text(
"durable realtime seed".to_string(),
)));
});
runtime_store
.commit_session_snapshot(
&LogicalRuntimeId::for_session(&created.session_id),
SerializedSessionSnapshot {
session_snapshot: (serde_json::to_vec(&durable)
.expect("serialize durable session"))
.into(),
},
)
.await
.expect("commit durable runtime snapshot");
let live_export = service.export_live_session(&created.session_id).await;
assert!(
matches!(live_export, Err(SessionError::NotFound { .. })),
"public live export must fail closed while durable semantic authority wins"
);
let realtime_snapshot = service
.export_realtime_open_session_snapshot(&created.session_id)
.await
.expect("realtime open should use durable authority after syncing live mechanics");
assert!(
realtime_snapshot.messages().iter().any(|message| matches!(
message,
Message::User(user) if user.text_content() == "durable realtime seed"
)),
"realtime-open snapshot must be the durable semantic authority"
);
assert!(
service
.has_live_session(&created.session_id)
.await
.expect("live-session status should succeed"),
"stale live transcript handle must remain installed after semantic sync"
);
assert!(
service.comms_runtime(&created.session_id).await.is_some(),
"semantic sync must preserve the live comms runtime"
);
assert!(
service
.interaction_event_injector(&created.session_id)
.await
.is_some(),
"semantic sync must preserve the live interaction event injector"
);
let synced_live = service.export_live_session(&created.session_id).await;
assert!(
synced_live
.expect("synced live session should export")
.messages()
.iter()
.any(|message| matches!(
message,
Message::User(user) if user.text_content() == "durable realtime seed"
)),
"live session semantics must match the durable snapshot after realtime-open recovery"
);
}
#[tokio::test]
async fn test_realtime_open_snapshot_hydrates_durable_blob_backed_user_image() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store: Arc<dyn RuntimeStore> = Arc::new(InMemoryRuntimeStore::new());
let blob_store = Arc::new(MemoryBlobStore::new());
let service_blob_store: Arc<dyn BlobStore> = blob_store.clone();
let service = PersistentSessionService::new(
CapabilityBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store),
service_blob_store,
);
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create_session should succeed");
let image_data = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAAB";
let blob_ref = blob_store
.put_image("image/png", image_data)
.await
.expect("image fixture should persist");
let durable = service
.export_live_session(&created.session_id)
.await
.expect("initial live session should export");
let durable = mutate_test_session(durable, "run boundary", |durable| {
durable.push(Message::User(UserMessage::with_blocks(vec![
ContentBlock::Image {
media_type: "image/png".to_string(),
data: ImageData::Blob {
blob_id: blob_ref.blob_id.clone(),
},
},
])));
durable.push(Message::ToolResults {
results: vec![ToolResult {
tool_use_id: "tool-image".to_string(),
content: vec![ContentBlock::Image {
media_type: "image/png".to_string(),
data: ImageData::Blob {
blob_id: blob_ref.blob_id.clone(),
},
}],
is_error: false,
}],
created_at: std::time::SystemTime::UNIX_EPOCH.into(),
});
});
runtime_store
.commit_session_snapshot(
&LogicalRuntimeId::for_session(&created.session_id),
SerializedSessionSnapshot {
session_snapshot: (serde_json::to_vec(&durable)
.expect("serialize durable image session"))
.into(),
},
)
.await
.expect("commit durable runtime snapshot");
let (realtime_snapshot, canonical_user_image_decoded_bytes) = service
.export_realtime_open_session_snapshot_with_image_usage(&created.session_id)
.await
.expect("realtime provider projection should hydrate durable image bytes");
assert_eq!(canonical_user_image_decoded_bytes, 24);
assert!(matches!(
realtime_snapshot.messages(),
[Message::User(user), Message::ToolResults { results, .. }]
if matches!(
user.content.as_slice(),
[ContentBlock::Image {
media_type,
data: ImageData::Inline { data },
}] if media_type == "image/png" && data == image_data
)
&& matches!(
results.as_slice(),
[ToolResult { content, .. }]
if matches!(
content.as_slice(),
[ContentBlock::Image {
data: ImageData::Blob { blob_id },
..
}] if blob_id == &blob_ref.blob_id
)
)
));
let synced_live = service
.export_live_session(&created.session_id)
.await
.expect("durable authority should synchronize into the live handle");
assert!(
matches!(
synced_live.messages(),
[Message::User(user), Message::ToolResults { results, .. }]
if matches!(
user.content.as_slice(),
[ContentBlock::Image {
data: ImageData::Blob { blob_id },
..
}] if blob_id == &blob_ref.blob_id
)
&& matches!(
results.as_slice(),
[ToolResult { content, .. }]
if matches!(
content.as_slice(),
[ContentBlock::Image {
data: ImageData::Blob { blob_id },
..
}] if blob_id == &blob_ref.blob_id
)
)
),
"provider hydration must not rewrite canonical durable/live history inline"
);
}
#[tokio::test]
async fn test_realtime_open_snapshot_fails_closed_when_durable_image_blob_is_missing() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store: Arc<dyn RuntimeStore> = Arc::new(InMemoryRuntimeStore::new());
let blob_store = Arc::new(MemoryBlobStore::new());
let service_blob_store: Arc<dyn BlobStore> = blob_store.clone();
let service = PersistentSessionService::new(
CapabilityBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store),
service_blob_store,
);
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create_session should succeed");
let blob_ref = blob_store
.put_image("image/png", "iVBORw0KGgoAAAANSUhEUgAAAAEAAAAB")
.await
.expect("image fixture should persist");
let durable = service
.export_live_session(&created.session_id)
.await
.expect("initial live session should export");
let durable = mutate_test_session(durable, "run boundary", |durable| {
durable.push(Message::User(UserMessage::with_blocks(vec![
ContentBlock::Image {
media_type: "image/png".to_string(),
data: ImageData::Blob {
blob_id: blob_ref.blob_id.clone(),
},
},
])));
});
runtime_store
.commit_session_snapshot(
&LogicalRuntimeId::for_session(&created.session_id),
SerializedSessionSnapshot {
session_snapshot: (serde_json::to_vec(&durable)
.expect("serialize durable image session"))
.into(),
},
)
.await
.expect("commit durable runtime snapshot");
blob_store
.delete(&blob_ref.blob_id)
.await
.expect("image fixture should be removable");
let error = service
.export_realtime_open_session_snapshot(&created.session_id)
.await
.expect_err("missing durable image bytes must fail provider projection closed");
match error {
SessionError::Agent(AgentError::InternalError(message)) => {
assert!(
message.contains(
"failed to hydrate realtime history images before provider projection"
),
"unexpected hydration error: {message}"
);
assert!(
message.contains(blob_ref.blob_id.as_str()),
"missing blob identity must survive the error boundary: {message}"
);
}
other => panic!("unexpected realtime image hydration error: {other}"),
}
let synced_live = service
.export_live_session(&created.session_id)
.await
.expect("failed provider hydration must leave canonical history intact");
assert!(
matches!(
synced_live.messages(),
[Message::User(user)]
if matches!(
user.content.as_slice(),
[ContentBlock::Image {
data: ImageData::Blob { blob_id },
..
}] if blob_id == &blob_ref.blob_id
)
),
"missing image bytes must not degrade canonical history to a placeholder"
);
}
#[tokio::test]
async fn test_realtime_user_image_rejects_unmaterialized_causal_predecessor() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store: Arc<dyn RuntimeStore> = Arc::new(InMemoryRuntimeStore::new());
let blob_store = Arc::new(MemoryBlobStore::new());
let service_blob_store: Arc<dyn BlobStore> = blob_store.clone();
let service = PersistentSessionService::new(
CapabilityBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store),
service_blob_store,
);
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create_session should succeed");
service
.append_realtime_transcript_event(
&created.session_id,
meerkat_core::RealtimeTranscriptEvent::AssistantTextDelta {
response_id: "response_before_image".to_string(),
delta_id: "delta_before_image".to_string(),
item_id: "assistant_before_image".to_string(),
previous_item_id: None,
content_index: 0,
delta: "before image".to_string(),
},
)
.await
.expect("assistant predecessor should stage");
let image_data = "iVBORw0KGgo=";
let image_blob_id = meerkat_core::blob::content_blob_id("image/png", image_data);
let image_event = meerkat_core::RealtimeTranscriptEvent::UserContentFinal {
idempotency_key: "image-request-1".to_string(),
item_id: "image_after_assistant".to_string(),
previous_item_id: Some("assistant_before_image".to_string()),
content_index: 0,
content: vec![ContentBlock::Image {
media_type: "image/png".to_string(),
data: ImageData::Inline {
data: image_data.to_string(),
},
}],
};
let outcome = service
.append_realtime_transcript_event(&created.session_id, image_event.clone())
.await
.expect("image rejection should be a typed reducer outcome");
assert!(matches!(
outcome.user_content,
Some(
meerkat_core::RealtimeUserContentApplyOutcome::RejectedUnmaterializedPredecessor { .. }
)
));
let replay = service
.append_realtime_transcript_event(&created.session_id, image_event)
.await
.expect("duplicate invalid image should remain a typed rejection");
assert!(matches!(
replay.user_content,
Some(
meerkat_core::RealtimeUserContentApplyOutcome::RejectedUnmaterializedPredecessor { .. }
)
));
assert!(matches!(
blob_store.get(&image_blob_id).await,
Err(meerkat_core::BlobStoreError::NotFound(_))
));
let staged = PersistentSessionService::<CapabilityBuilder>::
load_runtime_session_snapshot_for_session(&runtime_store, &created.session_id)
.await
.expect("runtime snapshot load should succeed")
.expect("runtime snapshot should exist");
let staged_json = serde_json::to_string(&staged).expect("serialize staged session");
assert!(
!staged_json.contains(image_data),
"causally waiting image bytes must never enter durable session metadata"
);
assert!(!staged_json.contains(image_blob_id.as_str()));
assert!(staged.messages().is_empty());
let completed = service
.append_realtime_transcript_event(
&created.session_id,
meerkat_core::RealtimeTranscriptEvent::AssistantTurnCompleted {
response_id: "response_before_image".to_string(),
stop_reason: StopReason::EndTurn,
usage: meerkat_core::Usage::default(),
},
)
.await
.expect("predecessor completion should materialize only the accepted predecessor");
assert_eq!(completed.materialized_messages.len(), 1);
let durable = PersistentSessionService::<CapabilityBuilder>::
load_runtime_session_snapshot_for_session(&runtime_store, &created.session_id)
.await
.expect("runtime snapshot load should succeed")
.expect("runtime snapshot should exist");
assert_eq!(durable.messages().len(), 1);
assert!(matches!(durable.messages()[0], Message::BlockAssistant(_)));
}
#[tokio::test]
async fn test_realtime_user_image_replay_and_conflict_preflight_before_blob_write() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store: Arc<dyn RuntimeStore> = Arc::new(InMemoryRuntimeStore::new());
let blob_store = Arc::new(MemoryBlobStore::new());
let service = PersistentSessionService::new(
CapabilityBuilder,
4,
store,
runtime_store,
blob_store.clone(),
);
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create session");
let image =
|item_id: &str, data: &str| meerkat_core::RealtimeTranscriptEvent::UserContentFinal {
idempotency_key: "stable-image-key".to_string(),
item_id: item_id.to_string(),
previous_item_id: None,
content_index: 0,
content: vec![ContentBlock::Image {
media_type: "image/png".to_string(),
data: ImageData::Inline {
data: data.to_string(),
},
}],
};
let canonical_image = "iVBORw0KGgo=";
let conflicting_image = "iVBORw0KGgoB";
let committed = service
.append_realtime_transcript_event(
&created.session_id,
image("canonical-item", canonical_image),
)
.await
.expect("first image commit");
assert!(matches!(
committed.user_content,
Some(meerkat_core::RealtimeUserContentApplyOutcome::Committed(_))
));
let replay = service
.append_realtime_transcript_event(
&created.session_id,
image("retry-item", canonical_image),
)
.await
.expect("exact retry");
assert!(matches!(
replay.user_content,
Some(meerkat_core::RealtimeUserContentApplyOutcome::AlreadyCommitted(
meerkat_core::RealtimeUserContentIdentity { ref item_id, .. }
)) if item_id == "canonical-item"
));
let conflicting_blob = meerkat_core::blob::content_blob_id("image/png", conflicting_image);
let conflict = service
.append_realtime_transcript_event(
&created.session_id,
image("conflict-item", conflicting_image),
)
.await
.expect("conflict is typed");
assert!(matches!(
conflict.user_content,
Some(meerkat_core::RealtimeUserContentApplyOutcome::RejectedConflict { .. })
));
assert!(matches!(
blob_store.get(&conflicting_blob).await,
Err(meerkat_core::BlobStoreError::NotFound(_))
));
let durable = service
.export_realtime_refresh_session_snapshot(&created.session_id)
.await
.expect("durable session remains valid");
assert_eq!(durable.messages().len(), 1);
assert_eq!(durable.realtime_user_content_identities().len(), 1);
}
#[tokio::test]
async fn test_realtime_raw_blob_reference_is_verified_before_stage_or_receipt() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store: Arc<dyn RuntimeStore> = Arc::new(InMemoryRuntimeStore::new());
let blob_store = Arc::new(MemoryBlobStore::new());
let service = PersistentSessionService::new(
CapabilityBuilder,
4,
store,
runtime_store.clone(),
blob_store,
);
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create session");
let missing_id = meerkat_core::blob::content_blob_id("image/png", "iVBORw0KGgo=");
let error = service
.append_realtime_transcript_event(
&created.session_id,
meerkat_core::RealtimeTranscriptEvent::UserContentFinal {
idempotency_key: "missing-raw-image".to_string(),
item_id: "missing-raw-item".to_string(),
previous_item_id: None,
content_index: 0,
content: vec![ContentBlock::Image {
media_type: "image/png".to_string(),
data: ImageData::Blob {
blob_id: missing_id,
},
}],
},
)
.await
.expect_err("missing raw blob must not receive a commit receipt");
assert!(error.to_string().contains("blob not found"));
let durable = PersistentSessionService::<CapabilityBuilder>::
load_runtime_session_snapshot_for_session(&runtime_store, &created.session_id)
.await
.expect("runtime snapshot load")
.expect("created snapshot");
assert!(durable.messages().is_empty());
assert!(durable.pending_realtime_user_content_blob().is_none());
assert!(durable.realtime_user_content_identities().is_empty());
}
#[tokio::test]
async fn test_realtime_exact_inline_replay_repairs_missing_committed_blob_before_receipt() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store: Arc<dyn RuntimeStore> = Arc::new(InMemoryRuntimeStore::new());
let blob_store = Arc::new(MemoryBlobStore::new());
let service = PersistentSessionService::new(
CapabilityBuilder,
4,
store,
runtime_store,
blob_store.clone(),
);
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create session");
let image_data = "iVBORw0KGgo=";
let blob_id = meerkat_core::blob::content_blob_id("image/png", image_data);
let image = |item_id: &str| meerkat_core::RealtimeTranscriptEvent::UserContentFinal {
idempotency_key: "repairable-image-key".to_string(),
item_id: item_id.to_string(),
previous_item_id: None,
content_index: 0,
content: vec![ContentBlock::Image {
media_type: "image/png".to_string(),
data: ImageData::Inline {
data: image_data.to_string(),
},
}],
};
service
.append_realtime_transcript_event(&created.session_id, image("canonical-item"))
.await
.expect("initial image commit");
blob_store
.delete(&blob_id)
.await
.expect("simulate lost committed object");
let replay = service
.append_realtime_transcript_event(&created.session_id, image("retry-item"))
.await
.expect("inline retry must repair before replay receipt");
assert!(matches!(
replay.user_content,
Some(meerkat_core::RealtimeUserContentApplyOutcome::AlreadyCommitted(_))
));
let repaired = blob_store
.get(&blob_id)
.await
.expect("exact replay must restore the object");
assert_eq!(repaired.media_type, "image/png");
assert_eq!(repaired.data, image_data);
}
#[tokio::test]
async fn test_realtime_blob_crash_before_put_leaves_no_object_or_anchor() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(GatedSnapshotRuntimeStore::new());
let runtime_store_authority: Arc<dyn RuntimeStore> = runtime_store.clone();
let blob_store = Arc::new(MemoryBlobStore::new());
let service = PersistentSessionService::new(
CapabilityBuilder,
4,
store,
runtime_store_authority.clone(),
blob_store.clone(),
);
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create session");
let data = "iVBORw0KGgo=";
let blob_id = meerkat_core::blob::content_blob_id("image/png", data);
runtime_store.fail_snapshot_commit_in(1);
let error = service
.append_realtime_transcript_event(
&created.session_id,
realtime_inline_image_event("crash-before-put", "crash-before-put-item", data),
)
.await
.expect_err("anchor persistence failure must stop before put");
assert!(error.to_string().contains("snapshot persistence failed"));
assert!(matches!(
blob_store.get(&blob_id).await,
Err(meerkat_core::BlobStoreError::NotFound(_))
));
let durable = PersistentSessionService::<CapabilityBuilder>::
load_runtime_session_snapshot_for_session(
&runtime_store_authority,
&created.session_id,
)
.await
.expect("runtime snapshot load")
.expect("initial durable snapshot remains");
assert!(durable.pending_realtime_user_content_blob().is_none());
assert!(durable.messages().is_empty());
}
#[tokio::test]
async fn test_realtime_blob_crash_after_directory_synced_put_recovers_exactly_once() {
let temp = tempfile::tempdir().expect("temporary filesystem blob root");
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store: Arc<dyn RuntimeStore> = Arc::new(InMemoryRuntimeStore::new());
let blob_store = Arc::new(FailOneBoundedReadBlobStore::new(temp.path().join("blobs")));
let service = PersistentSessionService::new(
CapabilityBuilder,
4,
store,
runtime_store.clone(),
blob_store.clone(),
);
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create session");
let data = "iVBORw0KGgo=";
let blob_id = meerkat_core::blob::content_blob_id("image/png", data);
let event = realtime_inline_image_event("crash-after-put", "crash-after-put-item", data);
let error = service
.append_realtime_transcript_event(&created.session_id, event.clone())
.await
.expect_err("synthetic post-put crash must withhold receipt");
assert!(
error
.to_string()
.contains("synthetic crash after durable put")
);
let staged = PersistentSessionService::<CapabilityBuilder>::
load_runtime_session_snapshot_for_session(&runtime_store, &created.session_id)
.await
.expect("runtime snapshot load")
.expect("staged anchor snapshot");
assert_eq!(staged.messages().len(), 0);
assert!(staged.pending_realtime_user_content_blob().is_some());
assert!(!serde_json::to_string(&staged).unwrap().contains(data));
assert_eq!(
blob_store
.inner
.get(&blob_id)
.await
.expect("fsynced blob")
.data,
data
);
let recovered = service
.append_realtime_transcript_event(&created.session_id, event)
.await
.expect("exact retry must consume the durable anchor");
assert!(matches!(
recovered.user_content,
Some(meerkat_core::RealtimeUserContentApplyOutcome::Committed(_))
));
let durable = PersistentSessionService::<CapabilityBuilder>::
load_runtime_session_snapshot_for_session(&runtime_store, &created.session_id)
.await
.expect("runtime snapshot load")
.expect("final snapshot");
assert!(durable.pending_realtime_user_content_blob().is_none());
assert_eq!(durable.messages().len(), 1);
assert_eq!(durable.realtime_user_content_identities().len(), 1);
}
#[tokio::test]
async fn test_realtime_open_reconciles_verified_pending_image_before_provider_seed() {
let temp = tempfile::tempdir().expect("temporary filesystem blob root");
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store: Arc<dyn RuntimeStore> = Arc::new(InMemoryRuntimeStore::new());
let blob_store = Arc::new(FailOneBoundedReadBlobStore::new(temp.path().join("blobs")));
let service = PersistentSessionService::new(
CapabilityBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store),
blob_store.clone(),
);
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create session");
let first_data = "iVBORw0KGgo=";
let first_event =
realtime_inline_image_event("pre-crash-image", "pre-crash-item", first_data);
service
.append_realtime_transcript_event(&created.session_id, first_event)
.await
.expect_err("synthetic post-put crash must leave a verified pending anchor");
let staged = PersistentSessionService::<CapabilityBuilder>::
load_runtime_session_snapshot_for_session(&runtime_store, &created.session_id)
.await
.expect("runtime snapshot load")
.expect("staged anchor snapshot");
assert!(staged.pending_realtime_user_content_blob().is_some());
assert!(staged.messages().is_empty());
drop(service);
let recovery = PersistentSessionService::new(
CapabilityBuilder,
4,
store,
Arc::clone(&runtime_store),
blob_store,
);
recovery
.create_session(resume_request(staged))
.await
.expect("cold resume must restore the pending anchor");
let projected = recovery
.export_realtime_open_session_snapshot(&created.session_id)
.await
.expect("realtime open must reconcile before projection");
assert!(projected.pending_realtime_user_content_blob().is_none());
assert_eq!(projected.messages().len(), 1);
assert!(matches!(
&projected.messages()[0],
Message::User(user)
if matches!(
user.content.as_slice(),
[ContentBlock::Image {
media_type,
data: ImageData::Inline { data },
}] if media_type == "image/png" && data == first_data
)
));
let second_data = "iVBORw0KGgoA";
recovery
.append_realtime_transcript_event(
&created.session_id,
realtime_inline_image_event("post-reopen-image", "post-reopen-item", second_data),
)
.await
.expect("different post-open image may commit only after old image was seeded");
let durable = PersistentSessionService::<CapabilityBuilder>::
load_runtime_session_snapshot_for_session(&runtime_store, &created.session_id)
.await
.expect("runtime snapshot load")
.expect("recovered durable snapshot");
assert!(durable.pending_realtime_user_content_blob().is_none());
assert_eq!(durable.messages().len(), 2);
assert_eq!(durable.realtime_user_content_identities().len(), 2);
}
#[tokio::test]
async fn test_realtime_blob_final_snapshot_failure_recovers_bounded_pending_anchor() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(GatedSnapshotRuntimeStore::new());
let runtime_store_authority: Arc<dyn RuntimeStore> = runtime_store.clone();
let blob_store = Arc::new(MemoryBlobStore::new());
let service = PersistentSessionService::new(
CapabilityBuilder,
4,
store.clone(),
runtime_store_authority.clone(),
blob_store.clone(),
);
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create session");
let data = "iVBORw0KGgo=";
let event = realtime_inline_image_event(
"crash-before-final-save",
"crash-before-final-save-item",
data,
);
runtime_store.fail_snapshot_commit_in(2);
let error = service
.append_realtime_transcript_event(&created.session_id, event.clone())
.await
.expect_err("final snapshot failure must withhold receipt");
assert!(error.to_string().contains("snapshot persistence failed"));
let staged = PersistentSessionService::<CapabilityBuilder>::
load_runtime_session_snapshot_for_session(
&runtime_store_authority,
&created.session_id,
)
.await
.expect("runtime snapshot load")
.expect("anchor snapshot must remain authority");
assert!(staged.pending_realtime_user_content_blob().is_some());
assert!(staged.messages().is_empty());
let staged_json = serde_json::to_string(&staged).expect("serialize staged snapshot");
assert!(!staged_json.contains(data));
let recovery = PersistentSessionService::new(
CapabilityBuilder,
4,
store,
runtime_store_authority.clone(),
blob_store,
);
let resumed = recovery
.create_session(resume_request(staged))
.await
.expect("cold resume must restore the bounded anchor");
assert_eq!(resumed.session_id, created.session_id);
let outcome = recovery
.append_realtime_transcript_event(&created.session_id, event)
.await
.expect("exact inline retry must complete after final-save crash");
assert!(matches!(
outcome.user_content,
Some(meerkat_core::RealtimeUserContentApplyOutcome::Committed(_))
));
let durable = PersistentSessionService::<CapabilityBuilder>::
load_runtime_session_snapshot_for_session(
&runtime_store_authority,
&created.session_id,
)
.await
.expect("runtime snapshot load")
.expect("recovered final snapshot");
assert!(durable.pending_realtime_user_content_blob().is_none());
assert_eq!(durable.messages().len(), 1);
assert_eq!(durable.realtime_user_content_identities().len(), 1);
}
#[tokio::test]
async fn test_transcript_rewrite_tombstone_survives_cold_restore_and_blocks_blob_write() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store: Arc<dyn RuntimeStore> = Arc::new(InMemoryRuntimeStore::new());
let blob_store = Arc::new(MemoryBlobStore::new());
let service = PersistentSessionService::new(
CapabilityBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store),
blob_store.clone(),
);
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create session");
let image = |key: &str, item_id: &str, data: &str| {
meerkat_core::RealtimeTranscriptEvent::UserContentFinal {
idempotency_key: key.to_string(),
item_id: item_id.to_string(),
previous_item_id: None,
content_index: 0,
content: vec![ContentBlock::Image {
media_type: "image/png".to_string(),
data: ImageData::Inline {
data: data.to_string(),
},
}],
}
};
let data = "iVBORw0KGgo=";
let blob_id = meerkat_core::blob::content_blob_id("image/png", data);
assert!(matches!(
service
.append_realtime_transcript_event(
&created.session_id,
image("removed-key", "removed-item", data),
)
.await
.expect("commit original image")
.user_content,
Some(meerkat_core::RealtimeUserContentApplyOutcome::Committed(_))
));
let before = service
.export_realtime_refresh_session_snapshot(&created.session_id)
.await
.expect("read parent");
let parent_revision = before.transcript_revision().expect("parent revision");
service
.rewrite_session_transcript(
&created.session_id,
SessionTranscriptRewriteRequest {
selection: TranscriptRewriteSelection::MessageRange { start: 0, end: 1 },
replacement: vec![Message::User(UserMessage::text("image removed"))],
reason: TranscriptRewriteReason::new("remove-image"),
actor: None,
expected_parent_revision: Some(parent_revision),
running_behavior: TranscriptEditRunningBehavior::Reject,
},
)
.await
.expect("rewrite should commit tombstone");
blob_store
.delete(&blob_id)
.await
.expect("remove old blob to prove retry does not rewrite it");
service
.discard_live_session(&created.session_id)
.await
.expect("force cold restore path");
drop(service);
let recovery = PersistentSessionService::new(
CapabilityBuilder,
4,
store,
runtime_store,
blob_store.clone(),
);
let resume_source = recovery
.load_authoritative_session(&created.session_id)
.await
.expect("cold-load rewritten runtime authority")
.expect("rewritten runtime authority remains present after restart");
recovery
.create_session(resume_request(resume_source))
.await
.expect("cold resume must materialize the rewritten runtime authority");
let retry = recovery
.append_realtime_transcript_event(
&created.session_id,
image("removed-key", "retry-item", data),
)
.await
.expect("tombstoned retry is a typed rejection");
assert!(matches!(
retry.user_content,
Some(meerkat_core::RealtimeUserContentApplyOutcome::RejectedConflict { .. })
));
assert!(retry.materialized_messages.is_empty());
assert!(matches!(
blob_store.get(&blob_id).await,
Err(meerkat_core::BlobStoreError::NotFound(_))
));
let fresh = recovery
.append_realtime_transcript_event(
&created.session_id,
image("fresh-key", "fresh-item", data),
)
.await
.expect("new key should commit after rewrite");
assert!(matches!(
fresh.user_content,
Some(meerkat_core::RealtimeUserContentApplyOutcome::Committed(_))
));
let durable = recovery
.export_realtime_refresh_session_snapshot(&created.session_id)
.await
.expect("rewritten durable session remains valid");
assert_eq!(durable.realtime_user_content_identities().len(), 1);
assert_eq!(durable.realtime_user_content_tombstones().len(), 1);
assert_eq!(durable.messages().len(), 2);
}
#[tokio::test]
async fn test_transcript_rewrite_retaining_image_preserves_exact_replay_after_cold_restore() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store: Arc<dyn RuntimeStore> = Arc::new(InMemoryRuntimeStore::new());
let blob_store = Arc::new(MemoryBlobStore::new());
let service = PersistentSessionService::new(
CapabilityBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store),
blob_store.clone(),
);
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create session");
let data = "iVBORw0KGgo=";
let image = |item_id: &str| meerkat_core::RealtimeTranscriptEvent::UserContentFinal {
idempotency_key: "retained-key".to_string(),
item_id: item_id.to_string(),
previous_item_id: None,
content_index: 0,
content: vec![ContentBlock::Image {
media_type: "image/png".to_string(),
data: ImageData::Inline {
data: data.to_string(),
},
}],
};
service
.append_realtime_transcript_event(&created.session_id, image("canonical-item"))
.await
.expect("commit original image");
let before = service
.export_realtime_refresh_session_snapshot(&created.session_id)
.await
.expect("read parent");
let parent_revision = before.transcript_revision().expect("parent revision");
service
.rewrite_session_transcript(
&created.session_id,
SessionTranscriptRewriteRequest {
selection: TranscriptRewriteSelection::MessageRange { start: 0, end: 1 },
replacement: vec![
before.messages()[0].clone(),
Message::User(UserMessage::text("neighbor changed")),
],
reason: TranscriptRewriteReason::new("retain-image"),
actor: None,
expected_parent_revision: Some(parent_revision),
running_behavior: TranscriptEditRunningBehavior::Reject,
},
)
.await
.expect("rewrite retaining canonical image");
service
.discard_live_session(&created.session_id)
.await
.expect("force cold restore path");
drop(service);
let recovery =
PersistentSessionService::new(CapabilityBuilder, 4, store, runtime_store, blob_store);
let resume_source = recovery
.load_authoritative_session(&created.session_id)
.await
.expect("cold-load rewritten runtime authority")
.expect("rewritten runtime authority remains present after restart");
recovery
.create_session(resume_request(resume_source))
.await
.expect("cold resume must materialize the rewritten runtime authority");
let replay = recovery
.append_realtime_transcript_event(&created.session_id, image("ignored-retry-item"))
.await
.expect("exact retained retry");
assert!(matches!(
replay.user_content,
Some(meerkat_core::RealtimeUserContentApplyOutcome::AlreadyCommitted(
meerkat_core::RealtimeUserContentIdentity { ref item_id, .. }
)) if item_id == "canonical-item"
));
assert!(replay.materialized_messages.is_empty());
}
#[tokio::test]
async fn test_authoritative_load_materializes_no_rewrite_record_bodies() {
async fn load_cost(rewrites: usize) -> (u64, u64) {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store: Arc<dyn RuntimeStore> = Arc::new(InMemoryRuntimeStore::new());
let dir = tempfile::tempdir().expect("temp dir");
let event_store = Arc::new(ObservingEventStore::new(dir.path().join("events")));
let service = PersistentSessionService::new(
CapabilityBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store),
memory_blob_store(),
)
.with_event_projection(
Arc::clone(&event_store) as Arc<dyn EventStore>,
Arc::new(SessionProjector::new(dir.path().join(".rkat"))),
);
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create session");
for generation in 0..=rewrites {
let parent = service
.export_realtime_refresh_session_snapshot(&created.session_id)
.await
.expect("read rewrite parent");
let parent_revision = parent.transcript_revision().expect("parent revision");
let (selection, replacement) = if generation == 0 {
(
TranscriptRewriteSelection::MessageRange { start: 0, end: 0 },
(0..12)
.map(|index| Message::User(UserMessage::text(format!("turn {index}"))))
.collect(),
)
} else {
(
TranscriptRewriteSelection::MessageRange { start: 0, end: 1 },
vec![Message::User(UserMessage::text(format!(
"system prompt generation {generation}"
)))],
)
};
service
.rewrite_session_transcript(
&created.session_id,
SessionTranscriptRewriteRequest {
selection,
replacement,
reason: TranscriptRewriteReason::new("adversarial-rewrite-chain"),
actor: None,
expected_parent_revision: Some(parent_revision),
running_behavior: TranscriptEditRunningBehavior::Reject,
},
)
.await
.expect("rewrite commits");
}
service
.discard_live_session(&created.session_id)
.await
.expect("force the durable load path");
service
.load_authoritative_session_base(&created.session_id)
.await
.expect("warm the per-shape decode memo");
let session = service
.load_committed_runtime_session_for_body(
&created.session_id,
"rewrite-body materialization probe",
)
.await
.expect("load current runtime authority")
.expect("session remains present");
session
.validate_transcript_history_state()
.expect("prove current compact graph before audit-tail measurement");
event_store.reset();
let hashed_before = meerkat_core::session_content_digest_bytes();
let replay = service
.apply_transcript_rewrite_replay(&created.session_id, Some(session))
.await
.expect("measured rewrite-audit replay");
service
.verify_transcript_rewrite_audit_events(
replay.session.as_ref().expect("session remains present"),
&replay,
)
.await
.expect("measured rewrite-audit verification");
let rows = event_store.raw_reads();
assert!(
rows.iter().all(|(scope, _)| *scope == 2),
"steady-state load must use only a store-authorized tail: {rows:?}"
);
(
meerkat_core::session_content_digest_bytes() - hashed_before,
rows.into_iter().map(|(_, rows)| rows as u64).sum(),
)
}
let (few_hashed, few_rows) = load_cost(2).await;
let (many_hashed, many_rows) = load_cost(8).await;
println!(
"authoritative load: 2 rewrites hashed {few_hashed} B / returned {few_rows} body rows, \
8 rewrites hashed {many_hashed} B / returned {many_rows} body rows"
);
assert_eq!(
(few_rows, many_rows),
(0, 0),
"a steady-state authoritative load folds no rewrite, so it must \
receive no rewrite record body: 2 rewrites returned \
{few_rows}, 8 returned {many_rows}"
);
assert_eq!(
few_hashed, many_hashed,
"a steady-state authoritative load must re-prove nothing the loaded \
session already proves, so its hashed bytes cannot grow with the \
retained revision count: 2 rewrites hashed {few_hashed} bytes, 8 \
hashed {many_hashed}"
);
}
struct ObservingEventStore {
inner: crate::event_store::FileEventStore,
reads: std::sync::Mutex<Vec<(u64, usize)>>,
raw_reads: std::sync::Mutex<Vec<(u64, usize)>>,
}
impl ObservingEventStore {
fn new(root: std::path::PathBuf) -> Self {
Self {
inner: crate::event_store::FileEventStore::new(root),
reads: std::sync::Mutex::new(Vec::new()),
raw_reads: std::sync::Mutex::new(Vec::new()),
}
}
fn reset(&self) {
self.reads.lock().expect("reads lock").clear();
self.raw_reads.lock().expect("raw reads lock").clear();
}
fn raw_reads(&self) -> Vec<(u64, usize)> {
self.raw_reads.lock().expect("raw reads lock").clone()
}
fn typed_reads(&self) -> Vec<(u64, usize)> {
self.reads.lock().expect("reads lock").clone()
}
fn rows_read(&self) -> usize {
self.raw_reads()
.into_iter()
.chain(self.typed_reads())
.map(|(_, rows)| rows)
.sum()
}
}
#[async_trait::async_trait]
impl EventStore for ObservingEventStore {
async fn append_envelopes(
&self,
session_id: &SessionId,
envelopes: &[meerkat_core::event::EventEnvelope<AgentEvent>],
) -> Result<u64, EventStoreError> {
self.inner.append_envelopes(session_id, envelopes).await
}
async fn append_transcript_rewrite_receipt_exact(
&self,
session_id: &SessionId,
receipt: &meerkat_core::TranscriptRewriteAuditReceiptBatch,
final_assistant_text: Option<&str>,
) -> Result<crate::event_store::ExactTranscriptRewriteReceiptAppend, EventStoreError>
{
self.inner
.append_transcript_rewrite_receipt_exact(session_id, receipt, final_assistant_text)
.await
}
async fn append_interaction_terminal_exact(
&self,
session_id: &SessionId,
interaction_id: meerkat_core::interaction::InteractionId,
envelope: &meerkat_core::event::EventEnvelope<AgentEvent>,
) -> Result<ExactInteractionAppend, EventStoreError> {
self.inner
.append_interaction_terminal_exact(session_id, interaction_id, envelope)
.await
}
async fn record_projection_halt(
&self,
session_id: &SessionId,
reason: &str,
) -> Result<(), EventStoreError> {
self.inner.record_projection_halt(session_id, reason).await
}
async fn projection_halt(
&self,
session_id: &SessionId,
) -> Result<Option<crate::event_store::EventProjectionHaltMarker>, EventStoreError>
{
self.inner.projection_halt(session_id).await
}
async fn read_from(
&self,
session_id: &SessionId,
from_seq: u64,
) -> Result<Vec<StoredEvent>, EventStoreError> {
let rows = self.inner.read_from(session_id, from_seq).await?;
self.reads
.lock()
.expect("reads lock")
.push((from_seq, rows.len()));
Ok(rows)
}
async fn read_raw_from(
&self,
session_id: &SessionId,
from_seq: u64,
) -> Result<Option<Vec<crate::event_store::RawStoredEvent>>, EventStoreError> {
let rows = self.inner.read_raw_from(session_id, from_seq).await?;
if let Some(rows) = rows.as_ref() {
self.raw_reads
.lock()
.expect("raw reads lock")
.push((from_seq, rows.len()));
}
Ok(rows)
}
async fn read_transcript_rewrite_audit(
&self,
session_id: &SessionId,
expectation: crate::event_store::TranscriptRewriteAuditExpectation<'_>,
) -> Result<Option<crate::event_store::TranscriptRewriteAuditRead>, EventStoreError>
{
let read = self
.inner
.read_transcript_rewrite_audit(session_id, expectation)
.await?;
if let Some(read) = read.as_ref() {
let (scope, rows) = match read {
crate::event_store::TranscriptRewriteAuditRead::AuthorizedTail(rows) => {
(2, rows.rewrite_rows().len())
}
crate::event_store::TranscriptRewriteAuditRead::FullReconciliation(rows) => {
(1, rows.rewrite_rows().len())
}
};
self.raw_reads
.lock()
.expect("raw reads lock")
.push((scope, rows));
}
Ok(read)
}
async fn finalize_transcript_rewrite_audit(
&self,
receipt: &crate::event_store::TranscriptRewritePrefixReceipt,
) -> Result<(), EventStoreError> {
self.inner.finalize_transcript_rewrite_audit(receipt).await
}
async fn read_from_bounded(
&self,
session_id: &SessionId,
from_seq: u64,
max_rows: usize,
) -> Result<Vec<StoredEvent>, EventStoreError> {
self.inner
.read_from_bounded(session_id, from_seq, max_rows)
.await
}
async fn last_seq(&self, session_id: &SessionId) -> Result<u64, EventStoreError> {
self.inner.last_seq(session_id).await
}
}
async fn audit_fixture(
dir: &tempfile::TempDir,
rewrites: usize,
) -> (
PersistentSessionService<CapabilityBuilder>,
Arc<ObservingEventStore>,
SessionId,
) {
let (service, event_store, id, _, _) = audit_fixture_with_stores(dir, rewrites).await;
(service, event_store, id)
}
async fn audit_fixture_with_stores(
dir: &tempfile::TempDir,
rewrites: usize,
) -> (
PersistentSessionService<CapabilityBuilder>,
Arc<ObservingEventStore>,
SessionId,
Arc<dyn SessionStore>,
Arc<InMemoryRuntimeStore>,
) {
let event_store = Arc::new(ObservingEventStore::new(dir.path().join("events")));
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
CapabilityBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store) as Arc<dyn RuntimeStore>,
memory_blob_store(),
)
.with_event_projection(
Arc::clone(&event_store) as Arc<dyn EventStore>,
Arc::new(SessionProjector::new(dir.path().join(".rkat"))),
);
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create session");
commit_resume_shaped_rewrites(&service, &created.session_id, 0..=rewrites).await;
service
.discard_live_session(&created.session_id)
.await
.expect("force the durable load path");
(
service,
event_store,
created.session_id,
store,
runtime_store,
)
}
async fn commit_resume_shaped_rewrites(
service: &PersistentSessionService<CapabilityBuilder>,
id: &SessionId,
generations: std::ops::RangeInclusive<usize>,
) {
for generation in generations {
let parent = service
.export_realtime_refresh_session_snapshot(id)
.await
.expect("read rewrite parent");
let parent_revision = parent.transcript_revision().expect("parent revision");
let (selection, replacement) = if generation == 0 {
(
TranscriptRewriteSelection::MessageRange { start: 0, end: 0 },
(0..12)
.map(|index| Message::User(UserMessage::text(format!("turn {index}"))))
.collect(),
)
} else {
(
TranscriptRewriteSelection::MessageRange { start: 0, end: 1 },
vec![Message::User(UserMessage::text(format!(
"system prompt generation {generation}"
)))],
)
};
service
.rewrite_session_transcript(
id,
SessionTranscriptRewriteRequest {
selection,
replacement,
reason: TranscriptRewriteReason::new("adversarial-rewrite-chain"),
actor: None,
expected_parent_revision: Some(parent_revision),
running_behavior: TranscriptEditRunningBehavior::Reject,
},
)
.await
.expect("rewrite commits");
}
}
#[tokio::test]
async fn test_steady_state_load_reads_no_rewrite_rows_it_already_folded() {
async fn rows_for(rewrites: usize) -> usize {
let dir = tempfile::tempdir().expect("temp dir");
let (service, event_store, id) = audit_fixture(&dir, rewrites).await;
let session = service
.load_committed_runtime_session_for_body(&id, "rewrite-audit cost probe")
.await
.expect("load measured runtime authority")
.expect("session is present");
session
.validate_transcript_history_state()
.expect("prove current compact graph before audit-tail measurement");
event_store.reset();
let before = meerkat_core::rewrite_record_body_decodes();
let replay = service
.apply_transcript_rewrite_replay(&id, Some(session))
.await
.expect("measured rewrite-audit replay");
service
.verify_transcript_rewrite_audit_events(
replay.session.as_ref().expect("session remains present"),
&replay,
)
.await
.expect("measured rewrite-audit verification");
let decoded = meerkat_core::rewrite_record_body_decodes() - before;
let reads = event_store.raw_reads();
assert!(
reads.iter().all(|(scope, _)| *scope == 2),
"steady state must use only store-authorized tails: {reads:?}"
);
let log_rows = event_store.last_seq(&id).await.expect("log high-water");
let rows = event_store.rows_read();
println!(
"{rewrites} rewrites: load read {rows} of {log_rows} log rows \
(from {reads:?}), materialized {decoded} record bodies"
);
rows
}
let few = rows_for(2).await;
let many = rows_for(8).await;
assert_eq!(
few, many,
"the rows an authoritative load reads must not grow with the retained \
rewrite count: 2 rewrites read {few} rows, 8 read {many}"
);
}
fn graph_test_rewrite_chain(count: usize) -> Vec<meerkat_core::TranscriptRewriteRecord> {
let mut messages = (0..6)
.map(|index| Message::User(UserMessage::text(format!("turn {index}"))))
.collect::<Vec<_>>();
let body = |messages: Vec<Message>, parent: Option<&str>| {
let revision =
meerkat_core::transcript_messages_digest(&messages).expect("digest body");
meerkat_core::TranscriptRevisionBody {
revision,
parent_revision: parent.map(str::to_string),
messages,
created_at: meerkat_core::time_compat::SystemTime::UNIX_EPOCH,
}
};
let mut records = Vec::with_capacity(count);
for generation in 0..count {
let parent_body = body(messages.clone(), None);
messages[0] = Message::User(UserMessage::text(format!(
"system prompt generation {generation}"
)));
let revision_body = body(messages.clone(), Some(&parent_body.revision));
let commit = meerkat_core::TranscriptRewriteCommit {
rewrite_generation: u64::try_from(generation)
.expect("test generation fits u64")
.checked_add(1)
.expect("test generation does not overflow"),
parent_revision: parent_body.revision.clone(),
revision: revision_body.revision.clone(),
selection: TranscriptRewriteSelection::MessageRange { start: 0, end: 1 },
original_span_digest: meerkat_core::transcript_messages_digest(
&parent_body.messages[..1],
)
.expect("digest original span"),
replacement_digest: meerkat_core::transcript_messages_digest(
&revision_body.messages[..1],
)
.expect("digest replacement span"),
messages_before: parent_body.messages.len(),
messages_after: revision_body.messages.len(),
reason: TranscriptRewriteReason::new("adversarial-rewrite-chain"),
actor: None,
committed_at: meerkat_core::time_compat::SystemTime::UNIX_EPOCH,
};
records.push(
meerkat_core::TranscriptRewriteRecord::new(commit, parent_body, revision_body)
.expect("chain record is valid"),
);
}
records
}
#[tokio::test]
async fn test_a_load_that_folds_nothing_materializes_no_bodies_and_so_checks_none() {
let dir = tempfile::tempdir().expect("temp dir");
let (service, event_store, id) = audit_fixture(&dir, 3).await;
service
.load_authoritative_session_base(&id)
.await
.expect("warming load")
.expect("session is present");
event_store.reset();
let before = meerkat_core::rewrite_record_body_decodes();
service
.load_authoritative_session_base(&id)
.await
.expect("steady-state load succeeds")
.expect("session is present");
let decoded = meerkat_core::rewrite_record_body_decodes() - before;
assert_eq!(
decoded, 0,
"a load with nothing to fold must not build a single record body; \
it built {decoded}"
);
assert!(
event_store.typed_reads().is_empty(),
"and must not take the typed read at all: {:?}",
event_store.typed_reads()
);
}
#[tokio::test]
async fn test_realtime_open_snapshot_rejects_repeated_images_over_cumulative_budget() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store: Arc<dyn RuntimeStore> = Arc::new(InMemoryRuntimeStore::new());
let blob_store = Arc::new(MemoryBlobStore::new());
let service_blob_store: Arc<dyn BlobStore> = blob_store.clone();
let service = PersistentSessionService::new(
CapabilityBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store),
service_blob_store,
);
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create_session should succeed");
let blob_ref = blob_store
.put_image("image/png", "AQID")
.await
.expect("image fixture should persist");
let durable = service
.export_live_session(&created.session_id)
.await
.expect("initial live session should export");
let durable = mutate_test_session(durable, "run boundary", |durable| {
for _ in 0..3 {
durable.push(Message::User(UserMessage::with_blocks(vec![
ContentBlock::Image {
media_type: "image/png".to_string(),
data: ImageData::Blob {
blob_id: blob_ref.blob_id.clone(),
},
},
])));
}
});
runtime_store
.commit_session_snapshot(
&LogicalRuntimeId::for_session(&created.session_id),
SerializedSessionSnapshot {
session_snapshot: (serde_json::to_vec(&durable)
.expect("serialize repeated-image session"))
.into(),
},
)
.await
.expect("commit durable runtime snapshot");
let error = service
.export_realtime_open_session_snapshot_with_image_budget(&created.session_id, 6)
.await
.expect_err("three repeated three-byte images must exceed a six-byte budget");
assert!(
matches!(
error,
SessionError::Agent(AgentError::InternalError(ref message))
if message.contains("exceeds the 6-byte cumulative decoded limit")
&& message.contains("attempted 9 bytes")
),
"unexpected cumulative image budget error: {error}"
);
let synced_live = service
.export_live_session(&created.session_id)
.await
.expect("failed projection hydration must leave canonical live history intact");
assert_eq!(synced_live.messages().len(), 3);
assert!(synced_live.messages().iter().all(|message| matches!(
message,
Message::User(user)
if matches!(
user.content.as_slice(),
[ContentBlock::Image {
data: ImageData::Blob { blob_id },
..
}] if blob_id == &blob_ref.blob_id
)
)));
}
#[tokio::test]
async fn test_runtime_turn_synchronizes_stale_live_transcript_authority() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store: Arc<dyn RuntimeStore> = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
CapabilityBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store),
memory_blob_store(),
);
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create_session should succeed");
let durable = service
.export_live_session(&created.session_id)
.await
.expect("initial live session should export");
let durable = mutate_test_session(durable, "run boundary", |durable| {
durable.push(Message::User(UserMessage::text(
"durable runtime turn seed".to_string(),
)));
});
runtime_store
.commit_session_snapshot(
&LogicalRuntimeId::for_session(&created.session_id),
SerializedSessionSnapshot {
session_snapshot: (serde_json::to_vec(&durable)
.expect("serialize durable session"))
.into(),
},
)
.await
.expect("commit durable runtime snapshot");
store
.save(&durable)
.await
.expect("align the durable projection with the committed runtime checkpoint");
let output = service
.apply_runtime_turn(
&created.session_id,
RunId::new(),
runtime_content_turn_request("next turn"),
RunApplyBoundary::RunStart,
vec![InputId::new()],
)
.await
.expect("runtime turn should synchronize live semantics instead of rebuilding");
let output_session: Session = serde_json::from_slice(
output
.whole_blob_bytes()
.expect("whole-blob runtime boundary should encode")
.expect("runtime output should carry a session snapshot"),
)
.expect("runtime output session snapshot should deserialize");
assert!(
output_session.messages().iter().any(|message| matches!(
message,
Message::User(user) if user.text_content() == "durable runtime turn seed"
)),
"runtime output must start from durable semantic authority"
);
assert!(
service
.has_live_session(&created.session_id)
.await
.expect("live-session status should succeed"),
"runtime turn sync must preserve the live handle"
);
assert!(
service.comms_runtime(&created.session_id).await.is_some(),
"runtime turn sync must preserve live comms mechanics"
);
}
#[tokio::test]
async fn test_runtime_backed_archive_discards_uncommitted_live_snapshot() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store: Arc<dyn RuntimeStore> = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store),
memory_blob_store(),
);
let machine = MeerkatMachine::persistent(Arc::clone(&runtime_store), memory_blob_store());
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create_session should succeed");
let output = service
.apply_runtime_turn(
&created.session_id,
RunId::new(),
runtime_content_turn_request("uncommitted runtime turn"),
RunApplyBoundary::Immediate,
vec![InputId::new()],
)
.await
.expect("runtime apply should build output without committing authority");
let staged_snapshot: Session = serde_json::from_slice(
output
.whole_blob_bytes()
.expect("staged whole-blob runtime boundary should encode")
.expect("runtime output should carry a staged session snapshot"),
)
.expect("staged session snapshot should deserialize");
assert!(
!staged_snapshot.messages().is_empty(),
"test must create an uncommitted live transcript before archive()"
);
service
.archive_with_machine_protocol(
&created.session_id,
MachineSessionArchiveProtocol::from_machine(&machine),
)
.await
.expect("archive should retire through machine authority");
let runtime_id =
PersistentSessionService::<DummyBuilder>::runtime_id_for_session(&created.session_id);
let archived_body = runtime_store
.load_session_snapshot(&runtime_id)
.await
.expect("RuntimeStore body load should succeed")
.expect("RuntimeStore body should remain present");
let archived_body =
Session::from_persisted_bytes(&archived_body).expect("RuntimeStore body should decode");
assert!(
!session_marks_archived(&archived_body),
"archive authority belongs to RuntimeStore lifecycle, not the Session body"
);
assert_eq!(
meerkat_runtime::store::load_runtime_state(runtime_store.as_ref(), &runtime_id)
.await
.expect("runtime state load should succeed"),
Some(RuntimeState::Retired),
"archive must commit the RuntimeStore-owned lifecycle terminal"
);
assert!(
archived_body.messages().is_empty(),
"archive() must not persist uncommitted live transcript as durable truth"
);
assert!(
store
.load(&created.session_id)
.await
.expect("SessionStore probe should succeed")
.is_none(),
"archive must not mint a second SessionStore authority"
);
assert!(
!service
.has_live_session(&created.session_id)
.await
.expect("live-session status should succeed"),
"archive() should discard the stale live session"
);
}
struct FailingSessionStoreProbe {
inner: MemoryStore,
fail_saves: AtomicBool,
fail_loads: AtomicBool,
}
impl FailingSessionStoreProbe {
fn new() -> Self {
Self {
inner: MemoryStore::new(),
fail_saves: AtomicBool::new(false),
fail_loads: AtomicBool::new(false),
}
}
fn reject_save(&self) -> Result<(), SessionStoreError> {
if self.fail_saves.load(Ordering::Acquire) {
return Err(SessionStoreError::Io(std::io::Error::other(
"injected SessionStore save failure",
)));
}
Ok(())
}
}
#[async_trait::async_trait]
impl SessionStore for FailingSessionStoreProbe {
async fn save(&self, session: &Session) -> Result<(), SessionStoreError> {
self.reject_save()?;
self.inner.save(session).await
}
async fn save_transcript_rewrite(
&self,
session: &Session,
commit: &meerkat_core::TranscriptRewriteCommit,
) -> Result<(), SessionStoreError> {
self.reject_save()?;
self.inner.save_transcript_rewrite(session, commit).await
}
async fn save_authoritative_projection(
&self,
session: &Session,
) -> Result<(), SessionStoreError> {
self.reject_save()?;
self.inner.save_authoritative_projection(session).await
}
async fn save_authoritative_projection_if_current_revision(
&self,
session: &Session,
expected_current_revision: Option<String>,
) -> Result<(), SessionStoreError> {
self.reject_save()?;
self.inner
.save_authoritative_projection_if_current_revision(
session,
expected_current_revision,
)
.await
}
async fn load(&self, id: &SessionId) -> Result<Option<Session>, SessionStoreError> {
if self.fail_loads.load(Ordering::Acquire) {
return Err(SessionStoreError::Io(std::io::Error::other(
"injected session-load failure",
)));
}
self.inner.load(id).await
}
async fn list(
&self,
filter: meerkat_store::SessionFilter,
) -> Result<Vec<meerkat_core::SessionMeta>, SessionStoreError> {
self.inner.list(filter).await
}
async fn delete(&self, id: &SessionId) -> Result<(), SessionStoreError> {
self.inner.delete(id).await
}
async fn delete_if_current_revision(
&self,
id: &SessionId,
expected_current_revision: &str,
) -> Result<bool, SessionStoreError> {
self.inner
.delete_if_current_revision(id, expected_current_revision)
.await
}
}
#[tokio::test]
async fn test_archived_runtime_checkpoint_is_machine_authorized_noop() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store: Arc<dyn RuntimeStore> = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store),
memory_blob_store(),
);
let machine = MeerkatMachine::persistent(Arc::clone(&runtime_store), memory_blob_store());
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create_session should succeed");
let id = created.session_id;
let runtime_id = PersistentSessionService::<DummyBuilder>::runtime_id_for_session(&id);
let retained_runtime_snapshot = runtime_store
.load_session_snapshot(&runtime_id)
.await
.expect("active RuntimeStore body load should succeed")
.expect("created session should have RuntimeStore authority");
service
.archive_with_machine_protocol(
&id,
MachineSessionArchiveProtocol::from_machine(&machine),
)
.await
.expect("archive should commit the absorbing RuntimeStore terminal");
assert_eq!(
meerkat_runtime::store::load_runtime_state(runtime_store.as_ref(), &runtime_id)
.await
.expect("runtime lifecycle should load"),
Some(RuntimeState::Retired)
);
assert!(
service.existing_gate_for_session(&id).await.is_none(),
"successful archive should retire the old checkpointer gate"
);
service
.checkpoint_committed_runtime_session_snapshot(
&id,
Arc::clone(&retained_runtime_snapshot),
)
.await
.expect("archived checkpoint disposition should be a valid no-op");
assert_eq!(
runtime_store
.load_session_snapshot(&runtime_id)
.await
.expect("post-checkpoint RuntimeStore body load should succeed")
.expect("RuntimeStore body should remain present")
.as_slice(),
retained_runtime_snapshot.as_slice(),
"retained body bytes must not overwrite or mutate archived authority"
);
assert_eq!(
meerkat_runtime::store::load_runtime_state(runtime_store.as_ref(), &runtime_id)
.await
.expect("post-checkpoint runtime lifecycle should load"),
Some(RuntimeState::Retired),
"retained body bytes must not resurrect the RuntimeStore lifecycle"
);
assert!(
service.existing_gate_for_session(&id).await.is_none(),
"the archived no-op must not recreate a checkpointer gate"
);
assert!(
store
.load(&id)
.await
.expect("SessionStore probe should succeed")
.is_none(),
"archived checkpoint must not consult SessionStore"
);
}
#[tokio::test]
async fn test_runtime_backed_archive_fails_closed_when_runtime_lifecycle_commit_fails() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let gated_runtime_store = Arc::new(GatedSnapshotRuntimeStore::new());
let runtime_store: Arc<dyn RuntimeStore> = gated_runtime_store.clone();
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store),
memory_blob_store(),
);
let machine = MeerkatMachine::persistent(Arc::clone(&runtime_store), memory_blob_store());
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create_session should succeed");
let id = created.session_id.clone();
let runtime_id = PersistentSessionService::<DummyBuilder>::runtime_id_for_session(&id);
let body_before = runtime_store
.load_session_snapshot(&runtime_id)
.await
.expect("RuntimeStore body load should succeed")
.expect("created session should have RuntimeStore authority");
gated_runtime_store.set_fail_machine_lifecycle_commits(true);
let err = service
.archive_with_machine_protocol(
&id,
MachineSessionArchiveProtocol::from_machine(&machine),
)
.await
.expect_err("archive must fail closed when RuntimeStore lifecycle commit fails");
assert!(
err.to_string()
.contains("synthetic machine lifecycle commit failure"),
"archive failure must surface the RuntimeStore lifecycle error, got: {err:?}"
);
assert_ne!(
meerkat_runtime::store::load_runtime_state(runtime_store.as_ref(), &runtime_id)
.await
.expect("runtime state load should succeed"),
Some(RuntimeState::Retired),
"failed archive must not leave the runtime retired"
);
let body_after_failure = runtime_store
.load_session_snapshot(&runtime_id)
.await
.expect("RuntimeStore body load should succeed")
.expect("RuntimeStore body should remain present");
assert_eq!(
body_after_failure.as_slice(),
body_before.as_slice(),
"failed lifecycle commit must preserve the committed content body"
);
service
.read(&id)
.await
.expect("session must remain readable after a failed archive");
gated_runtime_store.set_fail_machine_lifecycle_commits(false);
service
.archive_with_machine_protocol(
&id,
MachineSessionArchiveProtocol::from_machine(&machine),
)
.await
.expect("retried archive should converge after RuntimeStore heals");
assert_eq!(
meerkat_runtime::store::load_runtime_state(runtime_store.as_ref(), &runtime_id)
.await
.expect("runtime state load should succeed"),
Some(RuntimeState::Retired),
"retried archive must realize the RuntimeStore lifecycle terminal"
);
assert!(
store
.load(&id)
.await
.expect("SessionStore probe should succeed")
.is_none(),
"archive must not persist lifecycle into SessionStore"
);
assert!(
!service
.has_live_session(&id)
.await
.expect("live-session status should succeed"),
"archived session must not report a live session"
);
let reopened_service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store),
memory_blob_store(),
);
let reopened = reopened_service.read(&id).await;
assert!(
matches!(reopened, Err(SessionError::NotFound { .. })),
"reopened service must not resurrect the archived session, got: {reopened:?}"
);
}
#[tokio::test]
async fn test_runtime_retire_failure_keeps_archived_checkpointer_cancelled() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let gated_runtime_store = Arc::new(GatedSnapshotRuntimeStore::new());
let runtime_store: Arc<dyn RuntimeStore> = gated_runtime_store.clone();
let blob_store = memory_blob_store();
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store),
Arc::clone(&blob_store),
);
let machine =
MeerkatMachine::persistent(Arc::clone(&runtime_store), Arc::clone(&blob_store));
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create_session should succeed");
let id = created.session_id;
let gate = service
.existing_gate_for_session(&id)
.await
.expect("live session should own a checkpointer gate");
machine
.register_session(id.clone())
.await
.expect("seed runtime authority before archive");
gated_runtime_store.set_fail_machine_lifecycle_commits(true);
let first_error = service
.archive_with_machine_protocol(
&id,
MachineSessionArchiveProtocol::from_machine(&machine),
)
.await
.expect_err("injected runtime retire persistence failure must fail archive");
assert!(
first_error
.to_string()
.contains("synthetic machine lifecycle commit failure"),
"first archive failure must surface the injected lifecycle commit failure: {first_error:?}"
);
let durable = service
.load_authoritative_session_base(&id)
.await
.expect("RuntimeStore body should remain readable")
.expect("RuntimeStore body must remain durable");
assert!(
!session_marks_archived(&durable),
"session bodies must not forge lifecycle authority after retire persistence fails"
);
assert!(
*gate.cancelled.lock().await,
"retire failure must leave the stale checkpointer cancelled"
);
assert!(
!service
.has_live_session(&id)
.await
.expect("live-session status should succeed"),
"failed retirement must evict the stale live session even while runtime retry remains"
);
gated_runtime_store.set_fail_machine_lifecycle_commits(false);
let same_process_retry = service
.archive_with_machine_protocol(
&id,
MachineSessionArchiveProtocol::from_machine(&machine),
)
.await
.expect_err("the failed machine incarnation must require durable reload");
assert!(
same_process_retry
.to_string()
.contains("durability reload required"),
"same-process retry must preserve fail-stop reload authority: {same_process_retry:?}"
);
drop(service);
drop(machine);
let reopened_service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store),
Arc::clone(&blob_store),
);
let reopened_machine =
MeerkatMachine::persistent(Arc::clone(&runtime_store), Arc::clone(&blob_store));
reopened_service
.archive_with_machine_protocol(
&id,
MachineSessionArchiveProtocol::from_machine(&reopened_machine),
)
.await
.expect("cold archive retry should reload authority and converge retirement");
assert_eq!(
meerkat_runtime::store::load_runtime_state(
runtime_store.as_ref(),
&PersistentSessionService::<DummyBuilder>::runtime_id_for_session(&id),
)
.await
.expect("runtime state load should succeed"),
Some(RuntimeState::Retired)
);
assert!(
*gate.cancelled.lock().await,
"cold retry must not re-enable the failed incarnation's checkpointer"
);
assert!(
reopened_service
.existing_gate_for_session(&id)
.await
.is_none(),
"cold archive retry must not create a live checkpointer gate"
);
}
#[test]
fn test_session_document_archive_verdicts() {
use meerkat_core::session_document::SessionDocumentLifecycle;
fn verdict(
authority: &mut SessionDocumentMachineAuthority,
key: &SessionDocumentKey,
runtime_backed: bool,
durable_document_present: bool,
runtime_observation: SessionArchiveRuntimeObservation,
) -> (SessionArchiveDisposition, bool, bool) {
let effects = authority
.archive_session_document(
key.clone(),
runtime_backed,
durable_document_present,
runtime_observation,
)
.expect("seeded archive drive should resolve");
effects
.iter()
.find_map(|effect| match effect {
SessionDocumentEffect::SessionArchiveResolved {
disposition,
write_document,
retire_runtime,
} => Some((*disposition, *write_document, *retire_runtime)),
_ => None,
})
.expect("archive drive should emit a verdict")
}
fn seeded(
terminal: SessionDocumentLifecycle,
) -> (SessionDocumentMachineAuthority, SessionDocumentKey) {
let mut authority = SessionDocumentMachineAuthority::new();
let key = SessionDocumentKey::new("session-archive-verdicts");
authority
.recover_session_lifecycle_terminal(key.clone(), terminal)
.expect("lifecycle-terminal recovery should be authorized");
(authority, key)
}
let mut unseeded = SessionDocumentMachineAuthority::new();
assert!(
unseeded
.archive_session_document(
SessionDocumentKey::new("session-archive-verdicts"),
true,
true,
SessionArchiveRuntimeObservation::RetirementRequired,
)
.is_err(),
"archive drive against an unseeded session id must fail closed"
);
let (mut authority, key) = seeded(SessionDocumentLifecycle::Active);
assert_eq!(
verdict(
&mut authority,
&key,
true,
true,
SessionArchiveRuntimeObservation::RetirementRequired,
),
(SessionArchiveDisposition::Archive, true, true)
);
assert_eq!(
verdict(
&mut authority,
&key,
true,
true,
SessionArchiveRuntimeObservation::QuiescentTerminal,
),
(SessionArchiveDisposition::AlreadyArchived, false, false)
);
let (mut authority, key) = seeded(SessionDocumentLifecycle::Archived);
assert_eq!(
verdict(
&mut authority,
&key,
true,
true,
SessionArchiveRuntimeObservation::Absent,
),
(SessionArchiveDisposition::AlreadyArchived, false, false)
);
let (mut authority, key) = seeded(SessionDocumentLifecycle::Active);
assert_eq!(
verdict(
&mut authority,
&key,
true,
true,
SessionArchiveRuntimeObservation::Absent,
),
(SessionArchiveDisposition::Archive, true, true)
);
let (mut authority, key) = seeded(SessionDocumentLifecycle::Active);
assert_eq!(
verdict(
&mut authority,
&key,
true,
false,
SessionArchiveRuntimeObservation::RetirementRequired,
),
(SessionArchiveDisposition::Archive, false, true)
);
let (mut authority, key) = seeded(SessionDocumentLifecycle::Active);
assert_eq!(
verdict(
&mut authority,
&key,
true,
true,
SessionArchiveRuntimeObservation::QuiescentTerminal,
),
(SessionArchiveDisposition::Archive, true, false)
);
let (mut authority, key) = seeded(SessionDocumentLifecycle::Active);
assert_eq!(
verdict(
&mut authority,
&key,
false,
true,
SessionArchiveRuntimeObservation::RetirementRequired,
),
(SessionArchiveDisposition::Archive, true, false)
);
}
#[tokio::test]
async fn test_compacted_session_repersist_ignores_non_authoritative_session_store_lag() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store: Arc<dyn RuntimeStore> = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store),
memory_blob_store(),
);
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create_session should succeed");
let seed = service
.load_authoritative_session_base(&created.session_id)
.await
.expect("RuntimeStore authority load should succeed")
.expect("created session should have RuntimeStore authority");
let parent = mutate_test_session(seed, "run boundary", |parent| {
for turn in 0..2 {
parent.push(Message::User(UserMessage::text(format!(
"turn-{turn} question"
))));
parent.push(Message::BlockAssistant(
meerkat_core::BlockAssistantMessage {
blocks: vec![meerkat_core::AssistantBlock::Text {
text: format!("turn-{turn} answer"),
meta: None,
}],
stop_reason: StopReason::EndTurn,
identity: meerkat_core::types::TranscriptMessageIdentity::default(),
created_at: meerkat_core::types::message_timestamp_now(),
},
));
}
});
let parent_revision = parent.transcript_revision().expect("parent revision");
store
.save(&parent)
.await
.expect("foreign SessionStore should accept the lagged parent");
let replacement = vec![
parent.messages()[0].clone(),
Message::User(UserMessage::text(
"[Context compacted] singleton summary".to_string(),
)),
];
let compacted = mutate_test_session(parent.clone(), "transcript rewrite", |compacted| {
compacted
.commit_transcript_rewrite(
TranscriptRewriteSelection::MessageRange {
start: 0,
end: parent.messages().len(),
},
replacement,
TranscriptRewriteReason::new("compaction"),
Some("meerkat-core".to_string()),
Some(parent_revision),
)
.expect("compaction rewrite should commit");
});
let compacted_revision = compacted.transcript_revision().expect("compacted revision");
assert!(
compacted.messages().len() < parent.messages().len(),
"compaction must shrink the transcript to exercise the monotonicity guard"
);
let runtime_id =
PersistentSessionService::<DummyBuilder>::runtime_id_for_session(&created.session_id);
runtime_store
.commit_session_snapshot(
&runtime_id,
SerializedSessionSnapshot {
session_snapshot: (serde_json::to_vec(&compacted)
.expect("serialize compacted runtime snapshot"))
.into(),
},
)
.await
.expect("seed runtime_store at the compacted revision");
service
.save_normalized_session(compacted)
.await
.expect("compacted RuntimeStore body must ignore foreign SessionStore lag");
let foreign = store
.load(&created.session_id)
.await
.expect("load should succeed")
.expect("foreign SessionStore row should remain present");
assert_eq!(
foreign.transcript_revision().expect("foreign revision"),
parent.transcript_revision().expect("parent revision"),
"RuntimeStore persistence must not rewrite SessionStore"
);
let saved = service
.load_authoritative_session_base(&created.session_id)
.await
.expect("RuntimeStore authority load should succeed")
.expect("RuntimeStore authority should remain present");
assert_eq!(
saved.transcript_revision().expect("saved revision"),
compacted_revision
);
}
#[tokio::test]
async fn test_archive_ignores_lagging_non_authoritative_session_store_projection() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store: Arc<dyn RuntimeStore> = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store),
memory_blob_store(),
);
let machine = MeerkatMachine::persistent(Arc::clone(&runtime_store), memory_blob_store());
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create_session should succeed");
let seed = service
.load_authoritative_session_base(&created.session_id)
.await
.expect("RuntimeStore authority load should succeed")
.expect("created session should have RuntimeStore authority");
let parent = mutate_test_session(seed, "run boundary", |parent| {
for turn in 0..2 {
parent.push(Message::User(UserMessage::text(format!(
"turn-{turn} question"
))));
parent.push(Message::BlockAssistant(
meerkat_core::BlockAssistantMessage {
blocks: vec![meerkat_core::AssistantBlock::Text {
text: format!("turn-{turn} answer"),
meta: None,
}],
stop_reason: StopReason::EndTurn,
identity: meerkat_core::types::TranscriptMessageIdentity::default(),
created_at: meerkat_core::types::message_timestamp_now(),
},
));
}
});
let parent_revision = parent.transcript_revision().expect("parent revision");
store
.save(&parent)
.await
.expect("foreign SessionStore should accept the lagged parent");
let replacement = vec![
parent.messages()[0].clone(),
Message::User(UserMessage::text(
"[Context compacted] singleton summary".to_string(),
)),
];
let compacted = mutate_test_session(parent.clone(), "transcript rewrite", |compacted| {
compacted
.commit_transcript_rewrite(
TranscriptRewriteSelection::MessageRange {
start: 0,
end: parent.messages().len(),
},
replacement,
TranscriptRewriteReason::new("compaction"),
Some("meerkat-core".to_string()),
Some(parent_revision),
)
.expect("compaction rewrite should commit");
});
let compacted_revision = compacted.transcript_revision().expect("compacted revision");
let runtime_id =
PersistentSessionService::<DummyBuilder>::runtime_id_for_session(&created.session_id);
runtime_store
.commit_session_snapshot(
&runtime_id,
SerializedSessionSnapshot {
session_snapshot: (serde_json::to_vec(&compacted)
.expect("serialize compacted runtime snapshot"))
.into(),
},
)
.await
.expect("seed runtime_store at the compacted revision");
match service.discard_live_session(&created.session_id).await {
Ok(()) | Err(SessionError::NotFound { .. }) => {}
Err(error) => panic!("discard_live_session should succeed: {error}"),
}
service
.archive_with_machine_protocol(
&created.session_id,
MachineSessionArchiveProtocol::from_machine(&machine),
)
.await
.expect("archive must ignore a lagging SessionStore projection");
let archived_bytes = runtime_store
.load_session_snapshot(&runtime_id)
.await
.expect("RuntimeStore body should load")
.expect("RuntimeStore body should remain present");
let archived = Session::from_persisted_bytes(&archived_bytes)
.expect("RuntimeStore body should decode");
assert_eq!(
archived.transcript_revision().expect("archived revision"),
compacted_revision,
"archive must preserve the compacted RuntimeStore body"
);
assert!(
!session_marks_archived(&archived),
"RuntimeStore lifecycle must not be embedded into the content body"
);
assert_eq!(
meerkat_runtime::store::load_runtime_state(runtime_store.as_ref(), &runtime_id)
.await
.expect("runtime state load should succeed"),
Some(RuntimeState::Retired),
"archive must commit RuntimeStore lifecycle authority"
);
let foreign = store
.load(&created.session_id)
.await
.expect("foreign SessionStore row should load")
.expect("foreign SessionStore row should remain");
assert_eq!(
foreign.transcript_revision().expect("foreign revision"),
parent.transcript_revision().expect("parent revision"),
"archive must not rewrite foreign SessionStore content"
);
}
#[tokio::test]
async fn test_runtime_backed_public_interrupt_paths_are_not_callable() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
runtime_store,
memory_blob_store(),
);
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create_session should succeed");
let err = service
.interrupt(&created.session_id)
.await
.expect_err("public interrupt should not bypass MeerkatMachine");
assert!(
matches!(err, SessionError::Unsupported(_)),
"unexpected public interrupt error: {err:?}"
);
let err = service
.cancel_after_boundary(&created.session_id)
.await
.expect_err("public boundary cancel should not bypass MeerkatMachine");
assert!(
matches!(err, SessionError::Unsupported(_)),
"unexpected public boundary cancel error: {err:?}"
);
}
#[tokio::test]
async fn test_raw_store_delete_removes_seeded_session() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let session = Session::new();
let id = session.id().clone();
store.save(&session).await.unwrap();
assert!(store.load(&id).await.unwrap().is_some());
store.delete(&id).await.unwrap();
assert!(store.load(&id).await.unwrap().is_none());
}
#[tokio::test]
async fn test_persistent_deferred_create_runtime_commit_failure_discards_live_capacity() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(GatedSnapshotRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
1,
Arc::clone(&store),
runtime_store.clone(),
memory_blob_store(),
);
runtime_store.set_fail_snapshot_commits(true);
let failed = service
.create_session(create_request("staged", InitialTurnPolicy::Defer))
.await;
assert!(
failed.as_ref().err().is_some_and(|err| err
.to_string()
.contains("synthetic runtime snapshot commit failure")),
"deferred create should surface authoritative RuntimeStore failure: {failed:?}"
);
runtime_store.set_fail_snapshot_commits(false);
service
.create_session(create_request(
"after failed create",
InitialTurnPolicy::Defer,
))
.await
.expect("failed deferred create should discard live capacity");
}
#[tokio::test]
async fn test_ephemeral_archive_during_deferred_first_turn_keeps_capacity_until_turn_stops() {
let builder = BlockingRunBuilder::new();
let service = Arc::new(EphemeralSessionService::new(builder.clone(), 1));
let staged = service
.create_session(create_request("staged", InitialTurnPolicy::Defer))
.await
.expect("deferred session should be staged");
let service_for_turn = Arc::clone(&service);
let session_for_turn = staged.session_id.clone();
let first_turn = tokio::spawn(async move {
service_for_turn
.start_turn(&session_for_turn, start_turn_request("materialize"))
.await
});
builder.wait_for_entered_runs(1).await;
service
.archive(&staged.session_id)
.await
.expect("archive should request shutdown while first turn runs");
let blocked = service
.create_session(create_request("blocked", InitialTurnPolicy::Defer))
.await;
assert!(
blocked
.as_ref()
.err()
.is_some_and(|err| err.to_string().contains("Max sessions")),
"archive during a deferred first turn must not release capacity early: {blocked:?}"
);
builder.release_notify.add_permits(1);
let _ = first_turn.await.expect("first turn task should join");
service
.create_session(create_request("after turn", InitialTurnPolicy::Defer))
.await
.expect("capacity should release after archived turn stops");
}
#[tokio::test]
async fn test_append_system_context_keeps_missing_session_store_row_inert() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
runtime_store.clone(),
memory_blob_store(),
);
let result = service
.create_session(create_request(
"hello",
meerkat_core::service::InitialTurnPolicy::Defer,
))
.await
.expect("create_session should succeed");
let id = result.session_id;
store
.delete(&id)
.await
.expect("test should be able to remove persisted row");
assert!(
store.load(&id).await.unwrap().is_none(),
"store row should be absent before append"
);
let result = service
.append_system_context(
&id,
AppendSystemContextRequest {
content: meerkat_core::lifecycle::run_primitive::CoreRenderable::text(
"runtime notice".to_string(),
),
source: Some("mob".to_string()),
idempotency_key: Some("ctx-persistent-live".to_string()),
},
)
.await
.expect("live append should commit through RuntimeStore authority");
assert_eq!(
result.status,
meerkat_core::AppendSystemContextStatus::Applied
);
assert!(
store
.load(&id)
.await
.expect("SessionStore probe should succeed")
.is_none(),
"a runtime-backed control append must not mint SessionStore authority"
);
let stored = service
.load_authoritative_session(&id)
.await
.expect("authoritative load should succeed")
.expect("append should retain RuntimeStore authority");
assert!(stored.messages().iter().any(
|message| matches!(message, Message::System(system) if system.content == "runtime notice")
));
assert!(
runtime_store
.load_whole_blob_store_authority(
&PersistentSessionService::<DummyBuilder>::runtime_id_for_session(&id)
)
.await
.expect("RuntimeStore authority probe should succeed")
.is_some(),
"control append must retain store-issued WholeBlob authority"
);
}
#[tokio::test]
async fn test_append_system_context_unknown_session_does_not_allocate_gate() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::new(InMemoryRuntimeStore::new()),
memory_blob_store(),
);
let unknown = SessionId::new();
let err = service
.append_system_context(
&unknown,
AppendSystemContextRequest {
content: meerkat_core::lifecycle::run_primitive::CoreRenderable::text(
"runtime notice".to_string(),
),
source: Some("mob".to_string()),
idempotency_key: Some("ctx-unknown".to_string()),
},
)
.await
.expect_err("unknown session must fail");
assert_eq!(err.code(), "SESSION_NOT_FOUND");
assert!(
service.checkpointer_gates.lock().await.is_empty(),
"unknown-session append must not allocate a checkpointer gate"
);
}
#[tokio::test]
async fn test_runtime_backed_eager_create_session_is_rejected_before_authority_snapshot() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
runtime_store,
memory_blob_store(),
);
let error = service
.create_session(create_request(
"hello",
meerkat_core::service::InitialTurnPolicy::RunImmediately,
))
.await
.expect_err("runtime-backed eager create_session must be removed");
assert_runtime_backed_eager_create_rejected(&error);
assert!(
service.checkpointer_gates.lock().await.is_empty(),
"rejected eager create must not allocate persistence/checkpointer state"
);
}
#[tokio::test]
async fn test_runtime_backed_list_retains_projected_session_after_live_handle_discard() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
runtime_store,
memory_blob_store(),
);
let result = service
.create_session(create_request(
"hello",
meerkat_core::service::InitialTurnPolicy::Defer,
))
.await
.expect("create_session should succeed");
service
.discard_live_session(&result.session_id)
.await
.expect("test should be able to discard the live session");
let listed = service
.list(SessionQuery::default())
.await
.expect("list should succeed");
assert!(
listed
.iter()
.any(|summary| summary.session_id == result.session_id),
"runtime-backed sessions should remain listable through the RuntimeStore catalog after the live handle is gone"
);
}
#[tokio::test]
async fn test_runtime_backed_direct_start_turn_is_rejected_before_runtime_mutation() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
runtime_store,
memory_blob_store(),
);
let result = service
.create_session(create_request(
"hello",
meerkat_core::service::InitialTurnPolicy::Defer,
))
.await
.expect("create_session should succeed before projection failures");
let initial_authoritative = service
.load_authoritative_session(&result.session_id)
.await
.expect("authoritative load should succeed")
.expect("runtime authority should exist before rejected direct turn");
let error = service
.start_turn(
&result.session_id,
start_turn_request("runtime-backed direct start_turn must not commit"),
)
.await
.expect_err("runtime-backed direct start_turn must be removed");
assert_runtime_backed_direct_start_turn_rejected(&error);
let authoritative = service
.load_authoritative_session(&result.session_id)
.await
.expect("authoritative load should succeed")
.expect("runtime authority should remain present after rejected direct turn");
assert_eq!(
authoritative.messages().len(),
initial_authoritative.messages().len(),
"rejected direct start_turn must not advance runtime authority"
);
assert!(
store
.load(&result.session_id)
.await
.expect("SessionStore probe should succeed")
.is_none(),
"rejected direct start_turn must not mint a SessionStore authority row"
);
assert!(
service
.has_live_session(&result.session_id)
.await
.expect("status should succeed after rejection"),
"rejection happens before live state is mutated or evicted"
);
}
#[tokio::test]
async fn test_machine_committed_live_turn_rejects_mismatched_runtime_authority() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let service_runtime_store: Arc<dyn RuntimeStore> = Arc::new(InMemoryRuntimeStore::new());
let builder = BlockingRunBuilder::new();
let service = PersistentSessionService::new(
builder.clone(),
4,
Arc::clone(&store),
Arc::clone(&service_runtime_store),
memory_blob_store(),
);
let created = service
.create_session(create_request(
"hello",
meerkat_core::service::InitialTurnPolicy::Defer,
))
.await
.expect("create_session should succeed");
let authoritative_before = service
.load_authoritative_session(&created.session_id)
.await
.expect("authoritative load should succeed")
.expect("runtime authority should exist before rejected protocol");
let admission = service
.reserve_runtime_turn_admission(&created.session_id)
.await
.expect("runtime turn admission should be reserved");
let mismatched_runtime_store: Arc<dyn RuntimeStore> = Arc::new(InMemoryRuntimeStore::new());
let mismatched_machine =
meerkat_runtime::MeerkatMachine::persistent_without_blobs(mismatched_runtime_store);
let (error, returned_admission) = service
.run_machine_committed_live_turn(
MachineServiceTurnCommitProtocol::from_machine(&mismatched_machine),
&created.session_id,
start_turn_request("must not start"),
admission,
)
.await
.expect_err("mismatched machine/service runtime authority must fail closed");
assert!(
returned_admission.is_some(),
"authority rejection must return the unconsumed admission guard"
);
match error {
SessionError::Unsupported(message) => assert!(
message.contains("runtime authority does not match"),
"unexpected unsupported error: {message}"
),
other => panic!("expected unsupported authority rejection, got {other:?}"),
}
assert_eq!(
builder.entered_runs.load(Ordering::Acquire),
0,
"authority rejection must happen before the live turn starts"
);
let authoritative_after = service
.load_authoritative_session(&created.session_id)
.await
.expect("authoritative load should still succeed")
.expect("runtime authority should remain present");
assert_eq!(
authoritative_after.messages().len(),
authoritative_before.messages().len(),
"rejected protocol must not advance runtime authority"
);
assert!(
store
.load(&created.session_id)
.await
.expect("SessionStore probe should succeed")
.is_none(),
"rejected protocol must not mint a SessionStore authority row"
);
}
#[tokio::test]
async fn test_runtime_backed_callback_pending_direct_start_turn_cannot_bypass_machine_commit() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
CallbackPendingBuilder,
4,
Arc::clone(&store),
runtime_store,
memory_blob_store(),
);
let result = service
.create_session(create_request(
"hello",
meerkat_core::service::InitialTurnPolicy::Defer,
))
.await
.expect("create_session should succeed before projection failures");
let initial_authoritative = service
.load_authoritative_session(&result.session_id)
.await
.expect("authoritative load should succeed")
.expect("runtime authority should exist before rejected direct turn");
let error = service
.start_turn(
&result.session_id,
start_turn_request("callback pending direct path must not commit"),
)
.await
.expect_err("runtime-backed direct start_turn must be removed before builder output");
assert_runtime_backed_direct_start_turn_rejected(&error);
let authoritative = service
.load_authoritative_session(&result.session_id)
.await
.expect("authoritative load should succeed")
.expect("runtime authority should remain present after rejected direct turn");
assert_eq!(
authoritative.messages().len(),
initial_authoritative.messages().len(),
"rejected direct start_turn must not advance runtime authority"
);
assert!(
store
.load(&result.session_id)
.await
.expect("SessionStore probe should succeed")
.is_none(),
"rejected direct start_turn must not mint a SessionStore authority row"
);
}
#[tokio::test]
async fn test_runtime_backed_resume_eager_create_is_rejected_before_runtime_mutation() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
CallbackPendingBuilder,
4,
Arc::clone(&store),
runtime_store,
memory_blob_store(),
);
let result = service
.create_session(create_request(
"hello",
meerkat_core::service::InitialTurnPolicy::Defer,
))
.await
.expect("create_session should succeed before projection failures");
let resume_source = service
.load_authoritative_session(&result.session_id)
.await
.expect("authoritative load should succeed")
.expect("runtime authority should exist");
let authority_before = resume_source.clone();
assert!(
store
.load(&result.session_id)
.await
.expect("SessionStore probe should succeed")
.is_none(),
"runtime-backed creation must not require a second SessionStore authority row"
);
service
.discard_live_session(&result.session_id)
.await
.expect("test should model resume after live handle discard");
let mut req = create_request(
"resume callback pending after projection failure",
meerkat_core::service::InitialTurnPolicy::RunImmediately,
);
req.build = Some(SessionBuildOptions {
resume_session: Some(resume_source),
..Default::default()
});
let error = service
.create_session(req)
.await
.expect_err("runtime-backed eager resume create must be removed");
assert_runtime_backed_eager_create_rejected(&error);
assert!(
!service
.has_live_session(&result.session_id)
.await
.expect("status should succeed after eager create rejection"),
"eager create rejection happens before recovering live state"
);
let authoritative = service
.load_authoritative_session(&result.session_id)
.await
.expect("authoritative load should succeed")
.expect("runtime authority should remain unchanged after eager create rejection");
assert_eq!(
authoritative.messages().len(),
authority_before.messages().len(),
"runtime authority must not advance through rejected eager create"
);
}
#[tokio::test]
async fn test_runtime_backed_direct_start_turn_rejection_ignores_snapshot_store_failures() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(GatedSnapshotRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
runtime_store.clone(),
memory_blob_store(),
);
let result = service
.create_session(create_request(
"hello",
meerkat_core::service::InitialTurnPolicy::Defer,
))
.await
.expect("create_session should succeed");
let initial_authoritative = service
.load_authoritative_session(&result.session_id)
.await
.expect("authoritative load should succeed")
.expect("runtime authority should exist before rejected direct turn");
runtime_store.set_fail_snapshot_commits(true);
let error = service
.start_turn(
&result.session_id,
start_turn_request("removed direct path must not reach snapshot commit"),
)
.await
.expect_err("runtime-backed direct start_turn must be rejected before snapshot commit");
assert_runtime_backed_direct_start_turn_rejected(&error);
assert!(
service
.has_live_session(&result.session_id)
.await
.expect("live-session status should succeed"),
"rejection happens before live state is mutated or evicted"
);
runtime_store.set_fail_snapshot_commits(false);
let authoritative = service
.load_authoritative_session(&result.session_id)
.await
.expect("authoritative load should succeed")
.expect("created session should remain durable");
assert_eq!(
authoritative.messages().len(),
initial_authoritative.messages().len(),
"failed direct start_turn must not advance runtime authority"
);
assert!(
authoritative.messages().iter().all(|message| {
!format!("{message:?}")
.contains("removed direct path must not reach snapshot commit")
}),
"rejected direct start_turn must not leak into durable authority"
);
let view = service
.read(&result.session_id)
.await
.expect("read should fall back to durable runtime authority");
assert_eq!(
view.state.message_count,
initial_authoritative.messages().len(),
"read() must not report the rejected live mutation as clean truth"
);
let listed = service
.list(SessionQuery::default())
.await
.expect("list should succeed after rejected direct start_turn");
let summary = listed
.iter()
.find(|summary| summary.session_id == result.session_id)
.expect("durable session should remain listable");
assert_eq!(
summary.message_count,
initial_authoritative.messages().len(),
"list() must not report the failed live mutation as clean truth"
);
}
#[tokio::test]
async fn test_runtime_backed_control_commit_failure_discards_live_state() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(GatedSnapshotRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
runtime_store.clone(),
memory_blob_store(),
);
let append_result = service
.create_session(create_request(
"hello",
meerkat_core::service::InitialTurnPolicy::Defer,
))
.await
.expect("create_session should succeed before runtime failure injection");
runtime_store.set_fail_snapshot_commits(true);
let append_error = service
.append_system_context(
&append_result.session_id,
AppendSystemContextRequest {
content: meerkat_core::lifecycle::run_primitive::CoreRenderable::text(
"runtime-backed control context".to_string(),
),
source: Some("test".to_string()),
idempotency_key: Some("control-projection-failure".to_string()),
},
)
.await
.expect_err("authoritative control commit failure must fail closed");
assert!(
append_error
.to_string()
.contains("synthetic runtime snapshot commit failure"),
"RuntimeStore failure should surface, got {append_error:?}"
);
assert!(
!service
.has_live_session(&append_result.session_id)
.await
.expect("status should succeed after append commit failure"),
"failed append commit must evict uncommitted live state"
);
let append_authoritative = service
.load_authoritative_session(&append_result.session_id)
.await
.expect("authoritative load should succeed after append failure")
.expect("runtime predecessor authority should remain");
assert!(
append_authoritative
.messages()
.iter()
.all(|message| !matches!(
message,
Message::System(system) if system.content == "runtime-backed control context"
)),
"a failed RuntimeStore commit must preserve the durable predecessor"
);
assert!(
store
.load(&append_result.session_id)
.await
.expect("SessionStore probe after failed append")
.is_none(),
"failed append must not mint a second SessionStore authority"
);
runtime_store.set_fail_snapshot_commits(false);
let stage_result = service
.create_session(create_request(
"hello",
meerkat_core::service::InitialTurnPolicy::Defer,
))
.await
.expect("second create_session should succeed before runtime failure injection");
runtime_store.set_fail_snapshot_commits(true);
let stage_error = service
.stage_tool_results(
&stage_result.session_id,
StageToolResultsRequest {
results: vec![ToolResult::new(
"tool-call-1".to_string(),
"callback result".to_string(),
false,
)],
},
)
.await
.expect_err("staged tool-result authority commit failure must fail closed");
assert!(
stage_error
.to_string()
.contains("synthetic runtime snapshot commit failure"),
"RuntimeStore failure should surface, got {stage_error:?}"
);
assert!(
service
.has_live_session(&stage_result.session_id)
.await
.expect("status should succeed after stage commit failure"),
"failed staged-result commit must retain the unchanged live predecessor"
);
let live_deferred_state = service
.inner
.deferred_turn_state(&stage_result.session_id)
.await
.expect("deferred session should retain its live state");
assert_eq!(
live_deferred_state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.pending_tool_results()
.len(),
0,
"failed detached staging must not mutate the live deferred-state predecessor"
);
let stage_authoritative = service
.load_authoritative_session(&stage_result.session_id)
.await
.expect("authoritative load should succeed after stage failure")
.expect("runtime predecessor authority should remain");
let durable_tool_result_count = stage_authoritative
.deferred_turn_state()
.map_or(0, |state| state.pending_tool_results().len());
assert_eq!(
durable_tool_result_count, 0,
"failed staged-result commit must preserve the durable predecessor"
);
assert!(
store
.load(&stage_result.session_id)
.await
.expect("SessionStore probe after failed staging")
.is_none(),
"failed staging must not mint a second SessionStore authority"
);
}
#[tokio::test]
async fn test_runtime_backed_follow_up_start_turn_must_use_machine_commit_protocol() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
runtime_store,
memory_blob_store(),
);
let result = service
.create_session(create_request(
"hello",
meerkat_core::service::InitialTurnPolicy::Defer,
))
.await
.expect("create_session should succeed");
let id = result.session_id;
let initial_count = service
.load_authoritative_session(&id)
.await
.expect("authoritative load should succeed")
.expect("create_session should persist authoritative snapshot")
.messages()
.len();
let error = service
.start_turn(&id, start_turn_request("follow up"))
.await
.expect_err("runtime-backed follow-up start_turn must be removed");
assert_runtime_backed_direct_start_turn_rejected(&error);
let stored = service
.load_authoritative_session(&id)
.await
.expect("authoritative load should succeed")
.expect("runtime-backed session should remain durable");
assert_eq!(
stored.messages().len(),
initial_count,
"rejected follow-up direct start_turn must not update authoritative snapshot"
);
}
#[tokio::test]
async fn test_runtime_backed_append_system_context_persists_ordered_system_message() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
runtime_store.clone(),
memory_blob_store(),
);
let result = service
.create_session(create_request(
"hello",
meerkat_core::service::InitialTurnPolicy::Defer,
))
.await
.expect("create_session should succeed");
let checkpointer = Arc::new(StoreCheckpointer {
runtime_store: runtime_store.clone(),
incremental: service.incremental.clone(),
blob_store: Arc::clone(&service.blob_store),
gate: Arc::new(CheckpointerGate {
cancelled: Mutex::new(false),
}),
whole_blob_base_authority: std::sync::Mutex::new(None),
latest_run_checkpoint_receipt: Mutex::new(None),
});
service
.live_checkpointers
.lock()
.await
.insert(result.session_id.clone(), Arc::downgrade(&checkpointer));
let append = service
.append_system_context(
&result.session_id,
AppendSystemContextRequest {
content: meerkat_core::lifecycle::run_primitive::CoreRenderable::text(
"Remember runtime-backed context".to_string(),
),
source: Some("test".to_string()),
idempotency_key: Some("runtime-backed-ctx".to_string()),
},
)
.await
.expect("runtime-backed append_system_context should remain available");
assert_eq!(
append.status,
meerkat_core::AppendSystemContextStatus::Applied
);
let stored = service
.load_authoritative_session(&result.session_id)
.await
.expect("authoritative load should succeed")
.expect("append should persist the updated transcript");
assert!(stored.messages().iter().any(|message| matches!(
message,
Message::System(system) if system.content == "Remember runtime-backed context"
)));
let runtime_id =
PersistentSessionService::<DummyBuilder>::runtime_id_for_session(&result.session_id);
let committed_authority = runtime_store
.load_whole_blob_store_authority(&runtime_id)
.await
.expect("load committed control authority")
.expect("System control append must retain WholeBlob authority");
assert_eq!(
checkpointer
.whole_blob_base_authority
.lock()
.expect("checkpointer authority lock")
.as_ref(),
Some(&committed_authority),
"System control append must rebind the live actor to the exact store-issued authority; otherwise the next member turn loses its event stream when stale-base checkpointing discards the actor"
);
}
#[tokio::test]
async fn test_runtime_backed_append_system_context_persists_snapshot_without_boundary_receipt()
{
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(GatedSnapshotRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
runtime_store.clone(),
memory_blob_store(),
);
let result = service
.create_session(create_request(
"hello",
meerkat_core::service::InitialTurnPolicy::Defer,
))
.await
.expect("create_session should succeed");
runtime_store.reset_boundary_commits().await;
service
.append_system_context(
&result.session_id,
AppendSystemContextRequest {
content: meerkat_core::lifecycle::run_primitive::CoreRenderable::text(
"control snapshot should not mint a receipt".to_string(),
),
source: Some("api".to_string()),
idempotency_key: Some("no-receipt-control-snapshot".to_string()),
},
)
.await
.expect("append_system_context should succeed");
let boundary_commits = runtime_store.boundary_commits().await;
assert!(
boundary_commits.is_empty(),
"non-run control snapshots must not mint synthetic boundary receipts: {boundary_commits:?}"
);
let runtime_id =
PersistentSessionService::<DummyBuilder>::runtime_id_for_session(&result.session_id);
let snapshot = runtime_store
.load_session_snapshot(&runtime_id)
.await
.expect("runtime snapshot load should succeed")
.expect("control snapshot should still be durable");
let stored: Session =
serde_json::from_slice(&snapshot).expect("runtime snapshot should deserialize");
assert!(stored.messages().iter().any(
|message| matches!(
message,
Message::System(system) if system.content == "control snapshot should not mint a receipt"
)
));
}
#[tokio::test]
async fn test_runtime_backed_append_system_context_uses_newer_runtime_snapshot() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
runtime_store.clone(),
memory_blob_store(),
);
let result = service
.create_session(create_request(
"hello",
meerkat_core::service::InitialTurnPolicy::Defer,
))
.await
.expect("create_session should succeed");
let output = service
.apply_runtime_turn(
&result.session_id,
RunId::new(),
runtime_content_turn_request("runtime committed turn"),
RunApplyBoundary::Immediate,
vec![],
)
.await
.expect("runtime-backed turn should succeed");
machine_commit_runtime_output(
&service,
runtime_store.as_ref(),
&result.session_id,
&output,
)
.await
.expect("machine-owned runtime output should commit");
let stale_store_row = store
.load(&result.session_id)
.await
.expect("raw store load should succeed");
assert!(
stale_store_row
.as_ref()
.is_none_or(|session| session.messages().is_empty()),
"session store should still lag behind the runtime-backed committed snapshot before append"
);
assert!(
runtime_store
.load_session_snapshot(
&PersistentSessionService::<DummyBuilder>::runtime_id_for_session(
&result.session_id
)
)
.await
.expect("runtime snapshot load should succeed")
.is_some(),
"machine-owned runtime turn commit should persist a runtime snapshot"
);
service
.append_system_context(
&result.session_id,
AppendSystemContextRequest {
content: meerkat_core::lifecycle::run_primitive::CoreRenderable::text(
"runtime append".to_string(),
),
source: Some("api".to_string()),
idempotency_key: Some("runtime-base".to_string()),
},
)
.await
.expect("append_system_context should succeed");
let stored = service
.load_authoritative_session(&result.session_id)
.await
.expect("authoritative load should succeed")
.expect("append should persist a refreshed authoritative snapshot");
assert_eq!(
stored.messages().len(),
3,
"append must preserve the newest runtime-committed conversation state instead of rewinding to the stale session-store row"
);
assert!(matches!(
stored.messages().last(),
Some(Message::System(system)) if system.content == "runtime append"
));
}
#[tokio::test]
async fn test_runtime_backed_append_system_context_without_live_handle_uses_runtime_authority()
{
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
runtime_store.clone(),
memory_blob_store(),
);
let result = service
.create_session(create_request(
"hello",
meerkat_core::service::InitialTurnPolicy::Defer,
))
.await
.expect("create_session should succeed");
let output = service
.apply_runtime_turn(
&result.session_id,
RunId::new(),
runtime_content_turn_request("runtime committed turn"),
RunApplyBoundary::Immediate,
vec![],
)
.await
.expect("runtime-backed turn should succeed");
machine_commit_runtime_output(
&service,
runtime_store.as_ref(),
&result.session_id,
&output,
)
.await
.expect("machine-owned runtime output should commit");
service
.discard_live_session(&result.session_id)
.await
.expect("test should be able to discard the live session");
let append = service
.append_system_context(
&result.session_id,
AppendSystemContextRequest {
content: meerkat_core::lifecycle::run_primitive::CoreRenderable::text(
"persisted runtime append".to_string(),
),
source: Some("api".to_string()),
idempotency_key: Some("persisted-runtime-append".to_string()),
},
)
.await
.expect("persisted runtime-backed append should update runtime authority");
assert_eq!(
append.status,
meerkat_core::AppendSystemContextStatus::Applied
);
let authoritative = service
.load_authoritative_session(&result.session_id)
.await
.expect("authoritative load should succeed")
.expect("authoritative session should exist");
assert_eq!(
authoritative.messages().len(),
3,
"persisted append must preserve committed runtime transcript truth"
);
assert!(matches!(
authoritative.messages().last(),
Some(Message::System(system)) if system.content == "persisted runtime append"
));
let view = service
.read(&result.session_id)
.await
.expect("read should use durable runtime truth after live discard");
assert_eq!(view.state.message_count, authoritative.messages().len());
assert!(!view.state.is_active);
assert!(
!service
.has_live_session(&result.session_id)
.await
.expect("status should succeed"),
"status should not recreate a live handle"
);
let listed = service
.list(SessionQuery::default())
.await
.expect("list should succeed");
let summary = listed
.iter()
.find(|summary| summary.session_id == result.session_id)
.expect("list should include persisted runtime-backed session");
assert_eq!(summary.message_count, authoritative.messages().len());
assert!(
!summary.is_active,
"an Idle runtime remains catalog-visible without fabricating an active turn"
);
let restarted = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
runtime_store,
memory_blob_store(),
);
let resume_source = restarted
.load_authoritative_session(&result.session_id)
.await
.expect("resume source should load from runtime authority")
.expect("runtime authority should remain present after restart");
let resumed = restarted
.create_session(resume_request(resume_source))
.await
.expect("resume should materialize the runtime-authoritative snapshot");
assert_eq!(resumed.session_id, result.session_id);
let resumed_live = restarted
.export_live_session(&result.session_id)
.await
.expect("resumed live session should export");
assert!(matches!(
resumed_live.messages().last(),
Some(Message::System(system)) if system.content == "persisted runtime append"
));
}
#[tokio::test]
async fn test_authoritative_runtime_snapshot_ignores_store_owned_session_metadata() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
runtime_store.clone(),
memory_blob_store(),
);
let result = service
.create_session(create_request_with_metadata(
"hello",
meerkat_core::service::InitialTurnPolicy::Defer,
))
.await
.expect("create_session should succeed");
let store_session = service
.load_authoritative_session(&result.session_id)
.await
.expect("authoritative load should succeed")
.expect("initial runtime snapshot should exist");
let store_session = mutate_test_session(store_session, "run boundary", |store_session| {
store_session
.set_build_state(meerkat_core::SessionBuildState {
additional_instructions: Some(vec![
"store-owned recovery metadata".to_string(),
]),
..Default::default()
})
.expect("build state should serialize");
});
store
.save(&store_session)
.await
.expect("test should seed a compatibility store projection");
let mut runtime_session = without_session_owned_metadata(&store_session);
runtime_session.push(Message::User(UserMessage::text_with_render_metadata(
"runtime-only turn",
None,
)));
let session_snapshot =
serde_json::to_vec(&runtime_session).expect("runtime snapshot should serialize");
runtime_store
.commit_session_snapshot(
&PersistentSessionService::<DummyBuilder>::runtime_id_for_session(
&result.session_id,
),
meerkat_runtime::SerializedSessionSnapshot {
session_snapshot: session_snapshot.into(),
},
)
.await
.expect("runtime snapshot commit should succeed");
let authoritative = service
.load_authoritative_session(&result.session_id)
.await
.expect("authoritative load should succeed")
.expect("authoritative session should exist");
assert_eq!(
authoritative.messages().len(),
runtime_session.messages().len(),
"newer runtime snapshot still owns committed conversation state"
);
assert_eq!(
authoritative.updated_at(),
runtime_session.updated_at(),
"store projection metadata must not change runtime snapshot timestamps"
);
assert!(
authoritative.session_metadata().is_none(),
"runtime authority must not backfill metadata from the store projection"
);
assert!(
authoritative.build_state().is_none(),
"runtime authority must not backfill build state from the store projection"
);
}
#[tokio::test]
async fn test_persisted_runtime_state_loads_runtime_store_state() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
runtime_store.clone(),
memory_blob_store(),
);
let session_id = SessionId::new();
let runtime_id =
PersistentSessionService::<DummyBuilder>::runtime_id_for_session(&session_id);
let machine = meerkat_runtime::MeerkatMachine::persistent(
runtime_store.clone() as Arc<dyn RuntimeStore>,
memory_blob_store(),
);
machine
.register_session(session_id.clone())
.await
.expect("register session");
meerkat_runtime::RuntimeControlPlane::retire(&machine, &runtime_id)
.await
.expect("runtime state should persist");
assert_eq!(
service
.persisted_runtime_state(&session_id)
.await
.expect("runtime state load should succeed"),
Some(meerkat_runtime::RuntimeState::Retired)
);
}
#[tokio::test]
async fn test_runtime_backed_authoritative_load_ignores_store_only_projection() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
runtime_store.clone(),
memory_blob_store(),
);
let mut raw_projection = Session::new();
raw_projection.push(Message::User(UserMessage::text(
"legacy raw store row".to_string(),
)));
let id = raw_projection.id().clone();
store
.save(&raw_projection)
.await
.expect("test should seed a raw store projection");
let authoritative = service
.load_authoritative_session(&id)
.await
.expect("RuntimeStore absence should be a clean authoritative miss");
assert!(
authoritative.is_none(),
"a SessionStore-only projection must not become runtime authority"
);
assert!(
runtime_store
.load_session_snapshot(
&PersistentSessionService::<DummyBuilder>::runtime_id_for_session(&id)
)
.await
.expect("runtime snapshot load should succeed")
.is_none(),
"authoritative load must not create runtime authority from a store-only projection"
);
let raw_store_row = store
.load(&id)
.await
.expect("raw store load should succeed")
.expect("test projection should remain present");
assert_eq!(
raw_store_row.messages().len(),
raw_projection.messages().len(),
"compatibility projection should remain inert in the raw store"
);
}
#[tokio::test]
async fn test_runtime_backed_authoritative_load_ignores_legacy_session_uuid_runtime_alias() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(GatedSnapshotRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
runtime_store.clone(),
memory_blob_store(),
);
let mut legacy_runtime_session = Session::new();
legacy_runtime_session.push(Message::User(UserMessage::text(
"legacy runtime alias row".to_string(),
)));
let id = legacy_runtime_session.id().clone();
let snapshot =
serde_json::to_vec(&legacy_runtime_session).expect("legacy session should serialize");
let legacy_runtime_alias = LogicalRuntimeId::legacy_session_uuid_alias(&id);
let canonical_runtime_id =
PersistentSessionService::<DummyBuilder>::runtime_id_for_session(&id);
assert_ne!(
canonical_runtime_id, legacy_runtime_alias,
"canonical runtime id must be distinct from the legacy raw session UUID alias"
);
runtime_store
.override_session_snapshot(legacy_runtime_alias, snapshot)
.await;
let authoritative = service
.load_authoritative_session(&id)
.await
.expect("authoritative load should succeed");
assert!(
authoritative.is_none(),
"legacy raw alias snapshots must not drive runtime-backed resume authority"
);
}
#[tokio::test]
async fn test_runtime_backed_authoritative_load_ignores_newer_legacy_runtime_alias_snapshot() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(GatedSnapshotRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
runtime_store.clone(),
memory_blob_store(),
);
let canonical_session =
mutate_test_session(Session::new(), "session creation", |canonical_session| {
canonical_session.push(Message::User(UserMessage::text(
"canonical runtime alias row".to_string(),
)));
});
let id = canonical_session.id().clone();
let canonical_runtime_id =
PersistentSessionService::<DummyBuilder>::runtime_id_for_session(&id);
runtime_store
.commit_session_snapshot(
&canonical_runtime_id,
meerkat_runtime::store::SerializedSessionSnapshot {
session_snapshot: (serde_json::to_vec(&canonical_session)
.expect("canonical session should serialize"))
.into(),
},
)
.await
.expect("test should seed a canonical runtime alias snapshot");
tokio::time::sleep(std::time::Duration::from_millis(1)).await;
let mut legacy_session = canonical_session.clone();
legacy_session.push(Message::User(UserMessage::text(
"newer legacy runtime alias row".to_string(),
)));
runtime_store
.override_session_snapshot(
LogicalRuntimeId::legacy_session_uuid_alias(&id),
serde_json::to_vec(&legacy_session).expect("legacy session should serialize"),
)
.await;
let authoritative = service
.load_authoritative_session(&id)
.await
.expect("authoritative load should succeed")
.expect("runtime alias snapshot should exist");
assert_eq!(
authoritative.messages().len(),
canonical_session.messages().len(),
"canonical runtime authority must not be overwritten by a newer legacy raw alias snapshot"
);
}
#[tokio::test]
async fn test_runtime_authority_ignores_corrupt_legacy_alias_after_canonical_snapshot() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(GatedSnapshotRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
runtime_store.clone(),
memory_blob_store(),
);
let canonical_session =
mutate_test_session(Session::new(), "session creation", |canonical_session| {
canonical_session.push(Message::User(UserMessage::text(
"canonical runtime alias row".to_string(),
)));
});
let id = canonical_session.id().clone();
let canonical_runtime_id =
PersistentSessionService::<DummyBuilder>::runtime_id_for_session(&id);
runtime_store
.commit_session_snapshot(
&canonical_runtime_id,
meerkat_runtime::store::SerializedSessionSnapshot {
session_snapshot: (serde_json::to_vec(&canonical_session)
.expect("canonical session should serialize"))
.into(),
},
)
.await
.expect("test should seed a canonical runtime snapshot");
runtime_store
.override_session_snapshot(
LogicalRuntimeId::legacy_session_uuid_alias(&id),
b"{not valid json".to_vec(),
)
.await;
let authoritative = service
.load_authoritative_session(&id)
.await
.expect("valid canonical authority must not be poisoned by corrupt legacy fallback")
.expect("canonical runtime snapshot should exist");
assert_eq!(
authoritative.messages().len(),
canonical_session.messages().len(),
"canonical runtime authority must win over corrupt legacy fallback data"
);
}
#[tokio::test]
async fn test_runtime_input_updates_ignore_legacy_alias_load_error_after_canonical_states() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(GatedSnapshotRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
runtime_store.clone(),
memory_blob_store(),
);
let id = SessionId::new();
let input_id = InputId::new();
let canonical_runtime_id =
PersistentSessionService::<DummyBuilder>::runtime_id_for_session(&id);
let mut input_state = meerkat_runtime::InputState::new_accepted(input_id.clone());
input_state.durability = Some(meerkat_runtime::InputDurability::Durable);
let stored = StoredInputState {
state: input_state,
seed: meerkat_runtime::input_state::InputStateSeed::new_accepted(),
};
let stored = recovered_input_persistence_record(&canonical_runtime_id, stored);
runtime_store
.persist_input_state(&canonical_runtime_id, &stored)
.await
.expect("test should seed canonical input state");
runtime_store
.fail_input_state_load_for(LogicalRuntimeId::legacy_session_uuid_alias(&id))
.await;
let run_id = RunId::new();
let updates = service
.runtime_input_updates(&id, &run_id, 7, std::slice::from_ref(&input_id))
.await
.expect("legacy input-state load failure must not poison canonical updates");
assert_eq!(updates.len(), 1);
assert_eq!(updates[0].state.input_id, input_id);
assert_eq!(updates[0].seed.phase, InputLifecycleState::Consumed);
assert_eq!(updates[0].seed.last_run_id, Some(run_id));
assert_eq!(updates[0].seed.last_boundary_sequence, Some(7));
}
#[tokio::test]
async fn test_runtime_input_updates_ignore_legacy_alias_load_error_after_empty_canonical_read()
{
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(GatedSnapshotRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
runtime_store.clone(),
memory_blob_store(),
);
let id = SessionId::new();
runtime_store
.fail_input_state_load_for(LogicalRuntimeId::legacy_session_uuid_alias(&id))
.await;
let updates = service
.runtime_input_updates(&id, &RunId::new(), 0, &[])
.await
.expect("legacy input-state load failure must not poison empty canonical updates");
assert!(
updates.is_empty(),
"no-contributor boundary should not need legacy input-state data"
);
}
#[tokio::test]
async fn test_runtime_input_updates_ignores_legacy_alias_when_contributor_missing_from_canonical()
{
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(GatedSnapshotRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
runtime_store.clone(),
memory_blob_store(),
);
let id = SessionId::new();
let input_id = InputId::new();
runtime_store
.fail_input_state_load_for(LogicalRuntimeId::legacy_session_uuid_alias(&id))
.await;
let updates = service
.runtime_input_updates(&id, &RunId::new(), 0, std::slice::from_ref(&input_id))
.await
.expect("legacy input-state load failure must not poison canonical-only updates");
assert!(
updates.is_empty(),
"missing canonical contributor must not be recovered from the legacy alias"
);
}
#[tokio::test]
async fn test_runtime_input_updates_ignore_newer_stale_legacy_row() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(GatedSnapshotRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
runtime_store.clone(),
memory_blob_store(),
);
let id = SessionId::new();
let input_id = InputId::new();
let canonical_runtime_id =
PersistentSessionService::<DummyBuilder>::runtime_id_for_session(&id);
let legacy_runtime_id = LogicalRuntimeId::legacy_session_uuid_alias(&id);
let mut canonical_state = meerkat_runtime::InputState::new_accepted(input_id.clone());
canonical_state.durability = Some(meerkat_runtime::InputDurability::Durable);
canonical_state.recovery_count = 7;
let canonical_stored = StoredInputState {
state: canonical_state.clone(),
seed: meerkat_runtime::input_state::InputStateSeed::new_accepted(),
};
let canonical_stored =
recovered_input_persistence_record(&canonical_runtime_id, canonical_stored);
runtime_store
.persist_input_state(&canonical_runtime_id, &canonical_stored)
.await
.expect("test should seed canonical input state");
tokio::time::sleep(std::time::Duration::from_millis(1)).await;
let mut legacy_state = meerkat_runtime::InputState::new_accepted(input_id.clone());
legacy_state.durability = Some(meerkat_runtime::InputDurability::Durable);
legacy_state.recovery_count = 99;
runtime_store
.persist_input_state(
&legacy_runtime_id,
&recovered_input_persistence_record(
&legacy_runtime_id,
StoredInputState {
state: legacy_state,
seed: meerkat_runtime::input_state::InputStateSeed::new_accepted(),
},
),
)
.await
.expect("test should seed newer stale legacy input state");
let updates = service
.runtime_input_updates(&id, &RunId::new(), 3, std::slice::from_ref(&input_id))
.await
.expect("runtime input updates should read canonical input state only");
assert_eq!(updates.len(), 1);
assert_eq!(
updates[0].state.recovery_count, 7,
"canonical contributor state must win over newer stale legacy duplicate"
);
assert_eq!(updates[0].seed.phase, InputLifecycleState::Consumed);
assert_eq!(updates[0].seed.last_boundary_sequence, Some(3));
}
#[tokio::test]
async fn test_runtime_backed_read_and_list_ignore_store_only_projection() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
runtime_store.clone(),
memory_blob_store(),
);
let mut raw_projection = Session::new();
raw_projection.push(Message::User(UserMessage::text(
"store-only row must stay compatibility-only".to_string(),
)));
let id = raw_projection.id().clone();
store
.save(&raw_projection)
.await
.expect("test should seed a raw store projection");
let read_err = service
.read(&id)
.await
.expect_err("read must not promote a store-only projection");
assert!(
matches!(read_err, SessionError::NotFound { .. }),
"RuntimeStore absence is authoritative even when a stale SessionStore row exists: {read_err:?}"
);
let listed = service
.list(SessionQuery::default())
.await
.expect("list should be sourced only from the RuntimeStore catalog");
assert!(
listed.iter().all(|summary| summary.session_id != id),
"a SessionStore-only row must be absent from the RuntimeStore-owned catalog"
);
assert!(
runtime_store
.load_session_snapshot(
&PersistentSessionService::<DummyBuilder>::runtime_id_for_session(&id)
)
.await
.expect("runtime snapshot load should succeed")
.is_none(),
"read/list must not create runtime authority from a store-only projection"
);
}
#[tokio::test]
async fn test_live_authority_ignores_typed_intra_turn_store_successor() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
runtime_store,
memory_blob_store(),
);
let result = service
.create_session(create_request(
"hello",
meerkat_core::service::InitialTurnPolicy::Defer,
))
.await
.expect("create_session should succeed");
let runtime_authority = service
.load_authoritative_session(&result.session_id)
.await
.expect("authoritative load should succeed")
.expect("runtime authority should exist");
let store_projection = mutate_test_session(
runtime_authority.clone(),
"in-run provisional",
|store_projection| {
store_projection.push(Message::User(UserMessage::text(
"intra-turn store projection must not replace live authority".to_string(),
)));
},
);
store
.save(&store_projection)
.await
.expect("test should seed a typed intra-turn store projection");
let authoritative = service
.load_authoritative_session(&result.session_id)
.await
.expect("authoritative load should succeed")
.expect("authoritative session should exist");
assert_eq!(
authoritative.messages().len(),
runtime_authority.messages().len(),
"an intra-turn session-store successor must not override a live runtime authority snapshot"
);
}
#[tokio::test]
async fn test_live_runtime_read_ignores_typed_store_projection_when_snapshot_load_misses() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(GatedSnapshotRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
runtime_store.clone(),
memory_blob_store(),
);
let result = service
.create_session(create_request_with_metadata(
"hello",
meerkat_core::service::InitialTurnPolicy::Defer,
))
.await
.expect("create_session should succeed");
let output = service
.apply_runtime_turn(
&result.session_id,
RunId::new(),
runtime_content_turn_request("live runtime truth"),
RunApplyBoundary::Immediate,
vec![],
)
.await
.expect("machine-owned runtime turn should succeed");
machine_commit_runtime_output(
&service,
runtime_store.as_ref(),
&result.session_id,
&output,
)
.await
.expect("machine-owned runtime output should commit");
let live = service
.export_live_session(&result.session_id)
.await
.expect("live session should export");
let runtime_authority = service
.load_authoritative_session(&result.session_id)
.await
.expect("runtime authority should load")
.expect("runtime authority should exist");
let store_projection = mutate_test_session(
runtime_authority,
"in-run provisional",
|store_projection| {
store_projection.push(Message::User(UserMessage::text(
"intra-turn compatibility row".to_string(),
)));
store_projection.set_metadata(
SESSION_LABELS_KEY,
serde_json::json!({
"source": "store-projection",
}),
);
},
);
store
.save(&store_projection)
.await
.expect("test should seed a typed intra-turn store projection");
runtime_store.hide_next_session_snapshot_loads(1);
let view = service
.read(&result.session_id)
.await
.expect("read should preserve live runtime truth");
assert_eq!(
view.state.message_count,
live.messages().len(),
"read() must not expose an intra-turn store row when live runtime truth exists"
);
assert!(
view.state.labels.is_empty(),
"read() must not expose store projection metadata when live runtime truth exists"
);
assert!(
service
.has_live_session(&result.session_id)
.await
.expect("status should succeed"),
"status must not let a store projection evict a live runtime session"
);
let authoritative = service
.load_authoritative_session(&result.session_id)
.await
.expect("authoritative load should succeed")
.expect("runtime authority should remain present");
assert_eq!(
authoritative.messages().len(),
live.messages().len(),
"the store projection must not replace durable runtime authority"
);
}
#[tokio::test]
async fn test_live_runtime_list_and_cold_resume_ignore_session_store_successor() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(GatedSnapshotRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
runtime_store.clone(),
memory_blob_store(),
);
let result = service
.create_session(create_request_with_metadata(
"hello",
meerkat_core::service::InitialTurnPolicy::Defer,
))
.await
.expect("create_session should succeed");
let output = service
.apply_runtime_turn(
&result.session_id,
RunId::new(),
runtime_content_turn_request("live runtime truth"),
RunApplyBoundary::Immediate,
vec![],
)
.await
.expect("machine-owned runtime turn should succeed");
machine_commit_runtime_output(
&service,
runtime_store.as_ref(),
&result.session_id,
&output,
)
.await
.expect("machine-owned runtime output should commit");
let live = service
.export_live_session(&result.session_id)
.await
.expect("live session should export");
let runtime_authority = service
.load_authoritative_session(&result.session_id)
.await
.expect("runtime authority should load")
.expect("runtime authority should exist");
let store_projection =
mutate_test_session(runtime_authority, "run boundary", |store_projection| {
store_projection.push(Message::User(UserMessage::text(
"intra-turn compatibility row".to_string(),
)));
store_projection.set_metadata(
SESSION_LABELS_KEY,
serde_json::json!({
"source": "store-projection",
}),
);
});
store
.save(&store_projection)
.await
.expect("test should seed an inert SessionStore successor");
let listed = service
.list(SessionQuery::default())
.await
.expect("list should succeed");
let summary = listed
.iter()
.find(|summary| summary.session_id == result.session_id)
.expect("list should include the live runtime-backed session");
assert_eq!(
summary.message_count,
live.messages().len(),
"list() must report live/runtime metadata instead of the committed store successor"
);
assert!(
summary.labels.is_empty(),
"list() must not expose labels from the committed store successor"
);
assert!(
service
.has_live_session(&result.session_id)
.await
.expect("status should succeed"),
"status must remain live after list inspects stale raw projections"
);
service
.discard_live_session(&result.session_id)
.await
.expect("test should evict the live handle before resume");
let restarted = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
runtime_store,
memory_blob_store(),
);
let resume_source = restarted
.load_authoritative_session(&result.session_id)
.await
.expect("resume source should load")
.expect("resume source should remain present");
assert_eq!(
resume_source.messages().len(),
live.messages().len(),
"cold resume must ignore the SessionStore-only extension"
);
assert!(
!resume_source.metadata().contains_key(SESSION_LABELS_KEY),
"cold resume must not import SessionStore-only labels"
);
restarted
.create_session(resume_request(resume_source))
.await
.expect("resume from RuntimeStore authority must succeed");
assert!(
restarted
.has_live_session(&result.session_id)
.await
.expect("status should succeed after resume"),
"resume from RuntimeStore authority must materialize a live session"
);
}
#[tokio::test]
async fn steady_state_full_load_path_verification_cost_is_bounded_and_constant() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
runtime_store,
memory_blob_store(),
);
let result = service
.create_session(create_request(
"hello",
meerkat_core::service::InitialTurnPolicy::Defer,
))
.await
.expect("create_session should succeed");
service
.discard_live_session(&result.session_id)
.await
.expect("evict the live session so loads use durable state");
service
.load_authoritative_session(&result.session_id)
.await
.expect("first durable load succeeds")
.expect("runtime authority exists");
const DOCUMENTS_VERIFIED_PER_LOAD: u64 = 2;
let digests_before = meerkat_core::session_content_digest_computations();
for _ in 0..5 {
service
.load_authoritative_session(&result.session_id)
.await
.expect("steady-state durable load succeeds")
.expect("runtime authority exists");
}
let first_window = meerkat_core::session_content_digest_computations() - digests_before;
assert!(
first_window <= 5 * DOCUMENTS_VERIFIED_PER_LOAD,
"steady-state durable loads must verify at most one digest per distinct \
document read ({DOCUMENTS_VERIFIED_PER_LOAD} per load): {first_window} \
computations over 5 loads"
);
let second_before = meerkat_core::session_content_digest_computations();
for _ in 0..5 {
service
.load_authoritative_session(&result.session_id)
.await
.expect("steady-state durable load succeeds")
.expect("runtime authority exists");
}
let second_window = meerkat_core::session_content_digest_computations() - second_before;
assert_eq!(
second_window, first_window,
"steady-state per-load verification cost must be constant across repeated \
loads: {first_window} then {second_window}"
);
}
#[tokio::test]
async fn test_authoritative_load_ignores_session_store_metadata_projection() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
runtime_store,
memory_blob_store(),
);
let result = service
.create_session(create_request(
"hello",
meerkat_core::service::InitialTurnPolicy::Defer,
))
.await
.expect("create_session should succeed");
let runtime_authority = service
.load_authoritative_session(&result.session_id)
.await
.expect("authoritative load should succeed")
.expect("runtime authority should exist");
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
let mut raw_store_projection = runtime_authority.clone();
raw_store_projection.set_metadata(
SESSION_LABELS_KEY,
serde_json::json!({
"source": "raw-store",
}),
);
store
.save(&raw_store_projection)
.await
.expect("test should be able to write a stale raw projection");
service
.discard_live_session(&result.session_id)
.await
.expect("test should evict the live session so read/list use durable state");
let authoritative = service
.load_authoritative_session(&result.session_id)
.await
.expect("RuntimeStore authority must remain readable")
.expect("RuntimeStore authority must remain present");
assert_eq!(authoritative.messages(), runtime_authority.messages());
assert_ne!(
authoritative.metadata().get(SESSION_LABELS_KEY),
raw_store_projection.metadata().get(SESSION_LABELS_KEY),
"SessionStore metadata must not contaminate RuntimeStore body authority"
);
let read = service
.read(&result.session_id)
.await
.expect("read must use RuntimeStore body authority");
assert_eq!(read.state.message_count, runtime_authority.messages().len());
let listed = service
.list(SessionQuery::default())
.await
.expect("list must use RuntimeStore catalog authority");
assert!(
listed
.iter()
.any(|summary| summary.session_id == result.session_id),
"catalog listing must retain the RuntimeStore-owned session"
);
}
#[tokio::test]
async fn test_runtime_backed_append_system_context_uses_runtime_authority_after_runtime_turn() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
runtime_store.clone(),
memory_blob_store(),
);
let result = service
.create_session(create_request(
"hello",
meerkat_core::service::InitialTurnPolicy::Defer,
))
.await
.expect("create_session should succeed");
let output = service
.apply_runtime_turn(
&result.session_id,
RunId::new(),
runtime_content_turn_request("runtime committed turn"),
RunApplyBoundary::Immediate,
vec![],
)
.await
.expect("runtime-backed turn should succeed");
machine_commit_runtime_output(
&service,
runtime_store.as_ref(),
&result.session_id,
&output,
)
.await
.expect("machine-owned runtime output should commit");
service
.append_system_context(
&result.session_id,
AppendSystemContextRequest {
content: meerkat_core::lifecycle::run_primitive::CoreRenderable::text(
"prefer store".to_string(),
),
source: Some("api".to_string()),
idempotency_key: Some("store-base".to_string()),
},
)
.await
.expect("append_system_context should succeed");
let stored = service
.load_authoritative_session(&result.session_id)
.await
.expect("authoritative load should succeed")
.expect("append should preserve the latest machine-owned runtime turn");
assert_eq!(
stored.messages().len(),
3,
"append must preserve runtime authority instead of using the stale session-store projection"
);
assert!(matches!(
stored.messages().last(),
Some(Message::System(system)) if system.content == "prefer store"
));
}
#[tokio::test]
async fn test_runtime_backed_stage_tool_results_without_live_handle_updates_runtime_authority()
{
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
runtime_store.clone(),
memory_blob_store(),
);
let result = service
.create_session(create_request(
"hello",
meerkat_core::service::InitialTurnPolicy::Defer,
))
.await
.expect("create_session should succeed");
service
.discard_live_session(&result.session_id)
.await
.expect("test should be able to discard the live session");
let staged = service
.stage_tool_results(
&result.session_id,
StageToolResultsRequest {
results: vec![ToolResult::new(
"tool-call-1".to_string(),
"callback result".to_string(),
false,
)],
},
)
.await
.expect("stage_tool_results should update runtime authority");
assert_eq!(staged.accepted_result_count, 1);
assert_eq!(staged.disposition, StageToolResultsDisposition::Staged);
let authoritative = service
.load_authoritative_session(&result.session_id)
.await
.expect("authoritative load should succeed")
.expect("authoritative session should exist");
let deferred = authoritative
.deferred_turn_state()
.expect("runtime authority should carry deferred-turn state");
assert_eq!(deferred.pending_tool_results().len(), 1);
assert_eq!(deferred.pending_tool_results()[0].results.len(), 1);
assert!(
store
.load(&result.session_id)
.await
.expect("SessionStore probe should succeed")
.is_none(),
"staged results must remain listable through the RuntimeStore catalog without a duplicate SessionStore row"
);
let view = service
.read(&result.session_id)
.await
.expect("read should use durable runtime truth after live discard");
assert_eq!(view.state.message_count, authoritative.messages().len());
assert!(
!service
.has_live_session(&result.session_id)
.await
.expect("status should succeed"),
"status should not recreate a live handle"
);
let listed = service
.list(SessionQuery::default())
.await
.expect("list should succeed");
let summary = listed
.iter()
.find(|summary| summary.session_id == result.session_id)
.expect("list should include persisted runtime-backed session");
assert_eq!(summary.message_count, authoritative.messages().len());
let restarted = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
runtime_store,
memory_blob_store(),
);
let resume_source = restarted
.load_authoritative_session(&result.session_id)
.await
.expect("resume source should load from runtime authority")
.expect("runtime authority should remain present after restart");
let resumed = restarted
.create_session(resume_request(resume_source))
.await
.expect("resume should materialize the runtime-authoritative snapshot");
assert_eq!(resumed.session_id, result.session_id);
let resumed_live = restarted
.export_live_session(&result.session_id)
.await
.expect("resumed live session should export");
let resumed_deferred = resumed_live
.deferred_turn_state()
.expect("resumed session should preserve staged tool results");
assert_eq!(resumed_deferred.pending_tool_results().len(), 1);
let before_redelivery = serde_json::to_value(
restarted
.load_authoritative_session(&result.session_id)
.await
.expect("authoritative replay load should succeed")
.expect("authoritative replay session should exist"),
)
.expect("serialize replayed authoritative session");
let duplicate = restarted
.stage_tool_results(
&result.session_id,
StageToolResultsRequest {
results: vec![ToolResult::new(
"tool-call-1".to_string(),
"callback result".to_string(),
false,
)],
},
)
.await
.expect("identical persistent redelivery should coalesce");
assert_eq!(duplicate.accepted_result_count, 0);
assert_eq!(
duplicate.disposition,
StageToolResultsDisposition::AlreadyStaged,
"a crash after durable stage but before run must remain a runnable continuation"
);
assert_eq!(
serde_json::to_value(
restarted
.load_authoritative_session(&result.session_id)
.await
.expect("authoritative duplicate load should succeed")
.expect("authoritative duplicate session should exist"),
)
.unwrap(),
before_redelivery,
"identical persistent redelivery must be a byte-noop"
);
for rejected in [
ToolResult::new(
"tool-call-1".to_string(),
"conflicting callback result".to_string(),
false,
),
ToolResult::new(
"wrong-tool-call".to_string(),
"callback result".to_string(),
false,
),
] {
let error = restarted
.stage_tool_results(
&result.session_id,
StageToolResultsRequest {
results: vec![rejected],
},
)
.await
.expect_err("conflicting or wrong-id redelivery must fail typed");
assert!(matches!(
error,
SessionError::Agent(AgentError::ConfigError(_))
));
assert_eq!(
serde_json::to_value(
restarted
.load_authoritative_session(&result.session_id)
.await
.expect("authoritative rejection load should succeed")
.expect("authoritative rejection session should exist"),
)
.unwrap(),
before_redelivery,
"rejected persistent redelivery must not poison the pending continuation"
);
}
}
#[tokio::test]
async fn test_store_only_control_mutations_fail_closed_without_runtime_divergence() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store) as Arc<dyn RuntimeStore>,
memory_blob_store(),
);
let session = Session::new();
let id = session.id().clone();
store
.save(&session)
.await
.expect("test should seed a store-only compatibility projection");
let append_err = service
.append_system_context(
&id,
AppendSystemContextRequest {
content: meerkat_core::lifecycle::run_primitive::CoreRenderable::text(
"store-only append must fail closed".to_string(),
),
source: Some("api".to_string()),
idempotency_key: Some("store-only-append".to_string()),
},
)
.await
.expect_err("store-only projection must not be promoted by append");
assert!(
matches!(
append_err,
SessionControlError::Session(SessionError::NotFound { id: ref missing })
if missing == &id
),
"append must treat missing RuntimeStore authority as NotFound: {append_err:?}"
);
let stage_err = service
.stage_tool_results(
&id,
StageToolResultsRequest {
results: vec![ToolResult::new(
"tool-call-1".to_string(),
"callback result".to_string(),
false,
)],
},
)
.await
.expect_err("store-only projection must not be promoted by staged tool results");
assert!(
matches!(
stage_err,
SessionError::NotFound { id: ref missing } if missing == &id
),
"stage must treat missing RuntimeStore authority as NotFound: {stage_err:?}"
);
let archive_err = service
.archive(&id)
.await
.expect_err("runtime-backed archive must route through machine");
assert!(
matches!(
archive_err,
SessionError::Unsupported(ref message)
if message.contains("MachineSessionArchiveProtocol")
),
"archive error should require machine authority: {archive_err:?}"
);
let raw = store
.load(&id)
.await
.expect("raw store load should succeed")
.expect("store-only projection should remain present");
assert!(
raw.messages().is_empty(),
"append rejection must not mutate the store-only projection"
);
assert!(
raw.deferred_turn_state().is_none(),
"stage rejection must not mutate the store-only projection"
);
assert!(
!session_marks_archived(&raw),
"archive rejection must not persist archived lifecycle metadata"
);
assert!(
runtime_store
.load_session_snapshot(
&PersistentSessionService::<DummyBuilder>::runtime_id_for_session(&id)
)
.await
.expect("runtime snapshot load should succeed")
.is_none(),
"store-only control mutations must not create runtime authority"
);
}
#[tokio::test]
async fn test_machine_authorized_archive_accepts_never_run_session() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store) as Arc<dyn RuntimeStore>,
memory_blob_store(),
);
let created = service
.create_session(create_request("seed", InitialTurnPolicy::Defer))
.await
.expect("create generation-zero RuntimeStore authority");
let id = created.session_id;
let machine = std::sync::Arc::new(meerkat_runtime::MeerkatMachine::persistent(
Arc::clone(&runtime_store) as Arc<dyn RuntimeStore>,
memory_blob_store(),
));
machine
.register_session(id.clone())
.await
.expect("register the never-run session (the mob field shape)");
service
.archive_with_machine_protocol(
&id,
MachineSessionArchiveProtocol::from_machine(machine.as_ref()),
)
.await
.expect("archiving a never-run session must succeed, not strand it");
assert!(
store
.load(&id)
.await
.expect("SessionStore probe should succeed")
.is_none(),
"archive must not persist lifecycle metadata in SessionStore"
);
let runtime_state = meerkat_runtime::store::load_runtime_state(
runtime_store.as_ref(),
&PersistentSessionService::<DummyBuilder>::runtime_id_for_session(&id),
)
.await
.expect("runtime state load should succeed");
assert_eq!(
runtime_state,
Some(meerkat_runtime::RuntimeState::Retired),
"archive must durably retire the registered runtime session"
);
}
#[tokio::test]
async fn test_machine_archive_treats_retired_runtime_as_terminal_without_body_rewrite() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store) as Arc<dyn RuntimeStore>,
memory_blob_store(),
);
let session = Session::new();
let id = session.id().clone();
let runtime_id = PersistentSessionService::<DummyBuilder>::runtime_id_for_session(&id);
let session_bytes = serde_json::to_vec(&session).expect("serialize content body");
runtime_store
.commit_session_snapshot(
&runtime_id,
SerializedSessionSnapshot {
session_snapshot: session_bytes.clone().into(),
},
)
.await
.expect("seed RuntimeStore content body");
let machine = meerkat_runtime::MeerkatMachine::persistent(
Arc::clone(&runtime_store) as Arc<dyn RuntimeStore>,
memory_blob_store(),
);
machine
.register_session(id.clone())
.await
.expect("register active runtime");
meerkat_runtime::RuntimeControlPlane::retire(&machine, &runtime_id)
.await
.expect("seed retired RuntimeStore lifecycle");
assert!(
service
.session_archived_by_authority(&id, &session)
.await
.expect("public archive projection"),
"public reads must derive archived state from RuntimeStore lifecycle"
);
let duplicate = service
.archive_with_machine_protocol(
&id,
MachineSessionArchiveProtocol::from_machine(&machine),
)
.await;
assert!(
matches!(duplicate, Err(SessionError::NotFound { .. })),
"duplicate archive over Retired must keep the public NotFound contract: {duplicate:?}"
);
let retained = runtime_store
.load_session_snapshot(&runtime_id)
.await
.expect("load retained RuntimeStore body")
.expect("RuntimeStore body remains durable");
assert_eq!(
retained.as_slice(),
session_bytes.as_slice(),
"duplicate archive must not rewrite the content body"
);
assert!(
store
.load(&id)
.await
.expect("SessionStore probe should succeed")
.is_none(),
"duplicate archive must not mint SessionStore state"
);
}
#[tokio::test]
async fn test_archive_ownership_requires_catalog_or_physical_boundary() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store) as Arc<dyn RuntimeStore>,
memory_blob_store(),
);
let session = Session::new();
let id = session.id().clone();
let runtime_id = PersistentSessionService::<DummyBuilder>::runtime_id_for_session(&id);
runtime_store
.commit_session_snapshot(
&runtime_id,
SerializedSessionSnapshot {
session_snapshot: serde_json::to_vec(&session)
.expect("serialize session body")
.into(),
},
)
.await
.expect("seed exact session body and catalog");
let machine = meerkat_runtime::MeerkatMachine::persistent(
Arc::clone(&runtime_store) as Arc<dyn RuntimeStore>,
memory_blob_store(),
);
machine
.register_session(id.clone())
.await
.expect("register runtime lifecycle");
meerkat_runtime::RuntimeControlPlane::retire(&machine, &runtime_id)
.await
.expect("retire runtime lifecycle");
assert!(
service
.session_known_to_archive_authority(&id)
.await
.expect("observe body-backed archive ownership"),
"a real RuntimeStore body/catalog boundary must remain protected as archive-owned"
);
runtime_store
.clear_session_snapshot(&runtime_id)
.await
.expect("remove body, catalog, and physical boundary");
assert_eq!(
meerkat_runtime::store::load_runtime_state(runtime_store.as_ref(), &runtime_id)
.await
.expect("load retained runtime lifecycle"),
Some(RuntimeState::Retired),
"fixture must retain bare lifecycle residue"
);
assert!(
!service
.session_known_to_archive_authority(&id)
.await
.expect("observe bodyless archive ownership"),
"a bare runtime lifecycle registration must not resurrect durable archive ownership"
);
}
#[tokio::test]
async fn test_machine_archived_resume_requires_resume_snapshot_before_actor_admission() {
let (service, _machine, mut prepared, archived) = archived_resume_admission_fixture().await;
let id = archived.id().clone();
let authorization = prepared
.archived_resume_authorization()
.expect("mint exact archived-resume authorization");
let mut request = create_request("", InitialTurnPolicy::Defer);
request.build = Some(SessionBuildOptions {
runtime_build_mode: meerkat_core::RuntimeBuildMode::SessionOwned(
prepared.bindings_clone(),
),
..Default::default()
});
let admission = service
.reserve_create_session_admission()
.await
.expect("reserve archived-resume admission");
let error = service
.create_session_with_reserved_machine_archived_resume_admission(
request,
admission,
authorization,
)
.await
.expect_err("missing resume snapshot must fail before actor construction");
assert!(
error.to_string().contains("exact resume-session snapshot"),
"unexpected missing-snapshot error: {error}"
);
assert!(
!service
.inner
.has_live_session(&id)
.await
.expect("live actor lookup"),
"invalid archived admission must not publish a live actor"
);
prepared
.rollback_now()
.await
.expect("release rejected prepared materialization");
service
.create_session(create_request("capacity probe", InitialTurnPolicy::Defer))
.await
.expect("rejected archived admission must release reserved capacity");
}
#[tokio::test]
async fn test_machine_archived_resume_rejects_standalone_runtime_mode() {
let (service, _machine, mut prepared, archived) = archived_resume_admission_fixture().await;
let id = archived.id().clone();
let authorization = prepared
.archived_resume_authorization()
.expect("mint exact archived-resume authorization");
let request = resume_request(archived);
let admission = service
.reserve_create_session_admission()
.await
.expect("reserve archived-resume admission");
let error = service
.create_session_with_reserved_machine_archived_resume_admission(
request,
admission,
authorization,
)
.await
.expect_err("Standalone archived resume must fail before actor construction");
assert!(
error
.to_string()
.contains("exact SessionOwned runtime bindings"),
"unexpected Standalone error: {error}"
);
assert!(
!service
.inner
.has_live_session(&id)
.await
.expect("live actor lookup"),
"Standalone archived admission must not publish a live actor"
);
prepared
.rollback_now()
.await
.expect("release rejected prepared materialization");
service
.create_session(create_request("capacity probe", InitialTurnPolicy::Defer))
.await
.expect("rejected Standalone admission must release reserved capacity");
}
#[tokio::test]
async fn test_machine_archived_resume_rejects_resume_binding_identity_mismatch() {
let (service, _machine, mut prepared, archived) = archived_resume_admission_fixture().await;
let id = archived.id().clone();
let authorization = prepared
.archived_resume_authorization()
.expect("mint exact archived-resume authorization");
let different_session = Session::new();
service
.runtime_store
.commit_session_snapshot(
&PersistentSessionService::<DummyBuilder>::runtime_id_for_session(
different_session.id(),
),
SerializedSessionSnapshot {
session_snapshot: (serde_json::to_vec(&different_session)
.expect("serialize mismatched RuntimeStore body"))
.into(),
},
)
.await
.expect("seed exact mismatched RuntimeStore body");
let mut request = resume_request(different_session);
request
.build
.get_or_insert_with(Default::default)
.runtime_build_mode =
meerkat_core::RuntimeBuildMode::SessionOwned(prepared.bindings_clone());
let admission = service
.reserve_create_session_admission()
.await
.expect("reserve archived-resume admission");
let error = service
.create_session_with_reserved_machine_archived_resume_admission(
request,
admission,
authorization,
)
.await
.expect_err("mismatched resume snapshot must fail before actor construction");
assert!(
error.to_string().contains("does not match resumed session"),
"unexpected identity-mismatch error: {error}"
);
assert!(
!service
.inner
.has_live_session(&id)
.await
.expect("live actor lookup"),
"mismatched archived admission must not publish a live actor"
);
prepared
.rollback_now()
.await
.expect("release rejected prepared materialization");
service
.create_session(create_request("capacity probe", InitialTurnPolicy::Defer))
.await
.expect("rejected mismatched admission must release reserved capacity");
}
#[tokio::test]
async fn test_machine_authorized_revival_promotes_only_archived_retired_session() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store) as Arc<dyn RuntimeStore>,
memory_blob_store(),
);
let session = Session::new();
let id = session.id().clone();
let runtime_id = PersistentSessionService::<DummyBuilder>::runtime_id_for_session(&id);
runtime_store
.commit_session_snapshot(
&runtime_id,
SerializedSessionSnapshot {
session_snapshot: (serde_json::to_vec(&session)
.expect("serialize session body"))
.into(),
},
)
.await
.expect("seed RuntimeStore body");
let machine = meerkat_runtime::MeerkatMachine::persistent(
Arc::clone(&runtime_store) as Arc<dyn RuntimeStore>,
memory_blob_store(),
);
machine
.register_session(id.clone())
.await
.expect("register session");
service
.archive_with_machine_protocol(
&id,
MachineSessionArchiveProtocol::from_machine(&machine),
)
.await
.expect("archive session");
service
.revive_archived_session_with_machine_authority(
&id,
machine.session_control_authority(),
)
.await
.expect("machine-authorized archived+retired revival");
let revived = service
.load_authoritative_session_base(&id)
.await
.expect("load revived session")
.expect("revived session remains durable");
assert!(
!session_marks_archived(&revived),
"revival must preserve the content-only session body"
);
assert_eq!(
meerkat_runtime::store::load_runtime_state(runtime_store.as_ref(), &runtime_id)
.await
.expect("load runtime state"),
Some(meerkat_runtime::RuntimeState::Retired),
"session authorization must not silently reset runtime authority; the caller owns that next transition"
);
service
.revive_archived_session_with_machine_authority(
&id,
machine.session_control_authority(),
)
.await
.expect("Active+Retired crash boundary must resume idempotently");
let active = Session::new();
let active_id = active.id().clone();
runtime_store
.commit_session_snapshot(
&PersistentSessionService::<DummyBuilder>::runtime_id_for_session(&active_id),
SerializedSessionSnapshot {
session_snapshot: (serde_json::to_vec(&active)
.expect("serialize active runtime checkpoint"))
.into(),
},
)
.await
.expect("seed active runtime checkpoint");
machine
.register_session(active_id.clone())
.await
.expect("register active runtime");
let active_error = service
.revive_archived_session_with_machine_authority(
&active_id,
machine.session_control_authority(),
)
.await
.expect_err("active runtime must not enter retired-session revival");
assert!(
matches!(
&active_error,
SessionError::NotFound { id } if id == &active_id
) || matches!(
&active_error,
SessionError::Agent(AgentError::InternalError(message))
if message.contains("has no retired RuntimeStore lifecycle authority")
),
"unexpected active-session revival error: {active_error:?}"
);
}
#[tokio::test]
async fn test_machine_authorized_active_retired_revival_does_not_require_live_snapshot_sync() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
EventfulBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store) as Arc<dyn RuntimeStore>,
memory_blob_store(),
);
let session = Session::new();
let id = session.id().clone();
runtime_store
.commit_session_snapshot(
&PersistentSessionService::<EventfulBuilder>::runtime_id_for_session(&id),
SerializedSessionSnapshot {
session_snapshot: (serde_json::to_vec(&session)
.expect("serialize active RuntimeStore body"))
.into(),
},
)
.await
.expect("seed active RuntimeStore body");
let machine = Arc::new(meerkat_runtime::MeerkatMachine::persistent(
Arc::clone(&runtime_store) as Arc<dyn RuntimeStore>,
memory_blob_store(),
));
machine
.register_session(id.clone())
.await
.expect("register active runtime");
meerkat_runtime::RuntimeControlPlane::retire(
machine.as_ref(),
&PersistentSessionService::<EventfulBuilder>::runtime_id_for_session(&id),
)
.await
.expect("retire runtime while leaving the document active");
let mut prepared = machine
.prepare_session_materialization(id.clone())
.await
.expect("prepare exact retired-session materialization");
machine
.install_prepared_session_executor_handles(
prepared.bindings(),
Arc::new(TestPreparedInterruptHandle),
Arc::new(TestPreparedCleanupHandle),
)
.await
.expect("install provisional cleanup before actor construction");
let authorization = prepared
.archived_resume_authorization()
.expect("mint exact archived-resume authorization");
let mut request = resume_request(session);
request
.build
.get_or_insert_with(Default::default)
.runtime_build_mode =
meerkat_core::RuntimeBuildMode::SessionOwned(prepared.bindings_clone());
let admission = service
.reserve_create_session_admission()
.await
.expect("reserve live-session admission");
let created = service
.create_session_with_reserved_machine_archived_resume_admission(
request,
admission,
authorization,
)
.await
.expect("materialize the already-active compatibility snapshot");
assert_eq!(created.session_id, id);
assert!(
service
.inner
.has_live_session(&id)
.await
.expect("live-session status should succeed")
);
service
.revive_archived_session_with_machine_authority(
&id,
machine.session_control_authority(),
)
.await
.expect("Active+Retired compatibility revival must remain supported");
prepared
.rollback_now()
.await
.expect("clean exact test materialization");
assert!(
service
.inner
.has_live_session(&id)
.await
.expect("live-session status should succeed after revival")
);
let durable = service
.load_authoritative_session_base(&id)
.await
.expect("load active session")
.expect("active session remains durable");
assert!(!session_marks_archived(&durable));
}
#[tokio::test]
async fn test_machine_authorized_archive_retires_registered_session() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store) as Arc<dyn RuntimeStore>,
memory_blob_store(),
);
let active = Session::new();
let id = active.id().clone();
let runtime_id = PersistentSessionService::<DummyBuilder>::runtime_id_for_session(&id);
runtime_store
.commit_session_snapshot(
&runtime_id,
SerializedSessionSnapshot {
session_snapshot: (serde_json::to_vec(&active)
.expect("serialize active runtime authority"))
.into(),
},
)
.await
.expect("seed active runtime checkpoint authority");
let machine = std::sync::Arc::new(meerkat_runtime::MeerkatMachine::persistent(
Arc::clone(&runtime_store) as Arc<dyn RuntimeStore>,
memory_blob_store(),
));
machine
.register_session(id.clone())
.await
.expect("register the residual runtime session");
service
.read(&id)
.await
.expect("registered active session should be readable before archive");
service
.archive_with_machine_protocol(
&id,
MachineSessionArchiveProtocol::from_machine(machine.as_ref()),
)
.await
.expect("archive must complete RuntimeStore retirement");
let runtime_state =
meerkat_runtime::store::load_runtime_state(runtime_store.as_ref(), &runtime_id)
.await
.expect("runtime state load should succeed");
assert_eq!(
runtime_state,
Some(meerkat_runtime::RuntimeState::Retired),
"the residual runtime must be durably retired by the convergent re-archive"
);
assert!(matches!(
service.read(&id).await,
Err(SessionError::NotFound { id: ref missing }) if missing == &id
));
assert!(
store
.load(&id)
.await
.expect("SessionStore probe should succeed")
.is_none(),
"archive must not mint SessionStore lifecycle authority"
);
}
#[tokio::test]
async fn test_machine_authorized_archive_retires_clean_idle_after_cold_restart() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store) as Arc<dyn RuntimeStore>,
memory_blob_store(),
);
let session = Session::new();
let id = session.id().clone();
let runtime_id = PersistentSessionService::<DummyBuilder>::runtime_id_for_session(&id);
runtime_store
.commit_session_snapshot(
&runtime_id,
SerializedSessionSnapshot {
session_snapshot: (serde_json::to_vec(&session)
.expect("serialize RuntimeStore body"))
.into(),
},
)
.await
.expect("seed RuntimeStore body");
let first_process = meerkat_runtime::MeerkatMachine::persistent(
Arc::clone(&runtime_store) as Arc<dyn RuntimeStore>,
memory_blob_store(),
);
first_process
.register_session(id.clone())
.await
.expect("seed non-retired durable runtime residue");
first_process
.unregister_session(&id)
.await
.expect("converge the first process to a clean unbound Idle row");
assert_eq!(
meerkat_runtime::store::load_runtime_state(runtime_store.as_ref(), &runtime_id,)
.await
.expect("read seeded durable runtime state"),
Some(meerkat_runtime::RuntimeState::Idle)
);
drop(first_process);
let restarted = meerkat_runtime::MeerkatMachine::persistent(
Arc::clone(&runtime_store) as Arc<dyn RuntimeStore>,
memory_blob_store(),
);
assert!(
!restarted.contains_session(&id).await,
"cold restart must begin without an in-memory registration"
);
assert!(
!restarted
.archive_runtime_residue_present(&id)
.await
.expect("observe durable archive residue"),
"clean dead-process Idle must not become archive runtime authority"
);
service
.archive_with_machine_protocol(
&id,
MachineSessionArchiveProtocol::from_machine(&restarted),
)
.await
.expect("archive should retire a cold Idle RuntimeStore session");
assert_eq!(
meerkat_runtime::store::load_runtime_state(runtime_store.as_ref(), &runtime_id)
.await
.expect("read converged durable runtime state"),
Some(meerkat_runtime::RuntimeState::Retired),
"archive must commit the RuntimeStore-owned lifecycle terminal"
);
assert!(
!restarted.contains_session(&id).await,
"archive must release any reconstructed runtime shell"
);
}
#[tokio::test]
async fn test_machine_authorized_archive_promotes_body_to_retired_lifecycle_authority() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store) as Arc<dyn RuntimeStore>,
memory_blob_store(),
);
let session = Session::new();
let id = session.id().clone();
let runtime_id = PersistentSessionService::<DummyBuilder>::runtime_id_for_session(&id);
runtime_store
.commit_session_snapshot(
&runtime_id,
SerializedSessionSnapshot {
session_snapshot: (serde_json::to_vec(&session)
.expect("serialize runtime snapshot"))
.into(),
},
)
.await
.expect("seed runtime snapshot without lifecycle record");
assert_eq!(
meerkat_runtime::store::load_runtime_state(runtime_store.as_ref(), &runtime_id)
.await
.expect("read absent lifecycle record"),
None
);
let machine = meerkat_runtime::MeerkatMachine::persistent(
Arc::clone(&runtime_store) as Arc<dyn RuntimeStore>,
memory_blob_store(),
);
service
.archive_with_machine_protocol(
&id,
MachineSessionArchiveProtocol::from_machine(&machine),
)
.await
.expect("body authority must support archive lifecycle creation");
let archived = runtime_store
.load_session_snapshot(&runtime_id)
.await
.expect("load retained RuntimeStore body")
.expect("RuntimeStore body remains present");
let archived =
Session::from_persisted_bytes(&archived).expect("RuntimeStore body should decode");
assert!(!session_marks_archived(&archived));
assert_eq!(
meerkat_runtime::store::load_runtime_state(runtime_store.as_ref(), &runtime_id)
.await
.expect("read committed lifecycle"),
Some(RuntimeState::Retired)
);
assert!(
store
.load(&id)
.await
.expect("SessionStore probe should succeed")
.is_none(),
"archive must not mint SessionStore state"
);
}
#[tokio::test]
async fn test_machine_authorized_archive_keeps_body_and_lifecycle_authority_separate() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let gated_runtime_store = Arc::new(GatedSnapshotRuntimeStore::new());
let runtime_store: Arc<dyn RuntimeStore> = gated_runtime_store.clone();
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store),
memory_blob_store(),
);
let session = Session::new();
let id = session.id().clone();
let runtime_id = PersistentSessionService::<DummyBuilder>::runtime_id_for_session(&id);
runtime_store
.commit_session_snapshot(
&runtime_id,
SerializedSessionSnapshot {
session_snapshot: (serde_json::to_vec(&session)
.expect("serialize runtime snapshot"))
.into(),
},
)
.await
.expect("seed snapshot-only runtime authority");
let machine = meerkat_runtime::MeerkatMachine::persistent(
Arc::clone(&runtime_store),
memory_blob_store(),
);
service
.archive_with_machine_protocol(
&id,
MachineSessionArchiveProtocol::from_machine(&machine),
)
.await
.expect("body-only archive must converge through RuntimeStore");
let archived = runtime_store
.load_session_snapshot(&runtime_id)
.await
.expect("load retained RuntimeStore body")
.expect("RuntimeStore body remains present");
let archived =
Session::from_persisted_bytes(&archived).expect("RuntimeStore body should decode");
assert!(!session_marks_archived(&archived));
assert_eq!(
meerkat_runtime::store::load_runtime_state(runtime_store.as_ref(), &runtime_id)
.await
.expect("read committed lifecycle"),
Some(RuntimeState::Retired)
);
}
#[tokio::test]
async fn test_machine_authorized_duplicate_archive_preserves_live_destroyed_runtime() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store) as Arc<dyn RuntimeStore>,
memory_blob_store(),
);
let session = Session::new();
let id = session.id().clone();
let runtime_id = PersistentSessionService::<DummyBuilder>::runtime_id_for_session(&id);
let body_bytes = serde_json::to_vec(&session).expect("serialize RuntimeStore body");
runtime_store
.commit_session_snapshot(
&runtime_id,
SerializedSessionSnapshot {
session_snapshot: body_bytes.clone().into(),
},
)
.await
.expect("seed RuntimeStore body");
let machine = meerkat_runtime::MeerkatMachine::persistent(
Arc::clone(&runtime_store) as Arc<dyn RuntimeStore>,
memory_blob_store(),
);
machine
.prepare_bindings(id.clone())
.await
.expect("seed live bound runtime");
meerkat_runtime::RuntimeControlPlane::destroy(&machine, &runtime_id)
.await
.expect("destroy runtime before first archive");
let duplicate = service
.archive_with_machine_protocol(
&id,
MachineSessionArchiveProtocol::from_machine(&machine),
)
.await;
assert!(
matches!(duplicate, Err(SessionError::NotFound { .. })),
"Destroyed is already terminal and must map to NotFound: {duplicate:?}"
);
let retained = runtime_store
.load_session_snapshot(&runtime_id)
.await
.expect("load retained RuntimeStore body")
.expect("RuntimeStore body remains present");
assert_eq!(retained.as_slice(), body_bytes.as_slice());
assert_eq!(
meerkat_runtime::store::load_runtime_state(runtime_store.as_ref(), &runtime_id)
.await
.expect("read quiescent runtime terminal"),
Some(RuntimeState::Destroyed)
);
assert!(
machine.contains_session(&id).await,
"archive must not remove a live registration that predated lease preparation"
);
assert!(
store
.load(&id)
.await
.expect("SessionStore probe should succeed")
.is_none()
);
}
#[tokio::test]
async fn test_machine_authorized_duplicate_archive_keeps_cold_destroyed_runtime_quiescent() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store) as Arc<dyn RuntimeStore>,
memory_blob_store(),
);
let session = Session::new();
let id = session.id().clone();
let runtime_id = PersistentSessionService::<DummyBuilder>::runtime_id_for_session(&id);
let body_bytes = serde_json::to_vec(&session).expect("serialize RuntimeStore body");
runtime_store
.commit_session_snapshot(
&runtime_id,
SerializedSessionSnapshot {
session_snapshot: body_bytes.clone().into(),
},
)
.await
.expect("seed RuntimeStore body");
let first_process = meerkat_runtime::MeerkatMachine::persistent(
Arc::clone(&runtime_store) as Arc<dyn RuntimeStore>,
memory_blob_store(),
);
first_process
.prepare_bindings(id.clone())
.await
.expect("seed bound runtime authority");
meerkat_runtime::RuntimeControlPlane::destroy(&first_process, &runtime_id)
.await
.expect("destroy runtime before process boundary");
drop(first_process);
let restarted = meerkat_runtime::MeerkatMachine::persistent(
Arc::clone(&runtime_store) as Arc<dyn RuntimeStore>,
memory_blob_store(),
);
let duplicate = service
.archive_with_machine_protocol(
&id,
MachineSessionArchiveProtocol::from_machine(&restarted),
)
.await;
assert!(
matches!(duplicate, Err(SessionError::NotFound { .. })),
"cold Destroyed duplicate archive must map to NotFound: {duplicate:?}"
);
let retained = runtime_store
.load_session_snapshot(&runtime_id)
.await
.expect("load retained RuntimeStore body")
.expect("RuntimeStore body remains present");
assert_eq!(retained.as_slice(), body_bytes.as_slice());
assert!(!restarted.contains_session(&id).await);
assert_eq!(
meerkat_runtime::store::load_runtime_state(runtime_store.as_ref(), &runtime_id)
.await
.expect("read converged archive runtime"),
Some(RuntimeState::Destroyed),
"archive must not recreate or rewrite an already-quiescent cold runtime"
);
assert!(
store
.load(&id)
.await
.expect("SessionStore probe should succeed")
.is_none()
);
}
#[tokio::test]
async fn test_machine_authorized_cold_destroyed_ignores_session_store_save_outage() {
let failing_store = Arc::new(FailingSessionStoreProbe::new());
let store: Arc<dyn SessionStore> = Arc::clone(&failing_store) as Arc<dyn SessionStore>;
let runtime_store = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store) as Arc<dyn RuntimeStore>,
memory_blob_store(),
);
let session = Session::new();
let id = session.id().clone();
let original_bytes = serde_json::to_vec(&session).expect("serialize RuntimeStore body");
let runtime_id = PersistentSessionService::<DummyBuilder>::runtime_id_for_session(&id);
runtime_store
.commit_session_snapshot(
&runtime_id,
SerializedSessionSnapshot {
session_snapshot: original_bytes.clone().into(),
},
)
.await
.expect("seed RuntimeStore body");
let first_process = meerkat_runtime::MeerkatMachine::persistent(
Arc::clone(&runtime_store) as Arc<dyn RuntimeStore>,
memory_blob_store(),
);
first_process
.prepare_bindings(id.clone())
.await
.expect("seed bound runtime authority");
meerkat_runtime::RuntimeControlPlane::destroy(&first_process, &runtime_id)
.await
.expect("destroy runtime before process boundary");
drop(first_process);
let restarted = meerkat_runtime::MeerkatMachine::persistent(
Arc::clone(&runtime_store) as Arc<dyn RuntimeStore>,
memory_blob_store(),
);
failing_store.fail_saves.store(true, Ordering::Release);
let duplicate = service
.archive_with_machine_protocol(
&id,
MachineSessionArchiveProtocol::from_machine(&restarted),
)
.await;
assert!(
matches!(duplicate, Err(SessionError::NotFound { .. })),
"Destroyed duplicate archive must ignore SessionStore save outage: {duplicate:?}"
);
assert!(
!restarted.contains_session(&id).await,
"duplicate archive must not recreate a cold runtime shell"
);
assert_eq!(
meerkat_runtime::store::load_runtime_state(runtime_store.as_ref(), &runtime_id)
.await
.expect("read normalized runtime lifecycle"),
Some(RuntimeState::Destroyed),
"duplicate archive must preserve cold Destroyed evidence"
);
let retained = runtime_store
.load_session_snapshot(&runtime_id)
.await
.expect("load retained RuntimeStore body")
.expect("RuntimeStore body remains present");
assert_eq!(
retained.as_slice(),
original_bytes.as_slice(),
"duplicate archive must leave the content body byte-intact"
);
}
#[tokio::test]
async fn test_machine_authorized_cold_destroyed_ignores_session_store_read_outage() {
let failing_store = Arc::new(FailingSessionStoreProbe::new());
let store: Arc<dyn SessionStore> = Arc::clone(&failing_store) as Arc<dyn SessionStore>;
let runtime_store = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store) as Arc<dyn RuntimeStore>,
memory_blob_store(),
);
let session = Session::new();
let id = session.id().clone();
let runtime_id = PersistentSessionService::<DummyBuilder>::runtime_id_for_session(&id);
runtime_store
.commit_session_snapshot(
&runtime_id,
SerializedSessionSnapshot {
session_snapshot: (serde_json::to_vec(&session)
.expect("serialize RuntimeStore body"))
.into(),
},
)
.await
.expect("seed RuntimeStore body");
let first_process = meerkat_runtime::MeerkatMachine::persistent(
Arc::clone(&runtime_store) as Arc<dyn RuntimeStore>,
memory_blob_store(),
);
first_process
.prepare_bindings(id.clone())
.await
.expect("seed bound runtime authority");
meerkat_runtime::RuntimeControlPlane::destroy(&first_process, &runtime_id)
.await
.expect("destroy runtime before process boundary");
drop(first_process);
let restarted = meerkat_runtime::MeerkatMachine::persistent(
Arc::clone(&runtime_store) as Arc<dyn RuntimeStore>,
memory_blob_store(),
);
failing_store.fail_loads.store(true, Ordering::Release);
let duplicate = service
.archive_with_machine_protocol(
&id,
MachineSessionArchiveProtocol::from_machine(&restarted),
)
.await;
assert!(
matches!(duplicate, Err(SessionError::NotFound { .. })),
"Destroyed duplicate archive must ignore SessionStore read outage: {duplicate:?}"
);
assert!(
!restarted.contains_session(&id).await,
"duplicate archive must not recreate a cold runtime shell"
);
assert_eq!(
meerkat_runtime::store::load_runtime_state(runtime_store.as_ref(), &runtime_id)
.await
.expect("read normalized runtime lifecycle"),
Some(RuntimeState::Destroyed),
"duplicate archive must preserve quiescent cold runtime evidence"
);
}
#[tokio::test]
async fn test_machine_authorized_archive_retires_cold_runtime_without_retaining_shell() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store) as Arc<dyn RuntimeStore>,
memory_blob_store(),
);
let session = Session::new();
let id = session.id().clone();
let runtime_id = PersistentSessionService::<DummyBuilder>::runtime_id_for_session(&id);
runtime_store
.commit_session_snapshot(
&runtime_id,
SerializedSessionSnapshot {
session_snapshot: (serde_json::to_vec(&session)
.expect("serialize RuntimeStore body"))
.into(),
},
)
.await
.expect("seed RuntimeStore body");
let before_restart = meerkat_runtime::MeerkatMachine::persistent(
Arc::clone(&runtime_store) as Arc<dyn RuntimeStore>,
memory_blob_store(),
);
before_restart
.prepare_bindings(id.clone())
.await
.expect("persist a live runtime binding before restart");
let pre_restart_state =
meerkat_runtime::store::load_runtime_state(runtime_store.as_ref(), &runtime_id)
.await
.expect("load pre-restart runtime lifecycle")
.expect("binding preparation must persist runtime lifecycle authority");
assert_ne!(pre_restart_state, meerkat_runtime::RuntimeState::Retired);
drop(before_restart);
let after_restart = meerkat_runtime::MeerkatMachine::persistent(
Arc::clone(&runtime_store) as Arc<dyn RuntimeStore>,
memory_blob_store(),
);
assert!(
!after_restart.contains_session(&id).await,
"restart fixture must begin without a live runtime registration"
);
service
.archive_with_machine_protocol(
&id,
MachineSessionArchiveProtocol::from_machine(&after_restart),
)
.await
.expect("archive must converge the stored-only observation");
assert_eq!(
meerkat_runtime::store::load_runtime_state(runtime_store.as_ref(), &runtime_id)
.await
.expect("load post-archive runtime state"),
Some(meerkat_runtime::RuntimeState::Retired),
"archive must commit the singular RuntimeStore terminal"
);
assert!(
!after_restart.contains_session(&id).await,
"archive must not retain a process-local registration reconstructed from dead state"
);
assert!(
store
.load(&id)
.await
.expect("SessionStore probe should succeed")
.is_none(),
"archive must not persist a SessionStore terminal"
);
}
#[tokio::test]
async fn test_machine_archive_drains_stored_only_pending_terminal_exactly_once() {
struct InvalidReceiptArchiveTerminalPublisher;
#[async_trait::async_trait]
impl CoreExecutorPublicationHandle for InvalidReceiptArchiveTerminalPublisher {
async fn publish_interaction_terminals(
&self,
_events: &[AgentEvent],
) -> Result<
Vec<CoreInteractionTerminalPublicationReceipt>,
meerkat_core::lifecycle::CoreExecutorError,
> {
Ok(Vec::new())
}
}
struct ArchiveFixtureExecutor {
publisher: Arc<InvalidReceiptArchiveTerminalPublisher>,
}
#[async_trait::async_trait]
impl meerkat_core::lifecycle::CoreExecutor for ArchiveFixtureExecutor {
fn publication_handle(&self) -> Option<Arc<dyn CoreExecutorPublicationHandle>> {
Some(Arc::clone(&self.publisher) as Arc<_>)
}
async fn apply(
&mut self,
_run_id: RunId,
_primitive: meerkat_core::lifecycle::run_primitive::RunPrimitive,
) -> Result<
meerkat_core::lifecycle::core_executor::CoreApplyOutput,
meerkat_core::lifecycle::CoreExecutorError,
> {
Err(meerkat_core::lifecycle::CoreExecutorError::Internal(
"archive fixture input must remain queued until stop".to_string(),
))
}
async fn cancel_after_boundary(
&mut self,
_reason: String,
) -> Result<(), meerkat_core::lifecycle::CoreExecutorError> {
Ok(())
}
async fn stop_runtime_executor(
&mut self,
_reason: String,
) -> Result<(), meerkat_core::lifecycle::CoreExecutorError> {
Ok(())
}
}
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(InMemoryRuntimeStore::new());
let event_store = Arc::new(RecordingEventStore::default());
let event_store_trait: Arc<dyn EventStore> = event_store.clone();
let projection_dir = tempfile::tempdir().expect("projection tempdir");
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store) as Arc<dyn RuntimeStore>,
memory_blob_store(),
)
.with_event_projection(
event_store_trait,
Arc::new(SessionProjector::new(projection_dir.path().join(".rkat"))),
);
let session = Session::new();
let id = session.id().clone();
let runtime_id = PersistentSessionService::<DummyBuilder>::runtime_id_for_session(&id);
let body_bytes = serde_json::to_vec(&session).expect("serialize RuntimeStore body");
runtime_store
.commit_session_snapshot(
&runtime_id,
SerializedSessionSnapshot {
session_snapshot: body_bytes.clone().into(),
},
)
.await
.expect("seed RuntimeStore body");
let before_restart = Arc::new(meerkat_runtime::MeerkatMachine::persistent(
Arc::clone(&runtime_store) as Arc<dyn RuntimeStore>,
memory_blob_store(),
));
before_restart
.prepare_bindings(id.clone())
.await
.expect("prepare durable runtime authority");
before_restart
.ensure_session_with_executor(
id.clone(),
Box::new(ArchiveFixtureExecutor {
publisher: Arc::new(InvalidReceiptArchiveTerminalPublisher),
}),
)
.await
.expect("attach exact publication capability before directed admission");
let stable_input_id = InputId::new().0;
let directed = meerkat_runtime::mob_adapter::create_tracked_flow_step_input(
"archive-terminal-step",
ContentInput::Text("pending directed archive work".to_string()),
"archive-terminal-flow",
None,
&stable_input_id.to_string(),
)
.expect("build tracked directed input");
let accepted = before_restart
.accept_input_without_wake(&id, directed)
.await
.expect("admit the directed input");
assert!(accepted.is_accepted(), "directed input should be queued");
let stop_error = before_restart
.stop_runtime_executor(&id, "archive fixture stop")
.await
.expect_err("invalid exact receipts must leave the durable outbox pending");
assert!(
stop_error
.to_string()
.contains("publication receipts did not exactly match finalized outbox batch"),
"unexpected stop failure: {stop_error}"
);
drop(before_restart);
let after_restart = meerkat_runtime::MeerkatMachine::persistent(
Arc::clone(&runtime_store) as Arc<dyn RuntimeStore>,
memory_blob_store(),
);
event_store.fail_appends();
let first_archive_error = service
.archive_with_machine_protocol(
&id,
MachineSessionArchiveProtocol::from_machine(&after_restart),
)
.await
.expect_err("failed exact append must fail archive before lifecycle retirement");
assert!(
first_archive_error
.to_string()
.contains("pending-terminal drain"),
"unexpected first archive error: {first_archive_error}"
);
let still_active = runtime_store
.load_session_snapshot(&runtime_id)
.await
.expect("load body after failed archive")
.expect("RuntimeStore body remains present");
assert_eq!(
still_active.as_slice(),
body_bytes.as_slice(),
"pending-terminal publication failure must preserve body authority"
);
assert_ne!(
meerkat_runtime::store::load_runtime_state(runtime_store.as_ref(), &runtime_id)
.await
.expect("load lifecycle after failed drain"),
Some(RuntimeState::Retired),
"pending-terminal publication failure must precede lifecycle retirement"
);
event_store.allow_appends();
service
.archive_with_machine_protocol(
&id,
MachineSessionArchiveProtocol::from_machine(&after_restart),
)
.await
.expect("archive retry must publish the stored-only terminal and retire");
assert_eq!(
meerkat_runtime::store::load_runtime_state(runtime_store.as_ref(), &runtime_id)
.await
.expect("load lifecycle after successful drain"),
Some(RuntimeState::Retired),
"successful archive retry must commit the RuntimeStore lifecycle terminal"
);
let rows = event_store
.read_from(&id, 1)
.await
.expect("read exact terminal rows");
let interaction_id = meerkat_core::interaction::InteractionId(stable_input_id);
let exact_rows: Vec<_> = rows
.iter()
.filter(|row| {
matches!(
row.source,
meerkat_core::event::EventSourceIdentity::Interaction {
interaction_id: source_id
} if source_id == interaction_id
)
})
.collect();
assert_eq!(
exact_rows.len(),
1,
"archive retry must append exactly once"
);
assert!(matches!(
&exact_rows[0].event,
AgentEvent::InteractionFailed {
interaction_id: emitted_id,
reason: meerkat_core::event::InteractionFailureReason::Abandoned { detail },
} if *emitted_id == interaction_id && detail == "runtime stopped"
));
let duplicate = service
.archive_with_machine_protocol(
&id,
MachineSessionArchiveProtocol::from_machine(&after_restart),
)
.await;
assert!(
matches!(duplicate, Err(SessionError::NotFound { .. })),
"fully converged duplicate archive keeps the NotFound contract: {duplicate:?}"
);
assert_eq!(
event_store
.read_from(&id, 1)
.await
.expect("read rows after duplicate archive")
.iter()
.filter(|row| {
matches!(
row.source,
meerkat_core::event::EventSourceIdentity::Interaction {
interaction_id: source_id
} if source_id == interaction_id
)
})
.count(),
1,
"duplicate archive must not duplicate the exact interaction terminal"
);
}
#[tokio::test]
async fn test_machine_authorized_rearchive_treats_live_destroyed_runtime_as_quiescent() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store) as Arc<dyn RuntimeStore>,
memory_blob_store(),
);
let session = Session::new();
let id = session.id().clone();
let runtime_id = PersistentSessionService::<DummyBuilder>::runtime_id_for_session(&id);
runtime_store
.commit_session_snapshot(
&runtime_id,
SerializedSessionSnapshot {
session_snapshot: (serde_json::to_vec(&session)
.expect("serialize RuntimeStore body"))
.into(),
},
)
.await
.expect("seed RuntimeStore body");
let machine = meerkat_runtime::MeerkatMachine::persistent(
Arc::clone(&runtime_store) as Arc<dyn RuntimeStore>,
memory_blob_store(),
);
machine
.prepare_bindings(id.clone())
.await
.expect("seed bound durable runtime authority");
meerkat_runtime::RuntimeControlPlane::destroy(&machine, &runtime_id)
.await
.expect("destroy the live runtime");
assert!(
machine.contains_session(&id).await,
"destroy keeps the terminal registration available for cleanup"
);
assert!(
!machine
.archive_runtime_residue_present(&id)
.await
.expect("observe live destroyed runtime terminality"),
"live Destroyed must be quiescent for archive"
);
let error = service
.archive_with_machine_protocol(
&id,
MachineSessionArchiveProtocol::from_machine(&machine),
)
.await
.expect_err("quiescent live-destroyed re-archive keeps the NotFound contract");
assert!(matches!(error, SessionError::NotFound { .. }));
assert!(
machine.contains_session(&id).await,
"re-archive must preserve the live terminal registration it did not create"
);
assert_eq!(
meerkat_runtime::store::load_runtime_state(runtime_store.as_ref(), &runtime_id)
.await
.expect("destroyed state must remain readable"),
Some(meerkat_runtime::RuntimeState::Destroyed)
);
assert!(
store
.load(&id)
.await
.expect("SessionStore probe should succeed")
.is_none()
);
}
#[tokio::test]
async fn test_machine_authorized_rearchive_keeps_cold_destroyed_runtime_quiescent() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store) as Arc<dyn RuntimeStore>,
memory_blob_store(),
);
let session = Session::new();
let id = session.id().clone();
let body_bytes = serde_json::to_vec(&session).expect("serialize RuntimeStore body");
let runtime_id = PersistentSessionService::<DummyBuilder>::runtime_id_for_session(&id);
runtime_store
.commit_session_snapshot(
&runtime_id,
SerializedSessionSnapshot {
session_snapshot: body_bytes.clone().into(),
},
)
.await
.expect("seed RuntimeStore body");
let first_process = meerkat_runtime::MeerkatMachine::persistent(
Arc::clone(&runtime_store) as Arc<dyn RuntimeStore>,
memory_blob_store(),
);
first_process
.prepare_bindings(id.clone())
.await
.expect("seed bound durable runtime authority");
meerkat_runtime::RuntimeControlPlane::destroy(&first_process, &runtime_id)
.await
.expect("destroy runtime before process boundary");
assert_eq!(
meerkat_runtime::store::load_runtime_state(runtime_store.as_ref(), &runtime_id)
.await
.expect("read destroyed durable runtime state"),
Some(meerkat_runtime::RuntimeState::Destroyed)
);
drop(first_process);
let restarted = meerkat_runtime::MeerkatMachine::persistent(
Arc::clone(&runtime_store) as Arc<dyn RuntimeStore>,
memory_blob_store(),
);
assert!(
!restarted
.archive_runtime_residue_present(&id)
.await
.expect("observe destroyed runtime terminality"),
"cold Destroyed runtime state must be quiescent for archive"
);
let error = service
.archive_with_machine_protocol(
&id,
MachineSessionArchiveProtocol::from_machine(&restarted),
)
.await
.expect_err("cold quiescent re-archive keeps the NotFound contract");
assert!(matches!(error, SessionError::NotFound { .. }));
assert!(
!restarted.contains_session(&id).await,
"successful re-archive must remove its reconstructed runtime shell"
);
assert_eq!(
meerkat_runtime::store::load_runtime_state(runtime_store.as_ref(), &runtime_id)
.await
.expect("converged runtime state must remain readable"),
Some(meerkat_runtime::RuntimeState::Destroyed),
"re-archive must not rewrite already-quiescent cold runtime evidence"
);
let retained = runtime_store
.load_session_snapshot(&runtime_id)
.await
.expect("load RuntimeStore body after convergence")
.expect("RuntimeStore body remains present");
assert_eq!(
retained.as_slice(),
body_bytes.as_slice(),
"runtime-only convergence must not rewrite the content body"
);
assert!(
store
.load(&id)
.await
.expect("SessionStore probe should succeed")
.is_none()
);
}
#[tokio::test]
async fn test_machine_authorized_duplicate_archive_of_retired_session_is_not_found() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store) as Arc<dyn RuntimeStore>,
memory_blob_store(),
);
let session = Session::new();
let id = session.id().clone();
let runtime_id = PersistentSessionService::<DummyBuilder>::runtime_id_for_session(&id);
runtime_store
.commit_session_snapshot(
&runtime_id,
SerializedSessionSnapshot {
session_snapshot: (serde_json::to_vec(&session)
.expect("serialize RuntimeStore body"))
.into(),
},
)
.await
.expect("seed RuntimeStore body");
let machine = meerkat_runtime::MeerkatMachine::persistent(
Arc::clone(&runtime_store) as Arc<dyn RuntimeStore>,
memory_blob_store(),
);
machine
.register_session(id.clone())
.await
.expect("register RuntimeStore session");
meerkat_runtime::RuntimeControlPlane::retire(&machine, &runtime_id)
.await
.expect("seed Retired lifecycle");
let err = service
.archive_with_machine_protocol(
&id,
MachineSessionArchiveProtocol::from_machine(&machine),
)
.await
.expect_err("quiescent duplicate archive keeps the NotFound contract");
assert!(
matches!(err, SessionError::NotFound { .. }),
"unexpected duplicate-archive error: {err:?}"
);
assert!(
store
.load(&id)
.await
.expect("SessionStore probe should succeed")
.is_none()
);
}
fn recoverable_store_row() -> Session {
let mut session = Session::new();
session
.set_session_metadata(meerkat_core::SessionMetadata {
schema_version: meerkat_core::SESSION_METADATA_SCHEMA_VERSION,
model: "test-model".to_string(),
max_tokens: 1024,
structured_output_retries: 2,
provider: meerkat_core::Provider::Anthropic,
self_hosted_server_id: None,
provider_params: None,
tooling: meerkat_core::SessionTooling::default(),
keep_alive: false,
comms_name: None,
peer_meta: None,
realm_id: None,
instance_id: None,
backend: None,
config_generation: None,
auth_binding: None,
mob_member_binding: None,
})
.expect("test session metadata should serialize");
session
}
fn legacy_store_only_archived_row() -> Session {
let mut session = recoverable_store_row();
session.push(Message::User(UserMessage::text(
"legacy archived transcript".to_string(),
)));
session
.set_lifecycle_terminal(SessionLifecycleTerminal::Archived)
.expect("seed typed archived lifecycle-terminal fact");
session
}
#[tokio::test]
async fn test_unimported_store_only_archived_row_is_not_runtime_authority() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store: Arc<dyn RuntimeStore> = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store),
memory_blob_store(),
);
let live = service
.create_session(create_request("live", InitialTurnPolicy::Defer))
.await
.expect("create_session should succeed");
let legacy = legacy_store_only_archived_row();
let legacy_id = legacy.id().clone();
store
.save(&legacy)
.await
.expect("test should seed a legacy archived store row");
assert!(
!service
.session_archived_by_authority(&legacy_id, &legacy)
.await
.expect("RuntimeStore lifecycle read must not error"),
"unimported SessionStore lifecycle metadata is not runtime authority"
);
let summaries = service
.list(SessionQuery::default())
.await
.expect("a store-only archived row must not brick runtime catalog list()");
assert!(
summaries
.iter()
.any(|summary| summary.session_id == live.session_id),
"list must still surface RuntimeStore sessions"
);
assert!(
summaries
.iter()
.all(|summary| summary.session_id != legacy_id),
"store-only rows must stay out of the RuntimeStore catalog"
);
let read_error = service
.read(&legacy_id)
.await
.expect_err("unimported store-only row must not be readable");
assert!(
matches!(read_error, SessionError::NotFound { ref id } if *id == legacy_id),
"store-only read must reject with NotFound: {read_error:?}"
);
}
#[tokio::test]
async fn metadata_view_uses_runtime_store_and_ignores_session_store_projection() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store: Arc<dyn RuntimeStore> = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store),
memory_blob_store(),
);
let runtime_session = recoverable_store_row();
let id = runtime_session.id().clone();
runtime_store
.commit_session_snapshot(
&PersistentSessionService::<DummyBuilder>::runtime_id_for_session(&id),
SerializedSessionSnapshot {
session_snapshot: serde_json::to_vec(&runtime_session)
.expect("serialize RuntimeStore authority")
.into(),
},
)
.await
.expect("commit RuntimeStore authority");
let mut projection = runtime_session.clone();
let mut projection_metadata = projection
.session_metadata()
.expect("runtime fixture has session metadata");
projection_metadata.model = "session-store-only-model".to_string();
projection
.set_session_metadata(projection_metadata)
.expect("mutate compatibility projection metadata");
store
.save(&projection)
.await
.expect("write divergent SessionStore projection");
let view = service
.load_authoritative_session_metadata(&id)
.await
.expect("metadata authority load")
.expect("RuntimeStore metadata authority must exist");
assert_eq!(
view.session_metadata
.as_ref()
.map(|metadata| metadata.model.as_str()),
Some("test-model"),
"SessionStore metadata must not participate in ordinary metadata authority"
);
}
#[tokio::test]
async fn test_metadata_only_projection_does_not_discard_live_session() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::new(InMemoryRuntimeStore::new()),
memory_blob_store(),
);
let created = service
.create_session(create_request("hello", InitialTurnPolicy::Defer))
.await
.expect("create deferred session");
let id = created.session_id;
let mut projected = service
.export_live_session(&id)
.await
.expect("live session should export");
projected.set_metadata("projection_only", serde_json::json!(true));
store
.save(&projected)
.await
.expect("projection snapshot should save");
let discarded = service
.discard_stale_live_session_if_needed(&id)
.await
.expect("discard check should succeed");
assert!(
!discarded,
"metadata/timestamp-only durable projection must not evict live runtime mechanics"
);
service
.export_live_session(&id)
.await
.expect("live session should remain available");
}
#[tokio::test]
async fn exact_terminal_batch_sequences_once_broadcasts_inserts_and_silences_replay() {
use futures::StreamExt;
use meerkat_core::event::InteractionFailureReason;
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let event_store = Arc::new(RecordingEventStore::default());
let event_store_trait: Arc<dyn EventStore> = event_store.clone();
let dir = tempfile::tempdir().expect("tempdir");
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::new(InMemoryRuntimeStore::new()),
memory_blob_store(),
)
.with_event_projection(
event_store_trait,
Arc::new(SessionProjector::new(dir.path().join(".rkat"))),
);
let created = service
.create_session(create_request_with_metadata(
"hello",
InitialTurnPolicy::Defer,
))
.await
.expect("create session");
let session_id = created.session_id;
let mut stream = service
.subscribe_session_events(&session_id)
.await
.expect("subscribe exact terminal stream");
let ids = [
meerkat_core::interaction::InteractionId(meerkat_core::time_compat::new_uuid_v7()),
meerkat_core::interaction::InteractionId(meerkat_core::time_compat::new_uuid_v7()),
meerkat_core::interaction::InteractionId(meerkat_core::time_compat::new_uuid_v7()),
];
let terminals = vec![
AgentEvent::InteractionComplete {
interaction_id: ids[0],
result: "done".to_string(),
structured_output: None,
},
AgentEvent::InteractionCallbackPending {
interaction_id: ids[1],
tool_name: "external".to_string(),
args: serde_json::json!({"value": 1}),
pending_tool_calls: Vec::new(),
},
AgentEvent::InteractionFailed {
interaction_id: ids[2],
reason: InteractionFailureReason::Cancelled,
},
];
let first_receipts = service
.publish_interaction_terminals_exact_batch(&session_id, &terminals)
.await
.expect("publish exact terminal batch");
assert_eq!(
first_receipts
.iter()
.map(|receipt| receipt.interaction_id())
.collect::<Vec<_>>(),
ids
);
let mut published = Vec::new();
for _ in 0..terminals.len() {
published.push(
tokio::time::timeout(std::time::Duration::from_secs(2), stream.next())
.await
.expect("exact terminal broadcast timeout")
.expect("exact terminal stream closed"),
);
}
assert_eq!(
published
.iter()
.map(|envelope| match &envelope.payload {
AgentEvent::InteractionComplete { interaction_id, .. }
| AgentEvent::InteractionCallbackPending { interaction_id, .. }
| AgentEvent::InteractionFailed { interaction_id, .. } => *interaction_id,
other => panic!("unexpected batch broadcast: {other:?}"),
})
.collect::<Vec<_>>(),
ids
);
assert_eq!(published[1].seq, published[0].seq + 1);
assert_eq!(published[2].seq, published[1].seq + 1);
let replay_receipts = service
.publish_interaction_terminals_exact_batch(&session_id, &terminals)
.await
.expect("replay exact terminal batch");
assert_eq!(replay_receipts, first_receipts);
assert!(
tokio::time::timeout(std::time::Duration::from_millis(100), stream.next())
.await
.is_err(),
"canonical replay must not rebroadcast already-published terminals"
);
let next_id =
meerkat_core::interaction::InteractionId(meerkat_core::time_compat::new_uuid_v7());
let next_terminal = [AgentEvent::InteractionComplete {
interaction_id: next_id,
result: "next".to_string(),
structured_output: None,
}];
service
.publish_interaction_terminals_exact_batch(&session_id, &next_terminal)
.await
.expect("publish terminal after replay");
let next = tokio::time::timeout(std::time::Duration::from_secs(2), stream.next())
.await
.expect("next terminal broadcast timeout")
.expect("exact terminal stream closed");
assert_eq!(next.seq, published[2].seq + 1);
assert!(matches!(
next.payload,
AgentEvent::InteractionComplete { interaction_id, .. } if interaction_id == next_id
));
let rows = event_store
.read_from(&session_id, 0)
.await
.expect("read exact terminal rows");
let interaction_rows = rows
.iter()
.filter(|row| {
matches!(
row.event,
AgentEvent::InteractionComplete { .. }
| AgentEvent::InteractionCallbackPending { .. }
| AgentEvent::InteractionFailed { .. }
)
})
.count();
assert_eq!(
interaction_rows, 4,
"replay must not duplicate durable rows"
);
}
#[tokio::test]
async fn test_event_replay_projection_reads_ordered_session_events() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let event_store = Arc::new(RecordingEventStore::default());
let event_store_trait: Arc<dyn EventStore> = event_store.clone();
let dir = tempfile::tempdir().expect("tempdir");
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::new(InMemoryRuntimeStore::new()),
memory_blob_store(),
)
.with_event_projection(
event_store_trait,
Arc::new(SessionProjector::new(dir.path().join(".rkat"))),
);
let session_id = SessionId::new();
event_store
.append(
&session_id,
&[
AgentEvent::TurnStarted { turn_number: 0 },
AgentEvent::TextComplete {
content: "two".to_string(),
},
],
)
.await
.expect("append event fixtures");
assert_eq!(
service
.event_log_latest_seq(&session_id)
.await
.expect("latest seq"),
Some(2)
);
let events = service
.event_log_read_from(&session_id, 2)
.await
.expect("read event log")
.expect("event projection enabled");
assert_eq!(events.len(), 1);
assert_eq!(events[0].seq, 2);
assert!(matches!(events[0].event, AgentEvent::TextComplete { .. }));
}
#[tokio::test]
async fn test_event_replay_projection_reports_unsupported_when_not_installed() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::new(InMemoryRuntimeStore::new()),
memory_blob_store(),
);
let session_id = SessionId::new();
assert_eq!(
service
.event_log_latest_seq(&session_id)
.await
.expect("latest seq"),
None
);
assert!(
service
.event_log_read_from(&session_id, 1)
.await
.expect("read event log")
.is_none()
);
}
fn create_request_with_metadata(
prompt: &str,
initial_turn: InitialTurnPolicy,
) -> CreateSessionRequest {
let mut session = Session::new();
let metadata = meerkat_core::SessionMetadata {
schema_version: meerkat_core::SESSION_METADATA_SCHEMA_VERSION,
model: "test".to_string(),
max_tokens: 1024,
structured_output_retries: 2,
provider: meerkat_core::Provider::Anthropic,
self_hosted_server_id: None,
provider_params: None,
tooling: meerkat_core::SessionTooling::default(),
keep_alive: false,
comms_name: None,
peer_meta: None,
realm_id: None,
instance_id: None,
backend: None,
config_generation: None,
auth_binding: None,
mob_member_binding: None,
};
session.set_session_metadata(metadata).unwrap();
let mut req = create_request(prompt, initial_turn);
req.build = Some(SessionBuildOptions {
resume_session: Some(session),
..Default::default()
});
req
}
#[test]
fn rollback_snapshot_ignores_legacy_filter_metadata_when_canonical_state_is_absent() {
let mut session = Session::new();
session.set_metadata(
meerkat_core::EXTERNAL_TOOL_FILTER_METADATA_KEY,
serde_json::to_value(meerkat_core::ToolFilter::Deny(
["secret".to_string()].into_iter().collect(),
))
.unwrap(),
);
session.set_metadata(
meerkat_core::tool_scope::INHERITED_TOOL_FILTER_METADATA_KEY,
serde_json::to_value(meerkat_core::ToolFilter::Allow(
["visible".to_string()].into_iter().collect(),
))
.unwrap(),
);
let snapshot = rollback_tool_visibility_state_snapshot(&session)
.expect("legacy-only metadata should not fail canonical visibility parsing");
assert_eq!(
snapshot, None,
"legacy-only metadata must not become rollback visibility authority"
);
}
#[tokio::test]
async fn test_dispatch_external_tool_call_does_not_forge_tool_visibility_state() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
ToolDispatchBuilder,
4,
Arc::clone(&store),
runtime_store,
memory_blob_store(),
);
let created = service
.create_session(create_request("hello", InitialTurnPolicy::Defer))
.await
.expect("create session");
let id = created.session_id;
let outcome = service
.dispatch_external_tool_call(
&id,
ToolCall::new(
"call-1".to_string(),
"callback_probe_tool".to_string(),
serde_json::json!({}),
),
)
.await
.expect("dispatch external tool call");
assert_eq!(outcome.result.text_content(), "handled callback_probe_tool");
let persisted = service
.load_authoritative_session(&id)
.await
.expect("load runtime authority")
.expect("runtime authority should exist");
let visibility_state = persisted
.tool_visibility_state()
.expect("runtime visibility state should decode");
assert!(
visibility_state.is_none(),
"external dispatch must not forge generated tool-visibility authority: {visibility_state:?}"
);
assert!(
store
.load(&id)
.await
.expect("SessionStore probe after external dispatch")
.is_none(),
"external dispatch must not mint a second SessionStore authority"
);
}
#[tokio::test]
async fn test_dispatch_external_tool_call_discards_live_session_when_persist_fails() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(GatedSnapshotRuntimeStore::new());
let service = PersistentSessionService::new(
ToolDispatchBuilder,
4,
Arc::clone(&store),
runtime_store.clone(),
memory_blob_store(),
);
let created = service
.create_session(create_request("hello", InitialTurnPolicy::Defer))
.await
.expect("create session");
let id = created.session_id;
let authority_before = service
.load_authoritative_session(&id)
.await
.expect("load runtime authority before failure")
.expect("runtime authority should exist");
runtime_store.set_fail_snapshot_commits(true);
let error = service
.dispatch_external_tool_call(
&id,
ToolCall::new(
"call-2".to_string(),
"callback_probe_tool".to_string(),
serde_json::json!({}),
),
)
.await
.expect_err("persist failure should bubble out");
assert!(
error
.to_string()
.contains("synthetic runtime snapshot commit failure"),
"expected RuntimeStore error after persistence failure, got {error:?}"
);
let live = service.export_live_session(&id).await;
assert!(
matches!(live, Err(SessionError::NotFound { .. })),
"expected live session to be discarded after persist failure, got {live:?}"
);
let authority_after = service
.load_authoritative_session(&id)
.await
.expect("load runtime authority after failure")
.expect("runtime predecessor authority should remain");
assert_eq!(
authority_after.messages(),
authority_before.messages(),
"failed RuntimeStore commit must not advance durable authority"
);
assert!(
store
.load(&id)
.await
.expect("SessionStore probe after failed dispatch")
.is_none(),
"failed dispatch must not mint a second SessionStore authority"
);
}
#[tokio::test]
async fn cold_load_ignores_store_only_tail_without_atomic_physical_head_ownership() {
use meerkat_core::{AssistantBlock, BlockAssistantMessage, StopReason};
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
Arc::clone(&runtime_store) as Arc<dyn meerkat_runtime::RuntimeStore>,
memory_blob_store(),
);
let id = SessionId::new();
let mut snapshot = Session::with_id(id.clone());
snapshot.push(Message::User(UserMessage::text(
"committed turn".to_string(),
)));
let runtime_id = meerkat_runtime::LogicalRuntimeId::for_session(&id);
runtime_store
.commit_session_snapshot(
&runtime_id,
meerkat_runtime::SerializedSessionSnapshot {
session_snapshot: (serde_json::to_vec(&snapshot)
.expect("serialize committed snapshot"))
.into(),
},
)
.await
.expect("seed committed runtime snapshot");
let tail_run = meerkat_core::RunId::new();
let mut head = snapshot.clone();
head.push(Message::User(UserMessage::text(
"reply with the single word ACK".to_string(),
)));
let mut assistant = BlockAssistantMessage::new(
vec![AssistantBlock::Text {
text: "ACK".to_string(),
meta: None,
}],
StopReason::EndTurn,
);
assistant.identity = assistant.identity.with_run_id(tail_run.clone());
head.push(Message::BlockAssistant(assistant));
store.save(&head).await.expect("seed durable store head");
let loaded = service
.load_authoritative_session(&id)
.await
.expect("WholeBlob authoritative read should succeed")
.expect("committed runtime body should remain visible");
assert_eq!(
loaded.messages(),
snapshot.messages(),
"a SessionStore-only tail must not shadow WholeBlob authority"
);
let retained = store
.load(&id)
.await
.expect("store read succeeds")
.expect("held physical head must survive intact");
assert_eq!(
meerkat_core::session_store::session_projection_cas_token(&retained)
.expect("retained token"),
meerkat_core::session_store::session_projection_cas_token(&head)
.expect("original token"),
"ignoring a non-authoritative row must not mutate it"
);
assert!(
runtime_store
.load_boundary_receipt(&runtime_id, &tail_run, 1)
.await
.expect("receipt read succeeds")
.is_none(),
"an ignored store-only row must not mint a recovery receipt"
);
}
#[tokio::test]
async fn recover_committed_boundary_rejects_store_only_row_as_not_found() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
Arc::clone(&store),
runtime_store as Arc<dyn meerkat_runtime::RuntimeStore>,
memory_blob_store(),
);
let id = SessionId::new();
let mut session = Session::with_id(id.clone());
session.push(Message::User(UserMessage::text("held row".to_string())));
let session = session;
store.save(&session).await.expect("seed physical row");
let token = meerkat_core::session_store::session_projection_cas_token(&session)
.expect("physical-row token");
for _ in 0..2 {
let error = service
.recover_committed_boundary(&id)
.await
.expect_err("store-only rows have no runtime recovery authority");
assert!(
matches!(error, SessionError::NotFound { id: ref found } if *found == id),
"expected NotFound, got {error:?}"
);
}
let retained = store
.load(&id)
.await
.expect("store read succeeds")
.expect("held row survives");
assert_eq!(
meerkat_core::session_store::session_projection_cas_token(&retained)
.expect("retained token"),
token,
"NotFound classification must not mutate the physical row"
);
}
#[tokio::test]
async fn recover_committed_boundary_reports_missing_as_not_found() {
let store: Arc<dyn SessionStore> = Arc::new(MemoryStore::new());
let runtime_store = Arc::new(InMemoryRuntimeStore::new());
let service = PersistentSessionService::new(
DummyBuilder,
4,
store,
runtime_store as Arc<dyn meerkat_runtime::RuntimeStore>,
memory_blob_store(),
);
let missing = SessionId::new();
let error = service
.recover_committed_boundary(&missing)
.await
.expect_err("an unknown session is a typed error");
assert!(
matches!(error, SessionError::NotFound { id: ref found } if *found == missing),
"expected NotFound, got {error:?}"
);
}
fn incremental_store_fixture() -> (
Arc<MemoryStore>,
Arc<dyn SessionStore>,
Arc<dyn IncrementalSessionStore>,
) {
let memory_store = Arc::new(MemoryStore::new());
let store: Arc<dyn SessionStore> = memory_store.clone();
let incremental = Arc::clone(&memory_store)
.as_incremental()
.expect("memory store must expose the incremental capability");
(memory_store, store, incremental)
}
fn user_message(text: &str) -> Message {
Message::User(UserMessage::text(text.to_string()))
}
#[tokio::test]
async fn incremental_physical_boundary_flow_uses_delta_writes() {
let (memory_store, store, incremental) = incremental_store_fixture();
let baseline = memory_store.stats().await;
assert_eq!(
baseline.whole_blob_saves, 0,
"the incremental path must begin without a whole-blob row"
);
let mut session = Session::new();
let root = TranscriptStrandId::root();
let mut head = None;
for turn in 0..3u32 {
let before = memory_store.stats().await;
let base_seq = session.messages().len();
session = mutate_test_session(session, "run boundary", |session| {
session.push(user_message(&format!("turn-{turn} question")));
session.push(user_message(&format!("turn-{turn} answer")));
});
incremental
.append_messages(
session.id(),
&root,
u64::try_from(base_seq).expect("test message count"),
&session.messages()[base_seq..],
)
.await
.expect("append exact boundary delta");
let successor =
SessionHead::from_session(&session, root.clone(), 0).expect("successor head");
let expected = match head.as_ref() {
Some(predecessor) => SessionHeadCas::IfToken(
session_head_cas_token(predecessor).expect("predecessor token"),
),
None => SessionHeadCas::Create,
};
incremental
.save_head(&successor, expected)
.await
.expect("publish exact boundary head");
head = Some(successor);
let after = memory_store.stats().await;
assert_eq!(
after.appended_message_rows - before.appended_message_rows,
2,
"turn {turn} must append exactly the delta rows"
);
assert_eq!(
after.head_saves - before.head_saves,
1,
"turn {turn} must write exactly one head row"
);
assert_eq!(
after.whole_blob_saves, 0,
"no whole-blob saves on any boundary"
);
}
let final_stats = memory_store.stats().await;
assert_eq!(final_stats.whole_blob_saves, 0);
let slim = store
.load(session.id())
.await
.expect("load should succeed")
.expect("session persisted");
assert_eq!(slim.messages().len(), session.messages().len());
}
#[tokio::test]
async fn incremental_compaction_single_commit_single_audit() {
let memory_store = Arc::new(MemoryStore::new());
let store: Arc<dyn SessionStore> = memory_store.clone();
let incremental = Arc::clone(&memory_store)
.as_incremental()
.expect("memory store must expose the incremental capability");
let event_store = Arc::new(RecordingEventStore::default());
let event_store_trait: Arc<dyn EventStore> = event_store.clone();
let dir = tempfile::tempdir().expect("tempdir");
let projector = Arc::new(SessionProjector::new(dir.path().join(".rkat")));
let parent = mutate_test_session(Session::new(), "run boundary", |parent| {
for turn in 0..2 {
parent.push(user_message(&format!("turn-{turn} question")));
parent.push(user_message(&format!("turn-{turn} answer")));
}
});
let root = TranscriptStrandId::root();
incremental
.append_messages(parent.id(), &root, 0, parent.messages())
.await
.expect("persist parent rows");
let parent_head = SessionHead::from_session(&parent, root, 0).expect("prepare parent head");
incremental
.save_head(&parent_head, SessionHeadCas::Create)
.await
.expect("persist parent head");
let parent_revision = parent.transcript_revision().expect("parent revision");
let compacted = mutate_test_session(parent.clone(), "transcript rewrite", |compacted| {
compacted
.commit_transcript_rewrite(
TranscriptRewriteSelection::MessageRange {
start: 0,
end: parent.messages().len(),
},
vec![user_message("[Context compacted] singleton summary")],
TranscriptRewriteReason::new("compaction"),
Some("meerkat-core".to_string()),
Some(parent_revision),
)
.expect("compaction rewrite should commit");
});
assert!(compacted.messages().len() < parent.messages().len());
let before = memory_store.stats().await;
let commit = compacted
.validated_transcript_history_state()
.expect("compacted history")
.expect("compaction installs history")
.last_commit()
.cloned()
.expect("compaction commit");
let record =
transcript_rewrite_record_for_session(&compacted, &commit).expect("rewrite record");
let parent_token = session_head_cas_token(&parent_head).expect("parent token");
let next = incremental
.commit_rewrite(
parent.id(),
&record,
SessionHeadCas::IfToken(parent_token.clone()),
)
.await
.expect("commit compact rewrite");
let compacted_head = SessionHead::from_session(&compacted, next.strand, next.rewrite_count)
.expect("prepare compacted head");
incremental
.save_head(&compacted_head, SessionHeadCas::IfToken(parent_token))
.await
.expect("adopt compacted head");
append_transcript_rewrite_receipt_for_commits(
Some(&event_store_trait),
Some(&projector),
&compacted,
std::slice::from_ref(&commit),
)
.await
.expect("publish compact rewrite receipt");
let after = memory_store.stats().await;
assert_eq!(
after.rewrite_commits - before.rewrite_commits,
1,
"compaction must record exactly one rewrite commit"
);
assert_eq!(after.whole_blob_saves, 0);
let events = event_store
.read_from(parent.id(), 0)
.await
.expect("event replay");
let receipt_events = events
.iter()
.filter(|stored| {
matches!(
stored.event,
AgentEvent::TranscriptRewriteAuditReceiptCommitted { .. }
)
})
.count();
assert_eq!(receipt_events, 1, "exactly one rewrite audit receipt");
assert!(
events.iter().all(|stored| !matches!(
stored.event,
AgentEvent::TranscriptRewriteCommitted { .. }
)),
"current incremental writers must not emit full-body rewrite rows"
);
let slim = store
.load(parent.id())
.await
.expect("load should succeed")
.expect("compacted projection should exist");
assert_eq!(slim.messages().len(), compacted.messages().len());
}
#[tokio::test]
async fn incremental_out_of_line_history_reads_serve_rewrites_and_parent_rows() {
let (_memory_store, store, inc) = incremental_store_fixture();
let mut parent = recoverable_store_row();
parent.push(user_message("turn one"));
parent.push(user_message("turn two"));
let root = TranscriptStrandId::root();
inc.append_messages(parent.id(), &root, 0, parent.messages())
.await
.expect("seed rows");
let head = SessionHead::from_session(&parent, root, 0).expect("seed head");
inc.save_head(&head, SessionHeadCas::Create)
.await
.expect("seed head save");
let mut compacted = parent.clone();
let commit = compacted
.commit_transcript_rewrite(
TranscriptRewriteSelection::MessageRange { start: 0, end: 2 },
vec![user_message("[compacted] summary")],
TranscriptRewriteReason::new("compaction"),
Some("test".to_string()),
None,
)
.expect("rewrite should commit");
let record =
transcript_rewrite_record_for_session(&compacted, &commit).expect("record for commit");
let token = session_head_cas_token(&head).expect("token");
let next = inc
.commit_rewrite(parent.id(), &record, SessionHeadCas::IfToken(token.clone()))
.await
.expect("commit rewrite");
let adopted = SessionHead::from_session(&compacted, next.strand, next.rewrite_count)
.expect("adopted compact head");
inc.save_head(&adopted, SessionHeadCas::IfToken(token))
.await
.expect("adopt head");
let slim = store
.load(parent.id())
.await
.expect("load should succeed")
.expect("session persisted");
assert!(
slim.validated_transcript_history_state()
.expect("state read")
.is_none(),
"store-only incremental sessions resolve slim"
);
let listed = inc
.load_rewrite_commits(parent.id())
.await
.expect("adopted commits must be served out of line");
assert_eq!(listed.len(), 1);
assert_eq!(listed[0].revision, commit.revision);
assert_eq!(listed[0].parent_revision, commit.parent_revision);
let parent_rows = inc
.load_messages(parent.id(), &head.strand, 0..head.message_count)
.await
.expect("parent body must remain addressable out of line");
assert_eq!(parent_rows.len(), 2, "parent body served out-of-line");
}
}