#![allow(
dead_code,
unused_imports,
clippy::boxed_local,
clippy::expect_used,
clippy::implicit_clone,
clippy::large_futures,
clippy::redundant_clone,
clippy::unnested_or_patterns
)]
pub mod auth_endpoints;
mod schedule_host;
pub mod webhook;
use axum::{
Json, Router,
extract::{Path, Query, State},
http::StatusCode,
response::{
IntoResponse, Response,
sse::{Event, Sse},
},
routing::{get, post},
};
use chrono::{DateTime, Utc};
use futures::stream::Stream;
use meerkat::surface::{
RequestAdmissionError, RequestContext, RequestTerminal, RequestTerminalResolution,
SurfaceRequestExecutor, SurfaceSessionRecoveryContext, SurfaceSessionRecoveryOverrides,
build_recovered_session, noop_request_action, request_action,
run_runtime_backed_initial_turn_with_machine, split_runtime_backed_eager_create_request,
};
use meerkat::{
AgentEvent, AgentFactory, FactoryAgentBuilder, LlmClient, MachineSessionArchiveProtocol,
OutputSchema, PersistentSessionService, ScheduleService, ScheduleToolDispatcher, Session,
SessionId, SessionService, SessionServiceControlExt, SessionServiceHistoryExt,
WorkGraphService, encode_llm_client_override_for_service, handle_schedule_tools_call,
schedule_tools_list,
};
use meerkat_contracts::{
CommsSendParams, CommsSendResult, ErrorCode, RuntimeStateResult, SessionLocator, SkillsParams,
WireError, format_session_ref,
};
use meerkat_core::EventEnvelope;
#[cfg(test)]
use meerkat_core::lifecycle::core_executor::CoreApplyTerminal;
use meerkat_core::lifecycle::core_executor::{
CoreApplyOutput, CoreExecutor, CoreExecutorBoundaryHandle, CoreExecutorError,
CoreExecutorInterruptHandle, CoreExecutorPostStopCleanupHandle, CoreExecutorPublicationHandle,
};
use meerkat_core::lifecycle::run_primitive::{CoreRenderable, RunApplyBoundary, RunPrimitive};
use meerkat_core::service::{
AppendSystemContextRequest as SvcAppendSystemContextRequest,
CreateSessionRequest as SvcCreateSessionRequest, DeferredPromptPolicy, InitialTurnPolicy,
ResumeOverrideMask, SessionBuildOptions, SessionControlError, SessionError,
StartTurnRequest as SvcStartTurnRequest,
};
use meerkat_core::{
Config, ConfigDelta, ConfigEnvelope, ConfigEnvelopePolicy, ConfigStore, ContentInput,
FileConfigStore, HookRunOverrides, Provider, RealmSelection, RuntimeBootstrap,
SessionLlmIdentity, ToolCategoryOverride, agent_event_type, format_verbose_event,
};
#[cfg(feature = "mob")]
use meerkat_mob::MobSessionService as _;
use meerkat_runtime::SessionServiceRuntimeExt as _;
use meerkat_store::{RealmBackend, RealmOrigin};
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
use std::borrow::Cow;
use std::collections::{BTreeMap, HashMap};
use std::convert::Infallible;
use std::path::PathBuf;
use std::sync::{Arc, Mutex as StdMutex, Weak};
use tokio::sync::{broadcast, mpsc};
#[cfg(feature = "mcp")]
use meerkat::{
AgentToolDispatcher, McpLifecycleAction, McpLifecyclePhase, McpReloadTarget, McpRouter,
McpRouterAdapter,
};
#[cfg(feature = "mcp")]
use meerkat_core::ToolConfigChangeOperation;
#[cfg(feature = "mcp")]
use std::sync::atomic::{AtomicBool, Ordering};
#[cfg(feature = "mcp")]
use std::time::Duration;
#[cfg(feature = "mcp")]
use tokio::sync::RwLock;
#[cfg(feature = "mcp")]
pub struct SessionMcpState {
pub(crate) adapter: Arc<McpRouterAdapter>,
pub(crate) turn_counter: u32,
pub(crate) lifecycle_tx: mpsc::UnboundedSender<McpLifecycleAction>,
pub(crate) lifecycle_rx: mpsc::UnboundedReceiver<McpLifecycleAction>,
pub(crate) drain_task_running: Arc<AtomicBool>,
}
#[derive(Clone)]
pub struct AppState {
pub store_path: PathBuf,
pub max_tokens: u32,
pub rest_host: Cow<'static, str>,
pub rest_port: u16,
pub enable_builtins: bool,
pub enable_shell: bool,
pub project_root: Option<PathBuf>,
pub context_root: Option<PathBuf>,
pub user_config_root: Option<PathBuf>,
pub llm_client_override: Option<Arc<dyn LlmClient>>,
pub config_store: Arc<dyn ConfigStore>,
pub event_tx: broadcast::Sender<SessionEvent>,
pub session_service: Arc<PersistentSessionService<FactoryAgentBuilder>>,
pub schedule_service: ScheduleService,
pub workgraph_service: WorkGraphService,
pub webhook_auth: webhook::WebhookAuth,
pub realm: meerkat_core::RealmId,
pub realm_config_source: Arc<dyn meerkat_core::RealmConfigSource>,
pub instance_id: Option<String>,
pub backend: String,
pub resolved_paths: meerkat_core::ConfigResolvedPaths,
pub expose_paths: bool,
pub config_runtime: Arc<meerkat_core::ConfigRuntime>,
pub realm_lease: Arc<tokio::sync::Mutex<Option<meerkat_store::RealmLeaseGuard>>>,
pub skill_runtime: Option<Arc<meerkat_core::skills::SkillRuntime>>,
pub runtime_adapter: Arc<meerkat_runtime::MeerkatMachine>,
#[doc(hidden)]
pub runtime_pre_admissions: RestRuntimePreAdmissions,
#[doc(hidden)]
pub runtime_registration_locks: RestRuntimeRegistrationLocks,
pub schedule_host: Arc<schedule_host::ScheduleHostState>,
#[cfg(feature = "mob")]
pub mob_state: Arc<meerkat_mob_mcp::MobMcpState>,
#[cfg(feature = "mcp")]
pub mcp_sessions: Arc<RwLock<std::collections::HashMap<SessionId, SessionMcpState>>>,
pub request_executor: Arc<SurfaceRequestExecutor>,
pub provider_auth_persistence: meerkat_providers::auth_store::ProviderAuthPersistence,
pub auth_lease: meerkat_core::handles::GeneratedAuthLeaseHandle,
pub provider_registry: Arc<meerkat_providers::ProviderRuntimeRegistry>,
}
#[derive(Debug, Clone)]
pub struct SessionEvent {
session_id: SessionId,
event: EventEnvelope<AgentEvent>,
}
#[derive(Clone)]
struct RestRuntimeExecutorContext {
llm_client_override: Option<Arc<dyn LlmClient>>,
event_tx: broadcast::Sender<SessionEvent>,
session_service: Arc<PersistentSessionService<FactoryAgentBuilder>>,
workgraph_service: WorkGraphService,
realm: meerkat_core::RealmId,
instance_id: Option<String>,
backend: String,
config_runtime: Arc<meerkat_core::ConfigRuntime>,
realm_config_source: Arc<dyn meerkat_core::RealmConfigSource>,
runtime_adapter: Arc<meerkat_runtime::MeerkatMachine>,
runtime_pre_admissions: RestRuntimePreAdmissions,
}
impl RestRuntimeExecutorContext {
async fn effective_config(&self) -> Result<Config, meerkat_core::ConfigError> {
let head = self
.config_runtime
.get()
.await
.map_err(|err| meerkat_core::ConfigError::Validation(err.to_string()))?
.config;
meerkat_core::EffectiveConfigReader::new(Arc::clone(&self.realm_config_source))
.effective_config_over_head(&self.realm, head)
.await
}
}
struct RestSessionRuntimeExecutor {
context: RestRuntimeExecutorContext,
session_id: SessionId,
}
struct RestSessionRuntimeBoundaryHandle {
context: RestRuntimeExecutorContext,
session_id: SessionId,
}
#[async_trait::async_trait]
impl CoreExecutorBoundaryHandle for RestSessionRuntimeBoundaryHandle {
async fn cancel_after_boundary(
&self,
expected_run_id: &meerkat_core::lifecycle::RunId,
_reason: String,
) -> Result<(), CoreExecutorError> {
self.context
.session_service
.cancel_after_boundary_with_machine_authority(
&self.session_id,
expected_run_id,
self.context.runtime_adapter.session_control_authority(),
)
.await
.or_else(|err| match err {
SessionError::NotRunning { .. } => Ok(()),
err => Err(err),
})
.map_err(|err| CoreExecutorError::control_failed_runtime(err.to_string()))
}
async fn prepare_transient_turn_context_at_boundary(
&self,
expected_run_id: &meerkat_core::lifecycle::RunId,
contexts: Vec<meerkat_core::lifecycle::run_primitive::TurnRequestContext>,
) -> Result<
meerkat_core::lifecycle::CoreBoundaryStageOutput,
meerkat_core::lifecycle::CoreBoundaryStageError,
> {
self.context
.session_service
.prepare_live_transient_turn_context_boundary(
&self.session_id,
expected_run_id,
contexts,
)
.await
}
}
struct RestSessionRuntimeInterruptHandle {
context: RestRuntimeExecutorContext,
session_id: SessionId,
}
#[async_trait::async_trait]
impl CoreExecutorInterruptHandle for RestSessionRuntimeInterruptHandle {
async fn hard_cancel_current_run(&self, _reason: String) -> Result<(), CoreExecutorError> {
self.context
.session_service
.interrupt_with_machine_authority(
&self.session_id,
self.context.runtime_adapter.session_control_authority(),
)
.await
.or_else(|err| match err {
SessionError::NotRunning { .. } => Ok(()),
err => Err(err),
})
.map_err(|err| CoreExecutorError::control_failed_runtime(err.to_string()))
}
}
#[doc(hidden)]
pub type RestRuntimePreAdmissions =
Arc<tokio::sync::Mutex<HashMap<SessionId, Vec<RestRuntimePreAdmissionEntry>>>>;
#[doc(hidden)]
pub type RestRuntimeRegistrationLocks =
Arc<StdMutex<HashMap<SessionId, Weak<tokio::sync::Mutex<()>>>>>;
#[doc(hidden)]
pub struct RestRuntimePreAdmissionEntry {
input_id: meerkat_core::lifecycle::InputId,
admission: meerkat::RuntimeContextAdmissionGuard,
}
#[doc(hidden)]
pub fn default_rest_runtime_pre_admissions() -> RestRuntimePreAdmissions {
Arc::new(tokio::sync::Mutex::new(HashMap::new()))
}
#[doc(hidden)]
pub fn default_rest_runtime_registration_locks() -> RestRuntimeRegistrationLocks {
Arc::new(StdMutex::new(HashMap::new()))
}
struct RestRuntimePreAdmissionRegistration {
pre_admissions: RestRuntimePreAdmissions,
session_id: SessionId,
input_ids: Vec<meerkat_core::lifecycle::InputId>,
release_on_drop: bool,
}
type RestRuntimeRegistrationLockLease =
meerkat::session_runtime::admission::RuntimeRegistrationLockLease;
impl RestRuntimePreAdmissionRegistration {
fn new(
pre_admissions: RestRuntimePreAdmissions,
session_id: SessionId,
input_id: meerkat_core::lifecycle::InputId,
) -> Self {
Self {
pre_admissions,
session_id,
input_ids: vec![input_id],
release_on_drop: true,
}
}
fn track_input_id(&mut self, input_id: meerkat_core::lifecycle::InputId) {
if !self.input_ids.contains(&input_id) {
self.input_ids.push(input_id);
}
}
fn disarm(mut self) {
self.release_on_drop = false;
}
}
impl Drop for RestRuntimePreAdmissionRegistration {
fn drop(&mut self) {
if !self.release_on_drop {
return;
}
let pre_admissions = Arc::clone(&self.pre_admissions);
let session_id = self.session_id.clone();
let input_ids = self.input_ids.clone();
tokio::spawn(async move {
for input_id in input_ids {
discard_rest_runtime_pre_admission(&pre_admissions, &session_id, &input_id).await;
}
});
}
}
impl AppState {
pub async fn load() -> Result<Self, Box<dyn std::error::Error>> {
Self::load_with_bootstrap_and_options(RuntimeBootstrap::default(), false).await
}
#[cfg(test)]
async fn load_from(instance_root: PathBuf) -> Result<Self, Box<dyn std::error::Error>> {
let mut bootstrap = RuntimeBootstrap::default();
bootstrap.realm.state_root = Some(instance_root.join("realms"));
bootstrap.context.context_root = Some(instance_root.clone());
Self::load_from_with_bootstrap(instance_root, bootstrap, false).await
}
pub async fn load_with_bootstrap(
bootstrap: RuntimeBootstrap,
) -> Result<Self, Box<dyn std::error::Error>> {
Self::load_with_bootstrap_and_options(bootstrap, false).await
}
pub async fn load_with_bootstrap_and_options(
bootstrap: RuntimeBootstrap,
expose_paths: bool,
) -> Result<Self, Box<dyn std::error::Error>> {
Self::load_from_with_bootstrap(rest_instance_root(), bootstrap, expose_paths).await
}
async fn load_from_with_bootstrap(
instance_root: PathBuf,
bootstrap: RuntimeBootstrap,
expose_paths: bool,
) -> Result<Self, Box<dyn std::error::Error>> {
let (event_tx, _) = broadcast::channel(256);
let invocation_context = bootstrap
.context
.context_root
.clone()
.or_else(|| std::env::current_dir().ok())
.unwrap_or_else(|| PathBuf::from("."));
let meerkat_core::ResolvedStorage {
layout,
locator,
root_choice,
} = meerkat_core::StorageLayout::resolve(
meerkat_core::StorageLayoutInputs {
invocation_context,
explicit_state_root: bootstrap.realm.state_root.clone(),
user_config_root: bootstrap.context.user_config_root.clone(),
default_root: Some(meerkat_core::RealmRootDefault::UserGlobal),
probe_local_candidate: bootstrap.context.context_root.is_some(),
},
&bootstrap.realm,
)?;
tracing::info!(
realm = %locator.realm,
state_root = %locator.state_root.display(),
root_choice = ?root_choice,
"resolved realm storage"
);
let realm = locator.realm;
let realms_root = locator.state_root;
let instance_id = bootstrap.realm.instance_id.clone();
let backend_hint = bootstrap
.realm
.backend_hint
.as_deref()
.and_then(parse_backend_hint);
let origin_hint = Some(realm_origin_from_selection(&bootstrap.realm.selection));
let (manifest, persistence) =
meerkat::storage_provider::open_realm_persistence_with_layout(
layout.clone(),
realm.as_str(),
backend_hint,
origin_hint,
)
.await?;
let session_store = persistence.session_store();
let schedule_service = ScheduleService::new(persistence.schedule_store());
let workgraph_service = WorkGraphService::with_scope(
persistence.workgraph_store(),
realm.to_string(),
meerkat::WorkNamespace::default(),
);
let realm_paths = meerkat_store::realm_paths_in(&realms_root, realm.as_str());
let global_doc = layout
.global_config_path()
.unwrap_or_else(|| realms_root.join("__no_global__").join("config.toml"));
let fs_realm_config_source = meerkat_store::FilesystemRealmConfigSource::new(
realms_root.clone(),
global_doc,
meerkat_models::canonical(),
);
let head_config_doc = fs_realm_config_source.config_doc_path(&realm);
let realm_config_source: Arc<dyn meerkat_core::RealmConfigSource> =
Arc::new(fs_realm_config_source);
let resolved_paths = meerkat_core::ConfigResolvedPaths {
root: realm_paths.root.display().to_string(),
manifest_path: realm_paths.manifest_path.display().to_string(),
config_path: head_config_doc.display().to_string(),
sessions_sqlite_path: Some(realm_paths.sessions_sqlite_path.display().to_string()),
sessions_jsonl_dir: realm_paths.sessions_jsonl_dir.display().to_string(),
};
let base_config_store: Arc<dyn ConfigStore> = Arc::new(FileConfigStore::new(
head_config_doc,
meerkat_models::canonical(),
));
let config_store: Arc<dyn ConfigStore> = Arc::new(meerkat_core::TaggedConfigStore::new(
base_config_store,
meerkat_core::ConfigStoreMetadata {
realm_id: Some(realm.to_string()),
instance_id: instance_id.clone(),
backend: Some(manifest.backend.as_str().to_string()),
resolved_paths: Some(resolved_paths.clone()),
},
));
let config_runtime = Arc::new(meerkat_core::ConfigRuntime::new(
Arc::clone(&config_store),
realm_paths.root.join("config_state.json"),
));
let lease = meerkat_store::start_realm_lease_in(
&realms_root,
realm.as_str(),
instance_id.as_deref(),
"rkat-rest",
)
.await?;
let head_config = config_store.get().await.map_err(|err| {
Box::new(std::io::Error::new(
std::io::ErrorKind::InvalidData,
format!("failed to read head realm config: {err}"),
)) as Box<dyn std::error::Error>
})?;
let mut config = meerkat_core::EffectiveConfigReader::new(Arc::clone(&realm_config_source))
.effective_config_over_head(&realm, head_config)
.await
.map_err(|err| {
Box::new(std::io::Error::new(
std::io::ErrorKind::InvalidData,
format!("failed to compose effective config for realm '{realm}': {err}"),
)) as Box<dyn std::error::Error>
})?;
if let Err(err) = config.apply_env_overrides() {
tracing::warn!("Failed to apply env overrides: {}", err);
}
if let Err(err) = config.validate(meerkat_models::canonical()) {
return Err(Box::new(std::io::Error::new(
std::io::ErrorKind::InvalidData,
format!("Invalid config: {err}"),
)));
}
let store_path = persistence
.store_path()
.map(std::path::Path::to_path_buf)
.unwrap_or_else(|| match manifest.backend {
meerkat_store::RealmBackend::Jsonl => realm_paths.sessions_jsonl_dir.clone(),
meerkat_store::RealmBackend::Memory => realm_paths.root.clone(),
meerkat_store::RealmBackend::Sqlite => realm_paths.root.clone(),
});
let enable_builtins = config.tools.builtins_enabled;
let enable_shell = config.tools.shell_enabled;
let max_tokens = config.agent.resolved_max_tokens_per_turn();
let rest_host = Cow::Owned(config.rest.host.clone());
let rest_port = config.rest.port;
let provider_auth_persistence =
meerkat_providers::auth_store::TokenStoreBackend::default_auto()?
.open_with_refresh_authority()?;
let mut factory = AgentFactory::new(store_path.clone())
.with_provider_auth_persistence(provider_auth_persistence.clone())
.session_store(session_store.clone())
.runtime_root(realm_paths.root.clone())
.builtins(enable_builtins)
.shell(enable_shell)
.workgraph(config.tools.workgraph_enabled)
.schedule(config.tools.schedule_enabled);
let conventions_context_root = bootstrap.context.context_root.clone();
let conventions_user_root = bootstrap.context.user_config_root.clone();
let task_project_root = conventions_context_root
.clone()
.unwrap_or_else(|| instance_root.clone());
factory = factory.project_root(task_project_root.clone());
if let Some(context_root) = conventions_context_root {
factory = factory.context_root(context_root);
}
if let Some(user_root) = bootstrap.context.user_config_root.clone() {
factory = factory.user_config_root(user_root);
}
let skill_runtime = factory.build_skill_runtime(&config).await?;
let provider_registry = factory.provider_runtime_registry();
let max_sessions = config.max_sessions();
#[cfg(feature = "mob")]
let mob_host_config = config.mob_host.clone();
let builder =
FactoryAgentBuilder::new_with_config_store(factory, config, Arc::clone(&config_store))
.with_realm_inheritance(Arc::clone(&realm_config_source), realm.clone());
#[cfg(feature = "mob")]
let mob_tools_slot = Arc::clone(&builder.default_mob_tools);
meerkat::surface::set_default_schedule_tools(
&builder,
Some(Arc::new(ScheduleToolDispatcher::new(
schedule_service.clone(),
))),
);
meerkat::surface::set_default_workgraph_tools(
&builder,
Some(Arc::new(meerkat::WorkGraphToolSurface::new(
workgraph_service.clone(),
))),
);
let (session_service, runtime_adapter) =
meerkat::surface::build_runtime_backed_service(builder, max_sessions, persistence);
let auth_lease = runtime_adapter.generated_auth_lease_handle();
let session_service = Arc::new(session_service);
#[cfg(feature = "mob")]
let mob_session_service = session_service.clone();
Ok(Self {
store_path,
max_tokens,
rest_host,
rest_port,
enable_builtins,
enable_shell,
project_root: Some(task_project_root),
context_root: bootstrap.context.context_root.clone(),
user_config_root: conventions_user_root,
llm_client_override: None,
config_store,
event_tx,
session_service,
schedule_service,
workgraph_service: workgraph_service.clone(),
webhook_auth: webhook::WebhookAuth::from_env(),
realm,
realm_config_source,
instance_id,
backend: manifest.backend.as_str().to_string(),
resolved_paths,
expose_paths,
config_runtime,
realm_lease: Arc::new(tokio::sync::Mutex::new(Some(lease))),
skill_runtime,
runtime_adapter,
runtime_pre_admissions: Arc::new(tokio::sync::Mutex::new(HashMap::new())),
runtime_registration_locks: Arc::new(StdMutex::new(HashMap::new())),
schedule_host: Arc::new(schedule_host::ScheduleHostState::default()),
#[cfg(feature = "mob")]
mob_state: {
let acceptor_service: Arc<dyn meerkat_mob::MobSessionService> =
mob_session_service.clone();
let controlling_acceptor =
meerkat_mob::ControllingAcceptorConfig::from_mob_host_config(
&mob_host_config,
acceptor_service,
)
.map_err(|detail| {
std::io::Error::new(std::io::ErrorKind::InvalidInput, detail)
})?;
let mut state = meerkat_mob_mcp::MobMcpState::new(
mob_session_service,
meerkat_mob::MobControlPrincipal::Owner,
)
.with_persistent_storage_root(Some(realm_paths.root.clone()))
.with_workgraph_service(Some(workgraph_service.clone()));
if let Some(acceptor) = controlling_acceptor {
state = state.with_controlling_acceptor(acceptor);
}
let state = Arc::new(state);
*mob_tools_slot
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Arc::new(
meerkat_mob_mcp::AgentMobToolSurfaceFactory::new(Arc::clone(&state)),
));
state
},
#[cfg(feature = "mcp")]
mcp_sessions: Arc::new(RwLock::new(std::collections::HashMap::new())),
request_executor: Arc::new(SurfaceRequestExecutor::new(
std::time::Duration::from_secs(5),
)),
provider_auth_persistence,
auth_lease,
provider_registry,
})
}
pub fn oauth_flow_authority(
&self,
) -> Arc<dyn meerkat_providers::oauth_flow::OAuthFlowAuthority> {
self.runtime_adapter.oauth_flow_authority()
}
pub fn token_store(&self) -> Arc<dyn meerkat_providers::auth_store::TokenStore> {
self.provider_auth_persistence.token_store()
}
fn runtime_executor_context(&self) -> RestRuntimeExecutorContext {
RestRuntimeExecutorContext {
llm_client_override: self.llm_client_override.clone(),
event_tx: self.event_tx.clone(),
session_service: self.session_service.clone(),
workgraph_service: self.workgraph_service.clone(),
realm: self.realm.clone(),
instance_id: self.instance_id.clone(),
backend: self.backend.clone(),
config_runtime: self.config_runtime.clone(),
realm_config_source: Arc::clone(&self.realm_config_source),
runtime_adapter: self.runtime_adapter.clone(),
runtime_pre_admissions: self.runtime_pre_admissions.clone(),
}
}
}
impl RestSessionRuntimeExecutor {
fn new(context: RestRuntimeExecutorContext, session_id: SessionId) -> Self {
Self {
context,
session_id,
}
}
}
async fn ensure_rest_session_runtime_executor(
state: &AppState,
session_id: &SessionId,
) -> Result<(), meerkat_runtime::RuntimeDriverError> {
let executor = Box::new(RestSessionRuntimeExecutor::new(
state.runtime_executor_context(),
session_id.clone(),
));
state
.runtime_adapter
.ensure_session_with_executor(session_id.clone(), executor)
.await
}
async fn prepare_rest_session_runtime_executor_locked(
state: &AppState,
session_id: &SessionId,
input_id: &meerkat_core::lifecycle::InputId,
) -> Result<(), meerkat_runtime::RuntimeDriverError> {
if !state
.session_service
.has_live_session(session_id)
.await
.map_err(runtime_driver_error_from_session_error)?
{
let admission = take_rest_runtime_pre_admission(
&state.runtime_pre_admissions,
session_id,
std::slice::from_ref(input_id),
)
.await
.ok_or_else(|| {
meerkat_runtime::RuntimeDriverError::Internal(format!(
"persisted-only REST runtime preparation lost admission for input {input_id}"
))
})?;
let session = state
.session_service
.load_authoritative_session(session_id)
.await
.map_err(runtime_driver_error_from_session_error)?
.ok_or(meerkat_runtime::RuntimeDriverError::NotReady {
state: meerkat_runtime::RuntimeState::Destroyed,
})?;
if state
.session_service
.session_archived_by_authority(session_id, &session)
.await
.map_err(runtime_driver_error_from_session_error)?
{
return Err(meerkat_runtime::RuntimeDriverError::NotReady {
state: meerkat_runtime::RuntimeState::Retired,
});
}
let bindings = state
.runtime_adapter
.prepare_bindings(session_id.clone())
.await
.map_err(|error| {
meerkat_runtime::RuntimeDriverError::Internal(format!(
"failed to prepare runtime bindings for session {session_id}: {error}"
))
})?;
let recovered = build_recovered_session(
session,
&SurfaceSessionRecoveryOverrides::default(),
SurfaceSessionRecoveryContext {
llm_client_override: state
.llm_client_override
.clone()
.map(encode_llm_client_override_for_service),
runtime_build_mode: meerkat_core::RuntimeBuildMode::SessionOwned(bindings),
realm_id: Some(state.realm.clone()),
instance_id: state.instance_id.clone(),
backend: Some(state.backend.clone()),
config_generation: state.config_runtime.get().await.ok().map(|s| s.generation),
..Default::default()
},
)
.map_err(|error| meerkat_runtime::RuntimeDriverError::Internal(error.to_string()))?;
state
.session_service
.create_session_with_reserved_admission(
recovered.into_deferred_create_request(),
admission,
)
.await
.map_err(runtime_driver_error_from_session_error)?;
}
ensure_rest_session_runtime_executor(state, session_id).await
}
fn runtime_driver_error_from_session_error(
error: SessionError,
) -> meerkat_runtime::RuntimeDriverError {
meerkat_runtime::RuntimeDriverError::Internal(error.to_string())
}
fn runtime_executor_attach_error_to_api(error: meerkat_runtime::RuntimeDriverError) -> ApiError {
ApiError::Internal(format!("failed to attach REST runtime executor: {error}"))
}
fn runtime_driver_error_to_session_error(
error: meerkat_runtime::RuntimeDriverError,
) -> SessionError {
SessionError::Agent(meerkat_core::AgentError::InternalError(error.to_string()))
}
async fn unregister_rest_runtime_if_new(
state: &AppState,
session_id: &SessionId,
runtime_was_registered: bool,
) -> Result<(), meerkat_runtime::RuntimeDriverError> {
let lock = rest_runtime_registration_lock(state, session_id);
let _guard = lock.mutex().lock().await;
unregister_rest_runtime_if_new_idle_locked(state, session_id, runtime_was_registered).await
}
async fn discard_rebuilt_rest_session(
state: &AppState,
session_id: &SessionId,
runtime_was_registered: bool,
) -> Result<(), SessionError> {
let lock = rest_runtime_registration_lock(state, session_id);
let _guard = lock.mutex().lock().await;
discard_rebuilt_rest_session_locked(state, session_id, runtime_was_registered).await
}
#[cfg(test)]
async fn unregister_runtime_adapter_if_new(
adapter: &meerkat_runtime::MeerkatMachine,
session_id: &SessionId,
runtime_was_registered: bool,
) -> Result<(), meerkat_runtime::RuntimeDriverError> {
if !runtime_was_registered && adapter.contains_session(session_id).await {
adapter.unregister_session(session_id).await?;
}
Ok(())
}
fn runtime_cleanup_failure_message(
primary: impl std::fmt::Display,
cleanup: impl std::fmt::Display,
session_id: &SessionId,
operation: &str,
) -> String {
format!("{primary}; additionally failed to {operation} for session {session_id}: {cleanup}")
}
fn combine_runtime_cleanup_errors(
primary: meerkat_runtime::RuntimeDriverError,
cleanup: meerkat_runtime::RuntimeDriverError,
session_id: &SessionId,
operation: &str,
) -> meerkat_runtime::RuntimeDriverError {
meerkat_runtime::RuntimeDriverError::Internal(runtime_cleanup_failure_message(
primary, cleanup, session_id, operation,
))
}
#[cfg(test)]
fn combine_session_error_with_runtime_cleanup(
primary: SessionError,
cleanup: meerkat_runtime::RuntimeDriverError,
session_id: &SessionId,
operation: &str,
) -> SessionError {
SessionError::Agent(meerkat_core::AgentError::InternalError(
runtime_cleanup_failure_message(primary, cleanup, session_id, operation),
))
}
#[cfg(test)]
async fn unregister_runtime_adapter_if_new_after_session_error(
adapter: &meerkat_runtime::MeerkatMachine,
session_id: &SessionId,
runtime_was_registered: bool,
primary: SessionError,
) -> SessionError {
match unregister_runtime_adapter_if_new(adapter, session_id, runtime_was_registered).await {
Ok(()) => primary,
Err(cleanup) => combine_session_error_with_runtime_cleanup(
primary,
cleanup,
session_id,
"unregister newly prepared REST runtime",
),
}
}
fn rest_runtime_registration_lock(
state: &AppState,
session_id: &SessionId,
) -> RestRuntimeRegistrationLockLease {
let lock = {
let mut locks = state
.runtime_registration_locks
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if let Some(lock) = locks.get(session_id).and_then(Weak::upgrade) {
lock
} else {
let lock = Arc::new(tokio::sync::Mutex::new(()));
locks.insert(session_id.clone(), Arc::downgrade(&lock));
lock
}
};
RestRuntimeRegistrationLockLease {
locks: Arc::clone(&state.runtime_registration_locks),
session_id: session_id.clone(),
lock,
}
}
async fn unregister_rest_runtime_if_new_idle_locked(
state: &AppState,
session_id: &SessionId,
runtime_was_registered: bool,
) -> Result<(), meerkat_runtime::RuntimeDriverError> {
if runtime_was_registered {
return Ok(());
}
if !state.runtime_adapter.contains_session(session_id).await {
return Ok(());
}
if !state
.runtime_adapter
.list_active_inputs(session_id)
.await?
.is_empty()
{
return Ok(());
}
if state
.runtime_pre_admissions
.lock()
.await
.contains_key(session_id)
{
return Ok(());
}
state.runtime_adapter.unregister_session(session_id).await
}
async fn unregister_rest_runtime_after_api_error_locked(
state: &AppState,
session_id: &SessionId,
runtime_was_registered: bool,
primary: ApiError,
) -> ApiError {
match unregister_rest_runtime_if_new_idle_locked(state, session_id, runtime_was_registered)
.await
{
Ok(()) => primary,
Err(cleanup) => ApiError::Internal(runtime_cleanup_failure_message(
api_error_message(&primary),
cleanup,
session_id,
"unregister newly prepared REST runtime",
)),
}
}
async fn discard_rebuilt_rest_session_locked(
state: &AppState,
session_id: &SessionId,
runtime_was_registered: bool,
) -> Result<(), SessionError> {
let has_active_inputs = if state.runtime_adapter.contains_session(session_id).await {
!state
.runtime_adapter
.list_active_inputs(session_id)
.await
.map_err(runtime_driver_error_to_session_error)?
.is_empty()
} else {
false
};
if !has_active_inputs {
match state.session_service.discard_live_session(session_id).await {
Ok(()) | Err(SessionError::NotFound { .. }) => {}
Err(error) => return Err(error),
}
}
unregister_rest_runtime_if_new_idle_locked(state, session_id, runtime_was_registered)
.await
.map_err(runtime_driver_error_to_session_error)
}
async fn discard_rebuilt_rest_session_after_api_error_locked(
state: &AppState,
session_id: &SessionId,
runtime_was_registered: bool,
primary: ApiError,
) -> ApiError {
match discard_rebuilt_rest_session_locked(state, session_id, runtime_was_registered).await {
Ok(()) => primary,
Err(cleanup) => ApiError::Internal(runtime_cleanup_failure_message(
api_error_message(&primary),
cleanup,
session_id,
"discard rebuilt REST session and unregister its runtime",
)),
}
}
async fn cleanup_rest_session_after_api_error(
state: &AppState,
session_id: &SessionId,
primary: ApiError,
operation: &str,
) -> ApiError {
match cleanup_archived_session_runtime(state, session_id).await {
Ok(()) => primary,
Err(cleanup) => ApiError::Internal(runtime_cleanup_failure_message(
api_error_message(&primary),
cleanup,
session_id,
operation,
)),
}
}
async fn insert_rest_runtime_pre_admission(
pre_admissions: &RestRuntimePreAdmissions,
session_id: SessionId,
input_id: meerkat_core::lifecycle::InputId,
admission: meerkat::RuntimeContextAdmissionGuard,
) -> Result<(), SessionError> {
let mut pre_admissions = pre_admissions.lock().await;
let entries = pre_admissions.entry(session_id.clone()).or_default();
if entries.iter().any(|entry| entry.input_id == input_id) {
return Err(SessionError::Busy { id: session_id });
}
entries.push(RestRuntimePreAdmissionEntry {
input_id,
admission,
});
Ok(())
}
async fn take_rest_runtime_pre_admission(
pre_admissions: &RestRuntimePreAdmissions,
session_id: &SessionId,
input_ids: &[meerkat_core::lifecycle::InputId],
) -> Option<meerkat::RuntimeContextAdmissionGuard> {
if input_ids.is_empty() {
return None;
}
let mut pre_admissions = pre_admissions.lock().await;
let entries = pre_admissions.get_mut(session_id)?;
let index = entries
.iter()
.position(|entry| input_ids.contains(&entry.input_id))?;
let entry = entries.remove(index);
if entries.is_empty() {
pre_admissions.remove(session_id);
}
Some(entry.admission)
}
async fn discard_rest_runtime_pre_admission(
pre_admissions: &RestRuntimePreAdmissions,
session_id: &SessionId,
input_id: &meerkat_core::lifecycle::InputId,
) {
let mut pre_admissions = pre_admissions.lock().await;
let Some(entries) = pre_admissions.get_mut(session_id) else {
return;
};
if let Some(index) = entries.iter().position(|entry| &entry.input_id == input_id) {
entries.remove(index);
}
if entries.is_empty() {
pre_admissions.remove(session_id);
}
}
fn spawn_rest_runtime_pre_admission_rekey_and_cleanup(
state: AppState,
session_id: SessionId,
from_input_id: meerkat_core::lifecycle::InputId,
to_input_id: meerkat_core::lifecycle::InputId,
handle: meerkat_runtime::completion::CompletionHandle,
) {
tokio::spawn(async move {
rekey_rest_runtime_pre_admission(
&state.runtime_pre_admissions,
&session_id,
&from_input_id,
to_input_id.clone(),
)
.await;
let completion = handle.try_wait_with_cleanup_observation().await;
match completion {
Ok((_outcome, cleanup_observation)) => {
match cleanup_rest_runtime_after_completion_outcome(
&state,
&session_id,
cleanup_observation,
)
.await
{
Ok(true) => {
discard_rest_runtime_pre_admission(
&state.runtime_pre_admissions,
&session_id,
&from_input_id,
)
.await;
discard_rest_runtime_pre_admission(
&state.runtime_pre_admissions,
&session_id,
&to_input_id,
)
.await;
}
Ok(false) => {}
Err(error) => {
tracing::error!(
session_id = %session_id,
error = %error,
"REST runtime completion cleanup failed after background waiter completion"
);
}
}
}
Err(error) => {
if rest_runtime_wait_failure_releases_pre_admission(&state, &session_id, &error)
.await
{
discard_rest_runtime_pre_admission(
&state.runtime_pre_admissions,
&session_id,
&from_input_id,
)
.await;
discard_rest_runtime_pre_admission(
&state.runtime_pre_admissions,
&session_id,
&to_input_id,
)
.await;
}
tracing::warn!(
session_id = %session_id,
error = %error,
"REST runtime completion waiter failed before completion cleanup"
);
}
}
});
}
fn wrap_rest_runtime_completion_cleanup(
state: AppState,
session_id: SessionId,
handle: meerkat_runtime::completion::CompletionHandle,
) -> meerkat_runtime::completion::CompletionHandle {
handle.with_resultful_completion_cleanup(move |completion| async move {
let Ok(cleanup_observation) = completion else {
return Ok(());
};
cleanup_rest_runtime_after_completion_outcome(&state, &session_id, cleanup_observation)
.await
.map(|_| ())
.map_err(|error| {
meerkat_runtime::completion::CompletionWaitError::AuthorityUnavailable(format!(
"REST runtime completion cleanup failed for {session_id}: {error}"
))
})
})
}
fn rest_completion_cleanup_error(
session_id: &SessionId,
error: impl std::fmt::Display,
) -> meerkat_runtime::completion::CompletionWaitError {
meerkat_runtime::completion::CompletionWaitError::AuthorityUnavailable(format!(
"REST runtime completion cleanup failed for {session_id}: {error}"
))
}
async fn rest_runtime_wait_failure_releases_pre_admission_result(
state: &AppState,
session_id: &SessionId,
error: &meerkat_runtime::completion::CompletionWaitError,
) -> Result<bool, meerkat_runtime::completion::CompletionWaitError> {
state
.runtime_adapter
.resolve_runtime_completion_wait_failure(session_id, error)
.await
.map(|authority| authority.releases_pre_admission())
.map_err(|authority_error| {
meerkat_runtime::completion::CompletionWaitError::AuthorityUnavailable(format!(
"{error}; REST runtime wait-failure cleanup authority unavailable for {session_id}: {authority_error}"
))
})
}
async fn cleanup_rest_runtime_after_completion_outcome(
state: &AppState,
session_id: &SessionId,
cleanup_observation: meerkat_runtime::completion::CompletionCleanupObservation,
) -> Result<bool, SessionError> {
let runtime_registration_lock = rest_runtime_registration_lock(state, session_id);
let _runtime_registration_guard = runtime_registration_lock.mutex().lock().await;
let archived_now = match state
.session_service
.load_authoritative_session(session_id)
.await?
{
Some(session) => session_archived_by_authority(state, session_id, &session).await?,
None => false,
};
let live_session = if archived_now {
meerkat_runtime::meerkat_machine::dsl::RuntimeCompletionLiveSessionObservation::NotObserved
} else if state.session_service.has_live_session(session_id).await? {
meerkat_runtime::meerkat_machine::dsl::RuntimeCompletionLiveSessionObservation::Present
} else {
meerkat_runtime::meerkat_machine::dsl::RuntimeCompletionLiveSessionObservation::Absent
};
let runtime_termination_cleanup = cleanup_observation
.proves_runtime_termination_for(session_id)
&& matches!(
live_session,
meerkat_runtime::meerkat_machine::dsl::RuntimeCompletionLiveSessionObservation::Absent
);
let runtime_termination_proof = cleanup_observation.clone();
let cleanup_authority = match state
.runtime_adapter
.resolve_runtime_completion_cleanup(
session_id,
cleanup_observation,
archived_now,
live_session,
)
.await
{
Ok(authority) => authority,
Err(_error)
if archived_now && !state.runtime_adapter.contains_session(session_id).await =>
{
return Ok(true);
}
Err(_error)
if runtime_termination_cleanup
&& !state.runtime_adapter.contains_session(session_id).await =>
{
cleanup_rest_runtime_after_verified_termination(
state,
session_id,
&runtime_termination_proof,
)
.await?;
return Ok(true);
}
Err(error) => {
return Err(SessionError::Unsupported(format!(
"runtime completion cleanup authority failed: {error}"
)));
}
};
if cleanup_authority.requires_runtime_cleanup() {
match state.session_service.discard_live_session(session_id).await {
Ok(()) | Err(SessionError::NotFound { .. }) => {}
Err(error) => return Err(error),
}
if runtime_termination_cleanup && !state.runtime_adapter.contains_session(session_id).await
{
cleanup_rest_runtime_after_verified_termination(
state,
session_id,
&runtime_termination_proof,
)
.await?;
} else {
cleanup_archived_session_runtime(state, session_id).await?;
}
}
Ok(cleanup_authority.releases_pre_admission())
}
async fn rest_runtime_wait_failure_releases_pre_admission(
state: &AppState,
session_id: &SessionId,
error: &meerkat_runtime::completion::CompletionWaitError,
) -> bool {
match rest_runtime_wait_failure_releases_pre_admission_result(state, session_id, error).await {
Ok(release) => release,
Err(authority_error) => {
tracing::warn!(
session_id = %session_id,
error = %error,
authority_error = %authority_error,
"REST runtime pre-admission retained because generated wait-failure authority was unavailable"
);
false
}
}
}
async fn rekey_rest_runtime_pre_admission(
pre_admissions: &RestRuntimePreAdmissions,
session_id: &SessionId,
from_input_id: &meerkat_core::lifecycle::InputId,
to_input_id: meerkat_core::lifecycle::InputId,
) {
if from_input_id == &to_input_id {
return;
}
let mut pre_admissions = pre_admissions.lock().await;
if let Some(entries) = pre_admissions.get_mut(session_id)
&& let Some(entry) = entries
.iter_mut()
.find(|entry| &entry.input_id == from_input_id)
{
entry.input_id = to_input_id;
}
}
async fn require_rest_session_exists_for_read(
state: &AppState,
session_id: &SessionId,
) -> Result<(), Response> {
state
.session_service
.read(session_id)
.await
.map(|_| ())
.map_err(|err| {
(
StatusCode::NOT_FOUND,
Json(json!({"error": err.to_string()})),
)
.into_response()
})
}
fn resolve_validation_identity(
config: &Config,
model: &str,
provider: Option<meerkat_core::Provider>,
) -> Result<SessionLlmIdentity, String> {
let registry = config.model_registry(meerkat_models::canonical()).ok();
let entry = registry.as_ref().and_then(|registry| registry.entry(model));
if let (Some(registry), Some(provider)) = (registry.as_ref(), provider)
&& let Some(reason) = registry.provider_override_mismatch_reason(provider, model)
{
return Err(reason);
}
let provider = provider
.or_else(|| entry.map(|entry| entry.provider))
.ok_or_else(|| {
format!("model '{model}' requires an explicit provider or a registered model owner")
})?;
let self_hosted_server_id = if provider == meerkat_core::Provider::SelfHosted {
Some(
entry
.and_then(|entry| entry.self_hosted.as_ref())
.map(|server| server.server_id.clone())
.ok_or_else(|| {
format!("self-hosted model '{model}' requires a registered self-hosted alias")
})?,
)
} else {
None
};
Ok(SessionLlmIdentity {
model: model.to_string(),
provider,
self_hosted_server_id,
provider_params: None,
auth_binding: None,
})
}
async fn resolve_default_model(state: &AppState) -> Result<String, ApiError> {
let config = effective_config_for_state(state).await.map_err(|err| {
ApiError::Internal(format!("failed to compose realm config chain: {err}"))
})?;
resolve_rest_create_session_model(&config, None, None, None)
.map(|resolution| resolution.model)
.map_err(|err| ApiError::Configuration(err.to_string()))
}
fn resolve_rest_create_session_model(
config: &Config,
model: Option<String>,
provider: Option<Provider>,
auth_binding: Option<meerkat_core::AuthBindingRef>,
) -> Result<meerkat::CreateSessionModelResolution, meerkat::CreateSessionModelResolutionError> {
meerkat::resolve_create_session_model(
config,
meerkat::CreateSessionModelResolutionRequest {
model,
provider,
auth_binding,
},
)
}
fn create_session_model_resolution_error_to_api(
error: meerkat::CreateSessionModelResolutionError,
) -> ApiError {
if error.is_configuration_fault() {
ApiError::Configuration(error.to_string())
} else {
ApiError::BadRequest(error.to_string())
}
}
async fn resolve_rest_create_session_model_for_state(
state: &AppState,
model: Option<String>,
provider: Option<Provider>,
auth_binding: Option<meerkat_core::AuthBindingRef>,
) -> Result<meerkat::CreateSessionModelResolution, ApiError> {
let config = effective_config_for_state(state).await.map_err(|err| {
ApiError::Internal(format!("failed to compose realm config chain: {err}"))
})?;
resolve_rest_create_session_model(&config, model, provider, auth_binding)
.map_err(create_session_model_resolution_error_to_api)
}
#[derive(Debug, Clone)]
enum PromptVideoInputValidationError {
InvalidBlock(String),
UnsupportedCapability(meerkat_core::UnsupportedModelCapabilityEvidence),
}
impl PromptVideoInputValidationError {
#[cfg(test)]
fn unsupported_evidence(&self) -> Option<&meerkat_core::UnsupportedModelCapabilityEvidence> {
match self {
Self::UnsupportedCapability(evidence) => Some(evidence),
Self::InvalidBlock(_) => None,
}
}
}
impl std::fmt::Display for PromptVideoInputValidationError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::InvalidBlock(message) => f.write_str(message),
Self::UnsupportedCapability(evidence) => evidence.fmt(f),
}
}
}
fn inline_video_registry_unavailable_evidence(
identity: &SessionLlmIdentity,
) -> meerkat_core::UnsupportedModelCapabilityEvidence {
meerkat_core::UnsupportedModelCapabilityEvidence::inline_video(
identity.provider,
identity.model.clone(),
meerkat_core::UnsupportedModelCapabilityReason::CapabilityRegistryUnavailable,
)
}
fn require_inline_video_support(
config: &Config,
identity: &SessionLlmIdentity,
) -> Result<(), meerkat_core::UnsupportedModelCapabilityEvidence> {
let Ok(registry) = config.model_registry(meerkat_models::canonical()) else {
return Err(inline_video_registry_unavailable_evidence(identity));
};
registry.require_inline_video_for_provider(identity.provider, &identity.model)
}
fn validate_prompt_video_input(
config: &Config,
prompt: &ContentInput,
identity: &SessionLlmIdentity,
) -> Result<(), PromptVideoInputValidationError> {
let blocks = match prompt {
ContentInput::Text(_) => return Ok(()),
ContentInput::Blocks(blocks) => blocks,
};
meerkat_core::validate_inline_video_blocks(blocks)
.map_err(PromptVideoInputValidationError::InvalidBlock)?;
if meerkat_core::has_video(blocks) {
require_inline_video_support(config, identity)
.map_err(PromptVideoInputValidationError::UnsupportedCapability)?;
}
Ok(())
}
async fn resolve_validation_identity_for_state(
state: &AppState,
model: &str,
provider: Option<meerkat_core::Provider>,
) -> Result<SessionLlmIdentity, String> {
let config = effective_config_for_state(state)
.await
.map_err(|err| err.to_string())?;
resolve_validation_identity(&config, model, provider)
}
async fn validate_prompt_video_input_for_state(
state: &AppState,
prompt: &ContentInput,
identity: &SessionLlmIdentity,
) -> Result<(), PromptVideoInputValidationError> {
let config = effective_config_for_state(state).await.map_err(|err| {
PromptVideoInputValidationError::InvalidBlock(format!(
"failed to compose realm config chain: {err}"
))
})?;
validate_prompt_video_input(&config, prompt, identity)
}
fn prompt_video_input_error_to_api(error: PromptVideoInputValidationError) -> ApiError {
match error {
PromptVideoInputValidationError::InvalidBlock(message) => ApiError::BadRequest(message),
PromptVideoInputValidationError::UnsupportedCapability(evidence) => {
ApiError::BadRequestWithData {
message: evidence.to_string(),
code: "UNSUPPORTED_MODEL_CAPABILITY".to_string(),
details: evidence.details(),
}
}
}
}
async fn apply_runtime_turn_under_runtime_turn_boundary(
context: &RestRuntimeExecutorContext,
session_id: &SessionId,
run_id: meerkat_core::lifecycle::RunId,
primitive: &RunPrimitive,
prompt: ContentInput,
) -> Result<CoreApplyOutput, SessionError> {
if let Some(reason) = primitive.peer_response_terminal_apply_intent_violation() {
return Err(SessionError::Agent(
meerkat_core::error::AgentError::InternalError(reason.to_string()),
));
}
let archived_now = match context
.session_service
.load_authoritative_session(session_id)
.await?
{
Some(session) => {
context
.session_service
.session_archived_by_authority(session_id, &session)
.await?
}
None => false,
};
if archived_now {
return Err(SessionError::NotFound {
id: session_id.clone(),
});
}
let (event_tx, event_rx) = mpsc::channel::<EventEnvelope<AgentEvent>>(100);
let forwarder = spawn_event_forwarder(
event_rx,
context.event_tx.clone(),
session_id.clone(),
false,
);
let typed_turn_appends = primitive.typed_turn_appends();
let mut svc_req = SvcStartTurnRequest {
injected_context: Vec::new(),
prompt: prompt.clone(),
system_prompt: None,
event_tx: Some(event_tx.clone()),
runtime: meerkat_core::service::StartTurnRuntimeSemantics::new(
meerkat_core::types::HandlingMode::Queue,
primitive
.turn_metadata()
.and_then(|meta| meta.turn_tool_overlay.clone()),
primitive.turn_metadata().cloned(),
)
.with_typed_turn_appends(typed_turn_appends.clone()),
};
meerkat::surface::inject_workgraph_attention_turn_overlay(
context.session_service.as_ref(),
Some(&context.workgraph_service),
session_id,
&mut svc_req,
)
.await
.map_err(|error| {
SessionError::Agent(meerkat_core::AgentError::InternalError(error.to_string()))
})?;
let session_identity = context
.session_service
.load_authoritative_session(session_id)
.await
.ok()
.flatten()
.and_then(|session| {
session
.session_metadata()
.map(|metadata| metadata.llm_identity())
});
if let Some(identity) = session_identity {
let effective = context.effective_config().await.map_err(|err| {
SessionError::Agent(meerkat_core::AgentError::ConfigError(err.to_string()))
})?;
if let Err(error) = validate_prompt_video_input(&effective, &prompt, &identity) {
return Err(SessionError::Agent(meerkat_core::AgentError::ConfigError(
error.to_string(),
)));
}
}
let boundary = match primitive {
RunPrimitive::StagedInput(staged) => staged.boundary,
_ => RunApplyBoundary::Immediate,
};
let contributing_input_ids = primitive.contributing_input_ids().to_vec();
let mut pre_admission = take_rest_runtime_pre_admission(
&context.runtime_pre_admissions,
session_id,
&contributing_input_ids,
)
.await;
let apply_result = if let Some(admission) = pre_admission.take() {
match context
.session_service
.apply_runtime_turn_with_recoverable_reserved_admission(
session_id,
run_id.clone(),
svc_req,
boundary,
contributing_input_ids.clone(),
admission,
)
.await
{
Ok(output) => Ok(output),
Err((error, admission)) => {
drop(admission);
Err(error)
}
}
} else {
context
.session_service
.apply_runtime_turn(
session_id,
run_id.clone(),
svc_req,
boundary,
contributing_input_ids.clone(),
)
.await
};
let result = apply_result;
drop(event_tx);
drain_event_forwarder(session_id, forwarder).await;
result
}
#[cfg(test)]
async fn apply_runtime_turn(
context: &RestRuntimeExecutorContext,
session_id: &SessionId,
run_id: meerkat_core::lifecycle::RunId,
primitive: &RunPrimitive,
prompt: ContentInput,
) -> Result<CoreApplyOutput, SessionError> {
let _turn_finalization_guard = context
.session_service
.acquire_runtime_turn_finalization_guard(session_id)
.await;
apply_runtime_turn_under_runtime_turn_boundary(context, session_id, run_id, primitive, prompt)
.await
}
#[async_trait::async_trait]
impl CoreExecutor for RestSessionRuntimeExecutor {
fn boundary_handle(&self) -> Option<Arc<dyn CoreExecutorBoundaryHandle>> {
Some(Arc::new(RestSessionRuntimeBoundaryHandle {
context: self.context.clone(),
session_id: self.session_id.clone(),
}))
}
fn interrupt_handle(&self) -> Option<Arc<dyn CoreExecutorInterruptHandle>> {
Some(Arc::new(RestSessionRuntimeInterruptHandle {
context: self.context.clone(),
session_id: self.session_id.clone(),
}))
}
fn publication_handle(&self) -> Option<Arc<dyn CoreExecutorPublicationHandle>> {
Some(meerkat::surface::persistent_runtime_publication_handle(
Arc::clone(&self.context.session_service),
self.session_id.clone(),
))
}
fn machine_managed_post_stop_unregister(&self) -> bool {
true
}
fn post_stop_cleanup_handle(&self) -> Option<Arc<dyn CoreExecutorPostStopCleanupHandle>> {
Some(
meerkat::surface::persistent_runtime_post_stop_cleanup_handle(
Arc::clone(&self.context.session_service),
self.session_id.clone(),
),
)
}
fn turn_finalization_boundary_handle(
&self,
) -> Option<Arc<dyn meerkat_core::lifecycle::CoreExecutorTurnFinalizationBoundaryHandle>> {
Some(
meerkat::surface::persistent_runtime_turn_finalization_boundary_handle(
Arc::clone(&self.context.session_service),
self.session_id.clone(),
),
)
}
async fn apply(
&mut self,
run_id: meerkat_core::lifecycle::RunId,
primitive: RunPrimitive,
) -> Result<CoreApplyOutput, CoreExecutorError> {
let prompt = primitive.extract_content_input();
match apply_runtime_turn_under_runtime_turn_boundary(
&self.context,
&self.session_id,
run_id,
&primitive,
prompt,
)
.await
{
Ok(output) => Ok(output),
Err(error @ SessionError::NotFound { .. }) => {
let archived = match self
.context
.session_service
.load_authoritative_session(&self.session_id)
.await
{
Ok(Some(session)) => self
.context
.session_service
.session_archived_by_authority(&self.session_id, &session)
.await
.map_err(CoreExecutorError::apply_failed_from_session_error)?,
Ok(None) => false,
Err(authority_error) => {
return Err(CoreExecutorError::apply_failed_from_session_error(
authority_error,
));
}
};
if archived {
Err(CoreExecutorError::archived_session_requires_teardown(
error.to_string(),
))
} else {
Err(CoreExecutorError::session_unavailable_requires_teardown(
error.to_string(),
))
}
}
Err(error) => Err(CoreExecutorError::apply_failed_from_session_error(error)),
}
}
async fn reconcile_committed_compaction_projections(
&mut self,
intents: &[meerkat_core::CompactionProjectionIntent],
) -> Result<(), CoreExecutorError> {
self.context
.session_service
.reconcile_runtime_compaction_projections(&self.session_id, intents.to_vec())
.await
.map_err(|error| CoreExecutorError::Internal(error.to_string()))
}
async fn checkpoint_committed_session_snapshot(
&mut self,
session_snapshot: Arc<Vec<u8>>,
) -> Result<(), CoreExecutorError> {
self.context
.session_service
.checkpoint_committed_runtime_session_snapshot_under_runtime_turn_boundary(
&self.session_id,
session_snapshot,
)
.await
.map_err(CoreExecutorError::apply_failed_from_session_error)
}
async fn acknowledge_committed_session_boundary(
&mut self,
authority: &meerkat_core::CommittedSessionBoundaryAuthority,
) -> Result<(), CoreExecutorError> {
self.context
.session_service
.acknowledge_committed_runtime_session_boundary_under_runtime_turn_boundary(
&self.session_id,
authority,
)
.await
.map_err(CoreExecutorError::apply_failed_from_session_error)
}
async fn publish_interaction_terminals(
&mut self,
events: &[AgentEvent],
) -> Result<
Vec<meerkat_core::lifecycle::core_executor::CoreInteractionTerminalPublicationReceipt>,
CoreExecutorError,
> {
self.context
.session_service
.publish_interaction_terminals_exact_batch(&self.session_id, events)
.await
.map_err(|error| CoreExecutorError::Internal(error.to_string()))
}
async fn abort_uncommitted_compaction_projections(&mut self) -> Result<(), CoreExecutorError> {
self.context
.session_service
.abort_uncommitted_compaction_projections(&self.session_id)
.await
.map_err(|error| CoreExecutorError::Internal(error.to_string()))
}
async fn abort_rejected_run_projections(&mut self) -> Result<(), CoreExecutorError> {
self.context
.session_service
.abort_rejected_runtime_run_projections(&self.session_id)
.await
.map_err(|error| CoreExecutorError::Internal(error.to_string()))
}
async fn cancel_after_boundary(&mut self, _reason: String) -> Result<(), CoreExecutorError> {
self.context
.session_service
.cancel_current_after_boundary_with_machine_authority(
&self.session_id,
self.context.runtime_adapter.session_control_authority(),
)
.await
.or_else(|err| match err {
SessionError::NotRunning { .. } => Ok(()),
err => Err(err),
})
.map_err(|err| CoreExecutorError::control_failed_runtime(err.to_string()))
}
async fn stop_runtime_executor(&mut self, _reason: String) -> Result<(), CoreExecutorError> {
Ok(())
}
async fn cleanup_after_runtime_stop_terminalized(&mut self) -> Result<(), CoreExecutorError> {
meerkat::surface::persistent_runtime_post_stop_cleanup_handle(
Arc::clone(&self.context.session_service),
self.session_id.clone(),
)
.cleanup_after_runtime_stop_terminalized()
.await
}
}
fn rest_instance_root() -> PathBuf {
meerkat_core::runtime_bootstrap::default_surface_instance_root("rest")
}
fn parse_backend_hint(raw: &str) -> Option<RealmBackend> {
match raw {
"jsonl" => Some(RealmBackend::Jsonl),
"memory" => Some(RealmBackend::Memory),
"sqlite" => Some(RealmBackend::Sqlite),
_ => None,
}
}
fn realm_origin_from_selection(selection: &RealmSelection) -> RealmOrigin {
match selection {
RealmSelection::Explicit { .. } => RealmOrigin::Explicit,
RealmSelection::Isolated => RealmOrigin::Generated,
RealmSelection::WorkspaceDerived { .. } => RealmOrigin::Workspace,
}
}
fn resolve_keep_alive(requested: Option<bool>) -> Result<Option<bool>, ApiError> {
match requested {
Some(true) => meerkat::surface::resolve_keep_alive(true)
.map(Some)
.map_err(ApiError::BadRequest),
other => Ok(other), }
}
const REST_TURN_KEEP_ALIVE_TTL_SECS: u64 = 30;
fn resolve_turn_keep_alive_policy(
requested: Option<bool>,
) -> Option<meerkat_core::lifecycle::run_primitive::KeepAliveDirective> {
match requested {
Some(true) => Some(
meerkat_core::lifecycle::run_primitive::KeepAliveDirective::Enable(
meerkat_core::lifecycle::run_primitive::KeepAlivePolicy {
ttl: std::time::Duration::from_secs(REST_TURN_KEEP_ALIVE_TTL_SECS),
policy: meerkat_core::lifecycle::run_primitive::KeepAliveMode::Pinned,
},
),
),
Some(false) => Some(meerkat_core::lifecycle::run_primitive::KeepAliveDirective::Disable),
None => None,
}
}
fn resolve_turn_additional_instructions(
requested: Option<Vec<String>>,
) -> Option<Vec<meerkat_core::lifecycle::run_primitive::TurnInstruction>> {
requested.map(|instructions| {
instructions
.into_iter()
.map(
|body| meerkat_core::lifecycle::run_primitive::TurnInstruction {
kind: meerkat_core::lifecycle::run_primitive::TurnInstructionKind::User,
body,
},
)
.collect()
})
}
fn validate_public_peer_meta(peer_meta: Option<&meerkat_core::PeerMeta>) -> Result<(), ApiError> {
meerkat::surface::validate_public_peer_meta(peer_meta).map_err(ApiError::BadRequest)
}
fn validate_public_surface_metadata(
labels: Option<&BTreeMap<String, String>>,
app_context: Option<&Value>,
) -> Result<(), ApiError> {
meerkat::surface::validate_public_surface_metadata(labels, app_context)
.map_err(ApiError::BadRequest)
}
pub use meerkat_contracts::wire::RestCreateSessionRequest as CreateSessionRequest;
fn rest_continue_requires_rebuild(req: &ContinueSessionRequest) -> bool {
req.model.is_some()
|| req.provider.is_some()
|| req.auth_binding.is_some()
|| req.max_tokens.is_some()
|| req.output_schema.is_some()
|| req.structured_output_retries.is_some()
|| req.hooks_override.is_some()
|| req.enable_web_search.is_some()
|| req.comms_name.is_some()
|| req.peer_meta.is_some()
}
fn rest_continue_system_prompts(req: &ContinueSessionRequest) -> Vec<String> {
req.system_prompt.iter().cloned().collect()
}
fn create_session_resume_override_mask(
req: &CreateSessionRequest,
keep_alive_override: Option<bool>,
) -> ResumeOverrideMask {
ResumeOverrideMask {
model: req.model.is_some(),
provider: req.provider.is_some(),
auth_binding: req.auth_binding.is_some(),
max_tokens: req.max_tokens.is_some(),
structured_output_retries: req.structured_output_retries.is_some(),
provider_params: req.provider_params.is_some(),
preload_skills: req.preload_skills.is_some(),
keep_alive: keep_alive_override.is_some(),
comms_name: req.comms_name.is_some(),
peer_meta: req.peer_meta.is_some(),
override_schedule: req.enable_schedule.is_some(),
override_workgraph: req.enable_workgraph.is_some(),
override_web_search: req.enable_web_search.is_some(),
..Default::default()
}
}
async fn canonical_skill_keys_for_state(
state: &AppState,
skill_refs: Option<Vec<meerkat_core::skills::SkillRef>>,
) -> Result<Option<Vec<meerkat_core::skills::SkillKey>>, ApiError> {
let params = SkillsParams {
preload_skills: None,
skill_refs,
};
let snapshot = state
.config_runtime
.get()
.await
.map_err(config_runtime_err_to_api)?;
snapshot
.config
.skills
.build_source_identity_registry()
.map_err(|e| ApiError::BadRequest(format!("Invalid skills config: {e}")))?;
Ok(params.canonical_skill_keys())
}
pub use meerkat_contracts::wire::RestContinueSessionRequest as ContinueSessionRequest;
pub use meerkat_contracts::wire::RestAppendSystemContextRequest as AppendSystemContextRequest;
#[derive(Debug, Deserialize)]
pub struct ListSessionsQuery {
#[serde(default)]
pub limit: Option<usize>,
#[serde(default)]
pub offset: Option<usize>,
#[serde(default)]
pub label: Option<Vec<String>>,
}
#[derive(Debug, Deserialize)]
pub struct SessionHistoryQuery {
#[serde(default)]
pub offset: Option<usize>,
#[serde(default)]
pub limit: Option<usize>,
}
pub type SessionResponse = meerkat_contracts::WireRunResult;
pub type UsageResponse = meerkat_contracts::WireUsage;
pub use meerkat_contracts::wire::RestSessionDetailsResponse as SessionDetailsResponse;
pub type ScheduleListResponse = meerkat_contracts::wire::ScheduleListResult;
pub type ScheduleOccurrencesResponse = meerkat_contracts::wire::ScheduleOccurrencesResult;
#[derive(Debug, Serialize)]
pub struct ErrorResponse {
pub error: String,
pub code: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub details: Option<Value>,
}
fn workgraph_observability_router() -> Router<AppState> {
meerkat::workgraph_rest_path_catalog()
.iter()
.fold(Router::new(), |router, descriptor| match descriptor.route {
meerkat::WorkGraphRestRoute::Items => {
router.route(descriptor.path, get(workgraph_list_items))
}
meerkat::WorkGraphRestRoute::Item => {
router.route(descriptor.path, get(workgraph_get_item))
}
meerkat::WorkGraphRestRoute::Ready => {
router.route(descriptor.path, get(workgraph_ready))
}
meerkat::WorkGraphRestRoute::Snapshot => {
router.route(descriptor.path, get(workgraph_snapshot))
}
meerkat::WorkGraphRestRoute::Events => {
router.route(descriptor.path, get(workgraph_events))
}
meerkat::WorkGraphRestRoute::GoalStatus => {
router.route(descriptor.path, post(workgraph_goal_status))
}
meerkat::WorkGraphRestRoute::AttentionList => {
router.route(descriptor.path, post(workgraph_attention_list))
}
})
}
pub fn router(state: AppState) -> Router {
let schedule_state = state.clone();
tokio::spawn(async move {
if let Err(error) = schedule_state.ensure_schedule_host_started().await {
tracing::warn!("failed to start REST schedule host: {error}");
}
});
let r = Router::new()
.route("/help", post(help))
.route("/sessions", get(list_sessions).post(create_session))
.route("/sessions/{id}", get(get_session).delete(archive_session))
.route("/sessions/{id}/history", get(get_session_history))
.route("/sessions/{id}/interrupt", post(interrupt_session))
.route("/sessions/{id}/status", get(get_runtime_status))
.route("/sessions/{id}/system_context", post(append_system_context))
.route("/sessions/{id}/messages", post(continue_session))
.route("/sessions/{id}/external-events", post(post_external_event))
.route(
"/sessions/{id}/peer-response-terminal",
post(post_peer_response_terminal),
)
.route("/sessions/{id}/events", get(session_events))
.route("/schedule/tools", get(schedule_tools))
.route("/schedule/call", post(schedule_call))
.merge(workgraph_observability_router())
.route("/schedules", get(list_schedules).post(create_schedule))
.route(
"/schedules/{id}",
get(get_schedule)
.patch(update_schedule)
.delete(delete_schedule),
)
.route("/schedules/{id}/pause", post(pause_schedule))
.route("/schedules/{id}/resume", post(resume_schedule))
.route(
"/schedules/{id}/occurrences",
get(list_schedule_occurrences),
)
.route("/requests/{request_id}/cancel", post(cancel_request))
.route("/comms/send", post(comms_send))
.route("/comms/peers", get(comms_peers))
.route(
"/config",
get(get_config).put(set_config).patch(patch_config),
)
.route("/health", get(health_check))
.route("/skills", get(list_skills))
.route("/capabilities", get(get_capabilities))
.route("/runtime/host_info", get(get_runtime_host_info))
.route("/runtime/capabilities", get(get_runtime_capabilities))
.route("/runtime/health", get(get_runtime_health))
.route("/models/catalog", get(get_models_catalog))
.route(
"/auth/profiles",
get(crate::auth_endpoints::list_auth_profiles)
.post(crate::auth_endpoints::create_auth_profile),
)
.route(
"/auth/bindings/{binding_id}",
get(crate::auth_endpoints::get_auth_profile)
.delete(crate::auth_endpoints::delete_auth_profile),
)
.route(
"/auth/bindings/{binding_id}/test",
post(crate::auth_endpoints::test_auth_binding),
)
.route(
"/auth/login/start",
post(crate::auth_endpoints::start_login),
)
.route(
"/auth/login/complete",
post(crate::auth_endpoints::complete_login),
)
.route(
"/auth/login/device/start",
post(crate::auth_endpoints::start_device_login),
)
.route(
"/auth/login/device/complete",
post(crate::auth_endpoints::complete_device_login),
)
.route(
"/auth/bindings/{binding_id}/status",
get(crate::auth_endpoints::get_auth_status),
)
.route(
"/auth/bindings/{binding_id}/logout",
post(crate::auth_endpoints::logout),
)
.route("/realms", get(crate::auth_endpoints::list_realms))
.route("/realms/{id}", get(crate::auth_endpoints::get_realm));
#[cfg(feature = "mob")]
let r = r
.route("/mob/{id}/events", get(mob_event_stream))
.route("/mob/{id}/spawn-helper", post(mob_spawn_helper))
.route("/mob/{id}/fork-helper", post(mob_fork_helper))
.route("/mob/{id}/wait-kickoff", post(mob_wait_kickoff))
.route("/mob/{id}/wire-members-batch", post(mob_wire_members_batch))
.route(
"/mob/{id}/members/{agent_identity}/status",
get(mob_member_status),
)
.route(
"/mob/{id}/members/{agent_identity}/cancel",
post(mob_force_cancel),
)
.route(
"/mob/{id}/members/{agent_identity}/respawn",
post(mob_member_respawn),
)
.route(
"/mob/{id}/members/{agent_identity}/history",
get(mob_member_history),
)
.route("/mob/{id}/hosts", get(mob_hosts))
.route("/mob/{id}/route-installs", get(mob_route_installs));
#[cfg(feature = "mcp")]
let r = r
.route("/sessions/{id}/mcp/add", post(mcp_add))
.route("/sessions/{id}/mcp/remove", post(mcp_remove))
.route("/sessions/{id}/mcp/reload", post(mcp_reload));
r.with_state(state)
}
fn get_runtime_adapter(state: &AppState) -> &Arc<meerkat_runtime::MeerkatMachine> {
&state.runtime_adapter
}
async fn session_archived_by_authority(
state: &AppState,
session_id: &SessionId,
session: &Session,
) -> Result<bool, SessionError> {
state
.session_service
.session_archived_by_authority(session_id, session)
.await
}
async fn authoritative_session_archived_response(
state: &AppState,
session_id: &SessionId,
) -> Result<bool, Response> {
let session = state
.session_service
.load_authoritative_session(session_id)
.await
.map_err(|err| {
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(json!({"error": format!("failed to load session: {err}")})),
)
.into_response()
})?;
let Some(session) = session else {
return Ok(false);
};
session_archived_by_authority(state, session_id, &session)
.await
.map_err(|err| {
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(json!({"error": format!("failed to load session archive state: {err}")})),
)
.into_response()
})
}
fn session_metadata_marks_mob_member(session: &Session) -> bool {
session
.session_metadata()
.and_then(|metadata| metadata.peer_meta)
.is_some_and(|peer_meta| peer_meta.labels.contains_key("mob_id"))
}
fn runtime_state_to_wire(
state: meerkat_runtime::RuntimeState,
) -> Option<meerkat_contracts::WireRuntimeState> {
Some(match state {
meerkat_runtime::RuntimeState::Initializing => {
meerkat_contracts::WireRuntimeState::Initializing
}
meerkat_runtime::RuntimeState::Idle => meerkat_contracts::WireRuntimeState::Idle,
meerkat_runtime::RuntimeState::Attached => meerkat_contracts::WireRuntimeState::Attached,
meerkat_runtime::RuntimeState::Running => meerkat_contracts::WireRuntimeState::Running,
meerkat_runtime::RuntimeState::Retired => meerkat_contracts::WireRuntimeState::Retired,
meerkat_runtime::RuntimeState::Stopped => meerkat_contracts::WireRuntimeState::Stopped,
meerkat_runtime::RuntimeState::Destroyed => meerkat_contracts::WireRuntimeState::Destroyed,
_ => return None,
})
}
async fn get_runtime_status(
State(state): State<AppState>,
Path(id): Path<String>,
) -> Result<Json<RuntimeStateResult>, Response> {
let session_id =
resolve_session_id_for_state(&id, &state).map_err(IntoResponse::into_response)?;
let adapter = get_runtime_adapter(&state);
let runtime_state = adapter.runtime_state(&session_id).await.map_err(|err| {
(
StatusCode::NOT_FOUND,
Json(json!({"error": err.to_string()})),
)
.into_response()
})?;
let Some(runtime_state) = runtime_state_to_wire(runtime_state) else {
return Err((
StatusCode::INTERNAL_SERVER_ERROR,
Json(json!({"error": "runtime state projection is unsupported by this REST surface"})),
)
.into_response());
};
Ok(Json(RuntimeStateResult {
state: runtime_state,
}))
}
fn rest_wire_error(
code: ErrorCode,
message: impl Into<std::borrow::Cow<'static, str>>,
) -> Response {
let status =
StatusCode::from_u16(code.http_status()).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);
(status, Json(WireError::new(code, message))).into_response()
}
fn rest_wire_error_with_details(
code: ErrorCode,
message: impl Into<std::borrow::Cow<'static, str>>,
details: Option<Value>,
) -> Response {
let mut error = WireError::new(code, message);
if let Some(details) = details {
error = error.with_details(details);
}
let status =
StatusCode::from_u16(code.http_status()).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);
(status, Json(error)).into_response()
}
async fn health_check() -> &'static str {
"ok"
}
#[derive(Debug, Deserialize)]
#[cfg(feature = "mob")]
struct MobEventStreamQuery {
#[serde(default)]
member: Option<String>,
}
#[cfg(feature = "mob")]
async fn mob_event_stream(
State(state): State<AppState>,
Path(id): Path<String>,
Query(query): Query<MobEventStreamQuery>,
) -> Result<Sse<std::pin::Pin<Box<dyn Stream<Item = Result<Event, Infallible>> + Send>>>, Response>
{
let mob_id = meerkat_mob::MobId::from(id.as_str());
let stream: std::pin::Pin<Box<dyn Stream<Item = Result<Event, Infallible>> + Send>> =
if let Some(member) = query.member {
let identity = meerkat_mob::AgentIdentity::from(member.as_str());
let mut event_stream = state
.mob_state
.subscribe_agent_events(&mob_id, &identity)
.await
.map_err(|err| mob_rest_error(&err, ApiError::NotFound))?;
Box::pin(async_stream::stream! {
match serde_json::to_string(&json!({
"mob_id": id,
"member": member,
})) {
Ok(data) => yield Ok(Event::default().event("stream_opened").data(data)),
Err(e) => {
yield Ok(sse_serialization_error_event(&e));
return;
}
}
while let Some(envelope) = futures::StreamExt::next(&mut event_stream).await {
let event_type = agent_event_type(&envelope.payload);
match serde_json::to_string(&envelope) {
Ok(data) => yield Ok(Event::default().event(event_type).data(data)),
Err(e) => {
yield Ok(sse_serialization_error_event(&e));
return;
}
}
}
yield Ok(Event::default().event("done").data("{}"));
})
} else {
let mut handle = state
.mob_state
.subscribe_mob_events(&mob_id)
.await
.map_err(|err| mob_rest_error(&err, ApiError::NotFound))?;
Box::pin(async_stream::stream! {
match serde_json::to_string(&json!({
"mob_id": id,
})) {
Ok(data) => yield Ok(Event::default().event("stream_opened").data(data)),
Err(e) => {
yield Ok(sse_serialization_error_event(&e));
return;
}
}
while let Some(attributed) = handle.event_rx.recv().await {
let event_type = agent_event_type(&attributed.envelope.payload);
match serde_json::to_string(&attributed) {
Ok(data) => yield Ok(Event::default().event(event_type).data(data)),
Err(e) => {
yield Ok(sse_serialization_error_event(&e));
return;
}
}
}
yield Ok(Event::default().event("done").data("{}"));
})
};
Ok(Sse::new(stream))
}
#[cfg(feature = "mob")]
use meerkat_contracts::wire::RestMobHelperRequest as SpawnHelperRequest;
#[cfg(feature = "mob")]
fn mob_runtime_mode_from_wire(
mode: meerkat_contracts::WireMobRuntimeMode,
) -> meerkat_mob::MobRuntimeMode {
match mode {
meerkat_contracts::WireMobRuntimeMode::AutonomousHost => {
meerkat_mob::MobRuntimeMode::AutonomousHost
}
meerkat_contracts::WireMobRuntimeMode::TurnDriven => {
meerkat_mob::MobRuntimeMode::TurnDriven
}
}
}
#[cfg(feature = "mob")]
fn mob_backend_kind_from_wire(
kind: meerkat_contracts::WireMobBackendKind,
) -> meerkat_mob::MobBackendKind {
match kind {
meerkat_contracts::WireMobBackendKind::Session => meerkat_mob::MobBackendKind::Session,
meerkat_contracts::WireMobBackendKind::External => meerkat_mob::MobBackendKind::External,
}
}
#[cfg(feature = "mob")]
fn mob_fork_context_from_wire(
context: meerkat_contracts::wire::WireForkContext,
) -> meerkat_mob::ForkContext {
match context {
meerkat_contracts::wire::WireForkContext::FullHistory => {
meerkat_mob::ForkContext::FullHistory
}
meerkat_contracts::wire::WireForkContext::LastMessages { count } => {
meerkat_mob::ForkContext::LastMessages { count }
}
}
}
#[cfg(feature = "mob")]
async fn mob_spawn_helper(
State(state): State<AppState>,
Path(id): Path<String>,
Json(req): Json<SpawnHelperRequest>,
) -> Result<Json<Value>, Response> {
let mob_id = meerkat_mob::MobId::from(id.as_str());
let Some(agent_identity) = req.agent_identity else {
return Err(ApiError::BadRequest(
"spawn-helper requires agent_identity; the surface does not allocate member identity"
.to_string(),
)
.into_response());
};
let identity = meerkat_mob::AgentIdentity::from(agent_identity);
let mut options = meerkat_mob::HelperOptions::default();
if let Some(role) = req.role_name {
options.role_name = Some(meerkat_mob::ProfileName::from(role));
}
options.model_override = req.model_override;
options.auth_binding = req.auth_binding.map(Into::into);
options.runtime_mode = req.runtime_mode.map(mob_runtime_mode_from_wire);
options.backend = req.backend.map(mob_backend_kind_from_wire);
let result = state
.mob_state
.mob_spawn_helper(&mob_id, identity, req.prompt, options)
.await
.map_err(|err| mob_rest_error(&err, ApiError::BadRequest))?;
let payload = serde_json::to_value(result)
.map_err(|e| ApiError::Internal(format!("serialize helper result: {e}")).into_response())?;
Ok(Json(payload))
}
#[cfg(feature = "mob")]
use meerkat_contracts::wire::RestMobForkHelperRequest as ForkHelperRequest;
#[cfg(feature = "mob")]
use meerkat_contracts::wire::RestMobWaitRequest as WaitKickoffRequest;
#[cfg(feature = "mob")]
use meerkat_contracts::wire::RestMobWireMembersBatchRequest;
#[cfg(feature = "mob")]
fn rest_member_wire_edge(
edge: meerkat_mob::MemberWireEdge,
) -> meerkat_contracts::MobWireMembersBatchEdge {
meerkat_contracts::MobWireMembersBatchEdge {
a: edge.a.to_string(),
b: edge.b.to_string(),
}
}
#[cfg(feature = "mob")]
async fn mob_wait_kickoff(
State(state): State<AppState>,
Path(id): Path<String>,
body: Option<Json<WaitKickoffRequest>>,
) -> Result<Json<Value>, Response> {
let mob_id = meerkat_mob::MobId::from(id.as_str());
let request = body.map(|Json(value)| value).unwrap_or_default();
let member_ids = request.member_ids.map(|member_ids| {
member_ids
.into_iter()
.map(|member_id| meerkat_mob::AgentIdentity::from(member_id.as_str()))
.collect::<Vec<_>>()
});
let members = state
.mob_state
.mob_wait_kickoff(&mob_id, member_ids, request.timeout_ms)
.await
.map_err(|err| mob_rest_error(&err, ApiError::BadRequest))?;
Ok(Json(json!({ "members": members })))
}
#[cfg(feature = "mob")]
async fn mob_wire_members_batch(
State(state): State<AppState>,
Path(id): Path<String>,
Json(req): Json<RestMobWireMembersBatchRequest>,
) -> Result<Json<meerkat_contracts::MobWireMembersBatchResult>, Response> {
let mob_id = meerkat_mob::MobId::from(id.as_str());
let edges = req
.edges
.into_iter()
.map(|edge| {
(
meerkat_mob::AgentIdentity::from(edge.a),
meerkat_mob::AgentIdentity::from(edge.b),
)
})
.collect::<Vec<_>>();
let report = state
.mob_state
.handle_for(&mob_id)
.await
.map_err(|err| mob_rest_error(&err, ApiError::BadRequest))?
.wire_members_batch(edges)
.await
.map_err(|err| mob_rest_error(&err, ApiError::BadRequest))?;
Ok(Json(meerkat_contracts::MobWireMembersBatchResult {
requested: report.requested,
wired: report
.wired
.into_iter()
.map(rest_member_wire_edge)
.collect(),
already_wired: report
.already_wired
.into_iter()
.map(rest_member_wire_edge)
.collect(),
}))
}
#[cfg(feature = "mob")]
async fn mob_fork_helper(
State(state): State<AppState>,
Path(id): Path<String>,
Json(req): Json<ForkHelperRequest>,
) -> Result<Json<Value>, Response> {
let mob_id = meerkat_mob::MobId::from(id.as_str());
let source_identity = meerkat_mob::AgentIdentity::from(req.source_member_id.as_str());
let Some(agent_identity) = req.agent_identity else {
return Err(ApiError::BadRequest(
"fork-helper requires agent_identity; the surface does not allocate member identity"
.to_string(),
)
.into_response());
};
let identity = meerkat_mob::AgentIdentity::from(agent_identity);
let fork_context = req
.fork_context
.map(mob_fork_context_from_wire)
.unwrap_or(meerkat_mob::ForkContext::FullHistory);
let mut options = meerkat_mob::HelperOptions::default();
if let Some(role) = req.role_name {
options.role_name = Some(meerkat_mob::ProfileName::from(role));
}
options.model_override = req.model_override;
options.auth_binding = req.auth_binding.map(Into::into);
options.runtime_mode = req.runtime_mode.map(mob_runtime_mode_from_wire);
options.backend = req.backend.map(mob_backend_kind_from_wire);
let result = state
.mob_state
.mob_fork_helper(
&mob_id,
&source_identity,
identity,
req.prompt,
fork_context,
options,
)
.await
.map_err(|err| mob_rest_error(&err, ApiError::BadRequest))?;
let payload = serde_json::to_value(result)
.map_err(|e| ApiError::Internal(format!("serialize helper result: {e}")).into_response())?;
Ok(Json(payload))
}
#[cfg(feature = "mob")]
async fn mob_member_status(
State(state): State<AppState>,
Path((id, agent_identity)): Path<(String, String)>,
) -> Result<Json<Value>, Response> {
let mob_id = meerkat_mob::MobId::from(id.as_str());
let identity = meerkat_mob::AgentIdentity::from(agent_identity.as_str());
let snapshot = state
.mob_state
.mob_member_status(&mob_id, &identity)
.await
.map_err(|err| mob_rest_error(&err, ApiError::BadRequest))?;
let member_ref = meerkat_contracts::WireMemberRef::encode(mob_id.as_str(), identity.as_str());
let result = snapshot
.to_member_status_result(member_ref)
.map_err(|err| mob_rest_error(&err, ApiError::Internal))?;
Ok(Json(json!(result)))
}
#[cfg(feature = "mob")]
async fn mob_force_cancel(
State(state): State<AppState>,
Path((id, agent_identity)): Path<(String, String)>,
) -> Result<Json<Value>, Response> {
let mob_id = meerkat_mob::MobId::from(id.as_str());
let identity = meerkat_mob::AgentIdentity::from(agent_identity.as_str());
state
.mob_state
.mob_force_cancel(&mob_id, identity)
.await
.map_err(|err| mob_rest_error(&err, ApiError::BadRequest))?;
Ok(Json(json!({"cancelled": true})))
}
#[cfg(feature = "mob")]
trait MobRestWireErrorSource: std::fmt::Display {
fn mob_wire_detail(&self) -> Option<meerkat_contracts::wire::WireMobErrorDetail>;
fn structured_data(&self) -> Option<Value>;
}
#[cfg(feature = "mob")]
impl MobRestWireErrorSource for meerkat_mob::MobError {
fn mob_wire_detail(&self) -> Option<meerkat_contracts::wire::WireMobErrorDetail> {
self.wire_detail()
}
fn structured_data(&self) -> Option<Value> {
meerkat_mob::MobError::structured_data(self)
}
}
#[cfg(feature = "mob")]
impl MobRestWireErrorSource for meerkat_mob::MobRespawnError {
fn mob_wire_detail(&self) -> Option<meerkat_contracts::wire::WireMobErrorDetail> {
self.wire_detail()
}
fn structured_data(&self) -> Option<Value> {
meerkat_mob::MobRespawnError::structured_data(self)
}
}
#[cfg(feature = "mob")]
fn mob_rest_error<E>(err: &E, fallback: fn(String) -> ApiError) -> Response
where
E: MobRestWireErrorSource,
{
match err.mob_wire_detail() {
Some(detail) => match detail.detail_value() {
Ok(details) => {
rest_wire_error_with_details(detail.code(), err.to_string(), Some(details))
}
Err(serialize_error) => rest_wire_error(
ErrorCode::InternalError,
format!("failed to serialize mob error detail: {serialize_error}"),
),
},
None => {
let fallback_error = fallback(err.to_string());
match (fallback_error, err.structured_data()) {
(ApiError::BadRequest(message), Some(details)) => ApiError::BadRequestWithData {
message,
code: "BAD_REQUEST".to_string(),
details,
}
.into_response(),
(ApiError::Internal(message), Some(details)) => ApiError::InternalWithData {
message,
code: "INTERNAL_ERROR".to_string(),
details,
}
.into_response(),
(fallback_error, _) => fallback_error.into_response(),
}
}
}
}
#[cfg(feature = "mob")]
#[derive(Debug, Deserialize)]
struct MobMemberHistoryQuery {
#[serde(default)]
from_index: Option<u64>,
#[serde(default)]
limit: Option<u32>,
}
#[cfg(feature = "mob")]
async fn mob_member_history(
State(state): State<AppState>,
Path((id, agent_identity)): Path<(String, String)>,
Query(query): Query<MobMemberHistoryQuery>,
) -> Result<Json<meerkat_contracts::wire::MobMemberHistoryResult>, Response> {
let mob_id = meerkat_mob::MobId::from(id.as_str());
let identity = meerkat_mob::AgentIdentity::from(agent_identity.as_str());
let result = state
.mob_state
.mob_member_history(&mob_id, identity, query.from_index, query.limit)
.await
.map_err(|err| mob_rest_error(&err, ApiError::BadRequest))?;
Ok(Json(result))
}
#[cfg(feature = "mob")]
async fn mob_hosts(
State(state): State<AppState>,
Path(id): Path<String>,
) -> Result<Json<meerkat_contracts::wire::MobHostsResult>, Response> {
let mob_id = meerkat_mob::MobId::from(id.as_str());
let result = state
.mob_state
.mob_hosts(&mob_id)
.await
.map_err(|err| mob_rest_error(&err, ApiError::BadRequest))?;
Ok(Json(result))
}
#[cfg(feature = "mob")]
async fn mob_route_installs(
State(state): State<AppState>,
Path(id): Path<String>,
) -> Result<Json<meerkat_contracts::wire::MobRouteInstallsResult>, Response> {
let mob_id = meerkat_mob::MobId::from(id.as_str());
let result = state
.mob_state
.mob_route_installs(&mob_id)
.await
.map_err(|err| mob_rest_error(&err, ApiError::BadRequest))?;
Ok(Json(result))
}
#[cfg(feature = "mob")]
async fn mob_member_respawn(
State(state): State<AppState>,
Path((id, agent_identity)): Path<(String, String)>,
body: Option<Json<Value>>,
) -> Result<Json<Value>, Response> {
let mob_id = meerkat_mob::MobId::from(id.as_str());
let identity = meerkat_mob::AgentIdentity::from(agent_identity.as_str());
let initial_message = body
.and_then(|Json(v)| v.get("initial_message").cloned())
.map(serde_json::from_value::<ContentInput>)
.transpose()
.map_err(|e| {
ApiError::BadRequest(format!("invalid initial_message: {e}")).into_response()
})?;
match state
.mob_state
.mob_respawn(&mob_id, identity, initial_message)
.await
{
Ok(receipt) => Ok(Json(json!({
"status": "completed",
"receipt": receipt,
}))),
Err(meerkat_mob::MobRespawnError::TopologyRestoreFailed {
receipt,
failed_peer_ids,
}) => Ok(Json(json!({
"status": "topology_restore_failed",
"receipt": receipt,
"failed_peer_ids": failed_peer_ids.iter().map(std::string::ToString::to_string).collect::<Vec<_>>(),
}))),
Err(err) => Err(mob_rest_error(&err, ApiError::BadRequest)),
}
}
pub type CommsSendRequest = CommsSendParams;
#[derive(Debug, Deserialize)]
pub struct CommsPeersRequest {
pub session_id: String,
}
async fn comms_send(
State(state): State<AppState>,
Json(req): Json<CommsSendParams>,
) -> Result<Json<CommsSendResult>, ApiError> {
let session_id = resolve_session_id_for_state(req.session_id(), &state)?;
let comms = state
.session_service
.comms_runtime(&session_id)
.await
.ok_or_else(|| {
ApiError::NotFound(format!(
"Session not found or comms not enabled: {session_id}"
))
})?;
let peer_name = req.peer_label();
let cmd = req
.into_command()
.into_command(&session_id)
.map_err(|err| ApiError::BadRequest(err.to_string()))?;
match comms.send(cmd).await {
Ok(receipt) => Ok(Json(CommsSendResult::from(receipt))),
Err(e) => Err(normalize_rest_comms_send_error(peer_name.as_deref(), &e)),
}
}
fn normalize_rest_comms_send_error(
peer_name: Option<&str>,
error: &meerkat_core::comms::SendError,
) -> ApiError {
match error {
meerkat_core::comms::SendError::PeerNotFound(peer) => ApiError::InternalWithData {
message: format!(
"peer_not_found_or_not_trusted: peer '{peer}' is not found or not trusted"
),
code: "peer_not_found_or_not_trusted".to_string(),
details: json!({
"code": "peer_not_found_or_not_trusted",
"peer": peer,
"message": format!("peer '{peer}' is not found or not trusted"),
}),
},
meerkat_core::comms::SendError::PeerOffline => {
let peer = peer_name.unwrap_or("<unknown>");
ApiError::InternalWithData {
message: format!(
"peer_unreachable: peer '{peer}' is unreachable: offline_or_no_ack"
),
code: "peer_unreachable".to_string(),
details: json!({
"code": "peer_unreachable",
"peer": peer,
"reason": "offline_or_no_ack",
"message": format!("peer '{peer}' is unreachable: offline_or_no_ack"),
}),
}
}
meerkat_core::comms::SendError::Transport(details) => {
let peer = peer_name.unwrap_or("<unknown>");
ApiError::InternalWithData {
message: format!(
"peer_unreachable: peer '{peer}' is unreachable: transport_error ({details})"
),
code: "peer_unreachable".to_string(),
details: json!({
"code": "peer_unreachable",
"peer": peer,
"reason": "transport_error",
"message": format!("peer '{peer}' is unreachable: transport_error"),
"details": details,
}),
}
}
meerkat_core::comms::SendError::AdmissionDropped { reason } => {
let peer = peer_name.unwrap_or("<unknown>");
ApiError::InternalWithData {
message: format!(
"peer_admission_dropped: peer '{peer}' rejected envelope at ingress: {}",
reason.as_code()
),
code: "peer_admission_dropped".to_string(),
details: json!({
"code": "peer_admission_dropped",
"peer": peer,
"reason": reason,
"message": format!(
"peer '{peer}' rejected envelope at ingress: {}",
reason.as_code()
),
}),
}
}
other => ApiError::InternalWithData {
message: format!("send_failed: {other}"),
code: "send_failed".to_string(),
details: json!({
"code": "send_failed",
"message": other.to_string(),
}),
},
}
}
async fn comms_peers(
State(state): State<AppState>,
axum::extract::Query(params): axum::extract::Query<CommsPeersRequest>,
) -> Result<Json<Value>, ApiError> {
let session_id = resolve_session_id_for_state(¶ms.session_id, &state)?;
let comms = state
.session_service
.comms_runtime(&session_id)
.await
.ok_or_else(|| {
ApiError::NotFound(format!(
"Session not found or comms not enabled: {session_id}"
))
})?;
Ok(Json(comms_peers_payload(comms.peers().await)))
}
fn comms_peers_payload(peers: Vec<meerkat_core::comms::PeerDirectoryEntry>) -> Value {
json!(meerkat_contracts::CommsPeersResult::from_entries(&peers))
}
fn make_runtime_external_event_input(
event_type: &str,
payload: Value,
blocks: Option<Vec<meerkat_contracts::WireContentBlock>>,
) -> Result<meerkat_runtime::Input, ApiError> {
if event_type.trim().is_empty() {
return Err(ApiError::BadRequest(
"event_type cannot be empty".to_string(),
));
}
let blocks = blocks
.map(|blocks| {
blocks
.into_iter()
.map(meerkat_core::types::ContentBlock::try_from)
.collect::<Result<Vec<_>, _>>()
})
.transpose()
.map_err(|message| ApiError::BadRequest(message.to_string()))?;
Ok(meerkat_runtime::Input::ExternalEvent(
meerkat_runtime::ExternalEventInput {
objective_id: None,
header: meerkat_runtime::InputHeader {
id: meerkat_core::lifecycle::InputId::new(),
timestamp: chrono::Utc::now(),
source: meerkat_runtime::InputOrigin::External {
source_name: event_type.to_string(),
},
durability: meerkat_runtime::InputDurability::Durable,
visibility: meerkat_runtime::InputVisibility::default(),
idempotency_key: None,
supersession_key: None,
correlation_id: None,
},
event_type: event_type.to_string(),
payload,
handling_mode: meerkat_core::types::HandlingMode::Queue,
render_metadata: None,
blocks,
},
))
}
#[derive(Debug, Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case")]
enum RestSessionExternalEventEnvelope {
GenericJson {
event_type: String,
payload: Value,
#[serde(default)]
blocks: Option<Vec<meerkat_contracts::WireContentBlock>>,
},
PeerResponseTerminal {},
}
use meerkat_contracts::wire::RestPeerResponseTerminalRequest as RestPeerResponseTerminalBody;
async fn post_external_event(
State(state): State<AppState>,
Path(id): Path<String>,
headers: axum::http::HeaderMap,
Json(event): Json<RestSessionExternalEventEnvelope>,
) -> Result<(StatusCode, Json<Value>), Response> {
webhook::verify_webhook(&headers, &state.webhook_auth)
.map_err(|msg| ApiError::Unauthorized(msg.to_string()).into_response())?;
let session_id =
resolve_session_id_for_state(&id, &state).map_err(IntoResponse::into_response)?;
let input = match event {
RestSessionExternalEventEnvelope::GenericJson {
event_type,
payload,
blocks,
} => make_runtime_external_event_input(&event_type, payload, blocks),
RestSessionExternalEventEnvelope::PeerResponseTerminal {} => Err(ApiError::BadRequest(
"peer_response_terminal is reserved on /external-events; use /peer-response-terminal"
.to_string(),
)),
}
.map_err(IntoResponse::into_response)?;
admit_runtime_input_via_webhook(
&state,
&session_id,
input,
WebhookAdmissionMode::WithoutWake,
)
.await
}
async fn post_peer_response_terminal(
State(state): State<AppState>,
Path(id): Path<String>,
headers: axum::http::HeaderMap,
Json(body): Json<RestPeerResponseTerminalBody>,
) -> Result<(StatusCode, Json<Value>), Response> {
webhook::verify_webhook(&headers, &state.webhook_auth)
.map_err(|msg| ApiError::Unauthorized(msg.to_string()).into_response())?;
let session_id =
resolve_session_id_for_state(&id, &state).map_err(IntoResponse::into_response)?;
let RestPeerResponseTerminalBody {
peer_id,
display_name,
request_id,
status,
result,
} = body;
let input = meerkat_runtime::peer_response_terminal_input(
peer_id,
display_name,
request_id,
status,
result,
);
admit_runtime_input_via_webhook(&state, &session_id, input, WebhookAdmissionMode::Wakeful).await
}
#[derive(Debug, Clone, Copy)]
enum WebhookAdmissionMode {
WithoutWake,
Wakeful,
}
async fn admit_runtime_input_via_webhook(
state: &AppState,
session_id: &SessionId,
input: meerkat_runtime::Input,
mode: WebhookAdmissionMode,
) -> Result<(StatusCode, Json<Value>), Response> {
let outcome = match mode {
WebhookAdmissionMode::WithoutWake => {
if authoritative_session_archived_response(state, session_id).await? {
if let Err(error) = cleanup_archived_session_runtime(state, session_id).await {
return Err((
StatusCode::INTERNAL_SERVER_ERROR,
Json(json!({"error": format!("failed to clean up archived session runtime: {error}")})),
)
.into_response());
}
return Err((
StatusCode::NOT_FOUND,
Json(json!({"error": format!("session not found: {session_id}")})),
)
.into_response());
}
state
.runtime_adapter
.accept_input_without_wake(session_id, input)
.await
}
WebhookAdmissionMode::Wakeful => {
let input_id = input.id().clone();
let admission = match state
.session_service
.reserve_runtime_turn_admission(session_id)
.await
{
Ok(admission) => admission,
Err(err) => {
if authoritative_session_archived_response(state, session_id).await? {
if let Err(error) =
cleanup_archived_session_runtime(state, session_id).await
{
return Err((
StatusCode::INTERNAL_SERVER_ERROR,
Json(json!({"error": format!("failed to clean up archived session runtime: {error}")})),
)
.into_response());
}
return Err((
StatusCode::NOT_FOUND,
Json(json!({"error": format!("session not found: {session_id}")})),
)
.into_response());
}
return Err((
StatusCode::CONFLICT,
Json(json!({"error": err.to_string()})),
)
.into_response());
}
};
insert_rest_runtime_pre_admission(
&state.runtime_pre_admissions,
session_id.clone(),
input_id.clone(),
admission,
)
.await
.map_err(|err| {
(
StatusCode::CONFLICT,
Json(json!({"error": err.to_string()})),
)
.into_response()
})?;
let mut pre_admission_registration = Some(RestRuntimePreAdmissionRegistration::new(
state.runtime_pre_admissions.clone(),
session_id.clone(),
input_id.clone(),
));
if authoritative_session_archived_response(state, session_id).await? {
discard_rest_runtime_pre_admission(
&state.runtime_pre_admissions,
session_id,
&input_id,
)
.await;
if let Some(registration) = pre_admission_registration.take() {
registration.disarm();
}
if let Err(error) = cleanup_archived_session_runtime(state, session_id).await {
return Err((
StatusCode::INTERNAL_SERVER_ERROR,
Json(json!({"error": format!("failed to clean up archived session runtime: {error}")})),
)
.into_response());
}
return Err((
StatusCode::NOT_FOUND,
Json(json!({"error": format!("session not found: {session_id}")})),
)
.into_response());
}
let runtime_registration_lock = rest_runtime_registration_lock(state, session_id);
let runtime_registration_guard = runtime_registration_lock.mutex().lock().await;
let runtime_was_registered = state.runtime_adapter.contains_session(session_id).await;
if let Err(error) =
prepare_rest_session_runtime_executor_locked(state, session_id, &input_id).await
{
discard_rest_runtime_pre_admission(
&state.runtime_pre_admissions,
session_id,
&input_id,
)
.await;
if let Some(registration) = pre_admission_registration.take() {
registration.disarm();
}
let error = unregister_rest_runtime_after_api_error_locked(
state,
session_id,
runtime_was_registered,
runtime_executor_attach_error_to_api(error),
)
.await;
drop(runtime_registration_guard);
drop(runtime_registration_lock);
return Err(error.into_response());
}
let result = match state
.runtime_adapter
.accept_input_with_completion(session_id, input)
.await
{
Ok((outcome, handle)) => {
let cleanup_result = match (&outcome, handle) {
(
meerkat_runtime::AcceptOutcome::Accepted {
input_id: accepted_input_id,
..
},
Some(handle),
) => {
if let Some(registration) = pre_admission_registration.as_mut() {
registration.track_input_id(accepted_input_id.clone());
}
spawn_rest_runtime_pre_admission_rekey_and_cleanup(
state.clone(),
session_id.clone(),
input_id.clone(),
accepted_input_id.clone(),
handle,
);
if let Some(registration) = pre_admission_registration.take() {
registration.disarm();
}
Ok(())
}
(
meerkat_runtime::AcceptOutcome::Accepted {
input_id: accepted_input_id,
..
},
None,
) => {
if let Some(registration) = pre_admission_registration.as_mut() {
registration.track_input_id(accepted_input_id.clone());
}
discard_rest_runtime_pre_admission(
&state.runtime_pre_admissions,
session_id,
&input_id,
)
.await;
discard_rest_runtime_pre_admission(
&state.runtime_pre_admissions,
session_id,
accepted_input_id,
)
.await;
if let Some(registration) = pre_admission_registration.take() {
registration.disarm();
}
unregister_rest_runtime_if_new_idle_locked(
state,
session_id,
runtime_was_registered,
)
.await
}
_ => {
discard_rest_runtime_pre_admission(
&state.runtime_pre_admissions,
session_id,
&input_id,
)
.await;
if let Some(registration) = pre_admission_registration.take() {
registration.disarm();
}
unregister_rest_runtime_if_new_idle_locked(
state,
session_id,
runtime_was_registered,
)
.await
}
};
cleanup_result.map(|()| outcome)
}
Err(err) => {
discard_rest_runtime_pre_admission(
&state.runtime_pre_admissions,
session_id,
&input_id,
)
.await;
if let Some(registration) = pre_admission_registration.take() {
registration.disarm();
}
match unregister_rest_runtime_if_new_idle_locked(
state,
session_id,
runtime_was_registered,
)
.await
{
Ok(()) => Err(err),
Err(cleanup) => Err(combine_runtime_cleanup_errors(
err,
cleanup,
session_id,
"unregister newly prepared REST runtime",
)),
}
}
};
drop(runtime_registration_guard);
drop(runtime_registration_lock);
result
}
};
match outcome {
Ok(meerkat_runtime::AcceptOutcome::Accepted { .. })
| Ok(meerkat_runtime::AcceptOutcome::Deduplicated { .. }) => {
Ok((StatusCode::ACCEPTED, Json(json!({"queued": true}))))
}
Ok(meerkat_runtime::AcceptOutcome::Rejected { reason }) => {
Err((StatusCode::CONFLICT, Json(json!({"error": reason}))).into_response())
}
Ok(outcome) => Err((
StatusCode::INTERNAL_SERVER_ERROR,
Json(json!({
"error": format!("unexpected runtime accept outcome: {outcome:?}"),
})),
)
.into_response()),
Err(meerkat_runtime::RuntimeDriverError::NotReady { state }) => Err((
StatusCode::CONFLICT,
Json(json!({
"error": format!("runtime not accepting input while in state: {state}"),
})),
)
.into_response()),
Err(err) => Err((
StatusCode::INTERNAL_SERVER_ERROR,
Json(json!({"error": err.to_string()})),
)
.into_response()),
}
}
async fn list_skills(
State(state): State<AppState>,
) -> Result<Json<meerkat_contracts::SkillListResponse>, ApiError> {
let runtime = state
.skill_runtime
.as_ref()
.ok_or_else(|| ApiError::NotFound("skills not enabled".into()))?;
let entries = runtime
.list_all_with_provenance(&meerkat_core::skills::SkillFilter::default())
.await
.map_err(|e| ApiError::Internal(format!("skill list failed: {e}")))?;
let wire: Vec<meerkat_contracts::SkillEntry> =
entries.iter().map(skill_entry).collect::<Result<_, _>>()?;
Ok(Json(meerkat_contracts::SkillListResponse { skills: wire }))
}
fn skill_source_provenance(
identity: meerkat_core::skills::SourceIdentityRecord,
) -> meerkat_contracts::SkillSourceProvenance {
meerkat_contracts::SkillSourceProvenance { identity }
}
fn skill_entry(
e: &meerkat_core::skills::SkillIntrospectionEntry,
) -> Result<meerkat_contracts::SkillEntry, ApiError> {
let source_identity = e.source_identity.clone().ok_or_else(|| {
ApiError::Internal(format!(
"skill {} missing typed source identity",
e.descriptor.key
))
})?;
Ok(meerkat_contracts::SkillEntry {
key: e.descriptor.key.clone(),
name: e.descriptor.name.clone(),
description: e.descriptor.description.clone(),
scope: e.descriptor.scope,
source: skill_source_provenance(source_identity),
is_active: e.is_active,
shadowed_by: e.shadowed_by_identity.clone().map(skill_source_provenance),
})
}
async fn get_capabilities(
State(state): State<AppState>,
) -> Result<Json<meerkat_contracts::CapabilitiesResponse>, ApiError> {
let config = effective_config_for_state(&state)
.await
.map_err(|e| ApiError::Configuration(e.to_string()))?;
Ok(Json(meerkat::surface::build_capabilities_response(&config)))
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum RestPathFeature {
Always,
Mob,
Mcp,
Schedule,
}
impl RestPathFeature {
fn classify(path: &str) -> Self {
if path == "/mob" || path.starts_with("/mob/") {
RestPathFeature::Mob
} else if path.contains("/mcp/") {
RestPathFeature::Mcp
} else if path == "/schedule"
|| path.starts_with("/schedule/")
|| path == "/schedules"
|| path.starts_with("/schedules/")
{
RestPathFeature::Schedule
} else {
RestPathFeature::Always
}
}
fn is_enabled(self, options: &meerkat::surface::RuntimeHostSurfaceOptions) -> bool {
match self {
RestPathFeature::Always => true,
RestPathFeature::Mob => options.mobs,
RestPathFeature::Mcp => options.mcp_live,
RestPathFeature::Schedule => options.schedules,
}
}
}
fn rest_runtime_host_surface_options(
state: &AppState,
) -> meerkat::surface::RuntimeHostSurfaceOptions {
let mut options = meerkat::surface::RuntimeHostSurfaceOptions::process(
"meerkat-rest",
env!("CARGO_PKG_VERSION"),
);
options.runtime_backed_sessions = true;
options.mobs = cfg!(feature = "mob");
options.multi_host_mobs = cfg!(feature = "mob");
options.mcp_live = cfg!(feature = "mcp");
options.comms = cfg!(feature = "comms");
options.blobs = true;
options.artifacts = false;
options.session_events = true;
options.session_streams = true;
options.schedules = cfg!(feature = "schedule");
options.skills = state.skill_runtime.is_some();
options.approvals = false;
options.rest_base_url = Some(format!("http://{}:{}", state.rest_host, state.rest_port));
options.rest_paths = meerkat_contracts::rest_path_catalog()
.into_iter()
.filter(|path| RestPathFeature::classify(path.path).is_enabled(&options))
.map(|path| path.path.to_string())
.collect();
options
}
async fn get_runtime_host_info(
State(state): State<AppState>,
) -> Json<meerkat_contracts::RuntimeHostInfo> {
let options = rest_runtime_host_surface_options(&state);
let metadata = state.config_store.metadata();
let metadata = metadata
.as_ref()
.map(meerkat::surface::RuntimeHostMetadataProjection::from);
Json(meerkat::surface::build_runtime_host_info(
&options,
metadata.as_ref(),
state.context_root.clone(),
))
}
async fn get_runtime_capabilities(
State(state): State<AppState>,
) -> Json<meerkat_contracts::RuntimeHostCapabilities> {
let options = rest_runtime_host_surface_options(&state);
Json(meerkat::surface::build_runtime_host_capabilities(&options))
}
async fn get_runtime_health() -> Json<meerkat_contracts::RuntimeHostHealth> {
Json(meerkat::surface::build_runtime_host_health())
}
async fn get_models_catalog(
State(state): State<AppState>,
) -> Result<Json<meerkat_contracts::ModelsCatalogResponse>, ApiError> {
let config = effective_config_for_state(&state)
.await
.map_err(|e| ApiError::Configuration(e.to_string()))?;
let response = meerkat::surface::build_models_catalog_response(&config)
.map_err(|e| ApiError::Configuration(e.to_string()))?;
Ok(Json(response))
}
use meerkat_contracts::wire::RestSetConfigRequest as SetConfigRequest;
use meerkat_contracts::wire::RestPatchConfigRequest as PatchConfigRequest;
async fn get_config(State(state): State<AppState>) -> Result<Json<ConfigEnvelope>, ApiError> {
let snapshot = state
.config_runtime
.get()
.await
.map_err(config_runtime_err_to_api)?;
Ok(Json(ConfigEnvelope::from_snapshot(
snapshot,
if state.expose_paths {
ConfigEnvelopePolicy::Diagnostic
} else {
ConfigEnvelopePolicy::Public
},
)))
}
async fn set_config(
State(state): State<AppState>,
Json(req): Json<SetConfigRequest>,
) -> Result<Json<ConfigEnvelope>, ApiError> {
let (config, expected_generation) = match req {
SetConfigRequest::Wrapped {
config,
expected_generation,
} => (config, expected_generation),
SetConfigRequest::Direct(config) => (config, None),
};
validate_config_for_commit_with_roots(
&config,
state.context_root.as_deref(),
state.user_config_root.as_deref(),
)?;
let snapshot = state
.config_runtime
.set(config, expected_generation)
.await
.map_err(config_runtime_err_to_api)?;
Ok(Json(ConfigEnvelope::from_snapshot(
snapshot,
if state.expose_paths {
ConfigEnvelopePolicy::Diagnostic
} else {
ConfigEnvelopePolicy::Public
},
)))
}
async fn patch_config(
State(state): State<AppState>,
Json(req): Json<PatchConfigRequest>,
) -> Result<Json<ConfigEnvelope>, ApiError> {
let (delta, expected_generation) = match req {
PatchConfigRequest::Wrapped {
patch,
expected_generation,
} => (patch, expected_generation),
PatchConfigRequest::Direct(patch) => (patch, None),
};
let current = state
.config_runtime
.get()
.await
.map_err(config_runtime_err_to_api)?;
let preview = apply_patch_preview(¤t.config, delta.clone())?;
validate_config_for_commit_with_roots(
&preview,
state.context_root.as_deref(),
state.user_config_root.as_deref(),
)?;
let snapshot = state
.config_runtime
.patch(ConfigDelta(delta), expected_generation)
.await
.map_err(config_runtime_err_to_api)?;
Ok(Json(ConfigEnvelope::from_snapshot(
snapshot,
if state.expose_paths {
ConfigEnvelopePolicy::Diagnostic
} else {
ConfigEnvelopePolicy::Public
},
)))
}
fn config_runtime_err_to_api(err: meerkat_core::ConfigRuntimeError) -> ApiError {
match err {
meerkat_core::ConfigRuntimeError::GenerationConflict { expected, current } => {
ApiError::BadRequest(format!(
"Generation conflict: expected {expected}, current {current}"
))
}
other => ApiError::Configuration(other.to_string()),
}
}
async fn effective_config_for_state(state: &AppState) -> Result<Config, meerkat_core::ConfigError> {
let head = state.config_store.get().await?;
meerkat_core::EffectiveConfigReader::new(Arc::clone(&state.realm_config_source))
.effective_config_over_head(&state.realm, head)
.await
}
fn validate_config_for_commit_with_roots(
config: &Config,
_context_root: Option<&std::path::Path>,
_user_root: Option<&std::path::Path>,
) -> Result<(), ApiError> {
config
.validate(meerkat_models::canonical())
.map_err(|e| ApiError::BadRequest(format!("Invalid config: {e}")))?;
config
.skills
.build_source_identity_registry()
.map_err(|e| ApiError::BadRequest(format!("Invalid skills source-identity config: {e}")))?;
Ok(())
}
fn apply_patch_preview(config: &Config, patch: Value) -> Result<Config, ApiError> {
meerkat_core::apply_config_patch_preview(config, patch)
.map_err(|e| ApiError::BadRequest(format!("Invalid patch: {e}")))
}
fn spawn_event_forwarder(
mut event_rx: mpsc::Receiver<EventEnvelope<AgentEvent>>,
broadcast_tx: broadcast::Sender<SessionEvent>,
session_id: SessionId,
verbose: bool,
) -> tokio::task::JoinHandle<()> {
tokio::spawn(async move {
while let Some(event) = event_rx.recv().await {
if verbose && let Some(line) = format_verbose_event(&event.payload) {
tracing::info!("{}", line);
}
let _ = broadcast_tx.send(SessionEvent {
session_id: session_id.clone(),
event,
});
}
})
}
async fn drain_event_forwarder(session_id: &SessionId, forwarder: tokio::task::JoinHandle<()>) {
if tokio::time::timeout(std::time::Duration::from_millis(500), forwarder)
.await
.is_err()
{
tracing::debug!(
session_id = %session_id,
"event forwarder still draining after timeout; detaching task"
);
}
}
fn run_result_to_response(
result: meerkat_core::types::RunResult,
realm: &meerkat_core::RealmId,
) -> SessionResponse {
let mut response: SessionResponse = result.into();
response.session_ref = Some(format_session_ref(realm, &response.session_id));
response
}
fn callback_pending_api_error(
session_id: &SessionId,
realm: &meerkat_core::RealmId,
tool_use_id: String,
tool_name: String,
args: Value,
session_created: bool,
) -> ApiError {
callback_batch_pending_api_error(
session_id,
realm,
vec![meerkat_core::error::PendingCallbackToolCall {
tool_use_id,
tool_name,
args,
}],
session_created,
)
}
fn callback_batch_pending_api_error(
session_id: &SessionId,
realm: &meerkat_core::RealmId,
pending_tool_calls: Vec<meerkat_core::error::PendingCallbackToolCall>,
session_created: bool,
) -> ApiError {
let first = pending_tool_calls.first();
let tool_name = first
.map(|call| call.tool_name.clone())
.unwrap_or_else(|| "callback_batch".to_string());
let args = first.map(|call| call.args.clone()).unwrap_or(Value::Null);
ApiError::InternalWithData {
message: format!("callback pending for tool '{tool_name}'"),
code: "CALLBACK_PENDING".to_string(),
details: json!({
"session_id": session_id.to_string(),
"session_ref": format_session_ref(realm, session_id),
"session_created": session_created,
"resumable": true,
"tool_name": tool_name,
"args": args,
"pending_tool_calls": pending_tool_calls,
}),
}
}
fn session_created_with_turn_failure_api_error(
message: String,
session_id: &SessionId,
realm: &meerkat_core::RealmId,
turn_error_code: Option<String>,
turn_error_details: Option<Value>,
) -> ApiError {
let mut details = serde_json::Map::new();
details.insert("session_id".to_string(), json!(session_id.to_string()));
details.insert(
"session_ref".to_string(),
json!(format_session_ref(realm, session_id)),
);
details.insert("session_created".to_string(), json!(true));
details.insert("resumable".to_string(), json!(true));
if let Some(turn_error_code) = turn_error_code {
details.insert("turn_error_code".to_string(), json!(turn_error_code));
}
if let Some(turn_error_details) = turn_error_details {
if let Some(error) = turn_error_details.get("error") {
details.insert("error".to_string(), error.clone());
}
details.insert("turn_error_details".to_string(), turn_error_details);
}
ApiError::InternalWithData {
message,
code: "SESSION_CREATED_WITH_TURN_FAILURE".to_string(),
details: Value::Object(details),
}
}
fn completion_outcome_to_api_result(
outcome: meerkat_runtime::completion::CompletionOutcome,
session_id: &SessionId,
realm: &meerkat_core::RealmId,
session_created: bool,
) -> Result<meerkat_core::types::RunResult, ApiError> {
match outcome {
meerkat_runtime::completion::CompletionOutcome::Completed(run_result) => Ok(*run_result),
meerkat_runtime::completion::CompletionOutcome::CompletedWithoutResult => Err(
ApiError::Internal("turn completed without result".to_string()),
),
meerkat_runtime::completion::CompletionOutcome::CallbackPending {
tool_use_id,
tool_name,
args,
} => Err(callback_pending_api_error(
session_id,
realm,
tool_use_id,
tool_name,
args,
session_created,
)),
meerkat_runtime::completion::CompletionOutcome::CallbackBatchPending {
pending_tool_calls,
} => Err(callback_batch_pending_api_error(
session_id,
realm,
pending_tool_calls,
session_created,
)),
meerkat_runtime::completion::CompletionOutcome::Cancelled => {
Err(ApiError::RequestCancelled { details: None })
}
meerkat_runtime::completion::CompletionOutcome::Abandoned { reason, .. } => {
let message = format!("turn abandoned: {reason}");
if session_created {
Err(session_created_with_turn_failure_api_error(
message, session_id, realm, None, None,
))
} else {
Err(ApiError::Internal(message))
}
}
meerkat_runtime::completion::CompletionOutcome::AbandonedWithError { reason, error } => {
let message = format!("turn abandoned: {reason}");
let details = json!({
"error": error,
});
if session_created {
Err(session_created_with_turn_failure_api_error(
message,
session_id,
realm,
Some("TURN_ABANDONED".to_string()),
Some(details),
))
} else {
Err(ApiError::InternalWithData {
message,
code: "TURN_ABANDONED".to_string(),
details,
})
}
}
meerkat_runtime::completion::CompletionOutcome::CompletedWithFinalizationFailure {
error,
} => {
Err(ApiError::InternalWithData {
message: error
.detail
.clone()
.unwrap_or_else(|| "turn finalization failed".to_string()),
code: "TURN_FINALIZATION_FAILED".to_string(),
details: json!({
"error": error,
}),
})
}
meerkat_runtime::completion::CompletionOutcome::RuntimeTerminated { reason, .. } => {
Err(ApiError::Internal(format!("runtime terminated: {reason}")))
}
}
}
async fn completion_wait_error_to_api_error(
state: &AppState,
error: meerkat_runtime::completion::CompletionWaitError,
session_id: &SessionId,
realm: &meerkat_core::RealmId,
session_created: bool,
) -> ApiError {
match state
.runtime_adapter
.resolve_runtime_completion_wait_failure(session_id, &error)
.await
{
Ok(authority) => match authority.public_error_class {
meerkat_runtime::meerkat_machine::dsl::RuntimeCompletionWaitFailurePublicErrorClass::InternalError => {
ApiError::InternalWithData {
message: format!("runtime completion waiter failed: {error}"),
code: runtime_completion_wait_failure_api_code(authority.public_reason)
.to_string(),
details: json!({
"session_id": session_id.to_string(),
"session_ref": format_session_ref(realm, session_id),
"session_created": session_created,
"public_error_class": format!("{:?}", authority.public_error_class),
"resumable": authority.resumable,
}),
}
}
},
Err(authority_error) => ApiError::InternalWithData {
message: format!(
"runtime completion waiter failed without generated failure authority: {error}"
),
code: "RUNTIME_COMPLETION_WAIT_FAILURE_AUTHORITY_UNAVAILABLE".to_string(),
details: json!({
"session_id": session_id.to_string(),
"session_ref": format_session_ref(realm, session_id),
"session_created": session_created,
"authority_error": authority_error.to_string(),
}),
},
}
}
fn runtime_completion_wait_failure_api_code(
public_reason: meerkat_runtime::meerkat_machine::dsl::RuntimeCompletionWaitFailurePublicReason,
) -> &'static str {
match public_reason {
meerkat_runtime::meerkat_machine::dsl::RuntimeCompletionWaitFailurePublicReason::CompletionChannelClosed => {
"RUNTIME_COMPLETION_CHANNEL_CLOSED"
}
meerkat_runtime::meerkat_machine::dsl::RuntimeCompletionWaitFailurePublicReason::CompletionAuthorityUnavailable => {
"RUNTIME_COMPLETION_AUTHORITY_UNAVAILABLE"
}
}
}
fn missing_runtime_terminal_evidence_error(
request_kind: &'static str,
outcome: &meerkat_runtime::AcceptOutcome,
session_id: &SessionId,
realm: &meerkat_core::RealmId,
session_created: bool,
) -> ApiError {
ApiError::InternalWithData {
message: format!(
"runtime accepted REST {request_kind} input without completion terminal evidence"
),
code: "MISSING_RUNTIME_TERMINAL_EVIDENCE".to_string(),
details: json!({
"session_id": session_id.to_string(),
"session_ref": format_session_ref(realm, session_id),
"session_created": session_created,
"resumable": true,
"accept_outcome": format!("{outcome:?}"),
"authority": "runtime/session",
}),
}
}
async fn completion_handle_to_api_result(
request_kind: &'static str,
state: &AppState,
outcome: &meerkat_runtime::AcceptOutcome,
handle: Option<meerkat_runtime::completion::CompletionHandle>,
session_id: &SessionId,
realm: &meerkat_core::RealmId,
session_created: bool,
) -> Result<meerkat_core::types::RunResult, ApiError> {
let Some(handle) = handle else {
return Err(missing_runtime_terminal_evidence_error(
request_kind,
outcome,
session_id,
realm,
session_created,
));
};
let completion = match handle.wait().await {
Ok(completion) => completion,
Err(error) => {
return Err(completion_wait_error_to_api_error(
state,
error,
session_id,
realm,
session_created,
)
.await);
}
};
completion_outcome_to_api_result(completion, session_id, realm, session_created)
}
fn resolve_session_id_for_state(input: &str, state: &AppState) -> Result<SessionId, ApiError> {
let locator = SessionLocator::parse(input)
.map_err(|e| ApiError::BadRequest(format!("Invalid session locator '{input}': {e}")))?;
if let Some(locator_realm) = locator.realm_id.as_ref()
&& locator_realm != &state.realm
{
return Err(ApiError::BadRequest(format!(
"Session locator realm '{}' does not match active realm '{}'",
locator_realm, state.realm
)));
}
Ok(locator.session_id)
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum InterruptNoopTarget {
Present,
Missing,
}
fn interrupt_noop_target_for_presence(present: bool) -> InterruptNoopTarget {
if present {
InterruptNoopTarget::Present
} else {
InterruptNoopTarget::Missing
}
}
async fn interrupt_noop_target(
state: &AppState,
session_id: &SessionId,
) -> Result<InterruptNoopTarget, ApiError> {
match state
.session_service
.load_authoritative_session(session_id)
.await
{
Ok(Some(session)) => {
if session_archived_by_authority(state, session_id, &session)
.await
.map_err(|err| ApiError::Internal(err.to_string()))?
{
return Ok(InterruptNoopTarget::Missing);
}
if session_metadata_marks_mob_member(&session) {
#[cfg(feature = "mob")]
{
let owns_mob_session =
state.mob_state.owns_live_bridge_session(session_id).await
|| state
.mob_state
.owns_persisted_bridge_session(session_id)
.await;
return Ok(interrupt_noop_target_for_presence(owns_mob_session));
}
#[cfg(not(feature = "mob"))]
return Ok(InterruptNoopTarget::Missing);
}
return Ok(interrupt_noop_target_for_presence(true));
}
Ok(_) => {}
Err(SessionError::NotFound { .. }) => {}
Err(err) => {
return Err(ApiError::Internal(format!(
"Failed to load session before interrupt no-op: {err}"
)));
}
}
#[cfg(feature = "mob")]
if state.mob_state.owns_live_bridge_session(session_id).await
|| state
.mob_state
.owns_persisted_bridge_session(session_id)
.await
{
match state.mob_state.session_service().read(session_id).await {
Ok(_) => {
return Ok(interrupt_noop_target_for_presence(true));
}
Err(SessionError::NotFound { .. }) => {}
Err(err) => {
return Err(ApiError::Internal(format!(
"Failed to inspect mob session before interrupt no-op: {err}"
)));
}
}
}
match state.session_service.read(session_id).await {
Ok(_) => {
return Ok(interrupt_noop_target_for_presence(true));
}
Err(SessionError::NotFound { .. }) => {}
Err(err) => {
return Err(ApiError::Internal(format!(
"Failed to inspect session before interrupt no-op: {err}"
)));
}
}
Ok(interrupt_noop_target_for_presence(false))
}
fn resolve_schedule_id(input: &str) -> Result<meerkat::ScheduleId, ApiError> {
meerkat::ScheduleId::parse(input)
.map_err(|e| ApiError::BadRequest(format!("Invalid schedule id '{input}': {e}")))
}
fn schedule_error_to_api(error: meerkat::ScheduleDomainError) -> ApiError {
match error {
meerkat::ScheduleDomainError::Store(meerkat::ScheduleStoreError::ScheduleNotFound {
schedule_id,
}) => ApiError::NotFound(format!("Schedule not found: {schedule_id}")),
meerkat::ScheduleDomainError::Store(meerkat::ScheduleStoreError::UnsupportedBackend {
..
}) => ApiError::ServiceUnavailable(error.to_string()),
meerkat::ScheduleDomainError::InvalidSchedule(_)
| meerkat::ScheduleDomainError::InvalidTrigger(_)
| meerkat::ScheduleDomainError::InvalidCron(_) => ApiError::BadRequest(error.to_string()),
other => ApiError::Internal(other.to_string()),
}
}
fn schedule_tool_error_to_api(error: meerkat::ScheduleToolError) -> ApiError {
match error.code {
meerkat::SCHEDULE_TOOL_INVALID_ARGUMENTS => ApiError::BadRequest(error.message),
meerkat::SCHEDULE_TOOL_NOT_FOUND => ApiError::NotFound(error.message),
meerkat::SCHEDULE_TOOL_CAPABILITY_UNAVAILABLE => {
ApiError::ServiceUnavailable(error.message)
}
_ => ApiError::Internal(error.message),
}
}
fn workgraph_error_to_api(error: meerkat::WorkGraphError) -> ApiError {
let message = error.to_string();
match meerkat::WorkGraphMachine::public_error_class(&error) {
Ok(public_class) => workgraph_public_error_class_to_api(public_class, message),
Err(classification_error) => ApiError::Internal(format!(
"generated WorkGraph error classification failed: {classification_error}; original error: {message}"
)),
}
}
fn workgraph_public_error_class_to_api(
public_class: meerkat::WorkGraphPublicErrorClass,
message: String,
) -> ApiError {
match public_class {
meerkat::WorkGraphPublicErrorClass::NotFound => ApiError::NotFound(message),
meerkat::WorkGraphPublicErrorClass::Conflict
| meerkat::WorkGraphPublicErrorClass::InvalidTransition
| meerkat::WorkGraphPublicErrorClass::InvalidArguments => ApiError::BadRequest(message),
meerkat::WorkGraphPublicErrorClass::CapabilityUnavailable => {
ApiError::ServiceUnavailable(message)
}
meerkat::WorkGraphPublicErrorClass::StoreError => ApiError::Internal(message),
}
}
#[derive(Debug, Deserialize, Default)]
struct WorkGraphItemsQuery {
#[serde(default)]
realm_id: Option<String>,
#[serde(default)]
namespace: Option<meerkat::WorkNamespace>,
#[serde(default)]
all_namespaces: bool,
#[serde(default)]
statuses: Vec<meerkat::WorkStatus>,
#[serde(default)]
labels: Vec<String>,
#[serde(default)]
include_terminal: bool,
#[serde(default)]
limit: Option<usize>,
}
#[derive(Debug, Deserialize, Default)]
struct WorkGraphReadyQuery {
#[serde(default)]
realm_id: Option<String>,
#[serde(default)]
namespace: Option<meerkat::WorkNamespace>,
#[serde(default)]
labels: Vec<String>,
#[serde(default)]
limit: Option<usize>,
}
#[derive(Debug, Deserialize, Default)]
struct WorkGraphEventsQuery {
#[serde(default)]
realm_id: Option<String>,
#[serde(default)]
namespace: Option<meerkat::WorkNamespace>,
#[serde(default)]
all_namespaces: bool,
#[serde(default)]
after_seq: Option<i64>,
#[serde(default)]
limit: Option<usize>,
}
impl From<WorkGraphItemsQuery> for meerkat::WorkItemFilter {
fn from(value: WorkGraphItemsQuery) -> Self {
Self {
realm_id: value.realm_id,
namespace: value.namespace,
all_namespaces: value.all_namespaces,
statuses: value.statuses,
labels: value.labels,
include_terminal: value.include_terminal,
limit: value.limit,
}
}
}
impl From<WorkGraphItemsQuery> for meerkat::WorkGraphSnapshotFilter {
fn from(value: WorkGraphItemsQuery) -> Self {
Self {
realm_id: value.realm_id,
namespace: value.namespace,
all_namespaces: value.all_namespaces,
statuses: value.statuses,
labels: value.labels,
include_terminal: value.include_terminal,
limit: value.limit,
}
}
}
impl From<WorkGraphReadyQuery> for meerkat::ReadyWorkFilter {
fn from(value: WorkGraphReadyQuery) -> Self {
Self {
realm_id: value.realm_id,
namespace: value.namespace,
labels: value.labels,
limit: value.limit,
}
}
}
impl From<WorkGraphEventsQuery> for meerkat::WorkGraphEventFilter {
fn from(value: WorkGraphEventsQuery) -> Self {
Self {
realm_id: value.realm_id,
namespace: value.namespace,
all_namespaces: value.all_namespaces,
after_seq: value.after_seq,
limit: value.limit,
}
}
}
fn extract_request_context(
headers: &axum::http::HeaderMap,
executor: &SurfaceRequestExecutor,
) -> Result<Option<RequestContext>, ApiError> {
let Some(header_value) = headers.get("x-meerkat-request-id") else {
return Ok(None);
};
let request_id = header_value
.to_str()
.map_err(|_| ApiError::BadRequest("X-Meerkat-Request-Id must be valid UTF-8".into()))?
.trim();
if request_id.is_empty() {
return Err(ApiError::BadRequest(
"X-Meerkat-Request-Id must not be empty".into(),
));
}
match executor.try_begin_request(request_id, noop_request_action()) {
Ok(ctx) => Ok(Some(ctx)),
Err(RequestAdmissionError::AlreadyExists) => Err(ApiError::DuplicateRequestId {
request_id: request_id.to_string(),
}),
Err(RequestAdmissionError::AuthorityRejected { .. }) => Err(ApiError::Internal(format!(
"generated request authority rejected admission for request: {request_id}"
))),
}
}
async fn install_exact_rest_input_cancel(
context: &RequestContext,
adapter: Arc<meerkat_runtime::MeerkatMachine>,
session_id: SessionId,
input_id: meerkat_core::lifecycle::InputId,
) -> meerkat::surface::CancelActionInstallOutcome {
context
.install_cancel_action_or_cancelled(request_action(move || {
let adapter = Arc::clone(&adapter);
let session_id = session_id.clone();
let input_id = input_id.clone();
async move {
if let Err(error) = adapter
.cancel_input_if_present(&session_id, &input_id, "REST request cancelled")
.await
{
tracing::warn!(
%session_id,
%input_id,
%error,
"exact REST request input cancellation did not settle"
);
}
}
}))
.await
}
async fn with_request_lifecycle(
executor: &SurfaceRequestExecutor,
ctx: Option<RequestContext>,
outcome: RequestTerminal<Result<Json<SessionResponse>, ApiError>>,
) -> Result<Json<SessionResponse>, ApiError> {
let request_key = ctx.as_ref().map(|ctx| ctx.key().to_string());
match executor
.resolve_terminal(request_key.as_deref(), outcome)
.await
{
RequestTerminalResolution::Emit(val) => val,
RequestTerminalResolution::Cancelled => Err(ApiError::RequestCancelled { details: None }),
RequestTerminalResolution::LifecycleError(err) => Err(ApiError::Internal(format!(
"request lifecycle rejected publish response: {err}"
))),
}
}
async fn create_session(
State(state): State<AppState>,
headers: axum::http::HeaderMap,
Json(req): Json<CreateSessionRequest>,
) -> Result<Json<SessionResponse>, ApiError> {
let req_ctx = extract_request_context(&headers, &state.request_executor)?;
let executor = state.request_executor.clone();
let outcome = Box::pin(create_session_inner(&state, req, req_ctx.clone())).await;
with_request_lifecycle(&executor, req_ctx, outcome).await
}
async fn help(
State(state): State<AppState>,
headers: axum::http::HeaderMap,
Json(req): Json<meerkat_contracts::HelpRequest>,
) -> Result<Json<SessionResponse>, ApiError> {
let create_req = help_request_to_create_session(req)?;
let req_ctx = extract_request_context(&headers, &state.request_executor)?;
let executor = state.request_executor.clone();
let outcome = Box::pin(create_session_inner(&state, create_req, req_ctx.clone())).await;
with_request_lifecycle(&executor, req_ctx, outcome).await
}
fn help_request_to_create_session(
req: meerkat_contracts::HelpRequest,
) -> Result<CreateSessionRequest, ApiError> {
let prompt = meerkat::help::render_help_prompt(&req)
.map_err(|err| ApiError::BadRequest(err.to_string()))?;
let provider = req
.provider
.as_deref()
.map(|raw| {
Provider::parse_strict(raw)
.ok_or_else(|| ApiError::BadRequest(format!("invalid help provider `{raw}`")))
})
.transpose()?;
Ok(CreateSessionRequest {
prompt: prompt.into(),
injected_context: None,
transient_turn_context: None,
system_prompt: meerkat::SystemPromptOverride::Set(
meerkat::help::help_system_prompt().to_string(),
),
model: req.model,
provider,
max_tokens: req.max_tokens,
auth_binding: None,
output_schema: None,
structured_output_retries: None,
verbose: false,
keep_alive: Some(false),
comms_name: None,
peer_meta: None,
hooks_override: None,
enable_builtins: Some(false),
enable_shell: Some(false),
enable_memory: Some(false),
enable_mob: Some(false),
enable_schedule: Some(false),
enable_web_search: Some(false),
enable_workgraph: Some(false),
budget_limits: None,
provider_params: None,
preload_skills: Some(meerkat::help::platform_preload_skills()),
skill_refs: None,
labels: None,
additional_instructions: None,
app_context: None,
shell_env: None,
})
}
fn create_session_error_to_api(err: SessionError) -> ApiError {
let message = err.to_string();
match &err {
SessionError::NotFound { .. } => ApiError::NotFound(message),
SessionError::Busy { .. } => ApiError::BadRequest(message),
SessionError::Agent(meerkat_core::error::AgentError::Cancelled) => {
ApiError::RequestCancelled { details: None }
}
SessionError::Agent(meerkat_core::error::AgentError::ConfigError(_)) => {
ApiError::BadRequest(message)
}
_ => ApiError::Agent(message),
}
}
async fn create_session_inner(
state: &AppState,
req: CreateSessionRequest,
req_ctx: Option<RequestContext>,
) -> RequestTerminal<Result<Json<SessionResponse>, ApiError>> {
if let Err(e) = validate_public_peer_meta(req.peer_meta.as_ref()) {
return RequestTerminal::RespondWithoutPublish(Err(e));
}
if let Err(e) = validate_public_surface_metadata(req.labels.as_ref(), req.app_context.as_ref())
{
return RequestTerminal::RespondWithoutPublish(Err(e));
}
let keep_alive_override = match resolve_keep_alive(req.keep_alive) {
Ok(v) => v,
Err(e) => return RequestTerminal::RespondWithoutPublish(Err(e)),
};
let keep_alive = keep_alive_override.unwrap_or(false);
let resume_override_mask = create_session_resume_override_mask(&req, keep_alive_override);
let model_resolution = match resolve_rest_create_session_model_for_state(
state,
req.model.clone(),
req.provider,
req.auth_binding.clone().map(Into::into),
)
.await
{
Ok(resolution) => resolution,
Err(err) => return RequestTerminal::RespondWithoutPublish(Err(err)),
};
let model = model_resolution.model.clone();
let max_tokens = req.max_tokens.unwrap_or(state.max_tokens);
let skill_references = match canonical_skill_keys_for_state(state, req.skill_refs.clone()).await
{
Ok(v) => v,
Err(e) => return RequestTerminal::RespondWithoutPublish(Err(e)),
};
if let Some(ctx) = req_ctx.as_ref()
&& ctx.cancel_already_requested()
{
return RequestTerminal::RespondWithoutPublish(Err(ApiError::RequestCancelled {
details: None,
}));
}
let pre_session = Session::new();
let session_id = pre_session.id().clone();
let create_admission = match state
.session_service
.reserve_create_session_admission()
.await
{
Ok(admission) => admission,
Err(err) => {
return RequestTerminal::RespondWithoutPublish(Err(ApiError::Conflict(
err.to_string(),
)));
}
};
let bindings = match state
.runtime_adapter
.prepare_bindings(session_id.clone())
.await
{
Ok(v) => v,
Err(e) => {
let message = format!("failed to prepare runtime bindings: {e}");
return RequestTerminal::RespondWithoutPublish(Err(ApiError::Internal(message)));
}
};
let (caller_event_tx, caller_event_rx) = mpsc::channel::<EventEnvelope<AgentEvent>>(100);
let forward_task = spawn_event_forwarder(
caller_event_rx,
state.event_tx.clone(),
session_id.clone(),
req.verbose,
);
#[cfg(feature = "mcp")]
let mcp_external_tools = {
let adapter = Arc::new(McpRouterAdapter::new(McpRouter::new()));
let adapter_dispatcher: Arc<dyn AgentToolDispatcher> = adapter.clone();
let (lifecycle_tx, lifecycle_rx) = mpsc::unbounded_channel();
let mcp_state = SessionMcpState {
adapter,
turn_counter: 0,
lifecycle_tx,
lifecycle_rx,
drain_task_running: Arc::new(AtomicBool::new(false)),
};
state
.mcp_sessions
.write()
.await
.insert(session_id.clone(), mcp_state);
Some(adapter_dispatcher)
};
#[cfg(not(feature = "mcp"))]
let mcp_external_tools: Option<Arc<dyn meerkat_core::AgentToolDispatcher>> = None;
if let Some(ctx) = req_ctx.as_ref() {
if ctx.cancel_already_requested() {
let error = cleanup_rest_session_after_api_error(
state,
&session_id,
ApiError::RequestCancelled { details: None },
"clean up cancelled REST session preclaim",
)
.await;
drop(caller_event_tx);
drain_event_forwarder(&session_id, forward_task).await;
return RequestTerminal::RespondWithoutPublish(Err(error));
}
}
let current_generation = state.config_runtime.get().await.ok().map(|s| s.generation);
let initial_identity =
match resolve_validation_identity_for_state(state, &model, Some(model_resolution.provider))
.await
{
Ok(identity) => identity,
Err(err) => {
let error = cleanup_rest_session_after_api_error(
state,
&session_id,
ApiError::BadRequest(err),
"clean up REST runtime after validation error",
)
.await;
drop(caller_event_tx);
drain_event_forwarder(&session_id, forward_task).await;
return RequestTerminal::RespondWithoutPublish(Err(error));
}
};
let mut build = SessionBuildOptions {
tool_access_policy: None,
custom_models: std::collections::BTreeMap::new(),
image_generation_provider: None,
auto_compact_threshold_override: None,
provider: Some(model_resolution.provider),
override_comms: Default::default(),
self_hosted_server_id: initial_identity.self_hosted_server_id.clone(),
output_schema: req.output_schema,
structured_output_retries: req.structured_output_retries,
hooks_override: req.hooks_override.unwrap_or_default(),
comms_name: req.comms_name.clone(),
peer_meta: req.peer_meta.clone(),
resume_session: Some(pre_session),
budget_limits: req.budget_limits,
provider_params: req.provider_params.clone().map(Into::into),
external_tools: mcp_external_tools,
mcp_servers: Vec::new(),
recoverable_tool_defs: None,
llm_client_override: state
.llm_client_override
.clone()
.map(encode_llm_client_override_for_service),
session_comms_runtime_override: None,
host_prompt_sections: Default::default(),
agent_llm_client_decorator: None,
override_builtins: ToolCategoryOverride::from_override(req.enable_builtins),
override_shell: ToolCategoryOverride::from_override(req.enable_shell),
override_schedule: ToolCategoryOverride::from_override(req.enable_schedule),
override_workgraph: ToolCategoryOverride::from_override(req.enable_workgraph),
override_memory: ToolCategoryOverride::from_override(req.enable_memory),
override_mob: ToolCategoryOverride::Inherit,
override_image_generation: ToolCategoryOverride::Inherit,
override_web_search: ToolCategoryOverride::from_override(req.enable_web_search),
schedule_tools: None,
workgraph_tools: None,
mob_tool_authority_context: None,
preload_skills: req.preload_skills.clone(),
realm_id: Some(state.realm.clone()),
instance_id: state.instance_id.clone(),
backend: meerkat_core::RecoveryBackendKind::parse(&state.backend),
config_generation: current_generation,
auth_binding: model_resolution.auth_binding,
mob_member_binding: None,
keep_alive,
checkpointer: None,
silent_comms_intents: Vec::new(),
max_inline_peer_notifications: None,
app_context: req.app_context,
additional_instructions: req.additional_instructions,
initial_metadata_entries: std::collections::BTreeMap::new(),
initial_tool_filter: None,
shell_env: req.shell_env,
resume_override_mask,
call_timeout_override: Default::default(),
blob_store_override: None,
mob_tools: None,
runtime_build_mode: meerkat_core::RuntimeBuildMode::SessionOwned(bindings),
initial_turn_metadata: None,
};
build.apply_generated_create_only_mob_operator_access(ToolCategoryOverride::from_override(
req.enable_mob,
));
let create_provider = build.provider;
let svc_req = SvcCreateSessionRequest {
injected_context: Vec::new(),
model: model.to_string(),
prompt: req.prompt.clone(),
system_prompt: req.system_prompt,
max_tokens: Some(max_tokens),
event_tx: Some(caller_event_tx.clone()),
initial_turn: InitialTurnPolicy::Defer,
deferred_prompt_policy: DeferredPromptPolicy::Discard,
build: Some(build),
labels: req.labels,
};
let validation_identity =
match resolve_validation_identity_for_state(state, &svc_req.model, create_provider).await {
Ok(identity) => identity,
Err(err) => {
let error = cleanup_rest_session_after_api_error(
state,
&session_id,
ApiError::BadRequest(err),
"clean up REST runtime after validation error",
)
.await;
drop(caller_event_tx);
drain_event_forwarder(&session_id, forward_task).await;
return RequestTerminal::RespondWithoutPublish(Err(error));
}
};
if let Err(err) =
validate_prompt_video_input_for_state(state, &svc_req.prompt, &validation_identity).await
{
let error = cleanup_rest_session_after_api_error(
state,
&session_id,
prompt_video_input_error_to_api(err),
"clean up REST runtime after prompt validation error",
)
.await;
drop(caller_event_tx);
drain_event_forwarder(&session_id, forward_task).await;
return RequestTerminal::RespondWithoutPublish(Err(error));
}
let adapter = state.runtime_adapter.clone();
let create_result = match state
.session_service
.create_session_with_reserved_admission(svc_req, create_admission)
.await
{
Ok(result) => result,
Err(err) => {
let error = cleanup_rest_session_after_api_error(
state,
&session_id,
create_session_error_to_api(err),
"clean up REST runtime after session creation error",
)
.await;
drop(caller_event_tx);
drain_event_forwarder(&session_id, forward_task).await;
return RequestTerminal::RespondWithoutPublish(Err(error));
}
};
if let Err(error) = ensure_rest_session_runtime_executor(state, &create_result.session_id).await
{
let error = cleanup_rest_session_after_api_error(
state,
&session_id,
runtime_executor_attach_error_to_api(error),
"clean up REST runtime after executor attach failure",
)
.await;
drop(caller_event_tx);
drain_event_forwarder(&session_id, forward_task).await;
return RequestTerminal::RespondWithoutPublish(Err(error));
}
#[cfg(feature = "comms")]
{
let comms_rt = state.session_service.comms_runtime(&session_id).await;
if let Err(error) = adapter
.update_peer_ingress_context(&session_id, keep_alive, comms_rt)
.await
{
let error = cleanup_rest_session_after_api_error(
state,
&session_id,
ApiError::Internal(format!("failed to update peer ingress context: {error}")),
"clean up REST runtime after peer-ingress update failure",
)
.await;
drop(caller_event_tx);
drain_event_forwarder(&session_id, forward_task).await;
return RequestTerminal::RespondWithoutPublish(Err(error));
}
}
let input = meerkat_runtime::Input::Prompt(
meerkat_runtime::PromptInput::from_content_input(
req.prompt,
Some(
meerkat_core::lifecycle::run_primitive::RuntimeTurnMetadata {
keep_alive: resolve_turn_keep_alive_policy(keep_alive_override),
skill_references,
turn_tool_overlay: None,
additional_instructions: None,
transient_turn_context: req.transient_turn_context.clone(),
..Default::default()
},
),
)
.with_injected_context(req.injected_context.unwrap_or_default()),
);
if let Some(ctx) = req_ctx.as_ref()
&& ctx.cancel_already_requested()
{
drop(caller_event_tx);
drain_event_forwarder(&session_id, forward_task).await;
return RequestTerminal::RespondWithoutPublish(Err(ApiError::RequestCancelled {
details: None,
}));
}
let (outcome, handle) = match adapter
.accept_input_with_completion(&create_result.session_id, input)
.await
{
Ok(pair) => pair,
Err(err) => {
drop(caller_event_tx);
drain_event_forwarder(&session_id, forward_task).await;
return RequestTerminal::Publish(Err(ApiError::InternalWithData {
message: err.to_string(),
code: "SESSION_CREATED_WITH_TURN_FAILURE".to_string(),
details: json!({
"session_id": session_id.to_string(),
"session_ref": format_session_ref(&state.realm, &session_id),
"session_created": true,
"resumable": true,
}),
}));
}
};
if let meerkat_runtime::AcceptOutcome::Accepted { input_id, .. } = &outcome
&& let Some(ctx) = req_ctx.as_ref()
{
let _ = install_exact_rest_input_cancel(
ctx,
Arc::clone(&adapter),
session_id.clone(),
input_id.clone(),
)
.await;
}
let handle = handle.map(|handle| {
wrap_rest_runtime_completion_cleanup(state.clone(), session_id.clone(), handle)
});
let result = completion_handle_to_api_result(
"create",
state,
&outcome,
handle,
&session_id,
&state.realm,
true,
)
.await;
drop(caller_event_tx);
drain_event_forwarder(&session_id, forward_task).await;
match result {
Ok(run_result) => {
RequestTerminal::Publish(Ok(Json(run_result_to_response(run_result, &state.realm))))
}
Err(err) => {
let message = match err {
ApiError::Internal(msg) => msg,
other => return RequestTerminal::Publish(Err(other)),
};
RequestTerminal::Publish(Err(session_created_with_turn_failure_api_error(
message,
&session_id,
&state.realm,
None,
None,
)))
}
}
}
fn parse_label_filters(
raw: Option<Vec<String>>,
) -> Result<Option<BTreeMap<String, String>>, String> {
let labels = match raw {
Some(l) if !l.is_empty() => l,
_ => return Ok(None),
};
let mut map = BTreeMap::new();
for s in labels {
let (k, v) = s
.split_once('=')
.ok_or_else(|| format!("malformed label filter, expected key=value: {s}"))?;
map.insert(k.to_string(), v.to_string());
}
Ok(if map.is_empty() { None } else { Some(map) })
}
async fn create_schedule(
State(state): State<AppState>,
Json(request): Json<meerkat::CreateScheduleRequest>,
) -> Result<Json<meerkat_contracts::wire::Schedule>, ApiError> {
state
.ensure_schedule_host_started()
.await
.map_err(schedule_error_to_api)?;
state
.schedule_service
.create(request)
.await
.map(meerkat_contracts::wire::Schedule::from)
.map(Json)
.map_err(schedule_error_to_api)
}
async fn schedule_tools() -> Json<Value> {
Json(json!({ "tools": schedule_tools_list() }))
}
async fn schedule_call(
State(state): State<AppState>,
Json(req): Json<meerkat_contracts::wire::ScheduleToolCallParams>,
) -> Result<Json<Value>, ApiError> {
state
.ensure_schedule_host_started()
.await
.map_err(schedule_error_to_api)?;
handle_schedule_tools_call(&state.schedule_service, &req.name, &req.arguments)
.await
.map(Json)
.map_err(schedule_tool_error_to_api)
}
async fn workgraph_list_items(
State(state): State<AppState>,
Query(query): Query<WorkGraphItemsQuery>,
) -> Result<Json<meerkat::WorkGraphItemsResponse>, ApiError> {
state
.workgraph_service
.list(query.into())
.await
.map(|items| Json(meerkat::WorkGraphItemsResponse { items }))
.map_err(workgraph_error_to_api)
}
async fn workgraph_get_item(
State(state): State<AppState>,
Path(id): Path<String>,
Query(query): Query<WorkGraphReadyQuery>,
) -> Result<Json<meerkat::WorkItem>, ApiError> {
let id = meerkat::WorkItemId::new(id).map_err(workgraph_error_to_api)?;
state
.workgraph_service
.get(query.realm_id, query.namespace, id)
.await
.map(Json)
.map_err(workgraph_error_to_api)
}
async fn workgraph_ready(
State(state): State<AppState>,
Query(query): Query<WorkGraphReadyQuery>,
) -> Result<Json<meerkat::WorkGraphItemsResponse>, ApiError> {
state
.workgraph_service
.ready(query.into())
.await
.map(|items| Json(meerkat::WorkGraphItemsResponse { items }))
.map_err(workgraph_error_to_api)
}
async fn workgraph_snapshot(
State(state): State<AppState>,
Query(query): Query<WorkGraphItemsQuery>,
) -> Result<Json<meerkat::WorkGraphSnapshot>, ApiError> {
state
.workgraph_service
.snapshot(query.into())
.await
.map(Json)
.map_err(workgraph_error_to_api)
}
async fn workgraph_events(
State(state): State<AppState>,
Query(query): Query<WorkGraphEventsQuery>,
) -> Result<Json<meerkat::WorkGraphEventsResponse>, ApiError> {
state
.workgraph_service
.events(query.into())
.await
.map(|events| Json(meerkat::WorkGraphEventsResponse { events }))
.map_err(workgraph_error_to_api)
}
async fn workgraph_goal_status(
State(state): State<AppState>,
Json(request): Json<meerkat::GoalStatusRequest>,
) -> Result<Json<meerkat::GoalStatusResult>, ApiError> {
state
.workgraph_service
.goal_status(request)
.await
.map(Json)
.map_err(workgraph_error_to_api)
}
async fn workgraph_attention_list(
State(state): State<AppState>,
Json(request): Json<meerkat::AttentionListRequest>,
) -> Result<Json<meerkat::AttentionListResult>, ApiError> {
state
.workgraph_service
.list_attention(request)
.await
.map(Json)
.map_err(workgraph_error_to_api)
}
async fn list_schedules(
State(state): State<AppState>,
) -> Result<Json<ScheduleListResponse>, ApiError> {
state
.schedule_service
.list()
.await
.map(meerkat_contracts::wire::ScheduleListResult::from_domain)
.map(Json)
.map_err(schedule_error_to_api)
}
async fn get_schedule(
State(state): State<AppState>,
Path(id): Path<String>,
) -> Result<Json<meerkat_contracts::wire::Schedule>, ApiError> {
let schedule_id = resolve_schedule_id(&id)?;
state
.schedule_service
.get(&schedule_id)
.await
.map(meerkat_contracts::wire::Schedule::from)
.map(Json)
.map_err(schedule_error_to_api)
}
async fn update_schedule(
State(state): State<AppState>,
Path(id): Path<String>,
Json(request): Json<meerkat::UpdateScheduleRequest>,
) -> Result<Json<meerkat_contracts::wire::Schedule>, ApiError> {
let schedule_id = resolve_schedule_id(&id)?;
state
.ensure_schedule_host_started()
.await
.map_err(schedule_error_to_api)?;
state
.schedule_service
.update(&schedule_id, request)
.await
.map(meerkat_contracts::wire::Schedule::from)
.map(Json)
.map_err(schedule_error_to_api)
}
async fn pause_schedule(
State(state): State<AppState>,
Path(id): Path<String>,
) -> Result<Json<meerkat_contracts::wire::Schedule>, ApiError> {
let schedule_id = resolve_schedule_id(&id)?;
state
.ensure_schedule_host_started()
.await
.map_err(schedule_error_to_api)?;
state
.schedule_service
.pause(&schedule_id)
.await
.map(meerkat_contracts::wire::Schedule::from)
.map(Json)
.map_err(schedule_error_to_api)
}
async fn resume_schedule(
State(state): State<AppState>,
Path(id): Path<String>,
) -> Result<Json<meerkat_contracts::wire::Schedule>, ApiError> {
let schedule_id = resolve_schedule_id(&id)?;
state
.ensure_schedule_host_started()
.await
.map_err(schedule_error_to_api)?;
state
.schedule_service
.resume(&schedule_id)
.await
.map(meerkat_contracts::wire::Schedule::from)
.map(Json)
.map_err(schedule_error_to_api)
}
async fn delete_schedule(
State(state): State<AppState>,
Path(id): Path<String>,
) -> Result<Json<meerkat_contracts::wire::Schedule>, ApiError> {
let schedule_id = resolve_schedule_id(&id)?;
state
.schedule_service
.delete(&schedule_id)
.await
.map(meerkat_contracts::wire::Schedule::from)
.map(Json)
.map_err(schedule_error_to_api)
}
async fn list_schedule_occurrences(
State(state): State<AppState>,
Path(id): Path<String>,
) -> Result<Json<ScheduleOccurrencesResponse>, ApiError> {
let schedule_id = resolve_schedule_id(&id)?;
state
.schedule_service
.list_occurrences(&schedule_id)
.await
.map(|occurrences| Json(ScheduleOccurrencesResponse::from_domain(occurrences)))
.map_err(schedule_error_to_api)
}
async fn list_sessions(
State(state): State<AppState>,
Query(query): Query<ListSessionsQuery>,
) -> Result<Json<meerkat_contracts::wire::ListSessionsResult>, ApiError> {
let label_filters = parse_label_filters(query.label).map_err(ApiError::BadRequest)?;
let sessions = state
.session_service
.list(meerkat_core::service::SessionQuery {
limit: query.limit,
offset: query.offset,
labels: label_filters,
})
.await
.map_err(|e| ApiError::Internal(format!("Failed to list sessions: {e}")))?;
let wire_sessions: Vec<meerkat_contracts::WireSessionSummary> = sessions
.into_iter()
.map(|s| {
let session_ref = format_session_ref(&state.realm, &s.session_id);
let mut wire = meerkat_contracts::WireSessionSummary::from(s);
wire.session_ref = Some(session_ref);
wire
})
.collect();
Ok(Json(meerkat_contracts::wire::ListSessionsResult {
sessions: wire_sessions,
}))
}
async fn get_session(
State(state): State<AppState>,
Path(id): Path<String>,
) -> Result<Json<SessionDetailsResponse>, ApiError> {
let session_id = resolve_session_id_for_state(&id, &state)?;
let view = state
.session_service
.read(&session_id)
.await
.map_err(|e| match e {
SessionError::NotFound { .. } => ApiError::NotFound(format!("Session not found: {id}")),
_ => ApiError::Internal(format!("{e}")),
})?;
let created_at: DateTime<Utc> = view.state.created_at.into();
let updated_at: DateTime<Utc> = view.state.updated_at.into();
Ok(Json(SessionDetailsResponse {
session_id: view.state.session_id.to_string(),
session_ref: format_session_ref(&state.realm, &view.state.session_id),
created_at: created_at.to_rfc3339(),
updated_at: updated_at.to_rfc3339(),
message_count: view.state.message_count,
total_tokens: view.billing.total_tokens,
labels: view.state.labels,
}))
}
async fn get_session_history(
State(state): State<AppState>,
Path(id): Path<String>,
Query(query): Query<SessionHistoryQuery>,
) -> Result<Json<meerkat_contracts::WireSessionHistory>, ApiError> {
let session_id = resolve_session_id_for_state(&id, &state)?;
let history = state
.session_service
.read_history(
&session_id,
meerkat_core::service::SessionHistoryQuery {
offset: query.offset.unwrap_or(0),
limit: query.limit,
},
)
.await
.map_err(|e| match e {
SessionError::NotFound { .. } => ApiError::NotFound(format!("Session not found: {id}")),
SessionError::PersistenceDisabled => {
ApiError::BadRequest(format!("Session history is unavailable for session: {id}"))
}
_ => ApiError::Internal(format!("{e}")),
})?;
let mut wire: meerkat_contracts::WireSessionHistory = history.into();
wire.session_ref = Some(format_session_ref(&state.realm, &session_id));
Ok(Json(wire))
}
async fn interrupt_session(
State(state): State<AppState>,
Path(id): Path<String>,
) -> Result<Json<meerkat_contracts::wire::InterruptResult>, ApiError> {
let session_id = resolve_session_id_for_state(&id, &state)?;
let result = match state
.runtime_adapter
.hard_cancel_current_run(&session_id, "REST session interrupt")
.await
{
Ok(()) => meerkat_runtime::resolve_user_interrupt_public_result(
meerkat_runtime::UserInterruptObservation::Accepted,
true,
false,
),
Err(meerkat_runtime::RuntimeDriverError::NotReady {
state:
observed_state @ (meerkat_runtime::RuntimeState::Idle
| meerkat_runtime::RuntimeState::Attached),
}) => meerkat_runtime::resolve_user_interrupt_public_result(
meerkat_runtime::UserInterruptObservation::NotReady(observed_state),
true,
false,
),
Err(meerkat_runtime::RuntimeDriverError::NotReady {
state: meerkat_runtime::RuntimeState::Destroyed,
})
| Err(meerkat_runtime::RuntimeDriverError::Destroyed) => {
let target_present = matches!(
interrupt_noop_target(&state, &session_id).await?,
InterruptNoopTarget::Present
);
meerkat_runtime::resolve_user_interrupt_public_result(
meerkat_runtime::UserInterruptObservation::Destroyed,
target_present,
false,
)
}
Err(meerkat_runtime::RuntimeDriverError::NotReady { state }) => {
meerkat_runtime::resolve_user_interrupt_public_result(
meerkat_runtime::UserInterruptObservation::NotReady(state),
true,
false,
)
}
Err(e) => {
return Err(ApiError::Internal(format!(
"Failed to interrupt session: {e}"
)));
}
}
.map_err(|e| ApiError::Internal(format!("Failed to classify interrupt result: {e}")))?;
match result {
meerkat_runtime::UserInterruptPublicResult::Interrupted => Ok(Json(
meerkat_contracts::wire::InterruptResult::from_outcome(
session_id.to_string(),
meerkat_contracts::wire::WireInterruptOutcome::Interrupted,
),
)),
meerkat_runtime::UserInterruptPublicResult::StagedNoop => Ok(Json(
meerkat_contracts::wire::InterruptResult::from_outcome(
session_id.to_string(),
meerkat_contracts::wire::WireInterruptOutcome::StagedNoop,
),
)),
meerkat_runtime::UserInterruptPublicResult::NotFound => {
Err(ApiError::NotFound(format!("Session not found: {id}")))
}
meerkat_runtime::UserInterruptPublicResult::SessionBusy => Err(ApiError::Conflict(
"Session is still being materialized and cannot be interrupted yet".to_string(),
)),
meerkat_runtime::UserInterruptPublicResult::Conflict => Err(ApiError::Conflict(
"Session is not interruptible in its current runtime state".to_string(),
)),
}
}
async fn archive_session(
State(state): State<AppState>,
Path(id): Path<String>,
) -> Result<Json<Value>, ApiError> {
let session_id = resolve_session_id_for_state(&id, &state)?;
match archive_session_with_runtime_cleanup(state.clone(), session_id.clone()).await {
Ok(()) => Ok(Json(json!({
"session_id": session_id.to_string(),
"archived": true
}))),
Err(error) => Err(archive_session_error_to_api_error(&id, error)),
}
}
fn archive_session_error_to_api_error(id: &str, error: SessionError) -> ApiError {
match error {
SessionError::NotFound { .. } => ApiError::NotFound(format!("Session not found: {id}")),
SessionError::FailedWithData { message, data } => ApiError::InternalWithData {
message: format!("Failed to archive session: {message}"),
code: data
.get("code")
.and_then(Value::as_str)
.unwrap_or("INTERNAL_ERROR")
.to_string(),
details: data,
},
other => ApiError::Internal(format!("Failed to archive session: {other}")),
}
}
async fn cancel_request(
State(state): State<AppState>,
Path(request_id): Path<String>,
) -> Result<Json<Value>, ApiError> {
use meerkat::surface::CancelOutcome;
match state.request_executor.cancel_request(&request_id).await {
CancelOutcome::Cancelled | CancelOutcome::AlreadyCancelled => Ok(Json(json!({
"request_id": request_id,
"cancelled": true
}))),
CancelOutcome::AlreadyPublished | CancelOutcome::AlreadyCompleted => {
Ok(Json(json!({
"request_id": request_id,
"cancelled": false,
"reason": "already_terminal"
})))
}
CancelOutcome::NotFound => Err(ApiError::NotFound(format!(
"request not found: {request_id}"
))),
CancelOutcome::AuthorityRejected => Err(ApiError::Internal(format!(
"generated request authority rejected cancellation for request: {request_id}"
))),
}
}
fn system_context_error_to_api(err: SessionControlError) -> ApiError {
match err {
SessionControlError::Session(SessionError::NotFound { .. }) => {
ApiError::NotFound("Session not found".to_string())
}
SessionControlError::Session(other) => ApiError::Internal(other.to_string()),
SessionControlError::InvalidRequest { message } => ApiError::BadRequest(message),
SessionControlError::Conflict { key, .. } => ApiError::Conflict(format!(
"system-context idempotency conflict for key '{key}'"
)),
}
}
async fn append_system_context(
State(state): State<AppState>,
Path(id): Path<String>,
Json(req): Json<AppendSystemContextRequest>,
) -> Result<Json<Value>, ApiError> {
let session_id = resolve_session_id_for_state(&id, &state)?;
let svc_req = SvcAppendSystemContextRequest {
content: req.content,
source: req.source,
idempotency_key: req.idempotency_key,
};
let result = state
.session_service
.append_system_context(&session_id, svc_req)
.await
.map_err(system_context_error_to_api)?;
Ok(Json(json!({
"session_id": session_id.to_string(),
"status": result.status,
})))
}
async fn continue_session(
State(state): State<AppState>,
Path(id): Path<String>,
headers: axum::http::HeaderMap,
Json(req): Json<ContinueSessionRequest>,
) -> Result<Json<SessionResponse>, ApiError> {
let req_ctx = extract_request_context(&headers, &state.request_executor)?;
let executor = state.request_executor.clone();
let outcome = Box::pin(continue_session_inner(&state, &id, req, req_ctx.clone())).await;
with_request_lifecycle(&executor, req_ctx, outcome).await
}
async fn continue_session_inner(
state: &AppState,
id: &str,
req: ContinueSessionRequest,
req_ctx: Option<RequestContext>,
) -> RequestTerminal<Result<Json<SessionResponse>, ApiError>> {
if let Err(e) = validate_public_peer_meta(req.peer_meta.as_ref()) {
return RequestTerminal::RespondWithoutPublish(Err(e));
}
let path_session_id = match resolve_session_id_for_state(id, state) {
Ok(v) => v,
Err(e) => return RequestTerminal::RespondWithoutPublish(Err(e)),
};
let body_session_id = match resolve_session_id_for_state(&req.session_id, state) {
Ok(v) => v,
Err(e) => return RequestTerminal::RespondWithoutPublish(Err(e)),
};
if body_session_id != path_session_id {
return RequestTerminal::RespondWithoutPublish(Err(ApiError::BadRequest(format!(
"Session ID mismatch: path={} body={}",
id, req.session_id
))));
}
let session_id = body_session_id;
let keep_alive_override = match resolve_keep_alive(req.keep_alive) {
Ok(v) => v,
Err(e) => return RequestTerminal::RespondWithoutPublish(Err(e)),
};
let skill_references = match canonical_skill_keys_for_state(state, req.skill_refs.clone()).await
{
Ok(v) => v,
Err(e) => return RequestTerminal::RespondWithoutPublish(Err(e)),
};
if let Some(ctx) = req_ctx.as_ref()
&& ctx.cancel_already_requested()
{
return RequestTerminal::RespondWithoutPublish(Err(ApiError::RequestCancelled {
details: None,
}));
}
let (caller_event_tx, caller_event_rx) = mpsc::channel::<EventEnvelope<AgentEvent>>(100);
let forward_task = spawn_event_forwarder(
caller_event_rx,
state.event_tx.clone(),
session_id.clone(),
req.verbose,
);
let loaded_session = match state
.session_service
.load_authoritative_session(&session_id)
.await
{
Ok(v) => v,
Err(e) => {
drop(caller_event_tx);
drain_event_forwarder(&session_id, forward_task).await;
return RequestTerminal::RespondWithoutPublish(Err(ApiError::Internal(format!(
"Failed to load session: {e}"
))));
}
};
if let Some(session) = loaded_session.as_ref() {
match session_archived_by_authority(state, &session_id, session).await {
Ok(true) => {
let cleanup_result =
discard_and_cleanup_archived_session_runtime(state, &session_id).await;
drop(caller_event_tx);
drain_event_forwarder(&session_id, forward_task).await;
if let Err(error) = cleanup_result {
return RequestTerminal::RespondWithoutPublish(Err(ApiError::Internal(
format!(
"session {session_id} is archived but required runtime cleanup failed: {error}"
),
)));
}
return RequestTerminal::RespondWithoutPublish(Err(ApiError::NotFound(format!(
"Session not found: {session_id}"
))));
}
Ok(false) => {}
Err(err) => {
drop(caller_event_tx);
drain_event_forwarder(&session_id, forward_task).await;
return RequestTerminal::RespondWithoutPublish(Err(ApiError::Internal(format!(
"Failed to load session archive state: {err}"
))));
}
}
}
let mut turn_prompt = req.prompt.clone();
let adapter = state.runtime_adapter.clone();
let final_result = if rest_continue_requires_rebuild(&req) {
let runtime_registration_lock = rest_runtime_registration_lock(state, &session_id);
let runtime_registration_guard = runtime_registration_lock.mutex().lock().await;
let runtime_was_registered = state.runtime_adapter.contains_session(&session_id).await;
let session = match state
.session_service
.load_authoritative_session(&session_id)
.await
{
Ok(Some(session)) => {
match session_archived_by_authority(state, &session_id, &session).await {
Ok(true) => {
let error = unregister_rest_runtime_after_api_error_locked(
state,
&session_id,
runtime_was_registered,
ApiError::NotFound(format!("Session not found: {session_id}")),
)
.await;
drop(caller_event_tx);
drain_event_forwarder(&session_id, forward_task).await;
return RequestTerminal::RespondWithoutPublish(Err(error));
}
Ok(false) => session,
Err(err) => {
let error = unregister_rest_runtime_after_api_error_locked(
state,
&session_id,
runtime_was_registered,
ApiError::Internal(format!(
"Failed to load session archive state: {err}"
)),
)
.await;
drop(caller_event_tx);
drain_event_forwarder(&session_id, forward_task).await;
return RequestTerminal::RespondWithoutPublish(Err(error));
}
}
}
Ok(None) => {
let error = unregister_rest_runtime_after_api_error_locked(
state,
&session_id,
runtime_was_registered,
ApiError::NotFound(format!("Session not found: {session_id}")),
)
.await;
drop(caller_event_tx);
drain_event_forwarder(&session_id, forward_task).await;
return RequestTerminal::RespondWithoutPublish(Err(error));
}
Err(err) => {
let error = unregister_rest_runtime_after_api_error_locked(
state,
&session_id,
runtime_was_registered,
ApiError::Internal(format!("Failed to load session: {err}")),
)
.await;
drop(caller_event_tx);
drain_event_forwarder(&session_id, forward_task).await;
return RequestTerminal::RespondWithoutPublish(Err(error));
}
};
let session_defaults =
match meerkat_core::session_recovery::SessionDefaults::from_session(&session) {
Ok(defaults) => defaults,
Err(error) => {
let error = unregister_rest_runtime_after_api_error_locked(
state,
&session_id,
runtime_was_registered,
ApiError::Internal(error.to_string()),
)
.await;
drop(caller_event_tx);
drain_event_forwarder(&session_id, forward_task).await;
return RequestTerminal::RespondWithoutPublish(Err(error));
}
};
let stored_metadata = session_defaults.metadata.clone();
let keep_alive = match keep_alive_override {
Some(val) => val,
None => stored_metadata.keep_alive,
};
let effective_comms_name = req
.comms_name
.clone()
.or_else(|| stored_metadata.comms_name.clone());
if keep_alive
&& effective_comms_name
.as_ref()
.is_none_or(|name| name.trim().is_empty())
{
let error = unregister_rest_runtime_after_api_error_locked(
state,
&session_id,
runtime_was_registered,
ApiError::BadRequest("keep_alive requires comms_name".to_string()),
)
.await;
drop(caller_event_tx);
drain_event_forwarder(&session_id, forward_task).await;
return RequestTerminal::RespondWithoutPublish(Err(error));
}
let rebuild_admission = match state
.session_service
.reserve_runtime_turn_admission(&session_id)
.await
{
Ok(admission) => admission,
Err(err) => {
drop(caller_event_tx);
drain_event_forwarder(&session_id, forward_task).await;
return RequestTerminal::RespondWithoutPublish(Err(ApiError::Conflict(
err.to_string(),
)));
}
};
#[cfg(feature = "mcp")]
if let Err(e) = apply_mcp_boundary_to_turn_prompt(
state,
&session_id,
&caller_event_tx,
&mut turn_prompt,
)
.await
{
let error = unregister_rest_runtime_after_api_error_locked(
state,
&session_id,
runtime_was_registered,
e,
)
.await;
drop(caller_event_tx);
drain_event_forwarder(&session_id, forward_task).await;
return RequestTerminal::RespondWithoutPublish(Err(error));
}
let bindings = match state
.runtime_adapter
.prepare_bindings(session_id.clone())
.await
{
Ok(v) => v,
Err(e) => {
let message = format!("failed to prepare runtime bindings: {e}");
let error = unregister_rest_runtime_after_api_error_locked(
state,
&session_id,
runtime_was_registered,
ApiError::Internal(message),
)
.await;
drop(caller_event_tx);
drain_event_forwarder(&session_id, forward_task).await;
return RequestTerminal::RespondWithoutPublish(Err(error));
}
};
let llm_binding = match meerkat_core::session_recovery::resolve_resume_llm_binding(
stored_metadata.provider,
stored_metadata.self_hosted_server_id.clone(),
req.model.as_deref(),
req.provider,
) {
Ok(binding) => binding,
Err(e) => {
let error = unregister_rest_runtime_after_api_error_locked(
state,
&session_id,
runtime_was_registered,
ApiError::BadRequest(e.to_string()),
)
.await;
drop(caller_event_tx);
drain_event_forwarder(&session_id, forward_task).await;
return RequestTerminal::RespondWithoutPublish(Err(error));
}
};
let auth_binding_override = req.auth_binding.clone().map(Into::into);
let mut build = SessionBuildOptions {
tool_access_policy: None,
custom_models: std::collections::BTreeMap::new(),
image_generation_provider: None,
auto_compact_threshold_override: None,
provider: llm_binding.provider,
override_comms: Default::default(),
self_hosted_server_id: llm_binding.self_hosted_server_id,
output_schema: req.output_schema.clone(),
structured_output_retries: req.structured_output_retries,
hooks_override: req.hooks_override.clone().unwrap_or_default(),
comms_name: req.comms_name.clone(),
peer_meta: req.peer_meta.clone(),
resume_session: Some(session),
budget_limits: None,
provider_params: None,
external_tools: None,
mcp_servers: Vec::new(),
recoverable_tool_defs: None,
llm_client_override: state
.llm_client_override
.clone()
.map(encode_llm_client_override_for_service),
session_comms_runtime_override: None,
host_prompt_sections: Default::default(),
agent_llm_client_decorator: None,
override_builtins: ToolCategoryOverride::Inherit,
override_shell: ToolCategoryOverride::Inherit,
override_memory: ToolCategoryOverride::Inherit,
override_schedule: ToolCategoryOverride::Inherit,
override_workgraph: ToolCategoryOverride::Inherit,
override_mob: ToolCategoryOverride::Inherit,
override_image_generation: ToolCategoryOverride::Inherit,
override_web_search: ToolCategoryOverride::from_override(req.enable_web_search),
schedule_tools: None,
workgraph_tools: None,
mob_tool_authority_context: None,
preload_skills: None,
realm_id: Some(state.realm.clone()),
instance_id: state.instance_id.clone(),
backend: meerkat_core::RecoveryBackendKind::parse(&state.backend),
config_generation: state.config_runtime.get().await.ok().map(|s| s.generation),
auth_binding: auth_binding_override.clone(),
mob_member_binding: None,
keep_alive,
checkpointer: None,
silent_comms_intents: Vec::new(),
max_inline_peer_notifications: None,
app_context: None,
additional_instructions: None,
initial_metadata_entries: std::collections::BTreeMap::new(),
initial_tool_filter: None,
shell_env: None,
resume_override_mask: ResumeOverrideMask {
model: req.model.is_some(),
provider: llm_binding.provider_overridden,
auth_binding: auth_binding_override.is_some(),
max_tokens: req.max_tokens.is_some(),
structured_output_retries: req.structured_output_retries.is_some(),
keep_alive: keep_alive_override.is_some(),
comms_name: req.comms_name.is_some(),
peer_meta: req.peer_meta.is_some(),
override_web_search: req.enable_web_search.is_some(),
..Default::default()
},
call_timeout_override: Default::default(),
blob_store_override: None,
mob_tools: None,
runtime_build_mode: meerkat_core::RuntimeBuildMode::SessionOwned(bindings),
initial_turn_metadata: None,
};
build.apply_generated_create_only_mob_operator_access(ToolCategoryOverride::Inherit);
let model = match req.model.clone() {
Some(model) => model,
None => match resolve_default_model(state).await {
Ok(model) => model,
Err(e) => {
let error = unregister_rest_runtime_after_api_error_locked(
state,
&session_id,
runtime_was_registered,
e,
)
.await;
drop(caller_event_tx);
drain_event_forwarder(&session_id, forward_task).await;
return RequestTerminal::RespondWithoutPublish(Err(error));
}
},
};
let create_req = SvcCreateSessionRequest {
injected_context: Vec::new(),
model,
prompt: turn_prompt.clone(),
system_prompt: meerkat::SystemPromptOverride::Inherit,
max_tokens: req.max_tokens.or(Some(state.max_tokens)),
event_tx: Some(caller_event_tx.clone()),
initial_turn: InitialTurnPolicy::Defer,
deferred_prompt_policy: DeferredPromptPolicy::Discard,
build: Some(build),
labels: None,
};
let create_provider = create_req.build.as_ref().and_then(|build| build.provider);
let validation_identity =
match resolve_validation_identity_for_state(state, &create_req.model, create_provider)
.await
{
Ok(identity) => identity,
Err(err) => {
let error = unregister_rest_runtime_after_api_error_locked(
state,
&session_id,
runtime_was_registered,
ApiError::BadRequest(err),
)
.await;
drop(caller_event_tx);
drain_event_forwarder(&session_id, forward_task).await;
return RequestTerminal::RespondWithoutPublish(Err(error));
}
};
if let Err(err) =
validate_prompt_video_input_for_state(state, &create_req.prompt, &validation_identity)
.await
{
let error = unregister_rest_runtime_after_api_error_locked(
state,
&session_id,
runtime_was_registered,
prompt_video_input_error_to_api(err),
)
.await;
drop(caller_event_tx);
drain_event_forwarder(&session_id, forward_task).await;
return RequestTerminal::RespondWithoutPublish(Err(error));
}
let create_result = match state
.session_service
.create_session_with_reserved_admission(create_req, rebuild_admission)
.await
{
Ok(v) => v,
Err(e) => {
let error = unregister_rest_runtime_after_api_error_locked(
state,
&session_id,
runtime_was_registered,
ApiError::Internal(format!("Failed to rebuild session: {e}")),
)
.await;
drop(caller_event_tx);
drain_event_forwarder(&session_id, forward_task).await;
return RequestTerminal::RespondWithoutPublish(Err(error));
}
};
let rebuild_unpublished_cleanup_requested =
Arc::new(std::sync::atomic::AtomicBool::new(false));
if let Some(ctx) = req_ctx.as_ref() {
let cleanup_state = state.clone();
let cleanup_sid = session_id.clone();
let cleanup_runtime_was_registered = runtime_was_registered;
let cleanup_requested = Arc::clone(&rebuild_unpublished_cleanup_requested);
ctx.set_unpublished_cleanup(request_action(move || {
let state = cleanup_state.clone();
let sid = cleanup_sid.clone();
let cleanup_requested = Arc::clone(&cleanup_requested);
async move {
cleanup_requested.store(true, std::sync::atomic::Ordering::Release);
if let Err(error) =
discard_rebuilt_rest_session(&state, &sid, cleanup_runtime_was_registered)
.await
{
tracing::error!(
session_id = %sid,
error = %error,
"REST unpublished rebuild cleanup failed; registered runtime retains retry ownership"
);
}
}
}));
if ctx.cancel_already_requested() {
let error = discard_rebuilt_rest_session_after_api_error_locked(
state,
&session_id,
runtime_was_registered,
ApiError::RequestCancelled { details: None },
)
.await;
drop(caller_event_tx);
drain_event_forwarder(&session_id, forward_task).await;
return RequestTerminal::RespondWithoutPublish(Err(error));
}
}
#[cfg(feature = "comms")]
{
let comms_rt = state.session_service.comms_runtime(&session_id).await;
if let Err(error) = adapter
.update_peer_ingress_context(&session_id, keep_alive, comms_rt)
.await
{
let error = discard_rebuilt_rest_session_after_api_error_locked(
state,
&session_id,
runtime_was_registered,
ApiError::Internal(format!("failed to update peer ingress context: {error}")),
)
.await;
drop(caller_event_tx);
drain_event_forwarder(&session_id, forward_task).await;
return RequestTerminal::RespondWithoutPublish(Err(error));
}
}
if let Err(error) =
ensure_rest_session_runtime_executor(state, &create_result.session_id).await
{
let error = discard_rebuilt_rest_session_after_api_error_locked(
state,
&session_id,
runtime_was_registered,
runtime_executor_attach_error_to_api(error),
)
.await;
drop(caller_event_tx);
drain_event_forwarder(&session_id, forward_task).await;
return RequestTerminal::RespondWithoutPublish(Err(error));
}
let input = meerkat_runtime::Input::Prompt(
meerkat_runtime::PromptInput::from_content_input(
turn_prompt.clone(),
Some(
meerkat_core::lifecycle::run_primitive::RuntimeTurnMetadata {
keep_alive: resolve_turn_keep_alive_policy(keep_alive_override),
skill_references: skill_references.clone(),
system_prompts: rest_continue_system_prompts(&req),
turn_tool_overlay: req
.turn_tool_overlay
.clone()
.map(meerkat_core::service::TurnToolOverlay::from),
additional_instructions: resolve_turn_additional_instructions(
req.additional_instructions.clone(),
),
transient_turn_context: req.transient_turn_context.clone(),
..Default::default()
},
),
)
.with_injected_context(req.injected_context.clone().unwrap_or_default()),
);
if let Some(ctx) = req_ctx.as_ref()
&& ctx.cancel_already_requested()
{
let error = discard_rebuilt_rest_session_after_api_error_locked(
state,
&session_id,
runtime_was_registered,
ApiError::RequestCancelled { details: None },
)
.await;
drop(caller_event_tx);
drain_event_forwarder(&session_id, forward_task).await;
return RequestTerminal::RespondWithoutPublish(Err(error));
}
let (outcome, handle) = match adapter
.accept_input_with_completion(&create_result.session_id, input)
.await
{
Ok(pair) => pair,
Err(err) => {
let error = discard_rebuilt_rest_session_after_api_error_locked(
state,
&session_id,
runtime_was_registered,
ApiError::Internal(err.to_string()),
)
.await;
drop(caller_event_tx);
drain_event_forwarder(&session_id, forward_task).await;
return RequestTerminal::RespondWithoutPublish(Err(error));
}
};
if let meerkat_runtime::AcceptOutcome::Accepted { input_id, .. } = &outcome
&& let Some(ctx) = req_ctx.as_ref()
{
let _ = install_exact_rest_input_cancel(
ctx,
Arc::clone(&adapter),
session_id.clone(),
input_id.clone(),
)
.await;
}
drop(runtime_registration_guard);
drop(runtime_registration_lock);
let handle = handle.map(|handle| {
let cleanup_state = state.clone();
let cleanup_session_id = session_id.clone();
let cleanup_requested = Arc::clone(&rebuild_unpublished_cleanup_requested);
handle.with_resultful_completion_cleanup(move |completion| async move {
let Ok(cleanup_observation) = completion else {
return Ok(());
};
if cleanup_requested.load(std::sync::atomic::Ordering::Acquire) {
discard_rebuilt_rest_session(
&cleanup_state,
&cleanup_session_id,
runtime_was_registered,
)
.await
.map_err(|error| rest_completion_cleanup_error(&cleanup_session_id, error))?;
}
cleanup_rest_runtime_after_completion_outcome(
&cleanup_state,
&cleanup_session_id,
cleanup_observation,
)
.await
.map(|_| ())
.map_err(|error| rest_completion_cleanup_error(&cleanup_session_id, error))
})
});
completion_handle_to_api_result(
"continue_rebuild",
state,
&outcome,
handle,
&session_id,
&state.realm,
false,
)
.await
} else {
let runtime_registration_lock = rest_runtime_registration_lock(state, &session_id);
let runtime_registration_guard = runtime_registration_lock.mutex().lock().await;
let runtime_was_registered = state.runtime_adapter.contains_session(&session_id).await;
let session = match state
.session_service
.load_authoritative_session(&session_id)
.await
{
Ok(Some(session)) => {
match session_archived_by_authority(state, &session_id, &session).await {
Ok(true) => {
let cleanup_result =
discard_and_cleanup_archived_session_runtime(state, &session_id).await;
drop(caller_event_tx);
drain_event_forwarder(&session_id, forward_task).await;
if let Err(error) = cleanup_result {
return RequestTerminal::RespondWithoutPublish(Err(
ApiError::Internal(format!(
"session {session_id} is archived but required runtime cleanup failed: {error}"
)),
));
}
return RequestTerminal::RespondWithoutPublish(Err(ApiError::NotFound(
format!("Session not found: {session_id}"),
)));
}
Ok(false) => session,
Err(err) => {
drop(caller_event_tx);
drain_event_forwarder(&session_id, forward_task).await;
return RequestTerminal::RespondWithoutPublish(Err(ApiError::Internal(
format!("Failed to load session archive state: {err}"),
)));
}
}
}
Ok(None) => {
let error = unregister_rest_runtime_after_api_error_locked(
state,
&session_id,
runtime_was_registered,
ApiError::NotFound(format!("Session not found: {session_id}")),
)
.await;
drop(caller_event_tx);
drain_event_forwarder(&session_id, forward_task).await;
return RequestTerminal::RespondWithoutPublish(Err(error));
}
Err(err) => {
drop(caller_event_tx);
drain_event_forwarder(&session_id, forward_task).await;
return RequestTerminal::RespondWithoutPublish(Err(ApiError::Internal(format!(
"Failed to load session: {err}"
))));
}
};
let session_defaults =
match meerkat_core::session_recovery::SessionDefaults::from_session(&session) {
Ok(defaults) => defaults,
Err(error) => {
let error = unregister_rest_runtime_after_api_error_locked(
state,
&session_id,
runtime_was_registered,
ApiError::Internal(error.to_string()),
)
.await;
drop(caller_event_tx);
drain_event_forwarder(&session_id, forward_task).await;
return RequestTerminal::RespondWithoutPublish(Err(error));
}
};
let stored_metadata = session_defaults.metadata;
let keep_alive = match keep_alive_override {
Some(val) => val,
None => stored_metadata.keep_alive,
};
let effective_comms_name = req
.comms_name
.clone()
.or_else(|| stored_metadata.comms_name.clone());
if keep_alive
&& effective_comms_name
.as_ref()
.is_none_or(|name| name.trim().is_empty())
{
let error = unregister_rest_runtime_after_api_error_locked(
state,
&session_id,
runtime_was_registered,
ApiError::BadRequest("keep_alive requires comms_name".to_string()),
)
.await;
drop(caller_event_tx);
drain_event_forwarder(&session_id, forward_task).await;
return RequestTerminal::RespondWithoutPublish(Err(error));
}
#[cfg(feature = "comms")]
let comms_rt = {
let comms_rt = state.session_service.comms_runtime(&session_id).await;
if keep_alive && comms_rt.is_none() {
let error = unregister_rest_runtime_after_api_error_locked(
state,
&session_id,
runtime_was_registered,
ApiError::BadRequest(
"keep_alive requires a session created with comms_name".to_string(),
),
)
.await;
drop(caller_event_tx);
drain_event_forwarder(&session_id, forward_task).await;
return RequestTerminal::RespondWithoutPublish(Err(error));
}
comms_rt
};
let current_identity = stored_metadata.llm_identity();
if let Err(err) =
validate_prompt_video_input_for_state(state, &turn_prompt, ¤t_identity).await
{
let error = unregister_rest_runtime_after_api_error_locked(
state,
&session_id,
runtime_was_registered,
prompt_video_input_error_to_api(err),
)
.await;
drop(caller_event_tx);
drain_event_forwarder(&session_id, forward_task).await;
return RequestTerminal::RespondWithoutPublish(Err(error));
}
if let Some(ctx) = req_ctx.as_ref() {
if ctx.cancel_already_requested() {
let error = unregister_rest_runtime_after_api_error_locked(
state,
&session_id,
runtime_was_registered,
ApiError::RequestCancelled { details: None },
)
.await;
drop(caller_event_tx);
drain_event_forwarder(&session_id, forward_task).await;
return RequestTerminal::RespondWithoutPublish(Err(error));
}
}
let mut pending_pre_admission = Some(
match state
.session_service
.reserve_runtime_turn_admission(&session_id)
.await
{
Ok(admission) => admission,
Err(err) => {
let error = unregister_rest_runtime_after_api_error_locked(
state,
&session_id,
runtime_was_registered,
ApiError::Conflict(err.to_string()),
)
.await;
drop(caller_event_tx);
drain_event_forwarder(&session_id, forward_task).await;
return RequestTerminal::RespondWithoutPublish(Err(error));
}
},
);
if let Some(ctx) = req_ctx.as_ref()
&& ctx.cancel_already_requested()
{
let error = unregister_rest_runtime_after_api_error_locked(
state,
&session_id,
runtime_was_registered,
ApiError::RequestCancelled { details: None },
)
.await;
drop(caller_event_tx);
drain_event_forwarder(&session_id, forward_task).await;
return RequestTerminal::RespondWithoutPublish(Err(error));
}
#[cfg(feature = "mcp")]
if let Err(e) = apply_mcp_boundary_to_turn_prompt(
state,
&session_id,
&caller_event_tx,
&mut turn_prompt,
)
.await
{
let error = unregister_rest_runtime_after_api_error_locked(
state,
&session_id,
runtime_was_registered,
e,
)
.await;
drop(caller_event_tx);
drain_event_forwarder(&session_id, forward_task).await;
return RequestTerminal::RespondWithoutPublish(Err(error));
}
let input = meerkat_runtime::Input::Prompt(
meerkat_runtime::PromptInput::from_content_input(
turn_prompt.clone(),
Some(
meerkat_core::lifecycle::run_primitive::RuntimeTurnMetadata {
keep_alive: resolve_turn_keep_alive_policy(keep_alive_override),
skill_references: skill_references.clone(),
system_prompts: rest_continue_system_prompts(&req),
turn_tool_overlay: req
.turn_tool_overlay
.clone()
.map(meerkat_core::service::TurnToolOverlay::from),
additional_instructions: resolve_turn_additional_instructions(
req.additional_instructions.clone(),
),
transient_turn_context: req.transient_turn_context.clone(),
..Default::default()
},
),
)
.with_injected_context(req.injected_context.clone().unwrap_or_default()),
);
let input_id = input.id().clone();
let mut pre_admission_registration = None;
if let Some(admission) = pending_pre_admission.take() {
if let Err(err) = insert_rest_runtime_pre_admission(
&state.runtime_pre_admissions,
session_id.clone(),
input_id.clone(),
admission,
)
.await
{
let error = unregister_rest_runtime_after_api_error_locked(
state,
&session_id,
runtime_was_registered,
ApiError::Conflict(err.to_string()),
)
.await;
drop(caller_event_tx);
drain_event_forwarder(&session_id, forward_task).await;
return RequestTerminal::RespondWithoutPublish(Err(error));
}
pre_admission_registration = Some(RestRuntimePreAdmissionRegistration::new(
state.runtime_pre_admissions.clone(),
session_id.clone(),
input_id.clone(),
));
}
if let Some(ctx) = req_ctx.as_ref()
&& ctx.cancel_already_requested()
{
discard_rest_runtime_pre_admission(
&state.runtime_pre_admissions,
&session_id,
&input_id,
)
.await;
if let Some(registration) = pre_admission_registration.take() {
registration.disarm();
}
let error = unregister_rest_runtime_after_api_error_locked(
state,
&session_id,
runtime_was_registered,
ApiError::RequestCancelled { details: None },
)
.await;
drop(caller_event_tx);
drain_event_forwarder(&session_id, forward_task).await;
return RequestTerminal::RespondWithoutPublish(Err(error));
}
if let Err(error) =
prepare_rest_session_runtime_executor_locked(state, &session_id, &input_id).await
{
discard_rest_runtime_pre_admission(
&state.runtime_pre_admissions,
&session_id,
&input_id,
)
.await;
if let Some(registration) = pre_admission_registration.take() {
registration.disarm();
}
let error = unregister_rest_runtime_after_api_error_locked(
state,
&session_id,
runtime_was_registered,
runtime_executor_attach_error_to_api(error),
)
.await;
drop(caller_event_tx);
drain_event_forwarder(&session_id, forward_task).await;
return RequestTerminal::RespondWithoutPublish(Err(error));
}
#[cfg(feature = "comms")]
{
if let Err(error) = adapter
.update_peer_ingress_context(&session_id, keep_alive, comms_rt)
.await
{
let error = unregister_rest_runtime_after_api_error_locked(
state,
&session_id,
runtime_was_registered,
ApiError::Internal(format!("failed to update peer ingress context: {error}")),
)
.await;
drop(caller_event_tx);
drain_event_forwarder(&session_id, forward_task).await;
return RequestTerminal::RespondWithoutPublish(Err(error));
}
}
let (outcome, handle, cleanup_result) = match adapter
.accept_input_with_completion(&session_id, input)
.await
{
Ok((outcome, handle)) => {
let cleanup_result = match (&outcome, &handle) {
(
meerkat_runtime::AcceptOutcome::Accepted {
input_id: accepted_input_id,
..
},
Some(_),
) => {
if let Some(registration) = pre_admission_registration.as_mut() {
registration.track_input_id(accepted_input_id.clone());
}
rekey_rest_runtime_pre_admission(
&state.runtime_pre_admissions,
&session_id,
&input_id,
accepted_input_id.clone(),
)
.await;
if let Some(registration) = pre_admission_registration.take() {
registration.disarm();
}
Ok(())
}
(
meerkat_runtime::AcceptOutcome::Accepted {
input_id: accepted_input_id,
..
},
None,
) => {
if let Some(registration) = pre_admission_registration.as_mut() {
registration.track_input_id(accepted_input_id.clone());
}
discard_rest_runtime_pre_admission(
&state.runtime_pre_admissions,
&session_id,
&input_id,
)
.await;
discard_rest_runtime_pre_admission(
&state.runtime_pre_admissions,
&session_id,
accepted_input_id,
)
.await;
if let Some(registration) = pre_admission_registration.take() {
registration.disarm();
}
unregister_rest_runtime_if_new_idle_locked(
state,
&session_id,
runtime_was_registered,
)
.await
}
_ => {
discard_rest_runtime_pre_admission(
&state.runtime_pre_admissions,
&session_id,
&input_id,
)
.await;
if let Some(registration) = pre_admission_registration.take() {
registration.disarm();
}
unregister_rest_runtime_if_new_idle_locked(
state,
&session_id,
runtime_was_registered,
)
.await
}
};
(outcome, handle, cleanup_result)
}
Err(err) => {
discard_rest_runtime_pre_admission(
&state.runtime_pre_admissions,
&session_id,
&input_id,
)
.await;
if let Some(registration) = pre_admission_registration.take() {
registration.disarm();
}
let error = unregister_rest_runtime_after_api_error_locked(
state,
&session_id,
runtime_was_registered,
ApiError::Internal(err.to_string()),
)
.await;
drop(caller_event_tx);
drain_event_forwarder(&session_id, forward_task).await;
return RequestTerminal::RespondWithoutPublish(Err(error));
}
};
if let meerkat_runtime::AcceptOutcome::Accepted {
input_id: accepted_input_id,
..
} = &outcome
&& let Some(ctx) = req_ctx.as_ref()
{
let _ = install_exact_rest_input_cancel(
ctx,
Arc::clone(&adapter),
session_id.clone(),
accepted_input_id.clone(),
)
.await;
}
if let Err(cleanup) = cleanup_result {
let primary = missing_runtime_terminal_evidence_error(
"continue",
&outcome,
&session_id,
&state.realm,
false,
);
let error = ApiError::Internal(runtime_cleanup_failure_message(
api_error_message(&primary),
cleanup,
&session_id,
"unregister newly prepared REST runtime",
));
drop(caller_event_tx);
drain_event_forwarder(&session_id, forward_task).await;
return RequestTerminal::RespondWithoutPublish(Err(error));
}
let completion_cleanup_input_id = match (&outcome, &handle) {
(
meerkat_runtime::AcceptOutcome::Accepted {
input_id: accepted_input_id,
..
},
Some(_),
) => Some(accepted_input_id.clone()),
_ => None,
};
let handle = match (handle, completion_cleanup_input_id) {
(Some(handle), Some(accepted_input_id)) => {
let pre_admissions = state.runtime_pre_admissions.clone();
let release_session_id = session_id.clone();
let cleanup_input_id = input_id.clone();
let cleanup_state = state.clone();
Some(
handle.with_resultful_completion_cleanup(move |completion| async move {
let release_pre_admission = match completion {
Ok(cleanup_observation) => {
cleanup_rest_runtime_after_completion_outcome(
&cleanup_state,
&release_session_id,
cleanup_observation,
)
.await
.map_err(|error| {
rest_completion_cleanup_error(&release_session_id, error)
})?
}
Err(error) => {
rest_runtime_wait_failure_releases_pre_admission_result(
&cleanup_state,
&release_session_id,
&error,
)
.await?
}
};
if release_pre_admission {
discard_rest_runtime_pre_admission(
&pre_admissions,
&release_session_id,
&cleanup_input_id,
)
.await;
discard_rest_runtime_pre_admission(
&pre_admissions,
&release_session_id,
&accepted_input_id,
)
.await;
}
Ok(())
}),
)
}
(handle, _) => handle,
};
drop(runtime_registration_guard);
drop(runtime_registration_lock);
completion_handle_to_api_result(
"continue",
state,
&outcome,
handle,
&session_id,
&state.realm,
false,
)
.await
};
drop(caller_event_tx);
drain_event_forwarder(&session_id, forward_task).await;
match final_result {
Ok(run_result) => {
RequestTerminal::Publish(Ok(Json(run_result_to_response(run_result, &state.realm))))
}
Err(err) => {
let archived_now = match state
.session_service
.load_authoritative_session(&session_id)
.await
{
Ok(Some(session)) => {
match session_archived_by_authority(state, &session_id, &session).await {
Ok(archived) => archived,
Err(error) => {
return RequestTerminal::Publish(Err(ApiError::Internal(format!(
"Failed to load session archive state: {error}"
))));
}
}
}
Ok(None) => false,
Err(error) => {
return RequestTerminal::Publish(Err(ApiError::Internal(format!(
"Failed to load session: {error}"
))));
}
};
if archived_now {
let cleanup_result =
discard_and_cleanup_archived_session_runtime(state, &session_id).await;
if let Err(cleanup_error) = cleanup_result {
return RequestTerminal::Publish(Err(ApiError::Internal(format!(
"REST runtime cleanup failed after terminal error: {cleanup_error}"
))));
}
}
RequestTerminal::Publish(Err(err))
}
}
}
const SSE_SERIALIZATION_ERROR_EVENT: &str = "error";
fn sse_serialization_error_data(error: &serde_json::Error) -> String {
json!({
"error": "event_serialization_failed",
"message": error.to_string(),
})
.to_string()
}
fn sse_serialization_error_event(error: &serde_json::Error) -> Event {
Event::default()
.event(SSE_SERIALIZATION_ERROR_EVENT)
.data(sse_serialization_error_data(error))
}
async fn session_events(
State(state): State<AppState>,
Path(id): Path<String>,
) -> Result<Sse<impl Stream<Item = Result<Event, Infallible>>>, ApiError> {
let session_id = resolve_session_id_for_state(&id, &state)?;
let session = state
.session_service
.load_authoritative_session(&session_id)
.await
.map_err(|e| ApiError::Internal(format!("{e}")))?
.ok_or_else(|| ApiError::NotFound(format!("Session not found: {id}")))?;
let mut rx = state.event_tx.subscribe();
let stream = async_stream::stream! {
match serde_json::to_string(&json!({
"session_id": session_id.to_string(),
"message_count": session.messages().len(),
})) {
Ok(data) => yield Ok(Event::default().event("session_loaded").data(data)),
Err(e) => {
yield Ok(sse_serialization_error_event(&e));
return;
}
}
loop {
match rx.recv().await {
Ok(payload) => {
if payload.session_id != session_id {
continue;
}
let event_type = agent_event_type(&payload.event.payload);
match serde_json::to_string(&payload.event) {
Ok(data) => yield Ok(Event::default().event(event_type).data(data)),
Err(e) => {
yield Ok(sse_serialization_error_event(&e));
return;
}
}
}
Err(broadcast::error::RecvError::Lagged(_)) => {
continue;
}
Err(broadcast::error::RecvError::Closed) => {
break;
}
}
}
let event = Event::default().event("done").data("{}");
yield Ok(event);
};
Ok(Sse::new(stream))
}
#[cfg(feature = "mcp")]
async fn apply_mcp_boundary(
state: &AppState,
session_id: &SessionId,
event_tx: &mpsc::Sender<EventEnvelope<AgentEvent>>,
prompt: &mut String,
) -> Result<(), ApiError> {
let (adapter, turn_number, drain_task_running, lifecycle_tx, mut queued_actions) = {
let mut map = state.mcp_sessions.write().await;
let mcp_state = match map.get_mut(session_id) {
Some(s) => s,
None => return Ok(()),
};
mcp_state.turn_counter = mcp_state.turn_counter.saturating_add(1);
let mut queued = Vec::new();
while let Ok(action) = mcp_state.lifecycle_rx.try_recv() {
queued.push(action);
}
(
mcp_state.adapter.clone(),
mcp_state.turn_counter,
mcp_state.drain_task_running.clone(),
mcp_state.lifecycle_tx.clone(),
queued,
)
};
if !queued_actions.is_empty() {
let drained = std::mem::take(&mut queued_actions);
let source = meerkat_core::EventSourceIdentity::session(session_id.clone());
meerkat::surface::emit_mcp_lifecycle_events(
event_tx,
&source,
prompt,
turn_number,
drained,
)
.await;
}
let result = adapter
.apply_staged()
.await
.map_err(|e| ApiError::Internal(format!("failed to apply staged MCP operations: {e}")))?;
if result.delta.lifecycle_actions.iter().any(|action| {
action.operation == ToolConfigChangeOperation::Remove
&& action.phase == McpLifecyclePhase::Draining
}) {
spawn_mcp_drain_task(adapter, drain_task_running, lifecycle_tx);
}
queued_actions.extend(result.delta.lifecycle_actions);
if !queued_actions.is_empty() {
let source = meerkat_core::EventSourceIdentity::session(session_id.clone());
meerkat::surface::emit_mcp_lifecycle_events(
event_tx,
&source,
prompt,
turn_number,
queued_actions,
)
.await;
}
Ok(())
}
#[cfg(feature = "mcp")]
async fn apply_mcp_boundary_to_turn_prompt(
state: &AppState,
session_id: &SessionId,
event_tx: &mpsc::Sender<EventEnvelope<AgentEvent>>,
turn_prompt: &mut ContentInput,
) -> Result<(), ApiError> {
let mut mcp_text = String::new();
apply_mcp_boundary(state, session_id, event_tx, &mut mcp_text).await?;
if !mcp_text.is_empty() {
let prompt = std::mem::replace(turn_prompt, ContentInput::Text(String::new()));
let mut blocks = prompt.into_blocks();
blocks.insert(
0,
meerkat_core::types::ContentBlock::Text { text: mcp_text },
);
*turn_prompt = ContentInput::Blocks(blocks);
}
Ok(())
}
#[cfg(feature = "mcp")]
fn spawn_mcp_drain_task(
adapter: Arc<McpRouterAdapter>,
task_running: Arc<AtomicBool>,
lifecycle_tx: mpsc::UnboundedSender<McpLifecycleAction>,
) {
if task_running
.compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
.is_err()
{
return;
}
tokio::spawn(async move {
loop {
tokio::time::sleep(Duration::from_millis(100)).await;
let delta = match adapter.progress_removals().await {
Ok(d) => d,
Err(e) => {
tracing::warn!("background MCP drain apply failed: {e}");
break;
}
};
for action in delta.lifecycle_actions {
let _ = lifecycle_tx.send(action);
}
match adapter.has_removing_servers().await {
Ok(true) => continue,
Ok(false) => break,
Err(e) => {
tracing::warn!("background MCP drain state check failed: {e}");
break;
}
}
}
task_running.store(false, Ordering::Release);
});
}
#[cfg(feature = "mcp")]
async fn resolve_mcp_adapter(
state: &AppState,
session_id: &SessionId,
) -> Result<Arc<McpRouterAdapter>, ApiError> {
match state
.session_service
.load_authoritative_session(session_id)
.await
{
Ok(Some(session)) => {
if session_archived_by_authority(state, session_id, &session)
.await
.map_err(|err| ApiError::Internal(err.to_string()))?
{
cleanup_archived_session_runtime(state, session_id)
.await
.map_err(|error| {
ApiError::Internal(format!(
"failed to clean up archived session runtime: {error}"
))
})?;
return Err(ApiError::NotFound(format!(
"Session not found: {session_id}"
)));
}
}
Ok(None) => {
return Err(ApiError::NotFound(format!(
"Session not found: {session_id}"
)));
}
Err(err) => return Err(ApiError::Internal(format!("Failed to load session: {err}"))),
}
let map = state.mcp_sessions.read().await;
map.get(session_id)
.map(|s| s.adapter.clone())
.ok_or_else(|| {
ApiError::Conflict(
"Live MCP unavailable for this session. Recovery: create a new session \
(live MCP adapters are attached at session creation time; sessions from \
before the server started do not have them)."
.to_string(),
)
})
}
#[cfg(feature = "mcp")]
fn validate_session_id_consistency(
path_id: &str,
body_id: &str,
state: &AppState,
) -> Result<SessionId, ApiError> {
let path_sid = resolve_session_id_for_state(path_id, state)?;
let body_sid = resolve_session_id_for_state(body_id, state)?;
if path_sid != body_sid {
return Err(ApiError::BadRequest(format!(
"Session ID mismatch: path={path_id} body={body_id}"
)));
}
Ok(path_sid)
}
#[cfg(feature = "mcp")]
async fn mcp_add(
State(state): State<AppState>,
Path(id): Path<String>,
Json(req): Json<meerkat_contracts::McpAddParams>,
) -> Result<Json<meerkat_contracts::McpLiveOpResponse>, ApiError> {
let session_id = validate_session_id_consistency(&id, &req.session_id, &state)?;
let server_name = req.server_config.name.clone();
if server_name.trim().is_empty() {
return Err(ApiError::BadRequest(
"server_name cannot be empty".to_string(),
));
}
let adapter = resolve_mcp_adapter(&state, &session_id).await?;
let config = req.server_config;
let rollback = if req.persisted {
let authority = meerkat::surface::mcp_config_mutation_authority(
state.context_root.clone(),
state.user_config_root.clone(),
);
meerkat::surface::persist_mcp_add_if_requested(true, &authority, config.clone())
.await
.map_err(ApiError::Internal)?
} else {
None
};
if let Err(err) = adapter.stage_add(config).await {
let rollback_message =
match meerkat::surface::rollback_mcp_persisted_mutation(rollback).await {
Ok(()) => String::new(),
Err(rollback_err) => format!("; persisted rollback failed: {rollback_err}"),
};
return Err(ApiError::Internal(format!("{err}{rollback_message}")));
}
Ok(Json(meerkat::surface::mcp_live_response(
req.session_id,
meerkat_contracts::McpLiveOperation::Add,
Some(server_name),
rollback.is_some(),
)))
}
#[cfg(feature = "mcp")]
async fn mcp_remove(
State(state): State<AppState>,
Path(id): Path<String>,
Json(req): Json<meerkat_contracts::McpRemoveParams>,
) -> Result<Json<meerkat_contracts::McpLiveOpResponse>, ApiError> {
let session_id = validate_session_id_consistency(&id, &req.session_id, &state)?;
if req.server_name.trim().is_empty() {
return Err(ApiError::BadRequest(
"server_name cannot be empty".to_string(),
));
}
let adapter = resolve_mcp_adapter(&state, &session_id).await?;
let rollback = if req.persisted {
let authority = meerkat::surface::mcp_config_mutation_authority(
state.context_root.clone(),
state.user_config_root.clone(),
);
meerkat::surface::persist_mcp_remove_if_requested(true, &authority, &req.server_name)
.await
.map_err(ApiError::Internal)?
} else {
None
};
if let Err(err) = adapter.stage_remove(req.server_name.clone()).await {
let rollback_message =
match meerkat::surface::rollback_mcp_persisted_mutation(rollback).await {
Ok(()) => String::new(),
Err(rollback_err) => format!("; persisted rollback failed: {rollback_err}"),
};
return Err(ApiError::Internal(format!("{err}{rollback_message}")));
}
Ok(Json(meerkat::surface::mcp_live_response(
req.session_id,
meerkat_contracts::McpLiveOperation::Remove,
Some(req.server_name),
rollback.is_some(),
)))
}
#[cfg(feature = "mcp")]
async fn mcp_reload(
State(state): State<AppState>,
Path(id): Path<String>,
Json(req): Json<meerkat_contracts::McpReloadParams>,
) -> Result<Json<meerkat_contracts::McpLiveOpResponse>, ApiError> {
let session_id = validate_session_id_consistency(&id, &req.session_id, &state)?;
if let Some(name) = req.server_name.as_ref()
&& name.trim().is_empty()
{
return Err(ApiError::BadRequest(
"server_name cannot be empty".to_string(),
));
}
let adapter = resolve_mcp_adapter(&state, &session_id).await?;
match req.server_name.as_ref() {
Some(name) => {
meerkat::surface::validate_reload_target(&adapter, name)
.await
.map_err(ApiError::BadRequest)?;
adapter
.stage_reload(McpReloadTarget::ServerName(name.clone()))
.await
.map_err(ApiError::Internal)?;
}
None => {
let report = adapter
.stage_reload_all()
.await
.map_err(|e| ApiError::Internal(e.to_string()))?;
if !report.is_clean() {
let failed = report
.failed
.iter()
.map(|failure| format!("{}: {}", failure.server, failure.error))
.collect::<Vec<_>>()
.join("; ");
return Err(ApiError::Internal(format!(
"MCP reload-all staged {} server(s) but rejected: {failed}",
report.staged.len()
)));
}
}
}
Ok(Json(meerkat::surface::mcp_live_response(
req.session_id,
meerkat_contracts::McpLiveOperation::Reload,
req.server_name,
false,
)))
}
#[cfg(feature = "mcp")]
async fn cleanup_mcp_session(state: &AppState, session_id: &SessionId) {
if let Some(mcp_state) = state.mcp_sessions.write().await.remove(session_id) {
mcp_state.adapter.shutdown().await;
}
}
async fn cleanup_archived_session_runtime(
state: &AppState,
session_id: &SessionId,
) -> Result<(), SessionError> {
match state.runtime_adapter.unregister_session(session_id).await {
Ok(())
| Err(
meerkat_runtime::RuntimeDriverError::NotFound { .. }
| meerkat_runtime::RuntimeDriverError::Destroyed
| meerkat_runtime::RuntimeDriverError::NotReady { .. },
) => {}
Err(error) => return Err(runtime_driver_error_to_session_error(error)),
}
cleanup_rest_runtime_after_unregistered(state, session_id).await
}
async fn discard_and_cleanup_archived_session_runtime(
state: &AppState,
session_id: &SessionId,
) -> Result<(), SessionError> {
match state.session_service.discard_live_session(session_id).await {
Ok(()) | Err(SessionError::NotFound { .. }) => {}
Err(error) => return Err(error),
}
cleanup_archived_session_runtime(state, session_id).await
}
async fn cleanup_rest_runtime_after_verified_termination(
state: &AppState,
session_id: &SessionId,
cleanup_observation: &meerkat_runtime::CompletionCleanupObservation,
) -> Result<(), SessionError> {
if !cleanup_observation.proves_runtime_termination_for(session_id) {
return Err(SessionError::Unsupported(format!(
"REST runtime termination cleanup for {session_id} lacks machine-owned termination proof"
)));
}
if state.runtime_adapter.contains_session(session_id).await {
return Err(SessionError::Unsupported(format!(
"REST runtime termination cleanup for {session_id} expected an already-absent registration"
)));
}
cleanup_rest_runtime_after_unregistered(state, session_id).await
}
async fn cleanup_rest_runtime_after_unregistered(
state: &AppState,
session_id: &SessionId,
) -> Result<(), SessionError> {
#[cfg(feature = "mob")]
state
.mob_state
.destroy_bridge_session_mobs(&session_id.to_string())
.await
.map_err(|error| {
error.into_session_error(
"failed to destroy bridge session mobs during REST runtime cleanup",
)
})?;
#[cfg(feature = "mcp")]
cleanup_mcp_session(state, session_id).await;
#[cfg(feature = "comms")]
abort_comms_drain_for_archived_session(state, session_id).await?;
Ok(())
}
async fn cleanup_archived_session_surface_runtime(
state: &AppState,
session_id: &SessionId,
) -> Result<(), SessionError> {
match state.runtime_adapter.unregister_session(session_id).await {
Ok(())
| Err(
meerkat_runtime::RuntimeDriverError::NotFound { .. }
| meerkat_runtime::RuntimeDriverError::Destroyed
| meerkat_runtime::RuntimeDriverError::NotReady { .. },
) => {}
Err(error) => return Err(runtime_driver_error_to_session_error(error)),
}
#[cfg(feature = "mcp")]
cleanup_mcp_session(state, session_id).await;
#[cfg(feature = "comms")]
abort_comms_drain_for_archived_session(state, session_id).await?;
Ok(())
}
#[cfg(feature = "comms")]
async fn abort_comms_drain_for_archived_session(
state: &AppState,
session_id: &SessionId,
) -> Result<(), SessionError> {
match state.runtime_adapter.abort_comms_drain(session_id).await {
Ok(()) => Ok(()),
Err(
meerkat_runtime::RuntimeDriverError::NotFound { .. }
| meerkat_runtime::RuntimeDriverError::Destroyed
| meerkat_runtime::RuntimeDriverError::NotReady { .. },
) => Ok(()),
Err(error) => Err(SessionError::Agent(
meerkat_core::error::AgentError::InternalError(error.to_string()),
)),
}
}
async fn archive_session_with_runtime_cleanup(
state: AppState,
session_id: SessionId,
) -> Result<(), SessionError> {
let service = Arc::clone(&state.session_service);
let result_session_id = session_id.clone();
let (result_tx, result_rx) = tokio::sync::oneshot::channel();
tokio::spawn(async move {
#[cfg(feature = "mob")]
match state
.mob_state
.archive_mob_owned_bridge_session_with_cleanup(
&session_id,
"failed to destroy bridge session mobs during REST mob-owned archive cleanup",
)
.await
{
Ok(true) => {
let _ = result_tx
.send(cleanup_archived_session_surface_runtime(&state, &session_id).await);
return;
}
Ok(false) => {}
Err(error) => {
let _ = result_tx.send(Err(error));
return;
}
}
#[cfg(feature = "mob")]
let had_cleanup_anchor = state
.mob_state
.has_bridge_session_scoped_mobs(&session_id.to_string())
.await;
#[cfg(not(feature = "mob"))]
let had_cleanup_anchor = false;
let mut result = service
.archive_with_machine_protocol(
&session_id,
MachineSessionArchiveProtocol::from_machine(state.runtime_adapter.as_ref()),
)
.await;
let not_found = matches!(&result, Err(SessionError::NotFound { .. }));
if (result.is_ok() || not_found)
&& let Err(error) = cleanup_archived_session_runtime(&state, &session_id).await
{
let _ = result_tx.send(Err(error));
return;
}
if not_found && had_cleanup_anchor {
result = Ok(());
}
let _ = result_tx.send(result);
});
result_rx.await.map_err(|_| {
SessionError::Agent(meerkat_core::error::AgentError::InternalError(format!(
"REST archive task ended before reporting a result for {result_session_id}"
)))
})?
}
#[cfg(feature = "mcp")]
pub async fn shutdown_all_mcp_sessions(state: &AppState) {
let sessions: Vec<_> = state.mcp_sessions.write().await.drain().collect();
for (_, mcp_state) in sessions {
mcp_state.adapter.shutdown().await;
}
}
#[derive(Debug)]
pub enum ApiError {
BadRequest(String),
BadRequestWithData {
message: String,
code: String,
details: Value,
},
Unauthorized(String),
NotFound(String),
Conflict(String),
DuplicateInput {
existing_id: String,
},
Configuration(String),
Agent(String),
Internal(String),
InternalWithData {
message: String,
code: String,
details: Value,
},
RequestCancelled {
details: Option<Value>,
},
DuplicateRequestId {
request_id: String,
},
ServiceUnavailable(String),
Gone(String),
}
fn api_error_message(error: &ApiError) -> String {
match error {
ApiError::BadRequest(message)
| ApiError::Unauthorized(message)
| ApiError::NotFound(message)
| ApiError::Conflict(message)
| ApiError::Configuration(message)
| ApiError::Agent(message)
| ApiError::Internal(message)
| ApiError::ServiceUnavailable(message)
| ApiError::Gone(message) => message.clone(),
ApiError::BadRequestWithData { message, .. }
| ApiError::InternalWithData { message, .. } => message.clone(),
ApiError::DuplicateInput { existing_id } => {
format!("duplicate input: {existing_id}")
}
ApiError::RequestCancelled { .. } => "request cancelled".to_string(),
ApiError::DuplicateRequestId { request_id } => {
format!("duplicate request id: {request_id}")
}
}
}
impl IntoResponse for ApiError {
fn into_response(self) -> Response {
let (status, code, message, details) = match self {
ApiError::BadRequest(msg) => (
StatusCode::BAD_REQUEST,
"BAD_REQUEST".to_string(),
msg,
None,
),
ApiError::BadRequestWithData {
message,
code,
details,
} => (StatusCode::BAD_REQUEST, code, message, Some(details)),
ApiError::Unauthorized(msg) => (
StatusCode::UNAUTHORIZED,
"UNAUTHORIZED".to_string(),
msg,
None,
),
ApiError::NotFound(msg) => (StatusCode::NOT_FOUND, "NOT_FOUND".to_string(), msg, None),
ApiError::Conflict(msg) => (StatusCode::CONFLICT, "CONFLICT".to_string(), msg, None),
ApiError::DuplicateInput { existing_id } => {
let body = Json(serde_json::json!({
"error": "duplicate_input",
"code": "DUPLICATE_INPUT",
"existing_id": existing_id,
}));
return (StatusCode::CONFLICT, body).into_response();
}
ApiError::Configuration(msg) => (
StatusCode::INTERNAL_SERVER_ERROR,
"CONFIGURATION_ERROR".to_string(),
msg,
None,
),
ApiError::Agent(msg) => (
StatusCode::INTERNAL_SERVER_ERROR,
"AGENT_ERROR".to_string(),
msg,
None,
),
ApiError::Internal(msg) => (
StatusCode::INTERNAL_SERVER_ERROR,
"INTERNAL_ERROR".to_string(),
msg,
None,
),
ApiError::InternalWithData {
message,
code,
details,
} => (
StatusCode::INTERNAL_SERVER_ERROR,
code,
message,
Some(details),
),
ApiError::RequestCancelled { details } => (
StatusCode::from_u16(499).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR),
"REQUEST_CANCELLED".to_string(),
"request cancelled".to_string(),
details,
),
ApiError::DuplicateRequestId { request_id } => (
StatusCode::CONFLICT,
"DUPLICATE_REQUEST_ID".to_string(),
format!("request ID already in flight: {request_id}"),
None,
),
ApiError::ServiceUnavailable(msg) => (
StatusCode::SERVICE_UNAVAILABLE,
"SERVICE_UNAVAILABLE".to_string(),
msg,
None,
),
ApiError::Gone(msg) => (StatusCode::GONE, "GONE".to_string(), msg, None),
};
let body = Json(ErrorResponse {
error: message,
code,
details,
});
(status, body).into_response()
}
}
#[cfg(test)]
#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
mod tests {
use super::*;
use async_trait::async_trait;
use chrono::{Duration, Utc};
use futures::stream;
use meerkat::{OccurrenceFailureClass, OccurrencePhase, ScheduleId};
use meerkat_client::{LlmDoneOutcome, LlmError, LlmEvent, LlmRequest};
use meerkat_core::{
MemoryConfigStore, SelfHostedApiStyle, SelfHostedServerConfig, SelfHostedTransport,
SessionId,
};
use serde_json::json;
use std::path::PathBuf;
use std::pin::Pin;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering as AtomicOrdering};
use tempfile::TempDir;
struct FailDeleteOpsLifecycleOnceStore {
inner: Arc<meerkat_runtime::InMemoryRuntimeStore>,
fail_delete: AtomicBool,
}
#[async_trait]
impl meerkat_runtime::RuntimeStore for FailDeleteOpsLifecycleOnceStore {
fn session_authority_ops(&self) -> &dyn meerkat_runtime::store::RuntimeSessionAuthorityOps {
self.inner.session_authority_ops()
}
fn session_persistence_profile(
&self,
) -> meerkat_runtime::store::RuntimeSessionPersistenceProfile {
meerkat_runtime::RuntimeStore::session_persistence_profile(self.inner.as_ref())
}
fn session_boundary_authority_read_cost(
&self,
) -> meerkat_runtime::store::RuntimeSessionAuthorityReadCost {
self.inner.session_boundary_authority_read_cost()
}
async fn commit_prepared_session_boundary(
&self,
runtime_id: &meerkat_runtime::LogicalRuntimeId,
request: meerkat_runtime::store::PreparedRuntimeSessionCommit,
) -> Result<
meerkat_runtime::store::PreparedRuntimeSessionCommitResult,
meerkat_runtime::RuntimeStoreError,
> {
self.inner
.commit_prepared_session_boundary(runtime_id, request)
.await
}
async fn load_session_boundary_authority(
&self,
runtime_id: &meerkat_runtime::LogicalRuntimeId,
) -> Result<
Option<meerkat_runtime::store::RuntimeSessionAuthority>,
meerkat_runtime::RuntimeStoreError,
> {
self.inner.load_session_boundary_authority(runtime_id).await
}
async fn load_whole_blob_store_authority(
&self,
runtime_id: &meerkat_runtime::LogicalRuntimeId,
) -> Result<
Option<meerkat_runtime::store::WholeBlobStoreAuthority>,
meerkat_runtime::RuntimeStoreError,
> {
self.inner.load_whole_blob_store_authority(runtime_id).await
}
async fn load_committed_whole_blob_snapshot(
&self,
runtime_id: &meerkat_runtime::LogicalRuntimeId,
) -> Result<
Option<meerkat_runtime::store::CommittedWholeBlobSnapshot>,
meerkat_runtime::RuntimeStoreError,
> {
self.inner
.load_committed_whole_blob_snapshot(runtime_id)
.await
}
async fn commit_prepared_whole_blob_snapshot_cas(
&self,
runtime_id: &meerkat_runtime::LogicalRuntimeId,
prepared: meerkat_runtime::store::PreparedWholeBlobSnapshotCas,
) -> Result<
meerkat_runtime::store::WholeBlobSnapshotCasOutcome,
meerkat_runtime::RuntimeStoreError,
> {
self.inner
.commit_prepared_whole_blob_snapshot_cas(runtime_id, prepared)
.await
}
async fn delete_runtime_session_catalog_entry(
&self,
runtime_id: &meerkat_runtime::LogicalRuntimeId,
) -> Result<(), meerkat_runtime::RuntimeStoreError> {
self.inner
.delete_runtime_session_catalog_entry(runtime_id)
.await
}
async fn load_runtime_session_catalog_entry(
&self,
runtime_id: &meerkat_runtime::LogicalRuntimeId,
) -> Result<
Option<meerkat_runtime::store::RuntimeSessionCatalogEntry>,
meerkat_runtime::RuntimeStoreError,
> {
self.inner
.load_runtime_session_catalog_entry(runtime_id)
.await
}
async fn list_runtime_session_catalog_entries(
&self,
filter: meerkat_core::SessionFilter,
) -> Result<
Vec<meerkat_runtime::store::RuntimeSessionCatalogEntry>,
meerkat_runtime::RuntimeStoreError,
> {
self.inner
.list_runtime_session_catalog_entries(filter)
.await
}
async fn write_prepared_whole_blob_provisional_tail(
&self,
runtime_id: &meerkat_runtime::LogicalRuntimeId,
prepared: meerkat_runtime::store::PreparedWholeBlobProvisionalTail,
) -> Result<
meerkat_runtime::store::WholeBlobProvisionalTailAuthority,
meerkat_runtime::RuntimeStoreError,
> {
self.inner
.write_prepared_whole_blob_provisional_tail(runtime_id, prepared)
.await
}
async fn load_whole_blob_provisional_tail(
&self,
runtime_id: &meerkat_runtime::LogicalRuntimeId,
) -> Result<
Option<meerkat_runtime::store::CommittedWholeBlobProvisionalTail>,
meerkat_runtime::RuntimeStoreError,
> {
self.inner
.load_whole_blob_provisional_tail(runtime_id)
.await
}
async fn discard_whole_blob_provisional_tail(
&self,
runtime_id: &meerkat_runtime::LogicalRuntimeId,
expected: &meerkat_runtime::store::WholeBlobProvisionalTailAuthority,
) -> Result<bool, meerkat_runtime::RuntimeStoreError> {
self.inner
.discard_whole_blob_provisional_tail(runtime_id, expected)
.await
}
fn supports_compaction_projection_outbox(&self) -> bool {
meerkat_runtime::RuntimeStore::supports_compaction_projection_outbox(
self.inner.as_ref(),
)
}
fn input_state_batch_cas_implementation_profile(
&self,
) -> meerkat_runtime::store::InputStateBatchCasImplementationProfile {
self.inner.input_state_batch_cas_implementation_profile()
}
async fn observe_machine_lifecycle(
&self,
runtime_id: &meerkat_runtime::LogicalRuntimeId,
) -> Result<
meerkat_runtime::store::MachineLifecycleObservation,
meerkat_runtime::RuntimeStoreError,
> {
self.inner.observe_machine_lifecycle(runtime_id).await
}
async fn compare_and_swap_machine_lifecycle(
&self,
runtime_id: &meerkat_runtime::LogicalRuntimeId,
expected: meerkat_runtime::store::MachineLifecycleExpectedVersion,
replacement: meerkat_runtime::store::MachineLifecycleCommit,
) -> Result<
meerkat_runtime::store::MachineLifecycleCasOutcome,
meerkat_runtime::RuntimeStoreError,
> {
self.inner
.compare_and_swap_machine_lifecycle(runtime_id, expected, replacement)
.await
}
async fn compare_and_swap_machine_lifecycle_with_fence(
&self,
runtime_id: &meerkat_runtime::LogicalRuntimeId,
expected: meerkat_runtime::store::MachineLifecycleExpectedVersion,
replacement: meerkat_runtime::store::MachineLifecycleCommit,
write_fence: Arc<dyn meerkat_runtime::store::RuntimeStoreWriteFence>,
) -> Result<
meerkat_runtime::store::FencedMachineLifecycleCasOutcome,
meerkat_runtime::RuntimeStoreError,
> {
self.inner
.compare_and_swap_machine_lifecycle_with_fence(
runtime_id,
expected,
replacement,
write_fence,
)
.await
}
async fn commit_session_snapshot(
&self,
runtime_id: &meerkat_runtime::LogicalRuntimeId,
session_delta: meerkat_runtime::store::SerializedSessionSnapshot,
) -> Result<(), meerkat_runtime::RuntimeStoreError> {
meerkat_runtime::RuntimeStore::commit_session_snapshot(
self.inner.as_ref(),
runtime_id,
session_delta,
)
.await
}
async fn commit_prepared_whole_blob_rewrite_boundary(
&self,
runtime_id: &meerkat_runtime::LogicalRuntimeId,
boundary: meerkat_runtime::store::PreparedWholeBlobRewriteStoreParts,
) -> Result<
meerkat_runtime::store::WholeBlobStoreAuthority,
meerkat_runtime::RuntimeStoreError,
> {
self.inner
.commit_prepared_whole_blob_rewrite_boundary(runtime_id, boundary)
.await
}
async fn atomic_apply(
&self,
runtime_id: &meerkat_runtime::LogicalRuntimeId,
session_delta: Option<meerkat_runtime::store::SerializedSessionSnapshot>,
receipt: meerkat_core::lifecycle::RunBoundaryReceipt,
input_updates: Vec<meerkat_runtime::input_state::InputStatePersistenceRecord>,
session_store_key: Option<SessionId>,
) -> Result<(), meerkat_runtime::RuntimeStoreError> {
meerkat_runtime::RuntimeStore::atomic_apply(
self.inner.as_ref(),
runtime_id,
session_delta,
receipt,
input_updates,
session_store_key,
)
.await
}
async fn atomic_apply_with_machine_lifecycle(
&self,
runtime_id: &meerkat_runtime::LogicalRuntimeId,
session_delta: meerkat_runtime::store::SerializedSessionSnapshot,
receipt: meerkat_core::lifecycle::RunBoundaryReceipt,
machine_lifecycle: meerkat_runtime::store::MachineLifecycleCommit,
input_updates: Vec<meerkat_runtime::input_state::InputStatePersistenceRecord>,
session_store_key: SessionId,
) -> Result<(), meerkat_runtime::RuntimeStoreError> {
self.inner
.atomic_apply_with_machine_lifecycle(
runtime_id,
session_delta,
receipt,
machine_lifecycle,
input_updates,
session_store_key,
)
.await
}
async fn load_input_states(
&self,
runtime_id: &meerkat_runtime::LogicalRuntimeId,
) -> Result<Vec<meerkat_runtime::InputStateRow>, meerkat_runtime::RuntimeStoreError>
{
meerkat_runtime::RuntimeStore::load_input_states(self.inner.as_ref(), runtime_id).await
}
async fn load_input_states_with_versions(
&self,
runtime_id: &meerkat_runtime::LogicalRuntimeId,
) -> Result<
meerkat_runtime::store::PreparedRecoveryInputSnapshot,
meerkat_runtime::RuntimeStoreError,
> {
self.inner.load_input_states_with_versions(runtime_id).await
}
async fn load_boundary_receipt(
&self,
runtime_id: &meerkat_runtime::LogicalRuntimeId,
run_id: &meerkat_core::RunId,
sequence: u64,
) -> Result<
Option<meerkat_core::lifecycle::RunBoundaryReceipt>,
meerkat_runtime::RuntimeStoreError,
> {
meerkat_runtime::RuntimeStore::load_boundary_receipt(
self.inner.as_ref(),
runtime_id,
run_id,
sequence,
)
.await
}
async fn load_committed_boundary_receipts(
&self,
runtime_id: &meerkat_runtime::LogicalRuntimeId,
run_id: &meerkat_core::RunId,
) -> Result<
Vec<meerkat_core::lifecycle::RunBoundaryReceipt>,
meerkat_runtime::RuntimeStoreError,
> {
self.inner
.load_committed_boundary_receipts(runtime_id, run_id)
.await
}
async fn load_durable_tail_recovery_receipts(
&self,
runtime_id: &meerkat_runtime::LogicalRuntimeId,
run_id: &meerkat_core::RunId,
) -> Result<
Vec<meerkat_runtime::store::PreparedRecoveryReceiptSource>,
meerkat_runtime::RuntimeStoreError,
> {
self.inner
.load_durable_tail_recovery_receipts(runtime_id, run_id)
.await
}
async fn load_committed_recovery_boundary(
&self,
runtime_id: &meerkat_runtime::LogicalRuntimeId,
candidate_id: &str,
) -> Result<
Option<meerkat_runtime::store::CommittedRecoveryBoundary>,
meerkat_runtime::RuntimeStoreError,
> {
self.inner
.load_committed_recovery_boundary(runtime_id, candidate_id)
.await
}
async fn load_session_snapshot(
&self,
runtime_id: &meerkat_runtime::LogicalRuntimeId,
) -> Result<Option<std::sync::Arc<Vec<u8>>>, meerkat_runtime::RuntimeStoreError> {
meerkat_runtime::RuntimeStore::load_session_snapshot(self.inner.as_ref(), runtime_id)
.await
}
async fn load_pending_compaction_projections(
&self,
runtime_id: &meerkat_runtime::LogicalRuntimeId,
) -> Result<Vec<meerkat_core::CompactionProjectionIntent>, meerkat_runtime::RuntimeStoreError>
{
meerkat_runtime::RuntimeStore::load_pending_compaction_projections(
self.inner.as_ref(),
runtime_id,
)
.await
}
async fn mark_compaction_projection_finalized(
&self,
runtime_id: &meerkat_runtime::LogicalRuntimeId,
projection: &meerkat_core::CompactionProjectionId,
) -> Result<(), meerkat_runtime::RuntimeStoreError> {
meerkat_runtime::RuntimeStore::mark_compaction_projection_finalized(
self.inner.as_ref(),
runtime_id,
projection,
)
.await
}
async fn clear_session_snapshot(
&self,
runtime_id: &meerkat_runtime::LogicalRuntimeId,
) -> Result<(), meerkat_runtime::RuntimeStoreError> {
meerkat_runtime::RuntimeStore::clear_session_snapshot(self.inner.as_ref(), runtime_id)
.await
}
async fn replace_session_snapshot_if_current(
&self,
runtime_id: &meerkat_runtime::LogicalRuntimeId,
expected_current: &[u8],
replacement: Vec<u8>,
) -> Result<bool, meerkat_runtime::RuntimeStoreError> {
meerkat_runtime::RuntimeStore::replace_session_snapshot_if_current(
self.inner.as_ref(),
runtime_id,
expected_current,
replacement,
)
.await
}
async fn clear_session_snapshot_if_current(
&self,
runtime_id: &meerkat_runtime::LogicalRuntimeId,
expected_current: &[u8],
) -> Result<bool, meerkat_runtime::RuntimeStoreError> {
meerkat_runtime::RuntimeStore::clear_session_snapshot_if_current(
self.inner.as_ref(),
runtime_id,
expected_current,
)
.await
}
async fn is_runtime_projection_quarantined(
&self,
runtime_id: &meerkat_runtime::LogicalRuntimeId,
) -> Result<bool, meerkat_runtime::RuntimeStoreError> {
self.inner
.is_runtime_projection_quarantined(runtime_id)
.await
}
async fn persist_input_state(
&self,
runtime_id: &meerkat_runtime::LogicalRuntimeId,
state: &meerkat_runtime::input_state::InputStatePersistenceRecord,
) -> Result<(), meerkat_runtime::RuntimeStoreError> {
meerkat_runtime::RuntimeStore::persist_input_state(
self.inner.as_ref(),
runtime_id,
state,
)
.await
}
async fn persist_input_states_atomically(
&self,
runtime_id: &meerkat_runtime::LogicalRuntimeId,
states: &[meerkat_runtime::input_state::InputStatePersistenceRecord],
) -> Result<(), meerkat_runtime::RuntimeStoreError> {
self.inner
.persist_input_states_atomically(runtime_id, states)
.await
}
async fn compare_and_swap_input_states_atomically(
&self,
runtime_id: &meerkat_runtime::LogicalRuntimeId,
expected: &[meerkat_runtime::input_state::StoredInputState],
replacements: &[meerkat_runtime::input_state::InputStatePersistenceRecord],
) -> Result<
meerkat_runtime::store::InputStateBatchCasOutcome,
meerkat_runtime::RuntimeStoreError,
> {
self.inner
.compare_and_swap_input_states_atomically(runtime_id, expected, replacements)
.await
}
async fn compare_and_swap_input_states_atomically_with_fence(
&self,
runtime_id: &meerkat_runtime::LogicalRuntimeId,
expected: &[meerkat_runtime::input_state::StoredInputState],
replacements: &[meerkat_runtime::input_state::InputStatePersistenceRecord],
write_fence: Arc<dyn meerkat_runtime::store::RuntimeStoreWriteFence>,
) -> Result<
meerkat_runtime::store::FencedInputStateBatchCasOutcome,
meerkat_runtime::RuntimeStoreError,
> {
self.inner
.compare_and_swap_input_states_atomically_with_fence(
runtime_id,
expected,
replacements,
write_fence,
)
.await
}
async fn compare_and_swap_recovery_input_states_atomically(
&self,
runtime_id: &meerkat_runtime::LogicalRuntimeId,
expected_revision: meerkat_runtime::store::RecoveryInputSetRevision,
mutations: &[meerkat_runtime::store::RecoveryInputStateMutation],
) -> Result<
meerkat_runtime::store::InputStateBatchCasOutcome,
meerkat_runtime::RuntimeStoreError,
> {
self.inner
.compare_and_swap_recovery_input_states_atomically(
runtime_id,
expected_revision,
mutations,
)
.await
}
async fn compare_and_swap_recovery_input_states_atomically_with_fence(
&self,
runtime_id: &meerkat_runtime::LogicalRuntimeId,
expected_revision: meerkat_runtime::store::RecoveryInputSetRevision,
mutations: &[meerkat_runtime::store::RecoveryInputStateMutation],
write_fence: Arc<dyn meerkat_runtime::store::RuntimeStoreWriteFence>,
) -> Result<
meerkat_runtime::store::FencedInputStateBatchCasOutcome,
meerkat_runtime::RuntimeStoreError,
> {
self.inner
.compare_and_swap_recovery_input_states_atomically_with_fence(
runtime_id,
expected_revision,
mutations,
write_fence,
)
.await
}
async fn load_input_state(
&self,
runtime_id: &meerkat_runtime::LogicalRuntimeId,
input_id: &meerkat_core::InputId,
) -> Result<
Option<meerkat_runtime::input_state::StoredInputState>,
meerkat_runtime::RuntimeStoreError,
> {
meerkat_runtime::RuntimeStore::load_input_state(
self.inner.as_ref(),
runtime_id,
input_id,
)
.await
}
async fn load_input_state_by_idempotency_key(
&self,
runtime_id: &meerkat_runtime::LogicalRuntimeId,
key: &meerkat_runtime::IdempotencyKey,
) -> Result<
Option<meerkat_runtime::store::ExactInputStateObservation>,
meerkat_runtime::RuntimeStoreError,
> {
self.inner
.load_input_state_by_idempotency_key(runtime_id, key)
.await
}
async fn load_input_states_by_ids(
&self,
runtime_id: &meerkat_runtime::LogicalRuntimeId,
input_ids: &[meerkat_core::InputId],
) -> Result<
Vec<Option<meerkat_runtime::input_state::StoredInputState>>,
meerkat_runtime::RuntimeStoreError,
> {
self.inner
.load_input_states_by_ids(runtime_id, input_ids)
.await
}
async fn load_pending_terminal_owner_ids_page(
&self,
runtime_id: &meerkat_runtime::LogicalRuntimeId,
after: Option<&meerkat_core::InputId>,
limit: usize,
) -> Result<Vec<meerkat_core::InputId>, meerkat_runtime::RuntimeStoreError> {
self.inner
.load_pending_terminal_owner_ids_page(runtime_id, after, limit)
.await
}
async fn load_machine_lifecycle_record(
&self,
runtime_id: &meerkat_runtime::LogicalRuntimeId,
) -> Result<Option<Vec<u8>>, meerkat_runtime::RuntimeStoreError> {
meerkat_runtime::RuntimeStore::load_machine_lifecycle_record(
self.inner.as_ref(),
runtime_id,
)
.await
}
async fn commit_machine_lifecycle(
&self,
runtime_id: &meerkat_runtime::LogicalRuntimeId,
commit: meerkat_runtime::store::MachineLifecycleCommit,
input_states: &[meerkat_runtime::input_state::InputStatePersistenceRecord],
) -> Result<(), meerkat_runtime::RuntimeStoreError> {
meerkat_runtime::RuntimeStore::commit_machine_lifecycle(
self.inner.as_ref(),
runtime_id,
commit,
input_states,
)
.await
}
async fn commit_unregister_finalization(
&self,
runtime_id: &meerkat_runtime::LogicalRuntimeId,
finalization: meerkat_runtime::store::UnregisterFinalizationCommit,
) -> Result<(), meerkat_runtime::RuntimeStoreError> {
if self.fail_delete.swap(false, AtomicOrdering::AcqRel) {
return Err(meerkat_runtime::RuntimeStoreError::WriteFailed(
"synthetic REST unregister cleanup failure".to_string(),
));
}
meerkat_runtime::RuntimeStore::commit_unregister_finalization(
self.inner.as_ref(),
runtime_id,
finalization,
)
.await
}
async fn persist_ops_lifecycle(
&self,
runtime_id: &meerkat_runtime::LogicalRuntimeId,
snapshot: &meerkat_runtime::PersistedOpsSnapshot,
) -> Result<(), meerkat_runtime::RuntimeStoreError> {
meerkat_runtime::RuntimeStore::persist_ops_lifecycle(
self.inner.as_ref(),
runtime_id,
snapshot,
)
.await
}
async fn initialize_ops_lifecycle_if_absent(
&self,
runtime_id: &meerkat_runtime::LogicalRuntimeId,
candidate: &meerkat_runtime::PersistedOpsSnapshot,
) -> Result<meerkat_runtime::PersistedOpsSnapshot, meerkat_runtime::RuntimeStoreError>
{
meerkat_runtime::RuntimeStore::initialize_ops_lifecycle_if_absent(
self.inner.as_ref(),
runtime_id,
candidate,
)
.await
}
async fn load_ops_lifecycle(
&self,
runtime_id: &meerkat_runtime::LogicalRuntimeId,
) -> Result<Option<meerkat_runtime::PersistedOpsSnapshot>, meerkat_runtime::RuntimeStoreError>
{
meerkat_runtime::RuntimeStore::load_ops_lifecycle(self.inner.as_ref(), runtime_id).await
}
async fn delete_ops_lifecycle(
&self,
runtime_id: &meerkat_runtime::LogicalRuntimeId,
) -> Result<(), meerkat_runtime::RuntimeStoreError> {
if self.fail_delete.swap(false, AtomicOrdering::AcqRel) {
return Err(meerkat_runtime::RuntimeStoreError::WriteFailed(
"synthetic REST unregister cleanup failure".to_string(),
));
}
meerkat_runtime::RuntimeStore::delete_ops_lifecycle(self.inner.as_ref(), runtime_id)
.await
}
}
#[tokio::test]
async fn rest_new_runtime_compensation_combines_cleanup_failure_and_remains_retryable() {
let inner = Arc::new(meerkat_runtime::InMemoryRuntimeStore::new());
let store = Arc::new(FailDeleteOpsLifecycleOnceStore {
inner: Arc::clone(&inner),
fail_delete: AtomicBool::new(true),
});
let adapter = meerkat_runtime::MeerkatMachine::persistent(
store as Arc<dyn meerkat_runtime::RuntimeStore>,
Arc::new(meerkat_store::MemoryBlobStore::new()),
);
let session_id = SessionId::new();
let runtime_id = meerkat_runtime::LogicalRuntimeId::for_session(&session_id);
let snapshot = meerkat_runtime::RuntimeOpsLifecycleRegistry::new()
.capture_persistence_snapshot(
meerkat_core::RuntimeEpochId::new(),
&meerkat_core::EpochCursorState::new(),
)
.expect("capture ops lifecycle snapshot");
meerkat_runtime::RuntimeStore::persist_ops_lifecycle(
inner.as_ref(),
&runtime_id,
&snapshot,
)
.await
.expect("persist ops lifecycle snapshot");
adapter
.register_session(session_id.clone())
.await
.expect("register persistent runtime");
let primary = SessionError::Agent(meerkat_core::AgentError::InternalError(
"synthetic REST materialization failure".to_string(),
));
let error = unregister_runtime_adapter_if_new_after_session_error(
&adapter,
&session_id,
false,
primary,
)
.await;
let message = error.to_string();
assert!(
message.contains("synthetic REST materialization failure"),
"combined error must retain primary failure: {message}"
);
assert!(
message.contains("synthetic REST unregister cleanup failure"),
"combined error must retain cleanup failure: {message}"
);
assert!(
adapter.contains_session(&session_id).await,
"failed unregister must retain runtime registration for retry"
);
unregister_runtime_adapter_if_new(&adapter, &session_id, false)
.await
.expect("retry should complete retained unregister work");
assert!(
!adapter.contains_session(&session_id).await,
"successful retry must remove runtime registration"
);
}
async fn resolved_default_model(state: &AppState) -> String {
resolve_default_model(state)
.await
.expect("default model must resolve")
}
#[test]
fn sse_serialization_fault_surfaces_as_error_event_not_fake_success() {
let err = serde_json::from_str::<serde_json::Value>("{not json}")
.expect_err("invalid JSON must fail to parse");
let data = sse_serialization_error_data(&err);
assert_eq!(SSE_SERIALIZATION_ERROR_EVENT, "error");
let parsed: serde_json::Value =
serde_json::from_str(&data).expect("error payload must be valid JSON");
assert_eq!(parsed["error"], "event_serialization_failed");
assert!(
parsed["message"].as_str().is_some_and(|m| !m.is_empty()),
"typed cause must be carried in `message`, got: {data}"
);
assert_ne!(
parsed["payload"]["type"], "unknown",
"must not launder into a fabricated `unknown` success event"
);
assert!(
!data.is_empty(),
"must not launder into an empty data frame"
);
}
async fn runtime_terminated_completion_handle(
adapter: &Arc<meerkat_runtime::meerkat_machine::MeerkatMachine>,
session_id: &SessionId,
reason: &str,
) -> meerkat_runtime::CompletionHandle {
adapter
.register_session(session_id.clone())
.await
.expect("test machine should register runtime authority");
let input = meerkat_runtime::Input::Prompt(meerkat_runtime::PromptInput::new(
"pending completion fixture",
None,
));
let (_outcome, handle) = adapter
.accept_input_with_completion(session_id, input)
.await
.expect("test machine should accept pending completion input");
let handle = handle.expect("pending completion input should return a completion handle");
adapter
.stop_runtime_executor(session_id, reason)
.await
.expect("test machine should resolve pending completion as runtime terminated");
handle
}
#[test]
fn rest_peer_response_terminal_body_deserializes_typed_identity_and_correlation() {
let body: RestPeerResponseTerminalBody = serde_json::from_value(json!({
"peer_id": "00000000-0000-4000-8000-000000000161",
"display_name": "analyst",
"request_id": "00000000-0000-4000-8000-000000000162",
"status": "completed",
"result": {"ok": true},
}))
.unwrap();
assert_eq!(
body.peer_id.to_string(),
"00000000-0000-4000-8000-000000000161"
);
assert_eq!(body.display_name.unwrap().as_str(), "analyst");
assert_eq!(
body.request_id.to_string(),
"00000000-0000-4000-8000-000000000162"
);
assert_eq!(
body.status,
meerkat_contracts::PeerResponseTerminalStatusWire::Completed
);
assert_eq!(body.result["ok"], true);
}
#[test]
fn rest_context_system_notice_projects_via_typed_notice() {
let blocks = vec![meerkat_core::types::SystemNoticeBlock::Comms {
sender_taint: None,
kind: meerkat_core::types::CommsNoticeKind::ResponseTerminal,
direction: meerkat_core::types::SystemNoticeDirection::Incoming,
peer: None,
request_id: Some("req-1".to_string()),
intent: Some("checksum_token".to_string()),
status: Some("completed".to_string()),
summary: Some("Peer terminal response".to_string()),
payload: None,
content: Vec::new(),
}];
let content = CoreRenderable::SystemNotice {
kind: meerkat_core::types::SystemNoticeKind::Comms,
body: Some("Peer terminal response context".to_string()),
blocks: blocks.clone(),
};
assert_eq!(
content.render_text(),
meerkat_core::types::SystemNoticeMessage::with_blocks(
meerkat_core::types::SystemNoticeKind::Comms,
Some("Peer terminal response context".to_string()),
blocks,
)
.model_projection_text()
);
}
#[test]
fn interrupt_noop_target_for_presence_tracks_authoritative_presence_only() {
assert_eq!(
interrupt_noop_target_for_presence(true),
InterruptNoopTarget::Present
);
assert_eq!(
interrupt_noop_target_for_presence(false),
InterruptNoopTarget::Missing
);
}
#[test]
fn rest_peer_response_terminal_body_rejects_name_only_origin() {
let err = serde_json::from_value::<RestPeerResponseTerminalBody>(json!({
"peer_name": "analyst",
"request_id": "00000000-0000-4000-8000-000000000162",
"status": "completed",
"result": null,
}))
.unwrap_err();
assert!(
err.to_string().contains("peer_id"),
"unexpected error: {err}"
);
}
#[test]
fn rest_peer_response_terminal_body_rejects_mixed_peer_name_origin() {
let err = serde_json::from_value::<RestPeerResponseTerminalBody>(json!({
"peer_id": "00000000-0000-4000-8000-000000000161",
"peer_name": "analyst",
"request_id": "00000000-0000-4000-8000-000000000162",
"status": "completed",
"result": null,
}))
.unwrap_err();
assert!(
err.to_string().contains("peer_name"),
"unexpected error: {err}"
);
}
#[test]
fn rest_peer_response_terminal_body_rejects_stringly_request_id() {
let err = serde_json::from_value::<RestPeerResponseTerminalBody>(json!({
"peer_id": "00000000-0000-4000-8000-000000000161",
"request_id": "req-1",
"status": "completed",
"result": null,
}))
.unwrap_err();
assert!(
err.to_string().contains("UUID") || err.to_string().contains("uuid"),
"unexpected error: {err}"
);
}
struct MockLlmClient;
struct BlockingMockLlmClient {
calls: Arc<AtomicUsize>,
release: Arc<tokio::sync::Semaphore>,
}
struct ErrorLlmClient;
#[async_trait]
impl LlmClient for MockLlmClient {
fn project_replay_messages(
&self,
messages: &[meerkat_core::Message],
) -> Result<Vec<meerkat_core::Message>, meerkat_client::LlmError> {
Ok(messages.to_vec())
}
fn stream<'a>(
&'a self,
_request: &'a LlmRequest,
) -> Pin<Box<dyn futures::Stream<Item = Result<LlmEvent, LlmError>> + Send + 'a>> {
Box::pin(stream::iter(vec![
Ok(LlmEvent::TextDelta {
delta: "ok".to_string(),
meta: None,
}),
Ok(LlmEvent::Done {
outcome: LlmDoneOutcome::Success {
stop_reason: meerkat_core::StopReason::EndTurn,
},
}),
]))
}
fn provider(&self) -> meerkat_core::Provider {
meerkat_core::Provider::Other
}
async fn health_check(&self) -> Result<(), LlmError> {
Ok(())
}
}
#[async_trait]
impl LlmClient for BlockingMockLlmClient {
fn project_replay_messages(
&self,
messages: &[meerkat_core::Message],
) -> Result<Vec<meerkat_core::Message>, meerkat_client::LlmError> {
Ok(messages.to_vec())
}
fn stream<'a>(
&'a self,
_request: &'a LlmRequest,
) -> Pin<Box<dyn futures::Stream<Item = Result<LlmEvent, LlmError>> + Send + 'a>> {
let calls = Arc::clone(&self.calls);
let release = Arc::clone(&self.release);
Box::pin(async_stream::stream! {
calls.fetch_add(1, AtomicOrdering::SeqCst);
let permit = release
.acquire()
.await
.expect("blocking mock release semaphore should stay open");
drop(permit);
yield Ok(LlmEvent::TextDelta {
delta: "ok".to_string(),
meta: None,
});
yield Ok(LlmEvent::Done {
outcome: LlmDoneOutcome::Success {
stop_reason: meerkat_core::StopReason::EndTurn,
},
});
})
}
fn provider(&self) -> meerkat_core::Provider {
meerkat_core::Provider::Other
}
async fn health_check(&self) -> Result<(), LlmError> {
Ok(())
}
}
#[async_trait]
impl LlmClient for ErrorLlmClient {
fn project_replay_messages(
&self,
messages: &[meerkat_core::Message],
) -> Result<Vec<meerkat_core::Message>, meerkat_client::LlmError> {
Ok(messages.to_vec())
}
fn stream<'a>(
&'a self,
_request: &'a LlmRequest,
) -> Pin<Box<dyn futures::Stream<Item = Result<LlmEvent, LlmError>> + Send + 'a>> {
Box::pin(stream::iter(vec![Err(LlmError::Unknown {
message: "boom".to_string(),
})]))
}
fn provider(&self) -> meerkat_core::Provider {
meerkat_core::Provider::Other
}
async fn health_check(&self) -> Result<(), LlmError> {
Ok(())
}
}
fn hooks_override_fixture() -> HookRunOverrides {
let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("../test-fixtures/hooks/run_override.json");
let payload = std::fs::read_to_string(path).expect("hook override fixture must exist");
serde_json::from_str::<HookRunOverrides>(&payload)
.expect("hook override fixture must deserialize")
}
fn self_hosted_test_config(inline_video: bool) -> Config {
let mut config = Config::default();
config.self_hosted.servers.insert(
"local".to_string(),
SelfHostedServerConfig {
transport: SelfHostedTransport::OpenAiCompatible,
base_url: "http://127.0.0.1:11434".to_string(),
api_style: SelfHostedApiStyle::ChatCompletions,
},
);
config.self_hosted.models.insert(
"gemma-4-e2b".to_string(),
serde_json::from_value(json!({
"server": "local",
"remote_model": "gemma4:e2b",
"display_name": "Gemma 4 E2B",
"family": "gemma-4",
"tier": "supported",
"context_window": 128000,
"max_output_tokens": 8192,
"vision": true,
"image_tool_results": true,
"inline_video": inline_video,
"supports_temperature": true,
"supports_thinking": true,
"supports_reasoning": true,
"call_timeout_secs": 600
}))
.expect("self-hosted model config"),
);
config
}
fn inline_video_prompt() -> ContentInput {
ContentInput::Blocks(vec![meerkat_core::ContentBlock::Video {
media_type: "video/mp4".to_string(),
duration_ms: 1_000,
data: meerkat_core::VideoData::Inline {
data: "AAAA".to_string(),
},
}])
}
fn validation_identity(provider: Provider, model: &str) -> SessionLlmIdentity {
SessionLlmIdentity {
model: model.to_string(),
provider,
self_hosted_server_id: None,
provider_params: None,
auth_binding: None,
}
}
#[tokio::test]
async fn rest_app_state_reopen_preserves_persistent_oauth_flow_authority() {
let temp = TempDir::new().unwrap();
let mut bootstrap = RuntimeBootstrap::default();
bootstrap.realm.selection = RealmSelection::Explicit {
realm_id: "luc-194-rest-oauth".to_string(),
};
bootstrap.realm.backend_hint = Some("sqlite".to_string());
bootstrap.realm.state_root = Some(temp.path().join("realms"));
bootstrap.context.context_root = Some(temp.path().to_path_buf());
let target = meerkat_core::AuthBindingRef {
realm: meerkat_core::RealmId::parse("dev").expect("valid realm fixture"),
binding: meerkat_core::BindingId::parse("default_openai")
.expect("valid binding fixture"),
profile: None,
origin: meerkat_core::connection::BindingOrigin::Configured,
};
let provider = meerkat_providers::oauth_flow::OAuthProviderIdentity::OpenAiChatGpt;
let redirect_uri = "http://127.0.0.1:1455/callback";
let state_token = {
let state = AppState::load_from_with_bootstrap(
temp.path().to_path_buf(),
bootstrap.clone(),
false,
)
.await
.unwrap();
state
.oauth_flow_authority()
.start(
target.clone(),
provider,
redirect_uri.to_string(),
"rest-persistent-verifier".to_string(),
)
.expect("persistent REST authority admits OAuth flow")
};
let reopened =
AppState::load_from_with_bootstrap(temp.path().to_path_buf(), bootstrap, false)
.await
.unwrap();
let flow = reopened
.oauth_flow_authority()
.consume(&state_token, &target, provider, redirect_uri)
.expect("reopened REST state must preserve persistent OAuth authority");
assert_eq!(flow.pkce_verifier, "rest-persistent-verifier");
}
#[tokio::test]
async fn rest_apply_does_not_rematerialize_session_lost_after_preparation() {
let temp = TempDir::new().unwrap();
let mut state = AppState::load_from(temp.path().to_path_buf())
.await
.unwrap();
state.llm_client_override = Some(Arc::new(MockLlmClient));
let created = state
.session_service
.create_session(SvcCreateSessionRequest {
injected_context: Vec::new(),
model: resolved_default_model(&state).await,
prompt: "Hello".to_string().into(),
system_prompt: meerkat::SystemPromptOverride::Inherit,
max_tokens: Some(state.max_tokens),
event_tx: None,
initial_turn: InitialTurnPolicy::Defer,
deferred_prompt_policy: DeferredPromptPolicy::Discard,
build: Some(SessionBuildOptions {
llm_client_override: state
.llm_client_override
.clone()
.map(encode_llm_client_override_for_service),
..Default::default()
}),
labels: None,
})
.await
.expect("create session");
let session_id = created.session_id;
ensure_rest_session_runtime_executor(&state, &session_id)
.await
.expect("prepare REST runtime executor");
state
.session_service
.discard_live_session(&session_id)
.await
.expect("simulate live session loss after preparation");
let primitive =
RunPrimitive::StagedInput(meerkat_core::lifecycle::run_primitive::StagedRunInput {
boundary: RunApplyBoundary::RunCheckpoint,
appends: vec![meerkat_core::lifecycle::run_primitive::ConversationAppend {
role: meerkat_core::lifecycle::run_primitive::ConversationAppendRole::User,
content: CoreRenderable::Text {
text: "must not rematerialize from apply".to_string(),
},
identity: None,
}],
contributing_input_ids: vec![meerkat_core::lifecycle::InputId::new()],
turn_metadata: Some(
meerkat_core::lifecycle::run_primitive::RuntimeTurnMetadata {
execution_kind: Some(
meerkat_core::lifecycle::RuntimeExecutionKind::ContentTurn,
),
..Default::default()
},
),
});
let mut executor =
RestSessionRuntimeExecutor::new(state.runtime_executor_context(), session_id.clone());
let error = CoreExecutor::apply(&mut executor, meerkat_core::RunId::new(), primitive)
.await
.expect_err("post-preparation loss must surface to runtime-loop handoff");
assert!(error.requires_runtime_teardown());
assert!(
!state
.session_service
.has_live_session(&session_id)
.await
.expect("check live absence")
);
assert!(state.runtime_adapter.contains_session(&session_id).await);
state
.runtime_adapter
.unregister_session(&session_id)
.await
.expect("external owner unregisters after apply returns");
}
#[tokio::test]
async fn rest_cold_start_materializes_before_non_empty_compaction_outbox_validation() {
let temp = TempDir::new().unwrap();
let mut state = AppState::load_from(temp.path().to_path_buf())
.await
.unwrap();
state.llm_client_override = Some(Arc::new(MockLlmClient));
let session_id = create_deferred_rest_runtime_session(&state).await;
state
.session_service
.discard_live_session(&session_id)
.await
.expect("discard live session");
state
.runtime_adapter
.unregister_session(&session_id)
.await
.expect("remove runtime registration");
let input_id = meerkat_core::lifecycle::InputId::new();
let admission = state
.session_service
.reserve_runtime_turn_admission(&session_id)
.await
.expect("reserve cold REST recovery admission");
insert_rest_runtime_pre_admission(
&state.runtime_pre_admissions,
session_id.clone(),
input_id.clone(),
admission,
)
.await
.expect("record cold REST recovery admission");
let registration_lock = rest_runtime_registration_lock(&state, &session_id);
let registration_guard = registration_lock.mutex().lock().await;
prepare_rest_session_runtime_executor_locked(&state, &session_id, &input_id)
.await
.expect("cold startup must materialize before attaching the runtime loop");
drop(registration_guard);
drop(registration_lock);
let mut executor =
RestSessionRuntimeExecutor::new(state.runtime_executor_context(), session_id.clone());
let intent: meerkat_core::CompactionProjectionIntent =
serde_json::from_value(serde_json::json!({
"projection": {
"session_id": session_id,
"parent_revision": "cold-parent",
"revision": "cold-revision",
"commit_fingerprint": "sha256:rest-cold-start-outbox"
},
"summary_tokens": 3,
"messages_before": 2,
"messages_after": 1
}))
.expect("valid compaction intent fixture");
let error = tokio::time::timeout(
std::time::Duration::from_secs(2),
executor.reconcile_committed_compaction_projections(&[intent]),
)
.await
.expect("cold non-empty outbox reconciliation must not deadlock")
.expect_err("unbacked projection fixture must still fail closed");
assert!(
!error.to_string().contains("session not found"),
"non-empty outbox must reach the materialized memory owner: {error}"
);
assert!(
state
.session_service
.has_live_session(&session_id)
.await
.expect("check live session"),
"startup reconciliation must materialize the durable memory owner first"
);
}
#[cfg(feature = "mcp")]
#[tokio::test]
async fn rest_create_post_prepare_validation_failure_cleans_unpublished_state_without_request_context()
{
let temp = TempDir::new().unwrap();
let state = AppState::load_from(temp.path().to_path_buf())
.await
.unwrap();
let req: CreateSessionRequest = serde_json::from_value(json!({
"prompt": "hello",
"model": "gpt-5.4",
"provider": "anthropic"
}))
.expect("valid REST create request fixture");
let outcome = Box::pin(create_session_inner(&state, req, None)).await;
match outcome {
RequestTerminal::RespondWithoutPublish(Err(ApiError::BadRequest(message))) => {
assert!(
message.contains("provider"),
"expected provider validation failure after prepare: {message}"
);
}
other => panic!("expected bad request validation failure, got {other:?}"),
}
assert!(
state.mcp_sessions.read().await.is_empty(),
"post-prepare validation failure without request context should clean MCP state"
);
}
async fn create_archived_stale_rest_runtime_session(state: &AppState) -> SessionId {
let pre_session = Session::new();
let session_id = pre_session.id().clone();
let bindings = state
.runtime_adapter
.prepare_bindings(session_id.clone())
.await
.expect("runtime bindings should prepare");
state
.session_service
.create_session(SvcCreateSessionRequest {
injected_context: Vec::new(),
model: resolved_default_model(state).await,
prompt: "Hello".to_string().into(),
system_prompt: meerkat::SystemPromptOverride::Inherit,
max_tokens: Some(state.max_tokens),
event_tx: None,
initial_turn: InitialTurnPolicy::Defer,
deferred_prompt_policy: DeferredPromptPolicy::Discard,
build: Some(SessionBuildOptions {
custom_models: std::collections::BTreeMap::new(),
image_generation_provider: None,
auto_compact_threshold_override: None,
resume_session: Some(pre_session),
llm_client_override: state
.llm_client_override
.clone()
.map(encode_llm_client_override_for_service),
runtime_build_mode: meerkat_core::RuntimeBuildMode::SessionOwned(bindings),
..Default::default()
}),
labels: None,
})
.await
.expect("deferred session create should succeed");
state
.session_service
.archive_with_machine_protocol(
&session_id,
MachineSessionArchiveProtocol::from_machine(state.runtime_adapter.as_ref()),
)
.await
.expect("archive should retire through machine authority");
assert!(
state.runtime_adapter.contains_session(&session_id).await,
"machine archive leaves a retired runtime registration for cleanup"
);
session_id
}
async fn try_create_deferred_rest_runtime_session(
state: &AppState,
) -> Result<SessionId, SessionError> {
let pre_session = Session::new();
let session_id = pre_session.id().clone();
let bindings = state
.runtime_adapter
.prepare_bindings(session_id.clone())
.await
.expect("runtime bindings should prepare");
let created = match state
.session_service
.create_session(SvcCreateSessionRequest {
injected_context: Vec::new(),
model: resolved_default_model(state).await,
prompt: "Hello".to_string().into(),
system_prompt: meerkat::SystemPromptOverride::Inherit,
max_tokens: Some(state.max_tokens),
event_tx: None,
initial_turn: InitialTurnPolicy::Defer,
deferred_prompt_policy: DeferredPromptPolicy::Discard,
build: Some(SessionBuildOptions {
custom_models: std::collections::BTreeMap::new(),
image_generation_provider: None,
auto_compact_threshold_override: None,
resume_session: Some(pre_session),
llm_client_override: state
.llm_client_override
.clone()
.map(encode_llm_client_override_for_service),
runtime_build_mode: meerkat_core::RuntimeBuildMode::SessionOwned(bindings),
..Default::default()
}),
labels: None,
})
.await
{
Ok(created) => created,
Err(err) => {
if let Err(cleanup_error) =
state.runtime_adapter.unregister_session(&session_id).await
{
return Err(SessionError::Agent(
meerkat_core::AgentError::InternalError(format!(
"{err}; additionally failed to unregister prepared runtime session: {cleanup_error}"
)),
));
}
return Err(err);
}
};
Ok(created.session_id)
}
async fn create_deferred_rest_runtime_session(state: &AppState) -> SessionId {
try_create_deferred_rest_runtime_session(state)
.await
.expect("deferred session create should succeed")
}
#[cfg(feature = "mob")]
async fn insert_rest_archive_partial_destroy_mob(
mob_state: &Arc<meerkat_mob_mcp::MobMcpState>,
owner_session_id: &str,
) -> meerkat_mob::MobId {
insert_rest_archive_partial_destroy_mob_with_events(mob_state, owner_session_id)
.await
.0
}
#[cfg(feature = "mob")]
async fn insert_rest_archive_partial_destroy_mob_with_events(
mob_state: &Arc<meerkat_mob_mcp::MobMcpState>,
owner_session_id: &str,
) -> (
meerkat_mob::MobId,
Arc<meerkat_mob::store::InMemoryMobEventStore>,
) {
let mob_id = meerkat_mob::MobId::from("rest-session-archive-partial-destroy");
let mut definition = meerkat_mob::MobDefinition::explicit(mob_id.clone());
definition.profiles.insert(
meerkat_mob::ProfileName::from("worker"),
meerkat_mob::ProfileBinding::Inline(Box::new(meerkat_mob::Profile {
model: "claude-sonnet-4-5".to_string(),
provider: None,
self_hosted_server_id: None,
image_generation_provider: None,
auto_compact_threshold: None,
resume_overrides: Vec::new(),
skills: Vec::new(),
tools: meerkat_mob::ToolConfig::default(),
peer_description: "worker".to_string(),
external_addressable: false,
backend: None,
runtime_mode: meerkat_mob::MobRuntimeMode::TurnDriven,
max_inline_peer_notifications: None,
output_schema: None,
provider_params: None,
})),
);
let owner_session_id =
SessionId::parse(owner_session_id).expect("valid owner bridge session id");
let events = Arc::new(meerkat_mob::store::InMemoryMobEventStore::new());
events.fail_clear_until_allowed();
let storage = meerkat_mob::MobStorage::with_events(events.clone());
let handle = meerkat_mob::MobBuilder::new(definition, storage)
.with_owner_bridge_session_create_authority(owner_session_id, true, false)
.with_session_service(mob_state.session_service())
.allow_ephemeral_sessions(true)
.create()
.await
.expect("create archive-owned mob with failing event clear");
mob_state.mob_insert_handle(mob_id.clone(), handle).await;
(mob_id, events)
}
#[cfg(feature = "mob")]
async fn insert_rest_archive_live_member(
mob_state: &Arc<meerkat_mob_mcp::MobMcpState>,
) -> (meerkat_mob::MobId, SessionId) {
let mob_id = meerkat_mob::MobId::from("rest-session-archive-live-member");
let mut definition = meerkat_mob::MobDefinition::explicit(mob_id.clone());
definition.profiles.insert(
meerkat_mob::ProfileName::from("worker"),
meerkat_mob::ProfileBinding::Inline(Box::new(meerkat_mob::Profile {
model: "claude-sonnet-4-5".to_string(),
provider: None,
self_hosted_server_id: None,
image_generation_provider: None,
auto_compact_threshold: None,
resume_overrides: Vec::new(),
skills: Vec::new(),
tools: meerkat_mob::ToolConfig {
comms: true,
..meerkat_mob::ToolConfig::default()
},
peer_description: "worker".to_string(),
external_addressable: false,
backend: None,
runtime_mode: meerkat_mob::MobRuntimeMode::TurnDriven,
max_inline_peer_notifications: None,
output_schema: None,
provider_params: None,
})),
);
let handle = meerkat_mob::MobBuilder::new(definition, meerkat_mob::MobStorage::in_memory())
.with_session_service(mob_state.session_service())
.allow_ephemeral_sessions(true)
.create()
.await
.expect("create live archive mob");
let identity = meerkat_mob::AgentIdentity::from("worker-1");
handle
.spawn_spec(meerkat_mob::SpawnMemberSpec::new(
meerkat_mob::ProfileName::from("worker"),
identity.clone(),
))
.await
.expect("spawn live mob member");
let bridge_session_id = handle
.resolve_bridge_session_id(&identity)
.await
.expect("turn-driven worker should have a bridge session");
mob_state.mob_insert_handle(mob_id.clone(), handle).await;
(mob_id, bridge_session_id)
}
async fn load_rest_state_with_capacity(temp: &TempDir, max_sessions: usize) -> AppState {
let realm_id = "rest-capacity-test";
let realms_root = temp.path().join("realms");
let realm_root = realms_root.join(realm_id);
tokio::fs::create_dir_all(&realm_root)
.await
.expect("create realm config dir");
tokio::fs::write(
realm_root.join("config.toml"),
format!("[limits]\nmax_sessions = {max_sessions}\n"),
)
.await
.expect("write rest capacity config");
let mut bootstrap = RuntimeBootstrap::default();
bootstrap.realm.state_root = Some(realms_root);
bootstrap.realm.selection = meerkat_core::RealmSelection::Explicit {
realm_id: realm_id.to_string(),
};
bootstrap.context.context_root = Some(temp.path().to_path_buf());
AppState::load_from_with_bootstrap(temp.path().to_path_buf(), bootstrap, false)
.await
.expect("load rest app state with capacity")
}
async fn create_completed_rest_runtime_session(state: &AppState) -> SessionId {
let pre_session = Session::new();
let session_id = pre_session.id().clone();
let bindings = state
.runtime_adapter
.prepare_bindings(session_id.clone())
.await
.expect("runtime bindings should prepare");
let (request, initial_turn) =
split_runtime_backed_eager_create_request(SvcCreateSessionRequest {
injected_context: Vec::new(),
model: resolved_default_model(state).await,
prompt: "Hello".to_string().into(),
system_prompt: meerkat::SystemPromptOverride::Inherit,
max_tokens: Some(state.max_tokens),
event_tx: None,
initial_turn: InitialTurnPolicy::RunImmediately,
deferred_prompt_policy: DeferredPromptPolicy::Discard,
build: Some(SessionBuildOptions {
custom_models: std::collections::BTreeMap::new(),
image_generation_provider: None,
auto_compact_threshold_override: None,
resume_session: Some(pre_session),
llm_client_override: state
.llm_client_override
.clone()
.map(encode_llm_client_override_for_service),
runtime_build_mode: meerkat_core::RuntimeBuildMode::SessionOwned(bindings),
..Default::default()
}),
labels: None,
});
let created = state
.session_service
.create_session(request)
.await
.expect("deferred session create should succeed");
let initial_turn =
initial_turn.expect("runtime-backed eager fixture should split an initial turn");
let completed = run_runtime_backed_initial_turn_with_machine(
&state.session_service,
&state.runtime_adapter,
&created.session_id,
initial_turn,
)
.await
.expect("initial service turn should commit through machine receipt");
ensure_rest_session_runtime_executor(state, &created.session_id)
.await
.expect("REST runtime executor should attach");
completed.session_id
}
#[cfg(feature = "comms")]
async fn create_completed_rest_runtime_comms_session(state: &AppState) -> SessionId {
let pre_session = Session::new();
let session_id = pre_session.id().clone();
let bindings = state
.runtime_adapter
.prepare_bindings(session_id.clone())
.await
.expect("runtime bindings should prepare");
let (request, initial_turn) =
split_runtime_backed_eager_create_request(SvcCreateSessionRequest {
injected_context: Vec::new(),
model: resolved_default_model(state).await,
prompt: "Hello".to_string().into(),
system_prompt: meerkat::SystemPromptOverride::Inherit,
max_tokens: Some(state.max_tokens),
event_tx: None,
initial_turn: InitialTurnPolicy::RunImmediately,
deferred_prompt_policy: DeferredPromptPolicy::Discard,
build: Some(SessionBuildOptions {
custom_models: std::collections::BTreeMap::new(),
image_generation_provider: None,
auto_compact_threshold_override: None,
resume_session: Some(pre_session),
comms_name: Some("rest-capacity-target".to_string()),
keep_alive: false,
llm_client_override: state
.llm_client_override
.clone()
.map(encode_llm_client_override_for_service),
runtime_build_mode: meerkat_core::RuntimeBuildMode::SessionOwned(bindings),
..Default::default()
}),
labels: None,
});
let created = state
.session_service
.create_session(request)
.await
.expect("deferred comms session create should succeed");
let initial_turn =
initial_turn.expect("runtime-backed eager fixture should split an initial turn");
let completed = run_runtime_backed_initial_turn_with_machine(
&state.session_service,
&state.runtime_adapter,
&created.session_id,
initial_turn,
)
.await
.expect("initial comms service turn should commit through machine receipt");
ensure_rest_session_runtime_executor(state, &created.session_id)
.await
.expect("REST runtime executor should attach");
completed.session_id
}
async fn wait_for_rest_llm_calls(calls: &AtomicUsize, expected: usize, description: &str) {
for _ in 0..200 {
if calls.load(AtomicOrdering::SeqCst) >= expected {
return;
}
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
}
panic!("{description}: expected at least {expected} LLM calls");
}
async fn wait_for_rest_runtime_running(state: &AppState, session_id: &SessionId) {
for _ in 0..200 {
if matches!(
state.runtime_adapter.runtime_state(session_id).await,
Ok(meerkat_runtime::RuntimeState::Running)
) {
return;
}
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
}
let state = state.runtime_adapter.runtime_state(session_id).await;
panic!("runtime did not enter Running state: {state:?}");
}
async fn wait_for_rest_runtime_pre_admission(state: &AppState, session_id: &SessionId) {
for _ in 0..200 {
let has_pre_admission = state
.runtime_pre_admissions
.lock()
.await
.get(session_id)
.is_some_and(|entries| !entries.is_empty());
if has_pre_admission {
return;
}
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
}
panic!("runtime pre-admission was not registered for {session_id}");
}
#[tokio::test]
async fn rest_archived_runtime_input_webhooks_reject_and_unregister_stale_runtime() {
let temp = TempDir::new().unwrap();
let mut state = AppState::load_from(temp.path().to_path_buf())
.await
.unwrap();
state.llm_client_override = Some(Arc::new(MockLlmClient));
let generic_session_id = create_archived_stale_rest_runtime_session(&state).await;
let generic_input = make_runtime_external_event_input(
"archived_webhook_input",
json!({"archived": true}),
None,
)
.expect("external event input");
let generic_rejected = admit_runtime_input_via_webhook(
&state,
&generic_session_id,
generic_input,
WebhookAdmissionMode::WithoutWake,
)
.await
.expect_err("archived external event webhook should reject");
assert_eq!(generic_rejected.status(), StatusCode::NOT_FOUND);
assert!(
!state
.runtime_adapter
.contains_session(&generic_session_id)
.await,
"archived external event webhook should unregister stale runtime state"
);
let peer_session_id = create_archived_stale_rest_runtime_session(&state).await;
let peer_input = meerkat_runtime::peer_response_terminal_input(
meerkat_core::comms::PeerId::new(),
None,
meerkat_core::PeerCorrelationId::from_uuid(uuid::Uuid::new_v4()),
meerkat_contracts::PeerResponseTerminalStatusWire::Completed,
json!({"archived": true}),
);
let peer_rejected = admit_runtime_input_via_webhook(
&state,
&peer_session_id,
peer_input,
WebhookAdmissionMode::Wakeful,
)
.await
.expect_err("archived peer terminal webhook should reject");
assert_eq!(peer_rejected.status(), StatusCode::NOT_FOUND);
assert!(
!state
.runtime_adapter
.contains_session(&peer_session_id)
.await,
"archived peer terminal webhook should unregister stale runtime state"
);
}
#[tokio::test]
async fn rest_peer_terminal_webhook_allows_running_target_when_capacity_full() {
let temp = TempDir::new().unwrap();
let mut state = load_rest_state_with_capacity(&temp, 2).await;
let calls = Arc::new(AtomicUsize::new(0));
let release = Arc::new(tokio::sync::Semaphore::new(0));
state.llm_client_override = Some(Arc::new(BlockingMockLlmClient {
calls: Arc::clone(&calls),
release: Arc::clone(&release),
}));
let target_session_id = create_deferred_rest_runtime_session(&state).await;
let state_for_turn = state.clone();
let target_for_turn = target_session_id.to_string();
let running_turn = tokio::spawn(async move {
let body_session_id = target_for_turn.clone();
Box::pin(continue_session_inner(
&state_for_turn,
&target_for_turn,
ContinueSessionRequest {
injected_context: None,
transient_turn_context: None,
session_id: body_session_id,
prompt: ContentInput::Text("block while peer terminal arrives".to_string()),
system_prompt: None,
output_schema: None,
structured_output_retries: None,
keep_alive: None,
comms_name: None,
peer_meta: None,
verbose: false,
model: None,
provider: None,
auth_binding: None,
max_tokens: None,
hooks_override: None,
enable_web_search: None,
skill_refs: None,
turn_tool_overlay: None,
additional_instructions: None,
},
None,
))
.await
});
wait_for_rest_llm_calls(&calls, 1, "running target turn should reach LLM").await;
wait_for_rest_runtime_running(&state, &target_session_id).await;
let mut filler_sessions = Vec::new();
loop {
match try_create_deferred_rest_runtime_session(&state).await {
Ok(session_id) => filler_sessions.push(session_id),
Err(err) if err.to_string().contains("Max sessions") => break,
Err(err) => panic!("unexpected filler create error: {err:?}"),
}
}
assert!(
!filler_sessions.is_empty(),
"test should fill at least one additional active admission"
);
let peer_input = meerkat_runtime::peer_response_terminal_input(
meerkat_core::comms::PeerId::new(),
None,
meerkat_core::PeerCorrelationId::from_uuid(uuid::Uuid::new_v4()),
meerkat_contracts::PeerResponseTerminalStatusWire::Completed,
json!({"capacity": "full", "target": "running"}),
);
let webhook_state = state.clone();
let webhook_session_id = target_session_id.clone();
let webhook_task = tokio::spawn(async move {
admit_runtime_input_via_webhook(
&webhook_state,
&webhook_session_id,
peer_input,
WebhookAdmissionMode::Wakeful,
)
.await
});
wait_for_rest_runtime_pre_admission(&state, &target_session_id).await;
release.add_permits(1);
let completed = tokio::time::timeout(std::time::Duration::from_secs(5), running_turn)
.await
.expect("running turn should complete after releasing mock LLM")
.expect("running turn task should not panic");
assert!(
matches!(completed, RequestTerminal::Publish(Ok(_))),
"running turn should publish successfully: {completed:?}"
);
let admitted = tokio::time::timeout(std::time::Duration::from_secs(15), webhook_task)
.await
.expect("peer terminal webhook should finish after the running turn releases")
.expect("peer terminal webhook task should not panic")
.expect(
"running target peer terminal webhook should bypass new-session capacity precheck",
);
assert_eq!(admitted.0, StatusCode::ACCEPTED);
release.add_permits(1);
}
#[tokio::test]
async fn rest_peer_terminal_webhook_capacity_full_rejects_before_input_accept() {
let temp = TempDir::new().unwrap();
let mut state = load_rest_state_with_capacity(&temp, 1).await;
state.llm_client_override = Some(Arc::new(MockLlmClient));
let target_session_id = create_completed_rest_runtime_session(&state).await;
let mut filler_sessions = Vec::new();
loop {
match try_create_deferred_rest_runtime_session(&state).await {
Ok(session_id) => filler_sessions.push(session_id),
Err(err) if err.to_string().contains("Max sessions") => break,
Err(err) => panic!("unexpected filler create error: {err:?}"),
}
}
assert!(
!filler_sessions.is_empty(),
"test should fill at least one additional active admission"
);
let peer_input = meerkat_runtime::peer_response_terminal_input(
meerkat_core::comms::PeerId::new(),
None,
meerkat_core::PeerCorrelationId::from_uuid(uuid::Uuid::new_v4()),
meerkat_contracts::PeerResponseTerminalStatusWire::Completed,
json!({"capacity": "full"}),
);
let rejected = admit_runtime_input_via_webhook(
&state,
&target_session_id,
peer_input,
WebhookAdmissionMode::Wakeful,
)
.await
.expect_err("capacity-full peer terminal webhook should reject");
assert_eq!(rejected.status(), StatusCode::CONFLICT);
let active_inputs = state
.runtime_adapter
.list_active_inputs(&target_session_id)
.await
.expect("list active inputs");
assert!(
active_inputs.is_empty(),
"capacity rejection must not enqueue peer terminal input: {active_inputs:?}"
);
}
#[tokio::test]
async fn rest_peer_terminal_webhook_reserves_active_admission_before_registration_lock() {
let temp = TempDir::new().unwrap();
let mut state = load_rest_state_with_capacity(&temp, 1).await;
state.llm_client_override = Some(Arc::new(MockLlmClient));
let target_session_id = create_completed_rest_runtime_session(&state).await;
let registration_lock = rest_runtime_registration_lock(&state, &target_session_id);
let registration_guard = registration_lock.mutex().lock().await;
let webhook_state = state.clone();
let webhook_session_id = target_session_id.clone();
let peer_input = meerkat_runtime::peer_response_terminal_input(
meerkat_core::comms::PeerId::new(),
None,
meerkat_core::PeerCorrelationId::from_uuid(uuid::Uuid::new_v4()),
meerkat_contracts::PeerResponseTerminalStatusWire::Completed,
json!({"capacity": "lock-held"}),
);
let mut webhook_task = tokio::spawn(async move {
admit_runtime_input_via_webhook(
&webhook_state,
&webhook_session_id,
peer_input,
WebhookAdmissionMode::Wakeful,
)
.await
});
tokio::select! {
result = &mut webhook_task => {
panic!("wakeful webhook completed before registration lock release: {result:?}");
}
() = tokio::time::sleep(std::time::Duration::from_millis(50)) => {}
}
wait_for_rest_runtime_pre_admission(&state, &target_session_id).await;
let capacity_filler = try_create_deferred_rest_runtime_session(&state).await;
assert!(
capacity_filler
.as_ref()
.err()
.is_some_and(|err| err.to_string().contains("Max sessions")),
"blocked wakeful webhook must reserve active capacity before lock release: {capacity_filler:?}"
);
drop(registration_guard);
drop(registration_lock);
let admitted = tokio::time::timeout(std::time::Duration::from_secs(5), webhook_task)
.await
.expect("wakeful webhook should finish after lock release")
.expect("wakeful webhook task should not panic")
.expect("wakeful webhook should keep its reserved capacity after lock release");
assert_eq!(admitted.0, StatusCode::ACCEPTED);
}
#[tokio::test]
async fn rest_continue_capacity_full_rejects_before_input_accept() {
let temp = TempDir::new().unwrap();
let mut state = load_rest_state_with_capacity(&temp, 1).await;
state.llm_client_override = Some(Arc::new(MockLlmClient));
let target_session_id = create_completed_rest_runtime_session(&state).await;
let _capacity_filler = create_deferred_rest_runtime_session(&state).await;
let outcome = Box::pin(continue_session_inner(
&state,
&target_session_id.to_string(),
ContinueSessionRequest {
injected_context: None,
transient_turn_context: None,
session_id: target_session_id.to_string(),
prompt: ContentInput::Text("must not enter runtime queue".to_string()),
system_prompt: None,
output_schema: None,
structured_output_retries: None,
keep_alive: None,
comms_name: None,
peer_meta: None,
verbose: false,
model: None,
provider: None,
auth_binding: None,
max_tokens: None,
hooks_override: None,
enable_web_search: None,
skill_refs: None,
turn_tool_overlay: None,
additional_instructions: None,
},
None,
))
.await;
match outcome {
RequestTerminal::RespondWithoutPublish(Err(ApiError::Conflict(message))) => {
assert!(
message.contains("Max sessions"),
"capacity rejection should mention max sessions: {message}"
);
}
other => panic!("expected capacity conflict before runtime accept, got {other:?}"),
}
let active_inputs = state
.runtime_adapter
.list_active_inputs(&target_session_id)
.await
.expect("list active inputs");
assert!(
active_inputs.is_empty(),
"capacity rejection must not enqueue continue input: {active_inputs:?}"
);
}
#[tokio::test]
async fn rest_continue_dropped_waiter_cleans_pre_admission_after_completion() {
let temp = TempDir::new().unwrap();
let mut state = load_rest_state_with_capacity(&temp, 1).await;
let calls = Arc::new(AtomicUsize::new(0));
let release = Arc::new(tokio::sync::Semaphore::new(0));
state.llm_client_override = Some(Arc::new(BlockingMockLlmClient {
calls: Arc::clone(&calls),
release: Arc::clone(&release),
}));
let target_session_id = create_deferred_rest_runtime_session(&state).await;
let state_for_continue = state.clone();
let target_for_continue = target_session_id.to_string();
let continue_task = tokio::spawn(async move {
Box::pin(continue_session_inner(
&state_for_continue,
&target_for_continue,
ContinueSessionRequest {
injected_context: None,
transient_turn_context: None,
session_id: target_for_continue.clone(),
prompt: ContentInput::Text("complete after REST waiter drops".to_string()),
system_prompt: None,
output_schema: None,
structured_output_retries: None,
keep_alive: None,
comms_name: None,
peer_meta: None,
verbose: false,
model: None,
provider: None,
auth_binding: None,
max_tokens: None,
hooks_override: None,
enable_web_search: None,
skill_refs: None,
turn_tool_overlay: None,
additional_instructions: None,
},
None,
))
.await
});
wait_for_rest_llm_calls(&calls, 1, "continue should reach blocking LLM").await;
continue_task.abort();
let aborted = continue_task
.await
.expect_err("aborted REST continue task should report cancellation");
assert!(aborted.is_cancelled());
release.add_permits(1);
let replacement = tokio::time::timeout(std::time::Duration::from_secs(5), async {
loop {
if state
.runtime_pre_admissions
.lock()
.await
.get(&target_session_id)
.is_none()
{
match try_create_deferred_rest_runtime_session(&state).await {
Ok(session_id) => return session_id,
Err(err) if err.to_string().contains("Max sessions") => {}
Err(err) => panic!("unexpected replacement create error: {err:?}"),
}
}
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
}
})
.await
.expect("dropped REST waiter should not leak pre-admission or active capacity");
assert_ne!(
replacement, target_session_id,
"replacement should be a separate session admitted after cleanup"
);
}
#[tokio::test]
async fn rest_continue_rebuild_replaces_stale_stopped_projection() {
let temp = TempDir::new().unwrap();
let mut state = load_rest_state_with_capacity(&temp, 1).await;
state.llm_client_override = Some(Arc::new(MockLlmClient));
let target_session_id = create_completed_rest_runtime_session(&state).await;
state
.session_service
.discard_live_session(&target_session_id)
.await
.expect("discard live target session");
state
.runtime_adapter
.unregister_session(&target_session_id)
.await
.expect("target runtime session should unregister cleanly");
let runtime_store = state.session_service.runtime_store();
let persisted_input = meerkat_runtime::Input::Prompt(meerkat_runtime::PromptInput::new(
"conflicting recovered prompt",
Some(
meerkat_core::lifecycle::run_primitive::RuntimeTurnMetadata {
execution_kind: Some(
meerkat_core::lifecycle::RuntimeExecutionKind::ContentTurn,
),
..Default::default()
},
),
));
let policy = meerkat_runtime::DefaultPolicyTable::resolve(&persisted_input, true);
let mut input_state =
meerkat_runtime::InputState::new_accepted(persisted_input.header().id.clone());
input_state.runtime_semantics = Some(
meerkat_runtime::ingress_types::RuntimeInputSemantics::try_from_generated_admission(
&persisted_input,
true,
)
.expect("generated admission semantics"),
);
input_state.policy = Some(meerkat_runtime::PolicySnapshot {
version: policy.policy_version,
decision: policy,
});
input_state.persisted_input = Some(persisted_input);
let mut seed = meerkat_runtime::input_state::InputStateSeed::new_accepted();
seed.recovery_lane = Some(meerkat_core::types::HandlingMode::Queue);
let active_input = meerkat_runtime::input_state::StoredInputState {
state: input_state,
seed,
};
state
.runtime_adapter
.register_session(target_session_id.clone())
.await
.expect("register session");
state
.runtime_adapter
.stop_runtime_executor(&target_session_id, "seed stopped projection")
.await
.expect("persist conflicting stopped runtime projection");
let active_input = {
let mut driver = meerkat_runtime::EphemeralRuntimeDriver::new(
meerkat_runtime::LogicalRuntimeId::new(format!(
"rest-persistence-record-{target_session_id}"
)),
);
driver
.recover_input_state_persistence_record(active_input)
.expect("test input-state seed should pass generated recovery authority")
};
runtime_store
.persist_input_state(
&meerkat_runtime::LogicalRuntimeId::for_session(&target_session_id),
&active_input,
)
.await
.expect("persist conflicting active input state");
let outcome = Box::pin(continue_session_inner(
&state,
&target_session_id.to_string(),
ContinueSessionRequest {
injected_context: None,
transient_turn_context: None,
session_id: target_session_id.to_string(),
prompt: ContentInput::Text("rebuild against stopped runtime".to_string()),
system_prompt: None,
output_schema: None,
structured_output_retries: None,
keep_alive: None,
comms_name: None,
peer_meta: None,
verbose: false,
model: None,
provider: None,
auth_binding: None,
max_tokens: Some(state.max_tokens.saturating_add(1)),
hooks_override: None,
enable_web_search: None,
skill_refs: None,
turn_tool_overlay: None,
additional_instructions: None,
},
None,
))
.await;
assert!(
matches!(outcome, RequestTerminal::Publish(Ok(_))),
"rebuild should replace the stale stopped epoch and run on fresh authority: {outcome:?}"
);
assert!(
state
.session_service
.has_live_session(&target_session_id)
.await
.expect("check live target session"),
"successful rebuild should retain the recreated live session"
);
assert!(
state
.runtime_adapter
.contains_session(&target_session_id)
.await,
"successful rebuild should retain its fresh runtime registration"
);
state
.session_service
.discard_live_session(&target_session_id)
.await
.expect("test cleanup should discard rebuilt live session");
state
.runtime_adapter
.unregister_session(&target_session_id)
.await
.expect("test cleanup should unregister rebuilt runtime");
let replacement = try_create_deferred_rest_runtime_session(&state)
.await
.expect("post-rebuild cleanup should release active capacity");
assert_ne!(
replacement, target_session_id,
"replacement should be a separate active session admitted after cleanup"
);
}
#[tokio::test]
async fn rest_continue_rebuild_unpublished_cleanup_retries_after_active_completion() {
let temp = TempDir::new().unwrap();
let mut state = load_rest_state_with_capacity(&temp, 1).await;
state.llm_client_override = Some(Arc::new(MockLlmClient));
let target_session_id = create_completed_rest_runtime_session(&state).await;
state
.session_service
.discard_live_session(&target_session_id)
.await
.expect("discard live target session");
state
.runtime_adapter
.unregister_session(&target_session_id)
.await
.expect("target runtime session should unregister cleanly");
assert!(
!state
.runtime_adapter
.contains_session(&target_session_id)
.await,
"test requires rebuild to prepare a new runtime registration"
);
let calls = Arc::new(AtomicUsize::new(0));
let release = Arc::new(tokio::sync::Semaphore::new(0));
state.llm_client_override = Some(Arc::new(BlockingMockLlmClient {
calls: Arc::clone(&calls),
release: Arc::clone(&release),
}));
let executor = SurfaceRequestExecutor::new(std::time::Duration::from_millis(1));
let ctx = executor.begin_request("rest-rebuild-unpublished-cleanup", noop_request_action());
let request_key = ctx.key().to_string();
let continue_state = state.clone();
let target_for_continue = target_session_id.to_string();
let mut continue_task = tokio::spawn(async move {
Box::pin(continue_session_inner(
&continue_state,
&target_for_continue,
ContinueSessionRequest {
injected_context: None,
transient_turn_context: None,
session_id: target_for_continue.clone(),
prompt: ContentInput::Text("complete after rebuild cleanup".to_string()),
system_prompt: None,
output_schema: None,
structured_output_retries: None,
keep_alive: None,
comms_name: None,
peer_meta: None,
verbose: false,
model: None,
provider: None,
auth_binding: None,
max_tokens: Some(512),
hooks_override: None,
enable_web_search: None,
skill_refs: None,
turn_tool_overlay: None,
additional_instructions: None,
},
Some(ctx),
))
.await
});
tokio::time::timeout(std::time::Duration::from_secs(30), async {
loop {
if calls.load(AtomicOrdering::SeqCst) >= 1 {
break;
}
tokio::select! {
result = &mut continue_task => {
panic!(
"rebuild continue completed before reaching the blocking LLM: {result:?}"
);
}
() = tokio::time::sleep(std::time::Duration::from_millis(10)) => {}
}
}
})
.await
.expect("rebuild continue should reach the blocking LLM");
assert!(
state
.runtime_adapter
.contains_session(&target_session_id)
.await,
"accepted rebuild input should have a live runtime registration"
);
assert!(matches!(
executor.finish_unpublished(&request_key).await,
meerkat::surface::CompleteOutcome::Completed
));
assert!(
state
.runtime_adapter
.contains_session(&target_session_id)
.await,
"first cleanup pass should preserve the runtime while input is active"
);
tokio::time::timeout(std::time::Duration::from_secs(30), async {
loop {
release.add_permits(1);
tokio::select! {
result = &mut continue_task => break result,
() = tokio::time::sleep(std::time::Duration::from_millis(10)) => {}
}
}
})
.await
.expect("rebuild continue should finish after releasing mock LLM")
.expect("rebuild continue task should not panic");
for _ in 0..200 {
let runtime_registered = state
.runtime_adapter
.contains_session(&target_session_id)
.await;
let live_session = state
.session_service
.has_live_session(&target_session_id)
.await
.expect("check live target session");
if !runtime_registered && !live_session {
break;
}
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
}
assert!(
!state
.runtime_adapter
.contains_session(&target_session_id)
.await,
"completion-owned cleanup should retry unregister after active input drains"
);
assert!(
!state
.session_service
.has_live_session(&target_session_id)
.await
.expect("check live target session"),
"completion-owned cleanup should leave the rebuilt live session discarded"
);
let replacement = try_create_deferred_rest_runtime_session(&state)
.await
.expect("rebuild cleanup after completion should release active capacity");
assert_ne!(
replacement, target_session_id,
"replacement should be admitted after completion-owned cleanup"
);
}
#[cfg(feature = "comms")]
#[tokio::test]
async fn rest_continue_capacity_full_rejects_before_keep_alive_persist() {
let temp = TempDir::new().unwrap();
let mut state = load_rest_state_with_capacity(&temp, 1).await;
state.llm_client_override = Some(Arc::new(MockLlmClient));
let target_session_id = create_completed_rest_runtime_comms_session(&state).await;
let before = state
.session_service
.load_authoritative_session(&target_session_id)
.await
.expect("load target session")
.expect("target session should exist");
let before_metadata = before.session_metadata().expect("metadata should exist");
assert!(
!before_metadata.keep_alive,
"test requires a target whose keep_alive starts disabled"
);
assert!(
before_metadata.comms_name.is_some(),
"test requires a comms-enabled target so keep_alive true is valid"
);
let _capacity_filler = create_deferred_rest_runtime_session(&state).await;
let outcome = Box::pin(continue_session_inner(
&state,
&target_session_id.to_string(),
ContinueSessionRequest {
injected_context: None,
transient_turn_context: None,
session_id: target_session_id.to_string(),
prompt: ContentInput::Text("must not persist keep_alive".to_string()),
system_prompt: None,
output_schema: None,
structured_output_retries: None,
keep_alive: Some(true),
comms_name: None,
peer_meta: None,
verbose: false,
model: None,
provider: None,
auth_binding: None,
max_tokens: None,
hooks_override: None,
enable_web_search: None,
skill_refs: None,
turn_tool_overlay: None,
additional_instructions: None,
},
None,
))
.await;
match outcome {
RequestTerminal::RespondWithoutPublish(Err(ApiError::Conflict(message))) => {
assert!(
message.contains("Max sessions"),
"capacity rejection should mention max sessions: {message}"
);
}
other => panic!("expected capacity conflict before keep_alive persist, got {other:?}"),
}
let after = state
.session_service
.load_authoritative_session(&target_session_id)
.await
.expect("load target session after rejection")
.expect("target session should still exist");
let after_metadata = after.session_metadata().expect("metadata should exist");
assert!(
!after_metadata.keep_alive,
"capacity-rejected continue must not persist keep_alive"
);
}
#[cfg(feature = "mcp")]
#[tokio::test]
async fn rest_continue_capacity_full_rejects_before_mcp_boundary_apply() {
let temp = TempDir::new().unwrap();
let mut state = load_rest_state_with_capacity(&temp, 1).await;
state.llm_client_override = Some(Arc::new(MockLlmClient));
let target_session_id = create_completed_rest_runtime_session(&state).await;
let adapter = Arc::new(McpRouterAdapter::new(McpRouter::new()));
let (lifecycle_tx, lifecycle_rx) = mpsc::unbounded_channel();
state.mcp_sessions.write().await.insert(
target_session_id.clone(),
SessionMcpState {
adapter,
turn_counter: 0,
lifecycle_tx,
lifecycle_rx,
drain_task_running: Arc::new(AtomicBool::new(false)),
},
);
let _capacity_filler = create_deferred_rest_runtime_session(&state).await;
let outcome = Box::pin(continue_session_inner(
&state,
&target_session_id.to_string(),
ContinueSessionRequest {
injected_context: None,
transient_turn_context: None,
session_id: target_session_id.to_string(),
prompt: ContentInput::Text("must not apply MCP boundary".to_string()),
system_prompt: None,
output_schema: None,
structured_output_retries: None,
keep_alive: None,
comms_name: None,
peer_meta: None,
verbose: false,
model: None,
provider: None,
auth_binding: None,
max_tokens: None,
hooks_override: None,
enable_web_search: None,
skill_refs: None,
turn_tool_overlay: None,
additional_instructions: None,
},
None,
))
.await;
match outcome {
RequestTerminal::RespondWithoutPublish(Err(ApiError::Conflict(message))) => {
assert!(
message.contains("Max sessions"),
"capacity rejection should mention max sessions: {message}"
);
}
other => panic!("expected capacity conflict before MCP boundary, got {other:?}"),
}
let mcp_sessions = state.mcp_sessions.read().await;
let mcp_state = mcp_sessions
.get(&target_session_id)
.expect("target MCP state should remain registered");
assert_eq!(
mcp_state.turn_counter, 0,
"capacity-rejected continue must not apply the MCP boundary"
);
}
#[tokio::test]
async fn rest_peer_terminal_webhook_recovers_persisted_live_missing_session() {
let temp = TempDir::new().unwrap();
let mut state = AppState::load_from(temp.path().to_path_buf())
.await
.unwrap();
state.llm_client_override = Some(Arc::new(MockLlmClient));
let target_session_id = create_completed_rest_runtime_session(&state).await;
state
.session_service
.discard_live_session(&target_session_id)
.await
.expect("discard live target session");
state
.runtime_adapter
.unregister_session(&target_session_id)
.await
.expect("target runtime session should unregister cleanly");
assert!(
!state
.session_service
.has_live_session(&target_session_id)
.await
.expect("check live session"),
"test requires a persisted-only target session"
);
assert!(
!state
.runtime_adapter
.contains_session(&target_session_id)
.await,
"test requires no runtime registration so webhook admission must recreate it"
);
let peer_input = meerkat_runtime::peer_response_terminal_input(
meerkat_core::comms::PeerId::new(),
None,
meerkat_core::PeerCorrelationId::from_uuid(uuid::Uuid::new_v4()),
meerkat_contracts::PeerResponseTerminalStatusWire::Completed,
json!({"target": "persisted-live-missing"}),
);
let admitted = admit_runtime_input_via_webhook(
&state,
&target_session_id,
peer_input,
WebhookAdmissionMode::Wakeful,
)
.await
.expect("persisted-only peer terminal webhook should reserve and accept");
assert_eq!(admitted.0, StatusCode::ACCEPTED);
for _ in 0..200 {
let pre_admission_cleared = state
.runtime_pre_admissions
.lock()
.await
.get(&target_session_id)
.is_none();
let active_inputs = state
.runtime_adapter
.list_active_inputs(&target_session_id)
.await
.unwrap_or_default();
if pre_admission_cleared && active_inputs.is_empty() {
return;
}
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
}
panic!("persisted-only webhook input did not finish and clear pre-admission");
}
#[tokio::test]
async fn rest_runtime_recovery_no_pending_releases_capacity() {
let temp = TempDir::new().unwrap();
let mut state = load_rest_state_with_capacity(&temp, 1).await;
state.llm_client_override = Some(Arc::new(MockLlmClient));
let session_id = create_completed_rest_runtime_session(&state).await;
state
.session_service
.discard_live_session(&session_id)
.await
.expect("discard completed live session before recovery");
state
.runtime_adapter
.unregister_session(&session_id)
.await
.expect("runtime session should unregister cleanly");
let input_id = meerkat_core::lifecycle::InputId::new();
let admission = state
.session_service
.reserve_runtime_turn_admission(&session_id)
.await
.expect("reserve canonical recovery admission");
insert_rest_runtime_pre_admission(
&state.runtime_pre_admissions,
session_id.clone(),
input_id.clone(),
admission,
)
.await
.expect("stage canonical recovery admission");
let registration_lock = rest_runtime_registration_lock(&state, &session_id);
let registration_guard = registration_lock.mutex().lock().await;
prepare_rest_session_runtime_executor_locked(&state, &session_id, &input_id)
.await
.expect("canonical pre-attach recovery should rematerialize the session");
drop(registration_guard);
drop(registration_lock);
let primitive =
RunPrimitive::StagedInput(meerkat_core::lifecycle::run_primitive::StagedRunInput {
boundary: RunApplyBoundary::Immediate,
appends: Vec::new(),
contributing_input_ids: vec![input_id],
turn_metadata: Some(
meerkat_core::lifecycle::run_primitive::RuntimeTurnMetadata {
execution_kind: Some(
meerkat_core::lifecycle::RuntimeExecutionKind::ResumePending,
),
..Default::default()
},
),
});
let output = super::apply_runtime_turn(
&state.runtime_executor_context(),
&session_id,
meerkat_core::RunId::new(),
&primitive,
ContentInput::Text(String::new()),
)
.await
.expect("live-missing resume-pending recovery should return no-op output");
assert!(
matches!(output.terminal, Some(CoreApplyTerminal::NoPendingBoundary)),
"expected no-pending terminal from recovered completed session: {output:?}"
);
assert!(
state
.session_service
.has_live_session(&session_id)
.await
.expect("check recovered live session before handoff cleanup"),
"direct apply must retain the rebuilt live session until the exact executor is handed off"
);
let mut executor =
RestSessionRuntimeExecutor::new(state.runtime_executor_context(), session_id.clone());
meerkat_core::lifecycle::CoreExecutor::cleanup_after_runtime_stop_terminalized(
&mut executor,
)
.await
.expect("post-handoff REST executor cleanup should discard the rebuilt live session");
assert!(
!state
.session_service
.has_live_session(&session_id)
.await
.expect("check recovered live session"),
"no-op recovery should discard the rematerialized live session"
);
assert!(
state.runtime_adapter.contains_session(&session_id).await,
"direct CoreExecutor apply helper must leave runtime retirement to the post-handoff machine saga"
);
state
.runtime_adapter
.unregister_session(&session_id)
.await
.expect("test cleanup should drive canonical unregister");
try_create_deferred_rest_runtime_session(&state)
.await
.expect("no-op recovery must release active capacity");
}
#[tokio::test]
async fn rest_runtime_apply_archived_session_requests_post_handoff_teardown() {
let temp = TempDir::new().unwrap();
let mut state = AppState::load_from(temp.path().to_path_buf())
.await
.unwrap();
state.llm_client_override = Some(Arc::new(MockLlmClient));
let session_id = create_archived_stale_rest_runtime_session(&state).await;
let primitive = RunPrimitive::ImmediateAppend(
meerkat_core::lifecycle::run_primitive::ConversationAppend {
role: meerkat_core::lifecycle::run_primitive::ConversationAppendRole::User,
content: CoreRenderable::Text {
text: "after archive".to_string(),
},
identity: None,
},
);
let mut executor = RestSessionRuntimeExecutor {
context: state.runtime_executor_context(),
session_id: session_id.clone(),
};
let rejected =
CoreExecutor::apply(&mut executor, meerkat_core::RunId::new(), primitive).await;
assert!(matches!(
rejected,
Err(CoreExecutorError::TeardownRequired {
reason: meerkat_core::lifecycle::core_executor::CoreExecutorTeardownReason::ArchivedSession,
..
})
));
assert!(
state.runtime_adapter.contains_session(&session_id).await,
"archived REST apply must preserve the exact runtime registration until executor handoff"
);
}
#[cfg(feature = "mcp")]
#[tokio::test]
async fn rest_mcp_resolver_archived_session_cleans_stale_runtime_and_adapter() {
let temp = TempDir::new().unwrap();
let mut state = AppState::load_from(temp.path().to_path_buf())
.await
.unwrap();
state.llm_client_override = Some(Arc::new(MockLlmClient));
let session_id = create_archived_stale_rest_runtime_session(&state).await;
let adapter = Arc::new(McpRouterAdapter::new(McpRouter::new()));
let (lifecycle_tx, lifecycle_rx) = mpsc::unbounded_channel();
state.mcp_sessions.write().await.insert(
session_id.clone(),
SessionMcpState {
adapter,
turn_counter: 0,
lifecycle_tx,
lifecycle_rx,
drain_task_running: Arc::new(AtomicBool::new(false)),
},
);
let rejected = resolve_mcp_adapter(&state, &session_id).await;
assert!(
matches!(rejected, Err(ApiError::NotFound(_))),
"archived REST MCP resolver should reject as not found"
);
assert!(
!state.runtime_adapter.contains_session(&session_id).await,
"archived REST MCP resolver should unregister stale runtime state"
);
assert!(
!state.mcp_sessions.read().await.contains_key(&session_id),
"archived REST MCP resolver should remove stale live MCP adapter"
);
}
#[tokio::test]
async fn rest_runtime_recovery_rejects_archived_session_before_binding() {
let temp = TempDir::new().unwrap();
let mut state = AppState::load_from(temp.path().to_path_buf())
.await
.unwrap();
state.llm_client_override = Some(Arc::new(MockLlmClient));
let created = state
.session_service
.create_session(SvcCreateSessionRequest {
injected_context: Vec::new(),
model: resolved_default_model(&state).await,
prompt: "Hello".to_string().into(),
system_prompt: meerkat::SystemPromptOverride::Inherit,
max_tokens: Some(state.max_tokens),
event_tx: None,
initial_turn: InitialTurnPolicy::Defer,
deferred_prompt_policy: DeferredPromptPolicy::Discard,
build: Some(SessionBuildOptions {
custom_models: std::collections::BTreeMap::new(),
image_generation_provider: None,
auto_compact_threshold_override: None,
llm_client_override: state
.llm_client_override
.clone()
.map(encode_llm_client_override_for_service),
..Default::default()
}),
labels: None,
})
.await
.expect("deferred session create should succeed");
let session_id = created.session_id;
state
.session_service
.archive_with_machine_protocol(
&session_id,
MachineSessionArchiveProtocol::from_machine(state.runtime_adapter.as_ref()),
)
.await
.expect("archive should retire through machine authority");
state
.runtime_adapter
.unregister_session(&session_id)
.await
.expect("cold archived recovery starts without a runtime registration");
assert!(
!state.runtime_adapter.contains_session(&session_id).await,
"test must remove the archived runtime registration before recovery"
);
let primitive =
RunPrimitive::StagedInput(meerkat_core::lifecycle::run_primitive::StagedRunInput {
boundary: RunApplyBoundary::Immediate,
appends: Vec::new(),
contributing_input_ids: vec![meerkat_core::lifecycle::InputId::new()],
turn_metadata: Some(
meerkat_core::lifecycle::run_primitive::RuntimeTurnMetadata {
execution_kind: Some(
meerkat_core::lifecycle::RuntimeExecutionKind::ContentTurn,
),
..Default::default()
},
),
});
let rejected = super::apply_runtime_turn(
&state.runtime_executor_context(),
&session_id,
meerkat_core::RunId::new(),
&primitive,
ContentInput::Text("archived".to_string()),
)
.await;
assert!(
matches!(rejected, Err(SessionError::NotFound { .. })),
"archived runtime recovery should reject before materialization: {rejected:?}"
);
assert!(
!state.runtime_adapter.contains_session(&session_id).await,
"archived runtime recovery must not leave a runtime registration"
);
}
#[tokio::test]
async fn test_app_state_default() {
let temp = TempDir::new().unwrap();
let state = AppState::load_from(temp.path().to_path_buf())
.await
.unwrap();
assert!(!resolved_default_model(&state).await.is_empty());
assert!(state.max_tokens > 0);
}
#[tokio::test]
async fn create_session_default_model_resolves_ladder_when_agent_model_empty() {
let temp = TempDir::new().unwrap();
let state = AppState::load_from(temp.path().to_path_buf())
.await
.unwrap();
let mut config = state
.config_runtime
.get()
.await
.expect("config should read")
.config;
config.agent.model = String::new();
state
.config_runtime
.set(config.clone(), None)
.await
.expect("config should persist");
let resolved = resolve_default_model(&state)
.await
.expect("default model must resolve");
assert!(
!resolved.is_empty(),
"empty config.agent.model must resolve the catalog ladder, not ''"
);
assert_eq!(
resolved,
meerkat::resolve_create_session_default_model(&config),
"REST must serve exactly the canonical ladder over the current config"
);
}
fn rest_inherited_gemini_binding() -> (Config, meerkat_core::AuthBindingRef) {
let mut config = Config::default();
let mut global = meerkat_core::RealmConfigSection::default();
global.backend.insert(
"google".to_string(),
meerkat_core::BackendProfileConfig {
provider: "gemini".to_string(),
backend_kind: "google_code_assist".to_string(),
base_url: None,
options: serde_json::Value::Null,
},
);
global.auth.insert(
"oauth".to_string(),
meerkat_core::AuthProfileConfig {
provider: "gemini".to_string(),
auth_method: "google_oauth".to_string(),
source: meerkat_core::CredentialSourceSpec::ManagedStore,
constraints: Default::default(),
metadata_defaults: Default::default(),
},
);
global.binding.insert(
"primary".to_string(),
meerkat_core::ProviderBindingConfig {
backend_profile: "google".to_string(),
auth_profile: "oauth".to_string(),
default_model: Some("gemini-3.1-flash-lite-preview".to_string()),
policy: Default::default(),
provider_default: true,
},
);
config.realm.insert("global".to_string(), global);
config.realm.insert(
"dev".to_string(),
meerkat_core::RealmConfigSection {
parent: Some(meerkat_core::RealmId::global()),
..Default::default()
},
);
let binding = meerkat_core::AuthBindingRef {
realm: meerkat_core::RealmId::parse("dev").expect("realm"),
binding: meerkat_core::BindingId::parse("primary").expect("binding"),
profile: None,
origin: meerkat_core::BindingOrigin::Configured,
};
(config, binding)
}
#[test]
fn rest_create_session_resolution_preserves_supported_config_pin() {
let mut config = Config::default();
config.agent.model = "claude-opus-4-7".to_string();
config.models.anthropic = "claude-sonnet-4-6".to_string();
let resolved = resolve_rest_create_session_model(&config, None, None, None)
.expect("REST lowers through shared resolver");
assert_eq!(resolved.model, "claude-opus-4-7");
assert_eq!(resolved.provider, Provider::Anthropic);
}
#[test]
fn rest_create_session_resolution_preserves_inherited_binding_owner() {
let (config, binding) = rest_inherited_gemini_binding();
let resolved = resolve_rest_create_session_model(&config, None, None, Some(binding))
.expect("REST resolves inherited binding");
assert_eq!(resolved.model, "gemini-3.1-flash-lite-preview");
assert_eq!(resolved.provider, Provider::Gemini);
assert_eq!(
resolved
.auth_binding
.expect("owner-stamped binding")
.realm
.as_str(),
"global"
);
}
#[test]
fn rest_create_session_resolution_rejects_explicit_model_binding_mismatch() {
let (config, binding) = rest_inherited_gemini_binding();
let err = resolve_rest_create_session_model(
&config,
Some("gpt-5.5".to_string()),
None,
Some(binding),
)
.expect_err("known model owner must match binding provider");
assert!(err.to_string().contains("registered for provider 'openai'"));
}
#[test]
fn rest_create_session_resolution_preserves_config_fault_ownership() {
let config_error = meerkat::CreateSessionModelResolutionError::Config(
meerkat_core::ConfigError::Validation("broken model registry".to_string()),
);
assert!(matches!(
create_session_model_resolution_error_to_api(config_error),
ApiError::Configuration(_)
));
let binding_error =
meerkat::CreateSessionModelResolutionError::BindingMissingProviderDefault {
realm: meerkat_core::RealmId::parse("dev").expect("realm"),
binding: meerkat_core::BindingId::parse("primary").expect("binding"),
provider: meerkat_core::Provider::SelfHosted,
};
assert!(matches!(
create_session_model_resolution_error_to_api(binding_error),
ApiError::Configuration(_)
));
assert!(matches!(
create_session_model_resolution_error_to_api(
meerkat::CreateSessionModelResolutionError::EmptyExplicitModel,
),
ApiError::BadRequest(_)
));
assert!(matches!(
create_session_model_resolution_error_to_api(
meerkat::CreateSessionModelResolutionError::MissingProviderDefault {
provider: meerkat_core::Provider::SelfHosted,
},
),
ApiError::BadRequest(_)
));
}
#[tokio::test]
async fn validate_prompt_video_input_accepts_self_hosted_alias_from_runtime_registry() {
let config = self_hosted_test_config(true);
let identity = resolve_validation_identity(&config, "gemma-4-e2b", None)
.expect("self-hosted alias should resolve");
assert_eq!(identity.provider, Provider::SelfHosted);
validate_prompt_video_input(&config, &inline_video_prompt(), &identity)
.expect("self-hosted aliases should validate inline video against the active registry");
}
#[tokio::test]
async fn validate_prompt_video_input_rejects_inline_video_for_wrong_provider_known_model() {
let config = Config::default();
let identity = validation_identity(Provider::Anthropic, "gemini-3.5-flash");
let err = validate_prompt_video_input(&config, &inline_video_prompt(), &identity)
.expect_err("wrong typed provider must not inherit Gemini inline-video support");
let evidence = err
.unsupported_evidence()
.expect("unsupported inline-video error should carry typed evidence");
assert_eq!(
evidence.capability,
meerkat_core::ModelCapability::InlineVideo
);
assert_eq!(
evidence.reason,
meerkat_core::UnsupportedModelCapabilityReason::ProviderModelProfileMissing
);
assert_eq!(evidence.provider, Provider::Anthropic);
assert_eq!(evidence.model, "gemini-3.5-flash");
}
#[tokio::test]
async fn validate_prompt_video_input_rejects_inline_video_for_unknown_provider_model_pair() {
let config = Config::default();
let identity = validation_identity(Provider::Other, "uncatalogued-video-model");
let err = validate_prompt_video_input(&config, &inline_video_prompt(), &identity)
.expect_err("unknown provider/model pair must fail closed without defaults");
let evidence = err
.unsupported_evidence()
.expect("unknown provider/model must carry typed unsupported evidence");
assert_eq!(
evidence.reason,
meerkat_core::UnsupportedModelCapabilityReason::ProviderModelProfileMissing
);
assert_eq!(evidence.provider, Provider::Other);
assert_eq!(evidence.model, "uncatalogued-video-model");
}
#[tokio::test]
async fn validate_prompt_video_input_rejects_inline_video_without_typed_provider_authority() {
let config = Config::default();
let identity = validation_identity(Provider::Other, "gemini-3.5-flash");
let err = validate_prompt_video_input(&config, &inline_video_prompt(), &identity)
.expect_err(
"known model/display strings must not select capability without typed provider",
);
let evidence = err
.unsupported_evidence()
.expect("providerless display/model strings must carry typed unsupported evidence");
assert_eq!(
evidence.reason,
meerkat_core::UnsupportedModelCapabilityReason::ProviderModelProfileMissing
);
assert_eq!(evidence.provider, Provider::Other);
assert_eq!(evidence.model, "gemini-3.5-flash");
}
#[tokio::test]
async fn validation_identity_rejects_explicit_provider_that_contradicts_catalog_owner() {
let config = Config::default();
let err = resolve_validation_identity(&config, "gpt-5.4", Some(Provider::Anthropic))
.expect_err("validation identity should fail closed for wrong-provider overrides");
assert!(
err.contains("registered for provider 'openai'")
&& err.contains("not provider 'anthropic'")
&& err.contains("gpt-5.4"),
"error should identify the rejected provider/model pair: {err}"
);
}
#[tokio::test]
async fn validation_identity_rejects_uncatalogued_model_without_provider_authority() {
let config = Config::default();
let err = resolve_validation_identity(&config, "unknown-model", None)
.expect_err("validation identity should fail closed without a typed provider owner");
assert!(
err.contains("requires an explicit provider or a registered model owner")
&& err.contains("unknown-model"),
"error should identify the missing provider authority: {err}"
);
}
#[test]
fn test_error_response_serialization() {
let err = ErrorResponse {
error: "test error".to_string(),
code: "TEST_ERROR".to_string(),
details: None,
};
let json = serde_json::to_string(&err).unwrap();
assert!(json.contains("test error"));
assert!(json.contains("TEST_ERROR"));
}
#[cfg(feature = "comms")]
#[test]
fn test_normalize_rest_comms_send_error_includes_structured_details() {
let err = normalize_rest_comms_send_error(
Some("peer-a"),
&meerkat_core::comms::SendError::PeerOffline,
);
match err {
ApiError::InternalWithData {
message,
code,
details,
} => {
assert!(message.starts_with("peer_unreachable:"));
assert_eq!(code, "peer_unreachable");
assert_eq!(details.get("peer").and_then(Value::as_str), Some("peer-a"));
assert_eq!(
details.get("reason").and_then(Value::as_str),
Some("offline_or_no_ack")
);
}
other => panic!("expected structured internal error, got {other:?}"),
}
}
#[cfg(feature = "comms")]
#[test]
fn test_normalize_rest_comms_send_error_fallback_is_structured() {
let err = normalize_rest_comms_send_error(
Some("peer-a"),
&meerkat_core::comms::SendError::Internal("boom".to_string()),
);
match err {
ApiError::InternalWithData {
message,
code,
details,
} => {
assert_eq!(message, "send_failed: internal: boom");
assert_eq!(code, "send_failed");
assert_eq!(
details.get("code").and_then(Value::as_str),
Some("send_failed")
);
assert_eq!(
details.get("message").and_then(Value::as_str),
Some("internal: boom")
);
}
other => panic!("expected structured internal error, got {other:?}"),
}
}
#[cfg(feature = "comms")]
#[test]
fn test_normalize_rest_comms_send_error_transport_maps_to_peer_unreachable() {
let err = normalize_rest_comms_send_error(
Some("peer-a"),
&meerkat_core::comms::SendError::Transport(
"Transport error: connection refused".into(),
),
);
match err {
ApiError::InternalWithData {
message,
code,
details,
} => {
assert!(message.starts_with("peer_unreachable:"));
assert_eq!(code, "peer_unreachable");
assert_eq!(details.get("peer").and_then(Value::as_str), Some("peer-a"));
assert_eq!(
details.get("reason").and_then(Value::as_str),
Some("transport_error")
);
assert_eq!(
details.get("details").and_then(Value::as_str),
Some("Transport error: connection refused")
);
}
other => panic!("expected structured internal error, got {other:?}"),
}
}
#[cfg(feature = "comms")]
#[test]
fn test_normalize_rest_comms_send_error_preserves_admission_drop_reason() {
let err = normalize_rest_comms_send_error(
Some("peer-a"),
&meerkat_core::comms::SendError::AdmissionDropped {
reason: meerkat_core::comms::AdmissionDropReason::UntrustedSender,
},
);
match err {
ApiError::InternalWithData {
message,
code,
details,
} => {
assert!(message.starts_with("peer_admission_dropped:"));
assert_eq!(code, "peer_admission_dropped");
assert_eq!(details.get("peer").and_then(Value::as_str), Some("peer-a"));
assert_eq!(
details.get("reason").and_then(Value::as_str),
Some("untrusted_sender")
);
}
other => panic!("expected structured internal error, got {other:?}"),
}
}
#[cfg(feature = "comms")]
#[test]
fn test_rest_comms_send_result_uses_contract_peer_request_projection() {
let envelope_id = uuid::Uuid::new_v4();
let interaction_id = meerkat_core::interaction::InteractionId(uuid::Uuid::new_v4());
let payload = serde_json::to_value(CommsSendResult::from(
meerkat_core::comms::SendReceipt::PeerRequestSent {
envelope_id,
interaction_id,
stream_reserved: true,
},
))
.unwrap();
assert_eq!(
payload["request_id"],
serde_json::json!(envelope_id.to_string())
);
assert_eq!(
payload["interaction_id"],
serde_json::json!(interaction_id.0.to_string())
);
}
#[cfg(feature = "comms")]
#[test]
fn test_comms_peers_payload_uses_typed_core_wire_contract() {
let payload = comms_peers_payload(vec![sample_peer_directory_entry()]);
assert_peer_directory_wire(&payload);
}
#[cfg(feature = "comms")]
fn sample_peer_directory_entry() -> meerkat_core::comms::PeerDirectoryEntry {
meerkat_core::comms::PeerDirectoryEntry {
peer_id: meerkat_core::comms::PeerId::new(),
name: meerkat_core::comms::PeerName::new("agent").unwrap(),
address: meerkat_core::comms::PeerAddress::new(
meerkat_core::comms::PeerTransport::Inproc,
"agent",
),
source: meerkat_core::comms::PeerDirectorySource::Inproc,
sendable_kinds: vec![
meerkat_core::comms::PeerSendability::PeerMessage,
meerkat_core::comms::PeerSendability::PeerRequest,
],
capabilities: meerkat_core::comms::PeerCapabilitySet::default()
.with_extension("vendor.echo", serde_json::json!({ "enabled": true })),
meta: meerkat_core::PeerMeta::default(),
}
}
#[cfg(feature = "comms")]
fn assert_peer_directory_wire(result: &Value) {
let peer = &result["peers"][0];
assert_eq!(peer["source"], "inproc");
assert_eq!(
peer["sendable_kinds"],
serde_json::json!(["peer_message", "peer_request"])
);
assert_eq!(peer["capabilities"]["version"], 1);
assert_eq!(
peer["capabilities"]["extensions"]["vendor.echo"]["enabled"],
true
);
}
#[tokio::test]
async fn test_app_state_builtins_disabled_by_default() {
let temp = TempDir::new().unwrap();
let state = AppState::load_from(temp.path().to_path_buf())
.await
.unwrap();
assert!(!state.enable_builtins);
assert!(!state.enable_shell);
}
fn missing_target_schedule_tool_args() -> Value {
json!({
"name": "missing-target",
"description": "create a due schedule through the tool surface",
"trigger": {
"type": "once",
"due_at_utc": (Utc::now() - Duration::seconds(1)).to_rfc3339(),
},
"target": {
"target_kind": "session",
"type": "exact_session",
"session_id": SessionId::new(),
"action": {
"type": "prompt",
"prompt": "scheduled hello"
}
},
"missing_target_policy": "mark_misfired",
"planning_horizon_days": 1,
"planning_horizon_occurrences": 1
})
}
async fn wait_for_missing_target_misfire(
service: &meerkat::ScheduleService,
schedule_id: &ScheduleId,
) -> Option<meerkat::Occurrence> {
for _ in 0..200 {
let occurrences = service
.list_occurrences(schedule_id)
.await
.expect("list occurrences");
if let Some(occurrence) = occurrences.into_iter().find(|occurrence| {
occurrence.phase == OccurrencePhase::Misfired
&& occurrence.failure_class == Some(OccurrenceFailureClass::TargetMissing)
}) {
return Some(occurrence);
}
tokio::time::sleep(std::time::Duration::from_millis(25)).await;
}
None
}
#[tokio::test]
async fn schedule_call_starts_host_and_services_due_schedule() {
let temp = TempDir::new().unwrap();
let state = AppState::load_from(temp.path().to_path_buf())
.await
.unwrap();
let Json(created) = schedule_call(
State(state.clone()),
Json(meerkat_contracts::wire::ScheduleToolCallParams {
name: "meerkat_schedule_create".into(),
arguments: missing_target_schedule_tool_args(),
}),
)
.await
.expect("schedule tool create should succeed");
let schedule_id = ScheduleId::parse(
created["schedule_id"]
.as_str()
.expect("schedule_id should be returned"),
)
.expect("valid schedule id");
let occurrence = wait_for_missing_target_misfire(&state.schedule_service, &schedule_id)
.await
.expect("schedule/call should start the host and service due work");
assert_eq!(occurrence.phase, OccurrencePhase::Misfired);
}
#[tokio::test]
async fn test_config_envelope_redacts_paths_by_default() {
let temp = TempDir::new().unwrap();
let state = AppState::load_from(temp.path().to_path_buf())
.await
.unwrap();
let Json(envelope) = get_config(State(state)).await.unwrap();
assert!(envelope.resolved_paths.is_none());
}
#[tokio::test]
async fn test_config_envelope_includes_paths_when_enabled() {
let temp = TempDir::new().unwrap();
let mut state = AppState::load_from(temp.path().to_path_buf())
.await
.unwrap();
state.expose_paths = true;
let Json(envelope) = get_config(State(state)).await.unwrap();
assert!(envelope.resolved_paths.is_some());
}
#[tokio::test]
async fn test_config_set_and_patch_roundtrip_parity() {
let temp = TempDir::new().unwrap();
let state = AppState::load_from(temp.path().to_path_buf())
.await
.unwrap();
let Json(initial) = get_config(State(state.clone())).await.unwrap();
let mut config_value = serde_json::to_value(&initial.config).expect("serialize config");
config_value["max_tokens"] = serde_json::json!(2048);
let updated_config: Config =
serde_json::from_value(config_value).expect("deserialize config update");
let Json(after_set) = set_config(
State(state.clone()),
Json(SetConfigRequest::Wrapped {
config: updated_config,
expected_generation: None,
}),
)
.await
.expect("config set");
assert_eq!(after_set.config.max_tokens, Some(2048));
let Json(after_patch) = patch_config(
State(state.clone()),
Json(PatchConfigRequest::Wrapped {
patch: serde_json::json!({"max_tokens": 3072}),
expected_generation: None,
}),
)
.await
.expect("config patch");
assert_eq!(after_patch.config.max_tokens, Some(3072));
}
#[tokio::test]
async fn test_config_set_and_patch_reject_invalid_config() {
let temp = TempDir::new().unwrap();
let state = AppState::load_from(temp.path().to_path_buf())
.await
.unwrap();
let Json(initial) = get_config(State(state.clone())).await.unwrap();
let mut config_value = serde_json::to_value(&initial.config).expect("serialize config");
config_value["max_tokens"] = serde_json::json!(0);
let invalid_config: Config =
serde_json::from_value(config_value).expect("deserialize invalid config");
let set_err = set_config(
State(state.clone()),
Json(SetConfigRequest::Wrapped {
config: invalid_config,
expected_generation: None,
}),
)
.await
.expect_err("set should reject invalid config");
assert!(matches!(set_err, ApiError::BadRequest(_)));
let patch_err = patch_config(
State(state),
Json(PatchConfigRequest::Wrapped {
patch: serde_json::json!({"max_tokens": 0}),
expected_generation: None,
}),
)
.await
.expect_err("patch should reject invalid config");
assert!(matches!(patch_err, ApiError::BadRequest(_)));
}
#[test]
fn test_create_session_request_parsing_with_keep_alive() {
let req_json = serde_json::json!({
"prompt": "Hello",
"keep_alive": true,
"comms_name": "test-agent"
});
let req: CreateSessionRequest = serde_json::from_value(req_json).unwrap();
assert_eq!(req.prompt, ContentInput::Text("Hello".to_string()));
assert_eq!(req.keep_alive, Some(true));
assert_eq!(req.comms_name, Some("test-agent".to_string()));
}
#[test]
fn test_create_session_request_keep_alive_defaults_to_none() {
let req_json = serde_json::json!({
"prompt": "Hello"
});
let req: CreateSessionRequest = serde_json::from_value(req_json).unwrap();
assert_eq!(req.keep_alive, None);
assert!(req.comms_name.is_none());
}
#[test]
fn test_create_session_resume_override_mask_marks_workgraph_with_schedule() {
let req_json = serde_json::json!({
"prompt": "Hello",
"enable_schedule": false,
"enable_workgraph": true
});
let req: CreateSessionRequest = serde_json::from_value(req_json).unwrap();
let mask = create_session_resume_override_mask(&req, None);
assert!(mask.override_schedule);
assert!(mask.override_workgraph);
assert!(!mask.override_web_search);
}
#[test]
fn test_help_request_to_create_session_preloads_platform_skill() {
let req = meerkat_contracts::HelpRequest {
question: "How do I add an MCP server?".to_string(),
prompt: Some("Write a game".to_string()),
execution_mode: meerkat_contracts::HelpExecutionMode::PlanExecution,
model: Some("claude-sonnet-4-5".to_string()),
provider: Some("anthropic".to_string()),
max_tokens: Some(512),
};
let create = help_request_to_create_session(req).expect("valid help request");
assert!(
matches!(create.prompt, ContentInput::Text(ref text) if text.contains("Write a game"))
);
assert!(
create
.system_prompt
.as_set_prompt()
.is_some_and(|prompt| prompt.contains("dedicated help surface"))
);
assert_eq!(create.model.as_deref(), Some("claude-sonnet-4-5"));
assert_eq!(create.provider, Some(Provider::Anthropic));
assert_eq!(create.max_tokens, Some(512));
assert_eq!(
create.preload_skills,
Some(meerkat::help::platform_preload_skills())
);
assert_eq!(create.enable_builtins, Some(false));
assert_eq!(create.enable_shell, Some(false));
}
#[tokio::test]
async fn test_create_session_route_rejects_reserved_mob_peer_meta_labels() {
use axum::body::Body;
use http_body_util::BodyExt;
use tower::ServiceExt;
let temp = TempDir::new().unwrap();
let mut state = AppState::load_from(temp.path().to_path_buf())
.await
.unwrap();
let mock_client: Arc<dyn LlmClient> = Arc::new(MockLlmClient);
state.llm_client_override = Some(mock_client.clone());
state.mob_state = Arc::new(
meerkat_mob_mcp::MobMcpState::new_with_runtime_adapter(
state.session_service.clone(),
Some(state.runtime_adapter.clone()),
meerkat_mob::MobControlPrincipal::Owner,
)
.with_persistent_storage_root(Some(temp.path().to_path_buf()))
.with_default_llm_client(Some(mock_client)),
);
let app = router(state);
let response = app
.oneshot(
axum::http::Request::builder()
.method("POST")
.uri("/sessions")
.header("content-type", "application/json")
.body(Body::from(
serde_json::json!({
"prompt": "Hello",
"peer_meta": {
"labels": {
"mob_id": "team"
}
}
})
.to_string(),
))
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
let body = response.into_body().collect().await.unwrap().to_bytes();
let payload: serde_json::Value = serde_json::from_slice(&body).unwrap();
assert!(
payload["error"]
.as_str()
.is_some_and(|msg| msg.contains("reserved") && msg.contains("mob_id")),
"reserved mob label rejection should explain the trust boundary: {}",
String::from_utf8_lossy(&body)
);
}
#[cfg(feature = "mob")]
async fn insert_console_test_mob(
state: &Arc<meerkat_mob_mcp::MobMcpState>,
label: &str,
) -> meerkat_mob::MobId {
let mob_id = meerkat_mob::MobId::from(label);
let mut profiles = std::collections::BTreeMap::new();
profiles.insert(
meerkat_mob::ProfileName::from("worker"),
meerkat_mob::ProfileBinding::Inline(Box::new(meerkat_mob::Profile {
model: "claude-sonnet-4-5".to_string(),
provider: None,
self_hosted_server_id: None,
image_generation_provider: None,
auto_compact_threshold: None,
resume_overrides: Vec::new(),
skills: Vec::new(),
tools: meerkat_mob::ToolConfig::default(),
peer_description: "worker".to_string(),
external_addressable: false,
backend: None,
runtime_mode: meerkat_mob::MobRuntimeMode::TurnDriven,
max_inline_peer_notifications: None,
output_schema: None,
provider_params: None,
})),
);
let mut definition = meerkat_mob::MobDefinition::explicit(mob_id.clone());
definition.profiles = profiles;
let handle = meerkat_mob::MobBuilder::new(definition, meerkat_mob::MobStorage::in_memory())
.with_session_service(state.session_service())
.allow_ephemeral_sessions(true)
.create()
.await
.expect("create REST console test mob");
state.mob_insert_handle(mob_id.clone(), handle).await;
mob_id
}
#[cfg(feature = "mob")]
#[tokio::test]
async fn test_mob_observation_gets_serve_typed_payloads() {
use axum::body::Body;
use http_body_util::BodyExt;
use tower::ServiceExt;
let temp = TempDir::new().unwrap();
let state = AppState::load_from(temp.path().to_path_buf())
.await
.unwrap();
let mob_id = insert_console_test_mob(&state.mob_state, "rest-console-observation").await;
let app = router(state);
let get = |uri: String| {
axum::http::Request::builder()
.method("GET")
.uri(uri)
.body(Body::empty())
.unwrap()
};
let response = app
.clone()
.oneshot(get(format!("/mob/{}/hosts", mob_id.as_str())))
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
let body = response.into_body().collect().await.unwrap().to_bytes();
let hosts: meerkat_contracts::wire::MobHostsResult =
serde_json::from_slice(&body).expect("typed hosts payload");
assert!(hosts.hosts.is_empty());
let response = app
.clone()
.oneshot(get(format!("/mob/{}/route-installs", mob_id.as_str())))
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
let body = response.into_body().collect().await.unwrap().to_bytes();
let installs: meerkat_contracts::wire::MobRouteInstallsResult =
serde_json::from_slice(&body).expect("typed route-installs payload");
assert!(installs.outstanding.is_empty());
assert!(installs.complete);
let response = app
.clone()
.oneshot(get(format!(
"/mob/{}/members/nobody/history",
mob_id.as_str()
)))
.await
.unwrap();
assert_eq!(
response.status(),
StatusCode::BAD_REQUEST,
"an unclassified MemberNotFound keeps the BadRequest fallback"
);
}
#[cfg(feature = "mob")]
#[tokio::test]
async fn test_mob_observation_gets_render_scope_denied_as_403() {
use axum::body::Body;
use http_body_util::BodyExt;
use tower::ServiceExt;
let temp = TempDir::new().unwrap();
let mut state = AppState::load_from(temp.path().to_path_buf())
.await
.unwrap();
state.mob_state = meerkat_mob_mcp::MobMcpState::new_in_memory_as(
meerkat_mob::MobControlPrincipal::External(
meerkat_core::auth::PrincipalId::new("viewer").expect("valid principal"),
),
);
let mob_id = insert_console_test_mob(&state.mob_state, "rest-console-denied").await;
let app = router(state);
let response = app
.clone()
.oneshot(
axum::http::Request::builder()
.method("GET")
.uri(format!("/mob/{}/hosts", mob_id.as_str()))
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::FORBIDDEN);
let body = response.into_body().collect().await.unwrap().to_bytes();
let payload: serde_json::Value = serde_json::from_slice(&body).unwrap();
assert_eq!(payload["details"]["required"], json!("list"));
assert_eq!(payload["details"]["presented"], json!([]));
let response = app
.oneshot(
axum::http::Request::builder()
.method("POST")
.uri(format!("/mob/{}/wait-kickoff", mob_id.as_str()))
.header("content-type", "application/json")
.body(Body::from("{}"))
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::FORBIDDEN);
let body = response.into_body().collect().await.unwrap().to_bytes();
let payload: serde_json::Value = serde_json::from_slice(&body).unwrap();
assert_eq!(payload["code"], json!("SCOPE_DENIED"));
assert_eq!(payload["details"]["required"], json!("subscribe_events"));
assert_eq!(payload["details"]["presented"], json!([]));
}
#[cfg(feature = "mob")]
#[tokio::test]
async fn test_mob_rest_error_projects_all_typed_details_and_route_fallbacks() {
use http_body_util::BodyExt;
let scope_error = meerkat_mob::MobError::ScopeDenied(meerkat_mob::ScopeDenial {
required: meerkat_mob::ControlScope::AdminHost,
presented: std::collections::BTreeSet::from([meerkat_mob::ControlScope::List]),
});
let runtime_id =
meerkat_mob::AgentRuntimeId::initial(meerkat_mob::AgentIdentity::from("remote-worker"));
let cases = vec![
(
scope_error,
StatusCode::FORBIDDEN,
"SCOPE_DENIED",
json!({"required": "admin_host", "presented": ["list"]}),
),
(
meerkat_mob::MobError::BridgeRequestTimedOut {
request_envelope_id: "request-1".to_string(),
timeout_ms: 12_000,
},
StatusCode::SERVICE_UNAVAILABLE,
"HOST_UNAVAILABLE",
json!({"timeout_ms": 12_000}),
),
(
meerkat_mob::MobError::StaleEventCursor {
after_cursor: 41,
latest_cursor: 7,
},
StatusCode::GONE,
"STALE_CURSOR",
json!({"watermark": 7, "requested": 41}),
),
(
meerkat_mob::MobError::StaleFenceToken {
runtime_id: runtime_id.clone(),
expected: meerkat_mob::FenceToken::new(2),
actual: meerkat_mob::FenceToken::new(1),
},
StatusCode::CONFLICT,
"STALE_FENCE",
json!({
"runtime_id": runtime_id.to_string(),
"expected": 2,
"actual": 1,
}),
),
];
for (error, expected_status, expected_code, expected_details) in cases {
let response = mob_rest_error(&error, ApiError::BadRequest);
assert_eq!(response.status(), expected_status, "error: {error}");
let body = response.into_body().collect().await.unwrap().to_bytes();
let payload: serde_json::Value = serde_json::from_slice(&body).unwrap();
assert_eq!(payload["code"], json!(expected_code), "error: {error}");
assert_eq!(payload["details"], expected_details, "error: {error}");
}
let unclassified =
meerkat_mob::MobError::MemberNotFound(meerkat_mob::AgentIdentity::from("missing"));
let response = mob_rest_error(&unclassified, ApiError::NotFound);
assert_eq!(response.status(), StatusCode::NOT_FOUND);
let body = response.into_body().collect().await.unwrap().to_bytes();
let payload: serde_json::Value = serde_json::from_slice(&body).unwrap();
assert_eq!(payload["code"], json!("NOT_FOUND"));
let auth_failure = meerkat_core::service::SessionProviderAuthFailure {
kind: meerkat_core::AuthErrorKind::InteractiveLoginRequired,
provider: meerkat_core::Provider::OpenAI,
realm_id: Some(meerkat_core::RealmId::parse("project").unwrap()),
binding_id: Some(meerkat_core::BindingId::parse("openai").unwrap()),
};
let auth_respawn = meerkat_mob::MobRespawnError::SpawnAfterRetireWithCause {
identity: meerkat_mob::AgentIdentity::from("worker"),
cause: meerkat_mob::MobError::SessionError(
meerkat_core::SessionError::provider_auth_failure(auth_failure),
),
};
let response = mob_rest_error(&auth_respawn, ApiError::BadRequest);
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
let body = response.into_body().collect().await.unwrap().to_bytes();
let payload: serde_json::Value = serde_json::from_slice(&body).unwrap();
assert_eq!(payload["code"], json!("BAD_REQUEST"));
assert_eq!(payload["details"]["cause"], json!("provider_auth"));
assert_eq!(
payload["details"]["kind"],
json!("interactive_login_required")
);
assert_eq!(payload["details"]["provider"], json!("openai"));
assert_eq!(payload["details"]["realm_id"], json!("project"));
assert_eq!(payload["details"]["binding_id"], json!("openai"));
let respawn =
meerkat_mob::MobRespawnError::Mob(meerkat_mob::MobError::BridgeRequestTimedOut {
request_envelope_id: "respawn-request".to_string(),
timeout_ms: 5_000,
});
let response = mob_rest_error(&respawn, ApiError::BadRequest);
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
let body = response.into_body().collect().await.unwrap().to_bytes();
let payload: serde_json::Value = serde_json::from_slice(&body).unwrap();
assert_eq!(payload["code"], json!("HOST_UNAVAILABLE"));
assert_eq!(payload["details"], json!({"timeout_ms": 5_000}));
}
#[tokio::test]
async fn test_workgraph_rest_routes_are_read_only() {
use axum::body::Body;
use http_body_util::BodyExt;
use tower::ServiceExt;
let temp = TempDir::new().unwrap();
let state = AppState::load_from(temp.path().to_path_buf())
.await
.unwrap();
let item = state
.workgraph_service
.create(meerkat::CreateWorkItemRequest {
realm_id: None,
namespace: None,
title: "observe me".to_string(),
description: None,
priority: Default::default(),
completion_policy: Default::default(),
labels: Default::default(),
due_at: None,
not_before: None,
snoozed_until: None,
external_refs: Vec::new(),
evidence_refs: Vec::new(),
status: None,
})
.await
.expect("seed WorkGraph item");
state
.workgraph_service
.create(meerkat::CreateWorkItemRequest {
realm_id: None,
namespace: Some(meerkat::WorkNamespace::new("other").unwrap()),
title: "observe other namespace".to_string(),
description: None,
priority: Default::default(),
completion_policy: Default::default(),
labels: Default::default(),
due_at: None,
not_before: None,
snoozed_until: None,
external_refs: Vec::new(),
evidence_refs: Vec::new(),
status: None,
})
.await
.expect("seed other WorkGraph item");
let app = router(state);
for descriptor in meerkat::workgraph_rest_path_catalog() {
if descriptor
.operations
.iter()
.any(|operation| operation.method != "get")
{
continue;
}
let uri = match descriptor.route {
meerkat::WorkGraphRestRoute::Item => {
descriptor.path.replace("{id}", item.id.as_str())
}
_ => descriptor.path.to_string(),
};
let response = app
.clone()
.oneshot(
axum::http::Request::builder()
.method("GET")
.uri(uri.as_str())
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK, "GET {uri}");
}
let events_path = meerkat::workgraph_rest_path_catalog()
.iter()
.find(|descriptor| descriptor.route == meerkat::WorkGraphRestRoute::Events)
.expect("WorkGraph events route")
.path;
let response = app
.clone()
.oneshot(
axum::http::Request::builder()
.method("GET")
.uri(format!("{events_path}?all_namespaces=true"))
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
let body = response.into_body().collect().await.unwrap().to_bytes();
let payload: serde_json::Value = serde_json::from_slice(&body).unwrap();
let namespaces = payload["events"]
.as_array()
.unwrap()
.iter()
.filter_map(|event| event["namespace"].as_str())
.collect::<std::collections::BTreeSet<_>>();
assert!(namespaces.contains("default"));
assert!(namespaces.contains("other"));
for (method, uri) in [
("POST", "/workgraph/items"),
("POST", "/workgraph/claim"),
("POST", "/workgraph/release"),
("PATCH", "/workgraph/items"),
("POST", "/workgraph/close"),
("POST", "/workgraph/link"),
("POST", "/workgraph/evidence"),
] {
let response = app
.clone()
.oneshot(
axum::http::Request::builder()
.method(method)
.uri(uri)
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert!(
matches!(
response.status(),
StatusCode::NOT_FOUND | StatusCode::METHOD_NOT_ALLOWED
),
"{method} {uri} must not expose a WorkGraph mutation route; got {}",
response.status()
);
}
}
#[cfg(feature = "mob")]
#[tokio::test]
async fn test_archive_session_route_missing_without_retained_mob_cleanup_returns_not_found() {
use axum::body::Body;
use http_body_util::BodyExt;
use tower::ServiceExt;
let temp = TempDir::new().unwrap();
let mut state = AppState::load_from(temp.path().to_path_buf())
.await
.unwrap();
state.llm_client_override = Some(Arc::new(MockLlmClient));
let session_id = create_deferred_rest_runtime_session(&state).await;
let app = router(state);
let uri = format!("/sessions/{session_id}");
let first = app
.clone()
.oneshot(
axum::http::Request::builder()
.method("DELETE")
.uri(uri.as_str())
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(first.status(), StatusCode::OK);
let retry = app
.oneshot(
axum::http::Request::builder()
.method("DELETE")
.uri(uri.as_str())
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
let retry_status = retry.status();
let body = retry.into_body().collect().await.unwrap().to_bytes();
let payload: serde_json::Value = serde_json::from_slice(&body).unwrap();
assert_eq!(
retry_status,
StatusCode::NOT_FOUND,
"stale REST archive without retained mob cleanup must remain NotFound: {payload}"
);
assert!(
payload["error"]
.as_str()
.is_some_and(|message| message.contains("Session not found")),
"not found response should explain the missing session: {payload}"
);
}
#[cfg(feature = "mob")]
#[tokio::test]
async fn test_archive_session_route_surfaces_incomplete_mob_cleanup_data() {
use axum::body::Body;
use http_body_util::BodyExt;
use tower::ServiceExt;
let temp = TempDir::new().unwrap();
let mut state = AppState::load_from(temp.path().to_path_buf())
.await
.unwrap();
state.llm_client_override = Some(Arc::new(MockLlmClient));
let session_id = create_deferred_rest_runtime_session(&state).await;
let (mob_id, events) = insert_rest_archive_partial_destroy_mob_with_events(
&state.mob_state,
&session_id.to_string(),
)
.await;
let mob_state = Arc::clone(&state.mob_state);
let app = router(state);
let uri = format!("/sessions/{session_id}");
let response = app
.clone()
.oneshot(
axum::http::Request::builder()
.method("DELETE")
.uri(uri.as_str())
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
let body = response.into_body().collect().await.unwrap().to_bytes();
let payload: serde_json::Value = serde_json::from_slice(&body).unwrap();
assert_eq!(payload["code"], "mob_destroy_incomplete");
let details = &payload["details"];
assert_eq!(details["code"], "mob_destroy_incomplete");
assert_eq!(details["retryable"], true);
assert!(
details["destroy_report"]["errors"]
.as_array()
.is_some_and(|errors| !errors.is_empty()),
"archive error should carry the incomplete destroy report: {payload}"
);
assert!(
mob_state.handle_for(&mob_id).await.is_ok(),
"incomplete REST archive cleanup must retain the mob retry anchor"
);
let retry_response = app
.clone()
.oneshot(
axum::http::Request::builder()
.method("DELETE")
.uri(uri.as_str())
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(retry_response.status(), StatusCode::INTERNAL_SERVER_ERROR);
let retry_body = retry_response
.into_body()
.collect()
.await
.unwrap()
.to_bytes();
let retry_payload: serde_json::Value = serde_json::from_slice(&retry_body).unwrap();
assert_eq!(
retry_payload["code"], "mob_destroy_incomplete",
"REST archive retry must report retained cleanup state, not session not found: {retry_payload}"
);
assert_eq!(
retry_payload["details"]["retryable"], true,
"REST archive retry should keep typed retryable incomplete data: {retry_payload}"
);
assert!(
mob_state.handle_for(&mob_id).await.is_ok(),
"incomplete REST archive retry must still retain the mob retry anchor"
);
events.allow_clear();
let complete_retry_response = app
.oneshot(
axum::http::Request::builder()
.method("DELETE")
.uri(uri.as_str())
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
let complete_status = complete_retry_response.status();
let complete_body = complete_retry_response
.into_body()
.collect()
.await
.unwrap()
.to_bytes();
let complete_payload: serde_json::Value = serde_json::from_slice(&complete_body).unwrap();
assert_eq!(
complete_status,
StatusCode::OK,
"the completing REST archive retry must report success, not NotFound: {complete_payload}"
);
assert_eq!(complete_payload["archived"], true);
assert!(
mob_state.handle_for(&mob_id).await.is_err(),
"the completing REST archive retry must remove the mob retry anchor"
);
}
#[cfg(feature = "mob")]
#[tokio::test]
async fn test_archive_session_route_does_not_mask_mob_member_archive_failure_with_child_cleanup()
{
use axum::body::Body;
use http_body_util::BodyExt;
use tower::ServiceExt;
let temp = TempDir::new().unwrap();
let mut state = AppState::load_from(temp.path().to_path_buf())
.await
.unwrap();
let (mob_state, archive_failures) =
meerkat_mob_mcp::MobMcpState::new_in_memory_with_archive_failure_control();
state.mob_state = mob_state.clone();
let (_parent_mob_id, member_session_id) = insert_rest_archive_live_member(&mob_state).await;
archive_failures
.fail_archive(
member_session_id.clone(),
"forced REST mob archive failure after retire event",
)
.await;
let member_session_key = member_session_id.to_string();
let (child_mob_id, child_events) =
insert_rest_archive_partial_destroy_mob_with_events(&mob_state, &member_session_key)
.await;
child_events.allow_clear();
let app = router(state);
let uri = format!("/sessions/{member_session_key}");
let response = app
.clone()
.oneshot(
axum::http::Request::builder()
.method("DELETE")
.uri(uri.as_str())
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(
response.status(),
StatusCode::INTERNAL_SERVER_ERROR,
"REST archive must not success-classify a failed parent mob-member archive"
);
let body = response.into_body().collect().await.unwrap().to_bytes();
let payload: serde_json::Value = serde_json::from_slice(&body).unwrap();
assert!(
payload["error"].as_str().is_some_and(|message| {
message.contains("forced REST mob archive failure after retire event")
}),
"REST archive should surface the parent bridge-session archive failure: {payload}"
);
assert!(
mob_state
.session_service()
.has_live_session(&member_session_id)
.await
.expect("check failed parent bridge session"),
"failed ArchiveSession must retain the parent bridge session retry anchor"
);
assert!(
mob_state.handle_for(&child_mob_id).await.is_ok(),
"child cleanup must not be run as a success fallback while parent archive failed"
);
archive_failures
.clear_archive_failure(&member_session_id)
.await;
let retry_response = app
.oneshot(
axum::http::Request::builder()
.method("DELETE")
.uri(uri.as_str())
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(
retry_response.status(),
StatusCode::OK,
"REST archive retry should complete after parent archive failure clears"
);
assert!(
!mob_state
.session_service()
.has_live_session(&member_session_id)
.await
.expect("check retried parent bridge session"),
"successful retry must archive the parent bridge session"
);
assert!(
mob_state.handle_for(&child_mob_id).await.is_err(),
"successful retry must remove the child cleanup retry anchor"
);
}
#[test]
fn test_create_session_request_rejects_reserved_surface_metadata_keys() {
let app_context = serde_json::json!({
"meerkat.runtime_id": "spoof"
});
let result = validate_public_surface_metadata(None, Some(&app_context));
assert!(result.is_err());
match result.unwrap_err() {
ApiError::BadRequest(message) => assert!(message.contains("meerkat.runtime_id")),
other => panic!("expected bad request, got {other:?}"),
}
}
#[tokio::test]
async fn test_continue_session_route_rejects_reserved_mob_peer_meta_labels() {
use axum::body::Body;
use http_body_util::BodyExt;
use tower::ServiceExt;
let temp = TempDir::new().unwrap();
let mut state = AppState::load_from(temp.path().to_path_buf())
.await
.unwrap();
state.llm_client_override = Some(Arc::new(MockLlmClient));
let session_service = state.session_service.clone();
let created = session_service
.create_session(SvcCreateSessionRequest {
injected_context: Vec::new(),
model: resolved_default_model(&state).await,
prompt: "Hello".to_string().into(),
system_prompt: meerkat::SystemPromptOverride::Inherit,
max_tokens: Some(state.max_tokens),
event_tx: None,
initial_turn: InitialTurnPolicy::Defer,
deferred_prompt_policy: DeferredPromptPolicy::Discard,
build: Some(SessionBuildOptions {
custom_models: std::collections::BTreeMap::new(),
image_generation_provider: None,
auto_compact_threshold_override: None,
llm_client_override: state
.llm_client_override
.clone()
.map(encode_llm_client_override_for_service),
..Default::default()
}),
labels: None,
})
.await
.expect("deferred session create should succeed");
let session_id = created.session_id.to_string();
let app = router(state);
let response = app
.oneshot(
axum::http::Request::builder()
.method("POST")
.uri(format!("/sessions/{session_id}/messages"))
.header("content-type", "application/json")
.body(Body::from(
serde_json::json!({
"session_id": session_id,
"prompt": "Continue",
"peer_meta": {
"labels": {
"mob_id": "team"
}
}
})
.to_string(),
))
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
let body = response.into_body().collect().await.unwrap().to_bytes();
let payload: serde_json::Value = serde_json::from_slice(&body).unwrap();
assert!(
payload["error"]
.as_str()
.is_some_and(|msg| msg.contains("reserved for Meerkat-owned runtime facts")),
"reserved mob label rejection should explain the trust boundary: {}",
String::from_utf8_lossy(&body)
);
}
#[tokio::test]
async fn test_append_system_context_route_returns_applied_status() {
use axum::body::Body;
use http_body_util::BodyExt;
use tower::ServiceExt;
let temp = TempDir::new().unwrap();
let mut state = AppState::load_from(temp.path().to_path_buf())
.await
.unwrap();
state.llm_client_override = Some(Arc::new(MockLlmClient));
let session_service = state.session_service.clone();
let pre_session = Session::new();
let bindings = state
.runtime_adapter
.prepare_bindings(pre_session.id().clone())
.await
.expect("runtime bindings should prepare");
let create_result = session_service
.create_session(SvcCreateSessionRequest {
injected_context: Vec::new(),
model: resolved_default_model(&state).await,
prompt: "Hello".to_string().into(),
system_prompt: meerkat::SystemPromptOverride::Inherit,
max_tokens: Some(state.max_tokens),
event_tx: None,
initial_turn: InitialTurnPolicy::Defer,
deferred_prompt_policy: DeferredPromptPolicy::Discard,
build: Some(SessionBuildOptions {
custom_models: std::collections::BTreeMap::new(),
image_generation_provider: None,
auto_compact_threshold_override: None,
resume_session: Some(pre_session),
llm_client_override: state
.llm_client_override
.clone()
.map(encode_llm_client_override_for_service),
runtime_build_mode: meerkat_core::RuntimeBuildMode::SessionOwned(bindings),
..Default::default()
}),
labels: None,
})
.await
.expect("deferred session create should succeed");
let app = router(state);
let session_id = create_result.session_id.to_string();
let inject_request = axum::http::Request::builder()
.method("POST")
.uri(format!("/sessions/{session_id}/system_context"))
.header("content-type", "application/json")
.body(Body::from(
serde_json::json!({
"content": { "type": "text", "text": "Coordinate with the orchestrator." },
"source": "mob",
"idempotency_key": "ctx-rest-test"
})
.to_string(),
))
.unwrap();
let inject_response = app.clone().oneshot(inject_request).await.unwrap();
let inject_status = inject_response.status();
let inject_body = inject_response
.into_body()
.collect()
.await
.unwrap()
.to_bytes();
assert_eq!(
inject_status,
StatusCode::OK,
"append system context failed: {}",
String::from_utf8_lossy(&inject_body)
);
let inject_payload: serde_json::Value = serde_json::from_slice(&inject_body).unwrap();
assert_eq!(inject_payload["status"], "applied");
}
#[tokio::test]
async fn test_session_status_route_is_available_for_live_sessions() {
use axum::body::Body;
use http_body_util::BodyExt;
use tower::ServiceExt;
let temp = TempDir::new().unwrap();
let mut state = AppState::load_from(temp.path().to_path_buf())
.await
.unwrap();
state.llm_client_override = Some(Arc::new(MockLlmClient));
let session_service = state.session_service.clone();
let pre_session = Session::new();
let bindings = state
.runtime_adapter
.prepare_bindings(pre_session.id().clone())
.await
.expect("runtime bindings should prepare");
let create_result = session_service
.create_session(SvcCreateSessionRequest {
injected_context: Vec::new(),
model: resolved_default_model(&state).await,
prompt: "Hello".to_string().into(),
system_prompt: meerkat::SystemPromptOverride::Inherit,
max_tokens: Some(state.max_tokens),
event_tx: None,
initial_turn: InitialTurnPolicy::Defer,
deferred_prompt_policy: DeferredPromptPolicy::Discard,
build: Some(SessionBuildOptions {
custom_models: std::collections::BTreeMap::new(),
image_generation_provider: None,
auto_compact_threshold_override: None,
resume_session: Some(pre_session),
llm_client_override: state
.llm_client_override
.clone()
.map(encode_llm_client_override_for_service),
runtime_build_mode: meerkat_core::RuntimeBuildMode::SessionOwned(bindings),
..Default::default()
}),
labels: None,
})
.await
.expect("deferred session create should succeed");
let app = router(state);
let session_id = create_result.session_id.to_string();
let request = axum::http::Request::builder()
.method("GET")
.uri(format!("/sessions/{session_id}/status"))
.body(Body::empty())
.unwrap();
let response = app.oneshot(request).await.unwrap();
let status = response.status();
let body = response.into_body().collect().await.unwrap().to_bytes();
assert_eq!(
status,
StatusCode::OK,
"runtime state request failed: {}",
String::from_utf8_lossy(&body)
);
}
#[tokio::test]
async fn test_runtime_host_routes_report_read_only_projection() {
use axum::body::Body;
use http_body_util::BodyExt;
use tower::ServiceExt;
let temp = TempDir::new().unwrap();
let state = AppState::load_from(temp.path().to_path_buf())
.await
.unwrap();
let app = router(state);
let request = axum::http::Request::builder()
.method("GET")
.uri("/runtime/host_info")
.body(Body::empty())
.unwrap();
let response = app.clone().oneshot(request).await.unwrap();
let status = response.status();
let body = response.into_body().collect().await.unwrap().to_bytes();
assert_eq!(
status,
StatusCode::OK,
"runtime host info failed: {}",
String::from_utf8_lossy(&body)
);
let payload: Value = serde_json::from_slice(&body).unwrap();
assert_eq!(payload["process_name"], "meerkat-rest");
assert!(
payload["host_id"].as_str().unwrap().starts_with("process:")
|| payload["host_id"]
.as_str()
.unwrap()
.starts_with("realm-instance:")
);
assert!(
matches!(
payload["host_id_scope"].as_str(),
Some("process" | "realm_instance")
),
"unexpected host id scope: {payload}"
);
assert_eq!(
payload["capabilities"]["features"]["runtime_backed_sessions"],
true
);
assert_eq!(payload["capabilities"]["features"]["event_replay"], false);
assert_eq!(payload["capabilities"]["features"]["artifacts"], false);
assert_eq!(payload["capabilities"]["features"]["approvals"], false);
assert_eq!(
payload["capabilities"]["features"]["multi_host_mobs"],
cfg!(feature = "mob"),
"REST host projection must advertise the compiled multi-host mob surface"
);
let text = serde_json::to_string(&payload).unwrap();
for forbidden in ["topology", "registry", "lease", "claim", "project"] {
assert!(
!text.contains(forbidden),
"runtime host projection must not claim topology authority token `{forbidden}`: {text}"
);
}
let request = axum::http::Request::builder()
.method("GET")
.uri("/runtime/health")
.body(Body::empty())
.unwrap();
let response = app.oneshot(request).await.unwrap();
let body = response.into_body().collect().await.unwrap().to_bytes();
let health: Value = serde_json::from_slice(&body).unwrap();
assert_eq!(health["status"], "ok");
}
#[test]
fn test_rest_path_feature_classifier_maps_feature_families() {
for descriptor in meerkat_contracts::rest_path_catalog() {
let path = descriptor.path;
let expected = if path == "/mob" || path.starts_with("/mob/") {
RestPathFeature::Mob
} else if path.contains("/mcp/") {
RestPathFeature::Mcp
} else if path.starts_with("/schedule/")
|| path == "/schedules"
|| path.starts_with("/schedules/")
|| path == "/schedule"
{
RestPathFeature::Schedule
} else {
RestPathFeature::Always
};
assert_eq!(
RestPathFeature::classify(path),
expected,
"path `{path}` classified to the wrong feature family",
);
}
assert_eq!(
RestPathFeature::classify("/sessions"),
RestPathFeature::Always
);
assert_eq!(RestPathFeature::classify("/help"), RestPathFeature::Always);
}
#[test]
fn test_host_info_rest_paths_omit_disabled_feature_routes() {
let mut options = meerkat::surface::RuntimeHostSurfaceOptions::process(
"meerkat-rest",
env!("CARGO_PKG_VERSION"),
);
options.mobs = false;
options.mcp_live = false;
options.schedules = false;
let advertised: Vec<String> = meerkat_contracts::rest_path_catalog()
.into_iter()
.filter(|path| RestPathFeature::classify(path.path).is_enabled(&options))
.map(|path| path.path.to_string())
.collect();
assert!(
!advertised.is_empty(),
"always-on routes must still be advertised"
);
for path in &advertised {
assert!(
!(path == "/mob" || path.starts_with("/mob/")),
"no-mob build must not advertise mob route `{path}`",
);
assert!(
!path.contains("/mcp/"),
"no-mcp build must not advertise mcp route `{path}`",
);
assert!(
!(path.starts_with("/schedule/")
|| path == "/schedule"
|| path == "/schedules"
|| path.starts_with("/schedules/")),
"no-schedule build must not advertise schedule route `{path}`",
);
}
assert!(advertised.iter().any(|p| p == "/sessions"));
assert!(advertised.iter().any(|p| p == "/health"));
options.mobs = true;
options.mcp_live = true;
options.schedules = true;
let full: Vec<String> = meerkat_contracts::rest_path_catalog()
.into_iter()
.filter(|path| RestPathFeature::classify(path.path).is_enabled(&options))
.map(|path| path.path.to_string())
.collect();
assert!(full.iter().any(|p| p.starts_with("/mob/")));
assert!(full.iter().any(|p| p.contains("/mcp/")));
assert!(full.iter().any(|p| p.starts_with("/schedules")));
}
#[tokio::test]
async fn test_get_session_history_route_returns_messages_for_live_and_archived_sessions() {
use axum::body::Body;
use http_body_util::BodyExt;
use tower::ServiceExt;
let temp = TempDir::new().unwrap();
let mut state = AppState::load_from(temp.path().to_path_buf())
.await
.unwrap();
state.llm_client_override = Some(Arc::new(MockLlmClient));
let session_service = state.session_service.clone();
let created_session_id = create_completed_rest_runtime_session(&state).await;
let session_id = created_session_id.to_string();
let app = router(state.clone());
let continue_request = axum::http::Request::builder()
.method("POST")
.uri(format!("/sessions/{session_id}/messages"))
.header("content-type", "application/json")
.body(Body::from(
serde_json::to_vec(&serde_json::json!({
"session_id": session_id,
"prompt": "Follow up"
}))
.unwrap(),
))
.unwrap();
let continue_response = app.clone().oneshot(continue_request).await.unwrap();
let continue_status = continue_response.status();
let continue_body = continue_response
.into_body()
.collect()
.await
.unwrap()
.to_bytes();
assert_eq!(
continue_status,
StatusCode::OK,
"second turn should succeed through the REST/session-runtime composition path: {}",
String::from_utf8_lossy(&continue_body)
);
let request = axum::http::Request::builder()
.method("GET")
.uri(format!("/sessions/{session_id}/history?offset=1&limit=2"))
.body(Body::empty())
.unwrap();
let response = app.clone().oneshot(request).await.unwrap();
let status = response.status();
let body = response.into_body().collect().await.unwrap().to_bytes();
assert_eq!(status, StatusCode::OK);
let payload: serde_json::Value = serde_json::from_slice(&body).unwrap();
assert_eq!(payload["session_id"], session_id);
assert!(
payload["message_count"].as_u64().unwrap_or(0) >= 4,
"history should expose the full multi-turn transcript: {payload}"
);
assert_eq!(payload["offset"], 1);
assert_eq!(payload["limit"], 2);
assert_eq!(payload["has_more"], true);
assert_eq!(payload["messages"].as_array().unwrap().len(), 2);
session_service
.archive_with_machine_protocol(
&created_session_id,
MachineSessionArchiveProtocol::from_machine(state.runtime_adapter.as_ref()),
)
.await
.expect("archive should retire through machine authority");
let archived_request = axum::http::Request::builder()
.method("GET")
.uri(format!("/sessions/{session_id}/history"))
.body(Body::empty())
.unwrap();
let archived_response = app.clone().oneshot(archived_request).await.unwrap();
let archived_status = archived_response.status();
let archived_body = archived_response
.into_body()
.collect()
.await
.unwrap()
.to_bytes();
assert_eq!(archived_status, StatusCode::OK);
let archived_payload: serde_json::Value = serde_json::from_slice(&archived_body).unwrap();
assert!(
archived_payload["message_count"].as_u64().unwrap_or(0) >= 4,
"archived history should preserve the transcript: {archived_payload}"
);
assert!(
archived_payload["messages"].as_array().unwrap().len() >= 4,
"archived history should return the full transcript"
);
ensure_rest_session_runtime_executor(&state, &created_session_id)
.await
.expect("REST runtime executor should attach");
assert!(
state
.runtime_adapter
.contains_session(&created_session_id)
.await,
"test should start with a stale runtime registration"
);
let continue_request = axum::http::Request::builder()
.method("POST")
.uri(format!("/sessions/{session_id}/messages"))
.header("content-type", "application/json")
.body(Body::from(
serde_json::to_vec(&serde_json::json!({
"session_id": session_id,
"prompt": "should not resume archived session"
}))
.unwrap(),
))
.unwrap();
let continue_response = app.oneshot(continue_request).await.unwrap();
assert_eq!(continue_response.status(), StatusCode::NOT_FOUND);
assert!(
!state
.runtime_adapter
.contains_session(&created_session_id)
.await,
"archived REST continue must not register runtime state"
);
}
#[tokio::test]
async fn test_create_session_route_completes_in_runtime_backed_mode() {
use axum::body::Body;
use http_body_util::BodyExt;
use tower::ServiceExt;
let temp = TempDir::new().unwrap();
let mut state = AppState::load_from(temp.path().to_path_buf())
.await
.unwrap();
state.llm_client_override = Some(Arc::new(MockLlmClient));
let app = router(state);
let response = tokio::time::timeout(
std::time::Duration::from_secs(10),
app.oneshot(
axum::http::Request::builder()
.method("POST")
.uri("/sessions")
.header("content-type", "application/json")
.body(Body::from(
serde_json::json!({
"prompt": "Remember RuntimeRouteFox and reply briefly."
})
.to_string(),
))
.unwrap(),
),
)
.await
.expect("runtime-backed create route timed out")
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
let body = response.into_body().collect().await.unwrap().to_bytes();
let payload: serde_json::Value = serde_json::from_slice(&body).unwrap();
assert!(payload["session_id"].is_string());
assert_eq!(payload["text"], "ok");
}
#[tokio::test]
async fn test_create_session_injected_context_lands_before_user_message() {
use axum::body::Body;
use http_body_util::BodyExt;
use tower::ServiceExt;
let temp = TempDir::new().unwrap();
let mut state = AppState::load_from(temp.path().to_path_buf())
.await
.unwrap();
state.llm_client_override = Some(Arc::new(MockLlmClient));
let app = router(state);
let response = tokio::time::timeout(
std::time::Duration::from_secs(10),
app.clone().oneshot(
axum::http::Request::builder()
.method("POST")
.uri("/sessions")
.header("content-type", "application/json")
.body(Body::from(
serde_json::json!({
"prompt": "What changed?",
"injected_context": ["ambient alpha", "ambient beta"]
})
.to_string(),
))
.unwrap(),
),
)
.await
.expect("create with injected context timed out")
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
let body = response.into_body().collect().await.unwrap().to_bytes();
let payload: serde_json::Value = serde_json::from_slice(&body).unwrap();
let session_id = payload["session_id"]
.as_str()
.expect("session id")
.to_string();
let history_response = app
.clone()
.oneshot(
axum::http::Request::builder()
.method("GET")
.uri(format!("/sessions/{session_id}/history"))
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(history_response.status(), StatusCode::OK);
let history_body = history_response
.into_body()
.collect()
.await
.unwrap()
.to_bytes();
let history: serde_json::Value = serde_json::from_slice(&history_body).unwrap();
let user_channel: Vec<(String, String)> = history["messages"]
.as_array()
.expect("history messages")
.iter()
.filter(|message| message["role"] == "user")
.map(|message| {
(
message
.get("transcript_role")
.and_then(|role| role.as_str())
.unwrap_or("conversational")
.to_string(),
message["content"].as_str().unwrap_or_default().to_string(),
)
})
.collect();
assert_eq!(
user_channel,
vec![
("injected_context".to_string(), "ambient alpha".to_string()),
("injected_context".to_string(), "ambient beta".to_string()),
("conversational".to_string(), "What changed?".to_string()),
],
"injected context must land as typed injected-context messages \
immediately before the first turn's user message: {history}"
);
}
#[test]
fn test_comms_send_request_peer_request_invalid_stream_rejected_at_serde() {
let err = serde_json::from_value::<CommsSendRequest>(json!({
"session_id": "sid_123",
"kind": "peer_request",
"to": uuid::Uuid::new_v4().to_string(),
"intent": "supervisor.bridge",
"params": rest_supervisor_bridge_params(),
"stream": "invalid"
}))
.expect_err("invalid stream must fail deserialization");
assert!(
err.to_string().contains("stream") || err.to_string().contains("invalid"),
"expected serde error mentioning stream, got: {err}"
);
}
#[test]
fn test_comms_send_request_peer_response_invalid_status_rejected_at_serde() {
let json = format!(
r#"{{"session_id":"sid_123","kind":"peer_response","to":"{}","in_reply_to":"{}","status":"almost-done"}}"#,
uuid::Uuid::new_v4(),
uuid::Uuid::new_v4()
);
let err = serde_json::from_str::<CommsSendRequest>(&json)
.expect_err("invalid status must fail deserialization");
assert!(
err.to_string().contains("status") || err.to_string().contains("almost-done"),
"expected serde error mentioning status, got: {err}"
);
}
#[test]
fn test_comms_send_request_invalid_source_rejected_at_serde() {
let json = r#"{"session_id":"sid_123","kind":"input","body":"hi","source":"webhookd"}"#;
let err = serde_json::from_str::<CommsSendRequest>(json)
.expect_err("invalid source must fail deserialization");
assert!(
err.to_string().contains("source") || err.to_string().contains("webhookd"),
"expected serde error mentioning source, got: {err}"
);
}
#[cfg(feature = "comms")]
#[test]
fn test_rest_comms_send_request_unknown_intent_fails_closed() {
let err = serde_json::from_value::<CommsSendRequest>(json!({
"session_id": "sid_123",
"kind": "peer_request",
"to": uuid::Uuid::new_v4().to_string(),
"intent": "local.default",
"params": rest_supervisor_bridge_params()
}))
.expect_err("unknown intent must fail before REST can dispatch");
let message = err.to_string();
assert!(
message.contains("local.default") || message.contains("variant"),
"expected unknown intent error, got: {message}"
);
}
#[cfg(feature = "comms")]
#[test]
fn test_rest_comms_send_request_malformed_params_fails_closed() {
let mut params = rest_supervisor_bridge_params();
params["extra_behavior"] = json!(true);
let err = serde_json::from_value::<CommsSendRequest>(json!({
"session_id": "sid_123",
"kind": "peer_request",
"to": uuid::Uuid::new_v4().to_string(),
"intent": "supervisor.bridge",
"params": params
}))
.expect_err("malformed bridge params must fail before REST can dispatch");
let message = err.to_string();
assert!(
message.contains("extra_behavior")
|| message.contains("unknown field")
|| message.contains("did not match any variant"),
"expected unknown params field error, got: {message}"
);
}
#[cfg(feature = "comms")]
#[test]
fn test_rest_comms_send_request_malformed_result_fails_closed() {
let err = serde_json::from_value::<CommsSendRequest>(json!({
"session_id": "sid_123",
"kind": "peer_response",
"to": uuid::Uuid::new_v4().to_string(),
"in_reply_to": uuid::Uuid::new_v4().to_string(),
"status": "completed",
"result": {
"result": "ack",
"ok": true,
"extra_behavior": true
}
}))
.expect_err("malformed bridge result must fail before REST can dispatch");
let message = err.to_string();
assert!(
message.contains("extra_behavior")
|| message.contains("unknown field")
|| message.contains("did not match any variant"),
"expected unknown result field error, got: {message}"
);
}
#[cfg(feature = "comms")]
fn rest_supervisor_bridge_params() -> Value {
let pubkey = [7u8; 32];
json!({
"command": "observe_member",
"supervisor": {
"name": "supervisor",
"peer_id": meerkat_core::comms::PeerId::from_ed25519_pubkey(&pubkey).to_string(),
"address": "inproc://supervisor",
"pubkey": pubkey,
},
"epoch": 1,
"protocol_version": 2,
})
}
#[cfg(not(feature = "comms"))]
#[test]
fn test_resolve_keep_alive_rejects_when_comms_disabled() {
let err = resolve_keep_alive(Some(true)).expect_err("keep_alive should be rejected");
assert!(matches!(err, ApiError::BadRequest(_)));
}
#[cfg(feature = "comms")]
#[test]
fn test_resolve_keep_alive_allows_when_comms_enabled() {
assert_eq!(resolve_keep_alive(Some(true)).unwrap(), Some(true));
assert_eq!(resolve_keep_alive(Some(false)).unwrap(), Some(false));
assert_eq!(resolve_keep_alive(None).unwrap(), None);
}
#[test]
fn test_create_session_request_accepts_hooks_override_fixture() {
let hooks_override = hooks_override_fixture();
let req_json = serde_json::json!({
"prompt": "Hello",
"hooks_override": hooks_override,
});
let req: CreateSessionRequest = serde_json::from_value(req_json).unwrap();
assert!(req.hooks_override.is_some());
let overrides = req
.hooks_override
.expect("hooks override should be present");
assert_eq!(overrides.entries.len(), 2);
assert_eq!(
overrides.entries[0].point,
meerkat_core::HookPoint::PreToolExecution
);
}
#[test]
fn test_continue_session_request_accepts_hooks_override_fixture() {
let hooks_override = hooks_override_fixture();
let req_json = serde_json::json!({
"session_id": "01234567-89ab-cdef-0123-456789abcdef",
"prompt": "Continue",
"hooks_override": hooks_override,
});
let req: ContinueSessionRequest = serde_json::from_value(req_json).unwrap();
assert!(req.hooks_override.is_some());
let overrides = req
.hooks_override
.expect("hooks override should be present");
assert_eq!(overrides.entries.len(), 2);
assert_eq!(
overrides.entries[1].mode,
meerkat_core::HookExecutionMode::Background
);
}
#[test]
fn test_rest_continue_requires_rebuild_matches_surface_contract() {
let mut req = ContinueSessionRequest {
injected_context: None,
transient_turn_context: None,
session_id: "01234567-89ab-cdef-0123-456789abcdef".to_string(),
prompt: ContentInput::Text("Continue".to_string()),
system_prompt: None,
output_schema: None,
structured_output_retries: None,
keep_alive: None,
comms_name: None,
peer_meta: None,
verbose: false,
model: None,
provider: None,
auth_binding: None,
max_tokens: None,
hooks_override: None,
enable_web_search: None,
skill_refs: None,
turn_tool_overlay: None,
additional_instructions: None,
};
assert!(!rest_continue_requires_rebuild(&req));
req.model = Some("gpt-5.4".into());
assert!(rest_continue_requires_rebuild(&req));
req.model = None;
req.auth_binding = Some(meerkat_contracts::WireAuthBindingRef {
realm: meerkat_core::RealmId::parse("dev").expect("valid realm"),
binding: meerkat_core::BindingId::parse("default_openai").expect("valid binding"),
profile: None,
});
assert!(rest_continue_requires_rebuild(&req));
req.auth_binding = None;
req.turn_tool_overlay = Some(meerkat_core::service::PublicTurnToolOverlay::default());
assert!(
!rest_continue_requires_rebuild(&req),
"turn tool overlay stays on the live path"
);
req.turn_tool_overlay = None;
req.additional_instructions = Some(vec!["extra".to_string()]);
assert!(
!rest_continue_requires_rebuild(&req),
"additional instructions stay on the live path"
);
req.additional_instructions = None;
req.system_prompt = Some("ordinary turn-boundary system".to_string());
assert!(
!rest_continue_requires_rebuild(&req),
"system prompt stays on the live turn-boundary path"
);
assert_eq!(
rest_continue_system_prompts(&req),
vec!["ordinary turn-boundary system"],
"REST continue lowers the requested System exactly once"
);
req.system_prompt = None;
assert!(rest_continue_system_prompts(&req).is_empty());
req.enable_web_search = Some(true);
assert!(rest_continue_requires_rebuild(&req));
req.enable_web_search = None;
req.comms_name = Some("agent-a".to_string());
assert!(rest_continue_requires_rebuild(&req));
}
#[tokio::test]
async fn test_continue_session_invalid_keep_alive_is_side_effect_free() {
use axum::body::Body;
use http_body_util::BodyExt;
use tower::ServiceExt;
let temp = TempDir::new().unwrap();
let mut state = AppState::load_from(temp.path().to_path_buf())
.await
.unwrap();
state.llm_client_override = Some(Arc::new(MockLlmClient));
let session_service = state.session_service.clone();
let created = session_service
.create_session(SvcCreateSessionRequest {
injected_context: Vec::new(),
model: resolved_default_model(&state).await,
prompt: "Hello".to_string().into(),
system_prompt: meerkat::SystemPromptOverride::Inherit,
max_tokens: Some(state.max_tokens),
event_tx: None,
initial_turn: InitialTurnPolicy::Defer,
deferred_prompt_policy: DeferredPromptPolicy::Discard,
build: Some(SessionBuildOptions {
custom_models: std::collections::BTreeMap::new(),
image_generation_provider: None,
auto_compact_threshold_override: None,
llm_client_override: state
.llm_client_override
.clone()
.map(encode_llm_client_override_for_service),
..Default::default()
}),
labels: None,
})
.await
.expect("deferred session create should succeed");
let session_id = created.session_id.to_string();
let app = router(state);
let response = app
.oneshot(
axum::http::Request::builder()
.method("POST")
.uri(format!("/sessions/{session_id}/messages"))
.header("content-type", "application/json")
.body(Body::from(
serde_json::json!({
"session_id": session_id,
"prompt": "Continue",
"keep_alive": true
})
.to_string(),
))
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
let body = response.into_body().collect().await.unwrap().to_bytes();
let payload: serde_json::Value = serde_json::from_slice(&body).unwrap();
assert_eq!(payload["error"], "keep_alive requires comms_name");
let session = session_service
.load_authoritative_session(&created.session_id)
.await
.expect("load should succeed")
.expect("session should still exist");
let metadata = session.session_metadata().expect("metadata should exist");
assert!(
!metadata.keep_alive,
"failed request must not persist keep_alive"
);
assert!(metadata.comms_name.is_none());
}
#[cfg(feature = "comms")]
#[tokio::test]
async fn test_continue_session_keep_alive_live_missing_failure_unregisters_new_runtime() {
let temp = TempDir::new().unwrap();
let mut state = AppState::load_from(temp.path().to_path_buf())
.await
.unwrap();
state.llm_client_override = Some(Arc::new(MockLlmClient));
let created = state
.session_service
.create_session(SvcCreateSessionRequest {
injected_context: Vec::new(),
model: resolved_default_model(&state).await,
prompt: "Hello".to_string().into(),
system_prompt: meerkat::SystemPromptOverride::Inherit,
max_tokens: Some(state.max_tokens),
event_tx: None,
initial_turn: InitialTurnPolicy::Defer,
deferred_prompt_policy: DeferredPromptPolicy::Discard,
build: Some(SessionBuildOptions {
custom_models: std::collections::BTreeMap::new(),
image_generation_provider: None,
auto_compact_threshold_override: None,
comms_name: Some("stale-rest-agent".to_string()),
llm_client_override: state
.llm_client_override
.clone()
.map(encode_llm_client_override_for_service),
..Default::default()
}),
labels: None,
})
.await
.expect("deferred comms session create should succeed");
let session_id = created.session_id;
state
.session_service
.discard_live_session(&session_id)
.await
.expect("discard live session");
state
.runtime_adapter
.unregister_session(&session_id)
.await
.expect("runtime session should unregister cleanly");
assert!(
!state.runtime_adapter.contains_session(&session_id).await,
"test starts with no live runtime registration"
);
let outcome = Box::pin(continue_session_inner(
&state,
&session_id.to_string(),
ContinueSessionRequest {
injected_context: None,
transient_turn_context: None,
session_id: session_id.to_string(),
prompt: ContentInput::Text("Continue".to_string()),
system_prompt: None,
output_schema: None,
structured_output_retries: None,
keep_alive: Some(true),
comms_name: None,
peer_meta: None,
verbose: false,
model: None,
provider: None,
auth_binding: None,
max_tokens: None,
hooks_override: None,
enable_web_search: None,
skill_refs: None,
turn_tool_overlay: None,
additional_instructions: None,
},
None,
))
.await;
match outcome {
RequestTerminal::RespondWithoutPublish(Err(ApiError::BadRequest(message))) => {
assert!(
message.contains("session created with comms_name"),
"expected live-missing keep_alive rejection: {message}"
);
}
other => panic!("expected live-missing keep_alive rejection, got {other:?}"),
}
assert!(
!state.runtime_adapter.contains_session(&session_id).await,
"failed keep_alive continue must not leave the recovered runtime registered"
);
}
#[tokio::test]
async fn test_continue_session_validation_failure_unregisters_new_runtime() {
let temp = TempDir::new().unwrap();
let mut state = AppState::load_from(temp.path().to_path_buf())
.await
.unwrap();
state.llm_client_override = Some(Arc::new(MockLlmClient));
let created = state
.session_service
.create_session(SvcCreateSessionRequest {
injected_context: Vec::new(),
model: resolved_default_model(&state).await,
prompt: "Hello".to_string().into(),
system_prompt: meerkat::SystemPromptOverride::Inherit,
max_tokens: Some(state.max_tokens),
event_tx: None,
initial_turn: InitialTurnPolicy::Defer,
deferred_prompt_policy: DeferredPromptPolicy::Discard,
build: Some(SessionBuildOptions {
custom_models: std::collections::BTreeMap::new(),
image_generation_provider: None,
auto_compact_threshold_override: None,
llm_client_override: state
.llm_client_override
.clone()
.map(encode_llm_client_override_for_service),
..Default::default()
}),
labels: None,
})
.await
.expect("deferred session create should succeed");
let session_id = created.session_id;
state
.runtime_adapter
.unregister_session(&session_id)
.await
.expect("runtime session should unregister cleanly");
assert!(
!state.runtime_adapter.contains_session(&session_id).await,
"test starts with no runtime registration"
);
let outcome = Box::pin(continue_session_inner(
&state,
&session_id.to_string(),
ContinueSessionRequest {
injected_context: None,
transient_turn_context: None,
session_id: session_id.to_string(),
prompt: inline_video_prompt(),
system_prompt: None,
output_schema: None,
structured_output_retries: None,
keep_alive: None,
comms_name: None,
peer_meta: None,
verbose: false,
model: None,
provider: None,
auth_binding: None,
max_tokens: None,
hooks_override: None,
enable_web_search: None,
skill_refs: None,
turn_tool_overlay: None,
additional_instructions: None,
},
None,
))
.await;
match outcome {
RequestTerminal::RespondWithoutPublish(Err(ApiError::BadRequestWithData {
message,
code,
details,
})) => {
assert_eq!(code, "UNSUPPORTED_MODEL_CAPABILITY");
assert!(
message.contains("inline video"),
"expected inline video validation failure: {message}"
);
assert_eq!(
details["unsupported_capability"]["capability"],
serde_json::json!("inline_video")
);
}
other => panic!("expected bad request validation failure, got {other:?}"),
}
assert!(
!state.runtime_adapter.contains_session(&session_id).await,
"failed non-rebuild validation must not leave a new runtime registration"
);
}
#[tokio::test]
async fn test_continue_session_validation_failure_preserves_existing_runtime() {
let temp = TempDir::new().unwrap();
let mut state = AppState::load_from(temp.path().to_path_buf())
.await
.unwrap();
state.llm_client_override = Some(Arc::new(MockLlmClient));
let created = state
.session_service
.create_session(SvcCreateSessionRequest {
injected_context: Vec::new(),
model: resolved_default_model(&state).await,
prompt: "Hello".to_string().into(),
system_prompt: meerkat::SystemPromptOverride::Inherit,
max_tokens: Some(state.max_tokens),
event_tx: None,
initial_turn: InitialTurnPolicy::Defer,
deferred_prompt_policy: DeferredPromptPolicy::Discard,
build: Some(SessionBuildOptions {
custom_models: std::collections::BTreeMap::new(),
image_generation_provider: None,
auto_compact_threshold_override: None,
llm_client_override: state
.llm_client_override
.clone()
.map(encode_llm_client_override_for_service),
..Default::default()
}),
labels: None,
})
.await
.expect("deferred session create should succeed");
let session_id = created.session_id;
ensure_rest_session_runtime_executor(&state, &session_id)
.await
.expect("REST runtime executor should attach");
assert!(
state.runtime_adapter.contains_session(&session_id).await,
"test requires a pre-existing runtime registration"
);
let outcome = Box::pin(continue_session_inner(
&state,
&session_id.to_string(),
ContinueSessionRequest {
injected_context: None,
transient_turn_context: None,
session_id: session_id.to_string(),
prompt: inline_video_prompt(),
system_prompt: None,
output_schema: None,
structured_output_retries: None,
keep_alive: None,
comms_name: None,
peer_meta: None,
verbose: false,
model: None,
provider: None,
auth_binding: None,
max_tokens: None,
hooks_override: None,
enable_web_search: None,
skill_refs: None,
turn_tool_overlay: None,
additional_instructions: None,
},
None,
))
.await;
match outcome {
RequestTerminal::RespondWithoutPublish(Err(ApiError::BadRequestWithData {
message,
code,
details,
})) => {
assert_eq!(code, "UNSUPPORTED_MODEL_CAPABILITY");
assert!(
message.contains("inline video"),
"expected inline video validation failure: {message}"
);
assert_eq!(
details["unsupported_capability"]["capability"],
serde_json::json!("inline_video")
);
}
other => panic!("expected bad request validation failure, got {other:?}"),
}
assert!(
state.runtime_adapter.contains_session(&session_id).await,
"failed non-rebuild validation must preserve an existing runtime registration"
);
}
#[tokio::test]
async fn test_continue_session_rebuild_validation_failure_preserves_existing_runtime() {
let temp = TempDir::new().unwrap();
let mut state = AppState::load_from(temp.path().to_path_buf())
.await
.unwrap();
state.llm_client_override = Some(Arc::new(MockLlmClient));
let created = state
.session_service
.create_session(SvcCreateSessionRequest {
injected_context: Vec::new(),
model: resolved_default_model(&state).await,
prompt: "Hello".to_string().into(),
system_prompt: meerkat::SystemPromptOverride::Inherit,
max_tokens: Some(state.max_tokens),
event_tx: None,
initial_turn: InitialTurnPolicy::Defer,
deferred_prompt_policy: DeferredPromptPolicy::Discard,
build: Some(SessionBuildOptions {
custom_models: std::collections::BTreeMap::new(),
image_generation_provider: None,
auto_compact_threshold_override: None,
llm_client_override: state
.llm_client_override
.clone()
.map(encode_llm_client_override_for_service),
..Default::default()
}),
labels: None,
})
.await
.expect("deferred session create should succeed");
let session_id = created.session_id;
ensure_rest_session_runtime_executor(&state, &session_id)
.await
.expect("REST runtime executor should attach");
assert!(
state.runtime_adapter.contains_session(&session_id).await,
"test requires a pre-existing runtime registration"
);
let outcome = Box::pin(continue_session_inner(
&state,
&session_id.to_string(),
ContinueSessionRequest {
injected_context: None,
transient_turn_context: None,
session_id: session_id.to_string(),
prompt: ContentInput::Text("Continue".to_string()),
system_prompt: None,
output_schema: None,
structured_output_retries: None,
keep_alive: None,
comms_name: None,
peer_meta: None,
verbose: false,
model: Some("gpt-5.4".to_string()),
provider: Some(Provider::Anthropic),
auth_binding: None,
max_tokens: None,
hooks_override: None,
enable_web_search: None,
skill_refs: None,
turn_tool_overlay: None,
additional_instructions: None,
},
None,
))
.await;
match outcome {
RequestTerminal::RespondWithoutPublish(Err(ApiError::BadRequest(message))) => {
assert!(
message.contains("provider"),
"expected provider validation failure after prepare: {message}"
);
}
other => panic!("expected bad request validation failure, got {other:?}"),
}
assert!(
state.runtime_adapter.contains_session(&session_id).await,
"failed rebuild validation must preserve an existing runtime registration"
);
}
#[tokio::test]
async fn test_continue_session_rebuild_waits_for_runtime_registration_lock_before_prepare() {
let temp = TempDir::new().unwrap();
let mut state = AppState::load_from(temp.path().to_path_buf())
.await
.unwrap();
state.llm_client_override = Some(Arc::new(MockLlmClient));
let created = state
.session_service
.create_session(SvcCreateSessionRequest {
injected_context: Vec::new(),
model: resolved_default_model(&state).await,
prompt: "Hello".to_string().into(),
system_prompt: meerkat::SystemPromptOverride::Inherit,
max_tokens: Some(state.max_tokens),
event_tx: None,
initial_turn: InitialTurnPolicy::Defer,
deferred_prompt_policy: DeferredPromptPolicy::Discard,
build: Some(SessionBuildOptions {
custom_models: std::collections::BTreeMap::new(),
image_generation_provider: None,
auto_compact_threshold_override: None,
llm_client_override: state
.llm_client_override
.clone()
.map(encode_llm_client_override_for_service),
..Default::default()
}),
labels: None,
})
.await
.expect("deferred session create should succeed");
let session_id = created.session_id;
state
.runtime_adapter
.unregister_session(&session_id)
.await
.expect("runtime session should unregister cleanly");
let registration_lock = rest_runtime_registration_lock(&state, &session_id);
let registration_guard = registration_lock.mutex().lock().await;
let continue_state = state.clone();
let continue_session_id = session_id.clone();
let mut continue_task = tokio::spawn(async move {
Box::pin(continue_session_inner(
&continue_state,
&continue_session_id.to_string(),
ContinueSessionRequest {
injected_context: None,
transient_turn_context: None,
session_id: continue_session_id.to_string(),
prompt: ContentInput::Text("Continue".to_string()),
system_prompt: None,
output_schema: None,
structured_output_retries: None,
keep_alive: None,
comms_name: None,
peer_meta: None,
verbose: false,
model: Some("gpt-5.4".to_string()),
provider: Some(Provider::Anthropic),
auth_binding: None,
max_tokens: None,
hooks_override: None,
enable_web_search: None,
skill_refs: None,
turn_tool_overlay: None,
additional_instructions: None,
},
None,
))
.await
});
tokio::select! {
result = &mut continue_task => {
panic!("rebuild continue completed before taking the runtime registration lock: {result:?}");
}
() = tokio::time::sleep(std::time::Duration::from_millis(50)) => {}
}
assert!(
!state.runtime_adapter.contains_session(&session_id).await,
"rebuild continue must not prepare runtime bindings while registration lock is held"
);
drop(registration_guard);
drop(registration_lock);
let outcome = tokio::time::timeout(std::time::Duration::from_secs(5), continue_task)
.await
.expect("rebuild continue should finish after lock release")
.expect("rebuild continue task should not panic");
match outcome {
RequestTerminal::RespondWithoutPublish(Err(ApiError::BadRequest(message))) => {
assert!(
message.contains("provider"),
"expected provider validation failure after lock release: {message}"
);
}
other => panic!("expected bad request validation failure, got {other:?}"),
}
}
#[tokio::test]
async fn rest_new_runtime_cleanup_preserves_pending_pre_admission() {
let temp = TempDir::new().unwrap();
let mut state = load_rest_state_with_capacity(&temp, 1).await;
state.llm_client_override = Some(Arc::new(MockLlmClient));
let session_id = create_completed_rest_runtime_session(&state).await;
state
.runtime_adapter
.unregister_session(&session_id)
.await
.expect("runtime session should unregister cleanly");
let runtime_was_registered = state.runtime_adapter.contains_session(&session_id).await;
state
.runtime_adapter
.prepare_bindings(session_id.clone())
.await
.expect("prepare new runtime binding");
let input_id = meerkat_core::lifecycle::InputId::new();
let admission = state
.session_service
.reserve_runtime_turn_admission(&session_id)
.await
.expect("reserve pending runtime admission");
insert_rest_runtime_pre_admission(
&state.runtime_pre_admissions,
session_id.clone(),
input_id.clone(),
admission,
)
.await
.expect("insert pending pre-admission");
unregister_rest_runtime_if_new(&state, &session_id, runtime_was_registered)
.await
.expect("active runtime input must make cleanup a no-op");
assert!(
state.runtime_adapter.contains_session(&session_id).await,
"cleanup must preserve a new runtime registration with pending active admission"
);
discard_rest_runtime_pre_admission(&state.runtime_pre_admissions, &session_id, &input_id)
.await;
}
#[tokio::test]
async fn rest_runtime_cleanup_preserves_completion_outcome() {
let temp = TempDir::new().unwrap();
let state = load_rest_state_with_capacity(&temp, 1).await;
let session_id = SessionId::new();
let handle = runtime_terminated_completion_handle(
&state.runtime_adapter,
&session_id,
"cleanup preserves outcome",
)
.await;
let handle = wrap_rest_runtime_completion_cleanup(state, session_id, handle);
let outcome = handle
.wait()
.await
.expect("cleanup-preserved completion handle should resolve");
assert!(
matches!(
outcome,
meerkat_runtime::CompletionOutcome::RuntimeTerminated { .. }
),
"cleanup failure must not synthesize a runtime terminal outcome: {outcome:?}"
);
}
#[tokio::test]
async fn rest_runtime_cleanup_preserves_outcome_after_machine_saga_cleanup() {
let temp = TempDir::new().unwrap();
let state = load_rest_state_with_capacity(&temp, 1).await;
let session_id = SessionId::new();
let handle = runtime_terminated_completion_handle(
&state.runtime_adapter,
&session_id,
"REST external cleanup precedes machine saga",
)
.await;
let mut executor =
RestSessionRuntimeExecutor::new(state.runtime_executor_context(), session_id.clone());
meerkat_core::lifecycle::CoreExecutor::cleanup_after_runtime_stop_terminalized(
&mut executor,
)
.await
.expect("REST executor external cleanup should succeed");
assert!(
state.runtime_adapter.contains_session(&session_id).await,
"REST external cleanup must not recursively unregister machine authority"
);
state
.runtime_adapter
.unregister_session(&session_id)
.await
.expect("machine-owned saga should remove the stopped REST runtime");
assert!(!state.runtime_adapter.contains_session(&session_id).await);
let handle = wrap_rest_runtime_completion_cleanup(state, session_id, handle);
let outcome = handle
.wait()
.await
.expect("verified machine-saga cleanup should preserve the completion outcome");
assert!(
matches!(
outcome,
meerkat_runtime::CompletionOutcome::RuntimeTerminated { .. }
),
"REST machine-saga cleanup must publish RuntimeTerminated: {outcome:?}"
);
}
#[tokio::test]
async fn rest_runtime_cleanup_withholds_outcome_on_authority_mismatch() {
let temp = TempDir::new().unwrap();
let state = load_rest_state_with_capacity(&temp, 1).await;
let source_session_id = SessionId::new();
let target_session_id = SessionId::new();
let handle = runtime_terminated_completion_handle(
&state.runtime_adapter,
&source_session_id,
"mismatched cleanup authority",
)
.await;
let handle = wrap_rest_runtime_completion_cleanup(state, target_session_id, handle);
let error = handle
.wait()
.await
.expect_err("required REST cleanup mismatch must withhold the outcome");
assert!(
error.to_string().contains("cleanup authority failed"),
"unexpected cleanup error: {error}"
);
}
#[tokio::test]
async fn rest_runtime_pre_admission_cleanup_unregisters_after_runtime_termination() {
let temp = TempDir::new().unwrap();
let mut state = load_rest_state_with_capacity(&temp, 1).await;
state.llm_client_override = Some(Arc::new(MockLlmClient));
let session_id = SessionId::new();
let input_id = meerkat_core::lifecycle::InputId::new();
let admission = state
.session_service
.reserve_create_session_admission()
.await
.expect("reserve active admission");
insert_rest_runtime_pre_admission(
&state.runtime_pre_admissions,
session_id.clone(),
input_id.clone(),
admission,
)
.await
.expect("insert runtime pre-admission");
let handle = runtime_terminated_completion_handle(
&state.runtime_adapter,
&session_id,
"runtime stopped during cleanup",
)
.await;
spawn_rest_runtime_pre_admission_rekey_and_cleanup(
state.clone(),
session_id.clone(),
input_id.clone(),
input_id,
handle,
);
for _ in 0..200 {
if !state.runtime_adapter.contains_session(&session_id).await
&& !state
.runtime_pre_admissions
.lock()
.await
.contains_key(&session_id)
{
break;
}
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
}
assert!(
!state.runtime_adapter.contains_session(&session_id).await,
"runtime termination cleanup must unregister stale runtime state"
);
assert!(
!state
.runtime_pre_admissions
.lock()
.await
.contains_key(&session_id),
"completion cleanup must remove runtime pre-admission"
);
assert!(
!state
.session_service
.has_live_session(&session_id)
.await
.expect("check live session"),
"boundary commit failure cleanup must discard dirty live state"
);
let _replacement = tokio::time::timeout(std::time::Duration::from_secs(1), async {
loop {
match state
.session_service
.reserve_create_session_admission()
.await
{
Ok(guard) => break guard,
Err(err) if err.to_string().contains("Max sessions") => {
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
}
Err(err) => panic!("unexpected replacement admission error: {err:?}"),
}
}
})
.await
.expect("failed runtime input cleanup should release active admission");
}
#[tokio::test]
async fn test_create_session_route_returns_identity_on_post_commit_turn_failure() {
use axum::body::Body;
use http_body_util::BodyExt;
use tower::ServiceExt;
let temp = TempDir::new().unwrap();
let mut state = AppState::load_from(temp.path().to_path_buf())
.await
.unwrap();
state.llm_client_override = Some(Arc::new(ErrorLlmClient));
let app = router(state);
let response = tokio::time::timeout(
std::time::Duration::from_secs(30),
app.oneshot(
axum::http::Request::builder()
.method("POST")
.uri("/sessions")
.header("content-type", "application/json")
.body(Body::from(
serde_json::json!({
"prompt": "Trigger failure"
})
.to_string(),
))
.unwrap(),
),
)
.await
.expect("post-commit-failure create route timed out")
.unwrap();
assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
let body = response.into_body().collect().await.unwrap().to_bytes();
let payload: serde_json::Value = serde_json::from_slice(&body).unwrap();
assert_eq!(payload["code"], "SESSION_CREATED_WITH_TURN_FAILURE");
assert_eq!(payload["details"]["session_created"], true);
assert_eq!(payload["details"]["resumable"], true);
assert!(payload["details"]["session_id"].is_string());
assert!(payload["details"]["session_ref"].is_string());
assert_eq!(payload["details"]["turn_error_code"], "TURN_ABANDONED");
assert_eq!(payload["details"]["error"]["kind"], "llm_failure");
assert_eq!(payload["details"]["error"]["terminal"], true);
}
#[test]
fn test_skill_entry_uses_canonical_source_identity_records() {
use meerkat_core::skills::{
SkillDescriptor, SkillIntrospectionEntry, SkillKey, SkillName, SkillScope,
SourceIdentityRecord, SourceIdentityStatus, SourceTransportKind, SourceUuid,
};
let source_uuid =
SourceUuid::parse("33333333-3333-4333-8333-333333333333").expect("source uuid");
let shadow_uuid =
SourceUuid::parse("44444444-4444-4444-8444-444444444444").expect("shadow uuid");
let key = SkillKey::new(
source_uuid.clone(),
SkillName::parse("demo-skill").expect("skill name"),
);
let mut descriptor = SkillDescriptor::new(key, "Demo Skill", "Demo description");
descriptor.scope = SkillScope::Project;
descriptor.source_name = "canonical-source".to_string();
let source_identity = SourceIdentityRecord {
source_uuid: source_uuid.clone(),
display_name: "canonical-source".to_string(),
transport_kind: SourceTransportKind::Git,
fingerprint: "repo-canonical-source".to_string(),
status: SourceIdentityStatus::Retired,
};
let shadow_identity = SourceIdentityRecord {
source_uuid: shadow_uuid.clone(),
display_name: "shadow-source".to_string(),
transport_kind: SourceTransportKind::Http,
fingerprint: "repo-shadow-source".to_string(),
status: SourceIdentityStatus::Disabled,
};
let entry = SkillIntrospectionEntry {
descriptor,
source_identity: Some(source_identity.clone()),
shadowed_by: Some("shadow-source".to_string()),
shadowed_by_identity: Some(shadow_identity.clone()),
shadowed_by_source_uuid: Some(shadow_uuid),
is_active: false,
};
let wire = skill_entry(&entry).expect("skill entry");
assert_eq!(wire.source.identity, source_identity);
assert_eq!(
wire.shadowed_by.expect("shadowed by").identity,
shadow_identity
);
}
#[test]
fn test_validate_config_for_commit_rejects_invalid_skills_identity() {
let mut config = Config::default();
let source_uuid =
meerkat_core::skills::SourceUuid::parse("dc256086-0d2f-4f61-a307-320d4148107f")
.expect("uuid");
config.skills.repositories = vec![
meerkat_core::skills_config::SkillRepositoryConfig {
name: "a".to_string(),
source_uuid: source_uuid.clone(),
transport: meerkat_core::skills_config::SkillRepoTransport::Filesystem {
path: "/tmp/a".to_string(),
},
},
meerkat_core::skills_config::SkillRepositoryConfig {
name: "b".to_string(),
source_uuid,
transport: meerkat_core::skills_config::SkillRepoTransport::Filesystem {
path: "/tmp/b".to_string(),
},
},
];
let err = validate_config_for_commit_with_roots(&config, None, None)
.expect_err("duplicate source uuid");
assert!(matches!(&err, ApiError::BadRequest(_)));
if let ApiError::BadRequest(message) = err {
assert!(message.contains("Invalid skills source-identity config"));
}
}
#[test]
fn test_run_result_to_response_carries_skill_diagnostics() {
let session_id = SessionId::new();
let result = meerkat_core::RunResult {
text: "ok".to_string(),
session_id: session_id.clone(),
usage: Default::default(),
turns: 1,
tool_calls: 0,
terminal_cause_kind: None,
structured_output: None,
extraction_error: None,
schema_warnings: None,
skill_diagnostics: Some(meerkat_core::skills::SkillRuntimeDiagnostics {
source_health: meerkat_core::skills::SourceHealthSnapshot {
state: meerkat_core::skills::SourceHealthState::Degraded,
invalid_ratio: 0.50,
invalid_count: 1,
total_count: 2,
failure_streak: 3,
handshake_failed: false,
},
quarantined: vec![],
collection_fault: None,
}),
};
let realm = meerkat_core::RealmId::parse("test-realm").expect("valid test realm id");
let response = run_result_to_response(result, &realm);
assert!(response.skill_diagnostics.is_some());
assert_eq!(
response
.skill_diagnostics
.as_ref()
.expect("skill_diagnostics")
.source_health
.state,
meerkat_core::skills::SourceHealthState::Degraded
);
assert_eq!(
response.session_ref.as_deref().expect("session_ref"),
format_session_ref(&realm, &session_id)
);
}
#[test]
fn completion_outcome_to_api_result_surfaces_callback_pending_payload() {
let session_id = SessionId::new();
let realm = meerkat_core::RealmId::parse("test-realm").expect("valid test realm id");
let err = completion_outcome_to_api_result(
meerkat_runtime::completion::CompletionOutcome::CallbackPending {
tool_use_id: "call-1".to_string(),
tool_name: "external_mock".to_string(),
args: json!({ "value": "browser" }),
},
&session_id,
&realm,
false,
)
.expect_err("callback pending should map to an API error");
let ApiError::InternalWithData {
message,
code,
details,
} = err
else {
panic!("expected InternalWithData callback error");
};
assert_eq!(message, "callback pending for tool 'external_mock'");
assert_eq!(code, "CALLBACK_PENDING");
assert_eq!(details["session_id"], session_id.to_string());
assert_eq!(details["pending_tool_calls"][0]["tool_use_id"], "call-1");
assert_eq!(
details["session_ref"],
format_session_ref(&realm, &session_id)
);
assert_eq!(details["resumable"], true);
assert_eq!(details["tool_name"], "external_mock");
assert_eq!(details["args"], json!({ "value": "browser" }));
}
#[test]
fn completion_outcome_to_api_result_surfaces_cancelled() {
let session_id = SessionId::new();
let realm = meerkat_core::RealmId::parse("test-realm").expect("valid test realm id");
let err = completion_outcome_to_api_result(
meerkat_runtime::completion::CompletionOutcome::Cancelled,
&session_id,
&realm,
false,
)
.expect_err("cancelled completion should map to API cancellation");
assert!(matches!(err, ApiError::RequestCancelled { details: None }));
}
#[test]
fn completion_outcome_to_api_result_preserves_created_session_abandoned_metadata() {
let session_id = SessionId::new();
let realm = meerkat_core::RealmId::parse("test-realm").expect("valid test realm id");
let err = completion_outcome_to_api_result(
meerkat_runtime::completion::CompletionOutcome::AbandonedWithError {
reason: "apply failed: Terminal failure: Failed (LlmFailure)".to_string(),
error: meerkat_core::TurnErrorMetadata::terminal(
meerkat_core::TurnTerminalCauseKind::LlmFailure,
meerkat_core::TurnTerminalOutcome::Failed,
"LLM failure terminal turn",
),
},
&session_id,
&realm,
true,
)
.expect_err("created-session abandoned turn should map to resumable session failure");
let ApiError::InternalWithData {
code,
details,
message,
} = err
else {
panic!("expected created-session turn failure");
};
assert_eq!(code, "SESSION_CREATED_WITH_TURN_FAILURE");
assert!(message.contains("turn abandoned: apply failed"));
assert_eq!(details["session_created"], true);
assert_eq!(details["resumable"], true);
assert_eq!(details["turn_error_code"], "TURN_ABANDONED");
assert_eq!(details["error"]["kind"], "llm_failure");
assert_eq!(details["error"]["terminal"], true);
assert_eq!(
details["turn_error_details"]["error"]["kind"],
"llm_failure"
);
}
#[test]
fn completion_outcome_to_api_result_finalization_failure_hides_nondurable_result() {
let session_id = SessionId::new();
let realm = meerkat_core::RealmId::parse("test-realm").expect("valid test realm id");
let err = completion_outcome_to_api_result(
meerkat_runtime::completion::CompletionOutcome::CompletedWithFinalizationFailure {
error: meerkat_core::TurnErrorMetadata::runtime_apply_failure(
"runtime loop commit failed: synthetic finalization failure",
),
},
&session_id,
&realm,
false,
)
.expect_err("finalization failure should map to an API error");
let ApiError::InternalWithData {
code,
details,
message,
} = err
else {
panic!("expected typed finalization API error");
};
assert_eq!(code, "TURN_FINALIZATION_FAILED");
assert!(message.contains("synthetic finalization failure"));
assert_eq!(details["error"]["kind"], "runtime_apply_failure");
assert_eq!(details["error"]["terminal"], true);
assert!(
details.get("run_result").is_none(),
"finalization failure must not expose a non-durable run_result"
);
assert!(
details.get("structured_output").is_none(),
"finalization failure must not expose non-durable structured_output"
);
}
#[test]
fn create_session_error_to_api_surfaces_cancelled_as_request_cancelled() {
let err = create_session_error_to_api(SessionError::Agent(
meerkat_core::error::AgentError::Cancelled,
));
assert!(matches!(err, ApiError::RequestCancelled { details: None }));
}
#[cfg(feature = "mob")]
#[tokio::test]
async fn test_compatibility_mob_routes_are_not_found() {
use axum::body::Body;
use tower::ServiceExt;
let temp = TempDir::new().unwrap();
let state = AppState::load_from(temp.path().to_path_buf())
.await
.unwrap();
let app = router(state);
let tools_req = axum::http::Request::builder()
.method("GET")
.uri("/mob/tools")
.body(Body::empty())
.unwrap();
let tools_resp = app.clone().oneshot(tools_req).await.unwrap();
assert_eq!(tools_resp.status(), StatusCode::NOT_FOUND);
let call_req = axum::http::Request::builder()
.method("POST")
.uri("/mob/call")
.header("content-type", "application/json")
.body(Body::from(
serde_json::json!({
"name": "mob_create",
"arguments": { "definition": { "id": "test_mob", "profiles": { "worker": { "model": "claude-sonnet-4-6", "tools": { "comms": true } } } } }
})
.to_string(),
))
.unwrap();
let call_resp = app.oneshot(call_req).await.unwrap();
assert_eq!(call_resp.status(), StatusCode::NOT_FOUND);
}
#[cfg(feature = "mob")]
#[tokio::test]
async fn test_mob_wait_kickoff_route_returns_member_snapshots() {
use axum::body::Body;
use http_body_util::BodyExt;
use tower::ServiceExt;
let temp = TempDir::new().unwrap();
let state = AppState::load_from(temp.path().to_path_buf())
.await
.unwrap();
let definition = meerkat_mob::MobDefinition::from_toml(
"[mob]\nid = \"test_mob\"\n\n[profiles.worker]\nmodel = \"claude-sonnet-4-6\"\n\n[profiles.worker.tools]\ncomms = true\n",
)
.expect("minimal mob definition");
let mob_id = state
.mob_state
.mob_create_definition(definition)
.await
.expect("create mob");
let app = router(state);
let request = axum::http::Request::builder()
.method("POST")
.uri(format!("/mob/{mob_id}/wait-kickoff"))
.header("content-type", "application/json")
.body(Body::from("{}"))
.unwrap();
let response = app.oneshot(request).await.unwrap();
assert_eq!(response.status(), StatusCode::OK);
let body_bytes = response.into_body().collect().await.unwrap().to_bytes();
let payload: serde_json::Value = serde_json::from_slice(&body_bytes).unwrap();
let members = payload["members"]
.as_array()
.expect("members should be an array");
assert!(
members.is_empty(),
"empty mob should yield no member snapshots"
);
}
#[cfg(feature = "mob")]
#[tokio::test]
async fn test_mob_wait_kickoff_route_respects_member_filter() {
use axum::body::Body;
use http_body_util::BodyExt;
use tower::ServiceExt;
let temp = TempDir::new().unwrap();
let state = AppState::load_from(temp.path().to_path_buf())
.await
.unwrap();
let definition = meerkat_mob::MobDefinition::from_toml(
"[mob]\nid = \"test_mob\"\n\n[profiles.worker]\nmodel = \"claude-sonnet-4-6\"\n\n[profiles.worker.tools]\ncomms = true\n",
)
.expect("minimal mob definition");
let mob_id = state
.mob_state
.mob_create_definition(definition)
.await
.expect("create mob");
let app = router(state);
let body = serde_json::json!({
"member_ids": ["lead-filter"],
"timeout_ms": 10_000
});
let request = axum::http::Request::builder()
.method("POST")
.uri(format!("/mob/{mob_id}/wait-kickoff"))
.header("content-type", "application/json")
.body(Body::from(body.to_string()))
.unwrap();
let response = app.oneshot(request).await.unwrap();
assert_eq!(response.status(), StatusCode::OK);
let body_bytes = response.into_body().collect().await.unwrap().to_bytes();
let payload: serde_json::Value = serde_json::from_slice(&body_bytes).unwrap();
let members = payload["members"]
.as_array()
.expect("members should be an array");
assert_eq!(members.len(), 1);
assert_eq!(members[0]["agent_identity"], "lead-filter");
assert_eq!(members[0]["status"], "unknown");
}
#[cfg(feature = "mob")]
#[tokio::test]
async fn test_mob_wire_members_batch_route_returns_batch_report() {
use axum::body::Body;
use http_body_util::BodyExt;
use tower::ServiceExt;
let temp = TempDir::new().unwrap();
let mut state = AppState::load_from(temp.path().to_path_buf())
.await
.unwrap();
let mock_client: Arc<dyn LlmClient> = Arc::new(MockLlmClient);
state.mob_state = Arc::new(
meerkat_mob_mcp::MobMcpState::new(
state.session_service.clone(),
meerkat_mob::MobControlPrincipal::Owner,
)
.with_default_llm_client(Some(mock_client)),
);
let mut definition =
meerkat_mob::MobDefinition::explicit(meerkat_mob::MobId::from("test_mob"));
definition.profiles.insert(
meerkat_mob::ProfileName::from("worker"),
meerkat_mob::ProfileBinding::Inline(Box::new(meerkat_mob::Profile {
model: "claude-sonnet-4-5".to_string(),
provider: None,
self_hosted_server_id: None,
image_generation_provider: None,
auto_compact_threshold: None,
resume_overrides: Vec::new(),
skills: Vec::new(),
tools: meerkat_mob::ToolConfig {
comms: true,
..meerkat_mob::ToolConfig::default()
},
peer_description: "worker".to_string(),
external_addressable: false,
backend: None,
runtime_mode: meerkat_mob::MobRuntimeMode::TurnDriven,
max_inline_peer_notifications: None,
output_schema: None,
provider_params: None,
})),
);
let mob_id = state
.mob_state
.mob_create_definition(definition)
.await
.expect("create mob");
let handle = state
.mob_state
.handle_for(&mob_id)
.await
.expect("mob handle");
for identity in ["w-1", "w-2", "w-3"] {
handle
.spawn_spec(meerkat_mob::SpawnMemberSpec::new("worker", identity))
.await
.expect("spawn member");
}
let app = router(state);
let body = serde_json::json!({
"edges": [
{ "a": "w-1", "b": "w-2" },
{ "a": "w-2", "b": "w-1" },
{ "a": "w-1", "b": "w-3" }
]
});
let request = axum::http::Request::builder()
.method("POST")
.uri(format!("/mob/{mob_id}/wire-members-batch"))
.header("content-type", "application/json")
.body(Body::from(body.to_string()))
.unwrap();
let response = app.clone().oneshot(request).await.unwrap();
let status = response.status();
let body_bytes = response.into_body().collect().await.unwrap().to_bytes();
assert_eq!(
status,
StatusCode::OK,
"wire members batch route failed: {}",
String::from_utf8_lossy(&body_bytes)
);
let payload: serde_json::Value = serde_json::from_slice(&body_bytes).unwrap();
assert_eq!(payload["requested"], 3);
assert_eq!(
batch_edge_pairs(&payload, "wired"),
[
("w-1".to_string(), "w-2".to_string()),
("w-1".to_string(), "w-3".to_string())
]
.into_iter()
.collect()
);
assert!(batch_edge_pairs(&payload, "already_wired").is_empty());
let repeat = axum::http::Request::builder()
.method("POST")
.uri(format!("/mob/{mob_id}/wire-members-batch"))
.header("content-type", "application/json")
.body(Body::from(body.to_string()))
.unwrap();
let repeat_response = app.oneshot(repeat).await.unwrap();
assert_eq!(repeat_response.status(), StatusCode::OK);
let repeat_body = repeat_response
.into_body()
.collect()
.await
.unwrap()
.to_bytes();
let repeat_payload: serde_json::Value = serde_json::from_slice(&repeat_body).unwrap();
assert_eq!(repeat_payload["requested"], 3);
assert_eq!(
batch_edge_pairs(&repeat_payload, "already_wired"),
[
("w-1".to_string(), "w-2".to_string()),
("w-1".to_string(), "w-3".to_string())
]
.into_iter()
.collect()
);
assert!(batch_edge_pairs(&repeat_payload, "wired").is_empty());
}
#[cfg(feature = "mob")]
fn batch_edge_pairs(
payload: &serde_json::Value,
key: &str,
) -> std::collections::BTreeSet<(String, String)> {
payload[key]
.as_array()
.expect("edge array")
.iter()
.map(|edge| {
(
edge["a"].as_str().expect("edge a").to_string(),
edge["b"].as_str().expect("edge b").to_string(),
)
})
.collect()
}
#[cfg(feature = "mob")]
#[tokio::test]
#[ignore = "requires ANTHROPIC_API_KEY; run with cargo e2e-system"]
async fn test_mob_spawn_helper_route_returns_identity_native_fields() {
use axum::body::Body;
use http_body_util::BodyExt;
use tower::ServiceExt;
let temp = TempDir::new().unwrap();
let mut state = AppState::load_from(temp.path().to_path_buf())
.await
.unwrap();
state.llm_client_override = Some(Arc::new(MockLlmClient));
let definition = meerkat_mob::MobDefinition::from_toml(
"[mob]\nid = \"test_mob\"\n\n[profiles.worker]\nmodel = \"claude-sonnet-4-6\"\n\n[profiles.worker.tools]\nbuiltins = true\ncomms = true\n",
)
.expect("minimal mob definition");
let mob_id = state
.mob_state
.mob_create_definition(definition)
.await
.expect("create mob");
let app = router(state);
let request = axum::http::Request::builder()
.method("POST")
.uri(format!("/mob/{mob_id}/spawn-helper"))
.header("content-type", "application/json")
.body(Body::from(
serde_json::json!({
"prompt": "Hello from helper",
"agent_identity": "helper-rest",
"role_name": "worker",
})
.to_string(),
))
.unwrap();
let response = app.oneshot(request).await.unwrap();
let status = response.status();
let body = response.into_body().collect().await.unwrap().to_bytes();
assert_eq!(
status,
StatusCode::OK,
"spawn helper route failed: {}",
String::from_utf8_lossy(&body)
);
let payload: serde_json::Value = serde_json::from_slice(&body).unwrap();
assert_eq!(payload["agent_identity"], "helper-rest");
assert!(
payload["member_ref"]
.as_str()
.is_some_and(|s| !s.is_empty()),
"member_ref must be populated"
);
assert!(
payload.get("agent_runtime_id").is_none(),
"binding-era agent_runtime_id must not leak to app-facing responses"
);
}
#[cfg(feature = "mob")]
#[tokio::test]
async fn test_mob_spawn_helper_without_agent_identity_fails_closed() {
use axum::body::Body;
use http_body_util::BodyExt;
use tower::ServiceExt;
let temp = TempDir::new().unwrap();
let state = AppState::load_from(temp.path().to_path_buf())
.await
.unwrap();
let app = router(state);
let request = axum::http::Request::builder()
.method("POST")
.uri("/mob/some-mob/spawn-helper")
.header("content-type", "application/json")
.body(Body::from(
serde_json::json!({ "prompt": "Hello from helper" }).to_string(),
))
.unwrap();
let response = app.oneshot(request).await.unwrap();
let status = response.status();
let body = response.into_body().collect().await.unwrap().to_bytes();
assert_eq!(
status,
StatusCode::BAD_REQUEST,
"missing agent_identity must be a 400, got: {}",
String::from_utf8_lossy(&body)
);
assert!(
String::from_utf8_lossy(&body).contains("agent_identity"),
"error must name the missing field"
);
}
#[cfg(feature = "mcp")]
mod mcp_tests {
use super::*;
use axum::body::Body;
use http_body_util::BodyExt;
use tower::ServiceExt;
async fn make_test_state() -> (AppState, TempDir) {
let temp = TempDir::new().unwrap();
let state = AppState::load_from(temp.path().to_path_buf())
.await
.unwrap();
(state, temp)
}
#[tokio::test]
async fn test_mcp_nonexistent_session_404() {
let (state, _temp) = make_test_state().await;
let fake_id = SessionId::new();
let app = router(state);
let body = serde_json::json!({
"session_id": fake_id.to_string(),
"server_config": {"name": "test-server", "command": "echo", "args": ["hello"]}
});
let request = axum::http::Request::builder()
.method("POST")
.uri(format!("/sessions/{fake_id}/mcp/add"))
.header("content-type", "application/json")
.body(Body::from(serde_json::to_string(&body).unwrap()))
.unwrap();
let response = app.oneshot(request).await.unwrap();
assert_eq!(response.status(), StatusCode::NOT_FOUND);
}
#[tokio::test]
async fn test_mcp_add_empty_server_name_400() {
let (state, _temp) = make_test_state().await;
let session_id = SessionId::new();
{
let mut map = state.mcp_sessions.write().await;
let adapter = Arc::new(McpRouterAdapter::new(McpRouter::new()));
let (tx, rx) = mpsc::unbounded_channel();
map.insert(
session_id.clone(),
SessionMcpState {
adapter,
turn_counter: 0,
lifecycle_tx: tx,
lifecycle_rx: rx,
drain_task_running: Arc::new(AtomicBool::new(false)),
},
);
}
let app = router(state);
let body = serde_json::json!({
"session_id": session_id.to_string(),
"server_config": {"name": " ", "command": "echo"}
});
let request = axum::http::Request::builder()
.method("POST")
.uri(format!("/sessions/{session_id}/mcp/add"))
.header("content-type", "application/json")
.body(Body::from(serde_json::to_string(&body).unwrap()))
.unwrap();
let response = app.oneshot(request).await.unwrap();
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
let body_bytes = response.into_body().collect().await.unwrap().to_bytes();
let error: serde_json::Value = serde_json::from_slice(&body_bytes).unwrap();
assert!(
error["error"]
.as_str()
.unwrap()
.contains("server_name cannot be empty")
);
}
#[tokio::test]
async fn test_mcp_routes_registered() {
let (state, _temp) = make_test_state().await;
let fake_id = SessionId::new();
let app = router(state);
let body = serde_json::json!({
"session_id": fake_id.to_string(),
"server_config": {"name": "srv", "command": "echo"}
});
let request = axum::http::Request::builder()
.method("POST")
.uri(format!("/sessions/{fake_id}/mcp/add"))
.header("content-type", "application/json")
.body(Body::from(serde_json::to_string(&body).unwrap()))
.unwrap();
let response = app.oneshot(request).await.unwrap();
assert_eq!(response.status(), StatusCode::NOT_FOUND);
let body_bytes = response.into_body().collect().await.unwrap().to_bytes();
let error: serde_json::Value = serde_json::from_slice(&body_bytes).unwrap();
assert_eq!(error["code"].as_str().unwrap(), "NOT_FOUND");
}
#[tokio::test]
async fn test_mcp_response_shape() {
let resp = meerkat::surface::mcp_live_response(
"sid_123".to_string(),
meerkat_contracts::McpLiveOperation::Add,
Some("test-server".to_string()),
false,
);
assert_eq!(resp.session_id, "sid_123");
assert_eq!(resp.operation, meerkat_contracts::McpLiveOperation::Add);
assert_eq!(resp.server_name, Some("test-server".to_string()));
assert_eq!(resp.status, meerkat_contracts::McpLiveOpStatus::Staged);
assert!(!resp.persisted);
assert!(resp.applied_at_turn.is_none());
}
#[test]
fn test_mcp_response_shape_tracks_persisted_flag() {
let resp = meerkat::surface::mcp_live_response(
"sid_123".to_string(),
meerkat_contracts::McpLiveOperation::Add,
Some("test-server".to_string()),
true,
);
assert!(resp.persisted);
}
#[test]
fn test_conflict_error_is_409() {
let err = ApiError::Conflict("test conflict".to_string());
let response = err.into_response();
assert_eq!(response.status(), StatusCode::CONFLICT);
}
#[tokio::test]
async fn test_duplicate_input_error_returns_409_with_existing_id() {
let err = ApiError::DuplicateInput {
existing_id: "input-abc-123".to_string(),
};
let response = err.into_response();
assert_eq!(response.status(), StatusCode::CONFLICT);
let body = axum::body::to_bytes(response.into_body(), usize::MAX)
.await
.unwrap();
let json: serde_json::Value = serde_json::from_slice(&body).unwrap();
assert_eq!(json["error"], "duplicate_input");
assert_eq!(json["code"], "DUPLICATE_INPUT");
assert_eq!(json["existing_id"], "input-abc-123");
}
}
mod request_cancel_tests {
use super::*;
use axum::body::Body;
use http_body_util::BodyExt;
use tower::ServiceExt;
#[tokio::test]
async fn test_missing_header_works_normally() {
let temp = TempDir::new().unwrap();
let mut state = AppState::load_from(temp.path().to_path_buf())
.await
.unwrap();
state.llm_client_override = Some(Arc::new(MockLlmClient));
let app = router(state);
let response = tokio::time::timeout(
std::time::Duration::from_secs(10),
app.oneshot(
axum::http::Request::builder()
.method("POST")
.uri("/sessions")
.header("content-type", "application/json")
.body(Body::from(
serde_json::json!({"prompt": "Hello"}).to_string(),
))
.unwrap(),
),
)
.await
.expect("should not timeout")
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
}
#[tokio::test]
async fn test_empty_request_id_returns_400() {
let temp = TempDir::new().unwrap();
let mut state = AppState::load_from(temp.path().to_path_buf())
.await
.unwrap();
state.llm_client_override = Some(Arc::new(MockLlmClient));
let app = router(state);
let response = app
.oneshot(
axum::http::Request::builder()
.method("POST")
.uri("/sessions")
.header("content-type", "application/json")
.header("x-meerkat-request-id", "")
.body(Body::from(
serde_json::json!({"prompt": "Hello"}).to_string(),
))
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
let body = response.into_body().collect().await.unwrap().to_bytes();
let payload: serde_json::Value = serde_json::from_slice(&body).unwrap();
assert!(
payload["error"]
.as_str()
.unwrap()
.contains("must not be empty")
);
}
#[tokio::test]
async fn test_cancel_unknown_returns_404() {
let temp = TempDir::new().unwrap();
let state = AppState::load_from(temp.path().to_path_buf())
.await
.unwrap();
let app = router(state);
let response = app
.oneshot(
axum::http::Request::builder()
.method("POST")
.uri("/requests/nonexistent-req/cancel")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::NOT_FOUND);
}
#[tokio::test]
async fn test_interrupt_unknown_session_returns_404() {
use axum::body::Body;
use tower::ServiceExt;
let temp = TempDir::new().unwrap();
let state = AppState::load_from(temp.path().to_path_buf())
.await
.unwrap();
let app = router(state);
let unknown_session_id = SessionId::new();
let response = app
.oneshot(
axum::http::Request::builder()
.method("POST")
.uri(format!("/sessions/{unknown_session_id}/interrupt"))
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::NOT_FOUND);
}
#[tokio::test]
async fn test_interrupt_service_owned_idle_session_returns_ok() {
use axum::body::Body;
use http_body_util::BodyExt;
use tower::ServiceExt;
let temp = TempDir::new().unwrap();
let mut state = AppState::load_from(temp.path().to_path_buf())
.await
.unwrap();
state.llm_client_override = Some(Arc::new(MockLlmClient));
let pre_session = Session::new();
let bindings = state
.runtime_adapter
.prepare_bindings(pre_session.id().clone())
.await
.expect("runtime bindings should prepare");
let created = state
.session_service
.create_session(SvcCreateSessionRequest {
injected_context: Vec::new(),
model: resolved_default_model(&state).await,
prompt: "Hello".to_string().into(),
system_prompt: meerkat::SystemPromptOverride::Inherit,
max_tokens: Some(state.max_tokens),
event_tx: None,
initial_turn: InitialTurnPolicy::Defer,
deferred_prompt_policy: DeferredPromptPolicy::Discard,
build: Some(SessionBuildOptions {
custom_models: std::collections::BTreeMap::new(),
image_generation_provider: None,
auto_compact_threshold_override: None,
resume_session: Some(pre_session),
llm_client_override: state
.llm_client_override
.clone()
.map(encode_llm_client_override_for_service),
runtime_build_mode: meerkat_core::RuntimeBuildMode::SessionOwned(bindings),
..Default::default()
}),
labels: None,
})
.await
.expect("service-owned idle session should be created");
let app = router(state);
let response = app
.oneshot(
axum::http::Request::builder()
.method("POST")
.uri(format!("/sessions/{}/interrupt", created.session_id))
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
let status = response.status();
let body = response.into_body().collect().await.unwrap().to_bytes();
assert_eq!(
status,
StatusCode::OK,
"interrupt response body: {}",
String::from_utf8_lossy(&body)
);
let payload: serde_json::Value = serde_json::from_slice(&body).unwrap();
assert_eq!(payload["interrupted"], true);
}
#[tokio::test]
async fn test_interrupt_cold_persisted_stopped_projection_is_noop_when_session_exists() {
use axum::body::Body;
use http_body_util::BodyExt;
use tower::ServiceExt;
let temp = TempDir::new().unwrap();
let mut state = AppState::load_from(temp.path().to_path_buf())
.await
.unwrap();
state.llm_client_override = Some(Arc::new(MockLlmClient));
let pre_session = Session::new();
let bindings = state
.runtime_adapter
.prepare_bindings(pre_session.id().clone())
.await
.expect("runtime bindings should prepare");
let created = state
.session_service
.create_session(SvcCreateSessionRequest {
injected_context: Vec::new(),
model: resolved_default_model(&state).await,
prompt: "Hello".to_string().into(),
system_prompt: meerkat::SystemPromptOverride::Inherit,
max_tokens: Some(state.max_tokens),
event_tx: None,
initial_turn: InitialTurnPolicy::Defer,
deferred_prompt_policy: DeferredPromptPolicy::Discard,
build: Some(SessionBuildOptions {
custom_models: std::collections::BTreeMap::new(),
image_generation_provider: None,
auto_compact_threshold_override: None,
resume_session: Some(pre_session),
llm_client_override: state
.llm_client_override
.clone()
.map(encode_llm_client_override_for_service),
runtime_build_mode: meerkat_core::RuntimeBuildMode::SessionOwned(bindings),
..Default::default()
}),
labels: None,
})
.await
.expect("service-owned idle session should be created");
state
.runtime_adapter
.register_session(created.session_id.clone())
.await
.expect("register session");
state
.runtime_adapter
.stop_runtime_executor(&created.session_id, "seed stopped projection")
.await
.expect("runtime state should persist");
assert_eq!(
state
.session_service
.persisted_runtime_state(&created.session_id)
.await
.expect("runtime-state projection load should succeed"),
Some(meerkat_runtime::RuntimeState::Stopped)
);
state.runtime_adapter = Arc::new(meerkat_runtime::MeerkatMachine::persistent(
state.session_service.runtime_store(),
state.session_service.blob_store(),
));
assert!(
!state
.runtime_adapter
.contains_session(&created.session_id)
.await,
"cold REST runtime must not inherit process-local registration"
);
let app = router(state);
let response = app
.oneshot(
axum::http::Request::builder()
.method("POST")
.uri(format!("/sessions/{}/interrupt", created.session_id))
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
let status = response.status();
let body = response.into_body().collect().await.unwrap().to_bytes();
assert_eq!(status, StatusCode::OK);
let payload: serde_json::Value = serde_json::from_slice(&body).unwrap();
assert_eq!(payload["interrupted"], true);
}
#[tokio::test]
async fn test_publish_terminal_after_cancel_returns_request_cancelled() {
let executor = SurfaceRequestExecutor::new(std::time::Duration::from_millis(1));
let ctx = executor.begin_request("rest-cancel-before-publish", noop_request_action());
assert_eq!(
executor.cancel_request(ctx.key()).await,
meerkat::surface::CancelOutcome::Cancelled
);
let result = with_request_lifecycle(
&executor,
Some(ctx),
RequestTerminal::Publish(Err(ApiError::Internal(
"publish response must not leak after cancel".to_string(),
))),
)
.await;
assert!(matches!(
result,
Err(ApiError::RequestCancelled { details: None })
));
assert_eq!(executor.phase("rest-cancel-before-publish"), None);
}
#[tokio::test]
async fn test_unpublished_success_after_cancel_returns_request_cancelled() {
let executor = SurfaceRequestExecutor::new(std::time::Duration::from_millis(1));
let ctx = executor.begin_request("rest-cancel-before-success", noop_request_action());
let realm = meerkat_core::RealmId::parse("rest-cancel").unwrap();
let response = run_result_to_response(
meerkat_core::types::RunResult {
text: "must not leak".to_string(),
session_id: SessionId::new(),
usage: meerkat_core::types::Usage::default(),
turns: 1,
tool_calls: 0,
terminal_cause_kind: None,
structured_output: None,
extraction_error: None,
schema_warnings: None,
skill_diagnostics: None,
},
&realm,
);
assert_eq!(
executor.cancel_request(ctx.key()).await,
meerkat::surface::CancelOutcome::Cancelled
);
let result = with_request_lifecycle(
&executor,
Some(ctx),
RequestTerminal::RespondWithoutPublish(Ok(Json(response))),
)
.await;
assert!(matches!(
result,
Err(ApiError::RequestCancelled { details: None })
));
assert_eq!(executor.phase("rest-cancel-before-success"), None);
}
#[tokio::test]
async fn test_missing_runtime_terminal_evidence_fails_closed() {
let temp = TempDir::new().unwrap();
let state = load_rest_state_with_capacity(&temp, 1).await;
let session_id = SessionId::new();
let input_id = meerkat_core::lifecycle::InputId::new();
let existing_id = meerkat_core::lifecycle::InputId::new();
let realm = meerkat_core::RealmId::parse("rest-missing-terminal").unwrap();
let outcome = meerkat_runtime::AcceptOutcome::Deduplicated {
input_id,
existing_id,
existing_seed: meerkat_runtime::input_state::InputStateSeed::new_accepted(),
};
let result = completion_handle_to_api_result(
"continue",
&state,
&outcome,
None,
&session_id,
&realm,
false,
)
.await;
match result {
Err(ApiError::InternalWithData {
message,
code,
details,
}) => {
assert_eq!(code, "MISSING_RUNTIME_TERMINAL_EVIDENCE");
assert!(
message.contains("without completion terminal evidence"),
"unexpected message: {message}"
);
assert_eq!(details["authority"], "runtime/session");
assert_eq!(details["session_created"], false);
}
other => panic!("expected typed missing terminal evidence error, got {other:?}"),
}
}
#[derive(Clone)]
struct RequestLifecycleProbeState {
executor: SurfaceRequestExecutor,
}
async fn publish_after_cancel_probe(
State(state): State<RequestLifecycleProbeState>,
headers: axum::http::HeaderMap,
) -> Result<Json<SessionResponse>, ApiError> {
let ctx = extract_request_context(&headers, &state.executor)?;
if let Some(ctx) = ctx.as_ref() {
let _ = state.executor.cancel_request(ctx.key()).await;
}
with_request_lifecycle(
&state.executor,
ctx,
RequestTerminal::Publish(Err(ApiError::Internal(
"publish response must not cross HTTP after cancel".to_string(),
))),
)
.await
}
#[tokio::test]
async fn test_publish_terminal_after_cancel_returns_http_499() {
let app = Router::new()
.route("/probe", post(publish_after_cancel_probe))
.with_state(RequestLifecycleProbeState {
executor: SurfaceRequestExecutor::new(std::time::Duration::from_millis(1)),
});
let response = app
.oneshot(
axum::http::Request::builder()
.method("POST")
.uri("/probe")
.header("x-meerkat-request-id", "rest-http-cancel-before-publish")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(
response.status(),
StatusCode::from_u16(499).expect("499 should be a valid status")
);
let body = response.into_body().collect().await.unwrap().to_bytes();
let payload: serde_json::Value = serde_json::from_slice(&body).unwrap();
assert_eq!(payload["code"], "REQUEST_CANCELLED");
}
}
#[cfg(not(feature = "mcp"))]
mod mcp_feature_off_tests {
use super::*;
use axum::body::Body;
use tower::ServiceExt;
#[tokio::test]
async fn test_mcp_routes_not_registered_without_feature() {
let temp = TempDir::new().unwrap();
let state = AppState::load_from(temp.path().to_path_buf())
.await
.unwrap();
let app = router(state);
let body = serde_json::json!({
"session_id": "fake",
"server_config": {"name": "srv", "command": "echo"}
});
let request = axum::http::Request::builder()
.method("POST")
.uri("/sessions/fake/mcp/add")
.header("content-type", "application/json")
.body(Body::from(serde_json::to_string(&body).unwrap()))
.unwrap();
let response = app.oneshot(request).await.unwrap();
assert_ne!(response.status(), StatusCode::OK);
}
}
mod config_fault_propagation_tests {
use super::*;
use axum::body::Body;
use http_body_util::BodyExt;
use tower::ServiceExt;
struct FailingConfigStore;
#[async_trait]
impl ConfigStore for FailingConfigStore {
async fn get(&self) -> Result<Config, meerkat_core::ConfigError> {
Err(meerkat_core::ConfigError::Validation("boom".to_string()))
}
async fn set(&self, _config: Config) -> Result<(), meerkat_core::ConfigError> {
Err(meerkat_core::ConfigError::Validation("boom".to_string()))
}
async fn patch(
&self,
_delta: ConfigDelta,
) -> Result<Config, meerkat_core::ConfigError> {
Err(meerkat_core::ConfigError::Validation("boom".to_string()))
}
}
async fn assert_config_fault_endpoint(uri: &str) {
let temp = TempDir::new().unwrap();
let mut state = AppState::load_from(temp.path().to_path_buf())
.await
.unwrap();
state.config_store = Arc::new(FailingConfigStore);
let app = router(state);
let request = axum::http::Request::builder()
.method("GET")
.uri(uri)
.body(Body::empty())
.unwrap();
let response = app.oneshot(request).await.unwrap();
let status = response.status();
let body = response.into_body().collect().await.unwrap().to_bytes();
assert_eq!(
status,
StatusCode::INTERNAL_SERVER_ERROR,
"{uri} must return 500 on a config fault, got body: {}",
String::from_utf8_lossy(&body)
);
let payload: Value = serde_json::from_slice(&body).unwrap();
assert_eq!(
payload["code"], "CONFIGURATION_ERROR",
"{uri} must surface CONFIGURATION_ERROR on a config fault: {payload}"
);
}
#[tokio::test]
async fn capabilities_endpoint_propagates_config_fault() {
assert_config_fault_endpoint("/capabilities").await;
}
#[tokio::test]
async fn models_catalog_endpoint_propagates_config_fault() {
assert_config_fault_endpoint("/models/catalog").await;
}
}
}