use std::path::PathBuf;
use std::sync::Arc;
use std::time::Duration;
use axum::Router;
use meerkat::surface::ScheduleHostHandle;
use meerkat::{
PersistentSessionService, ScheduleRunnableHost, ScheduleService, SessionAgentBuilder,
WorkGraphService,
};
use meerkat_mob_mcp::MobMcpState;
use meerkat_runtime::MeerkatMachine;
use crate::mob_handle_runtime::MobBootstrapSpec;
use crate::runtime::cross_mob_control::ControlListenAddr;
use crate::runtime::{InMemoryMetadataStore, PersistentMetadataStore, RuntimeOptions};
use crate::schedule_wiring::{
ScheduleClaimWatchdogConfig, ScheduleFiringHostBinding, ScheduleMobTargetRegistry,
};
use crate::types::{EventEnvelope, MobKitConfig, UnifiedEvent};
use crate::unified_runtime::{
UnifiedRuntime, UnifiedRuntimeBootstrapError, UnifiedRuntimeShutdownReport,
};
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum GatewayCompatibilityProfile {
ConsoleHttp,
StdioRpc,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct GatewayCompatibilityContract {
pub top_level_permissive_init: bool,
pub registry_resume: bool,
pub post_init_stdin_rpc: bool,
pub callback_plane: bool,
pub single_shot: bool,
pub maintenance_verbs: bool,
}
impl GatewayCompatibilityProfile {
pub const fn contract(self) -> GatewayCompatibilityContract {
match self {
Self::ConsoleHttp => GatewayCompatibilityContract {
top_level_permissive_init: true,
registry_resume: true,
post_init_stdin_rpc: false,
callback_plane: false,
single_shot: false,
maintenance_verbs: true,
},
Self::StdioRpc => GatewayCompatibilityContract {
top_level_permissive_init: false,
registry_resume: false,
post_init_stdin_rpc: true,
callback_plane: true,
single_shot: true,
maintenance_verbs: false,
},
}
}
}
pub struct GatewayRuntimeBootstrapPlan {
mob_spec: MobBootstrapSpec,
module_config: MobKitConfig,
module_agent_events: Vec<EventEnvelope<UnifiedEvent>>,
timeout: Duration,
runtime_options: RuntimeOptions,
persistent_metadata: Arc<dyn PersistentMetadataStore>,
}
impl GatewayRuntimeBootstrapPlan {
pub fn console_http(
mob_spec: MobBootstrapSpec,
module_config: MobKitConfig,
timeout: Duration,
) -> Self {
Self {
mob_spec,
module_config,
module_agent_events: Vec::new(),
timeout,
runtime_options: RuntimeOptions::default(),
persistent_metadata: Arc::new(InMemoryMetadataStore::new()),
}
}
pub fn stdio_rpc(
mob_spec: MobBootstrapSpec,
module_config: MobKitConfig,
module_agent_events: Vec<EventEnvelope<UnifiedEvent>>,
timeout: Duration,
runtime_options: RuntimeOptions,
persistent_metadata: Arc<dyn PersistentMetadataStore>,
) -> Self {
Self {
mob_spec,
module_config,
module_agent_events,
timeout,
runtime_options,
persistent_metadata,
}
}
}
pub struct PreparedGateway {
plan: GatewayRuntimeBootstrapPlan,
}
pub struct BootstrappedGateway {
runtime: UnifiedRuntime,
}
pub struct ActiveGateway {
runtime: Arc<UnifiedRuntime>,
}
pub struct GatewayComposition<State> {
profile: GatewayCompatibilityProfile,
state: State,
}
impl GatewayComposition<PreparedGateway> {
pub fn prepare(
profile: GatewayCompatibilityProfile,
plan: GatewayRuntimeBootstrapPlan,
) -> Self {
Self {
profile,
state: PreparedGateway { plan },
}
}
pub async fn bootstrap(
self,
) -> Result<GatewayComposition<BootstrappedGateway>, UnifiedRuntimeBootstrapError> {
let plan = self.state.plan;
let runtime = UnifiedRuntime::bootstrap_with_options(
plan.mob_spec,
plan.module_config,
plan.module_agent_events,
plan.timeout,
plan.runtime_options,
plan.persistent_metadata,
)
.await?;
Ok(GatewayComposition {
profile: self.profile,
state: BootstrappedGateway { runtime },
})
}
}
impl GatewayComposition<BootstrappedGateway> {
pub fn runtime(&self) -> &UnifiedRuntime {
&self.state.runtime
}
pub fn runtime_mut(&mut self) -> &mut UnifiedRuntime {
&mut self.state.runtime
}
pub fn activate(self) -> GatewayComposition<ActiveGateway> {
GatewayComposition {
profile: self.profile,
state: ActiveGateway {
runtime: Arc::new(self.state.runtime),
},
}
}
}
impl GatewayComposition<ActiveGateway> {
pub fn profile(&self) -> GatewayCompatibilityProfile {
self.profile
}
pub fn runtime(&self) -> &Arc<UnifiedRuntime> {
&self.state.runtime
}
}
pub const GATEWAY_HTTP_DRAIN_TIMEOUT: Duration = Duration::from_secs(5);
pub const GATEWAY_RUNTIME_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(310);
pub struct GatewayHttpBinding {
listener: tokio::net::TcpListener,
port: u16,
}
impl GatewayHttpBinding {
pub async fn bind_loopback() -> std::io::Result<Self> {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?;
let port = listener.local_addr()?.port();
Ok(Self { listener, port })
}
pub fn port(&self) -> u16 {
self.port
}
pub fn http_base_url(&self) -> String {
format!("http://127.0.0.1:{}", self.port)
}
pub fn serve(self, app: Router) -> GatewayHttpServer {
let (shutdown_tx, mut shutdown_rx) = tokio::sync::watch::channel(false);
let task = tokio::spawn(async move {
axum::serve(self.listener, app)
.with_graceful_shutdown(async move {
let _ = shutdown_rx.changed().await;
})
.await
});
GatewayHttpServer {
shutdown_tx,
task: Some(task),
}
}
}
pub struct GatewayHttpServer {
shutdown_tx: tokio::sync::watch::Sender<bool>,
task: Option<tokio::task::JoinHandle<std::io::Result<()>>>,
}
#[derive(Debug)]
pub enum GatewayHttpDrainOutcome {
Completed(std::io::Result<()>),
TimedOut,
JoinFailed(String),
}
impl GatewayHttpServer {
pub async fn wait(&mut self) -> GatewayHttpDrainOutcome {
let Some(task) = self.task.as_mut() else {
return GatewayHttpDrainOutcome::Completed(Ok(()));
};
let outcome = match task.await {
Ok(result) => GatewayHttpDrainOutcome::Completed(result),
Err(error) => GatewayHttpDrainOutcome::JoinFailed(error.to_string()),
};
self.task = None;
outcome
}
}
pub struct GatewayShutdownOutcome<Cleanup> {
pub http: GatewayHttpDrainOutcome,
pub runtime: Option<UnifiedRuntimeShutdownReport>,
pub cleanup: Cleanup,
}
async fn ordered_shutdown_tail<
BeforeRuntime,
BeforeFuture,
RuntimeFuture,
RuntimeOutput,
Cleanup,
CleanupFuture,
CleanupOutput,
>(
before_runtime: BeforeRuntime,
runtime_shutdown: RuntimeFuture,
cleanup: Cleanup,
) -> (RuntimeOutput, CleanupOutput)
where
BeforeRuntime: FnOnce() -> BeforeFuture,
BeforeFuture: std::future::Future<Output = ()>,
RuntimeFuture: std::future::Future<Output = RuntimeOutput>,
Cleanup: FnOnce() -> CleanupFuture,
CleanupFuture: std::future::Future<Output = CleanupOutput>,
{
before_runtime().await;
let runtime = runtime_shutdown.await;
let cleanup = cleanup().await;
(runtime, cleanup)
}
impl GatewayComposition<ActiveGateway> {
pub async fn shutdown<BeforeRuntime, BeforeFuture, Cleanup, CleanupFuture, CleanupOutput>(
&self,
mut server: GatewayHttpServer,
before_runtime: BeforeRuntime,
cleanup: Cleanup,
) -> GatewayShutdownOutcome<CleanupOutput>
where
BeforeRuntime: FnOnce() -> BeforeFuture,
BeforeFuture: std::future::Future<Output = ()>,
Cleanup: FnOnce() -> CleanupFuture,
CleanupFuture: std::future::Future<Output = CleanupOutput>,
{
let _ = server.shutdown_tx.send(true);
let http = if let Some(mut task) = server.task.take() {
match tokio::time::timeout(GATEWAY_HTTP_DRAIN_TIMEOUT, &mut task).await {
Ok(Ok(result)) => GatewayHttpDrainOutcome::Completed(result),
Ok(Err(error)) => GatewayHttpDrainOutcome::JoinFailed(error.to_string()),
Err(_) => {
task.abort();
let _ = task.await;
GatewayHttpDrainOutcome::TimedOut
}
}
} else {
GatewayHttpDrainOutcome::Completed(Ok(()))
};
let runtime_shutdown = async {
match tokio::time::timeout(
GATEWAY_RUNTIME_SHUTDOWN_TIMEOUT,
self.state.runtime.shutdown(),
)
.await
{
Ok(report) => {
if !report.cleanup_completed() {
tracing::warn!(
drain_timed_out = report.drain.timed_out,
mob_stop = ?report.mob_stop,
identity_authority_release = ?report.identity_authority_release,
orphan_processes = report.module_shutdown.orphan_processes,
"gateway runtime shutdown completed without cleanup attestation"
);
}
Some(report)
}
Err(_) => {
tracing::warn!(
timeout_ms = GATEWAY_RUNTIME_SHUTDOWN_TIMEOUT.as_millis(),
"gateway runtime shutdown exceeded its bounded horizon"
);
None
}
}
};
let (runtime, cleanup) =
ordered_shutdown_tail(before_runtime, runtime_shutdown, cleanup).await;
GatewayShutdownOutcome {
http,
runtime,
cleanup,
}
}
}
const GATEWAY_WORKER_STACK_BYTES: usize = 16 * 1024 * 1024;
pub fn gateway_tokio_runtime() -> std::io::Result<tokio::runtime::Runtime> {
tokio::runtime::Builder::new_multi_thread()
.enable_all()
.thread_stack_size(GATEWAY_WORKER_STACK_BYTES)
.build()
}
pub fn default_tracing_filter(binary_target: &str) -> String {
format!("warn,meerkat_mobkit=info,{binary_target}=info")
}
pub fn init_gateway_tracing(binary_target: &str) {
tracing_subscriber::fmt()
.with_env_filter(
tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| {
tracing_subscriber::EnvFilter::new(default_tracing_filter(binary_target))
}),
)
.with_writer(std::io::stderr)
.with_ansi(false)
.init();
}
pub fn parse_control_listen_arg(args: &[String]) -> Result<Option<ControlListenAddr>, String> {
let Some(position) = args.iter().position(|arg| arg == "--control-listen") else {
return Ok(None);
};
let Some(value) = args.get(position + 1) else {
return Err(
"--control-listen requires an address (tcp://host:port or uds:///path)".to_string(),
);
};
ControlListenAddr::parse(value)
.map(Some)
.map_err(|error| format!("--control-listen: {error}"))
}
pub struct GatewayScheduleHostInputs<B: SessionAgentBuilder + 'static> {
pub schedule_service: ScheduleService,
pub session_service: Arc<PersistentSessionService<B>>,
pub runtime_adapter: Arc<MeerkatMachine>,
pub schedule_store_path: PathBuf,
pub firing_host_binding: ScheduleFiringHostBinding,
pub runnable_host: Option<Arc<dyn ScheduleRunnableHost>>,
pub workgraph_service: Option<WorkGraphService>,
pub owner_id: String,
}
pub async fn adopt_schedule_mob_targets(
runtime: &UnifiedRuntime,
schedule_service: &ScheduleService,
mob_target_registry: &ScheduleMobTargetRegistry,
) -> Option<Arc<MobMcpState>> {
let mob_state = runtime.mob_runtime().agent_mob_mcp_state();
mob_target_registry.set_mob_state(mob_state.clone());
match crate::schedule_wiring::repair_resumable_session_targets_to_mob_members(
schedule_service,
mob_target_registry,
)
.await
{
Ok(repaired) if repaired > 0 => {
tracing::info!(
repaired,
"repaired persisted resumable-session schedules to identity mob targets"
);
}
Ok(_) => {}
Err(error) => {
tracing::warn!(
error = %error,
"failed to repair persisted resumable-session schedules to identity mob targets",
);
}
}
mob_state
}
pub async fn spawn_gateway_schedule_host<B: SessionAgentBuilder + 'static>(
runtime: &UnifiedRuntime,
mob_state: Option<Arc<MobMcpState>>,
inputs: GatewayScheduleHostInputs<B>,
) -> (Option<ScheduleHostHandle>, tokio::task::JoinHandle<()>) {
let GatewayScheduleHostInputs {
schedule_service,
session_service,
runtime_adapter,
schedule_store_path,
firing_host_binding,
runnable_host,
workgraph_service,
owner_id,
} = inputs;
let watchdog_config = ScheduleClaimWatchdogConfig::default();
tracing::info!(
poll_interval_secs = watchdog_config.poll_interval.as_secs(),
overdue_threshold_secs = watchdog_config.overdue_threshold.as_secs(),
heartbeat_polls = watchdog_config.heartbeat_polls,
"schedule claim watchdog resident: probes the firing pipeline on this cadence, ERROR on \
a new or changed stall report, WARN heartbeat while it persists, INFO on recovery"
);
let watchdog = crate::schedule_wiring::spawn_schedule_claim_watchdog(
schedule_service.clone(),
schedule_store_path.clone(),
watchdog_config,
);
let schedule_service_for_probe = schedule_service.clone();
let schedule_host = crate::schedule_wiring::spawn_schedule_host_with_identity_runtime(
session_service,
runtime_adapter,
schedule_service,
mob_state,
runtime.mob_handle(),
runtime.identity_runtime().cloned(),
runnable_host,
workgraph_service,
owner_id,
);
if schedule_host.is_some() {
firing_host_binding.bind();
match crate::schedule_wiring::probe_schedule_firing_pipeline(
&schedule_service_for_probe,
&schedule_store_path,
watchdog_config.overdue_threshold,
)
.await
{
crate::schedule_wiring::ScheduleFiringProbe::Healthy => {
tracing::info!("schedule firing pipeline healthy at boot");
}
crate::schedule_wiring::ScheduleFiringProbe::Stalled { report } => {
tracing::warn!(
%report,
poll_interval_secs = watchdog_config.poll_interval.as_secs(),
"schedule firing pipeline is not delivering at boot; the resident claim \
watchdog re-probes on its cadence and escalates to ERROR if this persists \
(a restart backlog clears on the first host ticks)"
);
}
}
} else {
tracing::warn!(
"schedule host did not spawn over the attached schedule store: no firing driver is \
running in this gateway, and the firing-intent write gate is consequently still \
closed, so create/update/resume are being refused rather than accepted durably"
);
match crate::schedule_wiring::observe_schedule_firing_authority(&schedule_service_for_probe)
.await
{
crate::schedule_wiring::ScheduleFiringAuthority::Held {
owner_id: holder_id,
fencing_token,
expires_in_secs,
} => tracing::warn!(
executor_owner_id = %holder_id,
fencing_token,
lease_expires_in_secs = expires_in_secs,
"the schedule store itself reports SOME process holding the realm's singular \
firing authority, so durable schedules may still be drained elsewhere; note the \
holder can also be this deployment's own crashed predecessor, whose lease stays \
live until it expires, so check the owner id and expiry above before concluding \
anything is actually draining. This gateway's firing-intent write gate stays \
closed either way: it gates on a LOCAL host"
),
crate::schedule_wiring::ScheduleFiringAuthority::Vacant => tracing::warn!(
"the schedule store itself reports its firing authority vacant AT THIS INSTANT: \
no process holds the executor lease. A peer gateway mid-restart is vacant only \
until its first tick, so this is proof of an unattended store only if it \
persists - the resident claim watchdog is what escalates once durable work \
actually goes unclaimed"
),
crate::schedule_wiring::ScheduleFiringAuthority::Unobservable { detail } => {
tracing::warn!(
%detail,
"the schedule store cannot report firing authority, so whether any other \
process drains this store is unknown from here"
);
}
}
}
(schedule_host, watchdog)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn compatibility_profiles_pin_the_two_existing_command_surfaces() {
assert_eq!(
GatewayCompatibilityProfile::ConsoleHttp.contract(),
GatewayCompatibilityContract {
top_level_permissive_init: true,
registry_resume: true,
post_init_stdin_rpc: false,
callback_plane: false,
single_shot: false,
maintenance_verbs: true,
}
);
assert_eq!(
GatewayCompatibilityProfile::StdioRpc.contract(),
GatewayCompatibilityContract {
top_level_permissive_init: false,
registry_resume: false,
post_init_stdin_rpc: true,
callback_plane: true,
single_shot: true,
maintenance_verbs: false,
}
);
}
#[tokio::test]
async fn shutdown_tail_orders_profile_quiesce_runtime_cleanup_then_registry_cleanup() {
let observed = Arc::new(std::sync::Mutex::new(Vec::new()));
let before_observed = Arc::clone(&observed);
let runtime_observed = Arc::clone(&observed);
let cleanup_observed = Arc::clone(&observed);
let (runtime, cleanup) = ordered_shutdown_tail(
move || async move {
before_observed
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.push("profile_quiesce");
},
async move {
runtime_observed
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.push("runtime_shutdown");
"runtime_report"
},
move || async move {
cleanup_observed
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.push("registry_cleanup");
"cleanup_report"
},
)
.await;
assert_eq!(runtime, "runtime_report");
assert_eq!(cleanup, "cleanup_report");
assert_eq!(
*observed
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner),
["profile_quiesce", "runtime_shutdown", "registry_cleanup"]
);
}
#[test]
fn default_tracing_filter_reproduces_both_binary_constants() {
assert_eq!(
default_tracing_filter("rpc_gateway"),
"warn,meerkat_mobkit=info,rpc_gateway=info"
);
assert_eq!(
default_tracing_filter("mobkit_gateway"),
"warn,meerkat_mobkit=info,mobkit_gateway=info"
);
}
#[test]
fn default_tracing_filter_always_keeps_the_crate_at_info() {
for target in ["rpc_gateway", "mobkit_gateway", "some_future_gateway"] {
let filter = default_tracing_filter(target);
assert!(
filter.contains("meerkat_mobkit=info"),
"filter for {target} dropped the crate's own INFO target: {filter}"
);
assert!(
filter.starts_with("warn,"),
"filter for {target} lost the dependency WARN default: {filter}"
);
}
}
#[test]
fn control_listen_arg_absent_is_none() -> Result<(), String> {
let args = vec!["--persistent".to_string()];
assert!(parse_control_listen_arg(&args)?.is_none());
Ok(())
}
#[test]
fn control_listen_arg_parses_tcp() -> Result<(), String> {
let args = vec![
"--persistent".to_string(),
"--control-listen".to_string(),
"tcp://127.0.0.1:0".to_string(),
];
assert!(parse_control_listen_arg(&args)?.is_some());
Ok(())
}
#[test]
fn control_listen_arg_without_value_is_an_error() {
let args = vec!["--control-listen".to_string()];
let error = parse_control_listen_arg(&args).err().unwrap_or_default();
assert!(
error.contains("requires an address"),
"unexpected error text: {error}"
);
}
#[test]
fn control_listen_arg_rejects_inproc_and_names_the_flag() {
let args = vec!["--control-listen".to_string(), "inproc".to_string()];
let error = parse_control_listen_arg(&args).err().unwrap_or_default();
assert!(
error.starts_with("--control-listen: "),
"refusal must name the flag: {error}"
);
}
}