use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::Duration;
use axum::Router;
use meerkat::surface::ScheduleHostHandle;
use meerkat::{
PersistentSessionService, ScheduleRunnableHost, ScheduleService, SessionAgentBuilder,
WorkGraphService,
};
use meerkat_mob_mcp::MobMcpState;
use meerkat_runtime::MeerkatMachine;
use crate::mob_handle_runtime::MobBootstrapSpec;
use crate::runtime::cross_mob_control::ControlListenAddr;
use crate::runtime::{
InMemoryMetadataStore, PersistentMetadataStore, RuntimeDecisionState, RuntimeOptions,
};
use crate::schedule_wiring::{
ScheduleClaimWatchdogConfig, ScheduleFiringHostBinding, ScheduleMobTargetRegistry,
};
use crate::types::{EventEnvelope, MobKitConfig, UnifiedEvent};
use crate::unified_runtime::{
UnifiedRuntime, UnifiedRuntimeBootstrapError, UnifiedRuntimeShutdownReport,
};
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum GatewayCompatibilityProfile {
ConsoleHttp,
StdioRpc,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct GatewayCompatibilityContract {
pub top_level_permissive_init: bool,
pub registry_resume: bool,
pub post_init_stdin_rpc: bool,
pub callback_plane: bool,
pub single_shot: bool,
pub maintenance_verbs: bool,
}
impl GatewayCompatibilityProfile {
pub const fn contract(self) -> GatewayCompatibilityContract {
match self {
Self::ConsoleHttp => GatewayCompatibilityContract {
top_level_permissive_init: true,
registry_resume: true,
post_init_stdin_rpc: false,
callback_plane: false,
single_shot: false,
maintenance_verbs: true,
},
Self::StdioRpc => GatewayCompatibilityContract {
top_level_permissive_init: false,
registry_resume: false,
post_init_stdin_rpc: true,
callback_plane: true,
single_shot: true,
maintenance_verbs: false,
},
}
}
}
pub struct GatewayRuntimeBootstrapPlan {
mob_spec: MobBootstrapSpec,
module_config: MobKitConfig,
module_agent_events: Vec<EventEnvelope<UnifiedEvent>>,
timeout: Duration,
runtime_options: RuntimeOptions,
persistent_metadata: Arc<dyn PersistentMetadataStore>,
}
impl GatewayRuntimeBootstrapPlan {
pub fn console_http(
mob_spec: MobBootstrapSpec,
module_config: MobKitConfig,
timeout: Duration,
) -> Self {
Self {
mob_spec,
module_config,
module_agent_events: Vec::new(),
timeout,
runtime_options: RuntimeOptions::default(),
persistent_metadata: Arc::new(InMemoryMetadataStore::new()),
}
}
pub fn stdio_rpc(
mob_spec: MobBootstrapSpec,
module_config: MobKitConfig,
module_agent_events: Vec<EventEnvelope<UnifiedEvent>>,
timeout: Duration,
runtime_options: RuntimeOptions,
persistent_metadata: Arc<dyn PersistentMetadataStore>,
) -> Self {
Self {
mob_spec,
module_config,
module_agent_events,
timeout,
runtime_options,
persistent_metadata,
}
}
}
pub struct PreparedGateway {
plan: GatewayRuntimeBootstrapPlan,
}
pub struct BootstrappedGateway {
runtime: UnifiedRuntime,
}
pub struct ActiveGateway {
runtime: Arc<UnifiedRuntime>,
}
pub struct GatewayComposition<State> {
profile: GatewayCompatibilityProfile,
state: State,
}
impl GatewayComposition<PreparedGateway> {
pub fn prepare(
profile: GatewayCompatibilityProfile,
plan: GatewayRuntimeBootstrapPlan,
) -> Self {
Self {
profile,
state: PreparedGateway { plan },
}
}
pub async fn bootstrap(
self,
) -> Result<GatewayComposition<BootstrappedGateway>, UnifiedRuntimeBootstrapError> {
let plan = self.state.plan;
let runtime = UnifiedRuntime::bootstrap_with_options(
plan.mob_spec,
plan.module_config,
plan.module_agent_events,
plan.timeout,
plan.runtime_options,
plan.persistent_metadata,
)
.await?;
Ok(GatewayComposition {
profile: self.profile,
state: BootstrappedGateway { runtime },
})
}
}
impl GatewayComposition<BootstrappedGateway> {
pub fn runtime(&self) -> &UnifiedRuntime {
&self.state.runtime
}
pub fn runtime_mut(&mut self) -> &mut UnifiedRuntime {
&mut self.state.runtime
}
pub fn activate(self) -> GatewayComposition<ActiveGateway> {
GatewayComposition {
profile: self.profile,
state: ActiveGateway {
runtime: Arc::new(self.state.runtime),
},
}
}
}
impl GatewayComposition<ActiveGateway> {
pub fn profile(&self) -> GatewayCompatibilityProfile {
self.profile
}
pub fn runtime(&self) -> &Arc<UnifiedRuntime> {
&self.state.runtime
}
}
pub const GATEWAY_HTTP_DRAIN_TIMEOUT: Duration = Duration::from_secs(5);
pub const GATEWAY_RUNTIME_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(312);
pub const DEFAULT_GATEWAY_HTTP_LISTEN: SocketAddr =
SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0);
pub struct GatewayHttpBinding {
listener: tokio::net::TcpListener,
local_addr: SocketAddr,
advertised_base_url: Option<String>,
}
impl GatewayHttpBinding {
pub async fn bind_loopback() -> std::io::Result<Self> {
Self::bind(DEFAULT_GATEWAY_HTTP_LISTEN).await
}
pub async fn bind(listen: SocketAddr) -> std::io::Result<Self> {
let listener = tokio::net::TcpListener::bind(listen).await?;
let local_addr = listener.local_addr()?;
Ok(Self {
listener,
local_addr,
advertised_base_url: None,
})
}
pub fn with_advertised_base_url(mut self, base_url: Option<String>) -> Self {
self.advertised_base_url = base_url.map(|url| url.trim_end_matches('/').to_string());
self
}
pub fn port(&self) -> u16 {
self.local_addr.port()
}
pub fn local_addr(&self) -> SocketAddr {
self.local_addr
}
pub fn reachable_addr(&self) -> SocketAddr {
loopback_reachable_addr(self.local_addr)
}
pub fn http_base_url(&self) -> String {
format!("http://{}", self.reachable_addr())
}
pub fn advertised_base_url(&self) -> Option<&str> {
self.advertised_base_url.as_deref()
}
pub fn serve(self, app: Router) -> GatewayHttpServer {
let (shutdown_tx, mut shutdown_rx) = tokio::sync::watch::channel(false);
let task = tokio::spawn(async move {
axum::serve(self.listener, app)
.with_graceful_shutdown(async move {
let _ = shutdown_rx.changed().await;
})
.await
});
GatewayHttpServer {
shutdown_tx,
task: Some(task),
}
}
}
pub fn loopback_reachable_addr(bound: SocketAddr) -> SocketAddr {
match bound.ip() {
IpAddr::V4(ip) if ip.is_unspecified() => {
SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), bound.port())
}
IpAddr::V6(ip) if ip.is_unspecified() => {
SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), bound.port())
}
_ => bound,
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct HttpBindPolicy {
pub allow_remote: bool,
}
impl HttpBindPolicy {
pub const fn local_only() -> Self {
Self {
allow_remote: false,
}
}
pub const fn allow_remote() -> Self {
Self { allow_remote: true }
}
pub fn for_gateway(allow_remote: bool, decisions: &RuntimeDecisionState) -> Self {
Self {
allow_remote: allow_remote || ConsoleAuthPosture::of(decisions).is_enforced(),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum GatewaySurface {
RpcGateway,
MobkitGateway,
}
impl GatewaySurface {
pub const fn name(self) -> &'static str {
match self {
Self::RpcGateway => "rpc_gateway",
Self::MobkitGateway => "mobkit_gateway",
}
}
}
impl std::fmt::Display for GatewaySurface {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(self.name())
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ConsoleAuthPosture {
Open,
ClosedToEveryCaller,
Enforced,
}
impl ConsoleAuthPosture {
pub fn of(decisions: &RuntimeDecisionState) -> Self {
if !decisions.console.require_app_auth {
return Self::Open;
}
if crate::parse_jwks_json(&decisions.trusted_oidc.jwks_json).is_ok() {
Self::Enforced
} else {
Self::ClosedToEveryCaller
}
}
pub const fn is_enforced(self) -> bool {
matches!(self, Self::Enforced)
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub enum HttpBindPolicyError {
RemoteBindRequiresExplicitAllow {
surface: GatewaySurface,
listen: SocketAddr,
},
}
impl std::fmt::Display for HttpBindPolicyError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::RemoteBindRequiresExplicitAllow { surface, listen } => write!(
f,
"{surface} HTTP listen address `{listen}` is not loopback. MobKit binds 127.0.0.1 \
unless the console enforces app auth (auth_config with a trusted signing key) or \
the launch acknowledges the exposure with allow_remote \
(runtime_options.allow_remote = true on rpc_gateway; the allow_remote init param, \
--allow-remote, or MOBKIT_HTTP_ALLOW_REMOTE=1 on mobkit_gateway). allow_remote is \
an exposure acknowledgement, not an auth mechanism: pass it only with an \
authenticating proxy in front of this listener"
),
}
}
}
impl std::error::Error for HttpBindPolicyError {}
pub fn validate_http_bind_policy(
surface: GatewaySurface,
listen: SocketAddr,
policy: HttpBindPolicy,
) -> Result<(), HttpBindPolicyError> {
if policy.allow_remote || listen.ip().is_loopback() {
return Ok(());
}
Err(HttpBindPolicyError::RemoteBindRequiresExplicitAllow { surface, listen })
}
pub fn warn_on_non_loopback_bind(
surface: GatewaySurface,
bound: SocketAddr,
decisions: &RuntimeDecisionState,
) {
if bound.ip().is_loopback() {
return;
}
let console_auth = match ConsoleAuthPosture::of(decisions) {
ConsoleAuthPosture::Enforced => "console app auth is enforced",
ConsoleAuthPosture::ClosedToEveryCaller => {
"the console requires app auth but trusts no key (refuses every caller)"
}
ConsoleAuthPosture::Open => "the console is OPEN (no app auth)",
};
tracing::warn!(
%bound,
"{surface} HTTP listener is bound to a NON-LOOPBACK address; {console_auth}; every route \
on this listener (console, JSON-RPC, blobs, SSE, live) is reachable by anyone who can \
reach {bound}. Bind 127.0.0.1 (the default) unless an authenticating proxy fronts this \
listener"
);
}
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub enum HttpExposureParseError {
ListenAddress {
value: String,
source: std::net::AddrParseError,
},
ListenFlagMissingValue,
ListenFlagAddress {
value: String,
source: std::net::AddrParseError,
},
PublicBaseUrlNotAbsoluteHttp { value: String },
PublicBaseUrlNoHost { value: String },
}
impl std::fmt::Display for HttpExposureParseError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
fn listen_address(
f: &mut std::fmt::Formatter<'_>,
value: &str,
source: &std::net::AddrParseError,
) -> std::fmt::Result {
write!(
f,
"`{value}` is not a HOST:PORT socket address ({source}); use an IP literal such \
as 127.0.0.1:0 or 0.0.0.0:8080"
)
}
match self {
Self::ListenAddress { value, source } => listen_address(f, value, source),
Self::ListenFlagMissingValue => f.write_str(
"--http-listen requires an address (HOST:PORT, e.g. 127.0.0.1:8080 or 0.0.0.0:8080)",
),
Self::ListenFlagAddress { value, source } => {
f.write_str("--http-listen: ")?;
listen_address(f, value, source)
}
Self::PublicBaseUrlNotAbsoluteHttp { value } => write!(
f,
"`{value}` must be an absolute http:// or https:// URL, e.g. https://mob.example.com"
),
Self::PublicBaseUrlNoHost { value } => write!(f, "`{value}` names no host"),
}
}
}
impl std::error::Error for HttpExposureParseError {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
match self {
Self::ListenAddress { source, .. } | Self::ListenFlagAddress { source, .. } => {
Some(source)
}
Self::ListenFlagMissingValue
| Self::PublicBaseUrlNotAbsoluteHttp { .. }
| Self::PublicBaseUrlNoHost { .. } => None,
}
}
}
pub fn parse_http_listen_addr(value: &str) -> Result<SocketAddr, HttpExposureParseError> {
value
.trim()
.parse::<SocketAddr>()
.map_err(|source| HttpExposureParseError::ListenAddress {
value: value.to_string(),
source,
})
}
pub fn parse_http_public_base_url(value: &str) -> Result<String, HttpExposureParseError> {
let trimmed = value.trim();
if !(trimmed.starts_with("http://") || trimmed.starts_with("https://")) {
return Err(HttpExposureParseError::PublicBaseUrlNotAbsoluteHttp {
value: value.to_string(),
});
}
let base = trimmed.trim_end_matches('/');
if base == "http:" || base == "https:" {
return Err(HttpExposureParseError::PublicBaseUrlNoHost {
value: value.to_string(),
});
}
Ok(base.to_string())
}
pub fn parse_http_listen_arg(
args: &[String],
) -> Result<Option<SocketAddr>, HttpExposureParseError> {
let Some(position) = args.iter().position(|arg| arg == "--http-listen") else {
return Ok(None);
};
let Some(value) = args.get(position + 1) else {
return Err(HttpExposureParseError::ListenFlagMissingValue);
};
value
.trim()
.parse::<SocketAddr>()
.map(Some)
.map_err(|source| HttpExposureParseError::ListenFlagAddress {
value: value.clone(),
source,
})
}
pub struct GatewayHttpServer {
shutdown_tx: tokio::sync::watch::Sender<bool>,
task: Option<tokio::task::JoinHandle<std::io::Result<()>>>,
}
#[derive(Debug)]
pub enum GatewayHttpDrainOutcome {
Completed(std::io::Result<()>),
TimedOut,
JoinFailed(String),
}
impl GatewayHttpServer {
pub async fn wait(&mut self) -> GatewayHttpDrainOutcome {
let Some(task) = self.task.as_mut() else {
return GatewayHttpDrainOutcome::Completed(Ok(()));
};
let outcome = match task.await {
Ok(result) => GatewayHttpDrainOutcome::Completed(result),
Err(error) => GatewayHttpDrainOutcome::JoinFailed(error.to_string()),
};
self.task = None;
outcome
}
}
pub struct GatewayShutdownOutcome<Cleanup> {
pub http: GatewayHttpDrainOutcome,
pub runtime: Option<UnifiedRuntimeShutdownReport>,
pub cleanup: Cleanup,
}
async fn ordered_shutdown_tail<
BeforeRuntime,
BeforeFuture,
RuntimeFuture,
RuntimeOutput,
Cleanup,
CleanupFuture,
CleanupOutput,
>(
before_runtime: BeforeRuntime,
runtime_shutdown: RuntimeFuture,
cleanup: Cleanup,
) -> (RuntimeOutput, CleanupOutput)
where
BeforeRuntime: FnOnce() -> BeforeFuture,
BeforeFuture: std::future::Future<Output = ()>,
RuntimeFuture: std::future::Future<Output = RuntimeOutput>,
Cleanup: FnOnce() -> CleanupFuture,
CleanupFuture: std::future::Future<Output = CleanupOutput>,
{
before_runtime().await;
let runtime = runtime_shutdown.await;
let cleanup = cleanup().await;
(runtime, cleanup)
}
impl GatewayComposition<ActiveGateway> {
pub async fn shutdown<BeforeRuntime, BeforeFuture, Cleanup, CleanupFuture, CleanupOutput>(
&self,
mut server: GatewayHttpServer,
before_runtime: BeforeRuntime,
cleanup: Cleanup,
) -> GatewayShutdownOutcome<CleanupOutput>
where
BeforeRuntime: FnOnce() -> BeforeFuture,
BeforeFuture: std::future::Future<Output = ()>,
Cleanup: FnOnce() -> CleanupFuture,
CleanupFuture: std::future::Future<Output = CleanupOutput>,
{
let _ = server.shutdown_tx.send(true);
let http = if let Some(mut task) = server.task.take() {
match tokio::time::timeout(GATEWAY_HTTP_DRAIN_TIMEOUT, &mut task).await {
Ok(Ok(result)) => GatewayHttpDrainOutcome::Completed(result),
Ok(Err(error)) => GatewayHttpDrainOutcome::JoinFailed(error.to_string()),
Err(_) => {
task.abort();
let _ = task.await;
GatewayHttpDrainOutcome::TimedOut
}
}
} else {
GatewayHttpDrainOutcome::Completed(Ok(()))
};
let runtime_shutdown = async {
match tokio::time::timeout(
GATEWAY_RUNTIME_SHUTDOWN_TIMEOUT,
self.state.runtime.shutdown(),
)
.await
{
Ok(report) => {
if !report.cleanup_completed() {
tracing::warn!(
drain_timed_out = report.drain.timed_out,
mob_stop = ?report.mob_stop,
identity_authority_release = ?report.identity_authority_release,
orphan_processes = report.module_shutdown.orphan_processes,
"gateway runtime shutdown completed without cleanup attestation"
);
}
Some(report)
}
Err(_) => {
tracing::warn!(
timeout_ms = GATEWAY_RUNTIME_SHUTDOWN_TIMEOUT.as_millis(),
"gateway runtime shutdown exceeded its bounded horizon"
);
None
}
}
};
let (runtime, cleanup) =
ordered_shutdown_tail(before_runtime, runtime_shutdown, cleanup).await;
GatewayShutdownOutcome {
http,
runtime,
cleanup,
}
}
}
const GATEWAY_WORKER_STACK_BYTES: usize = 16 * 1024 * 1024;
pub fn gateway_tokio_runtime() -> std::io::Result<tokio::runtime::Runtime> {
tokio::runtime::Builder::new_multi_thread()
.enable_all()
.thread_stack_size(GATEWAY_WORKER_STACK_BYTES)
.build()
}
pub fn default_tracing_filter(binary_target: &str) -> String {
format!("warn,meerkat_mobkit=info,{binary_target}=info")
}
pub fn init_gateway_tracing(binary_target: &str) {
tracing_subscriber::fmt()
.with_env_filter(
tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| {
tracing_subscriber::EnvFilter::new(default_tracing_filter(binary_target))
}),
)
.with_writer(std::io::stderr)
.with_ansi(false)
.init();
install_gateway_panic_hook();
}
pub fn install_gateway_panic_hook() {
let previous = std::panic::take_hook();
std::panic::set_hook(Box::new(move |info| {
log_panic(info);
previous(info);
}));
}
fn log_panic(info: &std::panic::PanicHookInfo<'_>) {
let payload = panic_payload_text(info.payload());
let current = std::thread::current();
let thread = current.name().unwrap_or("<unnamed>");
match info.location() {
Some(location) => tracing::error!(
thread,
file = location.file(),
line = location.line(),
column = location.column(),
payload,
"panic"
),
None => tracing::error!(thread, payload, "panic (location unavailable)"),
}
}
fn panic_payload_text(payload: &(dyn std::any::Any + Send)) -> &str {
if let Some(text) = payload.downcast_ref::<&str>() {
text
} else if let Some(text) = payload.downcast_ref::<String>() {
text.as_str()
} else {
"<non-string panic payload>"
}
}
pub fn parse_control_listen_arg(args: &[String]) -> Result<Option<ControlListenAddr>, String> {
let Some(position) = args.iter().position(|arg| arg == "--control-listen") else {
return Ok(None);
};
let Some(value) = args.get(position + 1) else {
return Err(
"--control-listen requires an address (tcp://host:port or uds:///path)".to_string(),
);
};
ControlListenAddr::parse(value)
.map(Some)
.map_err(|error| format!("--control-listen: {error}"))
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum HostConfigTableInMobToml {
SelfHosted,
Realm,
}
impl HostConfigTableInMobToml {
pub const ALL: [Self; 2] = [Self::SelfHosted, Self::Realm];
pub fn table(self) -> &'static str {
match self {
Self::SelfHosted => "self_hosted",
Self::Realm => "realm",
}
}
}
impl std::fmt::Display for HostConfigTableInMobToml {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"mob.toml declares a top-level [{table}] table, which is meerkat host configuration \
that the mob definition parser ignores; move it into the meerkat config.toml and \
point the gateway at that file: runtime_options.meerkat_config_path on rpc_gateway, \
or the meerkat_config_path init param (default <workspace>/.rkat/config.toml) on \
mobkit_gateway",
table = self.table()
)
}
}
impl std::error::Error for HostConfigTableInMobToml {}
pub fn refuse_host_config_tables_in_mob_toml(
mob_toml: &str,
) -> Result<(), HostConfigTableInMobToml> {
let Ok(toml::Value::Table(document)) = toml::from_str::<toml::Value>(mob_toml) else {
return Ok(());
};
match HostConfigTableInMobToml::ALL
.into_iter()
.find(|table| document.contains_key(table.table()))
{
Some(table) => Err(table),
None => Ok(()),
}
}
#[derive(Debug)]
#[non_exhaustive]
pub enum GatewayHostConfigError {
Read {
path: PathBuf,
source: std::io::Error,
},
Parse {
path: PathBuf,
source: meerkat::ConfigError,
},
Invalid {
path: PathBuf,
source: meerkat::ConfigError,
},
}
impl std::fmt::Display for GatewayHostConfigError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Read { path, source } => {
write!(
f,
"failed to read meerkat config {}: {source}",
path.display()
)
}
Self::Parse { path, source } => {
write!(f, "meerkat config {} is invalid: {source}", path.display())
}
Self::Invalid { path, source } => {
write!(
f,
"meerkat config {} does not validate: {source}",
path.display()
)
}
}
}
}
impl std::error::Error for GatewayHostConfigError {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
match self {
Self::Read { source, .. } => Some(source),
Self::Parse { source, .. } | Self::Invalid { source, .. } => Some(source),
}
}
}
pub fn load_gateway_host_config(path: &Path) -> Result<meerkat::Config, GatewayHostConfigError> {
let text = std::fs::read_to_string(path).map_err(|source| GatewayHostConfigError::Read {
path: path.to_path_buf(),
source,
})?;
let mut config = meerkat::Config::default();
config
.merge_toml_str(&text)
.map_err(|source| GatewayHostConfigError::Parse {
path: path.to_path_buf(),
source,
})?;
config
.validate(meerkat_models::canonical())
.map_err(|source| GatewayHostConfigError::Invalid {
path: path.to_path_buf(),
source,
})?;
Ok(config)
}
pub struct GatewayScheduleHostInputs<B: SessionAgentBuilder + 'static> {
pub schedule_service: ScheduleService,
pub session_service: Arc<PersistentSessionService<B>>,
pub runtime_adapter: Arc<MeerkatMachine>,
pub schedule_store_path: PathBuf,
pub firing_host_binding: ScheduleFiringHostBinding,
pub runnable_host: Option<Arc<dyn ScheduleRunnableHost>>,
pub workgraph_service: Option<WorkGraphService>,
pub owner_id: String,
}
pub async fn adopt_schedule_mob_targets(
runtime: &UnifiedRuntime,
schedule_service: &ScheduleService,
mob_target_registry: &ScheduleMobTargetRegistry,
) -> Option<Arc<MobMcpState>> {
let mob_state = runtime.mob_runtime().agent_mob_mcp_state();
mob_target_registry.set_mob_state(mob_state.clone());
match crate::schedule_wiring::repair_resumable_session_targets_to_mob_members(
schedule_service,
mob_target_registry,
)
.await
{
Ok(repaired) if repaired > 0 => {
tracing::info!(
repaired,
"repaired persisted resumable-session schedules to identity mob targets"
);
}
Ok(_) => {}
Err(error) => {
tracing::warn!(
error = %error,
"failed to repair persisted resumable-session schedules to identity mob targets",
);
}
}
mob_state
}
pub async fn spawn_gateway_schedule_host<B: SessionAgentBuilder + 'static>(
runtime: &UnifiedRuntime,
mob_state: Option<Arc<MobMcpState>>,
inputs: GatewayScheduleHostInputs<B>,
) -> (Option<ScheduleHostHandle>, tokio::task::JoinHandle<()>) {
let GatewayScheduleHostInputs {
schedule_service,
session_service,
runtime_adapter,
schedule_store_path,
firing_host_binding,
runnable_host,
workgraph_service,
owner_id,
} = inputs;
let watchdog_config = ScheduleClaimWatchdogConfig::default();
tracing::info!(
poll_interval_secs = watchdog_config.poll_interval.as_secs(),
overdue_threshold_secs = watchdog_config.overdue_threshold.as_secs(),
heartbeat_polls = watchdog_config.heartbeat_polls,
"schedule claim watchdog resident: probes the firing pipeline on this cadence, ERROR on \
a new or changed stall report, WARN heartbeat while it persists, INFO on recovery"
);
let watchdog = crate::schedule_wiring::spawn_schedule_claim_watchdog(
schedule_service.clone(),
schedule_store_path.clone(),
watchdog_config,
);
let schedule_service_for_probe = schedule_service.clone();
let schedule_host = crate::schedule_wiring::spawn_schedule_host_with_identity_runtime(
session_service,
runtime_adapter,
schedule_service,
mob_state,
runtime.mob_handle(),
runtime.identity_runtime().cloned(),
runnable_host,
workgraph_service,
owner_id,
);
if schedule_host.is_some() {
firing_host_binding.bind();
crate::schedule_wiring::probe_schedule_firing_pipeline(
&schedule_service_for_probe,
&schedule_store_path,
watchdog_config.overdue_threshold,
)
.await
.log_at_boot(watchdog_config);
} else {
tracing::warn!(
"schedule host did not spawn over the attached schedule store: no firing driver is \
running in this gateway, and the firing-intent write gate is consequently still \
closed, so create/update/resume are being refused rather than accepted durably"
);
match crate::schedule_wiring::observe_schedule_firing_authority(&schedule_service_for_probe)
.await
{
crate::schedule_wiring::ScheduleFiringAuthority::Held {
owner_id: holder_id,
fencing_token,
expires_in_secs,
} => tracing::warn!(
executor_owner_id = %holder_id,
fencing_token,
lease_expires_in_secs = expires_in_secs,
"the schedule store itself reports SOME process holding the realm's singular \
firing authority, so durable schedules may still be drained elsewhere; note the \
holder can also be this deployment's own crashed predecessor, whose lease stays \
live until it expires, so check the owner id and expiry above before concluding \
anything is actually draining. This gateway's firing-intent write gate stays \
closed either way: it gates on a LOCAL host"
),
crate::schedule_wiring::ScheduleFiringAuthority::Vacant => tracing::warn!(
"the schedule store itself reports its firing authority vacant AT THIS INSTANT: \
no process holds the executor lease. A peer gateway mid-restart is vacant only \
until its first tick, so this is proof of an unattended store only if it \
persists - the resident claim watchdog is what escalates once durable work \
actually goes unclaimed"
),
crate::schedule_wiring::ScheduleFiringAuthority::Unobservable { detail } => {
tracing::warn!(
%detail,
"the schedule store cannot report firing authority, so whether any other \
process drains this store is unknown from here"
);
}
}
}
(schedule_host, watchdog)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn compatibility_profiles_pin_the_two_existing_command_surfaces() {
assert_eq!(
GatewayCompatibilityProfile::ConsoleHttp.contract(),
GatewayCompatibilityContract {
top_level_permissive_init: true,
registry_resume: true,
post_init_stdin_rpc: false,
callback_plane: false,
single_shot: false,
maintenance_verbs: true,
}
);
assert_eq!(
GatewayCompatibilityProfile::StdioRpc.contract(),
GatewayCompatibilityContract {
top_level_permissive_init: false,
registry_resume: false,
post_init_stdin_rpc: true,
callback_plane: true,
single_shot: true,
maintenance_verbs: false,
}
);
}
const HOST_CONFIG_FIXTURE: &str = r#"
[self_hosted]
default_model = "gemma-4-31b"
[self_hosted.servers.local]
transport = "openai_compatible"
base_url = "http://127.0.0.1:11434"
api_style = "chat_completions"
[self_hosted.models.gemma-4-31b]
server = "local"
remote_model = "gemma4:31b"
[models.house-model]
provider = "openai"
[realm.global]
default_binding = "local"
"#;
#[test]
fn host_config_carries_self_hosted_models_and_realm_tables()
-> Result<(), Box<dyn std::error::Error>> {
let dir = tempfile::tempdir()?;
let path = dir.path().join("config.toml");
std::fs::write(&path, HOST_CONFIG_FIXTURE)?;
let config = load_gateway_host_config(&path)?;
assert_eq!(
config
.self_hosted
.models
.get("gemma-4-31b")
.map(|model| model.server.as_str()),
Some("local")
);
assert!(config.self_hosted.servers.contains_key("local"));
assert_eq!(
config.self_hosted.default_model.as_deref(),
Some("gemma-4-31b")
);
assert_eq!(
config
.models
.custom
.get("house-model")
.map(|model| model.provider == meerkat_core::Provider::OpenAI),
Some(true)
);
assert!(config.realm.contains_key("global"));
Ok(())
}
#[test]
fn host_config_refuses_missing_and_malformed_files_by_path()
-> Result<(), Box<dyn std::error::Error>> {
let dir = tempfile::tempdir()?;
let missing = dir.path().join("absent.toml");
let error = load_gateway_host_config(&missing).err();
assert!(
matches!(&error, Some(GatewayHostConfigError::Read { path, .. }) if path == &missing),
"{error:?}"
);
assert!(
error
.map(|error| error.to_string())
.unwrap_or_default()
.contains("absent.toml")
);
let malformed = dir.path().join("broken.toml");
std::fs::write(&malformed, "[self_hosted\n")?;
let error = load_gateway_host_config(&malformed).err();
assert!(
matches!(&error, Some(GatewayHostConfigError::Parse { path, .. }) if path == &malformed),
"{error:?}"
);
Ok(())
}
#[test]
fn host_config_refuses_a_file_that_does_not_validate_by_path()
-> Result<(), Box<dyn std::error::Error>> {
let dir = tempfile::tempdir()?;
let dangling = dir.path().join("dangling.toml");
std::fs::write(
&dangling,
"[self_hosted.models.gemma-4-31b]\nserver = \"nowhere\"\nremote_model = \"gemma4:31b\"\n",
)?;
let error = load_gateway_host_config(&dangling).err();
assert!(
matches!(&error, Some(GatewayHostConfigError::Invalid { path, .. }) if path == &dangling),
"{error:?}"
);
let message = error.map(|error| error.to_string()).unwrap_or_default();
assert!(message.contains("dangling.toml"), "{message}");
assert!(message.contains("references unknown server"), "{message}");
assert!(message.contains("nowhere"), "{message}");
Ok(())
}
#[test]
fn mob_toml_host_config_tables_are_refused_by_name() {
let self_hosted = "[mob]\nid = \"m\"\n\n[self_hosted.servers.local]\n\
base_url = \"http://127.0.0.1:11434\"\n";
assert_eq!(
refuse_host_config_tables_in_mob_toml(self_hosted),
Err(HostConfigTableInMobToml::SelfHosted)
);
let realm = "[mob]\nid = \"m\"\n\n[realm.global]\ndefault_binding = \"local\"\n";
assert_eq!(
refuse_host_config_tables_in_mob_toml(realm),
Err(HostConfigTableInMobToml::Realm)
);
for table in HostConfigTableInMobToml::ALL {
let message = table.to_string();
assert!(
message.contains(&format!("[{}]", table.table())),
"{message}"
);
assert!(message.contains("meerkat_config_path"), "{message}");
}
let ordinary = "[mob]\nid = \"m\"\n\n[profiles.w]\nmodel = \"house-model\"\n\n\
[models.house-model]\nprovider = \"openai\"\n";
assert_eq!(refuse_host_config_tables_in_mob_toml(ordinary), Ok(()));
assert_eq!(refuse_host_config_tables_in_mob_toml("[mob\n"), Ok(()));
}
#[tokio::test]
async fn shutdown_tail_orders_profile_quiesce_runtime_cleanup_then_registry_cleanup() {
let observed = Arc::new(std::sync::Mutex::new(Vec::new()));
let before_observed = Arc::clone(&observed);
let runtime_observed = Arc::clone(&observed);
let cleanup_observed = Arc::clone(&observed);
let (runtime, cleanup) = ordered_shutdown_tail(
move || async move {
before_observed
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.push("profile_quiesce");
},
async move {
runtime_observed
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.push("runtime_shutdown");
"runtime_report"
},
move || async move {
cleanup_observed
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.push("registry_cleanup");
"cleanup_report"
},
)
.await;
assert_eq!(runtime, "runtime_report");
assert_eq!(cleanup, "cleanup_report");
assert_eq!(
*observed
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner),
["profile_quiesce", "runtime_shutdown", "registry_cleanup"]
);
}
#[test]
fn default_tracing_filter_reproduces_both_binary_constants() {
assert_eq!(
default_tracing_filter("rpc_gateway"),
"warn,meerkat_mobkit=info,rpc_gateway=info"
);
assert_eq!(
default_tracing_filter("mobkit_gateway"),
"warn,meerkat_mobkit=info,mobkit_gateway=info"
);
}
#[test]
fn default_tracing_filter_always_keeps_the_crate_at_info() {
for target in ["rpc_gateway", "mobkit_gateway", "some_future_gateway"] {
let filter = default_tracing_filter(target);
assert!(
filter.contains("meerkat_mobkit=info"),
"filter for {target} dropped the crate's own INFO target: {filter}"
);
assert!(
filter.starts_with("warn,"),
"filter for {target} lost the dependency WARN default: {filter}"
);
}
}
#[derive(Clone, Default)]
struct CaptureWriter(std::sync::Arc<std::sync::Mutex<Vec<u8>>>);
impl CaptureWriter {
fn contents(&self) -> String {
String::from_utf8_lossy(
&self
.0
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner),
)
.into_owned()
}
}
impl std::io::Write for CaptureWriter {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
self.0
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.extend_from_slice(buf);
Ok(buf.len())
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for CaptureWriter {
type Writer = CaptureWriter;
fn make_writer(&'a self) -> Self::Writer {
self.clone()
}
}
#[test]
#[allow(clippy::panic)]
fn panic_hook_reports_thread_location_and_payload_through_tracing() {
install_gateway_panic_hook();
let writer = CaptureWriter::default();
let subscriber = tracing_subscriber::fmt()
.with_writer(writer.clone())
.with_ansi(false)
.finish();
let current = std::thread::current();
let this_thread = current.name().unwrap_or("<unnamed>").to_string();
let outcomes = tracing::subscriber::with_default(subscriber, || {
let formatted = std::panic::catch_unwind(|| {
panic!("gateway panic hook marker {}", 40 + 2);
});
let literal = std::panic::catch_unwind(|| {
panic!("gateway panic hook literal marker");
});
(formatted, literal)
});
assert!(outcomes.0.is_err() && outcomes.1.is_err());
let log = writer.contents();
let error_lines: Vec<&str> = log.lines().filter(|line| line.contains("ERROR")).collect();
assert!(
error_lines
.iter()
.any(|line| line.contains("gateway panic hook marker 42")),
"formatted panic payload missing from the tracing stream:\n{log}"
);
assert!(
error_lines
.iter()
.any(|line| line.contains("gateway panic hook literal marker")),
"literal panic payload missing from the tracing stream:\n{log}"
);
for line in &error_lines {
assert!(
line.contains("gateway_composition.rs"),
"panic line does not name the source file: {line}"
);
assert!(
line.contains("line=") && line.contains("column="),
"panic line does not carry the source position: {line}"
);
assert!(
line.contains(&this_thread),
"panic line does not name the panicking thread {this_thread:?}: {line}"
);
}
}
#[test]
fn control_listen_arg_absent_is_none() -> Result<(), String> {
let args = vec!["--persistent".to_string()];
assert!(parse_control_listen_arg(&args)?.is_none());
Ok(())
}
#[test]
fn control_listen_arg_parses_tcp() -> Result<(), String> {
let args = vec![
"--persistent".to_string(),
"--control-listen".to_string(),
"tcp://127.0.0.1:0".to_string(),
];
assert!(parse_control_listen_arg(&args)?.is_some());
Ok(())
}
#[test]
fn control_listen_arg_without_value_is_an_error() {
let args = vec!["--control-listen".to_string()];
let error = parse_control_listen_arg(&args).err().unwrap_or_default();
assert!(
error.contains("requires an address"),
"unexpected error text: {error}"
);
}
#[test]
fn control_listen_arg_rejects_inproc_and_names_the_flag() {
let args = vec!["--control-listen".to_string(), "inproc".to_string()];
let error = parse_control_listen_arg(&args).err().unwrap_or_default();
assert!(
error.starts_with("--control-listen: "),
"refusal must name the flag: {error}"
);
}
const ONE_KEY_JWKS: &str =
r#"{"keys":[{"kid":"k1","kty":"oct","alg":"HS256","k":"c2VjcmV0LWJ5dGVz"}]}"#;
fn open_console() -> RuntimeDecisionState {
RuntimeDecisionState::local_console(
crate::decisions::ConsolePolicy {
require_app_auth: false,
..crate::decisions::ConsolePolicy::default()
},
None,
)
}
fn closed_console_without_keys() -> RuntimeDecisionState {
RuntimeDecisionState::local_console(crate::decisions::ConsolePolicy::default(), None)
}
fn authenticated_console() -> RuntimeDecisionState {
let mut state = closed_console_without_keys();
state.trusted_oidc.jwks_json = ONE_KEY_JWKS.to_string();
state
}
#[tokio::test]
async fn bind_loopback_reports_the_pre_existing_base_url_form() -> std::io::Result<()> {
let binding = GatewayHttpBinding::bind_loopback().await?;
assert_eq!(binding.local_addr().ip(), IpAddr::V4(Ipv4Addr::LOCALHOST));
assert_eq!(
binding.http_base_url(),
format!("http://127.0.0.1:{}", binding.port())
);
assert_eq!(binding.advertised_base_url(), None);
Ok(())
}
#[tokio::test]
async fn bind_unspecified_reports_a_reachable_loopback_base_url() -> std::io::Result<()> {
let binding = GatewayHttpBinding::bind("0.0.0.0:0".parse().map_err(std::io::Error::other)?)
.await?
.with_advertised_base_url(Some("https://mob.example.com/".to_string()));
assert!(binding.local_addr().ip().is_unspecified());
assert_ne!(binding.port(), 0);
assert_eq!(
binding.http_base_url(),
format!("http://127.0.0.1:{}", binding.port())
);
assert_eq!(
binding.reachable_addr(),
SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), binding.port())
);
assert_eq!(
binding.advertised_base_url(),
Some("https://mob.example.com")
);
Ok(())
}
#[test]
fn loopback_reachable_addr_maps_wildcards_only() -> Result<(), std::net::AddrParseError> {
assert_eq!(
loopback_reachable_addr("0.0.0.0:8080".parse()?),
"127.0.0.1:8080".parse()?
);
assert_eq!(
loopback_reachable_addr("[::]:8080".parse()?),
"[::1]:8080".parse()?
);
assert_eq!(
loopback_reachable_addr("192.0.2.10:8080".parse()?),
"192.0.2.10:8080".parse()?
);
assert_eq!(
loopback_reachable_addr("127.0.0.1:41".parse()?),
"127.0.0.1:41".parse()?
);
Ok(())
}
#[test]
fn http_bind_policy_refuses_non_loopback_without_allow_remote()
-> Result<(), std::net::AddrParseError> {
for loopback in ["127.0.0.1:0", "[::1]:8080", "127.0.0.2:9"] {
assert_eq!(
validate_http_bind_policy(
GatewaySurface::MobkitGateway,
loopback.parse()?,
HttpBindPolicy::local_only()
),
Ok(())
);
}
for remote in ["0.0.0.0:8080", "[::]:8080", "192.0.2.10:8080"] {
let listen: SocketAddr = remote.parse()?;
let error = validate_http_bind_policy(
GatewaySurface::RpcGateway,
listen,
HttpBindPolicy::local_only(),
);
assert_eq!(
error,
Err(HttpBindPolicyError::RemoteBindRequiresExplicitAllow {
surface: GatewaySurface::RpcGateway,
listen,
})
);
let message = error
.map(|()| String::new())
.unwrap_or_else(|error| error.to_string());
assert!(message.contains(remote), "{message}");
assert!(message.contains("rpc_gateway"), "{message}");
assert!(message.contains("allow_remote"), "{message}");
assert!(message.contains("auth_config"), "{message}");
assert!(message.contains("MOBKIT_HTTP_ALLOW_REMOTE"), "{message}");
assert_eq!(
validate_http_bind_policy(
GatewaySurface::MobkitGateway,
listen,
HttpBindPolicy::allow_remote()
),
Ok(())
);
}
Ok(())
}
#[test]
fn gateway_surface_display_is_the_binary_name() {
assert_eq!(GatewaySurface::RpcGateway.to_string(), "rpc_gateway");
assert_eq!(GatewaySurface::MobkitGateway.to_string(), "mobkit_gateway");
}
#[test]
fn http_bind_policy_for_gateway_treats_enforced_console_auth_as_allow() {
assert_eq!(
ConsoleAuthPosture::of(&authenticated_console()),
ConsoleAuthPosture::Enforced
);
assert_eq!(
ConsoleAuthPosture::of(&open_console()),
ConsoleAuthPosture::Open
);
assert_eq!(
ConsoleAuthPosture::of(&closed_console_without_keys()),
ConsoleAuthPosture::ClosedToEveryCaller
);
assert!(ConsoleAuthPosture::Enforced.is_enforced());
assert!(!ConsoleAuthPosture::Open.is_enforced());
assert!(!ConsoleAuthPosture::ClosedToEveryCaller.is_enforced());
assert_eq!(
HttpBindPolicy::for_gateway(false, &authenticated_console()),
HttpBindPolicy::allow_remote()
);
assert_eq!(
HttpBindPolicy::for_gateway(false, &open_console()),
HttpBindPolicy::local_only()
);
assert_eq!(
HttpBindPolicy::for_gateway(false, &closed_console_without_keys()),
HttpBindPolicy::local_only()
);
assert_eq!(
HttpBindPolicy::for_gateway(true, &open_console()),
HttpBindPolicy::allow_remote()
);
}
#[test]
fn parse_http_listen_addr_accepts_ip_literals_and_refuses_hostnames() {
assert_eq!(
parse_http_listen_addr(" 0.0.0.0:8080 ").ok(),
"0.0.0.0:8080".parse().ok()
);
assert_eq!(
parse_http_listen_addr("[::]:8080").ok(),
"[::]:8080".parse().ok()
);
for bad in ["localhost:8080", "8080", "0.0.0.0", ""] {
let error = parse_http_listen_addr(bad).err();
assert!(
matches!(&error, Some(HttpExposureParseError::ListenAddress { value, .. }) if value == bad),
"{bad:?}: {error:?}"
);
let message = error.map(|error| error.to_string()).unwrap_or_default();
assert!(message.contains("HOST:PORT"), "{bad:?}: {message}");
}
}
#[test]
fn parse_http_public_base_url_requires_an_absolute_http_url() {
assert_eq!(
parse_http_public_base_url(" https://mob.example.com/ ").as_deref(),
Ok("https://mob.example.com")
);
assert_eq!(
parse_http_public_base_url("http://192.168.0.10:8080").as_deref(),
Ok("http://192.168.0.10:8080")
);
for bad in ["mob.example.com", "ws://mob.example.com", ""] {
assert_eq!(
parse_http_public_base_url(bad),
Err(HttpExposureParseError::PublicBaseUrlNotAbsoluteHttp {
value: bad.to_string()
}),
"{bad:?}"
);
}
assert_eq!(
parse_http_public_base_url("https://"),
Err(HttpExposureParseError::PublicBaseUrlNoHost {
value: "https://".to_string()
})
);
}
#[test]
fn parse_http_listen_arg_mirrors_the_control_listen_flag() -> Result<(), HttpExposureParseError>
{
let absent = vec!["--persistent".to_string()];
assert!(parse_http_listen_arg(&absent)?.is_none());
let present = vec!["--http-listen".to_string(), "0.0.0.0:8080".to_string()];
assert_eq!(
parse_http_listen_arg(&present)?,
"0.0.0.0:8080".parse().ok()
);
let missing_value = vec!["--http-listen".to_string()];
let error = parse_http_listen_arg(&missing_value).err();
assert_eq!(error, Some(HttpExposureParseError::ListenFlagMissingValue));
let message = error.map(|error| error.to_string()).unwrap_or_default();
assert!(message.contains("--http-listen requires"), "{message}");
let malformed = vec!["--http-listen".to_string(), "localhost:8080".to_string()];
let error = parse_http_listen_arg(&malformed).err();
assert!(
matches!(&error, Some(HttpExposureParseError::ListenFlagAddress { value, .. }) if value == "localhost:8080"),
"{error:?}"
);
let message = error.map(|error| error.to_string()).unwrap_or_default();
assert!(message.starts_with("--http-listen:"), "{message}");
Ok(())
}
}