Skip to main content

meerkat_mobkit/
runtime.rs

1//! Runtime subsystem types — routing, delivery, gating, memory, scheduling, and session persistence.
2
3use std::collections::{BTreeMap, BTreeSet};
4use std::fs::{self, OpenOptions};
5use std::io::Write;
6use std::io::{BufRead, BufReader};
7use std::net::{TcpStream, ToSocketAddrs};
8use std::path::{Path, PathBuf};
9use std::process::{Child, Command, Stdio};
10use std::sync::Arc;
11use std::sync::mpsc;
12use std::time::Duration;
13use std::time::{SystemTime, UNIX_EPOCH};
14
15use chrono::{Datelike, Offset, Timelike, Utc};
16use serde::{Deserialize, Serialize};
17use serde_json::Value;
18
19use crate::auth::{
20    JwtClaimsValidationConfig, build_jwt_verification_key, inspect_jwt_header, parse_jwks_json,
21    parse_oidc_discovery_json, select_jwk_for_token, validate_jwt_with_verification_key,
22};
23use crate::baseline::{
24    BaselineVerificationError, BaselineVerificationReport, verify_meerkat_baseline_symbols,
25};
26use crate::decisions::{
27    AuthPolicy, AuthProvider, BigQueryNaming, ConsoleAccessRequest, ConsolePolicy,
28    DecisionPolicyError, ReleaseMetadata, RuntimeOpsPolicy, enforce_console_route_access,
29    load_trusted_mobkit_modules_from_toml, parse_release_metadata_json, validate_bigquery_naming,
30    validate_release_metadata, validate_runtime_ops_policy,
31};
32use crate::process::{ProcessBoundaryError, run_process_json_line};
33use crate::protocol::parse_unified_event_line;
34use crate::rpc::{RpcCapabilities, RpcCapabilitiesError, parse_rpc_capabilities};
35use crate::types::{
36    EventEnvelope, MobKitConfig, ModuleConfig, ModuleEvent, PreSpawnData, RestartPolicy,
37    UnifiedEvent,
38};
39
40mod bootstrap;
41mod console_ingress;
42pub mod cross_mob_control;
43pub mod cross_mob_remote;
44mod delivery;
45mod event_transport;
46mod gating;
47mod memory;
48pub mod metadata;
49mod module_boundary;
50mod routing;
51mod rpc;
52pub(crate) mod scheduling;
53mod session_store;
54mod supervisor;
55
56pub use bootstrap::{start_mobkit_runtime, start_mobkit_runtime_with_options};
57pub(crate) use console_ingress::resolve_authorized_console_auth_from_token;
58pub use console_ingress::{
59    ConsoleAgentLiveSnapshot, ConsoleLiveSnapshot, ConsoleMember, ConsoleModelCapabilities,
60    ConsoleRestJsonRequest, ConsoleRestJsonResponse, extract_bearer_token_from_header,
61    handle_console_rest_json_route, handle_console_rest_json_route_with_snapshot,
62    handle_console_rest_json_route_with_snapshot_access_and_memory,
63    handle_console_rest_json_route_with_snapshot_and_access, validate_console_token,
64};
65pub use event_transport::normalize_event_line;
66pub use metadata::{
67    InMemoryMetadataStore, LabelRpcResult, MetadataScope, MetadataStoreError,
68    PersistentMetadataStore, RuntimeMetadataTable, SqliteMetadataStore, dispatch_labels_delete,
69    dispatch_labels_get, dispatch_labels_set, labels_to_json_value, parse_labels_param,
70    parse_run_id_param,
71};
72pub use routing::WILDCARD_ROUTE;
73pub use routing::route_module_call;
74pub use rpc::{
75    route_module_call_rpc_json, route_module_call_rpc_subprocess,
76    run_rpc_capabilities_boundary_once,
77};
78pub use scheduling::evaluate_schedules_at_tick;
79pub use session_store::{
80    BigQueryGcConfig, BigQuerySessionStoreAdapter, BigQuerySessionStoreError, GcErrorCallback,
81    JsonFileSessionStore, JsonFileSessionStoreError, JsonStoreLockRecord, SessionPersistenceRow,
82    SessionStoreContract, SessionStoreKind, materialize_latest_session_rows,
83    materialize_live_session_rows, run_periodic_gc, run_periodic_gc_with_error_callback,
84    session_store_contracts,
85};
86pub use supervisor::{run_discovered_module_once, run_module_boundary_once};
87
88pub(crate) use scheduling::validate_schedules;
89
90use event_transport::{insert_event_sorted, merge_unified_events};
91use supervisor::supervise_module_start;
92
93#[derive(Debug, Clone, PartialEq, Eq)]
94pub enum NormalizationError {
95    InvalidJson,
96    InvalidSchema,
97    MissingField(&'static str),
98    InvalidFieldType(&'static str),
99    SourceMismatch { expected: &'static str, got: String },
100}
101
102impl std::fmt::Display for NormalizationError {
103    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
104        match self {
105            Self::InvalidJson => write!(f, "invalid JSON"),
106            Self::InvalidSchema => write!(f, "invalid schema"),
107            Self::MissingField(field) => write!(f, "missing field: {field}"),
108            Self::InvalidFieldType(field) => write!(f, "invalid field type: {field}"),
109            Self::SourceMismatch { expected, got } => {
110                write!(f, "source mismatch: expected {expected}, got {got}")
111            }
112        }
113    }
114}
115
116impl std::error::Error for NormalizationError {}
117
118#[derive(Debug, Clone, PartialEq, Eq)]
119pub enum RuntimeBoundaryError {
120    Process(ProcessBoundaryError),
121    Normalize(NormalizationError),
122    Mcp(McpBoundaryError),
123}
124
125impl std::fmt::Display for RuntimeBoundaryError {
126    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
127        match self {
128            Self::Process(err) => write!(f, "process boundary: {err}"),
129            Self::Normalize(err) => write!(f, "normalization: {err}"),
130            Self::Mcp(err) => write!(f, "MCP boundary: {err}"),
131        }
132    }
133}
134
135impl std::error::Error for RuntimeBoundaryError {
136    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
137        match self {
138            Self::Process(err) => Some(err),
139            Self::Normalize(err) => Some(err),
140            Self::Mcp(err) => Some(err),
141        }
142    }
143}
144
145#[derive(Debug, Clone, PartialEq, Eq)]
146pub enum McpBoundaryError {
147    RuntimeUnavailable(String),
148    McpRequired {
149        module_id: String,
150        flow: String,
151    },
152    Timeout {
153        module_id: String,
154        operation: String,
155        timeout_ms: u64,
156    },
157    ConnectionFailed {
158        module_id: String,
159        reason: String,
160    },
161    ToolListFailed {
162        module_id: String,
163        reason: String,
164    },
165    ToolNotFound {
166        module_id: String,
167        tool: String,
168        available_tools: Vec<String>,
169    },
170    ToolCallFailed {
171        module_id: String,
172        tool: String,
173        reason: String,
174    },
175    CloseFailed {
176        module_id: String,
177        reason: String,
178    },
179    OperationFailedWithCloseFailure {
180        primary: Box<McpBoundaryError>,
181        close: Box<McpBoundaryError>,
182    },
183    InvalidToolPayload {
184        module_id: String,
185        tool: String,
186        reason: String,
187    },
188    InvalidJsonResponse {
189        module_id: String,
190        tool: String,
191        response: String,
192    },
193}
194
195impl std::fmt::Display for McpBoundaryError {
196    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
197        match self {
198            Self::RuntimeUnavailable(msg) => write!(f, "runtime unavailable: {msg}"),
199            Self::McpRequired { module_id, flow } => {
200                write!(f, "MCP required for module {module_id} flow {flow}")
201            }
202            Self::Timeout {
203                module_id,
204                operation,
205                timeout_ms,
206            } => {
207                write!(
208                    f,
209                    "timeout for module {module_id} operation {operation} after {timeout_ms}ms"
210                )
211            }
212            Self::ConnectionFailed { module_id, reason } => {
213                write!(f, "connection failed for module {module_id}: {reason}")
214            }
215            Self::ToolListFailed { module_id, reason } => {
216                write!(f, "tool list failed for module {module_id}: {reason}")
217            }
218            Self::ToolNotFound {
219                module_id,
220                tool,
221                available_tools,
222            } => {
223                write!(
224                    f,
225                    "tool {tool} not found for module {module_id} (available: {})",
226                    available_tools.join(", ")
227                )
228            }
229            Self::ToolCallFailed {
230                module_id,
231                tool,
232                reason,
233            } => {
234                write!(
235                    f,
236                    "tool call {tool} failed for module {module_id}: {reason}"
237                )
238            }
239            Self::CloseFailed { module_id, reason } => {
240                write!(f, "close failed for module {module_id}: {reason}")
241            }
242            Self::OperationFailedWithCloseFailure { primary, close } => {
243                write!(f, "operation failed: {primary}; close also failed: {close}")
244            }
245            Self::InvalidToolPayload {
246                module_id,
247                tool,
248                reason,
249            } => {
250                write!(
251                    f,
252                    "invalid tool payload for module {module_id} tool {tool}: {reason}"
253                )
254            }
255            Self::InvalidJsonResponse {
256                module_id,
257                tool,
258                response,
259            } => {
260                write!(
261                    f,
262                    "invalid JSON response for module {module_id} tool {tool}: {response}"
263                )
264            }
265        }
266    }
267}
268
269impl std::error::Error for McpBoundaryError {}
270
271#[derive(Debug, Clone, PartialEq, Eq)]
272pub enum ConfigResolutionError {
273    ModuleNotConfigured(String),
274    ModuleNotDiscovered(String),
275}
276
277impl std::fmt::Display for ConfigResolutionError {
278    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
279        match self {
280            Self::ModuleNotConfigured(id) => write!(f, "module not configured: {id}"),
281            Self::ModuleNotDiscovered(id) => write!(f, "module not discovered: {id}"),
282        }
283    }
284}
285
286impl std::error::Error for ConfigResolutionError {}
287
288#[derive(Debug, Clone, PartialEq, Eq)]
289pub enum RuntimeFromConfigError {
290    Config(ConfigResolutionError),
291    Runtime(RuntimeBoundaryError),
292}
293
294impl std::fmt::Display for RuntimeFromConfigError {
295    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
296        match self {
297            Self::Config(err) => write!(f, "config resolution: {err}"),
298            Self::Runtime(err) => write!(f, "runtime boundary: {err}"),
299        }
300    }
301}
302
303impl std::error::Error for RuntimeFromConfigError {
304    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
305        match self {
306            Self::Config(err) => Some(err),
307            Self::Runtime(err) => Some(err),
308        }
309    }
310}
311
312#[derive(Debug, Clone, PartialEq, Eq)]
313pub enum RpcRuntimeError {
314    Process(ProcessBoundaryError),
315    Capabilities(RpcCapabilitiesError),
316}
317
318impl std::fmt::Display for RpcRuntimeError {
319    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
320        match self {
321            Self::Process(err) => write!(f, "process boundary: {err}"),
322            Self::Capabilities(err) => write!(f, "capabilities: {err}"),
323        }
324    }
325}
326
327impl std::error::Error for RpcRuntimeError {
328    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
329        match self {
330            Self::Process(err) => Some(err),
331            Self::Capabilities(err) => Some(err),
332        }
333    }
334}
335
336#[derive(Debug, Clone, PartialEq, Eq)]
337pub enum BaselineRuntimeError {
338    Process(ProcessBoundaryError),
339    InvalidRepoPathJson,
340    MissingRepoRoot,
341    InvalidRepoRoot,
342    Baseline(BaselineVerificationError),
343}
344
345impl std::fmt::Display for BaselineRuntimeError {
346    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
347        match self {
348            Self::Process(err) => write!(f, "process boundary: {err}"),
349            Self::InvalidRepoPathJson => write!(f, "invalid repo path JSON"),
350            Self::MissingRepoRoot => write!(f, "missing repo root"),
351            Self::InvalidRepoRoot => write!(f, "invalid repo root"),
352            Self::Baseline(err) => write!(f, "baseline verification: {err}"),
353        }
354    }
355}
356
357impl std::error::Error for BaselineRuntimeError {
358    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
359        match self {
360            Self::Process(err) => Some(err),
361            Self::Baseline(err) => Some(err),
362            _ => None,
363        }
364    }
365}
366
367#[derive(Debug, Clone, PartialEq, Eq)]
368pub enum MobkitRuntimeError {
369    Config(ConfigResolutionError),
370    MemoryBackend(LocalJsonMemoryStoreError),
371}
372
373impl std::fmt::Display for MobkitRuntimeError {
374    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
375        match self {
376            Self::Config(err) => write!(f, "config resolution: {err}"),
377            Self::MemoryBackend(err) => write!(f, "memory backend: {err}"),
378        }
379    }
380}
381
382impl std::error::Error for MobkitRuntimeError {
383    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
384        match self {
385            Self::Config(err) => Some(err),
386            Self::MemoryBackend(err) => Some(err),
387        }
388    }
389}
390
391#[derive(Debug, Clone, PartialEq, Eq)]
392pub enum DecisionRuntimeError {
393    Policy(DecisionPolicyError),
394}
395
396impl std::fmt::Display for DecisionRuntimeError {
397    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
398        match self {
399            Self::Policy(err) => write!(f, "decision policy: {err}"),
400        }
401    }
402}
403
404impl std::error::Error for DecisionRuntimeError {
405    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
406        match self {
407            Self::Policy(err) => Some(err),
408        }
409    }
410}
411
412#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
413pub struct RuntimeDecisionInputs {
414    pub bigquery: BigQueryNaming,
415    pub trusted_mobkit_toml: String,
416    pub auth: AuthPolicy,
417    pub trusted_oidc: TrustedOidcRuntimeConfig,
418    pub console: ConsolePolicy,
419    pub ops: RuntimeOpsPolicy,
420    pub release_metadata_json: String,
421}
422
423#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
424pub struct RuntimeDecisionState {
425    pub bigquery: BigQueryNaming,
426    pub modules: Vec<ModuleConfig>,
427    pub auth: AuthPolicy,
428    pub trusted_oidc: TrustedOidcRuntimeConfig,
429    pub console: ConsolePolicy,
430    pub ops: RuntimeOpsPolicy,
431    pub release_metadata: ReleaseMetadata,
432}
433
434#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
435pub struct TrustedOidcRuntimeConfig {
436    pub discovery_json: String,
437    pub jwks_json: String,
438    pub audience: String,
439}
440
441#[derive(Debug, Clone, PartialEq, Eq)]
442pub enum LocalJsonMemoryStoreError {
443    InvalidConfig(String),
444    Io(String),
445    Serialize(String),
446    InvalidStoreData(String),
447    ExternalCallFailed(String),
448}
449
450impl std::fmt::Display for LocalJsonMemoryStoreError {
451    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
452        match self {
453            Self::InvalidConfig(msg) => write!(f, "invalid config: {msg}"),
454            Self::Io(msg) => write!(f, "I/O error: {msg}"),
455            Self::Serialize(msg) => write!(f, "serialization error: {msg}"),
456            Self::InvalidStoreData(msg) => write!(f, "invalid store data: {msg}"),
457            Self::ExternalCallFailed(msg) => write!(f, "external call failed: {msg}"),
458        }
459    }
460}
461
462impl std::error::Error for LocalJsonMemoryStoreError {}
463
464/// Deprecated: kept as an alias for the pre-rename error type. The backend
465/// never sent data to Elephant, so the type is now named for what it does.
466pub type ElephantMemoryStoreError = LocalJsonMemoryStoreError;
467
468/// Operational-ledger backend that persists assertions/conflicts as local
469/// JSON, with an optional HTTP health gate applied before every read/write.
470#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
471pub struct LocalJsonMemoryBackendConfig {
472    pub state_path: String,
473    #[serde(default)]
474    pub health_check_endpoint: Option<String>,
475}
476
477/// Deprecated legacy config shape. Despite the name, this backend never
478/// wrote to Elephant: `endpoint` is only health-checked and the ledger is
479/// persisted as local JSON at `state_path`. Use
480/// [`LocalJsonMemoryBackendConfig`]; this shape is still accepted for wire
481/// and config compatibility.
482#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
483pub struct ElephantMemoryBackendConfig {
484    pub endpoint: String,
485    pub state_path: String,
486}
487
488impl From<ElephantMemoryBackendConfig> for LocalJsonMemoryBackendConfig {
489    fn from(legacy: ElephantMemoryBackendConfig) -> Self {
490        Self {
491            state_path: legacy.state_path,
492            health_check_endpoint: Some(legacy.endpoint),
493        }
494    }
495}
496
497#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
498#[serde(tag = "kind", rename_all = "snake_case")]
499pub enum MemoryBackendConfig {
500    LocalJson(LocalJsonMemoryBackendConfig),
501    /// Deprecated: legacy `kind = "elephant"` shape. Accepted and mapped to
502    /// the local-JSON backend (endpoint becomes the health-check endpoint);
503    /// a deprecation warning is logged at runtime start.
504    Elephant(ElephantMemoryBackendConfig),
505}
506
507#[derive(Debug, Clone, PartialEq, Eq)]
508struct LocalJsonMemoryStoreAdapter {
509    health_check_endpoint: Option<String>,
510    state_path: PathBuf,
511}
512
513#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
514pub struct RuntimeOptions {
515    pub on_failure_retry_budget: u32,
516    pub always_restart_budget: u32,
517    #[serde(default)]
518    pub supervisor_restart_backoff_ms: u64,
519    #[serde(default)]
520    pub supervisor_test_force_terminate_failure: bool,
521    #[serde(default)]
522    pub memory_backend: Option<MemoryBackendConfig>,
523    #[serde(default = "default_implicit_delegate_idle_retire_secs")]
524    pub implicit_delegate_idle_retire_secs: Option<u64>,
525    #[serde(default = "default_implicit_delegate_idle_sweep_interval_ms")]
526    pub implicit_delegate_idle_sweep_interval_ms: u64,
527}
528
529fn default_implicit_delegate_idle_retire_secs() -> Option<u64> {
530    Some(300)
531}
532
533fn default_implicit_delegate_idle_sweep_interval_ms() -> u64 {
534    10_000
535}
536
537impl Default for RuntimeOptions {
538    fn default() -> Self {
539        Self {
540            on_failure_retry_budget: 1,
541            always_restart_budget: 1,
542            supervisor_restart_backoff_ms: 0,
543            supervisor_test_force_terminate_failure: false,
544            memory_backend: None,
545            implicit_delegate_idle_retire_secs: default_implicit_delegate_idle_retire_secs(),
546            implicit_delegate_idle_sweep_interval_ms:
547                default_implicit_delegate_idle_sweep_interval_ms(),
548        }
549    }
550}
551
552#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
553pub enum LifecycleStage {
554    MobStarted,
555    ModulesStarted,
556    MergedStreamStarted,
557    ShutdownRequested,
558    ShutdownComplete,
559}
560
561#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
562pub struct LifecycleEvent {
563    pub seq: u64,
564    pub stage: LifecycleStage,
565}
566
567#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
568pub enum ModuleHealthState {
569    Starting,
570    Healthy,
571    Failed,
572    Restarting,
573    Stopped,
574}
575
576#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
577pub struct ModuleHealthTransition {
578    pub module_id: String,
579    pub from: Option<ModuleHealthState>,
580    pub to: ModuleHealthState,
581    pub attempt: u32,
582}
583
584#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
585pub struct SupervisorReport {
586    pub transitions: Vec<ModuleHealthTransition>,
587}
588
589#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
590pub struct RuntimeShutdownReport {
591    pub terminated_modules: Vec<String>,
592    pub orphan_processes: u32,
593}
594
595#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
596pub struct ScheduleDefinition {
597    pub schedule_id: String,
598    pub interval: String,
599    pub timezone: String,
600    pub enabled: bool,
601    #[serde(default)]
602    pub jitter_ms: u64,
603    #[serde(default)]
604    pub catch_up: bool,
605}
606
607#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
608pub struct ScheduleTrigger {
609    pub schedule_id: String,
610    pub interval: String,
611    pub timezone: String,
612    pub due_tick_ms: u64,
613}
614
615#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
616pub struct ScheduleEvaluation {
617    pub tick_ms: u64,
618    pub due_triggers: Vec<ScheduleTrigger>,
619}
620
621#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
622pub struct SchedulingSupervisorSignal {
623    pub module_id: String,
624    pub latest_state: ModuleHealthState,
625    pub latest_attempt: u32,
626    pub restart_observed: bool,
627}
628
629#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
630pub struct ScheduleDispatch {
631    pub claim_key: String,
632    pub schedule_id: String,
633    pub interval: String,
634    pub timezone: String,
635    pub due_tick_ms: u64,
636    pub tick_ms: u64,
637    pub event_id: String,
638    pub supervisor_signal: Option<SchedulingSupervisorSignal>,
639    #[serde(default)]
640    pub runtime_injection: Option<ScheduleRuntimeInjection>,
641    #[serde(default)]
642    pub runtime_injection_error: Option<String>,
643}
644
645#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
646pub struct ScheduleRuntimeInjection {
647    pub member_id: String,
648    pub message: String,
649    pub injection_event_id: String,
650}
651
652#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
653pub struct ScheduleDispatchReport {
654    pub tick_ms: u64,
655    pub due_count: usize,
656    pub dispatched: Vec<ScheduleDispatch>,
657    pub skipped_claims: Vec<String>,
658}
659
660#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
661pub struct RoutingResolveRequest {
662    pub recipient: String,
663    #[serde(default)]
664    pub channel: Option<String>,
665    #[serde(default)]
666    pub retry_max: Option<u32>,
667    #[serde(default)]
668    pub backoff_ms: Option<u64>,
669    #[serde(default)]
670    pub rate_limit_per_minute: Option<u32>,
671}
672
673#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
674pub struct RuntimeRoute {
675    pub route_key: String,
676    pub recipient: String,
677    #[serde(default)]
678    pub channel: Option<String>,
679    pub sink: String,
680    pub target_module: String,
681    #[serde(default)]
682    pub retry_max: Option<u32>,
683    #[serde(default)]
684    pub backoff_ms: Option<u64>,
685    #[serde(default)]
686    pub rate_limit_per_minute: Option<u32>,
687}
688
689#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
690pub struct RoutingResolution {
691    pub route_id: String,
692    pub recipient: String,
693    pub channel: String,
694    pub sink: String,
695    pub target_module: String,
696    pub retry_max: u32,
697    pub backoff_ms: u64,
698    pub rate_limit_per_minute: u32,
699}
700
701#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
702pub struct DeliverySendRequest {
703    pub resolution: RoutingResolution,
704    pub payload: Value,
705    #[serde(default)]
706    pub idempotency_key: Option<String>,
707}
708
709#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
710pub struct DeliveryAttempt {
711    pub attempt: u32,
712    pub status: String,
713    pub backoff_ms: u64,
714}
715
716#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
717pub struct DeliveryRecord {
718    pub delivery_id: String,
719    pub route_id: String,
720    pub recipient: String,
721    pub sink: String,
722    pub target_module: String,
723    pub payload: Value,
724    pub status: String,
725    pub attempts: Vec<DeliveryAttempt>,
726    pub first_attempt_ms: u64,
727    pub final_attempt_ms: u64,
728    pub idempotency_key: Option<String>,
729    pub sink_adapter: Option<String>,
730}
731
732#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
733pub struct DeliveryHistoryRequest {
734    #[serde(default)]
735    pub recipient: Option<String>,
736    #[serde(default)]
737    pub sink: Option<String>,
738    #[serde(default = "default_delivery_history_limit")]
739    pub limit: usize,
740}
741
742#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
743pub struct DeliveryHistoryResponse {
744    pub deliveries: Vec<DeliveryRecord>,
745}
746
747#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
748pub struct MemoryStoreInfo {
749    pub store: String,
750    pub record_count: usize,
751}
752
753#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
754pub struct MemoryIndexRequest {
755    pub entity: String,
756    pub topic: String,
757    #[serde(default)]
758    pub store: Option<String>,
759    #[serde(default)]
760    pub fact: Option<String>,
761    #[serde(default)]
762    pub metadata: Option<Value>,
763    #[serde(default)]
764    pub conflict: Option<bool>,
765    #[serde(default)]
766    pub conflict_reason: Option<String>,
767}
768
769#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
770pub struct MemoryAssertion {
771    pub assertion_id: String,
772    pub entity: String,
773    pub topic: String,
774    pub store: String,
775    pub fact: String,
776    #[serde(default)]
777    pub metadata: Option<Value>,
778    pub indexed_at_ms: u64,
779}
780
781#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
782pub struct MemoryConflictSignal {
783    pub entity: String,
784    pub topic: String,
785    pub store: String,
786    #[serde(default)]
787    pub reason: Option<String>,
788    pub updated_at_ms: u64,
789}
790
791#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
792pub struct MemoryIndexResult {
793    pub entity: String,
794    pub topic: String,
795    pub store: String,
796    #[serde(default)]
797    pub assertion_id: Option<String>,
798    pub conflict_active: bool,
799}
800
801#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
802pub struct MemoryQueryRequest {
803    #[serde(default)]
804    pub entity: Option<String>,
805    #[serde(default)]
806    pub topic: Option<String>,
807    #[serde(default)]
808    pub store: Option<String>,
809    /// Case-insensitive substring filter applied across entity, topic, and
810    /// fact (reason for conflict signals), after the exact filters above.
811    #[serde(default)]
812    pub query: Option<String>,
813}
814
815#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
816pub struct MemoryQueryResult {
817    pub assertions: Vec<MemoryAssertion>,
818    pub conflicts: Vec<MemoryConflictSignal>,
819}
820
821#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
822struct PersistedMemoryState {
823    #[serde(default)]
824    assertions: Vec<MemoryAssertion>,
825    #[serde(default)]
826    conflicts: Vec<MemoryConflictSignal>,
827}
828
829#[derive(Debug, Clone, PartialEq, Eq)]
830pub enum MemoryIndexError {
831    EntityRequired,
832    TopicRequired,
833    UnsupportedStore(String),
834    FactRequiredWhenConflictUnset,
835    BackendPersistFailed(LocalJsonMemoryStoreError),
836}
837
838impl std::fmt::Display for MemoryIndexError {
839    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
840        match self {
841            Self::EntityRequired => write!(f, "entity is required"),
842            Self::TopicRequired => write!(f, "topic is required"),
843            Self::UnsupportedStore(store) => write!(f, "unsupported store: {store}"),
844            Self::FactRequiredWhenConflictUnset => {
845                write!(f, "fact is required when conflict is unset")
846            }
847            Self::BackendPersistFailed(err) => write!(f, "backend persist failed: {err}"),
848        }
849    }
850}
851
852impl std::error::Error for MemoryIndexError {
853    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
854        match self {
855            Self::BackendPersistFailed(err) => Some(err),
856            _ => None,
857        }
858    }
859}
860
861#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
862#[serde(rename_all = "snake_case")]
863pub enum GatingRiskTier {
864    R0,
865    R1,
866    R2,
867    R3,
868}
869
870#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
871pub struct GatingEvaluateRequest {
872    pub action: String,
873    pub actor_id: String,
874    pub risk_tier: GatingRiskTier,
875    #[serde(default)]
876    pub rationale: Option<String>,
877    #[serde(default)]
878    pub requested_approver: Option<String>,
879    #[serde(default)]
880    pub approval_recipient: Option<String>,
881    #[serde(default)]
882    pub approval_channel: Option<String>,
883    #[serde(default)]
884    pub approval_timeout_ms: Option<u64>,
885    #[serde(default)]
886    pub entity: Option<String>,
887    #[serde(default)]
888    pub topic: Option<String>,
889}
890
891#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
892#[serde(rename_all = "snake_case")]
893pub enum GatingOutcome {
894    Allowed,
895    AllowedWithAudit,
896    PendingApproval,
897    SafeDraft,
898}
899
900#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
901pub struct GatingEvaluateResult {
902    pub action_id: String,
903    pub action: String,
904    pub actor_id: String,
905    pub risk_tier: GatingRiskTier,
906    pub outcome: GatingOutcome,
907    #[serde(default)]
908    pub pending_id: Option<String>,
909    #[serde(default)]
910    pub fallback_reason: Option<String>,
911}
912
913#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
914pub struct GatingPendingEntry {
915    pub pending_id: String,
916    pub action_id: String,
917    pub action: String,
918    pub actor_id: String,
919    pub risk_tier: GatingRiskTier,
920    #[serde(default)]
921    pub requested_approver: Option<String>,
922    #[serde(default)]
923    pub approval_recipient: Option<String>,
924    #[serde(default)]
925    pub approval_channel: Option<String>,
926    #[serde(default)]
927    pub approval_route_id: Option<String>,
928    #[serde(default)]
929    pub approval_delivery_id: Option<String>,
930    pub created_at_ms: u64,
931    pub deadline_at_ms: u64,
932}
933
934#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
935#[serde(rename_all = "snake_case")]
936pub enum GatingDecision {
937    Approve,
938    Reject,
939    Escalate,
940}
941
942#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
943pub struct GatingDecideRequest {
944    pub pending_id: String,
945    pub approver_id: String,
946    pub decision: GatingDecision,
947    #[serde(default)]
948    pub reason: Option<String>,
949}
950
951#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
952pub struct GatingDecisionResult {
953    pub pending_id: String,
954    pub action_id: String,
955    pub approver_id: String,
956    pub decision: GatingDecision,
957    pub outcome: GatingOutcome,
958    pub decided_at_ms: u64,
959    #[serde(default)]
960    pub reason: Option<String>,
961    #[serde(default)]
962    pub next_pending_id: Option<String>,
963}
964
965#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
966pub struct GatingAuditEntry {
967    pub audit_id: String,
968    pub timestamp_ms: u64,
969    pub event_type: String,
970    pub action_id: String,
971    #[serde(default)]
972    pub pending_id: Option<String>,
973    pub actor_id: String,
974    pub risk_tier: GatingRiskTier,
975    pub outcome: GatingOutcome,
976    pub detail: Value,
977}
978
979/// One resolved gating pending entry (decision or timeout), pushed
980/// synchronously to registered observers. The gating subsystem itself is
981/// poll-only; this seam exists so components that staged work behind a
982/// pending entry (the memory steward's quarantine-promotions, §10.2) can
983/// react without polling. Observers must not block — defer real work onto
984/// the runtime.
985#[derive(Debug, Clone, PartialEq, Eq)]
986pub struct GatingResolutionNotice {
987    pub pending_id: String,
988    pub action_id: String,
989    pub approved: bool,
990    /// Set when an escalation minted a successor pending entry.
991    pub next_pending_id: Option<String>,
992    /// `approval_decided`, `rejection_decided`, `escalation_decided`, or
993    /// `timeout_fallback`.
994    pub cause: String,
995}
996
997pub trait GatingResolutionObserver: Send + Sync {
998    fn on_gating_resolution(&self, notice: &GatingResolutionNotice);
999}
1000
1001/// Registered observers; newtype so `MobkitRuntimeHandle` keeps `Debug`.
1002#[derive(Default)]
1003pub(crate) struct GatingResolutionObservers(Vec<Arc<dyn GatingResolutionObserver>>);
1004
1005impl std::fmt::Debug for GatingResolutionObservers {
1006    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1007        write!(f, "GatingResolutionObservers({})", self.0.len())
1008    }
1009}
1010
1011impl GatingResolutionObservers {
1012    pub(crate) fn push(&mut self, observer: Arc<dyn GatingResolutionObserver>) {
1013        self.0.push(observer);
1014    }
1015
1016    pub(crate) fn notify(&self, notice: &GatingResolutionNotice) {
1017        for observer in &self.0 {
1018            observer.on_gating_resolution(notice);
1019        }
1020    }
1021}
1022
1023#[derive(Debug, Clone, PartialEq, Eq)]
1024pub enum GatingDecideError {
1025    UnknownPendingId(String),
1026    SelfApprovalForbidden,
1027    ApproverMismatch { expected: String, provided: String },
1028}
1029
1030impl std::fmt::Display for GatingDecideError {
1031    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1032        match self {
1033            Self::UnknownPendingId(id) => write!(f, "unknown pending id: {id}"),
1034            Self::SelfApprovalForbidden => write!(f, "self-approval is forbidden"),
1035            Self::ApproverMismatch { expected, provided } => {
1036                write!(f, "approver mismatch: expected {expected}, got {provided}")
1037            }
1038        }
1039    }
1040}
1041
1042impl std::error::Error for GatingDecideError {}
1043
1044#[derive(Debug, Clone, PartialEq, Eq)]
1045struct DeliveryIdempotencyEntry {
1046    delivery_id: String,
1047    payload: Value,
1048    canonical_resolution: RoutingResolution,
1049}
1050
1051#[derive(Debug, Clone, PartialEq, Eq, Default)]
1052struct RouterBoundaryOverrides {
1053    channel: Option<String>,
1054    sink: Option<String>,
1055    target_module: Option<String>,
1056    retry_max: Option<u32>,
1057    backoff_ms: Option<u64>,
1058    rate_limit_per_minute: Option<u32>,
1059}
1060
1061#[derive(Debug, Clone, PartialEq, Eq, Default)]
1062struct DeliveryBoundaryOutcome {
1063    sink_adapter: Option<String>,
1064    force_fail: bool,
1065}
1066
1067#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
1068struct DeliveryRateWindowKey {
1069    route_id: String,
1070    recipient: String,
1071    sink: String,
1072    window_start_ms: u64,
1073}
1074
1075#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
1076struct MemoryConflictKey {
1077    entity: String,
1078    topic: String,
1079    store: String,
1080}
1081
1082#[derive(Debug)]
1083pub struct MobkitRuntimeHandle {
1084    config: MobKitConfig,
1085    runtime_options: RuntimeOptions,
1086    loaded_modules: BTreeSet<String>,
1087    live_children: BTreeMap<String, Child>,
1088    pub lifecycle_events: Vec<LifecycleEvent>,
1089    pub supervisor_report: SupervisorReport,
1090    pub merged_events: Vec<EventEnvelope<UnifiedEvent>>,
1091    scheduling_claims: BTreeSet<String>,
1092    scheduling_claim_ticks: BTreeMap<u64, Vec<String>>,
1093    scheduling_last_due_ticks: BTreeMap<String, u64>,
1094    scheduling_dispatch_sequence: u64,
1095    routing_sequence: u64,
1096    routing_resolutions: BTreeMap<String, RoutingResolution>,
1097    routing_resolution_order: Vec<String>,
1098    runtime_routes: BTreeMap<String, RuntimeRoute>,
1099    delivery_sequence: u64,
1100    delivery_runtime_epoch_ms: u64,
1101    delivery_now_floor_ms: u64,
1102    delivery_clock_ms: u64,
1103    delivery_history: Vec<DeliveryRecord>,
1104    delivery_idempotency: BTreeMap<String, DeliveryIdempotencyEntry>,
1105    delivery_idempotency_by_delivery: BTreeMap<String, Vec<String>>,
1106    delivery_rate_window_counts: BTreeMap<DeliveryRateWindowKey, u32>,
1107    gating_sequence: u64,
1108    gating_pending: BTreeMap<String, GatingPendingEntry>,
1109    gating_pending_order: Vec<String>,
1110    gating_audit: Vec<GatingAuditEntry>,
1111    gating_resolution_observers: GatingResolutionObservers,
1112    memory_sequence: u64,
1113    memory_assertions: Vec<MemoryAssertion>,
1114    memory_conflicts: BTreeMap<MemoryConflictKey, MemoryConflictSignal>,
1115    memory_backend: Option<LocalJsonMemoryStoreAdapter>,
1116    running: bool,
1117}
1118
1119impl MobkitRuntimeHandle {
1120    /// Register a gating-resolution observer (see
1121    /// [`GatingResolutionObserver`]). Registration is append-only for the
1122    /// process lifetime.
1123    pub fn register_gating_resolution_observer(
1124        &mut self,
1125        observer: Arc<dyn GatingResolutionObserver>,
1126    ) {
1127        self.gating_resolution_observers.push(observer);
1128    }
1129
1130    pub fn lifecycle_events(&self) -> &[LifecycleEvent] {
1131        &self.lifecycle_events
1132    }
1133
1134    pub fn supervisor_report(&self) -> &SupervisorReport {
1135        &self.supervisor_report
1136    }
1137
1138    #[doc(hidden)]
1139    pub fn inject_test_events(&mut self, events: Vec<EventEnvelope<UnifiedEvent>>) {
1140        for event in events {
1141            insert_event_sorted(&mut self.merged_events, event);
1142        }
1143    }
1144
1145    fn next_sequence(counter: &mut u64) -> u64 {
1146        let seq = *counter;
1147        *counter = counter.saturating_add(1);
1148        seq
1149    }
1150}
1151
1152#[derive(Debug, Clone, PartialEq, Eq)]
1153pub enum ScheduleValidationError {
1154    EmptyScheduleId,
1155    DuplicateScheduleId(String),
1156    InvalidTickMs(u64),
1157    InvalidInterval {
1158        schedule_id: String,
1159        interval: String,
1160    },
1161    InvalidTimezone {
1162        schedule_id: String,
1163        timezone: String,
1164    },
1165    /// The total cron lookback work for this request exceeded
1166    /// `CRON_LOOKBACK_BUDGET_PER_REQUEST`. Surfaced (rather than silently
1167    /// stalling) when a request packs enough sparse crons to blow past the
1168    /// per-request iteration ceiling — a soft-DoS guard.
1169    LookbackBudgetExceeded {
1170        schedule_id: String,
1171    },
1172}
1173
1174impl std::fmt::Display for ScheduleValidationError {
1175    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1176        match self {
1177            Self::EmptyScheduleId => write!(f, "empty schedule id"),
1178            Self::DuplicateScheduleId(id) => write!(f, "duplicate schedule id: {id}"),
1179            Self::InvalidTickMs(ms) => write!(f, "invalid tick ms: {ms}"),
1180            Self::InvalidInterval {
1181                schedule_id,
1182                interval,
1183            } => {
1184                write!(f, "invalid interval for schedule {schedule_id}: {interval}")
1185            }
1186            Self::InvalidTimezone {
1187                schedule_id,
1188                timezone,
1189            } => {
1190                write!(f, "invalid timezone for schedule {schedule_id}: {timezone}")
1191            }
1192            Self::LookbackBudgetExceeded { schedule_id } => {
1193                write!(
1194                    f,
1195                    "cron lookback budget exceeded while resolving schedule {schedule_id}"
1196                )
1197            }
1198        }
1199    }
1200}
1201
1202impl std::error::Error for ScheduleValidationError {}
1203
1204#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1205pub struct ModuleRouteRequest {
1206    pub module_id: String,
1207    pub method: String,
1208    pub params: Value,
1209}
1210
1211#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1212pub struct ModuleRouteResponse {
1213    pub module_id: String,
1214    pub method: String,
1215    pub payload: Value,
1216}
1217
1218#[derive(Debug, Clone, PartialEq, Eq)]
1219pub enum ModuleRouteError {
1220    UnloadedModule(String),
1221    ModuleRuntime(RuntimeBoundaryError),
1222    UnexpectedRouteResponse,
1223}
1224
1225impl std::fmt::Display for ModuleRouteError {
1226    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1227        match self {
1228            Self::UnloadedModule(id) => write!(f, "unloaded module: {id}"),
1229            Self::ModuleRuntime(err) => write!(f, "module runtime: {err}"),
1230            Self::UnexpectedRouteResponse => write!(f, "unexpected route response"),
1231        }
1232    }
1233}
1234
1235impl std::error::Error for ModuleRouteError {
1236    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
1237        match self {
1238            Self::ModuleRuntime(err) => Some(err),
1239            _ => None,
1240        }
1241    }
1242}
1243
1244#[derive(Debug, Clone, PartialEq, Eq)]
1245pub enum RoutingResolveError {
1246    RouterModuleNotLoaded,
1247    DeliveryModuleNotLoaded,
1248    EmptyRecipient,
1249    InvalidChannel,
1250    InvalidRateLimitPerMinute,
1251    RetryMaxExceedsCap { provided: u32, cap: u32 },
1252    RouterBoundary(RuntimeBoundaryError),
1253}
1254
1255impl std::fmt::Display for RoutingResolveError {
1256    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1257        match self {
1258            Self::RouterModuleNotLoaded => write!(f, "router module not loaded"),
1259            Self::DeliveryModuleNotLoaded => write!(f, "delivery module not loaded"),
1260            Self::EmptyRecipient => write!(f, "empty recipient"),
1261            Self::InvalidChannel => write!(f, "invalid channel"),
1262            Self::InvalidRateLimitPerMinute => write!(f, "invalid rate limit per minute"),
1263            Self::RetryMaxExceedsCap { provided, cap } => {
1264                write!(f, "retry max {provided} exceeds cap {cap}")
1265            }
1266            Self::RouterBoundary(err) => write!(f, "router boundary: {err}"),
1267        }
1268    }
1269}
1270
1271impl std::error::Error for RoutingResolveError {
1272    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
1273        match self {
1274            Self::RouterBoundary(err) => Some(err),
1275            _ => None,
1276        }
1277    }
1278}
1279
1280#[derive(Debug, Clone, PartialEq, Eq)]
1281pub enum DeliverySendError {
1282    DeliveryModuleNotLoaded,
1283    InvalidRouteTarget(String),
1284    InvalidRouteId,
1285    UnknownRouteId(String),
1286    ForgedResolution,
1287    InvalidRecipient,
1288    InvalidSink,
1289    InvalidIdempotencyKey,
1290    IdempotencyPayloadMismatch,
1291    RateLimited {
1292        sink: String,
1293        window_start_ms: u64,
1294        limit: u32,
1295    },
1296    DeliveryBoundary(RuntimeBoundaryError),
1297}
1298
1299impl std::fmt::Display for DeliverySendError {
1300    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1301        match self {
1302            Self::DeliveryModuleNotLoaded => write!(f, "delivery module not loaded"),
1303            Self::InvalidRouteTarget(target) => write!(f, "invalid route target: {target}"),
1304            Self::InvalidRouteId => write!(f, "invalid route id"),
1305            Self::UnknownRouteId(id) => write!(f, "unknown route id: {id}"),
1306            Self::ForgedResolution => write!(f, "forged resolution"),
1307            Self::InvalidRecipient => write!(f, "invalid recipient"),
1308            Self::InvalidSink => write!(f, "invalid sink"),
1309            Self::InvalidIdempotencyKey => write!(f, "invalid idempotency key"),
1310            Self::IdempotencyPayloadMismatch => write!(f, "idempotency payload mismatch"),
1311            Self::RateLimited {
1312                sink,
1313                window_start_ms,
1314                limit,
1315            } => {
1316                write!(
1317                    f,
1318                    "rate limited on sink {sink} (window {window_start_ms}ms, limit {limit})"
1319                )
1320            }
1321            Self::DeliveryBoundary(err) => write!(f, "delivery boundary: {err}"),
1322        }
1323    }
1324}
1325
1326impl std::error::Error for DeliverySendError {
1327    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
1328        match self {
1329            Self::DeliveryBoundary(err) => Some(err),
1330            _ => None,
1331        }
1332    }
1333}
1334
1335#[derive(Debug, Clone, PartialEq, Eq)]
1336pub enum RuntimeRouteMutationError {
1337    EmptyRouteKey,
1338    EmptyRecipient,
1339    InvalidChannel,
1340    EmptySink,
1341    EmptyTargetModule,
1342    InvalidRateLimitPerMinute,
1343    RetryMaxExceedsCap { provided: u32, cap: u32 },
1344    RouteNotFound(String),
1345}
1346
1347impl std::fmt::Display for RuntimeRouteMutationError {
1348    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1349        match self {
1350            Self::EmptyRouteKey => write!(f, "empty route key"),
1351            Self::EmptyRecipient => write!(f, "empty recipient"),
1352            Self::InvalidChannel => write!(f, "invalid channel"),
1353            Self::EmptySink => write!(f, "empty sink"),
1354            Self::EmptyTargetModule => write!(f, "empty target module"),
1355            Self::InvalidRateLimitPerMinute => write!(f, "invalid rate limit per minute"),
1356            Self::RetryMaxExceedsCap { provided, cap } => {
1357                write!(f, "retry max {provided} exceeds cap {cap}")
1358            }
1359            Self::RouteNotFound(key) => write!(f, "route not found: {key}"),
1360        }
1361    }
1362}
1363
1364impl std::error::Error for RuntimeRouteMutationError {}
1365
1366#[derive(Debug, Clone, PartialEq, Eq)]
1367pub enum RpcRouteError {
1368    InvalidRequest,
1369    BoundaryProcess(ProcessBoundaryError),
1370    Route(ModuleRouteError),
1371    InvalidResponse,
1372}
1373
1374impl std::fmt::Display for RpcRouteError {
1375    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1376        match self {
1377            Self::InvalidRequest => write!(f, "invalid request"),
1378            Self::BoundaryProcess(err) => write!(f, "boundary process: {err}"),
1379            Self::Route(err) => write!(f, "route: {err}"),
1380            Self::InvalidResponse => write!(f, "invalid response"),
1381        }
1382    }
1383}
1384
1385impl std::error::Error for RpcRouteError {
1386    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
1387        match self {
1388            Self::BoundaryProcess(err) => Some(err),
1389            Self::Route(err) => Some(err),
1390            _ => None,
1391        }
1392    }
1393}
1394
1395#[derive(Debug, Clone, PartialEq, Eq)]
1396pub enum RuntimeMutationError {
1397    Config(ConfigResolutionError),
1398    Runtime(RuntimeBoundaryError),
1399}
1400
1401impl std::fmt::Display for RuntimeMutationError {
1402    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1403        match self {
1404            Self::Config(err) => write!(f, "config resolution: {err}"),
1405            Self::Runtime(err) => write!(f, "runtime boundary: {err}"),
1406        }
1407    }
1408}
1409
1410impl std::error::Error for RuntimeMutationError {
1411    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
1412        match self {
1413            Self::Config(err) => Some(err),
1414            Self::Runtime(err) => Some(err),
1415        }
1416    }
1417}
1418
1419#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1420#[serde(rename_all = "snake_case")]
1421pub enum SubscribeScope {
1422    Mob,
1423    Agent,
1424    Interaction,
1425}
1426
1427#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1428pub struct SubscribeRequest {
1429    pub scope: SubscribeScope,
1430    pub last_event_id: Option<String>,
1431    pub agent_id: Option<String>,
1432}
1433
1434impl Default for SubscribeRequest {
1435    fn default() -> Self {
1436        Self {
1437            scope: SubscribeScope::Mob,
1438            last_event_id: None,
1439            agent_id: None,
1440        }
1441    }
1442}
1443
1444#[derive(Debug, Clone, PartialEq, Eq)]
1445pub enum SubscribeError {
1446    EmptyCheckpoint,
1447    UnknownCheckpoint(String),
1448    MissingAgentId,
1449    InvalidAgentId,
1450}
1451
1452impl std::fmt::Display for SubscribeError {
1453    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1454        match self {
1455            Self::EmptyCheckpoint => write!(f, "empty checkpoint"),
1456            Self::UnknownCheckpoint(id) => write!(f, "unknown checkpoint: {id}"),
1457            Self::MissingAgentId => write!(f, "missing agent id"),
1458            Self::InvalidAgentId => write!(f, "invalid agent id"),
1459        }
1460    }
1461}
1462
1463impl std::error::Error for SubscribeError {}
1464
1465#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1466pub struct SubscribeKeepAlive {
1467    pub interval_ms: u64,
1468    pub event: String,
1469}
1470
1471#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1472pub struct SubscribeResponse {
1473    pub scope: SubscribeScope,
1474    pub replay_from_event_id: Option<String>,
1475    pub keep_alive: SubscribeKeepAlive,
1476    pub keep_alive_comment: String,
1477    pub event_frames: Vec<String>,
1478    pub events: Vec<EventEnvelope<UnifiedEvent>>,
1479}
1480
1481const SSE_KEEP_ALIVE_INTERVAL_MS: u64 = 15_000;
1482const SSE_KEEP_ALIVE_EVENT_NAME: &str = "keep-alive";
1483const SSE_KEEP_ALIVE_COMMENT_FRAME: &str = ": keep-alive\n\n";
1484const SUBSCRIBE_REPLAY_EVENT_CAP: usize = 3;
1485const SCHEDULING_CLAIM_RETENTION_WINDOW_MS: u64 = 86_400_000;
1486const SCHEDULING_CLAIMS_MAX_RETAINED: usize = 4_096;
1487const SCHEDULING_LAST_DUE_MAX_RETAINED: usize = 4_096;
1488const DELIVERY_HISTORY_LIMIT_DEFAULT: usize = 20;
1489const DELIVERY_HISTORY_LIMIT_MAX: usize = 200;
1490const ROUTING_RESOLUTION_LIMIT_MAX: usize = 512;
1491pub const ROUTING_RETRY_MAX_CAP: u32 = 10;
1492const DELIVERY_RATE_WINDOW_MS: u64 = 60_000;
1493const DELIVERY_RATE_WINDOWS_RETAINED: u64 = 2;
1494const DELIVERY_CLOCK_STEP_MS: u64 = 1_000;
1495const GATING_APPROVAL_TIMEOUT_DEFAULT_MS: u64 = 60_000;
1496/// Lower bound on a caller-supplied R3 approval timeout. Without it a value of
1497/// `0` (or a few ms) makes `deadline_at_ms == created_at_ms`, so the pending
1498/// entry is treated as already-expired by `refresh_gating_timeouts` on the very
1499/// next gating RPC and is replaced with a `timeout_fallback` SafeDraft before
1500/// any approver can act — R3 approval can never complete. 1 second is the floor
1501/// that keeps a `PendingApproval` outcome actually approvable.
1502const GATING_APPROVAL_TIMEOUT_MIN_MS: u64 = 1_000;
1503/// Upper bound on a caller-supplied R3 approval timeout. Without it a huge
1504/// `approval_timeout_ms` makes `created_at_ms.saturating_add(timeout_ms)`
1505/// saturate to `u64::MAX`, so `now_ms >= deadline_at_ms` is never true and the
1506/// pending approval can never time out to a safe draft. 7 days is far beyond
1507/// any real human-approval window.
1508const GATING_APPROVAL_TIMEOUT_MAX_MS: u64 = 7 * 24 * 60 * 60 * 1_000;
1509const GATING_AUDIT_MAX_RETAINED: usize = 512;
1510const GATING_PENDING_MAX_RETAINED: usize = 512;
1511const MEMORY_ASSERTIONS_MAX_RETAINED: usize = 4_096;
1512const MEMORY_SUPPORTED_STORES: [&str; 5] = [
1513    "knowledge_graph",
1514    "vector",
1515    "timeline",
1516    "todo",
1517    "top_of_mind",
1518];
1519const MEMORY_LEDGER_HEALTHCHECK_TIMEOUT: Duration = Duration::from_secs(2);
1520// Multi-year bounded lookback so sparse valid cron schedules (for example leap-day)
1521// are not silently skipped when polling cadence is coarse.
1522const CRON_LOOKBACK_MINUTES: u64 = 5_270_400;
1523// Per-request ceiling on TOTAL cron lookback iterations across all schedules in
1524// one evaluate/dispatch call. Each schedule may still walk back the full
1525// `CRON_LOOKBACK_MINUTES` (≈10y) for a legitimately-sparse cron such as
1526// `0 0 29 2 *` (Feb 29, ≈1.5M iterations), but a request packed with many such
1527// crons cannot multiply that cost without bound. The ceiling allows several
1528// worst-case sparse crons per request (well above any real schedule set) while
1529// capping the adversarial `MAX_SCHEDULES_PER_REQUEST * ~1.5M` (~380M) blow-up
1530// that would otherwise stall the tokio worker and starve the module mutex.
1531const CRON_LOOKBACK_BUDGET_PER_REQUEST: u64 = CRON_LOOKBACK_MINUTES.saturating_mul(8);
1532const CONSOLE_EXPERIENCE_ROUTE: &str = "/console/experience";
1533const CONSOLE_MODULES_ROUTE: &str = "/console/modules";
1534const EVENTS_SUBSCRIBE_METHOD: &str = "mobkit/events/subscribe";
1535
1536fn default_delivery_history_limit() -> usize {
1537    DELIVERY_HISTORY_LIMIT_DEFAULT
1538}
1539
1540pub fn run_meerkat_baseline_verification_once(
1541    command: &str,
1542    args: &[String],
1543    env: &[(String, String)],
1544    timeout: Duration,
1545) -> Result<BaselineVerificationReport, BaselineRuntimeError> {
1546    let line = run_process_json_line(command, args, env, timeout)
1547        .map_err(BaselineRuntimeError::Process)?;
1548    let value: Value =
1549        serde_json::from_str(&line).map_err(|_| BaselineRuntimeError::InvalidRepoPathJson)?;
1550    let repo = value
1551        .as_object()
1552        .and_then(|obj| obj.get("repo_root"))
1553        .ok_or(BaselineRuntimeError::MissingRepoRoot)?
1554        .as_str()
1555        .ok_or(BaselineRuntimeError::InvalidRepoRoot)?;
1556    if repo.trim().is_empty() {
1557        return Err(BaselineRuntimeError::InvalidRepoRoot);
1558    }
1559    verify_meerkat_baseline_symbols(Some(std::path::Path::new(repo)))
1560        .map_err(BaselineRuntimeError::Baseline)
1561}
1562
1563pub fn build_runtime_decision_state(
1564    input: RuntimeDecisionInputs,
1565) -> Result<RuntimeDecisionState, DecisionRuntimeError> {
1566    validate_bigquery_naming(&input.bigquery).map_err(DecisionRuntimeError::Policy)?;
1567    let modules = load_trusted_mobkit_modules_from_toml(&input.trusted_mobkit_toml)
1568        .map_err(DecisionRuntimeError::Policy)?;
1569    validate_runtime_ops_policy(&input.ops).map_err(DecisionRuntimeError::Policy)?;
1570    let release_metadata = parse_release_metadata_json(&input.release_metadata_json)
1571        .map_err(DecisionRuntimeError::Policy)?;
1572    validate_release_metadata(&release_metadata).map_err(DecisionRuntimeError::Policy)?;
1573    if input.trusted_oidc.audience.trim().is_empty() {
1574        return Err(DecisionRuntimeError::Policy(
1575            DecisionPolicyError::InvalidTrustedAuthConfig(
1576                "trusted OIDC audience must not be empty".to_string(),
1577            ),
1578        ));
1579    }
1580    parse_oidc_discovery_json(&input.trusted_oidc.discovery_json).map_err(|err| {
1581        DecisionRuntimeError::Policy(DecisionPolicyError::InvalidTrustedAuthConfig(format!(
1582            "invalid trusted OIDC discovery: {err:?}"
1583        )))
1584    })?;
1585    parse_jwks_json(&input.trusted_oidc.jwks_json).map_err(|err| {
1586        DecisionRuntimeError::Policy(DecisionPolicyError::InvalidTrustedAuthConfig(format!(
1587            "invalid trusted JWKS: {err:?}"
1588        )))
1589    })?;
1590
1591    Ok(RuntimeDecisionState {
1592        bigquery: input.bigquery,
1593        modules,
1594        auth: input.auth,
1595        trusted_oidc: input.trusted_oidc,
1596        console: input.console,
1597        ops: input.ops,
1598        release_metadata,
1599    })
1600}
1601
1602fn current_time_ms() -> u64 {
1603    SystemTime::now()
1604        .duration_since(UNIX_EPOCH)
1605        .unwrap_or_default()
1606        .as_millis() as u64
1607}