use std::collections::HashMap;
use std::path::PathBuf;
use std::sync::Arc;
use khive_runtime::{
config_from_env, run_migrations, runtime_config_from_khive_config, BackendConfig, BackendId,
BackendKind, ConnectionPool, KhiveConfig, KhiveRuntime, OutputFormat, RuntimeConfig,
StorageBackend,
};
use crate::args::{resolve_cli_namespace, Args};
use crate::server::KhiveMcpServer;
use crate::transport::{ServeOptions, TransportRegistry};
pub struct MultiBackendRegistry {
pub registry: khive_runtime::VerbRegistry,
pub default_namespace: String,
pub config_id: String,
pub per_pack_runtimes: HashMap<String, Arc<KhiveRuntime>>,
pub main_backend: Arc<StorageBackend>,
}
pub async fn run(args: Args, registry: &TransportRegistry) -> anyhow::Result<()> {
if let Some(generation) = args.resumed_generation {
tracing::warn!(
generation,
"bridge self-heal: this process is a resumed generation of an \
in-place re-exec triggered by a stale daemon-protocol mismatch (#714)"
);
}
#[cfg(unix)]
let boot_guard = if args.daemon {
Some(khive_runtime::daemon::acquire_daemon_boot_guard()?)
} else {
khive_runtime::daemon::acquire_recovery_lock()
};
let (server, schedule_rt) = build_server(&args)?;
#[cfg(feature = "channel-email")]
spawn_email_channel_loops_if_daemon(&server, &args);
spawn_schedule_tick_loop_if_daemon(&args, &server, schedule_rt);
#[cfg(unix)]
if args.daemon {
khive_runtime::daemon::run_daemon_with_boot_guard(server, boot_guard).await?;
return Ok(());
}
#[cfg(unix)]
drop(boot_guard);
#[cfg(not(unix))]
if args.daemon {
anyhow::bail!(
"--daemon mode requires Unix (macOS/Linux). On Windows, use the stdio transport."
);
}
let transport_name = args.transport.as_deref().unwrap_or("stdio");
let transport = registry.get(transport_name).ok_or_else(|| {
anyhow::anyhow!(
"unknown transport {transport_name:?}; registered: {}",
registry.names().join(", ")
)
})?;
let opts = ServeOptions {
bind: args.bind.clone(),
};
transport.serve(server, &opts).await
}
#[cfg(feature = "channel-email")]
fn is_daemon_role(args: &Args) -> bool {
args.daemon
}
#[cfg(feature = "channel-email")]
fn spawn_email_channel_loops_if_daemon(server: &KhiveMcpServer, args: &Args) {
if is_daemon_role(args) {
tracing::info!("email channel loops: spawning (daemon role)");
spawn_email_channel_loops(server);
} else {
tracing::info!("email channel loops: skipped (client role; daemon owns channel loops)");
}
}
fn spawn_schedule_tick_loop_if_daemon(
args: &Args,
server: &KhiveMcpServer,
schedule_rt: Option<KhiveRuntime>,
) {
if !args.daemon {
tracing::info!("schedule tick loop: skipped (client role; daemon owns the tick)");
return;
}
let Some(rt) = schedule_rt else {
tracing::info!(
"schedule tick loop: skipped (\"schedule\" pack is not in this daemon's \
resolved pack set)"
);
return;
};
let interval = crate::pending_events::tick_interval_from_env();
tracing::info!(
interval_secs = interval.as_secs(),
"schedule tick loop: spawning (daemon role)"
);
tokio::spawn(crate::pending_events::schedule_tick_loop(
rt,
server.clone(),
interval,
));
}
#[cfg(feature = "channel-email")]
fn spawn_email_channel_loops(server: &KhiveMcpServer) {
use khive_channel::ChannelRegistry;
use khive_channel_email::EmailChannel;
use std::sync::Arc;
match EmailChannel::from_env() {
Ok(email_ch) => {
let email_ch = Arc::new(email_ch);
let mut ch_registry = ChannelRegistry::new();
let dyn_ch: Arc<dyn khive_channel::Channel> = email_ch.clone();
ch_registry.register(dyn_ch);
let ch_registry = Arc::new(ch_registry);
let verb_reg = server.verb_registry_clone();
let ingest_ns = ingest_namespace_from_env();
let default_actor = default_inbound_actor_from_env();
let mut allowlist = allowed_recipients_from_env();
if allowlist.is_empty() {
allowlist.push(email_ch.maintainer_address().to_string());
}
let mailbox = email_ch.mailbox().to_string();
let ingest_ns_clone = ingest_ns.clone();
let default_actor_clone = default_actor.clone();
let verb_reg_poll = verb_reg.clone();
let verb_reg_outbox = verb_reg.clone();
let ingest_ns_outbox = ingest_ns.clone();
let allowlist_clone = allowlist.clone();
let mailbox_clone = mailbox.clone();
let email_ch_clone = Arc::clone(&email_ch);
let spawned = run_if_authorized(&ingest_ns, &verb_reg, || {
tokio::task::spawn(channel_poll_loop(
ch_registry,
verb_reg_poll,
ingest_ns_clone,
default_actor_clone,
));
tokio::task::spawn(channel_outbox_loop(
email_ch_clone,
verb_reg_outbox,
ingest_ns_outbox,
mailbox_clone,
allowlist_clone,
));
tracing::info!("email channel polling and outbox loops started");
});
if !spawned {
tracing::error!(
namespace = %ingest_ns,
"email channel loops NOT started: ingest namespace authorization failed (fail-closed)"
);
}
}
Err(e) => {
tracing::warn!(
"channel-email feature is enabled but configuration is incomplete: {e}; \
email polling is disabled"
);
}
}
}
#[cfg(feature = "channel-email")]
fn ingest_namespace_from_env() -> String {
std::env::var("KHIVE_EMAIL_INGEST_NAMESPACE")
.ok()
.filter(|s| !s.trim().is_empty())
.unwrap_or_else(|| "local".to_string())
}
#[cfg(feature = "channel-email")]
fn default_inbound_actor_from_env() -> String {
std::env::var("KHIVE_EMAIL_DEFAULT_ACTOR")
.ok()
.filter(|s| !s.trim().is_empty())
.unwrap_or_else(|| "lambda:leo".to_string())
}
#[cfg(feature = "channel-email")]
fn allowed_recipients_from_env() -> Vec<String> {
std::env::var("KHIVE_EMAIL_SEND_ALLOWED_RECIPIENTS")
.ok()
.map(|s| {
s.split(',')
.map(|r| r.trim().to_string())
.filter(|r| !r.is_empty())
.collect()
})
.unwrap_or_default()
}
#[cfg(feature = "channel-email")]
fn run_if_authorized(
ns_str: &str,
registry: &khive_runtime::VerbRegistry,
on_authorized: impl FnOnce(),
) -> bool {
if preflight_ingest_namespace(ns_str, registry) {
on_authorized();
true
} else {
false
}
}
#[cfg(feature = "channel-email")]
fn preflight_ingest_namespace(ns_str: &str, registry: &khive_runtime::VerbRegistry) -> bool {
match khive_runtime::Namespace::parse(ns_str) {
Ok(ns) => match registry.authorize_namespace(ns) {
Ok(()) => true,
Err(e) => {
tracing::error!(
namespace = %ns_str,
error = %e,
"ingest namespace authorization denied; email polling will not start"
);
false
}
},
Err(e) => {
tracing::error!(
namespace = %ns_str,
error = %e,
"invalid ingest namespace string; email polling will not start"
);
false
}
}
}
#[cfg(feature = "channel-email")]
async fn channel_poll_loop(
channels: std::sync::Arc<khive_channel::ChannelRegistry>,
registry: khive_runtime::VerbRegistry,
ingest_namespace: String,
default_inbound_actor: String,
) {
use chrono::{DateTime, Utc};
use khive_channel_email::{is_backoff_eligible, ImapBackoff};
use serde_json::json;
use std::collections::HashMap;
use std::time::Duration;
const HAPPY_PATH_INTERVAL: Duration = Duration::from_secs(5);
let mut bootstrap_since: HashMap<(String, String), DateTime<Utc>> = HashMap::new();
let mut backoffs: HashMap<(String, String), ImapBackoff> = HashMap::new();
let mut last_error_class: HashMap<(String, String), &'static str> = HashMap::new();
let mut next_interval = HAPPY_PATH_INTERVAL;
let event_store = registry.event_store();
let startup_since = Utc::now();
loop {
tokio::time::sleep(next_interval).await;
next_interval = HAPPY_PATH_INTERVAL;
let now = Utc::now();
for (kind, slug, channel) in channels.iter() {
let backoff_key = (kind.to_string(), slug.to_string());
let since = *bootstrap_since
.entry(backoff_key.clone())
.or_insert(startup_since);
let mut bootstrap_floor_advances = false;
append_channel_lifecycle_event(
event_store.as_ref(),
khive_types::EventKind::ChannelPollStarted,
khive_storage::ChannelPollStartedPayload {
channel_kind: kind.to_string(),
channel_slug: slug.to_string(),
since_rfc3339: since.to_rfc3339(),
},
)
.await;
let checkpoint = match load_channel_cursor(®istry, kind, slug).await {
Ok(cp) => cp,
Err(e) => {
tracing::warn!(
channel = kind,
"comm.cursor_get failed; skipping this channel's poll this cycle: {e}"
);
continue;
}
};
match channel.poll_page(since, checkpoint.as_ref()).await {
Ok(page) => {
let prior_attempt =
backoffs.get(&backoff_key).map(|b| b.attempt()).unwrap_or(0);
if let Some(backoff) = backoffs.get_mut(&backoff_key) {
backoff.record_success();
}
last_error_class.remove(&backoff_key);
if prior_attempt > 0 {
append_channel_lifecycle_event(
event_store.as_ref(),
khive_types::EventKind::ChannelPollSucceeded,
khive_storage::ChannelPollSucceededPayload {
channel_kind: kind.to_string(),
channel_slug: slug.to_string(),
envelope_count: page.envelopes.len(),
previous_backoff_attempt: prior_attempt,
},
)
.await;
append_channel_lifecycle_event(
event_store.as_ref(),
khive_types::EventKind::ChannelBackoffReset,
khive_storage::ChannelBackoffResetPayload {
channel_kind: kind.to_string(),
channel_slug: slug.to_string(),
previous_backoff_attempt: prior_attempt,
},
)
.await;
}
record_channel_heartbeat(
®istry,
kind,
slug,
HeartbeatOutcome::Success,
event_store.as_ref(),
)
.await;
let mut page_fully_ingested = true;
for env in page.envelopes {
let params = json!({
"namespace": ingest_namespace,
"from": env.from,
"to": env.to,
"content": env.content,
"subject": env.subject,
"channel_kind": kind,
"external_id": env.external_id,
"sent_at": env.sent_at.map(|ts| ts.to_rfc3339()),
"correlation_external_id": env.correlation_external_id,
"default_inbound_actor": default_inbound_actor,
"wire_message_id": env.wire_message_id,
"wire_references": env.wire_references,
"metadata": env.metadata,
});
if let Err(e) = registry.dispatch("comm.ingest", params).await {
tracing::warn!(
channel = kind,
"comm.ingest failed for inbound message: {e}"
);
page_fully_ingested = false;
}
}
if page_fully_ingested {
match page.next_checkpoint {
Some(next_checkpoint) => {
match commit_channel_cursor(®istry, kind, slug, &next_checkpoint)
.await
{
Ok(()) => bootstrap_floor_advances = true,
Err(e) => {
tracing::warn!(
channel = kind,
"comm.cursor_commit failed; progress not durably \
advanced, next poll will retry: {e}"
);
}
}
}
None => bootstrap_floor_advances = true,
}
} else {
tracing::warn!(
channel = kind,
"not committing IMAP cursor: at least one message in this page \
failed comm.ingest; the whole page will be retried next poll"
);
}
}
Err(e) => {
let class = channel_error_class(&e);
record_channel_heartbeat(
®istry,
kind,
slug,
HeartbeatOutcome::Failure {
class,
message: e.to_string(),
},
event_store.as_ref(),
)
.await;
if last_error_class.get(&backoff_key) != Some(&class) {
last_error_class.insert(backoff_key.clone(), class);
append_channel_lifecycle_event(
event_store.as_ref(),
khive_types::EventKind::ChannelPollFailed,
khive_storage::ChannelPollFailedPayload {
channel_kind: kind.to_string(),
channel_slug: slug.to_string(),
error_class: class.to_string(),
error_message: e.to_string(),
},
)
.await;
}
if is_backoff_eligible(&e) {
let backoff = backoffs.entry(backoff_key).or_default();
let tick = backoff.record_failure();
log_eligible_poll_failure(kind, &e, &tick);
next_interval = next_interval.max(tick.delay);
if tick.should_warn {
append_channel_lifecycle_event(
event_store.as_ref(),
khive_types::EventKind::ChannelBackoffArmed,
khive_storage::ChannelBackoffArmedPayload {
channel_kind: kind.to_string(),
channel_slug: slug.to_string(),
attempt: tick.attempt,
step_ms: tick.step.as_millis() as u64,
delay_ms: tick.delay.as_millis() as u64,
},
)
.await;
}
} else {
tracing::warn!(channel = kind, "channel poll failed: {e}");
}
}
}
if bootstrap_floor_advances {
bootstrap_since.insert((kind.to_string(), slug.to_string()), now);
}
}
}
}
#[cfg(feature = "channel-email")]
async fn append_channel_lifecycle_event<P: serde::Serialize>(
store: Option<&std::sync::Arc<dyn khive_storage::EventStore>>,
kind: khive_types::EventKind,
payload: P,
) {
let Some(store) = store else {
return;
};
let payload_value = match serde_json::to_value(&payload) {
Ok(v) => v,
Err(e) => {
tracing::warn!(
error = %e,
event_kind = %kind.name(),
"failed to serialize channel lifecycle event payload"
);
return;
}
};
let event = khive_storage::Event::new(
khive_pack_comm::CHANNEL_HEALTH_NAMESPACE,
"channel.poll_lifecycle",
kind,
khive_types::SubstrateKind::Event,
"daemon:channel_poll_loop",
)
.with_payload(payload_value);
if let Err(err) = store.append_event(event).await {
tracing::warn!(
error = %err,
event_kind = %kind.name(),
"channel lifecycle event append failed"
);
}
}
#[cfg(feature = "channel-email")]
enum HeartbeatOutcome {
Success,
Failure {
class: &'static str,
message: String,
},
}
#[cfg(feature = "channel-email")]
fn channel_error_class(err: &khive_channel::ChannelError) -> &'static str {
match err {
khive_channel::ChannelError::Auth(_) => "auth",
khive_channel::ChannelError::Transport(_) => "transport",
khive_channel::ChannelError::Config(_)
| khive_channel::ChannelError::UnauthorizedSender(_)
| khive_channel::ChannelError::InvalidEnvelope(_) => "config",
}
}
#[cfg(feature = "channel-email")]
async fn record_channel_heartbeat(
registry: &khive_runtime::VerbRegistry,
channel_kind: &str,
channel_slug: &str,
outcome: HeartbeatOutcome,
event_store: Option<&std::sync::Arc<dyn khive_storage::EventStore>>,
) {
use serde_json::json;
let namespace = khive_pack_comm::CHANNEL_HEALTH_NAMESPACE;
let params = match &outcome {
HeartbeatOutcome::Success => json!({
"namespace": namespace,
"channel_kind": channel_kind,
"channel_slug": channel_slug,
"outcome": "success",
}),
HeartbeatOutcome::Failure { class, message } => json!({
"namespace": namespace,
"channel_kind": channel_kind,
"channel_slug": channel_slug,
"outcome": "failure",
"error_class": class,
"error_message": message,
}),
};
if let Err(e) = registry.dispatch("comm.heartbeat", params).await {
tracing::warn!(
channel = channel_kind,
"comm.heartbeat failed to persist poll outcome: {e}"
);
append_channel_lifecycle_event(
event_store,
khive_types::EventKind::ChannelHeartbeatPersistFailed,
khive_storage::ChannelHeartbeatPersistFailedPayload {
channel_kind: channel_kind.to_string(),
channel_slug: channel_slug.to_string(),
error: e.to_string(),
},
)
.await;
}
}
#[cfg(feature = "channel-email")]
async fn load_channel_cursor(
registry: &khive_runtime::VerbRegistry,
channel_kind: &str,
channel_slug: &str,
) -> Result<Option<khive_channel::StoredChannelCheckpoint>, khive_runtime::RuntimeError> {
use serde_json::json;
let value = registry
.dispatch(
"comm.cursor_get",
json!({
"channel_kind": channel_kind,
"channel_slug": channel_slug,
}),
)
.await?;
if value.is_null() {
return Ok(None);
}
serde_json::from_value(value).map(Some).map_err(|e| {
khive_runtime::RuntimeError::Internal(format!(
"comm.cursor_get returned a malformed checkpoint: {e}"
))
})
}
#[cfg(feature = "channel-email")]
async fn commit_channel_cursor(
registry: &khive_runtime::VerbRegistry,
channel_kind: &str,
channel_slug: &str,
checkpoint: &khive_channel::ChannelCheckpoint,
) -> Result<(), khive_runtime::RuntimeError> {
use serde_json::json;
registry
.dispatch(
"comm.cursor_commit",
json!({
"channel_kind": channel_kind,
"channel_slug": channel_slug,
"source": checkpoint.source,
"generation": checkpoint.generation,
"high_water": checkpoint.high_water,
}),
)
.await?;
Ok(())
}
#[cfg(feature = "channel-email")]
fn log_eligible_poll_failure(
kind: &str,
err: &khive_channel::ChannelError,
tick: &khive_channel_email::BackoffTick,
) {
if tick.should_warn {
tracing::warn!(
channel = kind,
attempt = tick.attempt,
delay_secs = tick.delay.as_secs_f64(),
"IMAP poll backoff escalating after connect/auth failure: {err}"
);
} else {
tracing::debug!(
channel = kind,
attempt = tick.attempt,
delay_secs = tick.delay.as_secs_f64(),
"channel poll failed, holding at current backoff step: {err}"
);
}
}
#[cfg(feature = "channel-email")]
fn note_already_delivered(props: &serde_json::Map<String, serde_json::Value>) -> bool {
props
.get("delivered_at")
.map(|v| !v.is_null())
.unwrap_or(false)
}
#[cfg(feature = "channel-email")]
async fn channel_outbox_loop(
email_channel: std::sync::Arc<khive_channel_email::EmailChannel>,
registry: khive_runtime::VerbRegistry,
ingest_namespace: String,
mailbox: String,
allowlist: Vec<String>,
) {
use chrono::Utc;
use khive_channel::{Channel, ChannelEnvelope};
use serde_json::json;
let domain = mailbox.split('@').nth(1).unwrap_or("localhost").to_string();
loop {
tokio::time::sleep(std::time::Duration::from_secs(5)).await;
let list_params = json!({
"namespace": ingest_namespace,
"kind": "message",
"direction": "outbound",
"delivered": false,
"limit": 200,
});
let list_result = match registry.dispatch("list", list_params).await {
Ok(r) => r,
Err(e) => {
tracing::warn!(error = %e, "outbox loop: list failed");
continue;
}
};
let notes = match list_result.as_array() {
Some(arr) => arr.clone(),
None => continue,
};
for note_val in notes {
let props = match note_val.get("properties") {
Some(serde_json::Value::Object(m)) => m.clone(),
_ => continue,
};
if props.get("direction").and_then(|v| v.as_str()) != Some("outbound") {
continue;
}
let to_actor = match props.get("to_actor").and_then(|v| v.as_str()) {
Some(a) if a.starts_with("email:") => a.to_string(),
_ => continue,
};
if note_already_delivered(&props) {
continue;
}
let note_id = match note_val.get("id").and_then(|v| v.as_str()) {
Some(id) => id.to_string(),
None => continue,
};
let recipient = to_actor
.strip_prefix("email:")
.unwrap_or(to_actor.as_str())
.to_string();
if !allowlist.is_empty() && !allowlist.contains(&recipient) {
tracing::warn!(
note_id = %note_id,
recipient = %recipient,
"outbox loop: recipient not in allowlist; skipping"
);
continue;
}
let subject = props
.get("subject")
.and_then(|v| v.as_str())
.unwrap_or("(no subject)")
.to_string();
let content = match note_val.get("content").and_then(|v| v.as_str()) {
Some(c) => c.to_string(),
None => continue,
};
let thread_id = props
.get("thread_id")
.and_then(|v| v.as_str())
.map(|s| s.to_string());
let in_reply_to = props
.get("in_reply_to_message_id")
.and_then(|v| v.as_str())
.map(|s| s.to_string());
let references = props
.get("references_chain")
.and_then(|v| v.as_str())
.map(|s| s.to_string());
let message_id = match props.get("external_id").and_then(|v| v.as_str()) {
Some(eid) if !eid.is_empty() => eid.to_string(),
_ => {
let mid = format!("<{note_id}@{domain}>");
let claim_result = registry
.dispatch(
"update",
json!({
"namespace": ingest_namespace,
"id": note_id,
"properties": { "external_id": mid.clone() },
}),
)
.await;
if let Err(e) = claim_result {
tracing::warn!(
note_id = %note_id,
error = %e,
"outbox loop: failed to claim external_id; skipping"
);
continue;
}
mid
}
};
let mut env = ChannelEnvelope::new(
format!("email:{mailbox}"),
format!("email:{recipient}"),
content,
)
.with_subject(subject)
.with_message_id(message_id.clone());
if let Some(tid) = thread_id {
env = env.with_correlation(tid);
}
if let Some(irt) = in_reply_to {
env = env.with_in_reply_to(irt);
}
if let Some(refs) = references {
env = env.with_references(refs);
}
match email_channel.send(env).await {
Ok(()) => {
let delivered_at = Utc::now().to_rfc3339();
let mark_result = registry
.dispatch(
"update",
json!({
"namespace": ingest_namespace,
"id": note_id,
"properties": { "delivered_at": delivered_at },
}),
)
.await;
match mark_result {
Ok(_) => {
tracing::info!(
note_id = %note_id,
recipient = %recipient,
message_id = %message_id,
"outbox loop: delivered"
);
}
Err(e) => {
tracing::warn!(
note_id = %note_id,
error = %e,
"outbox loop: failed to set delivered_at (AT-LEAST-ONCE: will retry)"
);
}
}
}
Err(e) => {
tracing::warn!(
note_id = %note_id,
recipient = %recipient,
error = %e,
"outbox loop: send failed; will retry next cycle"
);
}
}
}
}
}
pub async fn serve_server(
server: KhiveMcpServer,
args: &Args,
registry: &TransportRegistry,
boot_guard: Option<std::fs::File>,
schedule_rt: Option<KhiveRuntime>,
) -> anyhow::Result<()> {
if let Some(generation) = args.resumed_generation {
tracing::warn!(
generation,
"bridge self-heal: this process is a resumed generation of an \
in-place re-exec triggered by a stale daemon-protocol mismatch (#714)"
);
}
#[cfg(feature = "channel-email")]
spawn_email_channel_loops_if_daemon(&server, args);
spawn_schedule_tick_loop_if_daemon(args, &server, schedule_rt);
#[cfg(unix)]
if args.daemon {
khive_runtime::daemon::run_daemon_with_boot_guard(server, boot_guard).await?;
return Ok(());
}
drop(boot_guard);
#[cfg(not(unix))]
if args.daemon {
anyhow::bail!(
"--daemon mode requires Unix (macOS/Linux). On Windows, use the stdio transport."
);
}
let transport_name = args.transport.as_deref().unwrap_or("stdio");
let transport = registry.get(transport_name).ok_or_else(|| {
anyhow::anyhow!(
"unknown transport {transport_name:?}; registered: {}",
registry.names().join(", ")
)
})?;
let opts = ServeOptions {
bind: args.bind.clone(),
};
transport.serve(server, &opts).await
}
pub fn build_registry_for_multi_backend(
base_config: RuntimeConfig,
khive_cfg: &KhiveConfig,
cli_db_override: Option<&str>,
) -> anyhow::Result<MultiBackendRegistry> {
khive_runtime::assert_db_anchor_consistent(base_config.db_path.as_deref(), cli_db_override)?;
let backend_count = khive_cfg.backends.len();
let force_memory = match cli_db_override {
Some(":memory:") => {
tracing::warn!(
"--db :memory: (or KHIVE_DB=:memory:) is overriding {backend_count} \
configured [[backends]] entries to in-memory storage for this invocation; \
khive.toml's declared backend paths will not be used this run"
);
true
}
Some(other) => {
anyhow::bail!(
"--db {other:?} (or KHIVE_DB) cannot be combined with [[backends]]: \
{backend_count} backend(s) are already declared in khive.toml, so applying \
this override here is ambiguous (it could silently collapse distinct \
declared backends onto a single file). Edit khive.toml directly to change \
backend paths, or pass --db :memory: to force all backends in-memory for \
this invocation."
);
}
None => false,
};
let mut backends: HashMap<String, Arc<StorageBackend>> = HashMap::new();
let mut path_to_backend: HashMap<std::path::PathBuf, Arc<StorageBackend>> = HashMap::new();
for backend_cfg in &khive_cfg.backends {
let owned_cfg = if force_memory {
BackendConfig {
kind: BackendKind::Memory,
path: None,
..backend_cfg.clone()
}
} else {
backend_cfg.clone()
};
let backend_cfg = &owned_cfg;
let canonical = canonical_backend_path(backend_cfg)?;
if let Some(ref canon) = canonical {
if let Some(existing) = path_to_backend.get(canon) {
backends.insert(backend_cfg.name.clone(), existing.clone());
continue;
}
}
let backend = open_backend(backend_cfg)?;
{
let mut writer = backend.pool().try_writer().map_err(|e| {
anyhow::anyhow!("backend {}: migration writer: {e}", backend_cfg.name)
})?;
run_migrations(writer.conn_mut())
.map_err(|e| anyhow::anyhow!("backend {}: migration: {e}", backend_cfg.name))?;
}
let arc = Arc::new(backend);
if let Some(canon) = canonical {
path_to_backend.insert(canon, arc.clone());
}
backends.insert(backend_cfg.name.clone(), arc);
}
let main_backend = backends
.get(BackendId::MAIN)
.ok_or_else(|| {
anyhow::anyhow!(
"[[backends]] is declared but no backend named \"main\" was found; \
add a [[backends]] entry with name = \"main\""
)
})?
.clone();
let pack_names = &base_config.packs;
let mut per_pack_runtimes_local: HashMap<String, KhiveRuntime> = HashMap::new();
for pack_name in pack_names {
let (backend_name, backend) = match khive_cfg.packs.get(pack_name.as_str()) {
None => (BackendId::MAIN, main_backend.clone()),
Some(pack_cfg) => {
let backend_name = pack_cfg.backend.as_str();
let backend = backends.get(backend_name).cloned().ok_or_else(|| {
let defined = backends.keys().cloned().collect::<Vec<_>>().join(", ");
anyhow::anyhow!(
"[packs.{pack_name}].backend = {backend_name:?} references an unknown backend; defined backends: {defined}"
)
})?;
(backend_name, backend)
}
};
let mut rt_config = base_config.clone();
rt_config.backend_id = BackendId::new(backend_name);
per_pack_runtimes_local.insert(
pack_name.clone(),
build_pack_runtime(backend, backend_name, rt_config, &main_backend),
);
}
let default_runtime = KhiveRuntime::from_backend(main_backend.clone(), {
let mut cfg = base_config.clone();
cfg.backend_id = BackendId::main();
cfg
});
#[cfg(feature = "bench-embedder")]
{
for rt in per_pack_runtimes_local.values() {
for name in rt.registered_embedding_model_names() {
rt.register_embedder(crate::bench_embedder::FeatureHashProvider::new(name));
}
}
for name in default_runtime.registered_embedding_model_names() {
default_runtime
.register_embedder(crate::bench_embedder::FeatureHashProvider::new(name));
}
}
enforce_strict_actor_mode(
default_runtime.config().actor_id.as_deref(),
&default_runtime.config().packs,
)?;
if should_warn_unattributed(
default_runtime.config().actor_id.as_deref(),
&default_runtime.config().packs,
) {
tracing::warn!(
"actor identity resolved to \"local\": comm sends will be stamped from \
\"local\" (unattributed) and comm.inbox will be unscoped (party-line). \
Set KHIVE_ACTOR or --actor to this lambda's id."
);
}
let gate = default_runtime.config().gate.clone();
let default_namespace = default_runtime.config().default_namespace.clone();
let config_id = crate::server::compute_config_id(default_runtime.config(), Some(khive_cfg));
let visible_namespaces = default_runtime.config().visible_namespaces.clone();
let mut builder = khive_runtime::VerbRegistryBuilder::new();
builder.with_gate(gate);
builder.with_default_namespace(default_namespace.as_str());
builder.with_visible_namespaces(visible_namespaces);
builder.with_actor_id(default_runtime.config().actor_id.clone());
if let Ok(tok) = default_runtime.authorize(khive_runtime::Namespace::local()) {
if let Ok(event_store) = default_runtime.events(&tok) {
builder.with_event_store(event_store);
}
}
khive_runtime::PackRegistry::register_packs_with_runtimes(
pack_names,
&per_pack_runtimes_local,
&default_runtime,
&mut builder,
)
.map_err(|e| anyhow::anyhow!("pack registration: {e}"))?;
let registry = builder
.build()
.map_err(|e| anyhow::anyhow!("registry build: {e}"))?;
default_runtime.install_edge_rules(registry.all_edge_rules());
for rt in per_pack_runtimes_local.values() {
rt.install_edge_rules(registry.all_edge_rules());
}
registry.call_register_embedders(&default_runtime);
registry.call_register_entity_type_validators(&default_runtime);
registry.call_register_note_mutation_hooks(&default_runtime);
let backend_for_pack: HashMap<&str, &StorageBackend> = per_pack_runtimes_local
.iter()
.map(|(name, rt)| (name.as_str(), rt.backend()))
.collect();
let main_ref: &StorageBackend = main_backend.as_ref();
registry
.apply_schema_plans_with_map(&backend_for_pack, main_ref)
.map_err(|e| anyhow::anyhow!("pack schema boot failure: {e}"))?;
let per_pack_runtimes_arc: HashMap<String, Arc<KhiveRuntime>> = per_pack_runtimes_local
.into_iter()
.map(|(k, v)| (k, Arc::new(v)))
.collect();
Ok(MultiBackendRegistry {
registry,
default_namespace: default_namespace.as_str().to_string(),
config_id,
per_pack_runtimes: per_pack_runtimes_arc,
main_backend,
})
}
pub(crate) fn should_warn_unattributed(actor_id: Option<&str>, loaded_packs: &[String]) -> bool {
khive_runtime::should_warn_unattributed_actor(actor_id, loaded_packs)
}
pub(crate) fn is_strict_actor_mode() -> bool {
std::env::var("KHIVE_REQUIRE_ATTRIBUTED_ACTOR")
.map(|v| v.trim() == "1")
.unwrap_or(false)
}
pub fn enforce_strict_actor_mode(
actor_id: Option<&str>,
loaded_packs: &[String],
) -> anyhow::Result<()> {
if is_strict_actor_mode() && should_warn_unattributed(actor_id, loaded_packs) {
anyhow::bail!(
"KHIVE_REQUIRE_ATTRIBUTED_ACTOR=1 is set but no actor identity is \
configured. Set KHIVE_ACTOR or --actor to this lambda's id before \
starting in strict mode (comm pack requires an attributed actor to \
prevent party-line inbox exposure)."
);
}
Ok(())
}
pub fn build_server(args: &Args) -> anyhow::Result<(KhiveMcpServer, Option<KhiveRuntime>)> {
let (cli_namespace_explicit, cli_namespace) =
resolve_cli_namespace(args).map_err(|e| anyhow::anyhow!("{e}"))?;
build_server_with_explicit_namespace(
args,
cli_namespace,
cli_namespace_explicit,
cli_namespace_explicit,
)
}
pub fn build_server_with_explicit_namespace(
args: &Args,
namespace: khive_runtime::Namespace,
namespace_explicit: bool,
actor_explicit: bool,
) -> anyhow::Result<(KhiveMcpServer, Option<KhiveRuntime>)> {
let config = resolve_runtime_config(RuntimeConfigInputs {
db: args.db.as_deref(),
config: args.config.as_deref(),
namespace,
namespace_explicit,
actor_explicit,
no_embed: args.no_embed,
packs: if args.pack.is_empty() {
None
} else {
Some(args.pack.clone())
},
brain_profile: args.brain_profile.clone(),
})?;
khive_runtime::assert_db_anchor_consistent(config.db_path.as_deref(), args.db.as_deref())?;
let db_path_for_config = config_discovery_db_anchor(args.db.as_deref());
let khive_cfg =
KhiveConfig::load_with_home_fallback(args.config.as_deref(), db_path_for_config.as_deref())
.map_err(|e| anyhow::anyhow!("config error: {e}"))?
.unwrap_or_default();
if khive_cfg.backends.is_empty() {
let runtime = KhiveRuntime::new(config)?;
#[cfg(feature = "bench-embedder")]
{
for name in runtime.registered_embedding_model_names() {
runtime.register_embedder(crate::bench_embedder::FeatureHashProvider::new(name));
}
}
enforce_strict_actor_mode(
runtime.config().actor_id.as_deref(),
&runtime.config().packs,
)?;
if should_warn_unattributed(
runtime.config().actor_id.as_deref(),
&runtime.config().packs,
) {
tracing::warn!(
"actor identity resolved to \"local\": comm sends will be stamped from \
\"local\" (unattributed) and comm.inbox will be unscoped (party-line). \
Set KHIVE_ACTOR or --actor to this lambda's id."
);
}
let schedule_rt = runtime
.config()
.packs
.iter()
.any(|p| p == "schedule")
.then(|| runtime.clone());
let fmt = apply_env_output_format(khive_cfg.runtime.default_output_format);
let server = KhiveMcpServer::new(runtime)
.map(|s| s.with_default_output_format(fmt))
.map_err(|e| anyhow::anyhow!("{e}"))?;
return Ok((server, schedule_rt));
}
let multi = build_registry_for_multi_backend(config, &khive_cfg, args.db.as_deref())?;
let schedule_rt = multi
.per_pack_runtimes
.get("schedule")
.map(|rt| (**rt).clone());
let server = build_server_from_multi_backend_registry(multi, &khive_cfg, None);
Ok((server, schedule_rt))
}
fn canonical_backend_path(cfg: &BackendConfig) -> anyhow::Result<Option<PathBuf>> {
if cfg.kind == BackendKind::Memory {
return Ok(None);
}
let path = match cfg.path.as_ref() {
Some(p) => expand_tilde(p),
None => return Ok(None),
};
let parent = path
.parent()
.ok_or_else(|| anyhow::anyhow!("backend {}: path has no parent directory", cfg.name))?;
let file_name = path
.file_name()
.ok_or_else(|| anyhow::anyhow!("backend {}: path has no file name", cfg.name))?;
std::fs::create_dir_all(parent).map_err(|e| {
anyhow::anyhow!(
"backend {}: cannot create parent dir {}: {e}",
cfg.name,
parent.display()
)
})?;
let canon_parent = parent.canonicalize().map_err(|e| {
anyhow::anyhow!(
"backend {}: cannot canonicalize parent dir {}: {e}",
cfg.name,
parent.display()
)
})?;
Ok(Some(canon_parent.join(file_name)))
}
pub fn build_server_multi_backend(
base_config: RuntimeConfig,
khive_cfg: &KhiveConfig,
cli_db_override: Option<&str>,
) -> anyhow::Result<KhiveMcpServer> {
let multi = build_registry_for_multi_backend(base_config, khive_cfg, cli_db_override)?;
Ok(build_server_from_multi_backend_registry(
multi, khive_cfg, None,
))
}
pub fn build_server_from_multi_backend_registry(
multi: MultiBackendRegistry,
khive_cfg: &KhiveConfig,
coordinator: Option<Arc<dyn crate::coordinator::CoordinatorService>>,
) -> KhiveMcpServer {
let pool = checkpoint_pool_for(multi.main_backend.as_ref());
let fmt = apply_env_output_format(khive_cfg.runtime.default_output_format);
let server = KhiveMcpServer::from_registry_with_meta(
multi.registry,
&multi.default_namespace,
&multi.config_id,
)
.with_default_output_format(fmt);
let server = match coordinator {
Some(c) => server.with_coordinator(c),
None => server,
};
match pool {
Some(p) => server.with_pool(p),
None => server,
}
}
#[derive(Debug, PartialEq, Eq)]
pub struct WiringSurface {
pub has_checkpoint_pool: bool,
pub output_format: OutputFormat,
#[cfg(feature = "channel-email")]
pub channel_loop_eligible: bool,
}
impl WiringSurface {
pub fn capture(server: &KhiveMcpServer) -> Self {
Self {
has_checkpoint_pool: server.pool().is_some(),
output_format: server.default_output_format(),
#[cfg(feature = "channel-email")]
channel_loop_eligible: preflight_ingest_namespace(
&ingest_namespace_from_env(),
&server.verb_registry_clone(),
),
}
}
}
pub fn checkpoint_pool_for(main_backend: &StorageBackend) -> Option<Arc<ConnectionPool>> {
if main_backend.is_file_backed() {
Some(main_backend.pool_arc())
} else {
None
}
}
fn build_pack_runtime(
backend: Arc<StorageBackend>,
backend_name: &str,
rt_config: RuntimeConfig,
main_backend: &Arc<StorageBackend>,
) -> KhiveRuntime {
let rt = KhiveRuntime::from_backend(backend, rt_config);
if backend_name != BackendId::MAIN {
rt.with_core_backend(main_backend.clone())
} else {
rt
}
}
fn open_backend(cfg: &BackendConfig) -> anyhow::Result<StorageBackend> {
match cfg.kind {
BackendKind::Memory => StorageBackend::memory()
.map_err(|e| anyhow::anyhow!("backend {}: memory open: {e}", cfg.name)),
BackendKind::Sqlite => {
let path = cfg.path.as_ref().ok_or_else(|| {
anyhow::anyhow!(
"backend {}: sqlite backend requires a `path` field",
cfg.name
)
})?;
let expanded = expand_tilde(path);
if let Some(parent) = expanded.parent() {
std::fs::create_dir_all(parent).map_err(|e| {
anyhow::anyhow!(
"backend {}: cannot create parent dir {}: {e}",
cfg.name,
parent.display()
)
})?;
}
if cfg.read_only {
StorageBackend::sqlite_read_only(&expanded).map_err(|e| {
anyhow::anyhow!("backend {}: sqlite read-only open: {e}", cfg.name)
})
} else {
StorageBackend::sqlite(&expanded)
.map_err(|e| anyhow::anyhow!("backend {}: sqlite open: {e}", cfg.name))
}
}
}
}
fn expand_tilde(path: &std::path::Path) -> PathBuf {
let s = path.to_string_lossy();
if let Some(rest) = s.strip_prefix("~/") {
let home = std::env::var("HOME").unwrap_or_else(|_| ".".into());
PathBuf::from(format!("{home}/{rest}"))
} else if s == "~" {
let home = std::env::var("HOME").unwrap_or_else(|_| ".".into());
PathBuf::from(home)
} else {
path.to_path_buf()
}
}
pub fn config_discovery_db_anchor(db: Option<&str>) -> Option<std::path::PathBuf> {
db.and_then(|d| khive_runtime::resolve_db_anchor(Some(d)))
}
pub struct RuntimeConfigInputs<'a> {
pub db: Option<&'a str>,
pub config: Option<&'a std::path::Path>,
pub namespace: khive_runtime::Namespace,
pub namespace_explicit: bool,
pub actor_explicit: bool,
pub no_embed: bool,
pub packs: Option<Vec<String>>,
pub brain_profile: Option<String>,
}
pub fn resolve_runtime_config(inputs: RuntimeConfigInputs<'_>) -> anyhow::Result<RuntimeConfig> {
let db_path = khive_runtime::resolve_db_anchor(inputs.db);
let packs = inputs
.packs
.unwrap_or_else(|| RuntimeConfig::default().packs);
let cli_brain_profile = inputs.brain_profile.filter(|s| !s.trim().is_empty());
let db_path_for_config = config_discovery_db_anchor(inputs.db);
let resolved = if inputs.no_embed {
let no_embed_base = RuntimeConfig {
db_path,
default_namespace: inputs.namespace,
packs,
brain_profile: cli_brain_profile,
..RuntimeConfig::no_embeddings()
};
resolve_actor_from_config(
inputs.config,
no_embed_base,
inputs.namespace_explicit,
db_path_for_config.as_deref(),
)?
} else {
let base_config = RuntimeConfig {
db_path,
default_namespace: inputs.namespace,
packs,
brain_profile: cli_brain_profile,
..RuntimeConfig::default()
};
resolve_config(inputs.config, base_config, db_path_for_config.as_deref())?
};
let resolved = {
let mut resolved = resolved;
let ns = resolved.default_namespace.as_str().to_string();
if inputs.namespace_explicit && ns != "local" {
resolved.actor_id = Some(ns);
} else if inputs.actor_explicit {
resolved.actor_id = None;
} else {
let project_actor = khive_runtime::resolve_project_actor_id(inputs.config)
.map_err(|e| anyhow::anyhow!("config error: {e}"))?;
resolved.actor_id = project_actor.or(resolved.actor_id);
}
resolved
};
Ok(apply_env_brain_profile(resolved))
}
fn apply_env_brain_profile(mut cfg: RuntimeConfig) -> RuntimeConfig {
if cfg.brain_profile.is_none() {
cfg.brain_profile = std::env::var("KHIVE_BRAIN_PROFILE")
.ok()
.filter(|s| !s.trim().is_empty());
}
cfg
}
pub fn apply_env_output_format(toml_default: Option<OutputFormat>) -> OutputFormat {
if let Ok(val) = std::env::var("KHIVE_OUTPUT_FORMAT") {
match val.trim() {
"json" => return OutputFormat::Json,
"auto" => return OutputFormat::Auto,
"table" => return OutputFormat::Table,
_ => {
tracing::warn!(
value = %val,
"KHIVE_OUTPUT_FORMAT has unknown value; falling back to TOML / builtin default"
);
}
}
}
toml_default.unwrap_or(OutputFormat::Json)
}
fn resolve_config(
config_path: Option<&std::path::Path>,
base: RuntimeConfig,
db_path: Option<&std::path::Path>,
) -> anyhow::Result<RuntimeConfig> {
match KhiveConfig::load_with_home_fallback(config_path, db_path)
.map_err(|e| anyhow::anyhow!("config error: {e}"))?
{
Some(khive_cfg) => {
let env_primary = std::env::var("KHIVE_EMBEDDING_MODEL").ok();
let env_additional = std::env::var("KHIVE_ADDITIONAL_EMBEDDING_MODELS").ok();
if !khive_cfg.engines.is_empty() && (env_primary.is_some() || env_additional.is_some())
{
tracing::warn!(
"khive config [[engines]] present; KHIVE_EMBEDDING_MODEL / \
KHIVE_ADDITIONAL_EMBEDDING_MODELS env vars are overridden"
);
}
Ok(runtime_config_from_khive_config(&khive_cfg, base))
}
None => {
let env_cfg = config_from_env();
if env_cfg.engines.is_empty() {
Ok(base)
} else {
Ok(runtime_config_from_khive_config(&env_cfg, base))
}
}
}
}
fn resolve_actor_from_config(
config_path: Option<&std::path::Path>,
base: RuntimeConfig,
cli_namespace_explicit: bool,
db_path: Option<&std::path::Path>,
) -> anyhow::Result<RuntimeConfig> {
if cli_namespace_explicit {
return Ok(base);
}
match KhiveConfig::load_with_home_fallback(config_path, db_path)
.map_err(|e| anyhow::anyhow!("config error: {e}"))?
{
Some(khive_cfg) => {
let resolved = runtime_config_from_khive_config(&khive_cfg, base);
Ok(RuntimeConfig {
embedding_model: None,
additional_embedding_models: vec![],
..resolved
})
}
None => Ok(base),
}
}
#[cfg(test)]
mod tests {
use super::*;
use khive_runtime::Namespace;
use serial_test::serial;
use std::io::Write;
#[test]
fn config_discovery_db_anchor_unset_is_none() {
assert_eq!(
config_discovery_db_anchor(None),
None,
"unset --db must not anchor discovery on the materialized home default"
);
}
#[test]
fn config_discovery_db_anchor_explicit_matches_resolve_db_anchor() {
assert_eq!(
config_discovery_db_anchor(Some("/tmp/explicit.db")),
khive_runtime::resolve_db_anchor(Some("/tmp/explicit.db")),
"an explicit --db must anchor discovery identically to resolve_db_anchor"
);
}
#[test]
fn config_discovery_db_anchor_memory_sentinel_is_none() {
assert_eq!(config_discovery_db_anchor(Some(":memory:")), None);
}
fn write_config(dir: &std::path::Path, body: &str) -> PathBuf {
let path = dir.join("khive.toml");
let mut f = std::fs::File::create(&path).expect("create config file");
f.write_all(body.as_bytes()).expect("write config");
path
}
#[test]
#[serial]
fn resolver_uses_config_file_engines_over_defaults() {
std::env::remove_var("KHIVE_EMBEDDING_MODEL");
std::env::remove_var("KHIVE_ADDITIONAL_EMBEDDING_MODELS");
let default_cfg = RuntimeConfig::default();
let default_primary = format!("{:?}", default_cfg.embedding_model);
assert!(
!default_cfg.additional_embedding_models.is_empty(),
"precondition: default config has additional engines"
);
let dir = tempfile::tempdir().expect("temp dir");
let path = write_config(
dir.path(),
r#"
[[engines]]
name = "primary"
model = "bge-small-en-v1.5"
default = true
"#,
);
let resolved = resolve_runtime_config(RuntimeConfigInputs {
db: Some(":memory:"),
config: Some(&path),
namespace: Namespace::parse("local").expect("ns"),
namespace_explicit: false,
actor_explicit: false,
no_embed: false,
packs: None,
brain_profile: None,
})
.expect("resolve config");
let resolved_primary = format!("{:?}", resolved.embedding_model);
assert_ne!(
resolved_primary, default_primary,
"resolved primary engine must come from the config file, not the default"
);
assert!(
resolved.embedding_model.is_some(),
"config-file engine must resolve to a primary embedding model"
);
assert!(
resolved.additional_embedding_models.is_empty(),
"config file declares one engine; additional list must be empty (not the default's)"
);
assert_eq!(resolved.db_path, None, ":memory: must map to in-memory db");
}
#[test]
#[serial]
fn resolver_falls_back_to_env_when_config_has_no_engines() {
std::env::remove_var("KHIVE_ADDITIONAL_EMBEDDING_MODELS");
std::env::set_var("KHIVE_EMBEDDING_MODEL", "bge-small-en-v1.5");
let dir = tempfile::tempdir().expect("temp dir");
let path = write_config(
dir.path(),
r#"
[runtime]
brain_profile = "unrelated"
"#,
);
let resolved = resolve_runtime_config(RuntimeConfigInputs {
db: Some(":memory:"),
config: Some(&path),
namespace: Namespace::parse("local").expect("ns"),
namespace_explicit: false,
actor_explicit: false,
no_embed: false,
packs: None,
brain_profile: None,
})
.expect("resolve config");
std::env::remove_var("KHIVE_EMBEDDING_MODEL");
assert_eq!(
format!("{:?}", resolved.embedding_model),
"Some(BgeSmallEnV15)",
"KHIVE_EMBEDDING_MODEL must be applied as the fallback when the \
config file has no [[engines]] block, not treated as ignored"
);
}
#[test]
#[serial]
fn brain_profile_config_beats_env() {
std::env::set_var("KHIVE_BRAIN_PROFILE", "env-profile");
let dir = tempfile::tempdir().expect("temp dir");
let path = write_config(
dir.path(),
r#"
[runtime]
brain_profile = "project-profile"
"#,
);
let resolved = resolve_runtime_config(RuntimeConfigInputs {
db: Some(":memory:"),
config: Some(&path),
namespace: Namespace::parse("local").expect("ns"),
namespace_explicit: false,
actor_explicit: false,
no_embed: false,
packs: None,
brain_profile: None, })
.expect("resolve config");
std::env::remove_var("KHIVE_BRAIN_PROFILE");
assert_eq!(
resolved.brain_profile.as_deref(),
Some("project-profile"),
"project TOML brain_profile must win over KHIVE_BRAIN_PROFILE env var"
);
}
#[test]
#[serial]
fn brain_profile_env_fallback_when_no_toml() {
std::env::set_var("KHIVE_BRAIN_PROFILE", "env-profile");
let dir = tempfile::tempdir().expect("temp dir");
let path = write_config(
dir.path(),
r#"
[[engines]]
name = "primary"
model = "bge-small-en-v1.5"
default = true
"#,
);
let resolved = resolve_runtime_config(RuntimeConfigInputs {
db: Some(":memory:"),
config: Some(&path),
namespace: Namespace::parse("local").expect("ns"),
namespace_explicit: false,
actor_explicit: false,
no_embed: false,
packs: None,
brain_profile: None,
})
.expect("resolve config");
std::env::remove_var("KHIVE_BRAIN_PROFILE");
assert_eq!(
resolved.brain_profile.as_deref(),
Some("env-profile"),
"env var must be used when no CLI flag and no TOML brain_profile is set"
);
}
#[test]
#[serial]
fn brain_profile_cli_wins_over_all() {
std::env::set_var("KHIVE_BRAIN_PROFILE", "env-profile");
let dir = tempfile::tempdir().expect("temp dir");
let path = write_config(
dir.path(),
r#"
[runtime]
brain_profile = "project-profile"
"#,
);
let resolved = resolve_runtime_config(RuntimeConfigInputs {
db: Some(":memory:"),
config: Some(&path),
namespace: Namespace::parse("local").expect("ns"),
namespace_explicit: false,
actor_explicit: false,
no_embed: false,
packs: None,
brain_profile: Some("cli-profile".to_string()), })
.expect("resolve config");
std::env::remove_var("KHIVE_BRAIN_PROFILE");
assert_eq!(
resolved.brain_profile.as_deref(),
Some("cli-profile"),
"CLI --brain-profile must win over both TOML and KHIVE_BRAIN_PROFILE env var"
);
}
#[test]
#[serial]
fn cli_actor_flag_populates_actor_id() {
std::env::remove_var("KHIVE_ACTOR");
let missing_config =
std::path::PathBuf::from("/nonexistent/khive-cli-actor-test/config.toml");
let resolved = resolve_runtime_config(RuntimeConfigInputs {
db: Some(":memory:"),
config: Some(&missing_config),
namespace: Namespace::parse("lambda:agent-x").expect("ns"),
namespace_explicit: true,
actor_explicit: true,
no_embed: true,
packs: None,
brain_profile: None,
})
.expect("resolve config");
assert_eq!(
resolved.actor_id.as_deref(),
Some("lambda:agent-x"),
"--actor flag must populate actor_id (flag==env parity), not just default_namespace"
);
assert_eq!(
resolved.default_namespace.as_str(),
"lambda:agent-x",
"the flag still sets the write namespace"
);
}
#[test]
#[serial]
fn cli_actor_flag_local_stays_anonymous() {
std::env::remove_var("KHIVE_ACTOR");
let missing_config =
std::path::PathBuf::from("/nonexistent/khive-cli-actor-local-test/config.toml");
let resolved = resolve_runtime_config(RuntimeConfigInputs {
db: Some(":memory:"),
config: Some(&missing_config),
namespace: Namespace::parse("local").expect("ns"),
namespace_explicit: true,
actor_explicit: true,
no_embed: true,
packs: None,
brain_profile: None,
})
.expect("resolve config");
assert_eq!(
resolved.actor_id, None,
"explicit --actor local must remain anonymous (no actor_id) so the \
unattributed-comm warning still fires"
);
}
struct SeatEnv {
original_cwd: PathBuf,
original_home: Option<std::ffi::OsString>,
_isolated_home: tempfile::TempDir,
}
impl SeatEnv {
fn enter(project_root: &std::path::Path) -> Self {
let original_cwd = std::env::current_dir().expect("read cwd");
let original_home = std::env::var_os("HOME");
let isolated_home = tempfile::tempdir().expect("isolated HOME tempdir");
std::env::set_current_dir(project_root).expect("chdir into seat project root");
std::env::set_var("HOME", isolated_home.path());
Self {
original_cwd,
original_home,
_isolated_home: isolated_home,
}
}
}
impl Drop for SeatEnv {
fn drop(&mut self) {
let _ = std::env::set_current_dir(&self.original_cwd);
match &self.original_home {
Some(h) => std::env::set_var("HOME", h),
None => std::env::remove_var("HOME"),
}
}
}
#[test]
#[serial]
fn resolve_project_actor_id_reads_cwd_anchored_project_config() {
std::env::remove_var("KHIVE_ACTOR");
let seat_dir = tempfile::tempdir().expect("seat tempdir");
std::fs::create_dir_all(seat_dir.path().join(".khive")).expect("mkdir seat .khive");
std::fs::write(
seat_dir.path().join(".khive/config.toml"),
"[actor]\nid = \"lambda:seat-actor\"\n",
)
.expect("write seat config");
let _seat_env = SeatEnv::enter(seat_dir.path());
assert_eq!(
khive_runtime::resolve_project_actor_id(None).expect("no config error"),
Some("lambda:seat-actor".to_string()),
"resolve_project_actor_id must read the cwd-anchored .khive/config.toml \
regardless of any database directory"
);
}
#[test]
#[serial]
fn seat_shaped_project_actor_resolves_through_full_tier_chain() {
std::env::remove_var("KHIVE_ACTOR");
let seat_dir = tempfile::tempdir().expect("seat tempdir");
std::fs::create_dir_all(seat_dir.path().join(".khive")).expect("mkdir seat .khive");
std::fs::write(
seat_dir.path().join(".khive/config.toml"),
"[actor]\nid = \"lambda:seat-actor\"\n",
)
.expect("write seat config");
let db_dir = tempfile::tempdir().expect("db tempdir");
let khive_dir = db_dir.path().join(".khive");
std::fs::create_dir_all(&khive_dir).expect("mkdir db .khive");
let db_path = khive_dir.join("khive.db");
std::fs::write(&db_path, b"").expect("touch db file");
let db_str = db_path.to_str().expect("utf8 path").to_string();
let _seat_env = SeatEnv::enter(seat_dir.path());
let resolved = resolve_runtime_config(RuntimeConfigInputs {
db: Some(&db_str),
config: None,
namespace: Namespace::parse("local").expect("ns"),
namespace_explicit: false,
actor_explicit: false,
no_embed: true,
packs: None,
brain_profile: None,
})
.expect("resolve seat-shaped config");
assert_eq!(
resolved.actor_id.as_deref(),
Some("lambda:seat-actor"),
"a seat-shaped cwd with its own [actor] must resolve that actor through \
the full discovery path even when the shared db-anchored config \
location carries none — got {:?}",
resolved.actor_id
);
assert_ne!(
resolved.actor_id.as_deref(),
Some("local"),
"must not collapse to the literal namespace string"
);
}
#[test]
#[serial]
fn resolve_runtime_config_unset_db_discovers_cwd_config_over_home() {
std::env::remove_var("KHIVE_ACTOR");
let project_dir = tempfile::tempdir().expect("project tempdir");
std::fs::create_dir_all(project_dir.path().join(".khive")).expect("mkdir project .khive");
std::fs::write(
project_dir.path().join(".khive/config.toml"),
"[runtime]\nbrain_profile = \"cwd-profile\"\n",
)
.expect("write project config");
let seat_env = SeatEnv::enter(project_dir.path());
std::fs::create_dir_all(seat_env._isolated_home.path().join(".khive"))
.expect("mkdir home .khive");
std::fs::write(
seat_env._isolated_home.path().join(".khive/config.toml"),
"[runtime]\nbrain_profile = \"home-profile\"\n",
)
.expect("write home config");
let resolved = resolve_runtime_config(RuntimeConfigInputs {
db: None,
config: None,
namespace: Namespace::parse("local").expect("ns"),
namespace_explicit: false,
actor_explicit: false,
no_embed: true,
packs: None,
brain_profile: None,
})
.expect("resolve unset-db config");
assert_eq!(
resolved.brain_profile.as_deref(),
Some("cwd-profile"),
"unset --db must resolve tier-3 discovery against the project cwd, \
not $HOME/.khive/khive.db's directory — got {:?}",
resolved.brain_profile
);
}
#[test]
#[serial]
fn cli_actor_flag_wins_over_project_config_actor() {
std::env::remove_var("KHIVE_ACTOR");
let seat_dir = tempfile::tempdir().expect("seat tempdir");
std::fs::create_dir_all(seat_dir.path().join(".khive")).expect("mkdir seat .khive");
std::fs::write(
seat_dir.path().join(".khive/config.toml"),
"[actor]\nid = \"lambda:project-actor\"\n",
)
.expect("write seat config");
let _seat_env = SeatEnv::enter(seat_dir.path());
let resolved = resolve_runtime_config(RuntimeConfigInputs {
db: Some(":memory:"),
config: None,
namespace: Namespace::parse("lambda:cli-actor").expect("ns"),
namespace_explicit: true,
actor_explicit: true,
no_embed: true,
packs: None,
brain_profile: None,
})
.expect("resolve config");
assert_eq!(
resolved.actor_id.as_deref(),
Some("lambda:cli-actor"),
"an explicit --actor flag must win over a discovered project-config actor"
);
}
#[test]
#[serial]
fn project_actor_config_beats_khive_actor_env_which_falls_back_to_anonymous() {
std::env::remove_var("KHIVE_ACTOR");
let dir = tempfile::tempdir().expect("temp dir");
let path = write_config(
dir.path(),
r#"
[actor]
id = "lambda:project-actor"
"#,
);
std::env::set_var("KHIVE_ACTOR", "lambda:env-actor");
let with_project_config = resolve_runtime_config(RuntimeConfigInputs {
db: Some(":memory:"),
config: Some(&path),
namespace: Namespace::parse("local").expect("ns"),
namespace_explicit: false,
actor_explicit: false,
no_embed: true,
packs: None,
brain_profile: None,
})
.expect("resolve config with project actor");
let missing_config =
std::path::PathBuf::from("/nonexistent/khive-project-vs-env-test/config.toml");
let without_project_config = resolve_runtime_config(RuntimeConfigInputs {
db: Some(":memory:"),
config: Some(&missing_config),
namespace: Namespace::parse("local").expect("ns"),
namespace_explicit: false,
actor_explicit: false,
no_embed: true,
packs: None,
brain_profile: None,
})
.expect("resolve config without project actor");
std::env::remove_var("KHIVE_ACTOR");
assert_eq!(
with_project_config.actor_id.as_deref(),
Some("lambda:project-actor"),
"a project-config [actor] id must win over KHIVE_ACTOR env"
);
assert_eq!(
without_project_config.actor_id.as_deref(),
Some("lambda:env-actor"),
"KHIVE_ACTOR env must still be used when no project config actor exists"
);
}
#[test]
#[serial]
fn real_clap_path_khive_actor_env_no_longer_wins_over_project_config() {
use clap::Parser;
std::env::remove_var("KHIVE_ACTOR");
let seat_dir = tempfile::tempdir().expect("seat tempdir");
std::fs::create_dir_all(seat_dir.path().join(".khive")).expect("mkdir seat .khive");
std::fs::write(
seat_dir.path().join(".khive/config.toml"),
"[actor]\nid = \"lambda:project-actor\"\n",
)
.expect("write seat config");
let _seat_env = SeatEnv::enter(seat_dir.path());
std::env::set_var("KHIVE_ACTOR", "lambda:env-actor");
let args = Args::try_parse_from(["mcp"]).expect("parse real mcp args");
let (namespace_explicit, namespace) =
resolve_cli_namespace(&args).expect("resolve cli namespace");
let resolved = resolve_runtime_config(RuntimeConfigInputs {
db: Some(":memory:"),
config: None,
namespace,
namespace_explicit,
actor_explicit: namespace_explicit,
no_embed: true,
packs: None,
brain_profile: None,
});
std::env::remove_var("KHIVE_ACTOR");
let resolved = resolved.expect("resolve config");
assert!(
!namespace_explicit,
"KHIVE_ACTOR env alone must NOT make the CLI namespace tier explicit"
);
assert_eq!(
resolved.actor_id.as_deref(),
Some("lambda:project-actor"),
"project-config [actor] id must win over KHIVE_ACTOR env on the real clap path"
);
assert_eq!(
resolved.default_namespace.as_str(),
"local",
"KHIVE_ACTOR env must never set default_namespace, only actor_id"
);
}
#[test]
#[serial]
fn real_clap_path_khive_actor_env_falls_back_to_tier3_actor_id() {
use clap::Parser;
std::env::remove_var("KHIVE_ACTOR");
let seat_dir = tempfile::tempdir().expect("seat tempdir");
let _seat_env = SeatEnv::enter(seat_dir.path());
std::env::set_var("KHIVE_ACTOR", "lambda:env-only-actor");
let args = Args::try_parse_from(["mcp"]).expect("parse real mcp args");
let (namespace_explicit, namespace) =
resolve_cli_namespace(&args).expect("resolve cli namespace");
let resolved = resolve_runtime_config(RuntimeConfigInputs {
db: Some(":memory:"),
config: None,
namespace,
namespace_explicit,
actor_explicit: namespace_explicit,
no_embed: true,
packs: None,
brain_profile: None,
});
std::env::remove_var("KHIVE_ACTOR");
let resolved = resolved.expect("resolve config");
assert!(
!namespace_explicit,
"KHIVE_ACTOR env alone must NOT make the CLI namespace tier explicit"
);
assert_eq!(
resolved.actor_id.as_deref(),
Some("lambda:env-only-actor"),
"KHIVE_ACTOR env must still land as the tier-3 actor_id fallback \
when no project config exists"
);
assert_eq!(
resolved.default_namespace.as_str(),
"local",
"KHIVE_ACTOR env must never set default_namespace, only actor_id"
);
}
#[test]
#[serial]
fn explicit_actor_local_suppresses_project_and_db_actor_tiers() {
std::env::remove_var("KHIVE_ACTOR");
let seat_dir = tempfile::tempdir().expect("seat tempdir");
std::fs::create_dir_all(seat_dir.path().join(".khive")).expect("mkdir seat .khive");
std::fs::write(
seat_dir.path().join(".khive/config.toml"),
"[actor]\nid = \"lambda:seat-actor\"\n",
)
.expect("write seat config");
let db_dir = tempfile::tempdir().expect("db tempdir");
let khive_dir = db_dir.path().join(".khive");
std::fs::create_dir_all(&khive_dir).expect("mkdir db .khive");
std::fs::write(
khive_dir.join("config.toml"),
"[actor]\nid = \"lambda:db-actor\"\n",
)
.expect("write db-anchored config");
let db_path = khive_dir.join("khive.db");
std::fs::write(&db_path, b"").expect("touch db file");
let db_str = db_path.to_str().expect("utf8 path").to_string();
let _seat_env = SeatEnv::enter(seat_dir.path());
let resolved = resolve_runtime_config(RuntimeConfigInputs {
db: Some(&db_str),
config: None,
namespace: Namespace::parse("local").expect("ns"),
namespace_explicit: true,
actor_explicit: true,
no_embed: false,
packs: None,
brain_profile: None,
})
.expect("resolve config");
assert_eq!(
resolved.actor_id, None,
"explicit --actor local must resolve to anonymous even when both a \
project-config and a db-anchored config declare an [actor] id — got {:?}",
resolved.actor_id
);
assert_eq!(
resolved.default_namespace.as_str(),
"local",
"explicit --actor local must keep default_namespace local"
);
}
#[test]
#[serial]
fn config_id_byte_identical_across_different_actor_ids() {
std::env::remove_var("KHIVE_ACTOR");
std::env::remove_var("KHIVE_EMBEDDING_MODEL");
std::env::remove_var("KHIVE_ADDITIONAL_EMBEDDING_MODELS");
let db_dir = tempfile::tempdir().expect("db tempdir");
let khive_dir = db_dir.path().join(".khive");
std::fs::create_dir_all(&khive_dir).expect("mkdir db .khive");
let db_path = khive_dir.join("khive.db");
std::fs::write(&db_path, b"").expect("touch db file");
let db_str = db_path.to_str().expect("utf8 path").to_string();
let seat_a = tempfile::tempdir().expect("seat a");
std::fs::create_dir_all(seat_a.path().join(".khive")).expect("mkdir seat a .khive");
std::fs::write(
seat_a.path().join(".khive/config.toml"),
"[actor]\nid = \"lambda:actor-a\"\n",
)
.expect("write seat a config");
let seat_b = tempfile::tempdir().expect("seat b");
std::fs::create_dir_all(seat_b.path().join(".khive")).expect("mkdir seat b .khive");
std::fs::write(
seat_b.path().join(".khive/config.toml"),
"[actor]\nid = \"lambda:actor-b\"\n",
)
.expect("write seat b config");
let cfg_a = {
let _seat_env = SeatEnv::enter(seat_a.path());
resolve_runtime_config(RuntimeConfigInputs {
db: Some(&db_str),
config: None,
namespace: Namespace::parse("local").expect("ns"),
namespace_explicit: false,
actor_explicit: false,
no_embed: true,
packs: None,
brain_profile: None,
})
.expect("resolve config a")
};
let cfg_b = {
let _seat_env = SeatEnv::enter(seat_b.path());
resolve_runtime_config(RuntimeConfigInputs {
db: Some(&db_str),
config: None,
namespace: Namespace::parse("local").expect("ns"),
namespace_explicit: false,
actor_explicit: false,
no_embed: true,
packs: None,
brain_profile: None,
})
.expect("resolve config b")
};
assert_eq!(cfg_a.actor_id.as_deref(), Some("lambda:actor-a"));
assert_eq!(cfg_b.actor_id.as_deref(), Some("lambda:actor-b"));
assert_ne!(
cfg_a.actor_id, cfg_b.actor_id,
"precondition: the two connections must actually declare different actors"
);
assert_eq!(
cfg_a.default_namespace.as_str(),
"local",
"default_namespace must stay local regardless of the configured actor"
);
assert_eq!(
cfg_b.default_namespace.as_str(),
"local",
"default_namespace must stay local regardless of the configured actor"
);
assert_eq!(
crate::server::compute_config_id(&cfg_a, None),
crate::server::compute_config_id(&cfg_b, None),
"config_id must be byte-identical across connections that differ ONLY \
in [actor] id and folded visibility — identity fields must never feed compute_config_id"
);
}
fn base_runtime_config_for_multi_backend() -> RuntimeConfig {
use khive_runtime::{AllowAllGate, BackendId, Namespace};
RuntimeConfig {
db_path: khive_runtime::resolve_db_anchor(None),
gate: std::sync::Arc::new(AllowAllGate),
default_namespace: Namespace::parse("local").expect("ns"),
embedding_model: None,
additional_embedding_models: vec![],
packs: vec!["kg".to_string(), "comm".to_string()],
backend_id: BackendId::main(),
..RuntimeConfig::default()
}
}
#[tokio::test]
#[serial]
async fn multi_backend_boots_ok_with_two_memory_backends() {
use crate::tools::request::RequestParams;
use khive_runtime::PackConfig;
let khive_cfg = KhiveConfig {
backends: vec![
BackendConfig {
name: "main".to_string(),
kind: BackendKind::Memory,
path: None,
cache_mb: None,
journal_mode: None,
read_only: false,
},
BackendConfig {
name: "secondary".to_string(),
kind: BackendKind::Memory,
path: None,
cache_mb: None,
journal_mode: None,
read_only: false,
},
],
packs: {
let mut m = std::collections::HashMap::new();
m.insert(
"comm".to_string(),
PackConfig {
backend: "secondary".to_string(),
},
);
m
},
..KhiveConfig::default()
};
let base_cfg = base_runtime_config_for_multi_backend();
let server = build_server_multi_backend(base_cfg, &khive_cfg, None)
.expect("multi-backend boot must succeed");
let kg_resp = server
.dispatch_request_local(RequestParams {
ops: r#"create(kind="concept", name="MultiBackendTestEntity")"#.to_string(),
presentation: None,
presentation_per_op: None,
save_to: None,
format: None,
format_per_op: None,
})
.await
.expect("kg dispatch must not error");
let kg_json: serde_json::Value =
serde_json::from_str(&kg_resp).expect("kg response is valid JSON");
let first_ok = kg_json["results"][0]["ok"].as_bool();
assert_eq!(
first_ok,
Some(true),
"kg create must succeed; response: {kg_resp}"
);
let comm_resp = server
.dispatch_request_local(RequestParams {
ops: r#"comm.send(to="local", content="multi-backend-test")"#.to_string(),
presentation: None,
presentation_per_op: None,
save_to: None,
format: None,
format_per_op: None,
})
.await
.expect("comm dispatch must not error");
let comm_json: serde_json::Value =
serde_json::from_str(&comm_resp).expect("comm response is valid JSON");
let first_comm_ok = comm_json["results"][0]["ok"].as_bool();
assert_eq!(
first_comm_ok,
Some(true),
"comm.send must succeed; response: {comm_resp}"
);
}
#[tokio::test]
#[serial]
async fn multi_backend_brain_dispatch_hook_updates_state_visible_through_same_instance() {
let khive_cfg = KhiveConfig {
backends: vec![BackendConfig {
name: "main".to_string(),
kind: BackendKind::Memory,
path: None,
cache_mb: None,
journal_mode: None,
read_only: false,
}],
..KhiveConfig::default()
};
let mut base_cfg = base_runtime_config_for_multi_backend();
base_cfg.packs = vec!["kg".to_string(), "brain".to_string()];
let multi = build_registry_for_multi_backend(base_cfg, &khive_cfg, None)
.expect("multi-backend registry build must succeed");
multi
.registry
.dispatch("brain.state", serde_json::Value::Null)
.await
.expect("brain.state loads the default namespace into the active slot");
multi
.registry
.dispatch("stats", serde_json::json!({}))
.await
.expect("kg.stats dispatch succeeds");
let state = multi
.registry
.dispatch("brain.state", serde_json::Value::Null)
.await
.expect("brain.state dispatch");
let total_events = state["balanced_recall"]["total_events"]
.as_u64()
.unwrap_or(0);
assert!(
total_events > 0,
"multi-backend dispatch hook must update the same BrainPack instance \
the registry dispatches brain.* verbs to; got snapshot {state:?}"
);
}
#[test]
#[serial]
fn kkernel_multi_backend_path_wires_pool_for_file_backed_main() {
let dir = tempfile::tempdir().expect("temp dir");
let main_path = dir.path().join("main.db");
let khive_cfg = KhiveConfig {
backends: vec![BackendConfig {
name: "main".to_string(),
kind: BackendKind::Sqlite,
path: Some(main_path.clone()),
cache_mb: None,
journal_mode: None,
read_only: false,
}],
..KhiveConfig::default()
};
let base_cfg = base_runtime_config_for_multi_backend();
let multi = build_registry_for_multi_backend(base_cfg, &khive_cfg, None)
.expect("multi-backend registry build must succeed");
let server = build_server_from_multi_backend_registry(multi, &khive_cfg, None);
assert!(
server.pool().is_some(),
"file-backed multi-backend main must wire a checkpoint pool onto the server"
);
}
#[test]
#[serial]
fn kkernel_multi_backend_path_leaves_pool_none_for_in_memory_main() {
let khive_cfg = KhiveConfig {
backends: vec![BackendConfig {
name: "main".to_string(),
kind: BackendKind::Memory,
path: None,
cache_mb: None,
journal_mode: None,
read_only: false,
}],
..KhiveConfig::default()
};
let base_cfg = base_runtime_config_for_multi_backend();
let multi = build_registry_for_multi_backend(base_cfg, &khive_cfg, None)
.expect("multi-backend registry build must succeed");
let server = build_server_from_multi_backend_registry(multi, &khive_cfg, None);
assert!(
server.pool().is_none(),
"in-memory multi-backend main must never carry a checkpoint pool"
);
}
#[test]
#[serial]
fn secondary_pack_runtime_core_resolves_to_main_after_build_registry() {
use khive_runtime::PackConfig;
let khive_cfg = KhiveConfig {
backends: vec![
BackendConfig {
name: "main".to_string(),
kind: BackendKind::Memory,
path: None,
cache_mb: None,
journal_mode: None,
read_only: false,
},
BackendConfig {
name: "secondary".to_string(),
kind: BackendKind::Memory,
path: None,
cache_mb: None,
journal_mode: None,
read_only: false,
},
],
packs: {
let mut m = std::collections::HashMap::new();
m.insert(
"comm".to_string(),
PackConfig {
backend: "secondary".to_string(),
},
);
m
},
..KhiveConfig::default()
};
let base_cfg = base_runtime_config_for_multi_backend();
let result = build_registry_for_multi_backend(base_cfg, &khive_cfg, None)
.expect("multi-backend registry must boot");
let comm_rt = result
.per_pack_runtimes
.get("comm")
.expect("comm pack runtime must be present in per_pack_runtimes");
assert_eq!(
comm_rt.backend_id().as_str(),
"secondary",
"comm pack runtime's own backend_id must be \"secondary\""
);
assert_eq!(
comm_rt.core().backend_id().as_str(),
BackendId::MAIN,
"secondary-backend pack must have core_backend wired to main (ADR-073); \
core().backend_id() returned {:?} — build_pack_runtime wiring missing",
comm_rt.core().backend_id().as_str()
);
}
#[test]
#[serial]
fn memory_override_forces_all_backends_in_memory_and_never_creates_sqlite_file() {
use khive_runtime::PackConfig;
let dir = tempfile::tempdir().unwrap();
let main_path = dir.path().join("main_should_never_be_created.db");
let secondary_path = dir.path().join("secondary_should_never_be_created.db");
let khive_cfg = KhiveConfig {
backends: vec![
BackendConfig {
name: "main".to_string(),
kind: BackendKind::Sqlite,
path: Some(main_path.clone()),
cache_mb: None,
journal_mode: None,
read_only: false,
},
BackendConfig {
name: "secondary".to_string(),
kind: BackendKind::Sqlite,
path: Some(secondary_path.clone()),
cache_mb: None,
journal_mode: None,
read_only: false,
},
],
packs: {
let mut m = std::collections::HashMap::new();
m.insert(
"comm".to_string(),
PackConfig {
backend: "secondary".to_string(),
},
);
m
},
..KhiveConfig::default()
};
let base_cfg = base_runtime_config_for_multi_backend();
let result = build_registry_for_multi_backend(base_cfg, &khive_cfg, Some(":memory:"));
if let Err(ref e) = result {
panic!(
"--db :memory: override must force both declared sqlite backends \
in-memory and boot successfully; got: {e}"
);
}
assert!(
!main_path.exists(),
"main backend's declared sqlite path must never be created on disk when \
--db :memory: overrides it; found file at {main_path:?}"
);
assert!(
!secondary_path.exists(),
"secondary backend's declared sqlite path must never be created on disk \
when --db :memory: overrides it; found file at {secondary_path:?}"
);
}
#[test]
#[serial]
fn concrete_db_override_with_backends_declared_is_rejected() {
use khive_runtime::PackConfig;
let khive_cfg = KhiveConfig {
backends: vec![
BackendConfig {
name: "main".to_string(),
kind: BackendKind::Memory,
path: None,
cache_mb: None,
journal_mode: None,
read_only: false,
},
BackendConfig {
name: "secondary".to_string(),
kind: BackendKind::Memory,
path: None,
cache_mb: None,
journal_mode: None,
read_only: false,
},
],
packs: {
let mut m = std::collections::HashMap::new();
m.insert(
"comm".to_string(),
PackConfig {
backend: "secondary".to_string(),
},
);
m
},
..KhiveConfig::default()
};
let base_cfg = RuntimeConfig {
db_path: khive_runtime::resolve_db_anchor(Some("/tmp/some-explicit-override.db")),
..base_runtime_config_for_multi_backend()
};
let result = build_registry_for_multi_backend(
base_cfg,
&khive_cfg,
Some("/tmp/some-explicit-override.db"),
);
assert!(
result.is_err(),
"a concrete --db path override combined with declared [[backends]] must \
be rejected as ambiguous"
);
if let Err(err) = result {
let msg = err.to_string();
assert!(
msg.contains("khive.toml"),
"error message must point at khive.toml as where to make the change \
instead; got: {msg}"
);
}
}
#[tokio::test]
#[serial]
async fn multi_backend_preserves_actor_filtering() {
use crate::tools::request::RequestParams;
use khive_runtime::PackConfig;
let khive_cfg = KhiveConfig {
backends: vec![
BackendConfig {
name: "main".to_string(),
kind: BackendKind::Memory,
path: None,
cache_mb: None,
journal_mode: None,
read_only: false,
},
BackendConfig {
name: "secondary".to_string(),
kind: BackendKind::Memory,
path: None,
cache_mb: None,
journal_mode: None,
read_only: false,
},
],
packs: {
let mut m = std::collections::HashMap::new();
m.insert(
"comm".to_string(),
PackConfig {
backend: "secondary".to_string(),
},
);
m
},
..KhiveConfig::default()
};
let base_cfg = RuntimeConfig {
actor_id: Some("actor-b".to_string()),
..base_runtime_config_for_multi_backend()
};
let server = build_server_multi_backend(base_cfg, &khive_cfg, None)
.expect("multi-backend boot must succeed");
let dispatch = |ops: String| {
let server = &server;
async move {
let resp = server
.dispatch_request_local(RequestParams {
ops,
presentation: None,
presentation_per_op: None,
save_to: None,
format: None,
format_per_op: None,
})
.await
.expect("dispatch must not error");
serde_json::from_str::<serde_json::Value>(&resp).expect("valid JSON")
}
};
let to_a = dispatch(r#"comm.send(to="actor-a", content="for-a")"#.to_string()).await;
assert_eq!(to_a["results"][0]["ok"].as_bool(), Some(true), "{to_a}");
let to_b = dispatch(r#"comm.send(to="actor-b", content="for-b")"#.to_string()).await;
assert_eq!(to_b["results"][0]["ok"].as_bool(), Some(true), "{to_b}");
let inbox = dispatch(r#"comm.inbox()"#.to_string()).await;
let result = &inbox["results"][0]["result"];
let messages = result["messages"]
.as_array()
.expect("inbox returns a messages array");
let contents: Vec<&str> = messages
.iter()
.filter_map(|m| m["content"].as_str())
.collect();
assert!(
contents.contains(&"for-b"),
"actor-b must see the message addressed to it; got {contents:?}"
);
assert!(
!contents.contains(&"for-a"),
"actor-b must NOT see the message addressed to actor-a (leak #75 / B-BLOCKER-1); \
got {contents:?} — actor identity was not threaded into the multi-backend registry"
);
}
#[test]
#[serial]
fn multi_backend_missing_main_returns_error_mentioning_main() {
let khive_cfg = KhiveConfig {
backends: vec![BackendConfig {
name: "secondary".to_string(), kind: BackendKind::Memory,
path: None,
cache_mb: None,
journal_mode: None,
read_only: false,
}],
packs: std::collections::HashMap::new(),
..KhiveConfig::default()
};
let base_cfg = base_runtime_config_for_multi_backend();
let result = build_server_multi_backend(base_cfg, &khive_cfg, None);
assert!(
result.is_err(),
"missing main backend must produce an error"
);
if let Err(err) = result {
assert!(
err.to_string().contains("main"),
"error message must mention \"main\"; got: {err}"
);
}
}
#[test]
#[serial]
fn multi_backend_registry_rejects_undefined_pack_backend() {
use khive_runtime::PackConfig;
let khive_cfg = KhiveConfig {
backends: vec![BackendConfig {
name: "main".to_string(),
kind: BackendKind::Memory,
path: None,
cache_mb: None,
journal_mode: None,
read_only: false,
}],
packs: {
let mut m = std::collections::HashMap::new();
m.insert(
"comm".to_string(),
PackConfig {
backend: "archive".to_string(),
},
);
m
},
..KhiveConfig::default()
};
let base_cfg = base_runtime_config_for_multi_backend();
let result = build_registry_for_multi_backend(base_cfg, &khive_cfg, None);
assert!(
result.is_err(),
"an undeclared configured pack backend must be a startup error, not a silent \
fallback to main"
);
if let Err(err) = result {
let msg = err.to_string();
assert!(
msg.contains("packs.comm"),
"error must name the pack; got: {msg}"
);
assert!(
msg.contains("archive"),
"error must name the undeclared backend; got: {msg}"
);
assert!(
msg.contains("main"),
"error must list the defined backends; got: {msg}"
);
}
}
#[test]
#[serial]
fn multi_backend_server_rejects_undefined_pack_backend() {
use khive_runtime::PackConfig;
let khive_cfg = KhiveConfig {
backends: vec![BackendConfig {
name: "main".to_string(),
kind: BackendKind::Memory,
path: None,
cache_mb: None,
journal_mode: None,
read_only: false,
}],
packs: {
let mut m = std::collections::HashMap::new();
m.insert(
"comm".to_string(),
PackConfig {
backend: "archive".to_string(),
},
);
m
},
..KhiveConfig::default()
};
let base_cfg = base_runtime_config_for_multi_backend();
let result = build_server_multi_backend(base_cfg, &khive_cfg, None);
assert!(
result.is_err(),
"an undeclared configured pack backend must be a startup error, not a silent \
fallback to main"
);
if let Err(err) = result {
let msg = err.to_string();
assert!(
msg.contains("packs.comm"),
"error must name the pack; got: {msg}"
);
assert!(
msg.contains("archive"),
"error must name the undeclared backend; got: {msg}"
);
assert!(
msg.contains("main"),
"error must list the defined backends; got: {msg}"
);
}
}
#[test]
fn read_only_backend_rejects_writes() {
let dir = tempfile::tempdir().unwrap();
let db_path = dir.path().join("ro_test.db");
let rw = StorageBackend::sqlite(&db_path).expect("rw backend");
rw.apply_pack_ddl_statements(&[
"CREATE TABLE IF NOT EXISTS ro_check (id INTEGER PRIMARY KEY)",
])
.expect("DDL on rw backend");
drop(rw);
let ro = StorageBackend::sqlite_read_only(&db_path).expect("ro backend");
let result = ro.apply_pack_ddl_statements(&["INSERT INTO ro_check (id) VALUES (1)"]);
assert!(
result.is_err(),
"write to a read-only backend must fail; got Ok(())"
);
}
#[test]
#[serial]
fn duplicate_sqlite_paths_deduplicated_to_single_backend() {
use khive_runtime::PackConfig;
let dir = tempfile::tempdir().unwrap();
let db_path = dir.path().join("shared.db");
let db_path_str = db_path.to_str().unwrap();
let khive_cfg = KhiveConfig {
backends: vec![
BackendConfig {
name: "main".to_string(),
kind: BackendKind::Sqlite,
path: Some(db_path.clone()),
cache_mb: None,
journal_mode: None,
read_only: false,
},
BackendConfig {
name: "alias".to_string(),
kind: BackendKind::Sqlite,
path: Some(db_path.clone()),
cache_mb: None,
journal_mode: None,
read_only: false,
},
],
packs: {
let mut m = std::collections::HashMap::new();
m.insert(
"comm".to_string(),
PackConfig {
backend: "alias".to_string(),
},
);
m
},
..KhiveConfig::default()
};
let _ = db_path_str;
let base_cfg = base_runtime_config_for_multi_backend();
let result = build_server_multi_backend(base_cfg, &khive_cfg, None);
if let Err(ref e) = result {
panic!(
"two backends with the same canonical path must share one Arc and boot ok; got: {e}"
);
}
}
#[test]
#[serial]
fn memory_override_forces_all_backends_in_memory_and_never_creates_sqlite_file_via_build_server_multi_backend(
) {
use khive_runtime::PackConfig;
let dir = tempfile::tempdir().unwrap();
let main_path = dir.path().join("main_should_never_be_created.db");
let secondary_path = dir.path().join("secondary_should_never_be_created.db");
let khive_cfg = KhiveConfig {
backends: vec![
BackendConfig {
name: "main".to_string(),
kind: BackendKind::Sqlite,
path: Some(main_path.clone()),
cache_mb: None,
journal_mode: None,
read_only: false,
},
BackendConfig {
name: "secondary".to_string(),
kind: BackendKind::Sqlite,
path: Some(secondary_path.clone()),
cache_mb: None,
journal_mode: None,
read_only: false,
},
],
packs: {
let mut m = std::collections::HashMap::new();
m.insert(
"comm".to_string(),
PackConfig {
backend: "secondary".to_string(),
},
);
m
},
..KhiveConfig::default()
};
let base_cfg = base_runtime_config_for_multi_backend();
let result = build_server_multi_backend(base_cfg, &khive_cfg, Some(":memory:"));
if let Err(ref e) = result {
panic!(
"--db :memory: override must force both declared sqlite backends \
in-memory and boot successfully; got: {e}"
);
}
assert!(
!main_path.exists(),
"main backend's declared sqlite path must never be created on disk when \
--db :memory: overrides it; found file at {main_path:?}"
);
assert!(
!secondary_path.exists(),
"secondary backend's declared sqlite path must never be created on disk \
when --db :memory: overrides it; found file at {secondary_path:?}"
);
}
#[test]
#[serial]
fn concrete_db_override_with_backends_declared_is_rejected_via_build_server_multi_backend() {
use khive_runtime::PackConfig;
let khive_cfg = KhiveConfig {
backends: vec![
BackendConfig {
name: "main".to_string(),
kind: BackendKind::Memory,
path: None,
cache_mb: None,
journal_mode: None,
read_only: false,
},
BackendConfig {
name: "secondary".to_string(),
kind: BackendKind::Memory,
path: None,
cache_mb: None,
journal_mode: None,
read_only: false,
},
],
packs: {
let mut m = std::collections::HashMap::new();
m.insert(
"comm".to_string(),
PackConfig {
backend: "secondary".to_string(),
},
);
m
},
..KhiveConfig::default()
};
let base_cfg = RuntimeConfig {
db_path: khive_runtime::resolve_db_anchor(Some("/tmp/some-explicit-override.db")),
..base_runtime_config_for_multi_backend()
};
let result = build_server_multi_backend(
base_cfg,
&khive_cfg,
Some("/tmp/some-explicit-override.db"),
);
assert!(
result.is_err(),
"a concrete --db path override combined with declared [[backends]] must \
be rejected as ambiguous"
);
if let Err(err) = result {
let msg = err.to_string();
assert!(
msg.contains("khive.toml"),
"error message must point at khive.toml as where to make the change \
instead; got: {msg}"
);
}
}
#[test]
fn config_id_folds_backend_topology_when_non_empty() {
use khive_runtime::{BackendId, KhiveConfig, Namespace, PackConfig, RuntimeConfig};
let base_rt = RuntimeConfig {
db_path: None,
default_namespace: Namespace::parse("local").unwrap(),
embedding_model: None,
packs: vec!["kg".to_string(), "comm".to_string()],
backend_id: BackendId::main(),
..RuntimeConfig::default()
};
let id_no_backends = crate::server::compute_config_id(&base_rt, None);
let id_empty_backends =
crate::server::compute_config_id(&base_rt, Some(&KhiveConfig::default()));
assert_eq!(
id_no_backends, id_empty_backends,
"empty-backends config_id must be byte-identical to None-config config_id"
);
let mut packs_a = std::collections::HashMap::new();
packs_a.insert(
"comm".to_string(),
PackConfig {
backend: "secondary".to_string(),
},
);
let cfg_a = KhiveConfig {
backends: vec![
BackendConfig {
name: "main".to_string(),
kind: BackendKind::Memory,
path: None,
cache_mb: None,
journal_mode: None,
read_only: false,
},
BackendConfig {
name: "secondary".to_string(),
kind: BackendKind::Memory,
path: None,
cache_mb: None,
journal_mode: None,
read_only: false,
},
],
packs: packs_a,
..KhiveConfig::default()
};
let cfg_b = KhiveConfig {
backends: cfg_a.backends.clone(),
packs: std::collections::HashMap::new(),
..KhiveConfig::default()
};
let id_a = crate::server::compute_config_id(&base_rt, Some(&cfg_a));
let id_b = crate::server::compute_config_id(&base_rt, Some(&cfg_b));
assert_ne!(
id_a, id_b,
"configs differing only in pack→backend routing must produce different config_ids; \
both produced: {id_a}"
);
}
#[tokio::test]
#[serial]
async fn multi_backend_isolates_pack_data_to_separate_files() {
use crate::tools::request::RequestParams;
use khive_runtime::PackConfig;
use rusqlite::Connection;
let dir = tempfile::tempdir().expect("temp dir");
let main_path = dir.path().join("main.db");
let second_path = dir.path().join("second.db");
let khive_cfg = KhiveConfig {
backends: vec![
BackendConfig {
name: "main".to_string(),
kind: BackendKind::Sqlite,
path: Some(main_path.clone()),
cache_mb: None,
journal_mode: None,
read_only: false,
},
BackendConfig {
name: "second".to_string(),
kind: BackendKind::Sqlite,
path: Some(second_path.clone()),
cache_mb: None,
journal_mode: None,
read_only: false,
},
],
packs: {
let mut m = std::collections::HashMap::new();
m.insert(
"comm".to_string(),
PackConfig {
backend: "second".to_string(),
},
);
m
},
..KhiveConfig::default()
};
let base_cfg = base_runtime_config_for_multi_backend();
let server = build_server_multi_backend(base_cfg, &khive_cfg, None)
.expect("multi-backend boot must succeed");
let dispatch = |ops: String| {
let server = &server;
async move {
server
.dispatch_request_local(RequestParams {
ops,
presentation: None,
presentation_per_op: None,
save_to: None,
format: None,
format_per_op: None,
})
.await
.expect("dispatch must not error")
}
};
let kg_resp =
dispatch(r#"create(kind="concept", name="MainOnlyEntity")"#.to_string()).await;
let kg_json: serde_json::Value =
serde_json::from_str(&kg_resp).expect("kg response is valid JSON");
assert_eq!(
kg_json["results"][0]["ok"].as_bool(),
Some(true),
"kg create must succeed; response: {kg_resp}"
);
let comm_resp =
dispatch(r#"comm.send(to="local", content="SecondOnlyMsg")"#.to_string()).await;
let comm_json: serde_json::Value =
serde_json::from_str(&comm_resp).expect("comm response is valid JSON");
assert_eq!(
comm_json["results"][0]["ok"].as_bool(),
Some(true),
"comm.send must succeed; response: {comm_resp}"
);
drop(server);
let main_conn = Connection::open(&main_path).expect("open main.db");
let main_entity_count: i64 = main_conn
.query_row(
"SELECT COUNT(*) FROM entities WHERE name = 'MainOnlyEntity' AND deleted_at IS NULL",
[],
|row| row.get(0),
)
.expect("query entities in main.db");
assert_eq!(
main_entity_count, 1,
"main.db MUST contain MainOnlyEntity (written via kg pack); got count={main_entity_count}"
);
let main_msg_count: i64 = main_conn
.query_row(
"SELECT COUNT(*) FROM notes WHERE kind = 'message'",
[],
|row| row.get(0),
)
.expect("query notes in main.db");
assert_eq!(
main_msg_count, 0,
"main.db MUST NOT contain any message notes (comm is pinned to second.db); \
got count={main_msg_count}"
);
let second_conn = Connection::open(&second_path).expect("open second.db");
let second_msg_count: i64 = second_conn
.query_row(
"SELECT COUNT(*) FROM notes WHERE kind = 'message' AND content = 'SecondOnlyMsg'",
[],
|row| row.get(0),
)
.expect("query notes in second.db");
assert_eq!(
second_msg_count, 2,
"second.db MUST contain SecondOnlyMsg (dual-write: 1 outbound + 1 inbound copy); \
got count={second_msg_count}"
);
let second_entity_count: i64 = second_conn
.query_row(
"SELECT COUNT(*) FROM entities WHERE name = 'MainOnlyEntity'",
[],
|row| row.get(0),
)
.expect("query entities in second.db");
assert_eq!(
second_entity_count, 0,
"second.db MUST NOT contain MainOnlyEntity (kg is pinned to main.db); \
got count={second_entity_count}"
);
}
#[cfg(feature = "channel-email")]
mod ingest_ns_tests {
use super::*;
#[test]
#[serial]
fn ingest_namespace_defaults_to_local() {
std::env::remove_var("KHIVE_EMAIL_INGEST_NAMESPACE");
assert_eq!(ingest_namespace_from_env(), "local");
}
#[test]
#[serial]
fn ingest_namespace_reads_env_var() {
std::env::set_var("KHIVE_EMAIL_INGEST_NAMESPACE", "lambda:mybot");
let ns = ingest_namespace_from_env();
std::env::remove_var("KHIVE_EMAIL_INGEST_NAMESPACE");
assert_eq!(ns, "lambda:mybot");
}
#[test]
#[serial]
fn ingest_namespace_ignores_blank_env_var() {
std::env::set_var("KHIVE_EMAIL_INGEST_NAMESPACE", " ");
let ns = ingest_namespace_from_env();
std::env::remove_var("KHIVE_EMAIL_INGEST_NAMESPACE");
assert_eq!(ns, "local", "blank env var must fall back to default");
}
#[test]
fn preflight_fails_on_invalid_namespace_string() {
let registry = khive_runtime::VerbRegistryBuilder::new()
.build()
.expect("build empty registry");
assert!(
!preflight_ingest_namespace("", ®istry),
"preflight must return false for an invalid namespace string"
);
}
#[test]
fn preflight_fails_when_gate_denies_namespace() {
use khive_runtime::{Gate, GateDecision, GateError, GateRequest};
use std::fmt;
#[derive(Debug)]
struct AlwaysDenyGate;
impl fmt::Display for AlwaysDenyGate {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "AlwaysDenyGate")
}
}
impl Gate for AlwaysDenyGate {
fn check(&self, _req: &GateRequest) -> Result<GateDecision, GateError> {
Ok(GateDecision::deny("test: always deny"))
}
}
let mut builder = khive_runtime::VerbRegistryBuilder::new();
builder.with_gate(std::sync::Arc::new(AlwaysDenyGate));
let registry = builder.build().expect("build registry with deny gate");
assert!(
!preflight_ingest_namespace("local", ®istry),
"preflight must return false when the gate denies the namespace"
);
}
#[test]
fn preflight_succeeds_with_allow_gate_and_valid_namespace() {
let registry = khive_runtime::VerbRegistryBuilder::new()
.build()
.expect("build registry with default allow-all gate");
assert!(
preflight_ingest_namespace("local", ®istry),
"preflight must return true for a valid namespace when the gate allows"
);
}
#[test]
fn spawn_not_called_when_gate_denies() {
use khive_runtime::{Gate, GateDecision, GateError, GateRequest};
use std::fmt;
#[derive(Debug)]
struct AlwaysDenyGate2;
impl fmt::Display for AlwaysDenyGate2 {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "AlwaysDenyGate2")
}
}
impl Gate for AlwaysDenyGate2 {
fn check(&self, _req: &GateRequest) -> Result<GateDecision, GateError> {
Ok(GateDecision::deny("spawn seam test: always deny"))
}
}
let mut builder = khive_runtime::VerbRegistryBuilder::new();
builder.with_gate(std::sync::Arc::new(AlwaysDenyGate2));
let registry = builder.build().expect("build registry with deny gate");
let mut spawn_count = 0usize;
let authorized = run_if_authorized("local", ®istry, || {
spawn_count += 1;
});
assert!(
!authorized,
"run_if_authorized must return false when gate denies"
);
assert_eq!(
spawn_count, 0,
"spawn must not be called when preflight fails"
);
}
#[test]
fn spawn_not_called_when_namespace_invalid() {
let registry = khive_runtime::VerbRegistryBuilder::new()
.build()
.expect("build empty registry");
let mut spawn_count = 0usize;
let authorized = run_if_authorized("", ®istry, || {
spawn_count += 1;
});
assert!(
!authorized,
"run_if_authorized must return false for invalid namespace"
);
assert_eq!(
spawn_count, 0,
"spawn must not be called when namespace is invalid"
);
}
}
#[cfg(feature = "channel-email")]
mod eligible_poll_failure_log_tests {
use super::*;
use khive_channel::ChannelError;
use khive_channel_email::BackoffTick;
use std::sync::{Arc, Mutex};
use std::time::Duration;
use tracing::field::{Field, Visit};
#[derive(Clone, Debug, Default)]
struct CapturedEvent {
level: Option<tracing::Level>,
message: Option<String>,
}
#[derive(Default)]
struct CapturedEventVisitor(Option<String>);
impl Visit for CapturedEventVisitor {
fn record_str(&mut self, field: &Field, value: &str) {
if field.name() == "message" {
self.0 = Some(value.to_string());
}
}
fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) {
if field.name() == "message" {
let formatted = format!("{value:?}");
self.0 = Some(
formatted
.trim_start_matches('"')
.trim_end_matches('"')
.to_string(),
);
}
}
}
struct CaptureSubscriber {
events: Arc<Mutex<Vec<CapturedEvent>>>,
}
impl tracing::Subscriber for CaptureSubscriber {
fn enabled(&self, _: &tracing::Metadata<'_>) -> bool {
true
}
fn new_span(&self, _: &tracing::span::Attributes<'_>) -> tracing::span::Id {
tracing::span::Id::from_u64(1)
}
fn record(&self, _: &tracing::span::Id, _: &tracing::span::Record<'_>) {}
fn record_follows_from(&self, _: &tracing::span::Id, _: &tracing::span::Id) {}
fn event(&self, event: &tracing::Event<'_>) {
let mut visitor = CapturedEventVisitor::default();
event.record(&mut visitor);
self.events.lock().unwrap().push(CapturedEvent {
level: Some(*event.metadata().level()),
message: visitor.0,
});
}
fn enter(&self, _: &tracing::span::Id) {}
fn exit(&self, _: &tracing::span::Id) {}
}
fn tick(should_warn: bool, attempt: u32) -> BackoffTick {
BackoffTick {
delay: Duration::from_secs(10),
step: Duration::from_secs(10),
attempt,
should_warn,
}
}
#[test]
fn escalation_edge_logs_warn_not_debug() {
let buffer = Arc::new(Mutex::new(Vec::new()));
let subscriber = CaptureSubscriber {
events: Arc::clone(&buffer),
};
let err = ChannelError::Transport("boom".into());
tracing::subscriber::with_default(subscriber, || {
log_eligible_poll_failure("email", &err, &tick(true, 1));
});
let events = buffer.lock().unwrap();
assert_eq!(
events.len(),
1,
"expected exactly one log event, got {events:?}"
);
assert_eq!(events[0].level, Some(tracing::Level::WARN));
}
#[test]
fn same_step_repeat_logs_debug_not_warn() {
let buffer = Arc::new(Mutex::new(Vec::new()));
let subscriber = CaptureSubscriber {
events: Arc::clone(&buffer),
};
let err = ChannelError::Transport("boom".into());
tracing::subscriber::with_default(subscriber, || {
log_eligible_poll_failure("email", &err, &tick(false, 2));
});
let events = buffer.lock().unwrap();
assert_eq!(
events.len(),
1,
"expected exactly one log event, got {events:?}"
);
assert_eq!(events[0].level, Some(tracing::Level::DEBUG));
}
#[test]
fn sustained_capped_pressure_produces_exactly_one_warn() {
let buffer = Arc::new(Mutex::new(Vec::new()));
let subscriber = CaptureSubscriber {
events: Arc::clone(&buffer),
};
let err = ChannelError::Auth("authenticated but not connected".into());
tracing::subscriber::with_default(subscriber, || {
log_eligible_poll_failure("email", &err, &tick(true, 8)); for attempt in 9..=15 {
log_eligible_poll_failure("email", &err, &tick(false, attempt));
}
});
let events = buffer.lock().unwrap();
let warn_count = events
.iter()
.filter(|e| e.level == Some(tracing::Level::WARN))
.count();
let debug_count = events
.iter()
.filter(|e| e.level == Some(tracing::Level::DEBUG))
.count();
assert_eq!(
warn_count, 1,
"exactly one WARN expected across the whole sequence, got {warn_count} in {events:?}"
);
assert_eq!(
debug_count, 7,
"the 7 same-step repeats must log at debug, not warn"
);
}
#[test]
fn warn_message_contains_error_text() {
let buffer = Arc::new(Mutex::new(Vec::new()));
let subscriber = CaptureSubscriber {
events: Arc::clone(&buffer),
};
let err = ChannelError::Auth("IMAP LOGIN failed: slot exhausted".into());
tracing::subscriber::with_default(subscriber, || {
log_eligible_poll_failure("email", &err, &tick(true, 1));
});
let events = buffer.lock().unwrap();
let message = events[0].message.as_deref().unwrap_or_default();
assert!(
message.contains("slot exhausted"),
"escalation warn must carry the underlying error text, got: {message}"
);
}
}
fn packs(names: &[&str]) -> Vec<String> {
names.iter().map(|s| s.to_string()).collect()
}
#[test]
fn warn_when_actor_is_none_and_comm_loaded() {
assert!(should_warn_unattributed(None, &packs(&["kg", "comm"])));
}
#[test]
fn warn_when_actor_is_local_and_comm_loaded() {
assert!(should_warn_unattributed(
Some("local"),
&packs(&["kg", "comm"])
));
}
#[test]
fn no_warn_when_actor_is_configured() {
assert!(!should_warn_unattributed(
Some("lambda:khive"),
&packs(&["kg", "comm"])
));
}
#[test]
fn no_warn_when_comm_not_loaded() {
assert!(!should_warn_unattributed(Some("local"), &packs(&["kg"])));
}
#[test]
fn no_warn_when_actor_none_and_no_comm() {
assert!(!should_warn_unattributed(None, &packs(&["kg", "memory"])));
}
#[test]
#[serial]
fn strict_mode_off_by_default() {
let prev = std::env::var("KHIVE_REQUIRE_ATTRIBUTED_ACTOR").ok();
std::env::remove_var("KHIVE_REQUIRE_ATTRIBUTED_ACTOR");
assert!(
!is_strict_actor_mode(),
"strict mode must be OFF when KHIVE_REQUIRE_ATTRIBUTED_ACTOR is unset"
);
if let Some(v) = prev {
std::env::set_var("KHIVE_REQUIRE_ATTRIBUTED_ACTOR", v);
}
}
#[test]
#[serial]
fn strict_mode_on_when_env_var_is_1() {
let prev = std::env::var("KHIVE_REQUIRE_ATTRIBUTED_ACTOR").ok();
std::env::set_var("KHIVE_REQUIRE_ATTRIBUTED_ACTOR", "1");
assert!(
is_strict_actor_mode(),
"strict mode must be ON when KHIVE_REQUIRE_ATTRIBUTED_ACTOR=1"
);
match prev {
Some(v) => std::env::set_var("KHIVE_REQUIRE_ATTRIBUTED_ACTOR", v),
None => std::env::remove_var("KHIVE_REQUIRE_ATTRIBUTED_ACTOR"),
}
}
#[test]
#[serial]
fn strict_mode_off_when_env_var_is_not_1() {
let prev = std::env::var("KHIVE_REQUIRE_ATTRIBUTED_ACTOR").ok();
std::env::set_var("KHIVE_REQUIRE_ATTRIBUTED_ACTOR", "0");
assert!(
!is_strict_actor_mode(),
"strict mode must be OFF when KHIVE_REQUIRE_ATTRIBUTED_ACTOR=0"
);
match prev {
Some(v) => std::env::set_var("KHIVE_REQUIRE_ATTRIBUTED_ACTOR", v),
None => std::env::remove_var("KHIVE_REQUIRE_ATTRIBUTED_ACTOR"),
}
}
#[test]
#[serial]
fn enforce_strict_actor_mode_returns_err_when_strict_and_no_actor() {
let prev = std::env::var("KHIVE_REQUIRE_ATTRIBUTED_ACTOR").ok();
std::env::set_var("KHIVE_REQUIRE_ATTRIBUTED_ACTOR", "1");
let result = enforce_strict_actor_mode(None, &packs(&["kg", "comm", "memory"]));
match prev {
Some(v) => std::env::set_var("KHIVE_REQUIRE_ATTRIBUTED_ACTOR", v),
None => std::env::remove_var("KHIVE_REQUIRE_ATTRIBUTED_ACTOR"),
}
assert!(
result.is_err(),
"enforce_strict_actor_mode must return Err when strict mode is ON \
and no actor is configured (comm pack loaded)"
);
let msg = result.unwrap_err().to_string();
assert!(
msg.contains("KHIVE_REQUIRE_ATTRIBUTED_ACTOR"),
"error message must name the env var; got: {msg}"
);
assert!(
msg.contains("KHIVE_ACTOR"),
"error message must name the remedy; got: {msg}"
);
}
#[test]
#[serial]
fn enforce_strict_actor_mode_ok_when_strict_and_actor_configured() {
let prev = std::env::var("KHIVE_REQUIRE_ATTRIBUTED_ACTOR").ok();
std::env::set_var("KHIVE_REQUIRE_ATTRIBUTED_ACTOR", "1");
let result = enforce_strict_actor_mode(Some("lambda:tenant-x"), &packs(&["kg", "comm"]));
match prev {
Some(v) => std::env::set_var("KHIVE_REQUIRE_ATTRIBUTED_ACTOR", v),
None => std::env::remove_var("KHIVE_REQUIRE_ATTRIBUTED_ACTOR"),
}
assert!(
result.is_ok(),
"enforce_strict_actor_mode must return Ok when actor is properly configured"
);
}
#[test]
#[serial]
fn enforce_strict_actor_mode_ok_when_strict_off_and_no_actor() {
let prev = std::env::var("KHIVE_REQUIRE_ATTRIBUTED_ACTOR").ok();
std::env::remove_var("KHIVE_REQUIRE_ATTRIBUTED_ACTOR");
let result = enforce_strict_actor_mode(None, &packs(&["kg", "comm", "memory"]));
match prev {
Some(v) => std::env::set_var("KHIVE_REQUIRE_ATTRIBUTED_ACTOR", v),
None => std::env::remove_var("KHIVE_REQUIRE_ATTRIBUTED_ACTOR"),
}
assert!(
result.is_ok(),
"enforce_strict_actor_mode must return Ok when strict mode is OFF \
(default OSS path must be completely unchanged)"
);
}
#[test]
#[serial]
fn enforce_strict_actor_mode_ok_when_strict_on_but_no_comm_pack() {
let prev = std::env::var("KHIVE_REQUIRE_ATTRIBUTED_ACTOR").ok();
std::env::set_var("KHIVE_REQUIRE_ATTRIBUTED_ACTOR", "1");
let result = enforce_strict_actor_mode(None, &packs(&["kg", "memory"]));
match prev {
Some(v) => std::env::set_var("KHIVE_REQUIRE_ATTRIBUTED_ACTOR", v),
None => std::env::remove_var("KHIVE_REQUIRE_ATTRIBUTED_ACTOR"),
}
assert!(
result.is_ok(),
"enforce_strict_actor_mode must return Ok when comm pack is not loaded \
(no party-line risk even without actor)"
);
}
#[test]
#[serial]
fn build_server_schedule_tick_uses_the_configured_backend_not_the_home_default() {
let seat_dir = tempfile::tempdir().expect("seat tempdir");
let _seat_env = SeatEnv::enter(seat_dir.path());
std::env::remove_var("KHIVE_DB");
std::env::remove_var("KHIVE_ACTOR");
std::env::remove_var("KHIVE_PACKS");
std::env::remove_var("KHIVE_REQUIRE_ATTRIBUTED_ACTOR");
let configured_db = seat_dir.path().join("configured-schedule-backend.db");
use clap::Parser;
let args = Args::parse_from(["mcp", "--db", configured_db.to_str().expect("utf8 path")]);
let (_server, schedule_rt) = build_server(&args).expect("build_server must succeed");
let rt = schedule_rt
.expect("the default pack set includes \"schedule\" — a runtime must be returned");
assert_eq!(
rt.config().db_path.as_deref(),
Some(configured_db.as_path()),
"the tick's runtime must target the exact --db this daemon was configured with, \
not RuntimeConfig::default()'s $HOME/.khive/khive.db fallback"
);
}
#[test]
#[serial]
fn build_server_schedule_tick_uses_the_configured_actor_identity() {
let seat_dir = tempfile::tempdir().expect("seat tempdir");
let _seat_env = SeatEnv::enter(seat_dir.path());
std::env::remove_var("KHIVE_DB");
std::env::remove_var("KHIVE_ACTOR");
std::env::remove_var("KHIVE_PACKS");
std::env::remove_var("KHIVE_REQUIRE_ATTRIBUTED_ACTOR");
use clap::Parser;
let args = Args::parse_from([
"mcp",
"--db",
":memory:",
"--actor",
"lambda:adr106-tick-actor",
]);
let (_server, schedule_rt) = build_server(&args).expect("build_server must succeed");
let rt = schedule_rt.expect("schedule pack is loaded by default");
assert_eq!(
rt.config().actor_id.as_deref(),
Some("lambda:adr106-tick-actor"),
"the tick's runtime must carry the daemon's own resolved --actor identity, \
not RuntimeConfig::default()'s unattributed actor_id=None"
);
}
#[test]
#[serial]
fn build_server_schedule_tick_is_none_when_schedule_pack_is_not_in_the_restricted_pack_set() {
let seat_dir = tempfile::tempdir().expect("seat tempdir");
let _seat_env = SeatEnv::enter(seat_dir.path());
std::env::remove_var("KHIVE_DB");
std::env::remove_var("KHIVE_ACTOR");
std::env::remove_var("KHIVE_PACKS");
std::env::remove_var("KHIVE_REQUIRE_ATTRIBUTED_ACTOR");
use clap::Parser;
let args = Args::parse_from(["mcp", "--db", ":memory:", "--pack", "kg"]);
let (_server, schedule_rt) = build_server(&args).expect("build_server must succeed");
assert!(
schedule_rt.is_none(),
"when the operator restricts --pack to exclude \"schedule\", the tick must have \
nothing to drain against — never silently falling back to a runtime that can \
dispatch through a pack the daemon was not configured to load"
);
}
#[test]
#[serial]
fn build_server_schedule_tick_runtime_satisfies_strict_actor_mode_like_the_live_server() {
let seat_dir = tempfile::tempdir().expect("seat tempdir");
let _seat_env = SeatEnv::enter(seat_dir.path());
std::env::remove_var("KHIVE_DB");
std::env::remove_var("KHIVE_ACTOR");
std::env::remove_var("KHIVE_PACKS");
let prev_strict = std::env::var("KHIVE_REQUIRE_ATTRIBUTED_ACTOR").ok();
std::env::set_var("KHIVE_REQUIRE_ATTRIBUTED_ACTOR", "1");
use clap::Parser;
let args = Args::parse_from([
"mcp",
"--db",
":memory:",
"--actor",
"lambda:strict-mode-tenant",
"--pack",
"kg",
"--pack",
"comm",
"--pack",
"schedule",
]);
let result = build_server(&args);
match prev_strict {
Some(v) => std::env::set_var("KHIVE_REQUIRE_ATTRIBUTED_ACTOR", v),
None => std::env::remove_var("KHIVE_REQUIRE_ATTRIBUTED_ACTOR"),
}
let (_server, schedule_rt) = result.expect(
"build_server must succeed under strict mode when --actor is properly configured",
);
let rt = schedule_rt.expect("\"schedule\" pack was explicitly requested");
assert_eq!(
rt.config().actor_id.as_deref(),
Some("lambda:strict-mode-tenant"),
"the tick's runtime must carry the same actor identity that satisfied strict \
mode at daemon boot, not a separately-resolved, unattributed default"
);
}
#[tokio::test]
#[serial]
async fn build_server_schedule_tick_uses_the_declared_multi_backend_not_main() {
let seat_dir = tempfile::tempdir().expect("seat tempdir");
let _seat_env = SeatEnv::enter(seat_dir.path());
std::env::remove_var("KHIVE_DB");
std::env::remove_var("KHIVE_ACTOR");
std::env::remove_var("KHIVE_PACKS");
std::env::remove_var("KHIVE_REQUIRE_ATTRIBUTED_ACTOR");
let main_db = seat_dir.path().join("main.db");
let schedule_db = seat_dir.path().join("schedule-backend.db");
let config_path = write_config(
seat_dir.path(),
&format!(
r#"
[[backends]]
name = "main"
kind = "sqlite"
path = "{main}"
[[backends]]
name = "schedule-backend"
kind = "sqlite"
path = "{schedule}"
[packs.schedule]
backend = "schedule-backend"
"#,
main = main_db.display(),
schedule = schedule_db.display(),
),
);
use clap::Parser;
let args = Args::parse_from(["mcp", "--config", config_path.to_str().expect("utf8 path")]);
let (_server, schedule_rt) = build_server(&args).expect("build_server must succeed");
let rt = schedule_rt.expect("schedule pack is loaded by default and declared here");
let marker_content = "adr106-multi-backend-schedule-marker";
let ns = Namespace::parse("local").expect("ns");
let token = rt.authorize(ns).expect("authorize schedule runtime");
let store = rt.notes(&token).expect("notes store");
store
.upsert_note(khive_storage::note::Note::new(
"local",
"observation",
marker_content,
))
.await
.expect("write marker note through the tick's runtime");
let count_marker_notes = |path: std::path::PathBuf| async move {
let cfg = RuntimeConfig {
db_path: Some(path),
default_namespace: Namespace::parse("local").unwrap(),
embedding_model: None,
additional_embedding_models: vec![],
..RuntimeConfig::default()
};
let probe_rt = KhiveRuntime::new(cfg).expect("reopen backend file");
let ns = Namespace::parse("local").unwrap();
let token = probe_rt.authorize(ns).expect("authorize probe");
let store = probe_rt.notes(&token).expect("notes store");
let page = store
.query_notes(
"local",
Some("observation"),
khive_storage::types::PageRequest {
limit: 10,
offset: 0,
},
)
.await
.expect("query observation notes");
page.items
.into_iter()
.filter(|n| n.content == marker_content)
.count()
};
assert_eq!(
count_marker_notes(schedule_db.clone()).await,
1,
"the marker written through the tick's runtime must be present in the declared \
\"schedule-backend\" backend file"
);
assert_eq!(
count_marker_notes(main_db.clone()).await,
0,
"the marker must be ABSENT from \"main\" — the tick's runtime must not have \
silently written into the main backend instead of the declared schedule backend"
);
}
#[tokio::test]
#[serial]
async fn build_server_schedule_tick_dispatches_actions_through_the_declared_multi_backend_not_schedule(
) {
let seat_dir = tempfile::tempdir().expect("seat tempdir");
let _seat_env = SeatEnv::enter(seat_dir.path());
std::env::remove_var("KHIVE_DB");
std::env::remove_var("KHIVE_ACTOR");
std::env::remove_var("KHIVE_PACKS");
std::env::remove_var("KHIVE_REQUIRE_ATTRIBUTED_ACTOR");
let main_db = seat_dir.path().join("main.db");
let kg_db = seat_dir.path().join("kg-backend.db");
let config_path = write_config(
seat_dir.path(),
&format!(
r#"
[[backends]]
name = "main"
kind = "sqlite"
path = "{main}"
[[backends]]
name = "kg-backend"
kind = "sqlite"
path = "{kg}"
[packs.kg]
backend = "kg-backend"
"#,
main = main_db.display(),
kg = kg_db.display(),
),
);
use clap::Parser;
let args = Args::parse_from(["mcp", "--config", config_path.to_str().expect("utf8 path")]);
let (server, schedule_rt) = build_server(&args).expect("build_server must succeed");
let rt = schedule_rt.expect("schedule pack is loaded by default");
let marker = "adr106-multi-backend-dispatch-marker";
let action_dsl = format!("create(kind=\"observation\", content=\"{marker}\")");
let past = (chrono::Utc::now() - chrono::Duration::seconds(5)).to_rfc3339();
let repeat: Option<&str> = None;
let fired_at: Option<&str> = None;
let cancelled_at: Option<&str> = None;
let props = serde_json::json!({
"trigger_at": past,
"repeat": repeat,
"status": "pending",
"event_type": "schedule",
"payload": action_dsl,
"fired_at": fired_at,
"cancelled_at": cancelled_at,
});
let ns = Namespace::parse("local").expect("ns");
let token = rt.authorize(ns).expect("authorize schedule runtime");
rt.create_note(
&token,
"scheduled_event",
None,
&action_dsl,
None,
Some(props),
vec![],
)
.await
.expect("create scheduled_event through the schedule runtime");
let summary = crate::pending_events::run_pending_events_on(&rt, &server, false)
.await
.expect("drain");
assert_eq!(
summary.fired + summary.advanced,
1,
"the due event must be dispatched, got summary={summary:?}"
);
assert_eq!(summary.failed, 0, "dispatch must not fail: {summary:?}");
let count_marker_notes = |path: std::path::PathBuf| async move {
let cfg = RuntimeConfig {
db_path: Some(path),
default_namespace: Namespace::parse("local").unwrap(),
embedding_model: None,
additional_embedding_models: vec![],
..RuntimeConfig::default()
};
let probe_rt = KhiveRuntime::new(cfg).expect("reopen backend file");
let ns = Namespace::parse("local").unwrap();
let token = probe_rt.authorize(ns).expect("authorize probe");
let store = probe_rt.notes(&token).expect("notes store");
let page = store
.query_notes(
"local",
Some("observation"),
khive_storage::types::PageRequest {
limit: 10,
offset: 0,
},
)
.await
.expect("query observation notes");
page.items
.into_iter()
.filter(|n| n.content == marker)
.count()
};
assert_eq!(
count_marker_notes(kg_db.clone()).await,
1,
"the replayed create(kind=\"observation\") action must land in the kg pack's OWN \
declared backend (\"kg-backend\"), not the schedule backend"
);
assert_eq!(
count_marker_notes(main_db.clone()).await,
0,
"the marker must be ABSENT from \"main\" (the schedule backend) — dispatching \
through a throwaway single-runtime server built from the schedule runtime alone \
would have written it here instead"
);
}
#[cfg(feature = "channel-email")]
mod channel_heartbeat_tests {
use super::*;
use khive_channel::ChannelError;
use khive_runtime::{KhiveRuntime, VerbRegistryBuilder};
#[test]
fn auth_maps_to_auth_class() {
assert_eq!(channel_error_class(&ChannelError::Auth("x".into())), "auth");
}
#[test]
fn transport_maps_to_transport_class() {
assert_eq!(
channel_error_class(&ChannelError::Transport("x".into())),
"transport"
);
}
#[test]
fn config_maps_to_config_class() {
assert_eq!(
channel_error_class(&ChannelError::Config("x".into())),
"config"
);
}
#[test]
fn unauthorized_sender_and_invalid_envelope_map_to_config_class() {
assert_eq!(
channel_error_class(&ChannelError::UnauthorizedSender("x".into())),
"config"
);
assert_eq!(
channel_error_class(&ChannelError::InvalidEnvelope("x".into())),
"config"
);
}
#[tokio::test]
async fn record_heartbeat_is_best_effort_when_comm_pack_absent() {
let registry = VerbRegistryBuilder::new()
.build()
.expect("empty registry builds");
record_channel_heartbeat(
®istry,
"email",
"recipient@example.com",
HeartbeatOutcome::Success,
None,
)
.await;
}
#[tokio::test]
async fn record_heartbeat_success_is_visible_via_comm_health() {
let runtime = KhiveRuntime::memory().expect("in-memory runtime");
let mut builder = VerbRegistryBuilder::new();
builder.register(khive_pack_kg::KgPack::new(runtime.clone()));
builder.register(khive_pack_comm::CommPack::new(runtime.clone()));
let registry = builder.build().expect("registry builds");
record_channel_heartbeat(
®istry,
"email",
"recipient@example.com",
HeartbeatOutcome::Success,
None,
)
.await;
let health = registry
.dispatch("comm.health", serde_json::json!({}))
.await
.expect("health succeeds");
let channels = health["channels"].as_array().expect("channels array");
assert_eq!(channels.len(), 1);
assert_eq!(channels[0]["channel_kind"].as_str(), Some("email"));
assert_eq!(
channels[0]["channel_slug"].as_str(),
Some("recipient@example.com")
);
}
#[tokio::test]
async fn heartbeat_visible_via_health_regardless_of_configured_ingest_namespace() {
let runtime = KhiveRuntime::memory().expect("in-memory runtime");
let mut builder = VerbRegistryBuilder::new();
builder.register(khive_pack_kg::KgPack::new(runtime.clone()));
builder.register(khive_pack_comm::CommPack::new(runtime.clone()));
let registry = builder.build().expect("registry builds");
let configured_ingest_namespace = "lambda:mybot";
let ingest_params = serde_json::json!({
"namespace": configured_ingest_namespace,
"from": "email:sender@example.com",
"to": "email:recipient@example.com",
"content": "hello",
"channel_kind": "email",
"external_id": "test-msg-1",
"default_inbound_actor": "lambda:leo",
});
registry
.dispatch("comm.ingest", ingest_params)
.await
.expect("message ingest into the configured namespace succeeds");
record_channel_heartbeat(
®istry,
"email",
"recipient@example.com",
HeartbeatOutcome::Success,
None,
)
.await;
let health = registry
.dispatch("comm.health", serde_json::json!({}))
.await
.expect("health succeeds");
assert_eq!(health["role"].as_str(), Some("daemon"));
let channels = health["channels"].as_array().expect("channels array");
assert_eq!(
channels.len(),
1,
"heartbeat row must be visible to a no-arg client comm.health() call \
regardless of the configured message-ingest namespace"
);
assert_eq!(channels[0]["channel_kind"].as_str(), Some("email"));
assert_eq!(
channels[0]["channel_slug"].as_str(),
Some("recipient@example.com")
);
}
}
#[cfg(feature = "channel-email")]
mod composite_key_registry_tests {
use super::*;
use async_trait::async_trait;
use chrono::{DateTime, Utc};
use khive_channel::{Channel, ChannelEnvelope, ChannelError, ChannelRegistry};
use khive_runtime::{KhiveRuntime, VerbRegistryBuilder};
use std::sync::Arc;
struct TwoMailboxChannel {
slug: String,
}
#[async_trait]
impl Channel for TwoMailboxChannel {
fn kind(&self) -> &'static str {
"email"
}
fn slug(&self) -> String {
self.slug.clone()
}
async fn send(&self, _envelope: ChannelEnvelope) -> Result<(), ChannelError> {
Ok(())
}
async fn poll(
&self,
_since: DateTime<Utc>,
) -> Result<Vec<ChannelEnvelope>, ChannelError> {
Ok(vec![])
}
}
#[tokio::test]
async fn two_same_kind_channels_both_poll_and_both_persist_health_rows() {
let mut ch_registry = ChannelRegistry::new();
ch_registry.register(Arc::new(TwoMailboxChannel {
slug: "mailbox-a@example.com".to_string(),
}));
ch_registry.register(Arc::new(TwoMailboxChannel {
slug: "mailbox-b@example.com".to_string(),
}));
assert_eq!(
ch_registry.len(),
2,
"two same-kind, different-slug adapters must both register, not collapse"
);
let runtime = KhiveRuntime::memory().expect("in-memory runtime");
let mut builder = VerbRegistryBuilder::new();
builder.register(khive_pack_kg::KgPack::new(runtime.clone()));
builder.register(khive_pack_comm::CommPack::new(runtime.clone()));
let registry = builder.build().expect("registry builds");
for (kind, slug, _channel) in ch_registry.iter() {
record_channel_heartbeat(®istry, kind, slug, HeartbeatOutcome::Success, None)
.await;
}
let health = registry
.dispatch("comm.health", serde_json::json!({}))
.await
.expect("health succeeds");
let channels = health["channels"].as_array().expect("channels array");
assert_eq!(
channels.len(),
2,
"both mailboxes must produce independent comm.health rows"
);
let slugs: std::collections::BTreeSet<&str> = channels
.iter()
.map(|c| c["channel_slug"].as_str().expect("channel_slug present"))
.collect();
assert_eq!(
slugs,
std::collections::BTreeSet::from([
"mailbox-a@example.com",
"mailbox-b@example.com"
])
);
}
#[test]
fn backoff_state_is_independent_per_kind_slug_pair() {
use khive_channel_email::ImapBackoff;
use std::collections::HashMap;
let mut backoffs: HashMap<(String, String), ImapBackoff> = HashMap::new();
let key_a = ("email".to_string(), "mailbox-a@example.com".to_string());
let key_b = ("email".to_string(), "mailbox-b@example.com".to_string());
let tick_a = backoffs.entry(key_a.clone()).or_default().record_failure();
assert!(
!backoffs.contains_key(&key_b),
"mailbox-b must have no backoff state after only mailbox-a fails"
);
assert!(tick_a.delay.as_secs() >= 1, "mailbox-a backoff engaged");
let backoff_b = backoffs.entry(key_b).or_default();
backoff_b.record_success();
assert_eq!(
backoffs.get(&key_a).unwrap().attempt(),
1,
"mailbox-a's backoff attempt count must be unaffected by mailbox-b's success"
);
}
}
#[cfg(feature = "channel-email")]
mod outbox_delivered_guard_tests {
use super::*;
use serde_json::json;
#[test]
fn missing_delivered_at_is_undelivered() {
let props = json!({}).as_object().unwrap().clone();
assert!(!note_already_delivered(&props));
}
#[test]
fn explicit_null_delivered_at_is_undelivered() {
let props = json!({ "delivered_at": null }).as_object().unwrap().clone();
assert!(!note_already_delivered(&props));
}
#[test]
fn present_non_null_delivered_at_is_delivered() {
let props = json!({ "delivered_at": "2026-06-30T12:00:00Z" })
.as_object()
.unwrap()
.clone();
assert!(note_already_delivered(&props));
}
}
#[cfg(feature = "channel-email")]
mod spawn_email_channel_loops_tests {
use super::*;
const EMAIL_ENV_VARS: [&str; 9] = [
"KHIVE_EMAIL_SMTP_HOST",
"KHIVE_EMAIL_IMAP_HOST",
"KHIVE_EMAIL_USERNAME",
"KHIVE_EMAIL_MAINTAINER_ADDRESS",
"KHIVE_EMAIL_AUTHSERV_ID",
"KHIVE_EMAIL_PASSWORD",
"KHIVE_EMAIL_OAUTH_TENANT_ID",
"KHIVE_EMAIL_OAUTH_CLIENT_ID",
"KHIVE_EMAIL_OAUTH_CLIENT_SECRET",
];
struct EmailEnvGuard {
snapshot: Vec<(&'static str, Option<String>)>,
}
impl EmailEnvGuard {
fn clear() -> Self {
let snapshot = EMAIL_ENV_VARS
.iter()
.map(|&var| (var, std::env::var(var).ok()))
.collect();
for var in EMAIL_ENV_VARS {
std::env::remove_var(var);
}
Self { snapshot }
}
}
impl Drop for EmailEnvGuard {
fn drop(&mut self) {
for (var, prev) in &self.snapshot {
match prev {
Some(v) => std::env::set_var(var, v),
None => std::env::remove_var(var),
}
}
}
}
#[tokio::test]
#[serial]
async fn missing_env_hits_err_arm_without_panic() {
let _env_guard = EmailEnvGuard::clear();
assert!(
khive_channel_email::EmailChannel::from_env().is_err(),
"with KHIVE_EMAIL_* cleared, from_env must fail closed (the Err arm the helper depends on)"
);
let config = RuntimeConfig {
db_path: None,
default_namespace: Namespace::parse("test").unwrap(),
embedding_model: None,
additional_embedding_models: vec![],
packs: vec!["kg".to_string()],
..RuntimeConfig::default()
};
let runtime = KhiveRuntime::new(config).expect("in-memory runtime");
let server = KhiveMcpServer::new(runtime).expect("server builds with kg");
spawn_email_channel_loops(&server);
}
#[test]
fn is_daemon_role_true_for_daemon_args() {
use clap::Parser;
let args = Args::parse_from(["mcp", "--daemon"]);
assert!(
is_daemon_role(&args),
"--daemon must resolve to daemon role"
);
}
#[test]
fn is_daemon_role_false_for_client_args() {
use clap::Parser;
let args = Args::parse_from(["mcp"]);
assert!(
!is_daemon_role(&args),
"a plain stdio client (no --daemon) must not resolve to daemon role"
);
}
#[tokio::test]
#[serial]
async fn daemon_role_gate_spawns_without_panic() {
use clap::Parser;
let _env_guard = EmailEnvGuard::clear();
let args = Args::parse_from(["mcp", "--daemon"]);
let config = RuntimeConfig {
db_path: None,
default_namespace: Namespace::parse("test").unwrap(),
embedding_model: None,
additional_embedding_models: vec![],
packs: vec!["kg".to_string()],
..RuntimeConfig::default()
};
let runtime = KhiveRuntime::new(config).expect("in-memory runtime");
let server = KhiveMcpServer::new(runtime).expect("server builds with kg");
spawn_email_channel_loops_if_daemon(&server, &args);
}
#[tokio::test]
#[serial]
async fn client_role_gate_skips_without_panic() {
use clap::Parser;
let _env_guard = EmailEnvGuard::clear();
let args = Args::parse_from(["mcp"]);
let config = RuntimeConfig {
db_path: None,
default_namespace: Namespace::parse("test").unwrap(),
embedding_model: None,
additional_embedding_models: vec![],
packs: vec!["kg".to_string()],
..RuntimeConfig::default()
};
let runtime = KhiveRuntime::new(config).expect("in-memory runtime");
let server = KhiveMcpServer::new(runtime).expect("server builds with kg");
spawn_email_channel_loops_if_daemon(&server, &args);
}
}
#[cfg(feature = "channel-email")]
mod channel_lifecycle_sequencing_tests {
use super::*;
use async_trait::async_trait;
use chrono::{DateTime, Utc};
use khive_channel::{Channel, ChannelEnvelope, ChannelError, ChannelRegistry};
use khive_runtime::{KhiveRuntime, VerbRegistryBuilder};
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
struct FlakyOnceChannel {
call_count: AtomicUsize,
}
#[async_trait]
impl Channel for FlakyOnceChannel {
fn kind(&self) -> &'static str {
"mock"
}
async fn send(&self, _envelope: ChannelEnvelope) -> Result<(), ChannelError> {
Ok(())
}
async fn poll(
&self,
_since: DateTime<Utc>,
) -> Result<Vec<ChannelEnvelope>, ChannelError> {
if self.call_count.fetch_add(1, Ordering::SeqCst) == 0 {
Err(ChannelError::Transport("synthetic connect failure".into()))
} else {
Ok(vec![])
}
}
}
#[derive(Default)]
struct FakeEventStore {
events: Mutex<Vec<khive_storage::Event>>,
}
#[async_trait]
impl khive_storage::EventStore for FakeEventStore {
async fn append_event(
&self,
event: khive_storage::Event,
) -> khive_storage::StorageResult<()> {
self.events.lock().unwrap().push(event);
Ok(())
}
async fn append_events(
&self,
events: Vec<khive_storage::Event>,
) -> khive_storage::StorageResult<khive_storage::BatchWriteSummary> {
let n = events.len() as u64;
self.events.lock().unwrap().extend(events);
Ok(khive_storage::BatchWriteSummary {
attempted: n,
affected: n,
failed: 0,
first_error: String::new(),
})
}
async fn get_event(
&self,
id: uuid::Uuid,
) -> khive_storage::StorageResult<Option<khive_storage::Event>> {
Ok(self
.events
.lock()
.unwrap()
.iter()
.find(|e| e.id == id)
.cloned())
}
async fn query_events(
&self,
_filter: khive_storage::EventFilter,
_page: khive_storage::PageRequest,
) -> khive_storage::StorageResult<khive_storage::Page<khive_storage::Event>>
{
let items = self.events.lock().unwrap().clone();
let total = items.len() as u64;
Ok(khive_storage::Page {
items,
total: Some(total),
})
}
async fn count_events(
&self,
_filter: khive_storage::EventFilter,
) -> khive_storage::StorageResult<u64> {
Ok(self.events.lock().unwrap().len() as u64)
}
}
fn lifecycle_sequence(store: &FakeEventStore) -> Vec<khive_types::EventKind> {
store
.events
.lock()
.unwrap()
.iter()
.map(|e| e.kind)
.filter(|k| {
matches!(
k,
khive_types::EventKind::ChannelPollStarted
| khive_types::EventKind::ChannelPollSucceeded
| khive_types::EventKind::ChannelPollFailed
| khive_types::EventKind::ChannelBackoffArmed
| khive_types::EventKind::ChannelBackoffReset
)
})
.collect()
}
async fn advance_until(store: &FakeEventStore, target: usize) {
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(60);
loop {
if lifecycle_sequence(store).len() >= target {
return;
}
assert!(
std::time::Instant::now() < deadline,
"lifecycle event sequence did not reach {target} events within 60s of \
wall-clock time; got {:?}",
lifecycle_sequence(store)
);
tokio::time::advance(std::time::Duration::from_millis(250)).await;
for _ in 0..10 {
tokio::task::yield_now().await;
}
}
}
#[tokio::test(start_paused = true)]
async fn channel_lifecycle_events_are_sequenced_across_a_failure_then_recovery() {
let mut ch_registry = ChannelRegistry::new();
ch_registry.register(Arc::new(FlakyOnceChannel {
call_count: AtomicUsize::new(0),
}));
let runtime = KhiveRuntime::memory().expect("in-memory runtime");
let mut builder = VerbRegistryBuilder::new();
builder.register(khive_pack_kg::KgPack::new(runtime.clone()));
builder.register(khive_pack_comm::CommPack::new(runtime.clone()));
let store = Arc::new(FakeEventStore::default());
builder.with_event_store(store.clone());
let registry = builder.build().expect("registry builds");
let task = tokio::spawn(channel_poll_loop(
Arc::new(ch_registry),
registry,
"test-ns".to_string(),
"actor:test".to_string(),
));
advance_until(&store, 6).await;
task.abort();
let sequence = lifecycle_sequence(&store);
assert_eq!(
sequence,
vec![
khive_types::EventKind::ChannelPollStarted,
khive_types::EventKind::ChannelPollFailed,
khive_types::EventKind::ChannelBackoffArmed,
khive_types::EventKind::ChannelPollStarted,
khive_types::EventKind::ChannelPollSucceeded,
khive_types::EventKind::ChannelBackoffReset,
],
"ADR-094 lifecycle events must be sequenced exactly as the poll \
loop drives them: started -> failed -> backoff armed -> \
started -> succeeded -> backoff reset. Got: {sequence:?}"
);
}
#[tokio::test(start_paused = true)]
async fn channel_lifecycle_events_are_a_no_op_without_an_event_store() {
let mut ch_registry = ChannelRegistry::new();
ch_registry.register(Arc::new(FlakyOnceChannel {
call_count: AtomicUsize::new(0),
}));
let runtime = KhiveRuntime::memory().expect("in-memory runtime");
let mut builder = VerbRegistryBuilder::new();
builder.register(khive_pack_kg::KgPack::new(runtime.clone()));
builder.register(khive_pack_comm::CommPack::new(runtime.clone()));
let registry = builder.build().expect("registry builds");
assert!(
registry.event_store().is_none(),
"no event store was configured for this registry"
);
let task = tokio::spawn(channel_poll_loop(
Arc::new(ch_registry),
registry,
"test-ns".to_string(),
"actor:test".to_string(),
));
for _ in 0..48 {
tokio::time::advance(std::time::Duration::from_millis(250)).await;
tokio::task::yield_now().await;
}
task.abort();
}
}
#[cfg(feature = "channel-email")]
mod cursor_commit_gating_tests {
use super::*;
use async_trait::async_trait;
use chrono::{DateTime, Utc};
use khive_channel::{
Channel, ChannelCheckpoint, ChannelEnvelope, ChannelError, ChannelPollPage,
ChannelRegistry, StoredChannelCheckpoint,
};
use khive_runtime::{KhiveRuntime, VerbRegistryBuilder};
use serde_json::json;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
const SOURCE: &str = "imap+tls:h:993:m:INBOX";
struct PartialFailureChannel {
call_count: AtomicUsize,
observed_checkpoints: Arc<Mutex<Vec<Option<StoredChannelCheckpoint>>>>,
}
#[async_trait]
impl Channel for PartialFailureChannel {
fn kind(&self) -> &'static str {
"mock"
}
async fn send(&self, _envelope: ChannelEnvelope) -> Result<(), ChannelError> {
Ok(())
}
async fn poll(
&self,
_since: DateTime<Utc>,
) -> Result<Vec<ChannelEnvelope>, ChannelError> {
panic!("the daemon poll loop must call poll_page, not poll");
}
async fn poll_page(
&self,
_since: DateTime<Utc>,
checkpoint: Option<&StoredChannelCheckpoint>,
) -> Result<ChannelPollPage, ChannelError> {
let call = self.call_count.fetch_add(1, Ordering::SeqCst);
self.observed_checkpoints
.lock()
.unwrap()
.push(checkpoint.cloned());
let good = ChannelEnvelope::new(
"email:sender@example.com",
"email:me@example.com",
"good body",
)
.with_external_id("imap:h:1:1");
if call == 0 {
let bad = ChannelEnvelope::new(
"email:sender@example.com",
"email:me@example.com",
"",
);
Ok(ChannelPollPage {
envelopes: vec![good, bad],
next_checkpoint: Some(ChannelCheckpoint {
source: SOURCE.to_string(),
generation: 1,
high_water: Some(2),
}),
})
} else {
Ok(ChannelPollPage {
envelopes: vec![good],
next_checkpoint: Some(ChannelCheckpoint {
source: SOURCE.to_string(),
generation: 1,
high_water: Some(1),
}),
})
}
}
}
#[tokio::test(start_paused = true)]
async fn partial_ingest_failure_does_not_advance_cursor_and_dedup_prevents_double_store() {
let observed_checkpoints = Arc::new(Mutex::new(Vec::new()));
let mut ch_registry = ChannelRegistry::new();
ch_registry.register(Arc::new(PartialFailureChannel {
call_count: AtomicUsize::new(0),
observed_checkpoints: observed_checkpoints.clone(),
}));
let runtime = KhiveRuntime::memory().expect("in-memory runtime");
let mut builder = VerbRegistryBuilder::new();
builder.register(khive_pack_kg::KgPack::new(runtime.clone()));
builder.register(khive_pack_comm::CommPack::new(runtime.clone()));
let registry = builder.build().expect("registry builds");
let task = tokio::spawn(channel_poll_loop(
Arc::new(ch_registry),
registry.clone(),
"test-ns".to_string(),
"actor:test".to_string(),
));
let expected_committed_checkpoint = ChannelCheckpoint {
source: SOURCE.to_string(),
generation: 1,
high_water: Some(1),
};
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(60);
loop {
if observed_checkpoints
.lock()
.unwrap()
.get(2)
.is_some_and(|c| {
c.as_ref().map(|stored| &stored.checkpoint)
== Some(&expected_committed_checkpoint)
})
{
break;
}
assert!(
std::time::Instant::now() < deadline,
"the retry must have committed its checkpoint (observable on the \
third poll_page call) within 60s of wall-clock time: {:?}",
observed_checkpoints.lock().unwrap()
);
tokio::time::advance(std::time::Duration::from_secs(5)).await;
for _ in 0..20 {
tokio::task::yield_now().await;
}
}
task.abort();
let calls = observed_checkpoints.lock().unwrap().clone();
assert!(
calls.len() >= 3,
"the loop must have retried and then re-polled with the retry's \
committed checkpoint: {calls:?}"
);
assert!(
calls[0].is_none(),
"the first poll must see no persisted checkpoint"
);
assert!(
calls[1].is_none(),
"the cursor must NOT have advanced past the partially-failed page \
-- the retry must still see no committed checkpoint: {calls:?}"
);
assert_eq!(
calls[2].as_ref().map(|stored| &stored.checkpoint),
Some(&expected_committed_checkpoint),
"the third poll must observe the checkpoint the retry committed, \
proving the retry's comm.ingest (and its dedup) actually completed: \
{calls:?}"
);
let inbox = registry
.dispatch(
"list",
json!({"namespace": "test-ns", "kind": "message", "limit": 50}),
)
.await
.expect("list must succeed");
let notes = inbox.as_array().expect("list returns an array").clone();
let matching: Vec<_> = notes
.iter()
.filter(|n| {
n.get("properties")
.and_then(|p| p.get("external_id"))
.and_then(|v| v.as_str())
== Some("imap:h:1:1")
})
.collect();
assert_eq!(
matching.len(),
1,
"the message that succeeded on the failed page must not be \
double-stored once the retry re-delivers the whole page: {notes:?}"
);
}
struct CommitRejectedChannel {
call_count: AtomicUsize,
}
#[async_trait]
impl Channel for CommitRejectedChannel {
fn kind(&self) -> &'static str {
"mock_commit_rejected"
}
async fn send(&self, _envelope: ChannelEnvelope) -> Result<(), ChannelError> {
Ok(())
}
async fn poll(
&self,
_since: DateTime<Utc>,
) -> Result<Vec<ChannelEnvelope>, ChannelError> {
panic!("the daemon poll loop must call poll_page, not poll");
}
async fn poll_page(
&self,
_since: DateTime<Utc>,
_checkpoint: Option<&StoredChannelCheckpoint>,
) -> Result<ChannelPollPage, ChannelError> {
self.call_count.fetch_add(1, Ordering::SeqCst);
Ok(ChannelPollPage {
envelopes: vec![],
next_checkpoint: Some(ChannelCheckpoint {
source: SOURCE.to_string(),
generation: 0,
high_water: Some(1),
}),
})
}
}
#[tokio::test(start_paused = true)]
async fn rejected_cursor_commit_leaves_no_committed_checkpoint() {
let channel = Arc::new(CommitRejectedChannel {
call_count: AtomicUsize::new(0),
});
let mut ch_registry = ChannelRegistry::new();
ch_registry.register(channel.clone());
let runtime = KhiveRuntime::memory().expect("in-memory runtime");
let mut builder = VerbRegistryBuilder::new();
builder.register(khive_pack_kg::KgPack::new(runtime.clone()));
builder.register(khive_pack_comm::CommPack::new(runtime.clone()));
let registry = builder.build().expect("registry builds");
let task = tokio::spawn(channel_poll_loop(
Arc::new(ch_registry),
registry.clone(),
"test-ns".to_string(),
"actor:test".to_string(),
));
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(60);
loop {
if channel.call_count.load(Ordering::SeqCst) >= 2 {
break;
}
assert!(
std::time::Instant::now() < deadline,
"poll_page was not called at least twice within 60s of wall-clock time"
);
tokio::time::advance(std::time::Duration::from_millis(250)).await;
for _ in 0..10 {
tokio::task::yield_now().await;
}
}
task.abort();
let restored =
load_channel_cursor(®istry, "mock_commit_rejected", "mock_commit_rejected")
.await
.expect("cursor_get must succeed");
assert!(
restored.is_none(),
"a rejected cursor_commit must not leave a committed checkpoint: {restored:?}"
);
}
struct QuarantineOnceChannel {
envelope: Mutex<Option<ChannelEnvelope>>,
}
#[async_trait]
impl Channel for QuarantineOnceChannel {
fn kind(&self) -> &'static str {
"mock_quarantine"
}
async fn send(&self, _envelope: ChannelEnvelope) -> Result<(), ChannelError> {
Ok(())
}
async fn poll(
&self,
_since: DateTime<Utc>,
) -> Result<Vec<ChannelEnvelope>, ChannelError> {
panic!("the daemon poll loop must call poll_page, not poll");
}
async fn poll_page(
&self,
_since: DateTime<Utc>,
_checkpoint: Option<&StoredChannelCheckpoint>,
) -> Result<ChannelPollPage, ChannelError> {
let Some(envelope) = self.envelope.lock().unwrap().take() else {
return Ok(ChannelPollPage {
envelopes: vec![],
next_checkpoint: None,
});
};
Ok(ChannelPollPage {
envelopes: vec![envelope],
next_checkpoint: Some(ChannelCheckpoint {
source: SOURCE.to_string(),
generation: 9,
high_water: Some(1),
}),
})
}
}
#[tokio::test(start_paused = true)]
async fn malformed_message_durably_quarantines_with_stable_external_id_and_metadata() {
let mut envelope = ChannelEnvelope::new(
"email:quarantine",
"email:maintainer@example.com",
"(khive: IMAP message UID 1 could not be parsed and was quarantined)",
)
.with_external_id("imap:h:9:1");
envelope
.metadata
.insert("quarantined".to_string(), "true".to_string());
envelope
.metadata
.insert("quarantine_reason".to_string(), "missing-body".to_string());
let mut ch_registry = ChannelRegistry::new();
ch_registry.register(Arc::new(QuarantineOnceChannel {
envelope: Mutex::new(Some(envelope)),
}));
let runtime = KhiveRuntime::memory().expect("in-memory runtime");
let mut builder = VerbRegistryBuilder::new();
builder.register(khive_pack_kg::KgPack::new(runtime.clone()));
builder.register(khive_pack_comm::CommPack::new(runtime.clone()));
let registry = builder.build().expect("registry builds");
let task = tokio::spawn(channel_poll_loop(
Arc::new(ch_registry),
registry.clone(),
"test-ns".to_string(),
"actor:test".to_string(),
));
for _ in 0..2000 {
let restored = load_channel_cursor(®istry, "mock_quarantine", "mock_quarantine")
.await
.expect("cursor_get must succeed");
if restored.is_some() {
break;
}
tokio::time::advance(std::time::Duration::from_millis(250)).await;
for _ in 0..10 {
tokio::task::yield_now().await;
}
}
task.abort();
let restored = load_channel_cursor(®istry, "mock_quarantine", "mock_quarantine")
.await
.expect("cursor_get must succeed")
.expect(
"the cursor must have committed -- a quarantine envelope must ingest \
durably like any other message",
);
assert_eq!(restored.checkpoint.high_water, Some(1));
let inbox = registry
.dispatch(
"list",
json!({"namespace": "test-ns", "kind": "message", "limit": 50}),
)
.await
.expect("list must succeed");
let notes = inbox.as_array().expect("list returns an array").clone();
let quarantined = notes
.iter()
.find(|n| {
n.get("properties")
.and_then(|p| p.get("external_id"))
.and_then(|v| v.as_str())
== Some("imap:h:9:1")
})
.expect(
"the quarantined message must be durably queryable by its stable \
external_id, not just held as an intermediate value",
);
let props = quarantined
.get("properties")
.expect("stored note must carry properties");
assert_eq!(
props.get("quarantined").and_then(|v| v.as_str()),
Some("true"),
"durable quarantine metadata must survive comm.ingest: {props:?}"
);
assert_eq!(
props.get("quarantine_reason").and_then(|v| v.as_str()),
Some("missing-body"),
"the quarantine reason must survive comm.ingest: {props:?}"
);
}
#[tokio::test]
async fn committed_cursor_round_trips_across_a_fresh_cursor_get() {
let runtime = KhiveRuntime::memory().expect("in-memory runtime");
let mut builder = VerbRegistryBuilder::new();
builder.register(khive_pack_kg::KgPack::new(runtime.clone()));
builder.register(khive_pack_comm::CommPack::new(runtime.clone()));
let registry = builder.build().expect("registry builds");
commit_channel_cursor(
®istry,
"mock",
"mailbox-a",
&ChannelCheckpoint {
source: SOURCE.to_string(),
generation: 7,
high_water: Some(123),
},
)
.await
.expect("cursor_commit must succeed");
let restored = load_channel_cursor(®istry, "mock", "mailbox-a")
.await
.expect("cursor_get must succeed")
.expect("a committed checkpoint must round-trip, not read back as absent");
assert_eq!(restored.checkpoint.source, SOURCE);
assert_eq!(restored.checkpoint.generation, 7);
assert_eq!(restored.checkpoint.high_water, Some(123));
}
}
#[cfg(feature = "channel-email")]
mod bootstrap_since_floor_tests {
use super::*;
use async_trait::async_trait;
use chrono::{DateTime, Utc};
use khive_channel::{
Channel, ChannelEnvelope, ChannelError, ChannelPollPage, ChannelRegistry,
StoredChannelCheckpoint,
};
use khive_runtime::{KhiveRuntime, VerbRegistryBuilder};
use khive_storage::types::{SqlStatement, SqlValue};
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
struct RecordingChannel {
kind: &'static str,
since_calls: Arc<Mutex<Vec<DateTime<Utc>>>>,
}
#[async_trait]
impl Channel for RecordingChannel {
fn kind(&self) -> &'static str {
self.kind
}
async fn send(&self, _envelope: ChannelEnvelope) -> Result<(), ChannelError> {
Ok(())
}
async fn poll(
&self,
_since: DateTime<Utc>,
) -> Result<Vec<ChannelEnvelope>, ChannelError> {
panic!("the daemon poll loop must call poll_page, not poll");
}
async fn poll_page(
&self,
since: DateTime<Utc>,
_checkpoint: Option<&StoredChannelCheckpoint>,
) -> Result<ChannelPollPage, ChannelError> {
self.since_calls.lock().unwrap().push(since);
Ok(ChannelPollPage {
envelopes: vec![],
next_checkpoint: None,
})
}
}
struct IngestFailsOnceChannel {
call_count: AtomicUsize,
since_calls: Arc<Mutex<Vec<DateTime<Utc>>>>,
}
#[async_trait]
impl Channel for IngestFailsOnceChannel {
fn kind(&self) -> &'static str {
"mock_ingest_fails_once"
}
async fn send(&self, _envelope: ChannelEnvelope) -> Result<(), ChannelError> {
Ok(())
}
async fn poll(
&self,
_since: DateTime<Utc>,
) -> Result<Vec<ChannelEnvelope>, ChannelError> {
panic!("the daemon poll loop must call poll_page, not poll");
}
async fn poll_page(
&self,
since: DateTime<Utc>,
_checkpoint: Option<&StoredChannelCheckpoint>,
) -> Result<ChannelPollPage, ChannelError> {
self.since_calls.lock().unwrap().push(since);
if self.call_count.fetch_add(1, Ordering::SeqCst) == 0 {
let bad = ChannelEnvelope::new(
"email:sender@example.com",
"email:me@example.com",
"",
);
Ok(ChannelPollPage {
envelopes: vec![bad],
next_checkpoint: None,
})
} else {
Ok(ChannelPollPage {
envelopes: vec![],
next_checkpoint: None,
})
}
}
}
async fn advance_until_calls(calls: &Mutex<Vec<DateTime<Utc>>>, target: usize) {
for _ in 0..2000 {
if calls.lock().unwrap().len() >= target {
return;
}
tokio::time::advance(std::time::Duration::from_millis(250)).await;
for _ in 0..10 {
tokio::task::yield_now().await;
}
}
}
#[tokio::test(start_paused = true)]
async fn cursor_get_failure_preserves_the_bootstrap_floor() {
const BROKEN_KIND: &str = "mock_broken_cursor_get";
let runtime = KhiveRuntime::memory().expect("in-memory runtime");
let mut builder = VerbRegistryBuilder::new();
builder.register(khive_pack_kg::KgPack::new(runtime.clone()));
builder.register(khive_pack_comm::CommPack::new(runtime.clone()));
let registry = builder.build().expect("registry builds");
commit_channel_cursor(
®istry,
BROKEN_KIND,
BROKEN_KIND,
&khive_channel::ChannelCheckpoint {
source: "seed".to_string(),
generation: 1,
high_water: Some(1),
},
)
.await
.expect("seed cursor_commit must succeed");
let sql = runtime.sql();
{
let mut w = sql.writer().await.expect("writer");
w.execute(SqlStatement {
sql: "UPDATE comm_channel_cursor SET generation = 1.5 \
WHERE channel_kind = ?1 AND channel_slug = ?2"
.into(),
params: vec![
SqlValue::Text(BROKEN_KIND.to_string()),
SqlValue::Text(BROKEN_KIND.to_string()),
],
label: Some("test_corrupt_generation".into()),
})
.await
.expect("corrupting update must succeed");
}
let control_calls = Arc::new(Mutex::new(Vec::new()));
let broken_calls = Arc::new(Mutex::new(Vec::new()));
let mut ch_registry = ChannelRegistry::new();
ch_registry.register(Arc::new(RecordingChannel {
kind: "mock_control",
since_calls: control_calls.clone(),
}));
ch_registry.register(Arc::new(RecordingChannel {
kind: BROKEN_KIND,
since_calls: broken_calls.clone(),
}));
let task = tokio::spawn(channel_poll_loop(
Arc::new(ch_registry),
registry.clone(),
"test-ns".to_string(),
"actor:test".to_string(),
));
advance_until_calls(&control_calls, 1).await;
assert_eq!(
control_calls.lock().unwrap().len(),
1,
"control channel must be polled on the first tick"
);
assert_eq!(
broken_calls.lock().unwrap().len(),
0,
"the broken channel must be skipped while cursor_get fails"
);
{
let mut w = sql.writer().await.expect("writer");
w.execute(SqlStatement {
sql: "UPDATE comm_channel_cursor SET generation = 1 \
WHERE channel_kind = ?1 AND channel_slug = ?2"
.into(),
params: vec![
SqlValue::Text(BROKEN_KIND.to_string()),
SqlValue::Text(BROKEN_KIND.to_string()),
],
label: Some("test_repair_generation".into()),
})
.await
.expect("repairing update must succeed");
}
advance_until_calls(&broken_calls, 1).await;
task.abort();
let control_first = control_calls.lock().unwrap()[0];
let broken_first = *broken_calls
.lock()
.unwrap()
.first()
.expect("the broken channel must have been polled after recovery");
assert_eq!(
broken_first, control_first,
"the broken channel's first poll_page call must see the SAME \
bootstrap floor as the control channel's very first call \
({control_first:?}), not a later tick's timestamp \
({broken_first:?}) -- the cursor_get failure must not have \
lost the earlier floor"
);
}
#[tokio::test(start_paused = true)]
async fn quarantine_ingest_failure_blocking_first_commit_preserves_the_bootstrap_floor() {
let runtime = KhiveRuntime::memory().expect("in-memory runtime");
let mut builder = VerbRegistryBuilder::new();
builder.register(khive_pack_kg::KgPack::new(runtime.clone()));
builder.register(khive_pack_comm::CommPack::new(runtime.clone()));
let registry = builder.build().expect("registry builds");
let control_calls = Arc::new(Mutex::new(Vec::new()));
let failing_calls = Arc::new(Mutex::new(Vec::new()));
let mut ch_registry = ChannelRegistry::new();
ch_registry.register(Arc::new(RecordingChannel {
kind: "mock_control",
since_calls: control_calls.clone(),
}));
ch_registry.register(Arc::new(IngestFailsOnceChannel {
call_count: AtomicUsize::new(0),
since_calls: failing_calls.clone(),
}));
let task = tokio::spawn(channel_poll_loop(
Arc::new(ch_registry),
registry.clone(),
"test-ns".to_string(),
"actor:test".to_string(),
));
advance_until_calls(&control_calls, 1).await;
advance_until_calls(&failing_calls, 1).await;
assert_eq!(control_calls.lock().unwrap().len(), 1);
assert_eq!(
failing_calls.lock().unwrap().len(),
1,
"poll_page is still called even though ingest will fail"
);
advance_until_calls(&failing_calls, 2).await;
task.abort();
let control_first = control_calls.lock().unwrap()[0];
let failing_calls = failing_calls.lock().unwrap();
assert_eq!(
failing_calls.len(),
2,
"the channel must have been polled again on the second tick"
);
assert_eq!(
failing_calls[0], control_first,
"the first poll_page call's `since` must match the control \
channel's first-tick floor"
);
assert_eq!(
failing_calls[1], control_first,
"the SECOND poll_page call's `since` must still match the \
same original floor ({control_first:?}), not a fresh \
timestamp from the tick where the ingest failure blocked \
the first commit ({:?}) -- otherwise a failure spanning a \
calendar-day boundary would permanently skip the earlier \
day's uncommitted mail",
failing_calls[1]
);
}
#[tokio::test]
async fn first_tick_uses_startup_time_not_post_sleep_time() {
let runtime = KhiveRuntime::memory().expect("in-memory runtime");
let mut builder = VerbRegistryBuilder::new();
builder.register(khive_pack_kg::KgPack::new(runtime.clone()));
builder.register(khive_pack_comm::CommPack::new(runtime.clone()));
let registry = builder.build().expect("registry builds");
let calls = Arc::new(Mutex::new(Vec::new()));
let mut ch_registry = ChannelRegistry::new();
ch_registry.register(Arc::new(RecordingChannel {
kind: "mock_startup_clock",
since_calls: calls.clone(),
}));
let startup = Utc::now();
let task = tokio::spawn(channel_poll_loop(
Arc::new(ch_registry),
registry.clone(),
"test-ns".to_string(),
"actor:test".to_string(),
));
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(15);
loop {
if !calls.lock().unwrap().is_empty() {
break;
}
assert!(
std::time::Instant::now() <= deadline,
"first poll_page call did not arrive within 15s"
);
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
}
task.abort();
let first_since = calls.lock().unwrap()[0];
let drift_ms = (first_since - startup).num_milliseconds().abs();
assert!(
drift_ms < 2_000,
"the first poll_page call's `since` ({first_since:?}) must \
reflect the daemon's startup time ({startup:?}), not a \
timestamp captured after the loop's first ~5s sleep -- a \
{drift_ms}ms drift means the floor is still seeded \
post-sleep, which would drop a full day of mail if that \
sleep happened to cross a calendar-day boundary"
);
}
}
}