pub mod admin;
pub mod auth;
pub mod config_monitor;
pub mod dedup;
pub mod fallback_replay;
pub mod handlers;
pub mod identity;
pub mod policy_poller;
pub mod reconciler;
pub mod watcher;
use std::path::Path;
use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU32, AtomicU64, Ordering};
use std::sync::{Arc, Mutex};
use axum::{extract::DefaultBodyLimit, middleware, routing::get, routing::post, Router};
use secrecy::SecretString;
use tokio::net::TcpListener;
use crate::cloud::{CloudState, CredentialProvider};
use crate::config::Config;
use crate::core::logging::tamper_log::{TamperLogger, TamperLoggerHandle};
use crate::core::supervision::task::{spawn_supervised, HealthRegistry, RestartPolicy, TaskSpec};
use crate::logging::{EventLogger, EventLoggerHandle};
use crate::privacy::PrivacyFilter;
use crate::update;
mod subsystem {
pub const BOUNDARY: &str = "boundary";
pub const BOUNDARY_WIRING: &str = "boundary-wiring";
pub const CLOUD_WORKER: &str = "cloud-worker";
pub const ALERTS_LONG_POLL: &str = "alerts-long-poll";
pub const POLICY_POLLER: &str = "policy-poller";
pub const RECONCILER: &str = "reconciler";
pub const DEDUP_EVICTOR: &str = "dedup-evictor";
pub const LOG_CLEANUP: &str = "log-cleanup";
pub const EVENT_LOG_WRITER: &str = "event-log-writer";
pub const TAMPER_LOG_WRITER: &str = "tamper-log-writer";
pub const FALLBACK_REPLAY: &str = "fallback-replay";
pub const UPDATE_CHECK: &str = "update-check";
pub const AUTO_UPDATE_WORKER: &str = "auto-update-worker";
pub const UPDATE_SENTINEL: &str = "update-sentinel";
}
struct CredentialStoreAdapter {
store: Arc<dyn crate::auth::CredentialStore>,
}
impl CredentialProvider for CredentialStoreAdapter {
fn retrieve(&self) -> Option<SecretString> {
match self.store.retrieve() {
Ok(key) => Some(key),
Err(e) => {
tracing::debug!(
code = e.code,
"credential provider: retrieve failed — returning None to worker"
);
None
}
}
}
}
pub struct PolicyRuntime {
pub handle: crate::core::policy::PolicyHandle,
pub last_fetch_ok: Arc<AtomicBool>,
pub last_poll_ok_at: Arc<AtomicI64>,
}
impl PolicyRuntime {
pub fn load_from_disk(base_dir: &Path) -> Self {
use crate::core::policy::{store, ResidentBundle};
let cached = match store::load(base_dir) {
Ok(cached) => cached,
Err(e) => {
tracing::warn!(
target: "policy",
code = e.code(),
error = %e,
"cached policy bundle rejected at startup; running with no policy until the next successful fetch"
);
None
}
};
let last_fetch_ok = Arc::new(AtomicBool::new(true));
let last_poll_ok_at = Arc::new(AtomicI64::new(0));
let resident = cached.map(|c| {
last_fetch_ok.store(c.meta.last_fetch_ok, Ordering::Relaxed);
if let Some(secs) = c
.meta
.last_poll_ok_at
.as_deref()
.and_then(|s| chrono::DateTime::parse_from_rfc3339(s).ok())
.map(|dt| dt.timestamp())
{
last_poll_ok_at.store(secs, Ordering::Relaxed);
}
ResidentBundle::from_bundle(&c.bundle)
});
Self {
handle: crate::core::policy::new_handle(resident),
last_fetch_ok,
last_poll_ok_at,
}
}
fn log_startup_state(&self) {
match self.handle.load().as_ref() {
Some(b) => tracing::info!(
target: "policy",
revision = b.revision,
rules = b.command_rules.len(),
request_rules = b.request_rules.len(),
enforcement_enabled = b.enforcement_enabled,
organization_id = %b.organization_id,
"policy engine enabled; enforcing the cached bundle"
),
None => tracing::info!(
target: "policy",
"policy engine enabled; no bundle — allowing everything until the first successful fetch"
),
}
}
}
pub struct AppState {
pub config: Arc<Config>,
pub token: String,
pub dedup: dedup::DedupStore,
pub event_logger: EventLogger,
pub privacy_filter: PrivacyFilter,
pub event_counter: AtomicU64,
pub shutdown_tx: tokio::sync::Mutex<Option<tokio::sync::oneshot::Sender<()>>>,
pub started_at: std::time::Instant,
pub available_update: Mutex<Option<String>>,
pub cloud_tx: Option<tokio::sync::mpsc::Sender<crate::cloud::CloudEvent>>,
pub cloud_state: Option<CloudState>,
pub local_ipv4: Option<std::net::Ipv4Addr>,
pub local_ipv6: Option<std::net::Ipv6Addr>,
pub public_ipv4: Option<std::net::Ipv4Addr>,
pub public_ipv6: Option<std::net::Ipv6Addr>,
pub tamper_logger: Option<TamperLogger>,
pub outbox: Option<Arc<crate::cloud::outbox::Outbox>>,
pub update_in_progress: Arc<AtomicBool>,
pub update_status: Arc<Mutex<update::UpdateStatusSnapshot>>,
pub admin_shutdown_request: Arc<tokio::sync::Notify>,
pub last_hook_at_unix_secs: Arc<AtomicU64>,
pub hooks_in_flight: Arc<AtomicU32>,
pub content_hash_cache: Arc<config_monitor::ContentHashCache>,
pub config_monitor_request_tx:
Option<tokio::sync::mpsc::Sender<config_monitor::ConfigChangeRequest>>,
pub pending_alerts: Arc<config_monitor::PendingAlerts>,
pub policy: Option<PolicyRuntime>,
pub registry: Arc<crate::boundary::session::SessionRegistry>,
pub health: Arc<HealthRegistry>,
}
impl AppState {
pub fn set_available_update(&self, version: String) {
if let Ok(mut guard) = self.available_update.lock() {
*guard = Some(version);
}
}
pub fn get_available_update(&self) -> Option<String> {
self.available_update.lock().ok().and_then(|g| g.clone())
}
}
pub async fn start_server(
config: Config,
token: String,
credential_store: Option<Arc<dyn crate::auth::CredentialStore>>,
spawn_boundary: bool,
) -> anyhow::Result<(u64, u64)> {
let bind_host = match std::env::var("OPENLATCH_BIND_ALL").as_deref() {
Ok("true") | Ok("1") => "0.0.0.0",
_ => "127.0.0.1",
};
let bind_addr = format!("{}:{}", bind_host, config.port);
let listener = TcpListener::bind(&bind_addr).await?;
tracing::info!(
port = config.port,
addr = %bind_addr,
"daemon listening"
);
if let Err(e) = crate::config::write_port_file(config.port) {
tracing::warn!(error = %e, "failed to write daemon.port file");
}
serve_with_listener(
listener,
config,
token,
credential_store,
spawn_boundary,
true,
)
.await
}
pub async fn start_server_with_listener(
listener: TcpListener,
config: Config,
token: String,
credential_store: Option<Arc<dyn crate::auth::CredentialStore>>,
spawn_boundary: bool,
) -> anyhow::Result<(u64, u64)> {
serve_with_listener(
listener,
config,
token,
credential_store,
spawn_boundary,
false,
)
.await
}
async fn serve_with_listener(
listener: TcpListener,
config: Config,
token: String,
credential_store: Option<Arc<dyn crate::auth::CredentialStore>>,
spawn_boundary: bool,
reconcile_wiring: bool,
) -> anyhow::Result<(u64, u64)> {
let health = Arc::new(HealthRegistry::new());
let (tasks_shutdown_tx, tasks_shutdown_rx) = tokio::sync::watch::channel(false);
let (_logs_shutdown_tx, logs_shutdown_rx) = tokio::sync::watch::channel(false);
let log_dir = config.log_dir.clone();
let (event_logger, event_log_rx) = EventLogger::channel();
let event_log_rx = Arc::new(tokio::sync::Mutex::new(event_log_rx));
let logger_handle = {
let log_dir = log_dir.clone();
EventLoggerHandle::from_task(spawn_supervised(
&health,
TaskSpec::new(subsystem::EVENT_LOG_WRITER, RestartPolicy::OnFailure),
logs_shutdown_rx.clone(),
move || {
let rx = event_log_rx.clone();
let log_dir = log_dir.clone();
async move {
let mut rx = rx.lock_owned().await;
crate::logging::run_event_writer(log_dir, &mut rx).await
}
},
))
};
let privacy_filter = PrivacyFilter::new(&config.extra_patterns);
let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel::<()>();
let pending_alerts = Arc::new(config_monitor::PendingAlerts::new());
let policy_runtime = if config.policy.enabled {
let runtime = PolicyRuntime::load_from_disk(&crate::config::openlatch_dir());
runtime.log_startup_state();
Some(runtime)
} else {
tracing::info!(
target: "policy",
"policy engine disabled by config ([policy] enabled = false); no bundle is fetched and no resident bundle is consulted"
);
None
};
let registry = Arc::new(crate::boundary::session::SessionRegistry::default());
#[allow(clippy::type_complexity)]
let (cloud_tx, cloud_state_opt, outbox_opt, cloud_worker_task): (
_,
_,
_,
Option<tokio::task::JoinHandle<()>>,
) = if config.cloud.enabled {
if let Some(store) = credential_store {
let (tx, rx) = tokio::sync::mpsc::channel(config.cloud.channel_size);
let cloud_state = CloudState::new();
let cloud_config = crate::cloud::CloudConfig {
api_url: config.cloud.api_url.clone(),
timeout_connect_ms: config.cloud.timeout_connect_ms,
timeout_total_ms: config.cloud.timeout_total_ms,
retry_delay_ms: config.cloud.retry_delay_ms,
channel_size: config.cloud.channel_size,
rate_limit_default_secs: 30,
credential_poll_interval_ms: config.cloud.credential_poll_interval_ms,
fallback_max_bytes: config.cloud.fallback_max_bytes,
batch_max_events: config.cloud.batch_max_events,
batch_max_wait_ms: config.cloud.batch_max_wait_ms,
};
let openlatch_dir = crate::config::openlatch_dir();
let provider: Arc<dyn CredentialProvider> = Arc::new(CredentialStoreAdapter { store });
let worker_state = cloud_state.clone();
let alerts_provider = provider.clone();
let alerts_state = cloud_state.clone();
let policy_provider = provider.clone();
let policy_state = cloud_state.clone();
let policy_dir = openlatch_dir.clone();
let outbox = if config.cloud.outbox_enabled {
Some(Arc::new(crate::cloud::outbox::Outbox::new(
&openlatch_dir,
config.cloud.outbox_max_bytes,
)))
} else {
None
};
let cloud_rx = Arc::new(tokio::sync::Mutex::new(rx));
let worker_outbox = outbox.clone();
let cloud_worker = spawn_supervised(
&health,
TaskSpec::new(subsystem::CLOUD_WORKER, RestartPolicy::Always),
tasks_shutdown_rx.clone(),
{
let shutdown_rx = tasks_shutdown_rx.clone();
move || {
let rx = cloud_rx.clone();
let provider = provider.clone();
let cloud_config = cloud_config.clone();
let worker_state = worker_state.clone();
let openlatch_dir = openlatch_dir.clone();
let outbox = worker_outbox.clone();
let shutdown_rx = shutdown_rx.clone();
async move {
let mut rx = rx.lock_owned().await;
crate::cloud::worker::run_cloud_worker_on(
&mut rx,
provider,
cloud_config,
worker_state,
openlatch_dir,
outbox,
Some(shutdown_rx),
)
.await
}
}
},
);
tracing::info!(
api_url = %config.cloud.api_url,
channel_size = config.cloud.channel_size,
"cloud forwarding worker started"
);
let alerts_url = config.cloud.api_url.clone();
let alerts_target = pending_alerts.clone();
let alerts_client = match reqwest::Client::builder()
.timeout(std::time::Duration::from_millis(
config.cloud.timeout_total_ms,
))
.build()
{
Ok(c) => Some(c),
Err(e) => {
tracing::warn!(error = %e, "alerts long-poll http client init failed");
None
}
};
match (alerts_client, config.agent_id.clone()) {
(Some(client), Some(machine_id)) => {
spawn_supervised(
&health,
TaskSpec::new(subsystem::ALERTS_LONG_POLL, RestartPolicy::Always),
tasks_shutdown_rx.clone(),
move || {
config_monitor::run_long_poll(
alerts_target.clone(),
alerts_state.clone(),
alerts_provider.clone(),
alerts_url.clone(),
machine_id.clone(),
client.clone(),
)
},
);
}
(Some(_), None) => {
tracing::warn!(
"alerts long-poll skipped: agent_id missing from config.toml; run `openlatch init` to provision"
);
}
_ => {}
}
if let Some(policy) = policy_runtime.as_ref() {
match reqwest::Client::builder()
.timeout(std::time::Duration::from_millis(
config.cloud.timeout_total_ms,
))
.build()
{
Ok(client) => {
let policy_handle = policy.handle.clone();
let policy_last_fetch_ok = policy.last_fetch_ok.clone();
let policy_last_poll_ok_at = policy.last_poll_ok_at.clone();
let policy_api_url = config.cloud.api_url.clone();
let policy_cfg = config.policy.clone();
let policy_agent_id = config.agent_id.clone();
spawn_supervised(
&health,
TaskSpec::new(subsystem::POLICY_POLLER, RestartPolicy::Always),
tasks_shutdown_rx.clone(),
move || {
policy_poller::run_policy_poller(
policy_handle.clone(),
policy_last_fetch_ok.clone(),
policy_last_poll_ok_at.clone(),
policy_state.clone(),
policy_provider.clone(),
policy_api_url.clone(),
policy_cfg.clone(),
policy_dir.clone(),
client.clone(),
policy_agent_id.clone(),
)
},
);
tracing::info!(
target: "policy",
poll_interval_secs = config.policy.poll_interval_secs,
"policy bundle poller started"
);
}
Err(e) => {
tracing::warn!(
target: "policy",
code = crate::error::ERR_BUNDLE_FETCH_FAILED,
error = %e,
"policy poller http client init failed; the resident bundle keeps enforcing but will not refresh"
);
}
}
}
(Some(tx), Some(cloud_state), outbox, Some(cloud_worker))
} else {
tracing::info!("cloud forwarding enabled in config but no credential store provided — cloud forwarding disabled");
(None, None, None, None)
}
} else {
(None, None, None, None)
};
if policy_runtime.is_some() && cloud_state_opt.is_none() {
tracing::warn!(
target: "policy",
"policy is enabled but no credential provider is available; running disk-bundle-only — the resident bundle keeps enforcing and will not refresh"
);
}
let startup_started = std::time::Instant::now();
let host_ips = crate::net::HostIps::detect().await;
tracing::info!(
local_ipv4 = host_ips
.local_ipv4
.map(|a| a.to_string())
.as_deref()
.unwrap_or("none"),
local_ipv6 = host_ips
.local_ipv6
.map(|a| a.to_string())
.as_deref()
.unwrap_or("none"),
public_ipv4 = host_ips
.public_ipv4
.map(|a| a.to_string())
.as_deref()
.unwrap_or("none"),
public_ipv6 = host_ips
.public_ipv6
.map(|a| a.to_string())
.as_deref()
.unwrap_or("none"),
"host ips detected"
);
tokio::task::spawn_blocking(crate::daemon::identity::os_user);
let openlatch_dir_for_tamper = crate::config::openlatch_dir();
let (tamper_logger, tamper_rx) = TamperLogger::channel();
let tamper_rx = Arc::new(tokio::sync::Mutex::new(tamper_rx));
let _tamper_logger_handle: TamperLoggerHandle =
TamperLoggerHandle::from_task(spawn_supervised(
&health,
TaskSpec::new(subsystem::TAMPER_LOG_WRITER, RestartPolicy::OnFailure),
logs_shutdown_rx.clone(),
move || {
let rx = tamper_rx.clone();
let dir = openlatch_dir_for_tamper.clone();
async move {
let mut rx = rx.lock_owned().await;
crate::core::logging::tamper_log::run_tamper_writer(dir, &mut rx).await
}
},
));
let cache_max = config.inventory_monitor.cache_max_entries.max(64);
let content_hash_cache = Arc::new(config_monitor::ContentHashCache::new(cache_max));
let mut config_monitor_handle: Option<config_monitor::ConfigMonitorHandle> = None;
let mut config_monitor_request_tx: Option<
tokio::sync::mpsc::Sender<config_monitor::ConfigChangeRequest>,
> = None;
if config.inventory_monitor.enabled {
match config_monitor::manifest::load_embedded() {
Ok(manifest) => {
let monitor = config_monitor::ConfigMonitor::new(
Arc::new(manifest),
content_hash_cache.clone(),
privacy_filter.clone(),
cloud_tx.clone(),
event_logger.clone(),
Arc::new(config.clone()),
);
match monitor.spawn().await {
Ok(handle) => {
config_monitor_request_tx = Some(handle.request_tx.clone());
config_monitor_handle = Some(handle);
tracing::info!("config monitor active");
}
Err(e) => {
tracing::error!(
code = crate::error::ERR_INVENTORY_INIT_FAILED,
error = %e,
"config monitor failed to start"
);
}
}
}
Err(e) => {
tracing::error!(
code = crate::error::ERR_INVENTORY_MANIFEST_PARSE,
error = %e,
"failed to load inventory manifest; config monitoring disabled"
);
}
}
} else {
tracing::info!("config monitor disabled by config");
}
#[cfg(feature = "boundary")]
#[allow(clippy::type_complexity)]
let (boundary_task, wiring_task): (
Option<tokio::task::JoinHandle<()>>,
Option<tokio::task::JoinHandle<()>>,
) = if spawn_boundary {
use crate::boundary;
let boundary_port = config.boundary.port;
let first_listener = boundary::bind_pinned(boundary_port).await?;
tracing::info!(
port = boundary_port,
"boundary listener bound (loopback only)"
);
let wiring = Arc::new(boundary::preflight::WiringState::default());
let boundary_patterns = config.extra_patterns.clone();
let boundary_registry = registry.clone();
let boundary_cloud_tx = cloud_tx.clone();
let boundary_shutdown_rx = tasks_shutdown_rx.clone();
let pre_bound = Arc::new(tokio::sync::Mutex::new(Some(first_listener)));
let boundary_policy = policy_runtime.as_ref().map(|p| p.handle.clone());
let boundary_wiring = wiring.clone();
let boundary_upstream = config.boundary.upstream_url();
let serve_task = spawn_supervised(
&health,
TaskSpec::new(subsystem::BOUNDARY, RestartPolicy::Always),
tasks_shutdown_rx.clone(),
move || {
let bstate = Arc::new(
boundary::BoundaryState::new(
boundary_upstream.clone(),
boundary_port,
boundary::DEFAULT_INFLIGHT,
&boundary_patterns,
)
.with_measurement(boundary_registry.clone(), boundary_cloud_tx.clone())
.with_policy(boundary_policy.clone())
.with_wiring(boundary_wiring.clone()),
);
boundary::serve_attempt(pre_bound.clone(), bstate, boundary_shutdown_rx.clone())
},
);
let wiring_config = Arc::new(config.clone());
let wiring_state = wiring.clone();
let wiring_shutdown = tasks_shutdown_rx.clone();
let wiring_task = spawn_supervised(
&health,
TaskSpec::new(subsystem::BOUNDARY_WIRING, RestartPolicy::Always),
tasks_shutdown_rx.clone(),
move || {
run_wiring_supervisor(
wiring_config.clone(),
boundary_port,
wiring_state.clone(),
wiring_shutdown.clone(),
)
},
);
(Some(serve_task), Some(wiring_task))
} else {
if reconcile_wiring && config.boundary.owns_agent_wiring() {
unwire_boundary_config();
}
(None, None)
};
#[cfg(feature = "boundary")]
let owns_agent_wiring = config.boundary.owns_agent_wiring();
let state = Arc::new(AppState {
config: Arc::new(config.clone()),
token,
dedup: dedup::DedupStore::new(),
event_logger,
privacy_filter,
event_counter: AtomicU64::new(0),
shutdown_tx: tokio::sync::Mutex::new(Some(shutdown_tx)),
started_at: std::time::Instant::now(),
available_update: Mutex::new(None),
cloud_tx,
cloud_state: cloud_state_opt,
local_ipv4: host_ips.local_ipv4,
local_ipv6: host_ips.local_ipv6,
public_ipv4: host_ips.public_ipv4,
public_ipv6: host_ips.public_ipv6,
tamper_logger: Some(tamper_logger),
outbox: outbox_opt,
update_in_progress: Arc::new(AtomicBool::new(false)),
update_status: Arc::new(Mutex::new(update::UpdateStatusSnapshot::idle())),
admin_shutdown_request: Arc::new(tokio::sync::Notify::new()),
last_hook_at_unix_secs: Arc::new(AtomicU64::new(0)),
hooks_in_flight: Arc::new(AtomicU32::new(0)),
content_hash_cache,
config_monitor_request_tx,
pending_alerts,
policy: policy_runtime,
registry,
health: health.clone(),
});
let _config_monitor_handle = config_monitor_handle;
crate::telemetry::capture_global(crate::telemetry::Event::daemon_started(
state.config.port,
startup_started
.elapsed()
.as_millis()
.min(u128::from(u64::MAX)) as u64,
state.config.cloud.enabled,
));
{
let state_for_replay = state.clone();
spawn_supervised(
&health,
TaskSpec::new(subsystem::FALLBACK_REPLAY, RestartPolicy::OnFailure),
tasks_shutdown_rx.clone(),
move || fallback_replay::run(state_for_replay.clone()),
);
}
if config.update.check {
let current = env!("CARGO_PKG_VERSION").to_string();
let state_for_update = state.clone();
spawn_supervised(
&health,
TaskSpec::new(subsystem::UPDATE_CHECK, RestartPolicy::OnFailure),
tasks_shutdown_rx.clone(),
move || {
let current = current.clone();
let state_for_update = state_for_update.clone();
async move {
if let Some(latest) = update::check_for_update(¤t).await {
tracing::warn!(code = crate::error::ERR_VERSION_OUTDATED, latest_version = %latest, "Update available: run `npx openlatch@latest`");
state_for_update.set_available_update(latest);
}
}
},
);
}
if config.update.auto_update {
let state_for_worker = state.clone();
spawn_supervised(
&health,
TaskSpec::new(subsystem::AUTO_UPDATE_WORKER, RestartPolicy::OnFailure),
tasks_shutdown_rx.clone(),
move || run_auto_update_worker(state_for_worker.clone()),
);
} else {
tracing::info!(target: "update", "auto-update worker disabled by config");
}
if let Some(sentinel) = update::read_sentinel() {
let port = config.port;
spawn_supervised(
&health,
TaskSpec::new(subsystem::UPDATE_SENTINEL, RestartPolicy::OnFailure),
tasks_shutdown_rx.clone(),
move || {
let sentinel = sentinel.clone();
async move {
tokio::time::sleep(std::time::Duration::from_secs(5)).await;
if !probe_self_health(port).await {
tracing::warn!(target: "update", from = %sentinel.from, to = %sentinel.to, "post-restart /health probe failed; leaving sentinel + .bak in place for restart-loop rollback");
return;
}
tracing::info!(target: "update", from = %sentinel.from, to = %sentinel.to, "post-restart healthz probe succeeded; cleaning up");
if let Err(e) = update::cleanup_bak_files() {
tracing::warn!(target: "update", error = %e, "cleanup of .bak siblings failed (non-fatal)");
}
if let Err(e) = update::delete_sentinel() {
tracing::warn!(target: "update", error = %e, "delete of update sentinel failed (non-fatal)");
}
crate::install_state::InstallState::stamp_for_running_binary(env!(
"CARGO_PKG_VERSION"
));
}
},
);
}
let state_for_evict = state.clone();
spawn_supervised::<_, _, ()>(
&health,
TaskSpec::new(subsystem::DEDUP_EVICTOR, RestartPolicy::Always),
tasks_shutdown_rx.clone(),
move || {
let state = state_for_evict.clone();
async move {
let mut interval = tokio::time::interval(std::time::Duration::from_secs(30));
loop {
interval.tick().await;
state.dedup.evict_expired();
}
}
},
);
let state_for_cleanup = state.clone();
spawn_supervised::<_, _, ()>(
&health,
TaskSpec::new(subsystem::LOG_CLEANUP, RestartPolicy::Always),
tasks_shutdown_rx.clone(),
move || {
let state = state_for_cleanup.clone();
async move {
let mut interval = tokio::time::interval(std::time::Duration::from_secs(86_400));
loop {
interval.tick().await;
let log_dir = state.config.log_dir.clone();
let retention = state.config.retention_days;
match tokio::task::spawn_blocking(move || {
crate::logging::cleanup_old_logs(&log_dir, retention)
})
.await
{
Ok(Ok(deleted)) if deleted > 0 => {
tracing::info!(
deleted,
retention_days = retention,
"cleaned up old log files"
);
}
Ok(Ok(_)) => {}
Ok(Err(e)) => {
tracing::warn!(error = %e, "periodic log cleanup failed");
}
Err(e) => {
tracing::warn!(error = %e, "periodic log cleanup task join error");
}
}
}
}
},
);
let _watcher_guard;
let _poll_handle;
if let Ok(agent) = crate::hooks::detect_agent() {
let settings_path = match &agent {
crate::hooks::DetectedAgent::ClaudeCode { settings_path, .. } => settings_path.clone(),
};
let openlatch_dir = crate::config::openlatch_dir();
let token_file = openlatch_dir.join("daemon.token");
reconciler::run_startup_reconcile(&settings_path, &openlatch_dir, config.port);
let (reconcile_tx, reconcile_rx) = tokio::sync::mpsc::channel(100);
let sinks = reconciler::TamperSinks {
logger: state.tamper_logger.clone(),
cloud_tx: state.cloud_tx.clone(),
agent_id: state.config.agent_id.clone().unwrap_or_default(),
client_version: env!("OPENLATCH_VERSION").to_string(),
};
let r = Arc::new(tokio::sync::Mutex::new(
reconciler::Reconciler::new_with_sinks(
reconcile_rx,
settings_path.clone(),
openlatch_dir,
config.port,
token_file,
sinks,
),
));
spawn_supervised(
&health,
TaskSpec::new(subsystem::RECONCILER, RestartPolicy::Always),
tasks_shutdown_rx.clone(),
move || {
let r = r.clone();
async move {
let mut guard = r.lock_owned().await;
guard.run().await
}
},
);
_watcher_guard = match watcher::spawn_watcher(&settings_path, reconcile_tx.clone()) {
Ok(w) => {
tracing::info!("filesystem watcher active");
Some(w)
}
Err(e) => {
tracing::warn!(error = %e, "filesystem watcher failed — falling back to poll-only");
None
}
};
_poll_handle = Some(watcher::spawn_poll_fallback(reconcile_tx));
tracing::info!("reconciler started (reactive watcher + 30s poll)");
} else {
_watcher_guard = None;
_poll_handle = None;
}
let hook_routes = Router::new()
.route("/hooks", post(handlers::ingest_cloudevent))
.route_layer(middleware::from_fn_with_state(
state.clone(),
auth::bearer_auth,
));
let shutdown_route = Router::new()
.route("/shutdown", post(handlers::shutdown_handler))
.route_layer(middleware::from_fn_with_state(
state.clone(),
auth::bearer_auth,
));
let public_routes = Router::new()
.route("/health", get(handlers::health))
.route("/metrics", get(handlers::metrics));
let admin_routes = admin::router(state.clone());
let app = Router::new()
.merge(hook_routes)
.merge(shutdown_route)
.merge(public_routes)
.merge(admin_routes)
.layer(DefaultBodyLimit::max(1_048_576))
.with_state(state.clone());
let admin_shutdown_request = state.admin_shutdown_request.clone();
axum::serve(listener, app)
.with_graceful_shutdown(async move {
tokio::select! {
_ = signal_handler() => {
tracing::info!("received OS shutdown signal");
}
_ = shutdown_rx => {
tracing::info!("received shutdown via /shutdown endpoint");
}
_ = admin_shutdown_request.notified() => {
tracing::info!(target: "update", "received shutdown for in-flight auto-update");
}
}
})
.await?;
let _ = tasks_shutdown_tx.send(true);
if let Some(cloud_worker) = cloud_worker_task {
if tokio::time::timeout(std::time::Duration::from_secs(5), cloud_worker)
.await
.is_err()
{
tracing::warn!(
"cloud worker did not finish its shutdown flush within 5s — abandoning it"
);
}
}
#[cfg(feature = "boundary")]
if let Some(boundary_task) = boundary_task {
if let Some(wiring_task) = wiring_task {
if tokio::time::timeout(std::time::Duration::from_secs(5), wiring_task)
.await
.is_err()
{
tracing::warn!("boundary wiring supervisor did not stop within 5s — abandoning it");
}
}
if tokio::time::timeout(std::time::Duration::from_secs(5), boundary_task)
.await
.is_err()
{
tracing::warn!("boundary listener did not shut down within 5s — abandoning it");
}
if owns_agent_wiring {
unwire_boundary_config();
}
}
let uptime_secs = state.started_at.elapsed().as_secs();
let events = state
.event_counter
.load(std::sync::atomic::Ordering::Relaxed);
crate::logging::daemon_log::log_shutdown(uptime_secs, events);
crate::telemetry::capture_global(crate::telemetry::Event::daemon_stopped(uptime_secs, events));
if !crate::telemetry::flush_global().await {
tracing::debug!("telemetry: final flush did not complete within budget");
}
match Arc::try_unwrap(state) {
Ok(_state) => {
if tokio::time::timeout(std::time::Duration::from_secs(5), logger_handle.shutdown())
.await
.is_err()
{
tracing::warn!("event-log drain did not finish within 5s — abandoning it");
}
}
Err(arc) => {
tracing::warn!(
strong_refs = Arc::strong_count(&arc),
"AppState still has references at shutdown — final event-log batch may be dropped"
);
drop(arc);
}
}
Ok((uptime_secs, events))
}
pub fn format_uptime(secs: u64) -> String {
let hours = secs / 3600;
let minutes = (secs % 3600) / 60;
let seconds = secs % 60;
if hours > 0 {
format!("{}h{}m", hours, minutes)
} else if minutes > 0 {
format!("{}m{}s", minutes, seconds)
} else {
format!("{}s", seconds)
}
}
async fn probe_self_health(port: u16) -> bool {
let Ok(client) = reqwest::Client::builder()
.timeout(std::time::Duration::from_secs(2))
.build()
else {
return false;
};
let url = format!("http://127.0.0.1:{port}/health");
match client.get(&url).send().await {
Ok(r) => r.status().is_success(),
Err(_) => false,
}
}
async fn run_auto_update_worker(state: Arc<AppState>) {
use crate::install_state::{detect_install_method, InstallMethod};
if crate::telemetry::is_ci_environment() {
tracing::debug!(target: "update", "CI environment detected; auto-update worker disabled");
return;
}
if matches!(detect_install_method(), InstallMethod::CargoInstall) {
tracing::info!(target: "update", "cargo-install path detected; auto-update worker disabled — use `cargo install --force --locked openlatch-client`");
return;
}
tokio::time::sleep(std::time::Duration::from_secs(10)).await;
let normal_interval =
std::time::Duration::from_secs(state.config.update.check_interval_secs.max(1));
let critical_interval = std::time::Duration::from_secs(3600);
let defer_interval = std::time::Duration::from_secs(300);
let mut pending_since: Option<std::time::Instant> = None;
let current_version = env!("CARGO_PKG_VERSION").to_string();
loop {
let next_sleep = match worker_iteration(&state, ¤t_version, pending_since).await {
WorkerOutcome::Idle => {
pending_since = None;
normal_interval
}
WorkerOutcome::Deferred {
severity: update::Severity::Critical,
} => {
if pending_since.is_none() {
pending_since = Some(std::time::Instant::now());
}
critical_interval
}
WorkerOutcome::Deferred { .. } => {
if pending_since.is_none() {
pending_since = Some(std::time::Instant::now());
}
defer_interval
}
WorkerOutcome::Failed {
severity: update::Severity::Critical,
} => {
pending_since = None;
critical_interval
}
WorkerOutcome::Failed { .. } => {
pending_since = None;
normal_interval
}
};
tokio::time::sleep(next_sleep).await;
}
}
#[derive(Debug, Clone, Copy)]
enum WorkerOutcome {
Idle,
Deferred { severity: update::Severity },
Failed { severity: update::Severity },
}
async fn worker_iteration(
state: &Arc<AppState>,
current_version: &str,
pending_since: Option<std::time::Instant>,
) -> WorkerOutcome {
let registry_origin = state.config.update.registry_origin.clone();
let download_timeout =
std::time::Duration::from_secs(state.config.update.download_timeout_secs.max(1));
let result = update::check(current_version, ®istry_origin).await;
let (latest, severity, min_supported) = match result {
update::CheckResult::Available {
latest,
severity,
min_supported,
..
} => (latest, severity, min_supported),
update::CheckResult::UpToDate { .. } | update::CheckResult::Failed { .. } => {
return WorkerOutcome::Idle;
}
};
if let Some(ref min) = min_supported {
if !update::version_at_least(current_version, min) {
crate::telemetry::capture_global(
crate::telemetry::Event::update_blocked_by_min_supported(
current_version,
&latest,
min,
),
);
tracing::info!(
target: "update",
latest = %latest,
min_supported = %min,
"auto-update blocked: client older than min_supported_client"
);
return WorkerOutcome::Idle;
}
}
let pending_age = pending_since
.map(|t| std::time::Instant::now().saturating_duration_since(t))
.unwrap_or_default();
if !update::should_apply_now(
severity,
&state.last_hook_at_unix_secs,
&state.hooks_in_flight,
pending_age,
state.config.update.quiet_window_secs,
state.config.update.max_defer_secs,
) {
tracing::debug!(target: "update", latest = %latest, severity = %severity.as_str(), "deferring update — agent active or quiet window not met");
return WorkerOutcome::Deferred { severity };
}
if state
.update_in_progress
.compare_exchange(
false,
true,
std::sync::atomic::Ordering::AcqRel,
std::sync::atomic::Ordering::Acquire,
)
.is_err()
{
tracing::info!(target: "update", "auto-update worker yielding to in-flight manual apply");
return WorkerOutcome::Deferred { severity };
}
{
let mut snap = state.update_status.lock().expect("status mutex poisoned");
*snap = update::UpdateStatusSnapshot {
status: update::UpdateStatusKind::InProgress,
stage: Some(update::ApplyStage::Check),
from: Some(current_version.to_string()),
to: Some(latest.clone()),
started_at: Some(crate::install_state::now_rfc3339()),
ended_at: None,
error: None,
};
}
let opts = update::ApplyOptions {
current_version: current_version.to_string(),
registry_origin,
download_timeout,
force_cargo_install: false,
mode: update::ApplyMode::Rpc,
};
admin::run_apply_in_daemon(state.clone(), opts, severity).await;
WorkerOutcome::Failed { severity }
}
#[cfg(feature = "boundary")]
fn agent_settings_path() -> Option<std::path::PathBuf> {
match crate::hooks::detect_agent() {
Ok(crate::hooks::DetectedAgent::ClaudeCode { settings_path, .. }) => Some(settings_path),
Err(e) => {
tracing::debug!(code = %e.code, "no agent detected — nothing to wire to the boundary");
None
}
}
}
#[cfg(feature = "boundary")]
const WIRING_TICK: std::time::Duration = std::time::Duration::from_secs(60);
#[cfg(feature = "boundary")]
const WIRING_BACKOFF_MAX: std::time::Duration = std::time::Duration::from_secs(300);
#[cfg(feature = "boundary")]
fn wiring_delay(base: std::time::Duration, consecutive_failures: u32) -> std::time::Duration {
if consecutive_failures == 0 {
return base;
}
let shift = (consecutive_failures - 1).min(8);
base.saturating_mul(1u32 << shift).min(WIRING_BACKOFF_MAX)
}
#[cfg(feature = "boundary")]
fn wiring_tick() -> std::time::Duration {
match std::env::var("OPENLATCH_BOUNDARY_WIRING_TICK_MS") {
Ok(v) => match v.parse::<u64>() {
Ok(ms) => std::time::Duration::from_millis(ms.max(50)),
Err(_) => WIRING_TICK,
},
Err(_) => WIRING_TICK,
}
}
#[cfg(feature = "boundary")]
async fn run_wiring_supervisor(
config: Arc<Config>,
port: u16,
wiring: Arc<crate::boundary::preflight::WiringState>,
mut shutdown: tokio::sync::watch::Receiver<bool>,
) {
use crate::boundary::preflight::{self, Verdict};
use crate::boundary::proxy::upstream_failures;
let mut last_failures = upstream_failures();
let mut consecutive_failures: u32 = 0;
let tick = wiring_tick();
loop {
if *shutdown.borrow() {
return;
}
let wired = wiring.is_wired();
let failures = upstream_failures();
let forwarding_broke = failures > last_failures;
last_failures = failures;
if !wired || forwarding_broke {
let upstream = config.boundary.upstream.clone();
match preflight::probe(port, &upstream, preflight::PREFLIGHT_TIMEOUT).await {
Ok(()) => {
consecutive_failures = 0;
wiring.set_verdict(Verdict::Ok);
if !wired {
wire_boundary_config(&config, port, &wiring);
}
}
Err(reason) => {
consecutive_failures = consecutive_failures.saturating_add(1);
if wired {
tracing::error!(
port,
reason = %reason,
"boundary preflight failed on a wired listener — removing the agent \
wiring so sessions fall back to a direct provider connection"
);
} else {
tracing::warn!(
port,
reason = %reason,
attempt = consecutive_failures,
"boundary preflight failed — the agent stays unwired and model calls \
go straight to the provider (nothing is captured)"
);
}
wiring.set_verdict(Verdict::Failed(reason));
if config.boundary.owns_agent_wiring() {
unwire_boundary_config();
wiring.set_wired(false);
}
}
}
}
let delay = wiring_delay(tick, consecutive_failures);
tokio::select! {
_ = tokio::time::sleep(delay) => {}
_ = shutdown.changed() => return,
}
}
}
#[cfg(feature = "boundary")]
fn wire_boundary_config(
config: &Config,
port: u16,
wiring: &crate::boundary::preflight::WiringState,
) {
if !config.boundary.owns_agent_wiring() {
tracing::info!(
port,
"isolated boundary instance — {} left untouched; run sessions through this \
listener with ANTHROPIC_BASE_URL=http://127.0.0.1:{port}",
agent_settings_path()
.map(|p| p.display().to_string())
.unwrap_or_else(|| "the agent config".into()),
);
return;
}
let Some(settings_path) = agent_settings_path() else {
return;
};
let install_id = match config.agent_id.clone() {
Some(id) => id,
None => crate::config::ensure_agent_id(&crate::config::openlatch_dir().join("config.toml"))
.unwrap_or_default(),
};
match crate::hooks::write_boundary_config(&settings_path, port, &install_id) {
Ok(()) => {
wiring.set_wired(true);
tracing::info!(
port,
path = %settings_path.display(),
"agent wired to the model boundary — model calls route via http://127.0.0.1:{port}"
);
tracing::info!(
"Claude Code disables Remote Control while ANTHROPIC_BASE_URL is set — \
stop the daemon (`openlatch stop`) to restore a direct connection"
);
}
Err(e) => {
tracing::error!(
code = %e.code,
error = %e.message,
path = %settings_path.display(),
"failed to wire agent to the model boundary — agents will talk to the provider directly"
);
}
}
}
#[cfg(feature = "boundary")]
fn unwire_boundary_config() {
let Some(settings_path) = agent_settings_path() else {
return;
};
match crate::hooks::remove_boundary_config(&settings_path) {
Ok(()) => tracing::info!(
path = %settings_path.display(),
"agent boundary wiring removed — agents connect to the provider directly"
),
Err(e) => tracing::warn!(
code = %e.code,
error = %e.message,
path = %settings_path.display(),
"failed to remove agent boundary wiring — ANTHROPIC_BASE_URL may point at a dead port"
),
}
}
async fn signal_handler() {
#[cfg(unix)]
{
use tokio::signal::unix::{signal, SignalKind};
let mut sigterm =
signal(SignalKind::terminate()).expect("failed to register SIGTERM handler");
let mut sighup = signal(SignalKind::hangup()).expect("failed to register SIGHUP handler");
tokio::select! {
_ = tokio::signal::ctrl_c() => {}
_ = sigterm.recv() => {}
_ = sighup.recv() => {
tracing::info!("received SIGHUP — draining (no config-reload path exists)");
}
}
}
#[cfg(not(unix))]
{
tokio::signal::ctrl_c()
.await
.expect("failed to register ctrl_c handler");
}
}
#[cfg(test)]
mod tests {
use super::*;
#[cfg(feature = "boundary")]
#[test]
fn wiring_delay_backs_off_and_caps() {
let t = WIRING_TICK;
assert_eq!(wiring_delay(t, 0), t, "a green gate idles at the tick");
assert_eq!(
wiring_delay(t, 1),
t,
"the first failure retries at the tick"
);
assert_eq!(wiring_delay(t, 2), t * 2);
assert_eq!(wiring_delay(t, 3), t * 4);
assert_eq!(wiring_delay(t, 4), WIRING_BACKOFF_MAX);
assert_eq!(wiring_delay(t, 50), WIRING_BACKOFF_MAX);
assert_eq!(wiring_delay(t, u32::MAX), WIRING_BACKOFF_MAX);
}
#[cfg(feature = "boundary")]
#[test]
fn wiring_tick_defaults_to_production_and_never_busy_loops() {
assert_eq!(wiring_tick(), WIRING_TICK, "unset must mean the real tick");
assert_eq!(
wiring_delay(std::time::Duration::from_millis(50), 0),
std::time::Duration::from_millis(50)
);
}
#[test]
fn test_openlatch_marker_detected_in_settings() {
let with_hooks = r#"{"hooks": {"_openlatch": true, "preToolUse": []}}"#;
assert!(with_hooks.contains("\"_openlatch\""));
let without_hooks = r#"{"hooks": {"preToolUse": []}}"#;
assert!(!without_hooks.contains("\"_openlatch\""));
}
#[test]
fn test_format_uptime_seconds_only() {
assert_eq!(format_uptime(0), "0s");
assert_eq!(format_uptime(45), "45s");
assert_eq!(format_uptime(59), "59s");
}
#[test]
fn test_format_uptime_minutes_and_seconds() {
assert_eq!(format_uptime(60), "1m0s");
assert_eq!(format_uptime(192), "3m12s");
assert_eq!(format_uptime(3599), "59m59s");
}
#[test]
fn test_format_uptime_hours_and_minutes() {
assert_eq!(format_uptime(3600), "1h0m");
assert_eq!(format_uptime(8094), "2h14m");
assert_eq!(format_uptime(7200), "2h0m");
}
mod policy_startup {
use super::*;
use crate::core::policy::store::{self, BundleMeta};
use crate::generated::types::PolicyBundle;
const BODY: &str = r#"{"schema_version":1,"revision":42,"organization_id":"0192f8a1-4c3b-7e2a-9f10-5d8c3b1a7e42","built_at":"2026-07-21T09:00:00Z","enforcement_enabled":true,"signature":null,"rules":[{"rule_id":"OL-CMD-ENF","kind":"command","match_pattern":"*olcanary-enforce*","action":"deny","mode":"enforce","severity":"high","reason":"Canary enforce"}]}"#;
fn seed(base: &std::path::Path, last_poll_ok_at: Option<&str>) {
let body = BODY.as_bytes();
let bundle: PolicyBundle = serde_json::from_slice(body).expect("fixture parses");
let mut meta = BundleMeta::activated(
&bundle,
store::digest_of(body),
Some("\"sha256:deadbeef\"".to_string()),
);
meta.last_poll_ok_at = last_poll_ok_at.map(str::to_string);
store::store(base, body, &meta).expect("cache writes");
}
#[test]
fn cached_bundle_is_resident_before_anything_can_serve() {
let tmp = tempfile::tempdir().expect("tempdir");
seed(tmp.path(), None);
let runtime = PolicyRuntime::load_from_disk(tmp.path());
let guard = runtime.handle.load();
let bundle = guard.as_ref().as_ref().expect("bundle resident at startup");
assert_eq!(bundle.revision, 42);
assert_eq!(bundle.command_rules.len(), 1);
assert_eq!(bundle.command_rules[0].rule_id, "OL-CMD-ENF");
assert!(bundle.enforcement_enabled);
}
#[test]
fn tampered_bundle_is_rejected_and_the_daemon_starts_with_no_policy() {
let tmp = tempfile::tempdir().expect("tempdir");
seed(tmp.path(), None);
std::fs::write(
store::bundle_path(tmp.path()),
BODY.replace(r#""rules":[{"#, r#""rules":[{"x":1,"#),
)
.expect("tamper writes");
let runtime = PolicyRuntime::load_from_disk(tmp.path());
assert!(
runtime.handle.load().is_none(),
"a tampered bundle must never activate"
);
}
#[test]
fn no_cache_starts_with_no_policy() {
let tmp = tempfile::tempdir().expect("tempdir");
let runtime = PolicyRuntime::load_from_disk(tmp.path());
assert!(runtime.handle.load().is_none());
assert_eq!(runtime.last_poll_ok_at.load(Ordering::Relaxed), 0);
}
#[test]
fn poll_clock_is_seeded_from_the_meta_file() {
let tmp = tempfile::tempdir().expect("tempdir");
seed(tmp.path(), Some("2026-07-21T09:00:00Z"));
let runtime = PolicyRuntime::load_from_disk(tmp.path());
assert_eq!(
runtime.last_poll_ok_at.load(Ordering::Relaxed),
1_784_624_400
);
assert!(runtime.last_fetch_ok.load(Ordering::Relaxed));
}
}
}