use std::process::Stdio;
use crate::cli::output::OutputConfig;
use crate::cli::StartArgs;
use crate::config;
use crate::error::{
OlError, ERR_ALREADY_RUNNING, ERR_DAEMON_START_FAILED, ERR_DAEMON_STOP_FAILED,
ERR_INVALID_CONFIG, ERR_PORT_IN_USE,
};
pub(crate) fn log_observability_status_from_env() {
let dir = config::openlatch_dir();
let telemetry_consent = crate::telemetry::consent::resolve(&dir.join("telemetry.json"));
let baked_key_present = crate::telemetry::network::key_is_present();
let telemetry_enabled = telemetry_consent.enabled() && baked_key_present;
let telemetry_decided_by = if !baked_key_present {
"NoBakedKey".to_string()
} else {
format!("{:?}", telemetry_consent.decided_by)
};
#[cfg(feature = "crash-report")]
let (crash_report_enabled, crash_report_decided_by) = {
let resolved = crate::crash_report::current_state(&dir);
(resolved.enabled(), format!("{:?}", resolved.decided_by))
};
#[cfg(not(feature = "crash-report"))]
let (crash_report_enabled, crash_report_decided_by) = (false, "BuildExcluded".to_string());
crate::logging::daemon_log::log_observability_status(
telemetry_enabled,
&telemetry_decided_by,
crash_report_enabled,
&crash_report_decided_by,
);
}
pub(crate) fn stop_via_supervisor() -> bool {
let Ok(cfg) = config::Config::load(None, None, false) else {
return false;
};
let Some(sup) = crate::supervision::lifecycle_owner(&cfg.supervision) else {
return false;
};
if let Err(e) = sup.stop() {
tracing::warn!(
error = %e.message, code = e.code,
"supervisor refused the stop; falling back to stopping the process directly"
);
return false;
}
if let Some(pid) = read_pid_file() {
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10);
while std::time::Instant::now() < deadline && is_process_alive(pid) {
std::thread::sleep(std::time::Duration::from_millis(100));
}
}
true
}
pub(crate) fn start_via_supervisor(port: u16) -> bool {
let Ok(cfg) = config::Config::load(None, None, false) else {
return false;
};
let Some(sup) = crate::supervision::lifecycle_owner(&cfg.supervision) else {
return false;
};
if let Err(e) = sup.start() {
tracing::warn!(
error = %e.message, code = e.code,
"supervisor refused the start; falling back to spawning the daemon directly"
);
return false;
}
wait_for_health(port, 10)
}
pub(crate) fn request_is_plain_start(args: &StartArgs) -> bool {
!args.foreground && args.port.is_none() && args.boundary_port.is_none()
}
pub fn run_start(args: &StartArgs, output: &OutputConfig) -> Result<(), OlError> {
if let Some(p) = args.boundary_port {
std::env::set_var("OPENLATCH_BOUNDARY_PORT", p.to_string());
}
let cfg = config::Config::load(args.port, None, false)?;
for key in config::unknown_config_keys_on_disk() {
output.print_info(&format!(
"Warning: ignoring unrecognized config key '{key}' in config.toml"
));
}
if request_is_plain_start(args) {
migrate_supervisor_artifact_if_stale(&cfg);
if let Some(sup) = crate::supervision::lifecycle_owner(&cfg.supervision) {
match sup.start() {
Ok(()) => {
if wait_for_health(cfg.port, 10) {
let pid = read_pid_file().unwrap_or(0);
output.print_step(&format!(
"Daemon started on port {} (PID {pid}, supervised)",
cfg.port
));
return Ok(());
}
return Err(OlError::new(
ERR_DAEMON_START_FAILED,
format!(
"Supervisor accepted the start but nothing answered /health on port {} within 10s",
cfg.port
),
)
.with_suggestion(
"Ask the supervisor what happened — `systemctl --user status openlatch.service` \
(Linux), `launchctl print gui/$UID/ai.openlatch.client` (macOS) — and check \
the newest ~/.openlatch/logs/daemon.log.<date>.",
)
.with_docs("https://docs.openlatch.ai/errors/OL-1502"));
}
Err(e) => {
tracing::warn!(
error = %e.message, code = e.code,
"supervisor refused the start; starting the daemon directly"
);
}
}
}
}
match read_pid_file() {
Some(pid) if is_process_alive(pid) => {
if args.foreground {
return Err(OlError::new(
ERR_ALREADY_RUNNING,
format!(
"A daemon is already running (PID {pid}); refusing to run a second one \
in the foreground"
),
)
.with_suggestion(
"Run `openlatch stop` first, or `openlatch restart` to cycle it. Under an OS \
supervisor the running daemon is the supervised one — leave it be.",
)
.with_docs("https://docs.openlatch.ai/errors/OL-1501"));
}
output.print_info(&format!("Daemon is already running (PID {pid})"));
return Ok(());
}
Some(pid) => {
let _ = std::fs::remove_file(config::openlatch_dir().join("daemon.pid"));
output.print_info(&format!("Cleared stale PID file (PID {pid} not running)"));
}
None => {}
}
if check_health(cfg.port) {
return Err(OlError::new(
ERR_ALREADY_RUNNING,
format!(
"A daemon is already answering on port {}; refusing to start a duplicate",
cfg.port
),
)
.with_suggestion("Run `openlatch stop` first, or `openlatch restart` to cycle it.")
.with_docs("https://docs.openlatch.ai/errors/OL-1501"));
}
let token = load_or_generate_token()?;
#[cfg(feature = "boundary")]
if cfg.boundary.enabled {
let boundary_port = cfg.boundary.port;
if let Err(e) = std::net::TcpListener::bind(("127.0.0.1", boundary_port)) {
return Err(OlError::port_occupied(boundary_port, e));
}
if !cfg.boundary.owns_agent_wiring() {
output.print_info(&format!(
"Isolated boundary instance on 127.0.0.1:{boundary_port} — the agent config is \
left untouched. Route a session through it with:\n \
ANTHROPIC_BASE_URL=http://127.0.0.1:{boundary_port} claude"
));
}
}
if args.foreground {
run_daemon_foreground(cfg.port, &token)?;
} else {
let pid = spawn_daemon_background(cfg.port, &token)?;
if !wait_for_health(cfg.port, 5) {
return Err(OlError::new(
ERR_DAEMON_START_FAILED,
format!("Daemon spawned (PID {pid}) but health check failed within 5s"),
)
.with_suggestion(
"Check the newest ~/.openlatch/logs/daemon.log.<date> for errors \
(the log rotates daily, so there is no unsuffixed daemon.log).",
)
.with_docs("https://docs.openlatch.ai/errors/OL-1502"));
}
output.print_step(&format!("Daemon started on port {} (PID {pid})", cfg.port));
}
Ok(())
}
pub fn run_stop(output: &OutputConfig) -> Result<(), OlError> {
if stop_via_supervisor() {
let _ = std::fs::remove_file(config::openlatch_dir().join("daemon.pid"));
announce_unwire(unwire_boundary_after_death(), output);
output.print_step("Daemon stopped (supervised)");
if output.format == crate::cli::output::OutputFormat::Human && !output.quiet {
eprintln!(
" It will start again at the next login. Run `openlatch supervision disable` \
to prevent that."
);
}
return Ok(());
}
let Some(pid) = read_pid_file() else {
output.print_info("Daemon is not running");
announce_unwire(unwire_boundary_after_death(), output);
return Ok(());
};
if !is_process_alive(pid) {
output.print_info("Daemon is not running");
let _ = std::fs::remove_file(config::openlatch_dir().join("daemon.pid"));
announce_unwire(unwire_boundary_after_death(), output);
return Ok(());
}
let cfg = config::Config::load(None, None, false)?;
let token = load_or_generate_token().unwrap_or_default();
if send_shutdown_request(cfg.port, &token) {
let start = std::time::Instant::now();
while start.elapsed() < std::time::Duration::from_secs(5) {
if !is_process_alive(pid) {
break;
}
std::thread::sleep(std::time::Duration::from_millis(200));
}
}
if !is_process_alive(pid) {
let _ = std::fs::remove_file(config::openlatch_dir().join("daemon.pid"));
announce_unwire(unwire_boundary_after_death(), output);
output.print_step("Daemon stopped");
return Ok(());
}
force_kill(pid);
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(3);
while std::time::Instant::now() < deadline && is_process_alive(pid) {
std::thread::sleep(std::time::Duration::from_millis(100));
}
if is_process_alive(pid) {
force_kill_hard(pid);
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(2);
while std::time::Instant::now() < deadline && is_process_alive(pid) {
std::thread::sleep(std::time::Duration::from_millis(100));
}
}
if is_process_alive(pid) {
return Err(OlError::new(
ERR_DAEMON_STOP_FAILED,
format!("Failed to stop daemon (pid {pid}); process still running after SIGKILL"),
)
.with_suggestion("Kill the process manually and remove ~/.openlatch/daemon.pid.")
.with_docs("https://docs.openlatch.ai/errors/OL-1507"));
}
let _ = std::fs::remove_file(config::openlatch_dir().join("daemon.pid"));
announce_unwire(unwire_boundary_after_death(), output);
output.print_step("Daemon stopped");
Ok(())
}
fn announce_unwire(removed: bool, output: &OutputConfig) {
if removed {
output.print_step(
"Boundary wiring removed from the agent config — model calls now go direct",
);
}
}
fn unwire_boundary_after_death() -> bool {
#[cfg(feature = "boundary")]
{
use crate::cli::commands::boundary::{verify_port_ownership, PortOwnership};
let Ok(cfg) = config::Config::load(None, None, false) else {
return false;
};
if !cfg.boundary.owns_agent_wiring() {
return false;
}
let port = cfg.boundary.port;
if verify_port_ownership(port) == PortOwnership::Owned {
return false;
}
unwire_all(crate::hooks::detect_agents().into_iter().map(|a| a.binding))
}
#[cfg(not(feature = "boundary"))]
false
}
#[cfg(feature = "boundary")]
fn unwire_all(
agents: impl IntoIterator<Item = std::sync::Arc<dyn crate::hooks::binding::AgentBinding>>,
) -> bool {
let mut was_wired = false;
for binding in agents {
let wired = crate::cli::commands::boundary::read_agent_wiring(&*binding).is_some();
match crate::hooks::remove_boundary_config(&*binding) {
Ok(()) if wired => was_wired = true,
Err(e) => tracing::warn!(
code = %e.code,
error = %e.message,
agent = binding.agent_type(),
"could not remove the agent's boundary wiring after stop"
),
_ => {}
}
}
was_wired
}
pub fn run_restart(output: &OutputConfig) -> Result<(), OlError> {
if let Ok(cfg) = config::Config::load(None, None, false) {
migrate_supervisor_artifact_if_stale(&cfg);
if let Some(sup) = crate::supervision::lifecycle_owner(&cfg.supervision) {
match sup.restart() {
Ok(()) => {
if wait_for_health(cfg.port, 10) {
let pid = read_pid_file().unwrap_or(0);
output.print_step(&format!(
"Daemon restarted on port {} (PID {pid}, supervised)",
cfg.port
));
return Ok(());
}
return Err(OlError::new(
ERR_DAEMON_START_FAILED,
format!(
"Supervisor accepted the restart but nothing answered /health on port {} within 10s",
cfg.port
),
)
.with_suggestion(
"Ask the supervisor what happened — `systemctl --user status openlatch.service` \
(Linux), `launchctl print gui/$UID/ai.openlatch.client` (macOS) — and check \
the newest ~/.openlatch/logs/daemon.log.<date>.",
)
.with_docs("https://docs.openlatch.ai/errors/OL-1502"));
}
Err(e) => {
tracing::warn!(
error = %e.message, code = e.code,
"supervisor refused the restart; cycling the daemon directly"
);
}
}
}
}
run_stop(output)?;
let timeout = std::time::Duration::from_secs(5);
let start = std::time::Instant::now();
let cfg = config::Config::load(None, None, false)?;
while start.elapsed() < timeout {
let pid_file_gone = read_pid_file().is_none();
let health_down = !check_health(cfg.port);
if pid_file_gone || health_down {
break;
}
std::thread::sleep(std::time::Duration::from_millis(200));
}
let start_args = StartArgs {
foreground: false,
port: None,
boundary_port: None,
};
run_start(&start_args, output)
}
pub fn spawn_daemon_background(port: u16, token: &str) -> Result<u32, OlError> {
let exe = std::env::current_exe().map_err(|e| {
OlError::new(
ERR_INVALID_CONFIG,
format!("Cannot locate current executable: {e}"),
)
})?;
#[cfg(unix)]
let child = {
use std::os::unix::process::CommandExt;
let mut cmd = std::process::Command::new(&exe);
cmd.args([
"daemon",
"start",
"--foreground",
"--port",
&port.to_string(),
])
.env("OPENLATCH_TOKEN", token)
.stdin(Stdio::null())
.stdout(Stdio::null())
.stderr(Stdio::null());
unsafe {
cmd.pre_exec(|| {
libc::setsid();
Ok(())
});
}
cmd.spawn().map_err(|e| {
OlError::new(
ERR_INVALID_CONFIG,
format!("Failed to spawn daemon process: {e}"),
)
.with_suggestion("Check that the openlatch binary is executable.")
})?
};
#[cfg(windows)]
let child = {
use std::os::windows::process::CommandExt;
const CREATE_NO_WINDOW: u32 = 0x0800_0000;
const CREATE_NEW_PROCESS_GROUP: u32 = 0x0000_0200;
std::process::Command::new(&exe)
.args([
"daemon",
"start",
"--foreground",
"--port",
&port.to_string(),
])
.env("OPENLATCH_TOKEN", token)
.stdin(Stdio::null())
.stdout(Stdio::null())
.stderr(Stdio::null())
.creation_flags(CREATE_NO_WINDOW | CREATE_NEW_PROCESS_GROUP)
.spawn()
.map_err(|e| {
OlError::new(
ERR_INVALID_CONFIG,
format!("Failed to spawn daemon process: {e}"),
)
.with_suggestion("Check that the openlatch binary is executable.")
})?
};
let pid = child.id();
Ok(pid)
}
pub(crate) fn read_pid_file() -> Option<u32> {
let pid_path = config::openlatch_dir().join("daemon.pid");
let content = std::fs::read_to_string(&pid_path).ok()?;
content.trim().parse::<u32>().ok()
}
pub(crate) fn is_process_alive(pid: u32) -> bool {
if pid == 0 {
return false;
}
#[cfg(unix)]
{
let result = unsafe { libc::kill(pid as libc::pid_t, 0) };
result == 0
}
#[cfg(windows)]
{
let handle = unsafe {
winapi::um::processthreadsapi::OpenProcess(
winapi::um::winnt::PROCESS_QUERY_INFORMATION,
0,
pid,
)
};
if handle.is_null() {
return false;
}
let mut exit_code: u32 = 0;
let alive = unsafe {
winapi::um::processthreadsapi::GetExitCodeProcess(handle, &mut exit_code) != 0
&& exit_code == winapi::um::minwinbase::STILL_ACTIVE
};
unsafe { winapi::um::handleapi::CloseHandle(handle) };
alive
}
#[cfg(not(any(unix, windows)))]
{
let _ = pid;
false
}
}
pub(crate) fn send_shutdown_request(port: u16, token: &str) -> bool {
let url = format!("http://127.0.0.1:{port}/shutdown");
let client = crate::egress::blocking_client_builder()
.timeout(std::time::Duration::from_secs(2))
.build();
match client {
Ok(c) => c
.post(&url)
.header("Authorization", format!("Bearer {token}"))
.send()
.map(|r| r.status().is_success() || r.status() == reqwest::StatusCode::GONE)
.unwrap_or(false),
Err(_) => false,
}
}
pub(crate) fn force_kill(pid: u32) {
#[cfg(unix)]
unsafe {
libc::kill(pid as libc::pid_t, libc::SIGTERM);
}
#[cfg(windows)]
{
let _ = std::process::Command::new("taskkill")
.args(["/F", "/T", "/PID", &pid.to_string()])
.stdout(Stdio::null())
.stderr(Stdio::null())
.status();
}
}
pub(crate) fn force_kill_hard(pid: u32) {
#[cfg(unix)]
unsafe {
libc::kill(pid as libc::pid_t, libc::SIGKILL);
}
#[cfg(windows)]
{
let _ = std::process::Command::new("taskkill")
.args(["/F", "/T", "/PID", &pid.to_string()])
.stdout(Stdio::null())
.stderr(Stdio::null())
.status();
}
}
fn probe_health_once(port: u16) -> bool {
let url = format!("http://127.0.0.1:{port}/health");
crate::egress::blocking_client_builder()
.timeout(std::time::Duration::from_secs(2))
.build()
.ok()
.and_then(|c| c.get(&url).send().ok())
.map(|r| r.status().is_success())
.unwrap_or(false)
}
pub(crate) fn wait_for_health(port: u16, timeout_secs: u64) -> bool {
let start = std::time::Instant::now();
let timeout = std::time::Duration::from_secs(timeout_secs);
while start.elapsed() < timeout {
if probe_health_once(port) {
return true;
}
std::thread::sleep(std::time::Duration::from_millis(200));
}
false
}
pub(crate) fn check_health(port: u16) -> bool {
probe_health_once(port)
}
pub(crate) fn build_credential_store() -> std::sync::Arc<dyn crate::auth::CredentialStore> {
let agent_id = config::Config::load(None, None, false)
.ok()
.and_then(|c| c.agent_id)
.unwrap_or_default();
let keyring = Box::new(crate::auth::KeyringCredentialStore::new());
let file = Box::new(crate::auth::FileCredentialStore::new(
config::openlatch_dir().join("credentials.enc"),
agent_id,
));
std::sync::Arc::new(crate::auth::FallbackCredentialStore::new(keyring, file))
}
fn load_or_generate_token() -> Result<String, OlError> {
let ol_dir = config::openlatch_dir();
config::ensure_token(&ol_dir)
}
fn artifact_needs_migration(status: &crate::supervision::SupervisorStatus) -> bool {
status.installed && !status.unit_current
}
fn migrate_supervisor_artifact_if_stale(cfg: &config::Config) {
let Some(sup) = crate::supervision::lifecycle_owner(&cfg.supervision) else {
return;
};
let Ok(status) = sup.status() else { return };
if !artifact_needs_migration(&status) {
return;
}
let Ok(exe) = std::env::current_exe() else {
return;
};
match sup.install(&exe) {
Ok(()) => tracing::info!(
exe = %exe.display(),
"supervisor artifact predated this binary; regenerated it — effective at the next start"
),
Err(e) => tracing::warn!(
error = %e.message, code = e.code,
"could not regenerate the stale supervisor artifact; it keeps its previous semantics"
),
}
}
fn run_daemon_foreground(port: u16, token: &str) -> Result<(), OlError> {
let mut serve_error: Option<String> = None;
let config_path = config::openlatch_dir().join("config.toml");
if config_path.exists() {
let _ = config::ensure_agent_id(&config_path);
}
let mut cfg = config::Config::load(Some(port), None, true)?;
cfg.foreground = true;
let rt = tokio::runtime::Runtime::new().map_err(|e| {
OlError::new(
ERR_INVALID_CONFIG,
format!("Failed to create async runtime: {e}"),
)
})?;
let token_owned = token.to_string();
let pid = std::process::id();
#[cfg(feature = "crash-report")]
crate::crash_report::enrich_daemon_scope(cfg.port, pid);
rt.block_on({
let serve_error = &mut serve_error;
async move {
use crate::daemon;
use crate::envelope;
use crate::logging;
use crate::privacy;
let mut cfg = cfg;
let _guard = logging::daemon_log::init_daemon_logging(&cfg.log_dir, cfg.foreground);
if let Ok(deleted) = logging::cleanup_old_logs(&cfg.log_dir, cfg.retention_days) {
if deleted > 0 {
tracing::info!(deleted = deleted, "cleaned up old log files");
}
}
privacy::init_filter(&cfg.extra_patterns);
{
let credentials = crate::egress::ProxyCredentialFile::new(
config::openlatch_dir().join(crate::egress::PROXY_CREDENTIALS_FILE),
cfg.agent_id.clone().unwrap_or_default(),
);
let api_url = cfg.cloud.api_url.clone();
cfg.egress =
crate::egress::resolve_auth(cfg.egress, Some(&api_url), Some(&credentials))
.await;
if let Some(resolved) = cfg.egress.resolved.as_ref() {
if let Some(warning) = resolved.warning.as_deref() {
tracing::warn!(
code = crate::error::ERR_PROXY_AUTH_FAILED,
proxy = ?cfg.egress.masked_url(),
"{warning}"
);
}
}
}
let pid_path = config::openlatch_dir().join("daemon.pid");
if let Err(e) = std::fs::write(&pid_path, pid.to_string()) {
tracing::warn!(error = %e, "failed to write PID file");
}
logging::daemon_log::log_startup(
env!("CARGO_PKG_VERSION"),
cfg.port,
pid,
envelope::os_string(),
envelope::arch_string(),
);
log_observability_status_from_env();
let header_output = crate::cli::output::OutputConfig {
format: crate::cli::output::OutputFormat::Human,
verbose: false,
debug: false,
quiet: false,
color: std::io::IsTerminal::is_terminal(&std::io::stderr()),
};
crate::cli::header::print(
&header_output,
&[
&format!("listening 127.0.0.1:{}", cfg.port),
&format!("pid {pid}"),
],
);
let credential_store = build_credential_store();
let spawn_boundary = cfg.boundary.enabled;
log_if_unsupervised(&cfg);
match daemon::start_server(
cfg.clone(),
token_owned,
Some(credential_store),
spawn_boundary,
)
.await
{
Ok((uptime_secs, events)) => {
eprintln!(
"openlatch daemon stopped \u{2022} uptime {} \u{2022} {} events processed",
daemon::format_uptime(uptime_secs),
events
);
}
Err(e) => {
tracing::error!(error = %e, "daemon exited with error");
eprintln!("Error: daemon exited unexpectedly: {e}");
*serve_error = Some(e.to_string());
}
}
let _ = std::fs::remove_file(&pid_path);
}
});
#[cfg(feature = "crash-report")]
crate::crash_report::flush(std::time::Duration::from_secs(2));
match serve_error {
None => Ok(()),
Some(message) => Err(OlError::new(
ERR_DAEMON_START_FAILED,
format!("Daemon exited unexpectedly: {message}"),
)
.with_suggestion(
"Check the newest ~/.openlatch/logs/daemon.log.<date> for the failure that \
preceded it (the log rotates daily, so there is no unsuffixed daemon.log).",
)
.with_docs("https://docs.openlatch.ai/errors/OL-1502")),
}
}
fn log_if_unsupervised(cfg: &config::Config) {
use crate::supervision::{absence_is_deliberate, SupervisionMode};
if cfg.supervision.mode == SupervisionMode::Active {
return;
}
if absence_is_deliberate(cfg.supervision.disabled_reason.as_deref()) {
return;
}
tracing::warn!(
code = crate::error::ERR_NO_SUPERVISOR,
supervision_mode = ?cfg.supervision.mode,
reason = cfg.supervision.disabled_reason.as_deref().unwrap_or("unknown"),
"No OS supervisor is installed and none was declined — nothing will restart this \
daemon after a crash or a reboot. Run `openlatch supervision install`."
);
}
#[cfg(test)]
mod tests {
#[test]
fn a_child_that_died_is_not_a_started_daemon() {
let dir = tempfile::tempdir().unwrap();
let log_path = dir.path().join("daemon-spawn.log");
std::fs::write(&log_path, "Error: A daemon is already running (OL-1501)\n").unwrap();
#[cfg(unix)]
let mut child = std::process::Command::new("/bin/sh")
.args(["-c", "exit 5"])
.spawn()
.expect("spawn");
#[cfg(windows)]
let mut child = std::process::Command::new("cmd")
.args(["/C", "exit 5"])
.spawn()
.expect("spawn");
let status = child.wait().expect("wait for child refusal");
assert_eq!(status.code(), Some(5), "the fixture must refuse its start");
let mut spawned = SpawnedDaemon::from_child_for_test(child, log_path);
let verdict = verify_started_daemon(&mut spawned, 17969, 2);
match verdict {
Err(StartFailure::ChildDied { log, .. }) => {
assert!(
log.iter().any(|l| l.contains("OL-1501")),
"the child's own words must reach the caller, not /dev/null: {log:?}"
);
}
other => panic!("a dead child must be reported as such, got {other:?}"),
}
}
#[test]
fn every_start_failure_names_a_next_step() {
let cases = [
StartFailure::ChildDied {
status: "exit status: 5".into(),
log: vec!["Error: A daemon is already running".into()],
},
StartFailure::NoHealth,
StartFailure::ForeignPid {
answering: 42,
expected: 43,
},
StartFailure::VersionMismatch {
serving: "0.1.16".into(),
expected: "0.1.18".into(),
},
];
for case in cases {
let err = start_failure_error(case.clone(), 7443);
assert!(err.suggestion.is_some(), "{case:?} must carry a suggestion");
assert!(!err.message.is_empty(), "{case:?} must say what happened");
}
}
use super::*;
static ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
fn quiet_output() -> OutputConfig {
OutputConfig {
format: crate::cli::output::OutputFormat::Human,
verbose: false,
debug: false,
quiet: true,
color: false,
}
}
#[cfg(unix)]
#[test]
fn force_kill_hard_escalates_to_sigkill_on_sigterm_ignoring_process() {
use std::time::{Duration, Instant};
let mut child = std::process::Command::new("sh")
.args(["-c", "trap '' TERM; sleep 30"])
.stdin(Stdio::null())
.stdout(Stdio::null())
.stderr(Stdio::null())
.spawn()
.expect("spawn SIGTERM-ignoring child");
let pid = child.id();
assert!(
is_process_alive(pid),
"child must be alive right after spawn"
);
force_kill(pid);
std::thread::sleep(Duration::from_millis(500));
assert!(
is_process_alive(pid),
"child ignores SIGTERM — force_kill alone must NOT stop it (the OL-1300 bug)"
);
force_kill_hard(pid);
let status = child.wait().expect("wait on killed child");
assert!(
!status.success(),
"a SIGKILL'd process must not report a success exit"
);
let deadline = Instant::now() + Duration::from_secs(2);
while Instant::now() < deadline && is_process_alive(pid) {
std::thread::sleep(Duration::from_millis(50));
}
assert!(
!is_process_alive(pid),
"SIGKILL must terminate a SIGTERM-ignoring process"
);
}
#[test]
fn foreground_refuses_when_already_running_but_background_stays_idempotent() {
let _env = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner());
let _dir_env = crate::config::OPENLATCH_DIR_ENV_LOCK
.lock()
.unwrap_or_else(|e| e.into_inner());
let tmp = tempfile::tempdir().expect("tempdir");
std::env::set_var("OPENLATCH_DIR", tmp.path());
std::fs::write(
tmp.path().join("daemon.pid"),
std::process::id().to_string(),
)
.expect("write pid file");
let foreground = run_start(
&StartArgs {
foreground: true,
port: Some(59_411),
boundary_port: None,
},
&quiet_output(),
);
let background = run_start(
&StartArgs {
foreground: false,
port: Some(59_411),
boundary_port: None,
},
&quiet_output(),
);
std::env::remove_var("OPENLATCH_DIR");
let err = foreground.expect_err("--foreground must refuse a second daemon");
assert_eq!(
err.code, ERR_ALREADY_RUNNING,
"the foreground refusal must carry OL-1501, got {}",
err.code
);
assert_eq!(
err.exit_code(),
5,
"OL-1501 must exit 5 — RestartPreventExitStatus=5 keys off it"
);
assert!(
background.is_ok(),
"background start must stay idempotent (exit 0), got {background:?}"
);
}
#[test]
fn a_looping_unit_is_migrated_even_though_it_reads_as_not_running() {
let looping = crate::supervision::SupervisorStatus {
installed: true,
running: false,
unit_current: false,
description: "systemd-user (unit present, state: activating)".into(),
};
assert!(
artifact_needs_migration(&looping),
"a looping unit reads as not-running; gating on that skipped the fix"
);
}
#[test]
fn nothing_to_migrate_is_left_alone() {
let absent = crate::supervision::SupervisorStatus {
installed: false,
running: false,
unit_current: false,
description: "not installed".into(),
};
assert!(!artifact_needs_migration(&absent));
let current = crate::supervision::SupervisorStatus {
installed: true,
running: true,
unit_current: true,
description: "systemd-user (Restart=always active)".into(),
};
assert!(
!artifact_needs_migration(¤t),
"a current artifact must not be rewritten on every start"
);
}
#[test]
fn only_a_plain_start_is_delegated_to_the_supervisor() {
let plain = StartArgs {
foreground: false,
port: None,
boundary_port: None,
};
assert!(request_is_plain_start(&plain));
assert!(!request_is_plain_start(&StartArgs {
foreground: true,
..plain
}));
assert!(!request_is_plain_start(&StartArgs {
port: Some(7444),
..plain
}));
assert!(!request_is_plain_start(&StartArgs {
boundary_port: Some(7600),
..plain
}));
}
#[test]
fn run_start_refuses_when_health_answers_without_pid_file() {
use std::io::{Read, Write};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use std::time::Duration;
let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind ephemeral port");
let port = listener.local_addr().unwrap().port();
listener
.set_nonblocking(true)
.expect("set listener non-blocking");
let stop = Arc::new(AtomicBool::new(false));
let stop_thread = stop.clone();
let responder = std::thread::spawn(move || {
while !stop_thread.load(Ordering::Relaxed) {
match listener.accept() {
Ok((mut stream, _)) => {
let mut buf = [0u8; 1024];
let _ = stream.read(&mut buf);
let _ = stream.write_all(
b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n",
);
let _ = stream.flush();
}
Err(ref e) if e.kind() == std::io::ErrorKind::WouldBlock => {
std::thread::sleep(Duration::from_millis(10));
}
Err(_) => break,
}
}
});
let _env = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner());
let _dir_env = crate::config::OPENLATCH_DIR_ENV_LOCK
.lock()
.unwrap_or_else(|e| e.into_inner());
let tmp = tempfile::tempdir().expect("tempdir");
std::env::set_var("OPENLATCH_DIR", tmp.path());
let args = StartArgs {
foreground: false,
port: Some(port),
boundary_port: None,
};
let result = run_start(&args, &quiet_output());
std::env::remove_var("OPENLATCH_DIR");
stop.store(true, Ordering::Relaxed);
let _ = responder.join();
let err = result.expect_err("start must refuse when a daemon answers /health");
assert_eq!(
err.code, ERR_ALREADY_RUNNING,
"duplicate refusal must carry OL-1501, got {}",
err.code
);
}
#[cfg(feature = "boundary")]
#[test]
fn stop_tears_down_every_wired_agent() {
let tmp = tempfile::tempdir().unwrap();
let mut agents: Vec<std::sync::Arc<dyn crate::hooks::binding::AgentBinding>> = Vec::new();
for (agent_type, dir_name) in [("claude-code", "claude"), ("cursor", "cursor")] {
let dir = tmp.path().join(dir_name);
std::fs::create_dir_all(&dir).unwrap();
std::fs::write(
dir.join("settings.json"),
r#"{"env": {"ANTHROPIC_BASE_URL": "http://127.0.0.1:7600"}}"#,
)
.unwrap();
agents.push(std::sync::Arc::new(
crate::hooks::binding::test_support::FakeBinding {
agent_type,
config_dir: dir,
boundary_wiring: Some(crate::hooks::binding::BoundaryWiring {
wire_format: crate::boundary::wire_format::WireFormat::AnthropicMessages,
endpoint: crate::hooks::binding::EndpointConvention::EnvVars {
base_url: crate::hooks::ANTHROPIC_BASE_URL_ENV,
headers: crate::hooks::ANTHROPIC_CUSTOM_HEADERS_ENV,
},
install_id_header: "x-openlatch-install-id",
}),
..Default::default()
},
));
}
for binding in &agents {
assert!(
crate::cli::commands::boundary::read_boundary_base_url(&binding.hook_config_path())
.is_some(),
"{} must start out wired",
binding.agent_type()
);
}
let was_wired = unwire_all(agents.clone());
assert!(
was_wired,
"wiring existed and was removed, so the caller must be told to print the step"
);
for binding in &agents {
assert!(
crate::cli::commands::boundary::read_boundary_base_url(&binding.hook_config_path())
.is_none(),
"{} is still wired after stop",
binding.agent_type()
);
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ReclaimAction {
Nothing,
Stopped,
ForceKilled,
}
#[derive(Debug, Clone, Default)]
pub struct ReclaimedIdentity {
pub pid: Option<u32>,
pub version: Option<String>,
pub uptime_secs: Option<u64>,
pub exe: Option<String>,
}
impl ReclaimedIdentity {
pub fn describe(&self) -> String {
let mut parts = Vec::new();
if let Some(pid) = self.pid {
parts.push(format!("PID {pid}"));
}
if let Some(v) = &self.version {
parts.push(format!("v{v}"));
}
if let Some(secs) = self.uptime_secs {
parts.push(format!("up {}", approx_duration(secs)));
}
if let Some(exe) = &self.exe {
parts.push(format!("from {exe}"));
}
if parts.is_empty() {
"an unidentified daemon".to_string()
} else {
parts.join(", ")
}
}
}
#[derive(Debug, Clone)]
pub struct ReclaimOutcome {
pub action: ReclaimAction,
pub identity: ReclaimedIdentity,
}
fn approx_duration(secs: u64) -> String {
match secs {
0..=90 => format!("{secs}s"),
91..=5400 => format!("{}m", secs / 60),
5401..=172_800 => format!("{}h", secs / 3600),
_ => format!("{}d", secs / 86_400),
}
}
fn process_exe(pid: u32) -> Option<String> {
#[cfg(target_os = "linux")]
{
std::fs::read_link(format!("/proc/{pid}/exe"))
.ok()
.map(|p| p.display().to_string())
}
#[cfg(not(target_os = "linux"))]
{
let _ = pid;
None
}
}
pub(crate) fn port_is_held(port: u16) -> bool {
std::net::TcpListener::bind(("127.0.0.1", port)).is_err()
}
fn wait_for_port_free(port: u16, timeout_secs: u64) -> bool {
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(timeout_secs);
while std::time::Instant::now() < deadline {
if !port_is_held(port) {
return true;
}
std::thread::sleep(std::time::Duration::from_millis(150));
}
!port_is_held(port)
}
fn identify_daemon(port: u16) -> Option<ReclaimedIdentity> {
let body: serde_json::Value = crate::egress::blocking_client()
.get(format!("http://127.0.0.1:{port}/health"))
.timeout(std::time::Duration::from_secs(2))
.send()
.ok()
.filter(|r| r.status().is_success())?
.json()
.ok()?;
body.get("version")?;
let pid = body
.get("pid")
.and_then(|v| v.as_u64())
.and_then(|v| u32::try_from(v).ok())
.or_else(read_pid_file);
Some(ReclaimedIdentity {
pid,
version: body
.get("version")
.and_then(|v| v.as_str())
.map(str::to_string),
uptime_secs: body.get("uptime_secs").and_then(|v| v.as_u64()),
exe: pid.and_then(process_exe),
})
}
pub fn reclaim_ports(
cfg: &config::Config,
output: &OutputConfig,
) -> Result<ReclaimOutcome, OlError> {
let mut outcome = ReclaimOutcome {
action: ReclaimAction::Nothing,
identity: ReclaimedIdentity::default(),
};
let daemon_held = port_is_held(cfg.port);
let stale_pid = read_pid_file().filter(|p| is_process_alive(*p));
if daemon_held || stale_pid.is_some() {
match identify_daemon(cfg.port) {
Some(identity) => {
output.print_substep(&format!("Reclaiming daemon ({})", identity.describe()));
outcome.identity = identity;
}
None if daemon_held => {
return Err(OlError::new(
ERR_PORT_IN_USE,
format!(
"127.0.0.1:{} is held by a process that is not an OpenLatch daemon",
cfg.port
),
)
.with_suggestion(format!(
"Identify it with `lsof -i :{}` (or `ss -tlnp | grep :{}`) and stop it, \
or point this install elsewhere with OPENLATCH_PORT.",
cfg.port, cfg.port
))
.with_docs("https://docs.openlatch.ai/errors/OL-1500"));
}
None => {
outcome.identity = ReclaimedIdentity {
pid: stale_pid,
exe: stale_pid.and_then(process_exe),
..Default::default()
};
}
}
run_stop(output)?;
let mut forced = false;
if port_is_held(cfg.port) {
if let Some(pid) = outcome.identity.pid.filter(|p| is_process_alive(*p)) {
forced = true;
output.print_substep(&format!(
"PID {pid} holds the port but is not the daemon on record — stopping it directly"
));
let token = load_or_generate_token().unwrap_or_default();
if send_shutdown_request(cfg.port, &token) {
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
while std::time::Instant::now() < deadline && is_process_alive(pid) {
std::thread::sleep(std::time::Duration::from_millis(150));
}
}
if is_process_alive(pid) {
force_kill(pid);
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(3);
while std::time::Instant::now() < deadline && is_process_alive(pid) {
std::thread::sleep(std::time::Duration::from_millis(100));
}
}
if is_process_alive(pid) {
force_kill_hard(pid);
}
}
}
if !wait_for_port_free(cfg.port, 10) {
return Err(OlError::new(
ERR_PORT_IN_USE,
format!(
"127.0.0.1:{} is still held 10s after stopping the daemon that held it",
cfg.port
),
)
.with_suggestion(format!(
"Something else took the port. Identify it with `lsof -i :{}` (or \
`ss -tlnp | grep :{}`) and stop it.",
cfg.port, cfg.port
))
.with_docs("https://docs.openlatch.ai/errors/OL-1500"));
}
outcome.action = if forced {
ReclaimAction::ForceKilled
} else {
ReclaimAction::Stopped
};
}
#[cfg(feature = "boundary")]
if cfg.boundary.enabled && port_is_held(cfg.boundary.port) {
if !wait_for_port_free(cfg.boundary.port, 5) {
return Err(OlError::new(
crate::error::ERR_BOUNDARY_PORT_FOREIGN,
format!(
"the model boundary's port 127.0.0.1:{} is held by another process",
cfg.boundary.port
),
)
.with_suggestion(format!(
"Identify it with `lsof -i :{}` and stop it, or run `openlatch init \
--no-boundary` to install without the model boundary.",
cfg.boundary.port
))
.with_docs("https://docs.openlatch.ai/errors/OL-BND-FOREIGN"));
}
}
Ok(outcome)
}
pub struct SpawnedDaemon {
pub pid: u32,
child: std::process::Child,
log_path: std::path::PathBuf,
}
impl SpawnedDaemon {
pub fn exited(&mut self) -> Option<std::process::ExitStatus> {
self.child.try_wait().ok().flatten()
}
pub fn stderr_tail(&self, lines: usize) -> Vec<String> {
let Ok(content) = std::fs::read_to_string(&self.log_path) else {
return Vec::new();
};
content
.lines()
.filter(|l| !l.trim().is_empty())
.rev()
.take(lines)
.map(str::to_string)
.collect::<Vec<_>>()
.into_iter()
.rev()
.collect()
}
pub fn log_path(&self) -> &std::path::Path {
&self.log_path
}
#[cfg(test)]
fn from_child_for_test(child: std::process::Child, log_path: std::path::PathBuf) -> Self {
Self {
pid: child.id(),
child,
log_path,
}
}
}
pub fn spawn_daemon_tracked(port: u16, token: &str) -> Result<SpawnedDaemon, OlError> {
let exe = std::env::current_exe().map_err(|e| {
OlError::new(
ERR_INVALID_CONFIG,
format!("Cannot locate current executable: {e}"),
)
})?;
let log_dir = config::openlatch_dir().join("logs");
let _ = std::fs::create_dir_all(&log_dir);
let log_path = log_dir.join("daemon-spawn.log");
let stderr_sink = std::fs::File::create(&log_path).map_err(|e| {
OlError::new(
ERR_INVALID_CONFIG,
format!("Cannot open {}: {e}", log_path.display()),
)
})?;
let mut cmd = std::process::Command::new(&exe);
cmd.args([
"daemon",
"start",
"--foreground",
"--port",
&port.to_string(),
])
.env("OPENLATCH_TOKEN", token)
.stdin(Stdio::null())
.stdout(Stdio::null())
.stderr(Stdio::from(stderr_sink));
#[cfg(unix)]
{
use std::os::unix::process::CommandExt;
unsafe {
cmd.pre_exec(|| {
libc::setsid();
Ok(())
});
}
}
#[cfg(windows)]
{
use std::os::windows::process::CommandExt;
const CREATE_NO_WINDOW: u32 = 0x0800_0000;
const CREATE_NEW_PROCESS_GROUP: u32 = 0x0000_0200;
cmd.creation_flags(CREATE_NO_WINDOW | CREATE_NEW_PROCESS_GROUP);
}
let child = cmd.spawn().map_err(|e| {
OlError::new(
ERR_INVALID_CONFIG,
format!("Failed to spawn daemon process: {e}"),
)
.with_suggestion("Check that the openlatch binary is executable.")
})?;
Ok(SpawnedDaemon {
pid: child.id(),
log_path,
child,
})
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum StartFailure {
ChildDied { status: String, log: Vec<String> },
NoHealth,
ForeignPid { answering: u32, expected: u32 },
VersionMismatch { serving: String, expected: String },
}
pub fn verify_started_daemon(
spawned: &mut SpawnedDaemon,
port: u16,
timeout_secs: u64,
) -> Result<(), StartFailure> {
let expected_version = env!("OPENLATCH_VERSION");
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(timeout_secs);
loop {
if let Some(status) = spawned.exited() {
return Err(StartFailure::ChildDied {
status: status.to_string(),
log: spawned.stderr_tail(20),
});
}
if let Some(identity) = identify_daemon(port) {
let serving = identity.version.unwrap_or_default();
if serving != expected_version {
return Err(StartFailure::VersionMismatch {
serving,
expected: expected_version.to_string(),
});
}
match identity.pid {
Some(pid) if pid == spawned.pid => return Ok(()),
Some(pid) if std::time::Instant::now() >= deadline => {
return Err(StartFailure::ForeignPid {
answering: pid,
expected: spawned.pid,
})
}
None if std::time::Instant::now() >= deadline => return Ok(()),
_ => {}
}
}
if std::time::Instant::now() >= deadline {
return Err(StartFailure::NoHealth);
}
std::thread::sleep(std::time::Duration::from_millis(150));
}
}
pub fn start_failure_error(failure: StartFailure, port: u16) -> OlError {
match failure {
StartFailure::ChildDied { status, log } => {
let detail = if log.is_empty() {
String::new()
} else {
format!("\n\n The daemon said:\n {}", log.join("\n "))
};
OlError::new(
ERR_DAEMON_START_FAILED,
format!("The daemon exited immediately ({status}){detail}"),
)
.with_suggestion(
"The full output is in ~/.openlatch/logs/daemon-spawn.log; the daemon's own \
log is the newest ~/.openlatch/logs/daemon.log.<date>.",
)
.with_docs("https://docs.openlatch.ai/errors/OL-1502")
}
StartFailure::NoHealth => OlError::new(
ERR_DAEMON_START_FAILED,
format!("Nothing answered /health on port {port}"),
)
.with_suggestion(
"Check ~/.openlatch/logs/daemon-spawn.log and the newest \
~/.openlatch/logs/daemon.log.<date> (the log rotates daily, so there is no \
unsuffixed daemon.log).",
)
.with_docs("https://docs.openlatch.ai/errors/OL-1502"),
StartFailure::ForeignPid {
answering,
expected,
} => OlError::new(
ERR_DAEMON_START_FAILED,
format!(
"Port {port} is served by PID {answering}, not the daemon just started (PID {expected})"
),
)
.with_suggestion("Run `openlatch stop`, then `openlatch init` again.")
.with_docs("https://docs.openlatch.ai/errors/OL-1502"),
StartFailure::VersionMismatch { serving, expected } => OlError::new(
ERR_DAEMON_START_FAILED,
format!(
"Port {port} is served by version {serving}, but this binary is {expected} — \
an older daemon still owns the port"
),
)
.with_suggestion("Run `openlatch stop`, confirm the port is free, then `openlatch init` again.")
.with_docs("https://docs.openlatch.ai/errors/OL-1502"),
}
}
pub fn verify_running_daemon(port: u16, timeout_secs: u64) -> Result<u32, StartFailure> {
let expected_version = env!("OPENLATCH_VERSION");
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(timeout_secs);
let mut last_seen: Option<String> = None;
loop {
if let Some(identity) = identify_daemon(port) {
let serving = identity.version.clone().unwrap_or_default();
if serving == expected_version {
return Ok(identity.pid.unwrap_or(0));
}
last_seen = Some(serving);
}
if std::time::Instant::now() >= deadline {
return Err(match last_seen {
Some(serving) => StartFailure::VersionMismatch {
serving,
expected: expected_version.to_string(),
},
None => StartFailure::NoHealth,
});
}
std::thread::sleep(std::time::Duration::from_millis(200));
}
}