Skip to main content

khive_runtime/daemon/
process_files.rs

1use super::*;
2
3// ── helpers ───────────────────────────────────────────────────────────────────
4
5/// Liveness verdict for a `kill(pid, 0)` probe.
6#[cfg(unix)]
7#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8pub(super) enum PidLiveness {
9    /// errno 0 — signal delivery succeeded, the process exists and this
10    /// caller may signal it.
11    Alive,
12    /// ESRCH (or any other non-EPERM errno) — no such process.
13    Dead,
14    /// EPERM — the process exists but this caller lacks permission to
15    /// signal it. Unknown-safe: treated as running so stale-daemon cleanup
16    /// never unlinks a live daemon's socket/PID file just because it is
17    /// owned by a different user/uid.
18    PermissionDenied,
19}
20
21#[cfg(unix)]
22impl PidLiveness {
23    pub(super) fn is_running(self) -> bool {
24        !matches!(self, PidLiveness::Dead)
25    }
26}
27
28/// Maps a `kill(pid, 0)` outcome (return code + errno) to a [`PidLiveness`].
29/// Pure and side-effect-free so the errno mapping can be unit tested without
30/// a real process probe.
31#[cfg(unix)]
32pub(super) fn classify_kill_result(rc: i32, errno: i32) -> PidLiveness {
33    if rc == 0 {
34        return PidLiveness::Alive;
35    }
36    match errno {
37        libc::EPERM => PidLiveness::PermissionDenied,
38        _ => PidLiveness::Dead,
39    }
40}
41
42#[cfg(unix)]
43pub(super) fn is_process_running(pid: u32) -> bool {
44    let Ok(pid) = i32::try_from(pid) else {
45        return false;
46    };
47    if pid <= 0 {
48        return false;
49    }
50    // SAFETY: signal 0 is an existence/permission probe with no side effects.
51    let rc = unsafe { libc::kill(pid, 0) };
52    let errno = std::io::Error::last_os_error().raw_os_error().unwrap_or(0);
53    classify_kill_result(rc, errno).is_running()
54}
55
56/// Whether a PID may identify an incumbent from this candidate's point of view.
57///
58/// Production rejects the current PID so a stale rendezvous left by a prior
59/// process whose PID was reused cannot protect an unrelated socket. The
60/// in-process daemon harness opts in to the same-PID case because all of its
61/// otherwise independent boot candidates necessarily share one OS process.
62#[cfg(unix)]
63pub(super) fn pid_can_name_incumbent(
64    pid: u32,
65    current_pid: u32,
66    allow_same_process_incumbent: bool,
67) -> bool {
68    allow_same_process_incumbent || pid != current_pid
69}
70
71/// Bounded timeout for the protocol-identity probe used by duplicate-daemon
72/// detection. Short enough that a hung or foreign listener does not stall
73/// startup; long enough for a live khived under normal load to answer a
74/// `probe_only` frame.
75#[cfg(unix)]
76pub(super) const DUPLICATE_PROBE_TIMEOUT: std::time::Duration =
77    std::time::Duration::from_millis(500);
78
79/// Whether the listener at `sock` actually speaks the khived wire protocol
80/// **as a khived this process can defer to** — identified by a configuration
81/// compatible with `expected_config_id`.
82///
83/// A live PID plus an accepting Unix socket is not proof of khived: any
84/// unrelated process that happens to have bound the same path also answers
85/// `connect()`. Nor is any well-formed [`DaemonResponseFrame`] proof: a
86/// `config_mismatch`/`version_mismatch` response, a `metrics_only` snapshot
87/// response, or a legacy pre-probe daemon that falls through to normal
88/// dispatch on the empty `ops` string, all deserialize cleanly without being
89/// the unambiguous "yes, alive and identity-matching" answer this check
90/// needs — the daemon's `metrics_only` arm in particular echoes the same
91/// `ok=true, result=None, error=None`, all-mismatch-flags-false, matching
92/// protocol version and `served_config_id` shape as the probe-ack arm, and
93/// is distinguished only by carrying `metrics: Some(...)`. This sends a
94/// bounded `probe_only` frame (the same identity probe the client-side
95/// recovery path uses, `crates/khive-mcp/src/daemon.rs::probe_daemon_identity`)
96/// carrying this process's own `config_id`, and requires a compatible
97/// probe-branch shape back: `ok=true`, `result=None`, `error=None`,
98/// `metrics=None`, `request_id=None` (this probe frame never sets one), no
99/// mismatch flags, matching protocol version, and matching
100/// `served_config_id` — mirroring the client probe's `is_probe_ack` check so
101/// both sides of the protocol agree on what "alive" means. Connect, write,
102/// and read are all inside the one bounded timeout: `UnixStream::connect`
103/// itself awaits write readiness, so a listener with a saturated accept
104/// backlog could otherwise hold this call open past the advertised bound.
105/// A connect that succeeds but never answers, times out, or answers with
106/// non-protocol bytes, a mismatched identity, a `metrics_only` snapshot, or
107/// any other non-probe-shaped response is not treated as the same khived
108/// and falls through to the stale-socket recovery path instead.
109#[cfg(unix)]
110pub(super) async fn socket_speaks_khived_protocol(
111    sock: &std::path::Path,
112    expected_config_id: &str,
113) -> bool {
114    let probe = DaemonRequestFrame {
115        probe_only: true,
116        protocol_version: PROTOCOL_VERSION,
117        config_id: expected_config_id.to_string(),
118        ..Default::default()
119    };
120    let Ok(payload) = serde_json::to_vec(&probe) else {
121        return false;
122    };
123    let response = tokio::time::timeout(DUPLICATE_PROBE_TIMEOUT, async {
124        let mut stream = UnixStream::connect(sock).await.ok()?;
125        write_frame(&mut stream, &payload).await.ok()?;
126        let raw = read_frame(&mut stream).await.ok()?;
127        serde_json::from_slice::<DaemonResponseFrame>(&raw).ok()
128    })
129    .await
130    .ok()
131    .flatten();
132
133    let Some(resp) = response else {
134        return false;
135    };
136    let is_probe_ack = resp.ok
137        && resp.result.is_none()
138        && resp.error.is_none()
139        && resp.metrics.is_none()
140        && resp.request_id.is_none();
141    is_probe_ack
142        && !resp.version_mismatch
143        && !resp.namespace_mismatch
144        && !resp.config_mismatch
145        && resp.daemon_protocol_version == PROTOCOL_VERSION
146        && resp
147            .served_config_id
148            .as_deref()
149            .is_some_and(|served| config_ids_compatible(expected_config_id, served))
150}
151
152/// Whether connecting to an existing socket path is definitely unreachable.
153/// Timeouts and other errors remain ambiguous so cleanup fails closed.
154#[cfg(unix)]
155pub(super) async fn socket_is_unreachable(sock: &std::path::Path) -> bool {
156    match tokio::time::timeout(DUPLICATE_PROBE_TIMEOUT, UnixStream::connect(sock)).await {
157        Ok(Err(error)) => matches!(
158            error.kind(),
159            std::io::ErrorKind::NotFound | std::io::ErrorKind::ConnectionRefused
160        ),
161        _ => false,
162    }
163}
164
165/// What owns the daemon PID file, from the point of view of a process that wants
166/// to start. A live owner is never cleaned up: a draining incumbent closes its
167/// listener before it releases writers, so an unanswered socket is ambiguous and
168/// deleting its PID file is how two daemons end up on one store.
169#[cfg(unix)]
170pub(super) enum Incumbent {
171    /// A live process that answered the khived protocol on the socket.
172    Serving(u32),
173    /// A live PID still has an active or ambiguous rendezvous. Nothing removed.
174    Live(u32),
175    /// A rendezvous could be live or could not be safely inspected.
176    Unknown,
177    /// Nothing live owns the store; the socket and PID file were removed.
178    Stale,
179}
180
181#[cfg(unix)]
182fn read_pid_contents(file: &std::fs::File) -> std::io::Result<String> {
183    use std::io::{Read, Seek, SeekFrom};
184
185    if !file.metadata()?.is_file() {
186        return Err(std::io::Error::new(
187            std::io::ErrorKind::InvalidData,
188            "daemon PID file is not a regular file",
189        ));
190    }
191    let mut file = file.try_clone()?;
192    file.seek(SeekFrom::Start(0))?;
193    let mut contents = String::new();
194    (&mut file).take(4097).read_to_string(&mut contents)?;
195    if contents.len() > 4096 {
196        return Err(std::io::Error::new(
197            std::io::ErrorKind::InvalidData,
198            "daemon PID file exceeds the inspection limit",
199        ));
200    }
201    Ok(contents)
202}
203
204#[cfg(unix)]
205fn read_pid_file(pid_file: &std::path::Path) -> std::io::Result<(std::fs::Metadata, String)> {
206    use std::os::unix::fs::OpenOptionsExt;
207
208    let file = std::fs::OpenOptions::new()
209        .read(true)
210        .custom_flags(libc::O_NONBLOCK | libc::O_NOFOLLOW)
211        .open(pid_file)?;
212    let metadata = file.metadata()?;
213    let contents = read_pid_contents(&file)?;
214    Ok((metadata, contents))
215}
216
217/// Check whether `pid_file`/`sock` already name a live daemon and, if not,
218/// remove the stale rendezvous files so the caller may bind fresh.
219///
220/// A live protocol responder, reachable socket, or live holder of the PID-file
221/// lock means the caller must refuse to start. An unlocked PID with no listener
222/// is a reused PID and may be reclaimed.
223#[cfg(unix)]
224pub(super) async fn cleanup_stale_daemon(
225    sock: &std::path::Path,
226    pid_file: &std::path::Path,
227    allow_same_process_incumbent: bool,
228    expected_config_id: &str,
229) -> Incumbent {
230    use std::os::unix::fs::{FileTypeExt, MetadataExt};
231
232    let mut stale_pid_file_guard = None;
233    let pid_snapshot = match read_pid_file(pid_file) {
234        Ok(snapshot) => Some(snapshot),
235        Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
236        Err(_) => return Incumbent::Unknown,
237    };
238    let pid_contents = pid_snapshot.as_ref().map(|(_, contents)| contents);
239    if let Some(pid_str) = pid_contents {
240        if let Ok(pid) = pid_str.trim().parse::<u32>() {
241            if pid_can_name_incumbent(pid, std::process::id(), allow_same_process_incumbent)
242                && is_process_running(pid)
243            {
244                if sock.exists() && socket_speaks_khived_protocol(sock, expected_config_id).await {
245                    return Incumbent::Serving(pid);
246                }
247                if sock.exists() && !socket_is_unreachable(sock).await {
248                    return Incumbent::Live(pid);
249                }
250                match try_acquire_pid_file_lock(pid_file) {
251                    Ok(Some(guard)) => stale_pid_file_guard = Some(guard),
252                    Ok(None) => return Incumbent::Live(pid),
253                    Err(e) => {
254                        tracing::warn!(
255                            error = %e,
256                            path = ?pid_file,
257                            "cannot check daemon PID-file lock"
258                        );
259                        return Incumbent::Live(pid);
260                    }
261                }
262            }
263        }
264    }
265    if pid_contents.is_some() && stale_pid_file_guard.is_none() {
266        match try_acquire_pid_file_lock(pid_file) {
267            Ok(Some(guard)) => stale_pid_file_guard = Some(guard),
268            Ok(None) | Err(_) => return Incumbent::Unknown,
269        }
270    }
271    if let (Some((observed, expected)), Some(guard)) = (&pid_snapshot, &stale_pid_file_guard) {
272        let (Ok(owned), Ok(current), Ok(contents)) = (
273            guard.metadata(),
274            std::fs::symlink_metadata(pid_file),
275            read_pid_contents(guard),
276        ) else {
277            return Incumbent::Unknown;
278        };
279        if !owned.is_file()
280            || !current.is_file()
281            || owned.dev() != observed.dev()
282            || owned.ino() != observed.ino()
283            || owned.dev() != current.dev()
284            || owned.ino() != current.ino()
285            || &contents != expected
286        {
287            return Incumbent::Unknown;
288        }
289    }
290
291    let socket_metadata = match std::fs::symlink_metadata(sock) {
292        Ok(metadata) if metadata.file_type().is_socket() => Some(metadata),
293        Ok(_) => return Incumbent::Unknown,
294        Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
295        Err(_) => return Incumbent::Unknown,
296    };
297    if let Some(observed) = socket_metadata {
298        if !socket_is_unreachable(sock).await {
299            return Incumbent::Unknown;
300        }
301        let current = match std::fs::symlink_metadata(sock) {
302            Ok(metadata) if metadata.file_type().is_socket() => metadata,
303            _ => return Incumbent::Unknown,
304        };
305        if observed.dev() != current.dev() || observed.ino() != current.ino() {
306            return Incumbent::Unknown;
307        }
308        if let Err(error) = std::fs::remove_file(sock) {
309            tracing::warn!(%error, path = ?sock, "failed to remove stale socket");
310            return Incumbent::Unknown;
311        }
312    }
313    if let Some(guard) = &stale_pid_file_guard {
314        let owned = match guard.metadata() {
315            Ok(metadata) if metadata.is_file() => metadata,
316            _ => return Incumbent::Unknown,
317        };
318        let current = match std::fs::symlink_metadata(pid_file) {
319            Ok(metadata) if metadata.is_file() => metadata,
320            _ => return Incumbent::Unknown,
321        };
322        if owned.dev() != current.dev() || owned.ino() != current.ino() {
323            return Incumbent::Unknown;
324        }
325        if let Err(error) = std::fs::remove_file(pid_file) {
326            tracing::warn!(%error, path = ?pid_file, "failed to remove stale PID file");
327            return Incumbent::Unknown;
328        }
329    }
330    drop(stale_pid_file_guard);
331    Incumbent::Stale
332}
333
334/// Create and lock `pid_file` exclusively (`O_EXCL`) and write this process's PID.
335///
336/// Uses `create_new(true)` rather than `create(true).truncate(true)` so
337/// this can never silently overwrite a PID file another process created —
338/// the held file lock also identifies a starting or draining daemon when its
339/// socket is not yet reachable.
340#[cfg(unix)]
341pub(super) fn write_pid_file_exclusive(
342    pid_file: &std::path::Path,
343) -> std::io::Result<std::fs::File> {
344    use std::os::unix::fs::OpenOptionsExt;
345
346    let mut opts = std::fs::OpenOptions::new();
347    opts.write(true).create_new(true).mode(0o600);
348    let mut f = opts.open(pid_file)?;
349    // SAFETY: flock is a POSIX advisory lock with no memory side effects.
350    let rc = unsafe { libc::flock(f.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) };
351    if rc != 0 {
352        return Err(std::io::Error::last_os_error());
353    }
354    f.write_all(std::process::id().to_string().as_bytes())?;
355    Ok(f)
356}
357
358/// Try to lock an existing PID file without creating it. `Some(file)` means
359/// there is no daemon lock holder; `None` means a daemon still owns the file.
360#[cfg(unix)]
361pub(super) fn try_acquire_pid_file_lock(
362    pid_file: &std::path::Path,
363) -> std::io::Result<Option<std::fs::File>> {
364    use std::os::unix::fs::OpenOptionsExt;
365
366    let open = |write| {
367        std::fs::OpenOptions::new()
368            .read(true)
369            .write(write)
370            .custom_flags(libc::O_NONBLOCK | libc::O_NOFOLLOW)
371            .open(pid_file)
372    };
373    let file = match open(true) {
374        Err(error) if error.kind() == std::io::ErrorKind::PermissionDenied => open(false)?,
375        result => result?,
376    };
377    if !file.metadata()?.is_file() {
378        return Err(std::io::Error::new(
379            std::io::ErrorKind::InvalidData,
380            "daemon PID file is not a regular file",
381        ));
382    }
383    // SAFETY: flock is a POSIX advisory lock with no memory side effects.
384    let rc = unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) };
385    if rc == 0 {
386        return Ok(Some(file));
387    }
388    let error = std::io::Error::last_os_error();
389    if error.kind() == std::io::ErrorKind::WouldBlock
390        || error.raw_os_error() == Some(libc::EWOULDBLOCK)
391    {
392        Ok(None)
393    } else {
394        Err(error)
395    }
396}
397
398/// Remove the PID file on setup failure only while its path still names the
399/// file this start attempt created.
400#[cfg(unix)]
401pub(super) fn remove_pid_file_if_owned(pid_file: &std::path::Path, guard: &std::fs::File) {
402    let Ok(owned) = guard.metadata() else {
403        return;
404    };
405    let Ok(current) = std::fs::metadata(pid_file) else {
406        return;
407    };
408    if owned.dev() == current.dev() && owned.ino() == current.ino() {
409        if let Err(e) = std::fs::remove_file(pid_file) {
410            tracing::warn!(error = %e, path = ?pid_file, "failed to remove unbound PID file");
411        }
412    }
413}
414
415/// Return `true` if `pid_file` currently names an eligible live process that
416/// still answers on `sock` — i.e. a daemon already owns this rendezvous and it
417/// is safe to defer to it rather than treat the `AlreadyExists` PID-file
418/// collision as a boot failure. Eligibility requires a different PID in
419/// production; the explicit in-process harness may allow the current PID.
420#[cfg(unix)]
421pub(super) async fn pid_file_names_a_reachable_daemon(
422    pid_file: &std::path::Path,
423    sock: &std::path::Path,
424    allow_same_process_incumbent: bool,
425    expected_config_id: &str,
426) -> bool {
427    let Ok(pid_str) = std::fs::read_to_string(pid_file) else {
428        return false;
429    };
430    let Ok(pid) = pid_str.trim().parse::<u32>() else {
431        return false;
432    };
433    pid_can_name_incumbent(pid, std::process::id(), allow_same_process_incumbent)
434        && is_process_running(pid)
435        && sock.exists()
436        && socket_speaks_khived_protocol(sock, expected_config_id).await
437}
438
439#[cfg(all(unix, test))]
440pub(super) async fn drain(active: &std::sync::atomic::AtomicUsize) -> bool {
441    drain_with_timeout(active, drain_timeout()).await
442}
443
444/// Voluntary retirement keeps admitted workers and rendezvous ownership alive.
445#[cfg(unix)]
446pub(super) async fn drain_for_idle(
447    active: &std::sync::atomic::AtomicUsize,
448    timeout: std::time::Duration,
449) {
450    let deadline = tokio::time::Instant::now() + timeout;
451    let mut warned = false;
452    while active.load(std::sync::atomic::Ordering::SeqCst) + background_task_count() != 0 {
453        if !warned && tokio::time::Instant::now() >= deadline {
454            tracing::warn!(
455                "idle drain interval elapsed; retaining workers and rendezvous until settled"
456            );
457            warned = true;
458        }
459        tokio::time::sleep(std::time::Duration::from_millis(100)).await;
460    }
461}
462
463#[cfg(unix)]
464pub(super) async fn drain_with_timeout(
465    active: &std::sync::atomic::AtomicUsize,
466    timeout: std::time::Duration,
467) -> bool {
468    use std::sync::atomic::Ordering;
469    // One sequentially-consistent order spans connection handoff and tracked
470    // task publication. Drain must not observe an ended connection and a
471    // stale pre-publication background count as two simultaneous zeroes.
472    let remaining = || active.load(Ordering::SeqCst) + background_task_count();
473    if remaining() == 0 {
474        return true;
475    }
476    let deadline = tokio::time::Instant::now() + timeout;
477    while remaining() > 0 {
478        if tokio::time::Instant::now() >= deadline {
479            tracing::warn!(
480                remaining_connections = active.load(Ordering::SeqCst),
481                remaining_background_tasks = background_task_count(),
482                outstanding_background_tasks = %background_task_names().join(", "),
483                "drain timeout reached; forcing shutdown"
484            );
485            return false;
486        }
487        tokio::select! {
488            _ = tokio::time::sleep(std::time::Duration::from_millis(100)) => {}
489            _ = tokio::time::sleep_until(deadline) => {}
490        }
491    }
492    true
493}
494
495#[cfg(unix)]
496pub(super) async fn finish_connection_tasks(
497    tasks: Vec<tokio::task::JoinHandle<()>>,
498    drained: bool,
499) {
500    if !drained {
501        for task in &tasks {
502            if !task.is_finished() {
503                task.abort();
504            }
505        }
506    }
507    for task in tasks {
508        let _ = task.await;
509    }
510}
511
512/// The bound `drain()` waits for tracked background tasks at daemon shutdown
513/// (`KHIVE_DRAIN_TIMEOUT_SECS`, default 10s). Public so component supervision
514/// can clamp per-component shutdown timeouts against it — a component timeout
515/// longer than the drain bound could never complete its abort/state
516/// transition before the daemon returns.
517pub fn drain_timeout() -> std::time::Duration {
518    let secs = khive_db::env::env_parse_or("KHIVE_DRAIN_TIMEOUT_SECS", DEFAULT_DRAIN_TIMEOUT_SECS);
519    std::time::Duration::from_secs(secs)
520}