Skip to main content

khive_runtime/
daemon.rs

1//! khived daemon server — persistent warm runtime over a Unix socket.
2//!
3//! The daemon binds `~/.khive/khived.sock`, accepts length-prefixed request
4//! frames, dispatches them through a [`DaemonDispatch`] implementor, and serves
5//! results back. It is transport-agnostic: the MCP crate provides the dispatch
6//! impl, but any future client (CLI, HTTP gateway) can reuse this server.
7//!
8//! The client side (forwarding, auto-spawn) lives in the transport crate
9//! (e.g. `khive-mcp`), not here.
10
11use std::sync::Arc;
12
13#[cfg(unix)]
14use std::io::Write as _;
15#[cfg(unix)]
16use std::os::unix::fs::PermissionsExt;
17#[cfg(unix)]
18use std::os::unix::io::AsRawFd;
19use std::path::PathBuf;
20
21#[cfg(unix)]
22use async_trait::async_trait;
23#[cfg(unix)]
24use libc;
25use serde::{Deserialize, Serialize};
26#[cfg(unix)]
27use tokio::io::{AsyncReadExt, AsyncWriteExt};
28#[cfg(unix)]
29use tokio::net::{UnixListener, UnixStream};
30
31#[cfg(unix)]
32use khive_db::{run_checkpoint_task, CheckpointConfig, CheckpointLifecycleOwner, ConnectionPool};
33
34#[cfg(unix)]
35use crate::pack::RequestIdentity;
36
37/// Maximum frame size accepted in either direction.
38pub const MAX_FRAME_BYTES: usize = 8 * 1024 * 1024;
39
40/// Wire protocol version for the daemon IPC framing.
41///
42/// Increment this constant whenever the request or response frame shape
43/// changes in a backward-incompatible way. The client sends its version
44/// in every request; the daemon rejects mismatches with an explicit error
45/// that names both sides so the operator knows exactly what to do
46/// (`make local` rebuilds the client binary).
47/// See `docs/api/daemon.md#protocol_version` for the version-by-version history.
48pub const PROTOCOL_VERSION: u32 = 4;
49
50const DEFAULT_DRAIN_TIMEOUT_SECS: u64 = 10;
51
52// ── paths ─────────────────────────────────────────────────────────────────────
53
54/// Base `.khive` directory used to anchor every advisory lock/socket/pid
55/// path below. Pure path computation — portable on every target, even
56/// though most of its callers (socket/pid paths) are unix-only.
57///
58/// Resolution order: `HOME`, then `USERPROFILE` (the conventional Windows
59/// home variable), then a platform-specific last resort. On non-unix targets
60/// the last resort is the OS temp directory (per-user on Windows), so the
61/// lock path can never be working-directory-relative there: two processes
62/// opening the same database from different working directories must resolve
63/// the same lock file. On unix the last resort stays the historical `"."` —
64/// a shared world-writable anchor such as `/tmp` would be worse, since a
65/// local attacker could pre-claim the directory and the socket/lock files
66/// under it before the daemon's first run.
67fn khive_dir() -> PathBuf {
68    khive_root_from(
69        std::env::var("HOME").ok(),
70        std::env::var("USERPROFILE").ok(),
71    )
72}
73
74/// Env-free core of [`khive_dir`], split out so the fallback chain is
75/// testable without mutating process-global environment variables.
76fn khive_root_from(home: Option<String>, userprofile: Option<String>) -> PathBuf {
77    home.filter(|v| !v.trim().is_empty())
78        .or_else(|| userprofile.filter(|v| !v.trim().is_empty()))
79        .map(PathBuf::from)
80        .unwrap_or_else(last_resort_root)
81        .join(".khive")
82}
83
84/// See [`khive_dir`] for why the two arms differ.
85#[cfg(unix)]
86fn last_resort_root() -> PathBuf {
87    PathBuf::from(".")
88}
89
90#[cfg(not(unix))]
91fn last_resort_root() -> PathBuf {
92    std::env::temp_dir()
93}
94
95/// Unix socket path the daemon binds and clients connect to.
96///
97/// Overridable via the `KHIVE_SOCKET` env var (for tests and ops).
98#[cfg(unix)]
99pub fn socket_path() -> PathBuf {
100    if let Ok(p) = std::env::var("KHIVE_SOCKET") {
101        if !p.is_empty() {
102            return PathBuf::from(p);
103        }
104    }
105    khive_dir().join("khived.sock")
106}
107
108/// PID file path written by the daemon.
109///
110/// Overridable via the `KHIVE_PID` env var.
111#[cfg(unix)]
112pub fn pid_path() -> PathBuf {
113    if let Ok(p) = std::env::var("KHIVE_PID") {
114        if !p.is_empty() {
115            return PathBuf::from(p);
116        }
117    }
118    khive_dir().join("khived.pid")
119}
120
121/// Advisory lock file used to serialize stale-daemon recovery across concurrent
122/// clients (flock/`File::lock` on the file; released when the lock file
123/// handle is dropped). Path computation is portable; `kkernel exec`'s
124/// non-unix local-construction guard (`kkernel::exec::acquire_local_construction_guard`)
125/// shares this exact path with the unix daemon-boot guard so the two stay
126/// mutually exclusive.
127///
128/// Overridable via the `KHIVE_LOCK` env var (for tests).
129pub fn lock_path() -> PathBuf {
130    if let Ok(p) = std::env::var("KHIVE_LOCK") {
131        if !p.is_empty() {
132            return PathBuf::from(p);
133        }
134    }
135    khive_dir().join("khived.recovery.lock")
136}
137
138/// Advisory lock file used to serialize RECOVERY (kill+respawn) attempts
139/// across concurrent clients only — the daemon's own boot sequence never
140/// acquires this file ([`lock_path`] / [`acquire_daemon_boot_guard`] is the
141/// boot-side lock). A recoverer holding this lock across dead-confirmation
142/// → kill → spawn (khive-mcp's `kill_and_respawn`) therefore can never
143/// deadlock against a peer daemon's boot, unlike holding the shared boot
144/// lock for that whole span would.
145///
146/// Overridable via the `KHIVE_RECOVERER_LOCK` env var (for tests).
147#[cfg(unix)]
148pub fn recoverer_lock_path() -> PathBuf {
149    if let Ok(p) = std::env::var("KHIVE_RECOVERER_LOCK") {
150        if !p.is_empty() {
151            return PathBuf::from(p);
152        }
153    }
154    khive_dir().join("khived.recoverer.lock")
155}
156
157#[cfg(unix)]
158fn open_lock_file(path: &std::path::Path) -> std::io::Result<std::fs::File> {
159    if let Some(parent) = path.parent() {
160        let _ = std::fs::create_dir_all(parent);
161    }
162    std::fs::OpenOptions::new()
163        .create(true)
164        .truncate(false)
165        .write(true)
166        .open(path)
167}
168
169#[cfg(unix)]
170fn acquire_flock_blocking(path: &std::path::Path, label: &str) -> Option<std::fs::File> {
171    let file = match open_lock_file(path) {
172        Ok(f) => f,
173        Err(e) => {
174            tracing::warn!(error = %e, path = ?path, "cannot open {label} lock file");
175            return None;
176        }
177    };
178    // SAFETY: flock is a POSIX advisory lock with no memory side-effects.
179    let rc = unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX) };
180    if rc != 0 {
181        tracing::warn!("flock LOCK_EX failed on {label} lock");
182        return None;
183    }
184    Some(file)
185}
186
187/// Acquire an exclusive advisory flock on the recovery/startup lock file.
188///
189/// The returned `File` holds the lock for its lifetime; dropping it releases
190/// it.  Used by both the client (serializing kill+spawn) and the daemon server
191/// (serializing cleanup+bind+pid-write) so the two critical sections are
192/// mutually exclusive across processes.
193#[cfg(unix)]
194pub fn acquire_recovery_lock() -> Option<std::fs::File> {
195    acquire_flock_blocking(&lock_path(), "recovery")
196}
197
198/// Attempt to acquire an exclusive advisory flock on `path`, retrying with a
199/// non-blocking `flock(LOCK_NB)` until `deadline` elapses. Bounded alternative
200/// to `acquire_recovery_lock`/`acquire_daemon_boot_guard`'s unbounded blocking
201/// flock — see `docs/api/daemon.md#try_acquire_flock_until` for why a caller
202/// merely detecting lock freedom needs a deadline instead.
203///
204/// - `Ok(Some(file))` — the lock was free within the deadline.
205/// - `Ok(None)` — `deadline` elapsed while the lock stayed held; an explicit
206///   "could not confirm" outcome, distinct from a hard I/O error.
207/// - `Err(_)` — the lock file could not be opened, or `flock` failed for a
208///   reason other than contention.
209///
210/// Blocking (paces retries with `std::thread::sleep`) — async callers must
211/// run this via `spawn_blocking`.
212#[cfg(unix)]
213fn try_acquire_flock_until(
214    path: &std::path::Path,
215    deadline: std::time::Instant,
216) -> std::io::Result<Option<std::fs::File>> {
217    let file = open_lock_file(path)?;
218    let poll_interval = std::time::Duration::from_millis(10);
219    loop {
220        // SAFETY: flock is a POSIX advisory lock with no memory side-effects.
221        let rc = unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) };
222        if rc == 0 {
223            return Ok(Some(file));
224        }
225        let err = std::io::Error::last_os_error();
226        if err.raw_os_error() != Some(libc::EWOULDBLOCK) {
227            return Err(err);
228        }
229        let now = std::time::Instant::now();
230        if now >= deadline {
231            return Ok(None);
232        }
233        std::thread::sleep(poll_interval.min(deadline - now));
234    }
235}
236
237/// Bounded, deadline-aware variant of [`acquire_daemon_boot_guard`]: attempts
238/// the SAME boot/recovery lock ([`lock_path`]) but gives up at `deadline`
239/// instead of blocking forever. For callers that need to detect "is a boot in
240/// progress right now" without risking an unbounded wait behind a wedged
241/// holder: e.g. khive-mcp's `confirm_genuinely_dead` re-probing rounds,
242/// where `DEAD_CONFIRM_ROUNDS` must bound elapsed time, not just probe count.
243#[cfg(unix)]
244pub fn try_acquire_daemon_boot_guard_until(
245    deadline: std::time::Instant,
246) -> std::io::Result<Option<DaemonBootGuard>> {
247    try_acquire_flock_until(&lock_path(), deadline)
248}
249
250/// Bounded, deadline-aware acquisition of the recoverer-only lock
251/// ([`recoverer_lock_path`]). See [`try_acquire_daemon_boot_guard_until`] for
252/// the shared rationale — a second recoverer waiting for a peer's dead
253/// confirmation/kill/spawn critical section must give up and report
254/// "uncertain" rather than block forever if that peer is itself wedged.
255#[cfg(unix)]
256pub fn try_acquire_recoverer_lock_until(
257    deadline: std::time::Instant,
258) -> std::io::Result<Option<std::fs::File>> {
259    try_acquire_flock_until(&recoverer_lock_path(), deadline)
260}
261
262/// Guard returned by [`acquire_daemon_boot_guard`], held across cold-boot
263/// schema initialization (migrations + pack schema plans / FTS DDL) through
264/// daemon bind + pid-write.
265#[cfg(unix)]
266pub type DaemonBootGuard = std::fs::File;
267
268/// Acquire the recovery/boot lock, treating failure as fatal.
269///
270/// Unlike [`acquire_recovery_lock`] (best-effort, `None` on failure: used by
271/// shutdown cleanup, where skipping unlink is safer than blocking forever),
272/// daemon-mode boot must hold this lock across migrations/FTS DDL through
273/// bind+pid-write. Silently continuing with no lock reopens the cold-boot FTS
274/// race this guard exists to close, so callers that are about to run
275/// daemon-mode boot (or wait for one to quiesce) must fail loudly instead of
276/// proceeding unguarded.
277#[cfg(unix)]
278pub fn acquire_daemon_boot_guard() -> anyhow::Result<DaemonBootGuard> {
279    acquire_recovery_lock()
280        .ok_or_else(|| anyhow::anyhow!("failed to acquire daemon boot/recovery lock"))
281}
282
283/// Identity of a bound Unix socket path, used to tell "the socket I bound" apart
284/// from "a same-path socket some other daemon bound after mine was removed".
285///
286/// A socket path can be recreated by a different process between the time
287/// this daemon captures its identity and the time it later checks it, so `dev`
288/// and `ino` (not the path) are what must match for cleanup to be safe.
289#[cfg(unix)]
290#[derive(Clone, Copy, PartialEq, Eq)]
291struct SocketIdentity {
292    dev: u64,
293    ino: u64,
294}
295
296#[cfg(unix)]
297fn socket_identity(path: &std::path::Path) -> Option<SocketIdentity> {
298    use std::os::unix::fs::MetadataExt;
299    let meta = std::fs::metadata(path).ok()?;
300    Some(SocketIdentity {
301        dev: meta.dev(),
302        ino: meta.ino(),
303    })
304}
305
306// ── connection principal ──────────────────────────────────────────────────────
307
308/// The uid on the other end of an accepted connection, read from the kernel.
309///
310/// This is the only identity on this socket the caller cannot choose. Every
311/// identity field on the request frame — `namespace`, `actor_id`,
312/// `visible_namespaces`, `config_id` — is supplied by the connecting process,
313/// so none of them can answer "who is this". A check reading self-asserted
314/// fields is not a weak gate, it is not a gate: anyone who wants to pass it
315/// asserts the passing values.
316///
317/// `getpeereid(2)` on macOS/BSD, `SO_PEERCRED` on Linux. Both report the peer's
318/// credentials as recorded by the kernel at connect time.
319#[cfg(unix)]
320fn peer_uid(stream: &UnixStream) -> std::io::Result<u32> {
321    use std::os::fd::AsRawFd;
322    let fd = stream.as_raw_fd();
323
324    #[cfg(any(target_os = "macos", target_os = "ios", target_vendor = "apple"))]
325    {
326        let mut uid: libc::uid_t = 0;
327        let mut gid: libc::gid_t = 0;
328        // SAFETY: `fd` is a live connected socket owned by `stream` for the
329        // duration of this call; both out-params are valid initialized locals.
330        let rc = unsafe { libc::getpeereid(fd, &mut uid, &mut gid) };
331        if rc != 0 {
332            return Err(std::io::Error::last_os_error());
333        }
334        Ok(uid as u32)
335    }
336
337    #[cfg(target_os = "linux")]
338    {
339        let mut cred = libc::ucred {
340            pid: 0,
341            uid: 0,
342            gid: 0,
343        };
344        let mut len = std::mem::size_of::<libc::ucred>() as libc::socklen_t;
345        // SAFETY: `fd` is a live connected socket owned by `stream`; `cred` is
346        // an initialized local of exactly `len` bytes, which is what
347        // SO_PEERCRED writes.
348        let rc = unsafe {
349            libc::getsockopt(
350                fd,
351                libc::SOL_SOCKET,
352                libc::SO_PEERCRED,
353                (&mut cred as *mut libc::ucred).cast::<libc::c_void>(),
354                &mut len,
355            )
356        };
357        if rc != 0 {
358            return Err(std::io::Error::last_os_error());
359        }
360        Ok(cred.uid)
361    }
362
363    #[cfg(not(any(
364        target_os = "linux",
365        target_os = "macos",
366        target_os = "ios",
367        target_vendor = "apple"
368    )))]
369    {
370        let _ = fd;
371        Err(std::io::Error::new(
372            std::io::ErrorKind::Unsupported,
373            "peer-credential capture is not implemented for this platform",
374        ))
375    }
376}
377
378/// Whether a connection from `uid` may be served by this daemon.
379///
380/// ADR-096 accepted per-request identity threading **for the single-principal
381/// owner-only socket only**, and rested that on three things: the `0600`
382/// socket, all connections being the same uid, and the database being already
383/// same-uid-accessible. The socket mode is asserted at bind. This asserts the
384/// second, which previously had no representation in the code at all — nothing
385/// read peer identity, so nothing could notice when it stopped being true.
386///
387/// **Principal is not attribution.** Many `actor_id`s over one socket is
388/// exactly what ADR-096 shipped and what every seat on a normal host does;
389/// refusing a second distinct actor would break the accepted design. The
390/// principal is the uid, and this refuses only a genuinely foreign one.
391///
392/// **There is deliberately no configuration escape hatch.** A flag permitting
393/// other uids would not weaken this assertion, it would delete it, in the way
394/// hardest to notice later: the check still exists, its tests still pass, and
395/// the deployment that matters has it off. A deployment that genuinely needs
396/// multiple uids needs a code change and a gated ADR — which is precisely the
397/// decision that should be impossible to make by accident.
398#[cfg(unix)]
399fn uid_is_permitted(peer: u32, daemon_euid: u32) -> bool {
400    peer == daemon_euid
401}
402
403// ── wire types ────────────────────────────────────────────────────────────────
404
405/// Request frame sent from a client to the daemon.
406#[derive(Serialize, Deserialize, Default)]
407pub struct DaemonRequestFrame {
408    pub ops: String,
409    pub presentation: Option<String>,
410    pub presentation_per_op: Option<Vec<Option<String>>>,
411    /// The client's resolved storage/gate default namespace for this request.
412    ///
413    /// As of protocol version 3 (ADR-096) the daemon serves the request under
414    /// this namespace instead of rejecting on mismatch: a per-request
415    /// identity input, not a same-process-identity assertion.
416    pub namespace: String,
417    /// The client's resolved write-stamp / gate actor identity (ADR-057),
418    /// carried on the frame so the warm daemon stamps writes with the
419    /// *caller's* actor instead of its own baked `actor_id` (ADR-096). `None`
420    /// mints `ActorRef::anonymous()`, matching an unconfigured actor.
421    #[serde(default)]
422    pub actor_id: Option<String>,
423    /// Opaque process provenance resolved in the originating client process.
424    /// It is carried per request because a shared warm daemon's environment
425    /// does not identify the worker that submitted the operation. Protocol v4
426    /// makes this field part of dispatch semantics: a v3 daemon must reject the
427    /// request rather than execute it while silently discarding provenance.
428    #[serde(default, skip_serializing_if = "Option::is_none")]
429    pub process_ref: Option<String>,
430    /// The client's resolved extra read-visibility namespaces (ADR-007 Rule
431    /// 3b), carried on the frame so the warm daemon widens read scope to
432    /// match the caller's own configuration rather than its own baked
433    /// `visible_namespaces` (ADR-096). Empty means no extra visibility beyond
434    /// `namespace` itself.
435    #[serde(default)]
436    pub visible_namespaces: Vec<String>,
437    /// Fingerprint of the client's engine-coherence config: packs, db target,
438    /// embedders, backend routing, and construction-baked outbound policy.
439    /// Identity fields are carried separately in this frame. The daemon rejects
440    /// a request whose `config_id` differs from its own so a restricted client
441    /// (e.g. `--pack kg`, `--db :memory:`) never dispatches through the broader
442    /// default daemon. See ADR-027 / ADR-049 / ADR-096.
443    #[serde(default)]
444    pub config_id: String,
445    /// IPC protocol version sent by the client. Pre-versioning clients omit
446    /// this field (deserializes to 0). The daemon compares against
447    /// [`PROTOCOL_VERSION`] and rejects mismatches with an explicit error.
448    #[serde(default)]
449    pub protocol_version: u32,
450    /// When `true`, the daemon returns an identity frame (ok=true, result=None)
451    /// immediately after identity validation — without calling the dispatcher.
452    /// Used by the client's under-lock recovery probe to confirm a daemon is
453    /// alive and identity-matching without dispatching any mutating verb.
454    /// Pre-probe clients omit this field (deserializes to false → normal dispatch).
455    #[serde(default)]
456    pub probe_only: bool,
457    /// When `true`, the daemon returns a point-in-time [`MetricsSnapshot`] of
458    /// its server-side gauges (a read-only measurement surface for the
459    /// load/perf harness) instead of dispatching any op. Handled before the
460    /// `config_id` equality reject: a gauge read is process-global and
461    /// namespace/config-agnostic, not a namespaced record operation.
462    /// READ-ONLY — this field is the only input the frame accepts for a
463    /// metrics request; there is no reset or mutation reachable over the
464    /// wire. Pre-metrics clients omit this field (deserializes to `false` →
465    /// normal dispatch, unaffected).
466    #[serde(default)]
467    pub metrics_only: bool,
468    /// Output format for this request (ADR-078). Forwarded to the daemon's
469    /// serialization seam. `None` means use the daemon's resolved default.
470    #[serde(default)]
471    pub format: Option<String>,
472    /// Per-operation output format overrides (ADR-078).
473    #[serde(default)]
474    pub format_per_op: Option<Vec<Option<String>>>,
475    /// Whether this request originated from the agent-facing MCP `request`
476    /// tool (the wire surface). When `true`, the daemon rejects
477    /// `Visibility::Subhandler` verbs: agents must not invoke internal
478    /// subhandlers. When `false` (the default, and the only value any
479    /// operator path sends), subhandlers are allowed: `kkernel exec` and
480    /// other in-process callers are trusted operator surfaces.
481    ///
482    /// This is the origin discriminator, not a daemon-vs-local one: operator
483    /// requests flow through the daemon by default too, so the gate cannot
484    /// key on transport.
485    #[serde(default)]
486    pub from_wire: bool,
487    /// Caller-supplied correlation id (khive#948): a `u64` from the caller's
488    /// own process-local monotonic counter, echoed back unchanged on
489    /// [`DaemonResponseFrame::request_id`] and stamped into the dispatch's
490    /// audit event (`resource.request_id`) so a benchmark harness can join
491    /// its own pre-send sample to the server-side audit row for the same
492    /// request. Purely additive — `#[serde(default)]` matches
493    /// `metrics_only`/`format`/`format_per_op` precedent, no
494    /// `PROTOCOL_VERSION` bump. `None` means the caller supplied no id.
495    #[serde(default)]
496    pub request_id: Option<u64>,
497}
498
499/// Response frame sent from the daemon back to a client.
500#[derive(Serialize, Deserialize, Debug)]
501pub struct DaemonResponseFrame {
502    pub ok: bool,
503    pub result: Option<String>,
504    pub error: Option<String>,
505    pub namespace_mismatch: bool,
506    /// Set when the request's `config_id` does not match the daemon's. Like
507    /// `namespace_mismatch`, this signals the client to fall back to local
508    /// dispatch rather than execute under a different runtime/config.
509    #[serde(default)]
510    pub config_mismatch: bool,
511    /// The `config_id` the daemon dispatched under, echoed back so the client
512    /// can positively confirm the result came from a matching runtime. A
513    /// pre-`config_id` daemon omits this field (deserializes to `None`), which
514    /// the client treats as a mismatch and falls back to local dispatch — this
515    /// closes the upgrade window where a new restricted client could otherwise
516    /// trust a still-warm legacy daemon's broader registry.
517    #[serde(default)]
518    pub served_config_id: Option<String>,
519    /// Set when the client's `protocol_version` does not match the daemon's
520    /// [`PROTOCOL_VERSION`]. The client must treat this as a hard error and
521    /// surface the human-readable `error` field rather than falling back to
522    /// local dispatch (which would hide the version skew).
523    #[serde(default)]
524    pub version_mismatch: bool,
525    /// The daemon's [`PROTOCOL_VERSION`], echoed in error responses so the
526    /// client can include both sides in the diagnostic message. Pre-versioning
527    /// daemons omit this field (deserializes to 0).
528    #[serde(default)]
529    pub daemon_protocol_version: u32,
530    /// Populated when the request set `metrics_only: true`: a point-in-time
531    /// snapshot of the daemon's server-side gauges. `None` on every other
532    /// response, and on any response from a daemon that predates this field
533    /// (client-side back-compat via `#[serde(default)]`, matching
534    /// `served_config_id`'s upgrade-window handling above).
535    #[serde(default, skip_serializing_if = "Option::is_none")]
536    pub metrics: Option<MetricsSnapshot>,
537    /// Echo of the request's `request_id` (khive#948), present whenever the
538    /// frame that produced this response carried one — including on every
539    /// error/denied arm, not only success, so a client can join a failure
540    /// the same way it joins a success. `#[serde(default)]` so an older
541    /// daemon's response (predating this field) deserializes to `None`
542    /// rather than a parse error.
543    #[serde(default)]
544    pub request_id: Option<u64>,
545}
546
547/// Point-in-time snapshot of the daemon's server-side gauges — the
548/// load/perf harness read-surface (measurement substrate, not a product feature).
549///
550/// Every field here is a **server-side** gauge reachable from `handle_conn`
551/// without any mutation: [`khive_storage::tx_registry`] (ADR-091 Plank 0,
552/// process-global singleton), the WAL checkpoint task's last-observed page
553/// count and TRUNCATE counters (`khive_db::checkpoint`), and the ADR-067
554/// Component A write queue depth (only when `KHIVE_WRITE_QUEUE=1`). There is
555/// no reset reachable through this type or through [`DaemonRequestFrame`] —
556/// gauges out, nothing in.
557#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq)]
558pub struct MetricsSnapshot {
559    /// Last-observed WAL page count from the periodic checkpoint tick.
560    /// `None` when the checkpoint task has never ticked in this process
561    /// (for example, an in-memory dispatcher with no pool, or a daemon that
562    /// just started and hasn't hit its first tick yet).
563    pub wal_pages: Option<u64>,
564    /// Total WAL TRUNCATE escalation attempts (ADR-091 Plank 2) made in this
565    /// process's lifetime, regardless of whether they succeeded in reclaiming
566    /// pages.
567    pub wal_truncate_attempts: u64,
568    /// Current consecutive-failure count for TRUNCATE attempts that failed to
569    /// bring the WAL back below `warn_pages`; resets to 0 the next time an
570    /// attempt clears it.
571    pub wal_truncate_consecutive_failures: u64,
572    /// Total checkpoint ticks skipped because the writer mutex was busy
573    /// (ADR-091 checkpoint-pressure telemetry), across this process's
574    /// lifetime. `#[serde(default)]` so an older client decoding a newer
575    /// daemon's snapshot (or vice versa) does not fail.
576    #[serde(default)]
577    pub wal_checkpoint_skipped_ticks: u64,
578    /// Current consecutive-skip run length; 0 once the next tick is observed.
579    #[serde(default)]
580    pub wal_checkpoint_consecutive_skips: u64,
581    /// WAL page count last known at the time of the most recent skip, if any
582    /// skip has occurred yet in this process.
583    #[serde(default)]
584    pub wal_checkpoint_last_skip_wal_pages: Option<u64>,
585    /// Age, in microseconds, of the oldest currently-open transaction
586    /// registry entry (ADR-091 Plank 0). `None` when no transaction is
587    /// currently open.
588    pub oldest_pinned_tx_micros: Option<u64>,
589    /// Diagnostic label of the oldest currently-open transaction registry
590    /// entry, if any and if it was registered with one.
591    pub oldest_pinned_tx_label: Option<String>,
592    /// Number of currently open transaction registry entries.
593    pub open_tx_count: usize,
594    /// Current write-queue backlog depth (ADR-067 Component A): requests
595    /// enqueued but not yet accepted by the `WriterTask` drain loop. `None`
596    /// unless the write queue is enabled (`KHIVE_WRITE_QUEUE=1`) and a
597    /// file-backed pool is available.
598    pub write_queue_depth: Option<usize>,
599    /// The write queue's configured bounded capacity
600    /// (`PoolConfig::write_queue_capacity`), gated the same as
601    /// `write_queue_depth`.
602    pub write_queue_capacity: Option<usize>,
603}
604
605// ── framing ───────────────────────────────────────────────────────────────────
606
607/// Read one length-prefixed frame (4-byte BE u32 length + JSON bytes).
608#[cfg(unix)]
609pub async fn read_frame(stream: &mut UnixStream) -> std::io::Result<Vec<u8>> {
610    let mut len_buf = [0u8; 4];
611    stream.read_exact(&mut len_buf).await?;
612    let len = u32::from_be_bytes(len_buf) as usize;
613    if len > MAX_FRAME_BYTES {
614        return Err(std::io::Error::new(
615            std::io::ErrorKind::InvalidData,
616            format!("daemon frame of {len} bytes exceeds {MAX_FRAME_BYTES} cap"),
617        ));
618    }
619    let mut buf = vec![0u8; len];
620    stream.read_exact(&mut buf).await?;
621    Ok(buf)
622}
623
624/// Write one length-prefixed frame.
625#[cfg(unix)]
626pub async fn write_frame(stream: &mut UnixStream, payload: &[u8]) -> std::io::Result<()> {
627    if payload.len() > MAX_FRAME_BYTES {
628        return Err(std::io::Error::new(
629            std::io::ErrorKind::InvalidData,
630            format!(
631                "daemon frame of {} bytes exceeds {MAX_FRAME_BYTES} cap",
632                payload.len()
633            ),
634        ));
635    }
636    let len = (payload.len() as u32).to_be_bytes();
637    stream.write_all(&len).await?;
638    stream.write_all(payload).await?;
639    stream.flush().await?;
640    Ok(())
641}
642
643// ── dispatch trait ────────────────────────────────────────────────────────────
644
645/// Transport-agnostic dispatch interface for the daemon server.
646///
647/// The MCP crate implements this by dispatching through the shared request body
648/// while honoring [`DaemonRequestFrame::from_wire`] (so subhandler visibility is
649/// gated by request origin, not by transport); any future transport can do the
650/// same.
651#[cfg(unix)]
652#[async_trait]
653pub trait DaemonDispatch: Clone + Send + Sync + 'static {
654    /// Dispatch a verb-DSL request string and return the rendered result.
655    ///
656    /// `from_wire` carries the origin discriminator from
657    /// [`DaemonRequestFrame::from_wire`]: when `true`, the implementor enforces
658    /// verb visibility (rejects `Visibility::Subhandler` verbs); when `false`,
659    /// the request is from a trusted operator surface and subhandlers pass.
660    ///
661    /// `identity` is the per-request identity context threaded from the frame
662    /// (ADR-096): `Some(..)` when serving a request forwarded over the
663    /// daemon socket (built from `frame.namespace` / `frame.actor_id` /
664    /// `frame.visible_namespaces` by the connection handler), `None` for any
665    /// other dispatch path. Implementors should mint the storage/gate token from
666    /// `identity` when present and fall back to their own construction-baked
667    /// identity when absent, so pure local (non-daemon) dispatch is unchanged.
668    #[allow(clippy::too_many_arguments)]
669    async fn dispatch(
670        &self,
671        ops: String,
672        presentation: Option<String>,
673        presentation_per_op: Option<Vec<Option<String>>>,
674        format: Option<String>,
675        format_per_op: Option<Vec<Option<String>>>,
676        from_wire: bool,
677        identity: Option<RequestIdentity>,
678    ) -> Result<String, String>;
679
680    /// Warm every pack's in-memory state (ANN indexes, etc.).
681    async fn warm_all(&self);
682
683    /// The namespace this dispatcher was configured for.
684    fn namespace(&self) -> &str;
685
686    /// Fingerprint of this dispatcher's resolved runtime config (packs, db
687    /// target, embedders). Used to reject forwarded requests from clients whose
688    /// config differs, so a restricted client cannot dispatch through a broader
689    /// daemon.
690    fn config_id(&self) -> &str;
691
692    /// Return the pool to use for background WAL checkpointing, if available.
693    ///
694    /// Implementors backed by a file-based SQLite database should return
695    /// `Some(pool_arc)`. In-memory or test dispatchers that have no pool
696    /// return `None` and the checkpoint task is not spawned.
697    ///
698    /// The default implementation returns `None`.
699    fn pool_for_checkpoint(&self) -> Option<Arc<ConnectionPool>> {
700        None
701    }
702
703    /// File-backed backend pools beyond [`Self::pool_for_checkpoint`]'s pool
704    /// (ADR-091 Amendment 3): one checkpoint task is spawned per entry here,
705    /// in addition to the one spawned for the primary pool, so a
706    /// multi-backend deployment gets PASSIVE/TRUNCATE checkpointing and
707    /// sidecar enumeration on every file-backed backend it wired, not only
708    /// the main one.
709    ///
710    /// The default implementation returns an empty `Vec` — an implementor
711    /// with only one backend (or none) needs no override.
712    fn secondary_pools_for_checkpoint(&self) -> Vec<Arc<ConnectionPool>> {
713        Vec::new()
714    }
715
716    /// Return the audit `EventStore` the checkpoint task should append
717    /// ADR-094 lifecycle events (`CheckpointOutcomeRecorded`) to, if any.
718    ///
719    /// Mirrors [`Self::pool_for_checkpoint`]'s default-`None` shape: an
720    /// implementor with no configured event store (or no pool at all) simply
721    /// gets a checkpoint task that never appends events — the checkpoint
722    /// task itself remains fully functional either way.
723    ///
724    /// The default implementation returns `None`.
725    fn event_store_for_checkpoint(&self) -> Option<Arc<dyn khive_storage::EventStore>> {
726        None
727    }
728}
729
730#[cfg(unix)]
731struct CheckpointTaskSpec {
732    pool: Arc<ConnectionPool>,
733    lifecycle_owner: Option<CheckpointLifecycleOwner>,
734    is_main: bool,
735}
736
737/// Build checkpoint-task fan-out and designate one lifecycle owner.
738///
739/// The main checkpoint task owns lifecycle emission when it exists. If the
740/// main backend is in-memory and therefore has no checkpoint task, the first
741/// file-backed secondary owns emission instead. All remaining tasks are
742/// explicit non-owners.
743#[cfg(unix)]
744fn checkpoint_task_specs(
745    main_pool: Option<Arc<ConnectionPool>>,
746    secondary_pools: Vec<Arc<ConnectionPool>>,
747    event_store: Option<Arc<dyn khive_storage::EventStore>>,
748    namespace: String,
749) -> Vec<CheckpointTaskSpec> {
750    let mut tasks = Vec::with_capacity(usize::from(main_pool.is_some()) + secondary_pools.len());
751    if let Some(pool) = main_pool {
752        tasks.push(CheckpointTaskSpec {
753            pool,
754            lifecycle_owner: None,
755            is_main: true,
756        });
757    }
758    tasks.extend(secondary_pools.into_iter().map(|pool| CheckpointTaskSpec {
759        pool,
760        lifecycle_owner: None,
761        is_main: false,
762    }));
763
764    if let (Some(task), Some(event_store)) = (tasks.first_mut(), event_store) {
765        task.lifecycle_owner = Some(CheckpointLifecycleOwner::new(event_store, namespace));
766    }
767    tasks
768}
769
770// ── tracked background tasks ─────────────────────────────────────────────────
771//
772// Pack handlers (e.g. memory.recall's ADR-081 serve-ledger append) fire
773// fire-and-forget `tokio::spawn`ed work off the response path so the caller
774// never waits on a cross-pack dispatch or a SQL write. Left untracked, that
775// work is invisible to `drain()`: a SIGTERM landing between the response
776// returning and the spawned task completing can abort it mid-flight with no
777// log and no row. `track_background_task` gives such spawns a process-wide
778// presence that `drain()` waits on, exactly like the `active` counter does
779// for in-flight connections: the caller still only pays for the spawn +
780// counter increment, never the task's own work.
781static BACKGROUND_TASKS: std::sync::OnceLock<Arc<std::sync::atomic::AtomicUsize>> =
782    std::sync::OnceLock::new();
783
784fn background_tasks() -> &'static Arc<std::sync::atomic::AtomicUsize> {
785    BACKGROUND_TASKS.get_or_init(|| Arc::new(std::sync::atomic::AtomicUsize::new(0)))
786}
787
788/// Decrements the shared background-task counter from `Drop`, so the count
789/// comes back down whether the tracked future returns normally, panics, or
790/// is cancelled — a plain post-`await` `fetch_sub` only covers the return
791/// path and leaks the count forever on a panic, since unwinding skips every
792/// statement after the panic point.
793struct BackgroundTaskGuard {
794    counter: Arc<std::sync::atomic::AtomicUsize>,
795}
796
797impl Drop for BackgroundTaskGuard {
798    fn drop(&mut self) {
799        self.counter
800            .fetch_sub(1, std::sync::atomic::Ordering::Relaxed);
801    }
802}
803
804/// Spawn a fire-and-forget background task that daemon shutdown's `drain()`
805/// waits for, instead of a bare `tokio::spawn` that a SIGTERM can abort
806/// mid-flight with no trace. Only the enqueue (an atomic increment) is
807/// synchronous on the caller's path — the future itself still runs fully
808/// off-path, unawaited. The decrement happens via `BackgroundTaskGuard`'s
809/// `Drop`, so a panic inside `fut` still restores the count.
810pub fn track_background_task<F>(fut: F)
811where
812    F: std::future::Future<Output = ()> + Send + 'static,
813{
814    background_tasks().fetch_add(1, std::sync::atomic::Ordering::Relaxed);
815    let guard = BackgroundTaskGuard {
816        counter: background_tasks().clone(),
817    };
818    tokio::spawn(async move {
819        let _guard = guard;
820        fut.await;
821    });
822}
823
824/// Current count of in-flight tasks started via [`track_background_task`].
825/// Exposed for tests; `drain()` reads the shared counter directly.
826pub fn background_task_count() -> usize {
827    background_tasks().load(std::sync::atomic::Ordering::Relaxed)
828}
829
830/// Process-wide daemon shutdown signal (ADR-119).
831///
832/// Cancelled exactly once, when the daemon's unified shutdown future resolves
833/// — before `drain()` begins waiting on tracked tasks — so long-running
834/// daemon components supervised outside this module observe shutdown through
835/// the same path the daemon itself does, rather than inventing their own.
836/// Clones share the underlying token; child tokens derived from it are
837/// cancelled transitively.
838///
839/// In non-daemon processes the token simply never fires.
840pub fn daemon_shutdown_token() -> tokio_util::sync::CancellationToken {
841    static TOKEN: std::sync::OnceLock<tokio_util::sync::CancellationToken> =
842        std::sync::OnceLock::new();
843    TOKEN
844        .get_or_init(tokio_util::sync::CancellationToken::new)
845        .clone()
846}
847
848// ── active background phase names (ADR-103) ──────────────────────────────────
849//
850// A lightweight, best-effort process-wide gauge of which named background
851// phases (e.g. `ann_warm`) are in flight right now, read by `comm.health`'s
852// resource self-report so a caller can see "what is the daemon doing" at a
853// glance without correlating timestamps across the event log itself. Counted
854// per name rather than boolean, since more than one occurrence of the same
855// named phase can legitimately overlap (e.g. two embedding models warming
856// concurrently) — the name only drops out of the reported set once every
857// concurrent occurrence has ended.
858static ACTIVE_PHASES: std::sync::OnceLock<
859    std::sync::Mutex<std::collections::HashMap<String, usize>>,
860> = std::sync::OnceLock::new();
861
862fn active_phases() -> &'static std::sync::Mutex<std::collections::HashMap<String, usize>> {
863    ACTIVE_PHASES.get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new()))
864}
865
866/// RAII guard for one occurrence of a named background phase. Increments the
867/// phase's count on creation (see [`register_active_phase`]); decrements on
868/// `Drop`, so the count comes back down whether the guarded work returns
869/// normally, panics, or is cancelled — the same rationale as
870/// `BackgroundTaskGuard` above.
871pub struct PhaseGuard {
872    name: String,
873}
874
875impl Drop for PhaseGuard {
876    fn drop(&mut self) {
877        let mut map = active_phases()
878            .lock()
879            .unwrap_or_else(std::sync::PoisonError::into_inner);
880        if let Some(count) = map.get_mut(&self.name) {
881            *count -= 1;
882            if *count == 0 {
883                map.remove(&self.name);
884            }
885        }
886    }
887}
888
889/// Register one occurrence of a named background phase as currently active.
890/// Returns a guard: drop it (or let it fall out of scope) when the phase
891/// ends. Best-effort process-wide gauge only, read by `comm.health` — never
892/// load-bearing for correctness.
893pub fn register_active_phase(name: &str) -> PhaseGuard {
894    let mut map = active_phases()
895        .lock()
896        .unwrap_or_else(std::sync::PoisonError::into_inner);
897    *map.entry(name.to_string()).or_insert(0) += 1;
898    PhaseGuard {
899        name: name.to_string(),
900    }
901}
902
903/// Currently active background-phase names, sorted for deterministic output.
904/// Empty when no tracked phase is in flight.
905pub fn active_phase_names() -> Vec<String> {
906    let map = active_phases()
907        .lock()
908        .unwrap_or_else(std::sync::PoisonError::into_inner);
909    let mut names: Vec<String> = map.keys().cloned().collect();
910    names.sort();
911    names
912}
913
914// ── server ────────────────────────────────────────────────────────────────────
915
916/// Build a point-in-time [`MetricsSnapshot`] of this process's server-side
917/// gauges. Called only from `handle_conn`'s `metrics_only` arm — a
918/// process-global, read-only assembly with no side effects of its own.
919/// See `docs/api/daemon.md#build_metrics_snapshot` for where each gauge is sourced
920/// from and why.
921#[cfg(unix)]
922fn build_metrics_snapshot<D: DaemonDispatch>(dispatcher: &D) -> MetricsSnapshot {
923    let open_tx_count = khive_storage::tx_registry::snapshot().len();
924    // ADR-091 Amendment 3: deliberately the process-wide aggregate, not a
925    // backend-attributed view — this gauge reports "oldest pinned tx in this
926    // process" across every wired backend, not an attribution claim about
927    // which database it belongs to. Attribution consumers (the session sweep,
928    // the per-backend checkpoint tasks) use the scoped `oldest_for` views in
929    // khive-db instead.
930    let (oldest_pinned_tx_micros, oldest_pinned_tx_label) =
931        match khive_storage::tx_registry::oldest() {
932            Some((_id, age, label)) => (Some(age.as_micros() as u64), label),
933            None => (None, None),
934        };
935
936    let (write_queue_depth, write_queue_capacity) = dispatcher
937        .pool_for_checkpoint()
938        .and_then(|pool| pool.writer_task_handle().ok().flatten())
939        .map(|handle| (Some(handle.queue_depth()), Some(handle.capacity())))
940        .unwrap_or((None, None));
941
942    MetricsSnapshot {
943        wal_pages: khive_db::checkpoint::last_observed_wal_pages(),
944        wal_truncate_attempts: khive_db::checkpoint::truncate_attempts(),
945        wal_truncate_consecutive_failures: khive_db::checkpoint::truncate_consecutive_failures(),
946        wal_checkpoint_skipped_ticks: khive_db::checkpoint::checkpoint_skipped_ticks(),
947        wal_checkpoint_consecutive_skips: khive_db::checkpoint::checkpoint_consecutive_skips(),
948        wal_checkpoint_last_skip_wal_pages: khive_db::checkpoint::checkpoint_last_skip_wal_pages(),
949        oldest_pinned_tx_micros,
950        oldest_pinned_tx_label,
951        open_tx_count,
952        write_queue_depth,
953        write_queue_capacity,
954    }
955}
956
957#[cfg(unix)]
958async fn handle_conn<D: DaemonDispatch>(mut stream: UnixStream, dispatcher: D) {
959    let raw = match read_frame(&mut stream).await {
960        Ok(r) => r,
961        Err(e) => {
962            tracing::debug!(error = %e, "failed to read daemon request frame");
963            return;
964        }
965    };
966    let frame: DaemonRequestFrame = match serde_json::from_slice(&raw) {
967        Ok(f) => f,
968        Err(e) => {
969            tracing::debug!(error = %e, "failed to decode daemon request frame");
970            return;
971        }
972    };
973
974    let served_config_id = Some(dispatcher.config_id().to_string());
975    let resp = if frame.protocol_version != PROTOCOL_VERSION {
976        let msg = format!(
977            "daemon protocol mismatch: client={} daemon={} — \
978             rebuild/update the client binary (make local)",
979            frame.protocol_version, PROTOCOL_VERSION,
980        );
981        tracing::warn!(
982            client_version = frame.protocol_version,
983            daemon_version = PROTOCOL_VERSION,
984            "daemon protocol version mismatch"
985        );
986        DaemonResponseFrame {
987            ok: false,
988            result: None,
989            error: Some(msg),
990            namespace_mismatch: false,
991            config_mismatch: false,
992            served_config_id,
993            version_mismatch: true,
994            daemon_protocol_version: PROTOCOL_VERSION,
995            metrics: None,
996            request_id: frame.request_id,
997        }
998    } else if frame.metrics_only {
999        // Process-global gauge read: namespace/config-agnostic, so this is
1000        // handled BEFORE the `config_id` equality reject below (unlike every
1001        // other arm) — a metrics probe must work regardless of which
1002        // client's config is asking, since it never touches the dispatcher's
1003        // packs/db/embed registry. READ-ONLY: builds a snapshot and returns
1004        // immediately, never reaching the ops-dispatch arm.
1005        DaemonResponseFrame {
1006            ok: true,
1007            result: None,
1008            error: None,
1009            namespace_mismatch: false,
1010            config_mismatch: false,
1011            served_config_id,
1012            version_mismatch: false,
1013            daemon_protocol_version: PROTOCOL_VERSION,
1014            metrics: Some(build_metrics_snapshot(&dispatcher)),
1015            request_id: frame.request_id,
1016        }
1017    // There is no `frame.namespace != dispatcher.namespace()` reject here.
1018    // The daemon accepts and serves the request under the frame's own
1019    // identity (namespace / actor / visible_namespaces, built into a
1020    // `RequestIdentity` below) over its one shared warm registry, rather
1021    // than rejecting a differently-attributed same-uid connection to a cold
1022    // local-dispatch fallback. `config_id`: which governs packs/db/embed
1023    // coherence for the shared warm engine: remains a hard reject; it is
1024    // not an identity field and softening it would let a restricted client
1025    // dispatch through an incompatible broader daemon.
1026    } else if frame.config_id != dispatcher.config_id() {
1027        DaemonResponseFrame {
1028            ok: false,
1029            result: None,
1030            error: None,
1031            namespace_mismatch: false,
1032            config_mismatch: true,
1033            served_config_id,
1034            version_mismatch: false,
1035            daemon_protocol_version: PROTOCOL_VERSION,
1036            metrics: None,
1037            request_id: frame.request_id,
1038        }
1039    } else if frame.probe_only {
1040        // Probe-only request: identity checks passed; return immediately without
1041        // dispatching any verb. The client uses this to confirm the daemon is
1042        // alive and identity-matching without triggering any mutation.
1043        DaemonResponseFrame {
1044            ok: true,
1045            result: None,
1046            error: None,
1047            namespace_mismatch: false,
1048            config_mismatch: false,
1049            served_config_id,
1050            version_mismatch: false,
1051            daemon_protocol_version: PROTOCOL_VERSION,
1052            metrics: None,
1053            request_id: frame.request_id,
1054        }
1055    } else {
1056        // Build the per-request identity context from the frame so the
1057        // implementor mints the storage/gate token from the CALLER's
1058        // identity, not the dispatcher's own construction-baked scalars.
1059        // This is always `Some` here: every frame that reaches this arm
1060        // carries a `namespace` (required on the wire) plus whatever
1061        // `actor_id`/`visible_namespaces` the client resolved (defaulting to
1062        // `None`/`vec![]` for an older, field-absent payload, which is
1063        // exactly the prior anonymous/no-extra-visibility behavior).
1064        let identity = RequestIdentity {
1065            namespace: frame.namespace.clone(),
1066            actor_id: frame.actor_id.clone(),
1067            visible_namespaces: frame.visible_namespaces.clone(),
1068            process_ref: frame.process_ref.clone(),
1069            request_id: frame.request_id,
1070        };
1071        match dispatcher
1072            .dispatch(
1073                frame.ops,
1074                frame.presentation,
1075                frame.presentation_per_op,
1076                frame.format,
1077                frame.format_per_op,
1078                frame.from_wire,
1079                Some(identity),
1080            )
1081            .await
1082        {
1083            Ok(result) => DaemonResponseFrame {
1084                ok: true,
1085                result: Some(result),
1086                error: None,
1087                namespace_mismatch: false,
1088                config_mismatch: false,
1089                served_config_id,
1090                version_mismatch: false,
1091                daemon_protocol_version: PROTOCOL_VERSION,
1092                metrics: None,
1093                request_id: frame.request_id,
1094            },
1095            Err(e) => DaemonResponseFrame {
1096                ok: false,
1097                result: None,
1098                error: Some(e),
1099                namespace_mismatch: false,
1100                config_mismatch: false,
1101                served_config_id,
1102                version_mismatch: false,
1103                daemon_protocol_version: PROTOCOL_VERSION,
1104                metrics: None,
1105                request_id: frame.request_id,
1106            },
1107        }
1108    };
1109
1110    match serde_json::to_vec(&resp) {
1111        Ok(payload) => {
1112            if payload.len() > MAX_FRAME_BYTES {
1113                // The serialized response exceeds the IPC frame cap.  Send a
1114                // small explicit error frame so the client can distinguish a
1115                // per-request payload-size failure from a daemon crash.  A
1116                // client that receives this error frame will NOT trigger
1117                // stale-daemon kill/respawn (ParseFailure requires a read_frame
1118                // error, not an ok=false result).
1119                tracing::warn!(
1120                    bytes = payload.len(),
1121                    limit = MAX_FRAME_BYTES,
1122                    "daemon response exceeds MAX_FRAME_BYTES; sending explicit error frame"
1123                );
1124                let err_resp = DaemonResponseFrame {
1125                    ok: false,
1126                    result: None,
1127                    error: Some(format!(
1128                        "response too large: {} bytes exceeds {} byte IPC cap",
1129                        payload.len(),
1130                        MAX_FRAME_BYTES,
1131                    )),
1132                    namespace_mismatch: false,
1133                    config_mismatch: false,
1134                    served_config_id: resp.served_config_id,
1135                    version_mismatch: false,
1136                    daemon_protocol_version: PROTOCOL_VERSION,
1137                    metrics: None,
1138                    request_id: resp.request_id,
1139                };
1140                if let Ok(err_payload) = serde_json::to_vec(&err_resp) {
1141                    if let Err(e) = write_frame(&mut stream, &err_payload).await {
1142                        tracing::debug!(error = %e, "failed to write oversized-response error frame");
1143                    }
1144                }
1145            } else if let Err(e) = write_frame(&mut stream, &payload).await {
1146                tracing::debug!(error = %e, "failed to write daemon response frame");
1147            }
1148        }
1149        Err(e) => tracing::warn!(error = %e, "failed to serialize daemon response frame"),
1150    }
1151}
1152
1153/// An accepted connection owns a drain slot from the accept loop until its
1154/// handler future is dropped. The claim happens before spawning so shutdown
1155/// cannot observe zero in the gap between `accept()` and the task's first poll.
1156#[cfg(unix)]
1157struct ActiveConnectionGuard {
1158    active: Arc<std::sync::atomic::AtomicUsize>,
1159}
1160
1161#[cfg(unix)]
1162impl ActiveConnectionGuard {
1163    fn claim(active: Arc<std::sync::atomic::AtomicUsize>) -> Self {
1164        active.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
1165        Self { active }
1166    }
1167}
1168
1169#[cfg(unix)]
1170impl Drop for ActiveConnectionGuard {
1171    fn drop(&mut self) {
1172        self.active
1173            .fetch_sub(1, std::sync::atomic::Ordering::Relaxed);
1174    }
1175}
1176
1177#[cfg(unix)]
1178fn spawn_connection_task<F>(
1179    active: Arc<std::sync::atomic::AtomicUsize>,
1180    future: F,
1181) -> tokio::task::JoinHandle<()>
1182where
1183    F: std::future::Future<Output = ()> + Send + 'static,
1184{
1185    let guard = ActiveConnectionGuard::claim(active);
1186    tokio::spawn(async move {
1187        let _guard = guard;
1188        future.await;
1189    })
1190}
1191
1192/// Run the daemon: bind the socket, warm in the background, serve request
1193/// frames until SIGTERM/SIGINT.
1194///
1195/// Fatally acquires its own startup lock, which only protects
1196/// cleanup→bind→pid-write — `dispatcher` has already run migrations and
1197/// applied pack schema plans while constructing itself, unguarded. Production
1198/// boot must go through [`run_daemon_with_boot_guard`] instead, which extends
1199/// the same lock back over construction. This entry point is for callers
1200/// (and tests) that build the dispatcher and start serving as one atomic
1201/// step with no separate boot-guard window to protect.
1202#[cfg(unix)]
1203pub async fn run_daemon<D: DaemonDispatch>(dispatcher: D) -> anyhow::Result<()> {
1204    let boot_guard = Some(acquire_daemon_boot_guard()?);
1205    run_daemon_with_boot_guard(dispatcher, boot_guard).await
1206}
1207
1208/// Vet the socket's parent directory, re-permissioning it only when it is the
1209/// directory khive owns by convention.
1210///
1211/// `KHIVE_SOCKET` takes an arbitrary path, and the previous unconditional
1212/// chmod-0700 of its parent was wrong in both directions: pointed at a shared
1213/// parent like `/tmp` it either failed outright for an ordinary user, or —
1214/// worse — succeeded when privileged and stripped access for every other
1215/// process on the machine. A directory we did not create is never modified.
1216///
1217/// What the directory must actually prevent is a *takeover of the socket
1218/// path*: the connection gate is the 0600 socket plus the accept-time
1219/// peer-uid check, but both defend the daemon's own socket — neither helps
1220/// once another local user can put *their* listener at the path clients
1221/// resolve. There are two ways a shared directory allows that, and the
1222/// sticky bit closes neither: a writer can *pre-bind* the predictable path
1223/// before this daemon starts (the sticky bit restricts unlinking, not
1224/// creating), and the directory's *owner* can unlink and rebind even in a
1225/// 1777 directory (sticky exempts the directory owner). So a caller-chosen
1226/// directory is served only when it is trusted end to end: owned by this
1227/// daemon's euid or root, and not writable by group or other at all. The
1228/// umask-default 0755 stays acceptable; shared sticky directories like
1229/// `/tmp` do not.
1230#[cfg(unix)]
1231fn ensure_socket_dir_is_trusted(parent: &std::path::Path) -> anyhow::Result<()> {
1232    // SAFETY: `geteuid` is always successful and takes no arguments.
1233    let daemon_euid = unsafe { libc::geteuid() } as u32;
1234
1235    if parent == khive_dir() {
1236        std::fs::set_permissions(parent, std::fs::Permissions::from_mode(0o700)).map_err(|e| {
1237            anyhow::anyhow!(
1238                "refusing to start: cannot chmod 0700 {}: {e}. The khive directory must be \
1239                 owner-only — it is half of the same-uid guarantee this daemon enforces.",
1240                parent.display()
1241            )
1242        })?;
1243        return ensure_socket_path_is_swap_resistant(parent, daemon_euid);
1244    }
1245
1246    // Fail closed on the stat itself: not being able to read the metadata is
1247    // not the same as the directory passing.
1248    let meta = std::fs::metadata(parent).map_err(|e| {
1249        anyhow::anyhow!(
1250            "refusing to start: cannot stat {}: {e}. The socket directory gates \
1251             socket-takeover safety, and unreadable metadata is not a passing state.",
1252            parent.display()
1253        )
1254    })?;
1255
1256    use std::os::unix::fs::MetadataExt;
1257    let owner = meta.uid();
1258    if owner != daemon_euid && owner != 0 {
1259        anyhow::bail!(
1260            "refusing to start: socket directory {} is owned by uid {owner}, not this \
1261             daemon's uid ({daemon_euid}) or root. A directory owner can replace the \
1262             socket regardless of mode bits. Point KHIVE_SOCKET at a directory you own, \
1263             or unset it for the default.",
1264            parent.display()
1265        );
1266    }
1267
1268    let mode = meta.permissions().mode();
1269    if mode & 0o022 != 0 {
1270        anyhow::bail!(
1271            "refusing to start: socket directory {} is mode {:04o} — writable by group or \
1272             other, so another local user could bind their own listener at the socket path \
1273             (before this daemon starts, the sticky bit does not prevent creating the \
1274             path). Use a directory only you can write, or unset KHIVE_SOCKET for the \
1275             default. This daemon is not changing the permissions of a directory it does \
1276             not own.",
1277            parent.display(),
1278            mode & 0o7777
1279        );
1280    }
1281
1282    ensure_socket_path_is_swap_resistant(parent, daemon_euid)
1283}
1284
1285/// Walk the socket directory path exactly as the kernel will traverse it at
1286/// bind time — component by component, following symlinks — and refuse any
1287/// node another local user could swap after this validation.
1288///
1289/// Checking only the canonicalized result is not enough: bind and every
1290/// client traverse the *original* path, so a symlink component owned by
1291/// another user can resolve somewhere trusted while it is being validated
1292/// and be retargeted before the socket is bound. Validating the nodes the
1293/// traversal actually visits closes that gap: a symlink component is
1294/// acceptable only when its owner — the only party besides root who can
1295/// retarget it — is this daemon's euid or root, and every directory
1296/// component is held to the ancestor rule below.
1297///
1298/// The directory rule is deliberately weaker than the immediate parent's.
1299/// The parent hosts the socket file, where the threat is *creation* of the
1300/// predictable path — the sticky bit does not restrict creating, so no
1301/// sticky exception is sound there. An ancestor only threatens via
1302/// *rename/unlink of an existing entry we own*, which the sticky bit does
1303/// restrict: in a sticky directory, only the entry's owner, the directory's
1304/// owner, or root may rename it. So a root-owned `/tmp` (1777) is an
1305/// acceptable ancestor of a user-owned 0700 socket directory, while a
1306/// non-sticky group/other-writable ancestor, or one owned by a third uid
1307/// (who could rename the entry, or chmod the directory first), is refused.
1308/// Every stat failure fails closed.
1309#[cfg(unix)]
1310fn ensure_socket_path_is_swap_resistant(
1311    parent: &std::path::Path,
1312    daemon_euid: u32,
1313) -> anyhow::Result<()> {
1314    use std::os::unix::fs::MetadataExt;
1315
1316    let absolute = if parent.is_absolute() {
1317        parent.to_path_buf()
1318    } else {
1319        std::env::current_dir()
1320            .map_err(|e| {
1321                anyhow::anyhow!(
1322                    "refusing to start: cannot resolve the working directory to absolutize \
1323                     socket directory {}: {e}.",
1324                    parent.display()
1325                )
1326            })?
1327            .join(parent)
1328    };
1329
1330    fn push_components(stack: &mut Vec<std::ffi::OsString>, path: &std::path::Path) {
1331        let components: Vec<_> = path
1332            .components()
1333            .map(|c| c.as_os_str().to_os_string())
1334            .collect();
1335        stack.extend(components.into_iter().rev());
1336    }
1337
1338    let mut stack: Vec<std::ffi::OsString> = Vec::new();
1339    push_components(&mut stack, &absolute);
1340    let mut resolved = std::path::PathBuf::new();
1341    let mut symlinks_followed = 0u32;
1342
1343    while let Some(component) = stack.pop() {
1344        if component == "/" {
1345            resolved = std::path::PathBuf::from("/");
1346            continue;
1347        }
1348        if component == "." {
1349            continue;
1350        }
1351        if component == ".." {
1352            resolved.pop();
1353            continue;
1354        }
1355        let candidate = resolved.join(&component);
1356        let meta = std::fs::symlink_metadata(&candidate).map_err(|e| {
1357            anyhow::anyhow!(
1358                "refusing to start: cannot stat socket-path component {}: {e}. An \
1359                 unreadable component is not a passing one.",
1360                candidate.display()
1361            )
1362        })?;
1363        let owner = meta.uid();
1364
1365        if meta.file_type().is_symlink() {
1366            symlinks_followed += 1;
1367            if symlinks_followed > 40 {
1368                anyhow::bail!(
1369                    "refusing to start: socket path resolves through more than 40 symlinks \
1370                     at {} — treating this as a loop.",
1371                    candidate.display()
1372                );
1373            }
1374            if owner != daemon_euid && owner != 0 {
1375                anyhow::bail!(
1376                    "refusing to start: socket-path symlink component {} is owned by uid \
1377                     {owner}, not this daemon's uid ({daemon_euid}) or root — its owner \
1378                     could retarget it after this check and re-root the socket path. Point \
1379                     KHIVE_SOCKET somewhere trusted end to end, or unset it for the \
1380                     default.",
1381                    candidate.display()
1382                );
1383            }
1384            let target = std::fs::read_link(&candidate).map_err(|e| {
1385                anyhow::anyhow!(
1386                    "refusing to start: cannot read socket-path symlink component {}: {e}.",
1387                    candidate.display()
1388                )
1389            })?;
1390            push_components(&mut stack, &target);
1391            continue;
1392        }
1393
1394        if meta.is_dir() {
1395            let mode = meta.permissions().mode();
1396            let sticky = mode & 0o1000 != 0;
1397            if owner != daemon_euid && owner != 0 {
1398                anyhow::bail!(
1399                    "refusing to start: socket-path ancestor {} is owned by uid {owner}, not \
1400                     this daemon's uid ({daemon_euid}) or root — its owner could rename the \
1401                     next path component and re-root the socket path. Point KHIVE_SOCKET \
1402                     somewhere trusted end to end, or unset it for the default.",
1403                    candidate.display()
1404                );
1405            }
1406            if mode & 0o022 != 0 && !sticky {
1407                anyhow::bail!(
1408                    "refusing to start: socket-path ancestor {} is mode {:04o} — writable by \
1409                     group or other without the sticky bit, so another local user could rename \
1410                     the next path component and re-root the socket path. Point KHIVE_SOCKET \
1411                     somewhere trusted end to end, or unset it for the default.",
1412                    candidate.display(),
1413                    mode & 0o7777
1414                );
1415            }
1416            resolved = candidate;
1417            continue;
1418        }
1419
1420        anyhow::bail!(
1421            "refusing to start: socket-path component {} is neither a directory nor a \
1422             symlink — the socket path cannot traverse it.",
1423            candidate.display()
1424        );
1425    }
1426
1427    Ok(())
1428}
1429
1430/// Run the daemon using a startup lock acquired by the caller *before*
1431/// building `dispatcher`, so a second process racing to boot (e.g. two
1432/// `kkernel mcp --daemon` spawns before either has bound its socket) cannot
1433/// run migrations/FTS DDL concurrently against the same database file.
1434/// `boot_guard` is only `None` on non-unix targets, where there is no
1435/// advisory boot lock to hold in the first place; every unix daemon-mode
1436/// caller passes `Some`.
1437///
1438/// The guard is held across cleanup → bind → pid-write, then dropped. The
1439/// caller must not still be holding a *different* handle to the same lock
1440/// file when this function is entered — see the "Deadlock note" on the
1441/// `_startup_lock` binding below for why that would self-deadlock on `flock`.
1442#[cfg(unix)]
1443pub async fn run_daemon_with_boot_guard<D: DaemonDispatch>(
1444    dispatcher: D,
1445    boot_guard: Option<std::fs::File>,
1446) -> anyhow::Result<()> {
1447    // ADR-119: components may have been started by the serve path before this
1448    // function established (or failed to establish) daemon ownership. Cancel
1449    // the process-wide shutdown token on EVERY exit — setup failures below,
1450    // early return when another daemon owns the socket, bind errors, and
1451    // normal shutdown alike — so supervisors never outlive this process's
1452    // claim to daemon role. Constructed before any fallible startup work so
1453    // no error path can precede it. The token is process-lifetime
1454    // single-shot; a process that stops being (or never becomes) the daemon
1455    // has no path back except exec.
1456    struct ComponentTeardown;
1457    impl Drop for ComponentTeardown {
1458        fn drop(&mut self) {
1459            daemon_shutdown_token().cancel();
1460        }
1461    }
1462    let _component_teardown = ComponentTeardown;
1463
1464    let sock = socket_path();
1465    let pid_file = pid_path();
1466
1467    if let Some(parent) = sock.parent() {
1468        std::fs::create_dir_all(parent)?;
1469        ensure_socket_dir_is_trusted(parent)?;
1470    }
1471
1472    // Hold the startup lock across cleanup → bind → pid-write so a concurrent
1473    // client's kill_and_respawn (which also holds this lock) cannot unlink the
1474    // socket between our bind and our pid-write.  The lock is released once the
1475    // listener is bound and the PID file is written — at that point any racing
1476    // client will find a live socket+pid and skip the stale-cleanup path.
1477    //
1478    // Deadlock note: the client holds this lock only during kill+spawn and
1479    // releases it before the spawned daemon process starts (the lock guard is
1480    // dropped when kill_and_respawn returns, before the readiness probe loop).
1481    // The daemon holds exactly one handle to this lock for its whole boot
1482    // sequence (received as `boot_guard`, extended from before `dispatcher`
1483    // was constructed) — never a second, independently-acquired handle in the
1484    // same process, which would self-deadlock on `flock`.
1485    let _startup_lock = boot_guard;
1486
1487    if !cleanup_stale_daemon(&sock, &pid_file).await {
1488        tracing::info!("a responsive khived is already running; exiting");
1489        return Ok(());
1490    }
1491
1492    let listener = UnixListener::bind(&sock)?;
1493    // Fail closed, same reason as the directory above. If this chmod fails the
1494    // socket is world-reachable in a way the accepted design never covered, so
1495    // the bound listener is dropped and the entry removed rather than served.
1496    if let Err(e) = std::fs::set_permissions(&sock, std::fs::Permissions::from_mode(0o600)) {
1497        drop(listener);
1498        let _ = std::fs::remove_file(&sock);
1499        return Err(anyhow::anyhow!(
1500            "refusing to start: cannot chmod 0600 {}: {e}. The daemon socket must be owner-only \
1501             — it is half of the single-principal guarantee this daemon enforces.",
1502            sock.display()
1503        ));
1504    }
1505    // Captured while still holding the startup lock, immediately after
1506    // bind, so shutdown cleanup can later prove "this is still the same socket
1507    // I bound" rather than trusting the path alone.
1508    let bound_identity = socket_identity(&sock);
1509
1510    if let Err(e) = write_pid_file_exclusive(&pid_file) {
1511        if e.kind() == std::io::ErrorKind::AlreadyExists {
1512            // A PID file appeared between our own `cleanup_stale_daemon`
1513            // removing it and this write — only possible if the boot lock did
1514            // not actually exclude a concurrent booter (e.g. `acquire_recovery_lock`
1515            // failed for one side). Never touch the winner's files: drop only
1516            // the socket entry we ourselves just bound (proven via identity,
1517            // not path), then decide by checking whether the PID now on disk
1518            // names a live, reachable daemon.
1519            if bound_identity.is_some() && socket_identity(&sock) == bound_identity {
1520                drop(listener);
1521                let _ = std::fs::remove_file(&sock);
1522            }
1523            if pid_file_names_a_reachable_daemon(&pid_file, &sock).await {
1524                tracing::info!(
1525                    "a replacement khived already claimed the pid/socket rendezvous; exiting"
1526                );
1527                return Ok(());
1528            }
1529            anyhow::bail!(
1530                "failed to claim daemon pid file at {pid_file:?}: it already exists \
1531                 and does not name a reachable daemon"
1532            );
1533        }
1534        return Err(e.into());
1535    }
1536    // Release the startup lock now: the listener is bound and the PID file is
1537    // written.  Any concurrent client or daemon startup will observe a live
1538    // socket+pid and take the non-recovery path.
1539    drop(_startup_lock);
1540    tracing::info!(
1541        socket = ?sock,
1542        pid = std::process::id(),
1543        source_revision = crate::BUILD_INFO.source_revision,
1544        build_time = crate::BUILD_INFO.build_time,
1545        "khived listening"
1546    );
1547
1548    {
1549        let warm = dispatcher.clone();
1550        tokio::spawn(async move {
1551            warm.warm_all().await;
1552        });
1553    }
1554
1555    // The checkpoint task's own strong-count-based exit is unreachable
1556    // whenever `event_store_for_checkpoint()` returns `Some` (the ordinary
1557    // production shape), because the `SqlEventStore` it wraps retains its
1558    // own clone of the same pool. An explicit watch channel replaces that
1559    // mechanism: the sender is held for the remainder of this function's
1560    // scope and signalled as the first action once shutdown is observed,
1561    // below.
1562    let (checkpoint_shutdown_tx, checkpoint_shutdown_rx) = tokio::sync::watch::channel(());
1563    // ADR-091 Amendment 3: one checkpoint task per file-backed backend the
1564    // dispatcher wired — the primary pool plus every entry
1565    // `secondary_pools_for_checkpoint` returns — sharing this one shutdown
1566    // channel (the sender broadcasts to every receiver clone), so the single
1567    // send below stops every spawned task before `drain()`.
1568    let checkpoint_tasks = checkpoint_task_specs(
1569        dispatcher.pool_for_checkpoint(),
1570        dispatcher.secondary_pools_for_checkpoint(),
1571        dispatcher.event_store_for_checkpoint(),
1572        dispatcher.namespace().to_string(),
1573    );
1574    if !checkpoint_tasks.is_empty() {
1575        let cfg = CheckpointConfig::from_env();
1576        let checkpoint_task_count = checkpoint_tasks.len();
1577        for task in checkpoint_tasks {
1578            track_background_task(run_checkpoint_task(
1579                task.pool,
1580                cfg.clone(),
1581                task.lifecycle_owner,
1582                checkpoint_shutdown_rx.clone(),
1583                task.is_main,
1584            ));
1585        }
1586        tracing::info!(checkpoint_task_count, "WAL checkpoint task(s) started");
1587    }
1588
1589    let active = Arc::new(std::sync::atomic::AtomicUsize::new(0));
1590
1591    let shutdown = async {
1592        // REASON: signal handler registration can only fail if the global Tokio runtime
1593        // is not running or the OS rejects the signal number — both are unrecoverable
1594        // at this point in startup, so panic is the correct response.
1595        let mut sigterm = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
1596            .expect("install SIGTERM handler");
1597        let mut sigint = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::interrupt())
1598            .expect("install SIGINT handler");
1599        tokio::select! {
1600            _ = sigterm.recv() => tracing::info!("received SIGTERM"),
1601            _ = sigint.recv() => tracing::info!("received SIGINT"),
1602        }
1603    };
1604
1605    // SAFETY: `geteuid` is always successful and takes no arguments.
1606    let daemon_euid = unsafe { libc::geteuid() } as u32;
1607
1608    tokio::select! {
1609        _ = async {
1610            loop {
1611                match listener.accept().await {
1612                    Ok((stream, _)) => {
1613                        // Refuse a foreign uid before any frame is read.
1614                        // Fails CLOSED: an error reading peer credentials is
1615                        // "cannot prove same-uid", which is the same answer as
1616                        // "is not same-uid" — never a pass.
1617                        //
1618                        // Scope: this is a same-UID check, which is strictly
1619                        // weaker than same-PRINCIPAL. Several distinct actors
1620                        // running under one uid all pass here, so this does not
1621                        // by itself establish that the daemon serves a single
1622                        // principal, and nothing downstream refuses or degrades
1623                        // on observing more than one. Do not describe it as a
1624                        // multi-principal guard — it bounds the process boundary,
1625                        // not the identity one.
1626                        match peer_uid(&stream) {
1627                            Ok(peer) if uid_is_permitted(peer, daemon_euid) => {}
1628                            Ok(peer) => {
1629                                tracing::error!(
1630                                    peer_uid = peer,
1631                                    daemon_euid,
1632                                    "refusing connection from a foreign uid: this daemon accepts \
1633                                     only peers running as its own uid"
1634                                );
1635                                drop(stream);
1636                                continue;
1637                            }
1638                            Err(e) => {
1639                                tracing::error!(
1640                                    error = %e,
1641                                    "refusing connection: cannot read peer credentials, so \
1642                                     same-uid cannot be proven"
1643                                );
1644                                drop(stream);
1645                                continue;
1646                            }
1647                        }
1648                        let d = dispatcher.clone();
1649                        spawn_connection_task(Arc::clone(&active), async move {
1650                            handle_conn(stream, d).await;
1651                        });
1652                    }
1653                    Err(e) => tracing::error!(error = %e, "accept failed"),
1654                }
1655            }
1656        } => {}
1657        _ = shutdown => {}
1658    }
1659
1660    // Signal the checkpoint task to exit before draining, so `drain()`
1661    // actually waits on it via `track_background_task` rather than the
1662    // task outliving the drain window (or the process) unsignalled.
1663    let _ = checkpoint_shutdown_tx.send(());
1664
1665    // Same ordering contract for ADR-119 daemon components: cancel before
1666    // drain, so each component's supervisor (itself a tracked task) can run
1667    // its bounded shutdown inside the drain wait.
1668    daemon_shutdown_token().cancel();
1669
1670    drain(&active).await;
1671
1672    // A concurrent client's `kill_and_respawn` may have already decided
1673    // this daemon looked stale, killed it, and spawned a replacement that
1674    // bound the same socket/PID paths while this daemon was draining above.
1675    // Reacquire the recovery lock (the same one that serializes startup) and
1676    // only unlink if the PID file still names this process AND the socket at
1677    // `sock` is still the exact one this daemon bound — otherwise a
1678    // replacement daemon owns those paths now and unlinking would delete its
1679    // live socket/PID out from under it.
1680    match acquire_recovery_lock() {
1681        Some(_shutdown_lock) => {
1682            shutdown_cleanup_if_owned(&sock, &pid_file, bound_identity);
1683        }
1684        None => {
1685            tracing::warn!(
1686                "could not acquire recovery lock for shutdown cleanup; \
1687                 skipping unlink to avoid deleting a replacement daemon's paths"
1688            );
1689        }
1690    }
1691    tracing::info!("khived stopped");
1692    Ok(())
1693}
1694
1695/// Remove `sock`/`pid_file` only if they still belong to this process: the PID
1696/// file must name `std::process::id()` AND the socket currently at `sock` must
1697/// still be the exact one identified by `bound_identity` (dev/ino, not path).
1698///
1699/// Returns `true` if cleanup ran, `false` if it was skipped because a
1700/// replacement daemon already owns those paths. The caller must hold
1701/// the recovery lock across this call — the same lock daemon startup holds
1702/// across cleanup+bind+pid-write — so no replacement can bind between this
1703/// function's checks and its unlinks.
1704#[cfg(unix)]
1705fn shutdown_cleanup_if_owned(
1706    sock: &std::path::Path,
1707    pid_file: &std::path::Path,
1708    bound_identity: Option<SocketIdentity>,
1709) -> bool {
1710    let pid_is_ours = std::fs::read_to_string(pid_file)
1711        .ok()
1712        .and_then(|s| s.trim().parse::<u32>().ok())
1713        == Some(std::process::id());
1714    let socket_is_ours = bound_identity.is_some() && socket_identity(sock) == bound_identity;
1715    if pid_is_ours && socket_is_ours {
1716        let _ = std::fs::remove_file(sock);
1717        let _ = std::fs::remove_file(pid_file);
1718        true
1719    } else {
1720        tracing::warn!(
1721            socket = ?sock,
1722            pid_file = ?pid_file,
1723            "skipping shutdown cleanup — a replacement daemon already owns this socket/PID"
1724        );
1725        false
1726    }
1727}
1728
1729// ── helpers ───────────────────────────────────────────────────────────────────
1730
1731/// Liveness verdict for a `kill(pid, 0)` probe.
1732#[cfg(unix)]
1733#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1734enum PidLiveness {
1735    /// errno 0 — signal delivery succeeded, the process exists and this
1736    /// caller may signal it.
1737    Alive,
1738    /// ESRCH (or any other non-EPERM errno) — no such process.
1739    Dead,
1740    /// EPERM — the process exists but this caller lacks permission to
1741    /// signal it. Unknown-safe: treated as running so stale-daemon cleanup
1742    /// never unlinks a live daemon's socket/PID file just because it is
1743    /// owned by a different user/uid.
1744    PermissionDenied,
1745}
1746
1747#[cfg(unix)]
1748impl PidLiveness {
1749    fn is_running(self) -> bool {
1750        !matches!(self, PidLiveness::Dead)
1751    }
1752}
1753
1754/// Maps a `kill(pid, 0)` outcome (return code + errno) to a [`PidLiveness`].
1755/// Pure and side-effect-free so the errno mapping can be unit tested without
1756/// a real process probe.
1757#[cfg(unix)]
1758fn classify_kill_result(rc: i32, errno: i32) -> PidLiveness {
1759    if rc == 0 {
1760        return PidLiveness::Alive;
1761    }
1762    match errno {
1763        libc::EPERM => PidLiveness::PermissionDenied,
1764        _ => PidLiveness::Dead,
1765    }
1766}
1767
1768#[cfg(unix)]
1769fn is_process_running(pid: u32) -> bool {
1770    let Ok(pid) = i32::try_from(pid) else {
1771        return false;
1772    };
1773    if pid <= 0 {
1774        return false;
1775    }
1776    // SAFETY: signal 0 is an existence/permission probe with no side effects.
1777    let rc = unsafe { libc::kill(pid, 0) };
1778    let errno = std::io::Error::last_os_error().raw_os_error().unwrap_or(0);
1779    classify_kill_result(rc, errno).is_running()
1780}
1781
1782#[cfg(unix)]
1783async fn cleanup_stale_daemon(sock: &std::path::Path, pid_file: &std::path::Path) -> bool {
1784    if let Ok(pid_str) = std::fs::read_to_string(pid_file) {
1785        if let Ok(pid) = pid_str.trim().parse::<u32>() {
1786            if pid != std::process::id()
1787                && is_process_running(pid)
1788                && sock.exists()
1789                && UnixStream::connect(sock).await.is_ok()
1790            {
1791                return false;
1792            }
1793        }
1794    }
1795    if sock.exists() {
1796        if let Err(e) = std::fs::remove_file(sock) {
1797            tracing::warn!(error = %e, path = ?sock, "failed to remove stale socket");
1798        }
1799    }
1800    if pid_file.exists() {
1801        if let Err(e) = std::fs::remove_file(pid_file) {
1802            tracing::warn!(error = %e, path = ?pid_file, "failed to remove stale PID file");
1803        }
1804    }
1805    true
1806}
1807
1808/// Create `pid_file` exclusively (`O_EXCL`) and write this process's PID.
1809///
1810/// Uses `create_new(true)` rather than `create(true).truncate(true)` so
1811/// this can never silently overwrite a PID file another process created —
1812/// under normal operation the boot lock already serializes cleanup → bind →
1813/// pid-write across processes, but that guarantee depends on
1814/// `acquire_recovery_lock` succeeding for every party. Exclusive creation is
1815/// the defense that holds even if the lock itself is unavailable to one side:
1816/// the loser observes `ErrorKind::AlreadyExists` instead of clobbering the
1817/// winner's PID out from under it.
1818#[cfg(unix)]
1819fn write_pid_file_exclusive(pid_file: &std::path::Path) -> std::io::Result<()> {
1820    use std::os::unix::fs::OpenOptionsExt;
1821
1822    let mut opts = std::fs::OpenOptions::new();
1823    opts.write(true).create_new(true).mode(0o600);
1824    let mut f = opts.open(pid_file)?;
1825    f.write_all(std::process::id().to_string().as_bytes())?;
1826    Ok(())
1827}
1828
1829/// Return `true` if `pid_file` currently names a different, live process that
1830/// still answers on `sock` — i.e. a daemon already owns this rendezvous and it
1831/// is safe to defer to it rather than treat the `AlreadyExists` PID-file
1832/// collision as a boot failure.
1833#[cfg(unix)]
1834async fn pid_file_names_a_reachable_daemon(
1835    pid_file: &std::path::Path,
1836    sock: &std::path::Path,
1837) -> bool {
1838    let Ok(pid_str) = std::fs::read_to_string(pid_file) else {
1839        return false;
1840    };
1841    let Ok(pid) = pid_str.trim().parse::<u32>() else {
1842        return false;
1843    };
1844    pid != std::process::id()
1845        && is_process_running(pid)
1846        && sock.exists()
1847        && UnixStream::connect(sock).await.is_ok()
1848}
1849
1850#[cfg(unix)]
1851async fn drain(active: &std::sync::atomic::AtomicUsize) {
1852    use std::sync::atomic::Ordering;
1853    let remaining = || active.load(Ordering::Relaxed) + background_task_count();
1854    if remaining() == 0 {
1855        return;
1856    }
1857    let deadline = tokio::time::Instant::now() + drain_timeout();
1858    while remaining() > 0 {
1859        if tokio::time::Instant::now() >= deadline {
1860            tracing::warn!(
1861                remaining_connections = active.load(Ordering::Relaxed),
1862                remaining_background_tasks = background_task_count(),
1863                "drain timeout reached; forcing shutdown"
1864            );
1865            break;
1866        }
1867        tokio::time::sleep(std::time::Duration::from_millis(100)).await;
1868    }
1869}
1870
1871/// The bound `drain()` waits for tracked background tasks at daemon shutdown
1872/// (`KHIVE_DRAIN_TIMEOUT_SECS`, default 10s). Public so component supervision
1873/// can clamp per-component shutdown timeouts against it — a component timeout
1874/// longer than the drain bound could never complete its abort/state
1875/// transition before the daemon returns.
1876pub fn drain_timeout() -> std::time::Duration {
1877    let secs = std::env::var("KHIVE_DRAIN_TIMEOUT_SECS")
1878        .ok()
1879        .and_then(|v| v.parse::<u64>().ok())
1880        .unwrap_or(DEFAULT_DRAIN_TIMEOUT_SECS);
1881    std::time::Duration::from_secs(secs)
1882}
1883
1884/// Returns `true` for non-empty env values that are not `"0"` or `"false"`.
1885#[cfg(unix)]
1886pub fn env_truthy(key: &str) -> bool {
1887    std::env::var(key)
1888        .map(|v| {
1889            let v = v.trim();
1890            !v.is_empty() && v != "0" && !v.eq_ignore_ascii_case("false")
1891        })
1892        .unwrap_or(false)
1893}
1894
1895#[cfg(test)]
1896mod khive_root_tests {
1897    use super::khive_root_from;
1898    use std::path::PathBuf;
1899
1900    #[test]
1901    fn home_wins_when_set() {
1902        assert_eq!(
1903            khive_root_from(Some("/base/home".into()), Some("/other".into())),
1904            PathBuf::from("/base/home/.khive")
1905        );
1906    }
1907
1908    #[test]
1909    fn userprofile_backfills_missing_home() {
1910        assert_eq!(
1911            khive_root_from(None, Some("/profile/home".into())),
1912            PathBuf::from("/profile/home/.khive")
1913        );
1914    }
1915
1916    #[test]
1917    fn blank_values_are_skipped() {
1918        assert_eq!(
1919            khive_root_from(Some("  ".into()), Some("/profile/home".into())),
1920            PathBuf::from("/profile/home/.khive")
1921        );
1922    }
1923
1924    #[cfg(not(unix))]
1925    #[test]
1926    fn fallback_is_never_cwd_relative() {
1927        // On non-unix the lock anchor must not depend on the caller's working
1928        // directory even when no home variable is set — a relative path here
1929        // would give the same database different lock files per cwd.
1930        assert!(khive_root_from(None, None).is_absolute());
1931    }
1932
1933    #[cfg(unix)]
1934    #[test]
1935    fn unix_fallback_stays_historical_not_shared_tmp() {
1936        // On unix the no-HOME last resort deliberately stays "./.khive"
1937        // rather than a shared world-writable anchor like /tmp, which a
1938        // local attacker could pre-claim to intercept the daemon socket.
1939        assert_eq!(khive_root_from(None, None), PathBuf::from("./.khive"));
1940    }
1941}
1942
1943#[cfg(all(test, unix))]
1944mod tests {
1945    use super::*;
1946    use serial_test::serial;
1947
1948    #[tokio::test]
1949    #[serial(checkpoint_skip_metrics)]
1950    async fn secondary_only_checkpoint_topology_emits_lifecycle_outcome() {
1951        let main_backend = khive_db::StorageBackend::memory().expect("in-memory main backend");
1952        let event_store = main_backend.events().expect("main event store");
1953        let secondary_dir = tempfile::tempdir().expect("secondary tempdir");
1954        let secondary_backend =
1955            khive_db::StorageBackend::sqlite(secondary_dir.path().join("secondary.db"))
1956                .expect("file-backed secondary backend");
1957
1958        let mut tasks = checkpoint_task_specs(
1959            None,
1960            vec![secondary_backend.pool_arc()],
1961            Some(Arc::clone(&event_store)),
1962            "local".to_string(),
1963        );
1964        assert_eq!(tasks.len(), 1);
1965        let task = tasks.pop().expect("one secondary checkpoint task");
1966        assert!(!task.is_main, "the only checkpoint task must be secondary");
1967        assert!(
1968            task.lifecycle_owner.is_some(),
1969            "the secondary task must own lifecycle emission when no main task exists"
1970        );
1971
1972        let config = CheckpointConfig {
1973            interval: std::time::Duration::from_millis(10),
1974            warn_pages: 0,
1975            ..CheckpointConfig::default()
1976        };
1977        let (shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(());
1978        let handle = tokio::spawn(run_checkpoint_task(
1979            task.pool,
1980            config,
1981            task.lifecycle_owner,
1982            shutdown_rx,
1983            task.is_main,
1984        ));
1985
1986        tokio::time::sleep(std::time::Duration::from_millis(60)).await;
1987        shutdown_tx.send(()).expect("send checkpoint shutdown");
1988        tokio::time::timeout(std::time::Duration::from_secs(1), handle)
1989            .await
1990            .expect("checkpoint task should exit within 1s")
1991            .expect("checkpoint task panicked");
1992
1993        let events = event_store
1994            .query_events(
1995                khive_storage::EventFilter::default(),
1996                khive_storage::PageRequest {
1997                    limit: 100,
1998                    offset: 0,
1999                },
2000            )
2001            .await
2002            .expect("query lifecycle events");
2003        assert!(
2004            !events.items.is_empty()
2005                && events
2006                    .items
2007                    .iter()
2008                    .all(|event| event.kind == khive_types::EventKind::CheckpointOutcomeRecorded),
2009            "the designated secondary owner must emit CheckpointOutcomeRecorded"
2010        );
2011
2012        let file_main_dir = tempfile::tempdir().expect("file-backed main tempdir");
2013        let file_main = khive_db::StorageBackend::sqlite(file_main_dir.path().join("main.db"))
2014            .expect("file-backed main backend");
2015        let tasks = checkpoint_task_specs(
2016            Some(file_main.pool_arc()),
2017            vec![secondary_backend.pool_arc()],
2018            Some(event_store),
2019            "local".to_string(),
2020        );
2021        assert!(tasks[0].is_main && tasks[0].lifecycle_owner.is_some());
2022        assert!(!tasks[1].is_main && tasks[1].lifecycle_owner.is_none());
2023    }
2024
2025    // Focused regression tests for the unsafe process probe (SAFETY: signal 0
2026    // is an existence check with no side effects; see is_process_running).
2027
2028    #[test]
2029    fn current_process_is_running() {
2030        // The current PID is always alive.
2031        let pid = std::process::id();
2032        assert!(
2033            is_process_running(pid),
2034            "current process {pid} should be detected as running"
2035        );
2036    }
2037
2038    #[test]
2039    fn pid_zero_is_not_running() {
2040        // PID 0 is the process group; kill(0, 0) sends to the group,
2041        // which we treat as invalid — the guard `pid <= 0` must block it.
2042        assert!(
2043            !is_process_running(0),
2044            "pid 0 must be rejected by the guard before the unsafe call"
2045        );
2046    }
2047
2048    #[test]
2049    fn very_large_pid_is_not_running() {
2050        // u32::MAX overflows i32 — try_from returns Err, guard returns false.
2051        assert!(
2052            !is_process_running(u32::MAX),
2053            "u32::MAX should fail i32 conversion and return false"
2054        );
2055    }
2056
2057    // EPERM (process exists, no permission to signal it) must not be
2058    // misread as "not running" during stale-daemon cleanup.
2059
2060    #[test]
2061    fn classify_kill_result_zero_is_alive() {
2062        assert_eq!(classify_kill_result(0, 0), PidLiveness::Alive);
2063        assert!(classify_kill_result(0, 0).is_running());
2064    }
2065
2066    #[test]
2067    fn classify_kill_result_esrch_is_dead() {
2068        assert_eq!(classify_kill_result(-1, libc::ESRCH), PidLiveness::Dead);
2069        assert!(!classify_kill_result(-1, libc::ESRCH).is_running());
2070    }
2071
2072    #[test]
2073    fn classify_kill_result_eperm_is_permission_denied_and_counts_as_running() {
2074        assert_eq!(
2075            classify_kill_result(-1, libc::EPERM),
2076            PidLiveness::PermissionDenied
2077        );
2078        assert!(
2079            classify_kill_result(-1, libc::EPERM).is_running(),
2080            "EPERM must be unknown-safe: treated as running, never as a basis \
2081             for stale cleanup to unlink a live daemon's rendezvous files"
2082        );
2083    }
2084
2085    #[test]
2086    fn pid_1_probe_is_running_regardless_of_permission_outcome() {
2087        // PID 1 (init/launchd) always exists. An unprivileged process gets
2088        // EPERM signaling it (never ESRCH); running as root would get 0
2089        // instead. Either way `is_process_running` must report true — this
2090        // is the live regression guard for EPERM being misread as dead;
2091        // `classify_kill_result` above is the pure-function unit coverage
2092        // for the same mapping, kept independent of process ownership so
2093        // it is never flaky in CI.
2094        assert!(
2095            is_process_running(1),
2096            "PID 1 always exists; EPERM must not read as dead"
2097        );
2098    }
2099
2100    #[test]
2101    fn env_truthy_recognises_set_values() {
2102        assert!(!env_truthy("__KHIVE_TEST_ABSENT_VAR_XYZ__"));
2103
2104        // env_truthy with a live value — set and unset atomically to avoid
2105        // cross-test pollution (not parallel-safe without serial_test, but these
2106        // are fast unit tests and the variable name is unique).
2107        let key = "__KHIVE_TEST_TRUTHY_ABC__";
2108        std::env::set_var(key, "1");
2109        assert!(env_truthy(key));
2110        std::env::set_var(key, "false");
2111        assert!(!env_truthy(key));
2112        std::env::set_var(key, "0");
2113        assert!(!env_truthy(key));
2114        std::env::remove_var(key);
2115    }
2116
2117    #[tokio::test(flavor = "current_thread", start_paused = true)]
2118    #[serial(background_tasks)]
2119    async fn accepted_connection_is_counted_before_first_poll_and_drain_waits() {
2120        use std::sync::atomic::Ordering;
2121
2122        let active = Arc::new(std::sync::atomic::AtomicUsize::new(0));
2123        let started = Arc::new(std::sync::atomic::AtomicBool::new(false));
2124        let (release_tx, release_rx) = tokio::sync::oneshot::channel::<()>();
2125        let started_in_task = Arc::clone(&started);
2126
2127        let handle = spawn_connection_task(Arc::clone(&active), async move {
2128            started_in_task.store(true, Ordering::Relaxed);
2129            let _ = release_rx.await;
2130        });
2131
2132        assert_eq!(active.load(Ordering::Relaxed), 1);
2133        assert!(
2134            !started.load(Ordering::Relaxed),
2135            "the current-thread runtime must leave the spawned handler unpolled"
2136        );
2137
2138        let drain_fut = drain(active.as_ref());
2139        tokio::pin!(drain_fut);
2140        let too_early =
2141            tokio::time::timeout(std::time::Duration::from_millis(150), &mut drain_fut).await;
2142        assert!(
2143            too_early.is_err(),
2144            "drain must wait for a connection claimed before its task's first poll"
2145        );
2146        assert!(started.load(Ordering::Relaxed));
2147
2148        release_tx.send(()).expect("handler still waiting");
2149        tokio::time::timeout(std::time::Duration::from_secs(1), handle)
2150            .await
2151            .expect("handler should finish promptly")
2152            .expect("handler should not panic");
2153        assert_eq!(active.load(Ordering::Relaxed), 0);
2154        tokio::time::timeout(std::time::Duration::from_secs(1), drain_fut)
2155            .await
2156            .expect("drain should finish once the handler releases its claim");
2157    }
2158
2159    #[tokio::test(flavor = "current_thread")]
2160    async fn cancelled_connection_releases_count_before_first_poll() {
2161        use std::sync::atomic::Ordering;
2162
2163        let active = Arc::new(std::sync::atomic::AtomicUsize::new(0));
2164        let started = Arc::new(std::sync::atomic::AtomicBool::new(false));
2165        let started_in_task = Arc::clone(&started);
2166        let handle = spawn_connection_task(Arc::clone(&active), async move {
2167            started_in_task.store(true, Ordering::Relaxed);
2168            std::future::pending::<()>().await;
2169        });
2170
2171        assert_eq!(active.load(Ordering::Relaxed), 1);
2172        handle.abort();
2173        let error = handle.await.expect_err("aborted handler must be cancelled");
2174        assert!(error.is_cancelled());
2175        assert!(!started.load(Ordering::Relaxed));
2176        assert_eq!(active.load(Ordering::Relaxed), 0);
2177    }
2178
2179    #[tokio::test(flavor = "current_thread")]
2180    async fn panicked_connection_releases_count() {
2181        use std::sync::atomic::Ordering;
2182
2183        let active = Arc::new(std::sync::atomic::AtomicUsize::new(0));
2184        let handle = spawn_connection_task(Arc::clone(&active), async move {
2185            panic!("intentional connection-handler panic");
2186        });
2187
2188        assert_eq!(active.load(Ordering::Relaxed), 1);
2189        let error = handle
2190            .await
2191            .expect_err("panicked handler must fail its join");
2192        assert!(error.is_panic());
2193        assert_eq!(active.load(Ordering::Relaxed), 0);
2194    }
2195
2196    #[test]
2197    fn connection_claim_releases_if_spawn_panics() {
2198        use std::sync::atomic::Ordering;
2199
2200        let active = Arc::new(std::sync::atomic::AtomicUsize::new(0));
2201        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
2202            drop(spawn_connection_task(Arc::clone(&active), async {}));
2203        }));
2204
2205        assert!(result.is_err(), "tokio::spawn outside a runtime must panic");
2206        assert_eq!(active.load(Ordering::Relaxed), 0);
2207    }
2208
2209    #[tokio::test]
2210    #[serial(background_tasks)]
2211    async fn drain_returns_promptly_with_no_accepted_connection() {
2212        let active = std::sync::atomic::AtomicUsize::new(0);
2213        tokio::time::timeout(std::time::Duration::from_secs(1), drain(&active))
2214            .await
2215            .expect("empty drain should return immediately");
2216    }
2217
2218    // `drain()` must wait for tracked background tasks (e.g. memory.recall's
2219    // serve-ledger append), not just in-flight connections, or a SIGTERM
2220    // lands mid-flight with no log and no row.
2221    //
2222    // `#[serial(background_tasks)]`: this test reads/asserts on the
2223    // process-wide `BACKGROUND_TASKS` static shared with the two counter
2224    // tests below. Under default parallel execution one test's increment
2225    // leaks into another's snapshot-then-assert window (reproduced: both
2226    // counter tests failed together, passed with `--test-threads=1`).
2227    // Serializing just this named group isolates them from each other
2228    // without forcing the whole test binary (including unrelated
2229    // `#[serial]` tests elsewhere in this crate) onto one thread.
2230    #[tokio::test]
2231    #[serial(background_tasks)]
2232    async fn drain_waits_for_tracked_background_tasks_before_returning() {
2233        let active = std::sync::atomic::AtomicUsize::new(0);
2234        let (tx, rx) = tokio::sync::oneshot::channel::<()>();
2235
2236        track_background_task(async move {
2237            let _ = rx.await;
2238        });
2239        assert!(
2240            background_task_count() >= 1,
2241            "track_background_task must make the in-flight task visible immediately"
2242        );
2243
2244        let drain_fut = drain(&active);
2245        tokio::pin!(drain_fut);
2246
2247        // Must NOT resolve while the tracked task is still pending.
2248        let too_early =
2249            tokio::time::timeout(std::time::Duration::from_millis(150), &mut drain_fut).await;
2250        assert!(
2251            too_early.is_err(),
2252            "drain() must not return while a tracked background task is still running"
2253        );
2254
2255        // Completing the task must let drain() proceed promptly.
2256        tx.send(())
2257            .expect("tracked task still awaiting the oneshot");
2258        let done = tokio::time::timeout(std::time::Duration::from_secs(5), drain_fut).await;
2259        assert!(
2260            done.is_ok(),
2261            "drain() must return once the tracked background task finishes"
2262        );
2263    }
2264
2265    // See the `#[serial(background_tasks)]` note on
2266    // `drain_waits_for_tracked_background_tasks_before_returning` above —
2267    // this test shares the same process-wide `BACKGROUND_TASKS` static and
2268    // races it (and the panic test below) under default parallelism.
2269    #[tokio::test]
2270    #[serial(background_tasks)]
2271    async fn track_background_task_count_returns_to_zero_after_completion() {
2272        // Sanity check on the counter's own bookkeeping, independent of drain().
2273        let before = background_task_count();
2274        let (tx, rx) = tokio::sync::oneshot::channel::<()>();
2275        track_background_task(async move {
2276            let _ = rx.await;
2277        });
2278        assert_eq!(background_task_count(), before + 1);
2279        tx.send(()).expect("still awaiting");
2280        // Yield until the spawned task's decrement has actually run.
2281        for _ in 0..100 {
2282            if background_task_count() == before {
2283                break;
2284            }
2285            tokio::time::sleep(std::time::Duration::from_millis(10)).await;
2286        }
2287        assert_eq!(background_task_count(), before);
2288    }
2289
2290    // See the `#[serial(background_tasks)]` note above — shares
2291    // `BACKGROUND_TASKS` with the other two tests in this group.
2292    #[tokio::test]
2293    #[serial(background_tasks)]
2294    async fn track_background_task_count_returns_to_baseline_after_panic() {
2295        // A panic inside the tracked future must still decrement the
2296        // counter (via BackgroundTaskGuard's Drop), not leak it forever.
2297        // `track_background_task` discards the spawned
2298        // task's `JoinHandle` (it is fire-and-forget by design — the caller
2299        // never awaits it), so this test does not await the panic directly;
2300        // tokio isolates the panic to the spawned task instead of aborting
2301        // the process, and we observe the recovery purely through the
2302        // shared counter returning to baseline after the guard's `Drop`
2303        // fires during that task's unwind.
2304        let before = background_task_count();
2305
2306        let (tx, rx) = tokio::sync::oneshot::channel::<()>();
2307        track_background_task(async move {
2308            let _ = rx.await;
2309            panic!("intentional panic to exercise the Drop-guard decrement path");
2310        });
2311        assert_eq!(background_task_count(), before + 1);
2312
2313        tx.send(()).expect("still awaiting");
2314        for _ in 0..100 {
2315            if background_task_count() == before {
2316                break;
2317            }
2318            tokio::time::sleep(std::time::Duration::from_millis(10)).await;
2319        }
2320        assert_eq!(
2321            background_task_count(),
2322            before,
2323            "background task counter must return to baseline after the tracked future panics"
2324        );
2325    }
2326
2327    // ── active background phase names (ADR-103) ──────────────────────────
2328
2329    // `#[serial(active_phases)]`: these tests read/assert on the process-wide
2330    // `ACTIVE_PHASES` static. No other test in this crate touches it today,
2331    // but the group mirrors the `background_tasks` precedent above so a
2332    // future addition does not silently reintroduce the same interleaving
2333    // hazard that motivated it there.
2334    #[test]
2335    #[serial(active_phases)]
2336    fn register_active_phase_appears_and_disappears_with_the_guard() {
2337        assert!(
2338            !active_phase_names().contains(&"adr103_test_phase".to_string()),
2339            "must start absent (leaked from a prior failed run would poison this test)"
2340        );
2341
2342        let guard = register_active_phase("adr103_test_phase");
2343        assert!(active_phase_names().contains(&"adr103_test_phase".to_string()));
2344
2345        drop(guard);
2346        assert!(
2347            !active_phase_names().contains(&"adr103_test_phase".to_string()),
2348            "the phase name must drop out of the gauge once its guard is dropped"
2349        );
2350    }
2351
2352    #[test]
2353    #[serial(active_phases)]
2354    fn register_active_phase_counts_concurrent_occurrences_of_the_same_name() {
2355        let first = register_active_phase("adr103_concurrent_phase");
2356        let second = register_active_phase("adr103_concurrent_phase");
2357        assert!(active_phase_names().contains(&"adr103_concurrent_phase".to_string()));
2358
2359        drop(first);
2360        assert!(
2361            active_phase_names().contains(&"adr103_concurrent_phase".to_string()),
2362            "one of two concurrent occurrences ending must not remove the name early"
2363        );
2364
2365        drop(second);
2366        assert!(
2367            !active_phase_names().contains(&"adr103_concurrent_phase".to_string()),
2368            "the name must be removed only once every concurrent occurrence has ended"
2369        );
2370    }
2371
2372    // ── metrics-only frame (load/perf harness read-surface) ────────────────
2373
2374    /// Minimal `DaemonDispatch` for the metrics tests: `dispatch` just counts
2375    /// how many times it was called (so tests can assert the ops path was
2376    /// never reached) and `pool_for_checkpoint` returns whatever pool the
2377    /// test wired in (or `None`, matching an in-memory/poolless dispatcher).
2378    #[derive(Clone)]
2379    struct MockDispatch {
2380        namespace: String,
2381        config_id: String,
2382        dispatch_calls: Arc<std::sync::atomic::AtomicUsize>,
2383        pool: Option<Arc<ConnectionPool>>,
2384        /// When `Some(msg)`, `dispatch` returns `Err(msg)` instead of the
2385        /// default `Ok("{}")` — lets a test drive `handle_conn`'s real
2386        /// dispatch-error arm (khive#948 request_id echo coverage).
2387        dispatch_err: Option<String>,
2388    }
2389
2390    #[async_trait]
2391    impl DaemonDispatch for MockDispatch {
2392        async fn dispatch(
2393            &self,
2394            _ops: String,
2395            _presentation: Option<String>,
2396            _presentation_per_op: Option<Vec<Option<String>>>,
2397            _format: Option<String>,
2398            _format_per_op: Option<Vec<Option<String>>>,
2399            _from_wire: bool,
2400            _identity: Option<RequestIdentity>,
2401        ) -> Result<String, String> {
2402            self.dispatch_calls
2403                .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
2404            match &self.dispatch_err {
2405                Some(msg) => Err(msg.clone()),
2406                None => Ok("{}".to_string()),
2407            }
2408        }
2409
2410        async fn warm_all(&self) {}
2411
2412        fn namespace(&self) -> &str {
2413            &self.namespace
2414        }
2415
2416        fn config_id(&self) -> &str {
2417            &self.config_id
2418        }
2419
2420        fn pool_for_checkpoint(&self) -> Option<Arc<ConnectionPool>> {
2421            self.pool.clone()
2422        }
2423    }
2424
2425    fn base_request_frame(config_id: &str) -> DaemonRequestFrame {
2426        DaemonRequestFrame {
2427            ops: String::new(),
2428            presentation: None,
2429            presentation_per_op: None,
2430            namespace: "local".to_string(),
2431            actor_id: None,
2432            process_ref: None,
2433            visible_namespaces: Vec::new(),
2434            config_id: config_id.to_string(),
2435            protocol_version: PROTOCOL_VERSION,
2436            probe_only: false,
2437            metrics_only: false,
2438            format: None,
2439            format_per_op: None,
2440            from_wire: false,
2441            request_id: None,
2442        }
2443    }
2444
2445    /// Drive `handle_conn` over an in-process `UnixStream::pair()` (no real
2446    /// socket file needed) and decode the response frame it writes back.
2447    async fn round_trip(dispatcher: MockDispatch, req: &DaemonRequestFrame) -> DaemonResponseFrame {
2448        let (mut client, server) = UnixStream::pair().expect("unix stream pair");
2449        let payload = serde_json::to_vec(req).expect("encode request frame");
2450        let handle = tokio::spawn(async move {
2451            handle_conn(server, dispatcher).await;
2452        });
2453        write_frame(&mut client, &payload)
2454            .await
2455            .expect("write request frame");
2456        let raw = read_frame(&mut client).await.expect("read response frame");
2457        handle.await.expect("handle_conn task panicked");
2458        serde_json::from_slice(&raw).expect("decode response frame")
2459    }
2460
2461    /// Protocol v4 makes `process_ref` part of dispatch semantics. A still-warm
2462    /// v3 daemon/client pairing must fail before the verb runs; otherwise the
2463    /// older peer can ignore the unknown field, persist a message without the
2464    /// requested provenance, and leave the caller unable to retry safely.
2465    #[tokio::test]
2466    async fn protocol_v3_frame_is_rejected_before_process_ref_dispatch() {
2467        assert_eq!(PROTOCOL_VERSION, 4, "process_ref is the protocol-v4 change");
2468        let dispatch_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
2469        let dispatcher = MockDispatch {
2470            namespace: "local".to_string(),
2471            config_id: "cfg-v4".to_string(),
2472            dispatch_calls: Arc::clone(&dispatch_calls),
2473            pool: None,
2474            dispatch_err: None,
2475        };
2476        let mut request = base_request_frame("cfg-v4");
2477        request.protocol_version = 3;
2478        request.process_ref = Some("worker/legacy-rollout".to_string());
2479
2480        let response = round_trip(dispatcher, &request).await;
2481        assert!(!response.ok);
2482        assert!(response.version_mismatch);
2483        assert_eq!(response.daemon_protocol_version, 4);
2484        assert_eq!(
2485            dispatch_calls.load(std::sync::atomic::Ordering::SeqCst),
2486            0,
2487            "a v3 frame must be rejected before a provenance-bearing mutation dispatches"
2488        );
2489        let error = response.error.expect("mismatch explains both versions");
2490        assert!(
2491            error.contains("client=3") && error.contains("daemon=4"),
2492            "mismatch must identify the exact rollout boundary; got {error:?}"
2493        );
2494    }
2495
2496    /// Test 1: a `metrics_only: true` request
2497    /// returns `metrics: Some(_)` and never reaches the ops-dispatch path; a
2498    /// normal request (the default `metrics_only: false`) still dispatches
2499    /// exactly as before and carries no metrics. Also proves `metrics_only`
2500    /// bypasses the `config_id` equality reject (a gauge read is
2501    /// process-global, not namespaced to a particular client config).
2502    #[tokio::test]
2503    async fn metrics_only_frame_returns_snapshot_without_dispatching() {
2504        let dispatch_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
2505        let dispatcher = MockDispatch {
2506            namespace: "local".to_string(),
2507            config_id: "cfg-a".to_string(),
2508            dispatch_calls: Arc::clone(&dispatch_calls),
2509            pool: None,
2510            dispatch_err: None,
2511        };
2512
2513        let mut metrics_req = base_request_frame("cfg-a");
2514        metrics_req.metrics_only = true;
2515        let metrics_resp = round_trip(dispatcher.clone(), &metrics_req).await;
2516
2517        assert!(metrics_resp.ok, "metrics_only response must be ok=true");
2518        assert!(
2519            metrics_resp.metrics.is_some(),
2520            "metrics_only=true must return Some(snapshot)"
2521        );
2522        assert_eq!(
2523            dispatch_calls.load(std::sync::atomic::Ordering::SeqCst),
2524            0,
2525            "metrics_only must never reach the ops-dispatch path"
2526        );
2527
2528        // metrics_only bypasses the config_id equality reject.
2529        let mut mismatched_req = base_request_frame("some-other-config");
2530        mismatched_req.metrics_only = true;
2531        let mismatched_resp = round_trip(dispatcher.clone(), &mismatched_req).await;
2532        assert!(mismatched_resp.ok);
2533        assert!(mismatched_resp.metrics.is_some());
2534        assert!(!mismatched_resp.config_mismatch);
2535        assert_eq!(
2536            dispatch_calls.load(std::sync::atomic::Ordering::SeqCst),
2537            0,
2538            "a mismatched-config metrics_only request must still skip dispatch"
2539        );
2540
2541        // A normal request (default metrics_only=false) is unaffected: it
2542        // still dispatches and carries no metrics.
2543        let normal_req = base_request_frame("cfg-a");
2544        let normal_resp = round_trip(dispatcher, &normal_req).await;
2545        assert!(normal_resp.ok);
2546        assert!(normal_resp.metrics.is_none());
2547        assert_eq!(dispatch_calls.load(std::sync::atomic::Ordering::SeqCst), 1);
2548    }
2549
2550    /// Group/other-writable socket directories are refused whether or not the
2551    /// sticky bit is set, and — the part that matters — are left exactly as
2552    /// they were found.
2553    ///
2554    /// The sticky `/tmp` shape (1777) is in the refusal set deliberately: the
2555    /// sticky bit restricts unlinking, not creating, so a shared directory
2556    /// lets another user pre-bind the predictable socket path before this
2557    /// daemon starts; and a sticky directory's owner may unlink and rebind
2558    /// regardless. Re-permissioning someone else's directory on the way past
2559    /// is what the second assertion pins against: run as a user who *can*
2560    /// chmod it, the old unconditional call succeeded and revoked access for
2561    /// every other process using that directory.
2562    #[test]
2563    fn shared_writable_socket_dirs_are_refused_without_being_modified() {
2564        let dir = tempfile::tempdir().expect("tempdir");
2565        for (name, mode) in [("open", 0o777u32), ("sticky-tmp", 0o1777u32)] {
2566            let shared = dir.path().join(name);
2567            std::fs::create_dir(&shared).expect("create");
2568            std::fs::set_permissions(&shared, std::fs::Permissions::from_mode(mode))
2569                .expect("chmod");
2570
2571            let err = ensure_socket_dir_is_trusted(&shared)
2572                .expect_err("group/other-writable must be refused, sticky or not");
2573            assert!(
2574                err.to_string().contains(&format!("{:04o}", mode & 0o7777)),
2575                "the refusal should name the mode it saw, got: {err}"
2576            );
2577            let after = std::fs::metadata(&shared)
2578                .expect("stat")
2579                .permissions()
2580                .mode()
2581                & 0o7777;
2582            assert_eq!(
2583                after, mode,
2584                "refusing must not re-permission a directory khive does not own"
2585            );
2586        }
2587    }
2588
2589    /// A vetted final directory is still refused when an ANCESTOR would let
2590    /// another local user rename it away and recreate it: a non-sticky
2591    /// group/other-writable ancestor re-roots the whole socket path without
2592    /// the final directory's own metadata ever changing. The sticky-ancestor
2593    /// arm (a root-owned 1777 `/tmp` above a user-owned 0700 directory is
2594    /// acceptable) is exercised implicitly by every accepting test below,
2595    /// whose tempdirs live under the platform temp root.
2596    #[test]
2597    fn writable_non_sticky_ancestor_is_refused() {
2598        let dir = tempfile::tempdir().expect("tempdir");
2599        let open_mid = dir.path().join("open-mid");
2600        std::fs::create_dir(&open_mid).expect("create mid");
2601        let inner = open_mid.join("private");
2602        std::fs::create_dir(&inner).expect("create inner");
2603        std::fs::set_permissions(&inner, std::fs::Permissions::from_mode(0o700)).expect("chmod");
2604        std::fs::set_permissions(&open_mid, std::fs::Permissions::from_mode(0o777))
2605            .expect("chmod mid");
2606
2607        let err = ensure_socket_dir_is_trusted(&inner)
2608            .expect_err("a 0777 non-sticky ancestor must be refused");
2609        assert!(
2610            err.to_string().contains("ancestor"),
2611            "the refusal should say it was an ancestor that failed, got: {err}"
2612        );
2613        assert!(
2614            err.to_string().contains("open-mid"),
2615            "the refusal should name the failing ancestor, got: {err}"
2616        );
2617    }
2618
2619    /// The path walk validates what a symlink component points AT, not just
2620    /// the symlink node: a trusted (self-owned) link into a group/other-
2621    /// writable non-sticky directory is refused on the target directory's
2622    /// metadata, proving the walk keeps traversing — and keeps applying the
2623    /// directory rules — past the link, exactly as the kernel will at bind
2624    /// time.
2625    #[test]
2626    fn symlink_component_to_untrusted_directory_is_refused() {
2627        let dir = tempfile::tempdir().expect("tempdir");
2628        let open = dir.path().join("open-target");
2629        std::fs::create_dir(&open).expect("create target");
2630        std::fs::set_permissions(&open, std::fs::Permissions::from_mode(0o777)).expect("chmod");
2631        let link = dir.path().join("link");
2632        std::os::unix::fs::symlink(&open, &link).expect("symlink");
2633
2634        // SAFETY: `geteuid` is always successful and takes no arguments.
2635        let euid = unsafe { libc::geteuid() } as u32;
2636        let err = ensure_socket_path_is_swap_resistant(&link, euid)
2637            .expect_err("a link into a 0777 non-sticky directory must be refused");
2638        assert!(
2639            err.to_string().contains("open-target"),
2640            "the refusal should name the untrusted target directory, got: {err}"
2641        );
2642    }
2643
2644    /// A symlink component owned by neither the daemon euid nor root is
2645    /// refused on the symlink itself: its owner can retarget it after
2646    /// validation, so where it currently points is irrelevant. A non-root
2647    /// test cannot create a foreign-owned symlink, so this injects a
2648    /// mismatched euid instead — and the fixture must live under an
2649    /// all-root-owned chain (the platform `/tmp`), because under the test's
2650    /// own tempdir the self-owned ancestor directories would already fail
2651    /// the injected euid before the walk ever reached the link.
2652    #[test]
2653    fn foreign_owned_symlink_component_is_refused() {
2654        let link =
2655            std::path::PathBuf::from(format!("/tmp/khive-swaptest-link-{}", std::process::id()));
2656        let _ = std::fs::remove_file(&link);
2657        std::os::unix::fs::symlink("/tmp", &link).expect("symlink");
2658
2659        // SAFETY: `geteuid` is always successful and takes no arguments.
2660        let not_my_euid = (unsafe { libc::geteuid() } as u32).wrapping_add(1);
2661        let result = ensure_socket_path_is_swap_resistant(&link, not_my_euid);
2662        std::fs::remove_file(&link).expect("cleanup");
2663
2664        let err = result.expect_err("a symlink owned by another uid must be refused");
2665        assert!(
2666            err.to_string().contains("symlink component"),
2667            "the refusal should strike the symlink itself, got: {err}"
2668        );
2669        assert!(
2670            err.to_string().contains("khive-swaptest-link"),
2671            "the refusal should name the offending link, got: {err}"
2672        );
2673    }
2674
2675    /// Directories this daemon can trust end to end are served as found:
2676    /// owner-only, and the umask-default 0755 every `tempfile::tempdir` and
2677    /// test runner produces (readable/traversable but writable only by the
2678    /// owner). On macOS these fixtures also implicitly exercise the
2679    /// symlink-accept arm, since the platform temp root itself resolves
2680    /// through root-owned symlinks. Foreign ownership of a directory is
2681    /// refused by the same helper, but a non-root test cannot chown a
2682    /// directory away from itself, so that arm is exercised by the mode
2683    /// checks' shared fail-closed path rather than a dedicated fixture.
2684    #[test]
2685    fn trusted_socket_dirs_are_accepted_unmodified() {
2686        let dir = tempfile::tempdir().expect("tempdir");
2687        for (name, mode) in [("private", 0o700), ("listable", 0o755)] {
2688            let d = dir.path().join(name);
2689            std::fs::create_dir(&d).expect("create");
2690            std::fs::set_permissions(&d, std::fs::Permissions::from_mode(mode)).expect("chmod");
2691
2692            ensure_socket_dir_is_trusted(&d)
2693                .unwrap_or_else(|e| panic!("mode {mode:04o} must be accepted, got: {e}"));
2694
2695            let after = std::fs::metadata(&d).expect("stat").permissions().mode() & 0o7777;
2696            assert_eq!(
2697                after, mode,
2698                "acceptance must not re-permission the directory either"
2699            );
2700        }
2701    }
2702
2703    /// Test 2: `wal_pages` reflects a real
2704    /// checkpoint observation after writes, deterministically forced via a
2705    /// direct `checkpoint_once` call rather than waiting on the async
2706    /// periodic task.
2707    #[tokio::test]
2708    async fn metrics_snapshot_wal_pages_reflects_recent_write() {
2709        let dir = tempfile::tempdir().expect("tempdir");
2710        let path = dir.path().join("metrics_wal_test.db");
2711        let pool = Arc::new(
2712            ConnectionPool::new(khive_db::PoolConfig {
2713                path: Some(path),
2714                ..khive_db::PoolConfig::default()
2715            })
2716            .expect("pool open"),
2717        );
2718
2719        {
2720            let writer = pool.try_writer().expect("writer");
2721            writer
2722                .conn()
2723                .execute_batch(
2724                    "CREATE TABLE t (x INTEGER); \
2725                     INSERT INTO t VALUES (1); \
2726                     INSERT INTO t VALUES (2);",
2727                )
2728                .expect("seed writes");
2729        }
2730
2731        let tick = khive_db::checkpoint_once(
2732            &pool,
2733            &CheckpointConfig::default(),
2734            &mut khive_db::checkpoint::TruncateState::default(),
2735        );
2736        assert!(
2737            matches!(tick, khive_db::CheckpointTick::Observed(_)),
2738            "checkpoint_once on a freshly-writer-held pool must observe, not skip: {tick:?}"
2739        );
2740
2741        let dispatcher = MockDispatch {
2742            namespace: "local".to_string(),
2743            config_id: "cfg-wal".to_string(),
2744            dispatch_calls: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
2745            pool: Some(pool),
2746            dispatch_err: None,
2747        };
2748
2749        let snapshot = build_metrics_snapshot(&dispatcher);
2750        assert!(
2751            snapshot.wal_pages.is_some(),
2752            "wal_pages must be observed after a real checkpoint tick, got {snapshot:?}"
2753        );
2754        // The snapshot carries the checkpoint-pressure fields read-only
2755        // (no mutation path reachable through `MetricsSnapshot`/`DaemonRequestFrame`);
2756        // an observed tick (not a skip) must report a zero-length skip streak.
2757        assert_eq!(
2758            snapshot.wal_checkpoint_consecutive_skips, 0,
2759            "an observed (non-skipped) tick must report zero consecutive skips, got {snapshot:?}"
2760        );
2761    }
2762
2763    /// Test 3: the tx-pin oracle. The registry is process-global, so an
2764    /// unrelated transaction can depart between snapshots and exactly offset
2765    /// this test's registration. Keep an owned handle live and assert the
2766    /// resulting count floor instead of comparing two points in time.
2767    #[test]
2768    #[serial(tx_registry)]
2769    fn metrics_snapshot_reflects_open_transaction_registry() {
2770        let dispatcher = MockDispatch {
2771            namespace: "local".to_string(),
2772            config_id: "cfg-tx".to_string(),
2773            dispatch_calls: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
2774            pool: None,
2775            dispatch_err: None,
2776        };
2777
2778        let departing_handle = khive_storage::tx_registry::register(Some(
2779            "daemon_metrics_snapshot_departing_test_tx".to_string(),
2780        ));
2781        let before = build_metrics_snapshot(&dispatcher).open_tx_count;
2782        assert!(before >= 1);
2783
2784        let handle = khive_storage::tx_registry::register(Some(
2785            "daemon_metrics_snapshot_owned_test_tx".to_string(),
2786        ));
2787        drop(departing_handle);
2788
2789        let during = build_metrics_snapshot(&dispatcher);
2790        assert!(
2791            during.open_tx_count >= 1,
2792            "open_tx_count must reflect the live owned transaction despite registry churn: \
2793             churn_baseline={before} during={}",
2794            during.open_tx_count
2795        );
2796        assert!(
2797            during.oldest_pinned_tx_micros.is_some(),
2798            "oldest_pinned_tx_micros must be Some while a transaction is open"
2799        );
2800
2801        drop(handle);
2802        assert!(
2803            !khive_storage::tx_registry::snapshot()
2804                .iter()
2805                .any(|(_, label)| label.as_deref()
2806                    == Some("daemon_metrics_snapshot_owned_test_tx")),
2807            "the owned registry entry must disappear when its handle is dropped"
2808        );
2809    }
2810
2811    /// Test 4: write-queue depth is flag-gated
2812    /// on `PoolConfig::write_queue_enabled` (the `KHIVE_WRITE_QUEUE=1`
2813    /// setting), never on a specific depth value (racy under concurrency).
2814    #[tokio::test]
2815    async fn metrics_snapshot_write_queue_depth_flag_gated() {
2816        let dir = tempfile::tempdir().expect("tempdir");
2817
2818        let enabled_pool = Arc::new(
2819            ConnectionPool::new(khive_db::PoolConfig {
2820                path: Some(dir.path().join("wq_enabled.db")),
2821                write_queue_enabled: true,
2822                ..khive_db::PoolConfig::default()
2823            })
2824            .expect("pool open"),
2825        );
2826        let enabled_dispatcher = MockDispatch {
2827            namespace: "local".to_string(),
2828            config_id: "cfg-wq-on".to_string(),
2829            dispatch_calls: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
2830            pool: Some(enabled_pool),
2831            dispatch_err: None,
2832        };
2833        let snapshot_on = build_metrics_snapshot(&enabled_dispatcher);
2834        assert!(
2835            snapshot_on.write_queue_depth.is_some(),
2836            "write_queue_depth must be Some when write_queue_enabled=true, got {snapshot_on:?}"
2837        );
2838        assert!(snapshot_on.write_queue_capacity.is_some());
2839
2840        let disabled_pool = Arc::new(
2841            ConnectionPool::new(khive_db::PoolConfig {
2842                path: Some(dir.path().join("wq_disabled.db")),
2843                write_queue_enabled: false,
2844                ..khive_db::PoolConfig::default()
2845            })
2846            .expect("pool open"),
2847        );
2848        let disabled_dispatcher = MockDispatch {
2849            namespace: "local".to_string(),
2850            config_id: "cfg-wq-off".to_string(),
2851            dispatch_calls: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
2852            pool: Some(disabled_pool),
2853            dispatch_err: None,
2854        };
2855        let snapshot_off = build_metrics_snapshot(&disabled_dispatcher);
2856        assert!(
2857            snapshot_off.write_queue_depth.is_none(),
2858            "write_queue_depth must be None when write_queue_enabled=false, got {snapshot_off:?}"
2859        );
2860        assert!(snapshot_off.write_queue_capacity.is_none());
2861
2862        // No pool at all (in-memory/poolless dispatcher): also None.
2863        let no_pool_dispatcher = MockDispatch {
2864            namespace: "local".to_string(),
2865            config_id: "cfg-no-pool".to_string(),
2866            dispatch_calls: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
2867            pool: None,
2868            dispatch_err: None,
2869        };
2870        let snapshot_no_pool = build_metrics_snapshot(&no_pool_dispatcher);
2871        assert!(snapshot_no_pool.write_queue_depth.is_none());
2872        assert!(snapshot_no_pool.write_queue_capacity.is_none());
2873    }
2874
2875    /// Test 5: serde default back-compat in both directions — a request JSON
2876    /// without additive request fields deserializes to their defaults, and a
2877    /// response JSON without `metrics` (an old daemon's shape) deserializes
2878    /// with it `None`.
2879    #[test]
2880    fn frame_serde_defaults_additive_fields_when_absent() {
2881        let req_json = serde_json::json!({
2882            "ops": "",
2883            "presentation": null,
2884            "presentation_per_op": null,
2885            "namespace": "local",
2886            "actor_id": null,
2887            "visible_namespaces": [],
2888            "config_id": "cfg",
2889            "protocol_version": PROTOCOL_VERSION,
2890            "probe_only": false,
2891            "format": null,
2892            "format_per_op": null,
2893            "from_wire": false
2894        });
2895        let frame: DaemonRequestFrame =
2896            serde_json::from_value(req_json).expect("decode a metrics_only-absent request frame");
2897        assert!(
2898            !frame.metrics_only,
2899            "metrics_only must default to false when absent from the wire payload"
2900        );
2901        assert_eq!(
2902            frame.request_id, None,
2903            "request_id must default to None when absent from the wire payload (khive#948)"
2904        );
2905        assert_eq!(
2906            frame.process_ref, None,
2907            "process_ref must default to None when absent from the wire payload (khive#1428)"
2908        );
2909        let encoded_frame = serde_json::to_value(&frame).expect("encode request frame");
2910        assert!(
2911            encoded_frame.get("process_ref").is_none(),
2912            "absent provenance must not change the serialized request wire shape"
2913        );
2914
2915        let resp_json = serde_json::json!({
2916            "ok": true,
2917            "result": null,
2918            "error": null,
2919            "namespace_mismatch": false,
2920            "config_mismatch": false,
2921            "served_config_id": "cfg",
2922            "version_mismatch": false,
2923            "daemon_protocol_version": PROTOCOL_VERSION
2924        });
2925        let resp: DaemonResponseFrame =
2926            serde_json::from_value(resp_json).expect("decode a metrics-absent response frame");
2927        assert!(
2928            resp.metrics.is_none(),
2929            "metrics must default to None when absent from the wire payload"
2930        );
2931        assert_eq!(
2932            resp.request_id, None,
2933            "request_id must default to None when absent from the wire payload (khive#948)"
2934        );
2935    }
2936
2937    /// khive#948: a request carrying `request_id: Some(n)` gets back a
2938    /// response with `request_id: Some(n)` on both the success and the
2939    /// error/denied dispatch arms — the echo must survive every branch of
2940    /// `handle_conn`, not only the happy path.
2941    #[tokio::test]
2942    async fn request_id_echoed_on_success_and_error_arms() {
2943        let dispatcher = MockDispatch {
2944            namespace: "local".to_string(),
2945            config_id: "cfg-a".to_string(),
2946            dispatch_calls: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
2947            pool: None,
2948            dispatch_err: None,
2949        };
2950        let mut ok_req = base_request_frame("cfg-a");
2951        ok_req.request_id = Some(42);
2952        let ok_resp = round_trip(dispatcher, &ok_req).await;
2953        assert!(ok_resp.ok, "expected successful dispatch: {ok_resp:?}");
2954        assert_eq!(
2955            ok_resp.request_id,
2956            Some(42),
2957            "request_id must be echoed back on a successful dispatch response"
2958        );
2959
2960        // config_mismatch is a rejection arm that never reaches dispatch —
2961        // must still echo the id so the client can join the failure.
2962        let mismatched_dispatcher = MockDispatch {
2963            namespace: "local".to_string(),
2964            config_id: "cfg-a".to_string(),
2965            dispatch_calls: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
2966            pool: None,
2967            dispatch_err: None,
2968        };
2969        let mut mismatch_req = base_request_frame("cfg-WRONG");
2970        mismatch_req.request_id = Some(99);
2971        let mismatch_resp = round_trip(mismatched_dispatcher, &mismatch_req).await;
2972        assert!(mismatch_resp.config_mismatch);
2973        assert_eq!(
2974            mismatch_resp.request_id,
2975            Some(99),
2976            "request_id must be echoed on the config_mismatch rejection arm too"
2977        );
2978
2979        // The real ops-dispatch error arm (`Err(e)` from `dispatcher.dispatch`)
2980        // must echo the id as well, not only the pre-dispatch rejection arms.
2981        let erroring_dispatcher = MockDispatch {
2982            namespace: "local".to_string(),
2983            config_id: "cfg-a".to_string(),
2984            dispatch_calls: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
2985            pool: None,
2986            dispatch_err: Some("simulated dispatch error".to_string()),
2987        };
2988        let mut err_req = base_request_frame("cfg-a");
2989        err_req.request_id = Some(7);
2990        let err_resp = round_trip(erroring_dispatcher, &err_req).await;
2991        assert!(!err_resp.ok, "expected a dispatch error: {err_resp:?}");
2992        assert_eq!(
2993            err_resp.request_id,
2994            Some(7),
2995            "request_id must be echoed on the real ops-dispatch error arm"
2996        );
2997    }
2998
2999    // ── owner-checked shutdown cleanup ────────────────────────────────────────
3000    //
3001    // A draining daemon must not unlink a socket/PID pair that a replacement
3002    // daemon has already bound. These tests exercise `shutdown_cleanup_if_owned`
3003    // directly (the pure decision the caller makes under the recovery lock)
3004    // rather than driving `run_daemon`'s real SIGTERM shutdown, which would
3005    // require sending a signal to the whole test process.
3006
3007    #[test]
3008    fn shutdown_cleanup_removes_paths_it_still_owns() {
3009        let dir = tempfile::tempdir().expect("tempdir");
3010        let sock = dir.path().join("khived.sock");
3011        let pid_file = dir.path().join("khived.pid");
3012
3013        let _listener = std::os::unix::net::UnixListener::bind(&sock).expect("bind socket");
3014        std::fs::write(&pid_file, std::process::id().to_string()).expect("write pid file");
3015        let identity = socket_identity(&sock);
3016        assert!(
3017            identity.is_some(),
3018            "must read identity of a freshly bound socket"
3019        );
3020
3021        let cleaned = shutdown_cleanup_if_owned(&sock, &pid_file, identity);
3022
3023        assert!(
3024            cleaned,
3025            "cleanup must proceed when PID and socket still match"
3026        );
3027        assert!(!sock.exists(), "owned socket must be removed");
3028        assert!(!pid_file.exists(), "owned pid file must be removed");
3029    }
3030
3031    #[test]
3032    fn shutdown_cleanup_skips_when_pid_file_names_a_different_process() {
3033        let dir = tempfile::tempdir().expect("tempdir");
3034        let sock = dir.path().join("khived.sock");
3035        let pid_file = dir.path().join("khived.pid");
3036
3037        let _listener = std::os::unix::net::UnixListener::bind(&sock).expect("bind socket");
3038        let identity = socket_identity(&sock);
3039        // A concurrent client's kill_and_respawn already replaced the PID file
3040        // with a different (replacement daemon's) PID before this daemon's
3041        // drain completed.
3042        std::fs::write(&pid_file, "1").expect("write foreign pid file");
3043
3044        let cleaned = shutdown_cleanup_if_owned(&sock, &pid_file, identity);
3045
3046        assert!(
3047            !cleaned,
3048            "cleanup must be skipped when the PID file no longer names this process"
3049        );
3050        assert!(sock.exists(), "replacement daemon's socket must survive");
3051        assert!(
3052            pid_file.exists(),
3053            "replacement daemon's pid file must survive"
3054        );
3055    }
3056
3057    #[test]
3058    fn shutdown_cleanup_skips_when_socket_was_rebound_by_a_replacement() {
3059        let dir = tempfile::tempdir().expect("tempdir");
3060        let sock = dir.path().join("khived.sock");
3061        let original_sock = dir.path().join("original.sock");
3062        let pid_file = dir.path().join("khived.pid");
3063
3064        // Bind two sockets at DIFFERENT paths, both alive at the same time,
3065        // so the OS cannot recycle an inode between them the way it could
3066        // across a bind/drop/rebind cycle at a single path (the flakiness a
3067        // prior version of this test hit on some filesystems). Both
3068        // identities are captured through the real production
3069        // `socket_identity()` path, not a synthetic/sentinel value, so a
3070        // regression where `socket_identity()` returns a constant identity
3071        // for every socket makes the `assert!` below fail loudly instead of
3072        // silently passing.
3073        let _original_listener =
3074            std::os::unix::net::UnixListener::bind(&original_sock).expect("bind original socket");
3075        let _replacement_listener =
3076            std::os::unix::net::UnixListener::bind(&sock).expect("bind replacement socket");
3077
3078        let original_identity = socket_identity(&original_sock);
3079        let replacement_identity = socket_identity(&sock);
3080        assert!(
3081            original_identity.is_some(),
3082            "must read identity of the original socket"
3083        );
3084        assert!(
3085            replacement_identity.is_some(),
3086            "must read identity of the replacement socket"
3087        );
3088        assert!(
3089            original_identity != replacement_identity,
3090            "two concurrently bound sockets must have distinct identities"
3091        );
3092
3093        std::fs::write(&pid_file, std::process::id().to_string())
3094            .expect("write pid file matching this process");
3095
3096        // `sock` (the replacement bind's path) is checked against
3097        // `original_identity` (a different, concurrently-alive socket's
3098        // identity) - the mismatch alone must be enough to block cleanup,
3099        // even though the pid file matches this process.
3100        let cleaned = shutdown_cleanup_if_owned(&sock, &pid_file, original_identity);
3101
3102        assert!(
3103            !cleaned,
3104            "cleanup must be skipped when the socket at this path is a different \
3105             inode than the one this daemon originally bound"
3106        );
3107        assert!(sock.exists(), "replacement daemon's socket must survive");
3108        assert!(
3109            pid_file.exists(),
3110            "replacement daemon's pid file must survive"
3111        );
3112    }
3113
3114    // ── the recovery lock actually serializes two boot sequences ─────────────
3115    //
3116    // Production wiring (`khive_mcp::serve::run` / `serve_server`) now acquires
3117    // this same lock *before* building a `KhiveMcpServer` (which runs
3118    // migrations and applies pack schema plans / FTS DDL) and holds it through
3119    // daemon bind+pid-write, via `run_daemon_with_boot_guard`. That closes the
3120    // cold-boot race only if `acquire_recovery_lock` genuinely provides mutual
3121    // exclusion across concurrent boot attempts — this test proves the
3122    // primitive itself: two "boot sequences" (each holding the lock across a
3123    // simulated schema-init critical section) must never run their critical
3124    // sections at the same time.
3125    #[test]
3126    #[serial]
3127    fn recovery_lock_serializes_two_concurrent_boot_sequences() {
3128        let dir = tempfile::tempdir().expect("tempdir");
3129        let lock_file = dir.path().join("khived.recovery.lock");
3130        std::env::set_var("KHIVE_LOCK", &lock_file);
3131
3132        let active = Arc::new(std::sync::atomic::AtomicUsize::new(0));
3133        let overlap_detected = Arc::new(std::sync::atomic::AtomicBool::new(false));
3134
3135        let run_one_boot =
3136            |active: Arc<std::sync::atomic::AtomicUsize>,
3137             overlap: Arc<std::sync::atomic::AtomicBool>| {
3138                move || {
3139                    let _guard = acquire_recovery_lock().expect("acquire recovery lock");
3140                    // Enter the "schema-init" critical section.
3141                    if active.fetch_add(1, std::sync::atomic::Ordering::SeqCst) != 0 {
3142                        overlap.store(true, std::sync::atomic::Ordering::SeqCst);
3143                    }
3144                    std::thread::sleep(std::time::Duration::from_millis(50));
3145                    active.fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
3146                    // `_guard` drops here, releasing the lock.
3147                }
3148            };
3149
3150        let t1 = std::thread::spawn(run_one_boot(active.clone(), overlap_detected.clone()));
3151        let t2 = std::thread::spawn(run_one_boot(active.clone(), overlap_detected.clone()));
3152        t1.join().expect("boot thread 1 must not panic");
3153        t2.join().expect("boot thread 2 must not panic");
3154
3155        assert!(
3156            !overlap_detected.load(std::sync::atomic::Ordering::SeqCst),
3157            "two concurrent boot sequences must never hold the schema-init \
3158             critical section at the same time (#667)"
3159        );
3160
3161        std::env::remove_var("KHIVE_LOCK");
3162    }
3163
3164    // ── acquire_daemon_boot_guard treats lock failure as fatal ───────────────
3165    // (unlike best-effort acquire_recovery_lock, whose `None` on failure is
3166    // correct for its own best-effort callers).
3167
3168    #[test]
3169    #[serial]
3170    fn acquire_daemon_boot_guard_returns_guard_when_lock_available() {
3171        let dir = tempfile::tempdir().expect("tempdir");
3172        let lock_file = dir.path().join("khived.recovery.lock");
3173        std::env::set_var("KHIVE_LOCK", &lock_file);
3174
3175        let guard = acquire_daemon_boot_guard();
3176        assert!(
3177            guard.is_ok(),
3178            "daemon boot guard must succeed when the lock file can be opened and flocked"
3179        );
3180        drop(guard);
3181
3182        std::env::remove_var("KHIVE_LOCK");
3183    }
3184
3185    #[test]
3186    #[serial]
3187    fn acquire_daemon_boot_guard_fails_loudly_when_lock_file_cannot_be_opened() {
3188        let dir = tempfile::tempdir().expect("tempdir");
3189        // Point KHIVE_LOCK at a directory, not a file: opening a directory
3190        // with `write(true)` fails (EISDIR), so `acquire_recovery_lock`
3191        // returns `None` here — the exact failure mode
3192        // `acquire_daemon_boot_guard` must turn into a hard `Err` instead of
3193        // silently letting daemon-mode boot proceed unguarded.
3194        std::env::set_var("KHIVE_LOCK", dir.path());
3195
3196        let result = acquire_daemon_boot_guard();
3197        assert!(
3198            result.is_err(),
3199            "daemon boot guard must fail loudly, never silently proceed unguarded, \
3200             when the underlying recovery lock cannot be acquired"
3201        );
3202
3203        std::env::remove_var("KHIVE_LOCK");
3204    }
3205
3206    // ── write_pid_file_exclusive never truncates a winner's pid file ────────
3207
3208    #[test]
3209    fn write_pid_file_exclusive_creates_new_file_with_own_pid() {
3210        let dir = tempfile::tempdir().expect("tempdir");
3211        let pid_file = dir.path().join("khived.pid");
3212        write_pid_file_exclusive(&pid_file).expect("first writer must win");
3213        let contents = std::fs::read_to_string(&pid_file).expect("read pid file");
3214        assert_eq!(contents, std::process::id().to_string());
3215    }
3216
3217    #[test]
3218    fn write_pid_file_exclusive_refuses_to_overwrite_an_existing_file() {
3219        let dir = tempfile::tempdir().expect("tempdir");
3220        let pid_file = dir.path().join("khived.pid");
3221        std::fs::write(&pid_file, "999999").expect("seed an existing pid file");
3222
3223        let err = write_pid_file_exclusive(&pid_file)
3224            .expect_err("must not silently overwrite an existing pid file");
3225        assert_eq!(err.kind(), std::io::ErrorKind::AlreadyExists);
3226
3227        // The existing content must be completely untouched — proving this is
3228        // `create_new`, not the old `create(true).truncate(true)`.
3229        let contents = std::fs::read_to_string(&pid_file).expect("read pid file");
3230        assert_eq!(
3231            contents, "999999",
3232            "an existing pid file must never be truncated by a losing writer"
3233        );
3234    }
3235
3236    // Real (not simulated) concurrency: two OS threads race to `create_new`
3237    // the exact same path, synchronized with a `Barrier` so they genuinely
3238    // overlap at the syscall rather than relying on a sleep-based ordering
3239    // guess. This is the deterministic race oracle for the convergence
3240    // requirement the atomic-creation primitive `write_pid_file_exclusive`
3241    // is built on: exactly one of two simultaneous daemon starters may claim
3242    // the pid file, and the loser must see `AlreadyExists`, never silently
3243    // clobber the winner's content.
3244    #[test]
3245    fn two_concurrent_writers_converge_on_exactly_one_pid_file_owner() {
3246        let dir = tempfile::tempdir().expect("tempdir");
3247        let pid_file = std::sync::Arc::new(dir.path().join("khived.pid"));
3248        let barrier = std::sync::Arc::new(std::sync::Barrier::new(2));
3249
3250        let spawn_writer =
3251            |pid_file: std::sync::Arc<std::path::PathBuf>,
3252             barrier: std::sync::Arc<std::sync::Barrier>| {
3253                std::thread::spawn(move || {
3254                    barrier.wait();
3255                    write_pid_file_exclusive(&pid_file)
3256                })
3257            };
3258
3259        let t1 = spawn_writer(pid_file.clone(), barrier.clone());
3260        let t2 = spawn_writer(pid_file.clone(), barrier.clone());
3261        let r1 = t1.join().expect("writer 1 must not panic");
3262        let r2 = t2.join().expect("writer 2 must not panic");
3263
3264        let results = [&r1, &r2];
3265        let ok_count = results.iter().filter(|r| r.is_ok()).count();
3266        let already_exists_count = results
3267            .iter()
3268            .filter(|r| matches!(r, Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists))
3269            .count();
3270        assert_eq!(
3271            ok_count, 1,
3272            "exactly one of two concurrent writers must win the pid file"
3273        );
3274        assert_eq!(
3275            already_exists_count, 1,
3276            "the other writer must observe AlreadyExists, never a silent overwrite"
3277        );
3278        assert!(pid_file.exists(), "the winner's pid file must exist");
3279        let contents = std::fs::read_to_string(&*pid_file).expect("read pid file");
3280        assert_eq!(
3281            contents,
3282            std::process::id().to_string(),
3283            "the surviving pid file must contain the winner's pid — both threads \
3284             share this process's pid, so an unexpected value would also prove a \
3285             lost/garbled write raced through"
3286        );
3287    }
3288
3289    // ── connection principal (ADR-096 condition 2) ────────────────────────────
3290
3291    /// The peer-credential syscall must actually work on this platform and
3292    /// report the real uid, not error or return a placeholder.
3293    ///
3294    /// This matters more than it looks because the accept path fails CLOSED: if
3295    /// `peer_uid` errored unconditionally — wrong syscall, wrong socket option,
3296    /// an unimplemented platform arm — every connection would be refused and
3297    /// the daemon would be silently unreachable. A test that only exercised the
3298    /// decision function would not catch that.
3299    #[tokio::test]
3300    async fn peer_uid_reports_the_connecting_process_uid() {
3301        let dir = tempfile::tempdir().expect("tempdir");
3302        let sock = dir.path().join("peer.sock");
3303        let listener = UnixListener::bind(&sock).expect("bind");
3304
3305        let connect_path = sock.clone();
3306        let client = tokio::spawn(async move { UnixStream::connect(&connect_path).await });
3307
3308        let (server_side, _) = listener.accept().await.expect("accept");
3309        let client_side = client.await.expect("join").expect("connect");
3310
3311        // SAFETY: `geteuid` is always successful and takes no arguments.
3312        let expected = unsafe { libc::geteuid() } as u32;
3313
3314        assert_eq!(
3315            peer_uid(&server_side).expect("peer_uid must succeed on a live connection"),
3316            expected,
3317            "the uid read from the kernel for a same-process connection must be \
3318             this process's euid"
3319        );
3320        // Symmetric: both ends report the same peer on a same-uid connection.
3321        assert_eq!(
3322            peer_uid(&client_side).expect("peer_uid must succeed on the client end"),
3323            expected
3324        );
3325    }
3326
3327    /// The refusal rule itself: the principal is the uid, and only a foreign
3328    /// uid is refused.
3329    ///
3330    /// The second assertion is the load-bearing one and it is a regression
3331    /// guard, not a formality. ADR-096 shipped many `actor_id`s over one
3332    /// socket; every seat on a normal host is a distinct attribution at the
3333    /// same uid. A check that conflated attribution with principal would refuse
3334    /// them all, so "same uid is permitted" must stay true no matter how the
3335    /// rule is later tightened.
3336    #[test]
3337    fn only_a_foreign_uid_is_refused() {
3338        // SAFETY: `geteuid` is always successful and takes no arguments.
3339        let euid = unsafe { libc::geteuid() } as u32;
3340
3341        assert!(
3342            uid_is_permitted(euid, euid),
3343            "a connection from the daemon's own uid must be served — this is \
3344             every seat on the host, and ADR-096 accepted exactly this shape"
3345        );
3346        assert!(
3347            !uid_is_permitted(euid.wrapping_add(1), euid),
3348            "a connection from any other uid must be refused"
3349        );
3350        assert!(
3351            !uid_is_permitted(0, euid.wrapping_add(1)),
3352            "root is not special-cased: the rule is equality with the daemon's \
3353             euid, not a privilege comparison"
3354        );
3355    }
3356}