Skip to main content

khive_runtime/
daemon.rs

1//! khived daemon server — persistent warm runtime over a Unix socket.
2//!
3//! The daemon binds `~/.khive/khived.sock`, accepts length-prefixed request
4//! frames, dispatches them through a [`DaemonDispatch`] implementor, and serves
5//! results back. It is transport-agnostic: the MCP crate provides the dispatch
6//! impl, but any future client (CLI, HTTP gateway) can reuse this server.
7//!
8//! The client side (forwarding, auto-spawn) lives in the transport crate
9//! (e.g. `khive-mcp`), not here.
10
11use std::sync::Arc;
12
13#[cfg(unix)]
14use std::io::Write as _;
15#[cfg(unix)]
16use std::os::unix::fs::PermissionsExt;
17#[cfg(unix)]
18use std::os::unix::io::AsRawFd;
19use std::path::PathBuf;
20
21#[cfg(unix)]
22use async_trait::async_trait;
23#[cfg(unix)]
24use libc;
25use serde::{Deserialize, Serialize};
26#[cfg(unix)]
27use tokio::io::{AsyncReadExt, AsyncWriteExt};
28#[cfg(unix)]
29use tokio::net::{UnixListener, UnixStream};
30
31#[cfg(unix)]
32use khive_db::{run_checkpoint_task, CheckpointConfig, CheckpointLifecycleOwner, ConnectionPool};
33
34#[cfg(unix)]
35use crate::pack::RequestIdentity;
36
37/// Maximum frame size accepted in either direction.
38pub const MAX_FRAME_BYTES: usize = 8 * 1024 * 1024;
39
40/// Wire protocol version for the daemon IPC framing.
41///
42/// Increment this constant whenever the request or response frame shape
43/// changes in a backward-incompatible way. The client sends its version
44/// in every request; the daemon rejects mismatches with an explicit error
45/// that names both sides so the operator knows exactly what to do
46/// (`make local` rebuilds the client binary).
47/// See `docs/api/daemon.md#protocol_version` for the version-by-version history.
48pub const PROTOCOL_VERSION: u32 = 4;
49
50const DEFAULT_DRAIN_TIMEOUT_SECS: u64 = 10;
51
52// ── paths ─────────────────────────────────────────────────────────────────────
53
54/// Base `.khive` directory used to anchor every advisory lock/socket/pid
55/// path below. Pure path computation — portable on every target, even
56/// though most of its callers (socket/pid paths) are unix-only.
57///
58/// Resolution order: `HOME`, then `USERPROFILE` (the conventional Windows
59/// home variable), then a platform-specific last resort. On non-unix targets
60/// the last resort is the OS temp directory (per-user on Windows), so the
61/// lock path can never be working-directory-relative there: two processes
62/// opening the same database from different working directories must resolve
63/// the same lock file. On unix the last resort stays the historical `"."` —
64/// a shared world-writable anchor such as `/tmp` would be worse, since a
65/// local attacker could pre-claim the directory and the socket/lock files
66/// under it before the daemon's first run.
67fn khive_dir() -> PathBuf {
68    khive_root_from(
69        std::env::var("HOME").ok(),
70        std::env::var("USERPROFILE").ok(),
71    )
72}
73
74/// Env-free core of [`khive_dir`], split out so the fallback chain is
75/// testable without mutating process-global environment variables.
76fn khive_root_from(home: Option<String>, userprofile: Option<String>) -> PathBuf {
77    home.filter(|v| !v.trim().is_empty())
78        .or_else(|| userprofile.filter(|v| !v.trim().is_empty()))
79        .map(PathBuf::from)
80        .unwrap_or_else(last_resort_root)
81        .join(".khive")
82}
83
84/// See [`khive_dir`] for why the two arms differ.
85#[cfg(unix)]
86fn last_resort_root() -> PathBuf {
87    PathBuf::from(".")
88}
89
90#[cfg(not(unix))]
91fn last_resort_root() -> PathBuf {
92    std::env::temp_dir()
93}
94
95/// Unix socket path the daemon binds and clients connect to.
96///
97/// Overridable via the `KHIVE_SOCKET` env var (for tests and ops).
98#[cfg(unix)]
99pub fn socket_path() -> PathBuf {
100    if let Ok(p) = std::env::var("KHIVE_SOCKET") {
101        if !p.is_empty() {
102            return PathBuf::from(p);
103        }
104    }
105    khive_dir().join("khived.sock")
106}
107
108/// PID file path written by the daemon.
109///
110/// Overridable via the `KHIVE_PID` env var.
111#[cfg(unix)]
112pub fn pid_path() -> PathBuf {
113    if let Ok(p) = std::env::var("KHIVE_PID") {
114        if !p.is_empty() {
115            return PathBuf::from(p);
116        }
117    }
118    khive_dir().join("khived.pid")
119}
120
121/// Advisory lock file used to serialize stale-daemon recovery across concurrent
122/// clients (flock/`File::lock` on the file; released when the lock file
123/// handle is dropped). Path computation is portable; `kkernel exec`'s
124/// non-unix local-construction guard (`kkernel::exec::acquire_local_construction_guard`)
125/// shares this exact path with the unix daemon-boot guard so the two stay
126/// mutually exclusive.
127///
128/// Overridable via the `KHIVE_LOCK` env var (for tests).
129pub fn lock_path() -> PathBuf {
130    if let Ok(p) = std::env::var("KHIVE_LOCK") {
131        if !p.is_empty() {
132            return PathBuf::from(p);
133        }
134    }
135    khive_dir().join("khived.recovery.lock")
136}
137
138/// Advisory lock file used to serialize RECOVERY (kill+respawn) attempts
139/// across concurrent clients only — the daemon's own boot sequence never
140/// acquires this file ([`lock_path`] / [`acquire_daemon_boot_guard`] is the
141/// boot-side lock). A recoverer holding this lock across dead-confirmation
142/// → kill → spawn (khive-mcp's `kill_and_respawn`) therefore can never
143/// deadlock against a peer daemon's boot, unlike holding the shared boot
144/// lock for that whole span would.
145///
146/// Overridable via the `KHIVE_RECOVERER_LOCK` env var (for tests).
147#[cfg(unix)]
148pub fn recoverer_lock_path() -> PathBuf {
149    if let Ok(p) = std::env::var("KHIVE_RECOVERER_LOCK") {
150        if !p.is_empty() {
151            return PathBuf::from(p);
152        }
153    }
154    khive_dir().join("khived.recoverer.lock")
155}
156
157#[cfg(unix)]
158fn open_lock_file(path: &std::path::Path) -> std::io::Result<std::fs::File> {
159    if let Some(parent) = path.parent() {
160        let _ = std::fs::create_dir_all(parent);
161    }
162    std::fs::OpenOptions::new()
163        .create(true)
164        .truncate(false)
165        .write(true)
166        .open(path)
167}
168
169#[cfg(unix)]
170fn acquire_flock_blocking(path: &std::path::Path, label: &str) -> Option<std::fs::File> {
171    let file = match open_lock_file(path) {
172        Ok(f) => f,
173        Err(e) => {
174            tracing::warn!(error = %e, path = ?path, "cannot open {label} lock file");
175            return None;
176        }
177    };
178    // SAFETY: flock is a POSIX advisory lock with no memory side-effects.
179    let rc = unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX) };
180    if rc != 0 {
181        tracing::warn!("flock LOCK_EX failed on {label} lock");
182        return None;
183    }
184    Some(file)
185}
186
187/// Acquire an exclusive advisory flock on the recovery/startup lock file.
188///
189/// The returned `File` holds the lock for its lifetime; dropping it releases
190/// it.  Used by both the client (serializing kill+spawn) and the daemon server
191/// (serializing cleanup+bind+pid-write) so the two critical sections are
192/// mutually exclusive across processes.
193#[cfg(unix)]
194pub fn acquire_recovery_lock() -> Option<std::fs::File> {
195    acquire_flock_blocking(&lock_path(), "recovery")
196}
197
198/// Attempt to acquire an exclusive advisory flock on `path`, retrying with a
199/// non-blocking `flock(LOCK_NB)` until `deadline` elapses. Bounded alternative
200/// to `acquire_recovery_lock`/`acquire_daemon_boot_guard`'s unbounded blocking
201/// flock — see `docs/api/daemon.md#try_acquire_flock_until` for why a caller
202/// merely detecting lock freedom needs a deadline instead.
203///
204/// - `Ok(Some(file))` — the lock was free within the deadline.
205/// - `Ok(None)` — `deadline` elapsed while the lock stayed held; an explicit
206///   "could not confirm" outcome, distinct from a hard I/O error.
207/// - `Err(_)` — the lock file could not be opened, or `flock` failed for a
208///   reason other than contention.
209///
210/// Blocking (paces retries with `std::thread::sleep`) — async callers must
211/// run this via `spawn_blocking`.
212#[cfg(unix)]
213fn try_acquire_flock_until(
214    path: &std::path::Path,
215    deadline: std::time::Instant,
216) -> std::io::Result<Option<std::fs::File>> {
217    let file = open_lock_file(path)?;
218    let poll_interval = std::time::Duration::from_millis(10);
219    loop {
220        // SAFETY: flock is a POSIX advisory lock with no memory side-effects.
221        let rc = unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) };
222        if rc == 0 {
223            return Ok(Some(file));
224        }
225        let err = std::io::Error::last_os_error();
226        if err.raw_os_error() != Some(libc::EWOULDBLOCK) {
227            return Err(err);
228        }
229        let now = std::time::Instant::now();
230        if now >= deadline {
231            return Ok(None);
232        }
233        std::thread::sleep(poll_interval.min(deadline - now));
234    }
235}
236
237/// Bounded, deadline-aware variant of [`acquire_daemon_boot_guard`]: attempts
238/// the SAME boot/recovery lock ([`lock_path`]) but gives up at `deadline`
239/// instead of blocking forever. For callers that need to detect "is a boot in
240/// progress right now" without risking an unbounded wait behind a wedged
241/// holder: e.g. khive-mcp's `confirm_genuinely_dead` re-probing rounds,
242/// where `DEAD_CONFIRM_ROUNDS` must bound elapsed time, not just probe count.
243#[cfg(unix)]
244pub fn try_acquire_daemon_boot_guard_until(
245    deadline: std::time::Instant,
246) -> std::io::Result<Option<DaemonBootGuard>> {
247    try_acquire_flock_until(&lock_path(), deadline)
248}
249
250/// Bounded, deadline-aware acquisition of the recoverer-only lock
251/// ([`recoverer_lock_path`]). See [`try_acquire_daemon_boot_guard_until`] for
252/// the shared rationale — a second recoverer waiting for a peer's dead
253/// confirmation/kill/spawn critical section must give up and report
254/// "uncertain" rather than block forever if that peer is itself wedged.
255#[cfg(unix)]
256pub fn try_acquire_recoverer_lock_until(
257    deadline: std::time::Instant,
258) -> std::io::Result<Option<std::fs::File>> {
259    try_acquire_flock_until(&recoverer_lock_path(), deadline)
260}
261
262/// Guard returned by [`acquire_daemon_boot_guard`], held across cold-boot
263/// schema initialization (migrations + pack schema plans / FTS DDL) through
264/// daemon bind + pid-write.
265#[cfg(unix)]
266pub type DaemonBootGuard = std::fs::File;
267
268/// Acquire the recovery/boot lock, treating failure as fatal.
269///
270/// Unlike [`acquire_recovery_lock`] (best-effort, `None` on failure: used by
271/// shutdown cleanup, where skipping unlink is safer than blocking forever),
272/// daemon-mode boot must hold this lock across migrations/FTS DDL through
273/// bind+pid-write. Silently continuing with no lock reopens the cold-boot FTS
274/// race this guard exists to close, so callers that are about to run
275/// daemon-mode boot (or wait for one to quiesce) must fail loudly instead of
276/// proceeding unguarded.
277#[cfg(unix)]
278pub fn acquire_daemon_boot_guard() -> anyhow::Result<DaemonBootGuard> {
279    acquire_recovery_lock()
280        .ok_or_else(|| anyhow::anyhow!("failed to acquire daemon boot/recovery lock"))
281}
282
283/// Identity of a bound Unix socket path, used to tell "the socket I bound" apart
284/// from "a same-path socket some other daemon bound after mine was removed".
285///
286/// A socket path can be recreated by a different process between the time
287/// this daemon captures its identity and the time it later checks it, so `dev`
288/// and `ino` (not the path) are what must match for cleanup to be safe.
289#[cfg(unix)]
290#[derive(Clone, Copy, PartialEq, Eq)]
291struct SocketIdentity {
292    dev: u64,
293    ino: u64,
294}
295
296#[cfg(unix)]
297fn socket_identity(path: &std::path::Path) -> Option<SocketIdentity> {
298    use std::os::unix::fs::MetadataExt;
299    let meta = std::fs::metadata(path).ok()?;
300    Some(SocketIdentity {
301        dev: meta.dev(),
302        ino: meta.ino(),
303    })
304}
305
306// ── connection principal ──────────────────────────────────────────────────────
307
308/// The uid on the other end of an accepted connection, read from the kernel.
309///
310/// This is the only identity on this socket the caller cannot choose. Every
311/// identity field on the request frame — `namespace`, `actor_id`,
312/// `visible_namespaces`, `config_id` — is supplied by the connecting process,
313/// so none of them can answer "who is this". A check reading self-asserted
314/// fields is not a weak gate, it is not a gate: anyone who wants to pass it
315/// asserts the passing values.
316///
317/// `getpeereid(2)` on macOS/BSD, `SO_PEERCRED` on Linux. Both report the peer's
318/// credentials as recorded by the kernel at connect time.
319#[cfg(unix)]
320pub(crate) fn peer_uid(stream: &UnixStream) -> std::io::Result<u32> {
321    use std::os::fd::AsRawFd;
322    let fd = stream.as_raw_fd();
323
324    #[cfg(any(target_os = "macos", target_os = "ios", target_vendor = "apple"))]
325    {
326        let mut uid: libc::uid_t = 0;
327        let mut gid: libc::gid_t = 0;
328        // SAFETY: `fd` is a live connected socket owned by `stream` for the
329        // duration of this call; both out-params are valid initialized locals.
330        let rc = unsafe { libc::getpeereid(fd, &mut uid, &mut gid) };
331        if rc != 0 {
332            return Err(std::io::Error::last_os_error());
333        }
334        Ok(uid as u32)
335    }
336
337    #[cfg(target_os = "linux")]
338    {
339        let mut cred = libc::ucred {
340            pid: 0,
341            uid: 0,
342            gid: 0,
343        };
344        let mut len = std::mem::size_of::<libc::ucred>() as libc::socklen_t;
345        // SAFETY: `fd` is a live connected socket owned by `stream`; `cred` is
346        // an initialized local of exactly `len` bytes, which is what
347        // SO_PEERCRED writes.
348        let rc = unsafe {
349            libc::getsockopt(
350                fd,
351                libc::SOL_SOCKET,
352                libc::SO_PEERCRED,
353                (&mut cred as *mut libc::ucred).cast::<libc::c_void>(),
354                &mut len,
355            )
356        };
357        if rc != 0 {
358            return Err(std::io::Error::last_os_error());
359        }
360        Ok(cred.uid)
361    }
362
363    #[cfg(not(any(
364        target_os = "linux",
365        target_os = "macos",
366        target_os = "ios",
367        target_vendor = "apple"
368    )))]
369    {
370        let _ = fd;
371        Err(std::io::Error::new(
372            std::io::ErrorKind::Unsupported,
373            "peer-credential capture is not implemented for this platform",
374        ))
375    }
376}
377
378/// Whether a connection from `uid` may be served by this daemon.
379///
380/// ADR-096 accepted per-request identity threading **for the single-principal
381/// owner-only socket only**, and rested that on three things: the `0600`
382/// socket, all connections being the same uid, and the database being already
383/// same-uid-accessible. The socket mode is asserted at bind. This asserts the
384/// second, which previously had no representation in the code at all — nothing
385/// read peer identity, so nothing could notice when it stopped being true.
386///
387/// **Principal is not attribution.** Many `actor_id`s over one socket is
388/// exactly what ADR-096 shipped and what every seat on a normal host does;
389/// refusing a second distinct actor would break the accepted design. The
390/// principal is the uid, and this refuses only a genuinely foreign one.
391///
392/// **There is deliberately no configuration escape hatch.** A flag permitting
393/// other uids would not weaken this assertion, it would delete it, in the way
394/// hardest to notice later: the check still exists, its tests still pass, and
395/// the deployment that matters has it off. A deployment that genuinely needs
396/// multiple uids needs a code change and a gated ADR — which is precisely the
397/// decision that should be impossible to make by accident.
398#[cfg(unix)]
399pub(crate) fn uid_is_permitted(peer: u32, daemon_euid: u32) -> bool {
400    peer == daemon_euid
401}
402
403// ── wire types ────────────────────────────────────────────────────────────────
404
405/// Request frame sent from a client to the daemon.
406#[derive(Serialize, Deserialize, Default)]
407pub struct DaemonRequestFrame {
408    pub ops: String,
409    pub presentation: Option<String>,
410    pub presentation_per_op: Option<Vec<Option<String>>>,
411    /// The client's resolved storage/gate default namespace for this request.
412    ///
413    /// As of protocol version 3 (ADR-096) the daemon serves the request under
414    /// this namespace instead of rejecting on mismatch: a per-request
415    /// identity input, not a same-process-identity assertion.
416    pub namespace: String,
417    /// The client's resolved write-stamp / gate actor identity (ADR-057),
418    /// carried on the frame so the warm daemon stamps writes with the
419    /// *caller's* actor instead of its own baked `actor_id` (ADR-096). `None`
420    /// mints `ActorRef::anonymous()`, matching an unconfigured actor.
421    #[serde(default)]
422    pub actor_id: Option<String>,
423    /// Opaque process provenance resolved in the originating client process.
424    /// It is carried per request because a shared warm daemon's environment
425    /// does not identify the worker that submitted the operation. Protocol v4
426    /// makes this field part of dispatch semantics: a v3 daemon must reject the
427    /// request rather than execute it while silently discarding provenance.
428    #[serde(default, skip_serializing_if = "Option::is_none")]
429    pub process_ref: Option<String>,
430    /// The client's resolved extra read-visibility namespaces (ADR-007 Rule
431    /// 3b), carried on the frame so the warm daemon widens read scope to
432    /// match the caller's own configuration rather than its own baked
433    /// `visible_namespaces` (ADR-096). Empty means no extra visibility beyond
434    /// `namespace` itself.
435    #[serde(default)]
436    pub visible_namespaces: Vec<String>,
437    /// Fingerprint of the client's engine-coherence config: packs, db target,
438    /// embedders, backend routing, and construction-baked outbound policy.
439    /// Identity fields are carried separately in this frame. The daemon rejects
440    /// a request whose `config_id` differs from its own so a restricted client
441    /// (e.g. `--pack kg`, `--db :memory:`) never dispatches through the broader
442    /// default daemon. See ADR-027 / ADR-049 / ADR-096.
443    #[serde(default)]
444    pub config_id: String,
445    /// IPC protocol version sent by the client. Pre-versioning clients omit
446    /// this field (deserializes to 0). The daemon compares against
447    /// [`PROTOCOL_VERSION`] and rejects mismatches with an explicit error.
448    #[serde(default)]
449    pub protocol_version: u32,
450    /// When `true`, the daemon returns an identity frame (ok=true, result=None)
451    /// immediately after identity validation — without calling the dispatcher.
452    /// Used by the client's under-lock recovery probe to confirm a daemon is
453    /// alive and identity-matching without dispatching any mutating verb.
454    /// Pre-probe clients omit this field (deserializes to false → normal dispatch).
455    #[serde(default)]
456    pub probe_only: bool,
457    /// When `true`, the daemon returns a point-in-time [`MetricsSnapshot`] of
458    /// its server-side gauges (a read-only measurement surface for the
459    /// load/perf harness) instead of dispatching any op. Handled before the
460    /// `config_id` equality reject: a gauge read is process-global and
461    /// namespace/config-agnostic, not a namespaced record operation.
462    /// READ-ONLY — this field is the only input the frame accepts for a
463    /// metrics request; there is no reset or mutation reachable over the
464    /// wire. Pre-metrics clients omit this field (deserializes to `false` →
465    /// normal dispatch, unaffected).
466    #[serde(default)]
467    pub metrics_only: bool,
468    /// Output format for this request (ADR-078). Forwarded to the daemon's
469    /// serialization seam. `None` means use the daemon's resolved default.
470    #[serde(default)]
471    pub format: Option<String>,
472    /// Per-operation output format overrides (ADR-078).
473    #[serde(default)]
474    pub format_per_op: Option<Vec<Option<String>>>,
475    /// Whether this request originated from the agent-facing MCP `request`
476    /// tool (the wire surface). When `true`, the daemon rejects
477    /// `Visibility::Subhandler` verbs: agents must not invoke internal
478    /// subhandlers. When `false` (the default, and the only value any
479    /// operator path sends), subhandlers are allowed: `kkernel exec` and
480    /// other in-process callers are trusted operator surfaces.
481    ///
482    /// This is the origin discriminator, not a daemon-vs-local one: operator
483    /// requests flow through the daemon by default too, so the gate cannot
484    /// key on transport.
485    #[serde(default)]
486    pub from_wire: bool,
487    /// Caller-supplied correlation id (khive#948): a `u64` from the caller's
488    /// own process-local monotonic counter, echoed back unchanged on
489    /// [`DaemonResponseFrame::request_id`] and stamped into the dispatch's
490    /// audit event (`resource.request_id`) so a benchmark harness can join
491    /// its own pre-send sample to the server-side audit row for the same
492    /// request. Purely additive — `#[serde(default)]` matches
493    /// `metrics_only`/`format`/`format_per_op` precedent, no
494    /// `PROTOCOL_VERSION` bump. `None` means the caller supplied no id.
495    #[serde(default)]
496    pub request_id: Option<u64>,
497}
498
499/// Response frame sent from the daemon back to a client.
500#[derive(Serialize, Deserialize, Debug)]
501pub struct DaemonResponseFrame {
502    pub ok: bool,
503    pub result: Option<String>,
504    pub error: Option<String>,
505    pub namespace_mismatch: bool,
506    /// Set when the request's `config_id` does not match the daemon's. Like
507    /// `namespace_mismatch`, this signals the client to fall back to local
508    /// dispatch rather than execute under a different runtime/config.
509    #[serde(default)]
510    pub config_mismatch: bool,
511    /// The `config_id` the daemon dispatched under, echoed back so the client
512    /// can positively confirm the result came from a matching runtime. A
513    /// pre-`config_id` daemon omits this field (deserializes to `None`), which
514    /// the client treats as a mismatch and falls back to local dispatch — this
515    /// closes the upgrade window where a new restricted client could otherwise
516    /// trust a still-warm legacy daemon's broader registry.
517    #[serde(default)]
518    pub served_config_id: Option<String>,
519    /// Set when the client's `protocol_version` does not match the daemon's
520    /// [`PROTOCOL_VERSION`]. The client must treat this as a hard error and
521    /// surface the human-readable `error` field rather than falling back to
522    /// local dispatch (which would hide the version skew).
523    #[serde(default)]
524    pub version_mismatch: bool,
525    /// The daemon's [`PROTOCOL_VERSION`], echoed in error responses so the
526    /// client can include both sides in the diagnostic message. Pre-versioning
527    /// daemons omit this field (deserializes to 0).
528    #[serde(default)]
529    pub daemon_protocol_version: u32,
530    /// Populated when the request set `metrics_only: true`: a point-in-time
531    /// snapshot of the daemon's server-side gauges. `None` on every other
532    /// response, and on any response from a daemon that predates this field
533    /// (client-side back-compat via `#[serde(default)]`, matching
534    /// `served_config_id`'s upgrade-window handling above).
535    #[serde(default, skip_serializing_if = "Option::is_none")]
536    pub metrics: Option<MetricsSnapshot>,
537    /// Echo of the request's `request_id` (khive#948), present whenever the
538    /// frame that produced this response carried one — including on every
539    /// error/denied arm, not only success, so a client can join a failure
540    /// the same way it joins a success. `#[serde(default)]` so an older
541    /// daemon's response (predating this field) deserializes to `None`
542    /// rather than a parse error.
543    #[serde(default)]
544    pub request_id: Option<u64>,
545}
546
547/// Point-in-time snapshot of the daemon's server-side gauges — the
548/// load/perf harness read-surface (measurement substrate, not a product feature).
549///
550/// Every field here is a **server-side** gauge reachable from `handle_conn`
551/// without any mutation: [`khive_storage::tx_registry`] (ADR-091 Plank 0,
552/// process-global singleton), the main pool's backend-keyed routine WAL
553/// sample and process TRUNCATE counters (`khive_db::checkpoint`), and the
554/// ADR-067 Component A write queue depth/latest writer-stage sample. There is
555/// no reset reachable through this type or through [`DaemonRequestFrame`] —
556/// gauges out, nothing in.
557#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq)]
558pub struct MetricsSnapshot {
559    /// Last-observed WAL page count from the periodic checkpoint tick.
560    /// `None` when the checkpoint task has never ticked in this process
561    /// (for example, an in-memory dispatcher with no pool, or a daemon that
562    /// just started and hasn't hit its first tick yet).
563    pub wal_pages: Option<u64>,
564    /// Logical frames present in the WAL at the main backend's most recent
565    /// periodic PASSIVE checkpoint. Kept separate from physical allocation.
566    #[serde(default)]
567    pub wal_log_frames: Option<u64>,
568    /// Frames backfilled by that same periodic PASSIVE pass.
569    #[serde(default)]
570    pub wal_checkpointed_frames: Option<u64>,
571    /// Logical frames still pending after that pass (`log - checkpointed`).
572    #[serde(default)]
573    pub wal_pending_frames: Option<u64>,
574    /// Physical `-wal` sidecar bytes captured at the same routine tick.
575    #[serde(default)]
576    pub wal_physical_bytes: Option<u64>,
577    /// Wall-clock timestamp of the routine WAL sample, for staleness checks.
578    #[serde(default)]
579    pub wal_observed_at_unix_ms: Option<u64>,
580    /// Total WAL TRUNCATE escalation attempts (ADR-091 Plank 2) made in this
581    /// process's lifetime, regardless of whether they succeeded in reclaiming
582    /// pages.
583    pub wal_truncate_attempts: u64,
584    /// Current consecutive-failure count for TRUNCATE attempts that failed to
585    /// bring the WAL back below `warn_pages`; resets to 0 the next time an
586    /// attempt clears it.
587    pub wal_truncate_consecutive_failures: u64,
588    /// Total checkpoint ticks skipped because the dedicated checkpoint
589    /// connection was unavailable (ADR-091 checkpoint-pressure telemetry),
590    /// across this process's lifetime. `#[serde(default)]` so an older client
591    /// decoding a newer daemon's snapshot (or vice versa) does not fail.
592    #[serde(default)]
593    pub wal_checkpoint_skipped_ticks: u64,
594    /// Current consecutive-skip run length; 0 once the next tick is observed.
595    #[serde(default)]
596    pub wal_checkpoint_consecutive_skips: u64,
597    /// WAL page count last known at the time of the most recent skip, if any
598    /// skip has occurred yet in this process.
599    #[serde(default)]
600    pub wal_checkpoint_last_skip_wal_pages: Option<u64>,
601    /// Age, in microseconds, of the oldest currently-open transaction
602    /// registry entry (ADR-091 Plank 0). `None` when no transaction is
603    /// currently open.
604    pub oldest_pinned_tx_micros: Option<u64>,
605    /// Diagnostic label of the oldest currently-open transaction registry
606    /// entry, if any and if it was registered with one.
607    pub oldest_pinned_tx_label: Option<String>,
608    /// Number of currently open transaction registry entries.
609    pub open_tx_count: usize,
610    /// Current write-queue backlog depth (ADR-067 Component A): requests
611    /// enqueued but not yet accepted by the `WriterTask` drain loop. `None`
612    /// unless the write queue is enabled (`KHIVE_WRITE_QUEUE=1`) and a
613    /// file-backed pool is available.
614    pub write_queue_depth: Option<usize>,
615    /// The write queue's configured bounded capacity
616    /// (`PoolConfig::write_queue_capacity`), gated the same as
617    /// `write_queue_depth`.
618    pub write_queue_capacity: Option<usize>,
619    /// Latest completed writer-task span: bounded-channel admission/backlog.
620    #[serde(default)]
621    pub write_last_queue_wait_micros: Option<u64>,
622    /// Latest completed writer-task span: `BEGIN IMMEDIATE` acquisition.
623    #[serde(default)]
624    pub write_last_transaction_acquire_micros: Option<u64>,
625    /// Latest completed writer-task span: application transaction body.
626    #[serde(default)]
627    pub write_last_body_micros: Option<u64>,
628    /// Latest completed writer-task span: SQLite COMMIT/fsync phase.
629    #[serde(default)]
630    pub write_last_commit_micros: Option<u64>,
631    /// Whole latest writer-task request span, retained for compatibility and
632    /// comparison with the decomposed stages.
633    #[serde(default)]
634    pub write_last_total_micros: Option<u64>,
635    /// Wall-clock timestamp of the writer-stage sample.
636    #[serde(default)]
637    pub write_last_observed_at_unix_ms: Option<u64>,
638}
639
640// ── framing ───────────────────────────────────────────────────────────────────
641
642/// Read one length-prefixed frame (4-byte BE u32 length + JSON bytes).
643#[cfg(unix)]
644pub async fn read_frame<R>(stream: &mut R) -> std::io::Result<Vec<u8>>
645where
646    R: tokio::io::AsyncRead + Unpin,
647{
648    let mut len_buf = [0u8; 4];
649    stream.read_exact(&mut len_buf).await?;
650    let len = u32::from_be_bytes(len_buf) as usize;
651    if len > MAX_FRAME_BYTES {
652        return Err(std::io::Error::new(
653            std::io::ErrorKind::InvalidData,
654            format!("daemon frame of {len} bytes exceeds {MAX_FRAME_BYTES} cap"),
655        ));
656    }
657    let mut buf = vec![0u8; len];
658    stream.read_exact(&mut buf).await?;
659    Ok(buf)
660}
661
662/// Write one length-prefixed frame.
663#[cfg(unix)]
664pub async fn write_frame<W>(stream: &mut W, payload: &[u8]) -> std::io::Result<()>
665where
666    W: tokio::io::AsyncWrite + Unpin,
667{
668    if payload.len() > MAX_FRAME_BYTES {
669        return Err(std::io::Error::new(
670            std::io::ErrorKind::InvalidData,
671            format!(
672                "daemon frame of {} bytes exceeds {MAX_FRAME_BYTES} cap",
673                payload.len()
674            ),
675        ));
676    }
677    let len = (payload.len() as u32).to_be_bytes();
678    stream.write_all(&len).await?;
679    stream.write_all(payload).await?;
680    stream.flush().await?;
681    Ok(())
682}
683
684// ── dispatch trait ────────────────────────────────────────────────────────────
685
686/// Transport-agnostic dispatch interface for the daemon server.
687///
688/// The MCP crate implements this by dispatching through the shared request body
689/// while honoring [`DaemonRequestFrame::from_wire`] (so subhandler visibility is
690/// gated by request origin, not by transport); any future transport can do the
691/// same.
692#[cfg(unix)]
693#[async_trait]
694pub trait DaemonDispatch: Clone + Send + Sync + 'static {
695    /// Dispatch a verb-DSL request string and return the rendered result.
696    ///
697    /// `from_wire` carries the origin discriminator from
698    /// [`DaemonRequestFrame::from_wire`]: when `true`, the implementor enforces
699    /// verb visibility (rejects `Visibility::Subhandler` verbs); when `false`,
700    /// the request is from a trusted operator surface and subhandlers pass.
701    ///
702    /// `identity` is the per-request identity context threaded from the frame
703    /// (ADR-096): `Some(..)` when serving a request forwarded over the
704    /// daemon socket (built from `frame.namespace` / `frame.actor_id` /
705    /// `frame.visible_namespaces` by the connection handler), `None` for any
706    /// other dispatch path. Implementors should mint the storage/gate token from
707    /// `identity` when present and fall back to their own construction-baked
708    /// identity when absent, so pure local (non-daemon) dispatch is unchanged.
709    #[allow(clippy::too_many_arguments)]
710    async fn dispatch(
711        &self,
712        ops: String,
713        presentation: Option<String>,
714        presentation_per_op: Option<Vec<Option<String>>>,
715        format: Option<String>,
716        format_per_op: Option<Vec<Option<String>>>,
717        from_wire: bool,
718        identity: Option<RequestIdentity>,
719    ) -> Result<String, String>;
720
721    /// Warm every pack's in-memory state (ANN indexes, etc.).
722    async fn warm_all(&self);
723
724    /// The namespace this dispatcher was configured for.
725    fn namespace(&self) -> &str;
726
727    /// Fingerprint of this dispatcher's resolved runtime config (packs, db
728    /// target, embedders). Used to reject forwarded requests from clients whose
729    /// config differs, so a restricted client cannot dispatch through a broader
730    /// daemon.
731    fn config_id(&self) -> &str;
732
733    /// Return the pool to use for background WAL checkpointing, if available.
734    ///
735    /// Implementors backed by a file-based SQLite database should return
736    /// `Some(pool_arc)`. In-memory or test dispatchers that have no pool
737    /// return `None` and the checkpoint task is not spawned.
738    ///
739    /// The default implementation returns `None`.
740    fn pool_for_checkpoint(&self) -> Option<Arc<ConnectionPool>> {
741        None
742    }
743
744    /// File-backed backend pools beyond [`Self::pool_for_checkpoint`]'s pool
745    /// (ADR-091 Amendment 3): one checkpoint task is spawned per entry here,
746    /// in addition to the one spawned for the primary pool, so a
747    /// multi-backend deployment gets PASSIVE/TRUNCATE checkpointing and
748    /// sidecar enumeration on every file-backed backend it wired, not only
749    /// the main one.
750    ///
751    /// The default implementation returns an empty `Vec` — an implementor
752    /// with only one backend (or none) needs no override.
753    fn secondary_pools_for_checkpoint(&self) -> Vec<Arc<ConnectionPool>> {
754        Vec::new()
755    }
756
757    /// Return the audit `EventStore` the checkpoint task should append
758    /// ADR-094 lifecycle events (`CheckpointOutcomeRecorded`) to, if any.
759    ///
760    /// Mirrors [`Self::pool_for_checkpoint`]'s default-`None` shape: an
761    /// implementor with no configured event store (or no pool at all) simply
762    /// gets a checkpoint task that never appends events — the checkpoint
763    /// task itself remains fully functional either way.
764    ///
765    /// The default implementation returns `None`.
766    fn event_store_for_checkpoint(&self) -> Option<Arc<dyn khive_storage::EventStore>> {
767        None
768    }
769}
770
771#[cfg(unix)]
772struct CheckpointTaskSpec {
773    pool: Arc<ConnectionPool>,
774    lifecycle_owner: Option<CheckpointLifecycleOwner>,
775    is_main: bool,
776}
777
778/// Build checkpoint-task fan-out and designate one lifecycle owner.
779///
780/// The main checkpoint task owns lifecycle emission when it exists. If the
781/// main backend is in-memory and therefore has no checkpoint task, the first
782/// file-backed secondary owns emission instead. All remaining tasks are
783/// explicit non-owners.
784#[cfg(unix)]
785fn checkpoint_task_specs(
786    main_pool: Option<Arc<ConnectionPool>>,
787    secondary_pools: Vec<Arc<ConnectionPool>>,
788    event_store: Option<Arc<dyn khive_storage::EventStore>>,
789    namespace: String,
790) -> Vec<CheckpointTaskSpec> {
791    let mut tasks = Vec::with_capacity(usize::from(main_pool.is_some()) + secondary_pools.len());
792    if let Some(pool) = main_pool {
793        tasks.push(CheckpointTaskSpec {
794            pool,
795            lifecycle_owner: None,
796            is_main: true,
797        });
798    }
799    tasks.extend(secondary_pools.into_iter().map(|pool| CheckpointTaskSpec {
800        pool,
801        lifecycle_owner: None,
802        is_main: false,
803    }));
804
805    if let (Some(task), Some(event_store)) = (tasks.first_mut(), event_store) {
806        task.lifecycle_owner = Some(CheckpointLifecycleOwner::new(event_store, namespace));
807    }
808    tasks
809}
810
811// ── tracked background tasks ─────────────────────────────────────────────────
812//
813// Pack handlers (e.g. memory.recall's ADR-081 serve-ledger append) fire
814// fire-and-forget `tokio::spawn`ed work off the response path so the caller
815// never waits on a cross-pack dispatch or a SQL write. Left untracked, that
816// work is invisible to `drain()`: a SIGTERM landing between the response
817// returning and the spawned task completing can abort it mid-flight with no
818// log and no row. `track_background_task` gives such spawns a process-wide
819// presence that `drain()` waits on, exactly like the `active` counter does
820// for in-flight connections: the caller still only pays for the spawn +
821// counter increment, never the task's own work.
822static BACKGROUND_TASKS: std::sync::OnceLock<Arc<std::sync::atomic::AtomicUsize>> =
823    std::sync::OnceLock::new();
824
825fn background_tasks() -> &'static Arc<std::sync::atomic::AtomicUsize> {
826    BACKGROUND_TASKS.get_or_init(|| Arc::new(std::sync::atomic::AtomicUsize::new(0)))
827}
828
829/// Decrements the shared background-task counter from `Drop`, so the count
830/// comes back down whether the tracked future returns normally, panics, or
831/// is cancelled — a plain post-`await` `fetch_sub` only covers the return
832/// path and leaks the count forever on a panic, since unwinding skips every
833/// statement after the panic point.
834struct BackgroundTaskGuard {
835    counter: Arc<std::sync::atomic::AtomicUsize>,
836}
837
838impl Drop for BackgroundTaskGuard {
839    fn drop(&mut self) {
840        self.counter
841            .fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
842    }
843}
844
845/// Spawn a task that daemon shutdown's `drain()` waits for and return its join
846/// handle. Retaining the handle lets boot coordinators form an explicit barrier;
847/// dropping it deliberately detaches the task while the background counter still
848/// keeps daemon drain aware of its lifetime. The decrement happens via
849/// `BackgroundTaskGuard`'s `Drop`, including panic and cancellation paths.
850pub fn spawn_tracked_task<F, T>(fut: F) -> tokio::task::JoinHandle<T>
851where
852    F: std::future::Future<Output = T> + Send + 'static,
853    T: Send + 'static,
854{
855    background_tasks().fetch_add(1, std::sync::atomic::Ordering::SeqCst);
856    let guard = BackgroundTaskGuard {
857        counter: background_tasks().clone(),
858    };
859    tokio::spawn(async move {
860        let _guard = guard;
861        fut.await
862    })
863}
864
865/// Spawn a fire-and-forget task through [`spawn_tracked_task`].
866///
867/// Callers that need a boot or shutdown barrier should retain and await the
868/// returned handle from [`spawn_tracked_task`] instead of detaching it here.
869pub fn track_background_task<F>(fut: F)
870where
871    F: std::future::Future<Output = ()> + Send + 'static,
872{
873    drop(spawn_tracked_task(fut));
874}
875
876/// Current count of in-flight tasks started via [`track_background_task`].
877/// Exposed for tests; `drain()` reads the shared counter directly.
878pub fn background_task_count() -> usize {
879    background_tasks().load(std::sync::atomic::Ordering::SeqCst)
880}
881
882/// Process-wide daemon shutdown signal (ADR-119).
883///
884/// Cancelled exactly once, when the daemon's unified shutdown future resolves
885/// — before `drain()` begins waiting on tracked tasks — so long-running
886/// daemon components supervised outside this module observe shutdown through
887/// the same path the daemon itself does, rather than inventing their own.
888/// Clones share the underlying token; child tokens derived from it are
889/// cancelled transitively.
890///
891/// In non-daemon processes the token simply never fires.
892pub fn daemon_shutdown_token() -> tokio_util::sync::CancellationToken {
893    static TOKEN: std::sync::OnceLock<tokio_util::sync::CancellationToken> =
894        std::sync::OnceLock::new();
895    TOKEN
896        .get_or_init(tokio_util::sync::CancellationToken::new)
897        .clone()
898}
899
900// ── active background phase names (ADR-103) ──────────────────────────────────
901//
902// A lightweight, best-effort process-wide gauge of which named background
903// phases (e.g. `ann_warm`) are in flight right now, read by `comm.health`'s
904// resource self-report so a caller can see "what is the daemon doing" at a
905// glance without correlating timestamps across the event log itself. Counted
906// per name rather than boolean, since more than one occurrence of the same
907// named phase can legitimately overlap (e.g. two embedding models warming
908// concurrently) — the name only drops out of the reported set once every
909// concurrent occurrence has ended.
910static ACTIVE_PHASES: std::sync::OnceLock<
911    std::sync::Mutex<std::collections::HashMap<String, usize>>,
912> = std::sync::OnceLock::new();
913
914fn active_phases() -> &'static std::sync::Mutex<std::collections::HashMap<String, usize>> {
915    ACTIVE_PHASES.get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new()))
916}
917
918/// RAII guard for one occurrence of a named background phase. Increments the
919/// phase's count on creation (see [`register_active_phase`]); decrements on
920/// `Drop`, so the count comes back down whether the guarded work returns
921/// normally, panics, or is cancelled — the same rationale as
922/// `BackgroundTaskGuard` above.
923pub struct PhaseGuard {
924    name: String,
925}
926
927impl Drop for PhaseGuard {
928    fn drop(&mut self) {
929        let mut map = active_phases()
930            .lock()
931            .unwrap_or_else(std::sync::PoisonError::into_inner);
932        if let Some(count) = map.get_mut(&self.name) {
933            *count -= 1;
934            if *count == 0 {
935                map.remove(&self.name);
936            }
937        }
938    }
939}
940
941/// Register one occurrence of a named background phase as currently active.
942/// Returns a guard: drop it (or let it fall out of scope) when the phase
943/// ends. Best-effort process-wide gauge only, read by `comm.health` — never
944/// load-bearing for correctness.
945pub fn register_active_phase(name: &str) -> PhaseGuard {
946    let mut map = active_phases()
947        .lock()
948        .unwrap_or_else(std::sync::PoisonError::into_inner);
949    *map.entry(name.to_string()).or_insert(0) += 1;
950    PhaseGuard {
951        name: name.to_string(),
952    }
953}
954
955/// Currently active background-phase names, sorted for deterministic output.
956/// Empty when no tracked phase is in flight.
957pub fn active_phase_names() -> Vec<String> {
958    let map = active_phases()
959        .lock()
960        .unwrap_or_else(std::sync::PoisonError::into_inner);
961    let mut names: Vec<String> = map.keys().cloned().collect();
962    names.sort();
963    names
964}
965
966// ── server ────────────────────────────────────────────────────────────────────
967
968/// Build a point-in-time [`MetricsSnapshot`] of this process's server-side
969/// gauges. Called only from `handle_conn`'s `metrics_only` arm — a
970/// process-global, read-only assembly with no side effects of its own.
971/// See `docs/api/daemon.md#build_metrics_snapshot` for where each gauge is sourced
972/// from and why.
973#[cfg(unix)]
974fn build_metrics_snapshot<D: DaemonDispatch>(dispatcher: &D) -> MetricsSnapshot {
975    let open_tx_count = khive_storage::tx_registry::snapshot().len();
976    // ADR-091 Amendment 3: deliberately the process-wide aggregate, not a
977    // backend-attributed view — this gauge reports "oldest pinned tx in this
978    // process" across every wired backend, not an attribution claim about
979    // which database it belongs to. Attribution consumers (the session sweep,
980    // the per-backend checkpoint tasks) use the scoped `oldest_for` views in
981    // khive-db instead.
982    let (oldest_pinned_tx_micros, oldest_pinned_tx_label) =
983        match khive_storage::tx_registry::oldest() {
984            Some((_id, age, label)) => (Some(age.as_micros() as u64), label),
985            None => (None, None),
986        };
987
988    let checkpoint_pool = dispatcher.pool_for_checkpoint();
989    let routine_wal = checkpoint_pool
990        .as_deref()
991        .and_then(khive_db::checkpoint::routine_wal_observation);
992    let writer_stages = checkpoint_pool
993        .as_deref()
994        .and_then(khive_db::writer_task::last_writer_stage_observation);
995    let (write_queue_depth, write_queue_capacity) = checkpoint_pool
996        .as_ref()
997        .and_then(|pool| pool.writer_task_handle().ok().flatten())
998        .map(|handle| (Some(handle.queue_depth()), Some(handle.capacity())))
999        .unwrap_or((None, None));
1000
1001    MetricsSnapshot {
1002        wal_pages: routine_wal.as_ref().map(|sample| sample.log_frames),
1003        wal_log_frames: routine_wal.as_ref().map(|sample| sample.log_frames),
1004        wal_checkpointed_frames: routine_wal
1005            .as_ref()
1006            .map(|sample| sample.checkpointed_frames),
1007        wal_pending_frames: routine_wal.as_ref().map(|sample| sample.pending_frames),
1008        wal_physical_bytes: routine_wal
1009            .as_ref()
1010            .and_then(|sample| sample.physical_wal_bytes),
1011        wal_observed_at_unix_ms: routine_wal
1012            .as_ref()
1013            .map(|sample| sample.observed_at_unix_ms),
1014        wal_truncate_attempts: khive_db::checkpoint::truncate_attempts(),
1015        wal_truncate_consecutive_failures: khive_db::checkpoint::truncate_consecutive_failures(),
1016        wal_checkpoint_skipped_ticks: khive_db::checkpoint::checkpoint_skipped_ticks(),
1017        wal_checkpoint_consecutive_skips: khive_db::checkpoint::checkpoint_consecutive_skips(),
1018        wal_checkpoint_last_skip_wal_pages: khive_db::checkpoint::checkpoint_last_skip_wal_pages(),
1019        oldest_pinned_tx_micros,
1020        oldest_pinned_tx_label,
1021        open_tx_count,
1022        write_queue_depth,
1023        write_queue_capacity,
1024        write_last_queue_wait_micros: writer_stages
1025            .as_ref()
1026            .map(|sample| sample.queue_wait_micros),
1027        write_last_transaction_acquire_micros: writer_stages
1028            .as_ref()
1029            .map(|sample| sample.transaction_acquire_micros),
1030        write_last_body_micros: writer_stages.as_ref().map(|sample| sample.body_micros),
1031        write_last_commit_micros: writer_stages.as_ref().map(|sample| sample.commit_micros),
1032        write_last_total_micros: writer_stages.as_ref().map(|sample| sample.total_micros),
1033        write_last_observed_at_unix_ms: writer_stages
1034            .as_ref()
1035            .map(|sample| sample.observed_at_unix_ms),
1036    }
1037}
1038
1039#[cfg(unix)]
1040async fn wait_for_peer_disconnect(read: &mut tokio::net::unix::OwnedReadHalf) {
1041    let mut byte = [0u8; 1];
1042    // One request is admitted per connection. EOF, a read error, or any
1043    // subsequent byte all make this connection no longer a valid response
1044    // peer, so the first completed read is the entire observation.
1045    let _ = read.read(&mut byte).await;
1046}
1047
1048#[cfg(all(unix, test))]
1049async fn handle_conn<D: DaemonDispatch>(stream: UnixStream, dispatcher: D) {
1050    handle_conn_with_shutdown(stream, dispatcher, None).await;
1051}
1052
1053#[cfg(unix)]
1054async fn handle_conn_with_shutdown<D: DaemonDispatch>(
1055    mut stream: UnixStream,
1056    dispatcher: D,
1057    shutdown: Option<tokio::sync::watch::Receiver<bool>>,
1058) {
1059    let (local_shutdown_tx, local_shutdown_rx) = tokio::sync::watch::channel(false);
1060    let shutdown = shutdown.unwrap_or(local_shutdown_rx);
1061    // Keeps the fallback receiver open in direct/test calls. Production owns
1062    // a sender at the daemon-run scope and passes its receiver above.
1063    let _local_shutdown_tx = local_shutdown_tx;
1064    let raw = match read_frame(&mut stream).await {
1065        Ok(r) => r,
1066        Err(e) => {
1067            tracing::debug!(error = %e, "failed to read daemon request frame");
1068            return;
1069        }
1070    };
1071    let frame: DaemonRequestFrame = match serde_json::from_slice(&raw) {
1072        Ok(f) => f,
1073        Err(e) => {
1074            tracing::debug!(error = %e, "failed to decode daemon request frame");
1075            return;
1076        }
1077    };
1078    let (mut peer_read, mut peer_write) = stream.into_split();
1079
1080    let served_config_id = Some(dispatcher.config_id().to_string());
1081    let resp = if frame.protocol_version != PROTOCOL_VERSION {
1082        let msg = format!(
1083            "daemon protocol mismatch: client={} daemon={} — \
1084             rebuild/update the client binary (make local)",
1085            frame.protocol_version, PROTOCOL_VERSION,
1086        );
1087        tracing::warn!(
1088            client_version = frame.protocol_version,
1089            daemon_version = PROTOCOL_VERSION,
1090            "daemon protocol version mismatch"
1091        );
1092        DaemonResponseFrame {
1093            ok: false,
1094            result: None,
1095            error: Some(msg),
1096            namespace_mismatch: false,
1097            config_mismatch: false,
1098            served_config_id,
1099            version_mismatch: true,
1100            daemon_protocol_version: PROTOCOL_VERSION,
1101            metrics: None,
1102            request_id: frame.request_id,
1103        }
1104    } else if frame.metrics_only {
1105        // Process-global gauge read: namespace/config-agnostic, so this is
1106        // handled BEFORE the `config_id` equality reject below (unlike every
1107        // other arm) — a metrics probe must work regardless of which
1108        // client's config is asking, since it never touches the dispatcher's
1109        // packs/db/embed registry. READ-ONLY: builds a snapshot and returns
1110        // immediately, never reaching the ops-dispatch arm.
1111        DaemonResponseFrame {
1112            ok: true,
1113            result: None,
1114            error: None,
1115            namespace_mismatch: false,
1116            config_mismatch: false,
1117            served_config_id,
1118            version_mismatch: false,
1119            daemon_protocol_version: PROTOCOL_VERSION,
1120            metrics: Some(build_metrics_snapshot(&dispatcher)),
1121            request_id: frame.request_id,
1122        }
1123    // There is no `frame.namespace != dispatcher.namespace()` reject here.
1124    // The daemon accepts and serves the request under the frame's own
1125    // identity (namespace / actor / visible_namespaces, built into a
1126    // `RequestIdentity` below) over its one shared warm registry, rather
1127    // than rejecting a differently-attributed same-uid connection to a cold
1128    // local-dispatch fallback. `config_id`: which governs packs/db/embed
1129    // coherence for the shared warm engine: remains a hard reject; it is
1130    // not an identity field and softening it would let a restricted client
1131    // dispatch through an incompatible broader daemon.
1132    } else if frame.config_id != dispatcher.config_id() {
1133        DaemonResponseFrame {
1134            ok: false,
1135            result: None,
1136            error: None,
1137            namespace_mismatch: false,
1138            config_mismatch: true,
1139            served_config_id,
1140            version_mismatch: false,
1141            daemon_protocol_version: PROTOCOL_VERSION,
1142            metrics: None,
1143            request_id: frame.request_id,
1144        }
1145    } else if frame.probe_only {
1146        // Probe-only request: identity checks passed; return immediately without
1147        // dispatching any verb. The client uses this to confirm the daemon is
1148        // alive and identity-matching without triggering any mutation.
1149        DaemonResponseFrame {
1150            ok: true,
1151            result: None,
1152            error: None,
1153            namespace_mismatch: false,
1154            config_mismatch: false,
1155            served_config_id,
1156            version_mismatch: false,
1157            daemon_protocol_version: PROTOCOL_VERSION,
1158            metrics: None,
1159            request_id: frame.request_id,
1160        }
1161    } else {
1162        // Build the per-request identity context from the frame so the
1163        // implementor mints the storage/gate token from the CALLER's
1164        // identity, not the dispatcher's own construction-baked scalars.
1165        // This is always `Some` here: every frame that reaches this arm
1166        // carries a `namespace` (required on the wire) plus whatever
1167        // `actor_id`/`visible_namespaces` the client resolved (defaulting to
1168        // `None`/`vec![]` for an older, field-absent payload, which is
1169        // exactly the prior anonymous/no-extra-visibility behavior).
1170        let identity = RequestIdentity {
1171            namespace: frame.namespace.clone(),
1172            actor_id: frame.actor_id.clone(),
1173            visible_namespaces: frame.visible_namespaces.clone(),
1174            process_ref: frame.process_ref.clone(),
1175            request_id: frame.request_id,
1176        };
1177        let (read_cancel_tx, read_cancel_rx) = tokio::sync::watch::channel(false);
1178        let dispatch = khive_storage::scope_request_read_cancellation(
1179            shutdown,
1180            khive_storage::scope_request_read_cancellation(
1181                read_cancel_rx,
1182                khive_storage::scope_request_read_deadline(
1183                    khive_storage::request_read_timeout_from_env(),
1184                    dispatcher.dispatch(
1185                        frame.ops,
1186                        frame.presentation,
1187                        frame.presentation_per_op,
1188                        frame.format,
1189                        frame.format_per_op,
1190                        frame.from_wire,
1191                        Some(identity),
1192                    ),
1193                ),
1194            ),
1195        );
1196        tokio::pin!(dispatch);
1197        let dispatch_result = tokio::select! {
1198            result = &mut dispatch => result,
1199            _ = wait_for_peer_disconnect(&mut peer_read) => {
1200                let _ = read_cancel_tx.send(true);
1201                dispatch.await
1202            }
1203        };
1204        match dispatch_result {
1205            Ok(result) => DaemonResponseFrame {
1206                ok: true,
1207                result: Some(result),
1208                error: None,
1209                namespace_mismatch: false,
1210                config_mismatch: false,
1211                served_config_id,
1212                version_mismatch: false,
1213                daemon_protocol_version: PROTOCOL_VERSION,
1214                metrics: None,
1215                request_id: frame.request_id,
1216            },
1217            Err(e) => DaemonResponseFrame {
1218                ok: false,
1219                result: None,
1220                error: Some(e),
1221                namespace_mismatch: false,
1222                config_mismatch: false,
1223                served_config_id,
1224                version_mismatch: false,
1225                daemon_protocol_version: PROTOCOL_VERSION,
1226                metrics: None,
1227                request_id: frame.request_id,
1228            },
1229        }
1230    };
1231
1232    match serde_json::to_vec(&resp) {
1233        Ok(payload) => {
1234            if payload.len() > MAX_FRAME_BYTES {
1235                // The serialized response exceeds the IPC frame cap.  Send a
1236                // small explicit error frame so the client can distinguish a
1237                // per-request payload-size failure from a daemon crash.  A
1238                // client that receives this error frame will NOT trigger
1239                // stale-daemon kill/respawn (ParseFailure requires a read_frame
1240                // error, not an ok=false result).
1241                tracing::warn!(
1242                    bytes = payload.len(),
1243                    limit = MAX_FRAME_BYTES,
1244                    "daemon response exceeds MAX_FRAME_BYTES; sending explicit error frame"
1245                );
1246                let err_resp = DaemonResponseFrame {
1247                    ok: false,
1248                    result: None,
1249                    error: Some(format!(
1250                        "response too large: {} bytes exceeds {} byte IPC cap",
1251                        payload.len(),
1252                        MAX_FRAME_BYTES,
1253                    )),
1254                    namespace_mismatch: false,
1255                    config_mismatch: false,
1256                    served_config_id: resp.served_config_id,
1257                    version_mismatch: false,
1258                    daemon_protocol_version: PROTOCOL_VERSION,
1259                    metrics: None,
1260                    request_id: resp.request_id,
1261                };
1262                if let Ok(err_payload) = serde_json::to_vec(&err_resp) {
1263                    if let Err(e) = write_frame(&mut peer_write, &err_payload).await {
1264                        tracing::debug!(error = %e, "failed to write oversized-response error frame");
1265                    }
1266                }
1267            } else if let Err(e) = write_frame(&mut peer_write, &payload).await {
1268                tracing::debug!(error = %e, "failed to write daemon response frame");
1269            }
1270        }
1271        Err(e) => tracing::warn!(error = %e, "failed to serialize daemon response frame"),
1272    }
1273}
1274
1275/// An accepted connection owns a drain slot from the accept loop until its
1276/// handler future is dropped. The claim happens before spawning so shutdown
1277/// cannot observe zero in the gap between `accept()` and the task's first poll.
1278#[cfg(unix)]
1279struct ActiveConnectionGuard {
1280    active: Arc<std::sync::atomic::AtomicUsize>,
1281}
1282
1283#[cfg(unix)]
1284impl ActiveConnectionGuard {
1285    fn claim(active: Arc<std::sync::atomic::AtomicUsize>) -> Self {
1286        active.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
1287        Self { active }
1288    }
1289}
1290
1291#[cfg(unix)]
1292impl Drop for ActiveConnectionGuard {
1293    fn drop(&mut self) {
1294        self.active
1295            .fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
1296    }
1297}
1298
1299#[cfg(unix)]
1300fn spawn_connection_task<F>(
1301    active: Arc<std::sync::atomic::AtomicUsize>,
1302    future: F,
1303) -> tokio::task::JoinHandle<()>
1304where
1305    F: std::future::Future<Output = ()> + Send + 'static,
1306{
1307    let guard = ActiveConnectionGuard::claim(active);
1308    tokio::spawn(async move {
1309        let _guard = guard;
1310        future.await;
1311    })
1312}
1313
1314/// Run the daemon: bind the socket, warm in the background, serve request
1315/// frames until SIGTERM/SIGINT.
1316///
1317/// Fatally acquires its own startup lock, which only protects
1318/// cleanup→bind→pid-write — `dispatcher` has already run migrations and
1319/// applied pack schema plans while constructing itself, unguarded. Production
1320/// boot must go through [`run_daemon_with_boot_guard`] instead, which extends
1321/// the same lock back over construction. This entry point is for callers
1322/// (and tests) that build the dispatcher and start serving as one atomic
1323/// step with no separate boot-guard window to protect.
1324#[cfg(unix)]
1325pub async fn run_daemon<D: DaemonDispatch>(dispatcher: D) -> anyhow::Result<()> {
1326    let boot_guard = Some(acquire_daemon_boot_guard()?);
1327    run_daemon_with_boot_guard_inner(dispatcher, boot_guard, false).await
1328}
1329
1330/// Run a real daemon server for an in-process multi-launch test.
1331///
1332/// Separate production daemon candidates have distinct PIDs, so the boot fence
1333/// recognizes a responsive incumbent and makes later candidates exit. Parallel
1334/// test launchers share one OS process and therefore one PID; this explicit
1335/// fault-injection entry point preserves the production fence semantics by
1336/// allowing a responsive same-PID incumbent to win. Ordinary daemon startup
1337/// continues to treat a same-PID rendezvous as stale, protecting PID-reuse
1338/// cleanup behavior.
1339///
1340/// A losing candidate still follows the ordinary daemon-exit path and cancels
1341/// the process-wide component shutdown token. Callers must therefore use a
1342/// component-free dispatcher; this seam validates socket/PID ownership, not
1343/// multi-candidate component lifecycle.
1344#[cfg(all(unix, any(test, feature = "fault-injection")))]
1345#[doc(hidden)]
1346pub async fn run_daemon_in_process_test<D: DaemonDispatch>(dispatcher: D) -> anyhow::Result<()> {
1347    let boot_guard = Some(acquire_daemon_boot_guard()?);
1348    run_daemon_with_boot_guard_inner(dispatcher, boot_guard, true).await
1349}
1350
1351/// Vet the socket's parent directory, re-permissioning it only when it is the
1352/// directory khive owns by convention.
1353///
1354/// `KHIVE_SOCKET` takes an arbitrary path, and the previous unconditional
1355/// chmod-0700 of its parent was wrong in both directions: pointed at a shared
1356/// parent like `/tmp` it either failed outright for an ordinary user, or —
1357/// worse — succeeded when privileged and stripped access for every other
1358/// process on the machine. A directory we did not create is never modified.
1359///
1360/// What the directory must actually prevent is a *takeover of the socket
1361/// path*: the connection gate is the 0600 socket plus the accept-time
1362/// peer-uid check, but both defend the daemon's own socket — neither helps
1363/// once another local user can put *their* listener at the path clients
1364/// resolve. There are two ways a shared directory allows that, and the
1365/// sticky bit closes neither: a writer can *pre-bind* the predictable path
1366/// before this daemon starts (the sticky bit restricts unlinking, not
1367/// creating), and the directory's *owner* can unlink and rebind even in a
1368/// 1777 directory (sticky exempts the directory owner). So a caller-chosen
1369/// directory is served only when it is trusted end to end: owned by this
1370/// daemon's euid or root, and not writable by group or other at all. The
1371/// umask-default 0755 stays acceptable; shared sticky directories like
1372/// `/tmp` do not.
1373#[cfg(unix)]
1374pub(crate) fn ensure_socket_dir_is_trusted(parent: &std::path::Path) -> anyhow::Result<()> {
1375    // SAFETY: `geteuid` is always successful and takes no arguments.
1376    let daemon_euid = unsafe { libc::geteuid() } as u32;
1377
1378    if parent == khive_dir() {
1379        std::fs::set_permissions(parent, std::fs::Permissions::from_mode(0o700)).map_err(|e| {
1380            anyhow::anyhow!(
1381                "refusing to start: cannot chmod 0700 {}: {e}. The khive directory must be \
1382                 owner-only — it is half of the same-uid guarantee this daemon enforces.",
1383                parent.display()
1384            )
1385        })?;
1386        return ensure_socket_path_is_swap_resistant(parent, daemon_euid);
1387    }
1388
1389    // Fail closed on the stat itself: not being able to read the metadata is
1390    // not the same as the directory passing.
1391    let meta = std::fs::metadata(parent).map_err(|e| {
1392        anyhow::anyhow!(
1393            "refusing to start: cannot stat {}: {e}. The socket directory gates \
1394             socket-takeover safety, and unreadable metadata is not a passing state.",
1395            parent.display()
1396        )
1397    })?;
1398
1399    use std::os::unix::fs::MetadataExt;
1400    let owner = meta.uid();
1401    if owner != daemon_euid && owner != 0 {
1402        anyhow::bail!(
1403            "refusing to start: socket directory {} is owned by uid {owner}, not this \
1404             daemon's uid ({daemon_euid}) or root. A directory owner can replace the \
1405             socket regardless of mode bits. Point KHIVE_SOCKET at a directory you own, \
1406             or unset it for the default.",
1407            parent.display()
1408        );
1409    }
1410
1411    let mode = meta.permissions().mode();
1412    if mode & 0o022 != 0 {
1413        anyhow::bail!(
1414            "refusing to start: socket directory {} is mode {:04o} — writable by group or \
1415             other, so another local user could bind their own listener at the socket path \
1416             (before this daemon starts, the sticky bit does not prevent creating the \
1417             path). Use a directory only you can write, or unset KHIVE_SOCKET for the \
1418             default. This daemon is not changing the permissions of a directory it does \
1419             not own.",
1420            parent.display(),
1421            mode & 0o7777
1422        );
1423    }
1424
1425    ensure_socket_path_is_swap_resistant(parent, daemon_euid)
1426}
1427
1428/// Walk the socket directory path exactly as the kernel will traverse it at
1429/// bind time — component by component, following symlinks — and refuse any
1430/// node another local user could swap after this validation.
1431///
1432/// Checking only the canonicalized result is not enough: bind and every
1433/// client traverse the *original* path, so a symlink component owned by
1434/// another user can resolve somewhere trusted while it is being validated
1435/// and be retargeted before the socket is bound. Validating the nodes the
1436/// traversal actually visits closes that gap: a symlink component is
1437/// acceptable only when its owner — the only party besides root who can
1438/// retarget it — is this daemon's euid or root, and every directory
1439/// component is held to the ancestor rule below.
1440///
1441/// The directory rule is deliberately weaker than the immediate parent's.
1442/// The parent hosts the socket file, where the threat is *creation* of the
1443/// predictable path — the sticky bit does not restrict creating, so no
1444/// sticky exception is sound there. An ancestor only threatens via
1445/// *rename/unlink of an existing entry we own*, which the sticky bit does
1446/// restrict: in a sticky directory, only the entry's owner, the directory's
1447/// owner, or root may rename it. So a root-owned `/tmp` (1777) is an
1448/// acceptable ancestor of a user-owned 0700 socket directory, while a
1449/// non-sticky group/other-writable ancestor, or one owned by a third uid
1450/// (who could rename the entry, or chmod the directory first), is refused.
1451/// Every stat failure fails closed.
1452#[cfg(unix)]
1453fn ensure_socket_path_is_swap_resistant(
1454    parent: &std::path::Path,
1455    daemon_euid: u32,
1456) -> anyhow::Result<()> {
1457    use std::os::unix::fs::MetadataExt;
1458
1459    let absolute = if parent.is_absolute() {
1460        parent.to_path_buf()
1461    } else {
1462        std::env::current_dir()
1463            .map_err(|e| {
1464                anyhow::anyhow!(
1465                    "refusing to start: cannot resolve the working directory to absolutize \
1466                     socket directory {}: {e}.",
1467                    parent.display()
1468                )
1469            })?
1470            .join(parent)
1471    };
1472
1473    fn push_components(stack: &mut Vec<std::ffi::OsString>, path: &std::path::Path) {
1474        let components: Vec<_> = path
1475            .components()
1476            .map(|c| c.as_os_str().to_os_string())
1477            .collect();
1478        stack.extend(components.into_iter().rev());
1479    }
1480
1481    let mut stack: Vec<std::ffi::OsString> = Vec::new();
1482    push_components(&mut stack, &absolute);
1483    let mut resolved = std::path::PathBuf::new();
1484    let mut symlinks_followed = 0u32;
1485
1486    while let Some(component) = stack.pop() {
1487        if component == "/" {
1488            resolved = std::path::PathBuf::from("/");
1489            continue;
1490        }
1491        if component == "." {
1492            continue;
1493        }
1494        if component == ".." {
1495            resolved.pop();
1496            continue;
1497        }
1498        let candidate = resolved.join(&component);
1499        let meta = std::fs::symlink_metadata(&candidate).map_err(|e| {
1500            anyhow::anyhow!(
1501                "refusing to start: cannot stat socket-path component {}: {e}. An \
1502                 unreadable component is not a passing one.",
1503                candidate.display()
1504            )
1505        })?;
1506        let owner = meta.uid();
1507
1508        if meta.file_type().is_symlink() {
1509            symlinks_followed += 1;
1510            if symlinks_followed > 40 {
1511                anyhow::bail!(
1512                    "refusing to start: socket path resolves through more than 40 symlinks \
1513                     at {} — treating this as a loop.",
1514                    candidate.display()
1515                );
1516            }
1517            if owner != daemon_euid && owner != 0 {
1518                anyhow::bail!(
1519                    "refusing to start: socket-path symlink component {} is owned by uid \
1520                     {owner}, not this daemon's uid ({daemon_euid}) or root — its owner \
1521                     could retarget it after this check and re-root the socket path. Point \
1522                     KHIVE_SOCKET somewhere trusted end to end, or unset it for the \
1523                     default.",
1524                    candidate.display()
1525                );
1526            }
1527            let target = std::fs::read_link(&candidate).map_err(|e| {
1528                anyhow::anyhow!(
1529                    "refusing to start: cannot read socket-path symlink component {}: {e}.",
1530                    candidate.display()
1531                )
1532            })?;
1533            push_components(&mut stack, &target);
1534            continue;
1535        }
1536
1537        if meta.is_dir() {
1538            let mode = meta.permissions().mode();
1539            let sticky = mode & 0o1000 != 0;
1540            if owner != daemon_euid && owner != 0 {
1541                anyhow::bail!(
1542                    "refusing to start: socket-path ancestor {} is owned by uid {owner}, not \
1543                     this daemon's uid ({daemon_euid}) or root — its owner could rename the \
1544                     next path component and re-root the socket path. Point KHIVE_SOCKET \
1545                     somewhere trusted end to end, or unset it for the default.",
1546                    candidate.display()
1547                );
1548            }
1549            if mode & 0o022 != 0 && !sticky {
1550                anyhow::bail!(
1551                    "refusing to start: socket-path ancestor {} is mode {:04o} — writable by \
1552                     group or other without the sticky bit, so another local user could rename \
1553                     the next path component and re-root the socket path. Point KHIVE_SOCKET \
1554                     somewhere trusted end to end, or unset it for the default.",
1555                    candidate.display(),
1556                    mode & 0o7777
1557                );
1558            }
1559            resolved = candidate;
1560            continue;
1561        }
1562
1563        anyhow::bail!(
1564            "refusing to start: socket-path component {} is neither a directory nor a \
1565             symlink — the socket path cannot traverse it.",
1566            candidate.display()
1567        );
1568    }
1569
1570    Ok(())
1571}
1572
1573/// Run the daemon using a startup lock acquired by the caller *before*
1574/// building `dispatcher`, so a second process racing to boot (e.g. two
1575/// `kkernel mcp --daemon` spawns before either has bound its socket) cannot
1576/// run migrations/FTS DDL concurrently against the same database file.
1577/// `boot_guard` is only `None` on non-unix targets, where there is no
1578/// advisory boot lock to hold in the first place; every unix daemon-mode
1579/// caller passes `Some`.
1580///
1581/// The guard is held across cleanup → bind → pid-write, then dropped. The
1582/// caller must not still be holding a *different* handle to the same lock
1583/// file when this function is entered — see the "Deadlock note" on the
1584/// `_startup_lock` binding below for why that would self-deadlock on `flock`.
1585#[cfg(unix)]
1586pub async fn run_daemon_with_boot_guard<D: DaemonDispatch>(
1587    dispatcher: D,
1588    boot_guard: Option<std::fs::File>,
1589) -> anyhow::Result<()> {
1590    run_daemon_with_boot_guard_inner(dispatcher, boot_guard, false).await
1591}
1592
1593#[cfg(unix)]
1594async fn run_daemon_with_boot_guard_inner<D: DaemonDispatch>(
1595    dispatcher: D,
1596    boot_guard: Option<std::fs::File>,
1597    allow_same_process_incumbent: bool,
1598) -> anyhow::Result<()> {
1599    // ADR-119: components may have been started by the serve path before this
1600    // function established (or failed to establish) daemon ownership. Cancel
1601    // the process-wide shutdown token on EVERY exit — setup failures below,
1602    // early return when another daemon owns the socket, bind errors, and
1603    // normal shutdown alike — so supervisors never outlive this process's
1604    // claim to daemon role. Constructed before any fallible startup work so
1605    // no error path can precede it. The token is process-lifetime
1606    // single-shot; a process that stops being (or never becomes) the daemon
1607    // has no path back except exec.
1608    struct ComponentTeardown;
1609    impl Drop for ComponentTeardown {
1610        fn drop(&mut self) {
1611            daemon_shutdown_token().cancel();
1612        }
1613    }
1614    let _component_teardown = ComponentTeardown;
1615
1616    let sock = socket_path();
1617    let pid_file = pid_path();
1618
1619    if let Some(parent) = sock.parent() {
1620        std::fs::create_dir_all(parent)?;
1621        ensure_socket_dir_is_trusted(parent)?;
1622    }
1623
1624    // Hold the startup lock across cleanup → bind → pid-write so a concurrent
1625    // client's kill_and_respawn (which also holds this lock) cannot unlink the
1626    // socket between our bind and our pid-write.  The lock is released once the
1627    // listener is bound and the PID file is written — at that point any racing
1628    // client will find a live socket+pid and skip the stale-cleanup path.
1629    //
1630    // Deadlock note: the client holds this lock only during kill+spawn and
1631    // releases it before the spawned daemon process starts (the lock guard is
1632    // dropped when kill_and_respawn returns, before the readiness probe loop).
1633    // The daemon holds exactly one handle to this lock for its whole boot
1634    // sequence (received as `boot_guard`, extended from before `dispatcher`
1635    // was constructed) — never a second, independently-acquired handle in the
1636    // same process, which would self-deadlock on `flock`.
1637    let _startup_lock = boot_guard;
1638
1639    if !cleanup_stale_daemon(&sock, &pid_file, allow_same_process_incumbent).await {
1640        tracing::info!("a responsive khived is already running; exiting");
1641        return Ok(());
1642    }
1643
1644    let listener = UnixListener::bind(&sock)?;
1645    // Fail closed, same reason as the directory above. If this chmod fails the
1646    // socket is world-reachable in a way the accepted design never covered, so
1647    // the bound listener is dropped and the entry removed rather than served.
1648    if let Err(e) = std::fs::set_permissions(&sock, std::fs::Permissions::from_mode(0o600)) {
1649        drop(listener);
1650        let _ = std::fs::remove_file(&sock);
1651        return Err(anyhow::anyhow!(
1652            "refusing to start: cannot chmod 0600 {}: {e}. The daemon socket must be owner-only \
1653             — it is half of the single-principal guarantee this daemon enforces.",
1654            sock.display()
1655        ));
1656    }
1657    // Captured while still holding the startup lock, immediately after
1658    // bind, so shutdown cleanup can later prove "this is still the same socket
1659    // I bound" rather than trusting the path alone.
1660    let bound_identity = socket_identity(&sock);
1661
1662    if let Err(e) = write_pid_file_exclusive(&pid_file) {
1663        if e.kind() == std::io::ErrorKind::AlreadyExists {
1664            // A PID file appeared between our own `cleanup_stale_daemon`
1665            // removing it and this write — only possible if the boot lock did
1666            // not actually exclude a concurrent booter (e.g. `acquire_recovery_lock`
1667            // failed for one side). Never touch the winner's files: drop only
1668            // the socket entry we ourselves just bound (proven via identity,
1669            // not path), then decide by checking whether the PID now on disk
1670            // names a live, reachable daemon.
1671            if bound_identity.is_some() && socket_identity(&sock) == bound_identity {
1672                drop(listener);
1673                let _ = std::fs::remove_file(&sock);
1674            }
1675            if pid_file_names_a_reachable_daemon(&pid_file, &sock, allow_same_process_incumbent)
1676                .await
1677            {
1678                tracing::info!(
1679                    "a replacement khived already claimed the pid/socket rendezvous; exiting"
1680                );
1681                return Ok(());
1682            }
1683            anyhow::bail!(
1684                "failed to claim daemon pid file at {pid_file:?}: it already exists \
1685                 and does not name a reachable daemon"
1686            );
1687        }
1688        return Err(e.into());
1689    }
1690    // Release the startup lock now: the listener is bound and the PID file is
1691    // written.  Any concurrent client or daemon startup will observe a live
1692    // socket+pid and take the non-recovery path.
1693    drop(_startup_lock);
1694    tracing::info!(
1695        socket = ?sock,
1696        pid = std::process::id(),
1697        source_revision = crate::BUILD_INFO.source_revision,
1698        build_time = crate::BUILD_INFO.build_time,
1699        "khived listening"
1700    );
1701
1702    {
1703        let warm = dispatcher.clone();
1704        tokio::spawn(async move {
1705            warm.warm_all().await;
1706        });
1707    }
1708
1709    // The checkpoint task's own strong-count-based exit is unreachable
1710    // whenever `event_store_for_checkpoint()` returns `Some` (the ordinary
1711    // production shape), because the `SqlEventStore` it wraps retains its
1712    // own clone of the same pool. An explicit watch channel replaces that
1713    // mechanism: the sender is held for the remainder of this function's
1714    // scope and signalled as the first action once shutdown is observed,
1715    // below.
1716    let (checkpoint_shutdown_tx, checkpoint_shutdown_rx) = tokio::sync::watch::channel(());
1717    // ADR-091 Amendment 3: one checkpoint task per file-backed backend the
1718    // dispatcher wired — the primary pool plus every entry
1719    // `secondary_pools_for_checkpoint` returns — sharing this one shutdown
1720    // channel (the sender broadcasts to every receiver clone), so the single
1721    // send below stops every spawned task before `drain()`.
1722    let checkpoint_tasks = checkpoint_task_specs(
1723        dispatcher.pool_for_checkpoint(),
1724        dispatcher.secondary_pools_for_checkpoint(),
1725        dispatcher.event_store_for_checkpoint(),
1726        dispatcher.namespace().to_string(),
1727    );
1728    if !checkpoint_tasks.is_empty() {
1729        let cfg = CheckpointConfig::from_env();
1730        let checkpoint_task_count = checkpoint_tasks.len();
1731        for task in checkpoint_tasks {
1732            track_background_task(run_checkpoint_task(
1733                task.pool,
1734                cfg.clone(),
1735                task.lifecycle_owner,
1736                checkpoint_shutdown_rx.clone(),
1737                task.is_main,
1738            ));
1739        }
1740        tracing::info!(checkpoint_task_count, "WAL checkpoint task(s) started");
1741    }
1742
1743    let active = Arc::new(std::sync::atomic::AtomicUsize::new(0));
1744    let connection_tasks = Arc::new(std::sync::Mutex::new(
1745        Vec::<tokio::task::JoinHandle<()>>::new(),
1746    ));
1747    let (request_shutdown_tx, request_shutdown_rx) = tokio::sync::watch::channel(false);
1748
1749    let shutdown = async {
1750        // REASON: signal handler registration can only fail if the global Tokio runtime
1751        // is not running or the OS rejects the signal number — both are unrecoverable
1752        // at this point in startup, so panic is the correct response.
1753        let mut sigterm = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
1754            .expect("install SIGTERM handler");
1755        let mut sigint = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::interrupt())
1756            .expect("install SIGINT handler");
1757        tokio::select! {
1758            _ = sigterm.recv() => tracing::info!("received SIGTERM"),
1759            _ = sigint.recv() => tracing::info!("received SIGINT"),
1760        }
1761    };
1762
1763    // SAFETY: `geteuid` is always successful and takes no arguments.
1764    let daemon_euid = unsafe { libc::geteuid() } as u32;
1765
1766    tokio::select! {
1767        _ = async {
1768            loop {
1769                match listener.accept().await {
1770                    Ok((stream, _)) => {
1771                        // Refuse a foreign uid before any frame is read.
1772                        // Fails CLOSED: an error reading peer credentials is
1773                        // "cannot prove same-uid", which is the same answer as
1774                        // "is not same-uid" — never a pass.
1775                        //
1776                        // Scope: this is a same-UID check, which is strictly
1777                        // weaker than same-PRINCIPAL. Several distinct actors
1778                        // running under one uid all pass here, so this does not
1779                        // by itself establish that the daemon serves a single
1780                        // principal, and nothing downstream refuses or degrades
1781                        // on observing more than one. Do not describe it as a
1782                        // multi-principal guard — it bounds the process boundary,
1783                        // not the identity one.
1784                        match peer_uid(&stream) {
1785                            Ok(peer) if uid_is_permitted(peer, daemon_euid) => {}
1786                            Ok(peer) => {
1787                                tracing::error!(
1788                                    peer_uid = peer,
1789                                    daemon_euid,
1790                                    "refusing connection from a foreign uid: this daemon accepts \
1791                                     only peers running as its own uid"
1792                                );
1793                                drop(stream);
1794                                continue;
1795                            }
1796                            Err(e) => {
1797                                tracing::error!(
1798                                    error = %e,
1799                                    "refusing connection: cannot read peer credentials, so \
1800                                     same-uid cannot be proven"
1801                                );
1802                                drop(stream);
1803                                continue;
1804                            }
1805                        }
1806                        let d = dispatcher.clone();
1807                        let shutdown = request_shutdown_rx.clone();
1808                        let handle = spawn_connection_task(Arc::clone(&active), async move {
1809                            handle_conn_with_shutdown(stream, d, Some(shutdown)).await;
1810                        });
1811                        let mut tasks = connection_tasks
1812                            .lock()
1813                            .unwrap_or_else(std::sync::PoisonError::into_inner);
1814                        tasks.retain(|task| !task.is_finished());
1815                        tasks.push(handle);
1816                    }
1817                    Err(e) => tracing::error!(error = %e, "accept failed"),
1818                }
1819            }
1820        } => {}
1821        _ = shutdown => {}
1822    }
1823
1824    // Signal the checkpoint task to exit before draining, so `drain()`
1825    // actually waits on it via `track_background_task` rather than the
1826    // task outliving the drain window (or the process) unsignalled.
1827    let _ = checkpoint_shutdown_tx.send(());
1828
1829    // Per-run signal: read scopes stop promptly, admitted writes ignore it and
1830    // retain the rest of the configured drain window to commit or roll back.
1831    let _ = request_shutdown_tx.send(true);
1832
1833    // Same ordering contract for ADR-119 daemon components: cancel before
1834    // drain, so each component's supervisor (itself a tracked task) can run
1835    // its bounded shutdown inside the drain wait.
1836    daemon_shutdown_token().cancel();
1837
1838    let drained = drain(&active).await;
1839    let tasks = {
1840        let mut retained = connection_tasks
1841            .lock()
1842            .unwrap_or_else(std::sync::PoisonError::into_inner);
1843        std::mem::take(&mut *retained)
1844    };
1845    finish_connection_tasks(tasks, drained).await;
1846
1847    // A concurrent client's `kill_and_respawn` may have already decided
1848    // this daemon looked stale, killed it, and spawned a replacement that
1849    // bound the same socket/PID paths while this daemon was draining above.
1850    // Reacquire the recovery lock (the same one that serializes startup) and
1851    // only unlink if the PID file still names this process AND the socket at
1852    // `sock` is still the exact one this daemon bound — otherwise a
1853    // replacement daemon owns those paths now and unlinking would delete its
1854    // live socket/PID out from under it.
1855    match acquire_recovery_lock() {
1856        Some(_shutdown_lock) => {
1857            shutdown_cleanup_if_owned(&sock, &pid_file, bound_identity);
1858        }
1859        None => {
1860            tracing::warn!(
1861                "could not acquire recovery lock for shutdown cleanup; \
1862                 skipping unlink to avoid deleting a replacement daemon's paths"
1863            );
1864        }
1865    }
1866    tracing::info!("khived stopped");
1867    Ok(())
1868}
1869
1870/// Remove `sock`/`pid_file` only if they still belong to this process: the PID
1871/// file must name `std::process::id()` AND the socket currently at `sock` must
1872/// still be the exact one identified by `bound_identity` (dev/ino, not path).
1873///
1874/// Returns `true` if cleanup ran, `false` if it was skipped because a
1875/// replacement daemon already owns those paths. The caller must hold
1876/// the recovery lock across this call — the same lock daemon startup holds
1877/// across cleanup+bind+pid-write — so no replacement can bind between this
1878/// function's checks and its unlinks.
1879#[cfg(unix)]
1880fn shutdown_cleanup_if_owned(
1881    sock: &std::path::Path,
1882    pid_file: &std::path::Path,
1883    bound_identity: Option<SocketIdentity>,
1884) -> bool {
1885    let pid_is_ours = std::fs::read_to_string(pid_file)
1886        .ok()
1887        .and_then(|s| s.trim().parse::<u32>().ok())
1888        == Some(std::process::id());
1889    let socket_is_ours = bound_identity.is_some() && socket_identity(sock) == bound_identity;
1890    if pid_is_ours && socket_is_ours {
1891        let _ = std::fs::remove_file(sock);
1892        let _ = std::fs::remove_file(pid_file);
1893        true
1894    } else {
1895        tracing::warn!(
1896            socket = ?sock,
1897            pid_file = ?pid_file,
1898            "skipping shutdown cleanup — a replacement daemon already owns this socket/PID"
1899        );
1900        false
1901    }
1902}
1903
1904// ── helpers ───────────────────────────────────────────────────────────────────
1905
1906/// Liveness verdict for a `kill(pid, 0)` probe.
1907#[cfg(unix)]
1908#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1909enum PidLiveness {
1910    /// errno 0 — signal delivery succeeded, the process exists and this
1911    /// caller may signal it.
1912    Alive,
1913    /// ESRCH (or any other non-EPERM errno) — no such process.
1914    Dead,
1915    /// EPERM — the process exists but this caller lacks permission to
1916    /// signal it. Unknown-safe: treated as running so stale-daemon cleanup
1917    /// never unlinks a live daemon's socket/PID file just because it is
1918    /// owned by a different user/uid.
1919    PermissionDenied,
1920}
1921
1922#[cfg(unix)]
1923impl PidLiveness {
1924    fn is_running(self) -> bool {
1925        !matches!(self, PidLiveness::Dead)
1926    }
1927}
1928
1929/// Maps a `kill(pid, 0)` outcome (return code + errno) to a [`PidLiveness`].
1930/// Pure and side-effect-free so the errno mapping can be unit tested without
1931/// a real process probe.
1932#[cfg(unix)]
1933fn classify_kill_result(rc: i32, errno: i32) -> PidLiveness {
1934    if rc == 0 {
1935        return PidLiveness::Alive;
1936    }
1937    match errno {
1938        libc::EPERM => PidLiveness::PermissionDenied,
1939        _ => PidLiveness::Dead,
1940    }
1941}
1942
1943#[cfg(unix)]
1944fn is_process_running(pid: u32) -> bool {
1945    let Ok(pid) = i32::try_from(pid) else {
1946        return false;
1947    };
1948    if pid <= 0 {
1949        return false;
1950    }
1951    // SAFETY: signal 0 is an existence/permission probe with no side effects.
1952    let rc = unsafe { libc::kill(pid, 0) };
1953    let errno = std::io::Error::last_os_error().raw_os_error().unwrap_or(0);
1954    classify_kill_result(rc, errno).is_running()
1955}
1956
1957/// Whether a PID may identify an incumbent from this candidate's point of view.
1958///
1959/// Production rejects the current PID so a stale rendezvous left by a prior
1960/// process whose PID was reused cannot protect an unrelated socket. The
1961/// in-process daemon harness opts in to the same-PID case because all of its
1962/// otherwise independent boot candidates necessarily share one OS process.
1963#[cfg(unix)]
1964fn pid_can_name_incumbent(pid: u32, current_pid: u32, allow_same_process_incumbent: bool) -> bool {
1965    allow_same_process_incumbent || pid != current_pid
1966}
1967
1968#[cfg(unix)]
1969async fn cleanup_stale_daemon(
1970    sock: &std::path::Path,
1971    pid_file: &std::path::Path,
1972    allow_same_process_incumbent: bool,
1973) -> bool {
1974    if let Ok(pid_str) = std::fs::read_to_string(pid_file) {
1975        if let Ok(pid) = pid_str.trim().parse::<u32>() {
1976            if pid_can_name_incumbent(pid, std::process::id(), allow_same_process_incumbent)
1977                && is_process_running(pid)
1978                && sock.exists()
1979                && UnixStream::connect(sock).await.is_ok()
1980            {
1981                return false;
1982            }
1983        }
1984    }
1985    if sock.exists() {
1986        if let Err(e) = std::fs::remove_file(sock) {
1987            tracing::warn!(error = %e, path = ?sock, "failed to remove stale socket");
1988        }
1989    }
1990    if pid_file.exists() {
1991        if let Err(e) = std::fs::remove_file(pid_file) {
1992            tracing::warn!(error = %e, path = ?pid_file, "failed to remove stale PID file");
1993        }
1994    }
1995    true
1996}
1997
1998/// Create `pid_file` exclusively (`O_EXCL`) and write this process's PID.
1999///
2000/// Uses `create_new(true)` rather than `create(true).truncate(true)` so
2001/// this can never silently overwrite a PID file another process created —
2002/// under normal operation the boot lock already serializes cleanup → bind →
2003/// pid-write across processes, but that guarantee depends on
2004/// `acquire_recovery_lock` succeeding for every party. Exclusive creation is
2005/// the defense that holds even if the lock itself is unavailable to one side:
2006/// the loser observes `ErrorKind::AlreadyExists` instead of clobbering the
2007/// winner's PID out from under it.
2008#[cfg(unix)]
2009fn write_pid_file_exclusive(pid_file: &std::path::Path) -> std::io::Result<()> {
2010    use std::os::unix::fs::OpenOptionsExt;
2011
2012    let mut opts = std::fs::OpenOptions::new();
2013    opts.write(true).create_new(true).mode(0o600);
2014    let mut f = opts.open(pid_file)?;
2015    f.write_all(std::process::id().to_string().as_bytes())?;
2016    Ok(())
2017}
2018
2019/// Return `true` if `pid_file` currently names an eligible live process that
2020/// still answers on `sock` — i.e. a daemon already owns this rendezvous and it
2021/// is safe to defer to it rather than treat the `AlreadyExists` PID-file
2022/// collision as a boot failure. Eligibility requires a different PID in
2023/// production; the explicit in-process harness may allow the current PID.
2024#[cfg(unix)]
2025async fn pid_file_names_a_reachable_daemon(
2026    pid_file: &std::path::Path,
2027    sock: &std::path::Path,
2028    allow_same_process_incumbent: bool,
2029) -> bool {
2030    let Ok(pid_str) = std::fs::read_to_string(pid_file) else {
2031        return false;
2032    };
2033    let Ok(pid) = pid_str.trim().parse::<u32>() else {
2034        return false;
2035    };
2036    pid_can_name_incumbent(pid, std::process::id(), allow_same_process_incumbent)
2037        && is_process_running(pid)
2038        && sock.exists()
2039        && UnixStream::connect(sock).await.is_ok()
2040}
2041
2042#[cfg(unix)]
2043async fn drain(active: &std::sync::atomic::AtomicUsize) -> bool {
2044    drain_with_timeout(active, drain_timeout()).await
2045}
2046
2047#[cfg(unix)]
2048async fn drain_with_timeout(
2049    active: &std::sync::atomic::AtomicUsize,
2050    timeout: std::time::Duration,
2051) -> bool {
2052    use std::sync::atomic::Ordering;
2053    // One sequentially-consistent order spans connection handoff and tracked
2054    // task publication. Drain must not observe an ended connection and a
2055    // stale pre-publication background count as two simultaneous zeroes.
2056    let remaining = || active.load(Ordering::SeqCst) + background_task_count();
2057    if remaining() == 0 {
2058        return true;
2059    }
2060    let deadline = tokio::time::Instant::now() + timeout;
2061    while remaining() > 0 {
2062        if tokio::time::Instant::now() >= deadline {
2063            tracing::warn!(
2064                remaining_connections = active.load(Ordering::SeqCst),
2065                remaining_background_tasks = background_task_count(),
2066                "drain timeout reached; forcing shutdown"
2067            );
2068            return false;
2069        }
2070        tokio::select! {
2071            _ = tokio::time::sleep(std::time::Duration::from_millis(100)) => {}
2072            _ = tokio::time::sleep_until(deadline) => {}
2073        }
2074    }
2075    true
2076}
2077
2078#[cfg(unix)]
2079async fn finish_connection_tasks(tasks: Vec<tokio::task::JoinHandle<()>>, drained: bool) {
2080    if !drained {
2081        for task in &tasks {
2082            if !task.is_finished() {
2083                task.abort();
2084            }
2085        }
2086    }
2087    for task in tasks {
2088        let _ = task.await;
2089    }
2090}
2091
2092/// The bound `drain()` waits for tracked background tasks at daemon shutdown
2093/// (`KHIVE_DRAIN_TIMEOUT_SECS`, default 10s). Public so component supervision
2094/// can clamp per-component shutdown timeouts against it — a component timeout
2095/// longer than the drain bound could never complete its abort/state
2096/// transition before the daemon returns.
2097pub fn drain_timeout() -> std::time::Duration {
2098    let secs = std::env::var("KHIVE_DRAIN_TIMEOUT_SECS")
2099        .ok()
2100        .and_then(|v| v.parse::<u64>().ok())
2101        .unwrap_or(DEFAULT_DRAIN_TIMEOUT_SECS);
2102    std::time::Duration::from_secs(secs)
2103}
2104
2105/// Returns `true` for non-empty env values that are not `"0"` or `"false"`.
2106#[cfg(unix)]
2107pub fn env_truthy(key: &str) -> bool {
2108    std::env::var(key)
2109        .map(|v| {
2110            let v = v.trim();
2111            !v.is_empty() && v != "0" && !v.eq_ignore_ascii_case("false")
2112        })
2113        .unwrap_or(false)
2114}
2115
2116#[cfg(test)]
2117mod khive_root_tests {
2118    use super::khive_root_from;
2119    use std::path::PathBuf;
2120
2121    #[test]
2122    fn home_wins_when_set() {
2123        assert_eq!(
2124            khive_root_from(Some("/base/home".into()), Some("/other".into())),
2125            PathBuf::from("/base/home/.khive")
2126        );
2127    }
2128
2129    #[test]
2130    fn userprofile_backfills_missing_home() {
2131        assert_eq!(
2132            khive_root_from(None, Some("/profile/home".into())),
2133            PathBuf::from("/profile/home/.khive")
2134        );
2135    }
2136
2137    #[test]
2138    fn blank_values_are_skipped() {
2139        assert_eq!(
2140            khive_root_from(Some("  ".into()), Some("/profile/home".into())),
2141            PathBuf::from("/profile/home/.khive")
2142        );
2143    }
2144
2145    #[cfg(not(unix))]
2146    #[test]
2147    fn fallback_is_never_cwd_relative() {
2148        // On non-unix the lock anchor must not depend on the caller's working
2149        // directory even when no home variable is set — a relative path here
2150        // would give the same database different lock files per cwd.
2151        assert!(khive_root_from(None, None).is_absolute());
2152    }
2153
2154    #[cfg(unix)]
2155    #[test]
2156    fn unix_fallback_stays_historical_not_shared_tmp() {
2157        // On unix the no-HOME last resort deliberately stays "./.khive"
2158        // rather than a shared world-writable anchor like /tmp, which a
2159        // local attacker could pre-claim to intercept the daemon socket.
2160        assert_eq!(khive_root_from(None, None), PathBuf::from("./.khive"));
2161    }
2162}
2163
2164#[cfg(all(test, unix))]
2165mod tests {
2166    use super::*;
2167    use serial_test::serial;
2168
2169    #[derive(Debug)]
2170    struct DrainBlockingBlobStore {
2171        started: std::sync::Mutex<Option<tokio::sync::oneshot::Sender<()>>>,
2172        release: Arc<tokio::sync::Semaphore>,
2173    }
2174
2175    #[async_trait]
2176    impl khive_storage::BlobStore for DrainBlockingBlobStore {
2177        async fn put(
2178            &self,
2179            _bytes: Vec<u8>,
2180        ) -> khive_storage::StorageResult<khive_storage::ContentRef> {
2181            panic!("put is not used by the hydration drain test")
2182        }
2183
2184        async fn get_bounded_verified(
2185            &self,
2186            _content_ref: &khive_storage::ContentRef,
2187            _max_bytes: u64,
2188        ) -> khive_storage::StorageResult<Vec<u8>> {
2189            if let Some(started) = self
2190                .started
2191                .lock()
2192                .unwrap_or_else(std::sync::PoisonError::into_inner)
2193                .take()
2194            {
2195                let _ = started.send(());
2196            }
2197            self.release
2198                .clone()
2199                .acquire_owned()
2200                .await
2201                .expect("test release semaphore remains open")
2202                .forget();
2203            Ok(b"late result".to_vec())
2204        }
2205
2206        async fn exists(
2207            &self,
2208            _content_ref: &khive_storage::ContentRef,
2209        ) -> khive_storage::StorageResult<bool> {
2210            panic!("exists is not used by the hydration drain test")
2211        }
2212
2213        async fn size(
2214            &self,
2215            _content_ref: &khive_storage::ContentRef,
2216        ) -> khive_storage::StorageResult<Option<u64>> {
2217            panic!("size is not used by the hydration drain test")
2218        }
2219
2220        async fn delete(
2221            &self,
2222            _content_ref: &khive_storage::ContentRef,
2223        ) -> khive_storage::StorageResult<bool> {
2224            panic!("delete is not used by the hydration drain test")
2225        }
2226    }
2227
2228    struct AppendCompletionEventStore {
2229        inner: Arc<dyn khive_storage::EventStore>,
2230        first_append: std::sync::Mutex<Option<tokio::sync::oneshot::Sender<()>>>,
2231    }
2232
2233    #[async_trait]
2234    impl khive_storage::EventStore for AppendCompletionEventStore {
2235        async fn append_event(
2236            &self,
2237            event: khive_storage::Event,
2238        ) -> khive_storage::StorageResult<()> {
2239            self.inner.append_event(event).await?;
2240            if let Some(completed) = self
2241                .first_append
2242                .lock()
2243                .unwrap_or_else(std::sync::PoisonError::into_inner)
2244                .take()
2245            {
2246                let _ = completed.send(());
2247            }
2248            Ok(())
2249        }
2250
2251        async fn append_events(
2252            &self,
2253            events: Vec<khive_storage::Event>,
2254        ) -> khive_storage::StorageResult<khive_storage::BatchWriteSummary> {
2255            self.inner.append_events(events).await
2256        }
2257
2258        async fn get_event(
2259            &self,
2260            id: uuid::Uuid,
2261        ) -> khive_storage::StorageResult<Option<khive_storage::Event>> {
2262            self.inner.get_event(id).await
2263        }
2264
2265        async fn query_events(
2266            &self,
2267            filter: khive_storage::EventFilter,
2268            page: khive_storage::PageRequest,
2269        ) -> khive_storage::StorageResult<khive_storage::Page<khive_storage::Event>> {
2270            self.inner.query_events(filter, page).await
2271        }
2272
2273        async fn count_events(
2274            &self,
2275            filter: khive_storage::EventFilter,
2276        ) -> khive_storage::StorageResult<u64> {
2277            self.inner.count_events(filter).await
2278        }
2279    }
2280
2281    #[tokio::test]
2282    #[serial(checkpoint_skip_metrics)]
2283    async fn secondary_only_checkpoint_topology_emits_lifecycle_outcome() {
2284        let main_backend = khive_db::StorageBackend::memory().expect("in-memory main backend");
2285        let inner_event_store = main_backend.events().expect("main event store");
2286        let (first_append_tx, first_append_rx) = tokio::sync::oneshot::channel();
2287        let event_store: Arc<dyn khive_storage::EventStore> =
2288            Arc::new(AppendCompletionEventStore {
2289                inner: inner_event_store,
2290                first_append: std::sync::Mutex::new(Some(first_append_tx)),
2291            });
2292        let secondary_dir = tempfile::tempdir().expect("secondary tempdir");
2293        let secondary_backend =
2294            khive_db::StorageBackend::sqlite(secondary_dir.path().join("secondary.db"))
2295                .expect("file-backed secondary backend");
2296
2297        let mut tasks = checkpoint_task_specs(
2298            None,
2299            vec![secondary_backend.pool_arc()],
2300            Some(Arc::clone(&event_store)),
2301            "local".to_string(),
2302        );
2303        assert_eq!(tasks.len(), 1);
2304        let task = tasks.pop().expect("one secondary checkpoint task");
2305        assert!(!task.is_main, "the only checkpoint task must be secondary");
2306        assert!(
2307            task.lifecycle_owner.is_some(),
2308            "the secondary task must own lifecycle emission when no main task exists"
2309        );
2310
2311        let config = CheckpointConfig {
2312            interval: std::time::Duration::from_millis(10),
2313            warn_pages: 0,
2314            ..CheckpointConfig::default()
2315        };
2316        let (shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(());
2317        let handle = tokio::spawn(run_checkpoint_task(
2318            task.pool,
2319            config,
2320            task.lifecycle_owner,
2321            shutdown_rx,
2322            task.is_main,
2323        ));
2324
2325        // The scheduler intentionally aborts its bounded append worker during
2326        // shutdown. Wait on the append's completion edge before requesting
2327        // shutdown, so the observation cannot race that deliberate abort.
2328        tokio::time::timeout(std::time::Duration::from_secs(10), first_append_rx)
2329            .await
2330            .expect("secondary checkpoint owner did not complete an append within 10s")
2331            .expect("checkpoint lifecycle append completion sender dropped");
2332
2333        let events = event_store
2334            .query_events(
2335                khive_storage::EventFilter::default(),
2336                khive_storage::PageRequest {
2337                    limit: 100,
2338                    offset: 0,
2339                },
2340            )
2341            .await
2342            .expect("query lifecycle events");
2343
2344        shutdown_tx.send(()).expect("send checkpoint shutdown");
2345        tokio::time::timeout(std::time::Duration::from_secs(1), handle)
2346            .await
2347            .expect("checkpoint task should exit within 1s")
2348            .expect("checkpoint task panicked");
2349        assert!(
2350            !events.items.is_empty()
2351                && events
2352                    .items
2353                    .iter()
2354                    .all(|event| event.kind == khive_types::EventKind::CheckpointOutcomeRecorded),
2355            "the designated secondary owner must emit CheckpointOutcomeRecorded"
2356        );
2357
2358        let file_main_dir = tempfile::tempdir().expect("file-backed main tempdir");
2359        let file_main = khive_db::StorageBackend::sqlite(file_main_dir.path().join("main.db"))
2360            .expect("file-backed main backend");
2361        let tasks = checkpoint_task_specs(
2362            Some(file_main.pool_arc()),
2363            vec![secondary_backend.pool_arc()],
2364            Some(event_store),
2365            "local".to_string(),
2366        );
2367        assert!(tasks[0].is_main && tasks[0].lifecycle_owner.is_some());
2368        assert!(!tasks[1].is_main && tasks[1].lifecycle_owner.is_none());
2369    }
2370
2371    // Focused regression tests for the unsafe process probe (SAFETY: signal 0
2372    // is an existence check with no side effects; see is_process_running).
2373
2374    #[test]
2375    fn current_process_is_running() {
2376        // The current PID is always alive.
2377        let pid = std::process::id();
2378        assert!(
2379            is_process_running(pid),
2380            "current process {pid} should be detected as running"
2381        );
2382    }
2383
2384    #[test]
2385    fn pid_zero_is_not_running() {
2386        // PID 0 is the process group; kill(0, 0) sends to the group,
2387        // which we treat as invalid — the guard `pid <= 0` must block it.
2388        assert!(
2389            !is_process_running(0),
2390            "pid 0 must be rejected by the guard before the unsafe call"
2391        );
2392    }
2393
2394    #[test]
2395    fn very_large_pid_is_not_running() {
2396        // u32::MAX overflows i32 — try_from returns Err, guard returns false.
2397        assert!(
2398            !is_process_running(u32::MAX),
2399            "u32::MAX should fail i32 conversion and return false"
2400        );
2401    }
2402
2403    // EPERM (process exists, no permission to signal it) must not be
2404    // misread as "not running" during stale-daemon cleanup.
2405
2406    #[test]
2407    fn classify_kill_result_zero_is_alive() {
2408        assert_eq!(classify_kill_result(0, 0), PidLiveness::Alive);
2409        assert!(classify_kill_result(0, 0).is_running());
2410    }
2411
2412    #[test]
2413    fn classify_kill_result_esrch_is_dead() {
2414        assert_eq!(classify_kill_result(-1, libc::ESRCH), PidLiveness::Dead);
2415        assert!(!classify_kill_result(-1, libc::ESRCH).is_running());
2416    }
2417
2418    #[test]
2419    fn classify_kill_result_eperm_is_permission_denied_and_counts_as_running() {
2420        assert_eq!(
2421            classify_kill_result(-1, libc::EPERM),
2422            PidLiveness::PermissionDenied
2423        );
2424        assert!(
2425            classify_kill_result(-1, libc::EPERM).is_running(),
2426            "EPERM must be unknown-safe: treated as running, never as a basis \
2427             for stale cleanup to unlink a live daemon's rendezvous files"
2428        );
2429    }
2430
2431    #[test]
2432    fn same_process_pid_requires_explicit_in_process_harness_opt_in() {
2433        let current = std::process::id();
2434        assert!(
2435            !pid_can_name_incumbent(current, current, false),
2436            "production startup must not trust a same-PID stale rendezvous"
2437        );
2438        assert!(
2439            pid_can_name_incumbent(current, current, true),
2440            "the in-process harness must let a responsive same-PID owner win"
2441        );
2442        // Keep the probe two away from `current` so the fixture preserves the
2443        // off-by-one regression check for adjacent PIDs; wrapping_add avoids
2444        // making the test overflow-sensitive at the u32 boundary.
2445        let distinct_probe_pid = current.wrapping_add(2);
2446        assert_ne!(
2447            distinct_probe_pid, current,
2448            "probe PID must differ from this process's PID"
2449        );
2450        assert!(
2451            pid_can_name_incumbent(distinct_probe_pid, current, false),
2452            "a distinct PID remains eligible under ordinary production rules"
2453        );
2454    }
2455
2456    #[test]
2457    fn pid_1_probe_is_running_regardless_of_permission_outcome() {
2458        // PID 1 (init/launchd) always exists. An unprivileged process gets
2459        // EPERM signaling it (never ESRCH); running as root would get 0
2460        // instead. Either way `is_process_running` must report true — this
2461        // is the live regression guard for EPERM being misread as dead;
2462        // `classify_kill_result` above is the pure-function unit coverage
2463        // for the same mapping, kept independent of process ownership so
2464        // it is never flaky in CI.
2465        assert!(
2466            is_process_running(1),
2467            "PID 1 always exists; EPERM must not read as dead"
2468        );
2469    }
2470
2471    #[test]
2472    fn env_truthy_recognises_set_values() {
2473        assert!(!env_truthy("__KHIVE_TEST_ABSENT_VAR_XYZ__"));
2474
2475        // env_truthy with a live value — set and unset atomically to avoid
2476        // cross-test pollution (not parallel-safe without serial_test, but these
2477        // are fast unit tests and the variable name is unique).
2478        let key = "__KHIVE_TEST_TRUTHY_ABC__";
2479        std::env::set_var(key, "1");
2480        assert!(env_truthy(key));
2481        std::env::set_var(key, "false");
2482        assert!(!env_truthy(key));
2483        std::env::set_var(key, "0");
2484        assert!(!env_truthy(key));
2485        std::env::remove_var(key);
2486    }
2487
2488    #[tokio::test(flavor = "current_thread", start_paused = true)]
2489    #[serial(background_tasks)]
2490    async fn accepted_connection_is_counted_before_first_poll_and_drain_waits() {
2491        use std::sync::atomic::Ordering;
2492
2493        let active = Arc::new(std::sync::atomic::AtomicUsize::new(0));
2494        let started = Arc::new(std::sync::atomic::AtomicBool::new(false));
2495        let (release_tx, release_rx) = tokio::sync::oneshot::channel::<()>();
2496        let started_in_task = Arc::clone(&started);
2497
2498        let handle = spawn_connection_task(Arc::clone(&active), async move {
2499            started_in_task.store(true, Ordering::Relaxed);
2500            let _ = release_rx.await;
2501        });
2502
2503        assert_eq!(active.load(Ordering::Relaxed), 1);
2504        assert!(
2505            !started.load(Ordering::Relaxed),
2506            "the current-thread runtime must leave the spawned handler unpolled"
2507        );
2508
2509        let drain_fut = drain(active.as_ref());
2510        tokio::pin!(drain_fut);
2511        let too_early =
2512            tokio::time::timeout(std::time::Duration::from_millis(150), &mut drain_fut).await;
2513        assert!(
2514            too_early.is_err(),
2515            "drain must wait for a connection claimed before its task's first poll"
2516        );
2517        assert!(started.load(Ordering::Relaxed));
2518
2519        release_tx.send(()).expect("handler still waiting");
2520        tokio::time::timeout(std::time::Duration::from_secs(1), handle)
2521            .await
2522            .expect("handler should finish promptly")
2523            .expect("handler should not panic");
2524        assert_eq!(active.load(Ordering::Relaxed), 0);
2525        tokio::time::timeout(std::time::Duration::from_secs(1), drain_fut)
2526            .await
2527            .expect("drain should finish once the handler releases its claim");
2528    }
2529
2530    #[tokio::test(flavor = "current_thread")]
2531    async fn cancelled_connection_releases_count_before_first_poll() {
2532        use std::sync::atomic::Ordering;
2533
2534        let active = Arc::new(std::sync::atomic::AtomicUsize::new(0));
2535        let started = Arc::new(std::sync::atomic::AtomicBool::new(false));
2536        let started_in_task = Arc::clone(&started);
2537        let handle = spawn_connection_task(Arc::clone(&active), async move {
2538            started_in_task.store(true, Ordering::Relaxed);
2539            std::future::pending::<()>().await;
2540        });
2541
2542        assert_eq!(active.load(Ordering::Relaxed), 1);
2543        handle.abort();
2544        let error = handle.await.expect_err("aborted handler must be cancelled");
2545        assert!(error.is_cancelled());
2546        assert!(!started.load(Ordering::Relaxed));
2547        assert_eq!(active.load(Ordering::Relaxed), 0);
2548    }
2549
2550    #[tokio::test(flavor = "current_thread")]
2551    async fn panicked_connection_releases_count() {
2552        use std::sync::atomic::Ordering;
2553
2554        let active = Arc::new(std::sync::atomic::AtomicUsize::new(0));
2555        let handle = spawn_connection_task(Arc::clone(&active), async move {
2556            panic!("intentional connection-handler panic");
2557        });
2558
2559        assert_eq!(active.load(Ordering::Relaxed), 1);
2560        let error = handle
2561            .await
2562            .expect_err("panicked handler must fail its join");
2563        assert!(error.is_panic());
2564        assert_eq!(active.load(Ordering::Relaxed), 0);
2565    }
2566
2567    #[test]
2568    fn connection_claim_releases_if_spawn_panics() {
2569        use std::sync::atomic::Ordering;
2570
2571        let active = Arc::new(std::sync::atomic::AtomicUsize::new(0));
2572        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
2573            drop(spawn_connection_task(Arc::clone(&active), async {}));
2574        }));
2575
2576        assert!(result.is_err(), "tokio::spawn outside a runtime must panic");
2577        assert_eq!(active.load(Ordering::Relaxed), 0);
2578    }
2579
2580    #[tokio::test]
2581    #[serial(background_tasks)]
2582    async fn drain_returns_promptly_with_no_accepted_connection() {
2583        let active = std::sync::atomic::AtomicUsize::new(0);
2584        tokio::time::timeout(std::time::Duration::from_secs(1), drain(&active))
2585            .await
2586            .expect("empty drain should return immediately");
2587    }
2588
2589    #[tokio::test(start_paused = true)]
2590    #[serial(background_tasks)]
2591    async fn graceful_drain_has_a_hard_upper_bound_with_stuck_work() {
2592        use std::sync::atomic::Ordering;
2593
2594        let active = Arc::new(std::sync::atomic::AtomicUsize::new(0));
2595        let task = spawn_connection_task(Arc::clone(&active), async {
2596            std::future::pending::<()>().await;
2597        });
2598        assert_eq!(active.load(Ordering::Relaxed), 1);
2599        let started = tokio::time::Instant::now();
2600
2601        let drained = drain_with_timeout(&active, std::time::Duration::from_millis(250)).await;
2602
2603        assert!(!drained, "stuck work must exhaust the drain bound");
2604        assert!(
2605            started.elapsed() >= std::time::Duration::from_millis(250)
2606                && started.elapsed() < std::time::Duration::from_millis(350),
2607            "graceful shutdown exceeded its configured bound: {:?}",
2608            started.elapsed()
2609        );
2610        finish_connection_tasks(vec![task], drained).await;
2611        assert_eq!(
2612            active.load(Ordering::Relaxed),
2613            0,
2614            "hard-bound escalation must abort, await, and release the handler"
2615        );
2616    }
2617
2618    #[tokio::test(start_paused = true)]
2619    #[serial(background_tasks)]
2620    async fn admitted_work_finishes_inside_drain_window_without_abort() {
2621        use std::sync::atomic::Ordering;
2622
2623        let active = Arc::new(std::sync::atomic::AtomicUsize::new(0));
2624        let committed = Arc::new(std::sync::atomic::AtomicBool::new(false));
2625        let committed_in_task = Arc::clone(&committed);
2626        let task = spawn_connection_task(Arc::clone(&active), async move {
2627            tokio::time::sleep(std::time::Duration::from_millis(100)).await;
2628            committed_in_task.store(true, Ordering::SeqCst);
2629        });
2630
2631        let drained = drain_with_timeout(&active, std::time::Duration::from_millis(250)).await;
2632        assert!(
2633            drained,
2634            "admitted work should finish inside the drain window"
2635        );
2636        finish_connection_tasks(vec![task], drained).await;
2637        assert!(committed.load(Ordering::SeqCst));
2638        assert_eq!(active.load(Ordering::Relaxed), 0);
2639    }
2640
2641    // `drain()` must wait for tracked background tasks (e.g. memory.recall's
2642    // serve-ledger append), not just in-flight connections, or a SIGTERM
2643    // lands mid-flight with no log and no row.
2644    //
2645    // `#[serial(background_tasks)]`: this test reads/asserts on the
2646    // process-wide `BACKGROUND_TASKS` static shared with the two counter
2647    // tests below. Under default parallel execution one test's increment
2648    // leaks into another's snapshot-then-assert window (reproduced: both
2649    // counter tests failed together, passed with `--test-threads=1`).
2650    // Serializing just this named group isolates them from each other
2651    // without forcing the whole test binary (including unrelated
2652    // `#[serial]` tests elsewhere in this crate) onto one thread.
2653    #[tokio::test]
2654    #[serial(background_tasks)]
2655    async fn drain_waits_for_tracked_background_tasks_before_returning() {
2656        let active = std::sync::atomic::AtomicUsize::new(0);
2657        let (tx, rx) = tokio::sync::oneshot::channel::<()>();
2658
2659        track_background_task(async move {
2660            let _ = rx.await;
2661        });
2662        assert!(
2663            background_task_count() >= 1,
2664            "track_background_task must make the in-flight task visible immediately"
2665        );
2666
2667        let drain_fut = drain(&active);
2668        tokio::pin!(drain_fut);
2669
2670        // Must NOT resolve while the tracked task is still pending.
2671        let too_early =
2672            tokio::time::timeout(std::time::Duration::from_millis(150), &mut drain_fut).await;
2673        assert!(
2674            too_early.is_err(),
2675            "drain() must not return while a tracked background task is still running"
2676        );
2677
2678        // Completing the task must let drain() proceed promptly.
2679        tx.send(())
2680            .expect("tracked task still awaiting the oneshot");
2681        let done = tokio::time::timeout(std::time::Duration::from_secs(5), drain_fut).await;
2682        assert!(
2683            done.is_ok(),
2684            "drain() must return once the tracked background task finishes"
2685        );
2686    }
2687
2688    #[tokio::test]
2689    #[serial(background_tasks)]
2690    async fn drain_waits_for_hydration_after_its_last_request_waiter_is_cancelled() {
2691        let before = background_task_count();
2692        let active = std::sync::atomic::AtomicUsize::new(0);
2693        let (started_tx, started_rx) = tokio::sync::oneshot::channel();
2694        let release = Arc::new(tokio::sync::Semaphore::new(0));
2695        let store = Arc::new(DrainBlockingBlobStore {
2696            started: std::sync::Mutex::new(Some(started_tx)),
2697            release: Arc::clone(&release),
2698        });
2699        let hydrator = Arc::new(
2700            crate::BlobHydrator::new(
2701                store as Arc<dyn khive_storage::BlobStore>,
2702                khive_storage::MAX_BLOB_WHOLE_BYTES,
2703            )
2704            .expect("minimum hydration budget is valid"),
2705        );
2706        let content_ref =
2707            khive_storage::ContentRef::from_hex("a".repeat(64)).expect("fixture content ref");
2708
2709        let request_hydrator = Arc::clone(&hydrator);
2710        let request = tokio::spawn(async move {
2711            request_hydrator
2712                .hydrate_verified(&content_ref, khive_storage::MAX_BLOB_WHOLE_BYTES)
2713                .await
2714        });
2715        started_rx.await.expect("backend work must begin");
2716        request.abort();
2717        assert!(request.await.unwrap_err().is_cancelled());
2718        assert_eq!(background_task_count(), before + 1);
2719
2720        let draining = drain_with_timeout(&active, std::time::Duration::from_secs(5));
2721        tokio::pin!(draining);
2722        assert!(
2723            tokio::time::timeout(std::time::Duration::from_millis(150), &mut draining)
2724                .await
2725                .is_err(),
2726            "drain must remain pending while cancelled-request hydration still runs"
2727        );
2728
2729        release.add_permits(1);
2730        assert!(
2731            tokio::time::timeout(std::time::Duration::from_secs(5), draining)
2732                .await
2733                .expect("drain should finish after native hydration ends"),
2734            "hydration should finish inside the drain window"
2735        );
2736        assert_eq!(background_task_count(), before);
2737    }
2738
2739    // See the `#[serial(background_tasks)]` note on
2740    // `drain_waits_for_tracked_background_tasks_before_returning` above —
2741    // this test shares the same process-wide `BACKGROUND_TASKS` static and
2742    // races it (and the panic test below) under default parallelism.
2743    #[tokio::test]
2744    #[serial(background_tasks)]
2745    async fn track_background_task_count_returns_to_zero_after_completion() {
2746        // Sanity check on the counter's own bookkeeping, independent of drain().
2747        let before = background_task_count();
2748        let (tx, rx) = tokio::sync::oneshot::channel::<()>();
2749        track_background_task(async move {
2750            let _ = rx.await;
2751        });
2752        assert_eq!(background_task_count(), before + 1);
2753        tx.send(()).expect("still awaiting");
2754        // Yield until the spawned task's decrement has actually run.
2755        for _ in 0..100 {
2756            if background_task_count() == before {
2757                break;
2758            }
2759            tokio::time::sleep(std::time::Duration::from_millis(10)).await;
2760        }
2761        assert_eq!(background_task_count(), before);
2762    }
2763
2764    // See the `#[serial(background_tasks)]` note above — shares
2765    // `BACKGROUND_TASKS` with the other two tests in this group.
2766    #[tokio::test]
2767    #[serial(background_tasks)]
2768    async fn track_background_task_count_returns_to_baseline_after_panic() {
2769        // A panic inside the tracked future must still decrement the
2770        // counter (via BackgroundTaskGuard's Drop), not leak it forever.
2771        // `track_background_task` discards the spawned
2772        // task's `JoinHandle` (it is fire-and-forget by design — the caller
2773        // never awaits it), so this test does not await the panic directly;
2774        // tokio isolates the panic to the spawned task instead of aborting
2775        // the process, and we observe the recovery purely through the
2776        // shared counter returning to baseline after the guard's `Drop`
2777        // fires during that task's unwind.
2778        let before = background_task_count();
2779
2780        let (tx, rx) = tokio::sync::oneshot::channel::<()>();
2781        track_background_task(async move {
2782            let _ = rx.await;
2783            panic!("intentional panic to exercise the Drop-guard decrement path");
2784        });
2785        assert_eq!(background_task_count(), before + 1);
2786
2787        tx.send(()).expect("still awaiting");
2788        for _ in 0..100 {
2789            if background_task_count() == before {
2790                break;
2791            }
2792            tokio::time::sleep(std::time::Duration::from_millis(10)).await;
2793        }
2794        assert_eq!(
2795            background_task_count(),
2796            before,
2797            "background task counter must return to baseline after the tracked future panics"
2798        );
2799    }
2800
2801    // ── active background phase names (ADR-103) ──────────────────────────
2802
2803    // `#[serial(active_phases)]`: these tests read/assert on the process-wide
2804    // `ACTIVE_PHASES` static. No other test in this crate touches it today,
2805    // but the group mirrors the `background_tasks` precedent above so a
2806    // future addition does not silently reintroduce the same interleaving
2807    // hazard that motivated it there.
2808    #[test]
2809    #[serial(active_phases)]
2810    fn register_active_phase_appears_and_disappears_with_the_guard() {
2811        assert!(
2812            !active_phase_names().contains(&"adr103_test_phase".to_string()),
2813            "must start absent (leaked from a prior failed run would poison this test)"
2814        );
2815
2816        let guard = register_active_phase("adr103_test_phase");
2817        assert!(active_phase_names().contains(&"adr103_test_phase".to_string()));
2818
2819        drop(guard);
2820        assert!(
2821            !active_phase_names().contains(&"adr103_test_phase".to_string()),
2822            "the phase name must drop out of the gauge once its guard is dropped"
2823        );
2824    }
2825
2826    #[test]
2827    #[serial(active_phases)]
2828    fn register_active_phase_counts_concurrent_occurrences_of_the_same_name() {
2829        let first = register_active_phase("adr103_concurrent_phase");
2830        let second = register_active_phase("adr103_concurrent_phase");
2831        assert!(active_phase_names().contains(&"adr103_concurrent_phase".to_string()));
2832
2833        drop(first);
2834        assert!(
2835            active_phase_names().contains(&"adr103_concurrent_phase".to_string()),
2836            "one of two concurrent occurrences ending must not remove the name early"
2837        );
2838
2839        drop(second);
2840        assert!(
2841            !active_phase_names().contains(&"adr103_concurrent_phase".to_string()),
2842            "the name must be removed only once every concurrent occurrence has ended"
2843        );
2844    }
2845
2846    // ── metrics-only frame (load/perf harness read-surface) ────────────────
2847
2848    /// Minimal `DaemonDispatch` for the metrics tests: `dispatch` just counts
2849    /// how many times it was called (so tests can assert the ops path was
2850    /// never reached) and `pool_for_checkpoint` returns whatever pool the
2851    /// test wired in (or `None`, matching an in-memory/poolless dispatcher).
2852    #[derive(Clone)]
2853    struct MockDispatch {
2854        namespace: String,
2855        config_id: String,
2856        dispatch_calls: Arc<std::sync::atomic::AtomicUsize>,
2857        pool: Option<Arc<ConnectionPool>>,
2858        /// When `Some(msg)`, `dispatch` returns `Err(msg)` instead of the
2859        /// default `Ok("{}")` — lets a test drive `handle_conn`'s real
2860        /// dispatch-error arm (khive#948 request_id echo coverage).
2861        dispatch_err: Option<String>,
2862    }
2863
2864    #[derive(Clone)]
2865    struct CancellationAwareDispatch {
2866        started: Arc<tokio::sync::Notify>,
2867        cancellation_observed: Arc<std::sync::atomic::AtomicBool>,
2868    }
2869
2870    #[async_trait]
2871    impl DaemonDispatch for CancellationAwareDispatch {
2872        async fn dispatch(
2873            &self,
2874            _ops: String,
2875            _presentation: Option<String>,
2876            _presentation_per_op: Option<Vec<Option<String>>>,
2877            _format: Option<String>,
2878            _format_per_op: Option<Vec<Option<String>>>,
2879            _from_wire: bool,
2880            _identity: Option<RequestIdentity>,
2881        ) -> Result<String, String> {
2882            self.started.notify_one();
2883            khive_storage::wait_for_request_read_cancellation().await;
2884            self.cancellation_observed
2885                .store(true, std::sync::atomic::Ordering::SeqCst);
2886            Ok("{}".to_string())
2887        }
2888
2889        async fn warm_all(&self) {}
2890
2891        fn namespace(&self) -> &str {
2892            "local"
2893        }
2894
2895        fn config_id(&self) -> &str {
2896            "disconnect-test"
2897        }
2898    }
2899
2900    #[async_trait]
2901    impl DaemonDispatch for MockDispatch {
2902        async fn dispatch(
2903            &self,
2904            _ops: String,
2905            _presentation: Option<String>,
2906            _presentation_per_op: Option<Vec<Option<String>>>,
2907            _format: Option<String>,
2908            _format_per_op: Option<Vec<Option<String>>>,
2909            _from_wire: bool,
2910            _identity: Option<RequestIdentity>,
2911        ) -> Result<String, String> {
2912            self.dispatch_calls
2913                .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
2914            match &self.dispatch_err {
2915                Some(msg) => Err(msg.clone()),
2916                None => Ok("{}".to_string()),
2917            }
2918        }
2919
2920        async fn warm_all(&self) {}
2921
2922        fn namespace(&self) -> &str {
2923            &self.namespace
2924        }
2925
2926        fn config_id(&self) -> &str {
2927            &self.config_id
2928        }
2929
2930        fn pool_for_checkpoint(&self) -> Option<Arc<ConnectionPool>> {
2931            self.pool.clone()
2932        }
2933    }
2934
2935    fn base_request_frame(config_id: &str) -> DaemonRequestFrame {
2936        DaemonRequestFrame {
2937            ops: String::new(),
2938            presentation: None,
2939            presentation_per_op: None,
2940            namespace: "local".to_string(),
2941            actor_id: None,
2942            process_ref: None,
2943            visible_namespaces: Vec::new(),
2944            config_id: config_id.to_string(),
2945            protocol_version: PROTOCOL_VERSION,
2946            probe_only: false,
2947            metrics_only: false,
2948            format: None,
2949            format_per_op: None,
2950            from_wire: false,
2951            request_id: None,
2952        }
2953    }
2954
2955    /// Drive `handle_conn` over an in-process `UnixStream::pair()` (no real
2956    /// socket file needed) and decode the response frame it writes back.
2957    async fn round_trip(dispatcher: MockDispatch, req: &DaemonRequestFrame) -> DaemonResponseFrame {
2958        let (mut client, server) = UnixStream::pair().expect("unix stream pair");
2959        let payload = serde_json::to_vec(req).expect("encode request frame");
2960        let handle = tokio::spawn(async move {
2961            handle_conn(server, dispatcher).await;
2962        });
2963        write_frame(&mut client, &payload)
2964            .await
2965            .expect("write request frame");
2966        let raw = read_frame(&mut client).await.expect("read response frame");
2967        handle.await.expect("handle_conn task panicked");
2968        serde_json::from_slice(&raw).expect("decode response frame")
2969    }
2970
2971    #[tokio::test]
2972    async fn daemon_peer_disconnect_signals_request_read_cancellation() {
2973        let started = Arc::new(tokio::sync::Notify::new());
2974        let cancellation_observed = Arc::new(std::sync::atomic::AtomicBool::new(false));
2975        let dispatcher = CancellationAwareDispatch {
2976            started: Arc::clone(&started),
2977            cancellation_observed: Arc::clone(&cancellation_observed),
2978        };
2979        let (mut client, server) = UnixStream::pair().expect("unix stream pair");
2980        let request = base_request_frame("disconnect-test");
2981        let payload = serde_json::to_vec(&request).expect("encode request frame");
2982        let handler = tokio::spawn(async move { handle_conn(server, dispatcher).await });
2983        write_frame(&mut client, &payload)
2984            .await
2985            .expect("write request frame");
2986        started.notified().await;
2987
2988        drop(client);
2989        tokio::time::timeout(std::time::Duration::from_millis(500), handler)
2990            .await
2991            .expect("daemon handler ignored peer disconnect")
2992            .expect("daemon handler panicked");
2993        assert!(
2994            cancellation_observed.load(std::sync::atomic::Ordering::SeqCst),
2995            "peer loss did not reach the request-scoped read cancellation signal"
2996        );
2997    }
2998
2999    /// Protocol v4 makes `process_ref` part of dispatch semantics. A still-warm
3000    /// v3 daemon/client pairing must fail before the verb runs; otherwise the
3001    /// older peer can ignore the unknown field, persist a message without the
3002    /// requested provenance, and leave the caller unable to retry safely.
3003    #[tokio::test]
3004    async fn protocol_v3_frame_is_rejected_before_process_ref_dispatch() {
3005        assert_eq!(PROTOCOL_VERSION, 4, "process_ref is the protocol-v4 change");
3006        let dispatch_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
3007        let dispatcher = MockDispatch {
3008            namespace: "local".to_string(),
3009            config_id: "cfg-v4".to_string(),
3010            dispatch_calls: Arc::clone(&dispatch_calls),
3011            pool: None,
3012            dispatch_err: None,
3013        };
3014        let mut request = base_request_frame("cfg-v4");
3015        request.protocol_version = 3;
3016        request.process_ref = Some("worker/legacy-rollout".to_string());
3017
3018        let response = round_trip(dispatcher, &request).await;
3019        assert!(!response.ok);
3020        assert!(response.version_mismatch);
3021        assert_eq!(response.daemon_protocol_version, 4);
3022        assert_eq!(
3023            dispatch_calls.load(std::sync::atomic::Ordering::SeqCst),
3024            0,
3025            "a v3 frame must be rejected before a provenance-bearing mutation dispatches"
3026        );
3027        let error = response.error.expect("mismatch explains both versions");
3028        assert!(
3029            error.contains("client=3") && error.contains("daemon=4"),
3030            "mismatch must identify the exact rollout boundary; got {error:?}"
3031        );
3032    }
3033
3034    /// Test 1: a `metrics_only: true` request
3035    /// returns `metrics: Some(_)` and never reaches the ops-dispatch path; a
3036    /// normal request (the default `metrics_only: false`) still dispatches
3037    /// exactly as before and carries no metrics. Also proves `metrics_only`
3038    /// bypasses the `config_id` equality reject (a gauge read is
3039    /// process-global, not namespaced to a particular client config).
3040    #[tokio::test]
3041    async fn metrics_only_frame_returns_snapshot_without_dispatching() {
3042        let dispatch_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
3043        let dispatcher = MockDispatch {
3044            namespace: "local".to_string(),
3045            config_id: "cfg-a".to_string(),
3046            dispatch_calls: Arc::clone(&dispatch_calls),
3047            pool: None,
3048            dispatch_err: None,
3049        };
3050
3051        let mut metrics_req = base_request_frame("cfg-a");
3052        metrics_req.metrics_only = true;
3053        let metrics_resp = round_trip(dispatcher.clone(), &metrics_req).await;
3054
3055        assert!(metrics_resp.ok, "metrics_only response must be ok=true");
3056        assert!(
3057            metrics_resp.metrics.is_some(),
3058            "metrics_only=true must return Some(snapshot)"
3059        );
3060        assert_eq!(
3061            dispatch_calls.load(std::sync::atomic::Ordering::SeqCst),
3062            0,
3063            "metrics_only must never reach the ops-dispatch path"
3064        );
3065
3066        // metrics_only bypasses the config_id equality reject.
3067        let mut mismatched_req = base_request_frame("some-other-config");
3068        mismatched_req.metrics_only = true;
3069        let mismatched_resp = round_trip(dispatcher.clone(), &mismatched_req).await;
3070        assert!(mismatched_resp.ok);
3071        assert!(mismatched_resp.metrics.is_some());
3072        assert!(!mismatched_resp.config_mismatch);
3073        assert_eq!(
3074            dispatch_calls.load(std::sync::atomic::Ordering::SeqCst),
3075            0,
3076            "a mismatched-config metrics_only request must still skip dispatch"
3077        );
3078
3079        // A normal request (default metrics_only=false) is unaffected: it
3080        // still dispatches and carries no metrics.
3081        let normal_req = base_request_frame("cfg-a");
3082        let normal_resp = round_trip(dispatcher, &normal_req).await;
3083        assert!(normal_resp.ok);
3084        assert!(normal_resp.metrics.is_none());
3085        assert_eq!(dispatch_calls.load(std::sync::atomic::Ordering::SeqCst), 1);
3086    }
3087
3088    /// Group/other-writable socket directories are refused whether or not the
3089    /// sticky bit is set, and — the part that matters — are left exactly as
3090    /// they were found.
3091    ///
3092    /// The sticky `/tmp` shape (1777) is in the refusal set deliberately: the
3093    /// sticky bit restricts unlinking, not creating, so a shared directory
3094    /// lets another user pre-bind the predictable socket path before this
3095    /// daemon starts; and a sticky directory's owner may unlink and rebind
3096    /// regardless. Re-permissioning someone else's directory on the way past
3097    /// is what the second assertion pins against: run as a user who *can*
3098    /// chmod it, the old unconditional call succeeded and revoked access for
3099    /// every other process using that directory.
3100    #[test]
3101    fn shared_writable_socket_dirs_are_refused_without_being_modified() {
3102        let dir = tempfile::tempdir().expect("tempdir");
3103        for (name, mode) in [("open", 0o777u32), ("sticky-tmp", 0o1777u32)] {
3104            let shared = dir.path().join(name);
3105            std::fs::create_dir(&shared).expect("create");
3106            std::fs::set_permissions(&shared, std::fs::Permissions::from_mode(mode))
3107                .expect("chmod");
3108
3109            let err = ensure_socket_dir_is_trusted(&shared)
3110                .expect_err("group/other-writable must be refused, sticky or not");
3111            assert!(
3112                err.to_string().contains(&format!("{:04o}", mode & 0o7777)),
3113                "the refusal should name the mode it saw, got: {err}"
3114            );
3115            let after = std::fs::metadata(&shared)
3116                .expect("stat")
3117                .permissions()
3118                .mode()
3119                & 0o7777;
3120            assert_eq!(
3121                after, mode,
3122                "refusing must not re-permission a directory khive does not own"
3123            );
3124        }
3125    }
3126
3127    /// A vetted final directory is still refused when an ANCESTOR would let
3128    /// another local user rename it away and recreate it: a non-sticky
3129    /// group/other-writable ancestor re-roots the whole socket path without
3130    /// the final directory's own metadata ever changing. The sticky-ancestor
3131    /// arm (a root-owned 1777 `/tmp` above a user-owned 0700 directory is
3132    /// acceptable) is exercised implicitly by every accepting test below,
3133    /// whose tempdirs live under the platform temp root.
3134    #[test]
3135    fn writable_non_sticky_ancestor_is_refused() {
3136        let dir = tempfile::tempdir().expect("tempdir");
3137        let open_mid = dir.path().join("open-mid");
3138        std::fs::create_dir(&open_mid).expect("create mid");
3139        let inner = open_mid.join("private");
3140        std::fs::create_dir(&inner).expect("create inner");
3141        std::fs::set_permissions(&inner, std::fs::Permissions::from_mode(0o700)).expect("chmod");
3142        std::fs::set_permissions(&open_mid, std::fs::Permissions::from_mode(0o777))
3143            .expect("chmod mid");
3144
3145        let err = ensure_socket_dir_is_trusted(&inner)
3146            .expect_err("a 0777 non-sticky ancestor must be refused");
3147        assert!(
3148            err.to_string().contains("ancestor"),
3149            "the refusal should say it was an ancestor that failed, got: {err}"
3150        );
3151        assert!(
3152            err.to_string().contains("open-mid"),
3153            "the refusal should name the failing ancestor, got: {err}"
3154        );
3155    }
3156
3157    /// The path walk validates what a symlink component points AT, not just
3158    /// the symlink node: a trusted (self-owned) link into a group/other-
3159    /// writable non-sticky directory is refused on the target directory's
3160    /// metadata, proving the walk keeps traversing — and keeps applying the
3161    /// directory rules — past the link, exactly as the kernel will at bind
3162    /// time.
3163    #[test]
3164    fn symlink_component_to_untrusted_directory_is_refused() {
3165        let dir = tempfile::tempdir().expect("tempdir");
3166        let open = dir.path().join("open-target");
3167        std::fs::create_dir(&open).expect("create target");
3168        std::fs::set_permissions(&open, std::fs::Permissions::from_mode(0o777)).expect("chmod");
3169        let link = dir.path().join("link");
3170        std::os::unix::fs::symlink(&open, &link).expect("symlink");
3171
3172        // SAFETY: `geteuid` is always successful and takes no arguments.
3173        let euid = unsafe { libc::geteuid() } as u32;
3174        let err = ensure_socket_path_is_swap_resistant(&link, euid)
3175            .expect_err("a link into a 0777 non-sticky directory must be refused");
3176        assert!(
3177            err.to_string().contains("open-target"),
3178            "the refusal should name the untrusted target directory, got: {err}"
3179        );
3180    }
3181
3182    /// A symlink component owned by neither the daemon euid nor root is
3183    /// refused on the symlink itself: its owner can retarget it after
3184    /// validation, so where it currently points is irrelevant. A non-root
3185    /// test cannot create a foreign-owned symlink, so this injects a
3186    /// mismatched euid instead — and the fixture must live under an
3187    /// all-root-owned chain (the platform `/tmp`), because under the test's
3188    /// own tempdir the self-owned ancestor directories would already fail
3189    /// the injected euid before the walk ever reached the link.
3190    #[test]
3191    fn foreign_owned_symlink_component_is_refused() {
3192        let link =
3193            std::path::PathBuf::from(format!("/tmp/khive-swaptest-link-{}", std::process::id()));
3194        let _ = std::fs::remove_file(&link);
3195        std::os::unix::fs::symlink("/tmp", &link).expect("symlink");
3196
3197        // SAFETY: `geteuid` is always successful and takes no arguments.
3198        let not_my_euid = (unsafe { libc::geteuid() } as u32).wrapping_add(1);
3199        let result = ensure_socket_path_is_swap_resistant(&link, not_my_euid);
3200        std::fs::remove_file(&link).expect("cleanup");
3201
3202        let err = result.expect_err("a symlink owned by another uid must be refused");
3203        assert!(
3204            err.to_string().contains("symlink component"),
3205            "the refusal should strike the symlink itself, got: {err}"
3206        );
3207        assert!(
3208            err.to_string().contains("khive-swaptest-link"),
3209            "the refusal should name the offending link, got: {err}"
3210        );
3211    }
3212
3213    /// Directories this daemon can trust end to end are served as found:
3214    /// owner-only, and the umask-default 0755 every `tempfile::tempdir` and
3215    /// test runner produces (readable/traversable but writable only by the
3216    /// owner). On macOS these fixtures also implicitly exercise the
3217    /// symlink-accept arm, since the platform temp root itself resolves
3218    /// through root-owned symlinks. Foreign ownership of a directory is
3219    /// refused by the same helper, but a non-root test cannot chown a
3220    /// directory away from itself, so that arm is exercised by the mode
3221    /// checks' shared fail-closed path rather than a dedicated fixture.
3222    #[test]
3223    fn trusted_socket_dirs_are_accepted_unmodified() {
3224        let dir = tempfile::tempdir().expect("tempdir");
3225        for (name, mode) in [("private", 0o700), ("listable", 0o755)] {
3226            let d = dir.path().join(name);
3227            std::fs::create_dir(&d).expect("create");
3228            std::fs::set_permissions(&d, std::fs::Permissions::from_mode(mode)).expect("chmod");
3229
3230            ensure_socket_dir_is_trusted(&d)
3231                .unwrap_or_else(|e| panic!("mode {mode:04o} must be accepted, got: {e}"));
3232
3233            let after = std::fs::metadata(&d).expect("stat").permissions().mode() & 0o7777;
3234            assert_eq!(
3235                after, mode,
3236                "acceptance must not re-permission the directory either"
3237            );
3238        }
3239    }
3240
3241    /// Test 2: `wal_pages` reflects a real
3242    /// checkpoint observation after writes, deterministically forced via a
3243    /// direct `checkpoint_once` call rather than waiting on the async
3244    /// periodic task.
3245    #[tokio::test]
3246    async fn metrics_snapshot_wal_pages_reflects_recent_write() {
3247        let dir = tempfile::tempdir().expect("tempdir");
3248        let path = dir.path().join("metrics_wal_test.db");
3249        let pool = Arc::new(
3250            ConnectionPool::new(khive_db::PoolConfig {
3251                path: Some(path),
3252                ..khive_db::PoolConfig::default()
3253            })
3254            .expect("pool open"),
3255        );
3256
3257        {
3258            let writer = pool.try_writer().expect("writer");
3259            writer
3260                .conn()
3261                .execute_batch(
3262                    "CREATE TABLE t (x INTEGER); \
3263                     INSERT INTO t VALUES (1); \
3264                     INSERT INTO t VALUES (2);",
3265                )
3266                .expect("seed writes");
3267        }
3268
3269        let dedicated_conn = pool
3270            .open_standalone_writer()
3271            .expect("open dedicated checkpoint connection");
3272        khive_db::checkpoint_once(
3273            &pool,
3274            &dedicated_conn,
3275            &CheckpointConfig::default(),
3276            &mut khive_db::checkpoint::TruncateState::default(),
3277        )
3278        .expect("checkpoint_once must observe on a healthy dedicated connection");
3279
3280        let dispatcher = MockDispatch {
3281            namespace: "local".to_string(),
3282            config_id: "cfg-wal".to_string(),
3283            dispatch_calls: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
3284            pool: Some(pool),
3285            dispatch_err: None,
3286        };
3287
3288        let snapshot = build_metrics_snapshot(&dispatcher);
3289        assert!(
3290            snapshot.wal_pages.is_some(),
3291            "wal_pages must be observed after a real checkpoint tick, got {snapshot:?}"
3292        );
3293        assert_eq!(snapshot.wal_log_frames, snapshot.wal_pages);
3294        assert!(snapshot.wal_checkpointed_frames.is_some());
3295        assert!(snapshot.wal_pending_frames.is_some());
3296        assert!(snapshot.wal_physical_bytes.is_some());
3297        assert!(snapshot.wal_observed_at_unix_ms.is_some());
3298        // The snapshot carries the checkpoint-pressure fields read-only
3299        // (no mutation path reachable through `MetricsSnapshot`/`DaemonRequestFrame`);
3300        // an observed tick (not a skip) must report a zero-length skip streak.
3301        assert_eq!(
3302            snapshot.wal_checkpoint_consecutive_skips, 0,
3303            "an observed (non-skipped) tick must report zero consecutive skips, got {snapshot:?}"
3304        );
3305    }
3306
3307    #[tokio::test]
3308    async fn metrics_snapshot_exposes_decomposed_writer_stages() {
3309        let dir = tempfile::tempdir().expect("tempdir");
3310        let path = dir.path().join("metrics_writer_stage_test.db");
3311        let pool = Arc::new(
3312            ConnectionPool::new(khive_db::PoolConfig {
3313                path: Some(path),
3314                ..khive_db::PoolConfig::default()
3315            })
3316            .expect("pool open"),
3317        );
3318        {
3319            let writer = pool.try_writer().unwrap();
3320            writer
3321                .conn()
3322                .execute_batch("CREATE TABLE t (id INTEGER PRIMARY KEY)")
3323                .unwrap();
3324        }
3325        let handle = pool
3326            .writer_task_handle()
3327            .unwrap()
3328            .expect("file-backed default writer task");
3329        handle
3330            .send(|conn| {
3331                std::thread::sleep(std::time::Duration::from_millis(30));
3332                conn.execute("INSERT INTO t VALUES (1)", [])
3333                    .map_err(|error| khive_storage::error::StorageError::Pool {
3334                        operation: "metrics_writer_stage_test".into(),
3335                        message: error.to_string(),
3336                    })
3337            })
3338            .await
3339            .unwrap();
3340
3341        let dispatcher = MockDispatch {
3342            namespace: "local".to_string(),
3343            config_id: "cfg-writer-stages".to_string(),
3344            dispatch_calls: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
3345            pool: Some(pool),
3346            dispatch_err: None,
3347        };
3348        let snapshot = build_metrics_snapshot(&dispatcher);
3349        assert!(snapshot.write_last_queue_wait_micros.is_some());
3350        assert!(snapshot.write_last_transaction_acquire_micros.is_some());
3351        assert!(snapshot.write_last_commit_micros.is_some());
3352        assert!(
3353            snapshot.write_last_body_micros >= Some(25_000),
3354            "synthetic delay must be attributed to the body: {snapshot:?}"
3355        );
3356        assert!(snapshot.write_last_total_micros >= snapshot.write_last_body_micros);
3357        assert!(snapshot.write_last_observed_at_unix_ms.is_some());
3358    }
3359
3360    /// Test 3: the tx-pin oracle. The registry is process-global, so an
3361    /// unrelated transaction can depart between snapshots and exactly offset
3362    /// this test's registration. Keep an owned handle live and assert the
3363    /// resulting count floor instead of comparing two points in time.
3364    #[test]
3365    #[serial(tx_registry)]
3366    fn metrics_snapshot_reflects_open_transaction_registry() {
3367        let dispatcher = MockDispatch {
3368            namespace: "local".to_string(),
3369            config_id: "cfg-tx".to_string(),
3370            dispatch_calls: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
3371            pool: None,
3372            dispatch_err: None,
3373        };
3374
3375        let departing_handle = khive_storage::tx_registry::register(Some(
3376            "daemon_metrics_snapshot_departing_test_tx".to_string(),
3377        ));
3378        let before = build_metrics_snapshot(&dispatcher).open_tx_count;
3379        assert!(before >= 1);
3380
3381        let handle = khive_storage::tx_registry::register(Some(
3382            "daemon_metrics_snapshot_owned_test_tx".to_string(),
3383        ));
3384        drop(departing_handle);
3385
3386        let during = build_metrics_snapshot(&dispatcher);
3387        assert!(
3388            during.open_tx_count >= 1,
3389            "open_tx_count must reflect the live owned transaction despite registry churn: \
3390             churn_baseline={before} during={}",
3391            during.open_tx_count
3392        );
3393        assert!(
3394            during.oldest_pinned_tx_micros.is_some(),
3395            "oldest_pinned_tx_micros must be Some while a transaction is open"
3396        );
3397
3398        drop(handle);
3399        assert!(
3400            !khive_storage::tx_registry::snapshot()
3401                .iter()
3402                .any(|(_, label)| label.as_deref()
3403                    == Some("daemon_metrics_snapshot_owned_test_tx")),
3404            "the owned registry entry must disappear when its handle is dropped"
3405        );
3406    }
3407
3408    /// Test 4: write-queue depth is flag-gated
3409    /// on `PoolConfig::write_queue_enabled` (the `KHIVE_WRITE_QUEUE=1`
3410    /// setting), never on a specific depth value (racy under concurrency).
3411    #[tokio::test]
3412    async fn metrics_snapshot_write_queue_depth_flag_gated() {
3413        let dir = tempfile::tempdir().expect("tempdir");
3414
3415        let enabled_pool = Arc::new(
3416            ConnectionPool::new(khive_db::PoolConfig {
3417                path: Some(dir.path().join("wq_enabled.db")),
3418                write_queue_enabled: Some(true),
3419                ..khive_db::PoolConfig::default()
3420            })
3421            .expect("pool open"),
3422        );
3423        let enabled_dispatcher = MockDispatch {
3424            namespace: "local".to_string(),
3425            config_id: "cfg-wq-on".to_string(),
3426            dispatch_calls: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
3427            pool: Some(enabled_pool),
3428            dispatch_err: None,
3429        };
3430        let snapshot_on = build_metrics_snapshot(&enabled_dispatcher);
3431        assert!(
3432            snapshot_on.write_queue_depth.is_some(),
3433            "write_queue_depth must be Some when write_queue_enabled=true, got {snapshot_on:?}"
3434        );
3435        assert!(snapshot_on.write_queue_capacity.is_some());
3436
3437        let disabled_pool = Arc::new(
3438            ConnectionPool::new(khive_db::PoolConfig {
3439                path: Some(dir.path().join("wq_disabled.db")),
3440                write_queue_enabled: Some(false),
3441                ..khive_db::PoolConfig::default()
3442            })
3443            .expect("pool open"),
3444        );
3445        let disabled_dispatcher = MockDispatch {
3446            namespace: "local".to_string(),
3447            config_id: "cfg-wq-off".to_string(),
3448            dispatch_calls: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
3449            pool: Some(disabled_pool),
3450            dispatch_err: None,
3451        };
3452        let snapshot_off = build_metrics_snapshot(&disabled_dispatcher);
3453        assert!(
3454            snapshot_off.write_queue_depth.is_none(),
3455            "write_queue_depth must be None when write_queue_enabled=false, got {snapshot_off:?}"
3456        );
3457        assert!(snapshot_off.write_queue_capacity.is_none());
3458
3459        // No pool at all (in-memory/poolless dispatcher): also None.
3460        let no_pool_dispatcher = MockDispatch {
3461            namespace: "local".to_string(),
3462            config_id: "cfg-no-pool".to_string(),
3463            dispatch_calls: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
3464            pool: None,
3465            dispatch_err: None,
3466        };
3467        let snapshot_no_pool = build_metrics_snapshot(&no_pool_dispatcher);
3468        assert!(snapshot_no_pool.write_queue_depth.is_none());
3469        assert!(snapshot_no_pool.write_queue_capacity.is_none());
3470    }
3471
3472    /// Test 5: serde default back-compat in both directions — a request JSON
3473    /// without additive request fields deserializes to their defaults, and a
3474    /// response JSON without `metrics` (an old daemon's shape) deserializes
3475    /// with it `None`.
3476    #[test]
3477    fn frame_serde_defaults_additive_fields_when_absent() {
3478        let req_json = serde_json::json!({
3479            "ops": "",
3480            "presentation": null,
3481            "presentation_per_op": null,
3482            "namespace": "local",
3483            "actor_id": null,
3484            "visible_namespaces": [],
3485            "config_id": "cfg",
3486            "protocol_version": PROTOCOL_VERSION,
3487            "probe_only": false,
3488            "format": null,
3489            "format_per_op": null,
3490            "from_wire": false
3491        });
3492        let frame: DaemonRequestFrame =
3493            serde_json::from_value(req_json).expect("decode a metrics_only-absent request frame");
3494        assert!(
3495            !frame.metrics_only,
3496            "metrics_only must default to false when absent from the wire payload"
3497        );
3498        assert_eq!(
3499            frame.request_id, None,
3500            "request_id must default to None when absent from the wire payload (khive#948)"
3501        );
3502        assert_eq!(
3503            frame.process_ref, None,
3504            "process_ref must default to None when absent from the wire payload (khive#1428)"
3505        );
3506        let encoded_frame = serde_json::to_value(&frame).expect("encode request frame");
3507        assert!(
3508            encoded_frame.get("process_ref").is_none(),
3509            "absent provenance must not change the serialized request wire shape"
3510        );
3511
3512        let resp_json = serde_json::json!({
3513            "ok": true,
3514            "result": null,
3515            "error": null,
3516            "namespace_mismatch": false,
3517            "config_mismatch": false,
3518            "served_config_id": "cfg",
3519            "version_mismatch": false,
3520            "daemon_protocol_version": PROTOCOL_VERSION
3521        });
3522        let resp: DaemonResponseFrame =
3523            serde_json::from_value(resp_json).expect("decode a metrics-absent response frame");
3524        assert!(
3525            resp.metrics.is_none(),
3526            "metrics must default to None when absent from the wire payload"
3527        );
3528        assert_eq!(
3529            resp.request_id, None,
3530            "request_id must default to None when absent from the wire payload (khive#948)"
3531        );
3532    }
3533
3534    /// khive#948: a request carrying `request_id: Some(n)` gets back a
3535    /// response with `request_id: Some(n)` on both the success and the
3536    /// error/denied dispatch arms — the echo must survive every branch of
3537    /// `handle_conn`, not only the happy path.
3538    #[tokio::test]
3539    async fn request_id_echoed_on_success_and_error_arms() {
3540        let dispatcher = MockDispatch {
3541            namespace: "local".to_string(),
3542            config_id: "cfg-a".to_string(),
3543            dispatch_calls: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
3544            pool: None,
3545            dispatch_err: None,
3546        };
3547        let mut ok_req = base_request_frame("cfg-a");
3548        ok_req.request_id = Some(42);
3549        let ok_resp = round_trip(dispatcher, &ok_req).await;
3550        assert!(ok_resp.ok, "expected successful dispatch: {ok_resp:?}");
3551        assert_eq!(
3552            ok_resp.request_id,
3553            Some(42),
3554            "request_id must be echoed back on a successful dispatch response"
3555        );
3556
3557        // config_mismatch is a rejection arm that never reaches dispatch —
3558        // must still echo the id so the client can join the failure.
3559        let mismatched_dispatcher = MockDispatch {
3560            namespace: "local".to_string(),
3561            config_id: "cfg-a".to_string(),
3562            dispatch_calls: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
3563            pool: None,
3564            dispatch_err: None,
3565        };
3566        let mut mismatch_req = base_request_frame("cfg-WRONG");
3567        mismatch_req.request_id = Some(99);
3568        let mismatch_resp = round_trip(mismatched_dispatcher, &mismatch_req).await;
3569        assert!(mismatch_resp.config_mismatch);
3570        assert_eq!(
3571            mismatch_resp.request_id,
3572            Some(99),
3573            "request_id must be echoed on the config_mismatch rejection arm too"
3574        );
3575
3576        // The real ops-dispatch error arm (`Err(e)` from `dispatcher.dispatch`)
3577        // must echo the id as well, not only the pre-dispatch rejection arms.
3578        let erroring_dispatcher = MockDispatch {
3579            namespace: "local".to_string(),
3580            config_id: "cfg-a".to_string(),
3581            dispatch_calls: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
3582            pool: None,
3583            dispatch_err: Some("simulated dispatch error".to_string()),
3584        };
3585        let mut err_req = base_request_frame("cfg-a");
3586        err_req.request_id = Some(7);
3587        let err_resp = round_trip(erroring_dispatcher, &err_req).await;
3588        assert!(!err_resp.ok, "expected a dispatch error: {err_resp:?}");
3589        assert_eq!(
3590            err_resp.request_id,
3591            Some(7),
3592            "request_id must be echoed on the real ops-dispatch error arm"
3593        );
3594    }
3595
3596    // ── owner-checked shutdown cleanup ────────────────────────────────────────
3597    //
3598    // A draining daemon must not unlink a socket/PID pair that a replacement
3599    // daemon has already bound. These tests exercise `shutdown_cleanup_if_owned`
3600    // directly (the pure decision the caller makes under the recovery lock)
3601    // rather than driving `run_daemon`'s real SIGTERM shutdown, which would
3602    // require sending a signal to the whole test process.
3603
3604    #[test]
3605    fn shutdown_cleanup_removes_paths_it_still_owns() {
3606        let dir = tempfile::tempdir().expect("tempdir");
3607        let sock = dir.path().join("khived.sock");
3608        let pid_file = dir.path().join("khived.pid");
3609
3610        let _listener = std::os::unix::net::UnixListener::bind(&sock).expect("bind socket");
3611        std::fs::write(&pid_file, std::process::id().to_string()).expect("write pid file");
3612        let identity = socket_identity(&sock);
3613        assert!(
3614            identity.is_some(),
3615            "must read identity of a freshly bound socket"
3616        );
3617
3618        let cleaned = shutdown_cleanup_if_owned(&sock, &pid_file, identity);
3619
3620        assert!(
3621            cleaned,
3622            "cleanup must proceed when PID and socket still match"
3623        );
3624        assert!(!sock.exists(), "owned socket must be removed");
3625        assert!(!pid_file.exists(), "owned pid file must be removed");
3626    }
3627
3628    #[test]
3629    fn shutdown_cleanup_skips_when_pid_file_names_a_different_process() {
3630        let dir = tempfile::tempdir().expect("tempdir");
3631        let sock = dir.path().join("khived.sock");
3632        let pid_file = dir.path().join("khived.pid");
3633
3634        let _listener = std::os::unix::net::UnixListener::bind(&sock).expect("bind socket");
3635        let identity = socket_identity(&sock);
3636        // A concurrent client's kill_and_respawn already replaced the PID file
3637        // with a different (replacement daemon's) PID before this daemon's
3638        // drain completed.
3639        std::fs::write(&pid_file, "1").expect("write foreign pid file");
3640
3641        let cleaned = shutdown_cleanup_if_owned(&sock, &pid_file, identity);
3642
3643        assert!(
3644            !cleaned,
3645            "cleanup must be skipped when the PID file no longer names this process"
3646        );
3647        assert!(sock.exists(), "replacement daemon's socket must survive");
3648        assert!(
3649            pid_file.exists(),
3650            "replacement daemon's pid file must survive"
3651        );
3652    }
3653
3654    #[test]
3655    fn shutdown_cleanup_skips_when_socket_was_rebound_by_a_replacement() {
3656        let dir = tempfile::tempdir().expect("tempdir");
3657        let sock = dir.path().join("khived.sock");
3658        let original_sock = dir.path().join("original.sock");
3659        let pid_file = dir.path().join("khived.pid");
3660
3661        // Bind two sockets at DIFFERENT paths, both alive at the same time,
3662        // so the OS cannot recycle an inode between them the way it could
3663        // across a bind/drop/rebind cycle at a single path (the flakiness a
3664        // prior version of this test hit on some filesystems). Both
3665        // identities are captured through the real production
3666        // `socket_identity()` path, not a synthetic/sentinel value, so a
3667        // regression where `socket_identity()` returns a constant identity
3668        // for every socket makes the `assert!` below fail loudly instead of
3669        // silently passing.
3670        let _original_listener =
3671            std::os::unix::net::UnixListener::bind(&original_sock).expect("bind original socket");
3672        let _replacement_listener =
3673            std::os::unix::net::UnixListener::bind(&sock).expect("bind replacement socket");
3674
3675        let original_identity = socket_identity(&original_sock);
3676        let replacement_identity = socket_identity(&sock);
3677        assert!(
3678            original_identity.is_some(),
3679            "must read identity of the original socket"
3680        );
3681        assert!(
3682            replacement_identity.is_some(),
3683            "must read identity of the replacement socket"
3684        );
3685        assert!(
3686            original_identity != replacement_identity,
3687            "two concurrently bound sockets must have distinct identities"
3688        );
3689
3690        std::fs::write(&pid_file, std::process::id().to_string())
3691            .expect("write pid file matching this process");
3692
3693        // `sock` (the replacement bind's path) is checked against
3694        // `original_identity` (a different, concurrently-alive socket's
3695        // identity) - the mismatch alone must be enough to block cleanup,
3696        // even though the pid file matches this process.
3697        let cleaned = shutdown_cleanup_if_owned(&sock, &pid_file, original_identity);
3698
3699        assert!(
3700            !cleaned,
3701            "cleanup must be skipped when the socket at this path is a different \
3702             inode than the one this daemon originally bound"
3703        );
3704        assert!(sock.exists(), "replacement daemon's socket must survive");
3705        assert!(
3706            pid_file.exists(),
3707            "replacement daemon's pid file must survive"
3708        );
3709    }
3710
3711    // ── the recovery lock actually serializes two boot sequences ─────────────
3712    //
3713    // Production wiring (`khive_mcp::serve::run` / `serve_server`) now acquires
3714    // this same lock *before* building a `KhiveMcpServer` (which runs
3715    // migrations and applies pack schema plans / FTS DDL) and holds it through
3716    // daemon bind+pid-write, via `run_daemon_with_boot_guard`. That closes the
3717    // cold-boot race only if `acquire_recovery_lock` genuinely provides mutual
3718    // exclusion across concurrent boot attempts — this test proves the
3719    // primitive itself: two "boot sequences" (each holding the lock across a
3720    // simulated schema-init critical section) must never run their critical
3721    // sections at the same time.
3722    #[test]
3723    #[serial]
3724    fn recovery_lock_serializes_two_concurrent_boot_sequences() {
3725        let dir = tempfile::tempdir().expect("tempdir");
3726        let lock_file = dir.path().join("khived.recovery.lock");
3727        std::env::set_var("KHIVE_LOCK", &lock_file);
3728
3729        let active = Arc::new(std::sync::atomic::AtomicUsize::new(0));
3730        let overlap_detected = Arc::new(std::sync::atomic::AtomicBool::new(false));
3731
3732        let run_one_boot =
3733            |active: Arc<std::sync::atomic::AtomicUsize>,
3734             overlap: Arc<std::sync::atomic::AtomicBool>| {
3735                move || {
3736                    let _guard = acquire_recovery_lock().expect("acquire recovery lock");
3737                    // Enter the "schema-init" critical section.
3738                    if active.fetch_add(1, std::sync::atomic::Ordering::SeqCst) != 0 {
3739                        overlap.store(true, std::sync::atomic::Ordering::SeqCst);
3740                    }
3741                    std::thread::sleep(std::time::Duration::from_millis(50));
3742                    active.fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
3743                    // `_guard` drops here, releasing the lock.
3744                }
3745            };
3746
3747        let t1 = std::thread::spawn(run_one_boot(active.clone(), overlap_detected.clone()));
3748        let t2 = std::thread::spawn(run_one_boot(active.clone(), overlap_detected.clone()));
3749        t1.join().expect("boot thread 1 must not panic");
3750        t2.join().expect("boot thread 2 must not panic");
3751
3752        assert!(
3753            !overlap_detected.load(std::sync::atomic::Ordering::SeqCst),
3754            "two concurrent boot sequences must never hold the schema-init \
3755             critical section at the same time (#667)"
3756        );
3757
3758        std::env::remove_var("KHIVE_LOCK");
3759    }
3760
3761    // ── acquire_daemon_boot_guard treats lock failure as fatal ───────────────
3762    // (unlike best-effort acquire_recovery_lock, whose `None` on failure is
3763    // correct for its own best-effort callers).
3764
3765    #[test]
3766    #[serial]
3767    fn acquire_daemon_boot_guard_returns_guard_when_lock_available() {
3768        let dir = tempfile::tempdir().expect("tempdir");
3769        let lock_file = dir.path().join("khived.recovery.lock");
3770        std::env::set_var("KHIVE_LOCK", &lock_file);
3771
3772        let guard = acquire_daemon_boot_guard();
3773        assert!(
3774            guard.is_ok(),
3775            "daemon boot guard must succeed when the lock file can be opened and flocked"
3776        );
3777        drop(guard);
3778
3779        std::env::remove_var("KHIVE_LOCK");
3780    }
3781
3782    #[test]
3783    #[serial]
3784    fn acquire_daemon_boot_guard_fails_loudly_when_lock_file_cannot_be_opened() {
3785        let dir = tempfile::tempdir().expect("tempdir");
3786        // Point KHIVE_LOCK at a directory, not a file: opening a directory
3787        // with `write(true)` fails (EISDIR), so `acquire_recovery_lock`
3788        // returns `None` here — the exact failure mode
3789        // `acquire_daemon_boot_guard` must turn into a hard `Err` instead of
3790        // silently letting daemon-mode boot proceed unguarded.
3791        std::env::set_var("KHIVE_LOCK", dir.path());
3792
3793        let result = acquire_daemon_boot_guard();
3794        assert!(
3795            result.is_err(),
3796            "daemon boot guard must fail loudly, never silently proceed unguarded, \
3797             when the underlying recovery lock cannot be acquired"
3798        );
3799
3800        std::env::remove_var("KHIVE_LOCK");
3801    }
3802
3803    // ── write_pid_file_exclusive never truncates a winner's pid file ────────
3804
3805    #[test]
3806    fn write_pid_file_exclusive_creates_new_file_with_own_pid() {
3807        let dir = tempfile::tempdir().expect("tempdir");
3808        let pid_file = dir.path().join("khived.pid");
3809        write_pid_file_exclusive(&pid_file).expect("first writer must win");
3810        let contents = std::fs::read_to_string(&pid_file).expect("read pid file");
3811        assert_eq!(contents, std::process::id().to_string());
3812    }
3813
3814    #[test]
3815    fn write_pid_file_exclusive_refuses_to_overwrite_an_existing_file() {
3816        let dir = tempfile::tempdir().expect("tempdir");
3817        let pid_file = dir.path().join("khived.pid");
3818        std::fs::write(&pid_file, "999999").expect("seed an existing pid file");
3819
3820        let err = write_pid_file_exclusive(&pid_file)
3821            .expect_err("must not silently overwrite an existing pid file");
3822        assert_eq!(err.kind(), std::io::ErrorKind::AlreadyExists);
3823
3824        // The existing content must be completely untouched — proving this is
3825        // `create_new`, not the old `create(true).truncate(true)`.
3826        let contents = std::fs::read_to_string(&pid_file).expect("read pid file");
3827        assert_eq!(
3828            contents, "999999",
3829            "an existing pid file must never be truncated by a losing writer"
3830        );
3831    }
3832
3833    // Real (not simulated) concurrency: two OS threads race to `create_new`
3834    // the exact same path, synchronized with a `Barrier` so they genuinely
3835    // overlap at the syscall rather than relying on a sleep-based ordering
3836    // guess. This is the deterministic race oracle for the convergence
3837    // requirement the atomic-creation primitive `write_pid_file_exclusive`
3838    // is built on: exactly one of two simultaneous daemon starters may claim
3839    // the pid file, and the loser must see `AlreadyExists`, never silently
3840    // clobber the winner's content.
3841    #[test]
3842    fn two_concurrent_writers_converge_on_exactly_one_pid_file_owner() {
3843        let dir = tempfile::tempdir().expect("tempdir");
3844        let pid_file = std::sync::Arc::new(dir.path().join("khived.pid"));
3845        let barrier = std::sync::Arc::new(std::sync::Barrier::new(2));
3846
3847        let spawn_writer =
3848            |pid_file: std::sync::Arc<std::path::PathBuf>,
3849             barrier: std::sync::Arc<std::sync::Barrier>| {
3850                std::thread::spawn(move || {
3851                    barrier.wait();
3852                    write_pid_file_exclusive(&pid_file)
3853                })
3854            };
3855
3856        let t1 = spawn_writer(pid_file.clone(), barrier.clone());
3857        let t2 = spawn_writer(pid_file.clone(), barrier.clone());
3858        let r1 = t1.join().expect("writer 1 must not panic");
3859        let r2 = t2.join().expect("writer 2 must not panic");
3860
3861        let results = [&r1, &r2];
3862        let ok_count = results.iter().filter(|r| r.is_ok()).count();
3863        let already_exists_count = results
3864            .iter()
3865            .filter(|r| matches!(r, Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists))
3866            .count();
3867        assert_eq!(
3868            ok_count, 1,
3869            "exactly one of two concurrent writers must win the pid file"
3870        );
3871        assert_eq!(
3872            already_exists_count, 1,
3873            "the other writer must observe AlreadyExists, never a silent overwrite"
3874        );
3875        assert!(pid_file.exists(), "the winner's pid file must exist");
3876        let contents = std::fs::read_to_string(&*pid_file).expect("read pid file");
3877        assert_eq!(
3878            contents,
3879            std::process::id().to_string(),
3880            "the surviving pid file must contain the winner's pid — both threads \
3881             share this process's pid, so an unexpected value would also prove a \
3882             lost/garbled write raced through"
3883        );
3884    }
3885
3886    // ── connection principal (ADR-096 condition 2) ────────────────────────────
3887
3888    /// The peer-credential syscall must actually work on this platform and
3889    /// report the real uid, not error or return a placeholder.
3890    ///
3891    /// This matters more than it looks because the accept path fails CLOSED: if
3892    /// `peer_uid` errored unconditionally — wrong syscall, wrong socket option,
3893    /// an unimplemented platform arm — every connection would be refused and
3894    /// the daemon would be silently unreachable. A test that only exercised the
3895    /// decision function would not catch that.
3896    #[tokio::test]
3897    async fn peer_uid_reports_the_connecting_process_uid() {
3898        let dir = tempfile::tempdir().expect("tempdir");
3899        let sock = dir.path().join("peer.sock");
3900        let listener = UnixListener::bind(&sock).expect("bind");
3901
3902        let connect_path = sock.clone();
3903        let client = tokio::spawn(async move { UnixStream::connect(&connect_path).await });
3904
3905        let (server_side, _) = listener.accept().await.expect("accept");
3906        let client_side = client.await.expect("join").expect("connect");
3907
3908        // SAFETY: `geteuid` is always successful and takes no arguments.
3909        let expected = unsafe { libc::geteuid() } as u32;
3910
3911        assert_eq!(
3912            peer_uid(&server_side).expect("peer_uid must succeed on a live connection"),
3913            expected,
3914            "the uid read from the kernel for a same-process connection must be \
3915             this process's euid"
3916        );
3917        // Symmetric: both ends report the same peer on a same-uid connection.
3918        assert_eq!(
3919            peer_uid(&client_side).expect("peer_uid must succeed on the client end"),
3920            expected
3921        );
3922    }
3923
3924    /// The refusal rule itself: the principal is the uid, and only a foreign
3925    /// uid is refused.
3926    ///
3927    /// The second assertion is the load-bearing one and it is a regression
3928    /// guard, not a formality. ADR-096 shipped many `actor_id`s over one
3929    /// socket; every seat on a normal host is a distinct attribution at the
3930    /// same uid. A check that conflated attribution with principal would refuse
3931    /// them all, so "same uid is permitted" must stay true no matter how the
3932    /// rule is later tightened.
3933    #[test]
3934    fn only_a_foreign_uid_is_refused() {
3935        // SAFETY: `geteuid` is always successful and takes no arguments.
3936        let euid = unsafe { libc::geteuid() } as u32;
3937
3938        assert!(
3939            uid_is_permitted(euid, euid),
3940            "a connection from the daemon's own uid must be served — this is \
3941             every seat on the host, and ADR-096 accepted exactly this shape"
3942        );
3943        assert!(
3944            !uid_is_permitted(euid.wrapping_add(1), euid),
3945            "a connection from any other uid must be refused"
3946        );
3947        assert!(
3948            !uid_is_permitted(0, euid.wrapping_add(1)),
3949            "root is not special-cased: the rule is equality with the daemon's \
3950             euid, not a privilege comparison"
3951        );
3952    }
3953}