1use std::sync::Arc;
12
13#[cfg(unix)]
14use std::io::Write as _;
15#[cfg(unix)]
16use std::os::unix::fs::PermissionsExt;
17#[cfg(unix)]
18use std::os::unix::io::AsRawFd;
19use std::path::PathBuf;
20
21#[cfg(unix)]
22use async_trait::async_trait;
23#[cfg(unix)]
24use libc;
25use serde::{Deserialize, Serialize};
26#[cfg(unix)]
27use tokio::io::{AsyncReadExt, AsyncWriteExt};
28#[cfg(unix)]
29use tokio::net::{UnixListener, UnixStream};
30
31#[cfg(unix)]
32use khive_db::{run_checkpoint_task, CheckpointConfig, CheckpointLifecycleOwner, ConnectionPool};
33
34#[cfg(unix)]
35use crate::pack::RequestIdentity;
36
37pub const MAX_FRAME_BYTES: usize = 8 * 1024 * 1024;
39
40pub const PROTOCOL_VERSION: u32 = 4;
49
50const DEFAULT_DRAIN_TIMEOUT_SECS: u64 = 10;
51
52fn khive_dir() -> PathBuf {
68 khive_root_from(
69 std::env::var("HOME").ok(),
70 std::env::var("USERPROFILE").ok(),
71 )
72}
73
74fn khive_root_from(home: Option<String>, userprofile: Option<String>) -> PathBuf {
77 home.filter(|v| !v.trim().is_empty())
78 .or_else(|| userprofile.filter(|v| !v.trim().is_empty()))
79 .map(PathBuf::from)
80 .unwrap_or_else(last_resort_root)
81 .join(".khive")
82}
83
84#[cfg(unix)]
86fn last_resort_root() -> PathBuf {
87 PathBuf::from(".")
88}
89
90#[cfg(not(unix))]
91fn last_resort_root() -> PathBuf {
92 std::env::temp_dir()
93}
94
95#[cfg(unix)]
99pub fn socket_path() -> PathBuf {
100 if let Ok(p) = std::env::var("KHIVE_SOCKET") {
101 if !p.is_empty() {
102 return PathBuf::from(p);
103 }
104 }
105 khive_dir().join("khived.sock")
106}
107
108#[cfg(unix)]
112pub fn pid_path() -> PathBuf {
113 if let Ok(p) = std::env::var("KHIVE_PID") {
114 if !p.is_empty() {
115 return PathBuf::from(p);
116 }
117 }
118 khive_dir().join("khived.pid")
119}
120
121pub fn lock_path() -> PathBuf {
130 if let Ok(p) = std::env::var("KHIVE_LOCK") {
131 if !p.is_empty() {
132 return PathBuf::from(p);
133 }
134 }
135 khive_dir().join("khived.recovery.lock")
136}
137
138#[cfg(unix)]
148pub fn recoverer_lock_path() -> PathBuf {
149 if let Ok(p) = std::env::var("KHIVE_RECOVERER_LOCK") {
150 if !p.is_empty() {
151 return PathBuf::from(p);
152 }
153 }
154 khive_dir().join("khived.recoverer.lock")
155}
156
157#[cfg(unix)]
158fn open_lock_file(path: &std::path::Path) -> std::io::Result<std::fs::File> {
159 if let Some(parent) = path.parent() {
160 let _ = std::fs::create_dir_all(parent);
161 }
162 std::fs::OpenOptions::new()
163 .create(true)
164 .truncate(false)
165 .write(true)
166 .open(path)
167}
168
169#[cfg(unix)]
170fn acquire_flock_blocking(path: &std::path::Path, label: &str) -> Option<std::fs::File> {
171 let file = match open_lock_file(path) {
172 Ok(f) => f,
173 Err(e) => {
174 tracing::warn!(error = %e, path = ?path, "cannot open {label} lock file");
175 return None;
176 }
177 };
178 let rc = unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX) };
180 if rc != 0 {
181 tracing::warn!("flock LOCK_EX failed on {label} lock");
182 return None;
183 }
184 Some(file)
185}
186
187#[cfg(unix)]
194pub fn acquire_recovery_lock() -> Option<std::fs::File> {
195 acquire_flock_blocking(&lock_path(), "recovery")
196}
197
198#[cfg(unix)]
213fn try_acquire_flock_until(
214 path: &std::path::Path,
215 deadline: std::time::Instant,
216) -> std::io::Result<Option<std::fs::File>> {
217 let file = open_lock_file(path)?;
218 let poll_interval = std::time::Duration::from_millis(10);
219 loop {
220 let rc = unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) };
222 if rc == 0 {
223 return Ok(Some(file));
224 }
225 let err = std::io::Error::last_os_error();
226 if err.raw_os_error() != Some(libc::EWOULDBLOCK) {
227 return Err(err);
228 }
229 let now = std::time::Instant::now();
230 if now >= deadline {
231 return Ok(None);
232 }
233 std::thread::sleep(poll_interval.min(deadline - now));
234 }
235}
236
237#[cfg(unix)]
244pub fn try_acquire_daemon_boot_guard_until(
245 deadline: std::time::Instant,
246) -> std::io::Result<Option<DaemonBootGuard>> {
247 try_acquire_flock_until(&lock_path(), deadline)
248}
249
250#[cfg(unix)]
256pub fn try_acquire_recoverer_lock_until(
257 deadline: std::time::Instant,
258) -> std::io::Result<Option<std::fs::File>> {
259 try_acquire_flock_until(&recoverer_lock_path(), deadline)
260}
261
262#[cfg(unix)]
266pub type DaemonBootGuard = std::fs::File;
267
268#[cfg(unix)]
278pub fn acquire_daemon_boot_guard() -> anyhow::Result<DaemonBootGuard> {
279 acquire_recovery_lock()
280 .ok_or_else(|| anyhow::anyhow!("failed to acquire daemon boot/recovery lock"))
281}
282
283#[cfg(unix)]
290#[derive(Clone, Copy, PartialEq, Eq)]
291struct SocketIdentity {
292 dev: u64,
293 ino: u64,
294}
295
296#[cfg(unix)]
297fn socket_identity(path: &std::path::Path) -> Option<SocketIdentity> {
298 use std::os::unix::fs::MetadataExt;
299 let meta = std::fs::metadata(path).ok()?;
300 Some(SocketIdentity {
301 dev: meta.dev(),
302 ino: meta.ino(),
303 })
304}
305
306#[cfg(unix)]
320fn peer_uid(stream: &UnixStream) -> std::io::Result<u32> {
321 use std::os::fd::AsRawFd;
322 let fd = stream.as_raw_fd();
323
324 #[cfg(any(target_os = "macos", target_os = "ios", target_vendor = "apple"))]
325 {
326 let mut uid: libc::uid_t = 0;
327 let mut gid: libc::gid_t = 0;
328 let rc = unsafe { libc::getpeereid(fd, &mut uid, &mut gid) };
331 if rc != 0 {
332 return Err(std::io::Error::last_os_error());
333 }
334 Ok(uid as u32)
335 }
336
337 #[cfg(target_os = "linux")]
338 {
339 let mut cred = libc::ucred {
340 pid: 0,
341 uid: 0,
342 gid: 0,
343 };
344 let mut len = std::mem::size_of::<libc::ucred>() as libc::socklen_t;
345 let rc = unsafe {
349 libc::getsockopt(
350 fd,
351 libc::SOL_SOCKET,
352 libc::SO_PEERCRED,
353 (&mut cred as *mut libc::ucred).cast::<libc::c_void>(),
354 &mut len,
355 )
356 };
357 if rc != 0 {
358 return Err(std::io::Error::last_os_error());
359 }
360 Ok(cred.uid)
361 }
362
363 #[cfg(not(any(
364 target_os = "linux",
365 target_os = "macos",
366 target_os = "ios",
367 target_vendor = "apple"
368 )))]
369 {
370 let _ = fd;
371 Err(std::io::Error::new(
372 std::io::ErrorKind::Unsupported,
373 "peer-credential capture is not implemented for this platform",
374 ))
375 }
376}
377
378#[cfg(unix)]
399fn uid_is_permitted(peer: u32, daemon_euid: u32) -> bool {
400 peer == daemon_euid
401}
402
403#[derive(Serialize, Deserialize, Default)]
407pub struct DaemonRequestFrame {
408 pub ops: String,
409 pub presentation: Option<String>,
410 pub presentation_per_op: Option<Vec<Option<String>>>,
411 pub namespace: String,
417 #[serde(default)]
422 pub actor_id: Option<String>,
423 #[serde(default, skip_serializing_if = "Option::is_none")]
429 pub process_ref: Option<String>,
430 #[serde(default)]
436 pub visible_namespaces: Vec<String>,
437 #[serde(default)]
444 pub config_id: String,
445 #[serde(default)]
449 pub protocol_version: u32,
450 #[serde(default)]
456 pub probe_only: bool,
457 #[serde(default)]
467 pub metrics_only: bool,
468 #[serde(default)]
471 pub format: Option<String>,
472 #[serde(default)]
474 pub format_per_op: Option<Vec<Option<String>>>,
475 #[serde(default)]
486 pub from_wire: bool,
487 #[serde(default)]
496 pub request_id: Option<u64>,
497}
498
499#[derive(Serialize, Deserialize, Debug)]
501pub struct DaemonResponseFrame {
502 pub ok: bool,
503 pub result: Option<String>,
504 pub error: Option<String>,
505 pub namespace_mismatch: bool,
506 #[serde(default)]
510 pub config_mismatch: bool,
511 #[serde(default)]
518 pub served_config_id: Option<String>,
519 #[serde(default)]
524 pub version_mismatch: bool,
525 #[serde(default)]
529 pub daemon_protocol_version: u32,
530 #[serde(default, skip_serializing_if = "Option::is_none")]
536 pub metrics: Option<MetricsSnapshot>,
537 #[serde(default)]
544 pub request_id: Option<u64>,
545}
546
547#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq)]
558pub struct MetricsSnapshot {
559 pub wal_pages: Option<u64>,
564 pub wal_truncate_attempts: u64,
568 pub wal_truncate_consecutive_failures: u64,
572 #[serde(default)]
577 pub wal_checkpoint_skipped_ticks: u64,
578 #[serde(default)]
580 pub wal_checkpoint_consecutive_skips: u64,
581 #[serde(default)]
584 pub wal_checkpoint_last_skip_wal_pages: Option<u64>,
585 pub oldest_pinned_tx_micros: Option<u64>,
589 pub oldest_pinned_tx_label: Option<String>,
592 pub open_tx_count: usize,
594 pub write_queue_depth: Option<usize>,
599 pub write_queue_capacity: Option<usize>,
603}
604
605#[cfg(unix)]
609pub async fn read_frame(stream: &mut UnixStream) -> std::io::Result<Vec<u8>> {
610 let mut len_buf = [0u8; 4];
611 stream.read_exact(&mut len_buf).await?;
612 let len = u32::from_be_bytes(len_buf) as usize;
613 if len > MAX_FRAME_BYTES {
614 return Err(std::io::Error::new(
615 std::io::ErrorKind::InvalidData,
616 format!("daemon frame of {len} bytes exceeds {MAX_FRAME_BYTES} cap"),
617 ));
618 }
619 let mut buf = vec![0u8; len];
620 stream.read_exact(&mut buf).await?;
621 Ok(buf)
622}
623
624#[cfg(unix)]
626pub async fn write_frame(stream: &mut UnixStream, payload: &[u8]) -> std::io::Result<()> {
627 if payload.len() > MAX_FRAME_BYTES {
628 return Err(std::io::Error::new(
629 std::io::ErrorKind::InvalidData,
630 format!(
631 "daemon frame of {} bytes exceeds {MAX_FRAME_BYTES} cap",
632 payload.len()
633 ),
634 ));
635 }
636 let len = (payload.len() as u32).to_be_bytes();
637 stream.write_all(&len).await?;
638 stream.write_all(payload).await?;
639 stream.flush().await?;
640 Ok(())
641}
642
643#[cfg(unix)]
652#[async_trait]
653pub trait DaemonDispatch: Clone + Send + Sync + 'static {
654 #[allow(clippy::too_many_arguments)]
669 async fn dispatch(
670 &self,
671 ops: String,
672 presentation: Option<String>,
673 presentation_per_op: Option<Vec<Option<String>>>,
674 format: Option<String>,
675 format_per_op: Option<Vec<Option<String>>>,
676 from_wire: bool,
677 identity: Option<RequestIdentity>,
678 ) -> Result<String, String>;
679
680 async fn warm_all(&self);
682
683 fn namespace(&self) -> &str;
685
686 fn config_id(&self) -> &str;
691
692 fn pool_for_checkpoint(&self) -> Option<Arc<ConnectionPool>> {
700 None
701 }
702
703 fn secondary_pools_for_checkpoint(&self) -> Vec<Arc<ConnectionPool>> {
713 Vec::new()
714 }
715
716 fn event_store_for_checkpoint(&self) -> Option<Arc<dyn khive_storage::EventStore>> {
726 None
727 }
728}
729
730#[cfg(unix)]
731struct CheckpointTaskSpec {
732 pool: Arc<ConnectionPool>,
733 lifecycle_owner: Option<CheckpointLifecycleOwner>,
734 is_main: bool,
735}
736
737#[cfg(unix)]
744fn checkpoint_task_specs(
745 main_pool: Option<Arc<ConnectionPool>>,
746 secondary_pools: Vec<Arc<ConnectionPool>>,
747 event_store: Option<Arc<dyn khive_storage::EventStore>>,
748 namespace: String,
749) -> Vec<CheckpointTaskSpec> {
750 let mut tasks = Vec::with_capacity(usize::from(main_pool.is_some()) + secondary_pools.len());
751 if let Some(pool) = main_pool {
752 tasks.push(CheckpointTaskSpec {
753 pool,
754 lifecycle_owner: None,
755 is_main: true,
756 });
757 }
758 tasks.extend(secondary_pools.into_iter().map(|pool| CheckpointTaskSpec {
759 pool,
760 lifecycle_owner: None,
761 is_main: false,
762 }));
763
764 if let (Some(task), Some(event_store)) = (tasks.first_mut(), event_store) {
765 task.lifecycle_owner = Some(CheckpointLifecycleOwner::new(event_store, namespace));
766 }
767 tasks
768}
769
770static BACKGROUND_TASKS: std::sync::OnceLock<Arc<std::sync::atomic::AtomicUsize>> =
782 std::sync::OnceLock::new();
783
784fn background_tasks() -> &'static Arc<std::sync::atomic::AtomicUsize> {
785 BACKGROUND_TASKS.get_or_init(|| Arc::new(std::sync::atomic::AtomicUsize::new(0)))
786}
787
788struct BackgroundTaskGuard {
794 counter: Arc<std::sync::atomic::AtomicUsize>,
795}
796
797impl Drop for BackgroundTaskGuard {
798 fn drop(&mut self) {
799 self.counter
800 .fetch_sub(1, std::sync::atomic::Ordering::Relaxed);
801 }
802}
803
804pub fn track_background_task<F>(fut: F)
811where
812 F: std::future::Future<Output = ()> + Send + 'static,
813{
814 background_tasks().fetch_add(1, std::sync::atomic::Ordering::Relaxed);
815 let guard = BackgroundTaskGuard {
816 counter: background_tasks().clone(),
817 };
818 tokio::spawn(async move {
819 let _guard = guard;
820 fut.await;
821 });
822}
823
824pub fn background_task_count() -> usize {
827 background_tasks().load(std::sync::atomic::Ordering::Relaxed)
828}
829
830pub fn daemon_shutdown_token() -> tokio_util::sync::CancellationToken {
841 static TOKEN: std::sync::OnceLock<tokio_util::sync::CancellationToken> =
842 std::sync::OnceLock::new();
843 TOKEN
844 .get_or_init(tokio_util::sync::CancellationToken::new)
845 .clone()
846}
847
848static ACTIVE_PHASES: std::sync::OnceLock<
859 std::sync::Mutex<std::collections::HashMap<String, usize>>,
860> = std::sync::OnceLock::new();
861
862fn active_phases() -> &'static std::sync::Mutex<std::collections::HashMap<String, usize>> {
863 ACTIVE_PHASES.get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new()))
864}
865
866pub struct PhaseGuard {
872 name: String,
873}
874
875impl Drop for PhaseGuard {
876 fn drop(&mut self) {
877 let mut map = active_phases()
878 .lock()
879 .unwrap_or_else(std::sync::PoisonError::into_inner);
880 if let Some(count) = map.get_mut(&self.name) {
881 *count -= 1;
882 if *count == 0 {
883 map.remove(&self.name);
884 }
885 }
886 }
887}
888
889pub fn register_active_phase(name: &str) -> PhaseGuard {
894 let mut map = active_phases()
895 .lock()
896 .unwrap_or_else(std::sync::PoisonError::into_inner);
897 *map.entry(name.to_string()).or_insert(0) += 1;
898 PhaseGuard {
899 name: name.to_string(),
900 }
901}
902
903pub fn active_phase_names() -> Vec<String> {
906 let map = active_phases()
907 .lock()
908 .unwrap_or_else(std::sync::PoisonError::into_inner);
909 let mut names: Vec<String> = map.keys().cloned().collect();
910 names.sort();
911 names
912}
913
914#[cfg(unix)]
922fn build_metrics_snapshot<D: DaemonDispatch>(dispatcher: &D) -> MetricsSnapshot {
923 let open_tx_count = khive_storage::tx_registry::snapshot().len();
924 let (oldest_pinned_tx_micros, oldest_pinned_tx_label) =
931 match khive_storage::tx_registry::oldest() {
932 Some((_id, age, label)) => (Some(age.as_micros() as u64), label),
933 None => (None, None),
934 };
935
936 let (write_queue_depth, write_queue_capacity) = dispatcher
937 .pool_for_checkpoint()
938 .and_then(|pool| pool.writer_task_handle().ok().flatten())
939 .map(|handle| (Some(handle.queue_depth()), Some(handle.capacity())))
940 .unwrap_or((None, None));
941
942 MetricsSnapshot {
943 wal_pages: khive_db::checkpoint::last_observed_wal_pages(),
944 wal_truncate_attempts: khive_db::checkpoint::truncate_attempts(),
945 wal_truncate_consecutive_failures: khive_db::checkpoint::truncate_consecutive_failures(),
946 wal_checkpoint_skipped_ticks: khive_db::checkpoint::checkpoint_skipped_ticks(),
947 wal_checkpoint_consecutive_skips: khive_db::checkpoint::checkpoint_consecutive_skips(),
948 wal_checkpoint_last_skip_wal_pages: khive_db::checkpoint::checkpoint_last_skip_wal_pages(),
949 oldest_pinned_tx_micros,
950 oldest_pinned_tx_label,
951 open_tx_count,
952 write_queue_depth,
953 write_queue_capacity,
954 }
955}
956
957#[cfg(unix)]
958async fn handle_conn<D: DaemonDispatch>(mut stream: UnixStream, dispatcher: D) {
959 let raw = match read_frame(&mut stream).await {
960 Ok(r) => r,
961 Err(e) => {
962 tracing::debug!(error = %e, "failed to read daemon request frame");
963 return;
964 }
965 };
966 let frame: DaemonRequestFrame = match serde_json::from_slice(&raw) {
967 Ok(f) => f,
968 Err(e) => {
969 tracing::debug!(error = %e, "failed to decode daemon request frame");
970 return;
971 }
972 };
973
974 let served_config_id = Some(dispatcher.config_id().to_string());
975 let resp = if frame.protocol_version != PROTOCOL_VERSION {
976 let msg = format!(
977 "daemon protocol mismatch: client={} daemon={} — \
978 rebuild/update the client binary (make local)",
979 frame.protocol_version, PROTOCOL_VERSION,
980 );
981 tracing::warn!(
982 client_version = frame.protocol_version,
983 daemon_version = PROTOCOL_VERSION,
984 "daemon protocol version mismatch"
985 );
986 DaemonResponseFrame {
987 ok: false,
988 result: None,
989 error: Some(msg),
990 namespace_mismatch: false,
991 config_mismatch: false,
992 served_config_id,
993 version_mismatch: true,
994 daemon_protocol_version: PROTOCOL_VERSION,
995 metrics: None,
996 request_id: frame.request_id,
997 }
998 } else if frame.metrics_only {
999 DaemonResponseFrame {
1006 ok: true,
1007 result: None,
1008 error: None,
1009 namespace_mismatch: false,
1010 config_mismatch: false,
1011 served_config_id,
1012 version_mismatch: false,
1013 daemon_protocol_version: PROTOCOL_VERSION,
1014 metrics: Some(build_metrics_snapshot(&dispatcher)),
1015 request_id: frame.request_id,
1016 }
1017 } else if frame.config_id != dispatcher.config_id() {
1027 DaemonResponseFrame {
1028 ok: false,
1029 result: None,
1030 error: None,
1031 namespace_mismatch: false,
1032 config_mismatch: true,
1033 served_config_id,
1034 version_mismatch: false,
1035 daemon_protocol_version: PROTOCOL_VERSION,
1036 metrics: None,
1037 request_id: frame.request_id,
1038 }
1039 } else if frame.probe_only {
1040 DaemonResponseFrame {
1044 ok: true,
1045 result: None,
1046 error: None,
1047 namespace_mismatch: false,
1048 config_mismatch: false,
1049 served_config_id,
1050 version_mismatch: false,
1051 daemon_protocol_version: PROTOCOL_VERSION,
1052 metrics: None,
1053 request_id: frame.request_id,
1054 }
1055 } else {
1056 let identity = RequestIdentity {
1065 namespace: frame.namespace.clone(),
1066 actor_id: frame.actor_id.clone(),
1067 visible_namespaces: frame.visible_namespaces.clone(),
1068 process_ref: frame.process_ref.clone(),
1069 request_id: frame.request_id,
1070 };
1071 match dispatcher
1072 .dispatch(
1073 frame.ops,
1074 frame.presentation,
1075 frame.presentation_per_op,
1076 frame.format,
1077 frame.format_per_op,
1078 frame.from_wire,
1079 Some(identity),
1080 )
1081 .await
1082 {
1083 Ok(result) => DaemonResponseFrame {
1084 ok: true,
1085 result: Some(result),
1086 error: None,
1087 namespace_mismatch: false,
1088 config_mismatch: false,
1089 served_config_id,
1090 version_mismatch: false,
1091 daemon_protocol_version: PROTOCOL_VERSION,
1092 metrics: None,
1093 request_id: frame.request_id,
1094 },
1095 Err(e) => DaemonResponseFrame {
1096 ok: false,
1097 result: None,
1098 error: Some(e),
1099 namespace_mismatch: false,
1100 config_mismatch: false,
1101 served_config_id,
1102 version_mismatch: false,
1103 daemon_protocol_version: PROTOCOL_VERSION,
1104 metrics: None,
1105 request_id: frame.request_id,
1106 },
1107 }
1108 };
1109
1110 match serde_json::to_vec(&resp) {
1111 Ok(payload) => {
1112 if payload.len() > MAX_FRAME_BYTES {
1113 tracing::warn!(
1120 bytes = payload.len(),
1121 limit = MAX_FRAME_BYTES,
1122 "daemon response exceeds MAX_FRAME_BYTES; sending explicit error frame"
1123 );
1124 let err_resp = DaemonResponseFrame {
1125 ok: false,
1126 result: None,
1127 error: Some(format!(
1128 "response too large: {} bytes exceeds {} byte IPC cap",
1129 payload.len(),
1130 MAX_FRAME_BYTES,
1131 )),
1132 namespace_mismatch: false,
1133 config_mismatch: false,
1134 served_config_id: resp.served_config_id,
1135 version_mismatch: false,
1136 daemon_protocol_version: PROTOCOL_VERSION,
1137 metrics: None,
1138 request_id: resp.request_id,
1139 };
1140 if let Ok(err_payload) = serde_json::to_vec(&err_resp) {
1141 if let Err(e) = write_frame(&mut stream, &err_payload).await {
1142 tracing::debug!(error = %e, "failed to write oversized-response error frame");
1143 }
1144 }
1145 } else if let Err(e) = write_frame(&mut stream, &payload).await {
1146 tracing::debug!(error = %e, "failed to write daemon response frame");
1147 }
1148 }
1149 Err(e) => tracing::warn!(error = %e, "failed to serialize daemon response frame"),
1150 }
1151}
1152
1153#[cfg(unix)]
1157struct ActiveConnectionGuard {
1158 active: Arc<std::sync::atomic::AtomicUsize>,
1159}
1160
1161#[cfg(unix)]
1162impl ActiveConnectionGuard {
1163 fn claim(active: Arc<std::sync::atomic::AtomicUsize>) -> Self {
1164 active.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
1165 Self { active }
1166 }
1167}
1168
1169#[cfg(unix)]
1170impl Drop for ActiveConnectionGuard {
1171 fn drop(&mut self) {
1172 self.active
1173 .fetch_sub(1, std::sync::atomic::Ordering::Relaxed);
1174 }
1175}
1176
1177#[cfg(unix)]
1178fn spawn_connection_task<F>(
1179 active: Arc<std::sync::atomic::AtomicUsize>,
1180 future: F,
1181) -> tokio::task::JoinHandle<()>
1182where
1183 F: std::future::Future<Output = ()> + Send + 'static,
1184{
1185 let guard = ActiveConnectionGuard::claim(active);
1186 tokio::spawn(async move {
1187 let _guard = guard;
1188 future.await;
1189 })
1190}
1191
1192#[cfg(unix)]
1203pub async fn run_daemon<D: DaemonDispatch>(dispatcher: D) -> anyhow::Result<()> {
1204 let boot_guard = Some(acquire_daemon_boot_guard()?);
1205 run_daemon_with_boot_guard(dispatcher, boot_guard).await
1206}
1207
1208#[cfg(unix)]
1231fn ensure_socket_dir_is_trusted(parent: &std::path::Path) -> anyhow::Result<()> {
1232 let daemon_euid = unsafe { libc::geteuid() } as u32;
1234
1235 if parent == khive_dir() {
1236 std::fs::set_permissions(parent, std::fs::Permissions::from_mode(0o700)).map_err(|e| {
1237 anyhow::anyhow!(
1238 "refusing to start: cannot chmod 0700 {}: {e}. The khive directory must be \
1239 owner-only — it is half of the same-uid guarantee this daemon enforces.",
1240 parent.display()
1241 )
1242 })?;
1243 return ensure_socket_path_is_swap_resistant(parent, daemon_euid);
1244 }
1245
1246 let meta = std::fs::metadata(parent).map_err(|e| {
1249 anyhow::anyhow!(
1250 "refusing to start: cannot stat {}: {e}. The socket directory gates \
1251 socket-takeover safety, and unreadable metadata is not a passing state.",
1252 parent.display()
1253 )
1254 })?;
1255
1256 use std::os::unix::fs::MetadataExt;
1257 let owner = meta.uid();
1258 if owner != daemon_euid && owner != 0 {
1259 anyhow::bail!(
1260 "refusing to start: socket directory {} is owned by uid {owner}, not this \
1261 daemon's uid ({daemon_euid}) or root. A directory owner can replace the \
1262 socket regardless of mode bits. Point KHIVE_SOCKET at a directory you own, \
1263 or unset it for the default.",
1264 parent.display()
1265 );
1266 }
1267
1268 let mode = meta.permissions().mode();
1269 if mode & 0o022 != 0 {
1270 anyhow::bail!(
1271 "refusing to start: socket directory {} is mode {:04o} — writable by group or \
1272 other, so another local user could bind their own listener at the socket path \
1273 (before this daemon starts, the sticky bit does not prevent creating the \
1274 path). Use a directory only you can write, or unset KHIVE_SOCKET for the \
1275 default. This daemon is not changing the permissions of a directory it does \
1276 not own.",
1277 parent.display(),
1278 mode & 0o7777
1279 );
1280 }
1281
1282 ensure_socket_path_is_swap_resistant(parent, daemon_euid)
1283}
1284
1285#[cfg(unix)]
1310fn ensure_socket_path_is_swap_resistant(
1311 parent: &std::path::Path,
1312 daemon_euid: u32,
1313) -> anyhow::Result<()> {
1314 use std::os::unix::fs::MetadataExt;
1315
1316 let absolute = if parent.is_absolute() {
1317 parent.to_path_buf()
1318 } else {
1319 std::env::current_dir()
1320 .map_err(|e| {
1321 anyhow::anyhow!(
1322 "refusing to start: cannot resolve the working directory to absolutize \
1323 socket directory {}: {e}.",
1324 parent.display()
1325 )
1326 })?
1327 .join(parent)
1328 };
1329
1330 fn push_components(stack: &mut Vec<std::ffi::OsString>, path: &std::path::Path) {
1331 let components: Vec<_> = path
1332 .components()
1333 .map(|c| c.as_os_str().to_os_string())
1334 .collect();
1335 stack.extend(components.into_iter().rev());
1336 }
1337
1338 let mut stack: Vec<std::ffi::OsString> = Vec::new();
1339 push_components(&mut stack, &absolute);
1340 let mut resolved = std::path::PathBuf::new();
1341 let mut symlinks_followed = 0u32;
1342
1343 while let Some(component) = stack.pop() {
1344 if component == "/" {
1345 resolved = std::path::PathBuf::from("/");
1346 continue;
1347 }
1348 if component == "." {
1349 continue;
1350 }
1351 if component == ".." {
1352 resolved.pop();
1353 continue;
1354 }
1355 let candidate = resolved.join(&component);
1356 let meta = std::fs::symlink_metadata(&candidate).map_err(|e| {
1357 anyhow::anyhow!(
1358 "refusing to start: cannot stat socket-path component {}: {e}. An \
1359 unreadable component is not a passing one.",
1360 candidate.display()
1361 )
1362 })?;
1363 let owner = meta.uid();
1364
1365 if meta.file_type().is_symlink() {
1366 symlinks_followed += 1;
1367 if symlinks_followed > 40 {
1368 anyhow::bail!(
1369 "refusing to start: socket path resolves through more than 40 symlinks \
1370 at {} — treating this as a loop.",
1371 candidate.display()
1372 );
1373 }
1374 if owner != daemon_euid && owner != 0 {
1375 anyhow::bail!(
1376 "refusing to start: socket-path symlink component {} is owned by uid \
1377 {owner}, not this daemon's uid ({daemon_euid}) or root — its owner \
1378 could retarget it after this check and re-root the socket path. Point \
1379 KHIVE_SOCKET somewhere trusted end to end, or unset it for the \
1380 default.",
1381 candidate.display()
1382 );
1383 }
1384 let target = std::fs::read_link(&candidate).map_err(|e| {
1385 anyhow::anyhow!(
1386 "refusing to start: cannot read socket-path symlink component {}: {e}.",
1387 candidate.display()
1388 )
1389 })?;
1390 push_components(&mut stack, &target);
1391 continue;
1392 }
1393
1394 if meta.is_dir() {
1395 let mode = meta.permissions().mode();
1396 let sticky = mode & 0o1000 != 0;
1397 if owner != daemon_euid && owner != 0 {
1398 anyhow::bail!(
1399 "refusing to start: socket-path ancestor {} is owned by uid {owner}, not \
1400 this daemon's uid ({daemon_euid}) or root — its owner could rename the \
1401 next path component and re-root the socket path. Point KHIVE_SOCKET \
1402 somewhere trusted end to end, or unset it for the default.",
1403 candidate.display()
1404 );
1405 }
1406 if mode & 0o022 != 0 && !sticky {
1407 anyhow::bail!(
1408 "refusing to start: socket-path ancestor {} is mode {:04o} — writable by \
1409 group or other without the sticky bit, so another local user could rename \
1410 the next path component and re-root the socket path. Point KHIVE_SOCKET \
1411 somewhere trusted end to end, or unset it for the default.",
1412 candidate.display(),
1413 mode & 0o7777
1414 );
1415 }
1416 resolved = candidate;
1417 continue;
1418 }
1419
1420 anyhow::bail!(
1421 "refusing to start: socket-path component {} is neither a directory nor a \
1422 symlink — the socket path cannot traverse it.",
1423 candidate.display()
1424 );
1425 }
1426
1427 Ok(())
1428}
1429
1430#[cfg(unix)]
1443pub async fn run_daemon_with_boot_guard<D: DaemonDispatch>(
1444 dispatcher: D,
1445 boot_guard: Option<std::fs::File>,
1446) -> anyhow::Result<()> {
1447 struct ComponentTeardown;
1457 impl Drop for ComponentTeardown {
1458 fn drop(&mut self) {
1459 daemon_shutdown_token().cancel();
1460 }
1461 }
1462 let _component_teardown = ComponentTeardown;
1463
1464 let sock = socket_path();
1465 let pid_file = pid_path();
1466
1467 if let Some(parent) = sock.parent() {
1468 std::fs::create_dir_all(parent)?;
1469 ensure_socket_dir_is_trusted(parent)?;
1470 }
1471
1472 let _startup_lock = boot_guard;
1486
1487 if !cleanup_stale_daemon(&sock, &pid_file).await {
1488 tracing::info!("a responsive khived is already running; exiting");
1489 return Ok(());
1490 }
1491
1492 let listener = UnixListener::bind(&sock)?;
1493 if let Err(e) = std::fs::set_permissions(&sock, std::fs::Permissions::from_mode(0o600)) {
1497 drop(listener);
1498 let _ = std::fs::remove_file(&sock);
1499 return Err(anyhow::anyhow!(
1500 "refusing to start: cannot chmod 0600 {}: {e}. The daemon socket must be owner-only \
1501 — it is half of the single-principal guarantee this daemon enforces.",
1502 sock.display()
1503 ));
1504 }
1505 let bound_identity = socket_identity(&sock);
1509
1510 if let Err(e) = write_pid_file_exclusive(&pid_file) {
1511 if e.kind() == std::io::ErrorKind::AlreadyExists {
1512 if bound_identity.is_some() && socket_identity(&sock) == bound_identity {
1520 drop(listener);
1521 let _ = std::fs::remove_file(&sock);
1522 }
1523 if pid_file_names_a_reachable_daemon(&pid_file, &sock).await {
1524 tracing::info!(
1525 "a replacement khived already claimed the pid/socket rendezvous; exiting"
1526 );
1527 return Ok(());
1528 }
1529 anyhow::bail!(
1530 "failed to claim daemon pid file at {pid_file:?}: it already exists \
1531 and does not name a reachable daemon"
1532 );
1533 }
1534 return Err(e.into());
1535 }
1536 drop(_startup_lock);
1540 tracing::info!(
1541 socket = ?sock,
1542 pid = std::process::id(),
1543 source_revision = crate::BUILD_INFO.source_revision,
1544 build_time = crate::BUILD_INFO.build_time,
1545 "khived listening"
1546 );
1547
1548 {
1549 let warm = dispatcher.clone();
1550 tokio::spawn(async move {
1551 warm.warm_all().await;
1552 });
1553 }
1554
1555 let (checkpoint_shutdown_tx, checkpoint_shutdown_rx) = tokio::sync::watch::channel(());
1563 let checkpoint_tasks = checkpoint_task_specs(
1569 dispatcher.pool_for_checkpoint(),
1570 dispatcher.secondary_pools_for_checkpoint(),
1571 dispatcher.event_store_for_checkpoint(),
1572 dispatcher.namespace().to_string(),
1573 );
1574 if !checkpoint_tasks.is_empty() {
1575 let cfg = CheckpointConfig::from_env();
1576 let checkpoint_task_count = checkpoint_tasks.len();
1577 for task in checkpoint_tasks {
1578 track_background_task(run_checkpoint_task(
1579 task.pool,
1580 cfg.clone(),
1581 task.lifecycle_owner,
1582 checkpoint_shutdown_rx.clone(),
1583 task.is_main,
1584 ));
1585 }
1586 tracing::info!(checkpoint_task_count, "WAL checkpoint task(s) started");
1587 }
1588
1589 let active = Arc::new(std::sync::atomic::AtomicUsize::new(0));
1590
1591 let shutdown = async {
1592 let mut sigterm = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
1596 .expect("install SIGTERM handler");
1597 let mut sigint = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::interrupt())
1598 .expect("install SIGINT handler");
1599 tokio::select! {
1600 _ = sigterm.recv() => tracing::info!("received SIGTERM"),
1601 _ = sigint.recv() => tracing::info!("received SIGINT"),
1602 }
1603 };
1604
1605 let daemon_euid = unsafe { libc::geteuid() } as u32;
1607
1608 tokio::select! {
1609 _ = async {
1610 loop {
1611 match listener.accept().await {
1612 Ok((stream, _)) => {
1613 match peer_uid(&stream) {
1627 Ok(peer) if uid_is_permitted(peer, daemon_euid) => {}
1628 Ok(peer) => {
1629 tracing::error!(
1630 peer_uid = peer,
1631 daemon_euid,
1632 "refusing connection from a foreign uid: this daemon accepts \
1633 only peers running as its own uid"
1634 );
1635 drop(stream);
1636 continue;
1637 }
1638 Err(e) => {
1639 tracing::error!(
1640 error = %e,
1641 "refusing connection: cannot read peer credentials, so \
1642 same-uid cannot be proven"
1643 );
1644 drop(stream);
1645 continue;
1646 }
1647 }
1648 let d = dispatcher.clone();
1649 spawn_connection_task(Arc::clone(&active), async move {
1650 handle_conn(stream, d).await;
1651 });
1652 }
1653 Err(e) => tracing::error!(error = %e, "accept failed"),
1654 }
1655 }
1656 } => {}
1657 _ = shutdown => {}
1658 }
1659
1660 let _ = checkpoint_shutdown_tx.send(());
1664
1665 daemon_shutdown_token().cancel();
1669
1670 drain(&active).await;
1671
1672 match acquire_recovery_lock() {
1681 Some(_shutdown_lock) => {
1682 shutdown_cleanup_if_owned(&sock, &pid_file, bound_identity);
1683 }
1684 None => {
1685 tracing::warn!(
1686 "could not acquire recovery lock for shutdown cleanup; \
1687 skipping unlink to avoid deleting a replacement daemon's paths"
1688 );
1689 }
1690 }
1691 tracing::info!("khived stopped");
1692 Ok(())
1693}
1694
1695#[cfg(unix)]
1705fn shutdown_cleanup_if_owned(
1706 sock: &std::path::Path,
1707 pid_file: &std::path::Path,
1708 bound_identity: Option<SocketIdentity>,
1709) -> bool {
1710 let pid_is_ours = std::fs::read_to_string(pid_file)
1711 .ok()
1712 .and_then(|s| s.trim().parse::<u32>().ok())
1713 == Some(std::process::id());
1714 let socket_is_ours = bound_identity.is_some() && socket_identity(sock) == bound_identity;
1715 if pid_is_ours && socket_is_ours {
1716 let _ = std::fs::remove_file(sock);
1717 let _ = std::fs::remove_file(pid_file);
1718 true
1719 } else {
1720 tracing::warn!(
1721 socket = ?sock,
1722 pid_file = ?pid_file,
1723 "skipping shutdown cleanup — a replacement daemon already owns this socket/PID"
1724 );
1725 false
1726 }
1727}
1728
1729#[cfg(unix)]
1733#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1734enum PidLiveness {
1735 Alive,
1738 Dead,
1740 PermissionDenied,
1745}
1746
1747#[cfg(unix)]
1748impl PidLiveness {
1749 fn is_running(self) -> bool {
1750 !matches!(self, PidLiveness::Dead)
1751 }
1752}
1753
1754#[cfg(unix)]
1758fn classify_kill_result(rc: i32, errno: i32) -> PidLiveness {
1759 if rc == 0 {
1760 return PidLiveness::Alive;
1761 }
1762 match errno {
1763 libc::EPERM => PidLiveness::PermissionDenied,
1764 _ => PidLiveness::Dead,
1765 }
1766}
1767
1768#[cfg(unix)]
1769fn is_process_running(pid: u32) -> bool {
1770 let Ok(pid) = i32::try_from(pid) else {
1771 return false;
1772 };
1773 if pid <= 0 {
1774 return false;
1775 }
1776 let rc = unsafe { libc::kill(pid, 0) };
1778 let errno = std::io::Error::last_os_error().raw_os_error().unwrap_or(0);
1779 classify_kill_result(rc, errno).is_running()
1780}
1781
1782#[cfg(unix)]
1783async fn cleanup_stale_daemon(sock: &std::path::Path, pid_file: &std::path::Path) -> bool {
1784 if let Ok(pid_str) = std::fs::read_to_string(pid_file) {
1785 if let Ok(pid) = pid_str.trim().parse::<u32>() {
1786 if pid != std::process::id()
1787 && is_process_running(pid)
1788 && sock.exists()
1789 && UnixStream::connect(sock).await.is_ok()
1790 {
1791 return false;
1792 }
1793 }
1794 }
1795 if sock.exists() {
1796 if let Err(e) = std::fs::remove_file(sock) {
1797 tracing::warn!(error = %e, path = ?sock, "failed to remove stale socket");
1798 }
1799 }
1800 if pid_file.exists() {
1801 if let Err(e) = std::fs::remove_file(pid_file) {
1802 tracing::warn!(error = %e, path = ?pid_file, "failed to remove stale PID file");
1803 }
1804 }
1805 true
1806}
1807
1808#[cfg(unix)]
1819fn write_pid_file_exclusive(pid_file: &std::path::Path) -> std::io::Result<()> {
1820 use std::os::unix::fs::OpenOptionsExt;
1821
1822 let mut opts = std::fs::OpenOptions::new();
1823 opts.write(true).create_new(true).mode(0o600);
1824 let mut f = opts.open(pid_file)?;
1825 f.write_all(std::process::id().to_string().as_bytes())?;
1826 Ok(())
1827}
1828
1829#[cfg(unix)]
1834async fn pid_file_names_a_reachable_daemon(
1835 pid_file: &std::path::Path,
1836 sock: &std::path::Path,
1837) -> bool {
1838 let Ok(pid_str) = std::fs::read_to_string(pid_file) else {
1839 return false;
1840 };
1841 let Ok(pid) = pid_str.trim().parse::<u32>() else {
1842 return false;
1843 };
1844 pid != std::process::id()
1845 && is_process_running(pid)
1846 && sock.exists()
1847 && UnixStream::connect(sock).await.is_ok()
1848}
1849
1850#[cfg(unix)]
1851async fn drain(active: &std::sync::atomic::AtomicUsize) {
1852 use std::sync::atomic::Ordering;
1853 let remaining = || active.load(Ordering::Relaxed) + background_task_count();
1854 if remaining() == 0 {
1855 return;
1856 }
1857 let deadline = tokio::time::Instant::now() + drain_timeout();
1858 while remaining() > 0 {
1859 if tokio::time::Instant::now() >= deadline {
1860 tracing::warn!(
1861 remaining_connections = active.load(Ordering::Relaxed),
1862 remaining_background_tasks = background_task_count(),
1863 "drain timeout reached; forcing shutdown"
1864 );
1865 break;
1866 }
1867 tokio::time::sleep(std::time::Duration::from_millis(100)).await;
1868 }
1869}
1870
1871pub fn drain_timeout() -> std::time::Duration {
1877 let secs = std::env::var("KHIVE_DRAIN_TIMEOUT_SECS")
1878 .ok()
1879 .and_then(|v| v.parse::<u64>().ok())
1880 .unwrap_or(DEFAULT_DRAIN_TIMEOUT_SECS);
1881 std::time::Duration::from_secs(secs)
1882}
1883
1884#[cfg(unix)]
1886pub fn env_truthy(key: &str) -> bool {
1887 std::env::var(key)
1888 .map(|v| {
1889 let v = v.trim();
1890 !v.is_empty() && v != "0" && !v.eq_ignore_ascii_case("false")
1891 })
1892 .unwrap_or(false)
1893}
1894
1895#[cfg(test)]
1896mod khive_root_tests {
1897 use super::khive_root_from;
1898 use std::path::PathBuf;
1899
1900 #[test]
1901 fn home_wins_when_set() {
1902 assert_eq!(
1903 khive_root_from(Some("/base/home".into()), Some("/other".into())),
1904 PathBuf::from("/base/home/.khive")
1905 );
1906 }
1907
1908 #[test]
1909 fn userprofile_backfills_missing_home() {
1910 assert_eq!(
1911 khive_root_from(None, Some("/profile/home".into())),
1912 PathBuf::from("/profile/home/.khive")
1913 );
1914 }
1915
1916 #[test]
1917 fn blank_values_are_skipped() {
1918 assert_eq!(
1919 khive_root_from(Some(" ".into()), Some("/profile/home".into())),
1920 PathBuf::from("/profile/home/.khive")
1921 );
1922 }
1923
1924 #[cfg(not(unix))]
1925 #[test]
1926 fn fallback_is_never_cwd_relative() {
1927 assert!(khive_root_from(None, None).is_absolute());
1931 }
1932
1933 #[cfg(unix)]
1934 #[test]
1935 fn unix_fallback_stays_historical_not_shared_tmp() {
1936 assert_eq!(khive_root_from(None, None), PathBuf::from("./.khive"));
1940 }
1941}
1942
1943#[cfg(all(test, unix))]
1944mod tests {
1945 use super::*;
1946 use serial_test::serial;
1947
1948 #[tokio::test]
1949 #[serial(checkpoint_skip_metrics)]
1950 async fn secondary_only_checkpoint_topology_emits_lifecycle_outcome() {
1951 let main_backend = khive_db::StorageBackend::memory().expect("in-memory main backend");
1952 let event_store = main_backend.events().expect("main event store");
1953 let secondary_dir = tempfile::tempdir().expect("secondary tempdir");
1954 let secondary_backend =
1955 khive_db::StorageBackend::sqlite(secondary_dir.path().join("secondary.db"))
1956 .expect("file-backed secondary backend");
1957
1958 let mut tasks = checkpoint_task_specs(
1959 None,
1960 vec![secondary_backend.pool_arc()],
1961 Some(Arc::clone(&event_store)),
1962 "local".to_string(),
1963 );
1964 assert_eq!(tasks.len(), 1);
1965 let task = tasks.pop().expect("one secondary checkpoint task");
1966 assert!(!task.is_main, "the only checkpoint task must be secondary");
1967 assert!(
1968 task.lifecycle_owner.is_some(),
1969 "the secondary task must own lifecycle emission when no main task exists"
1970 );
1971
1972 let config = CheckpointConfig {
1973 interval: std::time::Duration::from_millis(10),
1974 warn_pages: 0,
1975 ..CheckpointConfig::default()
1976 };
1977 let (shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(());
1978 let handle = tokio::spawn(run_checkpoint_task(
1979 task.pool,
1980 config,
1981 task.lifecycle_owner,
1982 shutdown_rx,
1983 task.is_main,
1984 ));
1985
1986 tokio::time::sleep(std::time::Duration::from_millis(60)).await;
1987 shutdown_tx.send(()).expect("send checkpoint shutdown");
1988 tokio::time::timeout(std::time::Duration::from_secs(1), handle)
1989 .await
1990 .expect("checkpoint task should exit within 1s")
1991 .expect("checkpoint task panicked");
1992
1993 let events = event_store
1994 .query_events(
1995 khive_storage::EventFilter::default(),
1996 khive_storage::PageRequest {
1997 limit: 100,
1998 offset: 0,
1999 },
2000 )
2001 .await
2002 .expect("query lifecycle events");
2003 assert!(
2004 !events.items.is_empty()
2005 && events
2006 .items
2007 .iter()
2008 .all(|event| event.kind == khive_types::EventKind::CheckpointOutcomeRecorded),
2009 "the designated secondary owner must emit CheckpointOutcomeRecorded"
2010 );
2011
2012 let file_main_dir = tempfile::tempdir().expect("file-backed main tempdir");
2013 let file_main = khive_db::StorageBackend::sqlite(file_main_dir.path().join("main.db"))
2014 .expect("file-backed main backend");
2015 let tasks = checkpoint_task_specs(
2016 Some(file_main.pool_arc()),
2017 vec![secondary_backend.pool_arc()],
2018 Some(event_store),
2019 "local".to_string(),
2020 );
2021 assert!(tasks[0].is_main && tasks[0].lifecycle_owner.is_some());
2022 assert!(!tasks[1].is_main && tasks[1].lifecycle_owner.is_none());
2023 }
2024
2025 #[test]
2029 fn current_process_is_running() {
2030 let pid = std::process::id();
2032 assert!(
2033 is_process_running(pid),
2034 "current process {pid} should be detected as running"
2035 );
2036 }
2037
2038 #[test]
2039 fn pid_zero_is_not_running() {
2040 assert!(
2043 !is_process_running(0),
2044 "pid 0 must be rejected by the guard before the unsafe call"
2045 );
2046 }
2047
2048 #[test]
2049 fn very_large_pid_is_not_running() {
2050 assert!(
2052 !is_process_running(u32::MAX),
2053 "u32::MAX should fail i32 conversion and return false"
2054 );
2055 }
2056
2057 #[test]
2061 fn classify_kill_result_zero_is_alive() {
2062 assert_eq!(classify_kill_result(0, 0), PidLiveness::Alive);
2063 assert!(classify_kill_result(0, 0).is_running());
2064 }
2065
2066 #[test]
2067 fn classify_kill_result_esrch_is_dead() {
2068 assert_eq!(classify_kill_result(-1, libc::ESRCH), PidLiveness::Dead);
2069 assert!(!classify_kill_result(-1, libc::ESRCH).is_running());
2070 }
2071
2072 #[test]
2073 fn classify_kill_result_eperm_is_permission_denied_and_counts_as_running() {
2074 assert_eq!(
2075 classify_kill_result(-1, libc::EPERM),
2076 PidLiveness::PermissionDenied
2077 );
2078 assert!(
2079 classify_kill_result(-1, libc::EPERM).is_running(),
2080 "EPERM must be unknown-safe: treated as running, never as a basis \
2081 for stale cleanup to unlink a live daemon's rendezvous files"
2082 );
2083 }
2084
2085 #[test]
2086 fn pid_1_probe_is_running_regardless_of_permission_outcome() {
2087 assert!(
2095 is_process_running(1),
2096 "PID 1 always exists; EPERM must not read as dead"
2097 );
2098 }
2099
2100 #[test]
2101 fn env_truthy_recognises_set_values() {
2102 assert!(!env_truthy("__KHIVE_TEST_ABSENT_VAR_XYZ__"));
2103
2104 let key = "__KHIVE_TEST_TRUTHY_ABC__";
2108 std::env::set_var(key, "1");
2109 assert!(env_truthy(key));
2110 std::env::set_var(key, "false");
2111 assert!(!env_truthy(key));
2112 std::env::set_var(key, "0");
2113 assert!(!env_truthy(key));
2114 std::env::remove_var(key);
2115 }
2116
2117 #[tokio::test(flavor = "current_thread", start_paused = true)]
2118 #[serial(background_tasks)]
2119 async fn accepted_connection_is_counted_before_first_poll_and_drain_waits() {
2120 use std::sync::atomic::Ordering;
2121
2122 let active = Arc::new(std::sync::atomic::AtomicUsize::new(0));
2123 let started = Arc::new(std::sync::atomic::AtomicBool::new(false));
2124 let (release_tx, release_rx) = tokio::sync::oneshot::channel::<()>();
2125 let started_in_task = Arc::clone(&started);
2126
2127 let handle = spawn_connection_task(Arc::clone(&active), async move {
2128 started_in_task.store(true, Ordering::Relaxed);
2129 let _ = release_rx.await;
2130 });
2131
2132 assert_eq!(active.load(Ordering::Relaxed), 1);
2133 assert!(
2134 !started.load(Ordering::Relaxed),
2135 "the current-thread runtime must leave the spawned handler unpolled"
2136 );
2137
2138 let drain_fut = drain(active.as_ref());
2139 tokio::pin!(drain_fut);
2140 let too_early =
2141 tokio::time::timeout(std::time::Duration::from_millis(150), &mut drain_fut).await;
2142 assert!(
2143 too_early.is_err(),
2144 "drain must wait for a connection claimed before its task's first poll"
2145 );
2146 assert!(started.load(Ordering::Relaxed));
2147
2148 release_tx.send(()).expect("handler still waiting");
2149 tokio::time::timeout(std::time::Duration::from_secs(1), handle)
2150 .await
2151 .expect("handler should finish promptly")
2152 .expect("handler should not panic");
2153 assert_eq!(active.load(Ordering::Relaxed), 0);
2154 tokio::time::timeout(std::time::Duration::from_secs(1), drain_fut)
2155 .await
2156 .expect("drain should finish once the handler releases its claim");
2157 }
2158
2159 #[tokio::test(flavor = "current_thread")]
2160 async fn cancelled_connection_releases_count_before_first_poll() {
2161 use std::sync::atomic::Ordering;
2162
2163 let active = Arc::new(std::sync::atomic::AtomicUsize::new(0));
2164 let started = Arc::new(std::sync::atomic::AtomicBool::new(false));
2165 let started_in_task = Arc::clone(&started);
2166 let handle = spawn_connection_task(Arc::clone(&active), async move {
2167 started_in_task.store(true, Ordering::Relaxed);
2168 std::future::pending::<()>().await;
2169 });
2170
2171 assert_eq!(active.load(Ordering::Relaxed), 1);
2172 handle.abort();
2173 let error = handle.await.expect_err("aborted handler must be cancelled");
2174 assert!(error.is_cancelled());
2175 assert!(!started.load(Ordering::Relaxed));
2176 assert_eq!(active.load(Ordering::Relaxed), 0);
2177 }
2178
2179 #[tokio::test(flavor = "current_thread")]
2180 async fn panicked_connection_releases_count() {
2181 use std::sync::atomic::Ordering;
2182
2183 let active = Arc::new(std::sync::atomic::AtomicUsize::new(0));
2184 let handle = spawn_connection_task(Arc::clone(&active), async move {
2185 panic!("intentional connection-handler panic");
2186 });
2187
2188 assert_eq!(active.load(Ordering::Relaxed), 1);
2189 let error = handle
2190 .await
2191 .expect_err("panicked handler must fail its join");
2192 assert!(error.is_panic());
2193 assert_eq!(active.load(Ordering::Relaxed), 0);
2194 }
2195
2196 #[test]
2197 fn connection_claim_releases_if_spawn_panics() {
2198 use std::sync::atomic::Ordering;
2199
2200 let active = Arc::new(std::sync::atomic::AtomicUsize::new(0));
2201 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
2202 drop(spawn_connection_task(Arc::clone(&active), async {}));
2203 }));
2204
2205 assert!(result.is_err(), "tokio::spawn outside a runtime must panic");
2206 assert_eq!(active.load(Ordering::Relaxed), 0);
2207 }
2208
2209 #[tokio::test]
2210 #[serial(background_tasks)]
2211 async fn drain_returns_promptly_with_no_accepted_connection() {
2212 let active = std::sync::atomic::AtomicUsize::new(0);
2213 tokio::time::timeout(std::time::Duration::from_secs(1), drain(&active))
2214 .await
2215 .expect("empty drain should return immediately");
2216 }
2217
2218 #[tokio::test]
2231 #[serial(background_tasks)]
2232 async fn drain_waits_for_tracked_background_tasks_before_returning() {
2233 let active = std::sync::atomic::AtomicUsize::new(0);
2234 let (tx, rx) = tokio::sync::oneshot::channel::<()>();
2235
2236 track_background_task(async move {
2237 let _ = rx.await;
2238 });
2239 assert!(
2240 background_task_count() >= 1,
2241 "track_background_task must make the in-flight task visible immediately"
2242 );
2243
2244 let drain_fut = drain(&active);
2245 tokio::pin!(drain_fut);
2246
2247 let too_early =
2249 tokio::time::timeout(std::time::Duration::from_millis(150), &mut drain_fut).await;
2250 assert!(
2251 too_early.is_err(),
2252 "drain() must not return while a tracked background task is still running"
2253 );
2254
2255 tx.send(())
2257 .expect("tracked task still awaiting the oneshot");
2258 let done = tokio::time::timeout(std::time::Duration::from_secs(5), drain_fut).await;
2259 assert!(
2260 done.is_ok(),
2261 "drain() must return once the tracked background task finishes"
2262 );
2263 }
2264
2265 #[tokio::test]
2270 #[serial(background_tasks)]
2271 async fn track_background_task_count_returns_to_zero_after_completion() {
2272 let before = background_task_count();
2274 let (tx, rx) = tokio::sync::oneshot::channel::<()>();
2275 track_background_task(async move {
2276 let _ = rx.await;
2277 });
2278 assert_eq!(background_task_count(), before + 1);
2279 tx.send(()).expect("still awaiting");
2280 for _ in 0..100 {
2282 if background_task_count() == before {
2283 break;
2284 }
2285 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
2286 }
2287 assert_eq!(background_task_count(), before);
2288 }
2289
2290 #[tokio::test]
2293 #[serial(background_tasks)]
2294 async fn track_background_task_count_returns_to_baseline_after_panic() {
2295 let before = background_task_count();
2305
2306 let (tx, rx) = tokio::sync::oneshot::channel::<()>();
2307 track_background_task(async move {
2308 let _ = rx.await;
2309 panic!("intentional panic to exercise the Drop-guard decrement path");
2310 });
2311 assert_eq!(background_task_count(), before + 1);
2312
2313 tx.send(()).expect("still awaiting");
2314 for _ in 0..100 {
2315 if background_task_count() == before {
2316 break;
2317 }
2318 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
2319 }
2320 assert_eq!(
2321 background_task_count(),
2322 before,
2323 "background task counter must return to baseline after the tracked future panics"
2324 );
2325 }
2326
2327 #[test]
2335 #[serial(active_phases)]
2336 fn register_active_phase_appears_and_disappears_with_the_guard() {
2337 assert!(
2338 !active_phase_names().contains(&"adr103_test_phase".to_string()),
2339 "must start absent (leaked from a prior failed run would poison this test)"
2340 );
2341
2342 let guard = register_active_phase("adr103_test_phase");
2343 assert!(active_phase_names().contains(&"adr103_test_phase".to_string()));
2344
2345 drop(guard);
2346 assert!(
2347 !active_phase_names().contains(&"adr103_test_phase".to_string()),
2348 "the phase name must drop out of the gauge once its guard is dropped"
2349 );
2350 }
2351
2352 #[test]
2353 #[serial(active_phases)]
2354 fn register_active_phase_counts_concurrent_occurrences_of_the_same_name() {
2355 let first = register_active_phase("adr103_concurrent_phase");
2356 let second = register_active_phase("adr103_concurrent_phase");
2357 assert!(active_phase_names().contains(&"adr103_concurrent_phase".to_string()));
2358
2359 drop(first);
2360 assert!(
2361 active_phase_names().contains(&"adr103_concurrent_phase".to_string()),
2362 "one of two concurrent occurrences ending must not remove the name early"
2363 );
2364
2365 drop(second);
2366 assert!(
2367 !active_phase_names().contains(&"adr103_concurrent_phase".to_string()),
2368 "the name must be removed only once every concurrent occurrence has ended"
2369 );
2370 }
2371
2372 #[derive(Clone)]
2379 struct MockDispatch {
2380 namespace: String,
2381 config_id: String,
2382 dispatch_calls: Arc<std::sync::atomic::AtomicUsize>,
2383 pool: Option<Arc<ConnectionPool>>,
2384 dispatch_err: Option<String>,
2388 }
2389
2390 #[async_trait]
2391 impl DaemonDispatch for MockDispatch {
2392 async fn dispatch(
2393 &self,
2394 _ops: String,
2395 _presentation: Option<String>,
2396 _presentation_per_op: Option<Vec<Option<String>>>,
2397 _format: Option<String>,
2398 _format_per_op: Option<Vec<Option<String>>>,
2399 _from_wire: bool,
2400 _identity: Option<RequestIdentity>,
2401 ) -> Result<String, String> {
2402 self.dispatch_calls
2403 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
2404 match &self.dispatch_err {
2405 Some(msg) => Err(msg.clone()),
2406 None => Ok("{}".to_string()),
2407 }
2408 }
2409
2410 async fn warm_all(&self) {}
2411
2412 fn namespace(&self) -> &str {
2413 &self.namespace
2414 }
2415
2416 fn config_id(&self) -> &str {
2417 &self.config_id
2418 }
2419
2420 fn pool_for_checkpoint(&self) -> Option<Arc<ConnectionPool>> {
2421 self.pool.clone()
2422 }
2423 }
2424
2425 fn base_request_frame(config_id: &str) -> DaemonRequestFrame {
2426 DaemonRequestFrame {
2427 ops: String::new(),
2428 presentation: None,
2429 presentation_per_op: None,
2430 namespace: "local".to_string(),
2431 actor_id: None,
2432 process_ref: None,
2433 visible_namespaces: Vec::new(),
2434 config_id: config_id.to_string(),
2435 protocol_version: PROTOCOL_VERSION,
2436 probe_only: false,
2437 metrics_only: false,
2438 format: None,
2439 format_per_op: None,
2440 from_wire: false,
2441 request_id: None,
2442 }
2443 }
2444
2445 async fn round_trip(dispatcher: MockDispatch, req: &DaemonRequestFrame) -> DaemonResponseFrame {
2448 let (mut client, server) = UnixStream::pair().expect("unix stream pair");
2449 let payload = serde_json::to_vec(req).expect("encode request frame");
2450 let handle = tokio::spawn(async move {
2451 handle_conn(server, dispatcher).await;
2452 });
2453 write_frame(&mut client, &payload)
2454 .await
2455 .expect("write request frame");
2456 let raw = read_frame(&mut client).await.expect("read response frame");
2457 handle.await.expect("handle_conn task panicked");
2458 serde_json::from_slice(&raw).expect("decode response frame")
2459 }
2460
2461 #[tokio::test]
2466 async fn protocol_v3_frame_is_rejected_before_process_ref_dispatch() {
2467 assert_eq!(PROTOCOL_VERSION, 4, "process_ref is the protocol-v4 change");
2468 let dispatch_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
2469 let dispatcher = MockDispatch {
2470 namespace: "local".to_string(),
2471 config_id: "cfg-v4".to_string(),
2472 dispatch_calls: Arc::clone(&dispatch_calls),
2473 pool: None,
2474 dispatch_err: None,
2475 };
2476 let mut request = base_request_frame("cfg-v4");
2477 request.protocol_version = 3;
2478 request.process_ref = Some("worker/legacy-rollout".to_string());
2479
2480 let response = round_trip(dispatcher, &request).await;
2481 assert!(!response.ok);
2482 assert!(response.version_mismatch);
2483 assert_eq!(response.daemon_protocol_version, 4);
2484 assert_eq!(
2485 dispatch_calls.load(std::sync::atomic::Ordering::SeqCst),
2486 0,
2487 "a v3 frame must be rejected before a provenance-bearing mutation dispatches"
2488 );
2489 let error = response.error.expect("mismatch explains both versions");
2490 assert!(
2491 error.contains("client=3") && error.contains("daemon=4"),
2492 "mismatch must identify the exact rollout boundary; got {error:?}"
2493 );
2494 }
2495
2496 #[tokio::test]
2503 async fn metrics_only_frame_returns_snapshot_without_dispatching() {
2504 let dispatch_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
2505 let dispatcher = MockDispatch {
2506 namespace: "local".to_string(),
2507 config_id: "cfg-a".to_string(),
2508 dispatch_calls: Arc::clone(&dispatch_calls),
2509 pool: None,
2510 dispatch_err: None,
2511 };
2512
2513 let mut metrics_req = base_request_frame("cfg-a");
2514 metrics_req.metrics_only = true;
2515 let metrics_resp = round_trip(dispatcher.clone(), &metrics_req).await;
2516
2517 assert!(metrics_resp.ok, "metrics_only response must be ok=true");
2518 assert!(
2519 metrics_resp.metrics.is_some(),
2520 "metrics_only=true must return Some(snapshot)"
2521 );
2522 assert_eq!(
2523 dispatch_calls.load(std::sync::atomic::Ordering::SeqCst),
2524 0,
2525 "metrics_only must never reach the ops-dispatch path"
2526 );
2527
2528 let mut mismatched_req = base_request_frame("some-other-config");
2530 mismatched_req.metrics_only = true;
2531 let mismatched_resp = round_trip(dispatcher.clone(), &mismatched_req).await;
2532 assert!(mismatched_resp.ok);
2533 assert!(mismatched_resp.metrics.is_some());
2534 assert!(!mismatched_resp.config_mismatch);
2535 assert_eq!(
2536 dispatch_calls.load(std::sync::atomic::Ordering::SeqCst),
2537 0,
2538 "a mismatched-config metrics_only request must still skip dispatch"
2539 );
2540
2541 let normal_req = base_request_frame("cfg-a");
2544 let normal_resp = round_trip(dispatcher, &normal_req).await;
2545 assert!(normal_resp.ok);
2546 assert!(normal_resp.metrics.is_none());
2547 assert_eq!(dispatch_calls.load(std::sync::atomic::Ordering::SeqCst), 1);
2548 }
2549
2550 #[test]
2563 fn shared_writable_socket_dirs_are_refused_without_being_modified() {
2564 let dir = tempfile::tempdir().expect("tempdir");
2565 for (name, mode) in [("open", 0o777u32), ("sticky-tmp", 0o1777u32)] {
2566 let shared = dir.path().join(name);
2567 std::fs::create_dir(&shared).expect("create");
2568 std::fs::set_permissions(&shared, std::fs::Permissions::from_mode(mode))
2569 .expect("chmod");
2570
2571 let err = ensure_socket_dir_is_trusted(&shared)
2572 .expect_err("group/other-writable must be refused, sticky or not");
2573 assert!(
2574 err.to_string().contains(&format!("{:04o}", mode & 0o7777)),
2575 "the refusal should name the mode it saw, got: {err}"
2576 );
2577 let after = std::fs::metadata(&shared)
2578 .expect("stat")
2579 .permissions()
2580 .mode()
2581 & 0o7777;
2582 assert_eq!(
2583 after, mode,
2584 "refusing must not re-permission a directory khive does not own"
2585 );
2586 }
2587 }
2588
2589 #[test]
2597 fn writable_non_sticky_ancestor_is_refused() {
2598 let dir = tempfile::tempdir().expect("tempdir");
2599 let open_mid = dir.path().join("open-mid");
2600 std::fs::create_dir(&open_mid).expect("create mid");
2601 let inner = open_mid.join("private");
2602 std::fs::create_dir(&inner).expect("create inner");
2603 std::fs::set_permissions(&inner, std::fs::Permissions::from_mode(0o700)).expect("chmod");
2604 std::fs::set_permissions(&open_mid, std::fs::Permissions::from_mode(0o777))
2605 .expect("chmod mid");
2606
2607 let err = ensure_socket_dir_is_trusted(&inner)
2608 .expect_err("a 0777 non-sticky ancestor must be refused");
2609 assert!(
2610 err.to_string().contains("ancestor"),
2611 "the refusal should say it was an ancestor that failed, got: {err}"
2612 );
2613 assert!(
2614 err.to_string().contains("open-mid"),
2615 "the refusal should name the failing ancestor, got: {err}"
2616 );
2617 }
2618
2619 #[test]
2626 fn symlink_component_to_untrusted_directory_is_refused() {
2627 let dir = tempfile::tempdir().expect("tempdir");
2628 let open = dir.path().join("open-target");
2629 std::fs::create_dir(&open).expect("create target");
2630 std::fs::set_permissions(&open, std::fs::Permissions::from_mode(0o777)).expect("chmod");
2631 let link = dir.path().join("link");
2632 std::os::unix::fs::symlink(&open, &link).expect("symlink");
2633
2634 let euid = unsafe { libc::geteuid() } as u32;
2636 let err = ensure_socket_path_is_swap_resistant(&link, euid)
2637 .expect_err("a link into a 0777 non-sticky directory must be refused");
2638 assert!(
2639 err.to_string().contains("open-target"),
2640 "the refusal should name the untrusted target directory, got: {err}"
2641 );
2642 }
2643
2644 #[test]
2653 fn foreign_owned_symlink_component_is_refused() {
2654 let link =
2655 std::path::PathBuf::from(format!("/tmp/khive-swaptest-link-{}", std::process::id()));
2656 let _ = std::fs::remove_file(&link);
2657 std::os::unix::fs::symlink("/tmp", &link).expect("symlink");
2658
2659 let not_my_euid = (unsafe { libc::geteuid() } as u32).wrapping_add(1);
2661 let result = ensure_socket_path_is_swap_resistant(&link, not_my_euid);
2662 std::fs::remove_file(&link).expect("cleanup");
2663
2664 let err = result.expect_err("a symlink owned by another uid must be refused");
2665 assert!(
2666 err.to_string().contains("symlink component"),
2667 "the refusal should strike the symlink itself, got: {err}"
2668 );
2669 assert!(
2670 err.to_string().contains("khive-swaptest-link"),
2671 "the refusal should name the offending link, got: {err}"
2672 );
2673 }
2674
2675 #[test]
2685 fn trusted_socket_dirs_are_accepted_unmodified() {
2686 let dir = tempfile::tempdir().expect("tempdir");
2687 for (name, mode) in [("private", 0o700), ("listable", 0o755)] {
2688 let d = dir.path().join(name);
2689 std::fs::create_dir(&d).expect("create");
2690 std::fs::set_permissions(&d, std::fs::Permissions::from_mode(mode)).expect("chmod");
2691
2692 ensure_socket_dir_is_trusted(&d)
2693 .unwrap_or_else(|e| panic!("mode {mode:04o} must be accepted, got: {e}"));
2694
2695 let after = std::fs::metadata(&d).expect("stat").permissions().mode() & 0o7777;
2696 assert_eq!(
2697 after, mode,
2698 "acceptance must not re-permission the directory either"
2699 );
2700 }
2701 }
2702
2703 #[tokio::test]
2708 async fn metrics_snapshot_wal_pages_reflects_recent_write() {
2709 let dir = tempfile::tempdir().expect("tempdir");
2710 let path = dir.path().join("metrics_wal_test.db");
2711 let pool = Arc::new(
2712 ConnectionPool::new(khive_db::PoolConfig {
2713 path: Some(path),
2714 ..khive_db::PoolConfig::default()
2715 })
2716 .expect("pool open"),
2717 );
2718
2719 {
2720 let writer = pool.try_writer().expect("writer");
2721 writer
2722 .conn()
2723 .execute_batch(
2724 "CREATE TABLE t (x INTEGER); \
2725 INSERT INTO t VALUES (1); \
2726 INSERT INTO t VALUES (2);",
2727 )
2728 .expect("seed writes");
2729 }
2730
2731 let tick = khive_db::checkpoint_once(
2732 &pool,
2733 &CheckpointConfig::default(),
2734 &mut khive_db::checkpoint::TruncateState::default(),
2735 );
2736 assert!(
2737 matches!(tick, khive_db::CheckpointTick::Observed(_)),
2738 "checkpoint_once on a freshly-writer-held pool must observe, not skip: {tick:?}"
2739 );
2740
2741 let dispatcher = MockDispatch {
2742 namespace: "local".to_string(),
2743 config_id: "cfg-wal".to_string(),
2744 dispatch_calls: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
2745 pool: Some(pool),
2746 dispatch_err: None,
2747 };
2748
2749 let snapshot = build_metrics_snapshot(&dispatcher);
2750 assert!(
2751 snapshot.wal_pages.is_some(),
2752 "wal_pages must be observed after a real checkpoint tick, got {snapshot:?}"
2753 );
2754 assert_eq!(
2758 snapshot.wal_checkpoint_consecutive_skips, 0,
2759 "an observed (non-skipped) tick must report zero consecutive skips, got {snapshot:?}"
2760 );
2761 }
2762
2763 #[test]
2768 #[serial(tx_registry)]
2769 fn metrics_snapshot_reflects_open_transaction_registry() {
2770 let dispatcher = MockDispatch {
2771 namespace: "local".to_string(),
2772 config_id: "cfg-tx".to_string(),
2773 dispatch_calls: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
2774 pool: None,
2775 dispatch_err: None,
2776 };
2777
2778 let departing_handle = khive_storage::tx_registry::register(Some(
2779 "daemon_metrics_snapshot_departing_test_tx".to_string(),
2780 ));
2781 let before = build_metrics_snapshot(&dispatcher).open_tx_count;
2782 assert!(before >= 1);
2783
2784 let handle = khive_storage::tx_registry::register(Some(
2785 "daemon_metrics_snapshot_owned_test_tx".to_string(),
2786 ));
2787 drop(departing_handle);
2788
2789 let during = build_metrics_snapshot(&dispatcher);
2790 assert!(
2791 during.open_tx_count >= 1,
2792 "open_tx_count must reflect the live owned transaction despite registry churn: \
2793 churn_baseline={before} during={}",
2794 during.open_tx_count
2795 );
2796 assert!(
2797 during.oldest_pinned_tx_micros.is_some(),
2798 "oldest_pinned_tx_micros must be Some while a transaction is open"
2799 );
2800
2801 drop(handle);
2802 assert!(
2803 !khive_storage::tx_registry::snapshot()
2804 .iter()
2805 .any(|(_, label)| label.as_deref()
2806 == Some("daemon_metrics_snapshot_owned_test_tx")),
2807 "the owned registry entry must disappear when its handle is dropped"
2808 );
2809 }
2810
2811 #[tokio::test]
2815 async fn metrics_snapshot_write_queue_depth_flag_gated() {
2816 let dir = tempfile::tempdir().expect("tempdir");
2817
2818 let enabled_pool = Arc::new(
2819 ConnectionPool::new(khive_db::PoolConfig {
2820 path: Some(dir.path().join("wq_enabled.db")),
2821 write_queue_enabled: true,
2822 ..khive_db::PoolConfig::default()
2823 })
2824 .expect("pool open"),
2825 );
2826 let enabled_dispatcher = MockDispatch {
2827 namespace: "local".to_string(),
2828 config_id: "cfg-wq-on".to_string(),
2829 dispatch_calls: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
2830 pool: Some(enabled_pool),
2831 dispatch_err: None,
2832 };
2833 let snapshot_on = build_metrics_snapshot(&enabled_dispatcher);
2834 assert!(
2835 snapshot_on.write_queue_depth.is_some(),
2836 "write_queue_depth must be Some when write_queue_enabled=true, got {snapshot_on:?}"
2837 );
2838 assert!(snapshot_on.write_queue_capacity.is_some());
2839
2840 let disabled_pool = Arc::new(
2841 ConnectionPool::new(khive_db::PoolConfig {
2842 path: Some(dir.path().join("wq_disabled.db")),
2843 write_queue_enabled: false,
2844 ..khive_db::PoolConfig::default()
2845 })
2846 .expect("pool open"),
2847 );
2848 let disabled_dispatcher = MockDispatch {
2849 namespace: "local".to_string(),
2850 config_id: "cfg-wq-off".to_string(),
2851 dispatch_calls: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
2852 pool: Some(disabled_pool),
2853 dispatch_err: None,
2854 };
2855 let snapshot_off = build_metrics_snapshot(&disabled_dispatcher);
2856 assert!(
2857 snapshot_off.write_queue_depth.is_none(),
2858 "write_queue_depth must be None when write_queue_enabled=false, got {snapshot_off:?}"
2859 );
2860 assert!(snapshot_off.write_queue_capacity.is_none());
2861
2862 let no_pool_dispatcher = MockDispatch {
2864 namespace: "local".to_string(),
2865 config_id: "cfg-no-pool".to_string(),
2866 dispatch_calls: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
2867 pool: None,
2868 dispatch_err: None,
2869 };
2870 let snapshot_no_pool = build_metrics_snapshot(&no_pool_dispatcher);
2871 assert!(snapshot_no_pool.write_queue_depth.is_none());
2872 assert!(snapshot_no_pool.write_queue_capacity.is_none());
2873 }
2874
2875 #[test]
2880 fn frame_serde_defaults_additive_fields_when_absent() {
2881 let req_json = serde_json::json!({
2882 "ops": "",
2883 "presentation": null,
2884 "presentation_per_op": null,
2885 "namespace": "local",
2886 "actor_id": null,
2887 "visible_namespaces": [],
2888 "config_id": "cfg",
2889 "protocol_version": PROTOCOL_VERSION,
2890 "probe_only": false,
2891 "format": null,
2892 "format_per_op": null,
2893 "from_wire": false
2894 });
2895 let frame: DaemonRequestFrame =
2896 serde_json::from_value(req_json).expect("decode a metrics_only-absent request frame");
2897 assert!(
2898 !frame.metrics_only,
2899 "metrics_only must default to false when absent from the wire payload"
2900 );
2901 assert_eq!(
2902 frame.request_id, None,
2903 "request_id must default to None when absent from the wire payload (khive#948)"
2904 );
2905 assert_eq!(
2906 frame.process_ref, None,
2907 "process_ref must default to None when absent from the wire payload (khive#1428)"
2908 );
2909 let encoded_frame = serde_json::to_value(&frame).expect("encode request frame");
2910 assert!(
2911 encoded_frame.get("process_ref").is_none(),
2912 "absent provenance must not change the serialized request wire shape"
2913 );
2914
2915 let resp_json = serde_json::json!({
2916 "ok": true,
2917 "result": null,
2918 "error": null,
2919 "namespace_mismatch": false,
2920 "config_mismatch": false,
2921 "served_config_id": "cfg",
2922 "version_mismatch": false,
2923 "daemon_protocol_version": PROTOCOL_VERSION
2924 });
2925 let resp: DaemonResponseFrame =
2926 serde_json::from_value(resp_json).expect("decode a metrics-absent response frame");
2927 assert!(
2928 resp.metrics.is_none(),
2929 "metrics must default to None when absent from the wire payload"
2930 );
2931 assert_eq!(
2932 resp.request_id, None,
2933 "request_id must default to None when absent from the wire payload (khive#948)"
2934 );
2935 }
2936
2937 #[tokio::test]
2942 async fn request_id_echoed_on_success_and_error_arms() {
2943 let dispatcher = MockDispatch {
2944 namespace: "local".to_string(),
2945 config_id: "cfg-a".to_string(),
2946 dispatch_calls: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
2947 pool: None,
2948 dispatch_err: None,
2949 };
2950 let mut ok_req = base_request_frame("cfg-a");
2951 ok_req.request_id = Some(42);
2952 let ok_resp = round_trip(dispatcher, &ok_req).await;
2953 assert!(ok_resp.ok, "expected successful dispatch: {ok_resp:?}");
2954 assert_eq!(
2955 ok_resp.request_id,
2956 Some(42),
2957 "request_id must be echoed back on a successful dispatch response"
2958 );
2959
2960 let mismatched_dispatcher = MockDispatch {
2963 namespace: "local".to_string(),
2964 config_id: "cfg-a".to_string(),
2965 dispatch_calls: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
2966 pool: None,
2967 dispatch_err: None,
2968 };
2969 let mut mismatch_req = base_request_frame("cfg-WRONG");
2970 mismatch_req.request_id = Some(99);
2971 let mismatch_resp = round_trip(mismatched_dispatcher, &mismatch_req).await;
2972 assert!(mismatch_resp.config_mismatch);
2973 assert_eq!(
2974 mismatch_resp.request_id,
2975 Some(99),
2976 "request_id must be echoed on the config_mismatch rejection arm too"
2977 );
2978
2979 let erroring_dispatcher = MockDispatch {
2982 namespace: "local".to_string(),
2983 config_id: "cfg-a".to_string(),
2984 dispatch_calls: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
2985 pool: None,
2986 dispatch_err: Some("simulated dispatch error".to_string()),
2987 };
2988 let mut err_req = base_request_frame("cfg-a");
2989 err_req.request_id = Some(7);
2990 let err_resp = round_trip(erroring_dispatcher, &err_req).await;
2991 assert!(!err_resp.ok, "expected a dispatch error: {err_resp:?}");
2992 assert_eq!(
2993 err_resp.request_id,
2994 Some(7),
2995 "request_id must be echoed on the real ops-dispatch error arm"
2996 );
2997 }
2998
2999 #[test]
3008 fn shutdown_cleanup_removes_paths_it_still_owns() {
3009 let dir = tempfile::tempdir().expect("tempdir");
3010 let sock = dir.path().join("khived.sock");
3011 let pid_file = dir.path().join("khived.pid");
3012
3013 let _listener = std::os::unix::net::UnixListener::bind(&sock).expect("bind socket");
3014 std::fs::write(&pid_file, std::process::id().to_string()).expect("write pid file");
3015 let identity = socket_identity(&sock);
3016 assert!(
3017 identity.is_some(),
3018 "must read identity of a freshly bound socket"
3019 );
3020
3021 let cleaned = shutdown_cleanup_if_owned(&sock, &pid_file, identity);
3022
3023 assert!(
3024 cleaned,
3025 "cleanup must proceed when PID and socket still match"
3026 );
3027 assert!(!sock.exists(), "owned socket must be removed");
3028 assert!(!pid_file.exists(), "owned pid file must be removed");
3029 }
3030
3031 #[test]
3032 fn shutdown_cleanup_skips_when_pid_file_names_a_different_process() {
3033 let dir = tempfile::tempdir().expect("tempdir");
3034 let sock = dir.path().join("khived.sock");
3035 let pid_file = dir.path().join("khived.pid");
3036
3037 let _listener = std::os::unix::net::UnixListener::bind(&sock).expect("bind socket");
3038 let identity = socket_identity(&sock);
3039 std::fs::write(&pid_file, "1").expect("write foreign pid file");
3043
3044 let cleaned = shutdown_cleanup_if_owned(&sock, &pid_file, identity);
3045
3046 assert!(
3047 !cleaned,
3048 "cleanup must be skipped when the PID file no longer names this process"
3049 );
3050 assert!(sock.exists(), "replacement daemon's socket must survive");
3051 assert!(
3052 pid_file.exists(),
3053 "replacement daemon's pid file must survive"
3054 );
3055 }
3056
3057 #[test]
3058 fn shutdown_cleanup_skips_when_socket_was_rebound_by_a_replacement() {
3059 let dir = tempfile::tempdir().expect("tempdir");
3060 let sock = dir.path().join("khived.sock");
3061 let original_sock = dir.path().join("original.sock");
3062 let pid_file = dir.path().join("khived.pid");
3063
3064 let _original_listener =
3074 std::os::unix::net::UnixListener::bind(&original_sock).expect("bind original socket");
3075 let _replacement_listener =
3076 std::os::unix::net::UnixListener::bind(&sock).expect("bind replacement socket");
3077
3078 let original_identity = socket_identity(&original_sock);
3079 let replacement_identity = socket_identity(&sock);
3080 assert!(
3081 original_identity.is_some(),
3082 "must read identity of the original socket"
3083 );
3084 assert!(
3085 replacement_identity.is_some(),
3086 "must read identity of the replacement socket"
3087 );
3088 assert!(
3089 original_identity != replacement_identity,
3090 "two concurrently bound sockets must have distinct identities"
3091 );
3092
3093 std::fs::write(&pid_file, std::process::id().to_string())
3094 .expect("write pid file matching this process");
3095
3096 let cleaned = shutdown_cleanup_if_owned(&sock, &pid_file, original_identity);
3101
3102 assert!(
3103 !cleaned,
3104 "cleanup must be skipped when the socket at this path is a different \
3105 inode than the one this daemon originally bound"
3106 );
3107 assert!(sock.exists(), "replacement daemon's socket must survive");
3108 assert!(
3109 pid_file.exists(),
3110 "replacement daemon's pid file must survive"
3111 );
3112 }
3113
3114 #[test]
3126 #[serial]
3127 fn recovery_lock_serializes_two_concurrent_boot_sequences() {
3128 let dir = tempfile::tempdir().expect("tempdir");
3129 let lock_file = dir.path().join("khived.recovery.lock");
3130 std::env::set_var("KHIVE_LOCK", &lock_file);
3131
3132 let active = Arc::new(std::sync::atomic::AtomicUsize::new(0));
3133 let overlap_detected = Arc::new(std::sync::atomic::AtomicBool::new(false));
3134
3135 let run_one_boot =
3136 |active: Arc<std::sync::atomic::AtomicUsize>,
3137 overlap: Arc<std::sync::atomic::AtomicBool>| {
3138 move || {
3139 let _guard = acquire_recovery_lock().expect("acquire recovery lock");
3140 if active.fetch_add(1, std::sync::atomic::Ordering::SeqCst) != 0 {
3142 overlap.store(true, std::sync::atomic::Ordering::SeqCst);
3143 }
3144 std::thread::sleep(std::time::Duration::from_millis(50));
3145 active.fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
3146 }
3148 };
3149
3150 let t1 = std::thread::spawn(run_one_boot(active.clone(), overlap_detected.clone()));
3151 let t2 = std::thread::spawn(run_one_boot(active.clone(), overlap_detected.clone()));
3152 t1.join().expect("boot thread 1 must not panic");
3153 t2.join().expect("boot thread 2 must not panic");
3154
3155 assert!(
3156 !overlap_detected.load(std::sync::atomic::Ordering::SeqCst),
3157 "two concurrent boot sequences must never hold the schema-init \
3158 critical section at the same time (#667)"
3159 );
3160
3161 std::env::remove_var("KHIVE_LOCK");
3162 }
3163
3164 #[test]
3169 #[serial]
3170 fn acquire_daemon_boot_guard_returns_guard_when_lock_available() {
3171 let dir = tempfile::tempdir().expect("tempdir");
3172 let lock_file = dir.path().join("khived.recovery.lock");
3173 std::env::set_var("KHIVE_LOCK", &lock_file);
3174
3175 let guard = acquire_daemon_boot_guard();
3176 assert!(
3177 guard.is_ok(),
3178 "daemon boot guard must succeed when the lock file can be opened and flocked"
3179 );
3180 drop(guard);
3181
3182 std::env::remove_var("KHIVE_LOCK");
3183 }
3184
3185 #[test]
3186 #[serial]
3187 fn acquire_daemon_boot_guard_fails_loudly_when_lock_file_cannot_be_opened() {
3188 let dir = tempfile::tempdir().expect("tempdir");
3189 std::env::set_var("KHIVE_LOCK", dir.path());
3195
3196 let result = acquire_daemon_boot_guard();
3197 assert!(
3198 result.is_err(),
3199 "daemon boot guard must fail loudly, never silently proceed unguarded, \
3200 when the underlying recovery lock cannot be acquired"
3201 );
3202
3203 std::env::remove_var("KHIVE_LOCK");
3204 }
3205
3206 #[test]
3209 fn write_pid_file_exclusive_creates_new_file_with_own_pid() {
3210 let dir = tempfile::tempdir().expect("tempdir");
3211 let pid_file = dir.path().join("khived.pid");
3212 write_pid_file_exclusive(&pid_file).expect("first writer must win");
3213 let contents = std::fs::read_to_string(&pid_file).expect("read pid file");
3214 assert_eq!(contents, std::process::id().to_string());
3215 }
3216
3217 #[test]
3218 fn write_pid_file_exclusive_refuses_to_overwrite_an_existing_file() {
3219 let dir = tempfile::tempdir().expect("tempdir");
3220 let pid_file = dir.path().join("khived.pid");
3221 std::fs::write(&pid_file, "999999").expect("seed an existing pid file");
3222
3223 let err = write_pid_file_exclusive(&pid_file)
3224 .expect_err("must not silently overwrite an existing pid file");
3225 assert_eq!(err.kind(), std::io::ErrorKind::AlreadyExists);
3226
3227 let contents = std::fs::read_to_string(&pid_file).expect("read pid file");
3230 assert_eq!(
3231 contents, "999999",
3232 "an existing pid file must never be truncated by a losing writer"
3233 );
3234 }
3235
3236 #[test]
3245 fn two_concurrent_writers_converge_on_exactly_one_pid_file_owner() {
3246 let dir = tempfile::tempdir().expect("tempdir");
3247 let pid_file = std::sync::Arc::new(dir.path().join("khived.pid"));
3248 let barrier = std::sync::Arc::new(std::sync::Barrier::new(2));
3249
3250 let spawn_writer =
3251 |pid_file: std::sync::Arc<std::path::PathBuf>,
3252 barrier: std::sync::Arc<std::sync::Barrier>| {
3253 std::thread::spawn(move || {
3254 barrier.wait();
3255 write_pid_file_exclusive(&pid_file)
3256 })
3257 };
3258
3259 let t1 = spawn_writer(pid_file.clone(), barrier.clone());
3260 let t2 = spawn_writer(pid_file.clone(), barrier.clone());
3261 let r1 = t1.join().expect("writer 1 must not panic");
3262 let r2 = t2.join().expect("writer 2 must not panic");
3263
3264 let results = [&r1, &r2];
3265 let ok_count = results.iter().filter(|r| r.is_ok()).count();
3266 let already_exists_count = results
3267 .iter()
3268 .filter(|r| matches!(r, Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists))
3269 .count();
3270 assert_eq!(
3271 ok_count, 1,
3272 "exactly one of two concurrent writers must win the pid file"
3273 );
3274 assert_eq!(
3275 already_exists_count, 1,
3276 "the other writer must observe AlreadyExists, never a silent overwrite"
3277 );
3278 assert!(pid_file.exists(), "the winner's pid file must exist");
3279 let contents = std::fs::read_to_string(&*pid_file).expect("read pid file");
3280 assert_eq!(
3281 contents,
3282 std::process::id().to_string(),
3283 "the surviving pid file must contain the winner's pid — both threads \
3284 share this process's pid, so an unexpected value would also prove a \
3285 lost/garbled write raced through"
3286 );
3287 }
3288
3289 #[tokio::test]
3300 async fn peer_uid_reports_the_connecting_process_uid() {
3301 let dir = tempfile::tempdir().expect("tempdir");
3302 let sock = dir.path().join("peer.sock");
3303 let listener = UnixListener::bind(&sock).expect("bind");
3304
3305 let connect_path = sock.clone();
3306 let client = tokio::spawn(async move { UnixStream::connect(&connect_path).await });
3307
3308 let (server_side, _) = listener.accept().await.expect("accept");
3309 let client_side = client.await.expect("join").expect("connect");
3310
3311 let expected = unsafe { libc::geteuid() } as u32;
3313
3314 assert_eq!(
3315 peer_uid(&server_side).expect("peer_uid must succeed on a live connection"),
3316 expected,
3317 "the uid read from the kernel for a same-process connection must be \
3318 this process's euid"
3319 );
3320 assert_eq!(
3322 peer_uid(&client_side).expect("peer_uid must succeed on the client end"),
3323 expected
3324 );
3325 }
3326
3327 #[test]
3337 fn only_a_foreign_uid_is_refused() {
3338 let euid = unsafe { libc::geteuid() } as u32;
3340
3341 assert!(
3342 uid_is_permitted(euid, euid),
3343 "a connection from the daemon's own uid must be served — this is \
3344 every seat on the host, and ADR-096 accepted exactly this shape"
3345 );
3346 assert!(
3347 !uid_is_permitted(euid.wrapping_add(1), euid),
3348 "a connection from any other uid must be refused"
3349 );
3350 assert!(
3351 !uid_is_permitted(0, euid.wrapping_add(1)),
3352 "root is not special-cased: the rule is equality with the daemon's \
3353 euid, not a privilege comparison"
3354 );
3355 }
3356}