1use std::sync::Arc;
12
13#[cfg(unix)]
14use std::io::Write as _;
15#[cfg(unix)]
16use std::os::unix::fs::PermissionsExt;
17#[cfg(unix)]
18use std::os::unix::io::AsRawFd;
19use std::path::PathBuf;
20
21#[cfg(unix)]
22use async_trait::async_trait;
23#[cfg(unix)]
24use libc;
25use serde::{Deserialize, Serialize};
26#[cfg(unix)]
27use tokio::io::{AsyncReadExt, AsyncWriteExt};
28#[cfg(unix)]
29use tokio::net::{UnixListener, UnixStream};
30
31#[cfg(unix)]
32use khive_db::{run_checkpoint_task, CheckpointConfig, CheckpointLifecycleOwner, ConnectionPool};
33
34#[cfg(unix)]
35use crate::pack::RequestIdentity;
36
37pub const MAX_FRAME_BYTES: usize = 8 * 1024 * 1024;
39
40pub const PROTOCOL_VERSION: u32 = 4;
49
50const DEFAULT_DRAIN_TIMEOUT_SECS: u64 = 10;
51
52fn khive_dir() -> PathBuf {
68 khive_root_from(
69 std::env::var("HOME").ok(),
70 std::env::var("USERPROFILE").ok(),
71 )
72}
73
74fn khive_root_from(home: Option<String>, userprofile: Option<String>) -> PathBuf {
77 home.filter(|v| !v.trim().is_empty())
78 .or_else(|| userprofile.filter(|v| !v.trim().is_empty()))
79 .map(PathBuf::from)
80 .unwrap_or_else(last_resort_root)
81 .join(".khive")
82}
83
84#[cfg(unix)]
86fn last_resort_root() -> PathBuf {
87 PathBuf::from(".")
88}
89
90#[cfg(not(unix))]
91fn last_resort_root() -> PathBuf {
92 std::env::temp_dir()
93}
94
95#[cfg(unix)]
99pub fn socket_path() -> PathBuf {
100 if let Ok(p) = std::env::var("KHIVE_SOCKET") {
101 if !p.is_empty() {
102 return PathBuf::from(p);
103 }
104 }
105 khive_dir().join("khived.sock")
106}
107
108#[cfg(unix)]
112pub fn pid_path() -> PathBuf {
113 if let Ok(p) = std::env::var("KHIVE_PID") {
114 if !p.is_empty() {
115 return PathBuf::from(p);
116 }
117 }
118 khive_dir().join("khived.pid")
119}
120
121pub fn lock_path() -> PathBuf {
130 if let Ok(p) = std::env::var("KHIVE_LOCK") {
131 if !p.is_empty() {
132 return PathBuf::from(p);
133 }
134 }
135 khive_dir().join("khived.recovery.lock")
136}
137
138#[cfg(unix)]
148pub fn recoverer_lock_path() -> PathBuf {
149 if let Ok(p) = std::env::var("KHIVE_RECOVERER_LOCK") {
150 if !p.is_empty() {
151 return PathBuf::from(p);
152 }
153 }
154 khive_dir().join("khived.recoverer.lock")
155}
156
157#[cfg(unix)]
158fn open_lock_file(path: &std::path::Path) -> std::io::Result<std::fs::File> {
159 if let Some(parent) = path.parent() {
160 let _ = std::fs::create_dir_all(parent);
161 }
162 std::fs::OpenOptions::new()
163 .create(true)
164 .truncate(false)
165 .write(true)
166 .open(path)
167}
168
169#[cfg(unix)]
170fn acquire_flock_blocking(path: &std::path::Path, label: &str) -> Option<std::fs::File> {
171 let file = match open_lock_file(path) {
172 Ok(f) => f,
173 Err(e) => {
174 tracing::warn!(error = %e, path = ?path, "cannot open {label} lock file");
175 return None;
176 }
177 };
178 let rc = unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX) };
180 if rc != 0 {
181 tracing::warn!("flock LOCK_EX failed on {label} lock");
182 return None;
183 }
184 Some(file)
185}
186
187#[cfg(unix)]
194pub fn acquire_recovery_lock() -> Option<std::fs::File> {
195 acquire_flock_blocking(&lock_path(), "recovery")
196}
197
198#[cfg(unix)]
213fn try_acquire_flock_until(
214 path: &std::path::Path,
215 deadline: std::time::Instant,
216) -> std::io::Result<Option<std::fs::File>> {
217 let file = open_lock_file(path)?;
218 let poll_interval = std::time::Duration::from_millis(10);
219 loop {
220 let rc = unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) };
222 if rc == 0 {
223 return Ok(Some(file));
224 }
225 let err = std::io::Error::last_os_error();
226 if err.raw_os_error() != Some(libc::EWOULDBLOCK) {
227 return Err(err);
228 }
229 let now = std::time::Instant::now();
230 if now >= deadline {
231 return Ok(None);
232 }
233 std::thread::sleep(poll_interval.min(deadline - now));
234 }
235}
236
237#[cfg(unix)]
244pub fn try_acquire_daemon_boot_guard_until(
245 deadline: std::time::Instant,
246) -> std::io::Result<Option<DaemonBootGuard>> {
247 try_acquire_flock_until(&lock_path(), deadline)
248}
249
250#[cfg(unix)]
256pub fn try_acquire_recoverer_lock_until(
257 deadline: std::time::Instant,
258) -> std::io::Result<Option<std::fs::File>> {
259 try_acquire_flock_until(&recoverer_lock_path(), deadline)
260}
261
262#[cfg(unix)]
266pub type DaemonBootGuard = std::fs::File;
267
268#[cfg(unix)]
278pub fn acquire_daemon_boot_guard() -> anyhow::Result<DaemonBootGuard> {
279 acquire_recovery_lock()
280 .ok_or_else(|| anyhow::anyhow!("failed to acquire daemon boot/recovery lock"))
281}
282
283#[cfg(unix)]
290#[derive(Clone, Copy, PartialEq, Eq)]
291struct SocketIdentity {
292 dev: u64,
293 ino: u64,
294}
295
296#[cfg(unix)]
297fn socket_identity(path: &std::path::Path) -> Option<SocketIdentity> {
298 use std::os::unix::fs::MetadataExt;
299 let meta = std::fs::metadata(path).ok()?;
300 Some(SocketIdentity {
301 dev: meta.dev(),
302 ino: meta.ino(),
303 })
304}
305
306#[cfg(unix)]
320pub(crate) fn peer_uid(stream: &UnixStream) -> std::io::Result<u32> {
321 use std::os::fd::AsRawFd;
322 let fd = stream.as_raw_fd();
323
324 #[cfg(any(target_os = "macos", target_os = "ios", target_vendor = "apple"))]
325 {
326 let mut uid: libc::uid_t = 0;
327 let mut gid: libc::gid_t = 0;
328 let rc = unsafe { libc::getpeereid(fd, &mut uid, &mut gid) };
331 if rc != 0 {
332 return Err(std::io::Error::last_os_error());
333 }
334 Ok(uid as u32)
335 }
336
337 #[cfg(target_os = "linux")]
338 {
339 let mut cred = libc::ucred {
340 pid: 0,
341 uid: 0,
342 gid: 0,
343 };
344 let mut len = std::mem::size_of::<libc::ucred>() as libc::socklen_t;
345 let rc = unsafe {
349 libc::getsockopt(
350 fd,
351 libc::SOL_SOCKET,
352 libc::SO_PEERCRED,
353 (&mut cred as *mut libc::ucred).cast::<libc::c_void>(),
354 &mut len,
355 )
356 };
357 if rc != 0 {
358 return Err(std::io::Error::last_os_error());
359 }
360 Ok(cred.uid)
361 }
362
363 #[cfg(not(any(
364 target_os = "linux",
365 target_os = "macos",
366 target_os = "ios",
367 target_vendor = "apple"
368 )))]
369 {
370 let _ = fd;
371 Err(std::io::Error::new(
372 std::io::ErrorKind::Unsupported,
373 "peer-credential capture is not implemented for this platform",
374 ))
375 }
376}
377
378#[cfg(unix)]
399pub(crate) fn uid_is_permitted(peer: u32, daemon_euid: u32) -> bool {
400 peer == daemon_euid
401}
402
403#[derive(Serialize, Deserialize, Default)]
407pub struct DaemonRequestFrame {
408 pub ops: String,
409 pub presentation: Option<String>,
410 pub presentation_per_op: Option<Vec<Option<String>>>,
411 pub namespace: String,
417 #[serde(default)]
422 pub actor_id: Option<String>,
423 #[serde(default, skip_serializing_if = "Option::is_none")]
429 pub process_ref: Option<String>,
430 #[serde(default)]
436 pub visible_namespaces: Vec<String>,
437 #[serde(default)]
444 pub config_id: String,
445 #[serde(default)]
449 pub protocol_version: u32,
450 #[serde(default)]
456 pub probe_only: bool,
457 #[serde(default)]
467 pub metrics_only: bool,
468 #[serde(default)]
471 pub format: Option<String>,
472 #[serde(default)]
474 pub format_per_op: Option<Vec<Option<String>>>,
475 #[serde(default)]
486 pub from_wire: bool,
487 #[serde(default)]
496 pub request_id: Option<u64>,
497}
498
499#[derive(Serialize, Deserialize, Debug)]
501pub struct DaemonResponseFrame {
502 pub ok: bool,
503 pub result: Option<String>,
504 pub error: Option<String>,
505 pub namespace_mismatch: bool,
506 #[serde(default)]
510 pub config_mismatch: bool,
511 #[serde(default)]
518 pub served_config_id: Option<String>,
519 #[serde(default)]
524 pub version_mismatch: bool,
525 #[serde(default)]
529 pub daemon_protocol_version: u32,
530 #[serde(default, skip_serializing_if = "Option::is_none")]
536 pub metrics: Option<MetricsSnapshot>,
537 #[serde(default)]
544 pub request_id: Option<u64>,
545}
546
547#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq)]
558pub struct MetricsSnapshot {
559 pub wal_pages: Option<u64>,
564 #[serde(default)]
567 pub wal_log_frames: Option<u64>,
568 #[serde(default)]
570 pub wal_checkpointed_frames: Option<u64>,
571 #[serde(default)]
573 pub wal_pending_frames: Option<u64>,
574 #[serde(default)]
576 pub wal_physical_bytes: Option<u64>,
577 #[serde(default)]
579 pub wal_observed_at_unix_ms: Option<u64>,
580 pub wal_truncate_attempts: u64,
584 pub wal_truncate_consecutive_failures: u64,
588 #[serde(default)]
593 pub wal_checkpoint_skipped_ticks: u64,
594 #[serde(default)]
596 pub wal_checkpoint_consecutive_skips: u64,
597 #[serde(default)]
600 pub wal_checkpoint_last_skip_wal_pages: Option<u64>,
601 pub oldest_pinned_tx_micros: Option<u64>,
605 pub oldest_pinned_tx_label: Option<String>,
608 pub open_tx_count: usize,
610 pub write_queue_depth: Option<usize>,
615 pub write_queue_capacity: Option<usize>,
619 #[serde(default)]
621 pub write_last_queue_wait_micros: Option<u64>,
622 #[serde(default)]
624 pub write_last_transaction_acquire_micros: Option<u64>,
625 #[serde(default)]
627 pub write_last_body_micros: Option<u64>,
628 #[serde(default)]
630 pub write_last_commit_micros: Option<u64>,
631 #[serde(default)]
634 pub write_last_total_micros: Option<u64>,
635 #[serde(default)]
637 pub write_last_observed_at_unix_ms: Option<u64>,
638}
639
640#[cfg(unix)]
644pub async fn read_frame<R>(stream: &mut R) -> std::io::Result<Vec<u8>>
645where
646 R: tokio::io::AsyncRead + Unpin,
647{
648 let mut len_buf = [0u8; 4];
649 stream.read_exact(&mut len_buf).await?;
650 let len = u32::from_be_bytes(len_buf) as usize;
651 if len > MAX_FRAME_BYTES {
652 return Err(std::io::Error::new(
653 std::io::ErrorKind::InvalidData,
654 format!("daemon frame of {len} bytes exceeds {MAX_FRAME_BYTES} cap"),
655 ));
656 }
657 let mut buf = vec![0u8; len];
658 stream.read_exact(&mut buf).await?;
659 Ok(buf)
660}
661
662#[cfg(unix)]
664pub async fn write_frame<W>(stream: &mut W, payload: &[u8]) -> std::io::Result<()>
665where
666 W: tokio::io::AsyncWrite + Unpin,
667{
668 if payload.len() > MAX_FRAME_BYTES {
669 return Err(std::io::Error::new(
670 std::io::ErrorKind::InvalidData,
671 format!(
672 "daemon frame of {} bytes exceeds {MAX_FRAME_BYTES} cap",
673 payload.len()
674 ),
675 ));
676 }
677 let len = (payload.len() as u32).to_be_bytes();
678 stream.write_all(&len).await?;
679 stream.write_all(payload).await?;
680 stream.flush().await?;
681 Ok(())
682}
683
684#[cfg(unix)]
693#[async_trait]
694pub trait DaemonDispatch: Clone + Send + Sync + 'static {
695 #[allow(clippy::too_many_arguments)]
710 async fn dispatch(
711 &self,
712 ops: String,
713 presentation: Option<String>,
714 presentation_per_op: Option<Vec<Option<String>>>,
715 format: Option<String>,
716 format_per_op: Option<Vec<Option<String>>>,
717 from_wire: bool,
718 identity: Option<RequestIdentity>,
719 ) -> Result<String, String>;
720
721 async fn warm_all(&self);
723
724 fn namespace(&self) -> &str;
726
727 fn config_id(&self) -> &str;
732
733 fn pool_for_checkpoint(&self) -> Option<Arc<ConnectionPool>> {
741 None
742 }
743
744 fn secondary_pools_for_checkpoint(&self) -> Vec<Arc<ConnectionPool>> {
754 Vec::new()
755 }
756
757 fn event_store_for_checkpoint(&self) -> Option<Arc<dyn khive_storage::EventStore>> {
767 None
768 }
769}
770
771#[cfg(unix)]
772struct CheckpointTaskSpec {
773 pool: Arc<ConnectionPool>,
774 lifecycle_owner: Option<CheckpointLifecycleOwner>,
775 is_main: bool,
776}
777
778#[cfg(unix)]
785fn checkpoint_task_specs(
786 main_pool: Option<Arc<ConnectionPool>>,
787 secondary_pools: Vec<Arc<ConnectionPool>>,
788 event_store: Option<Arc<dyn khive_storage::EventStore>>,
789 namespace: String,
790) -> Vec<CheckpointTaskSpec> {
791 let mut tasks = Vec::with_capacity(usize::from(main_pool.is_some()) + secondary_pools.len());
792 if let Some(pool) = main_pool {
793 tasks.push(CheckpointTaskSpec {
794 pool,
795 lifecycle_owner: None,
796 is_main: true,
797 });
798 }
799 tasks.extend(secondary_pools.into_iter().map(|pool| CheckpointTaskSpec {
800 pool,
801 lifecycle_owner: None,
802 is_main: false,
803 }));
804
805 if let (Some(task), Some(event_store)) = (tasks.first_mut(), event_store) {
806 task.lifecycle_owner = Some(CheckpointLifecycleOwner::new(event_store, namespace));
807 }
808 tasks
809}
810
811static BACKGROUND_TASKS: std::sync::OnceLock<Arc<std::sync::atomic::AtomicUsize>> =
823 std::sync::OnceLock::new();
824
825fn background_tasks() -> &'static Arc<std::sync::atomic::AtomicUsize> {
826 BACKGROUND_TASKS.get_or_init(|| Arc::new(std::sync::atomic::AtomicUsize::new(0)))
827}
828
829struct BackgroundTaskGuard {
835 counter: Arc<std::sync::atomic::AtomicUsize>,
836}
837
838impl Drop for BackgroundTaskGuard {
839 fn drop(&mut self) {
840 self.counter
841 .fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
842 }
843}
844
845pub fn spawn_tracked_task<F, T>(fut: F) -> tokio::task::JoinHandle<T>
851where
852 F: std::future::Future<Output = T> + Send + 'static,
853 T: Send + 'static,
854{
855 background_tasks().fetch_add(1, std::sync::atomic::Ordering::SeqCst);
856 let guard = BackgroundTaskGuard {
857 counter: background_tasks().clone(),
858 };
859 tokio::spawn(async move {
860 let _guard = guard;
861 fut.await
862 })
863}
864
865pub fn track_background_task<F>(fut: F)
870where
871 F: std::future::Future<Output = ()> + Send + 'static,
872{
873 drop(spawn_tracked_task(fut));
874}
875
876pub fn background_task_count() -> usize {
879 background_tasks().load(std::sync::atomic::Ordering::SeqCst)
880}
881
882pub fn daemon_shutdown_token() -> tokio_util::sync::CancellationToken {
893 static TOKEN: std::sync::OnceLock<tokio_util::sync::CancellationToken> =
894 std::sync::OnceLock::new();
895 TOKEN
896 .get_or_init(tokio_util::sync::CancellationToken::new)
897 .clone()
898}
899
900static ACTIVE_PHASES: std::sync::OnceLock<
911 std::sync::Mutex<std::collections::HashMap<String, usize>>,
912> = std::sync::OnceLock::new();
913
914fn active_phases() -> &'static std::sync::Mutex<std::collections::HashMap<String, usize>> {
915 ACTIVE_PHASES.get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new()))
916}
917
918pub struct PhaseGuard {
924 name: String,
925}
926
927impl Drop for PhaseGuard {
928 fn drop(&mut self) {
929 let mut map = active_phases()
930 .lock()
931 .unwrap_or_else(std::sync::PoisonError::into_inner);
932 if let Some(count) = map.get_mut(&self.name) {
933 *count -= 1;
934 if *count == 0 {
935 map.remove(&self.name);
936 }
937 }
938 }
939}
940
941pub fn register_active_phase(name: &str) -> PhaseGuard {
946 let mut map = active_phases()
947 .lock()
948 .unwrap_or_else(std::sync::PoisonError::into_inner);
949 *map.entry(name.to_string()).or_insert(0) += 1;
950 PhaseGuard {
951 name: name.to_string(),
952 }
953}
954
955pub fn active_phase_names() -> Vec<String> {
958 let map = active_phases()
959 .lock()
960 .unwrap_or_else(std::sync::PoisonError::into_inner);
961 let mut names: Vec<String> = map.keys().cloned().collect();
962 names.sort();
963 names
964}
965
966#[cfg(unix)]
974fn build_metrics_snapshot<D: DaemonDispatch>(dispatcher: &D) -> MetricsSnapshot {
975 let open_tx_count = khive_storage::tx_registry::snapshot().len();
976 let (oldest_pinned_tx_micros, oldest_pinned_tx_label) =
983 match khive_storage::tx_registry::oldest() {
984 Some((_id, age, label)) => (Some(age.as_micros() as u64), label),
985 None => (None, None),
986 };
987
988 let checkpoint_pool = dispatcher.pool_for_checkpoint();
989 let routine_wal = checkpoint_pool
990 .as_deref()
991 .and_then(khive_db::checkpoint::routine_wal_observation);
992 let writer_stages = checkpoint_pool
993 .as_deref()
994 .and_then(khive_db::writer_task::last_writer_stage_observation);
995 let (write_queue_depth, write_queue_capacity) = checkpoint_pool
996 .as_ref()
997 .and_then(|pool| pool.writer_task_handle().ok().flatten())
998 .map(|handle| (Some(handle.queue_depth()), Some(handle.capacity())))
999 .unwrap_or((None, None));
1000
1001 MetricsSnapshot {
1002 wal_pages: routine_wal.as_ref().map(|sample| sample.log_frames),
1003 wal_log_frames: routine_wal.as_ref().map(|sample| sample.log_frames),
1004 wal_checkpointed_frames: routine_wal
1005 .as_ref()
1006 .map(|sample| sample.checkpointed_frames),
1007 wal_pending_frames: routine_wal.as_ref().map(|sample| sample.pending_frames),
1008 wal_physical_bytes: routine_wal
1009 .as_ref()
1010 .and_then(|sample| sample.physical_wal_bytes),
1011 wal_observed_at_unix_ms: routine_wal
1012 .as_ref()
1013 .map(|sample| sample.observed_at_unix_ms),
1014 wal_truncate_attempts: khive_db::checkpoint::truncate_attempts(),
1015 wal_truncate_consecutive_failures: khive_db::checkpoint::truncate_consecutive_failures(),
1016 wal_checkpoint_skipped_ticks: khive_db::checkpoint::checkpoint_skipped_ticks(),
1017 wal_checkpoint_consecutive_skips: khive_db::checkpoint::checkpoint_consecutive_skips(),
1018 wal_checkpoint_last_skip_wal_pages: khive_db::checkpoint::checkpoint_last_skip_wal_pages(),
1019 oldest_pinned_tx_micros,
1020 oldest_pinned_tx_label,
1021 open_tx_count,
1022 write_queue_depth,
1023 write_queue_capacity,
1024 write_last_queue_wait_micros: writer_stages
1025 .as_ref()
1026 .map(|sample| sample.queue_wait_micros),
1027 write_last_transaction_acquire_micros: writer_stages
1028 .as_ref()
1029 .map(|sample| sample.transaction_acquire_micros),
1030 write_last_body_micros: writer_stages.as_ref().map(|sample| sample.body_micros),
1031 write_last_commit_micros: writer_stages.as_ref().map(|sample| sample.commit_micros),
1032 write_last_total_micros: writer_stages.as_ref().map(|sample| sample.total_micros),
1033 write_last_observed_at_unix_ms: writer_stages
1034 .as_ref()
1035 .map(|sample| sample.observed_at_unix_ms),
1036 }
1037}
1038
1039#[cfg(unix)]
1040async fn wait_for_peer_disconnect(read: &mut tokio::net::unix::OwnedReadHalf) {
1041 let mut byte = [0u8; 1];
1042 let _ = read.read(&mut byte).await;
1046}
1047
1048#[cfg(all(unix, test))]
1049async fn handle_conn<D: DaemonDispatch>(stream: UnixStream, dispatcher: D) {
1050 handle_conn_with_shutdown(stream, dispatcher, None).await;
1051}
1052
1053#[cfg(unix)]
1054async fn handle_conn_with_shutdown<D: DaemonDispatch>(
1055 mut stream: UnixStream,
1056 dispatcher: D,
1057 shutdown: Option<tokio::sync::watch::Receiver<bool>>,
1058) {
1059 let (local_shutdown_tx, local_shutdown_rx) = tokio::sync::watch::channel(false);
1060 let shutdown = shutdown.unwrap_or(local_shutdown_rx);
1061 let _local_shutdown_tx = local_shutdown_tx;
1064 let raw = match read_frame(&mut stream).await {
1065 Ok(r) => r,
1066 Err(e) => {
1067 tracing::debug!(error = %e, "failed to read daemon request frame");
1068 return;
1069 }
1070 };
1071 let frame: DaemonRequestFrame = match serde_json::from_slice(&raw) {
1072 Ok(f) => f,
1073 Err(e) => {
1074 tracing::debug!(error = %e, "failed to decode daemon request frame");
1075 return;
1076 }
1077 };
1078 let (mut peer_read, mut peer_write) = stream.into_split();
1079
1080 let served_config_id = Some(dispatcher.config_id().to_string());
1081 let resp = if frame.protocol_version != PROTOCOL_VERSION {
1082 let msg = format!(
1083 "daemon protocol mismatch: client={} daemon={} — \
1084 rebuild/update the client binary (make local)",
1085 frame.protocol_version, PROTOCOL_VERSION,
1086 );
1087 tracing::warn!(
1088 client_version = frame.protocol_version,
1089 daemon_version = PROTOCOL_VERSION,
1090 "daemon protocol version mismatch"
1091 );
1092 DaemonResponseFrame {
1093 ok: false,
1094 result: None,
1095 error: Some(msg),
1096 namespace_mismatch: false,
1097 config_mismatch: false,
1098 served_config_id,
1099 version_mismatch: true,
1100 daemon_protocol_version: PROTOCOL_VERSION,
1101 metrics: None,
1102 request_id: frame.request_id,
1103 }
1104 } else if frame.metrics_only {
1105 DaemonResponseFrame {
1112 ok: true,
1113 result: None,
1114 error: None,
1115 namespace_mismatch: false,
1116 config_mismatch: false,
1117 served_config_id,
1118 version_mismatch: false,
1119 daemon_protocol_version: PROTOCOL_VERSION,
1120 metrics: Some(build_metrics_snapshot(&dispatcher)),
1121 request_id: frame.request_id,
1122 }
1123 } else if frame.config_id != dispatcher.config_id() {
1133 DaemonResponseFrame {
1134 ok: false,
1135 result: None,
1136 error: None,
1137 namespace_mismatch: false,
1138 config_mismatch: true,
1139 served_config_id,
1140 version_mismatch: false,
1141 daemon_protocol_version: PROTOCOL_VERSION,
1142 metrics: None,
1143 request_id: frame.request_id,
1144 }
1145 } else if frame.probe_only {
1146 DaemonResponseFrame {
1150 ok: true,
1151 result: None,
1152 error: None,
1153 namespace_mismatch: false,
1154 config_mismatch: false,
1155 served_config_id,
1156 version_mismatch: false,
1157 daemon_protocol_version: PROTOCOL_VERSION,
1158 metrics: None,
1159 request_id: frame.request_id,
1160 }
1161 } else {
1162 let identity = RequestIdentity {
1171 namespace: frame.namespace.clone(),
1172 actor_id: frame.actor_id.clone(),
1173 visible_namespaces: frame.visible_namespaces.clone(),
1174 process_ref: frame.process_ref.clone(),
1175 request_id: frame.request_id,
1176 };
1177 let (read_cancel_tx, read_cancel_rx) = tokio::sync::watch::channel(false);
1178 let dispatch = khive_storage::scope_request_read_cancellation(
1179 shutdown,
1180 khive_storage::scope_request_read_cancellation(
1181 read_cancel_rx,
1182 khive_storage::scope_request_read_deadline(
1183 khive_storage::request_read_timeout_from_env(),
1184 dispatcher.dispatch(
1185 frame.ops,
1186 frame.presentation,
1187 frame.presentation_per_op,
1188 frame.format,
1189 frame.format_per_op,
1190 frame.from_wire,
1191 Some(identity),
1192 ),
1193 ),
1194 ),
1195 );
1196 tokio::pin!(dispatch);
1197 let dispatch_result = tokio::select! {
1198 result = &mut dispatch => result,
1199 _ = wait_for_peer_disconnect(&mut peer_read) => {
1200 let _ = read_cancel_tx.send(true);
1201 dispatch.await
1202 }
1203 };
1204 match dispatch_result {
1205 Ok(result) => DaemonResponseFrame {
1206 ok: true,
1207 result: Some(result),
1208 error: None,
1209 namespace_mismatch: false,
1210 config_mismatch: false,
1211 served_config_id,
1212 version_mismatch: false,
1213 daemon_protocol_version: PROTOCOL_VERSION,
1214 metrics: None,
1215 request_id: frame.request_id,
1216 },
1217 Err(e) => DaemonResponseFrame {
1218 ok: false,
1219 result: None,
1220 error: Some(e),
1221 namespace_mismatch: false,
1222 config_mismatch: false,
1223 served_config_id,
1224 version_mismatch: false,
1225 daemon_protocol_version: PROTOCOL_VERSION,
1226 metrics: None,
1227 request_id: frame.request_id,
1228 },
1229 }
1230 };
1231
1232 match serde_json::to_vec(&resp) {
1233 Ok(payload) => {
1234 if payload.len() > MAX_FRAME_BYTES {
1235 tracing::warn!(
1242 bytes = payload.len(),
1243 limit = MAX_FRAME_BYTES,
1244 "daemon response exceeds MAX_FRAME_BYTES; sending explicit error frame"
1245 );
1246 let err_resp = DaemonResponseFrame {
1247 ok: false,
1248 result: None,
1249 error: Some(format!(
1250 "response too large: {} bytes exceeds {} byte IPC cap",
1251 payload.len(),
1252 MAX_FRAME_BYTES,
1253 )),
1254 namespace_mismatch: false,
1255 config_mismatch: false,
1256 served_config_id: resp.served_config_id,
1257 version_mismatch: false,
1258 daemon_protocol_version: PROTOCOL_VERSION,
1259 metrics: None,
1260 request_id: resp.request_id,
1261 };
1262 if let Ok(err_payload) = serde_json::to_vec(&err_resp) {
1263 if let Err(e) = write_frame(&mut peer_write, &err_payload).await {
1264 tracing::debug!(error = %e, "failed to write oversized-response error frame");
1265 }
1266 }
1267 } else if let Err(e) = write_frame(&mut peer_write, &payload).await {
1268 tracing::debug!(error = %e, "failed to write daemon response frame");
1269 }
1270 }
1271 Err(e) => tracing::warn!(error = %e, "failed to serialize daemon response frame"),
1272 }
1273}
1274
1275#[cfg(unix)]
1279struct ActiveConnectionGuard {
1280 active: Arc<std::sync::atomic::AtomicUsize>,
1281}
1282
1283#[cfg(unix)]
1284impl ActiveConnectionGuard {
1285 fn claim(active: Arc<std::sync::atomic::AtomicUsize>) -> Self {
1286 active.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
1287 Self { active }
1288 }
1289}
1290
1291#[cfg(unix)]
1292impl Drop for ActiveConnectionGuard {
1293 fn drop(&mut self) {
1294 self.active
1295 .fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
1296 }
1297}
1298
1299#[cfg(unix)]
1300fn spawn_connection_task<F>(
1301 active: Arc<std::sync::atomic::AtomicUsize>,
1302 future: F,
1303) -> tokio::task::JoinHandle<()>
1304where
1305 F: std::future::Future<Output = ()> + Send + 'static,
1306{
1307 let guard = ActiveConnectionGuard::claim(active);
1308 tokio::spawn(async move {
1309 let _guard = guard;
1310 future.await;
1311 })
1312}
1313
1314#[cfg(unix)]
1325pub async fn run_daemon<D: DaemonDispatch>(dispatcher: D) -> anyhow::Result<()> {
1326 let boot_guard = Some(acquire_daemon_boot_guard()?);
1327 run_daemon_with_boot_guard_inner(dispatcher, boot_guard, false).await
1328}
1329
1330#[cfg(all(unix, any(test, feature = "fault-injection")))]
1345#[doc(hidden)]
1346pub async fn run_daemon_in_process_test<D: DaemonDispatch>(dispatcher: D) -> anyhow::Result<()> {
1347 let boot_guard = Some(acquire_daemon_boot_guard()?);
1348 run_daemon_with_boot_guard_inner(dispatcher, boot_guard, true).await
1349}
1350
1351#[cfg(unix)]
1374pub(crate) fn ensure_socket_dir_is_trusted(parent: &std::path::Path) -> anyhow::Result<()> {
1375 let daemon_euid = unsafe { libc::geteuid() } as u32;
1377
1378 if parent == khive_dir() {
1379 std::fs::set_permissions(parent, std::fs::Permissions::from_mode(0o700)).map_err(|e| {
1380 anyhow::anyhow!(
1381 "refusing to start: cannot chmod 0700 {}: {e}. The khive directory must be \
1382 owner-only — it is half of the same-uid guarantee this daemon enforces.",
1383 parent.display()
1384 )
1385 })?;
1386 return ensure_socket_path_is_swap_resistant(parent, daemon_euid);
1387 }
1388
1389 let meta = std::fs::metadata(parent).map_err(|e| {
1392 anyhow::anyhow!(
1393 "refusing to start: cannot stat {}: {e}. The socket directory gates \
1394 socket-takeover safety, and unreadable metadata is not a passing state.",
1395 parent.display()
1396 )
1397 })?;
1398
1399 use std::os::unix::fs::MetadataExt;
1400 let owner = meta.uid();
1401 if owner != daemon_euid && owner != 0 {
1402 anyhow::bail!(
1403 "refusing to start: socket directory {} is owned by uid {owner}, not this \
1404 daemon's uid ({daemon_euid}) or root. A directory owner can replace the \
1405 socket regardless of mode bits. Point KHIVE_SOCKET at a directory you own, \
1406 or unset it for the default.",
1407 parent.display()
1408 );
1409 }
1410
1411 let mode = meta.permissions().mode();
1412 if mode & 0o022 != 0 {
1413 anyhow::bail!(
1414 "refusing to start: socket directory {} is mode {:04o} — writable by group or \
1415 other, so another local user could bind their own listener at the socket path \
1416 (before this daemon starts, the sticky bit does not prevent creating the \
1417 path). Use a directory only you can write, or unset KHIVE_SOCKET for the \
1418 default. This daemon is not changing the permissions of a directory it does \
1419 not own.",
1420 parent.display(),
1421 mode & 0o7777
1422 );
1423 }
1424
1425 ensure_socket_path_is_swap_resistant(parent, daemon_euid)
1426}
1427
1428#[cfg(unix)]
1453fn ensure_socket_path_is_swap_resistant(
1454 parent: &std::path::Path,
1455 daemon_euid: u32,
1456) -> anyhow::Result<()> {
1457 use std::os::unix::fs::MetadataExt;
1458
1459 let absolute = if parent.is_absolute() {
1460 parent.to_path_buf()
1461 } else {
1462 std::env::current_dir()
1463 .map_err(|e| {
1464 anyhow::anyhow!(
1465 "refusing to start: cannot resolve the working directory to absolutize \
1466 socket directory {}: {e}.",
1467 parent.display()
1468 )
1469 })?
1470 .join(parent)
1471 };
1472
1473 fn push_components(stack: &mut Vec<std::ffi::OsString>, path: &std::path::Path) {
1474 let components: Vec<_> = path
1475 .components()
1476 .map(|c| c.as_os_str().to_os_string())
1477 .collect();
1478 stack.extend(components.into_iter().rev());
1479 }
1480
1481 let mut stack: Vec<std::ffi::OsString> = Vec::new();
1482 push_components(&mut stack, &absolute);
1483 let mut resolved = std::path::PathBuf::new();
1484 let mut symlinks_followed = 0u32;
1485
1486 while let Some(component) = stack.pop() {
1487 if component == "/" {
1488 resolved = std::path::PathBuf::from("/");
1489 continue;
1490 }
1491 if component == "." {
1492 continue;
1493 }
1494 if component == ".." {
1495 resolved.pop();
1496 continue;
1497 }
1498 let candidate = resolved.join(&component);
1499 let meta = std::fs::symlink_metadata(&candidate).map_err(|e| {
1500 anyhow::anyhow!(
1501 "refusing to start: cannot stat socket-path component {}: {e}. An \
1502 unreadable component is not a passing one.",
1503 candidate.display()
1504 )
1505 })?;
1506 let owner = meta.uid();
1507
1508 if meta.file_type().is_symlink() {
1509 symlinks_followed += 1;
1510 if symlinks_followed > 40 {
1511 anyhow::bail!(
1512 "refusing to start: socket path resolves through more than 40 symlinks \
1513 at {} — treating this as a loop.",
1514 candidate.display()
1515 );
1516 }
1517 if owner != daemon_euid && owner != 0 {
1518 anyhow::bail!(
1519 "refusing to start: socket-path symlink component {} is owned by uid \
1520 {owner}, not this daemon's uid ({daemon_euid}) or root — its owner \
1521 could retarget it after this check and re-root the socket path. Point \
1522 KHIVE_SOCKET somewhere trusted end to end, or unset it for the \
1523 default.",
1524 candidate.display()
1525 );
1526 }
1527 let target = std::fs::read_link(&candidate).map_err(|e| {
1528 anyhow::anyhow!(
1529 "refusing to start: cannot read socket-path symlink component {}: {e}.",
1530 candidate.display()
1531 )
1532 })?;
1533 push_components(&mut stack, &target);
1534 continue;
1535 }
1536
1537 if meta.is_dir() {
1538 let mode = meta.permissions().mode();
1539 let sticky = mode & 0o1000 != 0;
1540 if owner != daemon_euid && owner != 0 {
1541 anyhow::bail!(
1542 "refusing to start: socket-path ancestor {} is owned by uid {owner}, not \
1543 this daemon's uid ({daemon_euid}) or root — its owner could rename the \
1544 next path component and re-root the socket path. Point KHIVE_SOCKET \
1545 somewhere trusted end to end, or unset it for the default.",
1546 candidate.display()
1547 );
1548 }
1549 if mode & 0o022 != 0 && !sticky {
1550 anyhow::bail!(
1551 "refusing to start: socket-path ancestor {} is mode {:04o} — writable by \
1552 group or other without the sticky bit, so another local user could rename \
1553 the next path component and re-root the socket path. Point KHIVE_SOCKET \
1554 somewhere trusted end to end, or unset it for the default.",
1555 candidate.display(),
1556 mode & 0o7777
1557 );
1558 }
1559 resolved = candidate;
1560 continue;
1561 }
1562
1563 anyhow::bail!(
1564 "refusing to start: socket-path component {} is neither a directory nor a \
1565 symlink — the socket path cannot traverse it.",
1566 candidate.display()
1567 );
1568 }
1569
1570 Ok(())
1571}
1572
1573#[cfg(unix)]
1586pub async fn run_daemon_with_boot_guard<D: DaemonDispatch>(
1587 dispatcher: D,
1588 boot_guard: Option<std::fs::File>,
1589) -> anyhow::Result<()> {
1590 run_daemon_with_boot_guard_inner(dispatcher, boot_guard, false).await
1591}
1592
1593#[cfg(unix)]
1594async fn run_daemon_with_boot_guard_inner<D: DaemonDispatch>(
1595 dispatcher: D,
1596 boot_guard: Option<std::fs::File>,
1597 allow_same_process_incumbent: bool,
1598) -> anyhow::Result<()> {
1599 struct ComponentTeardown;
1609 impl Drop for ComponentTeardown {
1610 fn drop(&mut self) {
1611 daemon_shutdown_token().cancel();
1612 }
1613 }
1614 let _component_teardown = ComponentTeardown;
1615
1616 let sock = socket_path();
1617 let pid_file = pid_path();
1618
1619 if let Some(parent) = sock.parent() {
1620 std::fs::create_dir_all(parent)?;
1621 ensure_socket_dir_is_trusted(parent)?;
1622 }
1623
1624 let _startup_lock = boot_guard;
1638
1639 if !cleanup_stale_daemon(&sock, &pid_file, allow_same_process_incumbent).await {
1640 tracing::info!("a responsive khived is already running; exiting");
1641 return Ok(());
1642 }
1643
1644 let listener = UnixListener::bind(&sock)?;
1645 if let Err(e) = std::fs::set_permissions(&sock, std::fs::Permissions::from_mode(0o600)) {
1649 drop(listener);
1650 let _ = std::fs::remove_file(&sock);
1651 return Err(anyhow::anyhow!(
1652 "refusing to start: cannot chmod 0600 {}: {e}. The daemon socket must be owner-only \
1653 — it is half of the single-principal guarantee this daemon enforces.",
1654 sock.display()
1655 ));
1656 }
1657 let bound_identity = socket_identity(&sock);
1661
1662 if let Err(e) = write_pid_file_exclusive(&pid_file) {
1663 if e.kind() == std::io::ErrorKind::AlreadyExists {
1664 if bound_identity.is_some() && socket_identity(&sock) == bound_identity {
1672 drop(listener);
1673 let _ = std::fs::remove_file(&sock);
1674 }
1675 if pid_file_names_a_reachable_daemon(&pid_file, &sock, allow_same_process_incumbent)
1676 .await
1677 {
1678 tracing::info!(
1679 "a replacement khived already claimed the pid/socket rendezvous; exiting"
1680 );
1681 return Ok(());
1682 }
1683 anyhow::bail!(
1684 "failed to claim daemon pid file at {pid_file:?}: it already exists \
1685 and does not name a reachable daemon"
1686 );
1687 }
1688 return Err(e.into());
1689 }
1690 drop(_startup_lock);
1694 tracing::info!(
1695 socket = ?sock,
1696 pid = std::process::id(),
1697 source_revision = crate::BUILD_INFO.source_revision,
1698 build_time = crate::BUILD_INFO.build_time,
1699 "khived listening"
1700 );
1701
1702 {
1703 let warm = dispatcher.clone();
1704 tokio::spawn(async move {
1705 warm.warm_all().await;
1706 });
1707 }
1708
1709 let (checkpoint_shutdown_tx, checkpoint_shutdown_rx) = tokio::sync::watch::channel(());
1717 let checkpoint_tasks = checkpoint_task_specs(
1723 dispatcher.pool_for_checkpoint(),
1724 dispatcher.secondary_pools_for_checkpoint(),
1725 dispatcher.event_store_for_checkpoint(),
1726 dispatcher.namespace().to_string(),
1727 );
1728 if !checkpoint_tasks.is_empty() {
1729 let cfg = CheckpointConfig::from_env();
1730 let checkpoint_task_count = checkpoint_tasks.len();
1731 for task in checkpoint_tasks {
1732 track_background_task(run_checkpoint_task(
1733 task.pool,
1734 cfg.clone(),
1735 task.lifecycle_owner,
1736 checkpoint_shutdown_rx.clone(),
1737 task.is_main,
1738 ));
1739 }
1740 tracing::info!(checkpoint_task_count, "WAL checkpoint task(s) started");
1741 }
1742
1743 let active = Arc::new(std::sync::atomic::AtomicUsize::new(0));
1744 let connection_tasks = Arc::new(std::sync::Mutex::new(
1745 Vec::<tokio::task::JoinHandle<()>>::new(),
1746 ));
1747 let (request_shutdown_tx, request_shutdown_rx) = tokio::sync::watch::channel(false);
1748
1749 let shutdown = async {
1750 let mut sigterm = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
1754 .expect("install SIGTERM handler");
1755 let mut sigint = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::interrupt())
1756 .expect("install SIGINT handler");
1757 tokio::select! {
1758 _ = sigterm.recv() => tracing::info!("received SIGTERM"),
1759 _ = sigint.recv() => tracing::info!("received SIGINT"),
1760 }
1761 };
1762
1763 let daemon_euid = unsafe { libc::geteuid() } as u32;
1765
1766 tokio::select! {
1767 _ = async {
1768 loop {
1769 match listener.accept().await {
1770 Ok((stream, _)) => {
1771 match peer_uid(&stream) {
1785 Ok(peer) if uid_is_permitted(peer, daemon_euid) => {}
1786 Ok(peer) => {
1787 tracing::error!(
1788 peer_uid = peer,
1789 daemon_euid,
1790 "refusing connection from a foreign uid: this daemon accepts \
1791 only peers running as its own uid"
1792 );
1793 drop(stream);
1794 continue;
1795 }
1796 Err(e) => {
1797 tracing::error!(
1798 error = %e,
1799 "refusing connection: cannot read peer credentials, so \
1800 same-uid cannot be proven"
1801 );
1802 drop(stream);
1803 continue;
1804 }
1805 }
1806 let d = dispatcher.clone();
1807 let shutdown = request_shutdown_rx.clone();
1808 let handle = spawn_connection_task(Arc::clone(&active), async move {
1809 handle_conn_with_shutdown(stream, d, Some(shutdown)).await;
1810 });
1811 let mut tasks = connection_tasks
1812 .lock()
1813 .unwrap_or_else(std::sync::PoisonError::into_inner);
1814 tasks.retain(|task| !task.is_finished());
1815 tasks.push(handle);
1816 }
1817 Err(e) => tracing::error!(error = %e, "accept failed"),
1818 }
1819 }
1820 } => {}
1821 _ = shutdown => {}
1822 }
1823
1824 let _ = checkpoint_shutdown_tx.send(());
1828
1829 let _ = request_shutdown_tx.send(true);
1832
1833 daemon_shutdown_token().cancel();
1837
1838 let drained = drain(&active).await;
1839 let tasks = {
1840 let mut retained = connection_tasks
1841 .lock()
1842 .unwrap_or_else(std::sync::PoisonError::into_inner);
1843 std::mem::take(&mut *retained)
1844 };
1845 finish_connection_tasks(tasks, drained).await;
1846
1847 match acquire_recovery_lock() {
1856 Some(_shutdown_lock) => {
1857 shutdown_cleanup_if_owned(&sock, &pid_file, bound_identity);
1858 }
1859 None => {
1860 tracing::warn!(
1861 "could not acquire recovery lock for shutdown cleanup; \
1862 skipping unlink to avoid deleting a replacement daemon's paths"
1863 );
1864 }
1865 }
1866 tracing::info!("khived stopped");
1867 Ok(())
1868}
1869
1870#[cfg(unix)]
1880fn shutdown_cleanup_if_owned(
1881 sock: &std::path::Path,
1882 pid_file: &std::path::Path,
1883 bound_identity: Option<SocketIdentity>,
1884) -> bool {
1885 let pid_is_ours = std::fs::read_to_string(pid_file)
1886 .ok()
1887 .and_then(|s| s.trim().parse::<u32>().ok())
1888 == Some(std::process::id());
1889 let socket_is_ours = bound_identity.is_some() && socket_identity(sock) == bound_identity;
1890 if pid_is_ours && socket_is_ours {
1891 let _ = std::fs::remove_file(sock);
1892 let _ = std::fs::remove_file(pid_file);
1893 true
1894 } else {
1895 tracing::warn!(
1896 socket = ?sock,
1897 pid_file = ?pid_file,
1898 "skipping shutdown cleanup — a replacement daemon already owns this socket/PID"
1899 );
1900 false
1901 }
1902}
1903
1904#[cfg(unix)]
1908#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1909enum PidLiveness {
1910 Alive,
1913 Dead,
1915 PermissionDenied,
1920}
1921
1922#[cfg(unix)]
1923impl PidLiveness {
1924 fn is_running(self) -> bool {
1925 !matches!(self, PidLiveness::Dead)
1926 }
1927}
1928
1929#[cfg(unix)]
1933fn classify_kill_result(rc: i32, errno: i32) -> PidLiveness {
1934 if rc == 0 {
1935 return PidLiveness::Alive;
1936 }
1937 match errno {
1938 libc::EPERM => PidLiveness::PermissionDenied,
1939 _ => PidLiveness::Dead,
1940 }
1941}
1942
1943#[cfg(unix)]
1944fn is_process_running(pid: u32) -> bool {
1945 let Ok(pid) = i32::try_from(pid) else {
1946 return false;
1947 };
1948 if pid <= 0 {
1949 return false;
1950 }
1951 let rc = unsafe { libc::kill(pid, 0) };
1953 let errno = std::io::Error::last_os_error().raw_os_error().unwrap_or(0);
1954 classify_kill_result(rc, errno).is_running()
1955}
1956
1957#[cfg(unix)]
1964fn pid_can_name_incumbent(pid: u32, current_pid: u32, allow_same_process_incumbent: bool) -> bool {
1965 allow_same_process_incumbent || pid != current_pid
1966}
1967
1968#[cfg(unix)]
1969async fn cleanup_stale_daemon(
1970 sock: &std::path::Path,
1971 pid_file: &std::path::Path,
1972 allow_same_process_incumbent: bool,
1973) -> bool {
1974 if let Ok(pid_str) = std::fs::read_to_string(pid_file) {
1975 if let Ok(pid) = pid_str.trim().parse::<u32>() {
1976 if pid_can_name_incumbent(pid, std::process::id(), allow_same_process_incumbent)
1977 && is_process_running(pid)
1978 && sock.exists()
1979 && UnixStream::connect(sock).await.is_ok()
1980 {
1981 return false;
1982 }
1983 }
1984 }
1985 if sock.exists() {
1986 if let Err(e) = std::fs::remove_file(sock) {
1987 tracing::warn!(error = %e, path = ?sock, "failed to remove stale socket");
1988 }
1989 }
1990 if pid_file.exists() {
1991 if let Err(e) = std::fs::remove_file(pid_file) {
1992 tracing::warn!(error = %e, path = ?pid_file, "failed to remove stale PID file");
1993 }
1994 }
1995 true
1996}
1997
1998#[cfg(unix)]
2009fn write_pid_file_exclusive(pid_file: &std::path::Path) -> std::io::Result<()> {
2010 use std::os::unix::fs::OpenOptionsExt;
2011
2012 let mut opts = std::fs::OpenOptions::new();
2013 opts.write(true).create_new(true).mode(0o600);
2014 let mut f = opts.open(pid_file)?;
2015 f.write_all(std::process::id().to_string().as_bytes())?;
2016 Ok(())
2017}
2018
2019#[cfg(unix)]
2025async fn pid_file_names_a_reachable_daemon(
2026 pid_file: &std::path::Path,
2027 sock: &std::path::Path,
2028 allow_same_process_incumbent: bool,
2029) -> bool {
2030 let Ok(pid_str) = std::fs::read_to_string(pid_file) else {
2031 return false;
2032 };
2033 let Ok(pid) = pid_str.trim().parse::<u32>() else {
2034 return false;
2035 };
2036 pid_can_name_incumbent(pid, std::process::id(), allow_same_process_incumbent)
2037 && is_process_running(pid)
2038 && sock.exists()
2039 && UnixStream::connect(sock).await.is_ok()
2040}
2041
2042#[cfg(unix)]
2043async fn drain(active: &std::sync::atomic::AtomicUsize) -> bool {
2044 drain_with_timeout(active, drain_timeout()).await
2045}
2046
2047#[cfg(unix)]
2048async fn drain_with_timeout(
2049 active: &std::sync::atomic::AtomicUsize,
2050 timeout: std::time::Duration,
2051) -> bool {
2052 use std::sync::atomic::Ordering;
2053 let remaining = || active.load(Ordering::SeqCst) + background_task_count();
2057 if remaining() == 0 {
2058 return true;
2059 }
2060 let deadline = tokio::time::Instant::now() + timeout;
2061 while remaining() > 0 {
2062 if tokio::time::Instant::now() >= deadline {
2063 tracing::warn!(
2064 remaining_connections = active.load(Ordering::SeqCst),
2065 remaining_background_tasks = background_task_count(),
2066 "drain timeout reached; forcing shutdown"
2067 );
2068 return false;
2069 }
2070 tokio::select! {
2071 _ = tokio::time::sleep(std::time::Duration::from_millis(100)) => {}
2072 _ = tokio::time::sleep_until(deadline) => {}
2073 }
2074 }
2075 true
2076}
2077
2078#[cfg(unix)]
2079async fn finish_connection_tasks(tasks: Vec<tokio::task::JoinHandle<()>>, drained: bool) {
2080 if !drained {
2081 for task in &tasks {
2082 if !task.is_finished() {
2083 task.abort();
2084 }
2085 }
2086 }
2087 for task in tasks {
2088 let _ = task.await;
2089 }
2090}
2091
2092pub fn drain_timeout() -> std::time::Duration {
2098 let secs = std::env::var("KHIVE_DRAIN_TIMEOUT_SECS")
2099 .ok()
2100 .and_then(|v| v.parse::<u64>().ok())
2101 .unwrap_or(DEFAULT_DRAIN_TIMEOUT_SECS);
2102 std::time::Duration::from_secs(secs)
2103}
2104
2105#[cfg(unix)]
2107pub fn env_truthy(key: &str) -> bool {
2108 std::env::var(key)
2109 .map(|v| {
2110 let v = v.trim();
2111 !v.is_empty() && v != "0" && !v.eq_ignore_ascii_case("false")
2112 })
2113 .unwrap_or(false)
2114}
2115
2116#[cfg(test)]
2117mod khive_root_tests {
2118 use super::khive_root_from;
2119 use std::path::PathBuf;
2120
2121 #[test]
2122 fn home_wins_when_set() {
2123 assert_eq!(
2124 khive_root_from(Some("/base/home".into()), Some("/other".into())),
2125 PathBuf::from("/base/home/.khive")
2126 );
2127 }
2128
2129 #[test]
2130 fn userprofile_backfills_missing_home() {
2131 assert_eq!(
2132 khive_root_from(None, Some("/profile/home".into())),
2133 PathBuf::from("/profile/home/.khive")
2134 );
2135 }
2136
2137 #[test]
2138 fn blank_values_are_skipped() {
2139 assert_eq!(
2140 khive_root_from(Some(" ".into()), Some("/profile/home".into())),
2141 PathBuf::from("/profile/home/.khive")
2142 );
2143 }
2144
2145 #[cfg(not(unix))]
2146 #[test]
2147 fn fallback_is_never_cwd_relative() {
2148 assert!(khive_root_from(None, None).is_absolute());
2152 }
2153
2154 #[cfg(unix)]
2155 #[test]
2156 fn unix_fallback_stays_historical_not_shared_tmp() {
2157 assert_eq!(khive_root_from(None, None), PathBuf::from("./.khive"));
2161 }
2162}
2163
2164#[cfg(all(test, unix))]
2165mod tests {
2166 use super::*;
2167 use serial_test::serial;
2168
2169 #[derive(Debug)]
2170 struct DrainBlockingBlobStore {
2171 started: std::sync::Mutex<Option<tokio::sync::oneshot::Sender<()>>>,
2172 release: Arc<tokio::sync::Semaphore>,
2173 }
2174
2175 #[async_trait]
2176 impl khive_storage::BlobStore for DrainBlockingBlobStore {
2177 async fn put(
2178 &self,
2179 _bytes: Vec<u8>,
2180 ) -> khive_storage::StorageResult<khive_storage::ContentRef> {
2181 panic!("put is not used by the hydration drain test")
2182 }
2183
2184 async fn get_bounded_verified(
2185 &self,
2186 _content_ref: &khive_storage::ContentRef,
2187 _max_bytes: u64,
2188 ) -> khive_storage::StorageResult<Vec<u8>> {
2189 if let Some(started) = self
2190 .started
2191 .lock()
2192 .unwrap_or_else(std::sync::PoisonError::into_inner)
2193 .take()
2194 {
2195 let _ = started.send(());
2196 }
2197 self.release
2198 .clone()
2199 .acquire_owned()
2200 .await
2201 .expect("test release semaphore remains open")
2202 .forget();
2203 Ok(b"late result".to_vec())
2204 }
2205
2206 async fn exists(
2207 &self,
2208 _content_ref: &khive_storage::ContentRef,
2209 ) -> khive_storage::StorageResult<bool> {
2210 panic!("exists is not used by the hydration drain test")
2211 }
2212
2213 async fn size(
2214 &self,
2215 _content_ref: &khive_storage::ContentRef,
2216 ) -> khive_storage::StorageResult<Option<u64>> {
2217 panic!("size is not used by the hydration drain test")
2218 }
2219
2220 async fn delete(
2221 &self,
2222 _content_ref: &khive_storage::ContentRef,
2223 ) -> khive_storage::StorageResult<bool> {
2224 panic!("delete is not used by the hydration drain test")
2225 }
2226 }
2227
2228 struct AppendCompletionEventStore {
2229 inner: Arc<dyn khive_storage::EventStore>,
2230 first_append: std::sync::Mutex<Option<tokio::sync::oneshot::Sender<()>>>,
2231 }
2232
2233 #[async_trait]
2234 impl khive_storage::EventStore for AppendCompletionEventStore {
2235 async fn append_event(
2236 &self,
2237 event: khive_storage::Event,
2238 ) -> khive_storage::StorageResult<()> {
2239 self.inner.append_event(event).await?;
2240 if let Some(completed) = self
2241 .first_append
2242 .lock()
2243 .unwrap_or_else(std::sync::PoisonError::into_inner)
2244 .take()
2245 {
2246 let _ = completed.send(());
2247 }
2248 Ok(())
2249 }
2250
2251 async fn append_events(
2252 &self,
2253 events: Vec<khive_storage::Event>,
2254 ) -> khive_storage::StorageResult<khive_storage::BatchWriteSummary> {
2255 self.inner.append_events(events).await
2256 }
2257
2258 async fn get_event(
2259 &self,
2260 id: uuid::Uuid,
2261 ) -> khive_storage::StorageResult<Option<khive_storage::Event>> {
2262 self.inner.get_event(id).await
2263 }
2264
2265 async fn query_events(
2266 &self,
2267 filter: khive_storage::EventFilter,
2268 page: khive_storage::PageRequest,
2269 ) -> khive_storage::StorageResult<khive_storage::Page<khive_storage::Event>> {
2270 self.inner.query_events(filter, page).await
2271 }
2272
2273 async fn count_events(
2274 &self,
2275 filter: khive_storage::EventFilter,
2276 ) -> khive_storage::StorageResult<u64> {
2277 self.inner.count_events(filter).await
2278 }
2279 }
2280
2281 #[tokio::test]
2282 #[serial(checkpoint_skip_metrics)]
2283 async fn secondary_only_checkpoint_topology_emits_lifecycle_outcome() {
2284 let main_backend = khive_db::StorageBackend::memory().expect("in-memory main backend");
2285 let inner_event_store = main_backend.events().expect("main event store");
2286 let (first_append_tx, first_append_rx) = tokio::sync::oneshot::channel();
2287 let event_store: Arc<dyn khive_storage::EventStore> =
2288 Arc::new(AppendCompletionEventStore {
2289 inner: inner_event_store,
2290 first_append: std::sync::Mutex::new(Some(first_append_tx)),
2291 });
2292 let secondary_dir = tempfile::tempdir().expect("secondary tempdir");
2293 let secondary_backend =
2294 khive_db::StorageBackend::sqlite(secondary_dir.path().join("secondary.db"))
2295 .expect("file-backed secondary backend");
2296
2297 let mut tasks = checkpoint_task_specs(
2298 None,
2299 vec![secondary_backend.pool_arc()],
2300 Some(Arc::clone(&event_store)),
2301 "local".to_string(),
2302 );
2303 assert_eq!(tasks.len(), 1);
2304 let task = tasks.pop().expect("one secondary checkpoint task");
2305 assert!(!task.is_main, "the only checkpoint task must be secondary");
2306 assert!(
2307 task.lifecycle_owner.is_some(),
2308 "the secondary task must own lifecycle emission when no main task exists"
2309 );
2310
2311 let config = CheckpointConfig {
2312 interval: std::time::Duration::from_millis(10),
2313 warn_pages: 0,
2314 ..CheckpointConfig::default()
2315 };
2316 let (shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(());
2317 let handle = tokio::spawn(run_checkpoint_task(
2318 task.pool,
2319 config,
2320 task.lifecycle_owner,
2321 shutdown_rx,
2322 task.is_main,
2323 ));
2324
2325 tokio::time::timeout(std::time::Duration::from_secs(10), first_append_rx)
2329 .await
2330 .expect("secondary checkpoint owner did not complete an append within 10s")
2331 .expect("checkpoint lifecycle append completion sender dropped");
2332
2333 let events = event_store
2334 .query_events(
2335 khive_storage::EventFilter::default(),
2336 khive_storage::PageRequest {
2337 limit: 100,
2338 offset: 0,
2339 },
2340 )
2341 .await
2342 .expect("query lifecycle events");
2343
2344 shutdown_tx.send(()).expect("send checkpoint shutdown");
2345 tokio::time::timeout(std::time::Duration::from_secs(1), handle)
2346 .await
2347 .expect("checkpoint task should exit within 1s")
2348 .expect("checkpoint task panicked");
2349 assert!(
2350 !events.items.is_empty()
2351 && events
2352 .items
2353 .iter()
2354 .all(|event| event.kind == khive_types::EventKind::CheckpointOutcomeRecorded),
2355 "the designated secondary owner must emit CheckpointOutcomeRecorded"
2356 );
2357
2358 let file_main_dir = tempfile::tempdir().expect("file-backed main tempdir");
2359 let file_main = khive_db::StorageBackend::sqlite(file_main_dir.path().join("main.db"))
2360 .expect("file-backed main backend");
2361 let tasks = checkpoint_task_specs(
2362 Some(file_main.pool_arc()),
2363 vec![secondary_backend.pool_arc()],
2364 Some(event_store),
2365 "local".to_string(),
2366 );
2367 assert!(tasks[0].is_main && tasks[0].lifecycle_owner.is_some());
2368 assert!(!tasks[1].is_main && tasks[1].lifecycle_owner.is_none());
2369 }
2370
2371 #[test]
2375 fn current_process_is_running() {
2376 let pid = std::process::id();
2378 assert!(
2379 is_process_running(pid),
2380 "current process {pid} should be detected as running"
2381 );
2382 }
2383
2384 #[test]
2385 fn pid_zero_is_not_running() {
2386 assert!(
2389 !is_process_running(0),
2390 "pid 0 must be rejected by the guard before the unsafe call"
2391 );
2392 }
2393
2394 #[test]
2395 fn very_large_pid_is_not_running() {
2396 assert!(
2398 !is_process_running(u32::MAX),
2399 "u32::MAX should fail i32 conversion and return false"
2400 );
2401 }
2402
2403 #[test]
2407 fn classify_kill_result_zero_is_alive() {
2408 assert_eq!(classify_kill_result(0, 0), PidLiveness::Alive);
2409 assert!(classify_kill_result(0, 0).is_running());
2410 }
2411
2412 #[test]
2413 fn classify_kill_result_esrch_is_dead() {
2414 assert_eq!(classify_kill_result(-1, libc::ESRCH), PidLiveness::Dead);
2415 assert!(!classify_kill_result(-1, libc::ESRCH).is_running());
2416 }
2417
2418 #[test]
2419 fn classify_kill_result_eperm_is_permission_denied_and_counts_as_running() {
2420 assert_eq!(
2421 classify_kill_result(-1, libc::EPERM),
2422 PidLiveness::PermissionDenied
2423 );
2424 assert!(
2425 classify_kill_result(-1, libc::EPERM).is_running(),
2426 "EPERM must be unknown-safe: treated as running, never as a basis \
2427 for stale cleanup to unlink a live daemon's rendezvous files"
2428 );
2429 }
2430
2431 #[test]
2432 fn same_process_pid_requires_explicit_in_process_harness_opt_in() {
2433 let current = std::process::id();
2434 assert!(
2435 !pid_can_name_incumbent(current, current, false),
2436 "production startup must not trust a same-PID stale rendezvous"
2437 );
2438 assert!(
2439 pid_can_name_incumbent(current, current, true),
2440 "the in-process harness must let a responsive same-PID owner win"
2441 );
2442 let distinct_probe_pid = current.wrapping_add(2);
2446 assert_ne!(
2447 distinct_probe_pid, current,
2448 "probe PID must differ from this process's PID"
2449 );
2450 assert!(
2451 pid_can_name_incumbent(distinct_probe_pid, current, false),
2452 "a distinct PID remains eligible under ordinary production rules"
2453 );
2454 }
2455
2456 #[test]
2457 fn pid_1_probe_is_running_regardless_of_permission_outcome() {
2458 assert!(
2466 is_process_running(1),
2467 "PID 1 always exists; EPERM must not read as dead"
2468 );
2469 }
2470
2471 #[test]
2472 fn env_truthy_recognises_set_values() {
2473 assert!(!env_truthy("__KHIVE_TEST_ABSENT_VAR_XYZ__"));
2474
2475 let key = "__KHIVE_TEST_TRUTHY_ABC__";
2479 std::env::set_var(key, "1");
2480 assert!(env_truthy(key));
2481 std::env::set_var(key, "false");
2482 assert!(!env_truthy(key));
2483 std::env::set_var(key, "0");
2484 assert!(!env_truthy(key));
2485 std::env::remove_var(key);
2486 }
2487
2488 #[tokio::test(flavor = "current_thread", start_paused = true)]
2489 #[serial(background_tasks)]
2490 async fn accepted_connection_is_counted_before_first_poll_and_drain_waits() {
2491 use std::sync::atomic::Ordering;
2492
2493 let active = Arc::new(std::sync::atomic::AtomicUsize::new(0));
2494 let started = Arc::new(std::sync::atomic::AtomicBool::new(false));
2495 let (release_tx, release_rx) = tokio::sync::oneshot::channel::<()>();
2496 let started_in_task = Arc::clone(&started);
2497
2498 let handle = spawn_connection_task(Arc::clone(&active), async move {
2499 started_in_task.store(true, Ordering::Relaxed);
2500 let _ = release_rx.await;
2501 });
2502
2503 assert_eq!(active.load(Ordering::Relaxed), 1);
2504 assert!(
2505 !started.load(Ordering::Relaxed),
2506 "the current-thread runtime must leave the spawned handler unpolled"
2507 );
2508
2509 let drain_fut = drain(active.as_ref());
2510 tokio::pin!(drain_fut);
2511 let too_early =
2512 tokio::time::timeout(std::time::Duration::from_millis(150), &mut drain_fut).await;
2513 assert!(
2514 too_early.is_err(),
2515 "drain must wait for a connection claimed before its task's first poll"
2516 );
2517 assert!(started.load(Ordering::Relaxed));
2518
2519 release_tx.send(()).expect("handler still waiting");
2520 tokio::time::timeout(std::time::Duration::from_secs(1), handle)
2521 .await
2522 .expect("handler should finish promptly")
2523 .expect("handler should not panic");
2524 assert_eq!(active.load(Ordering::Relaxed), 0);
2525 tokio::time::timeout(std::time::Duration::from_secs(1), drain_fut)
2526 .await
2527 .expect("drain should finish once the handler releases its claim");
2528 }
2529
2530 #[tokio::test(flavor = "current_thread")]
2531 async fn cancelled_connection_releases_count_before_first_poll() {
2532 use std::sync::atomic::Ordering;
2533
2534 let active = Arc::new(std::sync::atomic::AtomicUsize::new(0));
2535 let started = Arc::new(std::sync::atomic::AtomicBool::new(false));
2536 let started_in_task = Arc::clone(&started);
2537 let handle = spawn_connection_task(Arc::clone(&active), async move {
2538 started_in_task.store(true, Ordering::Relaxed);
2539 std::future::pending::<()>().await;
2540 });
2541
2542 assert_eq!(active.load(Ordering::Relaxed), 1);
2543 handle.abort();
2544 let error = handle.await.expect_err("aborted handler must be cancelled");
2545 assert!(error.is_cancelled());
2546 assert!(!started.load(Ordering::Relaxed));
2547 assert_eq!(active.load(Ordering::Relaxed), 0);
2548 }
2549
2550 #[tokio::test(flavor = "current_thread")]
2551 async fn panicked_connection_releases_count() {
2552 use std::sync::atomic::Ordering;
2553
2554 let active = Arc::new(std::sync::atomic::AtomicUsize::new(0));
2555 let handle = spawn_connection_task(Arc::clone(&active), async move {
2556 panic!("intentional connection-handler panic");
2557 });
2558
2559 assert_eq!(active.load(Ordering::Relaxed), 1);
2560 let error = handle
2561 .await
2562 .expect_err("panicked handler must fail its join");
2563 assert!(error.is_panic());
2564 assert_eq!(active.load(Ordering::Relaxed), 0);
2565 }
2566
2567 #[test]
2568 fn connection_claim_releases_if_spawn_panics() {
2569 use std::sync::atomic::Ordering;
2570
2571 let active = Arc::new(std::sync::atomic::AtomicUsize::new(0));
2572 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
2573 drop(spawn_connection_task(Arc::clone(&active), async {}));
2574 }));
2575
2576 assert!(result.is_err(), "tokio::spawn outside a runtime must panic");
2577 assert_eq!(active.load(Ordering::Relaxed), 0);
2578 }
2579
2580 #[tokio::test]
2581 #[serial(background_tasks)]
2582 async fn drain_returns_promptly_with_no_accepted_connection() {
2583 let active = std::sync::atomic::AtomicUsize::new(0);
2584 tokio::time::timeout(std::time::Duration::from_secs(1), drain(&active))
2585 .await
2586 .expect("empty drain should return immediately");
2587 }
2588
2589 #[tokio::test(start_paused = true)]
2590 #[serial(background_tasks)]
2591 async fn graceful_drain_has_a_hard_upper_bound_with_stuck_work() {
2592 use std::sync::atomic::Ordering;
2593
2594 let active = Arc::new(std::sync::atomic::AtomicUsize::new(0));
2595 let task = spawn_connection_task(Arc::clone(&active), async {
2596 std::future::pending::<()>().await;
2597 });
2598 assert_eq!(active.load(Ordering::Relaxed), 1);
2599 let started = tokio::time::Instant::now();
2600
2601 let drained = drain_with_timeout(&active, std::time::Duration::from_millis(250)).await;
2602
2603 assert!(!drained, "stuck work must exhaust the drain bound");
2604 assert!(
2605 started.elapsed() >= std::time::Duration::from_millis(250)
2606 && started.elapsed() < std::time::Duration::from_millis(350),
2607 "graceful shutdown exceeded its configured bound: {:?}",
2608 started.elapsed()
2609 );
2610 finish_connection_tasks(vec![task], drained).await;
2611 assert_eq!(
2612 active.load(Ordering::Relaxed),
2613 0,
2614 "hard-bound escalation must abort, await, and release the handler"
2615 );
2616 }
2617
2618 #[tokio::test(start_paused = true)]
2619 #[serial(background_tasks)]
2620 async fn admitted_work_finishes_inside_drain_window_without_abort() {
2621 use std::sync::atomic::Ordering;
2622
2623 let active = Arc::new(std::sync::atomic::AtomicUsize::new(0));
2624 let committed = Arc::new(std::sync::atomic::AtomicBool::new(false));
2625 let committed_in_task = Arc::clone(&committed);
2626 let task = spawn_connection_task(Arc::clone(&active), async move {
2627 tokio::time::sleep(std::time::Duration::from_millis(100)).await;
2628 committed_in_task.store(true, Ordering::SeqCst);
2629 });
2630
2631 let drained = drain_with_timeout(&active, std::time::Duration::from_millis(250)).await;
2632 assert!(
2633 drained,
2634 "admitted work should finish inside the drain window"
2635 );
2636 finish_connection_tasks(vec![task], drained).await;
2637 assert!(committed.load(Ordering::SeqCst));
2638 assert_eq!(active.load(Ordering::Relaxed), 0);
2639 }
2640
2641 #[tokio::test]
2654 #[serial(background_tasks)]
2655 async fn drain_waits_for_tracked_background_tasks_before_returning() {
2656 let active = std::sync::atomic::AtomicUsize::new(0);
2657 let (tx, rx) = tokio::sync::oneshot::channel::<()>();
2658
2659 track_background_task(async move {
2660 let _ = rx.await;
2661 });
2662 assert!(
2663 background_task_count() >= 1,
2664 "track_background_task must make the in-flight task visible immediately"
2665 );
2666
2667 let drain_fut = drain(&active);
2668 tokio::pin!(drain_fut);
2669
2670 let too_early =
2672 tokio::time::timeout(std::time::Duration::from_millis(150), &mut drain_fut).await;
2673 assert!(
2674 too_early.is_err(),
2675 "drain() must not return while a tracked background task is still running"
2676 );
2677
2678 tx.send(())
2680 .expect("tracked task still awaiting the oneshot");
2681 let done = tokio::time::timeout(std::time::Duration::from_secs(5), drain_fut).await;
2682 assert!(
2683 done.is_ok(),
2684 "drain() must return once the tracked background task finishes"
2685 );
2686 }
2687
2688 #[tokio::test]
2689 #[serial(background_tasks)]
2690 async fn drain_waits_for_hydration_after_its_last_request_waiter_is_cancelled() {
2691 let before = background_task_count();
2692 let active = std::sync::atomic::AtomicUsize::new(0);
2693 let (started_tx, started_rx) = tokio::sync::oneshot::channel();
2694 let release = Arc::new(tokio::sync::Semaphore::new(0));
2695 let store = Arc::new(DrainBlockingBlobStore {
2696 started: std::sync::Mutex::new(Some(started_tx)),
2697 release: Arc::clone(&release),
2698 });
2699 let hydrator = Arc::new(
2700 crate::BlobHydrator::new(
2701 store as Arc<dyn khive_storage::BlobStore>,
2702 khive_storage::MAX_BLOB_WHOLE_BYTES,
2703 )
2704 .expect("minimum hydration budget is valid"),
2705 );
2706 let content_ref =
2707 khive_storage::ContentRef::from_hex("a".repeat(64)).expect("fixture content ref");
2708
2709 let request_hydrator = Arc::clone(&hydrator);
2710 let request = tokio::spawn(async move {
2711 request_hydrator
2712 .hydrate_verified(&content_ref, khive_storage::MAX_BLOB_WHOLE_BYTES)
2713 .await
2714 });
2715 started_rx.await.expect("backend work must begin");
2716 request.abort();
2717 assert!(request.await.unwrap_err().is_cancelled());
2718 assert_eq!(background_task_count(), before + 1);
2719
2720 let draining = drain_with_timeout(&active, std::time::Duration::from_secs(5));
2721 tokio::pin!(draining);
2722 assert!(
2723 tokio::time::timeout(std::time::Duration::from_millis(150), &mut draining)
2724 .await
2725 .is_err(),
2726 "drain must remain pending while cancelled-request hydration still runs"
2727 );
2728
2729 release.add_permits(1);
2730 assert!(
2731 tokio::time::timeout(std::time::Duration::from_secs(5), draining)
2732 .await
2733 .expect("drain should finish after native hydration ends"),
2734 "hydration should finish inside the drain window"
2735 );
2736 assert_eq!(background_task_count(), before);
2737 }
2738
2739 #[tokio::test]
2744 #[serial(background_tasks)]
2745 async fn track_background_task_count_returns_to_zero_after_completion() {
2746 let before = background_task_count();
2748 let (tx, rx) = tokio::sync::oneshot::channel::<()>();
2749 track_background_task(async move {
2750 let _ = rx.await;
2751 });
2752 assert_eq!(background_task_count(), before + 1);
2753 tx.send(()).expect("still awaiting");
2754 for _ in 0..100 {
2756 if background_task_count() == before {
2757 break;
2758 }
2759 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
2760 }
2761 assert_eq!(background_task_count(), before);
2762 }
2763
2764 #[tokio::test]
2767 #[serial(background_tasks)]
2768 async fn track_background_task_count_returns_to_baseline_after_panic() {
2769 let before = background_task_count();
2779
2780 let (tx, rx) = tokio::sync::oneshot::channel::<()>();
2781 track_background_task(async move {
2782 let _ = rx.await;
2783 panic!("intentional panic to exercise the Drop-guard decrement path");
2784 });
2785 assert_eq!(background_task_count(), before + 1);
2786
2787 tx.send(()).expect("still awaiting");
2788 for _ in 0..100 {
2789 if background_task_count() == before {
2790 break;
2791 }
2792 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
2793 }
2794 assert_eq!(
2795 background_task_count(),
2796 before,
2797 "background task counter must return to baseline after the tracked future panics"
2798 );
2799 }
2800
2801 #[test]
2809 #[serial(active_phases)]
2810 fn register_active_phase_appears_and_disappears_with_the_guard() {
2811 assert!(
2812 !active_phase_names().contains(&"adr103_test_phase".to_string()),
2813 "must start absent (leaked from a prior failed run would poison this test)"
2814 );
2815
2816 let guard = register_active_phase("adr103_test_phase");
2817 assert!(active_phase_names().contains(&"adr103_test_phase".to_string()));
2818
2819 drop(guard);
2820 assert!(
2821 !active_phase_names().contains(&"adr103_test_phase".to_string()),
2822 "the phase name must drop out of the gauge once its guard is dropped"
2823 );
2824 }
2825
2826 #[test]
2827 #[serial(active_phases)]
2828 fn register_active_phase_counts_concurrent_occurrences_of_the_same_name() {
2829 let first = register_active_phase("adr103_concurrent_phase");
2830 let second = register_active_phase("adr103_concurrent_phase");
2831 assert!(active_phase_names().contains(&"adr103_concurrent_phase".to_string()));
2832
2833 drop(first);
2834 assert!(
2835 active_phase_names().contains(&"adr103_concurrent_phase".to_string()),
2836 "one of two concurrent occurrences ending must not remove the name early"
2837 );
2838
2839 drop(second);
2840 assert!(
2841 !active_phase_names().contains(&"adr103_concurrent_phase".to_string()),
2842 "the name must be removed only once every concurrent occurrence has ended"
2843 );
2844 }
2845
2846 #[derive(Clone)]
2853 struct MockDispatch {
2854 namespace: String,
2855 config_id: String,
2856 dispatch_calls: Arc<std::sync::atomic::AtomicUsize>,
2857 pool: Option<Arc<ConnectionPool>>,
2858 dispatch_err: Option<String>,
2862 }
2863
2864 #[derive(Clone)]
2865 struct CancellationAwareDispatch {
2866 started: Arc<tokio::sync::Notify>,
2867 cancellation_observed: Arc<std::sync::atomic::AtomicBool>,
2868 }
2869
2870 #[async_trait]
2871 impl DaemonDispatch for CancellationAwareDispatch {
2872 async fn dispatch(
2873 &self,
2874 _ops: String,
2875 _presentation: Option<String>,
2876 _presentation_per_op: Option<Vec<Option<String>>>,
2877 _format: Option<String>,
2878 _format_per_op: Option<Vec<Option<String>>>,
2879 _from_wire: bool,
2880 _identity: Option<RequestIdentity>,
2881 ) -> Result<String, String> {
2882 self.started.notify_one();
2883 khive_storage::wait_for_request_read_cancellation().await;
2884 self.cancellation_observed
2885 .store(true, std::sync::atomic::Ordering::SeqCst);
2886 Ok("{}".to_string())
2887 }
2888
2889 async fn warm_all(&self) {}
2890
2891 fn namespace(&self) -> &str {
2892 "local"
2893 }
2894
2895 fn config_id(&self) -> &str {
2896 "disconnect-test"
2897 }
2898 }
2899
2900 #[async_trait]
2901 impl DaemonDispatch for MockDispatch {
2902 async fn dispatch(
2903 &self,
2904 _ops: String,
2905 _presentation: Option<String>,
2906 _presentation_per_op: Option<Vec<Option<String>>>,
2907 _format: Option<String>,
2908 _format_per_op: Option<Vec<Option<String>>>,
2909 _from_wire: bool,
2910 _identity: Option<RequestIdentity>,
2911 ) -> Result<String, String> {
2912 self.dispatch_calls
2913 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
2914 match &self.dispatch_err {
2915 Some(msg) => Err(msg.clone()),
2916 None => Ok("{}".to_string()),
2917 }
2918 }
2919
2920 async fn warm_all(&self) {}
2921
2922 fn namespace(&self) -> &str {
2923 &self.namespace
2924 }
2925
2926 fn config_id(&self) -> &str {
2927 &self.config_id
2928 }
2929
2930 fn pool_for_checkpoint(&self) -> Option<Arc<ConnectionPool>> {
2931 self.pool.clone()
2932 }
2933 }
2934
2935 fn base_request_frame(config_id: &str) -> DaemonRequestFrame {
2936 DaemonRequestFrame {
2937 ops: String::new(),
2938 presentation: None,
2939 presentation_per_op: None,
2940 namespace: "local".to_string(),
2941 actor_id: None,
2942 process_ref: None,
2943 visible_namespaces: Vec::new(),
2944 config_id: config_id.to_string(),
2945 protocol_version: PROTOCOL_VERSION,
2946 probe_only: false,
2947 metrics_only: false,
2948 format: None,
2949 format_per_op: None,
2950 from_wire: false,
2951 request_id: None,
2952 }
2953 }
2954
2955 async fn round_trip(dispatcher: MockDispatch, req: &DaemonRequestFrame) -> DaemonResponseFrame {
2958 let (mut client, server) = UnixStream::pair().expect("unix stream pair");
2959 let payload = serde_json::to_vec(req).expect("encode request frame");
2960 let handle = tokio::spawn(async move {
2961 handle_conn(server, dispatcher).await;
2962 });
2963 write_frame(&mut client, &payload)
2964 .await
2965 .expect("write request frame");
2966 let raw = read_frame(&mut client).await.expect("read response frame");
2967 handle.await.expect("handle_conn task panicked");
2968 serde_json::from_slice(&raw).expect("decode response frame")
2969 }
2970
2971 #[tokio::test]
2972 async fn daemon_peer_disconnect_signals_request_read_cancellation() {
2973 let started = Arc::new(tokio::sync::Notify::new());
2974 let cancellation_observed = Arc::new(std::sync::atomic::AtomicBool::new(false));
2975 let dispatcher = CancellationAwareDispatch {
2976 started: Arc::clone(&started),
2977 cancellation_observed: Arc::clone(&cancellation_observed),
2978 };
2979 let (mut client, server) = UnixStream::pair().expect("unix stream pair");
2980 let request = base_request_frame("disconnect-test");
2981 let payload = serde_json::to_vec(&request).expect("encode request frame");
2982 let handler = tokio::spawn(async move { handle_conn(server, dispatcher).await });
2983 write_frame(&mut client, &payload)
2984 .await
2985 .expect("write request frame");
2986 started.notified().await;
2987
2988 drop(client);
2989 tokio::time::timeout(std::time::Duration::from_millis(500), handler)
2990 .await
2991 .expect("daemon handler ignored peer disconnect")
2992 .expect("daemon handler panicked");
2993 assert!(
2994 cancellation_observed.load(std::sync::atomic::Ordering::SeqCst),
2995 "peer loss did not reach the request-scoped read cancellation signal"
2996 );
2997 }
2998
2999 #[tokio::test]
3004 async fn protocol_v3_frame_is_rejected_before_process_ref_dispatch() {
3005 assert_eq!(PROTOCOL_VERSION, 4, "process_ref is the protocol-v4 change");
3006 let dispatch_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
3007 let dispatcher = MockDispatch {
3008 namespace: "local".to_string(),
3009 config_id: "cfg-v4".to_string(),
3010 dispatch_calls: Arc::clone(&dispatch_calls),
3011 pool: None,
3012 dispatch_err: None,
3013 };
3014 let mut request = base_request_frame("cfg-v4");
3015 request.protocol_version = 3;
3016 request.process_ref = Some("worker/legacy-rollout".to_string());
3017
3018 let response = round_trip(dispatcher, &request).await;
3019 assert!(!response.ok);
3020 assert!(response.version_mismatch);
3021 assert_eq!(response.daemon_protocol_version, 4);
3022 assert_eq!(
3023 dispatch_calls.load(std::sync::atomic::Ordering::SeqCst),
3024 0,
3025 "a v3 frame must be rejected before a provenance-bearing mutation dispatches"
3026 );
3027 let error = response.error.expect("mismatch explains both versions");
3028 assert!(
3029 error.contains("client=3") && error.contains("daemon=4"),
3030 "mismatch must identify the exact rollout boundary; got {error:?}"
3031 );
3032 }
3033
3034 #[tokio::test]
3041 async fn metrics_only_frame_returns_snapshot_without_dispatching() {
3042 let dispatch_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
3043 let dispatcher = MockDispatch {
3044 namespace: "local".to_string(),
3045 config_id: "cfg-a".to_string(),
3046 dispatch_calls: Arc::clone(&dispatch_calls),
3047 pool: None,
3048 dispatch_err: None,
3049 };
3050
3051 let mut metrics_req = base_request_frame("cfg-a");
3052 metrics_req.metrics_only = true;
3053 let metrics_resp = round_trip(dispatcher.clone(), &metrics_req).await;
3054
3055 assert!(metrics_resp.ok, "metrics_only response must be ok=true");
3056 assert!(
3057 metrics_resp.metrics.is_some(),
3058 "metrics_only=true must return Some(snapshot)"
3059 );
3060 assert_eq!(
3061 dispatch_calls.load(std::sync::atomic::Ordering::SeqCst),
3062 0,
3063 "metrics_only must never reach the ops-dispatch path"
3064 );
3065
3066 let mut mismatched_req = base_request_frame("some-other-config");
3068 mismatched_req.metrics_only = true;
3069 let mismatched_resp = round_trip(dispatcher.clone(), &mismatched_req).await;
3070 assert!(mismatched_resp.ok);
3071 assert!(mismatched_resp.metrics.is_some());
3072 assert!(!mismatched_resp.config_mismatch);
3073 assert_eq!(
3074 dispatch_calls.load(std::sync::atomic::Ordering::SeqCst),
3075 0,
3076 "a mismatched-config metrics_only request must still skip dispatch"
3077 );
3078
3079 let normal_req = base_request_frame("cfg-a");
3082 let normal_resp = round_trip(dispatcher, &normal_req).await;
3083 assert!(normal_resp.ok);
3084 assert!(normal_resp.metrics.is_none());
3085 assert_eq!(dispatch_calls.load(std::sync::atomic::Ordering::SeqCst), 1);
3086 }
3087
3088 #[test]
3101 fn shared_writable_socket_dirs_are_refused_without_being_modified() {
3102 let dir = tempfile::tempdir().expect("tempdir");
3103 for (name, mode) in [("open", 0o777u32), ("sticky-tmp", 0o1777u32)] {
3104 let shared = dir.path().join(name);
3105 std::fs::create_dir(&shared).expect("create");
3106 std::fs::set_permissions(&shared, std::fs::Permissions::from_mode(mode))
3107 .expect("chmod");
3108
3109 let err = ensure_socket_dir_is_trusted(&shared)
3110 .expect_err("group/other-writable must be refused, sticky or not");
3111 assert!(
3112 err.to_string().contains(&format!("{:04o}", mode & 0o7777)),
3113 "the refusal should name the mode it saw, got: {err}"
3114 );
3115 let after = std::fs::metadata(&shared)
3116 .expect("stat")
3117 .permissions()
3118 .mode()
3119 & 0o7777;
3120 assert_eq!(
3121 after, mode,
3122 "refusing must not re-permission a directory khive does not own"
3123 );
3124 }
3125 }
3126
3127 #[test]
3135 fn writable_non_sticky_ancestor_is_refused() {
3136 let dir = tempfile::tempdir().expect("tempdir");
3137 let open_mid = dir.path().join("open-mid");
3138 std::fs::create_dir(&open_mid).expect("create mid");
3139 let inner = open_mid.join("private");
3140 std::fs::create_dir(&inner).expect("create inner");
3141 std::fs::set_permissions(&inner, std::fs::Permissions::from_mode(0o700)).expect("chmod");
3142 std::fs::set_permissions(&open_mid, std::fs::Permissions::from_mode(0o777))
3143 .expect("chmod mid");
3144
3145 let err = ensure_socket_dir_is_trusted(&inner)
3146 .expect_err("a 0777 non-sticky ancestor must be refused");
3147 assert!(
3148 err.to_string().contains("ancestor"),
3149 "the refusal should say it was an ancestor that failed, got: {err}"
3150 );
3151 assert!(
3152 err.to_string().contains("open-mid"),
3153 "the refusal should name the failing ancestor, got: {err}"
3154 );
3155 }
3156
3157 #[test]
3164 fn symlink_component_to_untrusted_directory_is_refused() {
3165 let dir = tempfile::tempdir().expect("tempdir");
3166 let open = dir.path().join("open-target");
3167 std::fs::create_dir(&open).expect("create target");
3168 std::fs::set_permissions(&open, std::fs::Permissions::from_mode(0o777)).expect("chmod");
3169 let link = dir.path().join("link");
3170 std::os::unix::fs::symlink(&open, &link).expect("symlink");
3171
3172 let euid = unsafe { libc::geteuid() } as u32;
3174 let err = ensure_socket_path_is_swap_resistant(&link, euid)
3175 .expect_err("a link into a 0777 non-sticky directory must be refused");
3176 assert!(
3177 err.to_string().contains("open-target"),
3178 "the refusal should name the untrusted target directory, got: {err}"
3179 );
3180 }
3181
3182 #[test]
3191 fn foreign_owned_symlink_component_is_refused() {
3192 let link =
3193 std::path::PathBuf::from(format!("/tmp/khive-swaptest-link-{}", std::process::id()));
3194 let _ = std::fs::remove_file(&link);
3195 std::os::unix::fs::symlink("/tmp", &link).expect("symlink");
3196
3197 let not_my_euid = (unsafe { libc::geteuid() } as u32).wrapping_add(1);
3199 let result = ensure_socket_path_is_swap_resistant(&link, not_my_euid);
3200 std::fs::remove_file(&link).expect("cleanup");
3201
3202 let err = result.expect_err("a symlink owned by another uid must be refused");
3203 assert!(
3204 err.to_string().contains("symlink component"),
3205 "the refusal should strike the symlink itself, got: {err}"
3206 );
3207 assert!(
3208 err.to_string().contains("khive-swaptest-link"),
3209 "the refusal should name the offending link, got: {err}"
3210 );
3211 }
3212
3213 #[test]
3223 fn trusted_socket_dirs_are_accepted_unmodified() {
3224 let dir = tempfile::tempdir().expect("tempdir");
3225 for (name, mode) in [("private", 0o700), ("listable", 0o755)] {
3226 let d = dir.path().join(name);
3227 std::fs::create_dir(&d).expect("create");
3228 std::fs::set_permissions(&d, std::fs::Permissions::from_mode(mode)).expect("chmod");
3229
3230 ensure_socket_dir_is_trusted(&d)
3231 .unwrap_or_else(|e| panic!("mode {mode:04o} must be accepted, got: {e}"));
3232
3233 let after = std::fs::metadata(&d).expect("stat").permissions().mode() & 0o7777;
3234 assert_eq!(
3235 after, mode,
3236 "acceptance must not re-permission the directory either"
3237 );
3238 }
3239 }
3240
3241 #[tokio::test]
3246 async fn metrics_snapshot_wal_pages_reflects_recent_write() {
3247 let dir = tempfile::tempdir().expect("tempdir");
3248 let path = dir.path().join("metrics_wal_test.db");
3249 let pool = Arc::new(
3250 ConnectionPool::new(khive_db::PoolConfig {
3251 path: Some(path),
3252 ..khive_db::PoolConfig::default()
3253 })
3254 .expect("pool open"),
3255 );
3256
3257 {
3258 let writer = pool.try_writer().expect("writer");
3259 writer
3260 .conn()
3261 .execute_batch(
3262 "CREATE TABLE t (x INTEGER); \
3263 INSERT INTO t VALUES (1); \
3264 INSERT INTO t VALUES (2);",
3265 )
3266 .expect("seed writes");
3267 }
3268
3269 let dedicated_conn = pool
3270 .open_standalone_writer()
3271 .expect("open dedicated checkpoint connection");
3272 khive_db::checkpoint_once(
3273 &pool,
3274 &dedicated_conn,
3275 &CheckpointConfig::default(),
3276 &mut khive_db::checkpoint::TruncateState::default(),
3277 )
3278 .expect("checkpoint_once must observe on a healthy dedicated connection");
3279
3280 let dispatcher = MockDispatch {
3281 namespace: "local".to_string(),
3282 config_id: "cfg-wal".to_string(),
3283 dispatch_calls: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
3284 pool: Some(pool),
3285 dispatch_err: None,
3286 };
3287
3288 let snapshot = build_metrics_snapshot(&dispatcher);
3289 assert!(
3290 snapshot.wal_pages.is_some(),
3291 "wal_pages must be observed after a real checkpoint tick, got {snapshot:?}"
3292 );
3293 assert_eq!(snapshot.wal_log_frames, snapshot.wal_pages);
3294 assert!(snapshot.wal_checkpointed_frames.is_some());
3295 assert!(snapshot.wal_pending_frames.is_some());
3296 assert!(snapshot.wal_physical_bytes.is_some());
3297 assert!(snapshot.wal_observed_at_unix_ms.is_some());
3298 assert_eq!(
3302 snapshot.wal_checkpoint_consecutive_skips, 0,
3303 "an observed (non-skipped) tick must report zero consecutive skips, got {snapshot:?}"
3304 );
3305 }
3306
3307 #[tokio::test]
3308 async fn metrics_snapshot_exposes_decomposed_writer_stages() {
3309 let dir = tempfile::tempdir().expect("tempdir");
3310 let path = dir.path().join("metrics_writer_stage_test.db");
3311 let pool = Arc::new(
3312 ConnectionPool::new(khive_db::PoolConfig {
3313 path: Some(path),
3314 ..khive_db::PoolConfig::default()
3315 })
3316 .expect("pool open"),
3317 );
3318 {
3319 let writer = pool.try_writer().unwrap();
3320 writer
3321 .conn()
3322 .execute_batch("CREATE TABLE t (id INTEGER PRIMARY KEY)")
3323 .unwrap();
3324 }
3325 let handle = pool
3326 .writer_task_handle()
3327 .unwrap()
3328 .expect("file-backed default writer task");
3329 handle
3330 .send(|conn| {
3331 std::thread::sleep(std::time::Duration::from_millis(30));
3332 conn.execute("INSERT INTO t VALUES (1)", [])
3333 .map_err(|error| khive_storage::error::StorageError::Pool {
3334 operation: "metrics_writer_stage_test".into(),
3335 message: error.to_string(),
3336 })
3337 })
3338 .await
3339 .unwrap();
3340
3341 let dispatcher = MockDispatch {
3342 namespace: "local".to_string(),
3343 config_id: "cfg-writer-stages".to_string(),
3344 dispatch_calls: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
3345 pool: Some(pool),
3346 dispatch_err: None,
3347 };
3348 let snapshot = build_metrics_snapshot(&dispatcher);
3349 assert!(snapshot.write_last_queue_wait_micros.is_some());
3350 assert!(snapshot.write_last_transaction_acquire_micros.is_some());
3351 assert!(snapshot.write_last_commit_micros.is_some());
3352 assert!(
3353 snapshot.write_last_body_micros >= Some(25_000),
3354 "synthetic delay must be attributed to the body: {snapshot:?}"
3355 );
3356 assert!(snapshot.write_last_total_micros >= snapshot.write_last_body_micros);
3357 assert!(snapshot.write_last_observed_at_unix_ms.is_some());
3358 }
3359
3360 #[test]
3365 #[serial(tx_registry)]
3366 fn metrics_snapshot_reflects_open_transaction_registry() {
3367 let dispatcher = MockDispatch {
3368 namespace: "local".to_string(),
3369 config_id: "cfg-tx".to_string(),
3370 dispatch_calls: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
3371 pool: None,
3372 dispatch_err: None,
3373 };
3374
3375 let departing_handle = khive_storage::tx_registry::register(Some(
3376 "daemon_metrics_snapshot_departing_test_tx".to_string(),
3377 ));
3378 let before = build_metrics_snapshot(&dispatcher).open_tx_count;
3379 assert!(before >= 1);
3380
3381 let handle = khive_storage::tx_registry::register(Some(
3382 "daemon_metrics_snapshot_owned_test_tx".to_string(),
3383 ));
3384 drop(departing_handle);
3385
3386 let during = build_metrics_snapshot(&dispatcher);
3387 assert!(
3388 during.open_tx_count >= 1,
3389 "open_tx_count must reflect the live owned transaction despite registry churn: \
3390 churn_baseline={before} during={}",
3391 during.open_tx_count
3392 );
3393 assert!(
3394 during.oldest_pinned_tx_micros.is_some(),
3395 "oldest_pinned_tx_micros must be Some while a transaction is open"
3396 );
3397
3398 drop(handle);
3399 assert!(
3400 !khive_storage::tx_registry::snapshot()
3401 .iter()
3402 .any(|(_, label)| label.as_deref()
3403 == Some("daemon_metrics_snapshot_owned_test_tx")),
3404 "the owned registry entry must disappear when its handle is dropped"
3405 );
3406 }
3407
3408 #[tokio::test]
3412 async fn metrics_snapshot_write_queue_depth_flag_gated() {
3413 let dir = tempfile::tempdir().expect("tempdir");
3414
3415 let enabled_pool = Arc::new(
3416 ConnectionPool::new(khive_db::PoolConfig {
3417 path: Some(dir.path().join("wq_enabled.db")),
3418 write_queue_enabled: Some(true),
3419 ..khive_db::PoolConfig::default()
3420 })
3421 .expect("pool open"),
3422 );
3423 let enabled_dispatcher = MockDispatch {
3424 namespace: "local".to_string(),
3425 config_id: "cfg-wq-on".to_string(),
3426 dispatch_calls: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
3427 pool: Some(enabled_pool),
3428 dispatch_err: None,
3429 };
3430 let snapshot_on = build_metrics_snapshot(&enabled_dispatcher);
3431 assert!(
3432 snapshot_on.write_queue_depth.is_some(),
3433 "write_queue_depth must be Some when write_queue_enabled=true, got {snapshot_on:?}"
3434 );
3435 assert!(snapshot_on.write_queue_capacity.is_some());
3436
3437 let disabled_pool = Arc::new(
3438 ConnectionPool::new(khive_db::PoolConfig {
3439 path: Some(dir.path().join("wq_disabled.db")),
3440 write_queue_enabled: Some(false),
3441 ..khive_db::PoolConfig::default()
3442 })
3443 .expect("pool open"),
3444 );
3445 let disabled_dispatcher = MockDispatch {
3446 namespace: "local".to_string(),
3447 config_id: "cfg-wq-off".to_string(),
3448 dispatch_calls: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
3449 pool: Some(disabled_pool),
3450 dispatch_err: None,
3451 };
3452 let snapshot_off = build_metrics_snapshot(&disabled_dispatcher);
3453 assert!(
3454 snapshot_off.write_queue_depth.is_none(),
3455 "write_queue_depth must be None when write_queue_enabled=false, got {snapshot_off:?}"
3456 );
3457 assert!(snapshot_off.write_queue_capacity.is_none());
3458
3459 let no_pool_dispatcher = MockDispatch {
3461 namespace: "local".to_string(),
3462 config_id: "cfg-no-pool".to_string(),
3463 dispatch_calls: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
3464 pool: None,
3465 dispatch_err: None,
3466 };
3467 let snapshot_no_pool = build_metrics_snapshot(&no_pool_dispatcher);
3468 assert!(snapshot_no_pool.write_queue_depth.is_none());
3469 assert!(snapshot_no_pool.write_queue_capacity.is_none());
3470 }
3471
3472 #[test]
3477 fn frame_serde_defaults_additive_fields_when_absent() {
3478 let req_json = serde_json::json!({
3479 "ops": "",
3480 "presentation": null,
3481 "presentation_per_op": null,
3482 "namespace": "local",
3483 "actor_id": null,
3484 "visible_namespaces": [],
3485 "config_id": "cfg",
3486 "protocol_version": PROTOCOL_VERSION,
3487 "probe_only": false,
3488 "format": null,
3489 "format_per_op": null,
3490 "from_wire": false
3491 });
3492 let frame: DaemonRequestFrame =
3493 serde_json::from_value(req_json).expect("decode a metrics_only-absent request frame");
3494 assert!(
3495 !frame.metrics_only,
3496 "metrics_only must default to false when absent from the wire payload"
3497 );
3498 assert_eq!(
3499 frame.request_id, None,
3500 "request_id must default to None when absent from the wire payload (khive#948)"
3501 );
3502 assert_eq!(
3503 frame.process_ref, None,
3504 "process_ref must default to None when absent from the wire payload (khive#1428)"
3505 );
3506 let encoded_frame = serde_json::to_value(&frame).expect("encode request frame");
3507 assert!(
3508 encoded_frame.get("process_ref").is_none(),
3509 "absent provenance must not change the serialized request wire shape"
3510 );
3511
3512 let resp_json = serde_json::json!({
3513 "ok": true,
3514 "result": null,
3515 "error": null,
3516 "namespace_mismatch": false,
3517 "config_mismatch": false,
3518 "served_config_id": "cfg",
3519 "version_mismatch": false,
3520 "daemon_protocol_version": PROTOCOL_VERSION
3521 });
3522 let resp: DaemonResponseFrame =
3523 serde_json::from_value(resp_json).expect("decode a metrics-absent response frame");
3524 assert!(
3525 resp.metrics.is_none(),
3526 "metrics must default to None when absent from the wire payload"
3527 );
3528 assert_eq!(
3529 resp.request_id, None,
3530 "request_id must default to None when absent from the wire payload (khive#948)"
3531 );
3532 }
3533
3534 #[tokio::test]
3539 async fn request_id_echoed_on_success_and_error_arms() {
3540 let dispatcher = MockDispatch {
3541 namespace: "local".to_string(),
3542 config_id: "cfg-a".to_string(),
3543 dispatch_calls: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
3544 pool: None,
3545 dispatch_err: None,
3546 };
3547 let mut ok_req = base_request_frame("cfg-a");
3548 ok_req.request_id = Some(42);
3549 let ok_resp = round_trip(dispatcher, &ok_req).await;
3550 assert!(ok_resp.ok, "expected successful dispatch: {ok_resp:?}");
3551 assert_eq!(
3552 ok_resp.request_id,
3553 Some(42),
3554 "request_id must be echoed back on a successful dispatch response"
3555 );
3556
3557 let mismatched_dispatcher = MockDispatch {
3560 namespace: "local".to_string(),
3561 config_id: "cfg-a".to_string(),
3562 dispatch_calls: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
3563 pool: None,
3564 dispatch_err: None,
3565 };
3566 let mut mismatch_req = base_request_frame("cfg-WRONG");
3567 mismatch_req.request_id = Some(99);
3568 let mismatch_resp = round_trip(mismatched_dispatcher, &mismatch_req).await;
3569 assert!(mismatch_resp.config_mismatch);
3570 assert_eq!(
3571 mismatch_resp.request_id,
3572 Some(99),
3573 "request_id must be echoed on the config_mismatch rejection arm too"
3574 );
3575
3576 let erroring_dispatcher = MockDispatch {
3579 namespace: "local".to_string(),
3580 config_id: "cfg-a".to_string(),
3581 dispatch_calls: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
3582 pool: None,
3583 dispatch_err: Some("simulated dispatch error".to_string()),
3584 };
3585 let mut err_req = base_request_frame("cfg-a");
3586 err_req.request_id = Some(7);
3587 let err_resp = round_trip(erroring_dispatcher, &err_req).await;
3588 assert!(!err_resp.ok, "expected a dispatch error: {err_resp:?}");
3589 assert_eq!(
3590 err_resp.request_id,
3591 Some(7),
3592 "request_id must be echoed on the real ops-dispatch error arm"
3593 );
3594 }
3595
3596 #[test]
3605 fn shutdown_cleanup_removes_paths_it_still_owns() {
3606 let dir = tempfile::tempdir().expect("tempdir");
3607 let sock = dir.path().join("khived.sock");
3608 let pid_file = dir.path().join("khived.pid");
3609
3610 let _listener = std::os::unix::net::UnixListener::bind(&sock).expect("bind socket");
3611 std::fs::write(&pid_file, std::process::id().to_string()).expect("write pid file");
3612 let identity = socket_identity(&sock);
3613 assert!(
3614 identity.is_some(),
3615 "must read identity of a freshly bound socket"
3616 );
3617
3618 let cleaned = shutdown_cleanup_if_owned(&sock, &pid_file, identity);
3619
3620 assert!(
3621 cleaned,
3622 "cleanup must proceed when PID and socket still match"
3623 );
3624 assert!(!sock.exists(), "owned socket must be removed");
3625 assert!(!pid_file.exists(), "owned pid file must be removed");
3626 }
3627
3628 #[test]
3629 fn shutdown_cleanup_skips_when_pid_file_names_a_different_process() {
3630 let dir = tempfile::tempdir().expect("tempdir");
3631 let sock = dir.path().join("khived.sock");
3632 let pid_file = dir.path().join("khived.pid");
3633
3634 let _listener = std::os::unix::net::UnixListener::bind(&sock).expect("bind socket");
3635 let identity = socket_identity(&sock);
3636 std::fs::write(&pid_file, "1").expect("write foreign pid file");
3640
3641 let cleaned = shutdown_cleanup_if_owned(&sock, &pid_file, identity);
3642
3643 assert!(
3644 !cleaned,
3645 "cleanup must be skipped when the PID file no longer names this process"
3646 );
3647 assert!(sock.exists(), "replacement daemon's socket must survive");
3648 assert!(
3649 pid_file.exists(),
3650 "replacement daemon's pid file must survive"
3651 );
3652 }
3653
3654 #[test]
3655 fn shutdown_cleanup_skips_when_socket_was_rebound_by_a_replacement() {
3656 let dir = tempfile::tempdir().expect("tempdir");
3657 let sock = dir.path().join("khived.sock");
3658 let original_sock = dir.path().join("original.sock");
3659 let pid_file = dir.path().join("khived.pid");
3660
3661 let _original_listener =
3671 std::os::unix::net::UnixListener::bind(&original_sock).expect("bind original socket");
3672 let _replacement_listener =
3673 std::os::unix::net::UnixListener::bind(&sock).expect("bind replacement socket");
3674
3675 let original_identity = socket_identity(&original_sock);
3676 let replacement_identity = socket_identity(&sock);
3677 assert!(
3678 original_identity.is_some(),
3679 "must read identity of the original socket"
3680 );
3681 assert!(
3682 replacement_identity.is_some(),
3683 "must read identity of the replacement socket"
3684 );
3685 assert!(
3686 original_identity != replacement_identity,
3687 "two concurrently bound sockets must have distinct identities"
3688 );
3689
3690 std::fs::write(&pid_file, std::process::id().to_string())
3691 .expect("write pid file matching this process");
3692
3693 let cleaned = shutdown_cleanup_if_owned(&sock, &pid_file, original_identity);
3698
3699 assert!(
3700 !cleaned,
3701 "cleanup must be skipped when the socket at this path is a different \
3702 inode than the one this daemon originally bound"
3703 );
3704 assert!(sock.exists(), "replacement daemon's socket must survive");
3705 assert!(
3706 pid_file.exists(),
3707 "replacement daemon's pid file must survive"
3708 );
3709 }
3710
3711 #[test]
3723 #[serial]
3724 fn recovery_lock_serializes_two_concurrent_boot_sequences() {
3725 let dir = tempfile::tempdir().expect("tempdir");
3726 let lock_file = dir.path().join("khived.recovery.lock");
3727 std::env::set_var("KHIVE_LOCK", &lock_file);
3728
3729 let active = Arc::new(std::sync::atomic::AtomicUsize::new(0));
3730 let overlap_detected = Arc::new(std::sync::atomic::AtomicBool::new(false));
3731
3732 let run_one_boot =
3733 |active: Arc<std::sync::atomic::AtomicUsize>,
3734 overlap: Arc<std::sync::atomic::AtomicBool>| {
3735 move || {
3736 let _guard = acquire_recovery_lock().expect("acquire recovery lock");
3737 if active.fetch_add(1, std::sync::atomic::Ordering::SeqCst) != 0 {
3739 overlap.store(true, std::sync::atomic::Ordering::SeqCst);
3740 }
3741 std::thread::sleep(std::time::Duration::from_millis(50));
3742 active.fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
3743 }
3745 };
3746
3747 let t1 = std::thread::spawn(run_one_boot(active.clone(), overlap_detected.clone()));
3748 let t2 = std::thread::spawn(run_one_boot(active.clone(), overlap_detected.clone()));
3749 t1.join().expect("boot thread 1 must not panic");
3750 t2.join().expect("boot thread 2 must not panic");
3751
3752 assert!(
3753 !overlap_detected.load(std::sync::atomic::Ordering::SeqCst),
3754 "two concurrent boot sequences must never hold the schema-init \
3755 critical section at the same time (#667)"
3756 );
3757
3758 std::env::remove_var("KHIVE_LOCK");
3759 }
3760
3761 #[test]
3766 #[serial]
3767 fn acquire_daemon_boot_guard_returns_guard_when_lock_available() {
3768 let dir = tempfile::tempdir().expect("tempdir");
3769 let lock_file = dir.path().join("khived.recovery.lock");
3770 std::env::set_var("KHIVE_LOCK", &lock_file);
3771
3772 let guard = acquire_daemon_boot_guard();
3773 assert!(
3774 guard.is_ok(),
3775 "daemon boot guard must succeed when the lock file can be opened and flocked"
3776 );
3777 drop(guard);
3778
3779 std::env::remove_var("KHIVE_LOCK");
3780 }
3781
3782 #[test]
3783 #[serial]
3784 fn acquire_daemon_boot_guard_fails_loudly_when_lock_file_cannot_be_opened() {
3785 let dir = tempfile::tempdir().expect("tempdir");
3786 std::env::set_var("KHIVE_LOCK", dir.path());
3792
3793 let result = acquire_daemon_boot_guard();
3794 assert!(
3795 result.is_err(),
3796 "daemon boot guard must fail loudly, never silently proceed unguarded, \
3797 when the underlying recovery lock cannot be acquired"
3798 );
3799
3800 std::env::remove_var("KHIVE_LOCK");
3801 }
3802
3803 #[test]
3806 fn write_pid_file_exclusive_creates_new_file_with_own_pid() {
3807 let dir = tempfile::tempdir().expect("tempdir");
3808 let pid_file = dir.path().join("khived.pid");
3809 write_pid_file_exclusive(&pid_file).expect("first writer must win");
3810 let contents = std::fs::read_to_string(&pid_file).expect("read pid file");
3811 assert_eq!(contents, std::process::id().to_string());
3812 }
3813
3814 #[test]
3815 fn write_pid_file_exclusive_refuses_to_overwrite_an_existing_file() {
3816 let dir = tempfile::tempdir().expect("tempdir");
3817 let pid_file = dir.path().join("khived.pid");
3818 std::fs::write(&pid_file, "999999").expect("seed an existing pid file");
3819
3820 let err = write_pid_file_exclusive(&pid_file)
3821 .expect_err("must not silently overwrite an existing pid file");
3822 assert_eq!(err.kind(), std::io::ErrorKind::AlreadyExists);
3823
3824 let contents = std::fs::read_to_string(&pid_file).expect("read pid file");
3827 assert_eq!(
3828 contents, "999999",
3829 "an existing pid file must never be truncated by a losing writer"
3830 );
3831 }
3832
3833 #[test]
3842 fn two_concurrent_writers_converge_on_exactly_one_pid_file_owner() {
3843 let dir = tempfile::tempdir().expect("tempdir");
3844 let pid_file = std::sync::Arc::new(dir.path().join("khived.pid"));
3845 let barrier = std::sync::Arc::new(std::sync::Barrier::new(2));
3846
3847 let spawn_writer =
3848 |pid_file: std::sync::Arc<std::path::PathBuf>,
3849 barrier: std::sync::Arc<std::sync::Barrier>| {
3850 std::thread::spawn(move || {
3851 barrier.wait();
3852 write_pid_file_exclusive(&pid_file)
3853 })
3854 };
3855
3856 let t1 = spawn_writer(pid_file.clone(), barrier.clone());
3857 let t2 = spawn_writer(pid_file.clone(), barrier.clone());
3858 let r1 = t1.join().expect("writer 1 must not panic");
3859 let r2 = t2.join().expect("writer 2 must not panic");
3860
3861 let results = [&r1, &r2];
3862 let ok_count = results.iter().filter(|r| r.is_ok()).count();
3863 let already_exists_count = results
3864 .iter()
3865 .filter(|r| matches!(r, Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists))
3866 .count();
3867 assert_eq!(
3868 ok_count, 1,
3869 "exactly one of two concurrent writers must win the pid file"
3870 );
3871 assert_eq!(
3872 already_exists_count, 1,
3873 "the other writer must observe AlreadyExists, never a silent overwrite"
3874 );
3875 assert!(pid_file.exists(), "the winner's pid file must exist");
3876 let contents = std::fs::read_to_string(&*pid_file).expect("read pid file");
3877 assert_eq!(
3878 contents,
3879 std::process::id().to_string(),
3880 "the surviving pid file must contain the winner's pid — both threads \
3881 share this process's pid, so an unexpected value would also prove a \
3882 lost/garbled write raced through"
3883 );
3884 }
3885
3886 #[tokio::test]
3897 async fn peer_uid_reports_the_connecting_process_uid() {
3898 let dir = tempfile::tempdir().expect("tempdir");
3899 let sock = dir.path().join("peer.sock");
3900 let listener = UnixListener::bind(&sock).expect("bind");
3901
3902 let connect_path = sock.clone();
3903 let client = tokio::spawn(async move { UnixStream::connect(&connect_path).await });
3904
3905 let (server_side, _) = listener.accept().await.expect("accept");
3906 let client_side = client.await.expect("join").expect("connect");
3907
3908 let expected = unsafe { libc::geteuid() } as u32;
3910
3911 assert_eq!(
3912 peer_uid(&server_side).expect("peer_uid must succeed on a live connection"),
3913 expected,
3914 "the uid read from the kernel for a same-process connection must be \
3915 this process's euid"
3916 );
3917 assert_eq!(
3919 peer_uid(&client_side).expect("peer_uid must succeed on the client end"),
3920 expected
3921 );
3922 }
3923
3924 #[test]
3934 fn only_a_foreign_uid_is_refused() {
3935 let euid = unsafe { libc::geteuid() } as u32;
3937
3938 assert!(
3939 uid_is_permitted(euid, euid),
3940 "a connection from the daemon's own uid must be served — this is \
3941 every seat on the host, and ADR-096 accepted exactly this shape"
3942 );
3943 assert!(
3944 !uid_is_permitted(euid.wrapping_add(1), euid),
3945 "a connection from any other uid must be refused"
3946 );
3947 assert!(
3948 !uid_is_permitted(0, euid.wrapping_add(1)),
3949 "root is not special-cased: the rule is equality with the daemon's \
3950 euid, not a privilege comparison"
3951 );
3952 }
3953}