1use std::sync::Arc;
12
13mod locks;
14#[cfg(all(test, unix))]
15use locks::open_lock_file;
16#[cfg(unix)]
17pub use locks::{
18 acquire_daemon_boot_guard, acquire_recovery_lock, try_acquire_daemon_boot_guard_until,
19 try_acquire_recoverer_lock_until, DaemonBootGuard,
20};
21#[cfg(unix)]
22pub(crate) use locks::{peer_uid, uid_is_permitted};
23#[cfg(unix)]
24use locks::{socket_identity, SocketIdentity};
25
26mod store_guard;
27mod store_identity;
28#[cfg(unix)]
29use store_guard::ensure_claimed_parent_identity;
30#[cfg(unix)]
31pub use store_guard::{acquire_daemon_store_guards, bind_daemon_store_files, claim_stores};
32pub use store_guard::{assert_daemon_store_identities, DaemonStoreGuard};
33#[cfg(unix)]
34pub use store_identity::claimed_daemon_store_identity;
35mod supervisor_marker;
36#[cfg(unix)]
37pub use supervisor_marker::supervisor_marker_path;
38
39#[cfg(unix)]
40use std::io::Write as _;
41#[cfg(unix)]
42use std::os::unix::fs::{MetadataExt, PermissionsExt};
43#[cfg(unix)]
44use std::os::unix::io::AsRawFd;
45#[cfg(all(test, unix))]
46use std::path::PathBuf;
47
48#[cfg(unix)]
49use async_trait::async_trait;
50#[cfg(unix)]
51use libc;
52#[cfg(unix)]
53use serde::Deserialize;
54#[cfg(unix)]
55use tokio::io::AsyncReadExt;
56#[cfg(all(test, unix))]
57use tokio::io::AsyncWriteExt;
58#[cfg(unix)]
59use tokio::net::{UnixListener, UnixStream};
60
61#[cfg(unix)]
62use crate::pack::RequestIdentity;
63#[cfg(unix)]
64use khive_db::{run_checkpoint_task, CheckpointConfig, CheckpointLifecycleOwner, ConnectionPool};
65
66mod exit_record;
69mod lifecycle;
70#[cfg(unix)]
71use lifecycle::khive_dir;
72#[cfg(test)]
73use lifecycle::khive_root_from;
74#[cfg(unix)]
75use lifecycle::{
76 current_supervisor_claim, default_socket_path, ensure_rendezvous_overrides_paired,
77 read_supervisor_marker_claim, DaemonLifecycle, PID_PATH_ENV, SOCKET_PATH_ENV,
78};
79pub use lifecycle::{
80 lock_path, volume_lock_dir, DaemonLifecyclePhase, DaemonLifecycleSnapshot, DaemonLifetime,
81 DaemonOptions, DaemonShutdownReason, DaemonStartupReport, DEFAULT_DEMAND_IDLE_SECS,
82};
83#[cfg(unix)]
84pub use lifecycle::{
85 pid_path, recoverer_lock_path, rendezvous_split, socket_path, RendezvousSplit,
86 SUPERVISOR_CLAIM_ENV,
87};
88
89mod load_limits;
90#[cfg(unix)]
91use load_limits::{admit_or_refuse_busy, ConnectionAdmission};
92pub use load_limits::{
93 recall_ledger_snapshot, track_recall_ledger_task, ConnectionCapSnapshot, RecallLedgerSnapshot,
94};
95
96pub const MAX_FRAME_BYTES: usize = 8 * 1024 * 1024;
98
99pub const PROTOCOL_VERSION: u32 = 8;
108
109#[doc(hidden)]
113pub const DAEMON_LEXICAL_TIMEOUT_MARKER: &str = "__khive_daemon_lexical_timeout";
114
115const DEFAULT_DRAIN_TIMEOUT_SECS: u64 = 10;
116#[cfg(unix)]
120const INITIAL_FRAME_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30);
121
122#[cfg(unix)]
123fn next_accept_error_backoff(previous: Option<std::time::Duration>) -> std::time::Duration {
124 previous
125 .map(|delay| delay.saturating_mul(2))
126 .unwrap_or_else(|| std::time::Duration::from_millis(10))
127 .min(std::time::Duration::from_secs(1))
128}
129
130mod config_id;
133#[cfg(test)]
134use config_id::parse_config_id;
135pub use config_id::{
136 config_id_extra_embedder_exclusions, config_ids_compatible, first_config_mismatch_field,
137};
138
139mod wire_frames;
140#[cfg(unix)]
141pub use wire_frames::{read_frame, write_frame};
142#[cfg(unix)]
143use wire_frames::{read_initial_frame, take_daemon_lexical_timeout_marker};
144pub use wire_frames::{
145 CheckpointStoreMetrics, DaemonDispatchError, DaemonRequestFrame, DaemonResponseFrame,
146 MetricsSnapshot, ERROR_DETAIL_NESTING_DEPTH_LIMIT,
147};
148
149#[cfg(unix)]
158#[async_trait]
159pub trait DaemonDispatch: Clone + Send + Sync + 'static {
160 fn idle_retirement_blockers(&self) -> Vec<String> {
163 vec!["dispatcher_resource_inventory_unknown".to_owned()]
164 }
165
166 fn plan(&self, ops: &str) -> String;
168
169 #[allow(clippy::too_many_arguments)]
184 async fn dispatch(
185 &self,
186 ops: String,
187 presentation: Option<String>,
188 presentation_per_op: Option<Vec<Option<String>>>,
189 format: Option<String>,
190 format_per_op: Option<Vec<Option<String>>>,
191 from_wire: bool,
192 identity: Option<RequestIdentity>,
193 ) -> Result<String, String>;
194
195 fn request_read_timeout(&self, _ops: &str) -> std::time::Duration {
199 khive_storage::request_read_timeout_from_env()
200 }
201
202 #[allow(clippy::too_many_arguments)]
204 async fn dispatch_with_error_detail(
205 &self,
206 ops: String,
207 presentation: Option<String>,
208 presentation_per_op: Option<Vec<Option<String>>>,
209 format: Option<String>,
210 format_per_op: Option<Vec<Option<String>>>,
211 from_wire: bool,
212 identity: Option<RequestIdentity>,
213 ) -> Result<String, DaemonDispatchError> {
214 self.dispatch(
215 ops,
216 presentation,
217 presentation_per_op,
218 format,
219 format_per_op,
220 from_wire,
221 identity,
222 )
223 .await
224 .map_err(|message| DaemonDispatchError::new(message, None))
225 }
226
227 fn events_readiness(&self) -> Option<crate::events_split::EventsReadiness> {
233 None
234 }
235
236 async fn warm_all(&self);
238
239 fn namespace(&self) -> &str;
241
242 fn config_id(&self) -> &str;
247
248 fn pool_for_checkpoint(&self) -> Option<Arc<ConnectionPool>> {
256 None
257 }
258
259 fn secondary_pools_for_checkpoint(&self) -> Vec<Arc<ConnectionPool>> {
269 Vec::new()
270 }
271
272 fn event_store_for_checkpoint(&self) -> Option<Arc<dyn khive_storage::EventStore>> {
282 None
283 }
284}
285
286#[cfg(unix)]
287struct CheckpointTaskSpec {
288 pool: Arc<ConnectionPool>,
289 lifecycle_owner: Option<CheckpointLifecycleOwner>,
290 is_main: bool,
291}
292
293#[cfg(unix)]
300fn checkpoint_task_specs(
301 main_pool: Option<Arc<ConnectionPool>>,
302 secondary_pools: Vec<Arc<ConnectionPool>>,
303 event_store: Option<Arc<dyn khive_storage::EventStore>>,
304 namespace: String,
305) -> Vec<CheckpointTaskSpec> {
306 let mut tasks = Vec::with_capacity(usize::from(main_pool.is_some()) + secondary_pools.len());
307 if let Some(pool) = main_pool {
308 tasks.push(CheckpointTaskSpec {
309 pool,
310 lifecycle_owner: None,
311 is_main: true,
312 });
313 }
314 tasks.extend(secondary_pools.into_iter().map(|pool| CheckpointTaskSpec {
315 pool,
316 lifecycle_owner: None,
317 is_main: false,
318 }));
319
320 if let (Some(task), Some(event_store)) = (tasks.first_mut(), event_store) {
321 task.lifecycle_owner = Some(CheckpointLifecycleOwner::new(event_store, namespace));
322 }
323 tasks
324}
325
326static WARM_INDEX_HOST: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
340
341pub fn mark_warm_index_host() {
345 WARM_INDEX_HOST.store(true, std::sync::atomic::Ordering::Release);
346}
347
348pub fn is_warm_index_host() -> bool {
357 WARM_INDEX_HOST.load(std::sync::atomic::Ordering::Acquire)
358}
359
360static BACKGROUND_TASKS: std::sync::OnceLock<Arc<std::sync::atomic::AtomicUsize>> =
361 std::sync::OnceLock::new();
362
363fn background_tasks() -> &'static Arc<std::sync::atomic::AtomicUsize> {
364 BACKGROUND_TASKS.get_or_init(|| Arc::new(std::sync::atomic::AtomicUsize::new(0)))
365}
366
367static BACKGROUND_TASK_NAMES: std::sync::OnceLock<
383 std::sync::Mutex<std::collections::HashMap<&'static str, usize>>,
384> = std::sync::OnceLock::new();
385
386fn background_task_names_registry(
387) -> &'static std::sync::Mutex<std::collections::HashMap<&'static str, usize>> {
388 BACKGROUND_TASK_NAMES.get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new()))
389}
390
391pub const UNNAMED_BACKGROUND_TASK: &str = "unnamed";
394
395fn register_background_task_name(name: &'static str) {
396 let mut names = background_task_names_registry()
397 .lock()
398 .unwrap_or_else(std::sync::PoisonError::into_inner);
399 *names.entry(name).or_insert(0) += 1;
400}
401
402fn release_background_task_name(name: &'static str) {
403 let mut names = background_task_names_registry()
404 .lock()
405 .unwrap_or_else(std::sync::PoisonError::into_inner);
406 if let Some(count) = names.get_mut(name) {
407 *count -= 1;
408 if *count == 0 {
409 names.remove(name);
410 }
411 }
412}
413
414pub fn background_task_names() -> Vec<String> {
418 let names = background_task_names_registry()
419 .lock()
420 .unwrap_or_else(std::sync::PoisonError::into_inner);
421 let mut out: Vec<String> = names.keys().map(|name| (*name).to_string()).collect();
422 out.sort();
423 out
424}
425
426#[cfg(unix)]
427fn idle_retirement_blockers<D: DaemonDispatch>(dispatcher: &D) -> Vec<String> {
428 let mut blockers = dispatcher.idle_retirement_blockers();
429 if !khive_storage::tx_registry::snapshot().is_empty() {
430 blockers.push("open_sql_transaction".to_owned());
431 }
432 blockers.extend(
433 active_phase_names()
434 .into_iter()
435 .map(|name| format!("active_phase:{name}")),
436 );
437 let count = background_task_count();
438 let names = background_task_names_registry()
439 .lock()
440 .unwrap_or_else(std::sync::PoisonError::into_inner);
441 if names.values().sum::<usize>() != count {
442 blockers.push("tracked_worker_inventory_unsettled".to_owned());
443 }
444 for name in names.keys() {
447 if !matches!(
448 *name,
449 "wal_checkpoint" | "memory_ann_rotation_watch" | "knowledge_ann_rotation_watch"
450 ) {
451 blockers.push(format!("unsettled_worker:{name}"));
452 }
453 }
454 blockers.sort();
455 blockers.dedup();
456 blockers
457}
458
459#[cfg(unix)]
460async fn wait_for_idle<D: DaemonDispatch>(dispatcher: &D, lifecycle: &DaemonLifecycle) {
461 if lifecycle.options.lifetime == DaemonLifetime::Persistent {
462 std::future::pending::<()>().await;
463 }
464 loop {
465 if lifecycle.try_idle(|| idle_retirement_blockers(dispatcher)) {
466 return;
467 }
468 tokio::time::sleep(std::time::Duration::from_millis(100)).await;
469 }
470}
471
472struct BackgroundTaskGuard {
473 counter: Arc<std::sync::atomic::AtomicUsize>,
474 name: &'static str,
475}
476
477impl Drop for BackgroundTaskGuard {
478 fn drop(&mut self) {
479 release_background_task_name(self.name);
480 self.counter
481 .fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
482 }
483}
484
485pub fn spawn_tracked_task<F, T>(fut: F) -> tokio::task::JoinHandle<T>
491where
492 F: std::future::Future<Output = T> + Send + 'static,
493 T: Send + 'static,
494{
495 spawn_named_tracked_task(UNNAMED_BACKGROUND_TASK, fut)
496}
497
498pub fn spawn_named_tracked_task<F, T>(name: &'static str, fut: F) -> tokio::task::JoinHandle<T>
505where
506 F: std::future::Future<Output = T> + Send + 'static,
507 T: Send + 'static,
508{
509 background_tasks().fetch_add(1, std::sync::atomic::Ordering::SeqCst);
510 register_background_task_name(name);
511 let guard = BackgroundTaskGuard {
512 counter: background_tasks().clone(),
513 name,
514 };
515 tokio::spawn(async move {
516 let _guard = guard;
517 fut.await
518 })
519}
520
521pub fn track_background_task<F>(fut: F)
526where
527 F: std::future::Future<Output = ()> + Send + 'static,
528{
529 track_named_background_task(UNNAMED_BACKGROUND_TASK, fut);
530}
531
532pub fn track_named_background_task<F>(name: &'static str, fut: F)
535where
536 F: std::future::Future<Output = ()> + Send + 'static,
537{
538 drop(spawn_named_tracked_task(name, fut));
539}
540
541pub fn background_task_count() -> usize {
544 background_tasks().load(std::sync::atomic::Ordering::SeqCst)
545}
546
547pub fn daemon_shutdown_token() -> tokio_util::sync::CancellationToken {
558 static TOKEN: std::sync::OnceLock<tokio_util::sync::CancellationToken> =
559 std::sync::OnceLock::new();
560 TOKEN
561 .get_or_init(tokio_util::sync::CancellationToken::new)
562 .clone()
563}
564
565static ACTIVE_PHASES: std::sync::OnceLock<
576 std::sync::Mutex<std::collections::HashMap<String, usize>>,
577> = std::sync::OnceLock::new();
578
579fn active_phases() -> &'static std::sync::Mutex<std::collections::HashMap<String, usize>> {
580 ACTIVE_PHASES.get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new()))
581}
582
583pub struct PhaseGuard {
589 name: String,
590}
591
592impl Drop for PhaseGuard {
593 fn drop(&mut self) {
594 let mut map = active_phases()
595 .lock()
596 .unwrap_or_else(std::sync::PoisonError::into_inner);
597 if let Some(count) = map.get_mut(&self.name) {
598 *count -= 1;
599 if *count == 0 {
600 map.remove(&self.name);
601 }
602 }
603 }
604}
605
606pub fn register_active_phase(name: &str) -> PhaseGuard {
611 let mut map = active_phases()
612 .lock()
613 .unwrap_or_else(std::sync::PoisonError::into_inner);
614 *map.entry(name.to_string()).or_insert(0) += 1;
615 PhaseGuard {
616 name: name.to_string(),
617 }
618}
619
620pub fn active_phase_names() -> Vec<String> {
623 let map = active_phases()
624 .lock()
625 .unwrap_or_else(std::sync::PoisonError::into_inner);
626 let mut names: Vec<String> = map.keys().cloned().collect();
627 names.sort();
628 names
629}
630
631#[cfg(unix)]
639fn build_metrics_snapshot<D: DaemonDispatch>(dispatcher: &D) -> MetricsSnapshot {
640 let open_tx_count = khive_storage::tx_registry::snapshot().len();
641 let (oldest_pinned_tx_micros, oldest_pinned_tx_label) =
648 match khive_storage::tx_registry::oldest() {
649 Some((_id, age, label)) => (Some(age.as_micros() as u64), label),
650 None => (None, None),
651 };
652
653 let checkpoint_pool = dispatcher.pool_for_checkpoint();
654 let mut secondary_index = 0;
655 let wal_checkpoint_stores = checkpoint_task_specs(
656 checkpoint_pool.clone(),
657 dispatcher.secondary_pools_for_checkpoint(),
658 None,
659 String::new(),
660 )
661 .into_iter()
662 .map(|task| {
663 let (store_id, role) = if task.is_main {
664 ("main".to_string(), "main".to_string())
665 } else {
666 let store_id = format!("secondary:{secondary_index}");
667 secondary_index += 1;
668 (store_id, "secondary".to_string())
669 };
670 CheckpointStoreMetrics {
671 store_id,
672 role,
673 database: task
674 .pool
675 .canonical_path()
676 .and_then(std::path::Path::file_name)
677 .map(|name| name.to_string_lossy().into_owned()),
678 timing: khive_db::checkpoint::checkpoint_timing(&task.pool),
679 }
680 })
681 .collect();
682 let routine_wal = checkpoint_pool
683 .as_deref()
684 .and_then(khive_db::checkpoint::routine_wal_observation);
685 let writer_stages = checkpoint_pool
686 .as_deref()
687 .and_then(khive_db::writer_task::last_writer_stage_observation);
688 let (write_queue_depth, write_queue_capacity) = checkpoint_pool
689 .as_ref()
690 .and_then(|pool| pool.writer_task_handle().ok().flatten())
691 .map(|handle| (Some(handle.queue_depth()), Some(handle.capacity())))
692 .unwrap_or((None, None));
693
694 MetricsSnapshot {
695 lifecycle: None,
696 wal_pages: routine_wal.as_ref().map(|sample| sample.log_frames),
697 wal_log_frames: routine_wal.as_ref().map(|sample| sample.log_frames),
698 wal_checkpointed_frames: routine_wal
699 .as_ref()
700 .map(|sample| sample.checkpointed_frames),
701 wal_pending_frames: routine_wal.as_ref().map(|sample| sample.pending_frames),
702 wal_physical_bytes: routine_wal
703 .as_ref()
704 .and_then(|sample| sample.physical_wal_bytes),
705 wal_observed_at_unix_ms: routine_wal
706 .as_ref()
707 .map(|sample| sample.observed_at_unix_ms),
708 wal_checkpoint_stores,
709 wal_truncate_attempts: khive_db::checkpoint::truncate_attempts(),
710 wal_truncate_consecutive_failures: khive_db::checkpoint::truncate_consecutive_failures(),
711 wal_checkpoint_skipped_ticks: khive_db::checkpoint::checkpoint_skipped_ticks(),
712 wal_checkpoint_consecutive_skips: khive_db::checkpoint::checkpoint_consecutive_skips(),
713 wal_checkpoint_last_skip_wal_pages: khive_db::checkpoint::checkpoint_last_skip_wal_pages(),
714 oldest_pinned_tx_micros,
715 oldest_pinned_tx_label,
716 open_tx_count,
717 write_queue_depth,
718 write_queue_capacity,
719 write_last_queue_wait_micros: writer_stages
720 .as_ref()
721 .map(|sample| sample.queue_wait_micros),
722 write_last_transaction_acquire_micros: writer_stages
723 .as_ref()
724 .map(|sample| sample.transaction_acquire_micros),
725 write_last_body_micros: writer_stages.as_ref().map(|sample| sample.body_micros),
726 write_last_commit_micros: writer_stages.as_ref().map(|sample| sample.commit_micros),
727 write_last_total_micros: writer_stages.as_ref().map(|sample| sample.total_micros),
728 write_last_observed_at_unix_ms: writer_stages
729 .as_ref()
730 .map(|sample| sample.observed_at_unix_ms),
731 connections: None,
732 recall_ledger: Some(recall_ledger_snapshot()),
733 }
734}
735
736#[cfg(unix)]
737async fn write_response_frame<W>(stream: &mut W, payload: &[u8]) -> std::io::Result<()>
738where
739 W: tokio::io::AsyncWrite + Unpin,
740{
741 tokio::time::timeout(INITIAL_FRAME_READ_TIMEOUT, write_frame(stream, payload))
742 .await
743 .map_err(|_| {
744 std::io::Error::new(
745 std::io::ErrorKind::TimedOut,
746 "daemon response write timed out",
747 )
748 })?
749}
750
751#[cfg(unix)]
752async fn wait_for_peer_disconnect(read: &mut tokio::net::unix::OwnedReadHalf) {
753 let mut byte = [0u8; 1];
754 let _ = read.read(&mut byte).await;
758}
759
760#[cfg(all(unix, test))]
761async fn handle_conn<D: DaemonDispatch>(stream: UnixStream, dispatcher: D) {
762 handle_conn_with_shutdown(
763 stream,
764 dispatcher,
765 None,
766 tokio::time::Instant::now() + INITIAL_FRAME_READ_TIMEOUT,
767 )
768 .await;
769}
770
771#[cfg(all(unix, feature = "fault-injection"))]
772#[doc(hidden)]
773pub async fn handle_conn_for_test<D: DaemonDispatch>(stream: UnixStream, dispatcher: D) {
774 handle_conn_with_shutdown(
775 stream,
776 dispatcher,
777 None,
778 tokio::time::Instant::now() + INITIAL_FRAME_READ_TIMEOUT,
779 )
780 .await;
781}
782
783#[cfg(unix)]
784fn plan_frame_companion(raw: &[u8]) -> Option<&'static str> {
785 let value: serde_json::Value = serde_json::from_slice(raw).ok()?;
786 if value.get("plan").and_then(serde_json::Value::as_bool) != Some(true)
787 || value
788 .get("protocol_version")
789 .and_then(serde_json::Value::as_u64)
790 != Some(u64::from(PROTOCOL_VERSION))
791 {
792 return None;
793 }
794 [
795 "presentation",
796 "presentation_per_op",
797 "format",
798 "format_per_op",
799 "request_id",
800 ]
801 .into_iter()
802 .find(|field| value.get(*field).is_some())
803}
804
805#[cfg(all(
806 unix,
807 any(test, feature = "fault-injection", feature = "test-internals")
808))]
809async fn handle_conn_with_shutdown<D: DaemonDispatch>(
810 stream: UnixStream,
811 dispatcher: D,
812 shutdown: Option<tokio::sync::watch::Receiver<bool>>,
813 initial_frame_deadline: tokio::time::Instant,
814) {
815 handle_conn_with_lifecycle(stream, dispatcher, shutdown, initial_frame_deadline, None).await;
816}
817
818#[cfg(unix)]
819async fn handle_conn_with_lifecycle<D: DaemonDispatch>(
820 mut stream: UnixStream,
821 dispatcher: D,
822 shutdown: Option<tokio::sync::watch::Receiver<bool>>,
823 initial_frame_deadline: tokio::time::Instant,
824 lifecycle: Option<Arc<DaemonLifecycle>>,
825) {
826 let mut ordinary_admission = None;
827 let production_shutdown = shutdown.is_some();
828 let handover_peer_allowed = peer_uid(&stream)
832 .ok()
833 .is_some_and(|uid| uid == unsafe { libc::geteuid() } as u32);
834 let (local_shutdown_tx, local_shutdown_rx) = tokio::sync::watch::channel(false);
835 let shutdown = shutdown.unwrap_or(local_shutdown_rx);
836 let _local_shutdown_tx = local_shutdown_tx;
839 let raw = match read_initial_frame(&mut stream, initial_frame_deadline).await {
840 Ok(r) => r,
841 Err(e) => {
842 tracing::debug!(error = %e, "failed to read daemon request frame");
843 return;
844 }
845 };
846 #[derive(Deserialize)]
847 struct SupervisorRequestEnvelope {
848 #[serde(flatten)]
849 frame: DaemonRequestFrame,
850 #[serde(default)]
851 supervisor_handover: bool,
852 }
853 let decoded: Result<SupervisorRequestEnvelope, _> = serde_json::from_slice(&raw);
854 if decoded.as_ref().ok().is_none_or(|item| item.frame.plan) {
855 if let Some(field) = plan_frame_companion(&raw) {
856 let response = DaemonResponseFrame {
857 ok: false,
858 result: None,
859 error: Some(format!(
860 "invalid_params: plan=true cannot be combined with {field}"
861 )),
862 error_detail: Some(serde_json::json!({
863 "kind": "protocol",
864 "code": "invalid_params",
865 "message": format!("plan=true cannot be combined with {field}"),
866 "domain_disposition": crate::DomainDisposition::NotCommitted.as_str(),
867 })),
868 namespace_mismatch: false,
869 config_mismatch: false,
870 served_config_id: Some(dispatcher.config_id().to_string()),
871 version_mismatch: false,
872 daemon_protocol_version: PROTOCOL_VERSION,
873 metrics: None,
874 request_id: None,
875 };
876 if let Ok(payload) = serde_json::to_vec(&response) {
877 if let Err(error) = write_response_frame(&mut stream, &payload).await {
878 tracing::debug!(%error, "failed to write plan envelope refusal");
879 }
880 }
881 return;
882 }
883 }
884 let (frame, handover_requested) = match decoded {
885 Ok(item) => (item.frame, item.supervisor_handover),
886 Err(e) => {
887 tracing::debug!(error = %e, "failed to decode daemon request frame");
888 return;
889 }
890 };
891 let supervisor_probe = frame.probe_only;
892 let handover_accepted = handover_requested
893 && supervisor_probe
894 && production_shutdown
895 && handover_peer_allowed
896 && frame.protocol_version == PROTOCOL_VERSION
897 && !frame.plan
898 && !frame.metrics_only
899 && frame.ops.is_empty()
900 && read_supervisor_marker_claim().is_some_and(|(pid, _)| pid != std::process::id());
901 let (mut peer_read, mut peer_write) = stream.into_split();
902
903 let served_config_id = Some(dispatcher.config_id().to_string());
904 let resp = if frame.protocol_version != PROTOCOL_VERSION {
905 let msg = format!(
906 "daemon protocol mismatch: client={} daemon={} — \
907 rebuild/update the client binary (make local)",
908 frame.protocol_version, PROTOCOL_VERSION,
909 );
910 tracing::warn!(
911 client_version = frame.protocol_version,
912 daemon_version = PROTOCOL_VERSION,
913 "daemon protocol version mismatch"
914 );
915 DaemonResponseFrame {
916 ok: false,
917 result: None,
918 error: Some(msg.clone()),
919 error_detail: Some(serde_json::json!({
920 "kind": "protocol",
921 "code": "version_mismatch",
922 "message": msg,
923 "domain_disposition": crate::DomainDisposition::Unknown.as_str(),
924 })),
925 namespace_mismatch: false,
926 config_mismatch: false,
927 served_config_id,
928 version_mismatch: frame.protocol_version > PROTOCOL_VERSION,
943 daemon_protocol_version: PROTOCOL_VERSION,
944 metrics: None,
945 request_id: frame.request_id,
946 }
947 } else if handover_requested {
948 if handover_accepted {
949 DaemonResponseFrame {
950 ok: true,
951 result: None,
952 error: None,
953 error_detail: None,
954 namespace_mismatch: false,
955 config_mismatch: false,
956 served_config_id,
957 version_mismatch: false,
958 daemon_protocol_version: PROTOCOL_VERSION,
959 metrics: None,
960 request_id: frame.request_id,
961 }
962 } else {
963 DaemonResponseFrame {
964 ok: false,
965 result: None,
966 error: Some("supervisor handover refused".to_string()),
967 error_detail: Some(serde_json::json!({
968 "kind": "protocol",
969 "code": "supervisor_handover_refused",
970 "message": "supervisor handover refused",
971 "domain_disposition": crate::DomainDisposition::NotCommitted.as_str(),
972 })),
973 namespace_mismatch: false,
974 config_mismatch: false,
975 served_config_id,
976 version_mismatch: false,
977 daemon_protocol_version: PROTOCOL_VERSION,
978 metrics: None,
979 request_id: frame.request_id,
980 }
981 }
982 } else if frame.metrics_only && !frame.plan {
983 DaemonResponseFrame {
990 ok: true,
991 result: None,
992 error: None,
993 error_detail: None,
994 namespace_mismatch: false,
995 config_mismatch: false,
996 served_config_id,
997 version_mismatch: false,
998 daemon_protocol_version: PROTOCOL_VERSION,
999 metrics: Some({
1000 let mut metrics = build_metrics_snapshot(&dispatcher);
1001 metrics.lifecycle = lifecycle.as_ref().map(|state| {
1002 let mut snapshot = state.snapshot();
1003 snapshot.idle_blockers = idle_retirement_blockers(&dispatcher);
1004 snapshot
1005 });
1006 metrics.connections = lifecycle.as_ref().map(|state| state.connections.snapshot());
1007 metrics
1008 }),
1009 request_id: frame.request_id,
1010 }
1011 } else if !config_ids_compatible(&frame.config_id, dispatcher.config_id()) {
1020 DaemonResponseFrame {
1021 ok: false,
1022 result: None,
1023 error: None,
1024 error_detail: Some(serde_json::json!({
1025 "kind": "protocol",
1026 "code": "config_mismatch",
1027 "message": "daemon configuration does not match the request",
1028 "domain_disposition": crate::DomainDisposition::NotCommitted.as_str(),
1029 })),
1030 namespace_mismatch: false,
1031 config_mismatch: true,
1032 served_config_id,
1033 version_mismatch: false,
1034 daemon_protocol_version: PROTOCOL_VERSION,
1035 metrics: None,
1036 request_id: frame.request_id,
1037 }
1038 } else if frame.plan {
1039 DaemonResponseFrame {
1040 ok: true,
1041 result: Some(dispatcher.plan(&frame.ops)),
1042 error: None,
1043 error_detail: None,
1044 namespace_mismatch: false,
1045 config_mismatch: false,
1046 served_config_id,
1047 version_mismatch: false,
1048 daemon_protocol_version: PROTOCOL_VERSION,
1049 metrics: None,
1050 request_id: None,
1051 }
1052 } else if frame.probe_only {
1053 DaemonResponseFrame {
1057 ok: true,
1058 result: None,
1059 error: None,
1060 error_detail: None,
1061 namespace_mismatch: false,
1062 config_mismatch: false,
1063 served_config_id,
1064 version_mismatch: false,
1065 daemon_protocol_version: PROTOCOL_VERSION,
1066 metrics: None,
1067 request_id: frame.request_id,
1068 }
1069 } else {
1070 if let Some(lifecycle) = &lifecycle {
1071 ordinary_admission = lifecycle.admit();
1072 if ordinary_admission.is_none() {
1073 let refusal = DaemonResponseFrame {
1074 ok: false,
1075 result: None,
1076 error: Some("daemon is draining; request was not admitted".to_owned()),
1077 error_detail: Some(serde_json::json!({
1078 "kind": "runtime", "code": "daemon_draining",
1079 "domain_disposition": crate::DomainDisposition::NotCommitted.as_str(),
1080 })),
1081 request_id: frame.request_id,
1082 daemon_protocol_version: PROTOCOL_VERSION,
1083 namespace_mismatch: false,
1084 config_mismatch: false,
1085 served_config_id: Some(dispatcher.config_id().to_owned()),
1086 version_mismatch: false,
1087 metrics: None,
1088 };
1089 if let Ok(payload) = serde_json::to_vec(&refusal) {
1090 let _ = write_response_frame(&mut peer_write, &payload).await;
1091 }
1092 return;
1093 }
1094 }
1095 if let Some(readiness) = dispatcher.events_readiness() {
1100 let not_ready = tokio::select! {
1101 outcome = readiness.wait() => outcome.err(),
1102 _ = wait_for_peer_disconnect(&mut peer_read) => return,
1103 };
1104 if let Some(not_ready) = not_ready {
1105 let message = not_ready.to_string();
1106 tracing::warn!(
1107 socket = %not_ready.socket.display(),
1108 reason = ?not_ready.reason,
1109 request_id = frame.request_id,
1110 "request refused: the supervised events daemon is not answering"
1111 );
1112 let refusal = DaemonResponseFrame {
1113 ok: false,
1114 result: None,
1115 error: Some(message.clone()),
1116 error_detail: Some(serde_json::json!({
1117 "kind": "runtime",
1118 "code": "events_daemon_unavailable",
1119 "message": message,
1120 "retryable": true,
1121 "socket": not_ready.socket.display().to_string(),
1122 "domain_disposition": crate::DomainDisposition::NotCommitted.as_str(),
1123 })),
1124 request_id: frame.request_id,
1125 daemon_protocol_version: PROTOCOL_VERSION,
1126 namespace_mismatch: false,
1127 config_mismatch: false,
1128 served_config_id: Some(dispatcher.config_id().to_owned()),
1129 version_mismatch: false,
1130 metrics: None,
1131 };
1132 if let Ok(payload) = serde_json::to_vec(&refusal) {
1133 let _ = write_response_frame(&mut peer_write, &payload).await;
1134 }
1135 return;
1136 }
1137 }
1138 let identity = RequestIdentity {
1150 namespace: frame.namespace.clone(),
1151 actor_id: frame.actor_id.clone(),
1152 visible_namespaces: frame.visible_namespaces.clone(),
1153 process_ref: frame.process_ref.clone(),
1154 request_id: frame.request_id,
1155 };
1156 tracing::debug!(
1157 request_id = frame.request_id,
1158 "daemon RequestIdentity constructed"
1159 );
1160 let (read_cancel_tx, read_cancel_rx) = tokio::sync::watch::channel(false);
1161 let read_timeout = dispatcher.request_read_timeout(&frame.ops);
1165 let excluded_embedder_names =
1166 config_id_extra_embedder_exclusions(&frame.config_id, dispatcher.config_id())
1167 .expect("compatible configuration ids must expose their extra embedder sets");
1168 let dispatch = crate::runtime::scope_request_embedder_exclusions(
1169 excluded_embedder_names,
1170 khive_storage::scope_request_read_cancellation(
1171 shutdown,
1172 khive_storage::scope_request_read_cancellation(
1173 read_cancel_rx,
1174 khive_storage::scope_request_read_deadline(
1175 read_timeout,
1176 dispatcher.dispatch_with_error_detail(
1177 frame.ops,
1178 frame.presentation,
1179 frame.presentation_per_op,
1180 frame.format,
1181 frame.format_per_op,
1182 frame.from_wire,
1183 Some(identity),
1184 ),
1185 ),
1186 ),
1187 ),
1188 );
1189 tokio::pin!(dispatch);
1190 let dispatch_result = tokio::select! {
1191 result = &mut dispatch => result,
1192 _ = wait_for_peer_disconnect(&mut peer_read) => {
1193 let _ = read_cancel_tx.send(true);
1194 dispatch.await
1195 }
1196 };
1197 match dispatch_result {
1198 Ok(result) => {
1199 let (result, detail) = take_daemon_lexical_timeout_marker(result);
1200 DaemonResponseFrame {
1201 ok: true,
1202 result: Some(result),
1203 error: None,
1204 error_detail: detail,
1205 namespace_mismatch: false,
1206 config_mismatch: false,
1207 served_config_id,
1208 version_mismatch: false,
1209 daemon_protocol_version: PROTOCOL_VERSION,
1210 metrics: None,
1211 request_id: frame.request_id,
1212 }
1213 }
1214 Err(error) => {
1215 let error = DaemonDispatchError::new(error.message, Some(error.error_detail));
1216 DaemonResponseFrame {
1217 ok: false,
1218 result: None,
1219 error: Some(error.message),
1220 error_detail: Some(error.error_detail),
1221 namespace_mismatch: false,
1222 config_mismatch: false,
1223 served_config_id,
1224 version_mismatch: false,
1225 daemon_protocol_version: PROTOCOL_VERSION,
1226 metrics: None,
1227 request_id: frame.request_id,
1228 }
1229 }
1230 }
1231 };
1232
1233 let payload = if supervisor_probe {
1234 serde_json::to_value(&resp).and_then(|mut value| {
1235 if let Some(claim) = current_supervisor_claim() {
1236 value["supervisor_claim"] = serde_json::Value::String(claim);
1237 }
1238 if handover_accepted {
1239 value["supervisor_handover_accepted"] = serde_json::Value::Bool(true);
1240 }
1241 serde_json::to_vec(&value)
1242 })
1243 } else {
1244 serde_json::to_vec(&resp)
1245 };
1246 let mut handover_ack_written = false;
1247 match payload {
1248 Ok(payload) => {
1249 if payload.len() > MAX_FRAME_BYTES {
1250 tracing::warn!(
1257 bytes = payload.len(),
1258 limit = MAX_FRAME_BYTES,
1259 "daemon response exceeds MAX_FRAME_BYTES; sending explicit error frame"
1260 );
1261 let message = format!(
1262 "response too large: {} bytes exceeds {} byte IPC cap",
1263 payload.len(),
1264 MAX_FRAME_BYTES,
1265 );
1266 let err_resp = DaemonResponseFrame {
1268 ok: false,
1269 result: None,
1270 error: Some(message.clone()),
1271 error_detail: Some(serde_json::json!({
1272 "kind": "transport",
1273 "code": "response_frame_size_limit",
1274 "message": message,
1275 "domain_disposition": crate::DomainDisposition::Unknown.as_str(),
1276 })),
1277 namespace_mismatch: false,
1278 config_mismatch: false,
1279 served_config_id: resp.served_config_id,
1280 version_mismatch: false,
1281 daemon_protocol_version: PROTOCOL_VERSION,
1282 metrics: None,
1283 request_id: resp.request_id,
1284 };
1285 if let Ok(err_payload) = serde_json::to_vec(&err_resp) {
1286 if let Err(e) = write_response_frame(&mut peer_write, &err_payload).await {
1287 tracing::debug!(error = %e, "failed to write oversized-response error frame");
1288 }
1289 }
1290 } else {
1291 match write_response_frame(&mut peer_write, &payload).await {
1292 Ok(()) => handover_ack_written = true,
1293 Err(e) => tracing::debug!(error = %e, "failed to write daemon response frame"),
1294 }
1295 }
1296 }
1297 Err(e) => tracing::warn!(error = %e, "failed to serialize daemon response frame"),
1298 }
1299 if handover_accepted && handover_ack_written {
1300 if unsafe { libc::raise(libc::SIGTERM) } != 0 {
1303 tracing::error!(error = %std::io::Error::last_os_error(), "self-directed handover signal failed");
1304 }
1305 }
1306 drop(ordinary_admission);
1307}
1308
1309#[cfg(unix)]
1313struct ActiveConnectionGuard {
1314 active: Arc<std::sync::atomic::AtomicUsize>,
1315}
1316
1317#[cfg(unix)]
1318impl ActiveConnectionGuard {
1319 fn claim(active: Arc<std::sync::atomic::AtomicUsize>) -> Self {
1320 active.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
1321 Self { active }
1322 }
1323}
1324
1325#[cfg(unix)]
1326impl Drop for ActiveConnectionGuard {
1327 fn drop(&mut self) {
1328 self.active
1329 .fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
1330 }
1331}
1332
1333#[cfg(unix)]
1334fn spawn_connection_task<F>(
1335 active: Arc<std::sync::atomic::AtomicUsize>,
1336 future: F,
1337) -> tokio::task::JoinHandle<()>
1338where
1339 F: std::future::Future<Output = ()> + Send + 'static,
1340{
1341 let guard = ActiveConnectionGuard::claim(active);
1342 tokio::spawn(async move {
1343 let _guard = guard;
1344 future.await;
1345 })
1346}
1347
1348#[cfg(unix)]
1359pub async fn run_daemon<D: DaemonDispatch>(dispatcher: D) -> anyhow::Result<()> {
1360 let boot_guard = Some(acquire_daemon_boot_guard()?);
1361 run_daemon_with_boot_guard_inner(
1362 dispatcher,
1363 boot_guard,
1364 false,
1365 DaemonOptions::default(),
1366 |_| DaemonStartupReport::default(),
1367 )
1368 .await
1369}
1370
1371#[cfg(all(unix, any(test, feature = "fault-injection")))]
1386#[doc(hidden)]
1387pub async fn run_daemon_in_process_test<D: DaemonDispatch>(dispatcher: D) -> anyhow::Result<()> {
1388 let boot_guard = Some(acquire_daemon_boot_guard()?);
1389 run_daemon_with_boot_guard_inner(
1390 dispatcher,
1391 boot_guard,
1392 true,
1393 DaemonOptions::default(),
1394 |_| DaemonStartupReport::default(),
1395 )
1396 .await
1397}
1398
1399#[cfg(unix)]
1400#[derive(Clone, Copy, PartialEq, Eq)]
1401enum RendezvousPathRole<'a> {
1402 Socket,
1403 PidFile,
1404 Store(&'a std::path::Path),
1405}
1406
1407#[cfg(unix)]
1408impl RendezvousPathRole<'_> {
1409 fn refusal(
1410 self,
1411 rendezvous: std::fmt::Arguments<'_>,
1412 store: std::fmt::Arguments<'_>,
1413 ) -> anyhow::Error {
1414 match self {
1415 Self::Store(db_path) => anyhow::anyhow!(
1416 "refusing to open events database {}: {store}. Check the offending directory's \
1417 ownership and permissions (remove group/other write access), or move the store \
1418 to a location trusted end to end. Only the first offending component is \
1419 reported; further untrusted ancestors or components may remain.",
1420 db_path.display()
1421 ),
1422 Self::Socket | Self::PidFile => anyhow::anyhow!("{rendezvous}"),
1423 }
1424 }
1425
1426 fn env_name(self) -> &'static str {
1427 match self {
1428 Self::Socket => SOCKET_PATH_ENV,
1429 Self::PidFile => PID_PATH_ENV,
1430 Self::Store(_) => "events database",
1431 }
1432 }
1433
1434 fn directory_name(self) -> &'static str {
1435 match self {
1436 Self::Socket => "socket directory",
1437 Self::PidFile => "PID-file directory",
1438 Self::Store(_) => "store directory",
1439 }
1440 }
1441
1442 fn path_name(self) -> &'static str {
1443 match self {
1444 Self::Socket => "socket path",
1445 Self::PidFile => "PID-file path",
1446 Self::Store(_) => "store path",
1447 }
1448 }
1449
1450 fn path_component_name(self) -> &'static str {
1451 match self {
1452 Self::Socket => "socket-path",
1453 Self::PidFile => "PID-file-path",
1454 Self::Store(_) => "store-path",
1455 }
1456 }
1457}
1458
1459#[cfg(unix)]
1482pub(crate) fn ensure_socket_dir_is_trusted(parent: &std::path::Path) -> anyhow::Result<()> {
1483 let daemon_euid = unsafe { libc::geteuid() } as u32;
1485 ensure_rendezvous_dir_is_trusted(parent, RendezvousPathRole::Socket, daemon_euid, true)
1486}
1487
1488#[cfg(unix)]
1490pub(crate) fn ensure_store_dir_is_trusted(
1491 parent: &std::path::Path,
1492 db_path: &std::path::Path,
1493) -> anyhow::Result<()> {
1494 let daemon_euid = unsafe { libc::geteuid() } as u32;
1496 ensure_rendezvous_dir_is_trusted(
1497 parent,
1498 RendezvousPathRole::Store(db_path),
1499 daemon_euid,
1500 true,
1501 )
1502}
1503
1504#[cfg(unix)]
1509pub fn ensure_pid_file_dir_is_trusted(pid_file: &std::path::Path) -> anyhow::Result<()> {
1510 let parent = pid_file
1511 .parent()
1512 .filter(|parent| !parent.as_os_str().is_empty())
1513 .unwrap_or_else(|| std::path::Path::new("."));
1514 let daemon_euid = unsafe { libc::geteuid() } as u32;
1516 ensure_rendezvous_dir_is_trusted(parent, RendezvousPathRole::PidFile, daemon_euid, false)
1517}
1518
1519#[cfg(unix)]
1520fn ensure_rendezvous_dir_is_trusted(
1521 parent: &std::path::Path,
1522 role: RendezvousPathRole<'_>,
1523 daemon_euid: u32,
1524 repair_owned_default: bool,
1525) -> anyhow::Result<()> {
1526 let env_name = role.env_name();
1527 let directory_name = role.directory_name();
1528
1529 if repair_owned_default && parent == khive_dir() {
1530 std::fs::set_permissions(parent, std::fs::Permissions::from_mode(0o700)).map_err(|e| {
1531 role.refusal(
1532 format_args!(
1533 "refusing to start: cannot chmod 0700 {}: {e}. The khive directory must be \
1534 owner-only as the {directory_name} for {env_name}; it is part of the \
1535 same-uid guarantee this daemon enforces.",
1536 parent.display()
1537 ),
1538 format_args!(
1539 "cannot chmod 0700 store directory {}: {e}",
1540 parent.display()
1541 ),
1542 )
1543 })?;
1544 return ensure_rendezvous_path_is_swap_resistant(parent, daemon_euid, role);
1545 }
1546
1547 let meta = std::fs::metadata(parent).map_err(|e| {
1550 role.refusal(
1551 format_args!(
1552 "refusing to start: cannot stat {directory_name} {} for {env_name}: {e}. \
1553 It gates rendezvous-path safety, and unreadable metadata is not a passing state.",
1554 parent.display()
1555 ),
1556 format_args!(
1557 "cannot stat store directory {}: {e}; unreadable metadata is not a passing state",
1558 parent.display()
1559 ),
1560 )
1561 })?;
1562
1563 use std::os::unix::fs::MetadataExt;
1564 let owner = meta.uid();
1565 if owner != daemon_euid && owner != 0 {
1566 return Err(role.refusal(
1567 format_args!(
1568 "refusing to start: {directory_name} {} for {env_name} is owned by uid {owner}, \
1569 not this daemon's uid ({daemon_euid}) or root. A directory owner can replace \
1570 the rendezvous path regardless of mode bits. Point {env_name} at a directory \
1571 you own, or unset it for the default.",
1572 parent.display()
1573 ),
1574 format_args!(
1575 "store directory {} is owned by uid {owner}, not this process's uid \
1576 ({daemon_euid}) or root; its owner can replace the database path",
1577 parent.display()
1578 ),
1579 ));
1580 }
1581
1582 let mode = meta.permissions().mode();
1583 if mode & 0o022 != 0 {
1584 return Err(role.refusal(
1585 format_args!(
1586 "refusing to start: {directory_name} {} for {env_name} is mode {:04o} — writable \
1587 by group or other, so another local user could replace the rendezvous path. \
1588 Use a directory only you can write, or unset {env_name} for the default. \
1589 This daemon is not changing the permissions of a directory it does not own.",
1590 parent.display(),
1591 mode & 0o7777
1592 ),
1593 format_args!(
1594 "store directory {} is mode {:04o}, writable by group or other, so another local \
1595 user could replace the database path",
1596 parent.display(),
1597 mode & 0o7777
1598 ),
1599 ));
1600 }
1601
1602 ensure_rendezvous_path_is_swap_resistant(parent, daemon_euid, role)
1603}
1604
1605#[cfg(all(unix, test))]
1608fn ensure_socket_path_is_swap_resistant(
1609 parent: &std::path::Path,
1610 daemon_euid: u32,
1611) -> anyhow::Result<()> {
1612 ensure_rendezvous_path_is_swap_resistant(parent, daemon_euid, RendezvousPathRole::Socket)
1613}
1614
1615#[cfg(unix)]
1640fn ensure_rendezvous_path_is_swap_resistant(
1641 parent: &std::path::Path,
1642 daemon_euid: u32,
1643 role: RendezvousPathRole<'_>,
1644) -> anyhow::Result<()> {
1645 use std::os::unix::fs::MetadataExt;
1646
1647 let env_name = role.env_name();
1648 let directory_name = role.directory_name();
1649 let path_name = role.path_name();
1650 let component_name = role.path_component_name();
1651
1652 let absolute = if parent.is_absolute() {
1653 parent.to_path_buf()
1654 } else {
1655 std::env::current_dir()
1656 .map_err(|e| {
1657 role.refusal(
1658 format_args!(
1659 "refusing to start: cannot resolve the working directory to absolutize \
1660 {directory_name} {} for {env_name}: {e}.",
1661 parent.display()
1662 ),
1663 format_args!(
1664 "cannot resolve the working directory for store directory {}: {e}",
1665 parent.display()
1666 ),
1667 )
1668 })?
1669 .join(parent)
1670 };
1671
1672 fn push_components(stack: &mut Vec<std::ffi::OsString>, path: &std::path::Path) {
1673 let components: Vec<_> = path
1674 .components()
1675 .map(|c| c.as_os_str().to_os_string())
1676 .collect();
1677 stack.extend(components.into_iter().rev());
1678 }
1679
1680 let mut stack: Vec<std::ffi::OsString> = Vec::new();
1681 push_components(&mut stack, &absolute);
1682 let mut resolved = std::path::PathBuf::new();
1683 let mut symlinks_followed = 0u32;
1684
1685 while let Some(component) = stack.pop() {
1686 if component == "/" {
1687 resolved = std::path::PathBuf::from("/");
1688 continue;
1689 }
1690 if component == "." {
1691 continue;
1692 }
1693 if component == ".." {
1694 resolved.pop();
1695 continue;
1696 }
1697 let candidate = resolved.join(&component);
1698 let meta = std::fs::symlink_metadata(&candidate).map_err(|e| {
1699 role.refusal(
1700 format_args!(
1701 "refusing to start: cannot stat {component_name} component {} for {env_name}: \
1702 {e}. An unreadable component is not a passing one.",
1703 candidate.display()
1704 ),
1705 format_args!(
1706 "cannot stat store-path component {}: {e}; unreadable metadata is not a \
1707 passing state",
1708 candidate.display()
1709 ),
1710 )
1711 })?;
1712 let owner = meta.uid();
1713
1714 if meta.file_type().is_symlink() {
1715 symlinks_followed += 1;
1716 if symlinks_followed > 40 {
1717 return Err(role.refusal(
1718 format_args!(
1719 "refusing to start: {path_name} for {env_name} resolves through more than \
1720 40 symlinks at {} — treating this as a loop.",
1721 candidate.display()
1722 ),
1723 format_args!(
1724 "store path resolves through more than 40 symlinks at {}; treating this as \
1725 a loop",
1726 candidate.display()
1727 ),
1728 ));
1729 }
1730 if owner != daemon_euid && owner != 0 {
1731 return Err(role.refusal(
1732 format_args!(
1733 "refusing to start: {component_name} symlink component {} for {env_name} \
1734 is owned by uid {owner}, not this daemon's uid ({daemon_euid}) or root — \
1735 its owner could retarget it after this check and re-root the {path_name}. \
1736 Point {env_name} somewhere trusted end to end, or unset it for the default.",
1737 candidate.display()
1738 ),
1739 format_args!(
1740 "store-path symlink component {} is owned by uid {owner}, not this \
1741 process's uid ({daemon_euid}) or root; its owner could retarget the \
1742 database path",
1743 candidate.display()
1744 ),
1745 ));
1746 }
1747 let target = std::fs::read_link(&candidate).map_err(|e| {
1748 role.refusal(
1749 format_args!(
1750 "refusing to start: cannot read {component_name} symlink component {} \
1751 for {env_name}: {e}.",
1752 candidate.display()
1753 ),
1754 format_args!(
1755 "cannot read store-path symlink component {}: {e}",
1756 candidate.display()
1757 ),
1758 )
1759 })?;
1760 push_components(&mut stack, &target);
1761 continue;
1762 }
1763
1764 if meta.is_dir() {
1765 let mode = meta.permissions().mode();
1766 let sticky = mode & 0o1000 != 0;
1767 if owner != daemon_euid && owner != 0 {
1768 return Err(role.refusal(
1769 format_args!(
1770 "refusing to start: {component_name} ancestor {} for {env_name} is owned by \
1771 uid {owner}, not this daemon's uid ({daemon_euid}) or root — its owner \
1772 could rename the next path component and re-root the {path_name}. Point \
1773 {env_name} somewhere trusted end to end, or unset it for the default.",
1774 candidate.display()
1775 ),
1776 format_args!(
1777 "store-path ancestor {} is owned by uid {owner}, not this process's uid \
1778 ({daemon_euid}) or root; its owner could rename the next path component",
1779 candidate.display()
1780 ),
1781 ));
1782 }
1783 if mode & 0o022 != 0 && !sticky {
1784 return Err(role.refusal(
1785 format_args!(
1786 "refusing to start: {component_name} ancestor {} for {env_name} is mode \
1787 {:04o} — writable by group or other without the sticky bit, so another \
1788 local user could rename the next path component and re-root the {path_name}. \
1789 Point {env_name} somewhere trusted end to end, or unset it for the default.",
1790 candidate.display(),
1791 mode & 0o7777
1792 ),
1793 format_args!(
1794 "store-path ancestor {} is mode {:04o}, writable by group or other without \
1795 the sticky bit, so another local user could rename the next path \
1796 component",
1797 candidate.display(), mode & 0o7777
1798 ),
1799 ));
1800 }
1801 resolved = candidate;
1802 continue;
1803 }
1804
1805 return Err(role.refusal(
1806 format_args!(
1807 "refusing to start: {component_name} component {} for {env_name} is neither a \
1808 directory nor a symlink — the {path_name} cannot traverse it.",
1809 candidate.display()
1810 ),
1811 format_args!(
1812 "store-path component {} is neither a directory nor a symlink; the database path \
1813 cannot traverse it",
1814 candidate.display()
1815 ),
1816 ));
1817 }
1818
1819 Ok(())
1820}
1821
1822#[cfg(unix)]
1835pub async fn run_daemon_with_boot_guard<D: DaemonDispatch>(
1836 dispatcher: D,
1837 boot_guard: Option<std::fs::File>,
1838) -> anyhow::Result<()> {
1839 run_daemon_with_boot_guard_inner(
1840 dispatcher,
1841 boot_guard,
1842 false,
1843 DaemonOptions::default(),
1844 |_| DaemonStartupReport::default(),
1845 )
1846 .await
1847}
1848
1849#[cfg(unix)]
1855pub async fn run_daemon_with_boot_guard_and_start<D, F>(
1856 dispatcher: D,
1857 boot_guard: Option<std::fs::File>,
1858 start: F,
1859) -> anyhow::Result<()>
1860where
1861 D: DaemonDispatch,
1862 F: FnOnce(&D) + Send,
1863{
1864 run_daemon_with_options_and_boot_guard_and_start(
1865 dispatcher,
1866 boot_guard,
1867 DaemonOptions::default(),
1868 |dispatcher| {
1869 start(dispatcher);
1870 DaemonStartupReport::default()
1871 },
1872 )
1873 .await
1874}
1875
1876#[cfg(unix)]
1878pub async fn run_daemon_with_options_and_boot_guard_and_start<D, F>(
1879 dispatcher: D,
1880 boot_guard: Option<std::fs::File>,
1881 options: DaemonOptions,
1882 start: F,
1883) -> anyhow::Result<()>
1884where
1885 D: DaemonDispatch,
1886 F: FnOnce(&D) -> DaemonStartupReport + Send,
1887{
1888 anyhow::ensure!(
1889 !options.idle_interval.is_zero(),
1890 "daemon idle interval must be positive"
1891 );
1892 run_daemon_with_boot_guard_inner(dispatcher, boot_guard, false, options, start).await
1893}
1894
1895#[cfg(unix)]
1896async fn run_daemon_with_boot_guard_inner<D, F>(
1897 dispatcher: D,
1898 boot_guard: Option<std::fs::File>,
1899 allow_same_process_incumbent: bool,
1900 options: DaemonOptions,
1901 start: F,
1902) -> anyhow::Result<()>
1903where
1904 D: DaemonDispatch,
1905 F: FnOnce(&D) -> DaemonStartupReport + Send,
1906{
1907 struct ComponentTeardown;
1912 impl Drop for ComponentTeardown {
1913 fn drop(&mut self) {
1914 daemon_shutdown_token().cancel();
1915 }
1916 }
1917 let _component_teardown = ComponentTeardown;
1918
1919 ensure_rendezvous_overrides_paired()?;
1923
1924 let sock = socket_path();
1925 let pid_file = pid_path();
1926 let socket_parent = sock.parent();
1927 let pid_parent = pid_file.parent();
1928
1929 if let Some(parent) = socket_parent {
1930 std::fs::create_dir_all(parent)?;
1931 ensure_socket_dir_is_trusted(parent)?;
1932 }
1933 if pid_parent != socket_parent {
1937 ensure_pid_file_dir_is_trusted(&pid_file)?;
1938 }
1939
1940 let _startup_lock = boot_guard;
1954
1955 match cleanup_stale_daemon(
1958 &sock,
1959 &pid_file,
1960 allow_same_process_incumbent,
1961 dispatcher.config_id(),
1962 )
1963 .await
1964 {
1965 Incumbent::Serving(incumbent_pid) => {
1966 tracing::error!(
1967 pid = incumbent_pid,
1968 socket = ?sock,
1969 "refusing to start: a khived instance is already serving this socket"
1970 );
1971 anyhow::bail!(
1972 "refusing to start: khived is already running as pid {incumbent_pid}, \
1973 serving socket {}. Stop that instance first if you intend to replace it.",
1974 sock.display()
1975 );
1976 }
1977 Incumbent::Live(incumbent_pid) => {
1978 tracing::error!(
1979 pid = incumbent_pid,
1980 socket = ?sock,
1981 "refusing to start: a live process owns the PID file but no khived answered"
1982 );
1983 anyhow::bail!(
1984 "refusing to start: pid {incumbent_pid} owns the daemon PID file and is alive, \
1985 but nothing answered the khived protocol on {}. It may be draining. Nothing \
1986 was removed; stop that process first if you intend to replace it.",
1987 sock.display()
1988 );
1989 }
1990 Incumbent::Stale => {}
1991 }
1992
1993 if let Err(error) = exit_record::capture_signal_senders() {
1998 tracing::warn!(target: "khive.boot", error = %error, "signal sender capture unavailable; the exit record will say sender_pid=unknown");
1999 }
2000 let mut sigterm = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())?;
2005 let mut sigint = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::interrupt())?;
2006
2007 let pid_file_guard = match write_pid_file_exclusive(&pid_file) {
2008 Ok(guard) => guard,
2009 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
2010 if pid_file_names_a_reachable_daemon(
2013 &pid_file,
2014 &sock,
2015 allow_same_process_incumbent,
2016 dispatcher.config_id(),
2017 )
2018 .await
2019 {
2020 tracing::info!(
2021 "a replacement khived already claimed the pid/socket rendezvous; exiting"
2022 );
2023 return Ok(());
2024 }
2025 anyhow::bail!(
2026 "failed to claim daemon pid file at {pid_file:?}: it already exists \
2027 and does not name a reachable daemon"
2028 );
2029 }
2030 Err(e) => return Err(e.into()),
2031 };
2032 let exit_record = exit_record::ExitRecord::claim(&pid_file);
2034
2035 let listener = match UnixListener::bind(&sock) {
2036 Ok(listener) => listener,
2037 Err(e) => {
2038 exit_record.exiting(exit_record::ExitReason::Error);
2039 remove_pid_file_if_owned(&pid_file, &pid_file_guard);
2040 return Err(e.into());
2041 }
2042 };
2043 if let Err(e) = std::fs::set_permissions(&sock, std::fs::Permissions::from_mode(0o600)) {
2047 drop(listener);
2048 let _ = std::fs::remove_file(&sock);
2049 exit_record.exiting(exit_record::ExitReason::Error);
2050 remove_pid_file_if_owned(&pid_file, &pid_file_guard);
2051 return Err(anyhow::anyhow!(
2052 "refusing to start: cannot chmod 0600 {}: {e}. The daemon socket must be owner-only \
2053 — it is half of the single-principal guarantee this daemon enforces.",
2054 sock.display()
2055 ));
2056 }
2057 let bound_identity = socket_identity(&sock);
2061
2062 let lifecycle = Arc::new(DaemonLifecycle::new(options, start(&dispatcher)));
2063
2064 drop(_startup_lock);
2067 tracing::info!(
2068 socket = ?sock,
2069 pid = std::process::id(),
2070 source_revision = crate::BUILD_INFO.source_revision,
2071 build_time = crate::BUILD_INFO.build_time,
2072 "khived listening"
2073 );
2074
2075 {
2076 let warm = dispatcher.clone();
2077 track_named_background_task("daemon_warmup", async move {
2078 warm.warm_all().await;
2079 });
2080 }
2081
2082 let (checkpoint_shutdown_tx, checkpoint_shutdown_rx) = tokio::sync::watch::channel(());
2090 let checkpoint_tasks = checkpoint_task_specs(
2096 dispatcher.pool_for_checkpoint(),
2097 dispatcher.secondary_pools_for_checkpoint(),
2098 dispatcher.event_store_for_checkpoint(),
2099 dispatcher.namespace().to_string(),
2100 );
2101 if !checkpoint_tasks.is_empty() {
2102 let cfg = CheckpointConfig::from_env();
2103 let checkpoint_task_count = checkpoint_tasks.len();
2104 for task in checkpoint_tasks {
2105 track_named_background_task(
2106 "wal_checkpoint",
2107 run_checkpoint_task(
2108 task.pool,
2109 cfg.clone(),
2110 task.lifecycle_owner,
2111 checkpoint_shutdown_rx.clone(),
2112 task.is_main,
2113 ),
2114 );
2115 }
2116 tracing::info!(checkpoint_task_count, "WAL checkpoint task(s) started");
2117 }
2118
2119 let active = Arc::new(std::sync::atomic::AtomicUsize::new(0));
2120 let connection_tasks = Arc::new(std::sync::Mutex::new(
2121 Vec::<tokio::task::JoinHandle<()>>::new(),
2122 ));
2123 let (request_shutdown_tx, request_shutdown_rx) = tokio::sync::watch::channel(false);
2124
2125 let shutdown = async {
2126 let signal = tokio::select! {
2127 _ = sigterm.recv() => "SIGTERM",
2128 _ = sigint.recv() => "SIGINT",
2129 };
2130 tracing::info!("received {signal}");
2131 for signal in [libc::SIGTERM, libc::SIGINT] {
2135 if unsafe { libc::signal(signal, libc::SIG_DFL) } == libc::SIG_ERR {
2138 return Err(std::io::Error::last_os_error());
2139 }
2140 }
2141 Ok::<&'static str, std::io::Error>(signal)
2142 };
2143 tokio::pin!(shutdown);
2144 lifecycle.ready();
2145
2146 let daemon_euid = unsafe { libc::geteuid() } as u32;
2148
2149 let reason = tokio::select! {
2150 _ = async {
2151 let mut accept_error_backoff = None;
2152 let mut last_accept_error_log: Option<std::time::Instant> = None;
2153 loop {
2154 match listener.accept().await {
2155 Ok((mut stream, _)) => {
2156 let initial_frame_deadline =
2157 tokio::time::Instant::now() + INITIAL_FRAME_READ_TIMEOUT;
2158 accept_error_backoff = None;
2159 last_accept_error_log = None;
2160 match peer_uid(&stream) {
2174 Ok(peer) if uid_is_permitted(peer, daemon_euid) => {}
2175 Ok(peer) => {
2176 tracing::error!(
2177 peer_uid = peer,
2178 daemon_euid,
2179 "refusing connection from a foreign uid: this daemon accepts \
2180 only peers running as its own uid"
2181 );
2182 drop(stream);
2183 continue;
2184 }
2185 Err(e) => {
2186 tracing::error!(
2187 error = %e,
2188 "refusing connection: cannot read peer credentials, so \
2189 same-uid cannot be proven"
2190 );
2191 drop(stream);
2192 continue;
2193 }
2194 }
2195 let Some(permit) = admit_or_refuse_busy(
2200 &lifecycle.connections,
2201 &mut stream,
2202 dispatcher.config_id(),
2203 )
2204 .await
2205 else {
2206 continue;
2207 };
2208 let d = dispatcher.clone();
2211 let shutdown = request_shutdown_rx.clone();
2212 let lifecycle = Arc::clone(&lifecycle);
2213 let handle = spawn_connection_task(Arc::clone(&active), async move {
2214 let _permit = permit;
2217 handle_conn_with_lifecycle(
2218 stream,
2219 d,
2220 Some(shutdown),
2221 initial_frame_deadline,
2222 Some(lifecycle),
2223 )
2224 .await;
2225 });
2226 let mut tasks = connection_tasks
2227 .lock()
2228 .unwrap_or_else(std::sync::PoisonError::into_inner);
2229 tasks.retain(|task| !task.is_finished());
2230 tasks.push(handle);
2231 }
2232 Err(e) => {
2233 let delay = next_accept_error_backoff(accept_error_backoff);
2234 accept_error_backoff = Some(delay);
2235 let capacity_exhausted = matches!(
2236 e.raw_os_error(),
2237 Some(libc::EMFILE) | Some(libc::ENFILE)
2238 );
2239 if last_accept_error_log.is_none_or(|last| {
2240 last.elapsed() >= std::time::Duration::from_secs(30)
2241 }) {
2242 tracing::error!(
2243 error = %e,
2244 capacity_exhausted,
2245 retry_ms = delay.as_millis(),
2246 "daemon accept failed; retrying with bounded backoff"
2247 );
2248 last_accept_error_log = Some(std::time::Instant::now());
2249 }
2250 tokio::time::sleep(delay).await;
2251 }
2252 }
2253 }
2254 } => DaemonShutdownReason::Signal,
2255 result = &mut shutdown => {
2256 exit_record.exiting(exit_record::ExitReason::Signal(result?));
2257 DaemonShutdownReason::Signal
2258 },
2259 _ = wait_for_idle(&dispatcher, &lifecycle) => {
2260 exit_record.exiting(exit_record::ExitReason::Idle);
2261 DaemonShutdownReason::Idle
2262 },
2263 };
2264
2265 lifecycle.draining(reason);
2266
2267 drop(listener);
2270
2271 let _ = checkpoint_shutdown_tx.send(());
2275
2276 if reason == DaemonShutdownReason::Signal {
2279 let _ = request_shutdown_tx.send(true);
2280 }
2281
2282 daemon_shutdown_token().cancel();
2286
2287 let drained = if reason == DaemonShutdownReason::Idle {
2288 tokio::select! {
2289 _ = drain_for_idle(&active, drain_timeout()) => true,
2290 result = &mut shutdown => {
2291 exit_record.exiting(exit_record::ExitReason::Signal(result?));
2292 lifecycle.draining(DaemonShutdownReason::Signal);
2293 let _ = request_shutdown_tx.send(true);
2294 drain(&active).await
2295 }
2296 }
2297 } else {
2298 drain(&active).await
2299 };
2300 let tasks = {
2301 let mut retained = connection_tasks
2302 .lock()
2303 .unwrap_or_else(std::sync::PoisonError::into_inner);
2304 std::mem::take(&mut *retained)
2305 };
2306 finish_connection_tasks(tasks, drained).await;
2307
2308 match acquire_recovery_lock() {
2317 Some(_shutdown_lock) => {
2318 shutdown_cleanup_if_owned(&sock, &pid_file, bound_identity);
2319 }
2320 None => {
2321 tracing::warn!(
2322 "could not acquire recovery lock for shutdown cleanup; \
2323 skipping unlink to avoid deleting a replacement daemon's paths"
2324 );
2325 }
2326 }
2327 lifecycle.stopped();
2328 tracing::info!("khived stopped");
2329 Ok(())
2330}
2331
2332#[cfg(unix)]
2342fn shutdown_cleanup_if_owned(
2343 sock: &std::path::Path,
2344 pid_file: &std::path::Path,
2345 bound_identity: Option<SocketIdentity>,
2346) -> bool {
2347 let pid_is_ours = std::fs::read_to_string(pid_file)
2348 .ok()
2349 .and_then(|s| s.trim().parse::<u32>().ok())
2350 == Some(std::process::id());
2351 let socket_is_ours = bound_identity.is_some() && socket_identity(sock) == bound_identity;
2352 if pid_is_ours && socket_is_ours {
2353 let _ = std::fs::remove_file(sock);
2354 let _ = std::fs::remove_file(pid_file);
2355 true
2356 } else {
2357 tracing::warn!(
2358 socket = ?sock,
2359 pid_file = ?pid_file,
2360 "skipping shutdown cleanup — a replacement daemon already owns this socket/PID"
2361 );
2362 false
2363 }
2364}
2365
2366#[cfg(unix)]
2370#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2371enum PidLiveness {
2372 Alive,
2375 Dead,
2377 PermissionDenied,
2382}
2383
2384#[cfg(unix)]
2385impl PidLiveness {
2386 fn is_running(self) -> bool {
2387 !matches!(self, PidLiveness::Dead)
2388 }
2389}
2390
2391#[cfg(unix)]
2395fn classify_kill_result(rc: i32, errno: i32) -> PidLiveness {
2396 if rc == 0 {
2397 return PidLiveness::Alive;
2398 }
2399 match errno {
2400 libc::EPERM => PidLiveness::PermissionDenied,
2401 _ => PidLiveness::Dead,
2402 }
2403}
2404
2405#[cfg(unix)]
2406fn is_process_running(pid: u32) -> bool {
2407 let Ok(pid) = i32::try_from(pid) else {
2408 return false;
2409 };
2410 if pid <= 0 {
2411 return false;
2412 }
2413 let rc = unsafe { libc::kill(pid, 0) };
2415 let errno = std::io::Error::last_os_error().raw_os_error().unwrap_or(0);
2416 classify_kill_result(rc, errno).is_running()
2417}
2418
2419#[cfg(unix)]
2426fn pid_can_name_incumbent(pid: u32, current_pid: u32, allow_same_process_incumbent: bool) -> bool {
2427 allow_same_process_incumbent || pid != current_pid
2428}
2429
2430#[cfg(unix)]
2435const DUPLICATE_PROBE_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(500);
2436
2437#[cfg(unix)]
2468async fn socket_speaks_khived_protocol(sock: &std::path::Path, expected_config_id: &str) -> bool {
2469 let probe = DaemonRequestFrame {
2470 probe_only: true,
2471 protocol_version: PROTOCOL_VERSION,
2472 config_id: expected_config_id.to_string(),
2473 ..Default::default()
2474 };
2475 let Ok(payload) = serde_json::to_vec(&probe) else {
2476 return false;
2477 };
2478 let response = tokio::time::timeout(DUPLICATE_PROBE_TIMEOUT, async {
2479 let mut stream = UnixStream::connect(sock).await.ok()?;
2480 write_frame(&mut stream, &payload).await.ok()?;
2481 let raw = read_frame(&mut stream).await.ok()?;
2482 serde_json::from_slice::<DaemonResponseFrame>(&raw).ok()
2483 })
2484 .await
2485 .ok()
2486 .flatten();
2487
2488 let Some(resp) = response else {
2489 return false;
2490 };
2491 let is_probe_ack = resp.ok
2492 && resp.result.is_none()
2493 && resp.error.is_none()
2494 && resp.metrics.is_none()
2495 && resp.request_id.is_none();
2496 is_probe_ack
2497 && !resp.version_mismatch
2498 && !resp.namespace_mismatch
2499 && !resp.config_mismatch
2500 && resp.daemon_protocol_version == PROTOCOL_VERSION
2501 && resp
2502 .served_config_id
2503 .as_deref()
2504 .is_some_and(|served| config_ids_compatible(expected_config_id, served))
2505}
2506
2507#[cfg(unix)]
2510async fn socket_is_unreachable(sock: &std::path::Path) -> bool {
2511 match tokio::time::timeout(DUPLICATE_PROBE_TIMEOUT, UnixStream::connect(sock)).await {
2512 Ok(Err(error)) => matches!(
2513 error.kind(),
2514 std::io::ErrorKind::NotFound | std::io::ErrorKind::ConnectionRefused
2515 ),
2516 _ => false,
2517 }
2518}
2519
2520#[cfg(unix)]
2525enum Incumbent {
2526 Serving(u32),
2528 Live(u32),
2530 Stale,
2532}
2533
2534#[cfg(unix)]
2541async fn cleanup_stale_daemon(
2542 sock: &std::path::Path,
2543 pid_file: &std::path::Path,
2544 allow_same_process_incumbent: bool,
2545 expected_config_id: &str,
2546) -> Incumbent {
2547 let mut stale_pid_file_guard = None;
2548 if let Ok(pid_str) = std::fs::read_to_string(pid_file) {
2549 if let Ok(pid) = pid_str.trim().parse::<u32>() {
2550 if pid_can_name_incumbent(pid, std::process::id(), allow_same_process_incumbent)
2551 && is_process_running(pid)
2552 {
2553 if sock.exists() && socket_speaks_khived_protocol(sock, expected_config_id).await {
2554 return Incumbent::Serving(pid);
2555 }
2556 if sock.exists() && !socket_is_unreachable(sock).await {
2557 return Incumbent::Live(pid);
2558 }
2559 match try_acquire_pid_file_lock(pid_file) {
2560 Ok(Some(guard)) => stale_pid_file_guard = Some(guard),
2561 Ok(None) => return Incumbent::Live(pid),
2562 Err(e) => {
2563 tracing::warn!(
2564 error = %e,
2565 path = ?pid_file,
2566 "cannot check daemon PID-file lock"
2567 );
2568 return Incumbent::Live(pid);
2569 }
2570 }
2571 }
2572 }
2573 }
2574 if sock.exists() {
2575 if let Err(e) = std::fs::remove_file(sock) {
2576 tracing::warn!(error = %e, path = ?sock, "failed to remove stale socket");
2577 }
2578 }
2579 if pid_file.exists() {
2580 if let Err(e) = std::fs::remove_file(pid_file) {
2581 tracing::warn!(error = %e, path = ?pid_file, "failed to remove stale PID file");
2582 }
2583 }
2584 drop(stale_pid_file_guard);
2585 Incumbent::Stale
2586}
2587
2588#[cfg(unix)]
2595fn write_pid_file_exclusive(pid_file: &std::path::Path) -> std::io::Result<std::fs::File> {
2596 use std::os::unix::fs::OpenOptionsExt;
2597
2598 let mut opts = std::fs::OpenOptions::new();
2599 opts.write(true).create_new(true).mode(0o600);
2600 let mut f = opts.open(pid_file)?;
2601 let rc = unsafe { libc::flock(f.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) };
2603 if rc != 0 {
2604 return Err(std::io::Error::last_os_error());
2605 }
2606 f.write_all(std::process::id().to_string().as_bytes())?;
2607 Ok(f)
2608}
2609
2610#[cfg(unix)]
2613fn try_acquire_pid_file_lock(pid_file: &std::path::Path) -> std::io::Result<Option<std::fs::File>> {
2614 let file = std::fs::OpenOptions::new()
2615 .read(true)
2616 .write(true)
2617 .open(pid_file)?;
2618 let rc = unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) };
2620 if rc == 0 {
2621 return Ok(Some(file));
2622 }
2623 let error = std::io::Error::last_os_error();
2624 if error.kind() == std::io::ErrorKind::WouldBlock
2625 || error.raw_os_error() == Some(libc::EWOULDBLOCK)
2626 {
2627 Ok(None)
2628 } else {
2629 Err(error)
2630 }
2631}
2632
2633#[cfg(unix)]
2636fn remove_pid_file_if_owned(pid_file: &std::path::Path, guard: &std::fs::File) {
2637 let Ok(owned) = guard.metadata() else {
2638 return;
2639 };
2640 let Ok(current) = std::fs::metadata(pid_file) else {
2641 return;
2642 };
2643 if owned.dev() == current.dev() && owned.ino() == current.ino() {
2644 if let Err(e) = std::fs::remove_file(pid_file) {
2645 tracing::warn!(error = %e, path = ?pid_file, "failed to remove unbound PID file");
2646 }
2647 }
2648}
2649
2650#[cfg(unix)]
2656async fn pid_file_names_a_reachable_daemon(
2657 pid_file: &std::path::Path,
2658 sock: &std::path::Path,
2659 allow_same_process_incumbent: bool,
2660 expected_config_id: &str,
2661) -> bool {
2662 let Ok(pid_str) = std::fs::read_to_string(pid_file) else {
2663 return false;
2664 };
2665 let Ok(pid) = pid_str.trim().parse::<u32>() else {
2666 return false;
2667 };
2668 pid_can_name_incumbent(pid, std::process::id(), allow_same_process_incumbent)
2669 && is_process_running(pid)
2670 && sock.exists()
2671 && socket_speaks_khived_protocol(sock, expected_config_id).await
2672}
2673
2674#[cfg(unix)]
2675async fn drain(active: &std::sync::atomic::AtomicUsize) -> bool {
2676 drain_with_timeout(active, drain_timeout()).await
2677}
2678
2679#[cfg(unix)]
2681async fn drain_for_idle(active: &std::sync::atomic::AtomicUsize, timeout: std::time::Duration) {
2682 let deadline = tokio::time::Instant::now() + timeout;
2683 let mut warned = false;
2684 while active.load(std::sync::atomic::Ordering::SeqCst) + background_task_count() != 0 {
2685 if !warned && tokio::time::Instant::now() >= deadline {
2686 tracing::warn!(
2687 "idle drain interval elapsed; retaining workers and rendezvous until settled"
2688 );
2689 warned = true;
2690 }
2691 tokio::time::sleep(std::time::Duration::from_millis(100)).await;
2692 }
2693}
2694
2695#[cfg(unix)]
2696async fn drain_with_timeout(
2697 active: &std::sync::atomic::AtomicUsize,
2698 timeout: std::time::Duration,
2699) -> bool {
2700 use std::sync::atomic::Ordering;
2701 let remaining = || active.load(Ordering::SeqCst) + background_task_count();
2705 if remaining() == 0 {
2706 return true;
2707 }
2708 let deadline = tokio::time::Instant::now() + timeout;
2709 while remaining() > 0 {
2710 if tokio::time::Instant::now() >= deadline {
2711 tracing::warn!(
2712 remaining_connections = active.load(Ordering::SeqCst),
2713 remaining_background_tasks = background_task_count(),
2714 outstanding_background_tasks = %background_task_names().join(", "),
2715 "drain timeout reached; forcing shutdown"
2716 );
2717 return false;
2718 }
2719 tokio::select! {
2720 _ = tokio::time::sleep(std::time::Duration::from_millis(100)) => {}
2721 _ = tokio::time::sleep_until(deadline) => {}
2722 }
2723 }
2724 true
2725}
2726
2727#[cfg(unix)]
2728async fn finish_connection_tasks(tasks: Vec<tokio::task::JoinHandle<()>>, drained: bool) {
2729 if !drained {
2730 for task in &tasks {
2731 if !task.is_finished() {
2732 task.abort();
2733 }
2734 }
2735 }
2736 for task in tasks {
2737 let _ = task.await;
2738 }
2739}
2740
2741pub fn drain_timeout() -> std::time::Duration {
2747 let secs = khive_db::env::env_parse_or("KHIVE_DRAIN_TIMEOUT_SECS", DEFAULT_DRAIN_TIMEOUT_SECS);
2748 std::time::Duration::from_secs(secs)
2749}
2750
2751#[cfg(unix)]
2753pub fn env_truthy(key: &str) -> bool {
2754 std::env::var(key)
2755 .map(|v| {
2756 let v = v.trim();
2757 !v.is_empty() && v != "0" && !v.eq_ignore_ascii_case("false")
2758 })
2759 .unwrap_or(false)
2760}
2761
2762include!("daemon_khive_root_tests.rs");
2763
2764#[cfg(all(unix, any(test, feature = "test-internals")))]
2770#[doc(hidden)]
2771pub async fn serve_connection_for_test<D: DaemonDispatch>(stream: UnixStream, dispatcher: D) {
2772 handle_conn_with_shutdown(
2773 stream,
2774 dispatcher,
2775 None,
2776 tokio::time::Instant::now() + INITIAL_FRAME_READ_TIMEOUT,
2777 )
2778 .await;
2779}
2780
2781#[cfg(all(test, unix))]
2782#[path = "daemon_tests.rs"]
2783mod tests;