use super::*;
#[cfg(unix)]
pub(super) async fn write_response_frame<W>(stream: &mut W, payload: &[u8]) -> std::io::Result<()>
where
W: tokio::io::AsyncWrite + Unpin,
{
tokio::time::timeout(INITIAL_FRAME_READ_TIMEOUT, write_frame(stream, payload))
.await
.map_err(|_| {
std::io::Error::new(
std::io::ErrorKind::TimedOut,
"daemon response write timed out",
)
})?
}
#[cfg(unix)]
pub(super) async fn wait_for_peer_disconnect(read: &mut tokio::net::unix::OwnedReadHalf) {
let mut byte = [0u8; 1];
let _ = read.read(&mut byte).await;
}
#[cfg(all(unix, test))]
pub(super) async fn handle_conn<D: DaemonDispatch>(stream: UnixStream, dispatcher: D) {
handle_conn_with_shutdown(
stream,
dispatcher,
None,
tokio::time::Instant::now() + INITIAL_FRAME_READ_TIMEOUT,
)
.await;
}
#[cfg(all(unix, feature = "fault-injection"))]
#[doc(hidden)]
pub async fn handle_conn_for_test<D: DaemonDispatch>(stream: UnixStream, dispatcher: D) {
handle_conn_with_shutdown(
stream,
dispatcher,
None,
tokio::time::Instant::now() + INITIAL_FRAME_READ_TIMEOUT,
)
.await;
}
#[cfg(unix)]
pub(super) fn plan_frame_companion(raw: &[u8]) -> Option<&'static str> {
let value: serde_json::Value = serde_json::from_slice(raw).ok()?;
if value.get("plan").and_then(serde_json::Value::as_bool) != Some(true)
|| value
.get("protocol_version")
.and_then(serde_json::Value::as_u64)
!= Some(u64::from(PROTOCOL_VERSION))
{
return None;
}
[
"presentation",
"presentation_per_op",
"format",
"format_per_op",
"request_id",
]
.into_iter()
.find(|field| value.get(*field).is_some())
}
#[cfg(all(
unix,
any(test, feature = "fault-injection", feature = "test-internals")
))]
pub(super) async fn handle_conn_with_shutdown<D: DaemonDispatch>(
stream: UnixStream,
dispatcher: D,
shutdown: Option<tokio::sync::watch::Receiver<bool>>,
initial_frame_deadline: tokio::time::Instant,
) {
handle_conn_with_lifecycle(stream, dispatcher, shutdown, initial_frame_deadline, None).await;
}
#[cfg(unix)]
pub(super) async fn handle_conn_with_lifecycle<D: DaemonDispatch>(
mut stream: UnixStream,
dispatcher: D,
shutdown: Option<tokio::sync::watch::Receiver<bool>>,
initial_frame_deadline: tokio::time::Instant,
lifecycle: Option<Arc<DaemonLifecycle>>,
) {
let mut ordinary_admission = None;
let production_shutdown = shutdown.is_some();
let handover_peer_allowed = peer_uid(&stream)
.ok()
.is_some_and(|uid| uid == unsafe { libc::geteuid() } as u32);
let (local_shutdown_tx, local_shutdown_rx) = tokio::sync::watch::channel(false);
let shutdown = shutdown.unwrap_or(local_shutdown_rx);
let _local_shutdown_tx = local_shutdown_tx;
let raw = match read_initial_frame(&mut stream, initial_frame_deadline).await {
Ok(r) => r,
Err(e) => {
tracing::debug!(error = %e, "failed to read daemon request frame");
return;
}
};
#[derive(Deserialize)]
struct SupervisorRequestEnvelope {
#[serde(flatten)]
frame: DaemonRequestFrame,
#[serde(default)]
supervisor_handover: bool,
}
let decoded: Result<SupervisorRequestEnvelope, _> = serde_json::from_slice(&raw);
if decoded.as_ref().ok().is_none_or(|item| item.frame.plan) {
if let Some(field) = plan_frame_companion(&raw) {
let response = DaemonResponseFrame {
ok: false,
result: None,
error: Some(format!(
"invalid_params: plan=true cannot be combined with {field}"
)),
error_detail: Some(serde_json::json!({
"kind": "protocol",
"code": "invalid_params",
"message": format!("plan=true cannot be combined with {field}"),
"domain_disposition": crate::DomainDisposition::NotCommitted.as_str(),
})),
namespace_mismatch: false,
config_mismatch: false,
served_config_id: Some(dispatcher.config_id().to_string()),
version_mismatch: false,
daemon_protocol_version: PROTOCOL_VERSION,
metrics: None,
request_id: None,
};
if let Ok(payload) = serde_json::to_vec(&response) {
if let Err(error) = write_response_frame(&mut stream, &payload).await {
tracing::debug!(%error, "failed to write plan envelope refusal");
}
}
return;
}
}
let (frame, handover_requested) = match decoded {
Ok(item) => (item.frame, item.supervisor_handover),
Err(e) => {
tracing::debug!(error = %e, "failed to decode daemon request frame");
return;
}
};
let supervisor_probe = frame.probe_only;
let handover_accepted = handover_requested
&& supervisor_probe
&& production_shutdown
&& handover_peer_allowed
&& frame.protocol_version == PROTOCOL_VERSION
&& !frame.plan
&& !frame.metrics_only
&& frame.ops.is_empty()
&& read_supervisor_marker_claim().is_some_and(|(pid, _)| pid != std::process::id());
let (mut peer_read, mut peer_write) = stream.into_split();
let served_config_id = Some(dispatcher.config_id().to_string());
let resp = if frame.protocol_version != PROTOCOL_VERSION {
let msg = format!(
"daemon protocol mismatch: client={} daemon={} — \
rebuild/update the client binary (make local)",
frame.protocol_version, PROTOCOL_VERSION,
);
tracing::warn!(
client_version = frame.protocol_version,
daemon_version = PROTOCOL_VERSION,
"daemon protocol version mismatch"
);
DaemonResponseFrame {
ok: false,
result: None,
error: Some(msg.clone()),
error_detail: Some(serde_json::json!({
"kind": "protocol",
"code": "version_mismatch",
"message": msg,
"domain_disposition": crate::DomainDisposition::Unknown.as_str(),
})),
namespace_mismatch: false,
config_mismatch: false,
served_config_id,
version_mismatch: frame.protocol_version > PROTOCOL_VERSION,
daemon_protocol_version: PROTOCOL_VERSION,
metrics: None,
request_id: frame.request_id,
}
} else if handover_requested {
if handover_accepted {
DaemonResponseFrame {
ok: true,
result: None,
error: None,
error_detail: None,
namespace_mismatch: false,
config_mismatch: false,
served_config_id,
version_mismatch: false,
daemon_protocol_version: PROTOCOL_VERSION,
metrics: None,
request_id: frame.request_id,
}
} else {
DaemonResponseFrame {
ok: false,
result: None,
error: Some("supervisor handover refused".to_string()),
error_detail: Some(serde_json::json!({
"kind": "protocol",
"code": "supervisor_handover_refused",
"message": "supervisor handover refused",
"domain_disposition": crate::DomainDisposition::NotCommitted.as_str(),
})),
namespace_mismatch: false,
config_mismatch: false,
served_config_id,
version_mismatch: false,
daemon_protocol_version: PROTOCOL_VERSION,
metrics: None,
request_id: frame.request_id,
}
}
} else if frame.metrics_only && !frame.plan {
DaemonResponseFrame {
ok: true,
result: None,
error: None,
error_detail: None,
namespace_mismatch: false,
config_mismatch: false,
served_config_id,
version_mismatch: false,
daemon_protocol_version: PROTOCOL_VERSION,
metrics: Some({
let mut metrics = build_metrics_snapshot(&dispatcher);
metrics.lifecycle = lifecycle.as_ref().map(|state| {
let mut snapshot = state.snapshot();
snapshot.idle_blockers = idle_retirement_blockers(&dispatcher);
snapshot
});
metrics.connections = lifecycle.as_ref().map(|state| state.connections.snapshot());
metrics
}),
request_id: frame.request_id,
}
} else if !config_ids_compatible(&frame.config_id, dispatcher.config_id()) {
DaemonResponseFrame {
ok: false,
result: None,
error: None,
error_detail: Some(serde_json::json!({
"kind": "protocol",
"code": "config_mismatch",
"message": "daemon configuration does not match the request",
"domain_disposition": crate::DomainDisposition::NotCommitted.as_str(),
})),
namespace_mismatch: false,
config_mismatch: true,
served_config_id,
version_mismatch: false,
daemon_protocol_version: PROTOCOL_VERSION,
metrics: None,
request_id: frame.request_id,
}
} else if frame.plan {
DaemonResponseFrame {
ok: true,
result: Some(dispatcher.plan(&frame.ops)),
error: None,
error_detail: None,
namespace_mismatch: false,
config_mismatch: false,
served_config_id,
version_mismatch: false,
daemon_protocol_version: PROTOCOL_VERSION,
metrics: None,
request_id: None,
}
} else if frame.probe_only {
DaemonResponseFrame {
ok: true,
result: None,
error: None,
error_detail: None,
namespace_mismatch: false,
config_mismatch: false,
served_config_id,
version_mismatch: false,
daemon_protocol_version: PROTOCOL_VERSION,
metrics: None,
request_id: frame.request_id,
}
} else {
if let Some(lifecycle) = &lifecycle {
ordinary_admission = lifecycle.admit();
if ordinary_admission.is_none() {
let refusal = DaemonResponseFrame {
ok: false,
result: None,
error: Some("daemon is draining; request was not admitted".to_owned()),
error_detail: Some(serde_json::json!({
"kind": "runtime", "code": "daemon_draining",
"domain_disposition": crate::DomainDisposition::NotCommitted.as_str(),
})),
request_id: frame.request_id,
daemon_protocol_version: PROTOCOL_VERSION,
namespace_mismatch: false,
config_mismatch: false,
served_config_id: Some(dispatcher.config_id().to_owned()),
version_mismatch: false,
metrics: None,
};
if let Ok(payload) = serde_json::to_vec(&refusal) {
let _ = write_response_frame(&mut peer_write, &payload).await;
}
return;
}
}
if let Some(readiness) = dispatcher.events_readiness() {
let not_ready = tokio::select! {
outcome = readiness.wait() => outcome.err(),
_ = wait_for_peer_disconnect(&mut peer_read) => return,
};
if let Some(not_ready) = not_ready {
let message = not_ready.to_string();
tracing::warn!(
socket = %not_ready.socket.display(),
reason = ?not_ready.reason,
request_id = frame.request_id,
"request refused: the supervised events daemon is not answering"
);
let refusal = DaemonResponseFrame {
ok: false,
result: None,
error: Some(message.clone()),
error_detail: Some(serde_json::json!({
"kind": "runtime",
"code": "events_daemon_unavailable",
"message": message,
"retryable": true,
"socket": not_ready.socket.display().to_string(),
"domain_disposition": crate::DomainDisposition::NotCommitted.as_str(),
})),
request_id: frame.request_id,
daemon_protocol_version: PROTOCOL_VERSION,
namespace_mismatch: false,
config_mismatch: false,
served_config_id: Some(dispatcher.config_id().to_owned()),
version_mismatch: false,
metrics: None,
};
if let Ok(payload) = serde_json::to_vec(&refusal) {
let _ = write_response_frame(&mut peer_write, &payload).await;
}
return;
}
}
let identity = RequestIdentity {
namespace: frame.namespace.clone(),
actor_id: frame.actor_id.clone(),
visible_namespaces: frame.visible_namespaces.clone(),
process_ref: frame.process_ref.clone(),
request_id: frame.request_id,
};
tracing::debug!(
request_id = frame.request_id,
"daemon RequestIdentity constructed"
);
let (read_cancel_tx, read_cancel_rx) = tokio::sync::watch::channel(false);
let read_timeout = dispatcher.request_read_timeout(&frame.ops);
let excluded_embedder_names =
config_id_extra_embedder_exclusions(&frame.config_id, dispatcher.config_id())
.expect("compatible configuration ids must expose their extra embedder sets");
let dispatch = crate::runtime::scope_request_embedder_exclusions(
excluded_embedder_names,
khive_storage::scope_request_read_cancellation(
shutdown,
khive_storage::scope_request_read_cancellation(
read_cancel_rx,
khive_storage::scope_request_read_deadline(
read_timeout,
dispatcher.dispatch_with_error_detail(
frame.ops,
frame.presentation,
frame.presentation_per_op,
frame.format,
frame.format_per_op,
frame.from_wire,
Some(identity),
),
),
),
),
);
tokio::pin!(dispatch);
let dispatch_result = tokio::select! {
result = &mut dispatch => result,
_ = wait_for_peer_disconnect(&mut peer_read) => {
let _ = read_cancel_tx.send(true);
dispatch.await
}
};
match dispatch_result {
Ok(result) => {
let (result, detail) = take_daemon_lexical_timeout_marker(result);
DaemonResponseFrame {
ok: true,
result: Some(result),
error: None,
error_detail: detail,
namespace_mismatch: false,
config_mismatch: false,
served_config_id,
version_mismatch: false,
daemon_protocol_version: PROTOCOL_VERSION,
metrics: None,
request_id: frame.request_id,
}
}
Err(error) => {
let error = DaemonDispatchError::new(error.message, Some(error.error_detail));
DaemonResponseFrame {
ok: false,
result: None,
error: Some(error.message),
error_detail: Some(error.error_detail),
namespace_mismatch: false,
config_mismatch: false,
served_config_id,
version_mismatch: false,
daemon_protocol_version: PROTOCOL_VERSION,
metrics: None,
request_id: frame.request_id,
}
}
}
};
let payload = if supervisor_probe {
serde_json::to_value(&resp).and_then(|mut value| {
if let Some(claim) = current_supervisor_claim() {
value["supervisor_claim"] = serde_json::Value::String(claim);
}
if handover_accepted {
value["supervisor_handover_accepted"] = serde_json::Value::Bool(true);
}
serde_json::to_vec(&value)
})
} else {
serde_json::to_vec(&resp)
};
let mut handover_ack_written = false;
match payload {
Ok(payload) => {
if payload.len() > MAX_FRAME_BYTES {
tracing::warn!(
bytes = payload.len(),
limit = MAX_FRAME_BYTES,
"daemon response exceeds MAX_FRAME_BYTES; sending explicit error frame"
);
let message = format!(
"response too large: {} bytes exceeds {} byte IPC cap",
payload.len(),
MAX_FRAME_BYTES,
);
let err_resp = DaemonResponseFrame {
ok: false,
result: None,
error: Some(message.clone()),
error_detail: Some(serde_json::json!({
"kind": "transport",
"code": "response_frame_size_limit",
"message": message,
"domain_disposition": crate::DomainDisposition::Unknown.as_str(),
})),
namespace_mismatch: false,
config_mismatch: false,
served_config_id: resp.served_config_id,
version_mismatch: false,
daemon_protocol_version: PROTOCOL_VERSION,
metrics: None,
request_id: resp.request_id,
};
if let Ok(err_payload) = serde_json::to_vec(&err_resp) {
if let Err(e) = write_response_frame(&mut peer_write, &err_payload).await {
tracing::debug!(error = %e, "failed to write oversized-response error frame");
}
}
} else {
match write_response_frame(&mut peer_write, &payload).await {
Ok(()) => handover_ack_written = true,
Err(e) => tracing::debug!(error = %e, "failed to write daemon response frame"),
}
}
}
Err(e) => tracing::warn!(error = %e, "failed to serialize daemon response frame"),
}
if handover_accepted && handover_ack_written {
if unsafe { libc::raise(libc::SIGTERM) } != 0 {
tracing::error!(error = %std::io::Error::last_os_error(), "self-directed handover signal failed");
}
}
drop(ordinary_admission);
}
#[cfg(unix)]
pub(super) struct ActiveConnectionGuard {
pub(super) active: Arc<std::sync::atomic::AtomicUsize>,
}
#[cfg(unix)]
impl ActiveConnectionGuard {
pub(super) fn claim(active: Arc<std::sync::atomic::AtomicUsize>) -> Self {
active.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
Self { active }
}
}
#[cfg(unix)]
impl Drop for ActiveConnectionGuard {
fn drop(&mut self) {
self.active
.fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
}
}
#[cfg(unix)]
pub(super) fn spawn_connection_task<F>(
active: Arc<std::sync::atomic::AtomicUsize>,
future: F,
) -> tokio::task::JoinHandle<()>
where
F: std::future::Future<Output = ()> + Send + 'static,
{
let guard = ActiveConnectionGuard::claim(active);
tokio::spawn(async move {
let _guard = guard;
future.await;
})
}