Skip to main content

khive_runtime/events_split/
forwarder.rs

1use super::events_count_read_budget::CountReadBudget;
2use super::paths::{
3    DEFAULT_APPEND_QUEUE_BATCHES, DEFAULT_APPEND_QUEUE_BYTES, EVENTS_PROTOCOL_VERSION,
4    FORWARDER_BACKOFF, REQUEST_TIMEOUT,
5};
6use super::protocol::{EventsRequest, EventsResponse};
7use super::socket_path::validate_events_socket_path;
8use crate::daemon::{read_frame, write_frame};
9use khive_db::StorageBackend;
10use khive_storage::{Event, EventStore, StorageError, StorageResult};
11use serde::Serialize;
12use std::path::{Path, PathBuf};
13use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
14use std::sync::Arc;
15use std::time::Duration;
16use tokio::net::UnixStream;
17
18/// Connect to the events socket and verify who answered. Pathname
19/// reachability is not identity: any process that can write the directory can
20/// pre-bind the path. The kernel-reported peer uid is the one identity the
21/// far side cannot choose, so every client connect — round-trips, the
22/// forwarder, the supervisor probe — refuses a socket answered by a foreign
23/// uid before a single frame is written.
24#[cfg(unix)]
25pub(super) async fn connect_verified(socket_path: &Path) -> std::io::Result<UnixStream> {
26    let stream = UnixStream::connect(socket_path).await?;
27    // SAFETY: `geteuid` is always successful and takes no arguments.
28    let own_euid = unsafe { libc::geteuid() } as u32;
29    let peer = crate::daemon::peer_uid(&stream)?;
30    if !crate::daemon::uid_is_permitted(peer, own_euid) {
31        return Err(std::io::Error::new(
32            std::io::ErrorKind::PermissionDenied,
33            format!(
34                "events socket {} answered by uid {peer}, not this process's uid {own_euid}; \
35                 refusing to exchange event frames with an unowned daemon",
36                socket_path.display()
37            ),
38        ));
39    }
40    Ok(stream)
41}
42
43/// Counters describing the fire-and-forget lane's degradation. Zero drops is
44/// the healthy state; any non-zero `dropped_batches` means the loss-tolerant
45/// contract was exercised and says so.
46#[cfg(unix)]
47#[derive(Debug, Clone, Copy, Serialize)]
48pub struct EventsForwardingMetrics {
49    pub forwarded_batches: u64,
50    pub forwarded_events: u64,
51    pub dropped_batches: u64,
52    pub dropped_events: u64,
53    /// Serialized request bytes reserved by queued and in-flight batches.
54    pub queued_bytes: usize,
55}
56
57#[cfg(unix)]
58#[derive(Debug, Default)]
59pub(super) struct ForwardingCounters {
60    pub(super) forwarded_batches: AtomicU64,
61    pub(super) forwarded_events: AtomicU64,
62    pub(super) dropped_batches: AtomicU64,
63    pub(super) dropped_events: AtomicU64,
64    pub(super) queued_bytes: AtomicUsize,
65}
66
67/// A reservation lives with the batch until delivery or a counted drop.
68/// Dropping a forwarder mid-delivery also returns its byte budget.
69#[cfg(unix)]
70#[derive(Debug)]
71pub(super) struct AppendByteReservation {
72    pub(super) counters: Arc<ForwardingCounters>,
73    pub(super) bytes: usize,
74}
75
76#[cfg(unix)]
77impl Drop for AppendByteReservation {
78    fn drop(&mut self) {
79        self.counters
80            .queued_bytes
81            .fetch_sub(self.bytes, Ordering::AcqRel);
82    }
83}
84
85#[cfg(unix)]
86#[derive(Debug)]
87pub(super) struct QueuedAppend {
88    pub(super) request: EventsRequest,
89    pub(super) count: u64,
90    pub(super) _reservation: AppendByteReservation,
91}
92
93#[cfg(all(test, unix))]
94impl QueuedAppend {
95    pub(super) fn unmetered(
96        namespace: &str,
97        events: Vec<Event>,
98        counters: Arc<ForwardingCounters>,
99    ) -> Self {
100        let count = events.len() as u64;
101        Self {
102            request: EventsRequest::AppendEvents {
103                protocol_version: EVENTS_PROTOCOL_VERSION,
104                namespace: namespace.to_string(),
105                events,
106            },
107            count,
108            _reservation: AppendByteReservation { counters, bytes: 0 },
109        }
110    }
111}
112
113/// Measure the actual wire request without retaining a second, potentially
114/// huge serialized copy of the caller's batch. Serde stops at the frame cap.
115#[cfg(unix)]
116#[derive(Default)]
117pub(super) struct BoundedFrameCounter {
118    pub(super) bytes: usize,
119    pub(super) exceeded: bool,
120}
121
122#[cfg(unix)]
123impl std::io::Write for BoundedFrameCounter {
124    fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
125        let next = self.bytes.saturating_add(buf.len());
126        if next > crate::daemon::MAX_FRAME_BYTES {
127            self.exceeded = true;
128            return Err(std::io::Error::new(
129                std::io::ErrorKind::InvalidData,
130                "events append request exceeds IPC frame cap",
131            ));
132        }
133        self.bytes = next;
134        Ok(buf.len())
135    }
136
137    fn flush(&mut self) -> std::io::Result<()> {
138        Ok(())
139    }
140}
141
142#[cfg(unix)]
143pub(super) fn reserve_append_bytes(
144    counters: &Arc<ForwardingCounters>,
145    bytes: usize,
146    budget: usize,
147) -> Option<AppendByteReservation> {
148    let mut used = counters.queued_bytes.load(Ordering::Acquire);
149    loop {
150        let next = used.checked_add(bytes)?;
151        if next > budget {
152            return None;
153        }
154        match counters.queued_bytes.compare_exchange_weak(
155            used,
156            next,
157            Ordering::AcqRel,
158            Ordering::Acquire,
159        ) {
160            Ok(_) => {
161                return Some(AppendByteReservation {
162                    counters: Arc::clone(counters),
163                    bytes,
164                });
165            }
166            Err(observed) => used = observed,
167        }
168    }
169}
170
171/// One per domain process: the connection to the events daemon plus the
172/// bounded fire-and-forget append queue.
173///
174/// Synchronous round-trips each own a fresh connection: there is no shared
175/// request connection and no lock in front of one, so concurrent reads and
176/// audit flushes never queue behind a single stalled call, and the round-trip
177/// timeout bounds each caller's whole attempt.
178#[cfg(unix)]
179pub struct EventsSplitClient {
180    pub(super) socket_path: PathBuf,
181    pub(super) append_tx: tokio::sync::mpsc::Sender<QueuedAppend>,
182    pub(super) append_queue_byte_budget: usize,
183    pub(super) counters: Arc<ForwardingCounters>,
184    /// Flipped by the forwarder while the daemon is unreachable so the drop
185    /// log fires once per outage, not once per batch.
186    pub(super) outage_logged: Arc<AtomicBool>,
187    /// In-memory validator backing `preflight_event` without I/O.
188    pub(super) preflight_store: Arc<dyn EventStore>,
189}
190
191#[cfg(unix)]
192impl std::fmt::Debug for EventsSplitClient {
193    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
194        f.debug_struct("EventsSplitClient")
195            .field("socket_path", &self.socket_path)
196            .finish_non_exhaustive()
197    }
198}
199
200#[cfg(unix)]
201impl EventsSplitClient {
202    /// Build the client and spawn its background forwarder.
203    pub fn new(socket_path: PathBuf) -> crate::error::RuntimeResult<Arc<Self>> {
204        Self::new_with_queue_depth(socket_path, DEFAULT_APPEND_QUEUE_BATCHES)
205    }
206
207    /// [`Self::new`] with an explicit fire-and-forget queue bound. Tests use a
208    /// tiny depth to exercise the overflow drop arm deterministically.
209    pub fn new_with_queue_depth(
210        socket_path: PathBuf,
211        queue_depth: usize,
212    ) -> crate::error::RuntimeResult<Arc<Self>> {
213        Self::new_with_queue_depth_and_delivery_timeout(socket_path, queue_depth, REQUEST_TIMEOUT)
214    }
215
216    /// [`Self::new_with_queue_depth`] with an explicit per-batch delivery
217    /// timeout, so tests can prove the forwarder abandons a hung peer without
218    /// waiting out the production clock.
219    pub(super) fn new_with_queue_depth_and_delivery_timeout(
220        socket_path: PathBuf,
221        queue_depth: usize,
222        delivery_timeout: Duration,
223    ) -> crate::error::RuntimeResult<Arc<Self>> {
224        Self::new_with_limits_and_delivery_timeout(
225            socket_path,
226            queue_depth,
227            DEFAULT_APPEND_QUEUE_BYTES,
228            delivery_timeout,
229        )
230    }
231
232    pub(super) fn new_with_limits_and_delivery_timeout(
233        socket_path: PathBuf,
234        queue_depth: usize,
235        byte_budget: usize,
236        delivery_timeout: Duration,
237    ) -> crate::error::RuntimeResult<Arc<Self>> {
238        validate_events_socket_path(&socket_path)?;
239        let preflight_backend = StorageBackend::memory()?;
240        let preflight_store = preflight_backend.events()?;
241        // The in-memory backend must outlive the store handle; the store holds
242        // the pool Arc internally, so dropping the backend wrapper here is fine.
243
244        let (append_tx, append_rx) = tokio::sync::mpsc::channel::<QueuedAppend>(queue_depth.max(1));
245        let counters = Arc::new(ForwardingCounters::default());
246        let outage_logged = Arc::new(AtomicBool::new(false));
247
248        let client = Arc::new(Self {
249            socket_path: socket_path.clone(),
250            append_tx,
251            append_queue_byte_budget: byte_budget,
252            counters: Arc::clone(&counters),
253            outage_logged: Arc::clone(&outage_logged),
254            preflight_store,
255        });
256
257        crate::daemon::spawn_named_tracked_task(
258            "events_forwarder",
259            run_forwarder(
260                socket_path,
261                append_rx,
262                counters,
263                outage_logged,
264                delivery_timeout,
265                crate::daemon::daemon_shutdown_token(),
266            ),
267        );
268        Ok(client)
269    }
270
271    /// Snapshot of the fire-and-forget lane's health.
272    pub fn metrics(&self) -> EventsForwardingMetrics {
273        EventsForwardingMetrics {
274            forwarded_batches: self.counters.forwarded_batches.load(Ordering::Relaxed),
275            forwarded_events: self.counters.forwarded_events.load(Ordering::Relaxed),
276            dropped_batches: self.counters.dropped_batches.load(Ordering::Relaxed),
277            dropped_events: self.counters.dropped_events.load(Ordering::Relaxed),
278            queued_bytes: self.counters.queued_bytes.load(Ordering::Acquire),
279        }
280    }
281
282    /// Enqueue a batch on the fire-and-forget lane. Never blocks; a full
283    /// queue is a counted, logged drop.
284    pub(super) fn enqueue(&self, namespace: &str, events: Vec<Event>) {
285        let count = events.len() as u64;
286        let request = EventsRequest::AppendEvents {
287            protocol_version: EVENTS_PROTOCOL_VERSION,
288            namespace: namespace.to_string(),
289            events,
290        };
291        let mut frame_size = BoundedFrameCounter::default();
292        if let Err(error) = serde_json::to_writer(&mut frame_size, &request) {
293            self.count_append_drop(
294                count,
295                if frame_size.exceeded {
296                    "frame cap"
297                } else {
298                    "serialization"
299                },
300            );
301            tracing::warn!(error = %error, "events append batch could not fit a wire frame");
302            return;
303        }
304        let Some(reservation) = reserve_append_bytes(
305            &self.counters,
306            frame_size.bytes,
307            self.append_queue_byte_budget,
308        ) else {
309            self.count_append_drop(count, "queue byte budget");
310            return;
311        };
312        let batch = QueuedAppend {
313            request,
314            count,
315            _reservation: reservation,
316        };
317        match self.append_tx.try_send(batch) {
318            Ok(()) => {}
319            Err(_) => {
320                // The rejected batch drops here and releases its reservation.
321                self.count_append_drop(count, "queue batch limit");
322            }
323        }
324    }
325
326    pub(super) fn count_append_drop(&self, count: u64, reason: &'static str) {
327        self.counters
328            .dropped_batches
329            .fetch_add(1, Ordering::Relaxed);
330        self.counters
331            .dropped_events
332            .fetch_add(count, Ordering::Relaxed);
333        if !self.outage_logged.swap(true, Ordering::Relaxed) {
334            tracing::warn!(
335                dropped_events = count,
336                reason,
337                "events append queue rejected loss-tolerant batch"
338            );
339        }
340    }
341
342    /// One synchronous framed round-trip with a bounded timeout, on its own
343    /// fresh connection. No connection is shared between concurrent callers,
344    /// so no caller ever waits on another's stall, and `REQUEST_TIMEOUT`
345    /// bounds the whole attempt — connect, peer verification, write, read.
346    pub(super) async fn round_trip(
347        &self,
348        request: &EventsRequest,
349    ) -> StorageResult<EventsResponse> {
350        let op = "events daemon round-trip";
351        let payload = serialize_events_request(request)?;
352        let attempt = events_round_trip_attempt(&self.socket_path, &payload);
353        let bytes = match tokio::time::timeout(REQUEST_TIMEOUT, attempt).await {
354            Ok(Ok(bytes)) => bytes,
355            Ok(Err(("read", error))) => {
356                return Err(StorageError::Serialization {
357                    capability: khive_storage::StorageCapability::Events,
358                    message: format!(
359                        "events daemon closed or broke the response after connection at {}: {error}",
360                        self.socket_path.display()
361                    ),
362                });
363            }
364            Ok(Err((stage, error))) => {
365                return Err(StorageError::Pool {
366                    operation: op.into(),
367                    message: format!(
368                        "events daemon {stage} failed at {}: {error}",
369                        self.socket_path.display()
370                    ),
371                });
372            }
373            Err(_elapsed) => {
374                return Err(StorageError::Timeout {
375                    operation: op.into(),
376                });
377            }
378        };
379
380        deserialize_events_response(&bytes)
381    }
382
383    pub(super) async fn round_trip_count(
384        &self,
385        request: &EventsRequest,
386        budget: &CountReadBudget,
387    ) -> StorageResult<EventsResponse> {
388        budget.scope(async {
389            let payload = serialize_events_request(request)?;
390            let attempt = events_round_trip_attempt(&self.socket_path, &payload);
391            let result = khive_storage::await_request_read_phase("events daemon count round-trip", attempt)
392                .await.map_err(|error| budget.attribute(error))?;
393            let bytes = match result {
394                Ok(bytes) => bytes,
395                Err(("read", error)) => return Err(StorageError::Serialization {
396                    capability: khive_storage::StorageCapability::Events,
397                    message: format!("events daemon closed or broke the response after connection at {}: {error}", self.socket_path.display()),
398                }),
399                Err((stage, error)) => return Err(StorageError::Pool {
400                    operation: "events daemon count round-trip".into(),
401                    message: format!("events daemon {stage} failed at {}: {error}", self.socket_path.display()),
402                }),
403            };
404            deserialize_events_response(&bytes)
405        }).await
406    }
407}
408
409#[cfg(unix)]
410pub(super) fn serialize_events_request(request: &EventsRequest) -> StorageResult<Vec<u8>> {
411    serde_json::to_vec(request).map_err(|error| StorageError::Serialization {
412        capability: khive_storage::StorageCapability::Events,
413        message: format!("events request serialization failed: {error}"),
414    })
415}
416
417#[cfg(unix)]
418pub(super) fn deserialize_events_response(bytes: &[u8]) -> StorageResult<EventsResponse> {
419    serde_json::from_slice(bytes).map_err(|error| StorageError::Serialization {
420        capability: khive_storage::StorageCapability::Events,
421        message: format!("events response deserialization failed: {error}"),
422    })
423}
424
425#[cfg(unix)]
426pub(super) async fn events_round_trip_attempt(
427    socket_path: &Path,
428    payload: &[u8],
429) -> Result<Vec<u8>, (&'static str, std::io::Error)> {
430    let mut stream = connect_verified(socket_path)
431        .await
432        .map_err(|error| ("connect", error))?;
433    write_frame(&mut stream, payload)
434        .await
435        .map_err(|error| ("write", error))?;
436    read_frame(&mut stream)
437        .await
438        .map_err(|error| ("read", error))
439}
440
441/// Drain every batch currently sitting in the fire-and-forget queue,
442/// counting each as a dropped batch/events and logging one summary line
443/// (never one line per batch — a full queue at shutdown is exactly the
444/// bursty case a per-batch log would flood). Non-blocking: `try_recv` only
445/// consumes what is already queued, so it terminates as soon as the queue
446/// (temporarily) empties even though the sender half is still live.
447#[cfg(unix)]
448pub(super) fn drain_dropped_queue(
449    rx: &mut tokio::sync::mpsc::Receiver<QueuedAppend>,
450    counters: &ForwardingCounters,
451) {
452    let mut dropped_batches = 0u64;
453    let mut dropped_events = 0u64;
454    while let Ok(batch) = rx.try_recv() {
455        dropped_batches += 1;
456        dropped_events += batch.count;
457        // `batch` drops its byte reservation at the end of this iteration.
458    }
459    if dropped_batches > 0 {
460        counters
461            .dropped_batches
462            .fetch_add(dropped_batches, Ordering::Relaxed);
463        counters
464            .dropped_events
465            .fetch_add(dropped_events, Ordering::Relaxed);
466        tracing::warn!(
467            dropped_batches,
468            dropped_events,
469            "events forwarder shutting down; dropping queued loss-tolerant batches"
470        );
471    }
472}
473
474/// The background fire-and-forget forwarder: drains the bounded queue into
475/// framed appends on its own connection, reconnecting with backoff. A batch
476/// that cannot be delivered is dropped and counted — never retried, never
477/// blocking the queue behind it.
478#[cfg(unix)]
479pub(super) async fn run_forwarder(
480    socket_path: PathBuf,
481    mut rx: tokio::sync::mpsc::Receiver<QueuedAppend>,
482    counters: Arc<ForwardingCounters>,
483    outage_logged: Arc<AtomicBool>,
484    delivery_timeout: Duration,
485    shutdown: tokio_util::sync::CancellationToken,
486) {
487    // The client lives in a process-global registry, so its sender is never
488    // dropped and `rx.recv()` alone would keep this tracked task alive
489    // through daemon shutdown, forcing `drain()` to its full timeout.
490    // Observe the shutdown token directly: queued batches at shutdown are a
491    // counted drop, exactly the lane's loss-tolerant contract — enforced by
492    // draining and counting `rx` below on every shutdown exit, not just
493    // implied by the comment.
494    let mut conn: Option<UnixStream> = None;
495    loop {
496        let batch = tokio::select! {
497            _ = shutdown.cancelled() => {
498                drain_dropped_queue(&mut rx, &counters);
499                break;
500            },
501            received = rx.recv() => match received {
502                Some(batch) => batch,
503                None => break,
504            },
505        };
506        let count = batch.count;
507        let payload = match serde_json::to_vec(&batch.request) {
508            Ok(payload) => payload,
509            Err(error) => {
510                counters.dropped_batches.fetch_add(1, Ordering::Relaxed);
511                counters.dropped_events.fetch_add(count, Ordering::Relaxed);
512                tracing::error!(error = %error, "events forwarder serialization failed; batch dropped");
513                continue;
514            }
515        };
516
517        // The delivery itself must stay both cancellable and bounded: a
518        // connected but non-responding daemon would otherwise park this
519        // tracked task inside write_frame/read_frame, beyond the reach of the
520        // recv-side shutdown select — and daemon drain would wait its full
521        // timeout on it. Shutdown mid-delivery abandons the batch (the
522        // lane's loss-tolerant contract); a timeout poisons the connection,
523        // since the peer may answer the abandoned frame later.
524        let delivered = tokio::select! {
525            _ = shutdown.cancelled() => {
526                counters.dropped_batches.fetch_add(1, Ordering::Relaxed);
527                counters.dropped_events.fetch_add(count, Ordering::Relaxed);
528                tracing::warn!(
529                    dropped_events = count,
530                    "events forwarder shutting down mid-delivery; in-flight batch dropped"
531                );
532                drain_dropped_queue(&mut rx, &counters);
533                break;
534            },
535            outcome = tokio::time::timeout(
536                delivery_timeout,
537                deliver_batch(&socket_path, &mut conn, &payload),
538            ) => match outcome {
539                Ok(delivered) => delivered,
540                Err(_elapsed) => {
541                    conn = None;
542                    false
543                }
544            },
545        };
546        if delivered {
547            counters.forwarded_batches.fetch_add(1, Ordering::Relaxed);
548            counters
549                .forwarded_events
550                .fetch_add(count, Ordering::Relaxed);
551            if outage_logged.swap(false, Ordering::Relaxed) {
552                tracing::info!("events daemon reachable again; forwarding resumed");
553            }
554        } else {
555            counters.dropped_batches.fetch_add(1, Ordering::Relaxed);
556            counters.dropped_events.fetch_add(count, Ordering::Relaxed);
557            if !outage_logged.swap(true, Ordering::Relaxed) {
558                tracing::warn!(
559                    socket = %socket_path.display(),
560                    "events daemon unreachable; dropping loss-tolerant events until it returns"
561                );
562            }
563            tokio::select! {
564                _ = shutdown.cancelled() => {
565                    drain_dropped_queue(&mut rx, &counters);
566                    break;
567                },
568                _ = tokio::time::sleep(FORWARDER_BACKOFF) => {}
569            }
570        }
571    }
572}
573
574/// Try to deliver one framed append over the forwarder connection,
575/// (re)connecting at most once. Returns whether the daemon acknowledged.
576/// Connections are peer-verified: a socket answered by a foreign uid is a
577/// failed connect, never a delivery target.
578#[cfg(unix)]
579pub(super) async fn deliver_batch(
580    socket_path: &Path,
581    conn: &mut Option<UnixStream>,
582    payload: &[u8],
583) -> bool {
584    for _attempt in 0..2u8 {
585        if conn.is_none() {
586            match connect_verified(socket_path).await {
587                Ok(stream) => *conn = Some(stream),
588                Err(_) => return false,
589            }
590        }
591        let stream = conn.as_mut().expect("connection populated above");
592        let ok = async {
593            write_frame(stream, payload).await?;
594            let bytes = read_frame(stream).await?;
595            std::io::Result::Ok(bytes)
596        }
597        .await;
598        match ok {
599            Ok(bytes) => {
600                return !matches!(
601                    serde_json::from_slice::<EventsResponse>(&bytes),
602                    Ok(EventsResponse::Error { .. }) | Err(_)
603                );
604            }
605            Err(_) => {
606                // Stale connection (daemon restarted): drop it and retry once
607                // with a fresh connect; a second failure is a real outage.
608                *conn = None;
609            }
610        }
611    }
612    false
613}