Skip to main content

orbit_events/
lib.rs

1//! `FleetEventBus` — append-only event stream over one Orbit ring.
2//!
3//! Events are not cache entries and not metrics snapshots. Cache reads
4//! ask "what is the newest value for this key?" Metrics reads ask "what
5//! is the newest sample for each node?" Event reads ask "which frames
6//! have appeared since my cursor?"
7//!
8//! One event SHM segment carries all topics, with one writer lane per fleet
9//! node. The topic is carried inside the frame payload, so adding a new event
10//! type does not allocate another SHM segment. Subscribers keep one cursor per
11//! node lane and do not assume a total order across nodes.
12
13use std::marker::PhantomData;
14use std::sync::Arc;
15
16use bytes::{BufMut, Bytes, BytesMut};
17#[cfg(any(
18    target_os = "linux",
19    target_os = "freebsd",
20    target_os = "macos"
21))]
22use orbit_core::RingEventFd;
23use orbit_core::fleet::FleetLaneCursor;
24use orbit_core::{Fleet, NetId64, NodeId, OrbitEpoch};
25
26mod layout;
27
28use layout::EventRecord;
29pub use layout::{
30    DefaultEventLayout, EVENT_PAYLOAD_MAX, EVENT_RING_CAPACITY,
31    EVENT_RING_KIND, EVENT_RING_PAYLOAD_CAPACITY, EVENT_RING_SPEC, EventLayout,
32};
33
34pub type Result<T, E = Error> = std::result::Result<T, E>;
35
36/// Failures produced by the Orbit event semantic layer.
37#[derive(Debug)]
38pub enum Error {
39    /// A topic and payload envelope cannot fit in one configured event frame.
40    FrameTooLarge { topic_len: usize, payload_len: usize, max_payload: usize },
41    /// The underlying Orbit ring operation failed.
42    Io(std::io::Error),
43    /// A custom [`EventLayout`] names a ring this bus cannot use.
44    InvalidLayout(&'static str),
45}
46
47impl std::fmt::Display for Error {
48    fn fmt(
49        &self,
50        f: &mut std::fmt::Formatter<'_>,
51    ) -> std::fmt::Result {
52        match self {
53            Self::FrameTooLarge { topic_len, payload_len, max_payload } => {
54                write!(
55                    f,
56                    "orbit event frame too large: topic_len={topic_len} payload_len={payload_len} max_payload={max_payload}"
57                )
58            }
59            Self::Io(error) => write!(f, "orbit event io error: {error}"),
60            Self::InvalidLayout(reason) => {
61                write!(f, "orbit event layout invalid: {reason}")
62            }
63        }
64    }
65}
66
67impl std::error::Error for Error {
68    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
69        match self {
70            Self::Io(error) => Some(error),
71            Self::FrameTooLarge { .. } | Self::InvalidLayout(_) => None,
72        }
73    }
74}
75
76const HEADER_LEN: usize = 2 + 2 + 8;
77const FRAME_KIND_EVENT: u8 = 1;
78
79/// Cursor for one event subscriber.
80///
81/// The cursor stores one next counter per node lane. It is intentionally
82/// caller-owned so different consumers can advance independently.
83#[derive(Clone, Debug, PartialEq, Eq)]
84pub struct FleetEventCursor {
85    inner: FleetLaneCursor,
86}
87
88impl FleetEventCursor {
89    /// Start from the beginning of the ring history that is still
90    /// available.
91    pub const fn from_start() -> Self {
92        Self { inner: FleetLaneCursor::from_counter(0) }
93    }
94
95    /// Start every node lane at the same lane-local next counter.
96    ///
97    /// This does not restore independently advanced lane positions. Retain or
98    /// clone the cursor to preserve them; a numerical minimum is not a checkpoint.
99    pub const fn from_counter(next_counter: u64) -> Self {
100        Self { inner: FleetLaneCursor::from_counter(next_counter) }
101    }
102
103    /// Numerical minimum of the lane-local next counters, including idle lanes.
104    ///
105    /// Each physical node lane advances independently. This is not a global
106    /// event cursor, event order, completion watermark, or multi-lane checkpoint.
107    /// An idle lane at zero keeps this value at zero even when all available
108    /// events have been consumed. Retain or clone this cursor to preserve each
109    /// lane's position; passing the minimum to [`Self::from_counter`] can replay
110    /// already consumed events. This reads only caller-owned cursor memory.
111    pub fn minimum_lane_next_counter(&self) -> u64 {
112        self.inner.minimum_lane_next_counter()
113    }
114
115    /// Numerical minimum of independent lane-local counters, not a global cursor.
116    #[deprecated(
117        note = "use minimum_lane_next_counter; this is not a global cursor or checkpoint"
118    )]
119    pub fn next_counter(&self) -> u64 {
120        self.minimum_lane_next_counter()
121    }
122}
123
124/// One decoded event from the shared event ring.
125#[derive(Clone, Debug, PartialEq, Eq)]
126pub struct FleetEvent {
127    pub id: NetId64,
128    pub topic: String,
129    pub payload: Vec<u8>,
130    pub timestamp_ms: u64,
131}
132
133/// Result of polling an event cursor.
134#[derive(Clone, Debug, Default, PartialEq, Eq)]
135pub struct FleetEventPoll {
136    pub events: Vec<FleetEvent>,
137    /// Number of counters that could not be delivered because the
138    /// subscriber lagged past the ring capacity, a slot was empty, or a
139    /// slot had already wrapped to another counter.
140    pub lagged: u64,
141}
142
143impl FleetEventPoll {
144    pub fn is_empty(&self) -> bool {
145        self.events.is_empty() && self.lagged == 0
146    }
147}
148
149/// Fleet-shared raw event bus over the ring named by `L`. Cheap to clone.
150pub struct FleetEventBus<L: EventLayout = DefaultEventLayout> {
151    fleet: Arc<Fleet>,
152    _layout: PhantomData<L>,
153}
154
155impl<L: EventLayout> Clone for FleetEventBus<L> {
156    fn clone(&self) -> Self {
157        Self { fleet: Arc::clone(&self.fleet), _layout: PhantomData }
158    }
159}
160
161impl FleetEventBus<DefaultEventLayout> {
162    /// A bus over the fleet's default event ring.
163    pub fn new(fleet: Arc<Fleet>) -> Self {
164        Self { fleet, _layout: PhantomData }
165    }
166}
167
168impl<L: EventLayout> FleetEventBus<L> {
169    /// Largest frame this bus can carry: the topic, the payload and the
170    /// frame header together.
171    pub const PAYLOAD_MAX: usize = L::RING_SPEC.payload_capacity;
172
173    /// A bus over the ring named by `L`.
174    pub fn with_layout(fleet: Arc<Fleet>) -> Result<Self> {
175        layout::validate::<L>()?;
176        Ok(Self { fleet, _layout: PhantomData })
177    }
178
179    pub fn node_id(&self) -> NodeId {
180        self.fleet.node_id()
181    }
182
183    /// Cursor that starts after all events currently in the ring.
184    /// Useful for subscribers that only want future events.
185    pub fn cursor_at_head(&self) -> FleetEventCursor {
186        FleetEventCursor {
187            inner: self.fleet.lane_cursor_at_head::<EventRecord<L>>(),
188        }
189    }
190
191    /// Cursor that starts at counter 0 and replays whatever history has
192    /// not wrapped out of the ring.
193    pub const fn cursor_from_start(&self) -> FleetEventCursor {
194        let _ = self;
195        FleetEventCursor::from_start()
196    }
197
198    /// Clear the shared event ring.
199    ///
200    /// Intended for owner-controlled boot-time cleanup before peer
201    /// processes publish events into the ring. This prevents stale
202    /// events from a previous process lifetime from being replayed or
203    /// counted as current runtime state.
204    pub fn reset_ring(&self) -> Result<()> {
205        self.fleet.reset_ring::<EventRecord<L>>().map_err(Error::Io)
206    }
207
208    /// Create this process' native fd readiness bridge for the event ring.
209    ///
210    /// The returned fd is local to this process. Publishers notify a shared
211    /// generation after committing an event; the bridge converts that
212    /// broadcast into local fd readiness suitable for epoll/kqueue/AsyncFd.
213    /// Drain the fd, then poll the ring with this subscriber's cursor.
214    #[cfg(any(
215        target_os = "linux",
216        target_os = "freebsd",
217        target_os = "macos"
218    ))]
219    pub fn event_fd(&self) -> Result<RingEventFd> {
220        self.fleet.ring_event_fd::<EventRecord<L>>().map_err(Error::Io)
221    }
222
223    /// Publish one event under `topic`.
224    pub fn publish(
225        &self,
226        topic: &str,
227        payload: &[u8],
228    ) -> Result<NetId64> {
229        let timestamp = OrbitEpoch::now();
230        let frame = encode_frame(
231            topic.as_bytes(),
232            payload,
233            timestamp,
234            Self::PAYLOAD_MAX,
235        )?;
236        #[cfg(any(
237            target_os = "linux",
238            target_os = "freebsd",
239            target_os = "macos"
240        ))]
241        let id = self
242            .fleet
243            .publish_notified::<EventRecord<L>>(
244                FRAME_KIND_EVENT,
245                timestamp.as_unix_ms(),
246                frame,
247            )
248            .map_err(Error::Io)?;
249        #[cfg(not(any(
250            target_os = "linux",
251            target_os = "freebsd",
252            target_os = "macos"
253        )))]
254        let id = self.fleet.publish::<EventRecord<L>>(
255            FRAME_KIND_EVENT,
256            timestamp.as_unix_ms(),
257            frame,
258        );
259        Ok(id)
260    }
261
262    /// Poll all events since `cursor`, advancing the cursor to the
263    /// current ring head. If the cursor has fallen behind the ring
264    /// capacity, older overwritten counters are reported as `lagged`.
265    pub fn poll(
266        &self,
267        cursor: &mut FleetEventCursor,
268    ) -> FleetEventPoll {
269        self.poll_where(cursor, |_| true)
270    }
271
272    /// Poll, building an event only for frames whose topic `keep` takes; a
273    /// skipped frame costs a comparison, not a copy of its topic and payload.
274    fn poll_where(
275        &self,
276        cursor: &mut FleetEventCursor,
277        keep: impl Fn(&[u8]) -> bool,
278    ) -> FleetEventPoll {
279        let ring_poll =
280            self.fleet.poll_lanes::<EventRecord<L>>(&mut cursor.inner);
281        let mut lagged = ring_poll.loss.total();
282        let mut events = Vec::new();
283        for frame in ring_poll.frames {
284            let Some(decoded) = decode_frame(&frame.payload) else {
285                lagged = lagged.saturating_add(1);
286                continue;
287            };
288            if !keep(decoded.topic) {
289                continue;
290            }
291            events.push(FleetEvent {
292                id: frame.id,
293                topic: String::from_utf8_lossy(decoded.topic).into_owned(),
294                payload: decoded.payload.to_vec(),
295                timestamp_ms: decoded.timestamp.as_unix_ms(),
296            });
297        }
298
299        FleetEventPoll { events, lagged }
300    }
301
302    /// Poll and keep only events whose topic matches `topic`.
303    ///
304    /// The cursor still advances past all events, including filtered
305    /// topics. Use separate cursors for independent consumers.
306    pub fn poll_topic(
307        &self,
308        cursor: &mut FleetEventCursor,
309        topic: &str,
310    ) -> FleetEventPoll {
311        self.poll_where(cursor, |candidate| candidate == topic.as_bytes())
312    }
313}
314
315struct DecodedFrame<'a> {
316    topic: &'a [u8],
317    payload: &'a [u8],
318    timestamp: OrbitEpoch,
319}
320
321fn encode_frame(
322    topic: &[u8],
323    payload: &[u8],
324    timestamp: OrbitEpoch,
325    max_payload: usize,
326) -> Result<Bytes> {
327    let total = HEADER_LEN + topic.len() + payload.len();
328    if topic.len() > u16::MAX as usize
329        || payload.len() > u16::MAX as usize
330        || total > max_payload
331    {
332        return Err(Error::FrameTooLarge {
333            topic_len: topic.len(),
334            payload_len: payload.len(),
335            max_payload,
336        });
337    }
338
339    let mut buf = BytesMut::with_capacity(total);
340    buf.put_u16_le(topic.len() as u16);
341    buf.put_u16_le(payload.len() as u16);
342    buf.put_u64_le(timestamp.as_unix_ms());
343    buf.put_slice(topic);
344    buf.put_slice(payload);
345    Ok(buf.freeze())
346}
347
348fn decode_frame(payload: &Bytes) -> Option<DecodedFrame<'_>> {
349    if payload.len() < HEADER_LEN {
350        return None;
351    }
352
353    let topic_len = u16::from_le_bytes(payload[0..2].try_into().ok()?) as usize;
354    let payload_len =
355        u16::from_le_bytes(payload[2..4].try_into().ok()?) as usize;
356    let timestamp = OrbitEpoch::from_unix_ms(u64::from_le_bytes(
357        payload[4..12].try_into().ok()?,
358    ));
359    let topic_start = HEADER_LEN;
360    let topic_end = topic_start.checked_add(topic_len)?;
361    let payload_end = topic_end.checked_add(payload_len)?;
362    if payload.len() < payload_end {
363        return None;
364    }
365
366    Some(DecodedFrame {
367        topic: &payload[topic_start..topic_end],
368        payload: &payload[topic_end..payload_end],
369        timestamp,
370    })
371}
372
373#[cfg(test)]
374mod tests {
375    use std::sync::Arc;
376
377    use orbit_core::{Fleet, OrbitEpoch, RingSpec};
378
379    use super::{EventLayout, FleetEventBus};
380
381    struct WideLayout;
382
383    impl EventLayout for WideLayout {
384        const RING_KIND: u8 = 221;
385        const RING_SPEC: RingSpec = RingSpec::per_node(16, 8_192);
386    }
387
388    #[test]
389    fn polls_events_since_cursor() {
390        let fleet = Arc::new(Fleet::join("event_poll", 1).expect("fleet"));
391        let bus = FleetEventBus::new(fleet);
392        let mut cursor = bus.cursor_from_start();
393
394        bus.publish("worker.booted", b"w1").expect("publish first");
395        bus.publish("cache.reset", b"all").expect("publish second");
396
397        let poll = bus.poll(&mut cursor);
398        assert_eq!(poll.lagged, 0);
399        assert_eq!(poll.events.len(), 2);
400        assert_eq!(poll.events[0].topic, "worker.booted");
401        assert_eq!(poll.events[0].payload, b"w1");
402        assert_eq!(poll.events[1].topic, "cache.reset");
403        assert_eq!(poll.events[1].payload, b"all");
404
405        assert!(bus.poll(&mut cursor).is_empty());
406    }
407
408    #[test]
409    fn minimum_lane_next_counter_can_stay_zero_after_all_events_are_read() {
410        let fleet = Arc::new(Fleet::join("event_lane_min", 2).expect("fleet"));
411        let bus = FleetEventBus::new(fleet);
412        let mut cursor = bus.cursor_from_start();
413
414        for _ in 0..3 {
415            bus.publish("test.event", b"payload").expect("publish");
416        }
417
418        let poll = bus.poll(&mut cursor);
419        assert_eq!(poll.events.len(), 3);
420        assert_eq!(poll.lagged, 0);
421        assert_eq!(cursor.minimum_lane_next_counter(), 0);
422        assert!(bus.poll(&mut cursor).is_empty());
423
424        bus.publish("test.event", b"next").expect("publish");
425        let poll = bus.poll(&mut cursor);
426        assert_eq!(poll.events.len(), 1);
427        assert_eq!(poll.events[0].payload, b"next");
428        assert_eq!(poll.lagged, 0);
429        assert_eq!(cursor.minimum_lane_next_counter(), 0);
430    }
431
432    #[test]
433    fn cursor_at_head_only_reads_future_events() {
434        let fleet = Arc::new(Fleet::join("event_head", 1).expect("fleet"));
435        let bus = FleetEventBus::new(fleet);
436
437        bus.publish("old", b"ignored").expect("publish old");
438        let mut cursor = bus.cursor_at_head();
439        bus.publish("new", b"seen").expect("publish new");
440
441        let poll = bus.poll(&mut cursor);
442        assert_eq!(poll.events.len(), 1);
443        assert_eq!(poll.events[0].topic, "new");
444    }
445
446    #[test]
447    fn topic_filter_advances_cursor() {
448        let fleet = Arc::new(Fleet::join("event_topic", 1).expect("fleet"));
449        let bus = FleetEventBus::new(fleet);
450        let mut cursor = bus.cursor_from_start();
451
452        bus.publish("a", b"1").expect("publish a");
453        bus.publish("b", b"2").expect("publish b");
454
455        let poll = bus.poll_topic(&mut cursor, "b");
456        assert_eq!(poll.events.len(), 1);
457        assert_eq!(poll.events[0].payload, b"2");
458        assert!(bus.poll(&mut cursor).is_empty());
459    }
460
461    #[test]
462    fn event_frame_contract_uses_the_compile_time_geometry() {
463        let expected = orbit_core::compile::usize_from_env(
464            option_env!("ORBIT_EVENT_RING_PAYLOAD_CAPACITY"),
465            512,
466        );
467        assert_eq!(super::EVENT_PAYLOAD_MAX, expected);
468
469        let largest_payload =
470            vec![0_u8; super::EVENT_PAYLOAD_MAX - super::HEADER_LEN - 1];
471        let frame = super::encode_frame(
472            b"x",
473            &largest_payload,
474            OrbitEpoch::ZERO,
475            expected,
476        )
477        .expect("frame must fit");
478        assert_eq!(frame.len(), super::EVENT_PAYLOAD_MAX);
479
480        let oversized_payload = vec![0_u8; largest_payload.len() + 1];
481        assert!(matches!(
482            super::encode_frame(b"x", &oversized_payload, OrbitEpoch::ZERO, expected),
483            Err(super::Error::FrameTooLarge {
484                max_payload,
485                ..
486            }) if max_payload == expected
487        ));
488    }
489
490    #[test]
491    fn reports_lag_when_cursor_falls_behind_capacity() {
492        let fleet = Arc::new(Fleet::join("event_lag", 1).expect("fleet"));
493        let bus = FleetEventBus::new(fleet);
494        let mut cursor = bus.cursor_from_start();
495
496        for value in 0..=super::EVENT_RING_SPEC.capacity {
497            bus.publish("n", value.to_string().as_bytes())
498                .expect("publish event");
499        }
500
501        let poll = bus.poll(&mut cursor);
502        assert_eq!(poll.lagged, 1);
503        assert_eq!(poll.events.len(), super::EVENT_RING_SPEC.capacity);
504        assert_eq!(poll.events[0].payload, b"1");
505    }
506
507    #[test]
508    fn a_second_layout_is_a_separate_ring_with_its_own_frame_size() {
509        let fleet = Arc::new(Fleet::join("event_layout", 1).expect("fleet"));
510        let fleet_bus = FleetEventBus::new(Arc::clone(&fleet));
511        let wide_bus =
512            FleetEventBus::<WideLayout>::with_layout(fleet).expect("layout");
513        let mut fleet_cursor = fleet_bus.cursor_from_start();
514        let mut wide_cursor = wide_bus.cursor_from_start();
515
516        let wide_payload = vec![7_u8; super::EVENT_PAYLOAD_MAX];
517        assert!(fleet_bus.publish("wide", &wide_payload).is_err());
518        wide_bus.publish("wide", &wide_payload).expect("wide frame fits");
519        fleet_bus.publish("small", b"1").expect("small frame fits");
520
521        let wide = wide_bus.poll(&mut wide_cursor);
522        assert_eq!(wide.events.len(), 1);
523        assert_eq!(wide.events[0].payload, wide_payload);
524
525        let small = fleet_bus.poll(&mut fleet_cursor);
526        assert_eq!(small.events.len(), 1);
527        assert_eq!(small.events[0].topic, "small");
528    }
529}