Skip to main content

rustfs_notify/
pipeline.rs

1// Copyright 2024 RustFS Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use crate::{Event, integration::LiveEventBatch, notifier::EventNotifier};
16use std::collections::VecDeque;
17use std::sync::Arc;
18use tokio::sync::{RwLock, broadcast};
19
20const MAX_RECENT_LIVE_EVENTS: usize = 1024;
21
22#[derive(Default)]
23pub struct LiveEventHistory {
24    next_sequence: u64,
25    events: VecDeque<(u64, Arc<Event>)>,
26}
27
28impl LiveEventHistory {
29    pub fn record(&mut self, event: Arc<Event>) {
30        self.next_sequence = self.next_sequence.saturating_add(1);
31        self.events.push_back((self.next_sequence, event));
32        while self.events.len() > MAX_RECENT_LIVE_EVENTS {
33            self.events.pop_front();
34        }
35    }
36
37    pub fn snapshot_since(&self, after_sequence: u64, limit: usize) -> LiveEventBatch {
38        let mut events = Vec::new();
39        let mut next_sequence = after_sequence;
40        let mut truncated = false;
41
42        // A gap means the consumer's cursor points before the oldest event we
43        // still retain: the events in between were evicted from the ring buffer
44        // and can never be delivered. Report it so the consumer knows it lost
45        // events instead of silently resuming from the oldest available one.
46        let gap = match self.events.front() {
47            Some((oldest, _)) => after_sequence.saturating_add(1) < *oldest,
48            None => false,
49        };
50
51        for (sequence, event) in self.events.iter() {
52            if *sequence <= after_sequence {
53                continue;
54            }
55            if events.len() >= limit {
56                truncated = true;
57                break;
58            }
59            next_sequence = *sequence;
60            events.push(event.clone());
61        }
62
63        LiveEventBatch {
64            events,
65            next_sequence,
66            truncated,
67            gap,
68        }
69    }
70}
71
72#[derive(Clone)]
73pub struct NotifyPipeline {
74    notifier: Arc<EventNotifier>,
75    live_event_sender: broadcast::Sender<Arc<Event>>,
76    live_event_history: Arc<RwLock<LiveEventHistory>>,
77}
78
79impl NotifyPipeline {
80    pub fn new(
81        notifier: Arc<EventNotifier>,
82        live_event_sender: broadcast::Sender<Arc<Event>>,
83        live_event_history: Arc<RwLock<LiveEventHistory>>,
84    ) -> Self {
85        Self {
86            notifier,
87            live_event_sender,
88            live_event_history,
89        }
90    }
91
92    pub fn has_live_listeners(&self) -> bool {
93        self.live_event_sender.receiver_count() > 0
94    }
95
96    pub fn subscribe_live_events(&self) -> broadcast::Receiver<Arc<Event>> {
97        self.live_event_sender.subscribe()
98    }
99
100    pub async fn recent_live_events_since(&self, after_sequence: u64, limit: usize) -> LiveEventBatch {
101        let history = self.live_event_history.read().await;
102        history.snapshot_since(after_sequence, limit.max(1))
103    }
104
105    pub async fn send_event(&self, event: Arc<Event>) {
106        // Assign the history sequence number and publish to live subscribers under
107        // the *same* critical section. `record` allocates a monotonically increasing
108        // sequence and `broadcast::Sender::send` is synchronous, so holding the write
109        // guard across both guarantees the broadcast order matches the recorded
110        // sequence order. Otherwise two concurrent senders could record seq 1 then 2
111        // but broadcast 2 then 1, so a cursor-based consumer would observe events out
112        // of order relative to the replayable history (backlog#984 / #969).
113        {
114            let mut history = self.live_event_history.write().await;
115            history.record(event.clone());
116            let _ = self.live_event_sender.send(event.clone());
117        }
118        self.notifier.send(event).await;
119    }
120}
121
122pub type NotifyEventBridge = NotifyPipeline;
123
124#[cfg(test)]
125mod tests {
126    use super::{LiveEventHistory, MAX_RECENT_LIVE_EVENTS, NotifyPipeline};
127    use crate::{Event, integration::NotificationMetrics, notifier::EventNotifier, rule_engine::NotifyRuleEngine};
128    use rustfs_s3_types::EventName;
129    use std::sync::Arc;
130    use tokio::sync::{RwLock, broadcast};
131
132    fn build_pipeline() -> NotifyPipeline {
133        let metrics = Arc::new(NotificationMetrics::new());
134        let notifier = Arc::new(EventNotifier::new(metrics, NotifyRuleEngine::new()));
135        let (live_event_sender, _) = broadcast::channel(16);
136        NotifyPipeline::new(notifier, live_event_sender, Arc::new(RwLock::new(LiveEventHistory::default())))
137    }
138
139    #[tokio::test]
140    async fn pipeline_reports_live_listener_subscription_state() {
141        let pipeline = build_pipeline();
142        assert!(!pipeline.has_live_listeners());
143
144        let _rx = pipeline.subscribe_live_events();
145        assert!(pipeline.has_live_listeners());
146    }
147
148    #[tokio::test]
149    async fn pipeline_records_recent_live_events() {
150        let pipeline = build_pipeline();
151        let event = Arc::new(Event::new_test_event("bucket", "one", EventName::ObjectCreatedPut));
152
153        pipeline.send_event(event).await;
154
155        let batch = pipeline.recent_live_events_since(0, 16).await;
156        assert_eq!(batch.next_sequence, 1);
157        assert!(!batch.truncated);
158        assert!(!batch.gap);
159        assert_eq!(batch.events.len(), 1);
160        assert_eq!(batch.events[0].s3.object.key, "one");
161    }
162
163    #[test]
164    fn snapshot_reports_gap_when_cursor_predates_evicted_history() {
165        let mut history = LiveEventHistory::default();
166        // Fill past the ring-buffer capacity so the oldest events are evicted.
167        let total = MAX_RECENT_LIVE_EVENTS + 128;
168        for i in 1..=total {
169            history.record(Arc::new(Event::new_test_event("bucket", &i.to_string(), EventName::ObjectCreatedPut)));
170        }
171
172        let oldest_retained = (total - MAX_RECENT_LIVE_EVENTS + 1) as u64;
173
174        // A consumer whose cursor sits inside the evicted range must be told a
175        // gap exists instead of silently resuming from the oldest available.
176        let batch = history.snapshot_since(10, 4096);
177        assert!(batch.gap, "cursor predating evicted history must report a gap");
178        assert_eq!(batch.events.first().unwrap().s3.object.key, oldest_retained.to_string());
179
180        // A cursor contiguous with the oldest retained event must not report a gap.
181        let contiguous = history.snapshot_since(oldest_retained - 1, 4096);
182        assert!(!contiguous.gap, "cursor contiguous with retained history must not report a gap");
183
184        // A caught-up cursor must not report a gap either.
185        let caught_up = history.snapshot_since(total as u64, 4096);
186        assert!(!caught_up.gap);
187        assert!(caught_up.events.is_empty());
188    }
189}