Skip to main content

rustfs_notify/
integration.rs

1// Copyright 2024 RustFS Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use crate::notification_system_subscriber::NotificationSystemSubscriberView;
16use crate::notifier::{EventNotifier, TargetList};
17use crate::services::NotifyServices;
18use crate::{
19    Event,
20    error::NotificationError,
21    lifecycle::{NotificationLifecycleTransition, NotificationRuntimeState},
22    pipeline::LiveEventHistory,
23    registry::TargetRegistry,
24    rule_engine::NotifyRuleEngine,
25    rules::BucketNotificationConfig,
26};
27use hashbrown::HashMap;
28use metrics::{counter, gauge};
29use rustfs_config::notify::{DEFAULT_NOTIFY_TARGET_STREAM_CONCURRENCY, ENV_NOTIFY_TARGET_STREAM_CONCURRENCY};
30use rustfs_config::server_config::{Config, KVS};
31use rustfs_s3_types::EventName;
32use rustfs_targets::arn::TargetID;
33use rustfs_targets::{ReplayWorkerManager, RuntimeTargetHealthSnapshot, SharedTarget};
34use std::sync::Arc;
35use std::sync::atomic::{AtomicUsize, Ordering};
36use std::time::{Duration, Instant};
37use tokio::sync::{RwLock, Semaphore, broadcast};
38use tracing::info;
39
40const METRIC_NOTIFICATION_CURRENT_SEND_IN_PROGRESS: &str = "rustfs_notification_current_send_in_progress";
41const METRIC_NOTIFICATION_EVENTS_ERRORS_TOTAL: &str = "rustfs_notification_events_errors_total";
42const METRIC_NOTIFICATION_EVENTS_SENT_TOTAL: &str = "rustfs_notification_events_sent_total";
43const METRIC_NOTIFICATION_EVENTS_SKIPPED_TOTAL: &str = "rustfs_notification_events_skipped_total";
44const LOG_COMPONENT_NOTIFY: &str = "notify";
45const LOG_SUBSYSTEM_INTEGRATION: &str = "integration";
46const EVENT_NOTIFY_SYSTEM_DROP: &str = "notify_system_drop";
47const EVENT_NOTIFY_SYSTEM_SHUTDOWN_METRIC: &str = "notify_system_shutdown_metric";
48const EVENT_NOTIFY_SYSTEM_STATUS_SNAPSHOT: &str = "notify_system_status_snapshot";
49
50#[derive(Clone)]
51pub struct LiveEventBatch {
52    pub events: Vec<Arc<Event>>,
53    pub next_sequence: u64,
54    pub truncated: bool,
55    /// Set when the consumer's cursor is older than the oldest event still
56    /// retained in the ring buffer: events between the cursor and the oldest
57    /// retained sequence were evicted and can never be delivered. Consumers
58    /// must treat a `gap` as lost events (e.g. trigger a full re-sync/alert)
59    /// instead of assuming the returned batch is contiguous with their cursor.
60    pub gap: bool,
61}
62
63/// Notify the system of monitoring indicators
64pub struct NotificationMetrics {
65    /// The number of events currently being processed
66    processing_events: AtomicUsize,
67    /// Number of events that have been successfully processed
68    processed_events: AtomicUsize,
69    /// Number of events that failed to handle
70    failed_events: AtomicUsize,
71    /// Number of dispatch attempts skipped before delivery
72    skipped_events: AtomicUsize,
73    /// System startup time
74    start_time: Instant,
75}
76
77#[derive(Debug, Clone, Default, PartialEq, Eq)]
78pub struct NotificationMetricSnapshot {
79    pub current_send_in_progress: u64,
80    pub events_errors_total: u64,
81    pub events_sent_total: u64,
82    pub events_skipped_total: u64,
83}
84
85#[derive(Debug, Clone, Default, PartialEq, Eq)]
86pub struct NotificationTargetMetricSnapshot {
87    pub failed_messages: u64,
88    pub failed_store_length: u64,
89    pub queue_length: u64,
90    pub target_id: String,
91    pub target_type: String,
92    pub total_messages: u64,
93}
94
95impl Default for NotificationMetrics {
96    fn default() -> Self {
97        Self::new()
98    }
99}
100
101impl NotificationMetrics {
102    pub fn new() -> Self {
103        NotificationMetrics {
104            processing_events: AtomicUsize::new(0),
105            processed_events: AtomicUsize::new(0),
106            failed_events: AtomicUsize::new(0),
107            skipped_events: AtomicUsize::new(0),
108            start_time: Instant::now(),
109        }
110    }
111
112    // Provide public methods to increase count
113    pub fn increment_processing(&self) {
114        self.processing_events.fetch_add(1, Ordering::Relaxed);
115    }
116
117    pub fn increment_processed(&self) {
118        self.processing_events.fetch_sub(1, Ordering::Relaxed);
119        self.processed_events.fetch_add(1, Ordering::Relaxed);
120    }
121
122    pub fn decrement_processing(&self) {
123        self.processing_events.fetch_sub(1, Ordering::Relaxed);
124    }
125
126    pub fn increment_failed(&self) {
127        self.processing_events.fetch_sub(1, Ordering::Relaxed);
128        self.failed_events.fetch_add(1, Ordering::Relaxed);
129    }
130
131    pub fn increment_skipped(&self) {
132        self.skipped_events.fetch_add(1, Ordering::Relaxed);
133    }
134
135    // Provide public methods to get count
136    pub fn processing_count(&self) -> usize {
137        self.processing_events.load(Ordering::Relaxed)
138    }
139
140    pub fn processed_count(&self) -> usize {
141        self.processed_events.load(Ordering::Relaxed)
142    }
143
144    pub fn failed_count(&self) -> usize {
145        self.failed_events.load(Ordering::Relaxed)
146    }
147
148    pub fn skipped_count(&self) -> usize {
149        self.skipped_events.load(Ordering::Relaxed)
150    }
151
152    pub fn uptime(&self) -> Duration {
153        self.start_time.elapsed()
154    }
155
156    pub fn snapshot(&self) -> NotificationMetricSnapshot {
157        NotificationMetricSnapshot {
158            current_send_in_progress: self.processing_count() as u64,
159            events_errors_total: self.failed_count() as u64,
160            events_sent_total: self.processed_count() as u64,
161            events_skipped_total: self.skipped_count() as u64,
162        }
163    }
164}
165
166/// The notification system that integrates all components
167pub struct NotificationSystem {
168    /// Event dispatcher. Runtime target mutation remains lifecycle-owned.
169    pub notifier: Arc<EventNotifier>,
170    /// Target factory registry. Creating a target does not publish it.
171    pub registry: Arc<TargetRegistry>,
172    /// The current cached configuration. Runtime publication must still go
173    /// through the lifecycle methods on this type.
174    pub config: Arc<RwLock<Config>>,
175    services: NotifyServices,
176}
177
178impl NotificationSystem {
179    /// Creates a new NotificationSystem
180    pub fn new(config: Config) -> Self {
181        let concurrency_limiter =
182            rustfs_utils::get_env_usize(ENV_NOTIFY_TARGET_STREAM_CONCURRENCY, DEFAULT_NOTIFY_TARGET_STREAM_CONCURRENCY);
183        let (live_event_sender, _) = broadcast::channel(1024);
184        let metrics = Arc::new(NotificationMetrics::new());
185        let subscriber_view = Arc::new(NotificationSystemSubscriberView::new());
186        let rule_engine = NotifyRuleEngine::new();
187        let notifier = Arc::new(EventNotifier::new(metrics.clone(), rule_engine.clone()));
188        let target_list = notifier.target_list();
189        let registry = Arc::new(TargetRegistry::new());
190        let config = Arc::new(RwLock::new(config));
191        let stream_cancellers = Arc::new(RwLock::new(ReplayWorkerManager::new()));
192        let concurrency_limiter = Arc::new(Semaphore::new(concurrency_limiter)); // Limit the maximum number of concurrent processing events to 20
193        let live_event_history = Arc::new(RwLock::new(LiveEventHistory::default()));
194        let services = NotifyServices::new(
195            notifier.clone(),
196            rule_engine,
197            target_list,
198            registry.clone(),
199            config.clone(),
200            stream_cancellers,
201            concurrency_limiter,
202            metrics,
203            subscriber_view,
204            live_event_sender,
205            live_event_history,
206        );
207
208        NotificationSystem {
209            notifier,
210            registry,
211            config,
212            services,
213        }
214    }
215
216    /// Initializes the notification system
217    pub async fn init(&self) -> Result<(), NotificationError> {
218        self.services.config_manager.init().await
219    }
220
221    /// Gets a list of Targets for all currently active (initialized).
222    ///
223    /// # Return
224    /// A Vec containing all active Targets `TargetID`.
225    pub async fn get_active_targets(&self) -> Vec<TargetID> {
226        self.services.runtime_view.get_active_targets().await
227    }
228
229    pub async fn config_snapshot(&self) -> Config {
230        self.config.read().await.clone()
231    }
232
233    /// Gets a read-only runtime container handle. Public mutation methods on
234    /// `TargetList` are intentionally unavailable outside this crate.
235    pub async fn get_all_targets(&self) -> Arc<RwLock<TargetList>> {
236        self.services.runtime_view.get_all_targets()
237    }
238
239    /// Gets all Target values, including both active and inactive Targets.
240    ///
241    /// # Return
242    /// A Vec containing all Targets.
243    pub async fn get_target_values(&self) -> Vec<SharedTarget<Event>> {
244        self.services.runtime_view.get_target_values().await
245    }
246
247    /// Checks if there are active subscribers for the given bucket and event name.
248    pub async fn has_subscriber(&self, bucket: &str, event: &EventName) -> bool {
249        self.services.bucket_config_manager.has_subscriber(bucket, event).await
250    }
251
252    /// Returns true when at least one in-process consumer is subscribed to live events.
253    pub fn has_live_listeners(&self) -> bool {
254        self.services.pipeline.has_live_listeners()
255    }
256
257    /// Subscribes to the in-process live event stream.
258    pub fn subscribe_live_events(&self) -> broadcast::Receiver<Arc<Event>> {
259        self.services.pipeline.subscribe_live_events()
260    }
261
262    pub async fn recent_live_events_since(&self, after_sequence: u64, limit: usize) -> LiveEventBatch {
263        self.services.pipeline.recent_live_events_since(after_sequence, limit).await
264    }
265
266    /// Accurately remove a Target and its related resources through TargetID.
267    ///
268    /// This process includes:
269    /// 1. Stop the event stream associated with the Target (if present).
270    /// 2. Remove the Target instance from the activity list of Notifier.
271    /// 3. Remove the configuration item of the Target from the system configuration.
272    ///
273    /// # Parameters
274    /// * `target_id` - The unique identifier of the Target to be removed.
275    ///
276    /// # return
277    /// If successful, return `Ok(())`.
278    pub async fn remove_target(&self, target_id: &TargetID, target_type: &str) -> Result<(), NotificationError> {
279        self.services.config_manager.remove_target(target_id, target_type).await
280    }
281
282    /// Set or update a Target configuration.
283    /// If the configuration is changed, the entire notification system will be automatically reloaded to apply the changes.
284    ///
285    /// # Arguments
286    /// * `target_type` - Target type, such as "notify_webhook" or "notify_mqtt".
287    /// * `target_name` - A unique name for a Target, such as "1".
288    /// * `kvs` - The full configuration of the Target.
289    ///
290    /// # Returns
291    /// Result<(), NotificationError>
292    /// If the target configuration is successfully set, it returns Ok(()).
293    /// If the target configuration is invalid, it returns Err(NotificationError::Configuration).
294    pub async fn set_target_config(&self, target_type: &str, target_name: &str, kvs: KVS) -> Result<(), NotificationError> {
295        self.services
296            .config_manager
297            .set_target_config(target_type, target_name, kvs)
298            .await
299    }
300
301    /// Removes all notification configurations for a bucket.
302    /// If the configuration is successfully removed, the entire notification system will be automatically reloaded.
303    ///
304    /// # Arguments
305    /// * `bucket` - The name of the bucket whose notification configuration is to be removed.
306    ///
307    pub async fn remove_bucket_notification_config(&self, bucket: &str) {
308        self.services
309            .bucket_config_manager
310            .remove_bucket_notification_config(bucket)
311            .await;
312    }
313
314    /// Removes a Target configuration.
315    /// If the configuration is successfully removed, the entire notification system will be automatically reloaded.
316    ///
317    /// # Arguments
318    /// * `target_type` - Target type, such as "notify_webhook" or "notify_mqtt".
319    /// * `target_name` - A unique name for a Target, such as "1".
320    ///
321    /// # Returns
322    /// Result<(), NotificationError>
323    ///
324    /// If the target configuration is successfully removed, it returns Ok(()).
325    /// If the target configuration does not exist, it returns Ok(()) without making any changes.
326    pub async fn remove_target_config(&self, target_type: &str, target_name: &str) -> Result<(), NotificationError> {
327        self.services
328            .config_manager
329            .remove_target_config(target_type, target_name)
330            .await
331    }
332
333    /// Reloads the configuration
334    pub async fn reload_config(&self, new_config: Config) -> Result<(), NotificationError> {
335        self.services.config_manager.reload_config(new_config).await
336    }
337
338    /// Synchronously publishes a config generation without changing target mode.
339    pub fn publish_config(&self, new_config: Config) -> NotificationLifecycleTransition {
340        self.services.config_manager.lifecycle().update_config(new_config)
341    }
342
343    /// Reconciles the cached and active configuration with the persisted
344    /// server config without changing the target-runtime mode.
345    pub async fn reload_persisted_config(&self) -> Result<(), NotificationError> {
346        self.services.config_manager.reload_persisted_config().await
347    }
348
349    /// Reconciles from an explicitly selected storage context.
350    pub async fn reload_persisted_config_from_store(&self, store: Arc<crate::NotifyStore>) -> Result<(), NotificationError> {
351        self.services.config_manager.reload_persisted_config_from_store(store).await
352    }
353
354    /// Enables or suspends configured notification targets without replacing
355    /// the process-wide live-event container.
356    pub async fn set_targets_enabled(&self, enabled: bool, config: Option<Config>) -> Result<(), NotificationError> {
357        self.publish_targets_enabled(enabled, config).wait().await
358    }
359
360    /// Synchronously accepts a target-runtime mode transition. The returned
361    /// receipt can be awaited after the caller releases its persistence lock.
362    pub fn publish_targets_enabled(&self, enabled: bool, config: Option<Config>) -> NotificationLifecycleTransition {
363        self.services.config_manager.lifecycle().set_mode(enabled, config)
364    }
365
366    pub fn runtime_lifecycle_state(&self) -> NotificationRuntimeState {
367        self.services.config_manager.lifecycle().state()
368    }
369
370    /// Returns whether the latest accepted lifecycle generation is active.
371    pub fn runtime_lifecycle_is_converged(&self) -> bool {
372        self.services.config_manager.lifecycle().is_converged()
373    }
374
375    /// Validates a bucket notification configuration against the runtime targets
376    /// without publishing it, so callers can reject a request before persisting.
377    pub async fn validate_bucket_notification_config(
378        &self,
379        bucket: &str,
380        cfg: &BucketNotificationConfig,
381    ) -> Result<(), NotificationError> {
382        self.services
383            .bucket_config_manager
384            .validate_bucket_notification_config(bucket, cfg)
385            .await
386    }
387
388    /// Loads the bucket notification configuration
389    pub async fn load_bucket_notification_config(
390        &self,
391        bucket: &str,
392        cfg: &BucketNotificationConfig,
393    ) -> Result<(), NotificationError> {
394        self.services
395            .bucket_config_manager
396            .load_bucket_notification_config(bucket, cfg)
397            .await
398    }
399
400    /// Sends an event
401    pub async fn send_event(&self, event: Arc<Event>) {
402        self.services.pipeline.send_event(event).await;
403    }
404
405    /// Obtain system status information
406    pub fn get_status(&self) -> HashMap<String, String> {
407        self.services.status_view.get_status()
408    }
409
410    pub fn snapshot_metrics(&self) -> NotificationMetricSnapshot {
411        self.services.status_view.snapshot_metrics()
412    }
413
414    pub async fn snapshot_target_metrics(&self) -> Vec<NotificationTargetMetricSnapshot> {
415        self.services.runtime_view.snapshot_target_metrics().await
416    }
417
418    pub async fn snapshot_target_health(&self) -> Vec<RuntimeTargetHealthSnapshot> {
419        self.services.runtime_view.snapshot_target_health().await
420    }
421
422    pub async fn runtime_status_snapshot(&self) -> rustfs_targets::RuntimeStatusSnapshot {
423        self.services.runtime_view.runtime_status_snapshot().await
424    }
425
426    pub async fn shutdown(&self) {
427        let _ = self.shutdown_checked().await;
428    }
429
430    /// Irreversibly terminates the notification target runtime for this process.
431    pub async fn shutdown_checked(&self) -> Result<(), NotificationError> {
432        self.services.config_manager.lifecycle().terminate().wait().await
433    }
434}
435
436impl Drop for NotificationSystem {
437    fn drop(&mut self) {
438        // Asynchronous operation cannot be used here, but logs can be recorded.
439        info!(
440            event = EVENT_NOTIFY_SYSTEM_DROP,
441            component = LOG_COMPONENT_NOTIFY,
442            subsystem = LOG_SUBSYSTEM_INTEGRATION,
443            state = "dropping",
444            "notify system integration state"
445        );
446
447        let snapshot = self.snapshot_metrics();
448        for (name, value, is_gauge) in [
449            (METRIC_NOTIFICATION_CURRENT_SEND_IN_PROGRESS, snapshot.current_send_in_progress, true),
450            (METRIC_NOTIFICATION_EVENTS_ERRORS_TOTAL, snapshot.events_errors_total, false),
451            (METRIC_NOTIFICATION_EVENTS_SENT_TOTAL, snapshot.events_sent_total, false),
452            (METRIC_NOTIFICATION_EVENTS_SKIPPED_TOTAL, snapshot.events_skipped_total, false),
453        ] {
454            if is_gauge {
455                gauge!(name).set(value as f64);
456            } else {
457                counter!(name).absolute(value);
458            }
459            info!(
460                event = EVENT_NOTIFY_SYSTEM_SHUTDOWN_METRIC,
461                component = LOG_COMPONENT_NOTIFY,
462                subsystem = LOG_SUBSYSTEM_INTEGRATION,
463                metric_name = name,
464                metric_value = value,
465                metric_kind = if is_gauge { "gauge" } else { "counter" },
466                "notify system integration state"
467            );
468        }
469
470        let status = self.get_status();
471        for (key, value) in status {
472            info!(
473                event = EVENT_NOTIFY_SYSTEM_STATUS_SNAPSHOT,
474                component = LOG_COMPONENT_NOTIFY,
475                subsystem = LOG_SUBSYSTEM_INTEGRATION,
476                status_key = %key,
477                status_value = %value,
478                "notify system integration state"
479            );
480        }
481    }
482}
483
484/// Loads configuration from a file
485pub async fn load_config_from_file(path: &str, system: &NotificationSystem) -> Result<(), NotificationError> {
486    let config_data = tokio::fs::read(path)
487        .await
488        .map_err(|e| NotificationError::Configuration(format!("Failed to read config file: {e}")))?;
489
490    let config = Config::unmarshal(config_data.as_slice())
491        .map_err(|e| NotificationError::Configuration(format!("Failed to parse config: {e}")))?;
492    system.reload_config(config).await
493}
494
495#[cfg(test)]
496mod tests {
497    use super::*;
498    use rustfs_s3_types::EventName;
499
500    #[test]
501    fn live_event_history_snapshots_from_sequence() {
502        let mut history = LiveEventHistory::default();
503        history.record(Arc::new(Event::new_test_event("bucket", "one", EventName::ObjectCreatedPut)));
504        history.record(Arc::new(Event::new_test_event("bucket", "two", EventName::ObjectCreatedPut)));
505
506        let batch = history.snapshot_since(1, 16);
507
508        assert_eq!(batch.next_sequence, 2);
509        assert!(!batch.truncated);
510        assert!(!batch.gap);
511        assert_eq!(batch.events.len(), 1);
512        assert_eq!(batch.events[0].s3.object.key, "two");
513    }
514
515    #[test]
516    fn live_event_history_marks_truncation() {
517        let mut history = LiveEventHistory::default();
518        history.record(Arc::new(Event::new_test_event("bucket", "one", EventName::ObjectCreatedPut)));
519        history.record(Arc::new(Event::new_test_event("bucket", "two", EventName::ObjectCreatedPut)));
520
521        let batch = history.snapshot_since(0, 1);
522
523        assert_eq!(batch.next_sequence, 1);
524        assert!(batch.truncated);
525        assert!(!batch.gap);
526        assert_eq!(batch.events.len(), 1);
527        assert_eq!(batch.events[0].s3.object.key, "one");
528    }
529
530    #[tokio::test]
531    async fn notification_system_exposes_live_event_pipeline() {
532        let system = NotificationSystem::new(Config::default());
533        assert!(!system.has_live_listeners());
534
535        let _rx = system.subscribe_live_events();
536        assert!(system.has_live_listeners());
537
538        system
539            .send_event(Arc::new(Event::new_test_event("bucket", "object", EventName::ObjectCreatedPut)))
540            .await;
541
542        let batch = system.recent_live_events_since(0, 16).await;
543        assert_eq!(batch.events.len(), 1);
544        assert_eq!(batch.events[0].s3.object.key, "object");
545        assert_eq!(batch.next_sequence, 1);
546        assert!(!batch.truncated);
547        assert!(!batch.gap);
548    }
549}