1use crate::notification_system_subscriber::NotificationSystemSubscriberView;
16use crate::notifier::{EventNotifier, TargetList};
17use crate::services::NotifyServices;
18use crate::{
19 Event,
20 error::NotificationError,
21 lifecycle::{NotificationLifecycleTransition, NotificationRuntimeState},
22 pipeline::LiveEventHistory,
23 registry::TargetRegistry,
24 rule_engine::NotifyRuleEngine,
25 rules::BucketNotificationConfig,
26};
27use hashbrown::HashMap;
28use metrics::{counter, gauge};
29use rustfs_config::notify::{DEFAULT_NOTIFY_TARGET_STREAM_CONCURRENCY, ENV_NOTIFY_TARGET_STREAM_CONCURRENCY};
30use rustfs_config::server_config::{Config, KVS};
31use rustfs_s3_types::EventName;
32use rustfs_targets::arn::TargetID;
33use rustfs_targets::{ReplayWorkerManager, RuntimeTargetHealthSnapshot, SharedTarget};
34use std::sync::Arc;
35use std::sync::atomic::{AtomicUsize, Ordering};
36use std::time::{Duration, Instant};
37use tokio::sync::{RwLock, Semaphore, broadcast};
38use tracing::info;
39
40const METRIC_NOTIFICATION_CURRENT_SEND_IN_PROGRESS: &str = "rustfs_notification_current_send_in_progress";
41const METRIC_NOTIFICATION_EVENTS_ERRORS_TOTAL: &str = "rustfs_notification_events_errors_total";
42const METRIC_NOTIFICATION_EVENTS_SENT_TOTAL: &str = "rustfs_notification_events_sent_total";
43const METRIC_NOTIFICATION_EVENTS_SKIPPED_TOTAL: &str = "rustfs_notification_events_skipped_total";
44const LOG_COMPONENT_NOTIFY: &str = "notify";
45const LOG_SUBSYSTEM_INTEGRATION: &str = "integration";
46const EVENT_NOTIFY_SYSTEM_DROP: &str = "notify_system_drop";
47const EVENT_NOTIFY_SYSTEM_SHUTDOWN_METRIC: &str = "notify_system_shutdown_metric";
48const EVENT_NOTIFY_SYSTEM_STATUS_SNAPSHOT: &str = "notify_system_status_snapshot";
49
50#[derive(Clone)]
51pub struct LiveEventBatch {
52 pub events: Vec<Arc<Event>>,
53 pub next_sequence: u64,
54 pub truncated: bool,
55 pub gap: bool,
61}
62
63pub struct NotificationMetrics {
65 processing_events: AtomicUsize,
67 processed_events: AtomicUsize,
69 failed_events: AtomicUsize,
71 skipped_events: AtomicUsize,
73 start_time: Instant,
75}
76
77#[derive(Debug, Clone, Default, PartialEq, Eq)]
78pub struct NotificationMetricSnapshot {
79 pub current_send_in_progress: u64,
80 pub events_errors_total: u64,
81 pub events_sent_total: u64,
82 pub events_skipped_total: u64,
83}
84
85#[derive(Debug, Clone, Default, PartialEq, Eq)]
86pub struct NotificationTargetMetricSnapshot {
87 pub failed_messages: u64,
88 pub failed_store_length: u64,
89 pub queue_length: u64,
90 pub target_id: String,
91 pub target_type: String,
92 pub total_messages: u64,
93}
94
95impl Default for NotificationMetrics {
96 fn default() -> Self {
97 Self::new()
98 }
99}
100
101impl NotificationMetrics {
102 pub fn new() -> Self {
103 NotificationMetrics {
104 processing_events: AtomicUsize::new(0),
105 processed_events: AtomicUsize::new(0),
106 failed_events: AtomicUsize::new(0),
107 skipped_events: AtomicUsize::new(0),
108 start_time: Instant::now(),
109 }
110 }
111
112 pub fn increment_processing(&self) {
114 self.processing_events.fetch_add(1, Ordering::Relaxed);
115 }
116
117 pub fn increment_processed(&self) {
118 self.processing_events.fetch_sub(1, Ordering::Relaxed);
119 self.processed_events.fetch_add(1, Ordering::Relaxed);
120 }
121
122 pub fn decrement_processing(&self) {
123 self.processing_events.fetch_sub(1, Ordering::Relaxed);
124 }
125
126 pub fn increment_failed(&self) {
127 self.processing_events.fetch_sub(1, Ordering::Relaxed);
128 self.failed_events.fetch_add(1, Ordering::Relaxed);
129 }
130
131 pub fn increment_skipped(&self) {
132 self.skipped_events.fetch_add(1, Ordering::Relaxed);
133 }
134
135 pub fn processing_count(&self) -> usize {
137 self.processing_events.load(Ordering::Relaxed)
138 }
139
140 pub fn processed_count(&self) -> usize {
141 self.processed_events.load(Ordering::Relaxed)
142 }
143
144 pub fn failed_count(&self) -> usize {
145 self.failed_events.load(Ordering::Relaxed)
146 }
147
148 pub fn skipped_count(&self) -> usize {
149 self.skipped_events.load(Ordering::Relaxed)
150 }
151
152 pub fn uptime(&self) -> Duration {
153 self.start_time.elapsed()
154 }
155
156 pub fn snapshot(&self) -> NotificationMetricSnapshot {
157 NotificationMetricSnapshot {
158 current_send_in_progress: self.processing_count() as u64,
159 events_errors_total: self.failed_count() as u64,
160 events_sent_total: self.processed_count() as u64,
161 events_skipped_total: self.skipped_count() as u64,
162 }
163 }
164}
165
166pub struct NotificationSystem {
168 pub notifier: Arc<EventNotifier>,
170 pub registry: Arc<TargetRegistry>,
172 pub config: Arc<RwLock<Config>>,
175 services: NotifyServices,
176}
177
178impl NotificationSystem {
179 pub fn new(config: Config) -> Self {
181 let concurrency_limiter =
182 rustfs_utils::get_env_usize(ENV_NOTIFY_TARGET_STREAM_CONCURRENCY, DEFAULT_NOTIFY_TARGET_STREAM_CONCURRENCY);
183 let (live_event_sender, _) = broadcast::channel(1024);
184 let metrics = Arc::new(NotificationMetrics::new());
185 let subscriber_view = Arc::new(NotificationSystemSubscriberView::new());
186 let rule_engine = NotifyRuleEngine::new();
187 let notifier = Arc::new(EventNotifier::new(metrics.clone(), rule_engine.clone()));
188 let target_list = notifier.target_list();
189 let registry = Arc::new(TargetRegistry::new());
190 let config = Arc::new(RwLock::new(config));
191 let stream_cancellers = Arc::new(RwLock::new(ReplayWorkerManager::new()));
192 let concurrency_limiter = Arc::new(Semaphore::new(concurrency_limiter)); let live_event_history = Arc::new(RwLock::new(LiveEventHistory::default()));
194 let services = NotifyServices::new(
195 notifier.clone(),
196 rule_engine,
197 target_list,
198 registry.clone(),
199 config.clone(),
200 stream_cancellers,
201 concurrency_limiter,
202 metrics,
203 subscriber_view,
204 live_event_sender,
205 live_event_history,
206 );
207
208 NotificationSystem {
209 notifier,
210 registry,
211 config,
212 services,
213 }
214 }
215
216 pub async fn init(&self) -> Result<(), NotificationError> {
218 self.services.config_manager.init().await
219 }
220
221 pub async fn get_active_targets(&self) -> Vec<TargetID> {
226 self.services.runtime_view.get_active_targets().await
227 }
228
229 pub async fn config_snapshot(&self) -> Config {
230 self.config.read().await.clone()
231 }
232
233 pub async fn get_all_targets(&self) -> Arc<RwLock<TargetList>> {
236 self.services.runtime_view.get_all_targets()
237 }
238
239 pub async fn get_target_values(&self) -> Vec<SharedTarget<Event>> {
244 self.services.runtime_view.get_target_values().await
245 }
246
247 pub async fn has_subscriber(&self, bucket: &str, event: &EventName) -> bool {
249 self.services.bucket_config_manager.has_subscriber(bucket, event).await
250 }
251
252 pub fn has_live_listeners(&self) -> bool {
254 self.services.pipeline.has_live_listeners()
255 }
256
257 pub fn subscribe_live_events(&self) -> broadcast::Receiver<Arc<Event>> {
259 self.services.pipeline.subscribe_live_events()
260 }
261
262 pub async fn recent_live_events_since(&self, after_sequence: u64, limit: usize) -> LiveEventBatch {
263 self.services.pipeline.recent_live_events_since(after_sequence, limit).await
264 }
265
266 pub async fn remove_target(&self, target_id: &TargetID, target_type: &str) -> Result<(), NotificationError> {
279 self.services.config_manager.remove_target(target_id, target_type).await
280 }
281
282 pub async fn set_target_config(&self, target_type: &str, target_name: &str, kvs: KVS) -> Result<(), NotificationError> {
295 self.services
296 .config_manager
297 .set_target_config(target_type, target_name, kvs)
298 .await
299 }
300
301 pub async fn remove_bucket_notification_config(&self, bucket: &str) {
308 self.services
309 .bucket_config_manager
310 .remove_bucket_notification_config(bucket)
311 .await;
312 }
313
314 pub async fn remove_target_config(&self, target_type: &str, target_name: &str) -> Result<(), NotificationError> {
327 self.services
328 .config_manager
329 .remove_target_config(target_type, target_name)
330 .await
331 }
332
333 pub async fn reload_config(&self, new_config: Config) -> Result<(), NotificationError> {
335 self.services.config_manager.reload_config(new_config).await
336 }
337
338 pub fn publish_config(&self, new_config: Config) -> NotificationLifecycleTransition {
340 self.services.config_manager.lifecycle().update_config(new_config)
341 }
342
343 pub async fn reload_persisted_config(&self) -> Result<(), NotificationError> {
346 self.services.config_manager.reload_persisted_config().await
347 }
348
349 pub async fn reload_persisted_config_from_store(&self, store: Arc<crate::NotifyStore>) -> Result<(), NotificationError> {
351 self.services.config_manager.reload_persisted_config_from_store(store).await
352 }
353
354 pub async fn set_targets_enabled(&self, enabled: bool, config: Option<Config>) -> Result<(), NotificationError> {
357 self.publish_targets_enabled(enabled, config).wait().await
358 }
359
360 pub fn publish_targets_enabled(&self, enabled: bool, config: Option<Config>) -> NotificationLifecycleTransition {
363 self.services.config_manager.lifecycle().set_mode(enabled, config)
364 }
365
366 pub fn runtime_lifecycle_state(&self) -> NotificationRuntimeState {
367 self.services.config_manager.lifecycle().state()
368 }
369
370 pub fn runtime_lifecycle_is_converged(&self) -> bool {
372 self.services.config_manager.lifecycle().is_converged()
373 }
374
375 pub async fn validate_bucket_notification_config(
378 &self,
379 bucket: &str,
380 cfg: &BucketNotificationConfig,
381 ) -> Result<(), NotificationError> {
382 self.services
383 .bucket_config_manager
384 .validate_bucket_notification_config(bucket, cfg)
385 .await
386 }
387
388 pub async fn load_bucket_notification_config(
390 &self,
391 bucket: &str,
392 cfg: &BucketNotificationConfig,
393 ) -> Result<(), NotificationError> {
394 self.services
395 .bucket_config_manager
396 .load_bucket_notification_config(bucket, cfg)
397 .await
398 }
399
400 pub async fn send_event(&self, event: Arc<Event>) {
402 self.services.pipeline.send_event(event).await;
403 }
404
405 pub fn get_status(&self) -> HashMap<String, String> {
407 self.services.status_view.get_status()
408 }
409
410 pub fn snapshot_metrics(&self) -> NotificationMetricSnapshot {
411 self.services.status_view.snapshot_metrics()
412 }
413
414 pub async fn snapshot_target_metrics(&self) -> Vec<NotificationTargetMetricSnapshot> {
415 self.services.runtime_view.snapshot_target_metrics().await
416 }
417
418 pub async fn snapshot_target_health(&self) -> Vec<RuntimeTargetHealthSnapshot> {
419 self.services.runtime_view.snapshot_target_health().await
420 }
421
422 pub async fn runtime_status_snapshot(&self) -> rustfs_targets::RuntimeStatusSnapshot {
423 self.services.runtime_view.runtime_status_snapshot().await
424 }
425
426 pub async fn shutdown(&self) {
427 let _ = self.shutdown_checked().await;
428 }
429
430 pub async fn shutdown_checked(&self) -> Result<(), NotificationError> {
432 self.services.config_manager.lifecycle().terminate().wait().await
433 }
434}
435
436impl Drop for NotificationSystem {
437 fn drop(&mut self) {
438 info!(
440 event = EVENT_NOTIFY_SYSTEM_DROP,
441 component = LOG_COMPONENT_NOTIFY,
442 subsystem = LOG_SUBSYSTEM_INTEGRATION,
443 state = "dropping",
444 "notify system integration state"
445 );
446
447 let snapshot = self.snapshot_metrics();
448 for (name, value, is_gauge) in [
449 (METRIC_NOTIFICATION_CURRENT_SEND_IN_PROGRESS, snapshot.current_send_in_progress, true),
450 (METRIC_NOTIFICATION_EVENTS_ERRORS_TOTAL, snapshot.events_errors_total, false),
451 (METRIC_NOTIFICATION_EVENTS_SENT_TOTAL, snapshot.events_sent_total, false),
452 (METRIC_NOTIFICATION_EVENTS_SKIPPED_TOTAL, snapshot.events_skipped_total, false),
453 ] {
454 if is_gauge {
455 gauge!(name).set(value as f64);
456 } else {
457 counter!(name).absolute(value);
458 }
459 info!(
460 event = EVENT_NOTIFY_SYSTEM_SHUTDOWN_METRIC,
461 component = LOG_COMPONENT_NOTIFY,
462 subsystem = LOG_SUBSYSTEM_INTEGRATION,
463 metric_name = name,
464 metric_value = value,
465 metric_kind = if is_gauge { "gauge" } else { "counter" },
466 "notify system integration state"
467 );
468 }
469
470 let status = self.get_status();
471 for (key, value) in status {
472 info!(
473 event = EVENT_NOTIFY_SYSTEM_STATUS_SNAPSHOT,
474 component = LOG_COMPONENT_NOTIFY,
475 subsystem = LOG_SUBSYSTEM_INTEGRATION,
476 status_key = %key,
477 status_value = %value,
478 "notify system integration state"
479 );
480 }
481 }
482}
483
484pub async fn load_config_from_file(path: &str, system: &NotificationSystem) -> Result<(), NotificationError> {
486 let config_data = tokio::fs::read(path)
487 .await
488 .map_err(|e| NotificationError::Configuration(format!("Failed to read config file: {e}")))?;
489
490 let config = Config::unmarshal(config_data.as_slice())
491 .map_err(|e| NotificationError::Configuration(format!("Failed to parse config: {e}")))?;
492 system.reload_config(config).await
493}
494
495#[cfg(test)]
496mod tests {
497 use super::*;
498 use rustfs_s3_types::EventName;
499
500 #[test]
501 fn live_event_history_snapshots_from_sequence() {
502 let mut history = LiveEventHistory::default();
503 history.record(Arc::new(Event::new_test_event("bucket", "one", EventName::ObjectCreatedPut)));
504 history.record(Arc::new(Event::new_test_event("bucket", "two", EventName::ObjectCreatedPut)));
505
506 let batch = history.snapshot_since(1, 16);
507
508 assert_eq!(batch.next_sequence, 2);
509 assert!(!batch.truncated);
510 assert!(!batch.gap);
511 assert_eq!(batch.events.len(), 1);
512 assert_eq!(batch.events[0].s3.object.key, "two");
513 }
514
515 #[test]
516 fn live_event_history_marks_truncation() {
517 let mut history = LiveEventHistory::default();
518 history.record(Arc::new(Event::new_test_event("bucket", "one", EventName::ObjectCreatedPut)));
519 history.record(Arc::new(Event::new_test_event("bucket", "two", EventName::ObjectCreatedPut)));
520
521 let batch = history.snapshot_since(0, 1);
522
523 assert_eq!(batch.next_sequence, 1);
524 assert!(batch.truncated);
525 assert!(!batch.gap);
526 assert_eq!(batch.events.len(), 1);
527 assert_eq!(batch.events[0].s3.object.key, "one");
528 }
529
530 #[tokio::test]
531 async fn notification_system_exposes_live_event_pipeline() {
532 let system = NotificationSystem::new(Config::default());
533 assert!(!system.has_live_listeners());
534
535 let _rx = system.subscribe_live_events();
536 assert!(system.has_live_listeners());
537
538 system
539 .send_event(Arc::new(Event::new_test_event("bucket", "object", EventName::ObjectCreatedPut)))
540 .await;
541
542 let batch = system.recent_live_events_since(0, 16).await;
543 assert_eq!(batch.events.len(), 1);
544 assert_eq!(batch.events[0].s3.object.key, "object");
545 assert_eq!(batch.next_sequence, 1);
546 assert!(!batch.truncated);
547 assert!(!batch.gap);
548 }
549}