1use crate::{
16 NotificationError,
17 lifecycle::{NotificationRuntimeState, NotifyLifecycleCoordinator},
18 registry::TargetRegistry,
19 resolve_notify_object_store_handle,
20 rule_engine::NotifyRuleEngine,
21 runtime_facade::NotifyRuntimeFacade,
22 with_notify_server_config_read_lock,
23};
24use rustfs_config::notify::{
25 NOTIFY_AMQP_SUB_SYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_MYSQL_SUB_SYS, NOTIFY_NATS_SUB_SYS,
26 NOTIFY_POSTGRES_SUB_SYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS,
27};
28use rustfs_config::server_config::{Config, KVS};
29use rustfs_targets::arn::TargetID;
30use std::sync::Arc;
31use tokio::sync::RwLock;
32use tracing::{debug, info};
33
34const LOG_COMPONENT_NOTIFY: &str = "notify";
35const LOG_SUBSYSTEM_CONFIG: &str = "config";
36const EVENT_NOTIFY_CONFIG_UPDATE: &str = "notify_config_update";
37
38#[derive(Debug)]
39enum NotifyConfigStoreError {
40 Lock(String),
41 StorageNotAvailable,
42 Read(String),
43 Save(String),
44 Converge { persisted: bool, error: String },
45}
46
47fn notification_convergence_error(persisted: bool, error: impl std::fmt::Display) -> NotificationError {
48 let durable_state = if persisted { "persisted" } else { "unchanged" };
49 NotificationError::Configuration(format!(
50 "configuration was {durable_state} but notification runtime convergence failed: {error}"
51 ))
52}
53
54async fn supervise_config_update<T>(
55 mutation: impl std::future::Future<Output = Result<T, NotifyConfigStoreError>> + Send + 'static,
56) -> Result<T, NotifyConfigStoreError>
57where
58 T: Send + 'static,
59{
60 tokio::spawn(mutation).await.map_err(|error| {
61 let outcome = if error.is_cancelled() { "cancelled" } else { "panicked" };
62 NotifyConfigStoreError::Lock(format!("notify config update task {outcome}"))
63 })?
64}
65
66async fn update_server_config<F>(
67 mut modifier: F,
68 lifecycle: NotifyLifecycleCoordinator,
69) -> Result<Option<(crate::lifecycle::NotificationLifecycleTransition, bool)>, NotifyConfigStoreError>
70where
71 F: FnMut(&mut Config) -> bool + Send + 'static,
72{
73 let Some(store) = resolve_notify_object_store_handle() else {
74 return Err(NotifyConfigStoreError::StorageNotAvailable);
75 };
76
77 supervise_config_update(async move {
78 let snapshot = crate::read_notify_server_config_snapshot(store.clone())
79 .await
80 .map_err(NotifyConfigStoreError::Read)?;
81 let mut config = snapshot.config.clone();
82 if !modifier(&mut config) {
83 return Ok(None);
84 }
85
86 let persisted = crate::save_notify_server_config_snapshot(store.clone(), &config, &snapshot)
87 .await
88 .map_err(NotifyConfigStoreError::Save)?;
89 drop(snapshot);
90
91 let read_store = store.clone();
92 with_notify_server_config_read_lock(store, move || async move {
93 let latest = crate::read_existing_notify_server_config_no_lock(read_store)
94 .await
95 .map_err(|error| NotifyConfigStoreError::Converge { persisted, error })?;
96 Ok::<_, NotifyConfigStoreError>(Some((lifecycle.update_config(latest), persisted)))
97 })
98 .await
99 .map_err(|error| NotifyConfigStoreError::Converge { persisted, error })?
100 })
101 .await
102}
103
104pub(crate) fn notify_configuration_hint() -> String {
105 let webhook_enable_primary = format!("{}_PRIMARY", rustfs_config::notify::ENV_NOTIFY_WEBHOOK_ENABLE);
106 let webhook_endpoint_primary = format!("{}_PRIMARY", rustfs_config::notify::ENV_NOTIFY_WEBHOOK_ENDPOINT);
107 format!(
108 "No notify targets configured. Check {}=true and instance-scoped target env vars (for example {webhook_enable_primary} + {webhook_endpoint_primary} for arn:rustfs:sqs::primary:webhook). If using default queue_dir, ensure {} is writable.",
109 rustfs_config::ENV_NOTIFY_ENABLE,
110 rustfs_config::EVENT_DEFAULT_DIR,
111 )
112}
113
114fn subsystem_target_type(target_type: &str) -> &str {
115 match target_type {
116 NOTIFY_AMQP_SUB_SYS => "amqp",
117 NOTIFY_WEBHOOK_SUB_SYS => "webhook",
118 NOTIFY_KAFKA_SUB_SYS => "kafka",
119 NOTIFY_MQTT_SUB_SYS => "mqtt",
120 NOTIFY_MYSQL_SUB_SYS => "mysql",
121 NOTIFY_NATS_SUB_SYS => "nats",
122 NOTIFY_POSTGRES_SUB_SYS => "postgres",
123 NOTIFY_PULSAR_SUB_SYS => "pulsar",
124 NOTIFY_REDIS_SUB_SYS => "redis",
125 _ => target_type,
126 }
127}
128
129pub fn runtime_target_id_for_subsystem(target_type: &str, target_name: &str) -> TargetID {
130 TargetID {
131 id: target_name.to_lowercase(),
132 name: subsystem_target_type(target_type).to_string(),
133 }
134}
135
136#[derive(Clone)]
137pub struct NotifyConfigManager {
138 config: Arc<RwLock<Config>>,
139 lifecycle: NotifyLifecycleCoordinator,
140 rule_engine: NotifyRuleEngine,
141}
142
143impl NotifyConfigManager {
144 pub fn new(
145 config: Arc<RwLock<Config>>,
146 registry: Arc<TargetRegistry>,
147 rule_engine: NotifyRuleEngine,
148 runtime_facade: NotifyRuntimeFacade,
149 ) -> Self {
150 let lifecycle = NotifyLifecycleCoordinator::new(config.clone(), registry, runtime_facade);
151 Self {
152 config,
153 lifecycle,
154 rule_engine,
155 }
156 }
157
158 pub(crate) fn lifecycle(&self) -> NotifyLifecycleCoordinator {
159 self.lifecycle.clone()
160 }
161
162 pub async fn init(&self) -> Result<(), NotificationError> {
163 let config = self.config.read().await.clone();
164 self.lifecycle.set_mode(true, Some(config)).wait().await
165 }
166
167 pub async fn remove_target(&self, target_id: &TargetID, target_type: &str) -> Result<(), NotificationError> {
168 debug!(
169 event = EVENT_NOTIFY_CONFIG_UPDATE,
170 component = LOG_COMPONENT_NOTIFY,
171 subsystem = LOG_SUBSYSTEM_CONFIG,
172 action = "remove_target",
173 target_id = %target_id,
174 target_type,
175 "Attempting to remove notification target"
176 );
177
178 let ttype = target_type.to_lowercase();
179 let tname = target_id.id.to_lowercase();
180 let log_target_id = target_id.clone();
181
182 let bound_target_id = runtime_target_id_for_subsystem(&ttype, &tname);
188 if self.rule_engine.is_target_bound_to_any_bucket(&bound_target_id).await {
189 return Err(NotificationError::Configuration(format!(
190 "Target is still bound to bucket rules and deletion is prohibited: type={ttype} name={tname}"
191 )));
192 }
193
194 self.update_config_and_reload(move |config| {
195 let mut changed = false;
196 if let Some(targets_of_type) = config.0.get_mut(&ttype) {
197 if targets_of_type.remove(&tname).is_some() {
198 info!(
199 event = EVENT_NOTIFY_CONFIG_UPDATE,
200 component = LOG_COMPONENT_NOTIFY,
201 subsystem = LOG_SUBSYSTEM_CONFIG,
202 action = "remove_target",
203 target_id = %log_target_id,
204 result = "removed",
205 "notify config update"
206 );
207 changed = true;
208 }
209 if targets_of_type.is_empty() {
210 config.0.remove(&ttype);
211 }
212 }
213 if !changed {
214 debug!(
215 event = EVENT_NOTIFY_CONFIG_UPDATE,
216 component = LOG_COMPONENT_NOTIFY,
217 subsystem = LOG_SUBSYSTEM_CONFIG,
218 action = "remove_target",
219 target_id = %log_target_id,
220 result = "not_found",
221 "notify config update"
222 );
223 }
224 changed
225 })
226 .await
227 }
228
229 pub async fn set_target_config(&self, target_type: &str, target_name: &str, kvs: KVS) -> Result<(), NotificationError> {
230 debug!(
231 event = EVENT_NOTIFY_CONFIG_UPDATE,
232 component = LOG_COMPONENT_NOTIFY,
233 subsystem = LOG_SUBSYSTEM_CONFIG,
234 action = "set_target_config",
235 target_type,
236 target_name,
237 "Setting notification target configuration"
238 );
239 let ttype = target_type.to_lowercase();
240 let tname = target_name.to_lowercase();
241 self.update_config_and_reload(move |config| {
242 config.0.entry(ttype.clone()).or_default().insert(tname.clone(), kvs.clone());
243 true
244 })
245 .await
246 }
247
248 pub async fn remove_target_config(&self, target_type: &str, target_name: &str) -> Result<(), NotificationError> {
249 debug!(
250 event = EVENT_NOTIFY_CONFIG_UPDATE,
251 component = LOG_COMPONENT_NOTIFY,
252 subsystem = LOG_SUBSYSTEM_CONFIG,
253 action = "remove_target_config",
254 target_type,
255 target_name,
256 "Removing notification target configuration"
257 );
258
259 let ttype = target_type.to_lowercase();
260 let tname = target_name.to_lowercase();
261 let target_id = runtime_target_id_for_subsystem(&ttype, &tname);
262
263 if self.rule_engine.is_target_bound_to_any_bucket(&target_id).await {
264 return Err(NotificationError::Configuration(format!(
265 "Target is still bound to bucket rules and deletion is prohibited: type={} name={}",
266 ttype, tname
267 )));
268 }
269
270 self.update_config_and_reload(move |config| {
271 let mut changed = false;
272 if let Some(targets) = config.0.get_mut(&ttype) {
273 if targets.remove(&tname).is_some() {
274 changed = true;
275 }
276 if targets.is_empty() {
277 config.0.remove(&ttype);
278 }
279 }
280 if !changed {
281 debug!(
282 event = EVENT_NOTIFY_CONFIG_UPDATE,
283 component = LOG_COMPONENT_NOTIFY,
284 subsystem = LOG_SUBSYSTEM_CONFIG,
285 action = "remove_target_config",
286 target_type = %ttype,
287 target_name = %tname,
288 result = "not_found",
289 "notify config update"
290 );
291 }
292 debug!(
293 subsystem_count = config.0.len(),
294 "Target config removal processed and configuration summary updated"
295 );
296 changed
297 })
298 .await
299 }
300
301 pub async fn reload_config(&self, new_config: Config) -> Result<(), NotificationError> {
302 self.lifecycle.set_mode(true, Some(new_config)).wait().await
303 }
304
305 pub async fn reload_persisted_config(&self) -> Result<(), NotificationError> {
306 let Some(store) = resolve_notify_object_store_handle() else {
307 return Err(NotificationError::StorageNotAvailable(
308 "Failed to load target configuration: server storage not initialized".to_string(),
309 ));
310 };
311 self.reload_persisted_config_from_store(store).await
312 }
313
314 pub async fn reload_persisted_config_from_store(&self, store: Arc<crate::NotifyStore>) -> Result<(), NotificationError> {
315 if self.lifecycle.state() == NotificationRuntimeState::Terminated {
316 return Err(NotificationError::Initialization("Notification runtime has terminated".to_string()));
317 }
318
319 let read_store = store.clone();
320 let config_cache = self.config.clone();
321 let lifecycle = self.lifecycle.clone();
322 let transition = with_notify_server_config_read_lock(store, move || async move {
323 let config = crate::read_existing_notify_server_config_no_lock(read_store)
324 .await
325 .map_err(NotificationError::ReadConfig)?;
326 Ok::<_, NotificationError>(if *config_cache.read().await == config && lifecycle.is_converged() {
327 None
328 } else {
329 Some(lifecycle.update_config(config))
330 })
331 })
332 .await
333 .map_err(NotificationError::StorageNotAvailable)??;
334
335 if let Some(transition) = transition {
336 transition.wait().await?;
337 }
338 Ok(())
339 }
340
341 async fn update_config_and_reload<F>(&self, modifier: F) -> Result<(), NotificationError>
342 where
343 F: FnMut(&mut Config) -> bool + Send + 'static,
344 {
345 if self.lifecycle.state() == NotificationRuntimeState::Terminated {
346 return Err(NotificationError::Initialization("Notification runtime has terminated".to_string()));
347 }
348 let Some((transition, persisted)) =
349 update_server_config(modifier, self.lifecycle.clone())
350 .await
351 .map_err(|err| match err {
352 NotifyConfigStoreError::Lock(err) => NotificationError::StorageNotAvailable(err),
353 NotifyConfigStoreError::StorageNotAvailable => NotificationError::StorageNotAvailable(
354 "Failed to save target configuration: server storage not initialized".to_string(),
355 ),
356 NotifyConfigStoreError::Read(err) => NotificationError::ReadConfig(err),
357 NotifyConfigStoreError::Save(err) => NotificationError::SaveConfig(err),
358 NotifyConfigStoreError::Converge { persisted, error } => notification_convergence_error(persisted, error),
359 })?
360 else {
361 debug!(
362 event = EVENT_NOTIFY_CONFIG_UPDATE,
363 component = LOG_COMPONENT_NOTIFY,
364 subsystem = LOG_SUBSYSTEM_CONFIG,
365 action = "reload_if_changed",
366 result = "unchanged",
367 "notify config update"
368 );
369 return Ok(());
370 };
371
372 let result = if persisted { "updated" } else { "unchanged" };
373 info!(
374 event = EVENT_NOTIFY_CONFIG_UPDATE,
375 component = LOG_COMPONENT_NOTIFY,
376 subsystem = LOG_SUBSYSTEM_CONFIG,
377 action = "reload_if_changed",
378 result,
379 "notify config update"
380 );
381 transition
382 .wait()
383 .await
384 .map_err(|err| notification_convergence_error(persisted, err))
385 }
386}
387
388#[cfg(test)]
389mod tests {
390 use super::{
391 NotifyConfigManager, NotifyConfigStoreError, notification_convergence_error, runtime_target_id_for_subsystem,
392 supervise_config_update,
393 };
394 use crate::rules::RulesMap;
395 use crate::{NotificationError, NotificationRuntimeState};
396
397 #[tokio::test]
398 async fn supervised_config_update_survives_waiter_cancellation() {
399 let (started_tx, started_rx) = tokio::sync::oneshot::channel();
400 let (release_tx, release_rx) = tokio::sync::oneshot::channel();
401 let (completed_tx, completed_rx) = tokio::sync::oneshot::channel();
402
403 let waiter = tokio::spawn(async move {
404 supervise_config_update(async move {
405 let _ = started_tx.send(());
406 let _ = release_rx.await;
407 let _ = completed_tx.send(());
408 Ok::<_, NotifyConfigStoreError>(())
409 })
410 .await
411 });
412
413 started_rx.await.expect("config update should start");
414 waiter.abort();
415 release_tx.send(()).expect("config update should be released");
416 let completed = tokio::time::timeout(std::time::Duration::from_secs(30), completed_rx).await;
417 assert!(matches!(completed, Ok(Ok(()))), "detached config update should complete");
418 }
419 use crate::{
420 integration::NotificationMetrics, notifier::EventNotifier, registry::TargetRegistry, rule_engine::NotifyRuleEngine,
421 runtime_facade::NotifyRuntimeFacade,
422 };
423 use rustfs_config::notify::{
424 NOTIFY_AMQP_SUB_SYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_NATS_SUB_SYS, NOTIFY_POSTGRES_SUB_SYS,
425 NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS,
426 };
427 use rustfs_config::server_config::Config;
428 use rustfs_s3_types::EventName;
429 use rustfs_targets::ReplayWorkerManager;
430 use rustfs_targets::arn::TargetID;
431 use std::sync::Arc;
432 use tokio::sync::{RwLock, Semaphore};
433
434 fn build_manager() -> NotifyConfigManager {
435 let config = Arc::new(RwLock::new(Config::default()));
436 let registry = Arc::new(TargetRegistry::new());
437 let metrics = Arc::new(NotificationMetrics::new());
438 let rule_engine = NotifyRuleEngine::new();
439 let notifier = Arc::new(EventNotifier::new(metrics.clone(), rule_engine.clone()));
440 let target_list = notifier.target_list();
441 let runtime_facade = NotifyRuntimeFacade::new_with_dispatch_gate(
442 target_list,
443 Arc::new(RwLock::new(ReplayWorkerManager::new())),
444 notifier.dispatch_gate(),
445 Arc::new(Semaphore::new(4)),
446 metrics,
447 );
448
449 NotifyConfigManager::new(config, registry, rule_engine, runtime_facade)
450 }
451
452 #[tokio::test]
459 async fn remove_target_refuses_when_bound_to_bucket_rules() {
460 let manager = build_manager();
461
462 let bound_id = TargetID::new("primary".to_string(), "webhook".to_string());
465 let mut rules_map = RulesMap::new();
466 rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), bound_id.clone());
467 manager.rule_engine.set_bucket_rules("bucket", rules_map).await;
468
469 let err = manager
470 .remove_target(&bound_id, NOTIFY_WEBHOOK_SUB_SYS)
471 .await
472 .expect_err("removing a bound target must be refused to avoid orphaning bucket rules");
473 assert!(
474 matches!(err, NotificationError::Configuration(_)),
475 "expected a Configuration refusal, got {err:?}"
476 );
477 }
478
479 #[tokio::test]
480 async fn config_manager_init_accepts_empty_target_set() {
481 let manager = build_manager();
482 manager.init().await.expect("init should succeed for empty targets");
483 }
484
485 #[tokio::test]
486 async fn config_manager_reload_accepts_empty_target_set() {
487 let manager = build_manager();
488 manager
489 .reload_config(Config::default())
490 .await
491 .expect("reload_config should succeed for empty targets");
492 assert!(matches!(manager.lifecycle().state(), NotificationRuntimeState::TargetsEnabled { .. }));
493 }
494
495 #[test]
496 fn runtime_target_id_for_subsystem_maps_notify_webhook_to_runtime_type() {
497 let target_id = runtime_target_id_for_subsystem(NOTIFY_WEBHOOK_SUB_SYS, "Primary");
498 assert_eq!(target_id.id, "primary");
499 assert_eq!(target_id.name, "webhook");
500 }
501
502 #[test]
503 fn runtime_target_id_for_subsystem_maps_notify_amqp_to_runtime_type() {
504 let target_id = runtime_target_id_for_subsystem(NOTIFY_AMQP_SUB_SYS, "Primary");
505 assert_eq!(target_id.id, "primary");
506 assert_eq!(target_id.name, "amqp");
507 }
508
509 #[test]
510 fn runtime_target_id_for_subsystem_maps_notify_mqtt_to_runtime_type() {
511 let target_id = runtime_target_id_for_subsystem(NOTIFY_MQTT_SUB_SYS, "Analytics");
512 assert_eq!(target_id.id, "analytics");
513 assert_eq!(target_id.name, "mqtt");
514 }
515
516 #[test]
517 fn runtime_target_id_for_subsystem_maps_notify_kafka_to_runtime_type() {
518 let target_id = runtime_target_id_for_subsystem(NOTIFY_KAFKA_SUB_SYS, "EventBus");
519 assert_eq!(target_id.id, "eventbus");
520 assert_eq!(target_id.name, "kafka");
521 }
522
523 #[test]
524 fn runtime_target_id_for_subsystem_maps_notify_nats_to_runtime_type() {
525 let target_id = runtime_target_id_for_subsystem(NOTIFY_NATS_SUB_SYS, "Bus");
526 assert_eq!(target_id.id, "bus");
527 assert_eq!(target_id.name, "nats");
528 }
529
530 #[test]
531 fn runtime_target_id_for_subsystem_maps_notify_pulsar_to_runtime_type() {
532 let target_id = runtime_target_id_for_subsystem(NOTIFY_PULSAR_SUB_SYS, "Ledger");
533 assert_eq!(target_id.id, "ledger");
534 assert_eq!(target_id.name, "pulsar");
535 }
536
537 #[test]
538 fn runtime_target_id_for_subsystem_maps_notify_redis_to_runtime_type() {
539 let target_id = runtime_target_id_for_subsystem(NOTIFY_REDIS_SUB_SYS, "Primary");
540 assert_eq!(target_id.id, "primary");
541 assert_eq!(target_id.name, "redis");
542 }
543
544 #[test]
545 fn runtime_target_id_for_subsystem_maps_notify_postgres_to_runtime_type() {
546 let target_id = runtime_target_id_for_subsystem(NOTIFY_POSTGRES_SUB_SYS, "AuditTrail");
547 assert_eq!(target_id.id, "audittrail");
548 assert_eq!(target_id.name, "postgres");
549 }
550
551 #[test]
552 fn convergence_error_reports_durable_write_state() {
553 for (persisted, expected) in [(true, "configuration was persisted"), (false, "configuration was unchanged")] {
554 let error = notification_convergence_error(persisted, "injected failure");
555 let NotificationError::Configuration(message) = error else {
556 panic!("convergence error should be a configuration error");
557 };
558 assert!(message.contains(expected), "unexpected convergence error: {message}");
559 assert!(message.contains("injected failure"));
560 }
561 }
562}