Skip to main content

rustfs_notify/
config_manager.rs

1// Copyright 2024 RustFS Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use crate::{
16    NotificationError,
17    lifecycle::{NotificationRuntimeState, NotifyLifecycleCoordinator},
18    registry::TargetRegistry,
19    resolve_notify_object_store_handle,
20    rule_engine::NotifyRuleEngine,
21    runtime_facade::NotifyRuntimeFacade,
22    with_notify_server_config_read_lock,
23};
24use rustfs_config::notify::{
25    NOTIFY_AMQP_SUB_SYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_MYSQL_SUB_SYS, NOTIFY_NATS_SUB_SYS,
26    NOTIFY_POSTGRES_SUB_SYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS,
27};
28use rustfs_config::server_config::{Config, KVS};
29use rustfs_targets::arn::TargetID;
30use std::sync::Arc;
31use tokio::sync::RwLock;
32use tracing::{debug, info};
33
34const LOG_COMPONENT_NOTIFY: &str = "notify";
35const LOG_SUBSYSTEM_CONFIG: &str = "config";
36const EVENT_NOTIFY_CONFIG_UPDATE: &str = "notify_config_update";
37
38#[derive(Debug)]
39enum NotifyConfigStoreError {
40    Lock(String),
41    StorageNotAvailable,
42    Read(String),
43    Save(String),
44    Converge { persisted: bool, error: String },
45}
46
47fn notification_convergence_error(persisted: bool, error: impl std::fmt::Display) -> NotificationError {
48    let durable_state = if persisted { "persisted" } else { "unchanged" };
49    NotificationError::Configuration(format!(
50        "configuration was {durable_state} but notification runtime convergence failed: {error}"
51    ))
52}
53
54async fn supervise_config_update<T>(
55    mutation: impl std::future::Future<Output = Result<T, NotifyConfigStoreError>> + Send + 'static,
56) -> Result<T, NotifyConfigStoreError>
57where
58    T: Send + 'static,
59{
60    tokio::spawn(mutation).await.map_err(|error| {
61        let outcome = if error.is_cancelled() { "cancelled" } else { "panicked" };
62        NotifyConfigStoreError::Lock(format!("notify config update task {outcome}"))
63    })?
64}
65
66async fn update_server_config<F>(
67    mut modifier: F,
68    lifecycle: NotifyLifecycleCoordinator,
69) -> Result<Option<(crate::lifecycle::NotificationLifecycleTransition, bool)>, NotifyConfigStoreError>
70where
71    F: FnMut(&mut Config) -> bool + Send + 'static,
72{
73    let Some(store) = resolve_notify_object_store_handle() else {
74        return Err(NotifyConfigStoreError::StorageNotAvailable);
75    };
76
77    supervise_config_update(async move {
78        let snapshot = crate::read_notify_server_config_snapshot(store.clone())
79            .await
80            .map_err(NotifyConfigStoreError::Read)?;
81        let mut config = snapshot.config.clone();
82        if !modifier(&mut config) {
83            return Ok(None);
84        }
85
86        let persisted = crate::save_notify_server_config_snapshot(store.clone(), &config, &snapshot)
87            .await
88            .map_err(NotifyConfigStoreError::Save)?;
89        drop(snapshot);
90
91        let read_store = store.clone();
92        with_notify_server_config_read_lock(store, move || async move {
93            let latest = crate::read_existing_notify_server_config_no_lock(read_store)
94                .await
95                .map_err(|error| NotifyConfigStoreError::Converge { persisted, error })?;
96            Ok::<_, NotifyConfigStoreError>(Some((lifecycle.update_config(latest), persisted)))
97        })
98        .await
99        .map_err(|error| NotifyConfigStoreError::Converge { persisted, error })?
100    })
101    .await
102}
103
104pub(crate) fn notify_configuration_hint() -> String {
105    let webhook_enable_primary = format!("{}_PRIMARY", rustfs_config::notify::ENV_NOTIFY_WEBHOOK_ENABLE);
106    let webhook_endpoint_primary = format!("{}_PRIMARY", rustfs_config::notify::ENV_NOTIFY_WEBHOOK_ENDPOINT);
107    format!(
108        "No notify targets configured. Check {}=true and instance-scoped target env vars (for example {webhook_enable_primary} + {webhook_endpoint_primary} for arn:rustfs:sqs::primary:webhook). If using default queue_dir, ensure {} is writable.",
109        rustfs_config::ENV_NOTIFY_ENABLE,
110        rustfs_config::EVENT_DEFAULT_DIR,
111    )
112}
113
114fn subsystem_target_type(target_type: &str) -> &str {
115    match target_type {
116        NOTIFY_AMQP_SUB_SYS => "amqp",
117        NOTIFY_WEBHOOK_SUB_SYS => "webhook",
118        NOTIFY_KAFKA_SUB_SYS => "kafka",
119        NOTIFY_MQTT_SUB_SYS => "mqtt",
120        NOTIFY_MYSQL_SUB_SYS => "mysql",
121        NOTIFY_NATS_SUB_SYS => "nats",
122        NOTIFY_POSTGRES_SUB_SYS => "postgres",
123        NOTIFY_PULSAR_SUB_SYS => "pulsar",
124        NOTIFY_REDIS_SUB_SYS => "redis",
125        _ => target_type,
126    }
127}
128
129pub fn runtime_target_id_for_subsystem(target_type: &str, target_name: &str) -> TargetID {
130    TargetID {
131        id: target_name.to_lowercase(),
132        name: subsystem_target_type(target_type).to_string(),
133    }
134}
135
136#[derive(Clone)]
137pub struct NotifyConfigManager {
138    config: Arc<RwLock<Config>>,
139    lifecycle: NotifyLifecycleCoordinator,
140    rule_engine: NotifyRuleEngine,
141}
142
143impl NotifyConfigManager {
144    pub fn new(
145        config: Arc<RwLock<Config>>,
146        registry: Arc<TargetRegistry>,
147        rule_engine: NotifyRuleEngine,
148        runtime_facade: NotifyRuntimeFacade,
149    ) -> Self {
150        let lifecycle = NotifyLifecycleCoordinator::new(config.clone(), registry, runtime_facade);
151        Self {
152            config,
153            lifecycle,
154            rule_engine,
155        }
156    }
157
158    pub(crate) fn lifecycle(&self) -> NotifyLifecycleCoordinator {
159        self.lifecycle.clone()
160    }
161
162    pub async fn init(&self) -> Result<(), NotificationError> {
163        let config = self.config.read().await.clone();
164        self.lifecycle.set_mode(true, Some(config)).wait().await
165    }
166
167    pub async fn remove_target(&self, target_id: &TargetID, target_type: &str) -> Result<(), NotificationError> {
168        debug!(
169            event = EVENT_NOTIFY_CONFIG_UPDATE,
170            component = LOG_COMPONENT_NOTIFY,
171            subsystem = LOG_SUBSYSTEM_CONFIG,
172            action = "remove_target",
173            target_id = %target_id,
174            target_type,
175            "Attempting to remove notification target"
176        );
177
178        let ttype = target_type.to_lowercase();
179        let tname = target_id.id.to_lowercase();
180        let log_target_id = target_id.clone();
181
182        // Guard against orphaning bucket notification rules (backlog#979). Removing a
183        // target while a bucket rule still references it would leave a dangling
184        // binding whose events can never be delivered. This mirrors the symmetric
185        // guard already applied in `remove_target_config`: refuse the removal while
186        // the target is still bound so the caller unbinds the bucket rules first.
187        let bound_target_id = runtime_target_id_for_subsystem(&ttype, &tname);
188        if self.rule_engine.is_target_bound_to_any_bucket(&bound_target_id).await {
189            return Err(NotificationError::Configuration(format!(
190                "Target is still bound to bucket rules and deletion is prohibited: type={ttype} name={tname}"
191            )));
192        }
193
194        self.update_config_and_reload(move |config| {
195            let mut changed = false;
196            if let Some(targets_of_type) = config.0.get_mut(&ttype) {
197                if targets_of_type.remove(&tname).is_some() {
198                    info!(
199                            event = EVENT_NOTIFY_CONFIG_UPDATE,
200                            component = LOG_COMPONENT_NOTIFY,
201                        subsystem = LOG_SUBSYSTEM_CONFIG,
202                        action = "remove_target",
203                        target_id = %log_target_id,
204                        result = "removed",
205                        "notify config update"
206                    );
207                    changed = true;
208                }
209                if targets_of_type.is_empty() {
210                    config.0.remove(&ttype);
211                }
212            }
213            if !changed {
214                debug!(
215                    event = EVENT_NOTIFY_CONFIG_UPDATE,
216                    component = LOG_COMPONENT_NOTIFY,
217                    subsystem = LOG_SUBSYSTEM_CONFIG,
218                    action = "remove_target",
219                    target_id = %log_target_id,
220                    result = "not_found",
221                    "notify config update"
222                );
223            }
224            changed
225        })
226        .await
227    }
228
229    pub async fn set_target_config(&self, target_type: &str, target_name: &str, kvs: KVS) -> Result<(), NotificationError> {
230        debug!(
231            event = EVENT_NOTIFY_CONFIG_UPDATE,
232            component = LOG_COMPONENT_NOTIFY,
233            subsystem = LOG_SUBSYSTEM_CONFIG,
234            action = "set_target_config",
235            target_type,
236            target_name,
237            "Setting notification target configuration"
238        );
239        let ttype = target_type.to_lowercase();
240        let tname = target_name.to_lowercase();
241        self.update_config_and_reload(move |config| {
242            config.0.entry(ttype.clone()).or_default().insert(tname.clone(), kvs.clone());
243            true
244        })
245        .await
246    }
247
248    pub async fn remove_target_config(&self, target_type: &str, target_name: &str) -> Result<(), NotificationError> {
249        debug!(
250            event = EVENT_NOTIFY_CONFIG_UPDATE,
251            component = LOG_COMPONENT_NOTIFY,
252            subsystem = LOG_SUBSYSTEM_CONFIG,
253            action = "remove_target_config",
254            target_type,
255            target_name,
256            "Removing notification target configuration"
257        );
258
259        let ttype = target_type.to_lowercase();
260        let tname = target_name.to_lowercase();
261        let target_id = runtime_target_id_for_subsystem(&ttype, &tname);
262
263        if self.rule_engine.is_target_bound_to_any_bucket(&target_id).await {
264            return Err(NotificationError::Configuration(format!(
265                "Target is still bound to bucket rules and deletion is prohibited: type={} name={}",
266                ttype, tname
267            )));
268        }
269
270        self.update_config_and_reload(move |config| {
271            let mut changed = false;
272            if let Some(targets) = config.0.get_mut(&ttype) {
273                if targets.remove(&tname).is_some() {
274                    changed = true;
275                }
276                if targets.is_empty() {
277                    config.0.remove(&ttype);
278                }
279            }
280            if !changed {
281                debug!(
282                    event = EVENT_NOTIFY_CONFIG_UPDATE,
283                    component = LOG_COMPONENT_NOTIFY,
284                    subsystem = LOG_SUBSYSTEM_CONFIG,
285                    action = "remove_target_config",
286                    target_type = %ttype,
287                    target_name = %tname,
288                    result = "not_found",
289                    "notify config update"
290                );
291            }
292            debug!(
293                subsystem_count = config.0.len(),
294                "Target config removal processed and configuration summary updated"
295            );
296            changed
297        })
298        .await
299    }
300
301    pub async fn reload_config(&self, new_config: Config) -> Result<(), NotificationError> {
302        self.lifecycle.set_mode(true, Some(new_config)).wait().await
303    }
304
305    pub async fn reload_persisted_config(&self) -> Result<(), NotificationError> {
306        let Some(store) = resolve_notify_object_store_handle() else {
307            return Err(NotificationError::StorageNotAvailable(
308                "Failed to load target configuration: server storage not initialized".to_string(),
309            ));
310        };
311        self.reload_persisted_config_from_store(store).await
312    }
313
314    pub async fn reload_persisted_config_from_store(&self, store: Arc<crate::NotifyStore>) -> Result<(), NotificationError> {
315        if self.lifecycle.state() == NotificationRuntimeState::Terminated {
316            return Err(NotificationError::Initialization("Notification runtime has terminated".to_string()));
317        }
318
319        let read_store = store.clone();
320        let config_cache = self.config.clone();
321        let lifecycle = self.lifecycle.clone();
322        let transition = with_notify_server_config_read_lock(store, move || async move {
323            let config = crate::read_existing_notify_server_config_no_lock(read_store)
324                .await
325                .map_err(NotificationError::ReadConfig)?;
326            Ok::<_, NotificationError>(if *config_cache.read().await == config && lifecycle.is_converged() {
327                None
328            } else {
329                Some(lifecycle.update_config(config))
330            })
331        })
332        .await
333        .map_err(NotificationError::StorageNotAvailable)??;
334
335        if let Some(transition) = transition {
336            transition.wait().await?;
337        }
338        Ok(())
339    }
340
341    async fn update_config_and_reload<F>(&self, modifier: F) -> Result<(), NotificationError>
342    where
343        F: FnMut(&mut Config) -> bool + Send + 'static,
344    {
345        if self.lifecycle.state() == NotificationRuntimeState::Terminated {
346            return Err(NotificationError::Initialization("Notification runtime has terminated".to_string()));
347        }
348        let Some((transition, persisted)) =
349            update_server_config(modifier, self.lifecycle.clone())
350                .await
351                .map_err(|err| match err {
352                    NotifyConfigStoreError::Lock(err) => NotificationError::StorageNotAvailable(err),
353                    NotifyConfigStoreError::StorageNotAvailable => NotificationError::StorageNotAvailable(
354                        "Failed to save target configuration: server storage not initialized".to_string(),
355                    ),
356                    NotifyConfigStoreError::Read(err) => NotificationError::ReadConfig(err),
357                    NotifyConfigStoreError::Save(err) => NotificationError::SaveConfig(err),
358                    NotifyConfigStoreError::Converge { persisted, error } => notification_convergence_error(persisted, error),
359                })?
360        else {
361            debug!(
362                event = EVENT_NOTIFY_CONFIG_UPDATE,
363                component = LOG_COMPONENT_NOTIFY,
364                subsystem = LOG_SUBSYSTEM_CONFIG,
365                action = "reload_if_changed",
366                result = "unchanged",
367                "notify config update"
368            );
369            return Ok(());
370        };
371
372        let result = if persisted { "updated" } else { "unchanged" };
373        info!(
374            event = EVENT_NOTIFY_CONFIG_UPDATE,
375            component = LOG_COMPONENT_NOTIFY,
376            subsystem = LOG_SUBSYSTEM_CONFIG,
377            action = "reload_if_changed",
378            result,
379            "notify config update"
380        );
381        transition
382            .wait()
383            .await
384            .map_err(|err| notification_convergence_error(persisted, err))
385    }
386}
387
388#[cfg(test)]
389mod tests {
390    use super::{
391        NotifyConfigManager, NotifyConfigStoreError, notification_convergence_error, runtime_target_id_for_subsystem,
392        supervise_config_update,
393    };
394    use crate::rules::RulesMap;
395    use crate::{NotificationError, NotificationRuntimeState};
396
397    #[tokio::test]
398    async fn supervised_config_update_survives_waiter_cancellation() {
399        let (started_tx, started_rx) = tokio::sync::oneshot::channel();
400        let (release_tx, release_rx) = tokio::sync::oneshot::channel();
401        let (completed_tx, completed_rx) = tokio::sync::oneshot::channel();
402
403        let waiter = tokio::spawn(async move {
404            supervise_config_update(async move {
405                let _ = started_tx.send(());
406                let _ = release_rx.await;
407                let _ = completed_tx.send(());
408                Ok::<_, NotifyConfigStoreError>(())
409            })
410            .await
411        });
412
413        started_rx.await.expect("config update should start");
414        waiter.abort();
415        release_tx.send(()).expect("config update should be released");
416        let completed = tokio::time::timeout(std::time::Duration::from_secs(30), completed_rx).await;
417        assert!(matches!(completed, Ok(Ok(()))), "detached config update should complete");
418    }
419    use crate::{
420        integration::NotificationMetrics, notifier::EventNotifier, registry::TargetRegistry, rule_engine::NotifyRuleEngine,
421        runtime_facade::NotifyRuntimeFacade,
422    };
423    use rustfs_config::notify::{
424        NOTIFY_AMQP_SUB_SYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_NATS_SUB_SYS, NOTIFY_POSTGRES_SUB_SYS,
425        NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS,
426    };
427    use rustfs_config::server_config::Config;
428    use rustfs_s3_types::EventName;
429    use rustfs_targets::ReplayWorkerManager;
430    use rustfs_targets::arn::TargetID;
431    use std::sync::Arc;
432    use tokio::sync::{RwLock, Semaphore};
433
434    fn build_manager() -> NotifyConfigManager {
435        let config = Arc::new(RwLock::new(Config::default()));
436        let registry = Arc::new(TargetRegistry::new());
437        let metrics = Arc::new(NotificationMetrics::new());
438        let rule_engine = NotifyRuleEngine::new();
439        let notifier = Arc::new(EventNotifier::new(metrics.clone(), rule_engine.clone()));
440        let target_list = notifier.target_list();
441        let runtime_facade = NotifyRuntimeFacade::new_with_dispatch_gate(
442            target_list,
443            Arc::new(RwLock::new(ReplayWorkerManager::new())),
444            notifier.dispatch_gate(),
445            Arc::new(Semaphore::new(4)),
446            metrics,
447        );
448
449        NotifyConfigManager::new(config, registry, rule_engine, runtime_facade)
450    }
451
452    // Regression test for backlog#979 (b): `remove_target` must refuse to remove a
453    // target that is still referenced by a bucket notification rule, otherwise the
454    // rule is left orphaned (pointing at a target that no longer exists). This
455    // mirrors the guard `remove_target_config` already enforces. The guard runs
456    // before the persisted config read-modify-write, so it returns the refusal
457    // without needing a live object store.
458    #[tokio::test]
459    async fn remove_target_refuses_when_bound_to_bucket_rules() {
460        let manager = build_manager();
461
462        // Bind the runtime target id (webhook/primary) that `remove_target` derives
463        // from (`NOTIFY_WEBHOOK_SUB_SYS`, id="primary") to a bucket rule.
464        let bound_id = TargetID::new("primary".to_string(), "webhook".to_string());
465        let mut rules_map = RulesMap::new();
466        rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), bound_id.clone());
467        manager.rule_engine.set_bucket_rules("bucket", rules_map).await;
468
469        let err = manager
470            .remove_target(&bound_id, NOTIFY_WEBHOOK_SUB_SYS)
471            .await
472            .expect_err("removing a bound target must be refused to avoid orphaning bucket rules");
473        assert!(
474            matches!(err, NotificationError::Configuration(_)),
475            "expected a Configuration refusal, got {err:?}"
476        );
477    }
478
479    #[tokio::test]
480    async fn config_manager_init_accepts_empty_target_set() {
481        let manager = build_manager();
482        manager.init().await.expect("init should succeed for empty targets");
483    }
484
485    #[tokio::test]
486    async fn config_manager_reload_accepts_empty_target_set() {
487        let manager = build_manager();
488        manager
489            .reload_config(Config::default())
490            .await
491            .expect("reload_config should succeed for empty targets");
492        assert!(matches!(manager.lifecycle().state(), NotificationRuntimeState::TargetsEnabled { .. }));
493    }
494
495    #[test]
496    fn runtime_target_id_for_subsystem_maps_notify_webhook_to_runtime_type() {
497        let target_id = runtime_target_id_for_subsystem(NOTIFY_WEBHOOK_SUB_SYS, "Primary");
498        assert_eq!(target_id.id, "primary");
499        assert_eq!(target_id.name, "webhook");
500    }
501
502    #[test]
503    fn runtime_target_id_for_subsystem_maps_notify_amqp_to_runtime_type() {
504        let target_id = runtime_target_id_for_subsystem(NOTIFY_AMQP_SUB_SYS, "Primary");
505        assert_eq!(target_id.id, "primary");
506        assert_eq!(target_id.name, "amqp");
507    }
508
509    #[test]
510    fn runtime_target_id_for_subsystem_maps_notify_mqtt_to_runtime_type() {
511        let target_id = runtime_target_id_for_subsystem(NOTIFY_MQTT_SUB_SYS, "Analytics");
512        assert_eq!(target_id.id, "analytics");
513        assert_eq!(target_id.name, "mqtt");
514    }
515
516    #[test]
517    fn runtime_target_id_for_subsystem_maps_notify_kafka_to_runtime_type() {
518        let target_id = runtime_target_id_for_subsystem(NOTIFY_KAFKA_SUB_SYS, "EventBus");
519        assert_eq!(target_id.id, "eventbus");
520        assert_eq!(target_id.name, "kafka");
521    }
522
523    #[test]
524    fn runtime_target_id_for_subsystem_maps_notify_nats_to_runtime_type() {
525        let target_id = runtime_target_id_for_subsystem(NOTIFY_NATS_SUB_SYS, "Bus");
526        assert_eq!(target_id.id, "bus");
527        assert_eq!(target_id.name, "nats");
528    }
529
530    #[test]
531    fn runtime_target_id_for_subsystem_maps_notify_pulsar_to_runtime_type() {
532        let target_id = runtime_target_id_for_subsystem(NOTIFY_PULSAR_SUB_SYS, "Ledger");
533        assert_eq!(target_id.id, "ledger");
534        assert_eq!(target_id.name, "pulsar");
535    }
536
537    #[test]
538    fn runtime_target_id_for_subsystem_maps_notify_redis_to_runtime_type() {
539        let target_id = runtime_target_id_for_subsystem(NOTIFY_REDIS_SUB_SYS, "Primary");
540        assert_eq!(target_id.id, "primary");
541        assert_eq!(target_id.name, "redis");
542    }
543
544    #[test]
545    fn runtime_target_id_for_subsystem_maps_notify_postgres_to_runtime_type() {
546        let target_id = runtime_target_id_for_subsystem(NOTIFY_POSTGRES_SUB_SYS, "AuditTrail");
547        assert_eq!(target_id.id, "audittrail");
548        assert_eq!(target_id.name, "postgres");
549    }
550
551    #[test]
552    fn convergence_error_reports_durable_write_state() {
553        for (persisted, expected) in [(true, "configuration was persisted"), (false, "configuration was unchanged")] {
554            let error = notification_convergence_error(persisted, "injected failure");
555            let NotificationError::Configuration(message) = error else {
556                panic!("convergence error should be a configuration error");
557            };
558            assert!(message.contains(expected), "unexpected convergence error: {message}");
559            assert!(message.contains("injected failure"));
560        }
561    }
562}