Skip to main content

rustfs_notify/
notifier.rs

1// Copyright 2024 RustFS Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use crate::error::NotificationError;
16use crate::{event::Event, integration::NotificationMetrics, rule_engine::NotifyRuleEngine};
17use rustfs_config::notify::{DEFAULT_NOTIFY_SEND_CONCURRENCY, ENV_NOTIFY_SEND_CONCURRENCY};
18use rustfs_targets::Target;
19use rustfs_targets::arn::TargetID;
20use rustfs_targets::target::EntityTarget;
21use rustfs_targets::{SharedTarget, TargetRuntimeManager};
22use std::collections::HashMap;
23use std::sync::{Arc, LazyLock, Mutex as StdMutex, Weak};
24use tokio::sync::{RwLock, Semaphore, watch};
25use tokio::task::JoinHandle;
26use tokio_util::sync::CancellationToken;
27use tracing::{debug, error, info, instrument, warn};
28
29const LOG_COMPONENT_NOTIFY: &str = "notify";
30const LOG_SUBSYSTEM_DISPATCH: &str = "dispatch";
31const EVENT_NOTIFY_DISPATCH_SKIPPED: &str = "notify_dispatch_skipped";
32const EVENT_NOTIFY_DISPATCH_FAILED: &str = "notify_dispatch_failed";
33const EVENT_NOTIFY_DISPATCH_STARTED: &str = "notify_dispatch_started";
34const EVENT_NOTIFY_DISPATCH_COMPLETED: &str = "notify_dispatch_completed";
35const EVENT_NOTIFY_RUNTIME_LIFECYCLE: &str = "notify_runtime_lifecycle";
36
37async fn wait_for_dispatch_tasks(handles: Vec<JoinHandle<()>>) {
38    for handle in handles {
39        if let Err(e) = handle.await {
40            let reason = if e.is_cancelled() { "join_cancelled" } else { "join_panicked" };
41            error!(
42                event = EVENT_NOTIFY_DISPATCH_FAILED,
43                component = LOG_COMPONENT_NOTIFY,
44                subsystem = LOG_SUBSYSTEM_DISPATCH,
45                reason,
46                "Notify dispatch task failed"
47            );
48        }
49    }
50}
51
52pub type SharedNotifyTargetList = Arc<RwLock<TargetList>>;
53
54static TARGET_LIST_DISPATCH_GATES: LazyLock<StdMutex<HashMap<usize, Weak<RwLock<()>>>>> =
55    LazyLock::new(|| StdMutex::new(HashMap::new()));
56
57pub(crate) fn shared_dispatch_gate(target_list: &SharedNotifyTargetList, preferred: Option<Arc<RwLock<()>>>) -> Arc<RwLock<()>> {
58    let key = Arc::as_ptr(target_list) as usize;
59    let mut gates = TARGET_LIST_DISPATCH_GATES.lock().unwrap_or_else(|err| err.into_inner());
60    gates.retain(|_, gate| gate.strong_count() != 0);
61    if let Some(gate) = gates.get(&key).and_then(Weak::upgrade) {
62        return gate;
63    }
64    let gate = preferred.unwrap_or_else(|| Arc::new(RwLock::new(())));
65    gates.insert(key, Arc::downgrade(&gate));
66    gate
67}
68
69pub(crate) struct DirectDispatchTracker {
70    cancellation: CancellationToken,
71    inflight: watch::Sender<usize>,
72}
73
74impl DirectDispatchTracker {
75    fn new() -> Self {
76        let (inflight, _) = watch::channel(0);
77        Self {
78            cancellation: CancellationToken::new(),
79            inflight,
80        }
81    }
82
83    fn acquire(self: &Arc<Self>) -> DirectDispatchLease {
84        self.inflight.send_modify(|count| *count += 1);
85        DirectDispatchLease {
86            tracker: Arc::clone(self),
87        }
88    }
89
90    pub(crate) async fn wait_idle(&self) {
91        let mut inflight = self.inflight.subscribe();
92        while *inflight.borrow_and_update() != 0 {
93            if inflight.changed().await.is_err() {
94                return;
95            }
96        }
97    }
98
99    fn cancel_pending(&self) {
100        self.cancellation.cancel();
101    }
102}
103
104struct DirectDispatchLease {
105    tracker: Arc<DirectDispatchTracker>,
106}
107
108impl DirectDispatchLease {
109    fn cancellation(&self) -> CancellationToken {
110        self.tracker.cancellation.clone()
111    }
112}
113
114impl Drop for DirectDispatchLease {
115    fn drop(&mut self) {
116        self.tracker.inflight.send_modify(|count| {
117            if let Some(next) = count.checked_sub(1) {
118                *count = next;
119            } else {
120                error!(
121                    event = EVENT_NOTIFY_DISPATCH_FAILED,
122                    component = LOG_COMPONENT_NOTIFY,
123                    subsystem = LOG_SUBSYSTEM_DISPATCH,
124                    reason = "direct_lease_underflow",
125                    "Notify direct dispatch lease accounting underflowed"
126                );
127            }
128        });
129    }
130}
131
132/// Resolves the effective send concurrency (semaphore permit count).
133///
134/// A value of `0` would build a zero-permit semaphore, so `acquire` never
135/// completes and every dispatch deadlocks. A misconfigured
136/// `RUSTFS_NOTIFY_SEND_CONCURRENCY=0` therefore coerces back to the default
137/// instead of silently wedging notifications (backlog#984).
138fn resolve_send_concurrency() -> usize {
139    let configured = rustfs_utils::get_env_usize(ENV_NOTIFY_SEND_CONCURRENCY, DEFAULT_NOTIFY_SEND_CONCURRENCY);
140    coerce_send_concurrency(configured)
141}
142
143/// Coerces a configured send concurrency into a valid semaphore permit count.
144/// `0` maps back to the default; any positive value is passed through. Kept as a
145/// pure function so the coercion is unit-testable without mutating process env.
146fn coerce_send_concurrency(configured: usize) -> usize {
147    if configured == 0 {
148        warn!(
149            event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
150            component = LOG_COMPONENT_NOTIFY,
151            subsystem = LOG_SUBSYSTEM_DISPATCH,
152            configured,
153            default = DEFAULT_NOTIFY_SEND_CONCURRENCY,
154            "invalid RUSTFS_NOTIFY_SEND_CONCURRENCY=0; falling back to default to avoid a zero-permit deadlock"
155        );
156        DEFAULT_NOTIFY_SEND_CONCURRENCY
157    } else {
158        configured
159    }
160}
161
162/// Manages event notification to targets based on rules
163pub struct EventNotifier {
164    dispatch_gate: Arc<RwLock<()>>,
165    enqueue_limiter: Arc<Semaphore>,
166    metrics: Arc<NotificationMetrics>,
167    rule_engine: NotifyRuleEngine,
168    target_list: SharedNotifyTargetList,
169    send_limiter: Arc<Semaphore>,
170}
171
172impl Default for EventNotifier {
173    fn default() -> Self {
174        Self::new(Arc::new(NotificationMetrics::new()), NotifyRuleEngine::new())
175    }
176}
177
178impl EventNotifier {
179    /// Creates a new EventNotifier
180    ///
181    /// # Returns
182    /// Returns a new instance of EventNotifier.
183    pub fn new(metrics: Arc<NotificationMetrics>, rule_engine: NotifyRuleEngine) -> Self {
184        let max_inflight = resolve_send_concurrency();
185        let target_list = Arc::new(RwLock::new(TargetList::new()));
186        let dispatch_gate = shared_dispatch_gate(&target_list, None);
187        EventNotifier {
188            dispatch_gate,
189            enqueue_limiter: Arc::new(Semaphore::new(max_inflight)),
190            metrics,
191            rule_engine,
192            target_list,
193            send_limiter: Arc::new(Semaphore::new(max_inflight)),
194        }
195    }
196
197    /// Returns a reference to the target list
198    /// This method provides access to the target list for external use.
199    ///
200    /// # Returns
201    /// Returns an `Arc<RwLock<TargetList>>` representing the target list.
202    pub fn target_list(&self) -> SharedNotifyTargetList {
203        Arc::clone(&self.target_list)
204    }
205
206    pub(crate) fn dispatch_gate(&self) -> Arc<RwLock<()>> {
207        self.dispatch_gate.clone()
208    }
209
210    /// Returns a list of ARNs for the registered targets
211    ///
212    /// # Arguments
213    /// * `region` - The region to use for generating the ARNs
214    ///
215    /// # Returns
216    /// Returns a vector of strings representing the ARNs of the registered targets
217    pub async fn get_arn_list(&self, region: &str) -> Vec<String> {
218        let target_list_guard = self.target_list.read().await;
219        target_list_guard
220            .keys()
221            .iter()
222            .map(|target_id| target_id.to_arn(region).to_string())
223            .collect()
224    }
225
226    pub async fn remove_all_bucket_targets(&self) {
227        let mut target_list_guard = self.target_list.write().await;
228        target_list_guard.clear_targets_only().await;
229        info!(
230            event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
231            component = LOG_COMPONENT_NOTIFY,
232            subsystem = LOG_SUBSYSTEM_DISPATCH,
233            state = "targets_cleared",
234            "notify runtime lifecycle"
235        );
236    }
237
238    /// Sends an event to the appropriate targets based on the bucket rules
239    ///
240    /// # Arguments
241    /// * `event` - The event to send
242    #[instrument(skip_all)]
243    pub async fn send(&self, event: Arc<Event>) {
244        let bucket_name = &event.s3.bucket.name;
245        let object_key = &event.s3.object.key;
246        let event_name = event.event_name;
247
248        let target_ids = self.rule_engine.match_targets(bucket_name, event_name, object_key).await;
249        if target_ids.is_empty() {
250            debug!(
251                event = EVENT_NOTIFY_DISPATCH_SKIPPED,
252                component = LOG_COMPONENT_NOTIFY,
253                subsystem = LOG_SUBSYSTEM_DISPATCH,
254                bucket = %bucket_name,
255                object = %object_key,
256                reason = "no_matching_targets",
257                "Skipped notify dispatch"
258            );
259            self.metrics.increment_skipped();
260            return;
261        }
262        let target_ids_len = target_ids.len();
263        let mut deferred_handles = Vec::new();
264        let mut direct_handles = Vec::new();
265
266        // A lifecycle writer holds this gate only while handing queue-store
267        // ownership from one runtime generation to the next. Taking the read
268        // guard before cloning targets means the writer both blocks new sends
269        // and drains every save already using the old generation.
270        let dispatch_guard = Arc::new(self.dispatch_gate.clone().read_owned().await);
271
272        // Use scope to limit the borrow scope of target_list
273        let target_list_guard = self.target_list.read().await;
274        debug!(
275            event = EVENT_NOTIFY_DISPATCH_STARTED,
276            component = LOG_COMPONENT_NOTIFY,
277            subsystem = LOG_SUBSYSTEM_DISPATCH,
278            bucket = %bucket_name,
279            object = %object_key,
280            target_count = target_ids_len,
281            "notify dispatch"
282        );
283        for target_id in target_ids {
284            // `get` now returns Option<Arc<dyn Target + Send + Sync>>
285            if let Some(target_arc) = target_list_guard.get(&target_id) {
286                // Clone an Arc<Box<dyn Target>> (which is where target_list is stored) to move into an asynchronous task
287                // target_arc is already Arc, clone it for the async task
288                let target_for_task = target_arc.clone();
289                if !target_for_task.is_enabled() {
290                    debug!(
291                        event = EVENT_NOTIFY_DISPATCH_SKIPPED,
292                        component = LOG_COMPONENT_NOTIFY,
293                        subsystem = LOG_SUBSYSTEM_DISPATCH,
294                        target_id = %target_for_task.id(),
295                        reason = "target_disabled",
296                        "Skipped notify dispatch target"
297                    );
298                    continue;
299                }
300                let is_deferred = target_for_task.store().is_some();
301                let direct_dispatch_lease = (!is_deferred).then(|| target_list_guard.direct_dispatch_lease());
302                let direct_cancellation = direct_dispatch_lease.as_ref().map(DirectDispatchLease::cancellation);
303                let deferred_dispatch_guard = is_deferred.then(|| Arc::clone(&dispatch_guard));
304                let limiter = if is_deferred {
305                    self.enqueue_limiter.clone()
306                } else {
307                    self.send_limiter.clone()
308                };
309                let metrics = self.metrics.clone();
310                let event_clone = event.clone();
311                let target_name_for_task = target_for_task.name(); // Get the name before generating the task
312                debug!(
313                    event = EVENT_NOTIFY_DISPATCH_STARTED,
314                    component = LOG_COMPONENT_NOTIFY,
315                    subsystem = LOG_SUBSYSTEM_DISPATCH,
316                    target_id = %target_for_task.id(),
317                    deferred = is_deferred,
318                    "notify dispatch"
319                );
320                // Use cloned data in closures to avoid borrowing conflicts
321                // Create an EntityTarget from the event
322                let entity_target: Arc<EntityTarget<Event>> = Arc::new(EntityTarget {
323                    object_name: object_key.to_string(),
324                    bucket_name: bucket_name.to_string(),
325                    event_name,
326                    data: event_clone.as_ref().clone(),
327                });
328                let handle = tokio::spawn(async move {
329                    let _direct_dispatch_lease = direct_dispatch_lease;
330                    let _deferred_dispatch_guard = deferred_dispatch_guard;
331                    metrics.increment_processing();
332                    let permit = if let Some(cancellation) = direct_cancellation {
333                        tokio::select! {
334                            biased;
335                            _ = cancellation.cancelled() => {
336                                metrics.decrement_processing();
337                                metrics.increment_skipped();
338                                debug!(
339                                    event = EVENT_NOTIFY_DISPATCH_SKIPPED,
340                                    component = LOG_COMPONENT_NOTIFY,
341                                    subsystem = LOG_SUBSYSTEM_DISPATCH,
342                                    target_id = %target_name_for_task,
343                                    reason = "runtime_generation_replaced",
344                                    "Skipped pending direct notify dispatch"
345                                );
346                                return;
347                            }
348                            permit = limiter.acquire_owned() => permit,
349                        }
350                    } else {
351                        limiter.acquire_owned().await
352                    };
353                    let _permit = match permit {
354                        Ok(permit) => permit,
355                        Err(e) => {
356                            error!(
357                                event = EVENT_NOTIFY_DISPATCH_FAILED,
358                                component = LOG_COMPONENT_NOTIFY,
359                                subsystem = LOG_SUBSYSTEM_DISPATCH,
360                                target_id = %target_name_for_task,
361                                error = %e,
362                                reason = "permit_acquire_failed",
363                                "Failed to acquire notify dispatch permit"
364                            );
365                            metrics.increment_failed();
366                            return;
367                        }
368                    };
369                    if let Err(e) = target_for_task.save(entity_target.clone()).await {
370                        metrics.increment_failed();
371                        error!(
372                            event = EVENT_NOTIFY_DISPATCH_FAILED,
373                            component = LOG_COMPONENT_NOTIFY,
374                            subsystem = LOG_SUBSYSTEM_DISPATCH,
375                            target_id = %target_name_for_task,
376                            error = %e,
377                            reason = "send_failed",
378                            "Failed to dispatch notify event"
379                        );
380                    } else {
381                        // Counting lifecycle (backlog#979): `increment_processing` above marks
382                        // the event as in-flight exactly once. For a non-deferred target, `save`
383                        // performs synchronous delivery, so the event is fully processed here and
384                        // we transition it from processing to processed. For a deferred
385                        // (store-backed) target, `save` only enqueues the event to the store: it
386                        // stays in-flight until the replay worker actually delivers it and emits
387                        // `ReplayEvent::Delivered`, which is the single place that calls
388                        // `increment_processed` for that event. Decrementing `processing_events`
389                        // here as well would double-count the completion and underflow the
390                        // counter back to `usize::MAX`, corrupting metrics and any in-flight
391                        // backpressure decisions.
392                        if !is_deferred {
393                            metrics.increment_processed();
394                        }
395                        debug!(
396                            event = EVENT_NOTIFY_DISPATCH_COMPLETED,
397                            component = LOG_COMPONENT_NOTIFY,
398                            subsystem = LOG_SUBSYSTEM_DISPATCH,
399                            target_id = %target_name_for_task,
400                            deferred = is_deferred,
401                            "notify dispatch"
402                        );
403                    }
404                });
405                if is_deferred {
406                    deferred_handles.push(handle);
407                } else {
408                    direct_handles.push(handle);
409                }
410            } else {
411                warn!(
412                    event = EVENT_NOTIFY_DISPATCH_FAILED,
413                    component = LOG_COMPONENT_NOTIFY,
414                    subsystem = LOG_SUBSYSTEM_DISPATCH,
415                    target_id = %target_id,
416                    reason = "target_missing_from_runtime",
417                    "Matched notify target is missing from runtime"
418                );
419                self.metrics.increment_skipped();
420            }
421        }
422        // target_list is automatically released here
423        drop(target_list_guard);
424
425        // Every store-backed save owns a share of the generation guard, so
426        // caller cancellation cannot race lifecycle handoff with an enqueue.
427        // Direct targets own no queue store and may finish against the detached
428        // target while lifecycle progresses.
429        drop(dispatch_guard);
430        wait_for_dispatch_tasks(deferred_handles).await;
431        wait_for_dispatch_tasks(direct_handles).await;
432        debug!(
433            event = EVENT_NOTIFY_DISPATCH_COMPLETED,
434            component = LOG_COMPONENT_NOTIFY,
435            subsystem = LOG_SUBSYSTEM_DISPATCH,
436            bucket = %bucket_name,
437            target_count = target_ids_len,
438            "notify dispatch"
439        );
440    }
441
442    /// Initializes the targets for buckets from shared target handles.
443    #[instrument(skip(self, targets_to_init))]
444    pub async fn init_bucket_targets_shared(&self, targets_to_init: Vec<SharedTarget<Event>>) -> Result<(), NotificationError> {
445        let mut target_list_guard = self.target_list.write().await;
446        // Close the currently registered targets before replacing them. A bare
447        // `clear()` drops the old targets without invoking `close()`, leaking their
448        // connections/streams; `clear_targets_only` closes each one first (backlog#984).
449        target_list_guard.clear_targets_only().await;
450
451        for target in targets_to_init {
452            debug!(
453                event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
454                component = LOG_COMPONENT_NOTIFY,
455                subsystem = LOG_SUBSYSTEM_DISPATCH,
456                state = "target_init",
457                target_id = %target.id(),
458                "Initializing notify runtime target"
459            );
460            target_list_guard.add(target)?;
461        }
462
463        tracing::info!(
464            event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
465            component = LOG_COMPONENT_NOTIFY,
466            subsystem = LOG_SUBSYSTEM_DISPATCH,
467            state = "targets_initialized",
468            target_count = target_list_guard.len(),
469            "notify runtime lifecycle"
470        );
471        Ok(())
472    }
473}
474
475/// A thread-safe list of targets
476pub struct TargetList {
477    direct_dispatches: Arc<DirectDispatchTracker>,
478    /// Map of TargetID to Target
479    runtime: TargetRuntimeManager<Event>,
480}
481
482impl Default for TargetList {
483    fn default() -> Self {
484        Self::new()
485    }
486}
487
488impl TargetList {
489    /// Creates a new TargetList
490    pub fn new() -> Self {
491        TargetList {
492            direct_dispatches: Arc::new(DirectDispatchTracker::new()),
493            runtime: TargetRuntimeManager::new(),
494        }
495    }
496
497    /// Adds a target to the list
498    ///
499    /// # Arguments
500    /// * `target` - The target to add
501    ///
502    /// # Returns
503    /// Returns `Ok(())` if the target was added successfully, or a `NotificationError` if an error occurred.
504    pub fn add(&mut self, target: Arc<dyn Target<Event> + Send + Sync>) -> Result<(), NotificationError> {
505        let id = target.id();
506        if self.runtime.get_by_target_id(&id).is_some() {
507            // Potentially update or log a warning/error if replacing an existing target.
508            warn!(
509                event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
510                component = LOG_COMPONENT_NOTIFY,
511                subsystem = LOG_SUBSYSTEM_DISPATCH,
512                state = "target_overwrite",
513                target_id = %id,
514                "Overwriting existing notify runtime target"
515            );
516        }
517        self.runtime.add_arc(target);
518        Ok(())
519    }
520
521    /// Clears all targets from the list
522    pub fn clear(&mut self) {
523        self.runtime.clear();
524    }
525
526    /// Removes a target by ID. Note: This does not stop its associated event stream.
527    /// Stream cancellation should be handled by EventNotifier.
528    ///
529    /// # Arguments
530    /// * `id` - The ID of the target to remove
531    ///
532    /// # Returns
533    /// Returns the removed target if it existed, otherwise `None`.
534    pub async fn remove_target_only(&mut self, id: &TargetID) -> Option<SharedTarget<Event>> {
535        self.runtime.remove_by_target_id_and_close(id).await
536    }
537
538    /// Clears all targets from the list. Note: This does not stop their associated event streams.
539    /// Stream cancellation should be handled by EventNotifier.
540    pub async fn clear_targets_only(&mut self) {
541        self.runtime.clear_and_close().await;
542    }
543
544    /// Returns a target by ID
545    ///
546    /// # Arguments
547    /// * `id` - The ID of the target to retrieve
548    ///
549    /// # Returns
550    /// Returns the target if it exists, otherwise `None`.
551    pub fn get(&self, id: &TargetID) -> Option<SharedTarget<Event>> {
552        self.runtime.get_by_target_id(id)
553    }
554
555    /// Returns all target IDs
556    pub fn keys(&self) -> Vec<TargetID> {
557        self.runtime.target_ids()
558    }
559
560    /// Returns all targets in the list
561    pub fn values(&self) -> Vec<SharedTarget<Event>> {
562        self.runtime.values()
563    }
564
565    pub fn runtime_snapshots(&self) -> Vec<rustfs_targets::RuntimeTargetSnapshot> {
566        self.runtime.snapshots()
567    }
568
569    pub async fn runtime_health_snapshots(&self) -> Vec<rustfs_targets::RuntimeTargetHealthSnapshot> {
570        self.runtime.health_snapshots().await
571    }
572
573    pub fn runtime_status_snapshot(
574        &self,
575        replay_workers: &rustfs_targets::ReplayWorkerManager,
576    ) -> rustfs_targets::RuntimeStatusSnapshot {
577        self.runtime.status_snapshot(replay_workers)
578    }
579
580    fn direct_dispatch_lease(&self) -> DirectDispatchLease {
581        self.direct_dispatches.acquire()
582    }
583
584    pub(crate) fn replace_runtime(
585        &mut self,
586        replacement: TargetRuntimeManager<Event>,
587    ) -> (TargetRuntimeManager<Event>, Arc<DirectDispatchTracker>) {
588        let runtime = std::mem::replace(&mut self.runtime, replacement);
589        let direct_dispatches = std::mem::replace(&mut self.direct_dispatches, Arc::new(DirectDispatchTracker::new()));
590        direct_dispatches.cancel_pending();
591        (runtime, direct_dispatches)
592    }
593
594    pub fn runtime_mut(&mut self) -> &mut TargetRuntimeManager<Event> {
595        &mut self.runtime
596    }
597
598    /// Returns the number of targets
599    pub fn len(&self) -> usize {
600        self.runtime.len()
601    }
602
603    /// is_empty can be derived from len()
604    pub fn is_empty(&self) -> bool {
605        self.runtime.is_empty()
606    }
607}
608
609#[cfg(test)]
610mod tests {
611    use super::*;
612    use crate::{rule_engine::NotifyRuleEngine, rules::RulesMap};
613    use rustfs_s3_types::EventName;
614    use rustfs_targets::testkit::MockTarget;
615    use rustfs_targets::{ReplayWorkerManager, store::QueueStore};
616    use std::sync::Arc;
617    use tokio::sync::Notify;
618
619    #[tokio::test]
620    async fn encoded_event_key_matches_raw_prefix_suffix_filter() {
621        let target_id = TargetID::new("primary".to_string(), "webhook".to_string());
622        let mut rules_map = RulesMap::new();
623        rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "uploads/*.csv".to_string(), target_id.clone());
624
625        let rule_engine = NotifyRuleEngine::new();
626        rule_engine.set_bucket_rules("test-bucket", rules_map).await;
627
628        let targets = rule_engine
629            .match_targets("test-bucket", EventName::ObjectCreatedPut, "uploads%2Freport.csv")
630            .await;
631
632        assert!(targets.contains(&target_id));
633    }
634
635    #[tokio::test]
636    async fn encoded_event_key_matches_raw_and_decoded_rule_targets() {
637        let raw_target = TargetID::new("raw".to_string(), "webhook".to_string());
638        let decoded_target = TargetID::new("decoded".to_string(), "webhook".to_string());
639        let mut rules_map = RulesMap::new();
640        rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "uploads%2F*.csv".to_string(), raw_target.clone());
641        rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "uploads/*.csv".to_string(), decoded_target.clone());
642
643        let rule_engine = NotifyRuleEngine::new();
644        rule_engine.set_bucket_rules("test-bucket", rules_map).await;
645
646        let targets = rule_engine
647            .match_targets("test-bucket", EventName::ObjectCreatedPut, "uploads%2Freport.csv")
648            .await;
649
650        assert_eq!(targets.len(), 2);
651        assert!(targets.contains(&raw_target));
652        assert!(targets.contains(&decoded_target));
653    }
654
655    #[tokio::test]
656    async fn encoded_event_key_does_not_bypass_suffix_filter() {
657        let target_id = TargetID::new("primary".to_string(), "webhook".to_string());
658        let mut rules_map = RulesMap::new();
659        rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "uploads/*.csv".to_string(), target_id);
660
661        let rule_engine = NotifyRuleEngine::new();
662        rule_engine.set_bucket_rules("test-bucket", rules_map).await;
663
664        let root_targets = rule_engine
665            .match_targets("test-bucket", EventName::ObjectCreatedPut, "report.csv")
666            .await;
667        let suffix_targets = rule_engine
668            .match_targets("test-bucket", EventName::ObjectCreatedPut, "uploads%2Freport.txt")
669            .await;
670
671        assert!(root_targets.is_empty());
672        assert!(suffix_targets.is_empty());
673    }
674
675    #[tokio::test]
676    async fn lifecycle_pause_drains_entered_deferred_dispatch_and_blocks_new_dispatch() {
677        let metrics = Arc::new(NotificationMetrics::new());
678        let rule_engine = NotifyRuleEngine::new();
679        let notifier = Arc::new(EventNotifier::new(metrics.clone(), rule_engine.clone()));
680        let save_entered = Arc::new(Notify::new());
681        let save_release = Arc::new(Notify::new());
682        let queue_dir = tempfile::tempdir().expect("queue tempdir should be created");
683        let target = MockTarget::new("gated-target", "webhook")
684            .with_first_save_gate(save_entered.clone(), save_release.clone())
685            .with_store(Arc::new(QueueStore::new(queue_dir.path(), 16, ".event")));
686
687        let mut rules_map = RulesMap::new();
688        rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), target.target_id());
689        rule_engine.set_bucket_rules("bucket", rules_map).await;
690        notifier
691            .target_list()
692            .write()
693            .await
694            .add(Arc::new(target.clone()))
695            .expect("target install should succeed");
696
697        let facade = crate::runtime_facade::NotifyRuntimeFacade::new_with_dispatch_gate(
698            notifier.target_list(),
699            Arc::new(RwLock::new(ReplayWorkerManager::new())),
700            notifier.dispatch_gate(),
701            Arc::new(Semaphore::new(1)),
702            metrics,
703        );
704        let first_dispatch = tokio::spawn({
705            let notifier = notifier.clone();
706            async move {
707                notifier
708                    .send(Arc::new(Event::new_test_event("bucket", "first", EventName::ObjectCreatedPut)))
709                    .await;
710            }
711        });
712        save_entered.notified().await;
713        assert_eq!(target.save_call_count(), 1);
714
715        let mut pause = Box::pin(facade.pause_dispatch());
716        tokio::select! {
717            biased;
718            _ = &mut pause => panic!("lifecycle pause crossed an in-flight dispatch"),
719            _ = std::future::ready(()) => {}
720        }
721
722        save_release.notify_one();
723        first_dispatch.await.expect("first dispatch task should finish");
724        let pause_guard = pause.await;
725
726        let replacement = MockTarget::new("gated-target", "webhook");
727        {
728            let target_list = notifier.target_list();
729            let mut target_list = target_list.write().await;
730            target_list.clear();
731            target_list
732                .add(Arc::new(replacement.clone()))
733                .expect("replacement target install should succeed");
734        }
735
736        let mut second_dispatch =
737            Box::pin(notifier.send(Arc::new(Event::new_test_event("bucket", "second", EventName::ObjectCreatedPut))));
738        tokio::select! {
739            biased;
740            _ = &mut second_dispatch => panic!("dispatch crossed the lifecycle pause"),
741            _ = std::future::ready(()) => {}
742        }
743        assert_eq!(
744            replacement.enabled_call_count(),
745            0,
746            "a paused dispatch must not select a target from the replacement generation early"
747        );
748        assert_eq!(target.save_call_count(), 1);
749        assert_eq!(replacement.save_call_count(), 0);
750
751        drop(pause_guard);
752        second_dispatch.await;
753        assert_eq!(target.enabled_call_count(), 1);
754        assert_eq!(target.save_call_count(), 1);
755        assert_eq!(replacement.enabled_call_count(), 1);
756        assert_eq!(replacement.save_call_count(), 1);
757    }
758
759    #[tokio::test]
760    async fn lifecycle_pause_does_not_wait_for_direct_network_dispatch() {
761        let metrics = Arc::new(NotificationMetrics::new());
762        let rule_engine = NotifyRuleEngine::new();
763        let notifier = Arc::new(EventNotifier::new(metrics.clone(), rule_engine.clone()));
764        let save_entered = Arc::new(Notify::new());
765        let save_release = Arc::new(Notify::new());
766        let target = MockTarget::new("direct-target", "webhook").with_first_save_gate(save_entered.clone(), save_release.clone());
767
768        let mut rules_map = RulesMap::new();
769        rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), target.target_id());
770        rule_engine.set_bucket_rules("bucket", rules_map).await;
771        notifier
772            .target_list()
773            .write()
774            .await
775            .add(Arc::new(target))
776            .expect("target install should succeed");
777
778        let facade = crate::runtime_facade::NotifyRuntimeFacade::new_with_dispatch_gate(
779            notifier.target_list(),
780            Arc::new(RwLock::new(ReplayWorkerManager::new())),
781            notifier.dispatch_gate(),
782            Arc::new(Semaphore::new(1)),
783            metrics,
784        );
785        let dispatch = tokio::spawn({
786            let notifier = notifier.clone();
787            async move {
788                notifier
789                    .send(Arc::new(Event::new_test_event("bucket", "object", EventName::ObjectCreatedPut)))
790                    .await;
791            }
792        });
793        save_entered.notified().await;
794
795        let mut pause = Box::pin(facade.pause_dispatch());
796        let pause_guard = tokio::select! {
797            biased;
798            guard = &mut pause => guard,
799            _ = std::future::ready(()) => panic!("a direct network send blocked lifecycle handoff"),
800        };
801        drop(pause_guard);
802        save_release.notify_one();
803        dispatch.await.expect("direct dispatch should finish after release");
804    }
805
806    #[tokio::test]
807    async fn replacement_cancels_permit_waiting_direct_dispatch_before_closing_target() {
808        let metrics = Arc::new(NotificationMetrics::new());
809        let rule_engine = NotifyRuleEngine::new();
810        let notifier = Arc::new(EventNotifier {
811            send_limiter: Arc::new(Semaphore::new(1)),
812            ..EventNotifier::new(metrics.clone(), rule_engine.clone())
813        });
814        let first_entered = Arc::new(Notify::new());
815        let first_release = Arc::new(Notify::new());
816        let target =
817            MockTarget::new("direct-target", "webhook").with_first_save_gate(first_entered.clone(), first_release.clone());
818
819        let mut rules_map = RulesMap::new();
820        rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), target.target_id());
821        rule_engine.set_bucket_rules("bucket", rules_map).await;
822        notifier
823            .target_list()
824            .write()
825            .await
826            .add(Arc::new(target.clone()))
827            .expect("target should install");
828
829        let facade = crate::runtime_facade::NotifyRuntimeFacade::new_with_dispatch_gate(
830            notifier.target_list(),
831            Arc::new(RwLock::new(ReplayWorkerManager::new())),
832            notifier.dispatch_gate(),
833            Arc::new(Semaphore::new(1)),
834            metrics.clone(),
835        );
836        let first = tokio::spawn({
837            let notifier = notifier.clone();
838            async move {
839                notifier
840                    .send(Arc::new(Event::new_test_event("bucket", "first", EventName::ObjectCreatedPut)))
841                    .await;
842            }
843        });
844        first_entered.notified().await;
845
846        // This task selects the old generation and acquires its lease before
847        // waiting for the saturated direct-send permit.
848        let second = tokio::spawn({
849            let notifier = notifier.clone();
850            async move {
851                notifier
852                    .send(Arc::new(Event::new_test_event("bucket", "second", EventName::ObjectCreatedPut)))
853                    .await;
854            }
855        });
856        tokio::time::timeout(std::time::Duration::from_secs(1), async {
857            while target.enabled_call_count() != 2 {
858                tokio::task::yield_now().await;
859            }
860        })
861        .await
862        .expect("both direct sends should select the old generation");
863
864        let activation = facade.activate_targets_with_replay(Vec::new()).await;
865        let mut replace = Box::pin(facade.replace_targets(activation));
866        tokio::select! {
867            biased;
868            result = &mut replace => panic!("replacement closed a generation with selected direct sends: {result:?}"),
869            _ = std::future::ready(()) => {}
870        }
871        assert_eq!(target.close_call_count(), 0);
872
873        first_release.notify_one();
874        first.await.expect("first direct dispatch should finish");
875        second.await.expect("permit-waiting direct dispatch should be cancelled");
876        replace.await.expect("replacement should close after direct leases drain");
877        assert_eq!(target.save_call_count(), 1);
878        assert_eq!(target.close_call_count(), 1);
879        assert_eq!(metrics.processing_count(), 0);
880        assert_eq!(metrics.processed_count(), 1);
881        assert_eq!(metrics.skipped_count(), 1);
882    }
883
884    #[tokio::test]
885    async fn caller_abort_does_not_release_deferred_generation_lease() {
886        let metrics = Arc::new(NotificationMetrics::new());
887        let rule_engine = NotifyRuleEngine::new();
888        let notifier = Arc::new(EventNotifier::new(metrics.clone(), rule_engine.clone()));
889        let save_entered = Arc::new(Notify::new());
890        let save_release = Arc::new(Notify::new());
891        let queue_dir = tempfile::tempdir().expect("queue tempdir should be created");
892        let target = MockTarget::new("deferred", "webhook")
893            .with_first_save_gate(save_entered.clone(), save_release.clone())
894            .with_store(Arc::new(QueueStore::new(queue_dir.path(), 16, ".event")));
895
896        let mut rules_map = RulesMap::new();
897        rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), target.target_id());
898        rule_engine.set_bucket_rules("bucket", rules_map).await;
899        notifier
900            .target_list()
901            .write()
902            .await
903            .add(Arc::new(target))
904            .expect("target should install");
905        let facade = crate::runtime_facade::NotifyRuntimeFacade::new_with_dispatch_gate(
906            notifier.target_list(),
907            Arc::new(RwLock::new(ReplayWorkerManager::new())),
908            notifier.dispatch_gate(),
909            Arc::new(Semaphore::new(1)),
910            metrics,
911        );
912
913        let dispatch = tokio::spawn({
914            let notifier = notifier.clone();
915            async move {
916                notifier
917                    .send(Arc::new(Event::new_test_event("bucket", "object", EventName::ObjectCreatedPut)))
918                    .await;
919            }
920        });
921        save_entered.notified().await;
922        dispatch.abort();
923        let _ = dispatch.await;
924
925        let mut pause = Box::pin(facade.pause_dispatch());
926        tokio::select! {
927            biased;
928            _ = &mut pause => panic!("caller abort released the deferred generation lease"),
929            _ = std::future::ready(()) => {}
930        }
931        save_release.notify_one();
932        let pause_guard = pause.await;
933        drop(pause_guard);
934    }
935
936    #[tokio::test]
937    async fn deferred_enqueue_concurrency_is_bounded() {
938        const LIMIT: usize = 2;
939        const TARGETS: usize = 3;
940
941        let metrics = Arc::new(NotificationMetrics::new());
942        let rule_engine = NotifyRuleEngine::new();
943        let notifier = Arc::new(EventNotifier {
944            enqueue_limiter: Arc::new(Semaphore::new(LIMIT)),
945            ..EventNotifier::new(metrics, rule_engine.clone())
946        });
947        let entered = Arc::new(Notify::new());
948        let release = Arc::new(Notify::new());
949        let queue_dir = tempfile::tempdir().expect("queue tempdir should be created");
950        let mut targets = Vec::new();
951        let mut rules_map = RulesMap::new();
952        for index in 0..TARGETS {
953            let target = MockTarget::new(&format!("deferred-{index}"), "webhook")
954                .with_first_save_gate(entered.clone(), release.clone())
955                .with_store(Arc::new(QueueStore::new(queue_dir.path().join(index.to_string()), 16, ".event")));
956            rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), target.target_id());
957            notifier
958                .target_list()
959                .write()
960                .await
961                .add(Arc::new(target.clone()))
962                .expect("target should install");
963            targets.push(target);
964        }
965        rule_engine.set_bucket_rules("bucket", rules_map).await;
966
967        let dispatch = tokio::spawn({
968            let notifier = notifier.clone();
969            async move {
970                notifier
971                    .send(Arc::new(Event::new_test_event("bucket", "object", EventName::ObjectCreatedPut)))
972                    .await;
973            }
974        });
975        let total_calls = || targets.iter().map(|target| target.save_call_count()).sum::<usize>();
976        tokio::time::timeout(std::time::Duration::from_secs(1), async {
977            while total_calls() != LIMIT {
978                tokio::task::yield_now().await;
979            }
980        })
981        .await
982        .expect("the configured number of enqueues should enter");
983        for _ in 0..10 {
984            tokio::task::yield_now().await;
985        }
986        assert_eq!(total_calls(), LIMIT, "enqueue concurrency exceeded its semaphore capacity");
987
988        release.notify_waiters();
989        tokio::time::timeout(std::time::Duration::from_secs(1), async {
990            while total_calls() != TARGETS {
991                tokio::task::yield_now().await;
992            }
993        })
994        .await
995        .expect("the waiting enqueue should enter after a permit is released");
996        release.notify_waiters();
997        dispatch.await.expect("all bounded enqueues should finish");
998    }
999
1000    #[tokio::test]
1001    async fn deferred_enqueue_does_not_wait_for_a_blocked_direct_send_permit() {
1002        let metrics = Arc::new(NotificationMetrics::new());
1003        let rule_engine = NotifyRuleEngine::new();
1004        let notifier = Arc::new(EventNotifier {
1005            send_limiter: Arc::new(Semaphore::new(1)),
1006            ..EventNotifier::new(metrics.clone(), rule_engine.clone())
1007        });
1008        let direct_entered = Arc::new(Notify::new());
1009        let direct_release = Arc::new(Notify::new());
1010        let direct = MockTarget::new("direct", "webhook").with_first_save_gate(direct_entered.clone(), direct_release.clone());
1011        let deferred_entered = Arc::new(Notify::new());
1012        let deferred_release = Arc::new(Notify::new());
1013        let queue_dir = tempfile::tempdir().expect("queue tempdir should be created");
1014        let deferred = MockTarget::new("deferred", "webhook")
1015            .with_first_save_gate(deferred_entered.clone(), deferred_release.clone())
1016            .with_store(Arc::new(QueueStore::new(queue_dir.path(), 16, ".event")));
1017
1018        let mut direct_rules = RulesMap::new();
1019        direct_rules.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), direct.target_id());
1020        rule_engine.set_bucket_rules("direct-bucket", direct_rules).await;
1021        let mut deferred_rules = RulesMap::new();
1022        deferred_rules.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), deferred.target_id());
1023        rule_engine.set_bucket_rules("deferred-bucket", deferred_rules).await;
1024        {
1025            let target_list = notifier.target_list();
1026            let mut target_list = target_list.write().await;
1027            target_list.add(Arc::new(direct)).expect("direct target should install");
1028            target_list.add(Arc::new(deferred)).expect("deferred target should install");
1029        }
1030
1031        let facade = crate::runtime_facade::NotifyRuntimeFacade::new_with_dispatch_gate(
1032            notifier.target_list(),
1033            Arc::new(RwLock::new(ReplayWorkerManager::new())),
1034            notifier.dispatch_gate(),
1035            Arc::new(Semaphore::new(1)),
1036            metrics,
1037        );
1038        let direct_dispatch = tokio::spawn({
1039            let notifier = notifier.clone();
1040            async move {
1041                notifier
1042                    .send(Arc::new(Event::new_test_event("direct-bucket", "object", EventName::ObjectCreatedPut)))
1043                    .await;
1044            }
1045        });
1046        direct_entered.notified().await;
1047        let deferred_dispatch = tokio::spawn({
1048            let notifier = notifier.clone();
1049            async move {
1050                notifier
1051                    .send(Arc::new(Event::new_test_event("deferred-bucket", "object", EventName::ObjectCreatedPut)))
1052                    .await;
1053            }
1054        });
1055        tokio::time::timeout(std::time::Duration::from_secs(1), deferred_entered.notified())
1056            .await
1057            .expect("queue persistence must not wait behind a direct network send permit");
1058
1059        let mut pause = Box::pin(facade.pause_dispatch());
1060        tokio::select! {
1061            biased;
1062            _ = &mut pause => panic!("lifecycle pause crossed the blocked deferred enqueue"),
1063            _ = std::future::ready(()) => {}
1064        }
1065        deferred_release.notify_one();
1066        deferred_dispatch
1067            .await
1068            .expect("deferred dispatch should finish after release");
1069        let pause_guard = tokio::select! {
1070            biased;
1071            guard = &mut pause => guard,
1072            _ = std::future::ready(()) => panic!("direct network delivery kept the lifecycle gate locked"),
1073        };
1074        drop(pause_guard);
1075
1076        direct_release.notify_one();
1077        direct_dispatch.await.expect("direct dispatch should finish after release");
1078    }
1079
1080    #[tokio::test]
1081    async fn test_send_event_skips_disabled_target() {
1082        let rule_engine = NotifyRuleEngine::new();
1083        let notifier = EventNotifier::new(Arc::new(NotificationMetrics::new()), rule_engine.clone());
1084
1085        let enabled_target = MockTarget::new("enabled-target", "webhook");
1086        let disabled_target = MockTarget::new("disabled-target", "webhook").disabled();
1087
1088        let mut rules_map = RulesMap::new();
1089        rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), enabled_target.target_id());
1090        rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), disabled_target.target_id());
1091
1092        rule_engine.set_bucket_rules("bucket", rules_map).await;
1093        notifier
1094            .target_list()
1095            .write()
1096            .await
1097            .add(Arc::new(enabled_target.clone()))
1098            .unwrap();
1099        notifier
1100            .target_list()
1101            .write()
1102            .await
1103            .add(Arc::new(disabled_target.clone()))
1104            .unwrap();
1105
1106        let event = Arc::new(Event::new_test_event("bucket", "object", EventName::ObjectCreatedPut));
1107        notifier.send(event).await;
1108
1109        assert_eq!(enabled_target.save_call_count(), 1);
1110        assert_eq!(disabled_target.save_call_count(), 0);
1111    }
1112
1113    #[tokio::test]
1114    async fn send_event_respects_prefix_suffix_filters() {
1115        let rule_engine = NotifyRuleEngine::new();
1116        let notifier = EventNotifier::new(Arc::new(NotificationMetrics::new()), rule_engine.clone());
1117        let target = MockTarget::new("filtered-target", "webhook");
1118        let mut rules_map = RulesMap::new();
1119        rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "uploads/*.csv".to_string(), target.target_id());
1120
1121        rule_engine.set_bucket_rules("bucket", rules_map).await;
1122        notifier.target_list().write().await.add(Arc::new(target.clone())).unwrap();
1123
1124        notifier
1125            .send(Arc::new(Event::new_test_event("bucket", "report.csv", EventName::ObjectCreatedPut)))
1126            .await;
1127        notifier
1128            .send(Arc::new(Event::new_test_event(
1129                "bucket",
1130                "uploads/report.txt",
1131                EventName::ObjectCreatedPut,
1132            )))
1133            .await;
1134
1135        assert_eq!(target.save_call_count(), 0);
1136
1137        notifier
1138            .send(Arc::new(Event::new_test_event(
1139                "bucket",
1140                "uploads/report.csv",
1141                EventName::ObjectCreatedPut,
1142            )))
1143            .await;
1144
1145        assert_eq!(target.save_call_count(), 1);
1146    }
1147
1148    /// Builds a store-backed (deferred) mock target. `save` only bumps the mock's counter, so the
1149    /// attached store is never written to: it just has to exist so the notifier treats the target
1150    /// as store-backed (deferred delivery), exercising the deferred counting path.
1151    fn deferred_test_target(id: &str, name: &str) -> MockTarget {
1152        MockTarget::new(id, name).with_store(Arc::new(QueueStore::new(
1153            std::env::temp_dir().join("rustfs-notify-979-noop-store"),
1154            0,
1155            "",
1156        )))
1157    }
1158
1159    /// Regression test for backlog#979 (a): dispatching to a store-backed
1160    /// (deferred) target must count the event as in-flight exactly once and must
1161    /// not decrement `processing_events` at enqueue time. The replay worker is the
1162    /// single owner of the completion decrement (`increment_processed`); an extra
1163    /// decrement here previously double-counted and underflowed the counter back
1164    /// to `usize::MAX`.
1165    #[tokio::test]
1166    async fn deferred_target_dispatch_does_not_underflow_processing_counter() {
1167        let metrics = Arc::new(NotificationMetrics::new());
1168        let rule_engine = NotifyRuleEngine::new();
1169        let notifier = EventNotifier::new(metrics.clone(), rule_engine.clone());
1170
1171        let target = deferred_test_target("deferred-target", "webhook");
1172        let mut rules_map = RulesMap::new();
1173        rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), target.target_id());
1174        rule_engine.set_bucket_rules("bucket", rules_map).await;
1175        notifier.target_list().write().await.add(Arc::new(target.clone())).unwrap();
1176
1177        // Dispatch enqueues to the store; the event stays in-flight (processing == 1).
1178        notifier
1179            .send(Arc::new(Event::new_test_event("bucket", "object", EventName::ObjectCreatedPut)))
1180            .await;
1181
1182        assert_eq!(target.save_call_count(), 1);
1183        assert_eq!(
1184            metrics.processing_count(),
1185            1,
1186            "a queued deferred event must remain counted as in-flight after enqueue"
1187        );
1188        assert_eq!(metrics.processed_count(), 0);
1189
1190        // Simulate the replay worker delivering the queued event (runtime_facade
1191        // maps `ReplayEvent::Delivered` to `increment_processed`).
1192        metrics.increment_processed();
1193
1194        assert_eq!(
1195            metrics.processing_count(),
1196            0,
1197            "processing counter must return to zero, not underflow to usize::MAX"
1198        );
1199        assert_ne!(metrics.processing_count(), usize::MAX);
1200        assert_eq!(metrics.processed_count(), 1);
1201    }
1202
1203    /// Regression test for backlog#984: a `RUSTFS_NOTIFY_SEND_CONCURRENCY=0`
1204    /// misconfiguration must not build a zero-permit semaphore (which would
1205    /// deadlock every dispatch); it must fall back to the default. Positive values
1206    /// pass through unchanged.
1207    #[test]
1208    fn zero_send_concurrency_falls_back_to_default() {
1209        assert_eq!(
1210            coerce_send_concurrency(0),
1211            DEFAULT_NOTIFY_SEND_CONCURRENCY,
1212            "zero concurrency must fall back to the default, never a zero-permit semaphore"
1213        );
1214        assert!(coerce_send_concurrency(0) > 0, "resolved concurrency must be strictly positive");
1215        assert_eq!(coerce_send_concurrency(1), 1);
1216        assert_eq!(coerce_send_concurrency(128), 128);
1217    }
1218
1219    /// Regression test for backlog#984: replacing the runtime target set via
1220    /// `init_bucket_targets_shared` must `close()` the previously registered
1221    /// targets instead of dropping them silently (connection/stream leak).
1222    #[tokio::test]
1223    async fn init_bucket_targets_shared_closes_replaced_targets() {
1224        let rule_engine = NotifyRuleEngine::new();
1225        let notifier = EventNotifier::new(Arc::new(NotificationMetrics::new()), rule_engine);
1226
1227        let old_target = MockTarget::new("old", "webhook");
1228        notifier
1229            .init_bucket_targets_shared(vec![Arc::new(old_target.clone()) as SharedTarget<Event>])
1230            .await
1231            .expect("initial install should succeed");
1232        assert_eq!(old_target.close_call_count(), 0, "target must not close on first install");
1233
1234        let new_target = MockTarget::new("new", "webhook");
1235        notifier
1236            .init_bucket_targets_shared(vec![Arc::new(new_target.clone()) as SharedTarget<Event>])
1237            .await
1238            .expect("replacement install should succeed");
1239
1240        assert_eq!(old_target.close_call_count(), 1, "the replaced target must be closed exactly once");
1241        assert_eq!(new_target.close_call_count(), 0, "the freshly installed target must stay open");
1242    }
1243}