1use crate::error::NotificationError;
16use crate::{event::Event, integration::NotificationMetrics, rule_engine::NotifyRuleEngine};
17use rustfs_config::notify::{DEFAULT_NOTIFY_SEND_CONCURRENCY, ENV_NOTIFY_SEND_CONCURRENCY};
18use rustfs_targets::Target;
19use rustfs_targets::arn::TargetID;
20use rustfs_targets::target::EntityTarget;
21use rustfs_targets::{SharedTarget, TargetRuntimeManager};
22use std::collections::HashMap;
23use std::sync::{Arc, LazyLock, Mutex as StdMutex, Weak};
24use tokio::sync::{RwLock, Semaphore, watch};
25use tokio::task::JoinHandle;
26use tokio_util::sync::CancellationToken;
27use tracing::{debug, error, info, instrument, warn};
28
29const LOG_COMPONENT_NOTIFY: &str = "notify";
30const LOG_SUBSYSTEM_DISPATCH: &str = "dispatch";
31const EVENT_NOTIFY_DISPATCH_SKIPPED: &str = "notify_dispatch_skipped";
32const EVENT_NOTIFY_DISPATCH_FAILED: &str = "notify_dispatch_failed";
33const EVENT_NOTIFY_DISPATCH_STARTED: &str = "notify_dispatch_started";
34const EVENT_NOTIFY_DISPATCH_COMPLETED: &str = "notify_dispatch_completed";
35const EVENT_NOTIFY_RUNTIME_LIFECYCLE: &str = "notify_runtime_lifecycle";
36
37async fn wait_for_dispatch_tasks(handles: Vec<JoinHandle<()>>) {
38 for handle in handles {
39 if let Err(e) = handle.await {
40 let reason = if e.is_cancelled() { "join_cancelled" } else { "join_panicked" };
41 error!(
42 event = EVENT_NOTIFY_DISPATCH_FAILED,
43 component = LOG_COMPONENT_NOTIFY,
44 subsystem = LOG_SUBSYSTEM_DISPATCH,
45 reason,
46 "Notify dispatch task failed"
47 );
48 }
49 }
50}
51
52pub type SharedNotifyTargetList = Arc<RwLock<TargetList>>;
53
54static TARGET_LIST_DISPATCH_GATES: LazyLock<StdMutex<HashMap<usize, Weak<RwLock<()>>>>> =
55 LazyLock::new(|| StdMutex::new(HashMap::new()));
56
57pub(crate) fn shared_dispatch_gate(target_list: &SharedNotifyTargetList, preferred: Option<Arc<RwLock<()>>>) -> Arc<RwLock<()>> {
58 let key = Arc::as_ptr(target_list) as usize;
59 let mut gates = TARGET_LIST_DISPATCH_GATES.lock().unwrap_or_else(|err| err.into_inner());
60 gates.retain(|_, gate| gate.strong_count() != 0);
61 if let Some(gate) = gates.get(&key).and_then(Weak::upgrade) {
62 return gate;
63 }
64 let gate = preferred.unwrap_or_else(|| Arc::new(RwLock::new(())));
65 gates.insert(key, Arc::downgrade(&gate));
66 gate
67}
68
69pub(crate) struct DirectDispatchTracker {
70 cancellation: CancellationToken,
71 inflight: watch::Sender<usize>,
72}
73
74impl DirectDispatchTracker {
75 fn new() -> Self {
76 let (inflight, _) = watch::channel(0);
77 Self {
78 cancellation: CancellationToken::new(),
79 inflight,
80 }
81 }
82
83 fn acquire(self: &Arc<Self>) -> DirectDispatchLease {
84 self.inflight.send_modify(|count| *count += 1);
85 DirectDispatchLease {
86 tracker: Arc::clone(self),
87 }
88 }
89
90 pub(crate) async fn wait_idle(&self) {
91 let mut inflight = self.inflight.subscribe();
92 while *inflight.borrow_and_update() != 0 {
93 if inflight.changed().await.is_err() {
94 return;
95 }
96 }
97 }
98
99 fn cancel_pending(&self) {
100 self.cancellation.cancel();
101 }
102}
103
104struct DirectDispatchLease {
105 tracker: Arc<DirectDispatchTracker>,
106}
107
108impl DirectDispatchLease {
109 fn cancellation(&self) -> CancellationToken {
110 self.tracker.cancellation.clone()
111 }
112}
113
114impl Drop for DirectDispatchLease {
115 fn drop(&mut self) {
116 self.tracker.inflight.send_modify(|count| {
117 if let Some(next) = count.checked_sub(1) {
118 *count = next;
119 } else {
120 error!(
121 event = EVENT_NOTIFY_DISPATCH_FAILED,
122 component = LOG_COMPONENT_NOTIFY,
123 subsystem = LOG_SUBSYSTEM_DISPATCH,
124 reason = "direct_lease_underflow",
125 "Notify direct dispatch lease accounting underflowed"
126 );
127 }
128 });
129 }
130}
131
132fn resolve_send_concurrency() -> usize {
139 let configured = rustfs_utils::get_env_usize(ENV_NOTIFY_SEND_CONCURRENCY, DEFAULT_NOTIFY_SEND_CONCURRENCY);
140 coerce_send_concurrency(configured)
141}
142
143fn coerce_send_concurrency(configured: usize) -> usize {
147 if configured == 0 {
148 warn!(
149 event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
150 component = LOG_COMPONENT_NOTIFY,
151 subsystem = LOG_SUBSYSTEM_DISPATCH,
152 configured,
153 default = DEFAULT_NOTIFY_SEND_CONCURRENCY,
154 "invalid RUSTFS_NOTIFY_SEND_CONCURRENCY=0; falling back to default to avoid a zero-permit deadlock"
155 );
156 DEFAULT_NOTIFY_SEND_CONCURRENCY
157 } else {
158 configured
159 }
160}
161
162pub struct EventNotifier {
164 dispatch_gate: Arc<RwLock<()>>,
165 enqueue_limiter: Arc<Semaphore>,
166 metrics: Arc<NotificationMetrics>,
167 rule_engine: NotifyRuleEngine,
168 target_list: SharedNotifyTargetList,
169 send_limiter: Arc<Semaphore>,
170}
171
172impl Default for EventNotifier {
173 fn default() -> Self {
174 Self::new(Arc::new(NotificationMetrics::new()), NotifyRuleEngine::new())
175 }
176}
177
178impl EventNotifier {
179 pub fn new(metrics: Arc<NotificationMetrics>, rule_engine: NotifyRuleEngine) -> Self {
184 let max_inflight = resolve_send_concurrency();
185 let target_list = Arc::new(RwLock::new(TargetList::new()));
186 let dispatch_gate = shared_dispatch_gate(&target_list, None);
187 EventNotifier {
188 dispatch_gate,
189 enqueue_limiter: Arc::new(Semaphore::new(max_inflight)),
190 metrics,
191 rule_engine,
192 target_list,
193 send_limiter: Arc::new(Semaphore::new(max_inflight)),
194 }
195 }
196
197 pub fn target_list(&self) -> SharedNotifyTargetList {
203 Arc::clone(&self.target_list)
204 }
205
206 pub(crate) fn dispatch_gate(&self) -> Arc<RwLock<()>> {
207 self.dispatch_gate.clone()
208 }
209
210 pub async fn get_arn_list(&self, region: &str) -> Vec<String> {
218 let target_list_guard = self.target_list.read().await;
219 target_list_guard
220 .keys()
221 .iter()
222 .map(|target_id| target_id.to_arn(region).to_string())
223 .collect()
224 }
225
226 pub async fn remove_all_bucket_targets(&self) {
227 let mut target_list_guard = self.target_list.write().await;
228 target_list_guard.clear_targets_only().await;
229 info!(
230 event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
231 component = LOG_COMPONENT_NOTIFY,
232 subsystem = LOG_SUBSYSTEM_DISPATCH,
233 state = "targets_cleared",
234 "notify runtime lifecycle"
235 );
236 }
237
238 #[instrument(skip_all)]
243 pub async fn send(&self, event: Arc<Event>) {
244 let bucket_name = &event.s3.bucket.name;
245 let object_key = &event.s3.object.key;
246 let event_name = event.event_name;
247
248 let target_ids = self.rule_engine.match_targets(bucket_name, event_name, object_key).await;
249 if target_ids.is_empty() {
250 debug!(
251 event = EVENT_NOTIFY_DISPATCH_SKIPPED,
252 component = LOG_COMPONENT_NOTIFY,
253 subsystem = LOG_SUBSYSTEM_DISPATCH,
254 bucket = %bucket_name,
255 object = %object_key,
256 reason = "no_matching_targets",
257 "Skipped notify dispatch"
258 );
259 self.metrics.increment_skipped();
260 return;
261 }
262 let target_ids_len = target_ids.len();
263 let mut deferred_handles = Vec::new();
264 let mut direct_handles = Vec::new();
265
266 let dispatch_guard = Arc::new(self.dispatch_gate.clone().read_owned().await);
271
272 let target_list_guard = self.target_list.read().await;
274 debug!(
275 event = EVENT_NOTIFY_DISPATCH_STARTED,
276 component = LOG_COMPONENT_NOTIFY,
277 subsystem = LOG_SUBSYSTEM_DISPATCH,
278 bucket = %bucket_name,
279 object = %object_key,
280 target_count = target_ids_len,
281 "notify dispatch"
282 );
283 for target_id in target_ids {
284 if let Some(target_arc) = target_list_guard.get(&target_id) {
286 let target_for_task = target_arc.clone();
289 if !target_for_task.is_enabled() {
290 debug!(
291 event = EVENT_NOTIFY_DISPATCH_SKIPPED,
292 component = LOG_COMPONENT_NOTIFY,
293 subsystem = LOG_SUBSYSTEM_DISPATCH,
294 target_id = %target_for_task.id(),
295 reason = "target_disabled",
296 "Skipped notify dispatch target"
297 );
298 continue;
299 }
300 let is_deferred = target_for_task.store().is_some();
301 let direct_dispatch_lease = (!is_deferred).then(|| target_list_guard.direct_dispatch_lease());
302 let direct_cancellation = direct_dispatch_lease.as_ref().map(DirectDispatchLease::cancellation);
303 let deferred_dispatch_guard = is_deferred.then(|| Arc::clone(&dispatch_guard));
304 let limiter = if is_deferred {
305 self.enqueue_limiter.clone()
306 } else {
307 self.send_limiter.clone()
308 };
309 let metrics = self.metrics.clone();
310 let event_clone = event.clone();
311 let target_name_for_task = target_for_task.name(); debug!(
313 event = EVENT_NOTIFY_DISPATCH_STARTED,
314 component = LOG_COMPONENT_NOTIFY,
315 subsystem = LOG_SUBSYSTEM_DISPATCH,
316 target_id = %target_for_task.id(),
317 deferred = is_deferred,
318 "notify dispatch"
319 );
320 let entity_target: Arc<EntityTarget<Event>> = Arc::new(EntityTarget {
323 object_name: object_key.to_string(),
324 bucket_name: bucket_name.to_string(),
325 event_name,
326 data: event_clone.as_ref().clone(),
327 });
328 let handle = tokio::spawn(async move {
329 let _direct_dispatch_lease = direct_dispatch_lease;
330 let _deferred_dispatch_guard = deferred_dispatch_guard;
331 metrics.increment_processing();
332 let permit = if let Some(cancellation) = direct_cancellation {
333 tokio::select! {
334 biased;
335 _ = cancellation.cancelled() => {
336 metrics.decrement_processing();
337 metrics.increment_skipped();
338 debug!(
339 event = EVENT_NOTIFY_DISPATCH_SKIPPED,
340 component = LOG_COMPONENT_NOTIFY,
341 subsystem = LOG_SUBSYSTEM_DISPATCH,
342 target_id = %target_name_for_task,
343 reason = "runtime_generation_replaced",
344 "Skipped pending direct notify dispatch"
345 );
346 return;
347 }
348 permit = limiter.acquire_owned() => permit,
349 }
350 } else {
351 limiter.acquire_owned().await
352 };
353 let _permit = match permit {
354 Ok(permit) => permit,
355 Err(e) => {
356 error!(
357 event = EVENT_NOTIFY_DISPATCH_FAILED,
358 component = LOG_COMPONENT_NOTIFY,
359 subsystem = LOG_SUBSYSTEM_DISPATCH,
360 target_id = %target_name_for_task,
361 error = %e,
362 reason = "permit_acquire_failed",
363 "Failed to acquire notify dispatch permit"
364 );
365 metrics.increment_failed();
366 return;
367 }
368 };
369 if let Err(e) = target_for_task.save(entity_target.clone()).await {
370 metrics.increment_failed();
371 error!(
372 event = EVENT_NOTIFY_DISPATCH_FAILED,
373 component = LOG_COMPONENT_NOTIFY,
374 subsystem = LOG_SUBSYSTEM_DISPATCH,
375 target_id = %target_name_for_task,
376 error = %e,
377 reason = "send_failed",
378 "Failed to dispatch notify event"
379 );
380 } else {
381 if !is_deferred {
393 metrics.increment_processed();
394 }
395 debug!(
396 event = EVENT_NOTIFY_DISPATCH_COMPLETED,
397 component = LOG_COMPONENT_NOTIFY,
398 subsystem = LOG_SUBSYSTEM_DISPATCH,
399 target_id = %target_name_for_task,
400 deferred = is_deferred,
401 "notify dispatch"
402 );
403 }
404 });
405 if is_deferred {
406 deferred_handles.push(handle);
407 } else {
408 direct_handles.push(handle);
409 }
410 } else {
411 warn!(
412 event = EVENT_NOTIFY_DISPATCH_FAILED,
413 component = LOG_COMPONENT_NOTIFY,
414 subsystem = LOG_SUBSYSTEM_DISPATCH,
415 target_id = %target_id,
416 reason = "target_missing_from_runtime",
417 "Matched notify target is missing from runtime"
418 );
419 self.metrics.increment_skipped();
420 }
421 }
422 drop(target_list_guard);
424
425 drop(dispatch_guard);
430 wait_for_dispatch_tasks(deferred_handles).await;
431 wait_for_dispatch_tasks(direct_handles).await;
432 debug!(
433 event = EVENT_NOTIFY_DISPATCH_COMPLETED,
434 component = LOG_COMPONENT_NOTIFY,
435 subsystem = LOG_SUBSYSTEM_DISPATCH,
436 bucket = %bucket_name,
437 target_count = target_ids_len,
438 "notify dispatch"
439 );
440 }
441
442 #[instrument(skip(self, targets_to_init))]
444 pub async fn init_bucket_targets_shared(&self, targets_to_init: Vec<SharedTarget<Event>>) -> Result<(), NotificationError> {
445 let mut target_list_guard = self.target_list.write().await;
446 target_list_guard.clear_targets_only().await;
450
451 for target in targets_to_init {
452 debug!(
453 event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
454 component = LOG_COMPONENT_NOTIFY,
455 subsystem = LOG_SUBSYSTEM_DISPATCH,
456 state = "target_init",
457 target_id = %target.id(),
458 "Initializing notify runtime target"
459 );
460 target_list_guard.add(target)?;
461 }
462
463 tracing::info!(
464 event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
465 component = LOG_COMPONENT_NOTIFY,
466 subsystem = LOG_SUBSYSTEM_DISPATCH,
467 state = "targets_initialized",
468 target_count = target_list_guard.len(),
469 "notify runtime lifecycle"
470 );
471 Ok(())
472 }
473}
474
475pub struct TargetList {
477 direct_dispatches: Arc<DirectDispatchTracker>,
478 runtime: TargetRuntimeManager<Event>,
480}
481
482impl Default for TargetList {
483 fn default() -> Self {
484 Self::new()
485 }
486}
487
488impl TargetList {
489 pub fn new() -> Self {
491 TargetList {
492 direct_dispatches: Arc::new(DirectDispatchTracker::new()),
493 runtime: TargetRuntimeManager::new(),
494 }
495 }
496
497 pub fn add(&mut self, target: Arc<dyn Target<Event> + Send + Sync>) -> Result<(), NotificationError> {
505 let id = target.id();
506 if self.runtime.get_by_target_id(&id).is_some() {
507 warn!(
509 event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
510 component = LOG_COMPONENT_NOTIFY,
511 subsystem = LOG_SUBSYSTEM_DISPATCH,
512 state = "target_overwrite",
513 target_id = %id,
514 "Overwriting existing notify runtime target"
515 );
516 }
517 self.runtime.add_arc(target);
518 Ok(())
519 }
520
521 pub fn clear(&mut self) {
523 self.runtime.clear();
524 }
525
526 pub async fn remove_target_only(&mut self, id: &TargetID) -> Option<SharedTarget<Event>> {
535 self.runtime.remove_by_target_id_and_close(id).await
536 }
537
538 pub async fn clear_targets_only(&mut self) {
541 self.runtime.clear_and_close().await;
542 }
543
544 pub fn get(&self, id: &TargetID) -> Option<SharedTarget<Event>> {
552 self.runtime.get_by_target_id(id)
553 }
554
555 pub fn keys(&self) -> Vec<TargetID> {
557 self.runtime.target_ids()
558 }
559
560 pub fn values(&self) -> Vec<SharedTarget<Event>> {
562 self.runtime.values()
563 }
564
565 pub fn runtime_snapshots(&self) -> Vec<rustfs_targets::RuntimeTargetSnapshot> {
566 self.runtime.snapshots()
567 }
568
569 pub async fn runtime_health_snapshots(&self) -> Vec<rustfs_targets::RuntimeTargetHealthSnapshot> {
570 self.runtime.health_snapshots().await
571 }
572
573 pub fn runtime_status_snapshot(
574 &self,
575 replay_workers: &rustfs_targets::ReplayWorkerManager,
576 ) -> rustfs_targets::RuntimeStatusSnapshot {
577 self.runtime.status_snapshot(replay_workers)
578 }
579
580 fn direct_dispatch_lease(&self) -> DirectDispatchLease {
581 self.direct_dispatches.acquire()
582 }
583
584 pub(crate) fn replace_runtime(
585 &mut self,
586 replacement: TargetRuntimeManager<Event>,
587 ) -> (TargetRuntimeManager<Event>, Arc<DirectDispatchTracker>) {
588 let runtime = std::mem::replace(&mut self.runtime, replacement);
589 let direct_dispatches = std::mem::replace(&mut self.direct_dispatches, Arc::new(DirectDispatchTracker::new()));
590 direct_dispatches.cancel_pending();
591 (runtime, direct_dispatches)
592 }
593
594 pub fn runtime_mut(&mut self) -> &mut TargetRuntimeManager<Event> {
595 &mut self.runtime
596 }
597
598 pub fn len(&self) -> usize {
600 self.runtime.len()
601 }
602
603 pub fn is_empty(&self) -> bool {
605 self.runtime.is_empty()
606 }
607}
608
609#[cfg(test)]
610mod tests {
611 use super::*;
612 use crate::{rule_engine::NotifyRuleEngine, rules::RulesMap};
613 use rustfs_s3_types::EventName;
614 use rustfs_targets::testkit::MockTarget;
615 use rustfs_targets::{ReplayWorkerManager, store::QueueStore};
616 use std::sync::Arc;
617 use tokio::sync::Notify;
618
619 #[tokio::test]
620 async fn encoded_event_key_matches_raw_prefix_suffix_filter() {
621 let target_id = TargetID::new("primary".to_string(), "webhook".to_string());
622 let mut rules_map = RulesMap::new();
623 rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "uploads/*.csv".to_string(), target_id.clone());
624
625 let rule_engine = NotifyRuleEngine::new();
626 rule_engine.set_bucket_rules("test-bucket", rules_map).await;
627
628 let targets = rule_engine
629 .match_targets("test-bucket", EventName::ObjectCreatedPut, "uploads%2Freport.csv")
630 .await;
631
632 assert!(targets.contains(&target_id));
633 }
634
635 #[tokio::test]
636 async fn encoded_event_key_matches_raw_and_decoded_rule_targets() {
637 let raw_target = TargetID::new("raw".to_string(), "webhook".to_string());
638 let decoded_target = TargetID::new("decoded".to_string(), "webhook".to_string());
639 let mut rules_map = RulesMap::new();
640 rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "uploads%2F*.csv".to_string(), raw_target.clone());
641 rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "uploads/*.csv".to_string(), decoded_target.clone());
642
643 let rule_engine = NotifyRuleEngine::new();
644 rule_engine.set_bucket_rules("test-bucket", rules_map).await;
645
646 let targets = rule_engine
647 .match_targets("test-bucket", EventName::ObjectCreatedPut, "uploads%2Freport.csv")
648 .await;
649
650 assert_eq!(targets.len(), 2);
651 assert!(targets.contains(&raw_target));
652 assert!(targets.contains(&decoded_target));
653 }
654
655 #[tokio::test]
656 async fn encoded_event_key_does_not_bypass_suffix_filter() {
657 let target_id = TargetID::new("primary".to_string(), "webhook".to_string());
658 let mut rules_map = RulesMap::new();
659 rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "uploads/*.csv".to_string(), target_id);
660
661 let rule_engine = NotifyRuleEngine::new();
662 rule_engine.set_bucket_rules("test-bucket", rules_map).await;
663
664 let root_targets = rule_engine
665 .match_targets("test-bucket", EventName::ObjectCreatedPut, "report.csv")
666 .await;
667 let suffix_targets = rule_engine
668 .match_targets("test-bucket", EventName::ObjectCreatedPut, "uploads%2Freport.txt")
669 .await;
670
671 assert!(root_targets.is_empty());
672 assert!(suffix_targets.is_empty());
673 }
674
675 #[tokio::test]
676 async fn lifecycle_pause_drains_entered_deferred_dispatch_and_blocks_new_dispatch() {
677 let metrics = Arc::new(NotificationMetrics::new());
678 let rule_engine = NotifyRuleEngine::new();
679 let notifier = Arc::new(EventNotifier::new(metrics.clone(), rule_engine.clone()));
680 let save_entered = Arc::new(Notify::new());
681 let save_release = Arc::new(Notify::new());
682 let queue_dir = tempfile::tempdir().expect("queue tempdir should be created");
683 let target = MockTarget::new("gated-target", "webhook")
684 .with_first_save_gate(save_entered.clone(), save_release.clone())
685 .with_store(Arc::new(QueueStore::new(queue_dir.path(), 16, ".event")));
686
687 let mut rules_map = RulesMap::new();
688 rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), target.target_id());
689 rule_engine.set_bucket_rules("bucket", rules_map).await;
690 notifier
691 .target_list()
692 .write()
693 .await
694 .add(Arc::new(target.clone()))
695 .expect("target install should succeed");
696
697 let facade = crate::runtime_facade::NotifyRuntimeFacade::new_with_dispatch_gate(
698 notifier.target_list(),
699 Arc::new(RwLock::new(ReplayWorkerManager::new())),
700 notifier.dispatch_gate(),
701 Arc::new(Semaphore::new(1)),
702 metrics,
703 );
704 let first_dispatch = tokio::spawn({
705 let notifier = notifier.clone();
706 async move {
707 notifier
708 .send(Arc::new(Event::new_test_event("bucket", "first", EventName::ObjectCreatedPut)))
709 .await;
710 }
711 });
712 save_entered.notified().await;
713 assert_eq!(target.save_call_count(), 1);
714
715 let mut pause = Box::pin(facade.pause_dispatch());
716 tokio::select! {
717 biased;
718 _ = &mut pause => panic!("lifecycle pause crossed an in-flight dispatch"),
719 _ = std::future::ready(()) => {}
720 }
721
722 save_release.notify_one();
723 first_dispatch.await.expect("first dispatch task should finish");
724 let pause_guard = pause.await;
725
726 let replacement = MockTarget::new("gated-target", "webhook");
727 {
728 let target_list = notifier.target_list();
729 let mut target_list = target_list.write().await;
730 target_list.clear();
731 target_list
732 .add(Arc::new(replacement.clone()))
733 .expect("replacement target install should succeed");
734 }
735
736 let mut second_dispatch =
737 Box::pin(notifier.send(Arc::new(Event::new_test_event("bucket", "second", EventName::ObjectCreatedPut))));
738 tokio::select! {
739 biased;
740 _ = &mut second_dispatch => panic!("dispatch crossed the lifecycle pause"),
741 _ = std::future::ready(()) => {}
742 }
743 assert_eq!(
744 replacement.enabled_call_count(),
745 0,
746 "a paused dispatch must not select a target from the replacement generation early"
747 );
748 assert_eq!(target.save_call_count(), 1);
749 assert_eq!(replacement.save_call_count(), 0);
750
751 drop(pause_guard);
752 second_dispatch.await;
753 assert_eq!(target.enabled_call_count(), 1);
754 assert_eq!(target.save_call_count(), 1);
755 assert_eq!(replacement.enabled_call_count(), 1);
756 assert_eq!(replacement.save_call_count(), 1);
757 }
758
759 #[tokio::test]
760 async fn lifecycle_pause_does_not_wait_for_direct_network_dispatch() {
761 let metrics = Arc::new(NotificationMetrics::new());
762 let rule_engine = NotifyRuleEngine::new();
763 let notifier = Arc::new(EventNotifier::new(metrics.clone(), rule_engine.clone()));
764 let save_entered = Arc::new(Notify::new());
765 let save_release = Arc::new(Notify::new());
766 let target = MockTarget::new("direct-target", "webhook").with_first_save_gate(save_entered.clone(), save_release.clone());
767
768 let mut rules_map = RulesMap::new();
769 rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), target.target_id());
770 rule_engine.set_bucket_rules("bucket", rules_map).await;
771 notifier
772 .target_list()
773 .write()
774 .await
775 .add(Arc::new(target))
776 .expect("target install should succeed");
777
778 let facade = crate::runtime_facade::NotifyRuntimeFacade::new_with_dispatch_gate(
779 notifier.target_list(),
780 Arc::new(RwLock::new(ReplayWorkerManager::new())),
781 notifier.dispatch_gate(),
782 Arc::new(Semaphore::new(1)),
783 metrics,
784 );
785 let dispatch = tokio::spawn({
786 let notifier = notifier.clone();
787 async move {
788 notifier
789 .send(Arc::new(Event::new_test_event("bucket", "object", EventName::ObjectCreatedPut)))
790 .await;
791 }
792 });
793 save_entered.notified().await;
794
795 let mut pause = Box::pin(facade.pause_dispatch());
796 let pause_guard = tokio::select! {
797 biased;
798 guard = &mut pause => guard,
799 _ = std::future::ready(()) => panic!("a direct network send blocked lifecycle handoff"),
800 };
801 drop(pause_guard);
802 save_release.notify_one();
803 dispatch.await.expect("direct dispatch should finish after release");
804 }
805
806 #[tokio::test]
807 async fn replacement_cancels_permit_waiting_direct_dispatch_before_closing_target() {
808 let metrics = Arc::new(NotificationMetrics::new());
809 let rule_engine = NotifyRuleEngine::new();
810 let notifier = Arc::new(EventNotifier {
811 send_limiter: Arc::new(Semaphore::new(1)),
812 ..EventNotifier::new(metrics.clone(), rule_engine.clone())
813 });
814 let first_entered = Arc::new(Notify::new());
815 let first_release = Arc::new(Notify::new());
816 let target =
817 MockTarget::new("direct-target", "webhook").with_first_save_gate(first_entered.clone(), first_release.clone());
818
819 let mut rules_map = RulesMap::new();
820 rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), target.target_id());
821 rule_engine.set_bucket_rules("bucket", rules_map).await;
822 notifier
823 .target_list()
824 .write()
825 .await
826 .add(Arc::new(target.clone()))
827 .expect("target should install");
828
829 let facade = crate::runtime_facade::NotifyRuntimeFacade::new_with_dispatch_gate(
830 notifier.target_list(),
831 Arc::new(RwLock::new(ReplayWorkerManager::new())),
832 notifier.dispatch_gate(),
833 Arc::new(Semaphore::new(1)),
834 metrics.clone(),
835 );
836 let first = tokio::spawn({
837 let notifier = notifier.clone();
838 async move {
839 notifier
840 .send(Arc::new(Event::new_test_event("bucket", "first", EventName::ObjectCreatedPut)))
841 .await;
842 }
843 });
844 first_entered.notified().await;
845
846 let second = tokio::spawn({
849 let notifier = notifier.clone();
850 async move {
851 notifier
852 .send(Arc::new(Event::new_test_event("bucket", "second", EventName::ObjectCreatedPut)))
853 .await;
854 }
855 });
856 tokio::time::timeout(std::time::Duration::from_secs(1), async {
857 while target.enabled_call_count() != 2 {
858 tokio::task::yield_now().await;
859 }
860 })
861 .await
862 .expect("both direct sends should select the old generation");
863
864 let activation = facade.activate_targets_with_replay(Vec::new()).await;
865 let mut replace = Box::pin(facade.replace_targets(activation));
866 tokio::select! {
867 biased;
868 result = &mut replace => panic!("replacement closed a generation with selected direct sends: {result:?}"),
869 _ = std::future::ready(()) => {}
870 }
871 assert_eq!(target.close_call_count(), 0);
872
873 first_release.notify_one();
874 first.await.expect("first direct dispatch should finish");
875 second.await.expect("permit-waiting direct dispatch should be cancelled");
876 replace.await.expect("replacement should close after direct leases drain");
877 assert_eq!(target.save_call_count(), 1);
878 assert_eq!(target.close_call_count(), 1);
879 assert_eq!(metrics.processing_count(), 0);
880 assert_eq!(metrics.processed_count(), 1);
881 assert_eq!(metrics.skipped_count(), 1);
882 }
883
884 #[tokio::test]
885 async fn caller_abort_does_not_release_deferred_generation_lease() {
886 let metrics = Arc::new(NotificationMetrics::new());
887 let rule_engine = NotifyRuleEngine::new();
888 let notifier = Arc::new(EventNotifier::new(metrics.clone(), rule_engine.clone()));
889 let save_entered = Arc::new(Notify::new());
890 let save_release = Arc::new(Notify::new());
891 let queue_dir = tempfile::tempdir().expect("queue tempdir should be created");
892 let target = MockTarget::new("deferred", "webhook")
893 .with_first_save_gate(save_entered.clone(), save_release.clone())
894 .with_store(Arc::new(QueueStore::new(queue_dir.path(), 16, ".event")));
895
896 let mut rules_map = RulesMap::new();
897 rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), target.target_id());
898 rule_engine.set_bucket_rules("bucket", rules_map).await;
899 notifier
900 .target_list()
901 .write()
902 .await
903 .add(Arc::new(target))
904 .expect("target should install");
905 let facade = crate::runtime_facade::NotifyRuntimeFacade::new_with_dispatch_gate(
906 notifier.target_list(),
907 Arc::new(RwLock::new(ReplayWorkerManager::new())),
908 notifier.dispatch_gate(),
909 Arc::new(Semaphore::new(1)),
910 metrics,
911 );
912
913 let dispatch = tokio::spawn({
914 let notifier = notifier.clone();
915 async move {
916 notifier
917 .send(Arc::new(Event::new_test_event("bucket", "object", EventName::ObjectCreatedPut)))
918 .await;
919 }
920 });
921 save_entered.notified().await;
922 dispatch.abort();
923 let _ = dispatch.await;
924
925 let mut pause = Box::pin(facade.pause_dispatch());
926 tokio::select! {
927 biased;
928 _ = &mut pause => panic!("caller abort released the deferred generation lease"),
929 _ = std::future::ready(()) => {}
930 }
931 save_release.notify_one();
932 let pause_guard = pause.await;
933 drop(pause_guard);
934 }
935
936 #[tokio::test]
937 async fn deferred_enqueue_concurrency_is_bounded() {
938 const LIMIT: usize = 2;
939 const TARGETS: usize = 3;
940
941 let metrics = Arc::new(NotificationMetrics::new());
942 let rule_engine = NotifyRuleEngine::new();
943 let notifier = Arc::new(EventNotifier {
944 enqueue_limiter: Arc::new(Semaphore::new(LIMIT)),
945 ..EventNotifier::new(metrics, rule_engine.clone())
946 });
947 let entered = Arc::new(Notify::new());
948 let release = Arc::new(Notify::new());
949 let queue_dir = tempfile::tempdir().expect("queue tempdir should be created");
950 let mut targets = Vec::new();
951 let mut rules_map = RulesMap::new();
952 for index in 0..TARGETS {
953 let target = MockTarget::new(&format!("deferred-{index}"), "webhook")
954 .with_first_save_gate(entered.clone(), release.clone())
955 .with_store(Arc::new(QueueStore::new(queue_dir.path().join(index.to_string()), 16, ".event")));
956 rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), target.target_id());
957 notifier
958 .target_list()
959 .write()
960 .await
961 .add(Arc::new(target.clone()))
962 .expect("target should install");
963 targets.push(target);
964 }
965 rule_engine.set_bucket_rules("bucket", rules_map).await;
966
967 let dispatch = tokio::spawn({
968 let notifier = notifier.clone();
969 async move {
970 notifier
971 .send(Arc::new(Event::new_test_event("bucket", "object", EventName::ObjectCreatedPut)))
972 .await;
973 }
974 });
975 let total_calls = || targets.iter().map(|target| target.save_call_count()).sum::<usize>();
976 tokio::time::timeout(std::time::Duration::from_secs(1), async {
977 while total_calls() != LIMIT {
978 tokio::task::yield_now().await;
979 }
980 })
981 .await
982 .expect("the configured number of enqueues should enter");
983 for _ in 0..10 {
984 tokio::task::yield_now().await;
985 }
986 assert_eq!(total_calls(), LIMIT, "enqueue concurrency exceeded its semaphore capacity");
987
988 release.notify_waiters();
989 tokio::time::timeout(std::time::Duration::from_secs(1), async {
990 while total_calls() != TARGETS {
991 tokio::task::yield_now().await;
992 }
993 })
994 .await
995 .expect("the waiting enqueue should enter after a permit is released");
996 release.notify_waiters();
997 dispatch.await.expect("all bounded enqueues should finish");
998 }
999
1000 #[tokio::test]
1001 async fn deferred_enqueue_does_not_wait_for_a_blocked_direct_send_permit() {
1002 let metrics = Arc::new(NotificationMetrics::new());
1003 let rule_engine = NotifyRuleEngine::new();
1004 let notifier = Arc::new(EventNotifier {
1005 send_limiter: Arc::new(Semaphore::new(1)),
1006 ..EventNotifier::new(metrics.clone(), rule_engine.clone())
1007 });
1008 let direct_entered = Arc::new(Notify::new());
1009 let direct_release = Arc::new(Notify::new());
1010 let direct = MockTarget::new("direct", "webhook").with_first_save_gate(direct_entered.clone(), direct_release.clone());
1011 let deferred_entered = Arc::new(Notify::new());
1012 let deferred_release = Arc::new(Notify::new());
1013 let queue_dir = tempfile::tempdir().expect("queue tempdir should be created");
1014 let deferred = MockTarget::new("deferred", "webhook")
1015 .with_first_save_gate(deferred_entered.clone(), deferred_release.clone())
1016 .with_store(Arc::new(QueueStore::new(queue_dir.path(), 16, ".event")));
1017
1018 let mut direct_rules = RulesMap::new();
1019 direct_rules.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), direct.target_id());
1020 rule_engine.set_bucket_rules("direct-bucket", direct_rules).await;
1021 let mut deferred_rules = RulesMap::new();
1022 deferred_rules.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), deferred.target_id());
1023 rule_engine.set_bucket_rules("deferred-bucket", deferred_rules).await;
1024 {
1025 let target_list = notifier.target_list();
1026 let mut target_list = target_list.write().await;
1027 target_list.add(Arc::new(direct)).expect("direct target should install");
1028 target_list.add(Arc::new(deferred)).expect("deferred target should install");
1029 }
1030
1031 let facade = crate::runtime_facade::NotifyRuntimeFacade::new_with_dispatch_gate(
1032 notifier.target_list(),
1033 Arc::new(RwLock::new(ReplayWorkerManager::new())),
1034 notifier.dispatch_gate(),
1035 Arc::new(Semaphore::new(1)),
1036 metrics,
1037 );
1038 let direct_dispatch = tokio::spawn({
1039 let notifier = notifier.clone();
1040 async move {
1041 notifier
1042 .send(Arc::new(Event::new_test_event("direct-bucket", "object", EventName::ObjectCreatedPut)))
1043 .await;
1044 }
1045 });
1046 direct_entered.notified().await;
1047 let deferred_dispatch = tokio::spawn({
1048 let notifier = notifier.clone();
1049 async move {
1050 notifier
1051 .send(Arc::new(Event::new_test_event("deferred-bucket", "object", EventName::ObjectCreatedPut)))
1052 .await;
1053 }
1054 });
1055 tokio::time::timeout(std::time::Duration::from_secs(1), deferred_entered.notified())
1056 .await
1057 .expect("queue persistence must not wait behind a direct network send permit");
1058
1059 let mut pause = Box::pin(facade.pause_dispatch());
1060 tokio::select! {
1061 biased;
1062 _ = &mut pause => panic!("lifecycle pause crossed the blocked deferred enqueue"),
1063 _ = std::future::ready(()) => {}
1064 }
1065 deferred_release.notify_one();
1066 deferred_dispatch
1067 .await
1068 .expect("deferred dispatch should finish after release");
1069 let pause_guard = tokio::select! {
1070 biased;
1071 guard = &mut pause => guard,
1072 _ = std::future::ready(()) => panic!("direct network delivery kept the lifecycle gate locked"),
1073 };
1074 drop(pause_guard);
1075
1076 direct_release.notify_one();
1077 direct_dispatch.await.expect("direct dispatch should finish after release");
1078 }
1079
1080 #[tokio::test]
1081 async fn test_send_event_skips_disabled_target() {
1082 let rule_engine = NotifyRuleEngine::new();
1083 let notifier = EventNotifier::new(Arc::new(NotificationMetrics::new()), rule_engine.clone());
1084
1085 let enabled_target = MockTarget::new("enabled-target", "webhook");
1086 let disabled_target = MockTarget::new("disabled-target", "webhook").disabled();
1087
1088 let mut rules_map = RulesMap::new();
1089 rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), enabled_target.target_id());
1090 rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), disabled_target.target_id());
1091
1092 rule_engine.set_bucket_rules("bucket", rules_map).await;
1093 notifier
1094 .target_list()
1095 .write()
1096 .await
1097 .add(Arc::new(enabled_target.clone()))
1098 .unwrap();
1099 notifier
1100 .target_list()
1101 .write()
1102 .await
1103 .add(Arc::new(disabled_target.clone()))
1104 .unwrap();
1105
1106 let event = Arc::new(Event::new_test_event("bucket", "object", EventName::ObjectCreatedPut));
1107 notifier.send(event).await;
1108
1109 assert_eq!(enabled_target.save_call_count(), 1);
1110 assert_eq!(disabled_target.save_call_count(), 0);
1111 }
1112
1113 #[tokio::test]
1114 async fn send_event_respects_prefix_suffix_filters() {
1115 let rule_engine = NotifyRuleEngine::new();
1116 let notifier = EventNotifier::new(Arc::new(NotificationMetrics::new()), rule_engine.clone());
1117 let target = MockTarget::new("filtered-target", "webhook");
1118 let mut rules_map = RulesMap::new();
1119 rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "uploads/*.csv".to_string(), target.target_id());
1120
1121 rule_engine.set_bucket_rules("bucket", rules_map).await;
1122 notifier.target_list().write().await.add(Arc::new(target.clone())).unwrap();
1123
1124 notifier
1125 .send(Arc::new(Event::new_test_event("bucket", "report.csv", EventName::ObjectCreatedPut)))
1126 .await;
1127 notifier
1128 .send(Arc::new(Event::new_test_event(
1129 "bucket",
1130 "uploads/report.txt",
1131 EventName::ObjectCreatedPut,
1132 )))
1133 .await;
1134
1135 assert_eq!(target.save_call_count(), 0);
1136
1137 notifier
1138 .send(Arc::new(Event::new_test_event(
1139 "bucket",
1140 "uploads/report.csv",
1141 EventName::ObjectCreatedPut,
1142 )))
1143 .await;
1144
1145 assert_eq!(target.save_call_count(), 1);
1146 }
1147
1148 fn deferred_test_target(id: &str, name: &str) -> MockTarget {
1152 MockTarget::new(id, name).with_store(Arc::new(QueueStore::new(
1153 std::env::temp_dir().join("rustfs-notify-979-noop-store"),
1154 0,
1155 "",
1156 )))
1157 }
1158
1159 #[tokio::test]
1166 async fn deferred_target_dispatch_does_not_underflow_processing_counter() {
1167 let metrics = Arc::new(NotificationMetrics::new());
1168 let rule_engine = NotifyRuleEngine::new();
1169 let notifier = EventNotifier::new(metrics.clone(), rule_engine.clone());
1170
1171 let target = deferred_test_target("deferred-target", "webhook");
1172 let mut rules_map = RulesMap::new();
1173 rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), target.target_id());
1174 rule_engine.set_bucket_rules("bucket", rules_map).await;
1175 notifier.target_list().write().await.add(Arc::new(target.clone())).unwrap();
1176
1177 notifier
1179 .send(Arc::new(Event::new_test_event("bucket", "object", EventName::ObjectCreatedPut)))
1180 .await;
1181
1182 assert_eq!(target.save_call_count(), 1);
1183 assert_eq!(
1184 metrics.processing_count(),
1185 1,
1186 "a queued deferred event must remain counted as in-flight after enqueue"
1187 );
1188 assert_eq!(metrics.processed_count(), 0);
1189
1190 metrics.increment_processed();
1193
1194 assert_eq!(
1195 metrics.processing_count(),
1196 0,
1197 "processing counter must return to zero, not underflow to usize::MAX"
1198 );
1199 assert_ne!(metrics.processing_count(), usize::MAX);
1200 assert_eq!(metrics.processed_count(), 1);
1201 }
1202
1203 #[test]
1208 fn zero_send_concurrency_falls_back_to_default() {
1209 assert_eq!(
1210 coerce_send_concurrency(0),
1211 DEFAULT_NOTIFY_SEND_CONCURRENCY,
1212 "zero concurrency must fall back to the default, never a zero-permit semaphore"
1213 );
1214 assert!(coerce_send_concurrency(0) > 0, "resolved concurrency must be strictly positive");
1215 assert_eq!(coerce_send_concurrency(1), 1);
1216 assert_eq!(coerce_send_concurrency(128), 128);
1217 }
1218
1219 #[tokio::test]
1223 async fn init_bucket_targets_shared_closes_replaced_targets() {
1224 let rule_engine = NotifyRuleEngine::new();
1225 let notifier = EventNotifier::new(Arc::new(NotificationMetrics::new()), rule_engine);
1226
1227 let old_target = MockTarget::new("old", "webhook");
1228 notifier
1229 .init_bucket_targets_shared(vec![Arc::new(old_target.clone()) as SharedTarget<Event>])
1230 .await
1231 .expect("initial install should succeed");
1232 assert_eq!(old_target.close_call_count(), 0, "target must not close on first install");
1233
1234 let new_target = MockTarget::new("new", "webhook");
1235 notifier
1236 .init_bucket_targets_shared(vec![Arc::new(new_target.clone()) as SharedTarget<Event>])
1237 .await
1238 .expect("replacement install should succeed");
1239
1240 assert_eq!(old_target.close_call_count(), 1, "the replaced target must be closed exactly once");
1241 assert_eq!(new_target.close_call_count(), 0, "the freshly installed target must stay open");
1242 }
1243}