1use crate::{
16 Event, NotificationError,
17 error::transition_join_error,
18 integration::NotificationMetrics,
19 notifier::{DirectDispatchTracker, SharedNotifyTargetList, shared_dispatch_gate},
20};
21use rustfs_targets::{
22 BuiltinPluginRuntimeAdapter, OpenedActivation, PluginRuntimeAdapter, PreparedActivation, ReplayEvent, ReplayWorkerManager,
23 RuntimeActivation, Target, TargetRuntimeManager,
24};
25use std::sync::{
26 Arc,
27 atomic::{AtomicBool, Ordering},
28};
29use std::time::Duration;
30use tokio::sync::{OwnedRwLockWriteGuard, RwLock, Semaphore};
31use tokio_util::sync::CancellationToken;
32use tracing::{debug, info, warn};
33
34const LOG_COMPONENT_NOTIFY: &str = "notify";
35const LOG_SUBSYSTEM_RUNTIME: &str = "runtime";
36const EVENT_NOTIFY_RUNTIME_LIFECYCLE: &str = "notify_runtime_lifecycle";
37const EVENT_NOTIFY_RUNTIME_SHUTDOWN_FAILED: &str = "notify_runtime_shutdown_failed";
38const EVENT_NOTIFY_REPLAY_RETRY_EXHAUSTED: &str = "notify_replay_retry_exhausted";
39const TARGET_CLOSE_TIMEOUT: Duration = Duration::from_secs(10);
40
41pub(crate) struct DetachedNotifyRuntime {
42 direct_dispatches: Arc<DirectDispatchTracker>,
43 runtime: TargetRuntimeManager<Event>,
44 replay_workers: ReplayWorkerManager,
45}
46
47#[derive(Clone)]
52pub struct NotifyRuntimeFacade {
53 dispatch_gate: Arc<RwLock<()>>,
54 legacy_terminated: Arc<AtomicBool>,
55 target_list: SharedNotifyTargetList,
56 replay_workers: Arc<RwLock<ReplayWorkerManager>>,
57 runtime_adapter: Arc<BuiltinPluginRuntimeAdapter<Event>>,
58}
59
60impl NotifyRuntimeFacade {
61 pub fn new(
62 target_list: SharedNotifyTargetList,
63 replay_workers: Arc<RwLock<ReplayWorkerManager>>,
64 concurrency_limiter: Arc<Semaphore>,
65 metrics: Arc<NotificationMetrics>,
66 ) -> Self {
67 let dispatch_gate = shared_dispatch_gate(&target_list, None);
68 Self::new_with_dispatch_gate(target_list, replay_workers, dispatch_gate, concurrency_limiter, metrics)
69 }
70
71 pub(crate) fn new_with_dispatch_gate(
72 target_list: SharedNotifyTargetList,
73 replay_workers: Arc<RwLock<ReplayWorkerManager>>,
74 dispatch_gate: Arc<RwLock<()>>,
75 concurrency_limiter: Arc<Semaphore>,
76 metrics: Arc<NotificationMetrics>,
77 ) -> Self {
78 let dispatch_gate = shared_dispatch_gate(&target_list, Some(dispatch_gate));
79 let replay_metrics = metrics;
80 let runtime_adapter = BuiltinPluginRuntimeAdapter::new(
81 Arc::new(move |event: ReplayEvent<Event>| {
82 let metrics = replay_metrics.clone();
83 Box::pin(async move {
84 match event {
85 ReplayEvent::Delivered { .. } => metrics.increment_processed(),
86 ReplayEvent::RetryableError { .. } => {}
87 ReplayEvent::RetryExhausted { detail, key, target } => {
88 warn!(
89 event = EVENT_NOTIFY_REPLAY_RETRY_EXHAUSTED,
90 component = LOG_COMPONENT_NOTIFY,
91 subsystem = LOG_SUBSYSTEM_RUNTIME,
92 target_id = %target.id(),
93 replay_key = %key,
94 error = %detail,
95 "notify replay retry budget exhausted, entry stays queued and retries"
96 );
97 }
98 ReplayEvent::Dropped { target, .. } | ReplayEvent::PermanentFailure { target, .. } => {
99 target.record_final_failure();
100 metrics.increment_failed();
101 }
102 ReplayEvent::UnreadableEntry { .. } => {}
103 }
104 })
105 }),
106 Arc::new(|target_id, has_replay| {
107 if has_replay {
108 info!(
109 event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
110 component = LOG_COMPONENT_NOTIFY,
111 subsystem = LOG_SUBSYSTEM_RUNTIME,
112 target_id = %target_id,
113 state = "replay_started",
114 "notify runtime lifecycle"
115 );
116 } else {
117 debug!(
118 event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
119 component = LOG_COMPONENT_NOTIFY,
120 subsystem = LOG_SUBSYSTEM_RUNTIME,
121 target_id = %target_id,
122 state = "replay_skipped",
123 reason = "no_store_configured",
124 "notify runtime lifecycle"
125 );
126 }
127 }),
128 Some(concurrency_limiter),
129 Duration::from_secs(5),
130 Duration::from_millis(500),
131 "Stop event stream processing for target",
132 );
133
134 Self {
135 dispatch_gate,
136 legacy_terminated: Arc::new(AtomicBool::new(false)),
137 target_list,
138 replay_workers,
139 runtime_adapter: Arc::new(runtime_adapter),
140 }
141 }
142
143 pub(crate) async fn pause_dispatch(&self) -> OwnedRwLockWriteGuard<()> {
144 self.dispatch_gate.clone().write_owned().await
145 }
146
147 pub async fn activate_targets_with_replay(
148 &self,
149 targets: Vec<Box<dyn Target<Event> + Send + Sync>>,
150 ) -> RuntimeActivation<Event> {
151 self.runtime_adapter.prepare_dormant_compat_activation(targets).await
154 }
155
156 pub(crate) async fn prepare_targets(
157 &self,
158 targets: Vec<Box<dyn Target<Event> + Send + Sync>>,
159 cancellation: &CancellationToken,
160 ) -> PreparedActivation<Event> {
161 self.runtime_adapter.prepare_targets_cancellable(targets, cancellation).await
162 }
163
164 pub(crate) async fn open_prepared_stores(
165 &self,
166 prepared: PreparedActivation<Event>,
167 ) -> Result<(OpenedActivation<Event>, PreparedActivation<Event>), NotificationError> {
168 let runtime_adapter = self.runtime_adapter.clone();
169 tokio::task::spawn_blocking(move || runtime_adapter.open_prepared_stores(prepared))
170 .await
171 .map_err(transition_join_error)
172 }
173
174 pub(crate) async fn commit_prepared<Committed>(
175 &self,
176 opened: OpenedActivation<Event>,
177 on_committed: Committed,
178 ) -> (DetachedNotifyRuntime, PreparedActivation<Event>)
179 where
180 Committed: FnOnce(bool),
181 {
182 let mut replay_workers = self.replay_workers.write().await;
188 let mut target_list = self.target_list.write().await;
189
190 let (activation, rejected) = self.runtime_adapter.try_activate_prepared(opened);
191 let fully_activated = rejected.failure_summary().is_none();
192 let (runtime, replay_workers, direct_dispatches) =
193 Self::swap_activation(&mut target_list, &mut replay_workers, activation);
194 on_committed(fully_activated);
195 (
196 DetachedNotifyRuntime {
197 direct_dispatches,
198 runtime,
199 replay_workers,
200 },
201 rejected,
202 )
203 }
204
205 pub(crate) async fn commit_disabled<Committed>(&self, on_committed: Committed) -> DetachedNotifyRuntime
206 where
207 Committed: FnOnce(),
208 {
209 let mut replay_workers = self.replay_workers.write().await;
214 let mut target_list = self.target_list.write().await;
215
216 let (runtime, direct_dispatches) = target_list.replace_runtime(TargetRuntimeManager::new());
217 let detached = DetachedNotifyRuntime {
218 direct_dispatches,
219 runtime,
220 replay_workers: std::mem::take(&mut *replay_workers),
221 };
222 on_committed();
223 detached
224 }
225
226 pub async fn replace_targets(&self, mut activation: RuntimeActivation<Event>) -> Result<(), NotificationError> {
227 self.runtime_adapter.stop_replay_workers(&mut activation.replay_workers).await;
232 let dispatch_guard = self.pause_dispatch().await;
233 if self.legacy_terminated.load(Ordering::Acquire) {
234 drop(dispatch_guard);
235 self.runtime_adapter
236 .close_compat_activation(activation)
237 .await
238 .map_err(NotificationError::Target)?;
239 return Err(NotificationError::Initialization("Notification runtime has terminated".to_string()));
240 }
241 self.stop_active_replay_workers().await;
242 let (activation, open_rejected, activation_rejected) = self.runtime_adapter.start_dormant_compat_activation(activation);
243 let activation_failures = [open_rejected.failure_summary(), activation_rejected.failure_summary()]
244 .into_iter()
245 .flatten()
246 .collect::<Vec<_>>();
247 let (old_runtime, old_replay_workers, old_direct_dispatches) = {
248 let mut replay_workers = self.replay_workers.write().await;
250 let mut target_list = self.target_list.write().await;
251 Self::swap_activation(&mut target_list, &mut replay_workers, activation)
252 };
253 drop(dispatch_guard);
254 let close_old = self.close_detached_targets(DetachedNotifyRuntime {
255 direct_dispatches: old_direct_dispatches,
256 runtime: old_runtime,
257 replay_workers: old_replay_workers,
258 });
259 let close_open_rejected = self.close_prepared(open_rejected);
260 let close_activation_rejected = self.close_prepared(activation_rejected);
261 let (close_old, close_open_rejected, close_activation_rejected) =
262 tokio::join!(close_old, close_open_rejected, close_activation_rejected);
263 close_old?;
264 close_open_rejected?;
265 close_activation_rejected?;
266 if !activation_failures.is_empty() {
267 return Err(NotificationError::Initialization(format!(
268 "one or more notification targets failed to activate: {}",
269 activation_failures.join("; ")
270 )));
271 }
272 Ok(())
273 }
274
275 pub async fn stop_replay_workers(&self) {
276 let _dispatch_guard = self.pause_dispatch().await;
277 self.stop_active_replay_workers().await;
278 }
279
280 pub(crate) async fn stop_active_replay_workers(&self) {
281 let mut detached = {
282 let mut replay_workers = self.replay_workers.write().await;
283 std::mem::take(&mut *replay_workers)
284 };
285 self.runtime_adapter.stop_replay_workers(&mut detached).await;
286 }
287
288 pub async fn shutdown(&self) {
289 let _ = self.shutdown_checked().await;
290 }
291
292 pub async fn shutdown_checked(&self) -> Result<(), NotificationError> {
293 let _dispatch_guard = self.pause_dispatch().await;
294 self.legacy_terminated.store(true, Ordering::Release);
295 info!(
296 event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
297 component = LOG_COMPONENT_NOTIFY,
298 subsystem = LOG_SUBSYSTEM_RUNTIME,
299 state = "stopping",
300 "notify runtime lifecycle"
301 );
302
303 let (detached_runtime, detached_replay_workers, detached_direct_dispatches) = {
304 let mut replay_workers = self.replay_workers.write().await;
306 let mut target_list = self.target_list.write().await;
307 let (runtime, direct_dispatches) = target_list.replace_runtime(TargetRuntimeManager::new());
308 (runtime, std::mem::take(&mut *replay_workers), direct_dispatches)
309 };
310 let active_targets = detached_replay_workers.len();
311 info!(
312 event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
313 component = LOG_COMPONENT_NOTIFY,
314 subsystem = LOG_SUBSYSTEM_RUNTIME,
315 state = "replay_stopping",
316 active_targets,
317 "notify runtime lifecycle"
318 );
319
320 let shutdown_result = self
321 .close_detached(DetachedNotifyRuntime {
322 direct_dispatches: detached_direct_dispatches,
323 runtime: detached_runtime,
324 replay_workers: detached_replay_workers,
325 })
326 .await;
327 if let Err(err) = &shutdown_result {
328 tracing::error!(
329 event = EVENT_NOTIFY_RUNTIME_SHUTDOWN_FAILED,
330 component = LOG_COMPONENT_NOTIFY,
331 subsystem = LOG_SUBSYSTEM_RUNTIME,
332 error = %err,
333 "Failed to shutdown notify runtime cleanly"
334 );
335 }
336
337 info!(
338 event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
339 component = LOG_COMPONENT_NOTIFY,
340 subsystem = LOG_SUBSYSTEM_RUNTIME,
341 state = "stopped",
342 "notify runtime lifecycle"
343 );
344 shutdown_result
345 }
346
347 fn swap_activation(
348 target_list: &mut crate::notifier::TargetList,
349 replay_workers: &mut ReplayWorkerManager,
350 activation: RuntimeActivation<Event>,
351 ) -> (TargetRuntimeManager<Event>, ReplayWorkerManager, Arc<DirectDispatchTracker>) {
352 let mut replacement = TargetRuntimeManager::new();
353 for target in activation.targets {
354 replacement.add_arc(target);
355 }
356 let (runtime, direct_dispatches) = target_list.replace_runtime(replacement);
357 (runtime, std::mem::replace(replay_workers, activation.replay_workers), direct_dispatches)
358 }
359
360 pub(crate) async fn stop_detached_replay(&self, detached: &mut DetachedNotifyRuntime) {
361 self.runtime_adapter.stop_replay_workers(&mut detached.replay_workers).await;
365 }
366
367 pub(crate) async fn close_detached_targets(&self, mut detached: DetachedNotifyRuntime) -> Result<(), NotificationError> {
368 detached.direct_dispatches.wait_idle().await;
372 match tokio::time::timeout(TARGET_CLOSE_TIMEOUT, detached.runtime.clear_and_close()).await {
373 Ok(close_errors) if close_errors.is_empty() => Ok(()),
374 Ok(close_errors) => {
375 let targets = close_errors
376 .into_iter()
377 .map(|(target_id, _)| target_id)
378 .collect::<Vec<_>>()
379 .join("; ");
380 Err(NotificationError::Target(rustfs_targets::TargetError::Storage(format!(
381 "Failed to close {targets}"
382 ))))
383 }
384 Err(_) => Err(NotificationError::Target(rustfs_targets::TargetError::Timeout(
385 "Timed out closing replaced notification targets".to_string(),
386 ))),
387 }
388 }
389
390 async fn close_detached(&self, mut detached: DetachedNotifyRuntime) -> Result<(), NotificationError> {
391 self.stop_detached_replay(&mut detached).await;
392 self.close_detached_targets(detached).await
393 }
394
395 pub(crate) async fn close_prepared(&self, prepared: PreparedActivation<Event>) -> Result<(), NotificationError> {
396 match tokio::time::timeout(TARGET_CLOSE_TIMEOUT, self.runtime_adapter.close_prepared(prepared)).await {
397 Ok(result) => result.map_err(NotificationError::Target),
398 Err(_) => Err(NotificationError::Target(rustfs_targets::TargetError::Timeout(
399 "Timed out closing superseded notification targets".to_string(),
400 ))),
401 }
402 }
403}
404
405#[cfg(test)]
406mod tests {
407 use super::NotifyRuntimeFacade;
408 use crate::{
409 Event, integration::NotificationMetrics, notifier::EventNotifier, rule_engine::NotifyRuleEngine,
410 runtime_view::NotifyRuntimeView,
411 };
412 use rustfs_targets::arn::TargetID;
413 use rustfs_targets::store::QueueStore;
414 use rustfs_targets::target::QueuedPayload;
415 use rustfs_targets::testkit::MockTarget;
416 use rustfs_targets::{ReplayWorkerManager, SharedTarget, TargetError};
417 use std::sync::Arc;
418 use tokio::sync::{Notify, RwLock, Semaphore};
419
420 fn build_facade() -> (NotifyRuntimeFacade, Arc<EventNotifier>, Arc<RwLock<ReplayWorkerManager>>) {
421 let metrics = Arc::new(NotificationMetrics::new());
422 let notifier = Arc::new(EventNotifier::new(metrics.clone(), NotifyRuleEngine::new()));
423 let target_list = notifier.target_list();
424 let replay_workers = Arc::new(RwLock::new(ReplayWorkerManager::new()));
425 let facade = NotifyRuntimeFacade::new_with_dispatch_gate(
426 target_list,
427 replay_workers.clone(),
428 notifier.dispatch_gate(),
429 Arc::new(Semaphore::new(4)),
430 metrics,
431 );
432 (facade, notifier, replay_workers)
433 }
434
435 #[tokio::test]
436 async fn stopping_replay_workers_is_a_no_op_when_there_are_none() {
437 let (facade, _, _) = build_facade();
438
439 facade.stop_replay_workers().await;
440
441 assert!(facade.replay_workers.read().await.is_empty());
445
446 facade.stop_replay_workers().await;
448 assert!(facade.replay_workers.read().await.is_empty());
449 }
450
451 #[tokio::test]
452 async fn runtime_facade_activates_empty_target_list() {
453 let (facade, _, _) = build_facade();
454 let activation = facade.activate_targets_with_replay(Vec::new()).await;
455
456 assert!(activation.targets.is_empty());
457 assert_eq!(activation.replay_workers.len(), 0);
458 }
459
460 #[tokio::test]
461 async fn compatibility_activation_stays_dormant_until_ordered_replace() {
462 let (facade, _, replay_workers) = build_facade();
463 let queue_root = tempfile::tempdir().expect("queue root");
464 let store = QueueStore::<QueuedPayload>::new_with_compression(queue_root.path(), 16, ".event", false);
465 let target = MockTarget::new("primary", "webhook").with_store(Arc::new(store));
466
467 let activation = facade.activate_targets_with_replay(vec![Box::new(target)]).await;
468 assert_eq!(activation.targets.len(), 1);
469 assert_eq!(activation.replay_workers.len(), 0, "compatibility prepare must not start replay early");
470 assert_eq!(replay_workers.read().await.len(), 0);
471
472 facade
473 .replace_targets(activation)
474 .await
475 .expect("ordered compatibility replace should succeed");
476 assert_eq!(replay_workers.read().await.len(), 1);
477 facade.shutdown_checked().await.expect("test runtime should shut down");
478 }
479
480 #[tokio::test]
481 async fn runtime_facade_replace_targets_commits_runtime_state() {
482 let (facade, notifier, replay_workers) = build_facade();
483 let target = MockTarget::new("primary", "webhook");
484 let activation = rustfs_targets::RuntimeActivation {
485 replay_workers: ReplayWorkerManager::new(),
486 targets: vec![Arc::new(target) as SharedTarget<Event>],
487 };
488
489 facade
490 .replace_targets(activation)
491 .await
492 .expect("replace_targets should succeed");
493
494 let runtime_view = NotifyRuntimeView::new(notifier.target_list(), replay_workers.clone());
495 let active_targets = runtime_view.get_active_targets().await;
496 assert_eq!(active_targets, vec![TargetID::new("primary".to_string(), "webhook".to_string())]);
497 assert_eq!(replay_workers.read().await.len(), 0);
498 }
499
500 #[tokio::test]
501 async fn runtime_queries_do_not_wait_for_target_close() {
502 let (facade, notifier, replay_workers) = build_facade();
503 let target = MockTarget::new("primary", "webhook");
504 target.set_block_on_close(true);
505 let observer = target.clone();
506 facade
507 .replace_targets(rustfs_targets::RuntimeActivation {
508 replay_workers: ReplayWorkerManager::new(),
509 targets: vec![Arc::new(target) as SharedTarget<Event>],
510 })
511 .await
512 .expect("target install should succeed");
513
514 let shutdown = tokio::spawn({
515 let facade = facade.clone();
516 async move { facade.shutdown_checked().await }
517 });
518 observer.close_started().notified().await;
519
520 let target_list = notifier.target_list();
521 assert!(target_list.try_read().is_ok(), "target list lock must not be held during close");
522 assert!(replay_workers.try_read().is_ok(), "replay manager lock must not be held during close");
523 observer.close_gate().add_permits(1);
524 shutdown
525 .await
526 .expect("shutdown task should not panic")
527 .expect("shutdown should succeed");
528
529 let snapshot = NotifyRuntimeView::new(target_list, replay_workers)
530 .runtime_status_snapshot()
531 .await;
532 assert_eq!(snapshot.target_count, 0);
533 assert_eq!(snapshot.replay_worker_count, 0);
534 }
535
536 #[tokio::test]
537 async fn runtime_locks_are_released_while_replay_worker_joins() {
538 let (facade, notifier, replay_workers) = build_facade();
539 let cancel_received = Arc::new(Notify::new());
540 let release = Arc::new(Notify::new());
541 let (cancel_tx, mut cancel_rx) = tokio::sync::mpsc::channel(1);
542 let join = tokio::spawn({
543 let cancel_received = cancel_received.clone();
544 let release = release.clone();
545 async move {
546 let _ = cancel_rx.recv().await;
547 cancel_received.notify_one();
548 release.notified().await;
549 }
550 });
551 replay_workers
552 .write()
553 .await
554 .insert_with_handle("blocked-worker".to_string(), cancel_tx, join);
555
556 let stop = tokio::spawn({
557 let facade = facade.clone();
558 async move { facade.stop_replay_workers().await }
559 });
560 cancel_received.notified().await;
561
562 assert!(replay_workers.try_read().is_ok(), "replay manager lock must not be held during join");
563 let target_list = notifier.target_list();
564 assert!(target_list.try_read().is_ok(), "target list lock must not be held during replay join");
565
566 release.notify_one();
567 stop.await.expect("stop task should finish");
568 }
569
570 #[tokio::test]
571 async fn shutdown_returns_close_error_after_detaching_runtime() {
572 let (facade, notifier, replay_workers) = build_facade();
573 let target = MockTarget::new("primary", "webhook").with_close_failures(usize::MAX);
574 facade
575 .replace_targets(rustfs_targets::RuntimeActivation {
576 replay_workers: ReplayWorkerManager::new(),
577 targets: vec![Arc::new(target) as SharedTarget<Event>],
578 })
579 .await
580 .expect("target install should succeed");
581
582 let err = facade.shutdown_checked().await.expect_err("close failure should propagate");
583 assert!(matches!(err, crate::NotificationError::Target(TargetError::Storage(_))));
584
585 let snapshot = NotifyRuntimeView::new(notifier.target_list(), replay_workers)
586 .runtime_status_snapshot()
587 .await;
588 assert_eq!(snapshot.target_count, 0);
589 assert_eq!(snapshot.replay_worker_count, 0);
590 }
591
592 #[tokio::test(start_paused = true)]
593 async fn shutdown_bounds_a_target_that_never_closes() {
594 let (facade, notifier, replay_workers) = build_facade();
595 let target = MockTarget::new("primary", "webhook");
597 target.set_block_on_close(true);
598 let observer = target.clone();
599 facade
600 .replace_targets(rustfs_targets::RuntimeActivation {
601 replay_workers: ReplayWorkerManager::new(),
602 targets: vec![Arc::new(target) as SharedTarget<Event>],
603 })
604 .await
605 .expect("target install should succeed");
606
607 let shutdown = tokio::spawn({
608 let facade = facade.clone();
609 async move { facade.shutdown_checked().await }
610 });
611 observer.close_started().notified().await;
612 tokio::time::advance(super::TARGET_CLOSE_TIMEOUT).await;
613
614 let err = shutdown
615 .await
616 .expect("shutdown task should not panic")
617 .expect_err("blocked close must time out");
618 assert!(matches!(err, crate::NotificationError::Target(TargetError::Timeout(_))));
619 let snapshot = NotifyRuntimeView::new(notifier.target_list(), replay_workers)
620 .runtime_status_snapshot()
621 .await;
622 assert_eq!(snapshot.target_count, 0);
623 assert_eq!(snapshot.replay_worker_count, 0);
624 }
625}