Skip to main content

rustfs_notify/
runtime_facade.rs

1// Copyright 2024 RustFS Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use crate::{
16    Event, NotificationError,
17    error::transition_join_error,
18    integration::NotificationMetrics,
19    notifier::{DirectDispatchTracker, SharedNotifyTargetList, shared_dispatch_gate},
20};
21use rustfs_targets::{
22    BuiltinPluginRuntimeAdapter, OpenedActivation, PluginRuntimeAdapter, PreparedActivation, ReplayEvent, ReplayWorkerManager,
23    RuntimeActivation, Target, TargetRuntimeManager,
24};
25use std::sync::{
26    Arc,
27    atomic::{AtomicBool, Ordering},
28};
29use std::time::Duration;
30use tokio::sync::{OwnedRwLockWriteGuard, RwLock, Semaphore};
31use tokio_util::sync::CancellationToken;
32use tracing::{debug, info, warn};
33
34const LOG_COMPONENT_NOTIFY: &str = "notify";
35const LOG_SUBSYSTEM_RUNTIME: &str = "runtime";
36const EVENT_NOTIFY_RUNTIME_LIFECYCLE: &str = "notify_runtime_lifecycle";
37const EVENT_NOTIFY_RUNTIME_SHUTDOWN_FAILED: &str = "notify_runtime_shutdown_failed";
38const EVENT_NOTIFY_REPLAY_RETRY_EXHAUSTED: &str = "notify_replay_retry_exhausted";
39const TARGET_CLOSE_TIMEOUT: Duration = Duration::from_secs(10);
40
41pub(crate) struct DetachedNotifyRuntime {
42    direct_dispatches: Arc<DirectDispatchTracker>,
43    runtime: TargetRuntimeManager<Event>,
44    replay_workers: ReplayWorkerManager,
45}
46
47// Multi-lock publication order: replay_workers -> target_list ->
48// publication_gate. The lifecycle may already hold dispatch_gate while
49// entering this facade; no path that needs these three locks may acquire
50// publication_gate before either runtime lock.
51#[derive(Clone)]
52pub struct NotifyRuntimeFacade {
53    dispatch_gate: Arc<RwLock<()>>,
54    legacy_terminated: Arc<AtomicBool>,
55    target_list: SharedNotifyTargetList,
56    replay_workers: Arc<RwLock<ReplayWorkerManager>>,
57    runtime_adapter: Arc<BuiltinPluginRuntimeAdapter<Event>>,
58}
59
60impl NotifyRuntimeFacade {
61    pub fn new(
62        target_list: SharedNotifyTargetList,
63        replay_workers: Arc<RwLock<ReplayWorkerManager>>,
64        concurrency_limiter: Arc<Semaphore>,
65        metrics: Arc<NotificationMetrics>,
66    ) -> Self {
67        let dispatch_gate = shared_dispatch_gate(&target_list, None);
68        Self::new_with_dispatch_gate(target_list, replay_workers, dispatch_gate, concurrency_limiter, metrics)
69    }
70
71    pub(crate) fn new_with_dispatch_gate(
72        target_list: SharedNotifyTargetList,
73        replay_workers: Arc<RwLock<ReplayWorkerManager>>,
74        dispatch_gate: Arc<RwLock<()>>,
75        concurrency_limiter: Arc<Semaphore>,
76        metrics: Arc<NotificationMetrics>,
77    ) -> Self {
78        let dispatch_gate = shared_dispatch_gate(&target_list, Some(dispatch_gate));
79        let replay_metrics = metrics;
80        let runtime_adapter = BuiltinPluginRuntimeAdapter::new(
81            Arc::new(move |event: ReplayEvent<Event>| {
82                let metrics = replay_metrics.clone();
83                Box::pin(async move {
84                    match event {
85                        ReplayEvent::Delivered { .. } => metrics.increment_processed(),
86                        ReplayEvent::RetryableError { .. } => {}
87                        ReplayEvent::RetryExhausted { detail, key, target } => {
88                            warn!(
89                                event = EVENT_NOTIFY_REPLAY_RETRY_EXHAUSTED,
90                                component = LOG_COMPONENT_NOTIFY,
91                                subsystem = LOG_SUBSYSTEM_RUNTIME,
92                                target_id = %target.id(),
93                                replay_key = %key,
94                                error = %detail,
95                                "notify replay retry budget exhausted, entry stays queued and retries"
96                            );
97                        }
98                        ReplayEvent::Dropped { target, .. } | ReplayEvent::PermanentFailure { target, .. } => {
99                            target.record_final_failure();
100                            metrics.increment_failed();
101                        }
102                        ReplayEvent::UnreadableEntry { .. } => {}
103                    }
104                })
105            }),
106            Arc::new(|target_id, has_replay| {
107                if has_replay {
108                    info!(
109                        event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
110                        component = LOG_COMPONENT_NOTIFY,
111                        subsystem = LOG_SUBSYSTEM_RUNTIME,
112                        target_id = %target_id,
113                        state = "replay_started",
114                        "notify runtime lifecycle"
115                    );
116                } else {
117                    debug!(
118                        event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
119                        component = LOG_COMPONENT_NOTIFY,
120                        subsystem = LOG_SUBSYSTEM_RUNTIME,
121                        target_id = %target_id,
122                        state = "replay_skipped",
123                        reason = "no_store_configured",
124                        "notify runtime lifecycle"
125                    );
126                }
127            }),
128            Some(concurrency_limiter),
129            Duration::from_secs(5),
130            Duration::from_millis(500),
131            "Stop event stream processing for target",
132        );
133
134        Self {
135            dispatch_gate,
136            legacy_terminated: Arc::new(AtomicBool::new(false)),
137            target_list,
138            replay_workers,
139            runtime_adapter: Arc::new(runtime_adapter),
140        }
141    }
142
143    pub(crate) async fn pause_dispatch(&self) -> OwnedRwLockWriteGuard<()> {
144        self.dispatch_gate.clone().write_owned().await
145    }
146
147    pub async fn activate_targets_with_replay(
148        &self,
149        targets: Vec<Box<dyn Target<Event> + Send + Sync>>,
150    ) -> RuntimeActivation<Event> {
151        // The compatibility pair must not start replacement replay before
152        // replace_targets has stopped and joined the current generation.
153        self.runtime_adapter.prepare_dormant_compat_activation(targets).await
154    }
155
156    pub(crate) async fn prepare_targets(
157        &self,
158        targets: Vec<Box<dyn Target<Event> + Send + Sync>>,
159        cancellation: &CancellationToken,
160    ) -> PreparedActivation<Event> {
161        self.runtime_adapter.prepare_targets_cancellable(targets, cancellation).await
162    }
163
164    pub(crate) async fn open_prepared_stores(
165        &self,
166        prepared: PreparedActivation<Event>,
167    ) -> Result<(OpenedActivation<Event>, PreparedActivation<Event>), NotificationError> {
168        let runtime_adapter = self.runtime_adapter.clone();
169        tokio::task::spawn_blocking(move || runtime_adapter.open_prepared_stores(prepared))
170            .await
171            .map_err(transition_join_error)
172    }
173
174    pub(crate) async fn commit_prepared<Committed>(
175        &self,
176        opened: OpenedActivation<Event>,
177        on_committed: Committed,
178    ) -> (DetachedNotifyRuntime, PreparedActivation<Event>)
179    where
180        Committed: FnOnce(bool),
181    {
182        // The lifecycle coordinator validates the generation immediately before
183        // entering the non-cancellable handoff. Once old replay workers have
184        // been joined, this generation must publish before a later accepted
185        // intent can run; abandoning it here would leave the old runtime
186        // visible without replay workers.
187        let mut replay_workers = self.replay_workers.write().await;
188        let mut target_list = self.target_list.write().await;
189
190        let (activation, rejected) = self.runtime_adapter.try_activate_prepared(opened);
191        let fully_activated = rejected.failure_summary().is_none();
192        let (runtime, replay_workers, direct_dispatches) =
193            Self::swap_activation(&mut target_list, &mut replay_workers, activation);
194        on_committed(fully_activated);
195        (
196            DetachedNotifyRuntime {
197                direct_dispatches,
198                runtime,
199                replay_workers,
200            },
201            rejected,
202        )
203    }
204
205    pub(crate) async fn commit_disabled<Committed>(&self, on_committed: Committed) -> DetachedNotifyRuntime
206    where
207        Committed: FnOnce(),
208    {
209        // Lock order: replay_workers -> target_list. The lifecycle coordinator
210        // crosses its publication barrier before stopping the old workers, so
211        // this non-cancellable commit must publish even if a newer intent was
212        // accepted while the workers joined.
213        let mut replay_workers = self.replay_workers.write().await;
214        let mut target_list = self.target_list.write().await;
215
216        let (runtime, direct_dispatches) = target_list.replace_runtime(TargetRuntimeManager::new());
217        let detached = DetachedNotifyRuntime {
218            direct_dispatches,
219            runtime,
220            replay_workers: std::mem::take(&mut *replay_workers),
221        };
222        on_committed();
223        detached
224    }
225
226    pub async fn replace_targets(&self, mut activation: RuntimeActivation<Event>) -> Result<(), NotificationError> {
227        // A caller may supply an activation created outside the compatibility
228        // prepare method. Stop any already-running replacement workers before
229        // entering the ordered handoff; the supported activate→replace pair is
230        // dormant here and therefore never overlaps the old generation.
231        self.runtime_adapter.stop_replay_workers(&mut activation.replay_workers).await;
232        let dispatch_guard = self.pause_dispatch().await;
233        if self.legacy_terminated.load(Ordering::Acquire) {
234            drop(dispatch_guard);
235            self.runtime_adapter
236                .close_compat_activation(activation)
237                .await
238                .map_err(NotificationError::Target)?;
239            return Err(NotificationError::Initialization("Notification runtime has terminated".to_string()));
240        }
241        self.stop_active_replay_workers().await;
242        let (activation, open_rejected, activation_rejected) = self.runtime_adapter.start_dormant_compat_activation(activation);
243        let activation_failures = [open_rejected.failure_summary(), activation_rejected.failure_summary()]
244            .into_iter()
245            .flatten()
246            .collect::<Vec<_>>();
247        let (old_runtime, old_replay_workers, old_direct_dispatches) = {
248            // Lock order: replay_workers -> target_list (matches notify AGENTS.md).
249            let mut replay_workers = self.replay_workers.write().await;
250            let mut target_list = self.target_list.write().await;
251            Self::swap_activation(&mut target_list, &mut replay_workers, activation)
252        };
253        drop(dispatch_guard);
254        let close_old = self.close_detached_targets(DetachedNotifyRuntime {
255            direct_dispatches: old_direct_dispatches,
256            runtime: old_runtime,
257            replay_workers: old_replay_workers,
258        });
259        let close_open_rejected = self.close_prepared(open_rejected);
260        let close_activation_rejected = self.close_prepared(activation_rejected);
261        let (close_old, close_open_rejected, close_activation_rejected) =
262            tokio::join!(close_old, close_open_rejected, close_activation_rejected);
263        close_old?;
264        close_open_rejected?;
265        close_activation_rejected?;
266        if !activation_failures.is_empty() {
267            return Err(NotificationError::Initialization(format!(
268                "one or more notification targets failed to activate: {}",
269                activation_failures.join("; ")
270            )));
271        }
272        Ok(())
273    }
274
275    pub async fn stop_replay_workers(&self) {
276        let _dispatch_guard = self.pause_dispatch().await;
277        self.stop_active_replay_workers().await;
278    }
279
280    pub(crate) async fn stop_active_replay_workers(&self) {
281        let mut detached = {
282            let mut replay_workers = self.replay_workers.write().await;
283            std::mem::take(&mut *replay_workers)
284        };
285        self.runtime_adapter.stop_replay_workers(&mut detached).await;
286    }
287
288    pub async fn shutdown(&self) {
289        let _ = self.shutdown_checked().await;
290    }
291
292    pub async fn shutdown_checked(&self) -> Result<(), NotificationError> {
293        let _dispatch_guard = self.pause_dispatch().await;
294        self.legacy_terminated.store(true, Ordering::Release);
295        info!(
296            event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
297            component = LOG_COMPONENT_NOTIFY,
298            subsystem = LOG_SUBSYSTEM_RUNTIME,
299            state = "stopping",
300            "notify runtime lifecycle"
301        );
302
303        let (detached_runtime, detached_replay_workers, detached_direct_dispatches) = {
304            // Lock order: replay_workers -> target_list (matches notify AGENTS.md).
305            let mut replay_workers = self.replay_workers.write().await;
306            let mut target_list = self.target_list.write().await;
307            let (runtime, direct_dispatches) = target_list.replace_runtime(TargetRuntimeManager::new());
308            (runtime, std::mem::take(&mut *replay_workers), direct_dispatches)
309        };
310        let active_targets = detached_replay_workers.len();
311        info!(
312            event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
313            component = LOG_COMPONENT_NOTIFY,
314            subsystem = LOG_SUBSYSTEM_RUNTIME,
315            state = "replay_stopping",
316            active_targets,
317            "notify runtime lifecycle"
318        );
319
320        let shutdown_result = self
321            .close_detached(DetachedNotifyRuntime {
322                direct_dispatches: detached_direct_dispatches,
323                runtime: detached_runtime,
324                replay_workers: detached_replay_workers,
325            })
326            .await;
327        if let Err(err) = &shutdown_result {
328            tracing::error!(
329                event = EVENT_NOTIFY_RUNTIME_SHUTDOWN_FAILED,
330                component = LOG_COMPONENT_NOTIFY,
331                subsystem = LOG_SUBSYSTEM_RUNTIME,
332                error = %err,
333                "Failed to shutdown notify runtime cleanly"
334            );
335        }
336
337        info!(
338            event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
339            component = LOG_COMPONENT_NOTIFY,
340            subsystem = LOG_SUBSYSTEM_RUNTIME,
341            state = "stopped",
342            "notify runtime lifecycle"
343        );
344        shutdown_result
345    }
346
347    fn swap_activation(
348        target_list: &mut crate::notifier::TargetList,
349        replay_workers: &mut ReplayWorkerManager,
350        activation: RuntimeActivation<Event>,
351    ) -> (TargetRuntimeManager<Event>, ReplayWorkerManager, Arc<DirectDispatchTracker>) {
352        let mut replacement = TargetRuntimeManager::new();
353        for target in activation.targets {
354            replacement.add_arc(target);
355        }
356        let (runtime, direct_dispatches) = target_list.replace_runtime(replacement);
357        (runtime, std::mem::replace(replay_workers, activation.replay_workers), direct_dispatches)
358    }
359
360    pub(crate) async fn stop_detached_replay(&self, detached: &mut DetachedNotifyRuntime) {
361        // Replay join is intentionally not wrapped in the target-close timeout:
362        // returning before a worker is confirmed stopped could let a
363        // replacement drain the same persistent queue concurrently.
364        self.runtime_adapter.stop_replay_workers(&mut detached.replay_workers).await;
365    }
366
367    pub(crate) async fn close_detached_targets(&self, mut detached: DetachedNotifyRuntime) -> Result<(), NotificationError> {
368        // Direct sends are allowed to finish after the replacement runtime is
369        // published, but the old targets must remain open until every task that
370        // selected that generation has released its lease.
371        detached.direct_dispatches.wait_idle().await;
372        match tokio::time::timeout(TARGET_CLOSE_TIMEOUT, detached.runtime.clear_and_close()).await {
373            Ok(close_errors) if close_errors.is_empty() => Ok(()),
374            Ok(close_errors) => {
375                let targets = close_errors
376                    .into_iter()
377                    .map(|(target_id, _)| target_id)
378                    .collect::<Vec<_>>()
379                    .join("; ");
380                Err(NotificationError::Target(rustfs_targets::TargetError::Storage(format!(
381                    "Failed to close {targets}"
382                ))))
383            }
384            Err(_) => Err(NotificationError::Target(rustfs_targets::TargetError::Timeout(
385                "Timed out closing replaced notification targets".to_string(),
386            ))),
387        }
388    }
389
390    async fn close_detached(&self, mut detached: DetachedNotifyRuntime) -> Result<(), NotificationError> {
391        self.stop_detached_replay(&mut detached).await;
392        self.close_detached_targets(detached).await
393    }
394
395    pub(crate) async fn close_prepared(&self, prepared: PreparedActivation<Event>) -> Result<(), NotificationError> {
396        match tokio::time::timeout(TARGET_CLOSE_TIMEOUT, self.runtime_adapter.close_prepared(prepared)).await {
397            Ok(result) => result.map_err(NotificationError::Target),
398            Err(_) => Err(NotificationError::Target(rustfs_targets::TargetError::Timeout(
399                "Timed out closing superseded notification targets".to_string(),
400            ))),
401        }
402    }
403}
404
405#[cfg(test)]
406mod tests {
407    use super::NotifyRuntimeFacade;
408    use crate::{
409        Event, integration::NotificationMetrics, notifier::EventNotifier, rule_engine::NotifyRuleEngine,
410        runtime_view::NotifyRuntimeView,
411    };
412    use rustfs_targets::arn::TargetID;
413    use rustfs_targets::store::QueueStore;
414    use rustfs_targets::target::QueuedPayload;
415    use rustfs_targets::testkit::MockTarget;
416    use rustfs_targets::{ReplayWorkerManager, SharedTarget, TargetError};
417    use std::sync::Arc;
418    use tokio::sync::{Notify, RwLock, Semaphore};
419
420    fn build_facade() -> (NotifyRuntimeFacade, Arc<EventNotifier>, Arc<RwLock<ReplayWorkerManager>>) {
421        let metrics = Arc::new(NotificationMetrics::new());
422        let notifier = Arc::new(EventNotifier::new(metrics.clone(), NotifyRuleEngine::new()));
423        let target_list = notifier.target_list();
424        let replay_workers = Arc::new(RwLock::new(ReplayWorkerManager::new()));
425        let facade = NotifyRuntimeFacade::new_with_dispatch_gate(
426            target_list,
427            replay_workers.clone(),
428            notifier.dispatch_gate(),
429            Arc::new(Semaphore::new(4)),
430            metrics,
431        );
432        (facade, notifier, replay_workers)
433    }
434
435    #[tokio::test]
436    async fn stopping_replay_workers_is_a_no_op_when_there_are_none() {
437        let (facade, _, _) = build_facade();
438
439        facade.stop_replay_workers().await;
440
441        // The stop path takes the worker list and hands it to the adapter, so an
442        // empty facade must come back with the list still empty and dispatch
443        // released rather than left paused (rustfs/backlog#1836).
444        assert!(facade.replay_workers.read().await.is_empty());
445
446        // Calling it twice must stay harmless: shutdown paths do exactly that.
447        facade.stop_replay_workers().await;
448        assert!(facade.replay_workers.read().await.is_empty());
449    }
450
451    #[tokio::test]
452    async fn runtime_facade_activates_empty_target_list() {
453        let (facade, _, _) = build_facade();
454        let activation = facade.activate_targets_with_replay(Vec::new()).await;
455
456        assert!(activation.targets.is_empty());
457        assert_eq!(activation.replay_workers.len(), 0);
458    }
459
460    #[tokio::test]
461    async fn compatibility_activation_stays_dormant_until_ordered_replace() {
462        let (facade, _, replay_workers) = build_facade();
463        let queue_root = tempfile::tempdir().expect("queue root");
464        let store = QueueStore::<QueuedPayload>::new_with_compression(queue_root.path(), 16, ".event", false);
465        let target = MockTarget::new("primary", "webhook").with_store(Arc::new(store));
466
467        let activation = facade.activate_targets_with_replay(vec![Box::new(target)]).await;
468        assert_eq!(activation.targets.len(), 1);
469        assert_eq!(activation.replay_workers.len(), 0, "compatibility prepare must not start replay early");
470        assert_eq!(replay_workers.read().await.len(), 0);
471
472        facade
473            .replace_targets(activation)
474            .await
475            .expect("ordered compatibility replace should succeed");
476        assert_eq!(replay_workers.read().await.len(), 1);
477        facade.shutdown_checked().await.expect("test runtime should shut down");
478    }
479
480    #[tokio::test]
481    async fn runtime_facade_replace_targets_commits_runtime_state() {
482        let (facade, notifier, replay_workers) = build_facade();
483        let target = MockTarget::new("primary", "webhook");
484        let activation = rustfs_targets::RuntimeActivation {
485            replay_workers: ReplayWorkerManager::new(),
486            targets: vec![Arc::new(target) as SharedTarget<Event>],
487        };
488
489        facade
490            .replace_targets(activation)
491            .await
492            .expect("replace_targets should succeed");
493
494        let runtime_view = NotifyRuntimeView::new(notifier.target_list(), replay_workers.clone());
495        let active_targets = runtime_view.get_active_targets().await;
496        assert_eq!(active_targets, vec![TargetID::new("primary".to_string(), "webhook".to_string())]);
497        assert_eq!(replay_workers.read().await.len(), 0);
498    }
499
500    #[tokio::test]
501    async fn runtime_queries_do_not_wait_for_target_close() {
502        let (facade, notifier, replay_workers) = build_facade();
503        let target = MockTarget::new("primary", "webhook");
504        target.set_block_on_close(true);
505        let observer = target.clone();
506        facade
507            .replace_targets(rustfs_targets::RuntimeActivation {
508                replay_workers: ReplayWorkerManager::new(),
509                targets: vec![Arc::new(target) as SharedTarget<Event>],
510            })
511            .await
512            .expect("target install should succeed");
513
514        let shutdown = tokio::spawn({
515            let facade = facade.clone();
516            async move { facade.shutdown_checked().await }
517        });
518        observer.close_started().notified().await;
519
520        let target_list = notifier.target_list();
521        assert!(target_list.try_read().is_ok(), "target list lock must not be held during close");
522        assert!(replay_workers.try_read().is_ok(), "replay manager lock must not be held during close");
523        observer.close_gate().add_permits(1);
524        shutdown
525            .await
526            .expect("shutdown task should not panic")
527            .expect("shutdown should succeed");
528
529        let snapshot = NotifyRuntimeView::new(target_list, replay_workers)
530            .runtime_status_snapshot()
531            .await;
532        assert_eq!(snapshot.target_count, 0);
533        assert_eq!(snapshot.replay_worker_count, 0);
534    }
535
536    #[tokio::test]
537    async fn runtime_locks_are_released_while_replay_worker_joins() {
538        let (facade, notifier, replay_workers) = build_facade();
539        let cancel_received = Arc::new(Notify::new());
540        let release = Arc::new(Notify::new());
541        let (cancel_tx, mut cancel_rx) = tokio::sync::mpsc::channel(1);
542        let join = tokio::spawn({
543            let cancel_received = cancel_received.clone();
544            let release = release.clone();
545            async move {
546                let _ = cancel_rx.recv().await;
547                cancel_received.notify_one();
548                release.notified().await;
549            }
550        });
551        replay_workers
552            .write()
553            .await
554            .insert_with_handle("blocked-worker".to_string(), cancel_tx, join);
555
556        let stop = tokio::spawn({
557            let facade = facade.clone();
558            async move { facade.stop_replay_workers().await }
559        });
560        cancel_received.notified().await;
561
562        assert!(replay_workers.try_read().is_ok(), "replay manager lock must not be held during join");
563        let target_list = notifier.target_list();
564        assert!(target_list.try_read().is_ok(), "target list lock must not be held during replay join");
565
566        release.notify_one();
567        stop.await.expect("stop task should finish");
568    }
569
570    #[tokio::test]
571    async fn shutdown_returns_close_error_after_detaching_runtime() {
572        let (facade, notifier, replay_workers) = build_facade();
573        let target = MockTarget::new("primary", "webhook").with_close_failures(usize::MAX);
574        facade
575            .replace_targets(rustfs_targets::RuntimeActivation {
576                replay_workers: ReplayWorkerManager::new(),
577                targets: vec![Arc::new(target) as SharedTarget<Event>],
578            })
579            .await
580            .expect("target install should succeed");
581
582        let err = facade.shutdown_checked().await.expect_err("close failure should propagate");
583        assert!(matches!(err, crate::NotificationError::Target(TargetError::Storage(_))));
584
585        let snapshot = NotifyRuntimeView::new(notifier.target_list(), replay_workers)
586            .runtime_status_snapshot()
587            .await;
588        assert_eq!(snapshot.target_count, 0);
589        assert_eq!(snapshot.replay_worker_count, 0);
590    }
591
592    #[tokio::test(start_paused = true)]
593    async fn shutdown_bounds_a_target_that_never_closes() {
594        let (facade, notifier, replay_workers) = build_facade();
595        // The close gate never receives a permit, so this target's close blocks forever.
596        let target = MockTarget::new("primary", "webhook");
597        target.set_block_on_close(true);
598        let observer = target.clone();
599        facade
600            .replace_targets(rustfs_targets::RuntimeActivation {
601                replay_workers: ReplayWorkerManager::new(),
602                targets: vec![Arc::new(target) as SharedTarget<Event>],
603            })
604            .await
605            .expect("target install should succeed");
606
607        let shutdown = tokio::spawn({
608            let facade = facade.clone();
609            async move { facade.shutdown_checked().await }
610        });
611        observer.close_started().notified().await;
612        tokio::time::advance(super::TARGET_CLOSE_TIMEOUT).await;
613
614        let err = shutdown
615            .await
616            .expect("shutdown task should not panic")
617            .expect_err("blocked close must time out");
618        assert!(matches!(err, crate::NotificationError::Target(TargetError::Timeout(_))));
619        let snapshot = NotifyRuntimeView::new(notifier.target_list(), replay_workers)
620            .runtime_status_snapshot()
621            .await;
622        assert_eq!(snapshot.target_count, 0);
623        assert_eq!(snapshot.replay_worker_count, 0);
624    }
625}