Skip to main content

rustfs_notify/
lifecycle.rs

1// Copyright 2024 RustFS Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use crate::{
16    Event, NotificationError, config_manager::notify_configuration_hint, error::transition_join_error, registry::TargetRegistry,
17    runtime_facade::NotifyRuntimeFacade,
18};
19use async_trait::async_trait;
20use rustfs_config::server_config::Config;
21use rustfs_targets::Target;
22use std::future::Future;
23use std::pin::Pin;
24use std::sync::atomic::{AtomicU64, Ordering};
25use std::sync::{Arc, Mutex as StdMutex};
26use tokio::sync::{Mutex, RwLock};
27use tokio::task::JoinHandle;
28use tokio_util::sync::CancellationToken;
29use tracing::{debug, info};
30
31type CurrentGeneration = Arc<dyn Fn() -> bool + Send + Sync>;
32type CommitGeneration = Box<dyn FnOnce(bool) + Send>;
33type CleanupFuture = Pin<Box<dyn Future<Output = Result<(), NotificationError>> + Send>>;
34
35const LOG_COMPONENT_NOTIFY: &str = "notify";
36const LOG_SUBSYSTEM_LIFECYCLE: &str = "lifecycle";
37const EVENT_NOTIFY_RUNTIME_LIFECYCLE: &str = "notify_runtime_lifecycle";
38
39#[derive(Clone, Copy, Debug, PartialEq, Eq)]
40pub enum NotificationRuntimeState {
41    LiveOnly,
42    TargetsEnabled { generation: u64 },
43    Terminated,
44}
45
46#[derive(Clone, Copy, Debug, PartialEq, Eq)]
47enum DesiredMode {
48    LiveOnly,
49    TargetsEnabled,
50    Terminated,
51}
52
53#[derive(Clone)]
54struct DesiredState {
55    cancellation: CancellationToken,
56    config: Config,
57    generation: u64,
58    mode: DesiredMode,
59}
60
61struct RuntimeApply {
62    cleanup: Option<CleanupFuture>,
63}
64
65impl RuntimeApply {
66    fn complete() -> Self {
67        Self { cleanup: None }
68    }
69
70    fn with_cleanup(cleanup: CleanupFuture) -> Self {
71        Self { cleanup: Some(cleanup) }
72    }
73
74    async fn finish(self) -> Result<(), NotificationError> {
75        match self.cleanup {
76            Some(cleanup) => cleanup.await,
77            None => Ok(()),
78        }
79    }
80}
81
82enum TransitionResult {
83    Ready(Result<(), NotificationError>),
84    Running(JoinHandle<Result<(), NotificationError>>),
85}
86
87pub struct NotificationLifecycleTransition {
88    result: TransitionResult,
89}
90
91impl NotificationLifecycleTransition {
92    fn ready(result: Result<(), NotificationError>) -> Self {
93        Self {
94            result: TransitionResult::Ready(result),
95        }
96    }
97
98    fn running(handle: JoinHandle<Result<(), NotificationError>>) -> Self {
99        Self {
100            result: TransitionResult::Running(handle),
101        }
102    }
103
104    pub async fn wait(self) -> Result<(), NotificationError> {
105        match self.result {
106            TransitionResult::Ready(result) => result,
107            TransitionResult::Running(handle) => handle.await.map_err(transition_join_error)?,
108        }
109    }
110}
111
112#[async_trait]
113trait LifecycleRuntime: Send + Sync {
114    async fn cache_config(&self, config: Config);
115
116    async fn enable_targets(
117        &self,
118        config: &Config,
119        cancellation: &CancellationToken,
120        publication_gate: Arc<StdMutex<()>>,
121        is_current: CurrentGeneration,
122        on_committed: CommitGeneration,
123    ) -> Result<RuntimeApply, NotificationError>;
124
125    async fn disable_targets(
126        &self,
127        publication_gate: Arc<StdMutex<()>>,
128        is_current: CurrentGeneration,
129        on_committed: CommitGeneration,
130    ) -> Result<RuntimeApply, NotificationError>;
131}
132
133struct DefaultLifecycleRuntime {
134    config: Arc<RwLock<Config>>,
135    registry: Arc<TargetRegistry>,
136    runtime_facade: NotifyRuntimeFacade,
137    #[cfg(test)]
138    handoff_observer: Option<Arc<dyn Fn() + Send + Sync>>,
139}
140
141#[async_trait]
142impl LifecycleRuntime for DefaultLifecycleRuntime {
143    async fn cache_config(&self, config: Config) {
144        *self.config.write().await = config;
145    }
146
147    async fn enable_targets(
148        &self,
149        config: &Config,
150        cancellation: &CancellationToken,
151        publication_gate: Arc<StdMutex<()>>,
152        is_current: CurrentGeneration,
153        on_committed: CommitGeneration,
154    ) -> Result<RuntimeApply, NotificationError> {
155        info!(
156            event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
157            component = LOG_COMPONENT_NOTIFY,
158            subsystem = LOG_SUBSYSTEM_LIFECYCLE,
159            state = "preparing",
160            "notify runtime lifecycle"
161        );
162
163        let (targets, creation_failures): (Vec<Box<dyn Target<Event> + Send + Sync>>, Vec<String>) =
164            self.registry.create_dormant_targets_from_config(config).await?;
165        if targets.is_empty() {
166            debug!(
167                event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
168                component = LOG_COMPONENT_NOTIFY,
169                subsystem = LOG_SUBSYSTEM_LIFECYCLE,
170                state = "idle",
171                reason = "no_targets_configured",
172                hint = %notify_configuration_hint(),
173                "notify runtime lifecycle"
174            );
175        }
176
177        let mut prepared = self.runtime_facade.prepare_targets(targets, cancellation).await;
178        prepared.extend_creation_failures(creation_failures);
179        if !is_current() {
180            let runtime_facade = self.runtime_facade.clone();
181            return Ok(RuntimeApply::with_cleanup(Box::pin(async move {
182                runtime_facade.close_prepared(prepared).await
183            })));
184        }
185
186        // Drain sends that cloned the old targets, then keep later sends queued
187        // until queue-store ownership and the target list move atomically. A
188        // successful current-generation check under publication_gate is the
189        // non-cancellable commit barrier: later intents serialize after this
190        // handoff instead of abandoning a workerless old runtime.
191        let dispatch_guard = self.runtime_facade.pause_dispatch().await;
192        {
193            let _publication_guard = publication_gate.lock().unwrap_or_else(|err| err.into_inner());
194            if !is_current() {
195                drop(_publication_guard);
196                drop(dispatch_guard);
197                let runtime_facade = self.runtime_facade.clone();
198                return Ok(RuntimeApply::with_cleanup(Box::pin(async move {
199                    runtime_facade.close_prepared(prepared).await
200                })));
201            }
202        }
203
204        #[cfg(test)]
205        if let Some(observer) = &self.handoff_observer {
206            observer();
207        }
208
209        self.runtime_facade.stop_active_replay_workers().await;
210        let (opened, rejected) = self.runtime_facade.open_prepared_stores(prepared).await?;
211        let open_complete = rejected.failure_summary().is_none();
212        // A newer accepted generation remains queued behind apply_lock and
213        // replaces this one next.
214        let (detached, activation_rejected) = self
215            .runtime_facade
216            .commit_prepared(opened, move |activation_complete| on_committed(open_complete && activation_complete))
217            .await;
218        let activation_failures = [rejected.failure_summary(), activation_rejected.failure_summary()]
219            .into_iter()
220            .flatten()
221            .collect::<Vec<_>>();
222        drop(dispatch_guard);
223        let runtime_facade = self.runtime_facade.clone();
224
225        info!(
226            event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
227            component = LOG_COMPONENT_NOTIFY,
228            subsystem = LOG_SUBSYSTEM_LIFECYCLE,
229            state = "enabled",
230            "notify runtime lifecycle"
231        );
232        Ok(RuntimeApply::with_cleanup(Box::pin(async move {
233            let (close_old, close_rejected, close_activation_rejected) = tokio::join!(
234                runtime_facade.close_detached_targets(detached),
235                runtime_facade.close_prepared(rejected),
236                runtime_facade.close_prepared(activation_rejected),
237            );
238            close_old?;
239            close_rejected?;
240            close_activation_rejected?;
241            if !activation_failures.is_empty() {
242                return Err(NotificationError::Initialization(format!(
243                    "one or more notification targets failed to activate: {}",
244                    activation_failures.join("; ")
245                )));
246            }
247            Ok(())
248        })))
249    }
250
251    async fn disable_targets(
252        &self,
253        publication_gate: Arc<StdMutex<()>>,
254        is_current: CurrentGeneration,
255        on_committed: CommitGeneration,
256    ) -> Result<RuntimeApply, NotificationError> {
257        let dispatch_guard = self.runtime_facade.pause_dispatch().await;
258        {
259            let _publication_guard = publication_gate.lock().unwrap_or_else(|err| err.into_inner());
260            if !is_current() {
261                drop(_publication_guard);
262                drop(dispatch_guard);
263                return Ok(RuntimeApply::complete());
264            }
265        }
266
267        // The generation check above is the non-cancellable publication
268        // barrier. Later intents serialize behind apply_lock while the old
269        // replay workers stop, so LiveOnly is not visible until every old
270        // worker has joined.
271        self.runtime_facade.stop_active_replay_workers().await;
272        let detached = self.runtime_facade.commit_disabled(move || on_committed(true)).await;
273        drop(dispatch_guard);
274        let runtime_facade = self.runtime_facade.clone();
275        Ok(RuntimeApply::with_cleanup(Box::pin(async move {
276            runtime_facade.close_detached_targets(detached).await
277        })))
278    }
279}
280
281struct LifecycleCoordinatorInner {
282    applied_generation: AtomicU64,
283    apply_lock: Mutex<()>,
284    desired: StdMutex<DesiredState>,
285    publication_gate: Arc<StdMutex<()>>,
286    runtime: Arc<dyn LifecycleRuntime>,
287    state: StdMutex<NotificationRuntimeState>,
288}
289
290#[derive(Clone)]
291pub(crate) struct NotifyLifecycleCoordinator {
292    inner: Arc<LifecycleCoordinatorInner>,
293}
294
295impl NotifyLifecycleCoordinator {
296    pub(crate) fn new(config: Arc<RwLock<Config>>, registry: Arc<TargetRegistry>, runtime_facade: NotifyRuntimeFacade) -> Self {
297        let runtime = Arc::new(DefaultLifecycleRuntime {
298            config,
299            registry,
300            runtime_facade,
301            #[cfg(test)]
302            handoff_observer: None,
303        });
304        Self::with_runtime(Config::new(), runtime)
305    }
306
307    fn with_runtime(initial_config: Config, runtime: Arc<dyn LifecycleRuntime>) -> Self {
308        Self {
309            inner: Arc::new(LifecycleCoordinatorInner {
310                applied_generation: AtomicU64::new(0),
311                apply_lock: Mutex::new(()),
312                desired: StdMutex::new(DesiredState {
313                    cancellation: CancellationToken::new(),
314                    config: initial_config,
315                    generation: 0,
316                    mode: DesiredMode::LiveOnly,
317                }),
318                publication_gate: Arc::new(StdMutex::new(())),
319                runtime,
320                state: StdMutex::new(NotificationRuntimeState::LiveOnly),
321            }),
322        }
323    }
324
325    pub(crate) fn set_mode(&self, enabled: bool, config: Option<Config>) -> NotificationLifecycleTransition {
326        self.accept(false, move |desired| {
327            desired.mode = if enabled {
328                DesiredMode::TargetsEnabled
329            } else {
330                DesiredMode::LiveOnly
331            };
332            if let Some(config) = config {
333                desired.config = config;
334            }
335            Ok(true)
336        })
337    }
338
339    pub(crate) fn update_config(&self, config: Config) -> NotificationLifecycleTransition {
340        self.accept(false, move |desired| {
341            desired.config = config;
342            Ok(true)
343        })
344    }
345
346    pub(crate) fn terminate(&self) -> NotificationLifecycleTransition {
347        self.accept(true, |desired| {
348            if desired.mode == DesiredMode::Terminated {
349                return Ok(false);
350            }
351            desired.mode = DesiredMode::Terminated;
352            Ok(true)
353        })
354    }
355
356    pub(crate) fn state(&self) -> NotificationRuntimeState {
357        *self.inner.state.lock().unwrap_or_else(|err| err.into_inner())
358    }
359
360    pub(crate) fn is_converged(&self) -> bool {
361        let desired_generation = self.inner.desired.lock().unwrap_or_else(|err| err.into_inner()).generation;
362        self.inner.applied_generation.load(Ordering::Acquire) >= desired_generation
363    }
364
365    fn accept<F>(&self, allow_terminated: bool, update: F) -> NotificationLifecycleTransition
366    where
367        F: FnOnce(&mut DesiredState) -> Result<bool, NotificationError>,
368    {
369        let Ok(runtime) = tokio::runtime::Handle::try_current() else {
370            return NotificationLifecycleTransition::ready(Err(NotificationError::Initialization(
371                "Notification lifecycle coordinator is unavailable".to_string(),
372            )));
373        };
374        let _publication_guard = self.inner.publication_gate.lock().unwrap_or_else(|err| err.into_inner());
375        let mut desired = self.inner.desired.lock().unwrap_or_else(|err| err.into_inner());
376        if desired.mode == DesiredMode::Terminated && !allow_terminated {
377            return NotificationLifecycleTransition::ready(Err(NotificationError::Initialization(
378                "Notification runtime has terminated".to_string(),
379            )));
380        }
381
382        let previous = desired.clone();
383        let changed = match update(&mut desired) {
384            Ok(changed) => changed,
385            Err(err) => return NotificationLifecycleTransition::ready(Err(err)),
386        };
387        if changed {
388            let Some(next_generation) = desired.generation.checked_add(1) else {
389                *desired = previous;
390                return NotificationLifecycleTransition::ready(Err(NotificationError::Initialization(
391                    "Notification lifecycle generation is exhausted".to_string(),
392                )));
393            };
394            desired.generation = next_generation;
395            previous.cancellation.cancel();
396            desired.cancellation = CancellationToken::new();
397        }
398        let accepted = desired.clone();
399        drop(desired);
400        drop(_publication_guard);
401        let inner = Arc::clone(&self.inner);
402        NotificationLifecycleTransition::running(runtime.spawn(async move { inner.converge(accepted).await }))
403    }
404}
405
406impl LifecycleCoordinatorInner {
407    async fn converge(self: &Arc<Self>, desired: DesiredState) -> Result<(), NotificationError> {
408        let _apply_guard = self.apply_lock.lock().await;
409        if self.applied_generation.load(Ordering::Acquire) >= desired.generation {
410            return Ok(());
411        }
412        if !self.current_matches(desired.generation, desired.mode) {
413            return Ok(());
414        }
415
416        self.runtime.cache_config(desired.config.clone()).await;
417        let current_inner = Arc::clone(self);
418        let generation = desired.generation;
419        let mode = desired.mode;
420        let is_current: CurrentGeneration = Arc::new(move || current_inner.current_matches(generation, mode));
421        let publication_gate = self.publication_gate.clone();
422
423        let state = match desired.mode {
424            DesiredMode::LiveOnly => NotificationRuntimeState::LiveOnly,
425            DesiredMode::TargetsEnabled => NotificationRuntimeState::TargetsEnabled {
426                generation: desired.generation,
427            },
428            DesiredMode::Terminated => NotificationRuntimeState::Terminated,
429        };
430        let committed_inner = Arc::clone(self);
431        let on_committed: CommitGeneration =
432            Box::new(move |complete| committed_inner.mark_published(generation, state, complete));
433
434        let applied = match desired.mode {
435            DesiredMode::LiveOnly | DesiredMode::Terminated => {
436                self.runtime
437                    .disable_targets(publication_gate, is_current, on_committed)
438                    .await?
439            }
440            DesiredMode::TargetsEnabled => {
441                self.runtime
442                    .enable_targets(&desired.config, &desired.cancellation, publication_gate, is_current, on_committed)
443                    .await?
444            }
445        };
446        drop(_apply_guard);
447        applied.finish().await
448    }
449
450    fn current_matches(&self, generation: u64, mode: DesiredMode) -> bool {
451        let desired = self.desired.lock().unwrap_or_else(|err| err.into_inner());
452        desired.generation == generation && desired.mode == mode
453    }
454
455    fn mark_published(&self, generation: u64, state: NotificationRuntimeState, complete: bool) {
456        *self.state.lock().unwrap_or_else(|err| err.into_inner()) = state;
457        if complete {
458            self.applied_generation.store(generation, Ordering::Release);
459        }
460    }
461}
462
463#[cfg(test)]
464mod tests {
465    use super::{
466        CommitGeneration, CurrentGeneration, DefaultLifecycleRuntime, LifecycleRuntime, NotificationRuntimeState,
467        NotifyLifecycleCoordinator, RuntimeApply,
468    };
469    use crate::{
470        Event, NotificationError, integration::NotificationMetrics, notifier::EventNotifier, registry::TargetRegistry,
471        rule_engine::NotifyRuleEngine, runtime_facade::NotifyRuntimeFacade, runtime_view::NotifyRuntimeView,
472    };
473    use async_trait::async_trait;
474    use rustfs_config::ENABLE_KEY;
475    use rustfs_config::server_config::{Config, KVS};
476    use rustfs_targets::arn::TargetID;
477    use rustfs_targets::store::{Key, QueueStore, Store};
478    use rustfs_targets::target::{EntityTarget, QueuedPayload, QueuedPayloadMeta};
479    use rustfs_targets::testkit::MockTarget;
480    use rustfs_targets::{
481        EventName, ReplayWorkerManager, StoreError, Target, TargetError, TargetPluginDescriptor, TargetPluginRegistry,
482    };
483    use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
484    use std::sync::{Arc, Mutex as StdMutex};
485    use tokio::sync::{Notify, RwLock, Semaphore, mpsc};
486    use tokio_util::sync::CancellationToken;
487
488    const BLOCKING_INIT_TARGET_TYPE: &str = "lifecycle_blocking_init";
489    const RETRY_INIT_TARGET_TYPE: &str = "lifecycle_retry_init";
490    const REPLAY_TARGET_TYPE: &str = "lifecycle_replay";
491
492    struct ReplayState {
493        active_workers: AtomicUsize,
494        completed_deliveries: AtomicUsize,
495        created_targets: AtomicUsize,
496        delivery_attempts: AtomicUsize,
497        delivery_release: Notify,
498        delivered_bodies: StdMutex<Vec<Vec<u8>>>,
499        blocked_init_generation: usize,
500        blocked_init_release: Notify,
501        init_tx: mpsc::UnboundedSender<usize>,
502        list_tx: mpsc::UnboundedSender<usize>,
503        peak_workers: AtomicUsize,
504        send_tx: mpsc::UnboundedSender<usize>,
505    }
506
507    struct ReplayWorkerLease {
508        state: Arc<ReplayState>,
509    }
510
511    impl ReplayWorkerLease {
512        fn acquire(state: Arc<ReplayState>) -> Self {
513            let active = state.active_workers.fetch_add(1, Ordering::SeqCst) + 1;
514            let mut peak = state.peak_workers.load(Ordering::SeqCst);
515            while active > peak {
516                match state
517                    .peak_workers
518                    .compare_exchange_weak(peak, active, Ordering::SeqCst, Ordering::SeqCst)
519                {
520                    Ok(_) => break,
521                    Err(observed) => peak = observed,
522                }
523            }
524            Self { state }
525        }
526    }
527
528    impl Drop for ReplayWorkerLease {
529        fn drop(&mut self) {
530            self.state.active_workers.fetch_sub(1, Ordering::SeqCst);
531        }
532    }
533
534    #[derive(Clone)]
535    struct ObservedQueueStore {
536        generation: usize,
537        inner: QueueStore<QueuedPayload>,
538        state: Arc<ReplayState>,
539        _worker_lease: Option<Arc<ReplayWorkerLease>>,
540    }
541
542    impl Store<QueuedPayload> for ObservedQueueStore {
543        type Error = StoreError;
544        type Key = Key;
545
546        fn open(&self) -> Result<(), Self::Error> {
547            self.inner.open()
548        }
549
550        fn put(&self, item: Arc<QueuedPayload>) -> Result<Self::Key, Self::Error> {
551            self.inner.put(item)
552        }
553
554        fn put_multiple(&self, items: Vec<QueuedPayload>) -> Result<Self::Key, Self::Error> {
555            self.inner.put_multiple(items)
556        }
557
558        fn put_raw(&self, data: &[u8]) -> Result<Self::Key, Self::Error> {
559            self.inner.put_raw(data)
560        }
561
562        fn get(&self, key: &Self::Key) -> Result<QueuedPayload, Self::Error> {
563            self.inner.get(key)
564        }
565
566        fn get_multiple(&self, key: &Self::Key) -> Result<Vec<QueuedPayload>, Self::Error> {
567            self.inner.get_multiple(key)
568        }
569
570        fn get_raw(&self, key: &Self::Key) -> Result<Vec<u8>, Self::Error> {
571            self.inner.get_raw(key)
572        }
573
574        fn del(&self, key: &Self::Key) -> Result<(), Self::Error> {
575            self.inner.del(key)
576        }
577
578        fn delete(&self) -> Result<(), Self::Error> {
579            self.inner.delete()
580        }
581
582        fn list(&self) -> Vec<Self::Key> {
583            let _ = self.state.list_tx.send(self.generation);
584            self.inner.list()
585        }
586
587        fn len(&self) -> usize {
588            self.inner.len()
589        }
590
591        fn is_empty(&self) -> bool {
592            self.inner.is_empty()
593        }
594
595        fn boxed_clone(&self) -> Box<dyn Store<QueuedPayload, Error = Self::Error, Key = Self::Key> + Send + Sync> {
596            Box::new(Self {
597                generation: self.generation,
598                inner: self.inner.clone(),
599                state: self.state.clone(),
600                _worker_lease: Some(Arc::new(ReplayWorkerLease::acquire(self.state.clone()))),
601            })
602        }
603    }
604
605    #[derive(Clone)]
606    struct ReplayTarget {
607        id: TargetID,
608        generation: usize,
609        state: Arc<ReplayState>,
610        store: ObservedQueueStore,
611    }
612
613    #[async_trait]
614    impl Target<Event> for ReplayTarget {
615        fn id(&self) -> TargetID {
616            self.id.clone()
617        }
618
619        async fn is_active(&self) -> Result<bool, TargetError> {
620            Ok(true)
621        }
622
623        async fn save(&self, _event: Arc<EntityTarget<Event>>) -> Result<(), TargetError> {
624            Ok(())
625        }
626
627        async fn send_raw_from_store(&self, _key: Key, body: Vec<u8>, _meta: QueuedPayloadMeta) -> Result<(), TargetError> {
628            let attempt = self.state.delivery_attempts.fetch_add(1, Ordering::SeqCst) + 1;
629            let _ = self.state.send_tx.send(attempt);
630            self.state.delivery_release.notified().await;
631            self.state
632                .delivered_bodies
633                .lock()
634                .unwrap_or_else(|err| err.into_inner())
635                .push(body);
636            self.state.completed_deliveries.fetch_add(1, Ordering::SeqCst);
637            Ok(())
638        }
639
640        async fn close(&self) -> Result<(), TargetError> {
641            Ok(())
642        }
643
644        fn store(&self) -> Option<&(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync)> {
645            Some(&self.store)
646        }
647
648        fn clone_dyn(&self) -> Box<dyn Target<Event> + Send + Sync> {
649            Box::new(self.clone())
650        }
651
652        async fn init(&self) -> Result<(), TargetError> {
653            let _ = self.state.init_tx.send(self.generation);
654            if self.generation == self.state.blocked_init_generation {
655                self.state.blocked_init_release.notified().await;
656            }
657            Ok(())
658        }
659
660        fn is_enabled(&self) -> bool {
661            true
662        }
663    }
664
665    struct TestRuntime {
666        block_cleanup: AtomicBool,
667        block_enable: AtomicBool,
668        cached_config: StdMutex<Config>,
669        cleanup_entered: Arc<Notify>,
670        cleanup_release: Arc<Notify>,
671        disable_calls: AtomicUsize,
672        enable_calls: AtomicUsize,
673        enable_entered: Notify,
674        fail_enable: AtomicBool,
675        enable_release: Notify,
676        enabled_configs: StdMutex<Vec<Config>>,
677    }
678
679    impl TestRuntime {
680        fn new(initial_config: Config) -> Self {
681            Self {
682                block_cleanup: AtomicBool::new(false),
683                block_enable: AtomicBool::new(false),
684                cached_config: StdMutex::new(initial_config),
685                cleanup_entered: Arc::new(Notify::new()),
686                cleanup_release: Arc::new(Notify::new()),
687                disable_calls: AtomicUsize::new(0),
688                enable_calls: AtomicUsize::new(0),
689                enable_entered: Notify::new(),
690                fail_enable: AtomicBool::new(false),
691                enable_release: Notify::new(),
692                enabled_configs: StdMutex::new(Vec::new()),
693            }
694        }
695
696        fn block_next_enable(&self) {
697            self.block_enable.store(true, Ordering::Release);
698        }
699
700        fn block_next_cleanup(&self) {
701            self.block_cleanup.store(true, Ordering::Release);
702        }
703
704        fn fail_next_enable(&self) {
705            self.fail_enable.store(true, Ordering::Release);
706        }
707
708        fn last_enabled_config(&self) -> Option<Config> {
709            self.enabled_configs
710                .lock()
711                .unwrap_or_else(|err| err.into_inner())
712                .last()
713                .cloned()
714        }
715    }
716
717    #[async_trait]
718    impl LifecycleRuntime for TestRuntime {
719        async fn cache_config(&self, config: Config) {
720            *self.cached_config.lock().unwrap_or_else(|err| err.into_inner()) = config;
721        }
722
723        async fn enable_targets(
724            &self,
725            config: &Config,
726            _cancellation: &CancellationToken,
727            publication_gate: Arc<StdMutex<()>>,
728            is_current: CurrentGeneration,
729            on_committed: CommitGeneration,
730        ) -> Result<RuntimeApply, NotificationError> {
731            self.enable_calls.fetch_add(1, Ordering::SeqCst);
732            if self.block_enable.swap(false, Ordering::AcqRel) {
733                self.enable_entered.notify_one();
734                self.enable_release.notified().await;
735            }
736            if self.fail_enable.swap(false, Ordering::AcqRel) {
737                return Err(NotificationError::Initialization("forced enable failure".to_string()));
738            }
739            let _publication_guard = publication_gate.lock().unwrap_or_else(|err| err.into_inner());
740            if !is_current() {
741                return Ok(RuntimeApply::complete());
742            }
743            self.enabled_configs
744                .lock()
745                .unwrap_or_else(|err| err.into_inner())
746                .push(config.clone());
747            on_committed(true);
748            if self.block_cleanup.swap(false, Ordering::AcqRel) {
749                let cleanup_entered = self.cleanup_entered.clone();
750                let cleanup_release = self.cleanup_release.clone();
751                return Ok(RuntimeApply::with_cleanup(Box::pin(async move {
752                    cleanup_entered.notify_one();
753                    cleanup_release.notified().await;
754                    Ok(())
755                })));
756            }
757            Ok(RuntimeApply::complete())
758        }
759
760        async fn disable_targets(
761            &self,
762            publication_gate: Arc<StdMutex<()>>,
763            is_current: CurrentGeneration,
764            on_committed: CommitGeneration,
765        ) -> Result<RuntimeApply, NotificationError> {
766            let _publication_guard = publication_gate.lock().unwrap_or_else(|err| err.into_inner());
767            if !is_current() {
768                return Ok(RuntimeApply::complete());
769            }
770            self.disable_calls.fetch_add(1, Ordering::SeqCst);
771            on_committed(true);
772            Ok(RuntimeApply::complete())
773        }
774    }
775
776    fn config_with_target(name: &str) -> Config {
777        let mut config = Config::new();
778        config
779            .0
780            .entry("notify_webhook".to_string())
781            .or_default()
782            .insert(name.to_string(), KVS::default());
783        config
784    }
785
786    fn config_with_enabled_test_target(target_type: &str, name: &str) -> Config {
787        let mut target = KVS::new();
788        target.insert(ENABLE_KEY.to_string(), "on".to_string());
789        let mut config = Config::new();
790        config
791            .0
792            .entry(format!("notify_{target_type}"))
793            .or_default()
794            .insert(name.to_string(), target);
795        config
796    }
797
798    fn real_runtime(
799        initial_config: Config,
800        registry: Arc<TargetRegistry>,
801        handoff_observer: Option<Arc<dyn Fn() + Send + Sync>>,
802    ) -> (NotifyLifecycleCoordinator, NotifyRuntimeView, Arc<RwLock<ReplayWorkerManager>>) {
803        let metrics = Arc::new(NotificationMetrics::new());
804        let rule_engine = NotifyRuleEngine::new();
805        let notifier = Arc::new(EventNotifier::new(metrics.clone(), rule_engine));
806        let target_list = notifier.target_list();
807        let replay_workers = Arc::new(RwLock::new(ReplayWorkerManager::new()));
808        let runtime_view = NotifyRuntimeView::new(target_list.clone(), replay_workers.clone());
809        let runtime_facade = NotifyRuntimeFacade::new_with_dispatch_gate(
810            target_list,
811            replay_workers.clone(),
812            notifier.dispatch_gate(),
813            Arc::new(Semaphore::new(4)),
814            metrics,
815        );
816        let config = Arc::new(RwLock::new(initial_config.clone()));
817        let runtime = Arc::new(DefaultLifecycleRuntime {
818            config,
819            registry,
820            runtime_facade,
821            handoff_observer,
822        });
823        (
824            NotifyLifecycleCoordinator::with_runtime(initial_config, runtime),
825            runtime_view,
826            replay_workers,
827        )
828    }
829
830    fn blocking_init_runtime() -> (
831        Arc<Notify>,
832        MockTarget,
833        NotifyLifecycleCoordinator,
834        NotifyRuntimeView,
835        Arc<AtomicUsize>,
836        Config,
837    ) {
838        // One observed template feeds every factory-constructed instance, so the returned
839        // observer sees the shared init signal and close counter across generations.
840        let init_entered = Arc::new(Notify::new());
841        let template = MockTarget::new("primary", BLOCKING_INIT_TARGET_TYPE).with_blocking_init(init_entered.clone());
842        let observer = template.clone();
843        let mut plugins = TargetPluginRegistry::<Event>::new();
844        plugins.register(TargetPluginDescriptor::new(
845            BLOCKING_INIT_TARGET_TYPE,
846            &[ENABLE_KEY],
847            |_config| Ok(()),
848            move |id, _config| Ok(Box::new(template.clone().with_id(&id, BLOCKING_INIT_TARGET_TYPE))),
849        ));
850        let config = config_with_enabled_test_target(BLOCKING_INIT_TARGET_TYPE, "primary");
851        let handoff_count = Arc::new(AtomicUsize::new(0));
852        let observer_count = handoff_count.clone();
853        let (coordinator, runtime_view, _) = real_runtime(
854            config.clone(),
855            Arc::new(TargetRegistry::with_plugins(plugins)),
856            Some(Arc::new(move || {
857                observer_count.fetch_add(1, Ordering::SeqCst);
858            })),
859        );
860        (init_entered, observer, coordinator, runtime_view, handoff_count, config)
861    }
862
863    async fn recv_until_generation(receiver: &mut mpsc::UnboundedReceiver<usize>, expected: usize, context: &str) {
864        loop {
865            let generation = receiver.recv().await.expect(context);
866            if generation == expected {
867                return;
868            }
869        }
870    }
871
872    fn coordinator(runtime: Arc<TestRuntime>, initial_config: Config) -> NotifyLifecycleCoordinator {
873        NotifyLifecycleCoordinator::with_runtime(initial_config, runtime)
874    }
875
876    #[tokio::test]
877    async fn slow_enable_then_disable_converges_to_live_only() {
878        let initial = config_with_target("initial");
879        let runtime = Arc::new(TestRuntime::new(initial.clone()));
880        runtime.block_next_enable();
881        let coordinator = coordinator(runtime.clone(), initial);
882
883        let enable = coordinator.set_mode(true, None);
884        runtime.enable_entered.notified().await;
885        let disable = coordinator.set_mode(false, None);
886        runtime.enable_release.notify_one();
887
888        enable.wait().await.expect("enable transition should finish");
889        disable.wait().await.expect("disable transition should finish");
890
891        assert_eq!(coordinator.state(), NotificationRuntimeState::LiveOnly);
892        assert_eq!(runtime.enable_calls.load(Ordering::SeqCst), 1);
893        assert_eq!(runtime.disable_calls.load(Ordering::SeqCst), 1);
894        assert_eq!(runtime.last_enabled_config(), None, "superseded enable must not publish targets");
895    }
896
897    #[tokio::test]
898    async fn real_runtime_disable_supersedes_target_init_and_leaves_no_runtime_state() {
899        let (init_entered, observer, coordinator, runtime_view, handoff_count, config) = blocking_init_runtime();
900
901        let enable = coordinator.set_mode(true, Some(config));
902        init_entered.notified().await;
903        let disable = coordinator.set_mode(false, None);
904
905        enable.wait().await.expect("superseded real enable should finish");
906        disable.wait().await.expect("real disable should converge");
907
908        let status = runtime_view.runtime_status_snapshot().await;
909        assert_eq!(status.target_count, 0);
910        assert_eq!(status.replay_worker_count, 0);
911        assert_eq!(observer.close_call_count(), 1);
912        assert_eq!(handoff_count.load(Ordering::SeqCst), 0);
913        assert_eq!(coordinator.state(), NotificationRuntimeState::LiveOnly);
914    }
915
916    #[tokio::test]
917    async fn real_runtime_disable_publishes_live_only_after_replay_join() {
918        let (coordinator, _, replay_workers) = real_runtime(Config::new(), Arc::new(TargetRegistry::new()), None);
919        coordinator
920            .set_mode(true, Some(Config::new()))
921            .wait()
922            .await
923            .expect("empty target runtime should enable");
924
925        let (cancel_tx, mut cancel_rx) = mpsc::channel(1);
926        let stop_entered = Arc::new(Notify::new());
927        let stop_release = Arc::new(Notify::new());
928        let worker_entered = stop_entered.clone();
929        let worker_release = stop_release.clone();
930        let join = tokio::spawn(async move {
931            cancel_rx.recv().await.expect("worker should receive cancellation");
932            worker_entered.notify_one();
933            worker_release.notified().await;
934        });
935        replay_workers
936            .write()
937            .await
938            .insert_with_handle("blocking-replay".to_string(), cancel_tx, join);
939
940        let disable = coordinator.set_mode(false, None);
941        stop_entered.notified().await;
942        assert_eq!(
943            coordinator.state(),
944            NotificationRuntimeState::TargetsEnabled { generation: 1 },
945            "LiveOnly must not publish while an old replay worker is still running"
946        );
947
948        stop_release.notify_one();
949        disable.wait().await.expect("disable should finish after replay joins");
950        assert_eq!(coordinator.state(), NotificationRuntimeState::LiveOnly);
951    }
952
953    #[tokio::test]
954    async fn real_runtime_terminate_cancels_target_init_and_is_final() {
955        let (init_entered, observer, coordinator, runtime_view, handoff_count, config) = blocking_init_runtime();
956
957        let enable = coordinator.set_mode(true, Some(config));
958        init_entered.notified().await;
959        let terminate = coordinator.terminate();
960
961        enable.wait().await.expect("superseded real enable should finish");
962        terminate.wait().await.expect("real terminate should converge");
963
964        let status = runtime_view.runtime_status_snapshot().await;
965        assert_eq!(status.target_count, 0);
966        assert_eq!(status.replay_worker_count, 0);
967        assert_eq!(observer.close_call_count(), 1);
968        assert_eq!(handoff_count.load(Ordering::SeqCst), 0);
969        assert_eq!(coordinator.state(), NotificationRuntimeState::Terminated);
970        let err = coordinator
971            .set_mode(false, None)
972            .wait()
973            .await
974            .expect_err("terminated runtime cannot be restarted");
975        assert!(matches!(
976            err,
977            NotificationError::Initialization(detail) if detail == "Notification runtime has terminated"
978        ));
979    }
980
981    #[tokio::test]
982    async fn partial_activation_reports_error_and_same_config_can_retry() {
983        // The template's single-failure init budget is shared by every clone the factory hands
984        // out, so the "bad" instance fails once in the first generation and recovers on retry.
985        let bad_template = MockTarget::new("bad", RETRY_INIT_TARGET_TYPE).with_init_failures(1);
986        let mut plugins = TargetPluginRegistry::<Event>::new();
987        plugins.register(TargetPluginDescriptor::new(
988            RETRY_INIT_TARGET_TYPE,
989            &[ENABLE_KEY],
990            |_config| Ok(()),
991            move |id, _config| {
992                Ok(Box::new(if id == "bad" {
993                    bad_template.clone()
994                } else {
995                    MockTarget::new(&id, RETRY_INIT_TARGET_TYPE)
996                }))
997            },
998        ));
999        let mut config = config_with_enabled_test_target(RETRY_INIT_TARGET_TYPE, "good");
1000        let mut bad = KVS::new();
1001        bad.insert(ENABLE_KEY.to_string(), "on".to_string());
1002        config
1003            .0
1004            .get_mut(&format!("notify_{RETRY_INIT_TARGET_TYPE}"))
1005            .expect("test target subsystem should exist")
1006            .insert("bad".to_string(), bad);
1007        let (coordinator, runtime_view, _) = real_runtime(config.clone(), Arc::new(TargetRegistry::with_plugins(plugins)), None);
1008
1009        coordinator
1010            .set_mode(true, Some(config.clone()))
1011            .wait()
1012            .await
1013            .expect_err("the originating transition must report the failed target");
1014        let partial = runtime_view.runtime_status_snapshot().await;
1015        assert_eq!(partial.target_count, 1, "the healthy target should remain fault-isolated and active");
1016        assert!(!coordinator.is_converged(), "a partial generation must remain retryable");
1017        assert_eq!(coordinator.state(), NotificationRuntimeState::TargetsEnabled { generation: 1 });
1018
1019        coordinator
1020            .update_config(config)
1021            .wait()
1022            .await
1023            .expect("the same persisted config should recover after the transient failure");
1024        let recovered = runtime_view.runtime_status_snapshot().await;
1025        assert_eq!(recovered.target_count, 2);
1026        assert!(coordinator.is_converged());
1027        assert_eq!(coordinator.state(), NotificationRuntimeState::TargetsEnabled { generation: 2 });
1028
1029        coordinator
1030            .set_mode(false, None)
1031            .wait()
1032            .await
1033            .expect("test runtime should shut down");
1034    }
1035
1036    #[tokio::test(start_paused = true)]
1037    async fn real_runtime_reload_never_overlaps_replay_workers_or_redelivers_queued_payload() {
1038        let queue_root = tempfile::tempdir().expect("queue root should be created");
1039        let queue_dir = queue_root.path().join("shared-queue");
1040        let body = br#"{"x":1}"#.to_vec();
1041        let seed_store = QueueStore::<QueuedPayload>::new_with_compression(&queue_dir, 16, ".event", false);
1042        seed_store.open().expect("seed queue should open");
1043        let meta = QueuedPayloadMeta::new(
1044            EventName::ObjectCreatedPut,
1045            "bucket-a".to_string(),
1046            "obj.txt".to_string(),
1047            "application/json",
1048            body.len(),
1049        );
1050        let payload = QueuedPayload::new(meta, body.clone());
1051        seed_store
1052            .put_raw(&payload.encode().expect("queued payload should encode"))
1053            .expect("queued payload should be seeded");
1054        assert_eq!(seed_store.len(), 1);
1055        drop(seed_store);
1056
1057        let (init_tx, mut init_rx) = mpsc::unbounded_channel();
1058        let (list_tx, mut list_rx) = mpsc::unbounded_channel();
1059        let (send_tx, mut send_rx) = mpsc::unbounded_channel();
1060        let state = Arc::new(ReplayState {
1061            active_workers: AtomicUsize::new(0),
1062            blocked_init_generation: 3,
1063            blocked_init_release: Notify::new(),
1064            completed_deliveries: AtomicUsize::new(0),
1065            created_targets: AtomicUsize::new(0),
1066            delivery_attempts: AtomicUsize::new(0),
1067            delivery_release: Notify::new(),
1068            delivered_bodies: StdMutex::new(Vec::new()),
1069            init_tx,
1070            list_tx,
1071            peak_workers: AtomicUsize::new(0),
1072            send_tx,
1073        });
1074        let mut plugins = TargetPluginRegistry::<Event>::new();
1075        let factory_state = state.clone();
1076        let factory_queue_dir = queue_dir.clone();
1077        plugins.register(TargetPluginDescriptor::new(
1078            REPLAY_TARGET_TYPE,
1079            &[ENABLE_KEY],
1080            |_config| Ok(()),
1081            move |id, _config| {
1082                let generation = factory_state.created_targets.fetch_add(1, Ordering::SeqCst) + 1;
1083                Ok(Box::new(ReplayTarget {
1084                    id: TargetID::new(id, REPLAY_TARGET_TYPE.to_string()),
1085                    generation,
1086                    state: factory_state.clone(),
1087                    store: ObservedQueueStore {
1088                        generation,
1089                        inner: QueueStore::new_with_compression(&factory_queue_dir, 16, ".event", false),
1090                        state: factory_state.clone(),
1091                        _worker_lease: None,
1092                    },
1093                }))
1094            },
1095        ));
1096
1097        let (handoff_tx, mut handoff_rx) = mpsc::unbounded_channel();
1098        let handoff_count = Arc::new(AtomicUsize::new(0));
1099        let observer_count = handoff_count.clone();
1100        let handoff_observer: Arc<dyn Fn() + Send + Sync> = Arc::new(move || {
1101            let generation = observer_count.fetch_add(1, Ordering::SeqCst) + 1;
1102            let _ = handoff_tx.send(generation);
1103        });
1104        let config = config_with_enabled_test_target(REPLAY_TARGET_TYPE, "primary");
1105        let (coordinator, runtime_view, _) =
1106            real_runtime(config.clone(), Arc::new(TargetRegistry::with_plugins(plugins)), Some(handoff_observer));
1107
1108        coordinator
1109            .set_mode(true, Some(config.clone()))
1110            .wait()
1111            .await
1112            .expect("initial real runtime enable should succeed");
1113        assert_eq!(init_rx.recv().await.expect("first target init should be observed"), 1);
1114        assert_eq!(handoff_rx.recv().await.expect("first handoff should be observed"), 1);
1115        recv_until_generation(&mut list_rx, 1, "first replay worker should scan the queue").await;
1116
1117        let status = runtime_view.runtime_status_snapshot().await;
1118        assert_eq!(status.target_count, 1);
1119        assert_eq!(status.replay_worker_count, 1);
1120        assert_eq!(state.active_workers.load(Ordering::SeqCst), 1);
1121        assert_eq!(state.peak_workers.load(Ordering::SeqCst), 1);
1122
1123        tokio::time::advance(std::time::Duration::from_secs(6)).await;
1124        assert_eq!(send_rx.recv().await.expect("seeded payload delivery should start"), 1);
1125
1126        let second_generation = coordinator.update_config(config.clone());
1127        assert_eq!(init_rx.recv().await.expect("replacement target init should be observed"), 2);
1128        assert_eq!(handoff_rx.recv().await.expect("replacement handoff should be observed"), 2);
1129
1130        let status_during_handoff = runtime_view.runtime_status_snapshot().await;
1131        assert_eq!(status_during_handoff.target_count, 1);
1132        assert_eq!(status_during_handoff.replay_worker_count, 0);
1133        assert_eq!(state.active_workers.load(Ordering::SeqCst), 1);
1134        assert_eq!(state.peak_workers.load(Ordering::SeqCst), 1);
1135
1136        let third_generation = coordinator.update_config(config);
1137        state.delivery_release.notify_one();
1138        second_generation
1139            .wait()
1140            .await
1141            .expect("second generation should commit after crossing the handoff barrier");
1142        assert_eq!(init_rx.recv().await.expect("third target init should be observed"), 3);
1143        assert_eq!(coordinator.state(), NotificationRuntimeState::TargetsEnabled { generation: 2 });
1144        recv_until_generation(&mut list_rx, 2, "second-generation replay worker should scan the queue").await;
1145
1146        let status_after_second_generation = runtime_view.runtime_status_snapshot().await;
1147        assert_eq!(status_after_second_generation.target_count, 1);
1148        assert_eq!(status_after_second_generation.replay_worker_count, 1);
1149        assert_eq!(state.active_workers.load(Ordering::SeqCst), 1);
1150        assert_eq!(state.peak_workers.load(Ordering::SeqCst), 1);
1151
1152        state.blocked_init_release.notify_one();
1153        assert_eq!(handoff_rx.recv().await.expect("third handoff should be observed"), 3);
1154        third_generation
1155            .wait()
1156            .await
1157            .expect("third generation should converge after the committed second generation");
1158        recv_until_generation(&mut list_rx, 3, "third-generation replay worker should scan the queue").await;
1159
1160        assert_eq!(state.delivery_attempts.load(Ordering::SeqCst), 1);
1161        assert_eq!(state.completed_deliveries.load(Ordering::SeqCst), 1);
1162        assert_eq!(state.peak_workers.load(Ordering::SeqCst), 1);
1163        assert_eq!(state.active_workers.load(Ordering::SeqCst), 1);
1164        assert_eq!(*state.delivered_bodies.lock().unwrap_or_else(|err| err.into_inner()), vec![body]);
1165        let status_after_reload = runtime_view.runtime_status_snapshot().await;
1166        assert_eq!(status_after_reload.target_count, 1);
1167        assert_eq!(status_after_reload.replay_worker_count, 1);
1168        assert_eq!(coordinator.state(), NotificationRuntimeState::TargetsEnabled { generation: 3 });
1169
1170        tokio::time::advance(std::time::Duration::from_secs(6)).await;
1171        recv_until_generation(&mut list_rx, 3, "third-generation replay worker should rescan after the batch timeout").await;
1172        assert!(matches!(send_rx.try_recv(), Err(mpsc::error::TryRecvError::Empty)));
1173
1174        coordinator
1175            .set_mode(false, None)
1176            .wait()
1177            .await
1178            .expect("real runtime disable should stop the replacement worker");
1179        let stopped = runtime_view.runtime_status_snapshot().await;
1180        assert_eq!(stopped.target_count, 0);
1181        assert_eq!(stopped.replay_worker_count, 0);
1182        assert_eq!(state.active_workers.load(Ordering::SeqCst), 0);
1183
1184        let verify_store = QueueStore::<QueuedPayload>::new_with_compression(&queue_dir, 16, ".event", false);
1185        verify_store.open().expect("queue should reopen after runtime shutdown");
1186        assert_eq!(verify_store.len(), 0);
1187    }
1188
1189    #[tokio::test]
1190    async fn later_config_generation_wins_after_slow_reload() {
1191        let first = config_with_target("first");
1192        let second = config_with_target("second");
1193        let runtime = Arc::new(TestRuntime::new(first.clone()));
1194        runtime.block_next_enable();
1195        let coordinator = coordinator(runtime.clone(), first.clone());
1196
1197        let enable = coordinator.set_mode(true, Some(first));
1198        runtime.enable_entered.notified().await;
1199        let reload = coordinator.update_config(second.clone());
1200        runtime.enable_release.notify_one();
1201
1202        enable.wait().await.expect("first generation should finish");
1203        reload.wait().await.expect("latest generation should finish");
1204
1205        assert_eq!(runtime.last_enabled_config(), Some(second));
1206        assert_eq!(coordinator.state(), NotificationRuntimeState::TargetsEnabled { generation: 2 });
1207    }
1208
1209    #[tokio::test]
1210    async fn accepted_transition_survives_caller_cancellation() {
1211        let initial = config_with_target("initial");
1212        let runtime = Arc::new(TestRuntime::new(initial.clone()));
1213        runtime.block_next_enable();
1214        let coordinator = coordinator(runtime.clone(), initial);
1215
1216        let receipt = coordinator.set_mode(true, None);
1217        runtime.enable_entered.notified().await;
1218        drop(receipt);
1219        runtime.enable_release.notify_one();
1220
1221        let apply_guard = coordinator.inner.apply_lock.lock().await;
1222        drop(apply_guard);
1223        assert!(matches!(coordinator.state(), NotificationRuntimeState::TargetsEnabled { generation: 1 }));
1224    }
1225
1226    #[tokio::test]
1227    async fn disabled_config_update_is_cached_until_reenabled() {
1228        let initial = config_with_target("initial");
1229        let latest = config_with_target("latest");
1230        let runtime = Arc::new(TestRuntime::new(initial.clone()));
1231        let coordinator = coordinator(runtime.clone(), initial);
1232
1233        coordinator
1234            .update_config(latest.clone())
1235            .wait()
1236            .await
1237            .expect("disabled update should be cached");
1238        assert_eq!(runtime.enable_calls.load(Ordering::SeqCst), 0);
1239        assert_eq!(coordinator.state(), NotificationRuntimeState::LiveOnly);
1240
1241        coordinator
1242            .set_mode(true, None)
1243            .wait()
1244            .await
1245            .expect("re-enable should use cached config");
1246        assert_eq!(runtime.last_enabled_config(), Some(latest));
1247    }
1248
1249    #[tokio::test]
1250    async fn concurrent_enable_publications_activate_once() {
1251        let initial = config_with_target("initial");
1252        let runtime = Arc::new(TestRuntime::new(initial.clone()));
1253        let coordinator = coordinator(runtime.clone(), initial);
1254
1255        let apply_guard = coordinator.inner.apply_lock.lock().await;
1256        let first = coordinator.set_mode(true, None);
1257        let second = coordinator.set_mode(true, None);
1258        drop(apply_guard);
1259
1260        first.wait().await.expect("first enable should converge");
1261        second.wait().await.expect("second enable should converge");
1262        assert_eq!(runtime.enable_calls.load(Ordering::SeqCst), 1);
1263        assert!(matches!(coordinator.state(), NotificationRuntimeState::TargetsEnabled { generation: 2 }));
1264    }
1265
1266    #[tokio::test]
1267    async fn committed_generation_cleanup_does_not_block_a_newer_generation() {
1268        let initial = config_with_target("initial");
1269        let latest = config_with_target("latest");
1270        let runtime = Arc::new(TestRuntime::new(initial.clone()));
1271        runtime.block_next_cleanup();
1272        let coordinator = coordinator(runtime.clone(), initial);
1273
1274        let first = coordinator.set_mode(true, None);
1275        runtime.cleanup_entered.notified().await;
1276
1277        coordinator
1278            .update_config(latest.clone())
1279            .wait()
1280            .await
1281            .expect("newer generation should not wait for old cleanup");
1282        assert_eq!(runtime.enable_calls.load(Ordering::SeqCst), 2);
1283        assert_eq!(runtime.last_enabled_config(), Some(latest));
1284
1285        runtime.cleanup_release.notify_one();
1286        first.wait().await.expect("old generation cleanup should finish");
1287    }
1288
1289    #[tokio::test]
1290    async fn superseded_receipt_does_not_retry_a_failed_latest_generation() {
1291        let initial = config_with_target("initial");
1292        let latest = config_with_target("latest");
1293        let runtime = Arc::new(TestRuntime::new(initial.clone()));
1294        let coordinator = coordinator(runtime.clone(), initial);
1295
1296        let apply_guard = coordinator.inner.apply_lock.lock().await;
1297        let superseded = coordinator.set_mode(true, None);
1298        tokio::task::yield_now().await;
1299        let latest_transition = coordinator.update_config(latest.clone());
1300        tokio::task::yield_now().await;
1301        runtime.fail_next_enable();
1302        drop(apply_guard);
1303
1304        superseded
1305            .wait()
1306            .await
1307            .expect("a superseded generation should not apply the latest intent");
1308        latest_transition
1309            .wait()
1310            .await
1311            .expect_err("the latest generation must own its apply error");
1312        assert_eq!(runtime.enable_calls.load(Ordering::SeqCst), 1);
1313        assert!(!coordinator.is_converged());
1314
1315        coordinator
1316            .update_config(latest.clone())
1317            .wait()
1318            .await
1319            .expect("an explicit publication should retry the failed configuration");
1320        assert_eq!(runtime.enable_calls.load(Ordering::SeqCst), 2);
1321        assert_eq!(runtime.last_enabled_config(), Some(latest));
1322    }
1323
1324    #[tokio::test]
1325    async fn failed_generation_can_retry_the_same_config() {
1326        let initial = config_with_target("initial");
1327        let latest = config_with_target("latest");
1328        let runtime = Arc::new(TestRuntime::new(initial.clone()));
1329        let coordinator = coordinator(runtime.clone(), initial);
1330
1331        coordinator
1332            .set_mode(true, None)
1333            .wait()
1334            .await
1335            .expect("initial enable should succeed");
1336        runtime.fail_next_enable();
1337        coordinator
1338            .update_config(latest.clone())
1339            .wait()
1340            .await
1341            .expect_err("forced reload should fail");
1342        assert!(!coordinator.is_converged());
1343
1344        coordinator
1345            .update_config(latest.clone())
1346            .wait()
1347            .await
1348            .expect("same config should be retryable");
1349        assert!(coordinator.is_converged());
1350        assert_eq!(runtime.last_enabled_config(), Some(latest));
1351    }
1352
1353    #[tokio::test]
1354    async fn suspend_is_restartable_but_terminate_is_final() {
1355        let initial = config_with_target("initial");
1356        let runtime = Arc::new(TestRuntime::new(initial.clone()));
1357        let coordinator = coordinator(runtime, initial);
1358
1359        coordinator.set_mode(true, None).wait().await.expect("enable should succeed");
1360        coordinator
1361            .set_mode(false, None)
1362            .wait()
1363            .await
1364            .expect("suspend should succeed");
1365        coordinator
1366            .set_mode(true, None)
1367            .wait()
1368            .await
1369            .expect("suspended runtime should restart");
1370        coordinator.terminate().wait().await.expect("terminate should succeed");
1371        coordinator.terminate().wait().await.expect("terminate should be idempotent");
1372
1373        let err = coordinator
1374            .set_mode(true, None)
1375            .wait()
1376            .await
1377            .expect_err("terminated runtime must reject restart");
1378        assert!(matches!(
1379            err,
1380            NotificationError::Initialization(detail) if detail == "Notification runtime has terminated"
1381        ));
1382        assert_eq!(coordinator.state(), NotificationRuntimeState::Terminated);
1383    }
1384
1385    #[tokio::test]
1386    async fn terminate_supersedes_blocked_enable_before_commit() {
1387        let initial = config_with_target("initial");
1388        let runtime = Arc::new(TestRuntime::new(initial.clone()));
1389        runtime.block_next_enable();
1390        let coordinator = coordinator(runtime.clone(), initial);
1391
1392        let enable = coordinator.set_mode(true, None);
1393        runtime.enable_entered.notified().await;
1394        let terminate = coordinator.terminate();
1395        runtime.enable_release.notify_one();
1396
1397        enable.wait().await.expect("superseded enable task should finish");
1398        terminate.wait().await.expect("terminate should finish");
1399        assert_eq!(runtime.last_enabled_config(), None);
1400        assert_eq!(coordinator.state(), NotificationRuntimeState::Terminated);
1401    }
1402
1403    #[test]
1404    fn publication_without_tokio_runtime_does_not_mutate_desired_state() {
1405        let initial = config_with_target("initial");
1406        let latest = config_with_target("latest");
1407        let runtime = Arc::new(TestRuntime::new(initial.clone()));
1408        let coordinator = coordinator(runtime, initial.clone());
1409
1410        let _transition = coordinator.update_config(latest);
1411        let desired = coordinator.inner.desired.lock().unwrap_or_else(|err| err.into_inner());
1412        assert_eq!(desired.generation, 0);
1413        assert_eq!(desired.config, initial);
1414    }
1415}