1use crate::{
16 Event, NotificationError, config_manager::notify_configuration_hint, error::transition_join_error, registry::TargetRegistry,
17 runtime_facade::NotifyRuntimeFacade,
18};
19use async_trait::async_trait;
20use rustfs_config::server_config::Config;
21use rustfs_targets::Target;
22use std::future::Future;
23use std::pin::Pin;
24use std::sync::atomic::{AtomicU64, Ordering};
25use std::sync::{Arc, Mutex as StdMutex};
26use tokio::sync::{Mutex, RwLock};
27use tokio::task::JoinHandle;
28use tokio_util::sync::CancellationToken;
29use tracing::{debug, info};
30
31type CurrentGeneration = Arc<dyn Fn() -> bool + Send + Sync>;
32type CommitGeneration = Box<dyn FnOnce(bool) + Send>;
33type CleanupFuture = Pin<Box<dyn Future<Output = Result<(), NotificationError>> + Send>>;
34
35const LOG_COMPONENT_NOTIFY: &str = "notify";
36const LOG_SUBSYSTEM_LIFECYCLE: &str = "lifecycle";
37const EVENT_NOTIFY_RUNTIME_LIFECYCLE: &str = "notify_runtime_lifecycle";
38
39#[derive(Clone, Copy, Debug, PartialEq, Eq)]
40pub enum NotificationRuntimeState {
41 LiveOnly,
42 TargetsEnabled { generation: u64 },
43 Terminated,
44}
45
46#[derive(Clone, Copy, Debug, PartialEq, Eq)]
47enum DesiredMode {
48 LiveOnly,
49 TargetsEnabled,
50 Terminated,
51}
52
53#[derive(Clone)]
54struct DesiredState {
55 cancellation: CancellationToken,
56 config: Config,
57 generation: u64,
58 mode: DesiredMode,
59}
60
61struct RuntimeApply {
62 cleanup: Option<CleanupFuture>,
63}
64
65impl RuntimeApply {
66 fn complete() -> Self {
67 Self { cleanup: None }
68 }
69
70 fn with_cleanup(cleanup: CleanupFuture) -> Self {
71 Self { cleanup: Some(cleanup) }
72 }
73
74 async fn finish(self) -> Result<(), NotificationError> {
75 match self.cleanup {
76 Some(cleanup) => cleanup.await,
77 None => Ok(()),
78 }
79 }
80}
81
82enum TransitionResult {
83 Ready(Result<(), NotificationError>),
84 Running(JoinHandle<Result<(), NotificationError>>),
85}
86
87pub struct NotificationLifecycleTransition {
88 result: TransitionResult,
89}
90
91impl NotificationLifecycleTransition {
92 fn ready(result: Result<(), NotificationError>) -> Self {
93 Self {
94 result: TransitionResult::Ready(result),
95 }
96 }
97
98 fn running(handle: JoinHandle<Result<(), NotificationError>>) -> Self {
99 Self {
100 result: TransitionResult::Running(handle),
101 }
102 }
103
104 pub async fn wait(self) -> Result<(), NotificationError> {
105 match self.result {
106 TransitionResult::Ready(result) => result,
107 TransitionResult::Running(handle) => handle.await.map_err(transition_join_error)?,
108 }
109 }
110}
111
112#[async_trait]
113trait LifecycleRuntime: Send + Sync {
114 async fn cache_config(&self, config: Config);
115
116 async fn enable_targets(
117 &self,
118 config: &Config,
119 cancellation: &CancellationToken,
120 publication_gate: Arc<StdMutex<()>>,
121 is_current: CurrentGeneration,
122 on_committed: CommitGeneration,
123 ) -> Result<RuntimeApply, NotificationError>;
124
125 async fn disable_targets(
126 &self,
127 publication_gate: Arc<StdMutex<()>>,
128 is_current: CurrentGeneration,
129 on_committed: CommitGeneration,
130 ) -> Result<RuntimeApply, NotificationError>;
131}
132
133struct DefaultLifecycleRuntime {
134 config: Arc<RwLock<Config>>,
135 registry: Arc<TargetRegistry>,
136 runtime_facade: NotifyRuntimeFacade,
137 #[cfg(test)]
138 handoff_observer: Option<Arc<dyn Fn() + Send + Sync>>,
139}
140
141#[async_trait]
142impl LifecycleRuntime for DefaultLifecycleRuntime {
143 async fn cache_config(&self, config: Config) {
144 *self.config.write().await = config;
145 }
146
147 async fn enable_targets(
148 &self,
149 config: &Config,
150 cancellation: &CancellationToken,
151 publication_gate: Arc<StdMutex<()>>,
152 is_current: CurrentGeneration,
153 on_committed: CommitGeneration,
154 ) -> Result<RuntimeApply, NotificationError> {
155 info!(
156 event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
157 component = LOG_COMPONENT_NOTIFY,
158 subsystem = LOG_SUBSYSTEM_LIFECYCLE,
159 state = "preparing",
160 "notify runtime lifecycle"
161 );
162
163 let (targets, creation_failures): (Vec<Box<dyn Target<Event> + Send + Sync>>, Vec<String>) =
164 self.registry.create_dormant_targets_from_config(config).await?;
165 if targets.is_empty() {
166 debug!(
167 event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
168 component = LOG_COMPONENT_NOTIFY,
169 subsystem = LOG_SUBSYSTEM_LIFECYCLE,
170 state = "idle",
171 reason = "no_targets_configured",
172 hint = %notify_configuration_hint(),
173 "notify runtime lifecycle"
174 );
175 }
176
177 let mut prepared = self.runtime_facade.prepare_targets(targets, cancellation).await;
178 prepared.extend_creation_failures(creation_failures);
179 if !is_current() {
180 let runtime_facade = self.runtime_facade.clone();
181 return Ok(RuntimeApply::with_cleanup(Box::pin(async move {
182 runtime_facade.close_prepared(prepared).await
183 })));
184 }
185
186 let dispatch_guard = self.runtime_facade.pause_dispatch().await;
192 {
193 let _publication_guard = publication_gate.lock().unwrap_or_else(|err| err.into_inner());
194 if !is_current() {
195 drop(_publication_guard);
196 drop(dispatch_guard);
197 let runtime_facade = self.runtime_facade.clone();
198 return Ok(RuntimeApply::with_cleanup(Box::pin(async move {
199 runtime_facade.close_prepared(prepared).await
200 })));
201 }
202 }
203
204 #[cfg(test)]
205 if let Some(observer) = &self.handoff_observer {
206 observer();
207 }
208
209 self.runtime_facade.stop_active_replay_workers().await;
210 let (opened, rejected) = self.runtime_facade.open_prepared_stores(prepared).await?;
211 let open_complete = rejected.failure_summary().is_none();
212 let (detached, activation_rejected) = self
215 .runtime_facade
216 .commit_prepared(opened, move |activation_complete| on_committed(open_complete && activation_complete))
217 .await;
218 let activation_failures = [rejected.failure_summary(), activation_rejected.failure_summary()]
219 .into_iter()
220 .flatten()
221 .collect::<Vec<_>>();
222 drop(dispatch_guard);
223 let runtime_facade = self.runtime_facade.clone();
224
225 info!(
226 event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
227 component = LOG_COMPONENT_NOTIFY,
228 subsystem = LOG_SUBSYSTEM_LIFECYCLE,
229 state = "enabled",
230 "notify runtime lifecycle"
231 );
232 Ok(RuntimeApply::with_cleanup(Box::pin(async move {
233 let (close_old, close_rejected, close_activation_rejected) = tokio::join!(
234 runtime_facade.close_detached_targets(detached),
235 runtime_facade.close_prepared(rejected),
236 runtime_facade.close_prepared(activation_rejected),
237 );
238 close_old?;
239 close_rejected?;
240 close_activation_rejected?;
241 if !activation_failures.is_empty() {
242 return Err(NotificationError::Initialization(format!(
243 "one or more notification targets failed to activate: {}",
244 activation_failures.join("; ")
245 )));
246 }
247 Ok(())
248 })))
249 }
250
251 async fn disable_targets(
252 &self,
253 publication_gate: Arc<StdMutex<()>>,
254 is_current: CurrentGeneration,
255 on_committed: CommitGeneration,
256 ) -> Result<RuntimeApply, NotificationError> {
257 let dispatch_guard = self.runtime_facade.pause_dispatch().await;
258 {
259 let _publication_guard = publication_gate.lock().unwrap_or_else(|err| err.into_inner());
260 if !is_current() {
261 drop(_publication_guard);
262 drop(dispatch_guard);
263 return Ok(RuntimeApply::complete());
264 }
265 }
266
267 self.runtime_facade.stop_active_replay_workers().await;
272 let detached = self.runtime_facade.commit_disabled(move || on_committed(true)).await;
273 drop(dispatch_guard);
274 let runtime_facade = self.runtime_facade.clone();
275 Ok(RuntimeApply::with_cleanup(Box::pin(async move {
276 runtime_facade.close_detached_targets(detached).await
277 })))
278 }
279}
280
281struct LifecycleCoordinatorInner {
282 applied_generation: AtomicU64,
283 apply_lock: Mutex<()>,
284 desired: StdMutex<DesiredState>,
285 publication_gate: Arc<StdMutex<()>>,
286 runtime: Arc<dyn LifecycleRuntime>,
287 state: StdMutex<NotificationRuntimeState>,
288}
289
290#[derive(Clone)]
291pub(crate) struct NotifyLifecycleCoordinator {
292 inner: Arc<LifecycleCoordinatorInner>,
293}
294
295impl NotifyLifecycleCoordinator {
296 pub(crate) fn new(config: Arc<RwLock<Config>>, registry: Arc<TargetRegistry>, runtime_facade: NotifyRuntimeFacade) -> Self {
297 let runtime = Arc::new(DefaultLifecycleRuntime {
298 config,
299 registry,
300 runtime_facade,
301 #[cfg(test)]
302 handoff_observer: None,
303 });
304 Self::with_runtime(Config::new(), runtime)
305 }
306
307 fn with_runtime(initial_config: Config, runtime: Arc<dyn LifecycleRuntime>) -> Self {
308 Self {
309 inner: Arc::new(LifecycleCoordinatorInner {
310 applied_generation: AtomicU64::new(0),
311 apply_lock: Mutex::new(()),
312 desired: StdMutex::new(DesiredState {
313 cancellation: CancellationToken::new(),
314 config: initial_config,
315 generation: 0,
316 mode: DesiredMode::LiveOnly,
317 }),
318 publication_gate: Arc::new(StdMutex::new(())),
319 runtime,
320 state: StdMutex::new(NotificationRuntimeState::LiveOnly),
321 }),
322 }
323 }
324
325 pub(crate) fn set_mode(&self, enabled: bool, config: Option<Config>) -> NotificationLifecycleTransition {
326 self.accept(false, move |desired| {
327 desired.mode = if enabled {
328 DesiredMode::TargetsEnabled
329 } else {
330 DesiredMode::LiveOnly
331 };
332 if let Some(config) = config {
333 desired.config = config;
334 }
335 Ok(true)
336 })
337 }
338
339 pub(crate) fn update_config(&self, config: Config) -> NotificationLifecycleTransition {
340 self.accept(false, move |desired| {
341 desired.config = config;
342 Ok(true)
343 })
344 }
345
346 pub(crate) fn terminate(&self) -> NotificationLifecycleTransition {
347 self.accept(true, |desired| {
348 if desired.mode == DesiredMode::Terminated {
349 return Ok(false);
350 }
351 desired.mode = DesiredMode::Terminated;
352 Ok(true)
353 })
354 }
355
356 pub(crate) fn state(&self) -> NotificationRuntimeState {
357 *self.inner.state.lock().unwrap_or_else(|err| err.into_inner())
358 }
359
360 pub(crate) fn is_converged(&self) -> bool {
361 let desired_generation = self.inner.desired.lock().unwrap_or_else(|err| err.into_inner()).generation;
362 self.inner.applied_generation.load(Ordering::Acquire) >= desired_generation
363 }
364
365 fn accept<F>(&self, allow_terminated: bool, update: F) -> NotificationLifecycleTransition
366 where
367 F: FnOnce(&mut DesiredState) -> Result<bool, NotificationError>,
368 {
369 let Ok(runtime) = tokio::runtime::Handle::try_current() else {
370 return NotificationLifecycleTransition::ready(Err(NotificationError::Initialization(
371 "Notification lifecycle coordinator is unavailable".to_string(),
372 )));
373 };
374 let _publication_guard = self.inner.publication_gate.lock().unwrap_or_else(|err| err.into_inner());
375 let mut desired = self.inner.desired.lock().unwrap_or_else(|err| err.into_inner());
376 if desired.mode == DesiredMode::Terminated && !allow_terminated {
377 return NotificationLifecycleTransition::ready(Err(NotificationError::Initialization(
378 "Notification runtime has terminated".to_string(),
379 )));
380 }
381
382 let previous = desired.clone();
383 let changed = match update(&mut desired) {
384 Ok(changed) => changed,
385 Err(err) => return NotificationLifecycleTransition::ready(Err(err)),
386 };
387 if changed {
388 let Some(next_generation) = desired.generation.checked_add(1) else {
389 *desired = previous;
390 return NotificationLifecycleTransition::ready(Err(NotificationError::Initialization(
391 "Notification lifecycle generation is exhausted".to_string(),
392 )));
393 };
394 desired.generation = next_generation;
395 previous.cancellation.cancel();
396 desired.cancellation = CancellationToken::new();
397 }
398 let accepted = desired.clone();
399 drop(desired);
400 drop(_publication_guard);
401 let inner = Arc::clone(&self.inner);
402 NotificationLifecycleTransition::running(runtime.spawn(async move { inner.converge(accepted).await }))
403 }
404}
405
406impl LifecycleCoordinatorInner {
407 async fn converge(self: &Arc<Self>, desired: DesiredState) -> Result<(), NotificationError> {
408 let _apply_guard = self.apply_lock.lock().await;
409 if self.applied_generation.load(Ordering::Acquire) >= desired.generation {
410 return Ok(());
411 }
412 if !self.current_matches(desired.generation, desired.mode) {
413 return Ok(());
414 }
415
416 self.runtime.cache_config(desired.config.clone()).await;
417 let current_inner = Arc::clone(self);
418 let generation = desired.generation;
419 let mode = desired.mode;
420 let is_current: CurrentGeneration = Arc::new(move || current_inner.current_matches(generation, mode));
421 let publication_gate = self.publication_gate.clone();
422
423 let state = match desired.mode {
424 DesiredMode::LiveOnly => NotificationRuntimeState::LiveOnly,
425 DesiredMode::TargetsEnabled => NotificationRuntimeState::TargetsEnabled {
426 generation: desired.generation,
427 },
428 DesiredMode::Terminated => NotificationRuntimeState::Terminated,
429 };
430 let committed_inner = Arc::clone(self);
431 let on_committed: CommitGeneration =
432 Box::new(move |complete| committed_inner.mark_published(generation, state, complete));
433
434 let applied = match desired.mode {
435 DesiredMode::LiveOnly | DesiredMode::Terminated => {
436 self.runtime
437 .disable_targets(publication_gate, is_current, on_committed)
438 .await?
439 }
440 DesiredMode::TargetsEnabled => {
441 self.runtime
442 .enable_targets(&desired.config, &desired.cancellation, publication_gate, is_current, on_committed)
443 .await?
444 }
445 };
446 drop(_apply_guard);
447 applied.finish().await
448 }
449
450 fn current_matches(&self, generation: u64, mode: DesiredMode) -> bool {
451 let desired = self.desired.lock().unwrap_or_else(|err| err.into_inner());
452 desired.generation == generation && desired.mode == mode
453 }
454
455 fn mark_published(&self, generation: u64, state: NotificationRuntimeState, complete: bool) {
456 *self.state.lock().unwrap_or_else(|err| err.into_inner()) = state;
457 if complete {
458 self.applied_generation.store(generation, Ordering::Release);
459 }
460 }
461}
462
463#[cfg(test)]
464mod tests {
465 use super::{
466 CommitGeneration, CurrentGeneration, DefaultLifecycleRuntime, LifecycleRuntime, NotificationRuntimeState,
467 NotifyLifecycleCoordinator, RuntimeApply,
468 };
469 use crate::{
470 Event, NotificationError, integration::NotificationMetrics, notifier::EventNotifier, registry::TargetRegistry,
471 rule_engine::NotifyRuleEngine, runtime_facade::NotifyRuntimeFacade, runtime_view::NotifyRuntimeView,
472 };
473 use async_trait::async_trait;
474 use rustfs_config::ENABLE_KEY;
475 use rustfs_config::server_config::{Config, KVS};
476 use rustfs_targets::arn::TargetID;
477 use rustfs_targets::store::{Key, QueueStore, Store};
478 use rustfs_targets::target::{EntityTarget, QueuedPayload, QueuedPayloadMeta};
479 use rustfs_targets::testkit::MockTarget;
480 use rustfs_targets::{
481 EventName, ReplayWorkerManager, StoreError, Target, TargetError, TargetPluginDescriptor, TargetPluginRegistry,
482 };
483 use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
484 use std::sync::{Arc, Mutex as StdMutex};
485 use tokio::sync::{Notify, RwLock, Semaphore, mpsc};
486 use tokio_util::sync::CancellationToken;
487
488 const BLOCKING_INIT_TARGET_TYPE: &str = "lifecycle_blocking_init";
489 const RETRY_INIT_TARGET_TYPE: &str = "lifecycle_retry_init";
490 const REPLAY_TARGET_TYPE: &str = "lifecycle_replay";
491
492 struct ReplayState {
493 active_workers: AtomicUsize,
494 completed_deliveries: AtomicUsize,
495 created_targets: AtomicUsize,
496 delivery_attempts: AtomicUsize,
497 delivery_release: Notify,
498 delivered_bodies: StdMutex<Vec<Vec<u8>>>,
499 blocked_init_generation: usize,
500 blocked_init_release: Notify,
501 init_tx: mpsc::UnboundedSender<usize>,
502 list_tx: mpsc::UnboundedSender<usize>,
503 peak_workers: AtomicUsize,
504 send_tx: mpsc::UnboundedSender<usize>,
505 }
506
507 struct ReplayWorkerLease {
508 state: Arc<ReplayState>,
509 }
510
511 impl ReplayWorkerLease {
512 fn acquire(state: Arc<ReplayState>) -> Self {
513 let active = state.active_workers.fetch_add(1, Ordering::SeqCst) + 1;
514 let mut peak = state.peak_workers.load(Ordering::SeqCst);
515 while active > peak {
516 match state
517 .peak_workers
518 .compare_exchange_weak(peak, active, Ordering::SeqCst, Ordering::SeqCst)
519 {
520 Ok(_) => break,
521 Err(observed) => peak = observed,
522 }
523 }
524 Self { state }
525 }
526 }
527
528 impl Drop for ReplayWorkerLease {
529 fn drop(&mut self) {
530 self.state.active_workers.fetch_sub(1, Ordering::SeqCst);
531 }
532 }
533
534 #[derive(Clone)]
535 struct ObservedQueueStore {
536 generation: usize,
537 inner: QueueStore<QueuedPayload>,
538 state: Arc<ReplayState>,
539 _worker_lease: Option<Arc<ReplayWorkerLease>>,
540 }
541
542 impl Store<QueuedPayload> for ObservedQueueStore {
543 type Error = StoreError;
544 type Key = Key;
545
546 fn open(&self) -> Result<(), Self::Error> {
547 self.inner.open()
548 }
549
550 fn put(&self, item: Arc<QueuedPayload>) -> Result<Self::Key, Self::Error> {
551 self.inner.put(item)
552 }
553
554 fn put_multiple(&self, items: Vec<QueuedPayload>) -> Result<Self::Key, Self::Error> {
555 self.inner.put_multiple(items)
556 }
557
558 fn put_raw(&self, data: &[u8]) -> Result<Self::Key, Self::Error> {
559 self.inner.put_raw(data)
560 }
561
562 fn get(&self, key: &Self::Key) -> Result<QueuedPayload, Self::Error> {
563 self.inner.get(key)
564 }
565
566 fn get_multiple(&self, key: &Self::Key) -> Result<Vec<QueuedPayload>, Self::Error> {
567 self.inner.get_multiple(key)
568 }
569
570 fn get_raw(&self, key: &Self::Key) -> Result<Vec<u8>, Self::Error> {
571 self.inner.get_raw(key)
572 }
573
574 fn del(&self, key: &Self::Key) -> Result<(), Self::Error> {
575 self.inner.del(key)
576 }
577
578 fn delete(&self) -> Result<(), Self::Error> {
579 self.inner.delete()
580 }
581
582 fn list(&self) -> Vec<Self::Key> {
583 let _ = self.state.list_tx.send(self.generation);
584 self.inner.list()
585 }
586
587 fn len(&self) -> usize {
588 self.inner.len()
589 }
590
591 fn is_empty(&self) -> bool {
592 self.inner.is_empty()
593 }
594
595 fn boxed_clone(&self) -> Box<dyn Store<QueuedPayload, Error = Self::Error, Key = Self::Key> + Send + Sync> {
596 Box::new(Self {
597 generation: self.generation,
598 inner: self.inner.clone(),
599 state: self.state.clone(),
600 _worker_lease: Some(Arc::new(ReplayWorkerLease::acquire(self.state.clone()))),
601 })
602 }
603 }
604
605 #[derive(Clone)]
606 struct ReplayTarget {
607 id: TargetID,
608 generation: usize,
609 state: Arc<ReplayState>,
610 store: ObservedQueueStore,
611 }
612
613 #[async_trait]
614 impl Target<Event> for ReplayTarget {
615 fn id(&self) -> TargetID {
616 self.id.clone()
617 }
618
619 async fn is_active(&self) -> Result<bool, TargetError> {
620 Ok(true)
621 }
622
623 async fn save(&self, _event: Arc<EntityTarget<Event>>) -> Result<(), TargetError> {
624 Ok(())
625 }
626
627 async fn send_raw_from_store(&self, _key: Key, body: Vec<u8>, _meta: QueuedPayloadMeta) -> Result<(), TargetError> {
628 let attempt = self.state.delivery_attempts.fetch_add(1, Ordering::SeqCst) + 1;
629 let _ = self.state.send_tx.send(attempt);
630 self.state.delivery_release.notified().await;
631 self.state
632 .delivered_bodies
633 .lock()
634 .unwrap_or_else(|err| err.into_inner())
635 .push(body);
636 self.state.completed_deliveries.fetch_add(1, Ordering::SeqCst);
637 Ok(())
638 }
639
640 async fn close(&self) -> Result<(), TargetError> {
641 Ok(())
642 }
643
644 fn store(&self) -> Option<&(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync)> {
645 Some(&self.store)
646 }
647
648 fn clone_dyn(&self) -> Box<dyn Target<Event> + Send + Sync> {
649 Box::new(self.clone())
650 }
651
652 async fn init(&self) -> Result<(), TargetError> {
653 let _ = self.state.init_tx.send(self.generation);
654 if self.generation == self.state.blocked_init_generation {
655 self.state.blocked_init_release.notified().await;
656 }
657 Ok(())
658 }
659
660 fn is_enabled(&self) -> bool {
661 true
662 }
663 }
664
665 struct TestRuntime {
666 block_cleanup: AtomicBool,
667 block_enable: AtomicBool,
668 cached_config: StdMutex<Config>,
669 cleanup_entered: Arc<Notify>,
670 cleanup_release: Arc<Notify>,
671 disable_calls: AtomicUsize,
672 enable_calls: AtomicUsize,
673 enable_entered: Notify,
674 fail_enable: AtomicBool,
675 enable_release: Notify,
676 enabled_configs: StdMutex<Vec<Config>>,
677 }
678
679 impl TestRuntime {
680 fn new(initial_config: Config) -> Self {
681 Self {
682 block_cleanup: AtomicBool::new(false),
683 block_enable: AtomicBool::new(false),
684 cached_config: StdMutex::new(initial_config),
685 cleanup_entered: Arc::new(Notify::new()),
686 cleanup_release: Arc::new(Notify::new()),
687 disable_calls: AtomicUsize::new(0),
688 enable_calls: AtomicUsize::new(0),
689 enable_entered: Notify::new(),
690 fail_enable: AtomicBool::new(false),
691 enable_release: Notify::new(),
692 enabled_configs: StdMutex::new(Vec::new()),
693 }
694 }
695
696 fn block_next_enable(&self) {
697 self.block_enable.store(true, Ordering::Release);
698 }
699
700 fn block_next_cleanup(&self) {
701 self.block_cleanup.store(true, Ordering::Release);
702 }
703
704 fn fail_next_enable(&self) {
705 self.fail_enable.store(true, Ordering::Release);
706 }
707
708 fn last_enabled_config(&self) -> Option<Config> {
709 self.enabled_configs
710 .lock()
711 .unwrap_or_else(|err| err.into_inner())
712 .last()
713 .cloned()
714 }
715 }
716
717 #[async_trait]
718 impl LifecycleRuntime for TestRuntime {
719 async fn cache_config(&self, config: Config) {
720 *self.cached_config.lock().unwrap_or_else(|err| err.into_inner()) = config;
721 }
722
723 async fn enable_targets(
724 &self,
725 config: &Config,
726 _cancellation: &CancellationToken,
727 publication_gate: Arc<StdMutex<()>>,
728 is_current: CurrentGeneration,
729 on_committed: CommitGeneration,
730 ) -> Result<RuntimeApply, NotificationError> {
731 self.enable_calls.fetch_add(1, Ordering::SeqCst);
732 if self.block_enable.swap(false, Ordering::AcqRel) {
733 self.enable_entered.notify_one();
734 self.enable_release.notified().await;
735 }
736 if self.fail_enable.swap(false, Ordering::AcqRel) {
737 return Err(NotificationError::Initialization("forced enable failure".to_string()));
738 }
739 let _publication_guard = publication_gate.lock().unwrap_or_else(|err| err.into_inner());
740 if !is_current() {
741 return Ok(RuntimeApply::complete());
742 }
743 self.enabled_configs
744 .lock()
745 .unwrap_or_else(|err| err.into_inner())
746 .push(config.clone());
747 on_committed(true);
748 if self.block_cleanup.swap(false, Ordering::AcqRel) {
749 let cleanup_entered = self.cleanup_entered.clone();
750 let cleanup_release = self.cleanup_release.clone();
751 return Ok(RuntimeApply::with_cleanup(Box::pin(async move {
752 cleanup_entered.notify_one();
753 cleanup_release.notified().await;
754 Ok(())
755 })));
756 }
757 Ok(RuntimeApply::complete())
758 }
759
760 async fn disable_targets(
761 &self,
762 publication_gate: Arc<StdMutex<()>>,
763 is_current: CurrentGeneration,
764 on_committed: CommitGeneration,
765 ) -> Result<RuntimeApply, NotificationError> {
766 let _publication_guard = publication_gate.lock().unwrap_or_else(|err| err.into_inner());
767 if !is_current() {
768 return Ok(RuntimeApply::complete());
769 }
770 self.disable_calls.fetch_add(1, Ordering::SeqCst);
771 on_committed(true);
772 Ok(RuntimeApply::complete())
773 }
774 }
775
776 fn config_with_target(name: &str) -> Config {
777 let mut config = Config::new();
778 config
779 .0
780 .entry("notify_webhook".to_string())
781 .or_default()
782 .insert(name.to_string(), KVS::default());
783 config
784 }
785
786 fn config_with_enabled_test_target(target_type: &str, name: &str) -> Config {
787 let mut target = KVS::new();
788 target.insert(ENABLE_KEY.to_string(), "on".to_string());
789 let mut config = Config::new();
790 config
791 .0
792 .entry(format!("notify_{target_type}"))
793 .or_default()
794 .insert(name.to_string(), target);
795 config
796 }
797
798 fn real_runtime(
799 initial_config: Config,
800 registry: Arc<TargetRegistry>,
801 handoff_observer: Option<Arc<dyn Fn() + Send + Sync>>,
802 ) -> (NotifyLifecycleCoordinator, NotifyRuntimeView, Arc<RwLock<ReplayWorkerManager>>) {
803 let metrics = Arc::new(NotificationMetrics::new());
804 let rule_engine = NotifyRuleEngine::new();
805 let notifier = Arc::new(EventNotifier::new(metrics.clone(), rule_engine));
806 let target_list = notifier.target_list();
807 let replay_workers = Arc::new(RwLock::new(ReplayWorkerManager::new()));
808 let runtime_view = NotifyRuntimeView::new(target_list.clone(), replay_workers.clone());
809 let runtime_facade = NotifyRuntimeFacade::new_with_dispatch_gate(
810 target_list,
811 replay_workers.clone(),
812 notifier.dispatch_gate(),
813 Arc::new(Semaphore::new(4)),
814 metrics,
815 );
816 let config = Arc::new(RwLock::new(initial_config.clone()));
817 let runtime = Arc::new(DefaultLifecycleRuntime {
818 config,
819 registry,
820 runtime_facade,
821 handoff_observer,
822 });
823 (
824 NotifyLifecycleCoordinator::with_runtime(initial_config, runtime),
825 runtime_view,
826 replay_workers,
827 )
828 }
829
830 fn blocking_init_runtime() -> (
831 Arc<Notify>,
832 MockTarget,
833 NotifyLifecycleCoordinator,
834 NotifyRuntimeView,
835 Arc<AtomicUsize>,
836 Config,
837 ) {
838 let init_entered = Arc::new(Notify::new());
841 let template = MockTarget::new("primary", BLOCKING_INIT_TARGET_TYPE).with_blocking_init(init_entered.clone());
842 let observer = template.clone();
843 let mut plugins = TargetPluginRegistry::<Event>::new();
844 plugins.register(TargetPluginDescriptor::new(
845 BLOCKING_INIT_TARGET_TYPE,
846 &[ENABLE_KEY],
847 |_config| Ok(()),
848 move |id, _config| Ok(Box::new(template.clone().with_id(&id, BLOCKING_INIT_TARGET_TYPE))),
849 ));
850 let config = config_with_enabled_test_target(BLOCKING_INIT_TARGET_TYPE, "primary");
851 let handoff_count = Arc::new(AtomicUsize::new(0));
852 let observer_count = handoff_count.clone();
853 let (coordinator, runtime_view, _) = real_runtime(
854 config.clone(),
855 Arc::new(TargetRegistry::with_plugins(plugins)),
856 Some(Arc::new(move || {
857 observer_count.fetch_add(1, Ordering::SeqCst);
858 })),
859 );
860 (init_entered, observer, coordinator, runtime_view, handoff_count, config)
861 }
862
863 async fn recv_until_generation(receiver: &mut mpsc::UnboundedReceiver<usize>, expected: usize, context: &str) {
864 loop {
865 let generation = receiver.recv().await.expect(context);
866 if generation == expected {
867 return;
868 }
869 }
870 }
871
872 fn coordinator(runtime: Arc<TestRuntime>, initial_config: Config) -> NotifyLifecycleCoordinator {
873 NotifyLifecycleCoordinator::with_runtime(initial_config, runtime)
874 }
875
876 #[tokio::test]
877 async fn slow_enable_then_disable_converges_to_live_only() {
878 let initial = config_with_target("initial");
879 let runtime = Arc::new(TestRuntime::new(initial.clone()));
880 runtime.block_next_enable();
881 let coordinator = coordinator(runtime.clone(), initial);
882
883 let enable = coordinator.set_mode(true, None);
884 runtime.enable_entered.notified().await;
885 let disable = coordinator.set_mode(false, None);
886 runtime.enable_release.notify_one();
887
888 enable.wait().await.expect("enable transition should finish");
889 disable.wait().await.expect("disable transition should finish");
890
891 assert_eq!(coordinator.state(), NotificationRuntimeState::LiveOnly);
892 assert_eq!(runtime.enable_calls.load(Ordering::SeqCst), 1);
893 assert_eq!(runtime.disable_calls.load(Ordering::SeqCst), 1);
894 assert_eq!(runtime.last_enabled_config(), None, "superseded enable must not publish targets");
895 }
896
897 #[tokio::test]
898 async fn real_runtime_disable_supersedes_target_init_and_leaves_no_runtime_state() {
899 let (init_entered, observer, coordinator, runtime_view, handoff_count, config) = blocking_init_runtime();
900
901 let enable = coordinator.set_mode(true, Some(config));
902 init_entered.notified().await;
903 let disable = coordinator.set_mode(false, None);
904
905 enable.wait().await.expect("superseded real enable should finish");
906 disable.wait().await.expect("real disable should converge");
907
908 let status = runtime_view.runtime_status_snapshot().await;
909 assert_eq!(status.target_count, 0);
910 assert_eq!(status.replay_worker_count, 0);
911 assert_eq!(observer.close_call_count(), 1);
912 assert_eq!(handoff_count.load(Ordering::SeqCst), 0);
913 assert_eq!(coordinator.state(), NotificationRuntimeState::LiveOnly);
914 }
915
916 #[tokio::test]
917 async fn real_runtime_disable_publishes_live_only_after_replay_join() {
918 let (coordinator, _, replay_workers) = real_runtime(Config::new(), Arc::new(TargetRegistry::new()), None);
919 coordinator
920 .set_mode(true, Some(Config::new()))
921 .wait()
922 .await
923 .expect("empty target runtime should enable");
924
925 let (cancel_tx, mut cancel_rx) = mpsc::channel(1);
926 let stop_entered = Arc::new(Notify::new());
927 let stop_release = Arc::new(Notify::new());
928 let worker_entered = stop_entered.clone();
929 let worker_release = stop_release.clone();
930 let join = tokio::spawn(async move {
931 cancel_rx.recv().await.expect("worker should receive cancellation");
932 worker_entered.notify_one();
933 worker_release.notified().await;
934 });
935 replay_workers
936 .write()
937 .await
938 .insert_with_handle("blocking-replay".to_string(), cancel_tx, join);
939
940 let disable = coordinator.set_mode(false, None);
941 stop_entered.notified().await;
942 assert_eq!(
943 coordinator.state(),
944 NotificationRuntimeState::TargetsEnabled { generation: 1 },
945 "LiveOnly must not publish while an old replay worker is still running"
946 );
947
948 stop_release.notify_one();
949 disable.wait().await.expect("disable should finish after replay joins");
950 assert_eq!(coordinator.state(), NotificationRuntimeState::LiveOnly);
951 }
952
953 #[tokio::test]
954 async fn real_runtime_terminate_cancels_target_init_and_is_final() {
955 let (init_entered, observer, coordinator, runtime_view, handoff_count, config) = blocking_init_runtime();
956
957 let enable = coordinator.set_mode(true, Some(config));
958 init_entered.notified().await;
959 let terminate = coordinator.terminate();
960
961 enable.wait().await.expect("superseded real enable should finish");
962 terminate.wait().await.expect("real terminate should converge");
963
964 let status = runtime_view.runtime_status_snapshot().await;
965 assert_eq!(status.target_count, 0);
966 assert_eq!(status.replay_worker_count, 0);
967 assert_eq!(observer.close_call_count(), 1);
968 assert_eq!(handoff_count.load(Ordering::SeqCst), 0);
969 assert_eq!(coordinator.state(), NotificationRuntimeState::Terminated);
970 let err = coordinator
971 .set_mode(false, None)
972 .wait()
973 .await
974 .expect_err("terminated runtime cannot be restarted");
975 assert!(matches!(
976 err,
977 NotificationError::Initialization(detail) if detail == "Notification runtime has terminated"
978 ));
979 }
980
981 #[tokio::test]
982 async fn partial_activation_reports_error_and_same_config_can_retry() {
983 let bad_template = MockTarget::new("bad", RETRY_INIT_TARGET_TYPE).with_init_failures(1);
986 let mut plugins = TargetPluginRegistry::<Event>::new();
987 plugins.register(TargetPluginDescriptor::new(
988 RETRY_INIT_TARGET_TYPE,
989 &[ENABLE_KEY],
990 |_config| Ok(()),
991 move |id, _config| {
992 Ok(Box::new(if id == "bad" {
993 bad_template.clone()
994 } else {
995 MockTarget::new(&id, RETRY_INIT_TARGET_TYPE)
996 }))
997 },
998 ));
999 let mut config = config_with_enabled_test_target(RETRY_INIT_TARGET_TYPE, "good");
1000 let mut bad = KVS::new();
1001 bad.insert(ENABLE_KEY.to_string(), "on".to_string());
1002 config
1003 .0
1004 .get_mut(&format!("notify_{RETRY_INIT_TARGET_TYPE}"))
1005 .expect("test target subsystem should exist")
1006 .insert("bad".to_string(), bad);
1007 let (coordinator, runtime_view, _) = real_runtime(config.clone(), Arc::new(TargetRegistry::with_plugins(plugins)), None);
1008
1009 coordinator
1010 .set_mode(true, Some(config.clone()))
1011 .wait()
1012 .await
1013 .expect_err("the originating transition must report the failed target");
1014 let partial = runtime_view.runtime_status_snapshot().await;
1015 assert_eq!(partial.target_count, 1, "the healthy target should remain fault-isolated and active");
1016 assert!(!coordinator.is_converged(), "a partial generation must remain retryable");
1017 assert_eq!(coordinator.state(), NotificationRuntimeState::TargetsEnabled { generation: 1 });
1018
1019 coordinator
1020 .update_config(config)
1021 .wait()
1022 .await
1023 .expect("the same persisted config should recover after the transient failure");
1024 let recovered = runtime_view.runtime_status_snapshot().await;
1025 assert_eq!(recovered.target_count, 2);
1026 assert!(coordinator.is_converged());
1027 assert_eq!(coordinator.state(), NotificationRuntimeState::TargetsEnabled { generation: 2 });
1028
1029 coordinator
1030 .set_mode(false, None)
1031 .wait()
1032 .await
1033 .expect("test runtime should shut down");
1034 }
1035
1036 #[tokio::test(start_paused = true)]
1037 async fn real_runtime_reload_never_overlaps_replay_workers_or_redelivers_queued_payload() {
1038 let queue_root = tempfile::tempdir().expect("queue root should be created");
1039 let queue_dir = queue_root.path().join("shared-queue");
1040 let body = br#"{"x":1}"#.to_vec();
1041 let seed_store = QueueStore::<QueuedPayload>::new_with_compression(&queue_dir, 16, ".event", false);
1042 seed_store.open().expect("seed queue should open");
1043 let meta = QueuedPayloadMeta::new(
1044 EventName::ObjectCreatedPut,
1045 "bucket-a".to_string(),
1046 "obj.txt".to_string(),
1047 "application/json",
1048 body.len(),
1049 );
1050 let payload = QueuedPayload::new(meta, body.clone());
1051 seed_store
1052 .put_raw(&payload.encode().expect("queued payload should encode"))
1053 .expect("queued payload should be seeded");
1054 assert_eq!(seed_store.len(), 1);
1055 drop(seed_store);
1056
1057 let (init_tx, mut init_rx) = mpsc::unbounded_channel();
1058 let (list_tx, mut list_rx) = mpsc::unbounded_channel();
1059 let (send_tx, mut send_rx) = mpsc::unbounded_channel();
1060 let state = Arc::new(ReplayState {
1061 active_workers: AtomicUsize::new(0),
1062 blocked_init_generation: 3,
1063 blocked_init_release: Notify::new(),
1064 completed_deliveries: AtomicUsize::new(0),
1065 created_targets: AtomicUsize::new(0),
1066 delivery_attempts: AtomicUsize::new(0),
1067 delivery_release: Notify::new(),
1068 delivered_bodies: StdMutex::new(Vec::new()),
1069 init_tx,
1070 list_tx,
1071 peak_workers: AtomicUsize::new(0),
1072 send_tx,
1073 });
1074 let mut plugins = TargetPluginRegistry::<Event>::new();
1075 let factory_state = state.clone();
1076 let factory_queue_dir = queue_dir.clone();
1077 plugins.register(TargetPluginDescriptor::new(
1078 REPLAY_TARGET_TYPE,
1079 &[ENABLE_KEY],
1080 |_config| Ok(()),
1081 move |id, _config| {
1082 let generation = factory_state.created_targets.fetch_add(1, Ordering::SeqCst) + 1;
1083 Ok(Box::new(ReplayTarget {
1084 id: TargetID::new(id, REPLAY_TARGET_TYPE.to_string()),
1085 generation,
1086 state: factory_state.clone(),
1087 store: ObservedQueueStore {
1088 generation,
1089 inner: QueueStore::new_with_compression(&factory_queue_dir, 16, ".event", false),
1090 state: factory_state.clone(),
1091 _worker_lease: None,
1092 },
1093 }))
1094 },
1095 ));
1096
1097 let (handoff_tx, mut handoff_rx) = mpsc::unbounded_channel();
1098 let handoff_count = Arc::new(AtomicUsize::new(0));
1099 let observer_count = handoff_count.clone();
1100 let handoff_observer: Arc<dyn Fn() + Send + Sync> = Arc::new(move || {
1101 let generation = observer_count.fetch_add(1, Ordering::SeqCst) + 1;
1102 let _ = handoff_tx.send(generation);
1103 });
1104 let config = config_with_enabled_test_target(REPLAY_TARGET_TYPE, "primary");
1105 let (coordinator, runtime_view, _) =
1106 real_runtime(config.clone(), Arc::new(TargetRegistry::with_plugins(plugins)), Some(handoff_observer));
1107
1108 coordinator
1109 .set_mode(true, Some(config.clone()))
1110 .wait()
1111 .await
1112 .expect("initial real runtime enable should succeed");
1113 assert_eq!(init_rx.recv().await.expect("first target init should be observed"), 1);
1114 assert_eq!(handoff_rx.recv().await.expect("first handoff should be observed"), 1);
1115 recv_until_generation(&mut list_rx, 1, "first replay worker should scan the queue").await;
1116
1117 let status = runtime_view.runtime_status_snapshot().await;
1118 assert_eq!(status.target_count, 1);
1119 assert_eq!(status.replay_worker_count, 1);
1120 assert_eq!(state.active_workers.load(Ordering::SeqCst), 1);
1121 assert_eq!(state.peak_workers.load(Ordering::SeqCst), 1);
1122
1123 tokio::time::advance(std::time::Duration::from_secs(6)).await;
1124 assert_eq!(send_rx.recv().await.expect("seeded payload delivery should start"), 1);
1125
1126 let second_generation = coordinator.update_config(config.clone());
1127 assert_eq!(init_rx.recv().await.expect("replacement target init should be observed"), 2);
1128 assert_eq!(handoff_rx.recv().await.expect("replacement handoff should be observed"), 2);
1129
1130 let status_during_handoff = runtime_view.runtime_status_snapshot().await;
1131 assert_eq!(status_during_handoff.target_count, 1);
1132 assert_eq!(status_during_handoff.replay_worker_count, 0);
1133 assert_eq!(state.active_workers.load(Ordering::SeqCst), 1);
1134 assert_eq!(state.peak_workers.load(Ordering::SeqCst), 1);
1135
1136 let third_generation = coordinator.update_config(config);
1137 state.delivery_release.notify_one();
1138 second_generation
1139 .wait()
1140 .await
1141 .expect("second generation should commit after crossing the handoff barrier");
1142 assert_eq!(init_rx.recv().await.expect("third target init should be observed"), 3);
1143 assert_eq!(coordinator.state(), NotificationRuntimeState::TargetsEnabled { generation: 2 });
1144 recv_until_generation(&mut list_rx, 2, "second-generation replay worker should scan the queue").await;
1145
1146 let status_after_second_generation = runtime_view.runtime_status_snapshot().await;
1147 assert_eq!(status_after_second_generation.target_count, 1);
1148 assert_eq!(status_after_second_generation.replay_worker_count, 1);
1149 assert_eq!(state.active_workers.load(Ordering::SeqCst), 1);
1150 assert_eq!(state.peak_workers.load(Ordering::SeqCst), 1);
1151
1152 state.blocked_init_release.notify_one();
1153 assert_eq!(handoff_rx.recv().await.expect("third handoff should be observed"), 3);
1154 third_generation
1155 .wait()
1156 .await
1157 .expect("third generation should converge after the committed second generation");
1158 recv_until_generation(&mut list_rx, 3, "third-generation replay worker should scan the queue").await;
1159
1160 assert_eq!(state.delivery_attempts.load(Ordering::SeqCst), 1);
1161 assert_eq!(state.completed_deliveries.load(Ordering::SeqCst), 1);
1162 assert_eq!(state.peak_workers.load(Ordering::SeqCst), 1);
1163 assert_eq!(state.active_workers.load(Ordering::SeqCst), 1);
1164 assert_eq!(*state.delivered_bodies.lock().unwrap_or_else(|err| err.into_inner()), vec![body]);
1165 let status_after_reload = runtime_view.runtime_status_snapshot().await;
1166 assert_eq!(status_after_reload.target_count, 1);
1167 assert_eq!(status_after_reload.replay_worker_count, 1);
1168 assert_eq!(coordinator.state(), NotificationRuntimeState::TargetsEnabled { generation: 3 });
1169
1170 tokio::time::advance(std::time::Duration::from_secs(6)).await;
1171 recv_until_generation(&mut list_rx, 3, "third-generation replay worker should rescan after the batch timeout").await;
1172 assert!(matches!(send_rx.try_recv(), Err(mpsc::error::TryRecvError::Empty)));
1173
1174 coordinator
1175 .set_mode(false, None)
1176 .wait()
1177 .await
1178 .expect("real runtime disable should stop the replacement worker");
1179 let stopped = runtime_view.runtime_status_snapshot().await;
1180 assert_eq!(stopped.target_count, 0);
1181 assert_eq!(stopped.replay_worker_count, 0);
1182 assert_eq!(state.active_workers.load(Ordering::SeqCst), 0);
1183
1184 let verify_store = QueueStore::<QueuedPayload>::new_with_compression(&queue_dir, 16, ".event", false);
1185 verify_store.open().expect("queue should reopen after runtime shutdown");
1186 assert_eq!(verify_store.len(), 0);
1187 }
1188
1189 #[tokio::test]
1190 async fn later_config_generation_wins_after_slow_reload() {
1191 let first = config_with_target("first");
1192 let second = config_with_target("second");
1193 let runtime = Arc::new(TestRuntime::new(first.clone()));
1194 runtime.block_next_enable();
1195 let coordinator = coordinator(runtime.clone(), first.clone());
1196
1197 let enable = coordinator.set_mode(true, Some(first));
1198 runtime.enable_entered.notified().await;
1199 let reload = coordinator.update_config(second.clone());
1200 runtime.enable_release.notify_one();
1201
1202 enable.wait().await.expect("first generation should finish");
1203 reload.wait().await.expect("latest generation should finish");
1204
1205 assert_eq!(runtime.last_enabled_config(), Some(second));
1206 assert_eq!(coordinator.state(), NotificationRuntimeState::TargetsEnabled { generation: 2 });
1207 }
1208
1209 #[tokio::test]
1210 async fn accepted_transition_survives_caller_cancellation() {
1211 let initial = config_with_target("initial");
1212 let runtime = Arc::new(TestRuntime::new(initial.clone()));
1213 runtime.block_next_enable();
1214 let coordinator = coordinator(runtime.clone(), initial);
1215
1216 let receipt = coordinator.set_mode(true, None);
1217 runtime.enable_entered.notified().await;
1218 drop(receipt);
1219 runtime.enable_release.notify_one();
1220
1221 let apply_guard = coordinator.inner.apply_lock.lock().await;
1222 drop(apply_guard);
1223 assert!(matches!(coordinator.state(), NotificationRuntimeState::TargetsEnabled { generation: 1 }));
1224 }
1225
1226 #[tokio::test]
1227 async fn disabled_config_update_is_cached_until_reenabled() {
1228 let initial = config_with_target("initial");
1229 let latest = config_with_target("latest");
1230 let runtime = Arc::new(TestRuntime::new(initial.clone()));
1231 let coordinator = coordinator(runtime.clone(), initial);
1232
1233 coordinator
1234 .update_config(latest.clone())
1235 .wait()
1236 .await
1237 .expect("disabled update should be cached");
1238 assert_eq!(runtime.enable_calls.load(Ordering::SeqCst), 0);
1239 assert_eq!(coordinator.state(), NotificationRuntimeState::LiveOnly);
1240
1241 coordinator
1242 .set_mode(true, None)
1243 .wait()
1244 .await
1245 .expect("re-enable should use cached config");
1246 assert_eq!(runtime.last_enabled_config(), Some(latest));
1247 }
1248
1249 #[tokio::test]
1250 async fn concurrent_enable_publications_activate_once() {
1251 let initial = config_with_target("initial");
1252 let runtime = Arc::new(TestRuntime::new(initial.clone()));
1253 let coordinator = coordinator(runtime.clone(), initial);
1254
1255 let apply_guard = coordinator.inner.apply_lock.lock().await;
1256 let first = coordinator.set_mode(true, None);
1257 let second = coordinator.set_mode(true, None);
1258 drop(apply_guard);
1259
1260 first.wait().await.expect("first enable should converge");
1261 second.wait().await.expect("second enable should converge");
1262 assert_eq!(runtime.enable_calls.load(Ordering::SeqCst), 1);
1263 assert!(matches!(coordinator.state(), NotificationRuntimeState::TargetsEnabled { generation: 2 }));
1264 }
1265
1266 #[tokio::test]
1267 async fn committed_generation_cleanup_does_not_block_a_newer_generation() {
1268 let initial = config_with_target("initial");
1269 let latest = config_with_target("latest");
1270 let runtime = Arc::new(TestRuntime::new(initial.clone()));
1271 runtime.block_next_cleanup();
1272 let coordinator = coordinator(runtime.clone(), initial);
1273
1274 let first = coordinator.set_mode(true, None);
1275 runtime.cleanup_entered.notified().await;
1276
1277 coordinator
1278 .update_config(latest.clone())
1279 .wait()
1280 .await
1281 .expect("newer generation should not wait for old cleanup");
1282 assert_eq!(runtime.enable_calls.load(Ordering::SeqCst), 2);
1283 assert_eq!(runtime.last_enabled_config(), Some(latest));
1284
1285 runtime.cleanup_release.notify_one();
1286 first.wait().await.expect("old generation cleanup should finish");
1287 }
1288
1289 #[tokio::test]
1290 async fn superseded_receipt_does_not_retry_a_failed_latest_generation() {
1291 let initial = config_with_target("initial");
1292 let latest = config_with_target("latest");
1293 let runtime = Arc::new(TestRuntime::new(initial.clone()));
1294 let coordinator = coordinator(runtime.clone(), initial);
1295
1296 let apply_guard = coordinator.inner.apply_lock.lock().await;
1297 let superseded = coordinator.set_mode(true, None);
1298 tokio::task::yield_now().await;
1299 let latest_transition = coordinator.update_config(latest.clone());
1300 tokio::task::yield_now().await;
1301 runtime.fail_next_enable();
1302 drop(apply_guard);
1303
1304 superseded
1305 .wait()
1306 .await
1307 .expect("a superseded generation should not apply the latest intent");
1308 latest_transition
1309 .wait()
1310 .await
1311 .expect_err("the latest generation must own its apply error");
1312 assert_eq!(runtime.enable_calls.load(Ordering::SeqCst), 1);
1313 assert!(!coordinator.is_converged());
1314
1315 coordinator
1316 .update_config(latest.clone())
1317 .wait()
1318 .await
1319 .expect("an explicit publication should retry the failed configuration");
1320 assert_eq!(runtime.enable_calls.load(Ordering::SeqCst), 2);
1321 assert_eq!(runtime.last_enabled_config(), Some(latest));
1322 }
1323
1324 #[tokio::test]
1325 async fn failed_generation_can_retry_the_same_config() {
1326 let initial = config_with_target("initial");
1327 let latest = config_with_target("latest");
1328 let runtime = Arc::new(TestRuntime::new(initial.clone()));
1329 let coordinator = coordinator(runtime.clone(), initial);
1330
1331 coordinator
1332 .set_mode(true, None)
1333 .wait()
1334 .await
1335 .expect("initial enable should succeed");
1336 runtime.fail_next_enable();
1337 coordinator
1338 .update_config(latest.clone())
1339 .wait()
1340 .await
1341 .expect_err("forced reload should fail");
1342 assert!(!coordinator.is_converged());
1343
1344 coordinator
1345 .update_config(latest.clone())
1346 .wait()
1347 .await
1348 .expect("same config should be retryable");
1349 assert!(coordinator.is_converged());
1350 assert_eq!(runtime.last_enabled_config(), Some(latest));
1351 }
1352
1353 #[tokio::test]
1354 async fn suspend_is_restartable_but_terminate_is_final() {
1355 let initial = config_with_target("initial");
1356 let runtime = Arc::new(TestRuntime::new(initial.clone()));
1357 let coordinator = coordinator(runtime, initial);
1358
1359 coordinator.set_mode(true, None).wait().await.expect("enable should succeed");
1360 coordinator
1361 .set_mode(false, None)
1362 .wait()
1363 .await
1364 .expect("suspend should succeed");
1365 coordinator
1366 .set_mode(true, None)
1367 .wait()
1368 .await
1369 .expect("suspended runtime should restart");
1370 coordinator.terminate().wait().await.expect("terminate should succeed");
1371 coordinator.terminate().wait().await.expect("terminate should be idempotent");
1372
1373 let err = coordinator
1374 .set_mode(true, None)
1375 .wait()
1376 .await
1377 .expect_err("terminated runtime must reject restart");
1378 assert!(matches!(
1379 err,
1380 NotificationError::Initialization(detail) if detail == "Notification runtime has terminated"
1381 ));
1382 assert_eq!(coordinator.state(), NotificationRuntimeState::Terminated);
1383 }
1384
1385 #[tokio::test]
1386 async fn terminate_supersedes_blocked_enable_before_commit() {
1387 let initial = config_with_target("initial");
1388 let runtime = Arc::new(TestRuntime::new(initial.clone()));
1389 runtime.block_next_enable();
1390 let coordinator = coordinator(runtime.clone(), initial);
1391
1392 let enable = coordinator.set_mode(true, None);
1393 runtime.enable_entered.notified().await;
1394 let terminate = coordinator.terminate();
1395 runtime.enable_release.notify_one();
1396
1397 enable.wait().await.expect("superseded enable task should finish");
1398 terminate.wait().await.expect("terminate should finish");
1399 assert_eq!(runtime.last_enabled_config(), None);
1400 assert_eq!(coordinator.state(), NotificationRuntimeState::Terminated);
1401 }
1402
1403 #[test]
1404 fn publication_without_tokio_runtime_does_not_mutate_desired_state() {
1405 let initial = config_with_target("initial");
1406 let latest = config_with_target("latest");
1407 let runtime = Arc::new(TestRuntime::new(initial.clone()));
1408 let coordinator = coordinator(runtime, initial.clone());
1409
1410 let _transition = coordinator.update_config(latest);
1411 let desired = coordinator.inner.desired.lock().unwrap_or_else(|err| err.into_inner());
1412 assert_eq!(desired.generation, 0);
1413 assert_eq!(desired.config, initial);
1414 }
1415}