Skip to main content

open_net/module/net_status/inner/
shared.rs

1//! Engine-scoped network observation ownership. Consumers only retain weak
2//! service contexts and explicit, idempotently releasable demand leases.
3use std::collections::HashSet;
4use std::future::Future;
5use std::pin::Pin;
6use std::sync::{Arc, Mutex, Weak};
7
8use tokio::sync::{mpsc, oneshot, watch};
9
10use super::inner_net_status_client::InnerNetStatusClient;
11use super::network_status_snapshot::NetworkStatusSnapshot;
12use crate::common::CommonEngine;
13use crate::error::{ErrorKind, NetError, NetworkMonitorPhase};
14use crate::net_status::{MonitorState, NetworkSnapshot};
15
16/// A read-only handle to the network facts owned by one `OpenNet` engine.
17/// Cloning or reading it never starts monitoring or extends the engine lifetime.
18/// A context remains tied to its original engine after that engine is dropped.
19#[derive(Clone)]
20pub struct NetworkStatusContext {
21    service: Weak<SharedNetworkService>,
22    facts: watch::Receiver<NetworkSnapshot>,
23    failure: watch::Receiver<Option<NetError>>,
24    gate: watch::Receiver<NetworkStatusSnapshot>,
25}
26impl std::fmt::Debug for NetworkStatusContext {
27    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
28        f.debug_struct("NetworkStatusContext")
29            .finish_non_exhaustive()
30    }
31}
32impl NetworkStatusContext {
33    pub(super) fn name_service(&self) -> Result<Arc<super::names::NameService>, NetError> {
34        self.service
35            .upgrade()
36            .map(|service| service.names.clone())
37            .ok_or_else(|| NetError::from(ErrorKind::Closed))
38    }
39    pub(super) fn observation_source(
40        &self,
41    ) -> Result<super::network_status_snapshot::NetworkStatusSource, NetError> {
42        self.service
43            .upgrade()
44            .map(|service| service.inner.observation_source())
45            .ok_or_else(|| NetError::from(ErrorKind::Closed))
46    }
47    #[cfg(any(windows, test))]
48    pub(super) fn windows_reachability_sampler(
49        &self,
50    ) -> Result<Arc<super::refresh::WindowsSampler>, NetError> {
51        self.service
52            .upgrade()
53            .map(|service| service.inner.windows_reachability_sampler())
54            .ok_or_else(|| NetError::from(ErrorKind::Closed))
55    }
56    /// Read the cached observation without starting or retrying monitoring.
57    pub fn snapshot(&self) -> Result<NetworkSnapshot, NetError> {
58        if let Some(error) = ::tokio::sync::watch::Receiver::borrow(&self.failure).clone() {
59            return Err(error);
60        }
61        match self.service.upgrade() {
62            Some(service) => service.inner.snapshot(),
63            None => Ok(::tokio::sync::watch::Receiver::borrow(&self.facts).clone()),
64        }
65    }
66    pub(crate) fn common_engine(&self) -> Result<Arc<CommonEngine>, NetError> {
67        self.service
68            .upgrade()
69            .map(|service| service.engine.clone())
70            .ok_or_else(|| NetError::from(ErrorKind::Closed))
71    }
72    pub(crate) fn is_closed(&self) -> bool {
73        self.service.upgrade().is_none_or(|service| {
74            match service.admission.lock() {
75                Ok(state) => state.closed,
76                Err(_) => {
77                    crate::log_e!(crate::common::log::log_def::LogType::Engine; "network_service_state", "error", "admission_lock_poisoned");
78                    true
79                }
80            }
81        })
82    }
83    pub(crate) fn acquire(&self) -> Result<NetworkLease, NetError> {
84        let service = self
85            .service
86            .upgrade()
87            .ok_or_else(|| NetError::from(ErrorKind::Closed))?;
88        // A recoverable Failed snapshot remains valid for explicit retry. A
89        // permanent publication error cannot admit new, unusable demand.
90        service.inner.snapshot()?;
91        let id = {
92            let mut state = service.admission.lock().map_err(NetError::from_poison)?;
93            if state.closed {
94                return Err(NetError::from(ErrorKind::Closed));
95            }
96            let id = state
97                .next_id
98                .checked_add(1)
99                .ok_or_else(|| NetError::from(ErrorKind::Internal))?;
100            service
101                .commands
102                .send(Command::Acquire(id))
103                .map_err(|_| NetError::from(ErrorKind::RuntimeUnavailable))?;
104            state.next_id = id;
105            state.active.insert(id);
106            id
107        };
108        Ok(NetworkLease {
109            inner: Arc::new(LeaseState {
110                id,
111                service: Arc::downgrade(&service),
112                released: Mutex::new(None),
113                closed: service.closed.clone(),
114            }),
115        })
116    }
117    pub(crate) fn subscribe_facts(&self) -> watch::Receiver<NetworkSnapshot> {
118        self.facts.clone()
119    }
120    pub(crate) fn subscribe_failure(&self) -> watch::Receiver<Option<NetError>> {
121        self.failure.clone()
122    }
123    pub(crate) fn subscribe_gate(&self) -> watch::Receiver<NetworkStatusSnapshot> {
124        self.gate.clone()
125    }
126}
127
128#[derive(Clone)]
129pub(crate) struct CleanupTicket {
130    finished: watch::Receiver<Option<Result<(), NetError>>>,
131}
132impl CleanupTicket {
133    fn pending() -> (watch::Sender<Option<Result<(), NetError>>>, Self) {
134        let (sender, finished) = watch::channel(None);
135        (sender, Self { finished })
136    }
137    pub(crate) fn is_finished(&self) -> bool {
138        matches!(
139            *::tokio::sync::watch::Receiver::borrow(&self.finished),
140            Some(Ok(()))
141        )
142    }
143    pub(crate) async fn wait(&self) -> Result<(), NetError> {
144        let mut finished = self.finished.clone();
145        loop {
146            if let Some(result) = finished.borrow_and_update().clone() {
147                return result;
148            }
149            finished
150                .changed()
151                .await
152                .map_err(|_| NetError::from(ErrorKind::RuntimeUnavailable))?;
153        }
154    }
155}
156
157#[derive(Clone)]
158pub(crate) struct NetworkLease {
159    inner: Arc<LeaseState>,
160}
161struct LeaseState {
162    id: u64,
163    service: Weak<SharedNetworkService>,
164    released: Mutex<Option<CleanupTicket>>,
165    closed: CleanupTicket,
166}
167impl NetworkLease {
168    pub(crate) fn ensure_started(&self) -> Result<(), NetError> {
169        self.inner.ensure(None)
170    }
171    pub(crate) async fn wait_started(&self) -> Result<NetworkSnapshot, NetError> {
172        let (sender, received) = oneshot::channel();
173        self.inner.ensure(Some(sender))?;
174        received.await.map_err(|source| {
175            NetError::with_source(ErrorKind::RuntimeUnavailable, source)
176                .with_network_monitor_phase(NetworkMonitorPhase::InitialResultReceive)
177        })?
178    }
179    pub(crate) fn release(&self) -> CleanupTicket {
180        self.inner.release()
181    }
182}
183
184impl LeaseState {
185    fn ensure(
186        &self,
187        waiter: Option<oneshot::Sender<Result<NetworkSnapshot, NetError>>>,
188    ) -> Result<(), NetError> {
189        let released = self.released.lock().map_err(NetError::from_poison)?;
190        if released.is_some() {
191            return Err(NetError::from(ErrorKind::Closed));
192        }
193        let service = self
194            .service
195            .upgrade()
196            .ok_or_else(|| NetError::from(ErrorKind::Closed))?;
197        let state = service.admission.lock().map_err(NetError::from_poison)?;
198        if state.closed || !state.active.contains(&self.id) {
199            return Err(NetError::from(ErrorKind::Closed));
200        }
201        service
202            .commands
203            .send(Command::Ensure(self.id, waiter))
204            .map_err(|_| {
205                // A SendError owns the command, waiter and application state;
206                // retain the fixed failure location without retaining that payload.
207                NetError::from(ErrorKind::RuntimeUnavailable)
208                    .with_network_monitor_phase(NetworkMonitorPhase::MonitorTaskSubmit)
209            })
210    }
211    fn release(&self) -> CleanupTicket {
212        let mut released = match self.released.lock() {
213            Ok(state) => state,
214            Err(poisoned) => {
215                crate::log_e!(crate::common::log::log_def::LogType::Engine; "network_lease_release", "error", "poisoned_lease_recovered");
216                poisoned.into_inner()
217            }
218        };
219        if let Some(ticket) = released.as_ref() {
220            return ticket.clone();
221        }
222        let ticket = match self.service.upgrade() {
223            Some(service) => {
224                let mut admission = match service.admission.lock() {
225                    Ok(state) => state,
226                    Err(poisoned) => {
227                        crate::log_e!(crate::common::log::log_def::LogType::Engine; "network_lease_release", "error", "admission_lock_poisoned_recovered");
228                        poisoned.into_inner()
229                    }
230                };
231                admission.active.remove(&self.id);
232                if admission.closed {
233                    service.closed.clone()
234                } else {
235                    let (sender, ticket) = CleanupTicket::pending();
236                    if let Err(error) = service.commands.send(Command::Release(self.id, sender)) {
237                        if let Command::Release(_, sender) = error.0 {
238                            Completion::new(sender, &service.engine)
239                                .finish(Err(NetError::from(ErrorKind::RuntimeUnavailable)));
240                        }
241                    }
242                    ticket
243                }
244            }
245            None => self.closed.clone(),
246        };
247        *released = Some(ticket.clone());
248        ticket
249    }
250}
251impl Drop for LeaseState {
252    fn drop(&mut self) {
253        self.release();
254    }
255}
256
257struct Admission {
258    closed: bool,
259    next_id: u64,
260    active: HashSet<u64>,
261}
262pub(crate) struct SharedNetworkService {
263    engine: Arc<CommonEngine>,
264    inner: Arc<InnerNetStatusClient>,
265    names: Arc<super::names::NameService>,
266    admission: Mutex<Admission>,
267    commands: mpsc::UnboundedSender<Command>,
268    closed: CleanupTicket,
269}
270enum Command {
271    Acquire(u64),
272    Ensure(
273        u64,
274        Option<oneshot::Sender<Result<NetworkSnapshot, NetError>>>,
275    ),
276    Release(u64, watch::Sender<Option<Result<(), NetError>>>),
277    Close,
278}
279impl SharedNetworkService {
280    pub(crate) fn new(engine: Arc<CommonEngine>) -> Result<Arc<Self>, NetError> {
281        let names = super::names::NameService::new(engine.clone());
282        Self::with_names(engine, names)
283    }
284
285    pub(super) fn with_names(
286        engine: Arc<CommonEngine>,
287        names: Arc<super::names::NameService>,
288    ) -> Result<Arc<Self>, NetError> {
289        let inner = Arc::new(InnerNetStatusClient::new(engine.clone())?);
290        let (commands, receiver) = mpsc::unbounded_channel();
291        let (closed_sender, closed) = CleanupTicket::pending();
292        engine.runtime_handle().spawn(coordinate(
293            inner.clone(),
294            engine.clone(),
295            receiver,
296            closed_sender,
297        ));
298        Ok(Arc::new(Self {
299            engine,
300            inner,
301            names,
302            commands,
303            admission: Mutex::new(Admission {
304                closed: false,
305                next_id: 0,
306                active: HashSet::new(),
307            }),
308            closed,
309        }))
310    }
311    pub(crate) fn context(self: &Arc<Self>) -> NetworkStatusContext {
312        NetworkStatusContext {
313            service: Arc::downgrade(self),
314            facts: self.inner.subscribe_facts(),
315            failure: self.inner.subscribe_failure(),
316            gate: self.inner.subscribe(),
317        }
318    }
319    pub(crate) fn request_close(&self) {
320        let newly_closed = {
321            let mut state = match self.admission.lock() {
322                Ok(state) => state,
323                Err(poisoned) => {
324                    crate::log_e!(crate::common::log::log_def::LogType::Engine; "network_service_close", "error", "admission_lock_poisoned_recovered");
325                    poisoned.into_inner()
326                }
327            };
328            if state.closed {
329                false
330            } else {
331                state.closed = true;
332                state.active.clear();
333                true
334            }
335        };
336        if newly_closed {
337            // Terminal facts become visible before returning. No admission lock
338            // is held while the source wakes its internal SDK readers.
339            self.inner.request_destroy();
340            let _ = self.commands.send(Command::Close);
341            self.names.request_close();
342        }
343    }
344    #[cfg(test)]
345    pub(crate) fn inner_for_test(&self) -> Arc<InnerNetStatusClient> {
346        self.inner.clone()
347    }
348    #[cfg(test)]
349    pub(crate) fn active_consumers_for_test(&self) -> Result<usize, NetError> {
350        Ok(self
351            .admission
352            .lock()
353            .map_err(NetError::from_poison)?
354            .active
355            .len())
356    }
357    pub(crate) async fn shutdown(&self) -> Result<(), NetError> {
358        self.request_close();
359        let monitor = self.closed.wait().await;
360        let names = self.names.wait_cleanup().await;
361        monitor.and(names)
362    }
363}
364
365impl Drop for SharedNetworkService {
366    fn drop(&mut self) {
367        self.request_close();
368    }
369}
370
371type StartFuture = Pin<Box<dyn Future<Output = Result<NetworkSnapshot, NetError>> + Send>>;
372/// This guard also reports task cancellation; dropping a sender is never treated
373/// as proof that provider resources have completed retirement.
374struct Completion {
375    sender: Option<watch::Sender<Option<Result<(), NetError>>>>,
376    engine: Arc<CommonEngine>,
377}
378impl Completion {
379    fn new(
380        sender: watch::Sender<Option<Result<(), NetError>>>,
381        engine: &Arc<CommonEngine>,
382    ) -> Self {
383        Self {
384            sender: Some(sender),
385            engine: engine.clone(),
386        }
387    }
388    fn finish(mut self, result: Result<(), NetError>) {
389        if let Some(sender) = self.sender.take() {
390            let engine = self.engine.clone();
391            self.engine.runtime_handle().spawn_blocking(move || {
392                // A Handle alone cannot keep accepted cleanup work alive.
393                let _runtime_owner = engine;
394                sender.send_replace(Some(result));
395            });
396        }
397    }
398}
399impl Drop for Completion {
400    fn drop(&mut self) {
401        if let Some(sender) = self.sender.take() {
402            let engine = self.engine.clone();
403            self.engine.runtime_handle().spawn_blocking(move || {
404                let _runtime_owner = engine;
405                sender.send_replace(Some(Err(NetError::from(ErrorKind::RuntimeUnavailable))));
406            });
407        }
408    }
409}
410fn notify_waiters(
411    engine: &Arc<CommonEngine>,
412    waiters: Vec<oneshot::Sender<Result<NetworkSnapshot, NetError>>>,
413    result: Result<NetworkSnapshot, NetError>,
414) {
415    for waiter in waiters {
416        let retained_engine = engine.clone();
417        let result = result.clone();
418        engine.runtime_handle().spawn_blocking(move || {
419            let _runtime_owner = retained_engine;
420            // Independent consumers must not share one user-waker stack.
421            let _ = waiter.send(result);
422        });
423    }
424}
425
426fn retire(
427    inner: &InnerNetStatusClient,
428    engine: &Arc<CommonEngine>,
429    permanent: bool,
430    sender: watch::Sender<Option<Result<(), NetError>>>,
431) {
432    let (finished, error) = inner.request_stop(permanent);
433    let completion = Completion::new(sender, engine);
434    engine.runtime_handle().spawn(async move {
435        let result = InnerNetStatusClient::wait_until_finished(finished).await;
436        completion.finish(result.and_then(|()| error.map_or(Ok(()), Err)));
437    });
438}
439async fn coordinate(
440    inner: Arc<InnerNetStatusClient>,
441    engine: Arc<CommonEngine>,
442    mut commands: mpsc::UnboundedReceiver<Command>,
443    closed: watch::Sender<Option<Result<(), NetError>>>,
444) {
445    let closed = Completion::new(closed, &engine);
446    let mut active = HashSet::new();
447    let mut start: Option<StartFuture> = None;
448    let mut waiters = Vec::<oneshot::Sender<Result<NetworkSnapshot, NetError>>>::new();
449    let mut retry_waiters = Vec::new();
450    let mut retry_requested = false;
451    let mut start_revision = 0;
452    loop {
453        tokio::select! {
454            biased;
455            command = commands.recv() => {
456                match command {
457                    Some(Command::Acquire(id)) => { active.insert(id); }
458                    Some(Command::Ensure(id, waiter)) if active.contains(&id) => {
459                        let snapshot = inner.snapshot();
460                        let failed_during_start = start.is_some() && snapshot.as_ref().is_ok_and(|snapshot| {
461                            snapshot.revision > start_revision && matches!(snapshot.state, MonitorState::Failed(_))
462                        });
463                        if failed_during_start {
464                            retry_requested = true;
465                            if let Some(waiter) = waiter { retry_waiters.push(waiter); }
466                        } else {
467                            if let Some(waiter) = waiter { waiters.push(waiter); }
468                            if start.is_none() {
469                                start_revision = snapshot.as_ref().map_or(0, |snapshot| snapshot.revision);
470                                let inner = inner.clone();
471                                start = Some(Box::pin(async move { inner.start().await }));
472                            }
473                        }
474                    }
475                    Some(Command::Ensure(_, waiter)) => {
476                        if let Some(waiter) = waiter { notify_waiters(&engine, vec![waiter], Err(NetError::from(ErrorKind::Closed))); }
477                    }
478                    Some(Command::Release(id, sender)) => {
479                        active.remove(&id);
480                        if active.is_empty() {
481                            drop(start.take());
482                            retry_requested = false;
483                            waiters.append(&mut retry_waiters);
484                            retire(&inner, &engine, false, sender);
485                            let result = inner.snapshot();
486                            notify_waiters(&engine, std::mem::take(&mut waiters), result);
487                        } else {
488                            Completion::new(sender, &engine).finish(Ok(()));
489                        }
490                    }
491                    Some(Command::Close) | None => {
492                        drop(start.take());
493                        waiters.append(&mut retry_waiters);
494                        notify_waiters(&engine, std::mem::take(&mut waiters), Err(NetError::from(ErrorKind::Closed)));
495                        let (finished, error) = inner.request_stop(true);
496                        // No commands need service after the terminal admission
497                        // transition. Only this final shutdown waits for resources.
498                        let result = InnerNetStatusClient::wait_until_finished(finished).await;
499                        closed.finish(result.and_then(|()| error.map_or(Ok(()), Err)));
500                        return;
501                    }
502                }
503            }
504            result = async {
505                match start.as_mut() {
506                    Some(start) => start.await,
507                    None => std::future::pending().await,
508                }
509            } => {
510                start = None;
511                notify_waiters(&engine, std::mem::take(&mut waiters), result);
512                if retry_requested && !active.is_empty() {
513                    retry_requested = false;
514                    waiters = std::mem::take(&mut retry_waiters);
515                    start_revision = inner.snapshot().map_or(0, |snapshot| snapshot.revision);
516                    let inner = inner.clone();
517                    start = Some(Box::pin(async move { inner.start().await }));
518                }
519            }
520        }
521    }
522}
523
524#[cfg(test)]
525mod tests {
526    use super::*;
527    type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
528    fn check(condition: bool, message: &str) -> TestResult {
529        if condition {
530            Ok(())
531        } else {
532            Err(std::io::Error::other(message).into())
533        }
534    }
535    fn service() -> Result<Arc<SharedNetworkService>, NetError> {
536        SharedNetworkService::new(Arc::new(CommonEngine::new(16, 16)?))
537    }
538    struct BlockingWake {
539        entered: Mutex<Option<oneshot::Sender<()>>>,
540        release: Mutex<std::sync::mpsc::Receiver<()>>,
541    }
542    impl std::task::Wake for BlockingWake {
543        fn wake(self: Arc<Self>) {
544            if let Ok(mut entered) = self.entered.lock() {
545                if let Some(entered) = entered.take() {
546                    let _ = entered.send(());
547                }
548            }
549            if let Ok(release) = self.release.lock() {
550                let _ = release.recv_timeout(std::time::Duration::from_secs(5));
551            }
552        }
553    }
554    #[tokio::test]
555    async fn accepted_waiters_are_not_blocked_by_another_callers_waker() -> TestResult {
556        let engine = Arc::new(CommonEngine::new(16, 16)?);
557        let (entered, entered_wait) = oneshot::channel();
558        let (release, released) = std::sync::mpsc::channel();
559        let waker = std::task::Waker::from(Arc::new(BlockingWake {
560            entered: Mutex::new(Some(entered)),
561            release: Mutex::new(released),
562        }));
563        let (first_sender, mut first) = oneshot::channel();
564        let (second_sender, second) = oneshot::channel();
565        let first_pending = Pin::new(&mut first)
566            .poll(&mut std::task::Context::from_waker(&waker))
567            .is_pending();
568        check(first_pending, "first waiter was already complete")?;
569        notify_waiters(
570            &engine,
571            vec![first_sender, second_sender],
572            Err(NetError::from(ErrorKind::Closed)),
573        );
574        tokio::time::timeout(std::time::Duration::from_secs(5), entered_wait).await??;
575        let second_result = tokio::time::timeout(std::time::Duration::from_secs(2), second).await;
576        let _ = release.send(());
577        check(
578            second_result??.is_err(),
579            "second caller did not receive terminal result",
580        )?;
581        check(
582            first.await?.is_err(),
583            "first caller did not receive terminal result",
584        )?;
585        Ok(())
586    }
587
588    #[tokio::test]
589    async fn accepted_cleanup_notification_keeps_runtime_until_delivery() -> TestResult {
590        let engine = Arc::new(CommonEngine::new(16, 16)?);
591        let (sender, ticket) = CleanupTicket::pending();
592        let completion = Completion::new(sender, &engine);
593        let runtime_owner = Arc::downgrade(&engine);
594        drop(engine);
595        check(
596            runtime_owner.upgrade().is_some(),
597            "accepted cleanup lost its runtime owner",
598        )?;
599        completion.finish(Ok(()));
600        tokio::time::timeout(std::time::Duration::from_secs(5), ticket.wait()).await??;
601        Ok(())
602    }
603
604    #[tokio::test]
605    async fn snapshots_are_inert_and_leases_are_logical_and_idempotent() -> TestResult {
606        let service = service()?;
607        let context = service.context();
608        check(
609            matches!(context.snapshot()?.state, MonitorState::Stopped),
610            "new source is not stopped",
611        )?;
612        let lease = context.acquire()?;
613        let clone = lease.clone();
614        check(
615            service
616                .admission
617                .lock()
618                .map_err(NetError::from_poison)?
619                .active
620                .len()
621                == 1,
622            "clone added demand",
623        )?;
624        lease.release().wait().await?;
625        clone.release().wait().await?;
626        check(
627            service
628                .admission
629                .lock()
630                .map_err(NetError::from_poison)?
631                .active
632                .is_empty(),
633            "released demand leaked",
634        )?;
635        check(
636            matches!(context.snapshot()?.state, MonitorState::Stopped),
637            "acquisition started monitoring",
638        )?;
639        service.shutdown().await?;
640        Ok(())
641    }
642    #[tokio::test]
643    async fn closed_context_rejects_new_leases_and_preserves_terminal_cache() -> TestResult {
644        let service = service()?;
645        let context = service.context();
646        service.request_close();
647        check(context.acquire().is_err(), "closed context admitted demand")?;
648        check(
649            matches!(context.snapshot()?.state, MonitorState::Closed),
650            "close was not synchronous",
651        )?;
652        service.shutdown().await?;
653        drop(service);
654        check(
655            matches!(context.snapshot()?.state, MonitorState::Closed),
656            "terminal cache was lost",
657        )?;
658        Ok(())
659    }
660    fn gated_service() -> Result<
661        (
662            Arc<SharedNetworkService>,
663            mpsc::UnboundedReceiver<oneshot::Sender<()>>,
664            Arc<std::sync::atomic::AtomicUsize>,
665        ),
666        NetError,
667    > {
668        let service = service()?;
669        let (attempts, receiver) = mpsc::unbounded_channel();
670        let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
671        let factory_calls = calls.clone();
672        service
673            .inner
674            .set_monitor_factory_for_test(Arc::new(move || {
675                factory_calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
676                let (release, ready) = oneshot::channel();
677                let _ = attempts.send(release);
678                Box::pin(async move {
679                    let _ = ready.await;
680                    Err(NetError::from(ErrorKind::Io))
681                })
682            }))?;
683        Ok((service, receiver, calls))
684    }
685    async fn attempt(
686        receiver: &mut mpsc::UnboundedReceiver<oneshot::Sender<()>>,
687    ) -> Result<oneshot::Sender<()>, Box<dyn std::error::Error + Send + Sync>> {
688        Ok(
689            tokio::time::timeout(std::time::Duration::from_secs(5), receiver.recv())
690                .await?
691                .ok_or("factory missing")?,
692        )
693    }
694    #[tokio::test]
695    async fn concurrent_demand_shares_pending_initialization_and_nonlast_release_is_isolated(
696    ) -> TestResult {
697        let (service, mut attempts, calls) = gated_service()?;
698        let a = service.context().acquire()?;
699        let b = service.context().acquire()?;
700        a.ensure_started()?;
701        b.ensure_started()?;
702        let release = attempt(&mut attempts).await?;
703        a.release().wait().await?;
704        check(
705            calls.load(std::sync::atomic::Ordering::SeqCst) == 1,
706            "concurrent demand duplicated provider",
707        )?;
708        check(
709            !release.is_closed(),
710            "nonlast release cancelled shared startup",
711        )?;
712        let mut waiting = Box::pin(b.wait_started());
713        check(
714            futures::poll!(&mut waiting).is_pending(),
715            "start waiter unexpectedly completed",
716        )?;
717        // Polling wait_started only queues Ensure. A subsequent nonlast release
718        // is a FIFO acknowledgement that the coordinator admitted this waiter
719        // before we fail initialization. Otherwise it could legitimately see a
720        // later Ensure as a retry, leaving that new generation gated forever.
721        let barrier = service.context().acquire()?;
722        barrier.release().wait().await?;
723        let _ = release.send(());
724        let result = tokio::time::timeout(std::time::Duration::from_secs(5), waiting).await?;
725        check(
726            matches!(result, Err(error) if error.kind() == ErrorKind::Io),
727            "shared startup error lost",
728        )?;
729        service.shutdown().await?;
730        Ok(())
731    }
732    #[tokio::test]
733    async fn failed_generation_can_retry_with_existing_lease_and_live_fact_stream() -> TestResult {
734        let (service, mut attempts, calls) = gated_service()?;
735        let context = service.context();
736        let lease = context.acquire()?;
737        let mut facts = context.subscribe_facts();
738        let first = lease.clone();
739        let waiting = tokio::spawn(async move { first.wait_started().await });
740        let _ = attempt(&mut attempts).await?.send(());
741        check(waiting.await?.is_err(), "failure was hidden")?;
742        check(
743            matches!(context.snapshot()?.state, MonitorState::Failed(_)),
744            "failure snapshot absent",
745        )?;
746        lease.ensure_started()?;
747        let second = attempt(&mut attempts).await?;
748        check(
749            calls.load(std::sync::atomic::Ordering::SeqCst) == 2,
750            "existing demand could not retry",
751        )?;
752        check(
753            facts.changed().await.is_ok(),
754            "recoverable failure closed stream",
755        )?;
756        lease.release().wait().await?;
757        check(second.is_closed(), "last release did not cancel retry")?;
758        service.shutdown().await?;
759        Ok(())
760    }
761    #[tokio::test]
762    async fn close_during_pending_start_finishes_waiters_and_prevents_revival() -> TestResult {
763        let (service, mut attempts, _) = gated_service()?;
764        let context = service.context();
765        let lease = context.acquire()?;
766        let waiter = lease.clone();
767        let waiting = tokio::spawn(async move { waiter.wait_started().await });
768        let _attempt = attempt(&mut attempts).await?;
769        service.request_close();
770        check(
771            matches!(context.snapshot()?.state, MonitorState::Closed),
772            "close did not publish terminal",
773        )?;
774        check(
775            matches!(waiting.await?, Err(error) if error.kind() == ErrorKind::Closed),
776            "pending start survived close",
777        )?;
778        lease.release().wait().await?;
779        check(context.acquire().is_err(), "closed service revived")?;
780        service.shutdown().await?;
781        Ok(())
782    }
783    #[tokio::test]
784    async fn service_drop_does_not_require_external_context_or_lease_drop() -> TestResult {
785        let (service, mut attempts, _) = gated_service()?;
786        let context = service.context();
787        let lease = context.acquire()?;
788        lease.ensure_started()?;
789        let _attempt = attempt(&mut attempts).await?;
790        drop(service);
791        check(
792            matches!(context.snapshot()?.state, MonitorState::Closed),
793            "external context kept engine alive",
794        )?;
795        lease.release().wait().await?;
796        check(
797            lease.ensure_started().is_err(),
798            "released context restarted engine",
799        )?;
800        Ok(())
801    }
802}
803
804#[cfg(test)]
805#[path = "shared_regression_tests.rs"]
806mod regression_tests;