Skip to main content

open_net/module/net_status/inner/
shared.rs

1//! Engine-scoped network observation ownership. Consumers only retain weak
2//! service contexts and explicit, idempotently releasable demand leases.
3use std::collections::HashSet;
4use std::future::Future;
5use std::pin::Pin;
6use std::sync::{Arc, Mutex, Weak};
7
8use tokio::sync::{mpsc, oneshot, watch};
9
10use super::inner_net_status_client::InnerNetStatusClient;
11use super::network_status_snapshot::NetworkStatusSnapshot;
12use crate::common::CommonEngine;
13use crate::error::{ErrorKind, NetError};
14use crate::net_status::{MonitorState, NetworkSnapshot};
15
16/// A read-only handle to the network facts owned by one `OpenNet` engine.
17/// Cloning or reading it never starts monitoring or extends the engine lifetime.
18/// A context remains tied to its original engine after that engine is dropped.
19#[derive(Clone)]
20pub struct NetworkStatusContext {
21    service: Weak<SharedNetworkService>,
22    facts: watch::Receiver<NetworkSnapshot>,
23    failure: watch::Receiver<Option<NetError>>,
24    gate: watch::Receiver<NetworkStatusSnapshot>,
25}
26impl std::fmt::Debug for NetworkStatusContext {
27    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
28        f.debug_struct("NetworkStatusContext")
29            .finish_non_exhaustive()
30    }
31}
32impl NetworkStatusContext {
33    /// Read the cached observation without starting or retrying monitoring.
34    pub fn snapshot(&self) -> Result<NetworkSnapshot, NetError> {
35        if let Some(error) = ::tokio::sync::watch::Receiver::borrow(&self.failure).clone() {
36            return Err(error);
37        }
38        match self.service.upgrade() {
39            Some(service) => service.inner.snapshot(),
40            None => Ok(::tokio::sync::watch::Receiver::borrow(&self.facts).clone()),
41        }
42    }
43    pub(crate) fn common_engine(&self) -> Result<Arc<CommonEngine>, NetError> {
44        self.service
45            .upgrade()
46            .map(|service| service.engine.clone())
47            .ok_or_else(|| NetError::from(ErrorKind::Closed))
48    }
49    pub(crate) fn is_closed(&self) -> bool {
50        self.service.upgrade().map_or(true, |service| {
51            match service.admission.lock() {
52                Ok(state) => state.closed,
53                Err(_) => {
54                    crate::log_e!(crate::common::log::log_def::LogType::Engine; "network_service_state", "error", "admission_lock_poisoned");
55                    true
56                }
57            }
58        })
59    }
60    pub(crate) fn acquire(&self) -> Result<NetworkLease, NetError> {
61        let service = self
62            .service
63            .upgrade()
64            .ok_or_else(|| NetError::from(ErrorKind::Closed))?;
65        // A recoverable Failed snapshot remains valid for explicit retry. A
66        // permanent publication error cannot admit new, unusable demand.
67        service.inner.snapshot()?;
68        let id = {
69            let mut state = service.admission.lock().map_err(NetError::from_poison)?;
70            if state.closed {
71                return Err(NetError::from(ErrorKind::Closed));
72            }
73            let id = state
74                .next_id
75                .checked_add(1)
76                .ok_or_else(|| NetError::from(ErrorKind::Internal))?;
77            service
78                .commands
79                .send(Command::Acquire(id))
80                .map_err(|_| NetError::from(ErrorKind::RuntimeUnavailable))?;
81            state.next_id = id;
82            state.active.insert(id);
83            id
84        };
85        Ok(NetworkLease {
86            inner: Arc::new(LeaseState {
87                id,
88                service: Arc::downgrade(&service),
89                released: Mutex::new(None),
90                closed: service.closed.clone(),
91            }),
92        })
93    }
94    pub(crate) fn subscribe_facts(&self) -> watch::Receiver<NetworkSnapshot> {
95        self.facts.clone()
96    }
97    pub(crate) fn subscribe_failure(&self) -> watch::Receiver<Option<NetError>> {
98        self.failure.clone()
99    }
100    pub(crate) fn subscribe_gate(&self) -> watch::Receiver<NetworkStatusSnapshot> {
101        self.gate.clone()
102    }
103}
104
105#[derive(Clone)]
106pub(crate) struct CleanupTicket {
107    finished: watch::Receiver<Option<Result<(), NetError>>>,
108}
109impl CleanupTicket {
110    fn pending() -> (watch::Sender<Option<Result<(), NetError>>>, Self) {
111        let (sender, finished) = watch::channel(None);
112        (sender, Self { finished })
113    }
114    pub(crate) fn is_finished(&self) -> bool {
115        matches!(
116            *::tokio::sync::watch::Receiver::borrow(&self.finished),
117            Some(Ok(()))
118        )
119    }
120    pub(crate) async fn wait(&self) -> Result<(), NetError> {
121        let mut finished = self.finished.clone();
122        loop {
123            if let Some(result) = finished.borrow_and_update().clone() {
124                return result;
125            }
126            finished
127                .changed()
128                .await
129                .map_err(|_| NetError::from(ErrorKind::RuntimeUnavailable))?;
130        }
131    }
132}
133
134#[derive(Clone)]
135pub(crate) struct NetworkLease {
136    inner: Arc<LeaseState>,
137}
138struct LeaseState {
139    id: u64,
140    service: Weak<SharedNetworkService>,
141    released: Mutex<Option<CleanupTicket>>,
142    closed: CleanupTicket,
143}
144impl NetworkLease {
145    pub(crate) fn ensure_started(&self) -> Result<(), NetError> {
146        self.inner.ensure(None)
147    }
148    pub(crate) async fn wait_started(&self) -> Result<NetworkSnapshot, NetError> {
149        let (sender, received) = oneshot::channel();
150        self.inner.ensure(Some(sender))?;
151        received
152            .await
153            .map_err(|_| NetError::from(ErrorKind::RuntimeUnavailable))?
154    }
155    pub(crate) fn release(&self) -> CleanupTicket {
156        self.inner.release()
157    }
158}
159
160impl LeaseState {
161    fn ensure(
162        &self,
163        waiter: Option<oneshot::Sender<Result<NetworkSnapshot, NetError>>>,
164    ) -> Result<(), NetError> {
165        let released = self.released.lock().map_err(NetError::from_poison)?;
166        if released.is_some() {
167            return Err(NetError::from(ErrorKind::Closed));
168        }
169        let service = self
170            .service
171            .upgrade()
172            .ok_or_else(|| NetError::from(ErrorKind::Closed))?;
173        let state = service.admission.lock().map_err(NetError::from_poison)?;
174        if state.closed || !state.active.contains(&self.id) {
175            return Err(NetError::from(ErrorKind::Closed));
176        }
177        service
178            .commands
179            .send(Command::Ensure(self.id, waiter))
180            .map_err(|_| NetError::from(ErrorKind::RuntimeUnavailable))
181    }
182    fn release(&self) -> CleanupTicket {
183        let mut released = match self.released.lock() {
184            Ok(state) => state,
185            Err(poisoned) => {
186                crate::log_e!(crate::common::log::log_def::LogType::Engine; "network_lease_release", "error", "poisoned_lease_recovered");
187                poisoned.into_inner()
188            }
189        };
190        if let Some(ticket) = released.as_ref() {
191            return ticket.clone();
192        }
193        let ticket = match self.service.upgrade() {
194            Some(service) => {
195                let mut admission = match service.admission.lock() {
196                    Ok(state) => state,
197                    Err(poisoned) => {
198                        crate::log_e!(crate::common::log::log_def::LogType::Engine; "network_lease_release", "error", "admission_lock_poisoned_recovered");
199                        poisoned.into_inner()
200                    }
201                };
202                admission.active.remove(&self.id);
203                if admission.closed {
204                    service.closed.clone()
205                } else {
206                    let (sender, ticket) = CleanupTicket::pending();
207                    if let Err(error) = service.commands.send(Command::Release(self.id, sender)) {
208                        if let Command::Release(_, sender) = error.0 {
209                            Completion::new(sender, &service.engine)
210                                .finish(Err(NetError::from(ErrorKind::RuntimeUnavailable)));
211                        }
212                    }
213                    ticket
214                }
215            }
216            None => self.closed.clone(),
217        };
218        *released = Some(ticket.clone());
219        ticket
220    }
221}
222impl Drop for LeaseState {
223    fn drop(&mut self) {
224        self.release();
225    }
226}
227
228struct Admission {
229    closed: bool,
230    next_id: u64,
231    active: HashSet<u64>,
232}
233pub(crate) struct SharedNetworkService {
234    engine: Arc<CommonEngine>,
235    inner: Arc<InnerNetStatusClient>,
236    admission: Mutex<Admission>,
237    commands: mpsc::UnboundedSender<Command>,
238    closed: CleanupTicket,
239}
240enum Command {
241    Acquire(u64),
242    Ensure(
243        u64,
244        Option<oneshot::Sender<Result<NetworkSnapshot, NetError>>>,
245    ),
246    Release(u64, watch::Sender<Option<Result<(), NetError>>>),
247    Close,
248}
249impl SharedNetworkService {
250    pub(crate) fn new(engine: Arc<CommonEngine>) -> Result<Arc<Self>, NetError> {
251        let inner = Arc::new(InnerNetStatusClient::new(engine.clone())?);
252        let (commands, receiver) = mpsc::unbounded_channel();
253        let (closed_sender, closed) = CleanupTicket::pending();
254        engine.runtime_handle().spawn(coordinate(
255            inner.clone(),
256            engine.clone(),
257            receiver,
258            closed_sender,
259        ));
260        Ok(Arc::new(Self {
261            engine,
262            inner,
263            commands,
264            admission: Mutex::new(Admission {
265                closed: false,
266                next_id: 0,
267                active: HashSet::new(),
268            }),
269            closed,
270        }))
271    }
272    pub(crate) fn context(self: &Arc<Self>) -> NetworkStatusContext {
273        NetworkStatusContext {
274            service: Arc::downgrade(self),
275            facts: self.inner.subscribe_facts(),
276            failure: self.inner.subscribe_failure(),
277            gate: self.inner.subscribe(),
278        }
279    }
280    pub(crate) fn request_close(&self) {
281        let newly_closed = {
282            let mut state = match self.admission.lock() {
283                Ok(state) => state,
284                Err(poisoned) => {
285                    crate::log_e!(crate::common::log::log_def::LogType::Engine; "network_service_close", "error", "admission_lock_poisoned_recovered");
286                    poisoned.into_inner()
287                }
288            };
289            if state.closed {
290                false
291            } else {
292                state.closed = true;
293                state.active.clear();
294                true
295            }
296        };
297        if newly_closed {
298            // Terminal facts become visible before returning. No admission lock
299            // is held while the source wakes its internal SDK readers.
300            self.inner.request_destroy();
301            let _ = self.commands.send(Command::Close);
302        }
303    }
304    #[cfg(test)]
305    pub(crate) fn inner_for_test(&self) -> Arc<InnerNetStatusClient> {
306        self.inner.clone()
307    }
308    #[cfg(test)]
309    pub(crate) fn active_consumers_for_test(&self) -> Result<usize, NetError> {
310        Ok(self
311            .admission
312            .lock()
313            .map_err(NetError::from_poison)?
314            .active
315            .len())
316    }
317    pub(crate) async fn shutdown(&self) -> Result<(), NetError> {
318        self.request_close();
319        self.closed.wait().await
320    }
321}
322
323impl Drop for SharedNetworkService {
324    fn drop(&mut self) {
325        self.request_close();
326    }
327}
328
329type StartFuture = Pin<Box<dyn Future<Output = Result<NetworkSnapshot, NetError>> + Send>>;
330/// This guard also reports task cancellation; dropping a sender is never treated
331/// as proof that provider resources have completed retirement.
332struct Completion {
333    sender: Option<watch::Sender<Option<Result<(), NetError>>>>,
334    engine: Arc<CommonEngine>,
335}
336impl Completion {
337    fn new(
338        sender: watch::Sender<Option<Result<(), NetError>>>,
339        engine: &Arc<CommonEngine>,
340    ) -> Self {
341        Self {
342            sender: Some(sender),
343            engine: engine.clone(),
344        }
345    }
346    fn finish(mut self, result: Result<(), NetError>) {
347        if let Some(sender) = self.sender.take() {
348            let engine = self.engine.clone();
349            self.engine.runtime_handle().spawn_blocking(move || {
350                // A Handle alone cannot keep accepted cleanup work alive.
351                let _runtime_owner = engine;
352                sender.send_replace(Some(result));
353            });
354        }
355    }
356}
357impl Drop for Completion {
358    fn drop(&mut self) {
359        if let Some(sender) = self.sender.take() {
360            let engine = self.engine.clone();
361            self.engine.runtime_handle().spawn_blocking(move || {
362                let _runtime_owner = engine;
363                sender.send_replace(Some(Err(NetError::from(ErrorKind::RuntimeUnavailable))));
364            });
365        }
366    }
367}
368fn notify_waiters(
369    engine: &Arc<CommonEngine>,
370    waiters: Vec<oneshot::Sender<Result<NetworkSnapshot, NetError>>>,
371    result: Result<NetworkSnapshot, NetError>,
372) {
373    for waiter in waiters {
374        let retained_engine = engine.clone();
375        let result = result.clone();
376        engine.runtime_handle().spawn_blocking(move || {
377            let _runtime_owner = retained_engine;
378            // Independent consumers must not share one user-waker stack.
379            let _ = waiter.send(result);
380        });
381    }
382}
383
384fn retire(
385    inner: &InnerNetStatusClient,
386    engine: &Arc<CommonEngine>,
387    permanent: bool,
388    sender: watch::Sender<Option<Result<(), NetError>>>,
389) {
390    let (finished, error) = inner.request_stop(permanent);
391    let completion = Completion::new(sender, engine);
392    engine.runtime_handle().spawn(async move {
393        let result = InnerNetStatusClient::wait_until_finished(finished).await;
394        completion.finish(result.and_then(|()| error.map_or(Ok(()), Err)));
395    });
396}
397async fn coordinate(
398    inner: Arc<InnerNetStatusClient>,
399    engine: Arc<CommonEngine>,
400    mut commands: mpsc::UnboundedReceiver<Command>,
401    closed: watch::Sender<Option<Result<(), NetError>>>,
402) {
403    let closed = Completion::new(closed, &engine);
404    let mut active = HashSet::new();
405    let mut start: Option<StartFuture> = None;
406    let mut waiters = Vec::<oneshot::Sender<Result<NetworkSnapshot, NetError>>>::new();
407    let mut retry_waiters = Vec::new();
408    let mut retry_requested = false;
409    let mut start_revision = 0;
410    loop {
411        tokio::select! {
412            biased;
413            command = commands.recv() => {
414                match command {
415                    Some(Command::Acquire(id)) => { active.insert(id); }
416                    Some(Command::Ensure(id, waiter)) if active.contains(&id) => {
417                        let snapshot = inner.snapshot();
418                        let failed_during_start = start.is_some() && snapshot.as_ref().is_ok_and(|snapshot| {
419                            snapshot.revision > start_revision && matches!(snapshot.state, MonitorState::Failed(_))
420                        });
421                        if failed_during_start {
422                            retry_requested = true;
423                            if let Some(waiter) = waiter { retry_waiters.push(waiter); }
424                        } else {
425                            if let Some(waiter) = waiter { waiters.push(waiter); }
426                            if start.is_none() {
427                                start_revision = snapshot.as_ref().map_or(0, |snapshot| snapshot.revision);
428                                let inner = inner.clone();
429                                start = Some(Box::pin(async move { inner.start().await }));
430                            }
431                        }
432                    }
433                    Some(Command::Ensure(_, waiter)) => {
434                        if let Some(waiter) = waiter { notify_waiters(&engine, vec![waiter], Err(NetError::from(ErrorKind::Closed))); }
435                    }
436                    Some(Command::Release(id, sender)) => {
437                        active.remove(&id);
438                        if active.is_empty() {
439                            drop(start.take());
440                            retry_requested = false;
441                            waiters.append(&mut retry_waiters);
442                            retire(&inner, &engine, false, sender);
443                            let result = inner.snapshot();
444                            notify_waiters(&engine, std::mem::take(&mut waiters), result);
445                        } else {
446                            Completion::new(sender, &engine).finish(Ok(()));
447                        }
448                    }
449                    Some(Command::Close) | None => {
450                        drop(start.take());
451                        waiters.append(&mut retry_waiters);
452                        notify_waiters(&engine, std::mem::take(&mut waiters), Err(NetError::from(ErrorKind::Closed)));
453                        let (finished, error) = inner.request_stop(true);
454                        // No commands need service after the terminal admission
455                        // transition. Only this final shutdown waits for resources.
456                        let result = InnerNetStatusClient::wait_until_finished(finished).await;
457                        closed.finish(result.and_then(|()| error.map_or(Ok(()), Err)));
458                        return;
459                    }
460                }
461            }
462            result = async {
463                match start.as_mut() {
464                    Some(start) => start.await,
465                    None => std::future::pending().await,
466                }
467            } => {
468                start = None;
469                notify_waiters(&engine, std::mem::take(&mut waiters), result);
470                if retry_requested && !active.is_empty() {
471                    retry_requested = false;
472                    waiters = std::mem::take(&mut retry_waiters);
473                    start_revision = inner.snapshot().map_or(0, |snapshot| snapshot.revision);
474                    let inner = inner.clone();
475                    start = Some(Box::pin(async move { inner.start().await }));
476                }
477            }
478        }
479    }
480}
481
482#[cfg(test)]
483mod tests {
484    use super::*;
485    type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
486    fn check(condition: bool, message: &str) -> TestResult {
487        if condition {
488            Ok(())
489        } else {
490            Err(std::io::Error::other(message).into())
491        }
492    }
493    fn service() -> Result<Arc<SharedNetworkService>, NetError> {
494        SharedNetworkService::new(Arc::new(CommonEngine::new(16, 16)?))
495    }
496    struct BlockingWake {
497        entered: Mutex<Option<oneshot::Sender<()>>>,
498        release: Mutex<std::sync::mpsc::Receiver<()>>,
499    }
500    impl std::task::Wake for BlockingWake {
501        fn wake(self: Arc<Self>) {
502            if let Ok(mut entered) = self.entered.lock() {
503                if let Some(entered) = entered.take() {
504                    let _ = entered.send(());
505                }
506            }
507            if let Ok(release) = self.release.lock() {
508                let _ = release.recv_timeout(std::time::Duration::from_secs(5));
509            }
510        }
511    }
512    #[tokio::test]
513    async fn accepted_waiters_are_not_blocked_by_another_callers_waker() -> TestResult {
514        let engine = Arc::new(CommonEngine::new(16, 16)?);
515        let (entered, entered_wait) = oneshot::channel();
516        let (release, released) = std::sync::mpsc::channel();
517        let waker = std::task::Waker::from(Arc::new(BlockingWake {
518            entered: Mutex::new(Some(entered)),
519            release: Mutex::new(released),
520        }));
521        let (first_sender, mut first) = oneshot::channel();
522        let (second_sender, second) = oneshot::channel();
523        let first_pending = Pin::new(&mut first)
524            .poll(&mut std::task::Context::from_waker(&waker))
525            .is_pending();
526        check(first_pending, "first waiter was already complete")?;
527        notify_waiters(
528            &engine,
529            vec![first_sender, second_sender],
530            Err(NetError::from(ErrorKind::Closed)),
531        );
532        tokio::time::timeout(std::time::Duration::from_secs(5), entered_wait).await??;
533        let second_result = tokio::time::timeout(std::time::Duration::from_secs(2), second).await;
534        let _ = release.send(());
535        check(
536            second_result??.is_err(),
537            "second caller did not receive terminal result",
538        )?;
539        check(
540            first.await?.is_err(),
541            "first caller did not receive terminal result",
542        )?;
543        Ok(())
544    }
545
546    #[tokio::test]
547    async fn accepted_cleanup_notification_keeps_runtime_until_delivery() -> TestResult {
548        let engine = Arc::new(CommonEngine::new(16, 16)?);
549        let (sender, ticket) = CleanupTicket::pending();
550        let completion = Completion::new(sender, &engine);
551        let runtime_owner = Arc::downgrade(&engine);
552        drop(engine);
553        check(
554            runtime_owner.upgrade().is_some(),
555            "accepted cleanup lost its runtime owner",
556        )?;
557        completion.finish(Ok(()));
558        tokio::time::timeout(std::time::Duration::from_secs(5), ticket.wait()).await??;
559        Ok(())
560    }
561
562    #[tokio::test]
563    async fn snapshots_are_inert_and_leases_are_logical_and_idempotent() -> TestResult {
564        let service = service()?;
565        let context = service.context();
566        check(
567            matches!(context.snapshot()?.state, MonitorState::Stopped),
568            "new source is not stopped",
569        )?;
570        let lease = context.acquire()?;
571        let clone = lease.clone();
572        check(
573            service
574                .admission
575                .lock()
576                .map_err(NetError::from_poison)?
577                .active
578                .len()
579                == 1,
580            "clone added demand",
581        )?;
582        lease.release().wait().await?;
583        clone.release().wait().await?;
584        check(
585            service
586                .admission
587                .lock()
588                .map_err(NetError::from_poison)?
589                .active
590                .is_empty(),
591            "released demand leaked",
592        )?;
593        check(
594            matches!(context.snapshot()?.state, MonitorState::Stopped),
595            "acquisition started monitoring",
596        )?;
597        service.shutdown().await?;
598        Ok(())
599    }
600    #[tokio::test]
601    async fn closed_context_rejects_new_leases_and_preserves_terminal_cache() -> TestResult {
602        let service = service()?;
603        let context = service.context();
604        service.request_close();
605        check(context.acquire().is_err(), "closed context admitted demand")?;
606        check(
607            matches!(context.snapshot()?.state, MonitorState::Closed),
608            "close was not synchronous",
609        )?;
610        service.shutdown().await?;
611        drop(service);
612        check(
613            matches!(context.snapshot()?.state, MonitorState::Closed),
614            "terminal cache was lost",
615        )?;
616        Ok(())
617    }
618    fn gated_service() -> Result<
619        (
620            Arc<SharedNetworkService>,
621            mpsc::UnboundedReceiver<oneshot::Sender<()>>,
622            Arc<std::sync::atomic::AtomicUsize>,
623        ),
624        NetError,
625    > {
626        let service = service()?;
627        let (attempts, receiver) = mpsc::unbounded_channel();
628        let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
629        let factory_calls = calls.clone();
630        service
631            .inner
632            .set_monitor_factory_for_test(Arc::new(move || {
633                factory_calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
634                let (release, ready) = oneshot::channel();
635                let _ = attempts.send(release);
636                Box::pin(async move {
637                    let _ = ready.await;
638                    Err(NetError::from(ErrorKind::Io))
639                })
640            }))?;
641        Ok((service, receiver, calls))
642    }
643    async fn attempt(
644        receiver: &mut mpsc::UnboundedReceiver<oneshot::Sender<()>>,
645    ) -> Result<oneshot::Sender<()>, Box<dyn std::error::Error + Send + Sync>> {
646        Ok(
647            tokio::time::timeout(std::time::Duration::from_secs(5), receiver.recv())
648                .await?
649                .ok_or("factory missing")?,
650        )
651    }
652    #[tokio::test]
653    async fn concurrent_demand_shares_pending_initialization_and_nonlast_release_is_isolated(
654    ) -> TestResult {
655        let (service, mut attempts, calls) = gated_service()?;
656        let a = service.context().acquire()?;
657        let b = service.context().acquire()?;
658        a.ensure_started()?;
659        b.ensure_started()?;
660        let release = attempt(&mut attempts).await?;
661        a.release().wait().await?;
662        check(
663            calls.load(std::sync::atomic::Ordering::SeqCst) == 1,
664            "concurrent demand duplicated provider",
665        )?;
666        check(
667            !release.is_closed(),
668            "nonlast release cancelled shared startup",
669        )?;
670        let mut waiting = Box::pin(b.wait_started());
671        check(
672            futures::poll!(&mut waiting).is_pending(),
673            "start waiter unexpectedly completed",
674        )?;
675        let _ = release.send(());
676        let result = tokio::time::timeout(std::time::Duration::from_secs(5), waiting).await?;
677        check(
678            matches!(result, Err(error) if error.kind() == ErrorKind::Io),
679            "shared startup error lost",
680        )?;
681        service.shutdown().await?;
682        Ok(())
683    }
684    #[tokio::test]
685    async fn failed_generation_can_retry_with_existing_lease_and_live_fact_stream() -> TestResult {
686        let (service, mut attempts, calls) = gated_service()?;
687        let context = service.context();
688        let lease = context.acquire()?;
689        let mut facts = context.subscribe_facts();
690        let first = lease.clone();
691        let waiting = tokio::spawn(async move { first.wait_started().await });
692        let _ = attempt(&mut attempts).await?.send(());
693        check(waiting.await?.is_err(), "failure was hidden")?;
694        check(
695            matches!(context.snapshot()?.state, MonitorState::Failed(_)),
696            "failure snapshot absent",
697        )?;
698        lease.ensure_started()?;
699        let second = attempt(&mut attempts).await?;
700        check(
701            calls.load(std::sync::atomic::Ordering::SeqCst) == 2,
702            "existing demand could not retry",
703        )?;
704        check(
705            facts.changed().await.is_ok(),
706            "recoverable failure closed stream",
707        )?;
708        lease.release().wait().await?;
709        check(second.is_closed(), "last release did not cancel retry")?;
710        service.shutdown().await?;
711        Ok(())
712    }
713    #[tokio::test]
714    async fn close_during_pending_start_finishes_waiters_and_prevents_revival() -> TestResult {
715        let (service, mut attempts, _) = gated_service()?;
716        let context = service.context();
717        let lease = context.acquire()?;
718        let waiter = lease.clone();
719        let waiting = tokio::spawn(async move { waiter.wait_started().await });
720        let _attempt = attempt(&mut attempts).await?;
721        service.request_close();
722        check(
723            matches!(context.snapshot()?.state, MonitorState::Closed),
724            "close did not publish terminal",
725        )?;
726        check(
727            matches!(waiting.await?, Err(error) if error.kind() == ErrorKind::Closed),
728            "pending start survived close",
729        )?;
730        lease.release().wait().await?;
731        check(context.acquire().is_err(), "closed service revived")?;
732        service.shutdown().await?;
733        Ok(())
734    }
735    #[tokio::test]
736    async fn service_drop_does_not_require_external_context_or_lease_drop() -> TestResult {
737        let (service, mut attempts, _) = gated_service()?;
738        let context = service.context();
739        let lease = context.acquire()?;
740        lease.ensure_started()?;
741        let _attempt = attempt(&mut attempts).await?;
742        drop(service);
743        check(
744            matches!(context.snapshot()?.state, MonitorState::Closed),
745            "external context kept engine alive",
746        )?;
747        lease.release().wait().await?;
748        check(
749            lease.ensure_started().is_err(),
750            "released context restarted engine",
751        )?;
752        Ok(())
753    }
754}
755
756#[cfg(test)]
757#[path = "shared_regression_tests.rs"]
758mod regression_tests;