Skip to main content

rigspark_runtime/application/
events.rs

1use crate::{
2    adapters::{BackendAdapter, BackendError, ServeRequest},
3    identity::ProcessIdentity,
4    state::ServerState,
5};
6use std::{
7    collections::VecDeque,
8    future::Future,
9    sync::{Arc, Mutex},
10};
11use tokio::sync::mpsc::{self, Receiver, Sender};
12use tokio_util::sync::CancellationToken;
13
14pub const EVENT_CAPACITY: usize = 64;
15
16#[derive(Clone, Copy)]
17pub(crate) enum LifecycleWarning {
18    InstalledBypass,
19    EstimatedFit,
20    SizeOnly,
21    DrafterSkipped,
22}
23
24impl LifecycleWarning {
25    fn message(self) -> &'static str {
26        match self {
27            Self::InstalledBypass => {
28                "up: bypassing estimated fit; local content integrity is not catalog verification; throughput may be unknown"
29            }
30            Self::EstimatedFit => {
31                "up: requested quantization may not fit this hardware; continuing because it was explicitly requested"
32            }
33            Self::SizeOnly => {
34                "up: weights passed a size-floor check; no catalog SHA-256 was available"
35            }
36            Self::DrafterSkipped => {
37                "up: skipped the speculative drafter because it does not fit beside the model"
38            }
39        }
40    }
41}
42
43#[derive(Clone, Default)]
44pub struct DiagnosticObserver {
45    state: Arc<Mutex<DiagnosticSnapshot>>,
46}
47
48#[derive(Clone, Default)]
49pub struct DiagnosticSnapshot {
50    warnings: [u64; 3],
51    progress: VecDeque<(u64, u64)>,
52    omitted_progress: u64,
53}
54
55impl DiagnosticObserver {
56    pub fn snapshot(&self) -> DiagnosticSnapshot {
57        self.state
58            .lock()
59            .unwrap_or_else(|error| error.into_inner())
60            .clone()
61    }
62
63    fn warning(&self, warning: LifecycleWarning) {
64        let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner());
65        let count = &mut state.warnings[warning as usize];
66        *count = count.saturating_add(1);
67    }
68
69    fn progress(&self, completed: u64, total: u64) {
70        let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner());
71        if state.progress.len() == EVENT_CAPACITY {
72            state.progress.pop_front();
73            state.omitted_progress = state.omitted_progress.saturating_add(1);
74        }
75        state.progress.push_back((completed, total));
76    }
77}
78
79impl DiagnosticSnapshot {
80    pub fn text(&self) -> String {
81        let mut lines = Vec::new();
82        for warning in [
83            LifecycleWarning::InstalledBypass,
84            LifecycleWarning::EstimatedFit,
85            LifecycleWarning::SizeOnly,
86        ] {
87            let count = self.warnings[warning as usize];
88            if count > 0 {
89                lines.push(format!("{} ({count} occurrences)", warning.message()));
90            }
91        }
92        if self.omitted_progress > 0 {
93            lines.push(format!(
94                "{} earlier progress diagnostics omitted",
95                self.omitted_progress
96            ));
97        }
98        for (completed, total) in &self.progress {
99            lines.push(format!("  [redacted filename]: {completed}/{total} bytes"));
100        }
101        if lines.is_empty() {
102            String::new()
103        } else {
104            format!("{}\n", lines.join("\n"))
105        }
106    }
107}
108
109pub(crate) fn warning(observer: Option<&LifecycleObserver>, warning: LifecycleWarning) {
110    match observer.and_then(|observer| observer.diagnostics.as_ref()) {
111        Some(diagnostics) => diagnostics.warning(warning),
112        None => eprintln!("{}", warning.message()),
113    }
114}
115
116pub(crate) fn download_progress(
117    diagnostics: Option<&DiagnosticObserver>,
118    completed: u64,
119    total: u64,
120    file: &str,
121) {
122    match diagnostics {
123        Some(diagnostics) => diagnostics.progress(completed, total),
124        None => eprintln!(
125            "  {}: {completed}/{total} bytes",
126            rigspark_core::reports::strip_control(file)
127        ),
128    }
129}
130
131#[derive(Debug, Clone, Copy, PartialEq, Eq)]
132pub enum LifecycleScope {
133    Runtime,
134    AcquisitionDaemon,
135}
136
137#[derive(Debug, Clone, Copy, PartialEq, Eq)]
138pub enum LifecycleStage {
139    Acquisition,
140    Verification,
141    AcquisitionVerification,
142    Start,
143    Attach,
144    Activation,
145    Readiness,
146    Stop,
147}
148
149#[derive(Debug, Clone, Copy, PartialEq, Eq)]
150pub enum LifecycleStatus {
151    Started,
152    Completed,
153    Failed,
154}
155
156/// Operation-level evidence, not command success or state-commit confirmation.
157/// Start covers the adapter's serve operation, which may reuse an existing server.
158/// AcquisitionVerification covers acquisition APIs that verify before returning.
159/// No strings are carried (maximum string payload: 0 bytes), including diagnostics,
160/// model identifiers, paths, endpoints, credentials, or subprocess output.
161#[derive(Debug, Clone, Copy, PartialEq, Eq)]
162pub struct LifecycleEvent {
163    pub scope: LifecycleScope,
164    pub stage: LifecycleStage,
165    pub status: LifecycleStatus,
166}
167
168/// Best-effort, nonblocking observations. Full or closed queues drop new events.
169/// Consumers must use the command result, not events, to determine final success.
170pub struct LifecycleObserver {
171    sender: Sender<LifecycleEvent>,
172    pub(crate) diagnostics: Option<DiagnosticObserver>,
173}
174
175impl LifecycleObserver {
176    pub fn channel() -> (Self, Receiver<LifecycleEvent>) {
177        let (sender, receiver) = mpsc::channel(EVENT_CAPACITY);
178        (
179            Self {
180                sender,
181                diagnostics: None,
182            },
183            receiver,
184        )
185    }
186
187    pub fn with_diagnostics(mut self, diagnostics: DiagnosticObserver) -> Self {
188        self.diagnostics = Some(diagnostics);
189        self
190    }
191
192    fn emit(&self, scope: LifecycleScope, stage: LifecycleStage, status: LifecycleStatus) {
193        let _ = self.sender.try_send(LifecycleEvent {
194            scope,
195            stage,
196            status,
197        });
198    }
199}
200
201pub(crate) async fn observe<T>(
202    observer: Option<&LifecycleObserver>,
203    scope: LifecycleScope,
204    stage: LifecycleStage,
205    operation: impl Future<Output = Result<T, BackendError>>,
206) -> Result<T, BackendError> {
207    if let Some(observer) = observer {
208        observer.emit(scope, stage, LifecycleStatus::Started);
209    }
210    let result = operation.await;
211    if let Some(observer) = observer {
212        observer.emit(
213            scope,
214            stage,
215            if result.is_ok() {
216                LifecycleStatus::Completed
217            } else {
218                LifecycleStatus::Failed
219            },
220        );
221    }
222    result
223}
224
225pub(super) struct ObservedAdapter<'runtime> {
226    inner: &'runtime dyn BackendAdapter,
227    observer: Option<&'runtime LifecycleObserver>,
228    scope: LifecycleScope,
229}
230
231impl<'runtime> ObservedAdapter<'runtime> {
232    pub(super) fn new(
233        inner: &'runtime dyn BackendAdapter,
234        observer: Option<&'runtime LifecycleObserver>,
235        scope: LifecycleScope,
236    ) -> Self {
237        Self {
238            inner,
239            observer,
240            scope,
241        }
242    }
243}
244
245#[async_trait::async_trait]
246impl BackendAdapter for ObservedAdapter<'_> {
247    fn name(&self) -> &'static str {
248        self.inner.name()
249    }
250    fn trusts(&self, identity: &ProcessIdentity) -> bool {
251        self.inner.trusts(identity)
252    }
253    fn can_embed(&self) -> bool {
254        self.inner.can_embed()
255    }
256    fn can_stream(&self) -> bool {
257        self.inner.can_stream()
258    }
259
260    async fn serve(
261        &self,
262        request: &ServeRequest,
263        cancel: &CancellationToken,
264    ) -> Result<ServerState, BackendError> {
265        observe(
266            self.observer,
267            self.scope,
268            LifecycleStage::Start,
269            self.inner.serve(request, cancel),
270        )
271        .await
272    }
273    async fn attach_only(
274        &self,
275        request: &ServeRequest,
276        cancel: &CancellationToken,
277    ) -> Result<ServerState, BackendError> {
278        observe(
279            self.observer,
280            self.scope,
281            LifecycleStage::Attach,
282            self.inner.attach_only(request, cancel),
283        )
284        .await
285    }
286    async fn ready(
287        &self,
288        request: &ServeRequest,
289        cancel: &CancellationToken,
290    ) -> Result<(), BackendError> {
291        observe(
292            self.observer,
293            self.scope,
294            LifecycleStage::Readiness,
295            self.inner.ready(request, cancel),
296        )
297        .await
298    }
299    async fn ready_handle(
300        &self,
301        request: &ServeRequest,
302        handle: &ServerState,
303        cancel: &CancellationToken,
304    ) -> Result<(), BackendError> {
305        observe(
306            self.observer,
307            self.scope,
308            LifecycleStage::Readiness,
309            self.inner.ready_handle(request, handle, cancel),
310        )
311        .await
312    }
313    async fn stop(
314        &self,
315        handle: &ServerState,
316        cancel: &CancellationToken,
317    ) -> Result<(), BackendError> {
318        observe(
319            self.observer,
320            self.scope,
321            LifecycleStage::Stop,
322            self.inner.stop(handle, cancel),
323        )
324        .await
325    }
326}
327
328#[cfg(test)]
329mod tests {
330    use super::*;
331    use crate::{
332        adapters::{BackendAdapter, ServeRequest},
333        identity::ProcessIdentity,
334        state::{RuntimeState, ServerState},
335    };
336    use std::sync::Mutex;
337    use tokio_util::sync::CancellationToken;
338
339    #[test]
340    fn diagnostic_sink_bounds_progress_without_evicting_security_warnings() {
341        let diagnostics = DiagnosticObserver::default();
342        diagnostics.warning(LifecycleWarning::InstalledBypass);
343        diagnostics.warning(LifecycleWarning::EstimatedFit);
344        diagnostics.warning(LifecycleWarning::SizeOnly);
345        for completed in 0..100 {
346            download_progress(
347                Some(&diagnostics),
348                completed,
349                100,
350                "\u{1b}]52;c;secret\u{7}/token=secret.bin",
351            );
352        }
353        let snapshot = diagnostics.snapshot();
354        assert_eq!(snapshot.progress.len(), EVENT_CAPACITY);
355        assert_eq!(snapshot.omitted_progress, 36);
356        let text = snapshot.text();
357        assert!(text.contains("no catalog SHA-256"));
358        assert!(text.contains("bypassing estimated fit"));
359        assert!(text.contains("may not fit"));
360        assert!(text.contains("99/100 bytes"));
361        assert!(text.contains("36 earlier progress diagnostics omitted"));
362        assert!(text.contains("[redacted filename]"));
363        assert!(!text.contains("secret"));
364        assert!(!text.contains('\u{1b}'));
365    }
366
367    #[test]
368    fn diagnostic_sink_is_injected_and_keeps_repeated_warning_counts() {
369        let (plain, _) = LifecycleObserver::channel();
370        assert!(plain.diagnostics.is_none());
371        let diagnostics = DiagnosticObserver::default();
372        let observed = plain.with_diagnostics(diagnostics.clone());
373        for _ in 0..100 {
374            warning(Some(&observed), LifecycleWarning::SizeOnly);
375        }
376        assert!(diagnostics.snapshot().text().contains("100 occurrences"));
377        assert!(DiagnosticObserver::default().snapshot().text().is_empty());
378    }
379
380    struct Adapter {
381        fail: Option<LifecycleStage>,
382        calls: Mutex<Vec<LifecycleStage>>,
383    }
384
385    impl Adapter {
386        fn attempt(
387            &self,
388            stage: LifecycleStage,
389            cancel: &CancellationToken,
390        ) -> Result<(), BackendError> {
391            self.calls.lock().unwrap().push(stage);
392            if cancel.is_cancelled() {
393                Err(BackendError("cancelled".into()))
394            } else if self.fail == Some(stage) {
395                Err(BackendError("mock operation failed".into()))
396            } else {
397                Ok(())
398            }
399        }
400    }
401
402    fn handle() -> ServerState {
403        RuntimeState::parse(r#"{"schemaVersion":2,"active":{"backend":"ollama","modelId":"model:latest","endpoint":"http://127.0.0.1:11434","port":11434,"ownedByUs":true,"pid":123}}"#)
404            .unwrap().active.unwrap()
405    }
406
407    fn request() -> ServeRequest {
408        ServeRequest {
409            draft_model_path: None,
410            model_id: "model:latest".into(),
411            endpoint: "http://127.0.0.1:11434".into(),
412            model_path: None,
413            context: Some(4096),
414            cache: None,
415            slot_save_path: None,
416        }
417    }
418
419    #[async_trait::async_trait]
420    impl BackendAdapter for Adapter {
421        fn name(&self) -> &'static str {
422            "ollama"
423        }
424        fn trusts(&self, _identity: &ProcessIdentity) -> bool {
425            false
426        }
427        async fn serve(
428            &self,
429            request: &ServeRequest,
430            cancel: &CancellationToken,
431        ) -> Result<ServerState, BackendError> {
432            assert_eq!(
433                request.context,
434                if request.model_id == "llmup-acquisition" {
435                    None
436                } else {
437                    Some(4096)
438                }
439            );
440            self.attempt(LifecycleStage::Start, cancel)?;
441            Ok(handle())
442        }
443        async fn attach_only(
444            &self,
445            _request: &ServeRequest,
446            cancel: &CancellationToken,
447        ) -> Result<ServerState, BackendError> {
448            self.attempt(LifecycleStage::Attach, cancel)?;
449            let mut handle = handle();
450            handle.owned_by_us = false;
451            Ok(handle)
452        }
453        async fn ready(
454            &self,
455            _request: &ServeRequest,
456            _cancel: &CancellationToken,
457        ) -> Result<(), BackendError> {
458            panic!("ready_handle override must be forwarded")
459        }
460        async fn ready_handle(
461            &self,
462            _request: &ServeRequest,
463            active: &ServerState,
464            cancel: &CancellationToken,
465        ) -> Result<(), BackendError> {
466            assert_eq!(active.pid, Some(123));
467            self.attempt(LifecycleStage::Readiness, cancel)
468        }
469        async fn stop(
470            &self,
471            active: &ServerState,
472            cancel: &CancellationToken,
473        ) -> Result<(), BackendError> {
474            assert!(active.owned_by_us);
475            self.attempt(LifecycleStage::Stop, cancel)
476        }
477    }
478
479    #[tokio::test]
480    async fn runtime_events_adapter_orders_real_calls_and_preserves_failures() {
481        for failure in [
482            None,
483            Some(LifecycleStage::Start),
484            Some(LifecycleStage::Readiness),
485            Some(LifecycleStage::Stop),
486        ] {
487            let (observer, mut receiver) = LifecycleObserver::channel();
488            let adapter = Adapter {
489                fail: failure,
490                calls: Mutex::new(Vec::new()),
491            };
492            let observed = ObservedAdapter::new(&adapter, Some(&observer), LifecycleScope::Runtime);
493            let cancel = CancellationToken::new();
494            let outcome = async {
495                let active = observed.serve(&request(), &cancel).await?;
496                observed.ready_handle(&request(), &active, &cancel).await?;
497                observed.stop(&active, &cancel).await
498            }
499            .await;
500            assert_eq!(outcome.is_err(), failure.is_some());
501            if let Err(error) = outcome {
502                assert_eq!(error.0, "mock operation failed");
503            }
504            let mut expected = Vec::new();
505            for stage in [
506                LifecycleStage::Start,
507                LifecycleStage::Readiness,
508                LifecycleStage::Stop,
509            ] {
510                expected.push(stage);
511                assert_eq!(
512                    receiver.try_recv().unwrap(),
513                    LifecycleEvent {
514                        scope: LifecycleScope::Runtime,
515                        stage,
516                        status: LifecycleStatus::Started
517                    }
518                );
519                assert_eq!(
520                    receiver.try_recv().unwrap(),
521                    LifecycleEvent {
522                        scope: LifecycleScope::Runtime,
523                        stage,
524                        status: if failure == Some(stage) {
525                            LifecycleStatus::Failed
526                        } else {
527                            LifecycleStatus::Completed
528                        }
529                    }
530                );
531                if failure == Some(stage) {
532                    break;
533                }
534            }
535            assert_eq!(*adapter.calls.lock().unwrap(), expected);
536            assert!(receiver.try_recv().is_err());
537        }
538    }
539
540    #[tokio::test]
541    async fn runtime_events_adapter_preserves_attach_and_cancellation() {
542        let (observer, mut receiver) = LifecycleObserver::channel();
543        let adapter = Adapter {
544            fail: None,
545            calls: Mutex::new(Vec::new()),
546        };
547        let observed =
548            ObservedAdapter::new(&adapter, Some(&observer), LifecycleScope::AcquisitionDaemon);
549        let cancel = CancellationToken::new();
550        assert!(
551            !observed
552                .attach_only(&request(), &cancel)
553                .await
554                .unwrap()
555                .owned_by_us
556        );
557        assert_eq!(receiver.try_recv().unwrap().stage, LifecycleStage::Attach);
558        assert_eq!(
559            receiver.try_recv().unwrap().status,
560            LifecycleStatus::Completed
561        );
562        cancel.cancel();
563        assert_eq!(
564            observed.serve(&request(), &cancel).await.unwrap_err().0,
565            "cancelled"
566        );
567        assert_eq!(
568            receiver.try_recv().unwrap().scope,
569            LifecycleScope::AcquisitionDaemon
570        );
571        assert_eq!(receiver.try_recv().unwrap().status, LifecycleStatus::Failed);
572        assert!(receiver.try_recv().is_err());
573    }
574
575    #[tokio::test]
576    async fn saturated_or_closed_observation_never_retries_or_replaces_runtime_result() {
577        for closed in [false, true] {
578            let (observer, mut receiver) = LifecycleObserver::channel();
579            for _ in 0..EVENT_CAPACITY {
580                observer.emit(
581                    LifecycleScope::Runtime,
582                    LifecycleStage::Start,
583                    LifecycleStatus::Started,
584                );
585            }
586            if closed {
587                receiver.close();
588            }
589            let calls = std::sync::atomic::AtomicUsize::new(0);
590            let result = tokio::time::timeout(
591                std::time::Duration::from_secs(1),
592                observe(
593                    Some(&observer),
594                    LifecycleScope::Runtime,
595                    LifecycleStage::Readiness,
596                    async {
597                        calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
598                        Err::<(), _>(BackendError("authoritative runtime failure".into()))
599                    },
600                ),
601            )
602            .await
603            .unwrap();
604            assert_eq!(result.unwrap_err().0, "authoritative runtime failure");
605            assert_eq!(calls.load(std::sync::atomic::Ordering::SeqCst), 1);
606            assert_eq!(receiver.len(), EVENT_CAPACITY);
607            let result = observe(
608                Some(&observer),
609                LifecycleScope::Runtime,
610                LifecycleStage::Stop,
611                async { Ok::<_, BackendError>("authoritative success") },
612            )
613            .await
614            .unwrap();
615            assert_eq!(result, "authoritative success");
616            assert_eq!(receiver.len(), EVENT_CAPACITY);
617        }
618    }
619
620    #[tokio::test]
621    async fn runtime_events_temporary_daemon_cleanup_does_not_claim_pull_success() {
622        let (observer, mut receiver) = LifecycleObserver::channel();
623        let adapter = Adapter {
624            fail: None,
625            calls: Mutex::new(Vec::new()),
626        };
627        let observed =
628            ObservedAdapter::new(&adapter, Some(&observer), LifecycleScope::AcquisitionDaemon);
629        let cancel = CancellationToken::new();
630        let operation = observe(
631            Some(&observer),
632            LifecycleScope::Runtime,
633            LifecycleStage::Acquisition,
634            async {
635                cancel.cancel();
636                Err::<(), _>(BackendError("pull cancelled".into()))
637            },
638        );
639        let result = crate::pull::with_ollama_daemon(
640            &observed,
641            "http://127.0.0.1:11434",
642            &cancel,
643            operation,
644        )
645        .await;
646        assert_eq!(result.unwrap_err().0, "pull cancelled");
647        assert_eq!(
648            *adapter.calls.lock().unwrap(),
649            [LifecycleStage::Start, LifecycleStage::Stop]
650        );
651        for (scope, stage, status) in [
652            (
653                LifecycleScope::AcquisitionDaemon,
654                LifecycleStage::Start,
655                LifecycleStatus::Started,
656            ),
657            (
658                LifecycleScope::AcquisitionDaemon,
659                LifecycleStage::Start,
660                LifecycleStatus::Completed,
661            ),
662            (
663                LifecycleScope::Runtime,
664                LifecycleStage::Acquisition,
665                LifecycleStatus::Started,
666            ),
667            (
668                LifecycleScope::Runtime,
669                LifecycleStage::Acquisition,
670                LifecycleStatus::Failed,
671            ),
672            (
673                LifecycleScope::AcquisitionDaemon,
674                LifecycleStage::Stop,
675                LifecycleStatus::Started,
676            ),
677            (
678                LifecycleScope::AcquisitionDaemon,
679                LifecycleStage::Stop,
680                LifecycleStatus::Completed,
681            ),
682        ] {
683            assert_eq!(
684                receiver.try_recv().unwrap(),
685                LifecycleEvent {
686                    scope,
687                    stage,
688                    status
689                }
690            );
691        }
692        assert!(receiver.try_recv().is_err());
693    }
694
695    #[tokio::test]
696    async fn runtime_events_bracket_success_and_failure_without_false_completion() {
697        let (observer, mut receiver) = LifecycleObserver::channel();
698        let operation = async {
699            assert_eq!(
700                receiver.try_recv().unwrap(),
701                LifecycleEvent {
702                    scope: LifecycleScope::Runtime,
703                    stage: LifecycleStage::Verification,
704                    status: LifecycleStatus::Started,
705                }
706            );
707            Ok::<_, BackendError>(42)
708        };
709        assert_eq!(
710            observe(
711                Some(&observer),
712                LifecycleScope::Runtime,
713                LifecycleStage::Verification,
714                operation
715            )
716            .await
717            .unwrap(),
718            42
719        );
720        assert_eq!(
721            receiver.try_recv().unwrap().status,
722            LifecycleStatus::Completed
723        );
724        let error = observe(
725            Some(&observer),
726            LifecycleScope::Runtime,
727            LifecycleStage::Start,
728            async { Err::<(), _>(BackendError("private diagnostic".into())) },
729        )
730        .await
731        .unwrap_err();
732        assert_eq!(error.0, "private diagnostic");
733        assert_eq!(
734            receiver.try_recv().unwrap().status,
735            LifecycleStatus::Started
736        );
737        let failed = receiver.try_recv().unwrap();
738        assert_eq!(failed.status, LifecycleStatus::Failed);
739        assert!(!format!("{failed:?}").contains("private diagnostic"));
740        assert!(receiver.try_recv().is_err());
741    }
742
743    #[tokio::test]
744    async fn runtime_events_full_closed_or_absent_observer_never_blocks_operations() {
745        let (observer, mut receiver) = LifecycleObserver::channel();
746        for _ in 0..EVENT_CAPACITY * 2 {
747            observe(
748                Some(&observer),
749                LifecycleScope::Runtime,
750                LifecycleStage::Readiness,
751                async { Ok::<_, BackendError>(()) },
752            )
753            .await
754            .unwrap();
755        }
756        assert_eq!(receiver.len(), EVENT_CAPACITY);
757        receiver.close();
758        for observer in [Some(&observer), None] {
759            assert_eq!(
760                observe(
761                    observer,
762                    LifecycleScope::Runtime,
763                    LifecycleStage::Stop,
764                    async { Ok::<_, BackendError>(7) }
765                )
766                .await
767                .unwrap(),
768                7
769            );
770        }
771    }
772
773    #[tokio::test]
774    async fn runtime_events_dropped_operation_has_no_completion() {
775        let (observer, mut receiver) = LifecycleObserver::channel();
776        {
777            let operation = observe(
778                Some(&observer),
779                LifecycleScope::Runtime,
780                LifecycleStage::Readiness,
781                std::future::pending::<Result<(), BackendError>>(),
782            );
783            tokio::pin!(operation);
784            tokio::select! {
785                biased;
786                _ = &mut operation => panic!("pending operation completed"),
787                event = receiver.recv() => assert_eq!(event.unwrap().status, LifecycleStatus::Started),
788            }
789        }
790        assert!(receiver.try_recv().is_err());
791    }
792}