Skip to main content

snip_sync/
orchestration.rs

1//! Server shutdown orchestration.
2//!
3//! Provides the single production implementation of service-lifetime
4//! coordination used by both `serve_inner` and deterministic tests.
5
6use std::time::Duration;
7
8/// Supervise Tonic and EggServe while preserving EggServe's typed terminal result.
9pub async fn run_eggserve_services_until_shutdown<F>(
10    mut shutdown_future: std::pin::Pin<&mut F>,
11    mut grpc_handle: tokio::task::JoinHandle<Result<(), tonic::transport::Error>>,
12    http_control: eggserve_server::ServerControl,
13    mut http_completion: eggserve_server::ServerCompletion,
14    shutdown_sender: tokio::sync::broadcast::Sender<()>,
15    drain_timeout: Duration,
16) -> ServiceShutdownOutcome
17where
18    F: std::future::Future<Output = ()>,
19{
20    let mut requested = false;
21    let mut grpc_result = None;
22    let mut http_result = None;
23
24    tokio::select! {
25        biased;
26        _ = &mut shutdown_future => requested = true,
27        result = &mut grpc_handle => grpc_result = Some(classify_result(result)),
28        result = http_completion.wait() => {
29            http_result = Some(match result {
30                Ok(_) => ServiceResult::Clean,
31                Err(error) => ServiceResult::ServiceError(error.to_string()),
32            });
33        }
34    }
35
36    let _ = shutdown_sender.send(());
37    http_control.shutdown();
38
39    let drain_result = tokio::time::timeout(drain_timeout, async {
40        while grpc_result.is_none() || http_result.is_none() {
41            tokio::select! {
42                result = &mut grpc_handle, if grpc_result.is_none() => {
43                    grpc_result = Some(classify_result(result));
44                }
45                result = http_completion.wait(), if http_result.is_none() => {
46                    http_result = Some(match result {
47                        Ok(_) => ServiceResult::Clean,
48                        Err(error) => ServiceResult::ServiceError(error.to_string()),
49                    });
50                }
51            }
52        }
53    })
54    .await;
55
56    let forced = drain_result.is_err();
57    if forced {
58        grpc_handle.abort();
59        if grpc_result.is_none() {
60            grpc_result = Some(classify_result(grpc_handle.await));
61        }
62        if http_result.is_none() {
63            // EggServe owns its bounded connection drain. Await its typed
64            // completion so no runtime task is detached from the process.
65            http_result = Some(match http_completion.wait().await {
66                Ok(_) => ServiceResult::Clean,
67                Err(error) => ServiceResult::ServiceError(error.to_string()),
68            });
69        }
70    }
71
72    ServiceShutdownOutcome {
73        requested,
74        forced,
75        grpc_result: grpc_result.unwrap_or_else(|| ServiceResult::Cancelled("not observed".into())),
76        http_result: http_result.unwrap_or_else(|| ServiceResult::Cancelled("not observed".into())),
77    }
78}
79
80/// Terminal classification of a service task result.
81#[derive(Debug, Clone, PartialEq, Eq)]
82pub enum ServiceResult {
83    /// Task returned `Ok(())`.
84    Clean,
85    /// Task returned `Ok(Err(e))` — a service-level error.
86    ServiceError(String),
87    /// Task returned `Err(JoinError)` where `is_panic()` is true.
88    Panic(String),
89    /// Task was cancelled (explicitly aborted).
90    Cancelled(String),
91}
92
93/// Result of the shutdown orchestration.
94#[derive(Debug)]
95pub struct ServiceShutdownOutcome {
96    /// `true` when the shutdown was triggered by a process signal
97    /// (Ctrl-C / SIGTERM). `false` when an unexpected service completion
98    /// triggered shutdown.
99    pub requested: bool,
100    /// `true` when the drain timed out and unfinished tasks were forcibly
101    /// aborted.
102    pub forced: bool,
103    /// Terminal result of the gRPC service.
104    pub grpc_result: ServiceResult,
105    /// Terminal result of the HTTP service.
106    pub http_result: ServiceResult,
107}
108
109impl ServiceShutdownOutcome {
110    /// A requested shutdown succeeds only when both services returned
111    /// cleanly without forced abort.
112    pub fn is_clean_requested_shutdown(&self) -> bool {
113        self.requested
114            && !self.forced
115            && self.grpc_result == ServiceResult::Clean
116            && self.http_result == ServiceResult::Clean
117    }
118
119    /// Production decision method used by `serve_inner`. Returns `Ok(())`
120    /// only for a requested, unforced, dual-clean shutdown. Returns an
121    /// `Err` containing both service classifications and the original
122    /// detail for every other case: forced abort, unexpected clean exit,
123    /// or any service error/panic.
124    pub fn ensure_clean_requested_shutdown(&self) -> Result<(), String> {
125        if self.is_clean_requested_shutdown() {
126            return Ok(());
127        }
128
129        Err(format!(
130            "service shutdown was not clean: requested={}, forced={}, grpc={:?}, http={:?}",
131            self.requested, self.forced, self.grpc_result, self.http_result,
132        ))
133    }
134}
135
136/// Classify a `JoinHandle` output into a `ServiceResult`.
137fn classify_result(
138    result: Result<Result<(), impl std::fmt::Display>, tokio::task::JoinError>,
139) -> ServiceResult {
140    match result {
141        Ok(Ok(())) => ServiceResult::Clean,
142        Ok(Err(e)) => ServiceResult::ServiceError(e.to_string()),
143        Err(e) if e.is_panic() => ServiceResult::Panic(format!("{e}")),
144        Err(e) => ServiceResult::Cancelled(format!("{e}")),
145    }
146}
147
148/// Run two service task handles until a shutdown signal or unexpected
149/// service completion, then drain with a bounded timeout.
150///
151/// This is the single production orchestration implementation used by
152/// both `serve_inner` and deterministic orchestration tests.
153///
154/// # Arguments
155/// * `shutdown_future` — a future that completes when a process signal
156///   is received.
157/// * `grpc_handle` — the gRPC service JoinHandle.
158/// * `http_handle` — the HTTP service JoinHandle.
159/// * `shutdown_sender` — broadcast sender to notify services of shutdown.
160/// * `drain_timeout` — maximum time to wait for services to drain after
161///   shutdown.
162#[cfg(test)]
163pub async fn run_services_until_shutdown<F>(
164    mut shutdown_future: std::pin::Pin<&mut F>,
165    grpc_handle: tokio::task::JoinHandle<Result<(), tonic::transport::Error>>,
166    http_handle: tokio::task::JoinHandle<Result<(), std::io::Error>>,
167    shutdown_sender: tokio::sync::broadcast::Sender<()>,
168    drain_timeout: Duration,
169) -> ServiceShutdownOutcome
170where
171    F: std::future::Future<Output = ()>,
172{
173    let mut requested = false;
174
175    let mut grpc_result: Option<ServiceResult> = None;
176    let mut http_result: Option<ServiceResult> = None;
177
178    // Keep abort handles for the original service tasks. The JoinSet below
179    // owns wrapper tasks that classify those handles; aborting only the
180    // wrappers would detach the actual service tasks on a forced drain.
181    let grpc_abort = grpc_handle.abort_handle();
182    let http_abort = http_handle.abort_handle();
183
184    #[derive(Clone, Copy)]
185    enum ServiceKind {
186        Grpc,
187        Http,
188    }
189
190    struct ServiceCompletion {
191        kind: ServiceKind,
192        result: ServiceResult,
193    }
194
195    let mut tasks = tokio::task::JoinSet::<ServiceCompletion>::new();
196    tasks.spawn(async move {
197        ServiceCompletion {
198            kind: ServiceKind::Grpc,
199            result: classify_result(grpc_handle.await),
200        }
201    });
202    tasks.spawn(async move {
203        ServiceCompletion {
204            kind: ServiceKind::Http,
205            result: classify_result(http_handle.await),
206        }
207    });
208
209    let record = |completion: ServiceCompletion,
210                  grpc_result: &mut Option<ServiceResult>,
211                  http_result: &mut Option<ServiceResult>,
212                  phase: &str| {
213        if completion.result != ServiceResult::Clean {
214            let name = match completion.kind {
215                ServiceKind::Grpc => "gRPC",
216                ServiceKind::Http => "HTTP",
217            };
218            tracing::error!("{name} service {phase}: {:?}", completion.result);
219        }
220        match completion.kind {
221            ServiceKind::Grpc => *grpc_result = Some(completion.result),
222            ServiceKind::Http => *http_result = Some(completion.result),
223        }
224    };
225
226    // Phase 1: Wait for the first terminal event — a shutdown signal,
227    // gRPC completion, or HTTP completion. No pre-signal lifetime
228    // timeout; the server runs indefinitely until a signal or failure.
229    tokio::select! {
230        biased;
231        _ = &mut shutdown_future => {
232            tracing::info!("Shutdown signal received");
233            requested = true;
234        }
235        completion = tasks.join_next() => {
236            if let Some(Ok(completion)) = completion {
237                record(completion, &mut grpc_result, &mut http_result, "completed");
238            }
239        }
240    }
241
242    // Broadcast shutdown to both services.
243    let _ = shutdown_sender.send(());
244
245    // Phase 2: drain every remaining task under one bounded timeout.
246    let drain_result = tokio::time::timeout(drain_timeout, async {
247        while let Some(joined) = tasks.join_next().await {
248            if let Ok(completion) = joined {
249                record(
250                    completion,
251                    &mut grpc_result,
252                    &mut http_result,
253                    "during drain",
254                );
255            }
256        }
257    })
258    .await;
259
260    // Phase 3: abort the original service tasks and drain the wrapper join
261    // records so cancellation is observed through the original handles.
262    let forced = drain_result.is_err();
263    if forced {
264        tracing::warn!(
265            "Graceful shutdown timed out after {}s, aborting remaining tasks",
266            drain_timeout.as_secs()
267        );
268
269        grpc_abort.abort();
270        http_abort.abort();
271        while let Some(joined) = tasks.join_next().await {
272            if let Ok(completion) = joined {
273                record(
274                    completion,
275                    &mut grpc_result,
276                    &mut http_result,
277                    "after abort",
278                );
279            }
280        }
281    }
282
283    ServiceShutdownOutcome {
284        requested,
285        forced,
286        grpc_result: grpc_result
287            .unwrap_or_else(|| ServiceResult::Cancelled("not observed".to_owned())),
288        http_result: http_result
289            .unwrap_or_else(|| ServiceResult::Cancelled("not observed".to_owned())),
290    }
291}
292
293#[cfg(test)]
294mod tests {
295    use super::*;
296    use std::sync::Arc;
297    use std::sync::atomic::{AtomicUsize, Ordering};
298
299    type GrpcHandle = tokio::task::JoinHandle<Result<(), tonic::transport::Error>>;
300    type HttpHandle = tokio::task::JoinHandle<Result<(), std::io::Error>>;
301
302    // ── Section 7.1: Requested clean shutdown ──────────────────────
303
304    #[tokio::test]
305    async fn requested_clean_shutdown() {
306        let (tx, _) = tokio::sync::broadcast::channel::<()>(1);
307
308        let grpc: GrpcHandle = tokio::spawn({
309            let mut rx = tx.subscribe();
310            async move {
311                let _ = rx.recv().await;
312                Ok(())
313            }
314        });
315        let http: HttpHandle = tokio::spawn({
316            let mut rx = tx.subscribe();
317            async move {
318                let _ = rx.recv().await;
319                Ok(())
320            }
321        });
322
323        tokio::time::sleep(Duration::from_millis(50)).await;
324
325        // Signal completes immediately so the helper itself must be the
326        // sole sender on the broadcast shutdown channel. Services cannot
327        // wake from any test-side send.
328        let signal = std::future::ready(());
329        tokio::pin!(signal);
330
331        let outcome =
332            run_services_until_shutdown(signal, grpc, http, tx, Duration::from_secs(2)).await;
333
334        assert!(outcome.requested);
335        assert!(!outcome.forced);
336        assert!(outcome.is_clean_requested_shutdown());
337        assert!(
338            outcome.ensure_clean_requested_shutdown().is_ok(),
339            "production decision method must return Ok for clean requested shutdown"
340        );
341        assert_eq!(outcome.grpc_result, ServiceResult::Clean);
342        assert_eq!(outcome.http_result, ServiceResult::Clean);
343    }
344
345    // ── Section 7.2: One service completes during drain, sibling times out ──
346
347    #[tokio::test]
348    async fn one_service_completes_sibling_times_out() {
349        let (tx, _) = tokio::sync::broadcast::channel::<()>(1);
350        let grpc_await_count = Arc::new(AtomicUsize::new(0));
351        let http_drop_count = Arc::new(AtomicUsize::new(0));
352
353        let grpc: GrpcHandle = {
354            let count = grpc_await_count.clone();
355            let mut rx = tx.subscribe();
356            tokio::spawn(async move {
357                let _ = rx.recv().await;
358                count.fetch_add(1, Ordering::SeqCst);
359                Ok(())
360            })
361        };
362        // HTTP never completes — it ignores the shutdown signal.
363        let http: HttpHandle = {
364            let drop_count = http_drop_count.clone();
365            tokio::spawn(async move {
366                struct DropProbe(Arc<AtomicUsize>);
367                impl Drop for DropProbe {
368                    fn drop(&mut self) {
369                        self.0.fetch_add(1, Ordering::SeqCst);
370                    }
371                }
372
373                let _probe = DropProbe(drop_count);
374                std::future::pending::<Result<(), std::io::Error>>().await
375            })
376        };
377
378        tokio::time::sleep(Duration::from_millis(50)).await;
379
380        let signal = std::future::ready(());
381        tokio::pin!(signal);
382
383        let outcome =
384            run_services_until_shutdown(signal, grpc, http, tx, Duration::from_millis(100)).await;
385
386        assert!(outcome.requested);
387        assert!(outcome.forced);
388        assert_eq!(outcome.grpc_result, ServiceResult::Clean);
389        assert_eq!(
390            grpc_await_count.load(Ordering::SeqCst),
391            1,
392            "gRPC future should have completed exactly once"
393        );
394        assert!(matches!(outcome.http_result, ServiceResult::Cancelled(_)));
395        assert_eq!(
396            http_drop_count.load(Ordering::SeqCst),
397            1,
398            "the underlying HTTP future must be dropped before the helper returns"
399        );
400        let err = outcome
401            .ensure_clean_requested_shutdown()
402            .expect_err("forced shutdown must fail production check");
403        assert!(
404            err.contains("forced=true"),
405            "diagnostic should mention forced: {err}"
406        );
407    }
408
409    // ── Section 7.4: Drain-time service panic ──────────────────────
410
411    #[tokio::test]
412    async fn drain_time_service_panic() {
413        let (tx, _) = tokio::sync::broadcast::channel::<()>(1);
414
415        let grpc: GrpcHandle = {
416            let mut rx = tx.subscribe();
417            tokio::spawn(async move {
418                let _ = rx.recv().await;
419                panic!("grpc drain panic");
420            })
421        };
422        let http: HttpHandle = {
423            let mut rx = tx.subscribe();
424            tokio::spawn(async move {
425                let _ = rx.recv().await;
426                Ok(())
427            })
428        };
429
430        tokio::time::sleep(Duration::from_millis(50)).await;
431
432        let signal = std::future::ready(());
433        tokio::pin!(signal);
434
435        let outcome =
436            run_services_until_shutdown(signal, grpc, http, tx, Duration::from_secs(2)).await;
437
438        assert!(outcome.requested);
439        assert!(!outcome.forced);
440        assert!(!outcome.is_clean_requested_shutdown());
441        assert!(matches!(outcome.grpc_result, ServiceResult::Panic(_)));
442        assert_eq!(outcome.http_result, ServiceResult::Clean);
443        let err = outcome
444            .ensure_clean_requested_shutdown()
445            .expect_err("panic must fail production check");
446        assert!(
447            err.contains("grpc"),
448            "diagnostic should mention grpc: {err}"
449        );
450        assert!(
451            err.contains("grpc drain panic"),
452            "diagnostic should retain original panic detail: {err}"
453        );
454    }
455
456    // ── Section 7.3: Drain-time service error ─────────────────────
457
458    #[tokio::test]
459    async fn drain_time_service_error() {
460        let (tx, _) = tokio::sync::broadcast::channel::<()>(1);
461
462        // gRPC returns cleanly after receiving shutdown.
463        let grpc: GrpcHandle = {
464            let mut rx = tx.subscribe();
465            tokio::spawn(async move {
466                let _ = rx.recv().await;
467                Ok(())
468            })
469        };
470        // HTTP returns an error after receiving shutdown.
471        let http: HttpHandle = {
472            let mut rx = tx.subscribe();
473            tokio::spawn(async move {
474                let _ = rx.recv().await;
475                Err(std::io::Error::other("http service error"))
476            })
477        };
478
479        tokio::time::sleep(Duration::from_millis(50)).await;
480
481        let signal = std::future::ready(());
482        tokio::pin!(signal);
483
484        let outcome =
485            run_services_until_shutdown(signal, grpc, http, tx, Duration::from_secs(2)).await;
486
487        assert!(outcome.requested);
488        assert!(!outcome.forced);
489        assert!(!outcome.is_clean_requested_shutdown());
490        assert_eq!(outcome.grpc_result, ServiceResult::Clean);
491        assert!(matches!(
492            outcome.http_result,
493            ServiceResult::ServiceError(_)
494        ));
495        let err = outcome
496            .ensure_clean_requested_shutdown()
497            .expect_err("drain-time error must fail production check");
498        assert!(
499            err.contains("http"),
500            "diagnostic should mention http: {err}"
501        );
502        assert!(
503            err.contains("http service error"),
504            "diagnostic should retain original error detail: {err}"
505        );
506    }
507
508    // ── Section 7.5: Unexpected service completion ─────────────────
509
510    #[tokio::test]
511    async fn unexpected_grpc_exit() {
512        let (tx, _) = tokio::sync::broadcast::channel::<()>(1);
513
514        let grpc: GrpcHandle = tokio::spawn(async { Ok(()) });
515        let http: HttpHandle = {
516            let mut rx = tx.subscribe();
517            tokio::spawn(async move {
518                let _ = rx.recv().await;
519                Ok(())
520            })
521        };
522
523        tokio::time::sleep(Duration::from_millis(50)).await;
524
525        // Signal held pending via oneshot — helper must wake from gRPC
526        // completion, not from a test-side broadcast send.
527        let (_signal_tx, signal_rx) = tokio::sync::oneshot::channel::<()>();
528        let signal = async move {
529            let _ = signal_rx.await;
530        };
531        tokio::pin!(signal);
532
533        let outcome =
534            run_services_until_shutdown(signal, grpc, http, tx, Duration::from_secs(2)).await;
535
536        assert!(!outcome.requested);
537        assert!(!outcome.forced);
538        assert!(!outcome.is_clean_requested_shutdown());
539        let err = outcome
540            .ensure_clean_requested_shutdown()
541            .expect_err("unexpected clean service exit must fail production check");
542        assert!(
543            err.contains("requested=false"),
544            "diagnostic should mention requested=false: {err}"
545        );
546    }
547
548    #[tokio::test]
549    async fn unexpected_http_error() {
550        let (tx, _) = tokio::sync::broadcast::channel::<()>(1);
551
552        let grpc: GrpcHandle = {
553            let mut rx = tx.subscribe();
554            tokio::spawn(async move {
555                let _ = rx.recv().await;
556                Ok(())
557            })
558        };
559        let http: HttpHandle = tokio::spawn(async { Err(std::io::Error::other("http boom")) });
560
561        tokio::time::sleep(Duration::from_millis(50)).await;
562
563        let (_signal_tx, signal_rx) = tokio::sync::oneshot::channel::<()>();
564        let signal = async move {
565            let _ = signal_rx.await;
566        };
567        tokio::pin!(signal);
568
569        let outcome =
570            run_services_until_shutdown(signal, grpc, http, tx, Duration::from_secs(2)).await;
571
572        assert!(!outcome.requested);
573        assert!(!outcome.forced);
574        assert!(!outcome.is_clean_requested_shutdown());
575        let err = outcome
576            .ensure_clean_requested_shutdown()
577            .expect_err("unexpected service error must fail production check");
578        assert!(
579            err.contains("http"),
580            "diagnostic should mention http: {err}"
581        );
582    }
583
584    // ── Section 7.6: Both services refuse to drain ─────────────────
585
586    #[tokio::test]
587    async fn both_refuse_to_drain() {
588        let (tx, _) = tokio::sync::broadcast::channel::<()>(1);
589
590        let grpc: GrpcHandle = tokio::spawn(async {
591            std::future::pending::<Result<(), tonic::transport::Error>>().await
592        });
593        let http: HttpHandle =
594            tokio::spawn(async { std::future::pending::<Result<(), std::io::Error>>().await });
595
596        tokio::time::sleep(Duration::from_millis(50)).await;
597
598        let signal = std::future::ready(());
599        tokio::pin!(signal);
600
601        let start = std::time::Instant::now();
602        let outcome =
603            run_services_until_shutdown(signal, grpc, http, tx, Duration::from_millis(100)).await;
604        let elapsed = start.elapsed();
605
606        assert!(outcome.requested);
607        assert!(outcome.forced);
608        assert!(!outcome.is_clean_requested_shutdown());
609        assert!(
610            elapsed < Duration::from_secs(2),
611            "test should complete quickly, took {elapsed:?}"
612        );
613        let err = outcome
614            .ensure_clean_requested_shutdown()
615            .expect_err("forced abort must fail production check");
616        assert!(
617            err.contains("forced=true"),
618            "diagnostic should mention forced: {err}"
619        );
620    }
621
622    // ── Section 7.7: No pre-signal lifetime timeout ────────────────
623
624    #[tokio::test]
625    async fn no_pre_signal_lifetime_timeout() {
626        let drain_timeout = Duration::from_millis(100);
627
628        // ── Phase A: helper remains pending while no terminal event ──
629        // The helper must not start the drain timeout until a terminal
630        // event has occurred. We construct the helper with a held-pending
631        // oneshot signal and no pre-completing services, then assert that
632        // it does not complete within 2× the drain timeout.
633        {
634            let (tx, _) = tokio::sync::broadcast::channel::<()>(1);
635            let (_signal_tx, signal_rx) = tokio::sync::oneshot::channel::<()>();
636            let grpc: GrpcHandle = {
637                let mut rx = tx.subscribe();
638                tokio::spawn(async move {
639                    let _ = rx.recv().await;
640                    Ok(())
641                })
642            };
643            let http: HttpHandle = {
644                let mut rx = tx.subscribe();
645                tokio::spawn(async move {
646                    let _ = rx.recv().await;
647                    Ok(())
648                })
649            };
650            let mut signal = Box::pin(async move {
651                let _ = signal_rx.await;
652            });
653            let helper_fut =
654                run_services_until_shutdown(signal.as_mut(), grpc, http, tx, drain_timeout);
655            let phase_a_result = tokio::time::timeout(drain_timeout * 2, helper_fut).await;
656            assert!(
657                phase_a_result.is_err(),
658                "helper must remain pending for at least 2× drain timeout when no terminal event has occurred"
659            );
660        }
661
662        // ── Phase B: signal triggers and helper completes cleanly ──
663        // Trigger the oneshot BEFORE polling the helper so the first
664        // poll sees the signal ready, then verify a clean requested
665        // shutdown.
666        {
667            let (tx, _) = tokio::sync::broadcast::channel::<()>(1);
668            let (signal_tx, signal_rx) = tokio::sync::oneshot::channel::<()>();
669            let grpc: GrpcHandle = {
670                let mut rx = tx.subscribe();
671                tokio::spawn(async move {
672                    let _ = rx.recv().await;
673                    Ok(())
674                })
675            };
676            let http: HttpHandle = {
677                let mut rx = tx.subscribe();
678                tokio::spawn(async move {
679                    let _ = rx.recv().await;
680                    Ok(())
681                })
682            };
683            let mut signal = Box::pin(async move {
684                let _ = signal_rx.await;
685            });
686            let _ = signal_tx.send(());
687            let helper_fut =
688                run_services_until_shutdown(signal.as_mut(), grpc, http, tx, drain_timeout);
689            let outcome = tokio::time::timeout(Duration::from_secs(5), helper_fut)
690                .await
691                .expect("helper should complete within 5s after signal trigger");
692            assert!(outcome.requested);
693            assert!(!outcome.forced);
694            assert!(outcome.is_clean_requested_shutdown());
695            assert!(
696                outcome.ensure_clean_requested_shutdown().is_ok(),
697                "production decision method must return Ok for clean requested shutdown"
698            );
699        }
700    }
701
702    // ── Regression: completed gRPC service is observed once ───────
703
704    #[tokio::test]
705    async fn grpc_service_completion_is_observed_once() {
706        let (tx, _) = tokio::sync::broadcast::channel::<()>(1);
707        let poll_count = Arc::new(AtomicUsize::new(0));
708
709        let grpc: GrpcHandle = {
710            let count = poll_count.clone();
711            let mut rx = tx.subscribe();
712            tokio::spawn(async move {
713                let _ = rx.recv().await;
714                count.fetch_add(1, Ordering::SeqCst);
715                Ok(())
716            })
717        };
718        let http: HttpHandle =
719            tokio::spawn(async { std::future::pending::<Result<(), std::io::Error>>().await });
720
721        tokio::time::sleep(Duration::from_millis(50)).await;
722
723        let signal = std::future::ready(());
724        tokio::pin!(signal);
725
726        let outcome =
727            run_services_until_shutdown(signal, grpc, http, tx, Duration::from_millis(100)).await;
728
729        assert!(outcome.forced);
730        assert_eq!(
731            poll_count.load(Ordering::SeqCst),
732            1,
733            "gRPC future should have been polled exactly once"
734        );
735    }
736
737    // ── Section 7.4 regression: drain-time panic preserves detail ──
738
739    #[tokio::test]
740    async fn drain_panic_preserved_in_outcome() {
741        let (tx, _) = tokio::sync::broadcast::channel::<()>(1);
742
743        let grpc: GrpcHandle = {
744            let mut rx = tx.subscribe();
745            tokio::spawn(async move {
746                let _ = rx.recv().await;
747                panic!("deliberate grpc panic");
748            })
749        };
750        let http: HttpHandle = {
751            let mut rx = tx.subscribe();
752            tokio::spawn(async move {
753                let _ = rx.recv().await;
754                Ok(())
755            })
756        };
757
758        tokio::time::sleep(Duration::from_millis(50)).await;
759
760        let signal = std::future::ready(());
761        tokio::pin!(signal);
762
763        let outcome =
764            run_services_until_shutdown(signal, grpc, http, tx, Duration::from_secs(2)).await;
765
766        assert!(outcome.requested);
767        assert!(!outcome.forced);
768        assert!(!outcome.is_clean_requested_shutdown());
769        match &outcome.grpc_result {
770            ServiceResult::Panic(msg) => {
771                assert!(
772                    msg.contains("deliberate grpc panic"),
773                    "panic message should be preserved"
774                );
775            }
776            other => panic!("expected Panic, got {other:?}"),
777        }
778    }
779
780    // ── Section 7.6 regression: HTTP aborted while gRPC clean ──────
781
782    #[tokio::test]
783    async fn http_aborted_while_grpc_clean() {
784        let (tx, _) = tokio::sync::broadcast::channel::<()>(1);
785
786        let grpc: GrpcHandle = {
787            let mut rx = tx.subscribe();
788            tokio::spawn(async move {
789                let _ = rx.recv().await;
790                Ok(())
791            })
792        };
793        let http: HttpHandle =
794            tokio::spawn(async { std::future::pending::<Result<(), std::io::Error>>().await });
795
796        tokio::time::sleep(Duration::from_millis(50)).await;
797
798        let signal = std::future::ready(());
799        tokio::pin!(signal);
800
801        let outcome =
802            run_services_until_shutdown(signal, grpc, http, tx, Duration::from_millis(100)).await;
803
804        assert!(outcome.requested);
805        assert!(outcome.forced);
806        assert_eq!(outcome.grpc_result, ServiceResult::Clean);
807        assert!(matches!(
808            outcome.http_result,
809            ServiceResult::Cancelled(_) | ServiceResult::Panic(_)
810        ));
811    }
812
813    // ── Section 7.1 regression: both complete during drain ──────────
814
815    #[tokio::test]
816    async fn both_complete_during_drain() {
817        let (tx, _) = tokio::sync::broadcast::channel::<()>(1);
818
819        let grpc: GrpcHandle = {
820            let mut rx = tx.subscribe();
821            tokio::spawn(async move {
822                let _ = rx.recv().await;
823                tokio::time::sleep(Duration::from_millis(10)).await;
824                Ok(())
825            })
826        };
827        let http: HttpHandle = {
828            let mut rx = tx.subscribe();
829            tokio::spawn(async move {
830                let _ = rx.recv().await;
831                tokio::time::sleep(Duration::from_millis(20)).await;
832                Ok(())
833            })
834        };
835
836        tokio::time::sleep(Duration::from_millis(50)).await;
837
838        let signal = std::future::ready(());
839        tokio::pin!(signal);
840
841        let outcome =
842            run_services_until_shutdown(signal, grpc, http, tx, Duration::from_secs(2)).await;
843
844        assert!(outcome.requested);
845        assert!(!outcome.forced);
846        assert!(outcome.is_clean_requested_shutdown());
847    }
848}
849
850#[cfg(test)]
851mod eggserve_tests {
852    use super::*;
853
854    #[tokio::test]
855    async fn eggserve_runtime_drains_after_requested_shutdown() {
856        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
857        let server = eggserve_server::Server::builder()
858            .from_listener(listener)
859            .build()
860            .unwrap();
861        let handle = server
862            .start_with_service(eggserve_server::service_fn(|_request| async {
863                Ok(eggserve_primitives::Response::builder()
864                    .status(eggserve_primitives::StatusCode::OK)
865                    .body(eggserve_primitives::ResponseBody::Empty)
866                    .unwrap())
867            }))
868            .await
869            .unwrap();
870        let (control, completion) = handle.into_parts();
871        let (shutdown_tx, _) = tokio::sync::broadcast::channel(1);
872        let grpc = tokio::spawn({
873            let mut shutdown_rx = shutdown_tx.subscribe();
874            async move {
875                let _ = shutdown_rx.recv().await;
876                Ok::<(), tonic::transport::Error>(())
877            }
878        });
879        let signal = std::future::ready(());
880        tokio::pin!(signal);
881
882        let outcome = run_eggserve_services_until_shutdown(
883            signal,
884            grpc,
885            control,
886            completion,
887            shutdown_tx,
888            Duration::from_secs(2),
889        )
890        .await;
891
892        assert!(outcome.is_clean_requested_shutdown(), "{outcome:?}");
893    }
894}