use std::time::Duration;
use async_trait::async_trait;
use tokio::sync::mpsc;
use tokio::task::{AbortHandle, JoinHandle, JoinSet};
use tokio_util::sync::CancellationToken;
use crate::chain::ProviderId;
use crate::error::ChainViewError;
use crate::terminal::TerminalGuard;
pub const DEFAULT_JOIN_BUDGET: Duration = Duration::from_secs(2);
const EXIT_REPORT_CAPACITY: usize = 64;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum TaskExit {
Completed,
Panicked,
}
#[async_trait]
pub trait SupervisedTask: Send {
async fn join(&mut self) -> TaskExit;
fn abort(&mut self);
}
#[derive(Debug)]
pub struct TokioTask {
handle: Option<tokio::task::JoinHandle<()>>,
}
impl TokioTask {
#[must_use]
pub fn new(handle: tokio::task::JoinHandle<()>) -> Self {
Self {
handle: Some(handle),
}
}
}
#[async_trait]
impl SupervisedTask for TokioTask {
async fn join(&mut self) -> TaskExit {
let Some(handle) = self.handle.as_mut() else {
return TaskExit::Completed;
};
let result = handle.await;
self.handle = None;
match result {
Ok(()) => TaskExit::Completed,
Err(error) if error.is_panic() => TaskExit::Panicked,
Err(_) => TaskExit::Completed,
}
}
fn abort(&mut self) {
if let Some(handle) = self.handle.as_ref() {
handle.abort();
}
}
}
pub trait FinalTeardown: Send {
fn run(self: Box<Self>);
}
pub struct GuardTeardown {
guard: TerminalGuard,
}
impl std::fmt::Debug for GuardTeardown {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("GuardTeardown").finish_non_exhaustive()
}
}
impl GuardTeardown {
#[must_use]
pub fn new(guard: TerminalGuard) -> Self {
Self { guard }
}
}
impl FinalTeardown for GuardTeardown {
fn run(self: Box<Self>) {
drop(self.guard);
}
}
#[derive(Debug)]
pub enum ExitCause {
Clean,
TaskPanicked,
Failed(ChainViewError),
}
impl ExitCause {
#[must_use]
pub fn exit_code(&self) -> i32 {
match self {
ExitCause::Clean => 0,
ExitCause::TaskPanicked | ExitCause::Failed(_) => 1,
}
}
#[must_use]
pub fn is_clean(&self) -> bool {
matches!(self, ExitCause::Clean)
}
#[must_use]
pub fn failure_message(&self) -> Option<String> {
match self {
ExitCause::Clean => None,
ExitCause::TaskPanicked => {
Some("chainview: a supervised task panicked; see the log".to_owned())
}
ExitCause::Failed(error) => Some(format!("chainview: {error}")),
}
}
}
#[derive(Debug, Clone)]
pub struct ExitReporter {
tx: mpsc::Sender<TaskExit>,
}
impl ExitReporter {
pub fn report(&self, exit: TaskExit) {
let _ = self.tx.try_send(exit);
}
}
struct Supervised {
child: CancellationToken,
task: Box<dyn SupervisedTask>,
}
pub struct Supervisor {
root: CancellationToken,
providers: Vec<(ProviderId, Supervised)>,
ancillary: Vec<Supervised>,
render: Option<Supervised>,
terminal: Box<dyn FinalTeardown>,
budget: Duration,
fatal: Option<ExitCause>,
rx_exit: mpsc::Receiver<TaskExit>,
tx_exit: mpsc::Sender<TaskExit>,
watchers: JoinSet<TaskExit>,
watch_aborts: Vec<AbortHandle>,
}
impl Supervisor {
#[must_use]
pub fn new(terminal: Box<dyn FinalTeardown>) -> Self {
Self::with_join_budget(terminal, DEFAULT_JOIN_BUDGET)
}
#[must_use]
pub fn with_join_budget(terminal: Box<dyn FinalTeardown>, budget: Duration) -> Self {
let (tx_exit, rx_exit) = mpsc::channel(EXIT_REPORT_CAPACITY);
Self {
root: CancellationToken::new(),
providers: Vec::new(),
ancillary: Vec::new(),
render: None,
terminal,
budget,
fatal: None,
rx_exit,
tx_exit,
watchers: JoinSet::new(),
watch_aborts: Vec::new(),
}
}
#[must_use]
pub fn root_token(&self) -> CancellationToken {
self.root.clone()
}
#[must_use]
pub fn child_token(&self) -> CancellationToken {
self.root.child_token()
}
#[must_use]
pub fn exit_reporter(&self) -> ExitReporter {
ExitReporter {
tx: self.tx_exit.clone(),
}
}
pub fn watch(&mut self, handle: JoinHandle<()>) -> AbortHandle {
let task_abort = handle.abort_handle();
self.watch_aborts.push(task_abort.clone());
self.watchers.spawn(async move {
match handle.await {
Ok(()) => TaskExit::Completed,
Err(error) if error.is_panic() => TaskExit::Panicked,
Err(_) => TaskExit::Completed,
}
});
task_abort
}
pub fn register_provider(
&mut self,
id: ProviderId,
child: CancellationToken,
task: Box<dyn SupervisedTask>,
) {
self.providers.push((id, Supervised { child, task }));
}
pub fn register_ancillary(&mut self, child: CancellationToken, task: Box<dyn SupervisedTask>) {
self.ancillary.push(Supervised { child, task });
}
pub fn set_render(&mut self, child: CancellationToken, task: Box<dyn SupervisedTask>) {
self.render = Some(Supervised { child, task });
}
pub fn cancel_provider(&self, id: &ProviderId) -> bool {
for (provider_id, supervised) in &self.providers {
if provider_id == id {
supervised.child.cancel();
return true;
}
}
false
}
pub fn request_quit(&self) {
self.root.cancel();
}
pub fn fail(&mut self, error: ChainViewError) {
self.trip_fatal(ExitCause::Failed(error));
}
fn trip_fatal(&mut self, cause: ExitCause) {
if self.fatal.is_none() {
self.fatal = Some(cause);
}
self.root.cancel();
}
#[must_use = "the returned ExitCause is main's exit code + post-restore stderr line"]
pub async fn run(mut self) -> ExitCause {
crate::terminal::set_supervisor_owns_restore(true);
self.supervise().await;
self.teardown().await
}
async fn supervise(&mut self) {
let root = self.root.clone();
tokio::select! {
() = root.cancelled_owned() => {}
report = self.rx_exit.recv() => match report {
Some(TaskExit::Panicked) => self.trip_fatal(ExitCause::TaskPanicked),
Some(TaskExit::Completed) | None => self.request_quit(),
},
watched = self.watchers.join_next(), if !self.watchers.is_empty() => match watched {
Some(Ok(TaskExit::Panicked)) => self.trip_fatal(ExitCause::TaskPanicked),
Some(Ok(TaskExit::Completed)) | Some(Err(_)) | None => self.request_quit(),
},
}
}
async fn teardown(mut self) -> ExitCause {
let budget = self.budget;
self.root.cancel();
let mut fatal = self.fatal.take();
drain_watchers(&mut self.watchers, &self.watch_aborts, budget, &mut fatal).await;
for (_id, supervised) in &mut self.providers {
let outcome = join_supervised(supervised, budget).await;
record_group_outcome(&mut fatal, outcome);
}
for supervised in &mut self.ancillary {
let outcome = join_supervised(supervised, budget).await;
record_group_outcome(&mut fatal, outcome);
}
if let Some(render) = self.render.as_mut() {
let outcome = join_supervised(render, budget).await;
record_group_outcome(&mut fatal, outcome);
}
let terminal = self.terminal;
terminal.run();
crate::terminal::set_supervisor_owns_restore(false);
fatal.unwrap_or(ExitCause::Clean)
}
}
async fn join_supervised(supervised: &mut Supervised, budget: Duration) -> JoinBudget {
supervised.child.cancel();
join_bounded(supervised.task.as_mut(), budget).await
}
async fn join_bounded(task: &mut dyn SupervisedTask, budget: Duration) -> JoinBudget {
match tokio::time::timeout(budget, task.join()).await {
Ok(exit) => JoinBudget::Returned(exit),
Err(_elapsed) => {
task.abort();
let _ = tokio::time::timeout(budget, task.join()).await;
JoinBudget::AbortedAfterBudget
}
}
}
async fn drain_watchers(
watchers: &mut JoinSet<TaskExit>,
watch_aborts: &[AbortHandle],
budget: Duration,
fatal: &mut Option<ExitCause>,
) {
while !watchers.is_empty() {
match tokio::time::timeout(budget, watchers.join_next()).await {
Ok(Some(Ok(exit))) => record_group_outcome(fatal, JoinBudget::Returned(exit)),
Ok(Some(Err(_))) => {}
Ok(None) => break,
Err(_elapsed) => {
for task_abort in watch_aborts {
task_abort.abort();
}
let _ = tokio::time::timeout(budget, async {
while watchers.join_next().await.is_some() {}
})
.await;
break;
}
}
}
}
enum JoinBudget {
Returned(TaskExit),
AbortedAfterBudget,
}
fn record_group_outcome(fatal: &mut Option<ExitCause>, outcome: JoinBudget) {
if let JoinBudget::Returned(TaskExit::Panicked) = outcome
&& fatal.is_none()
{
*fatal = Some(ExitCause::TaskPanicked);
}
}
#[cfg(test)]
mod tests {
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
use super::{
ExitCause, FinalTeardown, JoinBudget, SupervisedTask, Supervisor, TaskExit, TokioTask,
join_bounded,
};
use crate::chain::ProviderId;
use crate::error::ChainViewError;
type Log = Arc<Mutex<Vec<String>>>;
fn new_log() -> Log {
Arc::new(Mutex::new(Vec::new()))
}
fn record(log: &Log, step: String) {
if let Ok(mut guard) = log.lock() {
guard.push(step);
}
}
fn steps(log: &Log) -> Vec<String> {
match log.lock() {
Ok(guard) => guard.clone(),
Err(_) => Vec::new(),
}
}
#[track_caller]
fn position(steps: &[String], target: &str) -> usize {
match steps.iter().position(|step| step == target) {
Some(index) => index,
None => panic!("missing step `{target}` in {steps:?}"),
}
}
fn has_step(steps: &[String], target: &str) -> bool {
steps.iter().any(|step| step == target)
}
#[track_caller]
fn pid(id: &str) -> ProviderId {
match ProviderId::new(id) {
Ok(provider) => provider,
Err(error) => panic!("expected a valid provider id `{id}`, got: {error}"),
}
}
enum Behavior {
Cooperative,
Wedged,
}
struct MockTask {
label: String,
child: tokio_util::sync::CancellationToken,
behavior: Behavior,
exit: TaskExit,
log: Log,
}
impl MockTask {
fn cooperative(label: &str, child: tokio_util::sync::CancellationToken, log: Log) -> Self {
Self {
label: label.to_owned(),
child,
behavior: Behavior::Cooperative,
exit: TaskExit::Completed,
log,
}
}
fn panicking(label: &str, child: tokio_util::sync::CancellationToken, log: Log) -> Self {
Self {
label: label.to_owned(),
child,
behavior: Behavior::Cooperative,
exit: TaskExit::Panicked,
log,
}
}
fn wedged(label: &str, child: tokio_util::sync::CancellationToken, log: Log) -> Self {
Self {
label: label.to_owned(),
child,
behavior: Behavior::Wedged,
exit: TaskExit::Completed,
log,
}
}
}
#[async_trait::async_trait]
impl SupervisedTask for MockTask {
async fn join(&mut self) -> TaskExit {
match self.behavior {
Behavior::Cooperative => {
self.child.cancelled().await;
record(&self.log, format!("join:{}", self.label));
self.exit
}
Behavior::Wedged => std::future::pending::<TaskExit>().await,
}
}
fn abort(&mut self) {
record(&self.log, format!("abort:{}", self.label));
}
}
struct RecordingTeardown {
log: Log,
}
impl FinalTeardown for RecordingTeardown {
fn run(self: Box<Self>) {
record(&self.log, "terminal_restore".to_owned());
}
}
fn recording_supervisor(log: &Log) -> Supervisor {
Supervisor::new(Box::new(RecordingTeardown { log: log.clone() }))
}
#[test]
fn test_exit_cause_exit_code_is_zero_for_clean_nonzero_for_failure() {
assert_eq!(ExitCause::Clean.exit_code(), 0);
assert_eq!(ExitCause::TaskPanicked.exit_code(), 1);
let failed = ExitCause::Failed(ChainViewError::Terminal("boom".to_owned()));
assert_eq!(failed.exit_code(), 1);
}
#[test]
fn test_exit_cause_failure_message_absent_for_clean_present_for_failure() {
assert!(ExitCause::Clean.failure_message().is_none());
assert!(ExitCause::TaskPanicked.failure_message().is_some());
let failed = ExitCause::Failed(ChainViewError::Terminal("boom".to_owned()));
assert!(
failed
.failure_message()
.is_some_and(|message| message.contains("boom"))
);
}
#[test]
fn test_supervisor_request_quit_cascades_to_children_without_fatal() {
let log = new_log();
let supervisor = recording_supervisor(&log);
let child = supervisor.child_token();
supervisor.request_quit();
assert!(supervisor.root_token().is_cancelled());
assert!(
child.is_cancelled(),
"cancelling the root cascades to a child"
);
}
#[tokio::test]
async fn test_supervisor_cancel_provider_cancels_only_that_child() {
let log = new_log();
let mut supervisor = recording_supervisor(&log);
let first = supervisor.child_token();
supervisor.register_provider(
pid("deribit"),
first.clone(),
Box::new(MockTask::cooperative("deribit", first.clone(), log.clone())),
);
let second = supervisor.child_token();
supervisor.register_provider(
pid("dxlink"),
second.clone(),
Box::new(MockTask::cooperative("dxlink", second.clone(), log.clone())),
);
assert!(supervisor.cancel_provider(&pid("deribit")));
assert!(first.is_cancelled(), "the targeted provider is cancelled");
assert!(!second.is_cancelled(), "the other provider is untouched");
assert!(
!supervisor.root_token().is_cancelled(),
"a per-provider cancel must not trip the root"
);
assert!(
!supervisor.cancel_provider(&pid("alpaca")),
"an unregistered provider id reports not-found"
);
supervisor.request_quit();
let _ = supervisor.run().await;
}
#[tokio::test(start_paused = true)]
async fn test_join_bounded_wedged_task_is_aborted_after_budget() {
let log = new_log();
let child = tokio_util::sync::CancellationToken::new();
let mut task = MockTask::wedged("provider", child, log.clone());
let start = tokio::time::Instant::now();
let outcome = join_bounded(&mut task, super::DEFAULT_JOIN_BUDGET).await;
let elapsed = start.elapsed();
assert!(
matches!(outcome, JoinBudget::AbortedAfterBudget),
"a task ignoring cancellation past the budget is aborted"
);
assert!(
elapsed >= super::DEFAULT_JOIN_BUDGET,
"the budget was honored in virtual time (zero real wait): {elapsed:?}"
);
assert!(has_step(&steps(&log), "abort:provider"));
}
#[tokio::test(start_paused = true)]
async fn test_join_bounded_cooperative_task_returns_within_budget_without_abort() {
let log = new_log();
let child = tokio_util::sync::CancellationToken::new();
child.cancel();
let mut task = MockTask::cooperative("provider", child, log.clone());
let outcome = join_bounded(&mut task, super::DEFAULT_JOIN_BUDGET).await;
assert!(matches!(outcome, JoinBudget::Returned(TaskExit::Completed)));
let recorded = steps(&log);
assert!(has_step(&recorded, "join:provider"));
assert!(
!recorded.iter().any(|step| step.starts_with("abort:")),
"a task that returns in time is never aborted"
);
}
struct DropFlag(Arc<AtomicBool>);
impl Drop for DropFlag {
fn drop(&mut self) {
self.0.store(true, Ordering::SeqCst);
}
}
#[tokio::test(start_paused = true)]
async fn test_tokio_task_bounded_join_wedged_real_task_is_aborted_not_detached() {
let completed = Arc::new(AtomicBool::new(false));
let dropped = Arc::new(AtomicBool::new(false));
let guard = DropFlag(dropped.clone());
let ran = completed.clone();
let handle = tokio::spawn(async move {
let _guard = guard;
std::future::pending::<()>().await;
ran.store(true, Ordering::SeqCst);
});
let abort_handle = handle.abort_handle();
let mut task = TokioTask::new(handle);
let outcome = join_bounded(&mut task, super::DEFAULT_JOIN_BUDGET).await;
assert!(matches!(outcome, JoinBudget::AbortedAfterBudget));
for _ in 0..4 {
tokio::task::yield_now().await;
}
assert!(
!completed.load(Ordering::SeqCst),
"the wedged task must never run to completion"
);
assert!(
dropped.load(Ordering::SeqCst),
"the real task was TRULY aborted (its future dropped), not detached as an orphan"
);
assert!(
abort_handle.is_finished(),
"the real task handle reports finished after a genuine abort"
);
}
#[tokio::test(start_paused = true)]
async fn test_join_bounded_awaits_aborted_task_before_returning() {
let dropped = Arc::new(AtomicBool::new(false));
let guard = DropFlag(dropped.clone());
let handle = tokio::spawn(async move {
let _guard = guard;
std::future::pending::<()>().await;
});
let mut task = TokioTask::new(handle);
let outcome = join_bounded(&mut task, super::DEFAULT_JOIN_BUDGET).await;
assert!(matches!(outcome, JoinBudget::AbortedAfterBudget));
assert!(
dropped.load(Ordering::SeqCst),
"the aborted task was AWAITED to completion before join_bounded returned"
);
}
#[tokio::test]
async fn test_supervisor_teardown_joins_providers_before_ancillary_then_render_then_restore() {
let log = new_log();
let mut supervisor = recording_supervisor(&log);
let provider_child = supervisor.child_token();
supervisor.register_provider(
pid("deribit"),
provider_child.clone(),
Box::new(MockTask::cooperative(
"provider",
provider_child,
log.clone(),
)),
);
let input_child = supervisor.child_token();
supervisor.register_ancillary(
input_child.clone(),
Box::new(MockTask::cooperative("input", input_child, log.clone())),
);
let render_child = supervisor.child_token();
supervisor.set_render(
render_child.clone(),
Box::new(MockTask::cooperative("render", render_child, log.clone())),
);
supervisor.request_quit();
let cause = supervisor.run().await;
assert!(cause.is_clean());
let recorded = steps(&log);
let provider = position(&recorded, "join:provider");
let input = position(&recorded, "join:input");
let render = position(&recorded, "join:render");
let restore = position(&recorded, "terminal_restore");
assert!(
provider < input && input < render && render < restore,
"order must be provider -> ancillary -> render -> terminal restore: {recorded:?}"
);
}
#[tokio::test]
async fn test_supervisor_run_normal_quit_joins_every_task_and_restores_terminal() {
let log = new_log();
let mut supervisor = recording_supervisor(&log);
let provider_child = supervisor.child_token();
supervisor.register_provider(
pid("deribit"),
provider_child.clone(),
Box::new(MockTask::cooperative(
"provider",
provider_child,
log.clone(),
)),
);
let tick_child = supervisor.child_token();
supervisor.register_ancillary(
tick_child.clone(),
Box::new(MockTask::cooperative("tick", tick_child, log.clone())),
);
supervisor.request_quit();
let cause = supervisor.run().await;
assert!(cause.is_clean(), "a normal quit exits clean");
let recorded = steps(&log);
assert!(has_step(&recorded, "join:provider"), "the provider joined");
assert!(has_step(&recorded, "join:tick"), "the tick task joined");
assert_eq!(
recorded.last().map(String::as_str),
Some("terminal_restore"),
"terminal restore is the last step: {recorded:?}"
);
}
#[tokio::test]
async fn test_supervisor_run_reported_panic_exits_nonzero_and_restores_terminal_last() {
let log = new_log();
let mut supervisor = recording_supervisor(&log);
let provider_child = supervisor.child_token();
supervisor.register_provider(
pid("deribit"),
provider_child.clone(),
Box::new(MockTask::panicking("provider", provider_child, log.clone())),
);
let input_child = supervisor.child_token();
supervisor.register_ancillary(
input_child.clone(),
Box::new(MockTask::cooperative("input", input_child, log.clone())),
);
supervisor.exit_reporter().report(TaskExit::Panicked);
let cause = supervisor.run().await;
assert!(matches!(cause, ExitCause::TaskPanicked));
assert_eq!(cause.exit_code(), 1, "a supervised panic exits non-zero");
let recorded = steps(&log);
assert!(
has_step(&recorded, "join:input"),
"the other task still joined โ no orphan: {recorded:?}"
);
assert_eq!(
recorded.last().map(String::as_str),
Some("terminal_restore"),
"terminal restore is last even on a supervised failure: {recorded:?}"
);
}
#[tokio::test]
async fn test_supervisor_teardown_panic_at_join_records_fatal_over_clean_trigger() {
let log = new_log();
let mut supervisor = recording_supervisor(&log);
let provider_child = supervisor.child_token();
supervisor.register_provider(
pid("deribit"),
provider_child.clone(),
Box::new(MockTask::panicking("provider", provider_child, log.clone())),
);
supervisor.request_quit();
let cause = supervisor.run().await;
assert!(
matches!(cause, ExitCause::TaskPanicked),
"a panic discovered at join is fatal even under a clean trigger"
);
assert_eq!(
steps(&log).last().map(String::as_str),
Some("terminal_restore")
);
}
#[tokio::test]
async fn test_supervisor_fail_records_first_fatal_only() {
let log = new_log();
let mut supervisor = recording_supervisor(&log);
supervisor.fail(ChainViewError::Terminal("first".to_owned()));
supervisor.fail(ChainViewError::Terminal("second".to_owned()));
let cause = supervisor.run().await;
match cause {
ExitCause::Failed(error) => assert!(
error.to_string().contains("first"),
"the FIRST fatal cause is the recorded exit cause, got: {error}"
),
other => panic!("expected the first Failed cause, got {other:?}"),
}
}
#[tokio::test(start_paused = true)]
async fn test_supervisor_run_wedged_provider_is_aborted_and_exit_stays_clean() {
let log = new_log();
let mut supervisor = recording_supervisor(&log);
let provider_child = supervisor.child_token();
supervisor.register_provider(
pid("deribit"),
provider_child.clone(),
Box::new(MockTask::wedged("provider", provider_child, log.clone())),
);
supervisor.request_quit();
let cause = supervisor.run().await;
assert!(
cause.is_clean(),
"a wedged (non-panicking) task is aborted but does not fail the exit"
);
let recorded = steps(&log);
assert!(
has_step(&recorded, "abort:provider"),
"the wedged task was aborted so the exit cannot hang: {recorded:?}"
);
assert_eq!(
recorded.last().map(String::as_str),
Some("terminal_restore"),
"terminal restore is still last after an abort: {recorded:?}"
);
}
#[tokio::test]
async fn test_supervisor_watched_task_panic_mid_run_wakes_supervise_as_fatal() {
let log = new_log();
let mut supervisor = recording_supervisor(&log);
let handle = tokio::spawn(async { panic!("mid-run provider panic") });
let _abort = supervisor.watch(handle);
let cause = supervisor.run().await;
assert!(
matches!(cause, ExitCause::TaskPanicked),
"a watched task panicking mid-run wakes the supervisor as fatal"
);
assert_eq!(
steps(&log).last().map(String::as_str),
Some("terminal_restore"),
"terminal restore is still last on the watched-panic path"
);
}
#[tokio::test]
async fn test_supervisor_watched_task_return_mid_run_triggers_clean_shutdown() {
let log = new_log();
let mut supervisor = recording_supervisor(&log);
let handle = tokio::spawn(async {});
let _abort = supervisor.watch(handle);
let cause = supervisor.run().await;
assert!(
cause.is_clean(),
"a watched task self-completing mid-run is a clean shutdown trigger"
);
assert_eq!(
steps(&log).last().map(String::as_str),
Some("terminal_restore")
);
}
#[tokio::test(start_paused = true)]
async fn test_supervisor_watched_wedged_task_is_aborted_at_teardown() {
let dropped = Arc::new(AtomicBool::new(false));
let guard = DropFlag(dropped.clone());
let handle = tokio::spawn(async move {
let _guard = guard;
std::future::pending::<()>().await;
});
let log = new_log();
let mut supervisor = recording_supervisor(&log);
let _abort = supervisor.watch(handle);
supervisor.request_quit();
let cause = supervisor.run().await;
assert!(
cause.is_clean(),
"a wedged (non-panicking) watched task is aborted but does not fail the exit"
);
assert!(
dropped.load(Ordering::SeqCst),
"the wedged watched task was aborted AND awaited before restore"
);
assert_eq!(
steps(&log).last().map(String::as_str),
Some("terminal_restore")
);
}
#[tokio::test]
async fn test_supervisor_worker_panic_restores_terminal_exactly_once() {
let log = new_log();
let mut supervisor = recording_supervisor(&log);
let handle = tokio::spawn(async { panic!("worker panic") });
let _abort = supervisor.watch(handle);
let cause = supervisor.run().await;
assert!(matches!(cause, ExitCause::TaskPanicked));
let recorded = steps(&log);
assert_eq!(
recorded
.iter()
.filter(|step| step.as_str() == "terminal_restore")
.count(),
1,
"the terminal is restored by a single owner (exactly once): {recorded:?}"
);
assert_eq!(
recorded.last().map(String::as_str),
Some("terminal_restore"),
"the single restore is the last step: {recorded:?}"
);
}
}