use std::collections::HashMap;
use std::path::PathBuf;
use std::sync::{Arc, Mutex};
use chrono::Utc;
use tempfile::TempDir;
use crate::activity::cache::{ActivityState, ActivityVerdict};
use crate::activity::monitor::{ActivityError, ActivityMonitor, LlmClassifier};
use crate::session_manager::{
ManagedError, ManagedSessionId, ManagedSessionState, ManagedTmuxDriver, SessionManager,
SessionRecord, StopCause,
};
use super::Supervisor;
use super::config::{
DEFAULT_LLM_MODEL, ENV_AUTO_RESUME, ENV_CLASSIFY_IDLE, ENV_INTERVAL_SECS, ENV_LLM_MODEL,
SupervisorConfig,
};
use super::metrics::FleetMetrics;
use super::poller::run_tick;
use super::publish::{self, SupervisorMetricsStatus};
struct FakeTmux {
sessions: Mutex<HashMap<String, String>>,
create_calls: Mutex<u32>,
kill_calls: Mutex<u32>,
}
impl FakeTmux {
fn new() -> Arc<Self> {
Arc::new(Self {
sessions: Mutex::new(HashMap::new()),
create_calls: Mutex::new(0),
kill_calls: Mutex::new(0),
})
}
}
impl ManagedTmuxDriver for FakeTmux {
fn create_session(&self, name: &str, workdir: &str) -> Result<(), ManagedError> {
*self.create_calls.lock().expect("lock") += 1;
self.sessions
.lock()
.expect("lock")
.insert(name.to_owned(), workdir.to_owned());
Ok(())
}
fn kill_session(&self, name: &str) -> Result<(), ManagedError> {
*self.kill_calls.lock().expect("lock") += 1;
self.sessions.lock().expect("lock").remove(name);
Ok(())
}
fn send_line(&self, _name: &str, _text: &str) -> Result<(), ManagedError> {
Ok(())
}
fn capture(&self, _name: &str, _lines: usize) -> Result<String, ManagedError> {
Ok("working on the task...\n$ cargo test".to_owned())
}
fn list_sessions(&self) -> Result<Vec<String>, ManagedError> {
Ok(self
.sessions
.lock()
.expect("lock")
.keys()
.cloned()
.collect())
}
fn get_pane_id(&self, name: &str) -> Option<String> {
Some(fake_pane(name))
}
fn pane_identity(
&self,
pane_id: &str,
) -> Result<crate::session_manager::pane_identity::PaneIdentity, ManagedError> {
Ok(crate::test_support::self_describing_identity(pane_id))
}
fn capture_pane(&self, name: &str, _pane: &str, lines: usize) -> Result<String, ManagedError> {
self.capture(name, lines)
}
}
use crate::test_support::{FAKE_PANE_SERVER as FAKE_SERVER, self_describing_pane as fake_pane};
struct StubClassifier;
impl StubClassifier {
fn new() -> Self {
Self
}
}
impl LlmClassifier for StubClassifier {
async fn classify(
&self,
_pane_text: &str,
) -> Result<(ActivityVerdict, u32, u32), ActivityError> {
Ok((
ActivityVerdict {
state: ActivityState::Working,
summary: "stub".into(),
confidence: 1.0,
},
10,
2,
))
}
}
async fn make_manager(dir: &TempDir, tmux: Arc<FakeTmux>) -> Arc<SessionManager> {
Arc::new(
SessionManager::new(dir.path(), tmux)
.await
.expect("session manager"),
)
}
fn no_override(dir: &TempDir) -> PathBuf {
dir.path().join("auto_resume")
}
fn resume_cfg() -> SupervisorConfig {
SupervisorConfig {
auto_resume: true,
classify_idle: false,
..SupervisorConfig::default()
}
}
async fn set_stop_cause(
mgr: &Arc<SessionManager>,
id: &ManagedSessionId,
cause: Option<StopCause>,
) {
let mut store = mgr.store.write().await;
let mut record = store.cached_get(id).expect("seeded record");
record.stop_cause = cause;
store.upsert(record).await.expect("stamp stop_cause");
}
async fn seed_sessions(
mgr: &Arc<SessionManager>,
n: usize,
state: ManagedSessionState,
ws: &TempDir,
) -> Vec<ManagedSessionId> {
let mut ids = Vec::new();
let mut store = mgr.store.write().await;
for i in 0..n {
let id = ManagedSessionId::new();
let tmux_name = format!("tmpm-fleet-{i}");
let rec = SessionRecord {
id,
pane_id: Some(fake_pane(&tmux_name)),
tmux_server: Some(FAKE_SERVER.into()),
tmux_name,
cwd: ws.path().to_path_buf(),
task: format!("fleet task {i}"),
state: state.clone(),
created_at: Utc::now(),
last_activity_at: None,
workspace_path: Some(ws.path().to_path_buf()),
repo_url: None,
branch: None,
pending_decision: None,
proposed_default: None,
correlation: Default::default(),
runtime: Default::default(),
ephemeral: false,
workspace_owned: false,
source_id: None,
claude_session_id: None,
scrollback_path: None,
last_cwd: None,
deliverable_id: None,
injection_status: Default::default(),
worktree_owner: None,
terminal_at: None,
stop_cause: None,
kind: Default::default(),
};
store.upsert(rec).await.expect("seed upsert");
ids.push(id);
}
drop(store);
ids
}
fn fake_env(pairs: &[(&str, &str)]) -> impl Fn(&str) -> Option<String> {
let map: HashMap<String, String> = pairs
.iter()
.map(|(k, v)| ((*k).to_owned(), (*v).to_owned()))
.collect();
move |key: &str| map.get(key).cloned()
}
#[test]
fn config_defaults() {
let c = SupervisorConfig::default();
assert_eq!(c.interval.as_secs(), 30);
assert!(!c.auto_resume);
assert!(c.classify_idle);
assert_eq!(
c.pr_cleanup_interval, None,
"#7275: the post-merge cleanup sweep spawns `gh` and deletes branches, so it \
follows auto_resume's precedent and is OFF in a hand-constructed config"
);
assert_eq!(
c.tick_timeout.as_secs(),
super::config::DEFAULT_TICK_TIMEOUT_SECS
);
assert_eq!(
c.step_timeout.as_secs(),
super::config::DEFAULT_STEP_TIMEOUT_SECS
);
let from_empty = SupervisorConfig::from_env_with(fake_env(&[]));
assert_eq!(from_empty.interval, c.interval);
assert_eq!(from_empty.auto_resume, c.auto_resume);
assert_eq!(from_empty.classify_idle, c.classify_idle);
assert!(from_empty.pr_cleanup_interval.is_some());
}
#[test]
fn default_llm_model_is_documented() {
assert_eq!(ENV_LLM_MODEL, "TRUSTY_LLM_MODEL");
assert_eq!(DEFAULT_LLM_MODEL, "openai/gpt-4o-mini");
}
#[test]
fn auto_resume_env_parsing() {
let on = SupervisorConfig::from_env_with(fake_env(&[(ENV_AUTO_RESUME, "true")]));
assert!(on.auto_resume);
let off = SupervisorConfig::from_env_with(fake_env(&[(ENV_AUTO_RESUME, "0")]));
assert!(!off.auto_resume);
let unset = SupervisorConfig::from_env_with(fake_env(&[]));
assert!(!unset.auto_resume);
}
#[test]
fn interval_env_parsing() {
let five = SupervisorConfig::from_env_with(fake_env(&[(ENV_INTERVAL_SECS, "5")]));
assert_eq!(five.interval.as_secs(), 5);
let zero = SupervisorConfig::from_env_with(fake_env(&[(ENV_INTERVAL_SECS, "0")]));
assert_eq!(zero.interval.as_secs(), 30);
let garbage = SupervisorConfig::from_env_with(fake_env(&[(ENV_INTERVAL_SECS, "notanumber")]));
assert_eq!(garbage.interval.as_secs(), 30);
}
#[test]
fn pr_cleanup_interval_env_parsing() {
use crate::supervisor::config::{DEFAULT_PR_CLEANUP_SECS, ENV_PR_CLEANUP_SECS};
let unset = SupervisorConfig::from_env_with(fake_env(&[]));
assert_eq!(
unset.pr_cleanup_interval,
Some(std::time::Duration::from_secs(DEFAULT_PR_CLEANUP_SECS)),
"the real daemon sweeps every five minutes with no env set"
);
let slow = SupervisorConfig::from_env_with(fake_env(&[(ENV_PR_CLEANUP_SECS, "900")]));
assert_eq!(
slow.pr_cleanup_interval,
Some(std::time::Duration::from_secs(900))
);
let off = SupervisorConfig::from_env_with(fake_env(&[(ENV_PR_CLEANUP_SECS, "0")]));
assert_eq!(off.pr_cleanup_interval, None, "`0` is the off switch");
let garbage = SupervisorConfig::from_env_with(fake_env(&[(ENV_PR_CLEANUP_SECS, "soon")]));
assert_eq!(
garbage.pr_cleanup_interval,
Some(std::time::Duration::from_secs(DEFAULT_PR_CLEANUP_SECS)),
"an unparsable value falls back; only an explicit 0 disables"
);
assert_eq!(
SupervisorConfig::default().pr_cleanup_interval,
None,
"a hand-constructed config must not acquire a `gh`-spawning sweep by omission"
);
}
#[test]
fn pr_cleanup_is_due_only_after_the_interval() {
use crate::supervisor::pr_cleanup_tick::due;
use std::time::{Duration, Instant};
let now = Instant::now();
let interval = Duration::from_secs(300);
assert!(
due(Some(interval), None, now),
"the first tick after a restart sweeps, so a merge that landed while the \
daemon was down is not held for a full interval"
);
assert!(
!due(Some(interval), Some(now), now + Duration::from_secs(299)),
"a tick inside the interval must not sweep again"
);
assert!(due(Some(interval), Some(now), now + interval));
assert!(
!due(None, None, now),
"`None` is the off switch and never sweeps"
);
}
#[test]
fn classify_idle_env_parsing() {
let off = SupervisorConfig::from_env_with(fake_env(&[(ENV_CLASSIFY_IDLE, "off")]));
assert!(!off.classify_idle);
let unset = SupervisorConfig::from_env_with(fake_env(&[]));
assert!(unset.classify_idle);
}
#[test]
fn supervisor_config_ignores_a_stale_bind_address() {
let with_stale_addr = SupervisorConfig::from_env_with(fake_env(&[(
"TRUSTY_MPM_SUPERVISOR_ADDR",
"127.0.0.1:7881",
)]));
assert_eq!(
with_stale_addr,
SupervisorConfig::from_env_with(fake_env(&[])),
"a leftover TRUSTY_MPM_SUPERVISOR_ADDR in an operator's environment must \
change nothing — the supervisor publishes to a file (#6288)"
);
}
#[test]
fn env_bool_recognizes_truthy_and_falsy() {
for truthy in ["1", "true", "yes", "on", "TRUE", " On "] {
let c = SupervisorConfig::from_env_with(fake_env(&[(ENV_AUTO_RESUME, truthy)]));
assert!(c.auto_resume, "{truthy:?} should be truthy");
}
for falsy in ["0", "false", "no", "off", "FALSE"] {
let c = SupervisorConfig::from_env_with(fake_env(&[(ENV_AUTO_RESUME, falsy)]));
assert!(!c.auto_resume, "{falsy:?} should be falsy");
}
let unknown = SupervisorConfig::from_env_with(fake_env(&[(ENV_AUTO_RESUME, "maybe")]));
assert!(!unknown.auto_resume);
}
fn rec(state: ManagedSessionState, pending: Option<&str>) -> SessionRecord {
SessionRecord {
id: ManagedSessionId::new(),
tmux_name: "tmpm-x".into(),
cwd: PathBuf::from("/tmp"),
task: "t".into(),
state,
created_at: Utc::now(),
last_activity_at: None,
workspace_path: None,
repo_url: None,
branch: None,
pending_decision: pending.map(|s| s.to_owned()),
proposed_default: None,
correlation: Default::default(),
runtime: Default::default(),
ephemeral: false,
workspace_owned: false,
source_id: None,
claude_session_id: None,
scrollback_path: None,
last_cwd: None,
deliverable_id: None,
pane_id: None,
tmux_server: None,
injection_status: Default::default(),
worktree_owner: None,
terminal_at: None,
stop_cause: None,
kind: Default::default(),
}
}
#[test]
fn metrics_counts_by_state() {
let records = vec![
rec(ManagedSessionState::Active, None),
rec(ManagedSessionState::Active, None),
rec(ManagedSessionState::Stopped, None),
rec(ManagedSessionState::Errored, None),
rec(ManagedSessionState::Decommissioned, None),
rec(ManagedSessionState::Provisioning, None),
];
let m = FleetMetrics::from_records(&records);
assert_eq!(m.active, 2);
assert_eq!(m.stopped, 1);
assert_eq!(m.errored, 1);
assert_eq!(m.decommissioned, 1);
assert_eq!(m.provisioning, 1);
assert_eq!(m.total, 6);
}
#[test]
fn metrics_surfaces_pending_decisions() {
let records = vec![
rec(ManagedSessionState::Active, Some("Merge PR #42?")),
rec(ManagedSessionState::Active, None),
];
let m = FleetMetrics::from_records(&records);
assert_eq!(m.pending_decisions.len(), 1);
assert_eq!(m.pending_decisions[0].question, "Merge PR #42?");
}
#[test]
fn metrics_filters_pending_decisions_on_terminal_states() {
let records = vec![
rec(
ManagedSessionState::Decommissioned,
Some("stale — never acted on"),
),
rec(ManagedSessionState::Deleted, Some("also stale")),
rec(ManagedSessionState::Active, Some("Merge PR #42?")),
];
let m = FleetMetrics::from_records(&records);
assert_eq!(
m.pending_decisions.len(),
1,
"terminal-state pending_decisions must be filtered out: {:?}",
m.pending_decisions
);
assert_eq!(m.pending_decisions[0].question, "Merge PR #42?");
}
#[test]
fn metrics_last_activity_is_max() {
let early = Utc::now() - chrono::Duration::hours(2);
let late = Utc::now();
let mut r1 = rec(ManagedSessionState::Active, None);
r1.last_activity_at = Some(early);
let mut r2 = rec(ManagedSessionState::Active, None);
r2.last_activity_at = Some(late);
let m = FleetMetrics::from_records(&[r1, r2]);
assert_eq!(m.last_activity_at, Some(late));
}
#[tokio::test]
async fn tick_auto_resumes_stopped() {
let dir = TempDir::new().unwrap();
let ws = TempDir::new().unwrap();
let tmux = FakeTmux::new();
let mgr = make_manager(&dir, tmux.clone()).await;
seed_sessions(&mgr, 1, ManagedSessionState::Stopped, &ws).await;
let report = run_tick::<StubClassifier>(&mgr, &resume_cfg(), None).await;
assert_eq!(report.resumed.len(), 1);
assert_eq!(report.resume_failures, 0);
let after = mgr.list().await;
assert_eq!(after[0].state, ManagedSessionState::Active);
assert_eq!(*tmux.create_calls.lock().unwrap(), 1);
}
#[tokio::test]
async fn tick_never_resumes_a_deliberately_stopped_session() {
let dir = TempDir::new().unwrap();
let ws = TempDir::new().unwrap();
let tmux = FakeTmux::new();
let mgr = make_manager(&dir, tmux.clone()).await;
let ids = seed_sessions(&mgr, 1, ManagedSessionState::Stopped, &ws).await;
set_stop_cause(&mgr, &ids[0], Some(StopCause::Deliberate)).await;
let report = run_tick::<StubClassifier>(&mgr, &resume_cfg(), None).await;
assert!(
report.resumed.is_empty(),
"the sweep respawned a session the operator had stopped: {:?}",
report.resumed
);
assert_eq!(report.resume_failures, 0, "not resuming is not a failure");
assert_eq!(
*tmux.create_calls.lock().unwrap(),
0,
"no tmux session may be created for a deliberately stopped record"
);
assert_eq!(mgr.list().await[0].state, ManagedSessionState::Stopped);
}
#[tokio::test]
async fn tick_never_resumes_a_supervisor_record() {
use crate::session_manager::SessionKind;
for kind in [
SessionKind::Supervisor,
SessionKind::SupervisorAux,
SessionKind::Unknown,
] {
let dir = TempDir::new().unwrap();
let ws = TempDir::new().unwrap();
let tmux = FakeTmux::new();
let mgr = make_manager(&dir, tmux.clone()).await;
let ids = seed_sessions(&mgr, 1, ManagedSessionState::Stopped, &ws).await;
set_stop_cause(&mgr, &ids[0], Some(StopCause::Unexpected)).await;
{
let mut store = mgr.store.write().await;
let mut record = store.cached_get(&ids[0]).expect("seeded record");
record.kind = kind;
store.upsert(record).await.expect("stamp kind");
}
let report = run_tick::<StubClassifier>(&mgr, &resume_cfg(), None).await;
assert!(report.resumed.is_empty(), "{kind:?}: {:?}", report.resumed);
assert_eq!(*tmux.create_calls.lock().unwrap(), 0, "{kind:?}");
assert_eq!(mgr.list().await[0].state, ManagedSessionState::Stopped);
}
}
#[tokio::test]
async fn tick_still_resumes_a_session_whose_runtime_exited() {
let dir = TempDir::new().unwrap();
let ws = TempDir::new().unwrap();
let tmux = FakeTmux::new();
let mgr = make_manager(&dir, tmux.clone()).await;
let ids = seed_sessions(&mgr, 1, ManagedSessionState::Stopped, &ws).await;
set_stop_cause(&mgr, &ids[0], Some(StopCause::Unexpected)).await;
let report = run_tick::<StubClassifier>(&mgr, &resume_cfg(), None).await;
assert_eq!(report.resumed.len(), 1);
assert_eq!(*tmux.create_calls.lock().unwrap(), 1);
let after = mgr.list().await;
assert_eq!(after[0].state, ManagedSessionState::Active);
assert_eq!(
after[0].stop_cause, None,
"a resumed session carries no stop cause"
);
}
#[tokio::test]
async fn tick_skips_resume_when_disabled() {
let dir = TempDir::new().unwrap();
let ws = TempDir::new().unwrap();
let tmux = FakeTmux::new();
let mgr = make_manager(&dir, tmux.clone()).await;
seed_sessions(&mgr, 3, ManagedSessionState::Stopped, &ws).await;
let cfg = SupervisorConfig {
auto_resume: false,
classify_idle: false,
..SupervisorConfig::default()
};
let report = run_tick::<StubClassifier>(&mgr, &cfg, None).await;
assert!(report.resumed.is_empty());
assert!(
mgr.list()
.await
.iter()
.all(|r| r.state == ManagedSessionState::Stopped)
);
}
#[tokio::test]
async fn tick_fleet_of_n_resumed() {
const N: usize = 12;
let dir = TempDir::new().unwrap();
let ws = TempDir::new().unwrap();
let tmux = FakeTmux::new();
let mgr = make_manager(&dir, tmux.clone()).await;
seed_sessions(&mgr, N, ManagedSessionState::Stopped, &ws).await;
let report = run_tick::<StubClassifier>(&mgr, &resume_cfg(), None).await;
assert_eq!(report.observed, N);
assert_eq!(report.resumed.len(), N);
assert_eq!(report.resume_failures, 0);
let after = mgr.list().await;
assert_eq!(after.len(), N);
assert!(after.iter().all(|r| r.state == ManagedSessionState::Active));
assert_eq!(*tmux.create_calls.lock().unwrap(), N as u32);
}
#[tokio::test]
async fn tick_classifies_active() {
let dir = TempDir::new().unwrap();
let ws = TempDir::new().unwrap();
let tmux = FakeTmux::new();
let mgr = make_manager(&dir, tmux.clone()).await;
seed_sessions(&mgr, 2, ManagedSessionState::Active, &ws).await;
let monitor = ActivityMonitor::new(StubClassifier::new(), "test-model");
let cfg = SupervisorConfig {
auto_resume: false,
classify_idle: true,
..SupervisorConfig::default()
};
let report = run_tick(&mgr, &cfg, Some(&monitor)).await;
assert_eq!(report.classified, 2);
}
#[tokio::test]
async fn tick_never_answers_pending_decision() {
let dir = TempDir::new().unwrap();
let ws = TempDir::new().unwrap();
let tmux = FakeTmux::new();
let mgr = make_manager(&dir, tmux.clone()).await;
let ids = seed_sessions(&mgr, 1, ManagedSessionState::Active, &ws).await;
{
let mut store = mgr.store.write().await;
let mut r = store.get(&ids[0]).await.expect("get");
r.pending_decision = Some("Force-push to main?".into());
store.upsert(r).await.expect("upsert");
}
let monitor = ActivityMonitor::new(StubClassifier::new(), "test-model");
let cfg = SupervisorConfig {
auto_resume: true,
classify_idle: true,
..SupervisorConfig::default()
};
run_tick(&mgr, &cfg, Some(&monitor)).await;
let after = mgr.list().await;
assert_eq!(
after[0].pending_decision.as_deref(),
Some("Force-push to main?"),
"supervisor must never answer or clear a pending decision"
);
}
#[tokio::test]
async fn supervisor_tick_updates_stats() {
let dir = TempDir::new().unwrap();
let ws = TempDir::new().unwrap();
let tmux = FakeTmux::new();
let mgr = make_manager(&dir, tmux.clone()).await;
seed_sessions(&mgr, 4, ManagedSessionState::Stopped, &ws).await;
let mut sup: Supervisor<StubClassifier> =
Supervisor::new(mgr, resume_cfg(), None).with_auto_resume_path(no_override(&dir));
sup.tick().await;
assert_eq!(sup.stats().sweeps, 1);
assert_eq!(sup.stats().auto_resumed, 4);
sup.tick().await;
assert_eq!(sup.stats().sweeps, 2);
assert_eq!(sup.stats().auto_resumed, 4);
}
#[tokio::test]
async fn supervisor_snapshot_reflects_fleet() {
let dir = TempDir::new().unwrap();
let ws = TempDir::new().unwrap();
let tmux = FakeTmux::new();
let mgr = make_manager(&dir, tmux.clone()).await;
seed_sessions(&mgr, 3, ManagedSessionState::Stopped, &ws).await;
let mut sup: Supervisor<StubClassifier> =
Supervisor::new(mgr, resume_cfg(), None).with_auto_resume_path(no_override(&dir));
let before = sup.snapshot().await;
assert_eq!(before.stopped, 3);
assert_eq!(before.active, 0);
sup.tick().await;
let after = sup.snapshot().await;
assert_eq!(after.stopped, 0);
assert_eq!(after.active, 3);
assert_eq!(after.run_stats.auto_resumed, 3);
}
#[tokio::test]
async fn supervisor_classifier_invoked_on_active() {
let dir = TempDir::new().unwrap();
let ws = TempDir::new().unwrap();
let tmux = FakeTmux::new();
let mgr = make_manager(&dir, tmux.clone()).await;
seed_sessions(&mgr, 2, ManagedSessionState::Active, &ws).await;
let monitor = ActivityMonitor::new(StubClassifier::new(), "test-model");
let cfg = SupervisorConfig {
auto_resume: false,
classify_idle: true,
..SupervisorConfig::default()
};
let mut sup = Supervisor::new(mgr, cfg, Some(monitor)).with_auto_resume_path(no_override(&dir));
let report = sup.tick().await;
assert_eq!(report.classified, 2);
assert_eq!(sup.stats().classified, 2);
}
#[tokio::test]
async fn supervisor_honours_console_desired_state_without_restart() {
let dir = TempDir::new().unwrap();
let ws = TempDir::new().unwrap();
let state_dir = TempDir::new().unwrap();
let desired = state_dir.path().join("auto_resume");
let tmux = FakeTmux::new();
let mgr = make_manager(&dir, tmux.clone()).await;
seed_sessions(&mgr, 1, ManagedSessionState::Stopped, &ws).await;
let cfg = SupervisorConfig {
auto_resume: false,
classify_idle: false,
..SupervisorConfig::default()
};
let mut sup: Supervisor<StubClassifier> =
Supervisor::new(mgr.clone(), cfg, None).with_auto_resume_path(&desired);
let before = sup.tick().await;
assert!(
before.resumed.is_empty(),
"no desired-state file must leave the boot flag (off) in force"
);
assert_eq!(mgr.list().await[0].state, ManagedSessionState::Stopped);
assert_eq!(*tmux.create_calls.lock().unwrap(), 0);
crate::core::auto_resume::write_desired_at(&desired, true).expect("console write");
let after = sup.tick().await;
assert_eq!(
after.resumed.len(),
1,
"the persisted console setting must make the SAME running supervisor resume"
);
assert_eq!(
mgr.list().await[0].state,
ManagedSessionState::Active,
"the session must actually be resumed, not merely counted"
);
assert_eq!(
*tmux.create_calls.lock().unwrap(),
1,
"resume must have re-spawned a tmux session"
);
assert_eq!(sup.stats().auto_resumed, 1);
}
#[tokio::test]
async fn supervisor_console_disable_overrides_env_enabled() {
let dir = TempDir::new().unwrap();
let ws = TempDir::new().unwrap();
let state_dir = TempDir::new().unwrap();
let desired = state_dir.path().join("auto_resume");
let tmux = FakeTmux::new();
let mgr = make_manager(&dir, tmux.clone()).await;
seed_sessions(&mgr, 2, ManagedSessionState::Stopped, &ws).await;
crate::core::auto_resume::write_desired_at(&desired, false).expect("console write");
let mut sup: Supervisor<StubClassifier> =
Supervisor::new(mgr.clone(), resume_cfg(), None).with_auto_resume_path(&desired);
let report = sup.tick().await;
assert!(
report.resumed.is_empty(),
"an explicit console `false` must outrank the boot env flag"
);
assert!(
mgr.list()
.await
.iter()
.all(|r| r.state == ManagedSessionState::Stopped)
);
assert_eq!(*tmux.create_calls.lock().unwrap(), 0);
}
#[tokio::test]
async fn supervisor_absent_desired_file_keeps_boot_flag() {
let dir = TempDir::new().unwrap();
let ws = TempDir::new().unwrap();
let state_dir = TempDir::new().unwrap();
let tmux = FakeTmux::new();
let mgr = make_manager(&dir, tmux.clone()).await;
seed_sessions(&mgr, 1, ManagedSessionState::Stopped, &ws).await;
let mut sup: Supervisor<StubClassifier> = Supervisor::new(mgr.clone(), resume_cfg(), None)
.with_auto_resume_path(state_dir.path().join("auto_resume"));
let report = sup.tick().await;
assert_eq!(
report.resumed.len(),
1,
"no override file must leave the boot flag (on) in force"
);
assert_eq!(mgr.list().await[0].state, ManagedSessionState::Active);
}
#[tokio::test]
async fn supervisor_unreadable_desired_file_does_not_disable_resume() {
let dir = TempDir::new().unwrap();
let ws = TempDir::new().unwrap();
let state_dir = TempDir::new().unwrap();
let desired = state_dir.path().join("auto_resume");
std::fs::create_dir(&desired).expect("mkdir over the desired-state path");
let tmux = FakeTmux::new();
let mgr = make_manager(&dir, tmux.clone()).await;
seed_sessions(&mgr, 1, ManagedSessionState::Stopped, &ws).await;
let mut sup: Supervisor<StubClassifier> =
Supervisor::new(mgr.clone(), resume_cfg(), None).with_auto_resume_path(&desired);
let report = sup.tick().await;
assert_eq!(
report.resumed.len(),
1,
"an unreadable desired-state file must not fail open to auto-resume off"
);
assert_eq!(mgr.list().await[0].state, ManagedSessionState::Active);
}
#[tokio::test]
async fn supervisor_failed_resume_surfaces_as_errored() {
let dir = TempDir::new().unwrap();
let ws = TempDir::new().unwrap();
let state_dir = TempDir::new().unwrap();
let desired = state_dir.path().join("auto_resume");
let tmux = FakeTmux::new();
let mgr = make_manager(&dir, tmux.clone()).await;
seed_sessions(&mgr, 1, ManagedSessionState::Stopped, &ws).await;
ws.close()
.expect("drop the workspace the session resumes into");
crate::core::auto_resume::write_desired_at(&desired, true).expect("console write");
let mut sup: Supervisor<StubClassifier> = Supervisor::new(
mgr.clone(),
SupervisorConfig {
auto_resume: false,
classify_idle: false,
..SupervisorConfig::default()
},
None,
)
.with_auto_resume_path(&desired);
let report = sup.tick().await;
assert!(report.resumed.is_empty(), "the resume must have failed");
assert_eq!(report.resume_failures, 1);
let after = mgr.list().await;
assert_eq!(
after[0].state,
ManagedSessionState::Errored,
"a failed auto-resume must leave the session visibly errored, not silently stopped"
);
assert!(
after[0].task.contains("auto-resume failed"),
"the failure reason must be readable from the record: {}",
after[0].task
);
let snapshot = sup.snapshot().await;
assert_eq!(snapshot.errored, 1);
assert_eq!(snapshot.stopped, 0);
let second = sup.tick().await;
assert_eq!(
second.resume_failures, 0,
"an errored session must not be re-attempted every interval"
);
}
#[test]
fn published_metrics_round_trip() {
let tmp = TempDir::new().expect("tempdir");
let path = tmp.path().join("nested").join("supervisor-metrics.json");
let mut fleet = FleetMetrics::from_records(&[
rec(ManagedSessionState::Active, None),
rec(ManagedSessionState::Stopped, None),
]);
fleet.run_stats.sweeps = 7;
fleet.run_stats.auto_resumed = 3;
let now = Utc::now();
let interval = std::time::Duration::from_secs(45);
publish::write_at(&path, &fleet, interval, now).expect("publish writes");
let back = publish::read_at(&path)
.expect("read succeeds")
.expect("a snapshot exists");
assert_eq!(back.fleet, fleet);
assert_eq!(back.written_at.timestamp(), now.timestamp());
assert_eq!(
back.interval_secs, 45,
"the writer's cadence must travel with the snapshot — the reader sizes \
its staleness window from it"
);
let leftovers: Vec<_> = std::fs::read_dir(path.parent().expect("parent"))
.expect("readdir")
.filter_map(|e| e.ok().map(|e| e.file_name().to_string_lossy().into_owned()))
.filter(|n| n.ends_with(".tmp"))
.collect();
assert!(
leftovers.is_empty(),
"temp files left behind: {leftovers:?}"
);
}
#[test]
fn read_status_absent_is_unavailable() {
let tmp = TempDir::new().expect("tempdir");
let path = tmp.path().join("supervisor-metrics.json");
match publish::read_status_at(&path, Utc::now()) {
SupervisorMetricsStatus::Unavailable { reason } => {
assert!(
reason.contains("supervisor"),
"the reason must tell an operator what is missing: {reason}"
);
}
other => panic!("an absent snapshot must be Unavailable, got {other:?}"),
}
}
#[test]
fn read_status_corrupt_is_unavailable() {
let tmp = TempDir::new().expect("tempdir");
let path = tmp.path().join("supervisor-metrics.json");
std::fs::write(&path, "{ this is not json").expect("write corrupt");
match publish::read_status_at(&path, Utc::now()) {
SupervisorMetricsStatus::Unavailable { reason } => {
assert!(
reason.contains("json"),
"a corrupt file must report the parse failure: {reason}"
);
}
other => panic!("a corrupt snapshot must be Unavailable, got {other:?}"),
}
}
#[test]
fn read_status_old_snapshot_is_stale() {
let tmp = TempDir::new().expect("tempdir");
let path = tmp.path().join("supervisor-metrics.json");
let mut fleet = FleetMetrics::default();
fleet.run_stats.sweeps = 42;
let default_interval = SupervisorConfig::default().interval;
let threshold = publish::stale_after_secs(default_interval);
assert_eq!(
threshold,
publish::STALE_FLOOR_SECS,
"default cadence floors"
);
let written_at = Utc::now() - chrono::Duration::seconds(threshold + 60);
publish::write_at(&path, &fleet, default_interval, written_at).expect("publish writes");
match publish::read_status_at(&path, Utc::now()) {
SupervisorMetricsStatus::Stale {
snapshot,
age_secs,
stale_after_secs,
} => {
assert_eq!(stale_after_secs, threshold);
assert!(age_secs > threshold, "age {age_secs}");
assert_eq!(
snapshot.fleet.run_stats.sweeps, 42,
"a stale snapshot still carries the last real counters"
);
}
other => panic!("an old snapshot must be Stale, got {other:?}"),
}
publish::write_at(&path, &fleet, default_interval, Utc::now()).expect("republish");
assert!(
matches!(
publish::read_status_at(&path, Utc::now()),
SupervisorMetricsStatus::Current { .. }
),
"a freshly-published snapshot must be Current"
);
}
#[test]
fn stale_threshold_tracks_the_configured_interval() {
use std::time::Duration;
assert_eq!(
publish::stale_after_secs(Duration::from_secs(30)),
publish::STALE_FLOOR_SECS
);
assert_eq!(
publish::stale_after_secs(Duration::from_secs(1)),
publish::STALE_FLOOR_SECS,
"a 1s debug cadence must not flap on ordinary scheduling jitter"
);
assert_eq!(publish::stale_after_secs(Duration::from_secs(100)), 300);
assert_eq!(publish::stale_after_secs(Duration::from_secs(900)), 2700);
assert_eq!(publish::stale_after_secs(Duration::from_secs(3600)), 10_800);
assert!(publish::stale_after_secs(Duration::from_secs(u64::MAX)) > 0);
}
#[test]
fn read_status_respects_a_slow_configured_interval() {
let tmp = TempDir::new().expect("tempdir");
let path = tmp.path().join("supervisor-metrics.json");
let slow = std::time::Duration::from_secs(900);
let mut fleet = FleetMetrics::default();
fleet.run_stats.sweeps = 5;
let written_at = Utc::now();
publish::write_at(&path, &fleet, slow, written_at).expect("publish writes");
let at_400 = written_at + chrono::Duration::seconds(400);
match publish::read_status_at(&path, at_400) {
SupervisorMetricsStatus::Current {
stale_after_secs, ..
} => assert_eq!(stale_after_secs, 2700),
other => panic!(
"a 900s-cadence supervisor 400s after its publish is mid-cycle and healthy, not stale: {other:?}"
),
}
let at_3000 = written_at + chrono::Duration::seconds(3000);
match publish::read_status_at(&path, at_3000) {
SupervisorMetricsStatus::Stale {
age_secs,
stale_after_secs,
..
} => {
assert_eq!(age_secs, 3000);
assert_eq!(stale_after_secs, 2700);
}
other => panic!("three missed 900s sweeps must read Stale: {other:?}"),
}
}
#[test]
fn snapshot_without_interval_falls_back_to_the_default_cadence() {
let tmp = TempDir::new().expect("tempdir");
let path = tmp.path().join("supervisor-metrics.json");
let written_at = Utc::now() - chrono::Duration::seconds(120);
let legacy = serde_json::json!({
"written_at": written_at,
"fleet": FleetMetrics::default(),
});
std::fs::write(&path, legacy.to_string()).expect("write legacy snapshot");
let back = publish::read_at(&path)
.expect("a pre-interval snapshot must still parse")
.expect("a snapshot exists");
assert_eq!(
back.interval_secs,
crate::supervisor::config::DEFAULT_INTERVAL_SECS
);
match publish::read_status_at(&path, Utc::now()) {
SupervisorMetricsStatus::Current {
stale_after_secs, ..
} => assert_eq!(
stale_after_secs,
publish::STALE_FLOOR_SECS,
"the fallback must reproduce the pre-#6288 window exactly"
),
other => panic!("120s old at the default cadence is Current: {other:?}"),
}
}
#[test]
fn metrics_path_is_under_root() {
let paths = crate::core::paths::FrameworkPaths::under("/tmp/test-base");
let p = publish::metrics_path(&paths);
assert!(p.ends_with(".trusty-mpm/supervisor-metrics.json"), "{p:?}");
}
#[tokio::test]
async fn supervisor_publishes_run_stats_after_sweeps() {
let dir = TempDir::new().unwrap();
let ws = TempDir::new().unwrap();
let tmux = FakeTmux::new();
let mgr = make_manager(&dir, tmux.clone()).await;
seed_sessions(&mgr, 2, ManagedSessionState::Stopped, &ws).await;
let cfg = SupervisorConfig {
auto_resume: true,
classify_idle: false,
..SupervisorConfig::default()
};
let metrics_file = dir.path().join("supervisor-metrics.json");
let mut sup: Supervisor<StubClassifier> = Supervisor::new(mgr, cfg, None)
.with_auto_resume_path(no_override(&dir))
.with_metrics_path(&metrics_file);
assert!(
matches!(
publish::read_status_at(&metrics_file, Utc::now()),
SupervisorMetricsStatus::Unavailable { .. }
),
"before the first sweep there is no snapshot to read"
);
sup.tick().await;
sup.publish_snapshot().await;
sup.tick().await;
sup.publish_snapshot().await;
match publish::read_status_at(&metrics_file, Utc::now()) {
SupervisorMetricsStatus::Current { snapshot, .. } => {
let stats = &snapshot.fleet.run_stats;
assert_eq!(stats.sweeps, 2, "two real sweeps must be published");
assert_eq!(
stats.auto_resumed, 2,
"both seeded stopped sessions were auto-resumed; the published \
counters must say so rather than defaulting to zero"
);
}
other => panic!("a just-published snapshot must be Current, got {other:?}"),
}
}
#[tokio::test]
#[serial_test::serial]
async fn supervisor_run_until_stops_cleanly() {
use tracing::instrument::WithSubscriber;
let (buffer, subscriber) = capture_logs();
let dir = TempDir::new().unwrap();
let ws = TempDir::new().unwrap();
let tmux = FakeTmux::new();
let mgr = make_manager(&dir, tmux.clone()).await;
seed_sessions(&mgr, 2, ManagedSessionState::Stopped, &ws).await;
let cfg = SupervisorConfig {
interval: std::time::Duration::from_secs(3600),
auto_resume: true,
classify_idle: false,
pr_cleanup_interval: None,
..SupervisorConfig::default()
};
let metrics_file = dir.path().join("supervisor-metrics.json");
let sup: Supervisor<StubClassifier> = Supervisor::new(mgr, cfg, None)
.with_auto_resume_path(no_override(&dir))
.with_metrics_path(&metrics_file);
let (tx, rx) = tokio::sync::oneshot::channel::<()>();
tokio::spawn(async move {
let _ = tx.send(());
});
let shutdown = async move {
let _ = rx.await;
};
let result = tokio::time::timeout(
std::time::Duration::from_secs(5),
sup.run_until(shutdown).with_subscriber(subscriber),
)
.await
.expect("run_until must return promptly after shutdown");
result.expect("clean shutdown returns Ok");
let errors = error_lines(&buffer);
assert!(
errors.is_empty(),
"a clean shutdown logs no ERROR: {errors:#?}"
);
let published = publish::read_at(&metrics_file)
.expect("read")
.expect("the loop published before parking on the timer");
assert_eq!(published.fleet.stopped, 2);
}
struct HangsOnceClassifier {
calls: std::sync::atomic::AtomicUsize,
}
impl LlmClassifier for HangsOnceClassifier {
async fn classify(
&self,
pane_text: &str,
) -> Result<(ActivityVerdict, u32, u32), ActivityError> {
if self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) == 0 {
std::future::pending::<()>().await;
}
StubClassifier::new().classify(pane_text).await
}
}
#[tokio::test]
async fn a_wedged_sweep_times_out_and_the_loop_continues() {
let dir = TempDir::new().unwrap();
let ws = TempDir::new().unwrap();
let mgr = make_manager(&dir, FakeTmux::new()).await;
seed_sessions(&mgr, 1, ManagedSessionState::Active, &ws).await;
let cfg = SupervisorConfig {
interval: std::time::Duration::from_millis(20),
classify_idle: true,
tick_timeout: std::time::Duration::from_millis(200),
..SupervisorConfig::default()
};
let classifier = HangsOnceClassifier {
calls: std::sync::atomic::AtomicUsize::new(0),
};
let monitor = ActivityMonitor::new(classifier, "test-model");
let metrics_file = dir.path().join("supervisor-metrics.json");
let sup = Supervisor::new(mgr, cfg, Some(monitor))
.with_auto_resume_path(no_override(&dir))
.with_metrics_path(&metrics_file);
let watched = metrics_file.clone();
let later_sweep_completed = async move {
loop {
let classified = publish::read_at(&watched)
.ok()
.flatten()
.map_or(0, |m| m.fleet.run_stats.classified);
if classified >= 1 {
return;
}
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
}
};
let result = tokio::time::timeout(
std::time::Duration::from_secs(10),
sup.run_until(later_sweep_completed),
)
.await
.expect("the loop must abandon the wedged sweep and complete a later one");
result.expect("clean shutdown returns Ok");
}
#[test]
fn heartbeat_watchdog_flags_a_stale_snapshot_once() {
use super::watchdog::check_heartbeat;
let dir = TempDir::new().unwrap();
let path = dir.path().join("supervisor-metrics.json");
let interval = std::time::Duration::from_secs(30);
let now = Utc::now();
assert!(
!check_heartbeat(&path, now, false),
"an absent file is not stale"
);
let old = now - chrono::Duration::seconds(3600);
publish::write_at(&path, &FleetMetrics::default(), interval, old).expect("write");
assert!(
check_heartbeat(&path, now, false),
"a frozen heartbeat is stale"
);
assert!(check_heartbeat(&path, now, true), "it stays stale");
publish::write_at(&path, &FleetMetrics::default(), interval, now).expect("write");
assert!(!check_heartbeat(&path, now, true), "a fresh write recovers");
std::fs::write(&path, "not json").expect("corrupt");
assert!(
check_heartbeat(&path, now, true),
"unreadable keeps the verdict"
);
}
fn capture_logs() -> (
trusty_common::log_buffer::LogBuffer,
impl tracing::Subscriber + Send + Sync + 'static,
) {
use tracing_subscriber::layer::SubscriberExt;
crate::test_support::enable_event_capture();
let buffer = trusty_common::log_buffer::LogBuffer::new(256);
let subscriber = tracing_subscriber::registry().with(
trusty_common::log_buffer::LogBufferLayer::new(buffer.clone()),
);
(buffer, subscriber)
}
fn error_lines(buffer: &trusty_common::log_buffer::LogBuffer) -> Vec<String> {
buffer
.tail(256)
.into_iter()
.filter(|l| l.contains("[ERROR"))
.collect()
}
#[tokio::test]
#[serial_test::serial]
async fn an_armed_loop_exit_guard_logs_an_error_when_dropped() {
use super::watchdog::LoopExitGuard;
let (buffer, subscriber) = capture_logs();
let task = tokio::spawn(std::future::pending::<()>());
let abort = task.abort_handle();
tracing::subscriber::with_default(subscriber, || {
drop(LoopExitGuard::new(Some(task)));
let mut clean = LoopExitGuard::new(None);
clean.disarm();
drop(clean);
});
let errors = error_lines(&buffer);
assert_eq!(errors.len(), 1, "exactly the armed drop logs: {errors:#?}");
assert!(
errors[0].contains("exited without a shutdown signal"),
"{errors:#?}"
);
for _ in 0..100 {
if abort.is_finished() {
break;
}
tokio::task::yield_now().await;
}
assert!(abort.is_finished(), "dropping the guard stops the watchdog");
}
struct HungGh {
gate: Mutex<std::sync::mpsc::Receiver<()>>,
calls: std::sync::atomic::AtomicUsize,
}
impl crate::core::pr_cleanup::Gh for HungGh {
fn run(&self, _args: &[String]) -> anyhow::Result<crate::core::pr_cleanup::CmdOut> {
self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
let _ = self
.gate
.lock()
.expect("lock")
.recv_timeout(std::time::Duration::from_secs(20));
Ok(crate::core::pr_cleanup::CmdOut {
success: false,
stdout: String::new(),
stderr: "fake gh released".into(),
})
}
}
#[tokio::test]
async fn a_hung_gh_in_the_cleanup_sweep_is_abandoned_and_the_loop_continues() {
use crate::core::pr_cleanup::auth_backoff::AuthBackoff;
use crate::core::pr_cleanup::{CleanupRegistry, CleanupScope, OpenedPr};
let dir = TempDir::new().unwrap();
let mgr = make_manager(&dir, FakeTmux::new()).await;
let registry_path = dir.path().join("pr-cleanup.json");
CleanupRegistry::at(®istry_path)
.record_open(OpenedPr {
pr: 1,
repo: "owner/repo".into(),
repo_root: dir.path().to_path_buf(),
opened_at: Utc::now(),
cleaned_at: None,
scope: CleanupScope::Wide,
})
.expect("record the pending PR");
let (release, gate) = std::sync::mpsc::channel();
let gh = Arc::new(HungGh {
gate: Mutex::new(gate),
calls: std::sync::atomic::AtomicUsize::new(0),
});
let seam_gh = Arc::clone(&gh);
let cleanup: super::CleanupFn = Arc::new(move |mgr| {
let (gh, path) = (Arc::clone(&seam_gh), registry_path.clone());
Box::pin(async move {
let registry = CleanupRegistry::at(path);
super::pr_cleanup_tick::run_sweep_with_gh(&mgr, &*gh, ®istry, &AuthBackoff::new())
.await
})
});
let cfg = SupervisorConfig {
classify_idle: false,
pr_cleanup_interval: Some(std::time::Duration::ZERO),
tick_timeout: std::time::Duration::from_millis(200),
..SupervisorConfig::default()
};
let mut sup: Supervisor<StubClassifier> = Supervisor::new(mgr, cfg, None)
.with_auto_resume_path(no_override(&dir))
.with_metrics_path(dir.path().join("supervisor-metrics.json"))
.with_cleanup(cleanup);
for tick in 1..=2 {
let started = std::time::Instant::now();
sup.tick().await;
assert!(
started.elapsed() < std::time::Duration::from_secs(5),
"tick {tick} waited {:?} on a hung `gh`",
started.elapsed()
);
}
assert_eq!(sup.stats().sweeps, 2, "both fleet sweeps completed");
assert_eq!(sup.stats().cleanup_sweeps_abandoned, 2);
assert_eq!(
gh.calls.load(std::sync::atomic::Ordering::SeqCst),
1,
"the second tick skips the cleanup sweep while the first one is blocked"
);
drop(release);
}
struct BlockingCaptureTmux {
inner: Arc<FakeTmux>,
gate: Mutex<std::sync::mpsc::Receiver<()>>,
}
impl ManagedTmuxDriver for BlockingCaptureTmux {
fn create_session(&self, name: &str, workdir: &str) -> Result<(), ManagedError> {
self.inner.create_session(name, workdir)
}
fn kill_session(&self, name: &str) -> Result<(), ManagedError> {
self.inner.kill_session(name)
}
fn send_line(&self, name: &str, text: &str) -> Result<(), ManagedError> {
self.inner.send_line(name, text)
}
fn capture(&self, name: &str, lines: usize) -> Result<String, ManagedError> {
let _ = self
.gate
.lock()
.expect("lock")
.recv_timeout(std::time::Duration::from_secs(20));
self.inner.capture(name, lines)
}
fn capture_pane(&self, name: &str, _pane: &str, lines: usize) -> Result<String, ManagedError> {
self.capture(name, lines)
}
fn pane_identity(
&self,
pane_id: &str,
) -> Result<crate::session_manager::pane_identity::PaneIdentity, ManagedError> {
self.inner.pane_identity(pane_id)
}
fn list_sessions(&self) -> Result<Vec<String>, ManagedError> {
self.inner.list_sessions()
}
}
#[tokio::test]
async fn abandoned_fleet_sweeps_are_counted_and_read_as_stale() {
let dir = TempDir::new().unwrap();
let ws = TempDir::new().unwrap();
let (release, gate) = std::sync::mpsc::channel::<()>();
let tmux = Arc::new(BlockingCaptureTmux {
inner: FakeTmux::new(),
gate: Mutex::new(gate),
});
let mgr = Arc::new(SessionManager::new(dir.path(), tmux).await.expect("mgr"));
seed_sessions(&mgr, 1, ManagedSessionState::Active, &ws).await;
let cfg = SupervisorConfig {
classify_idle: true,
tick_timeout: std::time::Duration::from_millis(200),
..SupervisorConfig::default()
};
let monitor = ActivityMonitor::new(StubClassifier::new(), "test-model");
let metrics_file = dir.path().join("supervisor-metrics.json");
let mut sup = Supervisor::new(mgr, cfg, Some(monitor))
.with_auto_resume_path(no_override(&dir))
.with_metrics_path(&metrics_file);
for _ in 0..2 {
let started = std::time::Instant::now();
sup.tick().await;
assert!(started.elapsed() < std::time::Duration::from_secs(5));
}
let stats = sup.stats().clone();
assert_eq!(
(
stats.sweeps,
stats.sweeps_abandoned,
stats.consecutive_sweeps_abandoned
),
(0, 2, 2)
);
sup.publish_snapshot().await;
match publish::read_status_at(&metrics_file, Utc::now()) {
SupervisorMetricsStatus::Stale { snapshot, .. } => {
assert_eq!(snapshot.fleet.run_stats.sweeps_abandoned, 2);
}
other => panic!("a fresh snapshot of abandoned sweeps must be stale: {other:?}"),
}
drop(release);
for _ in 0..100 {
sup.tick().await;
if sup.stats().sweeps > 0 {
break;
}
tokio::time::sleep(std::time::Duration::from_millis(20)).await;
}
assert_eq!(sup.stats().consecutive_sweeps_abandoned, 0);
sup.publish_snapshot().await;
assert!(matches!(
publish::read_status_at(&metrics_file, Utc::now()),
SupervisorMetricsStatus::Current { .. }
));
}
#[tokio::test]
#[serial_test::serial]
async fn a_hung_classify_step_is_abandoned_and_the_sweep_moves_on() {
use tracing::instrument::WithSubscriber;
let (buffer, subscriber) = capture_logs();
let dir = TempDir::new().unwrap();
let ws = TempDir::new().unwrap();
let mgr = make_manager(&dir, FakeTmux::new()).await;
let ids = seed_sessions(&mgr, 2, ManagedSessionState::Active, &ws).await;
let cfg = SupervisorConfig {
classify_idle: true,
step_timeout: std::time::Duration::from_millis(100),
..SupervisorConfig::default()
};
let classifier = HangsOnceClassifier {
calls: std::sync::atomic::AtomicUsize::new(0),
};
let monitor = ActivityMonitor::new(classifier, "test-model");
let report = tokio::time::timeout(
std::time::Duration::from_secs(5),
run_tick(&mgr, &cfg, Some(&monitor)).with_subscriber(subscriber),
)
.await
.expect("the sweep must not wait on the hung step");
assert_eq!(
report.classified, 1,
"the other session is still classified"
);
let errors = error_lines(&buffer);
let line = errors
.iter()
.find(|l| l.contains("sweep step did not finish"))
.unwrap_or_else(|| panic!("no step-expiry line: {errors:#?}"));
assert!(line.contains("classify"), "{line}");
assert!(line.contains("tmpm-fleet-"), "{line}");
assert!(
ids.iter().any(|id| line.contains(&id.to_string())),
"{line}"
);
}
#[test]
fn heartbeat_watch_skips_its_verdict_across_a_wall_clock_jump() {
use super::watchdog::HeartbeatWatch;
let dir = TempDir::new().unwrap();
let path = dir.path().join("supervisor-metrics.json");
let period = std::time::Duration::from_secs(30);
let t0 = Utc::now();
publish::write_at(&path, &FleetMetrics::default(), period, t0).expect("write");
let mut watch = HeartbeatWatch::new(period);
assert!(!watch.observe(&path, t0), "a fresh heartbeat is current");
let woke = t0 + chrono::Duration::hours(1);
assert!(
!watch.observe(&path, woke),
"the first check after a wall-clock jump gives no verdict"
);
assert!(
watch.observe(&path, woke + chrono::Duration::seconds(30)),
"a normal gap judges again, and the old heartbeat is stale"
);
}
#[cfg(feature = "daemon")]
#[tokio::test]
async fn supervisor_metrics_merge_reports_real_run_stats() {
let dir = TempDir::new().unwrap();
let ws = TempDir::new().unwrap();
let tmux = FakeTmux::new();
let mgr = make_manager(&dir, tmux.clone()).await;
seed_sessions(&mgr, 3, ManagedSessionState::Stopped, &ws).await;
let cfg = SupervisorConfig {
auto_resume: true,
classify_idle: false,
..SupervisorConfig::default()
};
let metrics_file = dir.path().join("supervisor-metrics.json");
let mut sup: Supervisor<StubClassifier> = Supervisor::new(mgr, cfg, None)
.with_auto_resume_path(no_override(&dir))
.with_metrics_path(&metrics_file);
sup.tick().await;
sup.publish_snapshot().await;
let mut fleet = FleetMetrics::default();
assert_eq!(
fleet.run_stats.sweeps, 0,
"precondition: the daemon starts from zeroed counters"
);
let block =
crate::daemon::mcp_console::merge_supervisor_metrics(&mut fleet, &metrics_file, Utc::now());
assert_eq!(block["status"], "current");
assert_eq!(
fleet.run_stats.sweeps, 1,
"the daemon must report the supervisor's real sweep count, not zero"
);
assert_eq!(
fleet.run_stats.auto_resumed, 3,
"the daemon must report the supervisor's real auto-resume count: {block}"
);
}
struct ScriptedRelauncher {
calls: std::sync::atomic::AtomicUsize,
verdict: Result<(), String>,
}
#[async_trait::async_trait]
impl crate::session_manager::relaunch::RuntimeRelauncher for ScriptedRelauncher {
async fn relaunch(&self, _record: &SessionRecord) -> Result<(), String> {
self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
self.verdict.clone()
}
}
#[tokio::test]
async fn one_failed_auto_resume_appends_exactly_one_error_note() {
let dir = TempDir::new().unwrap();
let ws = TempDir::new().unwrap();
let tmux = FakeTmux::new();
let mgr = make_manager(&dir, tmux.clone()).await;
let ids = seed_sessions(&mgr, 1, ManagedSessionState::Stopped, &ws).await;
set_stop_cause(&mgr, &ids[0], Some(StopCause::Unexpected)).await;
let relauncher = Arc::new(ScriptedRelauncher {
calls: std::sync::atomic::AtomicUsize::new(0),
verdict: Err("no runtime came up in pane 'tmpm-fleet-0'".to_owned()),
});
assert!(mgr.install_relauncher(relauncher.clone()));
let report = run_tick::<StubClassifier>(&mgr, &resume_cfg(), None).await;
assert_eq!(
report.resume_failures, 1,
"the failure must still be counted — the fix removes a note, not the count"
);
let record = mgr.get(&ids[0]).await.expect("record");
assert_eq!(
record.task.matches("[error:").count(),
1,
"one failure, one note — the task is what the next relaunch is handed: {}",
record.task
);
assert_eq!(
record.state,
ManagedSessionState::Errored,
"and the record is still errored, which `resume_auto` did"
);
}
#[tokio::test]
async fn a_supervisor_tick_does_nothing_to_a_session_being_resumed() {
let dir = TempDir::new().unwrap();
let ws = TempDir::new().unwrap();
let tmux = FakeTmux::new();
let mgr = make_manager(&dir, tmux.clone()).await;
let ids = seed_sessions(&mgr, 1, ManagedSessionState::Stopped, &ws).await;
set_stop_cause(&mgr, &ids[0], Some(StopCause::Unexpected)).await;
let before = mgr.get(&ids[0]).await.expect("record").task;
let claim = mgr
.begin_resume(&ids[0])
.expect("the first claim is granted");
let report = run_tick::<StubClassifier>(&mgr, &resume_cfg(), None).await;
assert!(report.resumed.is_empty(), "no launch: {:?}", report.resumed);
assert_eq!(report.resume_failures, 0, "and nothing failed either");
let record = mgr.get(&ids[0]).await.expect("record");
assert_eq!(
record.state,
ManagedSessionState::Stopped,
"the tick must leave the record to the path that holds it"
);
assert_eq!(record.task, before, "and append nothing to its task");
assert_eq!(
*tmux.create_calls.lock().unwrap(),
0,
"above all, no second launch line reaches the pane"
);
drop(claim);
let after = run_tick::<StubClassifier>(&mgr, &resume_cfg(), None).await;
assert_eq!(
after.resumed.len(),
1,
"once the claim is gone the sweep resumes it as usual: {after:?}"
);
}
#[tokio::test]
async fn an_already_active_session_is_not_marked_errored_by_a_stale_resume() {
let dir = TempDir::new().unwrap();
let ws = TempDir::new().unwrap();
let mgr = make_manager(&dir, FakeTmux::new()).await;
let ids = seed_sessions(&mgr, 1, ManagedSessionState::Active, &ws).await;
let fresh = mgr.get(&ids[0]).await.expect("record");
let mut stale = fresh.clone();
stale.state = ManagedSessionState::Stopped;
let mut report = super::poller::TickReport::default();
for _ in 0..2 {
let refusal = mgr
.resume_auto(&ids[0])
.await
.expect_err("an active session cannot be resumed");
assert!(
matches!(refusal, ManagedError::InvalidState(..)),
"{refusal:?}"
);
super::poller::settle_failed_resume(&mgr, &stale, refusal, &mut report).await;
}
assert_eq!(report.resume_failures, 0, "nothing failed: {report:?}");
let after = mgr.get(&ids[0]).await.expect("record");
assert_eq!(after.state, ManagedSessionState::Active);
assert_eq!(after.task, fresh.task, "no note accumulates on the task");
}
#[tokio::test]
async fn a_resume_refusal_for_a_non_active_state_is_still_recorded() {
let dir = TempDir::new().unwrap();
let ws = TempDir::new().unwrap();
let mgr = make_manager(&dir, FakeTmux::new()).await;
let ids = seed_sessions(&mgr, 1, ManagedSessionState::Provisioning, &ws).await;
let record = mgr.get(&ids[0]).await.expect("record");
let refusal = mgr
.resume_auto(&ids[0])
.await
.expect_err("a provisioning session cannot be resumed");
assert!(
matches!(refusal, ManagedError::InvalidState(..)),
"{refusal:?}"
);
let mut report = super::poller::TickReport::default();
super::poller::settle_failed_resume(&mgr, &record, refusal, &mut report).await;
assert_eq!(report.resume_failures, 1);
let after = mgr.get(&ids[0]).await.expect("record");
assert_eq!(after.state, ManagedSessionState::Errored);
assert_eq!(after.task.matches("[error: auto-resume failed").count(), 1);
}
#[tokio::test]
async fn a_non_state_resume_error_is_still_recorded() {
let dir = TempDir::new().unwrap();
let ws = TempDir::new().unwrap();
let mgr = make_manager(&dir, FakeTmux::new()).await;
let ids = seed_sessions(&mgr, 1, ManagedSessionState::Stopped, &ws).await;
let record = mgr.get(&ids[0]).await.expect("record");
let mut report = super::poller::TickReport::default();
let err = ManagedError::TmuxUnavailable("tmux server went away".to_owned());
super::poller::settle_failed_resume(&mgr, &record, err, &mut report).await;
assert_eq!(report.resume_failures, 1);
let after = mgr.get(&ids[0]).await.expect("record");
assert_eq!(after.state, ManagedSessionState::Errored);
assert!(
after.task.contains("tmux server went away"),
"{}",
after.task
);
}
#[tokio::test]
async fn session_claims_match_a_workspace_spelled_through_a_symlink() {
use crate::core::pr_cleanup::ClaimEnder;
use crate::supervisor::pr_cleanup_tick::SessionClaims;
let dir = TempDir::new().expect("store dir");
let ws = TempDir::new().expect("workspace dir");
let mgr = make_manager(&dir, FakeTmux::new()).await;
let tree = ws.path().join("tree");
std::fs::create_dir(&tree).expect("create the tree");
let alias = ws.path().join("alias");
std::os::unix::fs::symlink(&tree, &alias).expect("symlink the tree");
let gone = ws.path().join("gone");
let mut via_alias = rec(ManagedSessionState::Active, None);
via_alias.workspace_path = Some(alias);
let mut on_gone = rec(ManagedSessionState::Active, None);
on_gone.workspace_path = Some(gone.clone());
let (alias_id, gone_id) = (via_alias.id.to_string(), on_gone.id.to_string());
{
let mut store = mgr.store.write().await;
store
.upsert(via_alias)
.await
.expect("upsert the alias record");
store.upsert(on_gone).await.expect("upsert the gone record");
}
let claims = SessionClaims::new(&mgr);
let on_tree = claims.claims_on(&tree).await.expect("claims on the tree");
assert_eq!(on_tree, [alias_id], "the symlinked spelling holds the tree");
let on_missing = claims
.claims_on(&gone)
.await
.expect("claims on a gone path");
assert_eq!(
on_missing,
[gone_id],
"an unresolvable path keeps its holder"
);
}