use std::sync::Arc;
use std::time::Duration;
use chrono::{TimeDelta, Utc};
use super::record::{ManagedSessionId, ManagedSessionState, StopCause};
use super::resume_breaker::{
BreakerVerdict, DEFAULT_MAX_CONSECUTIVE, ENV_FLAP_WINDOW_SECS, ENV_MAX_CONSECUTIVE, FlapState,
ResumeBreakerConfig, ResumeBreakerStore, evaluate,
};
use super::{FakeNoopTmuxDriver, SessionManager};
#[test]
fn config_defaults() {
let cfg = ResumeBreakerConfig::default();
assert_eq!(cfg.flap_window, Duration::from_secs(120));
assert_eq!(cfg.max_consecutive, DEFAULT_MAX_CONSECUTIVE);
}
#[test]
fn config_env_parsing() {
let cfg = ResumeBreakerConfig::from_env_with(|k| match k {
ENV_FLAP_WINDOW_SECS => Some(" 45 ".to_string()),
ENV_MAX_CONSECUTIVE => Some("3".to_string()),
_ => None,
});
assert_eq!(cfg.flap_window, Duration::from_secs(45));
assert_eq!(cfg.max_consecutive, 3);
let cfg = ResumeBreakerConfig::from_env_with(|_| Some("nope".to_string()));
assert_eq!(cfg, ResumeBreakerConfig::default());
let cfg =
ResumeBreakerConfig::from_env_with(|k| (k == ENV_MAX_CONSECUTIVE).then(|| "0".to_string()));
assert_eq!(cfg.max_consecutive, 1);
}
fn cfg(window_secs: u64, k: u32) -> ResumeBreakerConfig {
ResumeBreakerConfig {
flap_window: Duration::from_secs(window_secs),
max_consecutive: k,
}
}
#[test]
fn evaluate_resets_without_a_prior_auto_resume() {
let v = evaluate(&cfg(120, 3), &FlapState::default(), Utc::now());
assert_eq!(v, BreakerVerdict::Reset);
}
#[test]
fn evaluate_resets_for_a_slow_death() {
let now = Utc::now();
let state = FlapState {
last_auto_resume_at: Some(now - TimeDelta::seconds(500)),
consecutive: 2,
};
assert_eq!(evaluate(&cfg(120, 3), &state, now), BreakerVerdict::Reset);
}
#[test]
fn evaluate_counts_a_fast_death() {
let now = Utc::now();
let state = FlapState {
last_auto_resume_at: Some(now - TimeDelta::seconds(60)),
consecutive: 1,
};
assert_eq!(
evaluate(&cfg(120, 5), &state, now),
BreakerVerdict::Counting { consecutive: 2 }
);
}
#[test]
fn evaluate_parks_at_the_threshold() {
let now = Utc::now();
let state = FlapState {
last_auto_resume_at: Some(now - TimeDelta::seconds(60)),
consecutive: 2,
};
assert_eq!(
evaluate(&cfg(120, 3), &state, now),
BreakerVerdict::Park { consecutive: 3 }
);
}
#[test]
fn evaluate_resets_on_a_backwards_clock() {
let now = Utc::now();
let state = FlapState {
last_auto_resume_at: Some(now + TimeDelta::seconds(600)),
consecutive: 4,
};
assert_eq!(evaluate(&cfg(120, 5), &state, now), BreakerVerdict::Reset);
}
#[test]
fn a_zero_window_disables_the_breaker() {
let now = Utc::now();
let state = FlapState {
last_auto_resume_at: Some(now),
consecutive: 99,
};
assert_eq!(evaluate(&cfg(0, 1), &state, now), BreakerVerdict::Reset);
}
#[test]
fn verdict_reports_its_streak_length() {
assert_eq!(BreakerVerdict::Reset.consecutive(), 0);
assert_eq!(BreakerVerdict::Counting { consecutive: 2 }.consecutive(), 2);
assert_eq!(BreakerVerdict::Park { consecutive: 7 }.consecutive(), 7);
}
#[tokio::test]
async fn store_round_trips_state() {
let tmp = tempfile::TempDir::new().expect("tempdir");
let id = ManagedSessionId::new();
let now = Utc::now();
{
let mut store = ResumeBreakerStore::load(tmp.path()).await;
store.note_auto_resume(&id, now).await;
assert_eq!(
store.record_death(&id, &cfg(120, 3), now).await,
BreakerVerdict::Counting { consecutive: 1 }
);
}
let mut reloaded = ResumeBreakerStore::load(tmp.path()).await;
assert_eq!(reloaded.state_of(&id).await.consecutive, 1);
}
#[tokio::test]
async fn an_external_write_is_picked_up_by_the_next_read() {
let tmp = tempfile::TempDir::new().expect("tempdir");
let id = ManagedSessionId::new();
let now = Utc::now();
let mut store_a = ResumeBreakerStore::load(tmp.path()).await;
let mut store_b = ResumeBreakerStore::load(tmp.path()).await;
assert_eq!(store_b.state_of(&id).await, FlapState::default());
store_a.note_auto_resume(&id, now).await;
assert_eq!(
store_b.state_of(&id).await.last_auto_resume_at,
Some(now),
"a read must reconcile with disk before answering (#6568)"
);
}
#[tokio::test]
async fn a_same_length_external_rewrite_is_still_picked_up() {
let tmp = tempfile::TempDir::new().expect("tempdir");
let id = ManagedSessionId::new();
let now = Utc::now();
let mut writer = ResumeBreakerStore::load(tmp.path()).await;
let mut reader = ResumeBreakerStore::load(tmp.path()).await;
writer.note_auto_resume(&id, now).await;
let first = writer.record_death(&id, &cfg(3600, 9), now).await;
assert_eq!(first, BreakerVerdict::Counting { consecutive: 1 });
let len_after_first = std::fs::metadata(tmp.path().join(ResumeBreakerStore::FILE_NAME))
.expect("sidecar exists")
.len();
assert_eq!(reader.state_of(&id).await.consecutive, 1);
let second = writer.record_death(&id, &cfg(3600, 9), now).await;
assert_eq!(second, BreakerVerdict::Counting { consecutive: 2 });
let len_after_second = std::fs::metadata(tmp.path().join(ResumeBreakerStore::FILE_NAME))
.expect("sidecar exists")
.len();
assert_eq!(
len_after_first, len_after_second,
"the premise of this test is that the length does not move"
);
assert_eq!(
reader.state_of(&id).await.consecutive,
2,
"a same-length rewrite must still reach the other instance (#6568)"
);
}
#[tokio::test]
async fn a_write_does_not_erase_another_instances_rows() {
let tmp = tempfile::TempDir::new().expect("tempdir");
let (a, b) = (ManagedSessionId::new(), ManagedSessionId::new());
let now = Utc::now();
let mut store_a = ResumeBreakerStore::load(tmp.path()).await;
let mut store_b = ResumeBreakerStore::load(tmp.path()).await;
store_a.note_auto_resume(&a, now).await;
store_b.note_auto_resume(&b, now).await;
let mut reader = ResumeBreakerStore::load(tmp.path()).await;
assert_eq!(reader.state_of(&a).await.last_auto_resume_at, Some(now));
assert_eq!(reader.state_of(&b).await.last_auto_resume_at, Some(now));
}
#[tokio::test]
async fn store_survives_a_missing_file() {
let tmp = tempfile::TempDir::new().expect("tempdir");
let mut store = ResumeBreakerStore::load(tmp.path()).await;
assert_eq!(
store.state_of(&ManagedSessionId::new()).await,
FlapState::default()
);
}
#[tokio::test]
async fn store_survives_a_corrupt_file() {
let tmp = tempfile::TempDir::new().expect("tempdir");
std::fs::write(tmp.path().join(ResumeBreakerStore::FILE_NAME), "{ not json")
.expect("write corrupt sidecar");
let mut store = ResumeBreakerStore::load(tmp.path()).await;
assert_eq!(
store.state_of(&ManagedSessionId::new()).await,
FlapState::default()
);
}
#[tokio::test]
async fn a_reset_forgets_the_stamp_as_well_as_the_count() {
let tmp = tempfile::TempDir::new().expect("tempdir");
let id = ManagedSessionId::new();
let now = Utc::now();
let mut store = ResumeBreakerStore::load(tmp.path()).await;
store
.note_auto_resume(&id, now - TimeDelta::seconds(500))
.await;
assert_eq!(
store.record_death(&id, &cfg(120, 2), now).await,
BreakerVerdict::Reset
);
assert_eq!(store.state_of(&id).await, FlapState::default());
assert_eq!(
store.record_death(&id, &cfg(120, 2), now).await,
BreakerVerdict::Reset
);
}
#[tokio::test]
async fn a_fast_death_after_an_auto_resume_counts() {
let tmp = tempfile::TempDir::new().expect("tempdir");
let id = ManagedSessionId::new();
let now = Utc::now();
let mut store = ResumeBreakerStore::load(tmp.path()).await;
store.note_auto_resume(&id, now).await;
assert_eq!(
store
.record_death(&id, &cfg(120, 9), now + TimeDelta::seconds(30))
.await,
BreakerVerdict::Counting { consecutive: 1 }
);
}
fn fixture_workspace(dir: &tempfile::TempDir) -> std::path::PathBuf {
let ws = dir.path().join("ws");
std::fs::create_dir_all(&ws).expect("create fixture workspace");
ws
}
async fn seeded(dir: &tempfile::TempDir, k: u32) -> (Arc<SessionManager>, ManagedSessionId) {
let mut mgr = SessionManager::new(dir.path(), Arc::new(FakeNoopTmuxDriver))
.await
.expect("session manager");
mgr.set_resume_breaker_config(ResumeBreakerConfig {
flap_window: Duration::from_secs(3600),
max_consecutive: k,
});
let mgr = Arc::new(mgr);
let ws = fixture_workspace(dir);
let record = mgr
.create("flap-test".into(), Some(ws.clone()), None, None, None, None)
.await
.expect("create");
let id = record.id;
mgr.set_workspace(&id, ws, ManagedSessionState::Active)
.await
.expect("set Active");
(mgr, id)
}
async fn one_cycle(mgr: &Arc<SessionManager>, id: &ManagedSessionId) -> super::SessionRecord {
let stopped = mgr
.mark_runtime_exited_stopped(id)
.await
.expect("mark stopped");
if stopped.is_auto_resumable() {
mgr.resume_auto(id).await.expect("auto-resume");
}
stopped
}
async fn attach(dir: &tempfile::TempDir, k: u32) -> Arc<SessionManager> {
let mut mgr = SessionManager::new(dir.path(), Arc::new(FakeNoopTmuxDriver))
.await
.expect("session manager");
mgr.set_resume_breaker_config(ResumeBreakerConfig {
flap_window: Duration::from_secs(3600),
max_consecutive: k,
});
Arc::new(mgr)
}
#[tokio::test]
async fn two_managers_over_one_data_dir_still_park_a_flapping_session() {
let tmp = tempfile::TempDir::new().expect("tempdir");
let (daemon, id) = seeded(&tmp, 3).await;
let supervisor = attach(&tmp, 3).await;
let first = daemon
.mark_runtime_exited_stopped(&id)
.await
.expect("mark stopped");
assert_eq!(first.stop_cause, Some(StopCause::Unexpected));
supervisor.resume_auto(&id).await.expect("auto-resume");
for cycle in 2..=3 {
let r = daemon
.mark_runtime_exited_stopped(&id)
.await
.expect("mark stopped");
assert_eq!(
r.stop_cause,
Some(StopCause::Unexpected),
"cycle {cycle} must still be resumable"
);
supervisor.resume_auto(&id).await.expect("auto-resume");
}
let parked = daemon
.mark_runtime_exited_stopped(&id)
.await
.expect("mark stopped");
assert_eq!(
parked.stop_cause,
Some(StopCause::ResumeFlapping),
"the breaker must trip when the two halves of the cycle run in different \
processes — the only topology that exists in production (#6568)"
);
assert!(!parked.is_auto_resumable());
let seen_by_supervisor = supervisor.get(&id).await.expect("get");
assert!(
!seen_by_supervisor.is_auto_resumable(),
"the process that does the resuming must be the one that sees the park"
);
}
#[tokio::test]
async fn the_supervisor_parks_a_flapping_session_after_k_cycles() {
let tmp = tempfile::TempDir::new().expect("tempdir");
let (mgr, id) = seeded(&tmp, 3).await;
let first = one_cycle(&mgr, &id).await;
assert_eq!(first.stop_cause, Some(StopCause::Unexpected));
assert!(first.is_auto_resumable());
for cycle in 1..=2 {
let r = one_cycle(&mgr, &id).await;
assert_eq!(
r.stop_cause,
Some(StopCause::Unexpected),
"cycle {cycle} must still be resumable"
);
assert!(r.is_auto_resumable(), "cycle {cycle}");
}
let parked = one_cycle(&mgr, &id).await;
assert_eq!(
parked.stop_cause,
Some(StopCause::ResumeFlapping),
"the breaker must trip at K consecutive fast deaths (#6568)"
);
assert!(
!parked.is_auto_resumable(),
"a parked session must be invisible to every automatic resume path"
);
let after = mgr.get(&id).await.expect("get");
assert_eq!(after.stop_cause, Some(StopCause::ResumeFlapping));
assert!(!after.is_auto_resumable());
}
#[tokio::test]
async fn an_operator_resume_forgives_the_streak() {
let tmp = tempfile::TempDir::new().expect("tempdir");
let (mgr, id) = seeded(&tmp, 2).await;
one_cycle(&mgr, &id).await; one_cycle(&mgr, &id).await; let parked = mgr
.mark_runtime_exited_stopped(&id)
.await
.expect("mark stopped");
assert_eq!(parked.stop_cause, Some(StopCause::ResumeFlapping));
let revived = mgr.resume(&id).await.expect("operator resume");
assert_eq!(revived.state, ManagedSessionState::Active);
assert_eq!(revived.stop_cause, None);
let next = mgr
.mark_runtime_exited_stopped(&id)
.await
.expect("mark stopped");
assert_eq!(
next.stop_cause,
Some(StopCause::Unexpected),
"an operator resume must restore the full budget, not one death of it"
);
assert!(next.is_auto_resumable());
}
#[tokio::test]
async fn an_in_place_reactivate_forgives_the_streak() {
let tmp = tempfile::TempDir::new().expect("tempdir");
let (mgr, id) = seeded(&tmp, 2).await;
one_cycle(&mgr, &id).await; let counted = mgr
.mark_runtime_exited_stopped(&id)
.await
.expect("mark stopped");
assert_eq!(counted.stop_cause, Some(StopCause::Unexpected));
let revived = mgr.mark_reactivated(&id).await.expect("reactivate");
assert_eq!(revived.state, ManagedSessionState::Active);
assert_eq!(revived.stop_cause, None);
let next = mgr
.mark_runtime_exited_stopped(&id)
.await
.expect("mark stopped");
assert_eq!(
next.stop_cause,
Some(StopCause::Unexpected),
"an in-place reactivate must restore the full budget, like `resume` does"
);
assert!(next.is_auto_resumable());
}
#[tokio::test]
async fn a_session_that_dies_slowly_is_never_parked() {
let tmp = tempfile::TempDir::new().expect("tempdir");
let mut mgr = SessionManager::new(tmp.path(), Arc::new(FakeNoopTmuxDriver))
.await
.expect("session manager");
mgr.set_resume_breaker_config(ResumeBreakerConfig {
flap_window: Duration::from_secs(0),
max_consecutive: 2,
});
let mgr = Arc::new(mgr);
let ws = fixture_workspace(&tmp);
let record = mgr
.create("slow".into(), Some(ws.clone()), None, None, None, None)
.await
.expect("create");
let id = record.id;
mgr.set_workspace(&id, ws, ManagedSessionState::Active)
.await
.expect("set Active");
for cycle in 1..=6 {
let r = one_cycle(&mgr, &id).await;
assert_eq!(
r.stop_cause,
Some(StopCause::Unexpected),
"cycle {cycle}: a slow death must never park the session"
);
assert!(r.is_auto_resumable(), "cycle {cycle}");
}
}
async fn stopped_record_with(
dir: &tempfile::TempDir,
cause: Option<StopCause>,
) -> super::SessionRecord {
let mgr = SessionManager::new(dir.path(), Arc::new(FakeNoopTmuxDriver))
.await
.expect("session manager");
let mut r = mgr
.create(
"cause".into(),
Some(fixture_workspace(dir)),
None,
None,
None,
None,
)
.await
.expect("create");
r.state = ManagedSessionState::Stopped;
r.stop_cause = cause;
r
}
#[tokio::test]
async fn park_reason_is_set_only_for_a_flapping_record() {
let tmp = tempfile::TempDir::new().expect("tempdir");
for cause in [
None,
Some(StopCause::Unexpected),
Some(StopCause::Deliberate),
] {
let r = stopped_record_with(&tmp, cause).await;
assert!(
r.auto_resume_park_reason().is_none(),
"{cause:?} is not a breaker trip"
);
}
let parked = stopped_record_with(&tmp, Some(StopCause::ResumeFlapping)).await;
let reason = parked
.auto_resume_park_reason()
.expect("a flapping record must name its park reason");
assert!(reason.contains("#6568"), "reason was: {reason}");
assert!(
reason.contains("tm session resume"),
"the reason must tell an operator how to clear it: {reason}"
);
}
#[tokio::test]
async fn a_stopped_flap_session_writes_its_scrollback_inside_its_own_tempdir() {
let tmp = tempfile::TempDir::new().expect("tempdir");
let (mgr, id) = seeded(&tmp, 3).await;
let stopped = one_cycle(&mgr, &id).await;
let written = stopped
.scrollback_path
.expect("the stop path captured scrollback");
assert!(
written.starts_with(tmp.path()),
"scrollback escaped the fixture into a shared directory: {}",
written.display()
);
}
#[tokio::test]
async fn summary_carries_the_resume_park_reason() {
use crate::daemon::managed_routes::summary::record_to_summary;
let tmp = tempfile::TempDir::new().expect("tempdir");
let ordinary = stopped_record_with(&tmp, Some(StopCause::Unexpected)).await;
assert!(record_to_summary(&ordinary).auto_resume_parked.is_none());
let parked = stopped_record_with(&tmp, Some(StopCause::ResumeFlapping)).await;
assert_eq!(
record_to_summary(&parked).auto_resume_parked.as_deref(),
parked.auto_resume_park_reason(),
"the summary must not restate the reason in its own words"
);
}