use super::{HeldJob, History, HostHold, Run, digest, insights};
use crate::api::{self, AutoNote, InsightKind, JobState, Severity};
use crate::config::Config;
const G: u64 = 1 << 30;
const MINUTE: u64 = 60 * 1000;
const HOUR: u64 = 60 * MINUTE;
const NOW: u64 = 1_800_000_000_000;
const CONFIG: &str = r#"
[host]
slice = "runnerd.slice"
isolation = "sysbox"
[caches]
mirror = { enabled = false, memory = "512M", cpus = 1, ttl = "168h" }
pnpm = { max_size = "5G" }
maven = { max_size = "5G" }
[profiles.app]
image_stage = "node"
cpus = 4
memory = "4G"
slots = 2
[[repositories]]
name = "halyard/api"
profile = "app"
labels = ["app"]
"#;
fn config() -> Config {
Config::parse(CONFIG).unwrap()
}
fn test_run(n: u64, hours_ago: u64, seconds: u64) -> Run {
let finished = NOW - hours_ago * HOUR;
Run {
id: 1000 + n,
repository: "halyard/api".to_owned(),
workflow: Some("CI".to_owned()),
name: "test".to_owned(),
branch: Some("main".to_owned()),
url: Some(format!(
"https://github.com/halyard/api/actions/runs/{n}/job/{}",
1000 + n
)),
run_id: n,
run_attempt: 1,
state: JobState::Completed,
conclusion: Some("success".to_owned()),
queued_at: finished - seconds * 1000 - 20_000,
started_at: Some(finished - seconds * 1000),
finished_at: Some(finished),
runner_started_at: Some(finished - seconds * 1000 - 15_000),
profile: Some("app".to_owned()),
memory_peak: Some(G),
build_cache: None,
sock_throttled: Some(0),
unseen: 0,
cpu_cap: Some(4000),
cpu_periods: None,
cpu_throttled: None,
}
}
fn whole(runs: &[Run]) -> History {
History {
runs: runs.to_vec(),
kept_since: 0,
host: None,
}
}
fn history(durations: &[u64]) -> Vec<Run> {
let count = durations.len() as u64;
durations
.iter()
.zip(0..)
.map(|(seconds, n)| test_run(n, 1 + (count - 1 - n) * 6, *seconds))
.collect()
}
#[test]
fn a_job_that_got_slower_and_stayed_slower_is_slower_since_its_first_slow_run() {
let mut durations = vec![240; 8];
durations.extend([360; 6]);
let runs = history(&durations);
let found = insights(&whole(&runs), &config(), 14, NOW).insights;
assert_eq!(found.len(), 1, "{found:#?}");
let slower = &found[0];
assert_eq!(slower.key, "slower:halyard/api/CI/test");
assert_eq!(slower.severity, Severity::Warning);
let InsightKind::Slower {
branch,
before,
now,
since,
} = &slower.kind
else {
panic!("not slower: {slower:#?}");
};
assert_eq!((branch.as_deref(), *before, *now), (Some("main"), 240, 360));
assert_eq!(since.as_ref().map(|since| since.job), Some(runs[8].id));
assert_eq!(
slower.sentence,
"halyard/api: CI / test on main takes 6m now, up from 4m."
);
}
#[test]
fn a_one_off_slow_run_is_not_a_regression() {
let mut durations = vec![240; 13];
durations[10] = 900;
assert!(
insights(&whole(&history(&durations)), &config(), 14, NOW)
.insights
.is_empty(),
"one slow run among fast ones says nothing"
);
}
#[test]
fn a_job_fast_again_in_its_last_two_runs_is_not_slower() {
let mut durations = vec![240; 8];
durations.extend([360, 360, 360, 240, 240]);
assert!(
insights(&whole(&history(&durations)), &config(), 14, NOW)
.insights
.is_empty(),
"the slow stretch ended"
);
}
#[test]
fn a_job_that_flips_between_cache_hits_and_misses_is_not_slower() {
let durations = [
112, 37, 44, 46, 146, 39, 124, 134, 41, 156, 122, 126, 147, 40,
];
assert!(
insights(&whole(&history(&durations)), &config(), 14, NOW)
.insights
.is_empty(),
"a recent run as fast as before says the job did not get slower"
);
}
#[test]
fn a_small_slowdown_is_not_a_regression() {
let mut durations = vec![120; 8];
durations.extend([144; 6]);
assert!(
insights(&whole(&history(&durations)), &config(), 14, NOW)
.insights
.is_empty(),
"under 30 seconds is noise"
);
}
#[test]
fn slower_runs_on_a_branch_with_few_runs_do_not_count() {
let mut runs = history(&[240; 10]);
for n in 0..6 {
let mut feature = test_run(100 + n, 2, 600);
feature.branch = Some("feature/slow".to_owned());
runs.push(feature);
}
assert!(
insights(&whole(&runs), &config(), 14, NOW)
.insights
.is_empty(),
"only the busiest branch is compared"
);
}
#[test]
fn main_is_compared_even_when_pull_requests_run_more() {
let mut runs = history(&[240; 10]);
for n in 0..14 {
let mut feature = test_run(100 + n, 30 - n, if n < 7 { 240 } else { 600 });
feature.branch = Some("feature/more-tests".to_owned());
runs.push(feature);
}
assert!(
insights(&whole(&runs), &config(), 14, NOW)
.insights
.is_empty(),
"main runs as fast as it did"
);
}
fn attempt(run_id: u64, attempt: u32, conclusion: &str) -> Run {
let mut run = test_run(run_id * 10 + u64::from(attempt), 3, 240);
run.run_id = run_id;
run.run_attempt = attempt;
run.conclusion = Some(conclusion.to_owned());
run
}
#[test]
fn a_job_that_failed_and_passed_on_a_re_run_is_flaky() {
let runs = vec![
attempt(1, 1, "success"),
attempt(2, 1, "failure"),
attempt(2, 2, "success"),
attempt(3, 1, "success"),
];
let found = insights(&whole(&runs), &config(), 14, NOW).insights;
assert_eq!(found.len(), 1, "{found:#?}");
assert_eq!(found[0].key, "flaky:halyard/api/CI/test");
assert_eq!(found[0].kind, InsightKind::Flaky { flaky: 1, total: 3 });
assert_eq!(
found[0].severity,
Severity::Notice,
"once may be chance: shown, not sent"
);
assert_eq!(found[0].runs[0].job, runs[1].id, "links the failed attempt");
assert_eq!(
found[0].sentence,
"halyard/api: CI / test failed and then passed on a re-run in 1 of its 3 runs."
);
}
#[test]
fn a_job_flaky_twice_is_a_warning() {
let runs = vec![
attempt(1, 1, "failure"),
attempt(1, 2, "success"),
attempt(2, 1, "timed_out"),
attempt(2, 2, "success"),
];
let found = insights(&whole(&runs), &config(), 14, NOW).insights;
assert_eq!(found[0].kind, InsightKind::Flaky { flaky: 2, total: 2 });
assert_eq!(found[0].severity, Severity::Warning);
}
#[test]
fn a_job_that_failed_again_on_a_re_run_is_not_flaky() {
let runs = vec![attempt(2, 1, "failure"), attempt(2, 2, "failure")];
assert!(
insights(&whole(&runs), &config(), 14, NOW)
.insights
.is_empty(),
"failing twice is broken, not flaky"
);
}
#[test]
fn a_lost_job_that_passed_on_a_re_run_is_not_flaky() {
let mut lost = attempt(2, 1, "failure");
lost.state = JobState::Lost;
lost.conclusion = None;
let runs = vec![lost, attempt(2, 2, "success")];
assert!(
insights(&whole(&runs), &config(), 14, NOW)
.insights
.is_empty(),
"a vanished runner is the host's fault"
);
}
fn cached(branch: &str, hits: u64, writer: bool) -> Vec<Run> {
history(&[240; 5])
.into_iter()
.map(|run| Run {
branch: Some(branch.to_owned()),
build_cache: Some(api::BuildCacheCounts {
hits,
misses: 40 - hits,
writer,
..api::BuildCacheCounts::default()
}),
..run
})
.collect()
}
#[test]
fn main_runs_that_find_nothing_their_predecessors_stored_have_a_cold_cache() {
let found = insights(&whole(&cached("main", 1, true)), &config(), 14, NOW).insights;
assert_eq!(found.len(), 1, "{found:?}");
assert_eq!(
found[0].kind,
InsightKind::ColdCache {
branch: Some("main".to_owned()),
hits: 5,
lookups: 200,
}
);
assert!(
found[0].sentence.contains("2% of its build cache lookups"),
"{}",
found[0].sentence
);
}
#[test]
fn a_warm_cache_or_runs_that_only_read_are_no_insight() {
for runs in [cached("main", 30, true), cached("feature", 0, false)] {
let found = insights(&whole(&runs), &config(), 14, NOW).insights;
assert!(found.is_empty(), "{found:?}");
}
}
fn throttled_at(throttled: &[usize]) -> Vec<Run> {
let mut runs = history(&[240; 10]);
for at in throttled {
runs[*at].sock_throttled = Some(846);
}
runs
}
#[test]
fn downloads_memory_pressure_throttled_in_several_runs_are_a_warning() {
let found = insights(&whole(&throttled_at(&[3, 8])), &config(), 14, NOW).insights;
assert_eq!(found.len(), 1, "{found:?}");
assert_eq!(found[0].severity, Severity::Warning);
assert_eq!(
found[0].kind,
InsightKind::Throttled {
throttled: 2,
total: 10
}
);
let linked: Vec<u64> = found[0].runs.iter().map(|run| run.job).collect();
assert_eq!(linked, [1008, 1003], "the throttled runs, newest first");
}
#[test]
fn one_throttled_run_is_no_insight() {
let found = insights(&whole(&throttled_at(&[5])), &config(), 14, NOW).insights;
assert!(found.is_empty(), "{found:?}");
}
fn peaking(peaks_in_tenths_of_g: &[u64]) -> Vec<Run> {
let mut runs = history(&vec![240; peaks_in_tenths_of_g.len()]);
for (run, peak) in runs.iter_mut().zip(peaks_in_tenths_of_g) {
run.memory_peak = Some(peak * G / 10);
}
runs
}
#[test]
fn memory_near_the_cap_is_flagged_with_what_auto_sizing_does() {
let runs = peaking(&[30, 31, 33, 35, 36, 36]);
let found = insights(&whole(&runs), &config(), 14, NOW).insights;
assert_eq!(found.len(), 1, "{found:#?}");
let InsightKind::NearCap {
profile, cap, auto, ..
} = &found[0].kind
else {
panic!("not near the cap: {found:#?}");
};
assert_eq!(
(profile.as_str(), *cap, *auto),
("app", 4 * G, AutoNote::Off)
);
assert_eq!(
found[0].severity,
Severity::Warning,
"three runs near the cap and nothing raises it"
);
assert!(
found[0]
.sentence
.ends_with("Auto sizing is off: raise app's memory, or turn it on."),
"{}",
found[0].sentence
);
let auto = Config::parse(&format!("{CONFIG}\n[sizing]\nauto = true\n")).unwrap();
let found = insights(&whole(&runs), &auto, 14, NOW).insights;
assert!(
matches!(
found[0].kind,
InsightKind::NearCap {
auto: AutoNote::Raises,
..
}
),
"auto sizing raises it: {found:#?}"
);
assert_eq!(
found[0].severity,
Severity::Notice,
"auto sizing deals with it: shown, not sent"
);
}
#[test]
fn one_spike_near_the_cap_is_a_notice() {
let found = insights(
&whole(&peaking(&[30, 30, 31, 30, 30, 36])),
&config(),
14,
NOW,
)
.insights;
assert!(
matches!(found[0].kind, InsightKind::NearCap { .. }),
"{found:#?}"
);
assert_eq!(found[0].severity, Severity::Notice);
}
#[test]
fn memory_well_under_the_cap_says_nothing() {
assert!(
insights(
&whole(&peaking(&[20, 21, 22, 20, 24, 25])),
&config(),
14,
NOW
)
.insights
.is_empty(),
"62% of the cap is fine"
);
}
#[test]
fn a_job_served_in_two_profiles_is_near_the_cap_by_its_last_profiles_runs_alone() {
let config = Config::parse(&format!(
"{CONFIG}\n[profiles.light]\nimage_stage = \"node\"\ncpus = 1\nmemory = \"1G\"\nslots = 1\n"
))
.unwrap();
let mut runs = peaking(&[30, 30, 30, 30, 30, 30, 30, 30, 30, 2]);
runs.last_mut().unwrap().profile = Some("light".to_owned());
assert!(
insights(&whole(&runs), &config, 14, NOW)
.insights
.is_empty(),
"nothing is near a cap"
);
}
fn throttled_runs(shares: &[Option<u64>]) -> Vec<Run> {
let mut runs = history(&vec![240; shares.len()]);
for (run, share) in runs.iter_mut().zip(shares) {
run.cpu_periods = share.map(|_| 2400);
run.cpu_throttled = share.map(|share| share * 2400 / 1000);
}
runs
}
fn measured(auto: bool) -> Config {
Config::parse(&format!(
"{}\n[sizing]\nauto = {auto}\n",
CONFIG.replace(
"isolation = \"sysbox\"",
"isolation = \"sysbox\"\nadmission = \"measured\""
)
))
.unwrap()
}
fn cpu_held(history: &History, config: &Config) -> Vec<api::Insight> {
insights(history, config, 14, NOW)
.insights
.into_iter()
.filter(|insight| matches!(insight.kind, InsightKind::CpuHeld { .. }))
.collect()
}
#[test]
fn a_job_its_cpu_cap_held_back_three_times_says_the_cpus_to_set_under_caps() {
let runs = throttled_runs(&[
Some(0),
Some(150),
Some(149),
Some(40),
Some(300),
Some(150),
Some(0),
]);
let found = cpu_held(&whole(&runs), &config());
assert_eq!(found.len(), 1, "{found:#?}");
assert_eq!(found[0].key, "cpu-held:halyard/api/CI/test");
assert_eq!(found[0].severity, Severity::Warning, "the owner acts");
assert_eq!(
found[0].kind,
InsightKind::CpuHeld {
profile: "app".to_owned(),
cpus: 4,
held: 3,
total: 7,
raised_from: None,
set: Some(6),
}
);
assert_eq!(
found[0].sentence,
"halyard/api: CI / test was held back by app's 4 CPUs in 3 of its last 7 runs, its quota throttling it in 15% \
of its run or more. Under caps admission each CPU takes budget, so auto sizing does not raise cpus for this: \
set app's cpus to 6 if runnerctl sizing shows the budget has room."
);
let linked: Vec<u64> = found[0].runs.iter().map(|run| run.job).collect();
assert_eq!(linked, [1005, 1004, 1001], "the held runs, newest first");
let raised = Config::parse(&CONFIG.replace("cpus = 4", "cpus = 6")).unwrap();
assert!(
cpu_held(&whole(&runs), &raised).is_empty(),
"runs at 4 CPUs say nothing of 6"
);
}
#[test]
fn runs_without_a_throttling_figure_are_neither_held_nor_free() {
let unknown = [None; 8];
assert!(
cpu_held(&whole(&[]), &config()).is_empty(),
"no runs say nothing"
);
assert!(
cpu_held(&whole(&throttled_runs(&unknown)), &config()).is_empty(),
"runs from before schema 17 say nothing"
);
let mut shares = unknown.to_vec();
shares.extend([Some(200), Some(200)]);
assert!(
cpu_held(&whole(&throttled_runs(&shares)), &config()).is_empty(),
"two held runs are no insight"
);
shares.insert(3, Some(200));
let found = cpu_held(&whole(&throttled_runs(&shares)), &config());
assert!(
matches!(
found[0].kind,
InsightKind::CpuHeld {
held: 3,
total: 3,
..
}
),
"{found:#?}"
);
}
#[test]
fn under_measured_admission_auto_sizing_raises_cpus_until_a_raise_does_not_help() {
let held = throttled_runs(&[Some(300); 4]);
let found = cpu_held(&whole(&held), &measured(true));
assert_eq!(
found[0].severity,
Severity::Notice,
"auto sizing deals with it"
);
assert!(
found[0].sentence.ends_with(
"Auto sizing raises app's cpus while each raise makes it faster or less throttled and the slice \
has room; runnerctl sizing says when it cannot."
),
"{}",
found[0].sentence
);
let off = cpu_held(&whole(&held), &measured(false));
assert_eq!(off[0].severity, Severity::Warning);
assert!(
off[0]
.sentence
.ends_with("Auto sizing is off: set app's cpus to 6, or turn it on."),
"{}",
off[0].sentence
);
let mut runs = throttled_runs(&[Some(300); 7]);
for run in &mut runs[..3] {
run.cpu_cap = Some(3000);
}
let unhelped = cpu_held(&whole(&runs), &measured(true));
assert_eq!(
unhelped[0].severity,
Severity::Warning,
"its runs ask for no further raise"
);
assert!(
matches!(
unhelped[0].kind,
InsightKind::CpuHeld {
raised_from: Some(3),
set: None,
held: 4,
..
}
),
"{unhelped:#?}"
);
assert!(
unhelped[0].sentence.ends_with(UNHELPED),
"{}",
unhelped[0].sentence
);
for run in &mut runs[..3] {
run.cpu_throttled = Some(960);
}
let helped = cpu_held(&whole(&runs), &measured(true));
assert_eq!(helped[0].severity, Severity::Notice, "{helped:#?}");
}
const UNHELPED: &str = "The raise of app from 3 to 4 CPUs did not make it faster or less throttled, as when its tools \
start a worker per CPU, so its runs ask for no further raise; runnerctl sizing says what auto \
sizing does next.";
fn as_sized(runs: &[Run], average: u64) -> Vec<crate::sizing::Ran> {
runs.iter()
.map(|run| crate::sizing::Ran {
repository: run.repository.clone(),
workflow: run.workflow.clone(),
name: run.name.clone(),
profile: run.profile.clone().unwrap(),
memory_cap: Some(4 * G),
cpu_cap: run.cpu_cap,
memory_peak: G,
cpu_peak: run.cpu_cap.unwrap(),
cpu_average: Some(average),
queued_at: run.queued_at,
started_at: run.started_at,
runner_started_at: run.runner_started_at.unwrap(),
finished_at: run.finished_at,
oom_killed: false,
hit_cap: None,
own_cap: None,
measure: crate::sizing::Measure::WithoutPageCache,
succeeded: true,
failed: false,
run_id: run.run_id,
run_attempt: 1,
branch: run.branch.clone(),
reserved: None,
host_waited: 0,
slot_waited: 0,
unseen: 0,
cpu_periods: run.cpu_periods,
cpu_throttled: run.cpu_throttled,
})
.collect()
}
fn sizing_raises_app_to(config: &Config, jobs: Vec<crate::sizing::Ran>) -> Option<u32> {
let history = crate::sizing::History {
jobs,
..crate::sizing::History::default()
};
let (last, running) = (
std::collections::BTreeMap::new(),
std::collections::BTreeMap::new(),
);
let changes = crate::sizing::auto::changes(
config,
&history,
crate::sizing::auto::Context {
memory_max: Some(33 * G),
cpus: Some(32_000),
window: crate::sizing::Window {
since: NOW - 30 * 24 * HOUR,
now: NOW,
sample_interval: 2,
},
last: &last,
cpus_set: &[],
running: &running,
disk: None,
},
);
changes
.iter()
.filter(|change| change.profile == "app")
.find_map(|change| change.to.cpus.filter(|to| *to > 4))
}
fn raised(name: &str, ids: u64, before: u64, now: u64) -> Vec<Run> {
let mut runs = throttled_runs(&[
Some(before),
Some(before),
Some(before),
Some(now),
Some(now),
Some(now),
Some(now),
]);
for run in &mut runs {
run.name = name.to_owned();
run.id += ids;
run.run_id += ids;
}
for run in &mut runs[..3] {
run.cpu_cap = Some(3000);
}
runs
}
#[test]
fn a_raise_that_did_not_help_a_job_whose_runs_average_near_the_cap_never_says_auto_sizing_stopped()
{
let runs = raised("test", 0, 300, 300);
let config = measured(true);
assert_eq!(
sizing_raises_app_to(&config, as_sized(&runs, 3000)),
Some(6),
"auto sizing raises app on the runs' averages"
);
let found = cpu_held(&whole(&runs), &config);
assert!(
found[0].sentence.ends_with(UNHELPED),
"{}",
found[0].sentence
);
assert!(
!found[0].sentence.contains("stopped"),
"it says what this job's runs ask for, not what auto sizing does: {}",
found[0].sentence
);
}
#[test]
fn a_raise_that_did_not_help_one_job_but_helped_another_never_says_auto_sizing_stopped() {
let mut runs = raised("test", 0, 300, 300);
runs.extend(raised("lint", 100, 400, 200));
let config = measured(true);
assert_eq!(
sizing_raises_app_to(&config, as_sized(&runs, 1000)),
Some(6),
"auto sizing raises app for lint"
);
let found = cpu_held(&whole(&runs), &config);
let sentence = |job: &str| {
found
.iter()
.find(|insight| insight.job.as_deref() == Some(job))
.map(|insight| (insight.severity, insight.sentence.as_str()))
.unwrap()
};
let (_, test) = sentence("test");
assert!(test.ends_with(UNHELPED), "{test}");
assert!(!test.contains("stopped"), "{test}");
let (severity, lint) = sentence("lint");
assert_eq!(severity, Severity::Notice, "{lint}");
assert!(lint.contains("Auto sizing raises app's cpus"), "{lint}");
}
fn waited(n: u64, hours_ago: u64, waited: u64, warm: bool) -> Run {
let mut run = test_run(n, hours_ago, 60);
let started = run.started_at.unwrap();
run.queued_at = started - waited * 1000;
run.runner_started_at = Some(if warm {
run.queued_at - 60_000
} else {
started - 5_000
});
run
}
#[test]
fn waits_are_split_warm_and_cold_and_longer_ones_are_a_notice() {
let mut runs: Vec<Run> = (0..6).map(|n| waited(n, 200, 20, false)).collect();
runs.extend((10..16).map(|n| waited(n, 10, 300, false)));
runs.extend((20..24).map(|n| waited(n, 10, 2, true)));
let found = insights(&whole(&runs), &config(), 14, NOW);
let api = &found.waits[0];
assert_eq!(api.repository, "halyard/api");
assert_eq!((api.now.warm.jobs, api.now.cold.jobs), (4, 6));
assert_eq!((api.now.warm.p50, api.now.cold.p50), (Some(2), Some(300)));
assert_eq!(api.before.all.p95, Some(20));
let notice = found
.insights
.iter()
.find(|insight| insight.key == "waits:halyard/api")
.expect("waits went up");
assert_eq!(notice.severity, Severity::Notice);
assert_eq!(
notice.kind,
InsightKind::Waits {
before: 20,
now: 300
}
);
}
#[test]
fn a_week_of_fewer_than_20_jobs_tells_its_longest_wait_and_one_of_more_its_1_in_20() {
let sentence = |now: u64| {
let mut runs: Vec<Run> = (0..6).map(|n| waited(n, 200, 20, false)).collect();
runs.extend((10..10 + now).map(|n| waited(n, 10, 300, false)));
insights(&whole(&runs), &config(), 14, NOW)
.insights
.into_iter()
.find(|insight| insight.key == "waits:halyard/api")
.expect("waits went up")
.sentence
};
assert_eq!(
sentence(19),
"halyard/api: the longest wait for a runner this week is 5m, up from 20s the week before."
);
assert_eq!(
sentence(20),
"halyard/api: 1 job in 20 waits 5m or more for a runner this week, up from 20s the week before."
);
}
#[test]
fn a_week_before_the_history_holds_every_job_is_not_compared() {
let mut runs: Vec<Run> = (0..6).map(|n| waited(n, 8 * 24 - 1, 20, false)).collect();
runs.extend((10..16).map(|n| waited(n, 10, 300, false)));
let young = History {
runs,
kept_since: NOW - 8 * 24 * HOUR,
host: None,
};
let found = insights(&young, &config(), 14, NOW);
let api = &found.waits[0];
assert_eq!((api.before.all.jobs, api.before.all.p95), (0, None));
assert_eq!(
api.now.all.p95,
Some(300),
"this week is what the history has"
);
assert!(
found
.insights
.iter()
.all(|insight| !insight.key.starts_with("waits:")),
"no waits insight: {:?}",
found.insights
);
assert!(
digest(&young, &config(), NOW)
.new
.iter()
.all(|insight| !insight.key.starts_with("waits:")),
"nor in the digest"
);
let kept = History {
kept_since: NOW - 14 * 24 * HOUR,
host: None,
..young
};
assert!(
insights(&kept, &config(), 14, NOW)
.insights
.iter()
.any(|insight| insight.key == "waits:halyard/api"),
"a history that holds the whole week before compares it"
);
}
#[test]
fn a_job_queued_before_runnerd_first_ran_waits_from_when_it_did() {
let first = NOW - 10 * HOUR;
let runs: Vec<Run> = (0..6)
.map(|n| {
let mut run = waited(n, 9, 60, false);
run.started_at = Some(first + 60_000);
run.finished_at = Some(first + 120_000);
run.queued_at = first - 3 * HOUR;
run
})
.collect();
let history = History {
runs,
kept_since: first,
host: None,
};
let found = insights(&history, &config(), 14, NOW);
assert_eq!(found.waits[0].now.all.p95, Some(60));
let day = digest(&history, &config(), NOW);
assert_eq!(day.waits[0].waited, 60, "the digest's longest wait too");
}
#[test]
fn the_digest_counts_the_day_and_lists_only_new_insights() {
let mut runs = vec![attempt(900, 1, "failure"), attempt(900, 2, "success")];
for run in &mut runs {
run.finished_at = Some(NOW - 48 * HOUR);
}
let mut durations = vec![240; 9];
durations.extend([400; 5]);
let mut slow = history(&durations);
for (run, hours) in slow.iter_mut().rev().zip([1, 3, 5, 7, 9]) {
run.finished_at = Some(NOW - hours * HOUR);
run.started_at = Some(NOW - hours * HOUR - 400_000);
}
let mut failed = test_run(500, 2, 60);
failed.run_id = 500;
failed.conclusion = Some("failure".to_owned());
slow.push(failed);
runs.extend(slow);
let day = digest(&whole(&runs), &config(), NOW);
assert_eq!((day.finished, day.failed), (6, 1));
let new: Vec<&str> = day.new.iter().map(|insight| insight.key.as_str()).collect();
assert_eq!(new, ["slower:halyard/api/CI/test"]);
assert_eq!(day.waits.len(), 3);
}
#[test]
fn every_kind_of_insight_reads_back_as_the_daemon_sends_it() {
let mut runs = history(&[
240, 240, 240, 240, 240, 240, 240, 240, 360, 360, 360, 360, 360, 360,
]);
for run in &mut runs {
run.memory_peak = Some(37 * G / 10);
}
for run in &mut runs[12..] {
run.sock_throttled = Some(12);
}
for run in &mut runs[11..] {
run.cpu_periods = Some(3600);
run.cpu_throttled = Some(1800);
}
let mut others = vec![attempt(900, 1, "failure"), attempt(900, 2, "success")];
others.extend((0..6).map(|n| waited(n + 2000, 200, 20, false)));
others.extend((10..16).map(|n| waited(n + 2000, 10, 300, false)));
for run in &mut others {
run.name = "lint".to_owned();
run.memory_peak = Some(G);
}
runs.extend(others);
let found = insights(&whole(&runs), &config(), 14, NOW);
let kinds: Vec<&str> = found
.insights
.iter()
.map(|insight| insight.key.split(':').next().unwrap())
.collect();
assert_eq!(
kinds,
[
"throttled",
"cpu-held",
"near-cap",
"slower",
"waits",
"flaky"
],
"warnings first: {found:#?}"
);
let sent = serde_json::to_string(&found).unwrap();
let read: api::Insights = serde_json::from_str(&sent).unwrap();
assert_eq!(read, found);
}
#[test]
fn an_insight_of_a_kind_from_a_later_release_still_reads_by_its_sentence() {
let sent = r#"{"since": 0, "until": 86400, "waits": [], "insights": [{"key": "later:halyard/api",
"severity": "warning", "repository": "halyard/api", "workflow": null, "job": null,
"sentence": "halyard/api: something a later release finds.", "kind": "later", "figure": 3, "runs": []}]}"#;
let read: api::Insights = serde_json::from_str(sent).unwrap();
assert_eq!(read.insights[0].kind, InsightKind::Unknown);
assert_eq!(
read.insights[0].sentence,
"halyard/api: something a later release finds."
);
}
fn held(waited: u64) -> HostHold {
HostHold {
memory: api::HostMemory {
at: NOW / 1000 - 2,
total: 64 * G,
available: 8 * G,
slice: 18 * G,
some: None,
full: None,
holders: vec![
api::Holder {
name: "java".to_owned(),
memory: 9 * G,
},
api::Holder {
name: "claude".to_owned(),
memory: 5 * G,
},
],
ceiling: None,
swap: None,
},
seen: NOW - 3_000,
jobs: vec![
HeldJob {
repository: "halyard/web".to_owned(),
workflow: Some("CI".to_owned()),
name: "lint".to_owned(),
waited: 5 * MINUTE,
},
HeldJob {
repository: "halyard/api".to_owned(),
workflow: Some("CI".to_owned()),
name: "test".to_owned(),
waited,
},
],
}
}
fn holding(hold: Option<HostHold>, at: u64) -> Vec<api::Insight> {
holding_by(hold, &config(), at)
}
fn holding_by(hold: Option<HostHold>, config: &Config, at: u64) -> Vec<api::Insight> {
let history = History {
host: hold,
..History::default()
};
insights(&history, config, 14, at).insights
}
#[test]
fn a_job_held_half_an_hour_for_the_hosts_memory_is_a_warning_naming_who_holds_it() {
let found = holding(Some(held(42 * MINUTE)), NOW);
let [insight] = found.as_slice() else {
panic!("one insight: {found:#?}");
};
assert_eq!(insight.severity, Severity::Warning);
assert_eq!(insight.repository, "halyard/api", "the longest waiter's");
assert_eq!(
insight.kind,
InsightKind::HostHeld {
jobs: 2,
waited: 42 * 60
}
);
assert_eq!(
insight.sentence,
"2 jobs wait for the host's memory, the longest for 42m 00s (halyard/api: CI / test); programs outside CI \
hold 38G, most java 9G and claude 5G. Closing programs frees only part of that."
);
let key = |at: u64| {
let mut hold = held(42 * MINUTE);
hold.memory.at = at / 1000;
hold.seen = at;
holding(Some(hold), at)[0].key.clone()
};
let window = 6 * HOUR;
let start = NOW / window * window;
assert_eq!(key(start), key(start + window - 1000));
assert_ne!(key(start + window - 1000), key(start + window));
}
#[test]
fn nothing_is_said_of_a_host_wait_that_is_short_unseen_stale_or_from_after_the_look() {
let said = |hold, at| !holding(hold, at).is_empty();
assert!(said(Some(held(30 * MINUTE)), NOW), "30 minutes is enough");
assert!(!said(Some(held(30 * MINUTE - 1)), NOW), "29m 59s is not");
assert!(!said(None, NOW), "no job waits for it now");
let mut no_jobs = held(HOUR);
no_jobs.jobs.clear();
assert!(!said(Some(no_jobs), NOW));
let mut old_reading = held(HOUR);
old_reading.memory.at = NOW / 1000 - 61;
assert!(!said(Some(old_reading), NOW), "a reading of a minute ago");
let mut old_look = held(HOUR);
old_look.seen = NOW - 61_000;
assert!(!said(Some(old_look), NOW), "the engine's last look");
let slow = Config::parse(&CONFIG.replace(
"isolation = \"sysbox\"",
"isolation = \"sysbox\"\npoll_interval = \"1m\"",
))
.unwrap();
let mut slow_plan = held(HOUR);
slow_plan.seen = NOW - 170_000;
assert!(
holding_by(Some(slow_plan.clone()), &config(), NOW).is_empty(),
"170 s is over the default's minute"
);
assert!(
!holding_by(Some(slow_plan.clone()), &slow, NOW).is_empty(),
"170 s is under three 1-minute polls"
);
slow_plan.seen = NOW - 190_000;
assert!(
holding_by(Some(slow_plan), &slow, NOW).is_empty(),
"190 s is over"
);
let mut this_second = held(HOUR);
this_second.seen = NOW + 400;
assert!(
said(Some(this_second), NOW),
"a plan in the second of the look is not from after it"
);
let mut next_second = held(HOUR);
next_second.seen = NOW + 1_000;
assert!(!said(Some(next_second), NOW), "one in the next is");
assert!(
!said(Some(held(HOUR)), NOW - 24 * HOUR),
"a look from before the reading"
);
}
#[test]
fn a_host_held_insight_without_names_says_how_much_the_host_holds_and_no_more() {
let mut hold = held(HOUR);
hold.memory.holders.clear();
let found = holding(Some(hold), NOW);
assert_eq!(
found[0].sentence,
"2 jobs wait for the host's memory, the longest for 1h 00m (halyard/api: CI / test); programs outside CI \
hold 38G. Closing programs frees only part of that."
);
}