use super::*;
use crate::admission::pooled::{self, MOST_CREDIT};
use crate::admission::trace::{Outside, Point, Reading, RunId, RunTrace, STEP, Trace};
const SLICE: u64 = 8 * G + G / 2;
const USED: u64 = G / 2;
fn flat(memory: u64, minutes: u64) -> Vec<Point> {
(0..minutes * MINUTE / STEP)
.map(|step| Point {
at: step * STEP,
memory,
})
.collect()
}
fn check(queued: u64, minutes: u64) -> Ran {
run("halyard/ledger", "check", "jvm", queued, minutes, 3 * G)
}
fn site(queued: u64, minutes: u64) -> Ran {
run("halyard/storefront", "site", "node", queued, minutes, 3 * G)
}
fn page(queued: u64, minutes: u64) -> Ran {
run("halyard/storefront", "page", "node", queued, minutes, 2 * G)
}
fn known() -> History {
let before = NOW - 900 * MINUTE;
let jobs = (0..10)
.flat_map(|i| {
let at = before + i * 40 * MINUTE;
[check(at, 30), site(at, 30), page(at, 5)]
})
.collect();
History {
jobs,
..History::default()
}
}
fn jobs(page_at: u64) -> Vec<Ran> {
let mut jobs = vec![
in_run(check(NOW, 30), 1, 1),
in_run(site(NOW, 30), 2, 1),
in_run(page(page_at, 5), 3, 1),
];
jobs.sort_by_key(|run| run.queued_at);
jobs
}
fn trace_of(jobs: &[Ran]) -> Trace {
let runs = jobs
.iter()
.map(|run| {
let trace = RunTrace {
samples: flat(USED, 40),
named_after: 0,
};
(RunId::of(run), trace)
})
.collect();
Trace {
runs,
outside: vec![
Outside {
at: NOW - MINUTE,
memory: 8 * G,
},
Outside {
at: NOW + 60 * MINUTE,
memory: 8 * G,
},
],
total: 64 * G,
overhead: 0,
guard_overhead: 0,
host_overhead: 0,
}
}
fn replay(jobs: &[Ran], trace: &Trace, most_credit: u64) -> Replay {
let known = known();
simulate(
&config(),
Some(SLICE),
None,
jobs,
Policy::Measured,
&BTreeMap::new(),
Replaying {
barriers: Barriers::On,
arrivals: Arrivals::AsQueued,
known: Some(&known),
trace: Some(trace),
most_credit,
host_hold: u64::MAX,
credit_room: CreditRoom::Counted,
band: Band::NoWaiterFits,
},
)
}
fn page_waited(replay: &Replay) -> u64 {
let pool = pool("halyard/storefront", "node");
*replay.pools[&pool].waits.last().unwrap()
}
#[test]
fn a_pooled_replay_starts_what_only_the_pooled_count_fits() {
let jobs = jobs(NOW + 5 * MINUTE);
let trace = trace_of(&jobs);
let plain = replay(&jobs, &trace, 0);
assert_eq!(page_waited(&plain), 25 * 60, "until one of the two ends");
assert_eq!(plain.hazard.most_credit, 0);
let pooled = replay(&jobs, &trace, MOST_CREDIT);
assert!(page_waited(&pooled) <= 15, "at the next plan");
let (check, site, page) = (Key::of(&jobs[0]), Key::of(&jobs[1]), Key::of(&jobs[2]));
let runner = |key, takes| pooled::Runner {
key: Some(key),
takes,
used: Some(USED),
ran: Some(pooled::POOL_AFTER),
basis: Some(Basis::Job {
runs: 10,
peak: JobPeak::P95,
}),
};
let direct = pooled::count(&pooled::Snapshot {
runners: &[
runner(&check, 3584 * M),
runner(&site, 3584 * M),
runner(&page, 2560 * M),
],
committed: 0,
slice: Some(pooled::Slice {
memory: 3 * USED,
memory_max: SLICE,
}),
host: Some(pooled::Host {
available: 50 * G,
keeps: 20 * G,
stalled: false,
}),
most: MOST_CREDIT,
});
assert_eq!(
pooled.hazard.most_credit, direct.credit,
"the replay's count is the daemon's"
);
}
#[test]
fn a_replayed_run_counts_in_full_for_its_first_minute() {
let jobs = jobs(NOW + 20_000);
let trace = trace_of(&jobs);
let waited = page_waited(&replay(&jobs, &trace, MOST_CREDIT));
assert!(
(40..=60).contains(&waited),
"from 20 s to a plan past the minute, waited {waited}"
);
}
#[test]
fn a_replayed_run_is_not_pooled_before_the_daemon_could_name_its_job() {
let jobs = jobs(NOW + 5 * MINUTE);
let mut trace = trace_of(&jobs);
let check = RunId::of(&jobs[0]);
trace.runs.get_mut(&check).unwrap().named_after = 10 * MINUTE;
let waited = page_waited(&replay(&jobs, &trace, MOST_CREDIT));
assert!(
(300..=320).contains(&waited),
"until check is known at minute 10, waited {waited}"
);
}
#[test]
fn a_sampler_gap_takes_the_credit_back_and_the_replay_says_so() {
let jobs = jobs(NOW + 5 * MINUTE);
let mut trace = trace_of(&jobs);
let check = RunId::of(&jobs[0]);
let run = trace.runs.get_mut(&check).unwrap();
run.samples
.retain(|point| !(8 * MINUTE..9 * MINUTE).contains(&point.at));
let hazard = replay(&jobs, &trace, MOST_CREDIT).hazard;
assert!(hazard.credit_falls >= 1, "the gap took it back: {hazard:?}");
}
#[test]
fn a_credit_the_host_takes_back_leaves_the_count_past_the_budget_and_the_replay_says_so() {
let jobs = jobs(NOW + 5 * MINUTE);
let mut trace = trace_of(&jobs);
trace.outside.insert(
1,
Outside {
at: NOW + 8 * MINUTE,
memory: 43 * G,
},
);
let hazard = replay(&jobs, &trace, MOST_CREDIT).hazard;
assert_eq!(hazard.overdrawn, 1, "{hazard:?}");
}
#[test]
fn a_host_short_of_memory_keeps_a_replay_from_pooling() {
let jobs = jobs(NOW + 5 * MINUTE);
let mut trace = trace_of(&jobs);
for reading in &mut trace.outside {
reading.memory = 43 * G;
}
let replay = replay(&jobs, &trace, MOST_CREDIT);
assert!(replay.hazard.most_credit < G, "{:?}", replay.hazard);
assert_eq!(page_waited(&replay), 25 * 60);
}
#[test]
fn a_replay_finds_the_slice_past_its_lines_and_how_long() {
let jobs = jobs(NOW + 40 * MINUTE);
let mut trace = trace_of(&jobs);
let check = RunId::of(&jobs[0]);
let run = trace.runs.get_mut(&check).unwrap();
for point in &mut run.samples {
if (10 * MINUTE..11 * MINUTE).contains(&point.at) {
point.memory = 9 * G;
}
}
let hazard = replay(&jobs, &trace, 0).hazard;
assert_eq!(hazard.over_mark.count, 1, "{hazard:?}");
assert_eq!(hazard.over_max.count, 1, "{hazard:?}");
assert_eq!(hazard.over_max.time, MINUTE);
assert_eq!(hazard.over_max.worst, 9 * G + USED - SLICE);
assert_eq!(hazard.slice_peak, 9 * G + USED);
}
#[test]
fn a_cold_window_gives_every_run_its_jobs_heaviest_samples() {
let first = in_run(check(NOW, 30), 1, 1);
let second = in_run(check(NOW + MINUTE, 30), 2, 1);
let other = in_run(site(NOW, 30), 3, 1);
let jobs = [first, second, other];
let mut trace = trace_of(&jobs);
trace.runs.get_mut(&RunId::of(&jobs[1])).unwrap().samples = flat(4 * G, 1);
let cold = trace.cold(&jobs);
let peak = |at: usize| cold.runs[&RunId::of(&jobs[at])].at(0);
assert_eq!(peak(0), Reading::Used(4 * G), "the job's heaviest run");
assert_eq!(peak(1), Reading::Used(4 * G));
assert_eq!(peak(2), Reading::Used(USED), "another job keeps its own");
}
#[test]
fn a_cold_run_shorter_than_the_heaviest_reaches_its_peak_in_the_time_it_has() {
let long = in_run(check(NOW, 30), 1, 1);
let short = in_run(check(NOW + MINUTE, 5), 2, 1);
let jobs = [long, short];
let mut trace = trace_of(&jobs);
let heavy = &mut trace.runs.get_mut(&RunId::of(&jobs[0])).unwrap().samples;
heavy
.iter_mut()
.find(|point| point.at == 20 * MINUTE)
.unwrap()
.memory = 4 * G;
let cold = trace.cold(&jobs);
let lasted = jobs[1].finished_at.unwrap() - jobs[1].started_at.unwrap();
let run = &cold.runs[&RunId::of(&jobs[1])];
let at = run.samples.iter().find(|point| point.memory == 4 * G);
assert!(
at.is_some_and(|point| point.at < lasted),
"the peak falls inside its {lasted} ms: {at:?}"
);
assert_eq!(run.at(0), Reading::Used(USED), "it reads from its start");
}
#[test]
fn a_shifted_host_keeps_every_reading_and_comes_round_the_window() {
let trace = Trace {
outside: [(0, 1), (10, 2), (20, 3), (30, 4)]
.map(|(at, memory)| Outside { at, memory })
.to_vec(),
..Trace::default()
};
let shifted = trace.shifted(10);
assert_eq!(shifted.outside_at(0), Some(4), "what ended the window");
assert_eq!(shifted.outside_at(10), Some(1));
assert_eq!(shifted.outside_at(30), Some(3));
assert_eq!(shifted.outside.len(), 4);
}
#[test]
fn a_slice_that_reads_low_at_a_decision_goes_past_the_mark_it_did_not_see() {
let jobs = jobs(NOW + 40 * MINUTE);
let mut trace = trace_of(&jobs);
trace.overhead = 8 * G;
trace.guard_overhead = 0;
let replay = replay(&jobs, &trace, 0);
assert!(replay.hazard.over_mark.count > 0, "{:?}", replay.hazard);
assert_eq!(replay.held_back, 0, "the guards saw room");
}
#[test]
fn a_run_between_samples_reads_the_last_and_a_gap_reads_nothing() {
let run = RunTrace {
samples: vec![
Point {
at: 1000,
memory: G,
},
Point {
at: 3000,
memory: 2 * G,
},
Point {
at: 30_000,
memory: 3 * G,
},
],
named_after: 0,
};
assert_eq!(run.at(0), Reading::NotYet);
assert_eq!(run.at(2000), Reading::Used(G));
assert_eq!(run.at(9000), Reading::Used(2 * G), "at the edge of fresh");
assert_eq!(run.at(9001), Reading::Stale);
assert_eq!(run.at(90_000), Reading::Used(3 * G), "its end is not a gap");
}