use crate::admission::Key;
use crate::admission::recent::Recent;
use crate::api::{self, Advice, Peaks, Spread};
use crate::config::{Admission, ByteSize, Config};
use crate::observe::metrics::SAMPLE_INTERVAL;
pub(crate) use crate::api::GRANULE;
const AT_CAP: u64 = 95;
const WORTH_LOWERING: u64 = 10;
const WAITED: u64 = 60;
const BUSY: u32 = 50;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Ran {
pub repository: String,
pub workflow: Option<String>,
pub name: String,
pub profile: String,
pub memory_cap: Option<u64>,
pub cpu_cap: Option<u64>,
pub memory_peak: u64,
pub cpu_peak: u64,
pub cpu_average: Option<u64>,
pub queued_at: u64,
pub started_at: Option<u64>,
pub runner_started_at: u64,
pub finished_at: Option<u64>,
pub oom_killed: bool,
pub hit_cap: Option<bool>,
pub own_cap: Option<bool>,
pub measure: Measure,
pub succeeded: bool,
pub failed: bool,
pub run_id: u64,
pub branch: Option<String>,
pub reserved: Option<u64>,
pub host_waited: u64,
pub unseen: u64,
pub cpu_periods: Option<u64>,
pub cpu_throttled: Option<u64>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Wait {
pub queued_at: u64,
pub started_at: u64,
pub unseen: u64,
}
impl Wait {
pub fn counted(self, kept_since: u64) -> u64 {
self.started_at
.saturating_sub(self.queued_at.max(kept_since))
.saturating_sub(self.unseen)
}
}
pub const HELD: u64 = 15;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Throttling {
pub started_at: Option<u64>,
pub finished_at: Option<u64>,
pub periods: Option<u64>,
pub throttled: Option<u64>,
}
impl Throttling {
pub fn took(self) -> Option<u64> {
Some(self.finished_at?.saturating_sub(self.started_at?))
}
pub fn share(self) -> Option<u64> {
let (periods, throttled) = self.periods.zip(self.throttled)?;
let run = self.took().map_or(0, |took| took / 100).max(periods);
Some(throttled.saturating_mul(1000) / run.max(1))
}
pub fn held(self) -> Option<bool> {
let share = self.share()?;
Some(self.throttled.is_some_and(|throttled| throttled > 0) && share >= HELD * 10)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Measure {
WithActiveFile,
WithoutPageCache,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Capped {
Met,
Assumed,
Killed,
Clear,
}
impl Capped {
pub fn met(self) -> bool {
matches!(self, Capped::Met | Capped::Assumed)
}
pub fn holds(self) -> bool {
self != Capped::Clear
}
}
impl Ran {
pub fn wait(&self) -> Option<Wait> {
Some(Wait {
queued_at: self.queued_at,
started_at: self.started_at?,
unseen: self.unseen,
})
}
pub fn throttling(&self) -> Throttling {
Throttling {
started_at: self.started_at,
finished_at: self.finished_at,
periods: self.cpu_periods,
throttled: self.cpu_throttled,
}
}
pub fn capped(&self, cap: u64) -> Capped {
match self.own_cap {
Some(true) => Capped::Met,
Some(false)
if self.oom_killed
&& self.hit_cap == Some(false)
&& self.memory_peak.saturating_mul(100) >= cap.saturating_mul(AT_CAP) =>
{
Capped::Met
}
Some(false) if self.oom_killed && self.hit_cap != Some(true) => Capped::Killed,
None if self.oom_killed
|| self.hit_cap.unwrap_or_else(|| {
self.memory_peak.saturating_mul(100) >= cap.saturating_mul(AT_CAP)
}) =>
{
Capped::Assumed
}
Some(false) | None => Capped::Clear,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Held {
pub profile: String,
pub started_at: u64,
pub removed_at: Option<u64>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CpusSet {
pub profile: String,
pub cpus: u32,
pub at: u64,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct MemorySet {
pub profile: String,
pub memory: u64,
pub at: u64,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct History {
pub jobs: Vec<Ran>,
pub cancelled: Vec<Ran>,
pub runners: Vec<Held>,
pub changes: Vec<api::SizingChange>,
pub memory_set: Vec<MemorySet>,
pub kept_since: u64,
}
#[derive(Debug, Clone, Copy)]
pub struct Window {
pub since: u64,
pub now: u64,
pub sample_interval: u64,
}
impl Window {
pub fn new(since: Option<u64>, now: u64) -> Window {
Window {
since: since
.unwrap_or_else(|| now.saturating_sub(30 * 24 * 3600))
.min(now)
.saturating_mul(1000),
now: now.saturating_mul(1000),
sample_interval: SAMPLE_INTERVAL.as_secs(),
}
}
}
pub fn advise(
config: &Config,
memory_max: Option<u64>,
history: &History,
window: Window,
) -> api::Sizing {
let since = first_runner(history, window.since);
let caps = config.memory_caps().map_or(u64::MAX, |caps| caps.0);
let mirror = config.mirror_memory();
let rules = &config.sizing;
let recent = memory::windows(config, history, window.now);
let ceiling = Bound::ceiling(config);
let profiles = config
.profiles
.iter()
.map(|(name, profile)| {
let jobs: Vec<&Ran> = history
.jobs
.iter()
.filter(|job| job.profile == *name && job.queued_at >= window.since)
.collect();
let share = profile.memory.0.saturating_mul(u64::from(profile.slots));
let busy = busy(history, name, profile.slots, since, window.now);
let wait = waits(&jobs, history.kept_since);
let starts = starts(&jobs, history.kept_since);
let cap = profile.memory.0;
let took = memory::took(&history.memory_set, name, cap, window.now);
let sized = Sized {
cap,
cpus: u64::from(profile.cpus) * 1000,
slots: profile.slots,
share,
admission: config.host.admission,
room: memory_max.map(|max| match config.host.admission {
Admission::Caps => max.saturating_sub(caps.saturating_sub(share)),
Admission::Measured => max.saturating_sub(mirror),
}),
wants_slots: wait.p95 >= WAITED || busy >= BUSY,
min_jobs: rules.min_jobs,
upper: upper(
ceiling,
Bound::of(profile.max_memory, api::MemoryBound::MaxMemory),
),
lower: Bound::of(profile.min_memory, api::MemoryBound::MinMemory),
max_slots: profile.max_slots.unwrap_or(u32::MAX),
};
let raises = Raises {
history,
recent: &recent,
profile: name,
cap,
took,
margin: rules.margin,
};
let memory = job_memory(&raises, &jobs, &sized, window.now, busy);
let peaks = memory::peaks(
recent.jobs_of(name).flat_map(|(_, runs)| runs.window()),
memory.on_cap,
sized.cpus,
);
let current = api::Resize {
memory: cap,
slots: profile.slots,
cpus: None,
};
let revert = memory::revert(config, history, name, current);
let advised = if let Some((_, to)) = &revert {
sized.reverted(*to)
} else {
let needs = Needs {
raise: raises.need(None),
floor: memory.floor.max(memory::revert_floor(history, name)),
};
sized.advice(&peaks, needs, sized.wants_slots)
};
api::ProfileSizing {
name: name.clone(),
memory: cap,
cpus: profile.cpus,
slots: profile.slots,
advice: advised.advice,
revert: revert.map(|(revert, _)| revert),
hold: advised.hold,
peaks,
wait,
starts,
busy,
wants_slots: sized.wants_slots,
jobs: memory.jobs,
}
})
.collect();
api::Sizing {
since: since / 1000,
until: window.now / 1000,
margin: rules.margin,
sample_interval: window.sample_interval,
min_jobs: rules.min_jobs,
memory_max,
admission: config.host.admission.policy(),
profiles,
auto: config.sizing.auto,
changes: Vec::new(),
auto_plan: None,
}
}
fn upper(ceiling: Option<Bound>, max_memory: Option<Bound>) -> Option<Bound> {
match (ceiling, max_memory) {
(Some(ceiling), Some(max)) if max.memory < ceiling.memory => Some(max),
(ceiling, max) => ceiling.or(max),
}
}
struct Raises<'history> {
history: &'history History,
recent: &'history Recent,
profile: &'history str,
cap: u64,
took: u64,
margin: u32,
}
impl Raises<'_> {
fn need(&self, key: Option<&Key>) -> u64 {
memory::raise_need(
self.history,
self.recent,
self.profile,
self.cap,
self.took,
self.margin,
key,
)
}
}
struct JobMemory {
jobs: Vec<api::JobSizing>,
on_cap: u64,
floor: u64,
}
fn job_memory(raises: &Raises<'_>, jobs: &[&Ran], sized: &Sized, now: u64, busy: u32) -> JobMemory {
let mut on_cap = 0;
let mut floor = 0;
let sizings = raises
.recent
.jobs_of(raises.profile)
.filter(|(_, runs)| runs.runs() > 0)
.map(|(key, runs)| {
let held = memory::on_cap(runs, now);
on_cap += u64::from(held);
let peaks = memory::peaks(runs.window(), u64::from(held), sized.cpus);
let needs = Needs {
raise: raises.need(Some(key)),
floor: memory::floor(runs, raises.margin),
};
floor = floor.max(needs.floor);
let runs_of: Vec<&Ran> = jobs
.iter()
.copied()
.filter(|job| Key::of(job) == *key)
.collect();
let wait = waits(&runs_of, raises.history.kept_since);
let unbounded = Sized {
upper: None,
lower: None,
..*sized
};
api::JobSizing {
repository: key.repository.clone(),
workflow: key.workflow.clone(),
name: key.name.clone(),
advice: unbounded
.advice(&peaks, needs, wait.p95 >= WAITED || busy >= BUSY)
.advice,
peaks,
wait,
}
})
.collect();
JobMemory {
jobs: sizings,
on_cap,
floor,
}
}
fn first_runner(history: &History, start: u64) -> u64 {
history
.runners
.iter()
.map(|runner| runner.started_at)
.min()
.map_or(start, |first| first.max(start))
}
fn busy(history: &History, profile: &str, slots: u32, since: u64, now: u64) -> u32 {
let span = now.saturating_sub(since).max(1);
let held: u64 = history
.runners
.iter()
.filter(|runner| runner.profile == profile)
.map(|runner| {
let end = runner.removed_at.unwrap_or(now).min(now);
end.saturating_sub(runner.started_at.max(since))
})
.sum();
u32::try_from(
(held.saturating_mul(100) / span.saturating_mul(u64::from(slots).max(1))).min(100),
)
.unwrap_or(100)
}
#[derive(Clone, Copy)]
struct Sized {
cap: u64,
cpus: u64,
slots: u32,
share: u64,
admission: Admission,
room: Option<u64>,
wants_slots: bool,
min_jobs: u32,
upper: Option<Bound>,
lower: Option<Bound>,
max_slots: u32,
}
#[derive(Clone, Copy)]
struct Bound {
memory: u64,
by: api::MemoryBound,
}
impl Bound {
fn ceiling(config: &Config) -> Option<Bound> {
let most = config.runner_ceiling()?;
Some(Bound {
memory: most / GRANULE * GRANULE,
by: api::MemoryBound::Ceiling {
ceiling: config.host.memory_ceiling?.0,
mirror: config.mirror_memory(),
},
})
}
fn of(size: Option<ByteSize>, by: api::MemoryBound) -> Option<Bound> {
size.map(|size| Bound { memory: size.0, by })
}
}
struct Advised {
advice: Advice,
hold: Option<api::MemoryHold>,
}
impl From<Advice> for Advised {
fn from(advice: Advice) -> Advised {
Advised { advice, hold: None }
}
}
#[derive(Debug, Clone, Copy)]
struct Needs {
raise: u64,
floor: u64,
}
impl Sized {
fn advice(&self, peaks: &Peaks, needs: Needs, wants_slots: bool) -> Advised {
if peaks.jobs < u64::from(self.min_jobs) {
return Advice::TooFew { jobs: peaks.jobs }.into();
}
if needs.raise > self.cap {
return self.raise(needs.raise, self.slots);
}
let need = needs.floor.max(GRANULE);
if peaks.on_cap == 0
&& need <= self.cap
&& self.cap - need >= self.cap * WORTH_LOWERING / 100
{
let hold = self
.lower
.filter(|min| min.memory > need)
.map(|min| api::MemoryHold {
need,
bound: min.memory,
by: min.by,
});
let memory = hold.map_or(need, |hold| hold.bound);
if memory >= self.cap {
return Advised {
advice: Advice::Keep { memory: need },
hold,
};
}
let fit = |room: u64| u32::try_from(room / memory).unwrap_or(u32::MAX);
let advice = Advice::Lower {
memory,
slots: if wants_slots && self.admission == Admission::Caps {
fit(self.share).min(self.max_slots).max(self.slots)
} else {
self.slots
},
};
return Advised { advice, hold };
}
Advice::Keep { memory: need }.into()
}
fn reverted(&self, to: api::Resize) -> Advised {
self.raise(to.memory, to.slots.min(self.max_slots))
}
fn raise(&self, need: u64, slots: u32) -> Advised {
let hold = self
.upper
.filter(|most| need > most.memory)
.map(|most| api::MemoryHold {
need,
bound: most.memory,
by: most.by,
});
let memory = hold.map_or(need, |hold| hold.bound);
let too_small = Advised {
advice: Advice::SliceTooSmall { memory: need },
hold,
};
if memory <= self.cap {
return too_small;
}
let fit = |room: u64| u32::try_from(room / memory).unwrap_or(u32::MAX);
let slots = match (self.admission, self.room) {
(Admission::Caps, Some(room)) => fit(room).min(slots),
(Admission::Measured, Some(room)) if room < memory => 0,
(_, _) => slots,
};
if slots == 0 {
return too_small;
}
Advised {
advice: Advice::Raise { memory, slots },
hold,
}
}
}
fn waits(jobs: &[&Ran], kept_since: u64) -> Spread {
spread(
jobs.iter()
.filter_map(|job| {
job.wait()
.map(|wait| wait.counted(kept_since).saturating_sub(job.host_waited) / 1000)
})
.collect(),
)
}
fn starts(jobs: &[&Ran], kept_since: u64) -> api::Starts {
let (warm, cold): (Vec<&Ran>, Vec<&Ran>) = jobs
.iter()
.filter(|job| job.started_at.is_some())
.partition(|job| job.runner_started_at < job.queued_at);
let count = |jobs: &[&Ran]| u32::try_from(jobs.len()).unwrap_or(u32::MAX);
api::Starts {
warm: count(&warm),
warm_p50: waits(&warm, kept_since).p50,
cold: count(&cold),
cold_p50: waits(&cold, kept_since).p50,
}
}
pub(crate) fn spread(mut values: Vec<u64>) -> Spread {
values.sort_unstable();
let rank = |percent: usize| {
let index = (values.len() * percent).div_ceil(100).saturating_sub(1);
values.get(index).copied().unwrap_or(0)
};
Spread {
p50: rank(50),
p95: rank(95),
p99: rank(99),
max: values.last().copied().unwrap_or(0),
}
}
pub mod auto;
pub mod memory;
#[cfg(test)]
mod tests;