use serde::{Deserialize, Serialize};
use crate::api::AdmissionPolicy;
use crate::config::ByteSize;
mod shown;
pub use shown::{GRANULE, Need, ProfilePlan, SAMPLE_INTERVAL, need, sampled_note};
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct SizingQuery {
pub since: Option<u64>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Sizing {
pub since: u64,
pub until: u64,
pub margin: u32,
pub sample_interval: u64,
pub min_jobs: u32,
pub memory_max: Option<u64>,
pub admission: AdmissionPolicy,
pub profiles: Vec<ProfileSizing>,
pub auto: bool,
pub changes: Vec<SizingChange>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub auto_plan: Option<AutoPlan>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct AutoPlan {
pub every: u64,
pub interval: u64,
pub raise_interval: u64,
pub max_step: u32,
pub next: Vec<SizingChange>,
pub held: Vec<SizingHold>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub blocked: Option<String>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub refused: Vec<SizingRefused>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub refused_slots: Vec<SizingRefused>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub waited_before_change: Vec<String>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct SizingRefused {
pub profile: String,
pub reason: String,
}
impl SizingRefused {
pub fn new(profile: &str, reason: String) -> SizingRefused {
SizingRefused {
profile: profile.to_owned(),
reason,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct SizingHold {
pub profile: String,
pub changed: u64,
pub until: u64,
pub lower_until: u64,
}
impl SizingHold {
pub fn lowered_after(&self) -> u64 {
self.lower_until.max(self.until)
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct SizingChange {
pub at: u64,
pub profile: String,
pub from: Resize,
pub to: Resize,
pub reason: String,
pub uncertain: bool,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ProfileSizing {
pub name: String,
pub memory: u64,
pub cpus: u32,
pub slots: u32,
pub peaks: Peaks,
pub wait: Spread,
#[serde(default)]
pub slot_wait: Spread,
pub starts: Starts,
pub busy: u32,
pub wants_slots: bool,
pub advice: Advice,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub revert: Option<SizingRevert>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub hold: Option<MemoryHold>,
pub jobs: Vec<JobSizing>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub struct MemoryHold {
pub need: u64,
pub bound: u64,
pub by: MemoryBound,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case", tag = "kind")]
pub enum MemoryBound {
Ceiling {
ceiling: u64,
mirror: u64,
},
MaxMemory,
MinMemory,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct SizingRevert {
pub from: u64,
pub to: u64,
pub job: String,
pub cause: RevertCause,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case", tag = "kind")]
pub enum RevertCause {
Met,
Failed {
runs: u32,
},
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct Starts {
pub warm: u32,
pub warm_p50: u64,
pub cold: u32,
pub cold_p50: u64,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct JobSizing {
pub repository: String,
pub workflow: Option<String>,
pub name: String,
pub peaks: Peaks,
pub wait: Spread,
pub advice: Advice,
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct Peaks {
pub jobs: u64,
pub at_cap: u64,
#[serde(default, skip_serializing_if = "is_zero")]
pub on_cap: u64,
pub memory: Spread,
pub cpu: Spread,
#[serde(default)]
pub cpu_runs: u64,
}
#[expect(
clippy::trivially_copy_pass_by_ref,
reason = "serde's skip_serializing_if passes a reference"
)]
fn is_zero(count: &u64) -> bool {
*count == 0
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct Spread {
pub p50: u64,
pub p95: u64,
pub p99: u64,
pub max: u64,
}
pub const SLOT_DISK: u64 = 2 << 30;
pub const PAGE_CACHE_ROOM: u64 = 1 << 30;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case", tag = "kind")]
pub enum Advice {
TooFew {
jobs: u64,
},
Keep {
memory: u64,
},
Lower {
memory: u64,
slots: u32,
},
Raise {
memory: u64,
slots: u32,
},
SliceTooSmall {
memory: u64,
},
}
impl Advice {
pub fn change(&self) -> Option<Resize> {
match *self {
Advice::Lower { memory, slots } | Advice::Raise { memory, slots } => Some(Resize {
memory,
slots,
cpus: None,
}),
Advice::TooFew { .. } | Advice::Keep { .. } | Advice::SliceTooSmall { .. } => None,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub struct Resize {
pub memory: u64,
pub slots: u32,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cpus: Option<u32>,
}
impl AutoPlan {
pub fn rules(&self) -> String {
if self.raise_interval == 0 || self.raise_interval >= self.interval {
return format!(
"looks every {} and changes a profile at most once every {}, moving its memory by at most {}% at a \
time",
period(self.every),
period(self.interval),
self.max_step
);
}
format!(
"looks every {}, raises a profile at most once every {} and lowers it at most once every {}, moving its \
memory and CPUs by at most {}% at a time",
period(self.every),
period(self.raise_interval),
period(self.interval),
self.max_step
)
}
}
pub fn period(seconds: u64) -> String {
let plural = |count: u64, unit: &str| {
if count == 1 {
format!("{count} {unit}")
} else {
format!("{count} {unit}s")
}
};
match seconds {
0 => "0 s".to_owned(),
_ if seconds.is_multiple_of(3600) => plural(seconds / 3600, "hour"),
_ if seconds.is_multiple_of(60) => plural(seconds / 60, "minute"),
_ => format!("{seconds} s"),
}
}
impl Resize {
pub fn line_from(&self, from: Option<&Resize>) -> String {
let memory = match from {
Some(from) if from.memory != self.memory => format!(
"memory {} → {}",
ByteSize(from.memory),
ByteSize(self.memory)
),
_ => format!("memory {}", ByteSize(self.memory)),
};
let slots = match from {
Some(from) if from.slots != self.slots => {
format!("slots {} → {}", from.slots, self.slots)
}
_ => format!("slots {}", self.slots),
};
let cpus = match (from.and_then(|from| from.cpus), self.cpus) {
(Some(from), Some(to)) if from != to => format!(", cpus {from} → {to}"),
_ => String::new(),
};
format!("{memory}, {slots}{cpus}")
}
}
impl SizingChange {
pub fn line(&self) -> String {
let line = self.to.line_from(Some(&self.from));
if self.uncertain {
format!("{line} (may not have been written)")
} else {
line
}
}
}
impl Sizing {
pub fn advice_line(&self, profile: &ProfileSizing) -> String {
let line = self.memory_line(profile);
if self.admission == AdmissionPolicy::Measured
&& profile.wants_slots
&& !matches!(profile.advice, Advice::TooFew { .. })
{
let plan = self.auto_plan.as_ref();
let refused = plan.is_some_and(|plan| {
plan.refused_slots
.iter()
.any(|refused| refused.profile == profile.name)
});
let before_change =
plan.is_some_and(|plan| plan.waited_before_change.contains(&profile.name));
let then = if refused {
"auto sizing cannot add one now"
} else if before_change {
"auto sizing changed it since, and adds more only once jobs queued after that change wait for a slot \
too"
} else if self.auto {
&format!(
"auto sizing adds as many as waited at once, up to max_slots and while each slot keeps {} of \
Docker's disk",
ByteSize(SLOT_DISK)
)
} else {
"measured admission reserves predicted peaks, not caps, so more slots start them sooner: raise slots \
by hand"
};
format!(
"{line}; jobs waited {}s for a slot (p95) while memory and CPU had room: {then}",
profile.slot_wait.p95
)
} else {
line
}
}
pub(crate) fn memory_line(&self, profile: &ProfileSizing) -> String {
let size = |bytes: u64| ByteSize(bytes).to_string();
let peaks = &profile.peaks;
let need = self.need_reason(peaks);
let at_cap = format!(
"{} of {} jobs ran out of memory at their cap",
peaks.at_cap, peaks.jobs
);
match profile.advice {
Advice::TooFew { jobs } => format!(
"{jobs} jobs, too few to advise on (it takes {}); largest peak {} of a {} cap",
self.min_jobs,
size(peaks.memory.max),
size(profile.memory)
),
Advice::Keep { .. } if peaks.on_cap > 0 => format!(
"keep {}: {} of its jobs met their cap or were killed for memory, and fewer than 3 runs since ran \
clear of it",
size(profile.memory),
peaks.on_cap
),
Advice::Keep { memory } if memory > profile.memory => format!(
"keep {}: its recent runs need {}, but none is known to have run out of memory at {} since it was set",
size(profile.memory),
size(memory),
size(profile.memory)
),
Advice::Keep { memory } => match profile.hold {
Some(hold) => format!(
"keep {}, {}: {need} is {}",
size(profile.memory),
hold.bound_line(),
size(memory)
),
None => format!("keep {}: {need} is {}", size(profile.memory), size(memory)),
},
Advice::Lower { memory, slots } => {
let share = profile.memory * u64::from(profile.slots);
let need = match profile.hold {
Some(hold) => format!("{}: {need} is {}", hold.bound_line(), size(hold.need)),
None => need,
};
if self.admission == AdmissionPolicy::Measured {
return format!(
"lower memory to {} from {}, {need}",
size(memory),
size(profile.memory)
);
}
let then = if slots > profile.slots {
format!("{slots} slots fit in the same {}", size(share))
} else {
format!(
"frees {} of the budget",
size(share - memory * u64::from(slots))
)
};
format!(
"lower memory to {} from {}, {need}; {then}",
size(memory),
size(profile.memory)
)
}
Advice::Raise { memory, .. } if let Some(revert) = &profile.revert => {
let held = match profile.hold {
Some(hold) if memory < revert.to => format!(", {}", hold.bound_line()),
_ => String::new(),
};
format!(
"raise memory back to {}{held}: {}",
size(memory),
revert.why()
)
}
Advice::Raise { memory, slots } => {
let why = if peaks.at_cap > 0 {
at_cap
} else {
format!("{need} is above the {} cap", size(profile.memory))
};
let slots = if slots < profile.slots {
format!("; {slots} of its {} slots fit the slice", profile.slots)
} else {
String::new()
};
match profile.hold {
Some(hold) => format!(
"raise memory to {}, {}, short of the {} its jobs need: {why}{slots}",
size(memory),
hold.bound_line(),
size(hold.need)
),
None => format!("raise memory to {}: {why}{slots}", size(memory)),
}
}
Advice::SliceTooSmall { memory } => too_small_line(profile, memory, need),
}
}
}
fn too_small_line(profile: &ProfileSizing, memory: u64, need: String) -> String {
let why = profile.revert.as_ref().map_or(need, SizingRevert::why);
if let Some(hold) = profile.held() {
let then = match hold.by {
MemoryBound::Ceiling { .. } => {
"raise [host] memory_ceiling if the host can grow further"
}
MemoryBound::MaxMemory | MemoryBound::MinMemory => "raise max_memory to give them more",
};
return format!(
"keep {}, {}: its jobs need {} ({why}); {then}",
ByteSize(profile.memory),
hold.bound_line(),
ByteSize(memory)
);
}
let then = profile
.hold
.map(|hold| {
format!(
"; {} then holds it at {}",
hold.by.name(),
ByteSize(hold.bound)
)
})
.unwrap_or_default();
format!(
"needs {} ({why}), more than the slice holds for one slot: raise the slice's MemoryMax first{then}",
ByteSize(memory)
)
}
impl MemoryHold {
pub fn bound_line(&self) -> String {
match self.by {
MemoryBound::Ceiling { ceiling, mirror: 0 } => format!(
"the most the host's {} memory_ceiling leaves one runner",
ByteSize(ceiling)
),
MemoryBound::Ceiling { ceiling, mirror } => format!(
"the most the host's {} memory_ceiling leaves one runner beside the mirror's {}",
ByteSize(ceiling),
ByteSize(mirror)
),
MemoryBound::MaxMemory => "its max_memory".to_owned(),
MemoryBound::MinMemory => "its min_memory".to_owned(),
}
}
}
impl MemoryBound {
pub fn name(self) -> &'static str {
match self {
MemoryBound::Ceiling { .. } => "[host] memory_ceiling",
MemoryBound::MaxMemory => "max_memory",
MemoryBound::MinMemory => "min_memory",
}
}
}
impl SizingRevert {
fn why(&self) -> String {
let how = match self.cause {
RevertCause::Met => format!("met the {} cap", ByteSize(self.from)),
RevertCause::Failed { runs } => {
format!("failed {runs} times in a row at {}", ByteSize(self.from))
}
};
format!(
"{} {how} since auto sizing lowered it from {}",
self.job,
ByteSize(self.to)
)
}
}