runnerd_core/api/sizing.rs
1//! Right-sizing as clients read it: per profile and per job (repository, workflow and job name), the spread of the
2//! job history's sampled memory and CPU peaks, how long jobs waited for a slot, how busy the slots were, and what
3//! to set each profile's `memory` and `slots` to. Plain data; the daemon computes it from the history. Sizes are
4//! bytes, CPU millicores, times seconds since the Unix epoch.
5
6use serde::{Deserialize, Serialize};
7
8use crate::api::AdmissionPolicy;
9use crate::config::ByteSize;
10
11/// `GET /v1/sizing` query parameters.
12#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
13pub struct SizingQuery {
14 /// Only jobs queued at or after this time; default 30 days ago.
15 pub since: Option<u64>,
16}
17
18/// `GET /v1/sizing`: what the job history says each configured profile needs.
19#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
20pub struct Sizing {
21 /// The start of the history read: `since`, or the first runner recorded after it.
22 pub since: u64,
23 /// When it was computed.
24 pub until: u64,
25 /// The percent added to a sampled peak, `[sizing] margin`: peaks are sampled every `sample_interval`
26 /// seconds, so a shorter spike between two samples goes unseen.
27 pub margin: u32,
28 /// Seconds between two samples of a job's container.
29 pub sample_interval: u64,
30 /// The fewest jobs a profile or job needs before its percentiles become advice, `[sizing] min_jobs`.
31 pub min_jobs: u32,
32 /// The slice's `MemoryMax`; `None` when it is unlimited.
33 pub memory_max: Option<u64>,
34 /// `[host] admission`, which the advice budgets `MemoryMax` by.
35 #[serde(default)]
36 pub admission: AdmissionPolicy,
37 /// Every configured profile, by name.
38 pub profiles: Vec<ProfileSizing>,
39 /// Whether the daemon applies advice itself, `[sizing] auto`.
40 #[serde(default)]
41 pub auto: bool,
42 /// The changes auto sizing made since `since`, oldest first.
43 #[serde(default)]
44 pub changes: Vec<SizingChange>,
45}
46
47/// A profile's `memory` and `slots` as auto sizing changed them in the config file.
48#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
49pub struct SizingChange {
50 /// When it was written.
51 pub at: u64,
52 /// The profile.
53 pub profile: String,
54 /// What they were: writing these back undoes it.
55 pub from: Resize,
56 /// What they are now.
57 pub to: Resize,
58 /// Why, as the advice said it then.
59 pub reason: String,
60 /// Whether it may not have been written: recorded before its file was written, and the config no longer said
61 /// whether it was when runnerd looked again. It still holds the interval, and `from` is still what it replaced.
62 #[serde(default)]
63 pub uncertain: bool,
64}
65
66/// One profile: its limits now, its jobs' peaks and waits, and the advice.
67#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
68pub struct ProfileSizing {
69 /// The profile.
70 pub name: String,
71 /// Its `memory` now: each container's hard limit.
72 pub memory: u64,
73 /// Its `cpus` now.
74 pub cpus: u32,
75 /// Its `slots` now.
76 pub slots: u32,
77 /// Its jobs' peaks.
78 pub peaks: Peaks,
79 /// Seconds from queued to started.
80 pub wait: Spread,
81 /// The percent of the time since `since` its slots held a runner, over its `slots` now.
82 pub busy: u32,
83 /// Whether its jobs waited a minute or more for a slot at p95, or its slots were busy half the time: more slots
84 /// would help.
85 #[serde(default)]
86 pub wants_slots: bool,
87 /// What to set `memory` and `slots` to. Sized for its largest peak, so no job gets less than it used.
88 pub advice: Advice,
89 /// Each job that ran in it, by repository, workflow and name.
90 pub jobs: Vec<JobSizing>,
91}
92
93/// The jobs of one name in one workflow of one repository.
94#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
95pub struct JobSizing {
96 /// `owner/name` on GitHub.
97 pub repository: String,
98 /// The workflow's name, when the history knew it.
99 pub workflow: Option<String>,
100 /// The job's name in the workflow.
101 pub name: String,
102 /// Its runs' peaks.
103 pub peaks: Peaks,
104 /// Seconds from queued to started.
105 pub wait: Spread,
106 /// What its profile's `memory` would be for this job alone, and the `slots` that would fit in the memory the
107 /// profile's slots take now.
108 pub advice: Advice,
109}
110
111/// The peaks of a set of jobs, each the highest of its samples.
112#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
113pub struct Peaks {
114 /// How many jobs: the number of peaks the percentiles are taken over.
115 pub jobs: u64,
116 /// Of those, how many met their container's memory cap: killed for memory, or out of memory at it with
117 /// nothing left to reclaim (for a run whose container's events were not recorded, a peak within 5% of it). They
118 /// may need more than they show.
119 pub at_cap: u64,
120 /// Memory without page cache, in bytes.
121 pub memory: Spread,
122 /// CPU, in millicores.
123 pub cpu: Spread,
124}
125
126/// Percentiles of a set of values, by nearest rank: each is one of the values, and with fewer than 100 values
127/// `p99` is the largest.
128#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
129pub struct Spread {
130 /// The median.
131 pub p50: u64,
132 /// The 95th percentile.
133 pub p95: u64,
134 /// The 99th percentile.
135 pub p99: u64,
136 /// The largest.
137 pub max: u64,
138}
139
140/// What to set a profile's `memory` and `slots` to. `memory` is the largest sampled peak plus the margin, rounded
141/// up to 256M: the cap kills a job that goes over it, so it is sized for the heaviest run, not a percentile.
142#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
143#[serde(rename_all = "kebab-case", tag = "kind")]
144pub enum Advice {
145 /// Fewer than `min_jobs` jobs: too few to advise on.
146 TooFew {
147 /// How many there are.
148 jobs: u64,
149 },
150 /// The cap fits, and lowering it would save less than a tenth or a run met its cap.
151 Keep {
152 /// What the jobs need.
153 memory: u64,
154 },
155 /// The cap is a tenth or more above what the jobs need, and no run met its cap.
156 Lower {
157 /// The new `memory`.
158 memory: u64,
159 /// The new `slots`: under `caps` admission, as many as fit in the memory the slots take now when jobs waited
160 /// for one; else the same.
161 slots: u32,
162 },
163 /// The jobs need more than the cap.
164 Raise {
165 /// The new `memory`.
166 memory: u64,
167 /// The new `slots`: the same, or under `caps` admission as many as the slice's `MemoryMax` still holds.
168 slots: u32,
169 },
170 /// The jobs need more than the cap, and the slice's `MemoryMax` does not hold one slot of what they need (under
171 /// `measured` admission, one runner beside the mirror): the slice has to grow first.
172 SliceTooSmall {
173 /// What the jobs need.
174 memory: u64,
175 },
176}
177
178impl Advice {
179 /// The `memory` and `slots` to write, when they differ from the profile's.
180 pub fn change(&self) -> Option<Resize> {
181 match *self {
182 Advice::Lower { memory, slots } | Advice::Raise { memory, slots } => {
183 Some(Resize { memory, slots })
184 }
185 Advice::TooFew { .. } | Advice::Keep { .. } | Advice::SliceTooSmall { .. } => None,
186 }
187 }
188}
189
190/// A profile's new `memory` and `slots`.
191#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
192pub struct Resize {
193 /// Bytes.
194 pub memory: u64,
195 /// Slots.
196 pub slots: u32,
197}
198
199impl SizingChange {
200 /// What it changed, such as `memory 9G → 6.75G, slots 2`.
201 pub fn line(&self) -> String {
202 let memory = if self.from.memory == self.to.memory {
203 format!("memory {}", ByteSize(self.to.memory))
204 } else {
205 format!(
206 "memory {} → {}",
207 ByteSize(self.from.memory),
208 ByteSize(self.to.memory)
209 )
210 };
211 let slots = if self.from.slots == self.to.slots {
212 format!("slots {}", self.to.slots)
213 } else {
214 format!("slots {} → {}", self.from.slots, self.to.slots)
215 };
216 if self.uncertain {
217 format!("{memory}, {slots} (may not have been written)")
218 } else {
219 format!("{memory}, {slots}")
220 }
221 }
222}
223
224impl Sizing {
225 /// One line on what to do with a profile and why, as `runnerctl sizing`, the dashboard and `runnerd doctor`
226 /// say it, such as `lower memory to 2.5G from 7G, its largest peak + 15%; 5 slots fit in the same 14G`.
227 /// Under `measured` admission, when jobs wanted more slots, it adds that `slots` is not bought with caps there.
228 pub fn advice_line(&self, profile: &ProfileSizing) -> String {
229 let line = self.memory_line(profile);
230 if self.admission == AdmissionPolicy::Measured
231 && profile.wants_slots
232 && !matches!(profile.advice, Advice::TooFew { .. })
233 {
234 format!(
235 "{line}; jobs waited {}s for a slot (p95) and slots were busy {}%: measured admission reserves \
236 predicted peaks, not caps, so more slots may fit: raise slots by hand",
237 profile.wait.p95, profile.busy
238 )
239 } else {
240 line
241 }
242 }
243
244 /// `advice_line` without what it says about slots under measured admission.
245 pub(crate) fn memory_line(&self, profile: &ProfileSizing) -> String {
246 let size = |bytes: u64| ByteSize(bytes).to_string();
247 let peaks = &profile.peaks;
248 let need = format!("its largest peak + {}%", self.margin);
249 let at_cap = format!(
250 "{} of {} jobs ran out of memory at their cap",
251 peaks.at_cap, peaks.jobs
252 );
253 match profile.advice {
254 Advice::TooFew { jobs } => format!(
255 "{jobs} jobs, too few to advise on (it takes {}); largest peak {} of a {} cap",
256 self.min_jobs,
257 size(peaks.memory.max),
258 size(profile.memory)
259 ),
260 Advice::Keep { .. } if peaks.at_cap > 0 => format!(
261 "keep {}: {at_cap}, so they may need more than they show",
262 size(profile.memory)
263 ),
264 Advice::Keep { memory } => {
265 format!("keep {}: {need} is {}", size(profile.memory), size(memory))
266 }
267 Advice::Lower { memory, slots } => {
268 let share = profile.memory * u64::from(profile.slots);
269 if self.admission == AdmissionPolicy::Measured {
270 return format!(
271 "lower memory to {} from {}, {need}",
272 size(memory),
273 size(profile.memory)
274 );
275 }
276 let then = if slots > profile.slots {
277 format!("{slots} slots fit in the same {}", size(share))
278 } else {
279 format!(
280 "frees {} of the budget",
281 size(share - memory * u64::from(slots))
282 )
283 };
284 format!(
285 "lower memory to {} from {}, {need}; {then}",
286 size(memory),
287 size(profile.memory)
288 )
289 }
290 Advice::Raise { memory, slots } => {
291 let why = if peaks.at_cap > 0 {
292 at_cap
293 } else {
294 format!("{need} is above the {} cap", size(profile.memory))
295 };
296 let slots = if slots < profile.slots {
297 format!("; {slots} of its {} slots fit the slice", profile.slots)
298 } else {
299 String::new()
300 };
301 format!("raise memory to {}: {why}{slots}", size(memory))
302 }
303 Advice::SliceTooSmall { memory } => format!(
304 "needs {memory_needed} ({need}), more than the slice holds for one slot: raise the slice's \
305 MemoryMax first",
306 memory_needed = size(memory)
307 ),
308 }
309 }
310}
311
312// The wire format is JSON, which only the host and client features read.
313#[cfg(all(test, any(feature = "host", feature = "client")))]
314mod tests;