runnerd_core/api/sizing.rs
1//! Right-sizing as clients read it: per profile and per job (repository, workflow and job name), the spread of the
2//! job history's sampled memory and CPU peaks, how long jobs waited for a slot, how busy the slots were, and what
3//! to set each profile's `memory` and `slots` to. Plain data; the daemon computes it from the history. Sizes are
4//! bytes, CPU millicores, times seconds since the Unix epoch.
5
6use serde::{Deserialize, Serialize};
7
8use crate::api::AdmissionPolicy;
9use crate::config::ByteSize;
10
11mod shown;
12
13pub use shown::{GRANULE, Need, ProfilePlan, SAMPLE_INTERVAL, need, sampled_note};
14
15/// `GET /v1/sizing` query parameters.
16#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
17pub struct SizingQuery {
18 /// Only jobs queued at or after this time; default 30 days ago.
19 pub since: Option<u64>,
20}
21
22/// `GET /v1/sizing`: what the job history says each configured profile needs.
23#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
24pub struct Sizing {
25 /// The start of the history read: `since`, or the first runner recorded after it.
26 pub since: u64,
27 /// When it was computed.
28 pub until: u64,
29 /// The percent added to a sampled peak, `[sizing] margin`: peaks are sampled every `sample_interval`
30 /// seconds, so a shorter spike between two samples goes unseen.
31 pub margin: u32,
32 /// Seconds between two samples of a job's container.
33 pub sample_interval: u64,
34 /// The fewest jobs a profile or job needs before its percentiles become advice, `[sizing] min_jobs`.
35 pub min_jobs: u32,
36 /// The slice's `MemoryMax`; `None` when it is unlimited.
37 pub memory_max: Option<u64>,
38 /// `[host] admission`, which the advice budgets `MemoryMax` by.
39 pub admission: AdmissionPolicy,
40 /// Every configured profile, by name.
41 pub profiles: Vec<ProfileSizing>,
42 /// Whether the daemon applies advice itself, `[sizing] auto`.
43 pub auto: bool,
44 /// The changes auto sizing made since `since`, oldest first.
45 pub changes: Vec<SizingChange>,
46 /// Under auto sizing, what it does next and when. `None` while it is off.
47 #[serde(default, skip_serializing_if = "Option::is_none")]
48 pub auto_plan: Option<AutoPlan>,
49}
50
51/// What auto sizing will do: its rules, the changes its next look would make as the history and config are now, and
52/// the profiles a recent change holds.
53#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
54pub struct AutoPlan {
55 /// Seconds between two looks of the daemon's.
56 pub every: u64,
57 /// `[sizing] interval`: the least time between a change to one profile and one that lowers it, in seconds.
58 pub interval: u64,
59 /// `[sizing] raise_interval`: the least time between a change to one profile and one that only raises it, in
60 /// seconds.
61 pub raise_interval: u64,
62 /// `[sizing] max_step`: the most one change moves a profile's `memory`, in percent of it.
63 pub max_step: u32,
64 /// The changes the next look would make, in the order it would make them; `at` is when this was worked out.
65 /// None of them is made while `blocked` says why.
66 pub next: Vec<SizingChange>,
67 /// The profiles a change in the last `interval` holds from some change, the hold that ends first first.
68 pub held: Vec<SizingHold>,
69 /// Why no look changes anything until the owner acts, as when the file on disk differs from the config runnerd
70 /// runs with. `None` when the next look may write `next`.
71 #[serde(default, skip_serializing_if = "Option::is_none")]
72 pub blocked: Option<String>,
73 /// The profiles whose jobs the CPU cap holds back and whose `cpus` the next look cannot raise, and why.
74 #[serde(default, skip_serializing_if = "Vec::is_empty")]
75 pub refused: Vec<SizingRefused>,
76}
77
78/// A raise of a profile's `cpus` its jobs ask for that auto sizing cannot make, as when it does not fit the CPU
79/// budget.
80#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
81pub struct SizingRefused {
82 /// The profile.
83 pub profile: String,
84 /// Which job the cap holds back, and what keeps it from a raise.
85 pub reason: String,
86}
87
88/// A profile auto sizing changes nothing in before `until`, and lowers nothing in before `lower_until`.
89#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
90pub struct SizingHold {
91 /// The profile.
92 pub profile: String,
93 /// When its last change was made.
94 pub changed: u64,
95 /// When it may be raised again, in seconds since the Unix epoch.
96 pub until: u64,
97 /// When it may be lowered again.
98 pub lower_until: u64,
99}
100
101impl SizingHold {
102 /// When it may be lowered again.
103 pub fn lowered_after(&self) -> u64 {
104 self.lower_until.max(self.until)
105 }
106}
107
108/// A profile's `memory` and `slots` as auto sizing changed them in the config file.
109#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
110pub struct SizingChange {
111 /// When it was written.
112 pub at: u64,
113 /// The profile.
114 pub profile: String,
115 /// What they were: writing these back undoes it.
116 pub from: Resize,
117 /// What they are now.
118 pub to: Resize,
119 /// Why, as the advice said it then.
120 pub reason: String,
121 /// Whether it may not have been written: recorded before its file was written, and the config no longer said
122 /// whether it was when runnerd looked again. It still holds the interval, and `from` is still what it replaced.
123 pub uncertain: bool,
124}
125
126/// One profile: its limits now, its jobs' peaks and waits, and the advice.
127#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
128pub struct ProfileSizing {
129 /// The profile.
130 pub name: String,
131 /// Its `memory` now: each container's hard limit.
132 pub memory: u64,
133 /// Its `cpus` now.
134 pub cpus: u32,
135 /// Its `slots` now.
136 pub slots: u32,
137 /// Its jobs' peaks.
138 pub peaks: Peaks,
139 /// Seconds from queued to started, leaving out the time before the history's start and the time no runnerd
140 /// ran (ADR 0081).
141 pub wait: Spread,
142 /// Its jobs by whether a warm runner took them, and how long each kind waited.
143 pub starts: Starts,
144 /// The percent of the time since `since` its slots held a runner for a job, from when the job was queued, over
145 /// its `slots` now.
146 pub busy: u32,
147 /// Whether its jobs waited a minute or more for a slot at p95, or its slots were busy half the time: more slots
148 /// would help.
149 pub wants_slots: bool,
150 /// What to set `memory` and `slots` to. Sized for its largest peak, so no job gets less than it used.
151 pub advice: Advice,
152 /// When the advice is to take back a lowering auto sizing made that went too far: what showed it.
153 #[serde(default, skip_serializing_if = "Option::is_none")]
154 pub revert: Option<SizingRevert>,
155 /// Each job that ran in it, by repository, workflow and name.
156 pub jobs: Vec<JobSizing>,
157}
158
159/// A lowering of a profile's `memory` by auto sizing that went too far, which the advice takes back (ADR 0070).
160#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
161pub struct SizingRevert {
162 /// The `memory` it lowered to, in bytes.
163 pub from: u64,
164 /// The `memory` it lowered from, which the advice goes back to.
165 pub to: u64,
166 /// The job that showed it, as `owner/name workflow / job`.
167 pub job: String,
168 /// How.
169 pub cause: RevertCause,
170}
171
172/// What showed a lowering went too far.
173#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
174#[serde(rename_all = "kebab-case", tag = "kind")]
175pub enum RevertCause {
176 /// A run met the lowered cap.
177 Met,
178 /// The job, which succeeded before the lowering, failed this many times in a row at it, in different workflow
179 /// runs and on more than one branch: a heap sized from the cgroup's limit fails without the kernel's limit.
180 Failed {
181 /// How many.
182 runs: u32,
183 },
184}
185
186/// A profile's jobs by whether their runner was already up when GitHub queued them (ADR 0050).
187#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
188pub struct Starts {
189 /// Jobs that started on a warm runner: one that started before the job was queued.
190 pub warm: u32,
191 /// Their median wait from queued to started, in seconds.
192 pub warm_p50: u64,
193 /// Jobs whose runner started after they were queued.
194 pub cold: u32,
195 /// Their median wait, in seconds.
196 pub cold_p50: u64,
197}
198
199/// The jobs of one name in one workflow of one repository.
200#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
201pub struct JobSizing {
202 /// `owner/name` on GitHub.
203 pub repository: String,
204 /// The workflow's name, when the history knew it.
205 pub workflow: Option<String>,
206 /// The job's name in the workflow.
207 pub name: String,
208 /// Its runs' peaks.
209 pub peaks: Peaks,
210 /// Seconds from queued to started, leaving out the time before the history's start and the time no runnerd
211 /// ran (ADR 0081).
212 pub wait: Spread,
213 /// What its profile's `memory` would be for this job alone, and the `slots` that would fit in the memory the
214 /// profile's slots take now.
215 pub advice: Advice,
216}
217
218/// The peaks of a set of jobs, each the highest of its samples.
219#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
220pub struct Peaks {
221 /// How many jobs: the number of peaks the percentiles are taken over.
222 pub jobs: u64,
223 /// Of those, how many met their container's memory cap: its own limit failed reclaim (for a run recorded
224 /// before that was told apart, a kill or a limit hit in the container, or without events a peak within 5% of it).
225 /// They may need more than they show.
226 pub at_cap: u64,
227 /// How many of their jobs are on their cap: one of their runs met it or was killed for memory, and fewer than 3
228 /// successful runs since were clear of it. No lowering while any is.
229 #[serde(default, skip_serializing_if = "is_zero")]
230 pub on_cap: u64,
231 /// Memory without page cache, in bytes.
232 pub memory: Spread,
233 /// CPU, in millicores, of the runs at the profile's `cpus` now: a run at a larger cap may have peaked past them.
234 pub cpu: Spread,
235 /// How many runs `cpu` is over: those of `jobs` at the profile's `cpus` now.
236 #[serde(default)]
237 pub cpu_runs: u64,
238}
239
240#[expect(
241 clippy::trivially_copy_pass_by_ref,
242 reason = "serde's skip_serializing_if passes a reference"
243)]
244fn is_zero(count: &u64) -> bool {
245 *count == 0
246}
247
248/// Percentiles of a set of values, by nearest rank: each is one of the values, and with fewer than 100 values
249/// `p99` is the largest.
250#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
251pub struct Spread {
252 /// The median.
253 pub p50: u64,
254 /// The 95th percentile.
255 pub p95: u64,
256 /// The 99th percentile.
257 pub p99: u64,
258 /// The largest.
259 pub max: u64,
260}
261
262/// What a cap keeps at least over the largest peak, for page cache: 1G. Peaks leave page cache out, and a container
263/// whose memory past them leaves the kernel less than about 500M of page cache to reclaim at its cap reclaims badly,
264/// which throttles its sockets and slows its downloads (ADR 0055).
265pub const PAGE_CACHE_ROOM: u64 = 1 << 30;
266
267/// What to set a profile's `memory` and `slots` to. `memory` is the largest sampled peak plus the margin, or plus
268/// `PAGE_CACHE_ROOM` when that is more, rounded up to 256M: the cap kills a job that goes over it, so it is sized for
269/// the heaviest run, not a percentile.
270#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
271#[serde(rename_all = "kebab-case", tag = "kind")]
272pub enum Advice {
273 /// Fewer than `min_jobs` jobs: too few to advise on.
274 TooFew {
275 /// How many there are.
276 jobs: u64,
277 },
278 /// The cap fits, and lowering it would save less than a tenth or a run met its cap.
279 Keep {
280 /// What the jobs need.
281 memory: u64,
282 },
283 /// The cap is a tenth or more above what the jobs need, and no run met its cap.
284 Lower {
285 /// The new `memory`.
286 memory: u64,
287 /// The new `slots`: under `caps` admission, as many as fit in the memory the slots take now when jobs waited
288 /// for one; else the same.
289 slots: u32,
290 },
291 /// The jobs need more than the cap.
292 Raise {
293 /// The new `memory`.
294 memory: u64,
295 /// The new `slots`: the same, or under `caps` admission as many as the slice's `MemoryMax` still holds.
296 slots: u32,
297 },
298 /// The jobs need more than the cap, and the slice's `MemoryMax` does not hold one slot of what they need (under
299 /// `measured` admission, one runner beside the mirror): the slice has to grow first.
300 SliceTooSmall {
301 /// What the jobs need.
302 memory: u64,
303 },
304}
305
306impl Advice {
307 /// The `memory` and `slots` to write, when they differ from the profile's.
308 pub fn change(&self) -> Option<Resize> {
309 match *self {
310 Advice::Lower { memory, slots } | Advice::Raise { memory, slots } => Some(Resize {
311 memory,
312 slots,
313 cpus: None,
314 }),
315 Advice::TooFew { .. } | Advice::Keep { .. } | Advice::SliceTooSmall { .. } => None,
316 }
317 }
318}
319
320/// A profile's new `memory`, `slots` and, when auto sizing changed them, `cpus`.
321#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
322pub struct Resize {
323 /// Bytes.
324 pub memory: u64,
325 /// Slots.
326 pub slots: u32,
327 /// CPUs per slot; `None` leaves them as they are, as advice does and as auto sizing's changes recorded before it
328 /// changed them.
329 #[serde(default, skip_serializing_if = "Option::is_none")]
330 pub cpus: Option<u32>,
331}
332
333impl AutoPlan {
334 /// Its rules in words, as clients say them after `runnerd`: `looks every 5 minutes, raises a profile at most
335 /// once every 1 hour and lowers it at most once every 24 hours, moving its memory and CPUs by at most 25% at a
336 /// time`.
337 pub fn rules(&self) -> String {
338 if self.raise_interval == 0 || self.raise_interval >= self.interval {
339 return format!(
340 "looks every {} and changes a profile at most once every {}, moving its memory by at most {}% at a \
341 time",
342 period(self.every),
343 period(self.interval),
344 self.max_step
345 );
346 }
347 format!(
348 "looks every {}, raises a profile at most once every {} and lowers it at most once every {}, moving its \
349 memory and CPUs by at most {}% at a time",
350 period(self.every),
351 period(self.raise_interval),
352 period(self.interval),
353 self.max_step
354 )
355 }
356}
357
358/// A setting's length in words, as a config's periods read: `5 minutes`, `24 hours`, `90 s`.
359pub fn period(seconds: u64) -> String {
360 let plural = |count: u64, unit: &str| {
361 if count == 1 {
362 format!("{count} {unit}")
363 } else {
364 format!("{count} {unit}s")
365 }
366 };
367 match seconds {
368 0 => "0 s".to_owned(),
369 _ if seconds.is_multiple_of(3600) => plural(seconds / 3600, "hour"),
370 _ if seconds.is_multiple_of(60) => plural(seconds / 60, "minute"),
371 _ => format!("{seconds} s"),
372 }
373}
374
375impl SizingChange {
376 /// What it changed, such as `memory 9G → 6.75G, slots 2`, and `cpus 12 → 15` when it changed them.
377 pub fn line(&self) -> String {
378 let memory = if self.from.memory == self.to.memory {
379 format!("memory {}", ByteSize(self.to.memory))
380 } else {
381 format!(
382 "memory {} → {}",
383 ByteSize(self.from.memory),
384 ByteSize(self.to.memory)
385 )
386 };
387 let slots = if self.from.slots == self.to.slots {
388 format!("slots {}", self.to.slots)
389 } else {
390 format!("slots {} → {}", self.from.slots, self.to.slots)
391 };
392 let cpus = match (self.from.cpus, self.to.cpus) {
393 (Some(from), Some(to)) if from != to => format!(", cpus {from} → {to}"),
394 _ => String::new(),
395 };
396 if self.uncertain {
397 format!("{memory}, {slots}{cpus} (may not have been written)")
398 } else {
399 format!("{memory}, {slots}{cpus}")
400 }
401 }
402}
403
404impl Sizing {
405 /// One line on what to do with a profile and why, as `runnerctl sizing`, the dashboard and `runnerd doctor`
406 /// say it, such as `lower memory to 2.5G from 7G, its largest peak + 15%; 5 slots fit in the same 14G`.
407 /// Under `measured` admission, when jobs wanted more slots, it adds that `slots` is not bought with caps there, or
408 /// with auto sizing on, that it adds them.
409 pub fn advice_line(&self, profile: &ProfileSizing) -> String {
410 let line = self.memory_line(profile);
411 if self.admission == AdmissionPolicy::Measured
412 && profile.wants_slots
413 && !matches!(profile.advice, Advice::TooFew { .. })
414 {
415 let then = if self.auto {
416 "auto sizing adds one at a time while every profile's slots fit the slice at their jobs' peaks"
417 } else {
418 "measured admission reserves predicted peaks, not caps, so more slots may fit: raise slots by hand"
419 };
420 format!(
421 "{line}; jobs waited {}s for a slot (p95) and slots were busy {}%: {then}",
422 profile.wait.p95, profile.busy
423 )
424 } else {
425 line
426 }
427 }
428
429 /// `advice_line` without what it says about slots under measured admission.
430 pub(crate) fn memory_line(&self, profile: &ProfileSizing) -> String {
431 let size = |bytes: u64| ByteSize(bytes).to_string();
432 let peaks = &profile.peaks;
433 let need = self.need_reason(peaks);
434 let at_cap = format!(
435 "{} of {} jobs ran out of memory at their cap",
436 peaks.at_cap, peaks.jobs
437 );
438 match profile.advice {
439 Advice::TooFew { jobs } => format!(
440 "{jobs} jobs, too few to advise on (it takes {}); largest peak {} of a {} cap",
441 self.min_jobs,
442 size(peaks.memory.max),
443 size(profile.memory)
444 ),
445 Advice::Keep { .. } if peaks.on_cap > 0 => format!(
446 "keep {}: {} of its jobs met their cap or were killed for memory, and fewer than 3 runs since ran \
447 clear of it",
448 size(profile.memory),
449 peaks.on_cap
450 ),
451 Advice::Keep { memory } if memory > profile.memory => format!(
452 "keep {}: its recent runs need {}, but none is known to have run out of memory at {} since it was set",
453 size(profile.memory),
454 size(memory),
455 size(profile.memory)
456 ),
457 Advice::Keep { memory } => {
458 format!("keep {}: {need} is {}", size(profile.memory), size(memory))
459 }
460 Advice::Lower { memory, slots } => {
461 let share = profile.memory * u64::from(profile.slots);
462 if self.admission == AdmissionPolicy::Measured {
463 return format!(
464 "lower memory to {} from {}, {need}",
465 size(memory),
466 size(profile.memory)
467 );
468 }
469 let then = if slots > profile.slots {
470 format!("{slots} slots fit in the same {}", size(share))
471 } else {
472 format!(
473 "frees {} of the budget",
474 size(share - memory * u64::from(slots))
475 )
476 };
477 format!(
478 "lower memory to {} from {}, {need}; {then}",
479 size(memory),
480 size(profile.memory)
481 )
482 }
483 Advice::Raise { memory, .. } if let Some(revert) = &profile.revert => {
484 let how = match revert.cause {
485 RevertCause::Met => format!("met the {} cap", size(revert.from)),
486 RevertCause::Failed { runs } => {
487 format!("failed {runs} times in a row at {}", size(revert.from))
488 }
489 };
490 format!(
491 "raise memory back to {}: {} {how} since auto sizing lowered it from {}",
492 size(memory),
493 revert.job,
494 size(revert.to)
495 )
496 }
497 Advice::Raise { memory, slots } => {
498 let why = if peaks.at_cap > 0 {
499 at_cap
500 } else {
501 format!("{need} is above the {} cap", size(profile.memory))
502 };
503 let slots = if slots < profile.slots {
504 format!("; {slots} of its {} slots fit the slice", profile.slots)
505 } else {
506 String::new()
507 };
508 format!("raise memory to {}: {why}{slots}", size(memory))
509 }
510 Advice::SliceTooSmall { memory } => format!(
511 "needs {memory_needed} ({need}), more than the slice holds for one slot: raise the slice's \
512 MemoryMax first",
513 memory_needed = size(memory)
514 ),
515 }
516 }
517}