runnerd_core/api/sizing.rs
1//! Right-sizing as clients read it: per profile and per job (repository, workflow and job name), the spread of the
2//! job history's sampled memory and CPU peaks, how long jobs waited for a slot, how busy the slots were, and what
3//! to set each profile's `memory` and `slots` to. Plain data; the daemon computes it from the history. Sizes are
4//! bytes, CPU millicores, times seconds since the Unix epoch.
5
6use serde::{Deserialize, Serialize};
7
8use crate::api::AdmissionPolicy;
9use crate::config::ByteSize;
10
11mod shown;
12
13pub use shown::{GRANULE, Need, ProfilePlan, SAMPLE_INTERVAL, need, sampled_note};
14
15/// `GET /v1/sizing` query parameters.
16#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
17pub struct SizingQuery {
18 /// Only jobs queued at or after this time; default 30 days ago.
19 pub since: Option<u64>,
20}
21
22/// `GET /v1/sizing`: what the job history says each configured profile needs.
23#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
24pub struct Sizing {
25 /// The start of the history read: `since`, or the first runner recorded after it.
26 pub since: u64,
27 /// When it was computed.
28 pub until: u64,
29 /// The percent added to a sampled peak, `[sizing] margin`: peaks are sampled every `sample_interval`
30 /// seconds, so a shorter spike between two samples goes unseen.
31 pub margin: u32,
32 /// Seconds between two samples of a job's container.
33 pub sample_interval: u64,
34 /// The fewest jobs a profile or job needs before its percentiles become advice, `[sizing] min_jobs`.
35 pub min_jobs: u32,
36 /// The slice's `MemoryMax`; `None` when it is unlimited.
37 pub memory_max: Option<u64>,
38 /// `[host] admission`, which the advice budgets `MemoryMax` by.
39 #[serde(default)]
40 pub admission: AdmissionPolicy,
41 /// Every configured profile, by name.
42 pub profiles: Vec<ProfileSizing>,
43 /// Whether the daemon applies advice itself, `[sizing] auto`.
44 #[serde(default)]
45 pub auto: bool,
46 /// The changes auto sizing made since `since`, oldest first.
47 #[serde(default)]
48 pub changes: Vec<SizingChange>,
49 /// Under auto sizing, what it does next and when. `None` while it is off, and from a daemon before it said.
50 #[serde(default, skip_serializing_if = "Option::is_none")]
51 pub auto_plan: Option<AutoPlan>,
52}
53
54/// What auto sizing will do: its rules, the changes its next look would make as the history and config are now, and
55/// the profiles a recent change holds.
56#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
57pub struct AutoPlan {
58 /// Seconds between two looks of the daemon's.
59 pub every: u64,
60 /// `[sizing] interval`: the least time between a change to one profile and one that lowers it, in seconds.
61 pub interval: u64,
62 /// `[sizing] raise_interval`: the least time between a change to one profile and one that only raises it, in
63 /// seconds. 0 from a daemon before it said: `interval` then holds every change.
64 #[serde(default)]
65 pub raise_interval: u64,
66 /// `[sizing] max_step`: the most one change moves a profile's `memory`, in percent of it.
67 pub max_step: u32,
68 /// The changes the next look would make, in the order it would make them; `at` is when this was worked out.
69 /// None of them is made while `blocked` says why.
70 pub next: Vec<SizingChange>,
71 /// The profiles a change in the last `interval` holds from some change, the hold that ends first first.
72 pub held: Vec<SizingHold>,
73 /// Why no look changes anything until the owner acts, as when the file on disk differs from the config runnerd
74 /// runs with. `None` when the next look may write `next`, and from a daemon before it said.
75 #[serde(default, skip_serializing_if = "Option::is_none")]
76 pub blocked: Option<String>,
77 /// The profiles whose jobs the CPU cap holds back and whose `cpus` the next look cannot raise, and why. Empty
78 /// from a daemon before it said.
79 #[serde(default, skip_serializing_if = "Vec::is_empty")]
80 pub refused: Vec<SizingRefused>,
81}
82
83/// A raise of a profile's `cpus` its jobs ask for that auto sizing cannot make, as when it does not fit the CPU
84/// budget.
85#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
86pub struct SizingRefused {
87 /// The profile.
88 pub profile: String,
89 /// Which job the cap holds back, and what keeps it from a raise.
90 pub reason: String,
91}
92
93/// A profile auto sizing changes nothing in before `until`, and lowers nothing in before `lower_until`.
94#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
95pub struct SizingHold {
96 /// The profile.
97 pub profile: String,
98 /// When its last change was made.
99 pub changed: u64,
100 /// When it may be raised again, in seconds since the Unix epoch.
101 pub until: u64,
102 /// When it may be lowered again; 0 from a daemon before it said, which held every change until `until`.
103 #[serde(default)]
104 pub lower_until: u64,
105}
106
107impl SizingHold {
108 /// When it may be lowered again.
109 pub fn lowered_after(&self) -> u64 {
110 self.lower_until.max(self.until)
111 }
112}
113
114/// A profile's `memory` and `slots` as auto sizing changed them in the config file.
115#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
116pub struct SizingChange {
117 /// When it was written.
118 pub at: u64,
119 /// The profile.
120 pub profile: String,
121 /// What they were: writing these back undoes it.
122 pub from: Resize,
123 /// What they are now.
124 pub to: Resize,
125 /// Why, as the advice said it then.
126 pub reason: String,
127 /// Whether it may not have been written: recorded before its file was written, and the config no longer said
128 /// whether it was when runnerd looked again. It still holds the interval, and `from` is still what it replaced.
129 #[serde(default)]
130 pub uncertain: bool,
131}
132
133/// One profile: its limits now, its jobs' peaks and waits, and the advice.
134#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
135pub struct ProfileSizing {
136 /// The profile.
137 pub name: String,
138 /// Its `memory` now: each container's hard limit.
139 pub memory: u64,
140 /// Its `cpus` now.
141 pub cpus: u32,
142 /// Its `slots` now.
143 pub slots: u32,
144 /// Its jobs' peaks.
145 pub peaks: Peaks,
146 /// Seconds from queued to started.
147 pub wait: Spread,
148 /// Its jobs by whether a warm runner took them, and how long each kind waited. Empty from a daemon before warm
149 /// runners.
150 #[serde(default)]
151 pub starts: Starts,
152 /// The percent of the time since `since` its slots held a runner for a job, from when the job was queued, over
153 /// its `slots` now.
154 pub busy: u32,
155 /// Whether its jobs waited a minute or more for a slot at p95, or its slots were busy half the time: more slots
156 /// would help.
157 #[serde(default)]
158 pub wants_slots: bool,
159 /// What to set `memory` and `slots` to. Sized for its largest peak, so no job gets less than it used.
160 pub advice: Advice,
161 /// Each job that ran in it, by repository, workflow and name.
162 pub jobs: Vec<JobSizing>,
163}
164
165/// A profile's jobs by whether their runner was already up when GitHub queued them (ADR 0050).
166#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
167pub struct Starts {
168 /// Jobs that started on a warm runner: one that started before the job was queued.
169 pub warm: u32,
170 /// Their median wait from queued to started, in seconds.
171 pub warm_p50: u64,
172 /// Jobs whose runner started after they were queued.
173 pub cold: u32,
174 /// Their median wait, in seconds.
175 pub cold_p50: u64,
176}
177
178/// The jobs of one name in one workflow of one repository.
179#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
180pub struct JobSizing {
181 /// `owner/name` on GitHub.
182 pub repository: String,
183 /// The workflow's name, when the history knew it.
184 pub workflow: Option<String>,
185 /// The job's name in the workflow.
186 pub name: String,
187 /// Its runs' peaks.
188 pub peaks: Peaks,
189 /// Seconds from queued to started.
190 pub wait: Spread,
191 /// What its profile's `memory` would be for this job alone, and the `slots` that would fit in the memory the
192 /// profile's slots take now.
193 pub advice: Advice,
194}
195
196/// The peaks of a set of jobs, each the highest of its samples.
197#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
198pub struct Peaks {
199 /// How many jobs: the number of peaks the percentiles are taken over.
200 pub jobs: u64,
201 /// Of those, how many met their container's memory cap: killed for memory, or out of memory at it with
202 /// nothing left to reclaim (for a run whose container's events were not recorded, a peak within 5% of it). They
203 /// may need more than they show.
204 pub at_cap: u64,
205 /// Memory without page cache, in bytes.
206 pub memory: Spread,
207 /// CPU, in millicores.
208 pub cpu: Spread,
209}
210
211/// Percentiles of a set of values, by nearest rank: each is one of the values, and with fewer than 100 values
212/// `p99` is the largest.
213#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
214pub struct Spread {
215 /// The median.
216 pub p50: u64,
217 /// The 95th percentile.
218 pub p95: u64,
219 /// The 99th percentile.
220 pub p99: u64,
221 /// The largest.
222 pub max: u64,
223}
224
225/// What a cap keeps at least over the largest peak, for page cache: 1G. Peaks leave page cache out, and a container
226/// whose memory past them leaves the kernel less than about 500M of page cache to reclaim at its cap reclaims badly,
227/// which throttles its sockets and slows its downloads (ADR 0055).
228pub const PAGE_CACHE_ROOM: u64 = 1 << 30;
229
230/// What to set a profile's `memory` and `slots` to. `memory` is the largest sampled peak plus the margin, or plus
231/// `PAGE_CACHE_ROOM` when that is more, rounded up to 256M: the cap kills a job that goes over it, so it is sized for
232/// the heaviest run, not a percentile.
233#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
234#[serde(rename_all = "kebab-case", tag = "kind")]
235pub enum Advice {
236 /// Fewer than `min_jobs` jobs: too few to advise on.
237 TooFew {
238 /// How many there are.
239 jobs: u64,
240 },
241 /// The cap fits, and lowering it would save less than a tenth or a run met its cap.
242 Keep {
243 /// What the jobs need.
244 memory: u64,
245 },
246 /// The cap is a tenth or more above what the jobs need, and no run met its cap.
247 Lower {
248 /// The new `memory`.
249 memory: u64,
250 /// The new `slots`: under `caps` admission, as many as fit in the memory the slots take now when jobs waited
251 /// for one; else the same.
252 slots: u32,
253 },
254 /// The jobs need more than the cap.
255 Raise {
256 /// The new `memory`.
257 memory: u64,
258 /// The new `slots`: the same, or under `caps` admission as many as the slice's `MemoryMax` still holds.
259 slots: u32,
260 },
261 /// The jobs need more than the cap, and the slice's `MemoryMax` does not hold one slot of what they need (under
262 /// `measured` admission, one runner beside the mirror): the slice has to grow first.
263 SliceTooSmall {
264 /// What the jobs need.
265 memory: u64,
266 },
267}
268
269impl Advice {
270 /// The `memory` and `slots` to write, when they differ from the profile's.
271 pub fn change(&self) -> Option<Resize> {
272 match *self {
273 Advice::Lower { memory, slots } | Advice::Raise { memory, slots } => Some(Resize {
274 memory,
275 slots,
276 cpus: None,
277 }),
278 Advice::TooFew { .. } | Advice::Keep { .. } | Advice::SliceTooSmall { .. } => None,
279 }
280 }
281}
282
283/// A profile's new `memory`, `slots` and, when auto sizing changed them, `cpus`.
284#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
285pub struct Resize {
286 /// Bytes.
287 pub memory: u64,
288 /// Slots.
289 pub slots: u32,
290 /// CPUs per slot; `None` leaves them as they are, as advice does and as auto sizing's changes recorded before it
291 /// changed them.
292 #[serde(default, skip_serializing_if = "Option::is_none")]
293 pub cpus: Option<u32>,
294}
295
296impl AutoPlan {
297 /// Its rules in words, as clients say them after `runnerd`: `looks every 5 minutes, raises a profile at most
298 /// once every 1 hour and lowers it at most once every 24 hours, moving its memory and CPUs by at most 25% at a
299 /// time`.
300 pub fn rules(&self) -> String {
301 if self.raise_interval == 0 || self.raise_interval >= self.interval {
302 return format!(
303 "looks every {} and changes a profile at most once every {}, moving its memory by at most {}% at a \
304 time",
305 period(self.every),
306 period(self.interval),
307 self.max_step
308 );
309 }
310 format!(
311 "looks every {}, raises a profile at most once every {} and lowers it at most once every {}, moving its \
312 memory and CPUs by at most {}% at a time",
313 period(self.every),
314 period(self.raise_interval),
315 period(self.interval),
316 self.max_step
317 )
318 }
319}
320
321/// A setting's length in words, as a config's periods read: `5 minutes`, `24 hours`, `90 s`.
322pub fn period(seconds: u64) -> String {
323 let plural = |count: u64, unit: &str| {
324 if count == 1 {
325 format!("{count} {unit}")
326 } else {
327 format!("{count} {unit}s")
328 }
329 };
330 match seconds {
331 0 => "0 s".to_owned(),
332 _ if seconds.is_multiple_of(3600) => plural(seconds / 3600, "hour"),
333 _ if seconds.is_multiple_of(60) => plural(seconds / 60, "minute"),
334 _ => format!("{seconds} s"),
335 }
336}
337
338impl SizingChange {
339 /// What it changed, such as `memory 9G → 6.75G, slots 2`, and `cpus 12 → 15` when it changed them.
340 pub fn line(&self) -> String {
341 let memory = if self.from.memory == self.to.memory {
342 format!("memory {}", ByteSize(self.to.memory))
343 } else {
344 format!(
345 "memory {} → {}",
346 ByteSize(self.from.memory),
347 ByteSize(self.to.memory)
348 )
349 };
350 let slots = if self.from.slots == self.to.slots {
351 format!("slots {}", self.to.slots)
352 } else {
353 format!("slots {} → {}", self.from.slots, self.to.slots)
354 };
355 let cpus = match (self.from.cpus, self.to.cpus) {
356 (Some(from), Some(to)) if from != to => format!(", cpus {from} → {to}"),
357 _ => String::new(),
358 };
359 if self.uncertain {
360 format!("{memory}, {slots}{cpus} (may not have been written)")
361 } else {
362 format!("{memory}, {slots}{cpus}")
363 }
364 }
365}
366
367impl Sizing {
368 /// One line on what to do with a profile and why, as `runnerctl sizing`, the dashboard and `runnerd doctor`
369 /// say it, such as `lower memory to 2.5G from 7G, its largest peak + 15%; 5 slots fit in the same 14G`.
370 /// Under `measured` admission, when jobs wanted more slots, it adds that `slots` is not bought with caps there, or
371 /// with auto sizing on, that it adds them.
372 pub fn advice_line(&self, profile: &ProfileSizing) -> String {
373 let line = self.memory_line(profile);
374 if self.admission == AdmissionPolicy::Measured
375 && profile.wants_slots
376 && !matches!(profile.advice, Advice::TooFew { .. })
377 {
378 let then = if self.auto {
379 "auto sizing adds one at a time while every profile's slots fit the slice at their jobs' peaks"
380 } else {
381 "measured admission reserves predicted peaks, not caps, so more slots may fit: raise slots by hand"
382 };
383 format!(
384 "{line}; jobs waited {}s for a slot (p95) and slots were busy {}%: {then}",
385 profile.wait.p95, profile.busy
386 )
387 } else {
388 line
389 }
390 }
391
392 /// `advice_line` without what it says about slots under measured admission.
393 pub(crate) fn memory_line(&self, profile: &ProfileSizing) -> String {
394 let size = |bytes: u64| ByteSize(bytes).to_string();
395 let peaks = &profile.peaks;
396 let need = self.need_reason(peaks);
397 let at_cap = format!(
398 "{} of {} jobs ran out of memory at their cap",
399 peaks.at_cap, peaks.jobs
400 );
401 match profile.advice {
402 Advice::TooFew { jobs } => format!(
403 "{jobs} jobs, too few to advise on (it takes {}); largest peak {} of a {} cap",
404 self.min_jobs,
405 size(peaks.memory.max),
406 size(profile.memory)
407 ),
408 Advice::Keep { .. } if peaks.at_cap > 0 => format!(
409 "keep {}: {at_cap}, so they may need more than they show",
410 size(profile.memory)
411 ),
412 Advice::Keep { memory } => {
413 format!("keep {}: {need} is {}", size(profile.memory), size(memory))
414 }
415 Advice::Lower { memory, slots } => {
416 let share = profile.memory * u64::from(profile.slots);
417 if self.admission == AdmissionPolicy::Measured {
418 return format!(
419 "lower memory to {} from {}, {need}",
420 size(memory),
421 size(profile.memory)
422 );
423 }
424 let then = if slots > profile.slots {
425 format!("{slots} slots fit in the same {}", size(share))
426 } else {
427 format!(
428 "frees {} of the budget",
429 size(share - memory * u64::from(slots))
430 )
431 };
432 format!(
433 "lower memory to {} from {}, {need}; {then}",
434 size(memory),
435 size(profile.memory)
436 )
437 }
438 Advice::Raise { memory, slots } => {
439 let why = if peaks.at_cap > 0 {
440 at_cap
441 } else {
442 format!("{need} is above the {} cap", size(profile.memory))
443 };
444 let slots = if slots < profile.slots {
445 format!("; {slots} of its {} slots fit the slice", profile.slots)
446 } else {
447 String::new()
448 };
449 format!("raise memory to {}: {why}{slots}", size(memory))
450 }
451 Advice::SliceTooSmall { memory } => format!(
452 "needs {memory_needed} ({need}), more than the slice holds for one slot: raise the slice's \
453 MemoryMax first",
454 memory_needed = size(memory)
455 ),
456 }
457 }
458}
459
460// The wire format is JSON, which only the host and client features read.
461#[cfg(all(test, any(feature = "host", feature = "client")))]
462mod tests;