runnerd_core/api/sizing.rs
1//! Right-sizing as clients read it: per profile and per job (repository, workflow and job name), the spread of the
2//! job history's sampled memory and CPU peaks, how long jobs waited for a slot, how busy the slots were, and what
3//! to set each profile's `memory` and `slots` to. Plain data; the daemon computes it from the history. Sizes are
4//! bytes, CPU millicores, times seconds since the Unix epoch.
5
6use serde::{Deserialize, Serialize};
7
8use crate::config::ByteSize;
9
10/// `GET /v1/sizing` query parameters.
11#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
12pub struct SizingQuery {
13 /// Only jobs queued at or after this time; default 30 days ago.
14 pub since: Option<u64>,
15}
16
17/// `GET /v1/sizing`: what the job history says each configured profile needs.
18#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
19pub struct Sizing {
20 /// The start of the history read: `since`, or the first runner recorded after it.
21 pub since: u64,
22 /// When it was computed.
23 pub until: u64,
24 /// The percent added to a sampled peak, `[sizing] margin`: peaks are sampled every `sample_interval`
25 /// seconds, so a shorter spike between two samples goes unseen.
26 pub margin: u32,
27 /// Seconds between two samples of a job's container.
28 pub sample_interval: u64,
29 /// The fewest jobs a profile or job needs before its percentiles become advice, `[sizing] min_jobs`.
30 pub min_jobs: u32,
31 /// The slice's `MemoryMax`; `None` when it is unlimited.
32 pub memory_max: Option<u64>,
33 /// Every configured profile, by name.
34 pub profiles: Vec<ProfileSizing>,
35}
36
37/// One profile: its limits now, its jobs' peaks and waits, and the advice.
38#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
39pub struct ProfileSizing {
40 /// The profile.
41 pub name: String,
42 /// Its `memory` now: each container's hard limit.
43 pub memory: u64,
44 /// Its `cpus` now.
45 pub cpus: u32,
46 /// Its `slots` now.
47 pub slots: u32,
48 /// Its jobs' peaks.
49 pub peaks: Peaks,
50 /// Seconds from queued to started.
51 pub wait: Spread,
52 /// The percent of the time since `since` its slots held a runner, over its `slots` now.
53 pub busy: u32,
54 /// What to set `memory` and `slots` to. Sized for its largest peak, so no job gets less than it used.
55 pub advice: Advice,
56 /// Each job that ran in it, by repository, workflow and name.
57 pub jobs: Vec<JobSizing>,
58}
59
60/// The jobs of one name in one workflow of one repository.
61#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
62pub struct JobSizing {
63 /// `owner/name` on GitHub.
64 pub repository: String,
65 /// The workflow's name, when the history knew it.
66 pub workflow: Option<String>,
67 /// The job's name in the workflow.
68 pub name: String,
69 /// Its runs' peaks.
70 pub peaks: Peaks,
71 /// Seconds from queued to started.
72 pub wait: Spread,
73 /// What its profile's `memory` would be for this job alone, and the `slots` that would fit in the memory the
74 /// profile's slots take now.
75 pub advice: Advice,
76}
77
78/// The peaks of a set of jobs, each the highest of its samples.
79#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
80pub struct Peaks {
81 /// How many jobs: the number of peaks the percentiles are taken over.
82 pub jobs: u64,
83 /// Of those, how many peaked within 5% of their container's memory cap: likely held back or killed by it, so
84 /// they may need more than they show.
85 pub at_cap: u64,
86 /// Memory without page cache, in bytes.
87 pub memory: Spread,
88 /// CPU, in millicores.
89 pub cpu: Spread,
90}
91
92/// Percentiles of a set of values, by nearest rank: each is one of the values, and with fewer than 100 values
93/// `p99` is the largest.
94#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
95pub struct Spread {
96 /// The median.
97 pub p50: u64,
98 /// The 95th percentile.
99 pub p95: u64,
100 /// The 99th percentile.
101 pub p99: u64,
102 /// The largest.
103 pub max: u64,
104}
105
106/// What to set a profile's `memory` and `slots` to. `memory` is the largest sampled peak plus the margin, rounded
107/// up to 256M: the cap kills a job that goes over it, so it is sized for the heaviest run, not a percentile.
108#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
109#[serde(rename_all = "kebab-case", tag = "kind")]
110pub enum Advice {
111 /// Fewer than `min_jobs` jobs: too few to advise on.
112 TooFew {
113 /// How many there are.
114 jobs: u64,
115 },
116 /// The cap fits, and lowering it would save less than a tenth or a run peaked at its cap.
117 Keep {
118 /// What the jobs need.
119 memory: u64,
120 },
121 /// The cap is a tenth or more above what the jobs need, and no run peaked at its cap.
122 Lower {
123 /// The new `memory`.
124 memory: u64,
125 /// The new `slots`: as many as fit in the memory the slots take now when jobs waited for one, else the
126 /// same.
127 slots: u32,
128 },
129 /// The jobs need more than the cap.
130 Raise {
131 /// The new `memory`.
132 memory: u64,
133 /// The new `slots`: the same, or as many as the slice's `MemoryMax` still holds.
134 slots: u32,
135 },
136 /// The jobs need more than the cap, and the slice's `MemoryMax` does not hold one slot of what they need: the
137 /// slice has to grow first.
138 SliceTooSmall {
139 /// What the jobs need.
140 memory: u64,
141 },
142}
143
144impl Advice {
145 /// The `memory` and `slots` to write, when they differ from the profile's.
146 pub fn change(&self) -> Option<Resize> {
147 match *self {
148 Advice::Lower { memory, slots } | Advice::Raise { memory, slots } => {
149 Some(Resize { memory, slots })
150 }
151 Advice::TooFew { .. } | Advice::Keep { .. } | Advice::SliceTooSmall { .. } => None,
152 }
153 }
154}
155
156/// A profile's new `memory` and `slots`.
157#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
158pub struct Resize {
159 /// Bytes.
160 pub memory: u64,
161 /// Slots.
162 pub slots: u32,
163}
164
165impl Sizing {
166 /// One line on what to do with a profile and why, as `runnerctl sizing`, the dashboard and `runnerd doctor`
167 /// say it, such as `lower memory to 2.5G from 7G, its largest peak + 15%; 5 slots fit in the same 14G`.
168 pub fn advice_line(&self, profile: &ProfileSizing) -> String {
169 let size = |bytes: u64| ByteSize(bytes).to_string();
170 let peaks = &profile.peaks;
171 let need = format!("its largest peak + {}%", self.margin);
172 let at_cap = format!(
173 "{} of {} jobs peaked at their cap",
174 peaks.at_cap, peaks.jobs
175 );
176 match profile.advice {
177 Advice::TooFew { jobs } => format!(
178 "{jobs} jobs, too few to advise on (it takes {}); largest peak {} of a {} cap",
179 self.min_jobs,
180 size(peaks.memory.max),
181 size(profile.memory)
182 ),
183 Advice::Keep { .. } if peaks.at_cap > 0 => format!(
184 "keep {}: {at_cap}, so they may need more than they show",
185 size(profile.memory)
186 ),
187 Advice::Keep { memory } => {
188 format!("keep {}: {need} is {}", size(profile.memory), size(memory))
189 }
190 Advice::Lower { memory, slots } => {
191 let share = profile.memory * u64::from(profile.slots);
192 let then = if slots > profile.slots {
193 format!("{slots} slots fit in the same {}", size(share))
194 } else {
195 format!(
196 "frees {} of the budget",
197 size(share - memory * u64::from(slots))
198 )
199 };
200 format!(
201 "lower memory to {} from {}, {need}; {then}",
202 size(memory),
203 size(profile.memory)
204 )
205 }
206 Advice::Raise { memory, slots } => {
207 let why = if peaks.at_cap > 0 {
208 at_cap
209 } else {
210 format!("{need} is above the {} cap", size(profile.memory))
211 };
212 let slots = if slots < profile.slots {
213 format!("; {slots} of its {} slots fit the slice", profile.slots)
214 } else {
215 String::new()
216 };
217 format!("raise memory to {}: {why}{slots}", size(memory))
218 }
219 Advice::SliceTooSmall { memory } => format!(
220 "needs {memory_needed} ({need}), more than the slice holds for one slot: raise the slice's \
221 MemoryMax first",
222 memory_needed = size(memory)
223 ),
224 }
225 }
226}