runnerd_core/
config.rs

1//! `runnerd.toml`: the repositories to serve, the runner profiles they use, the shared caches and the host
2//! budget they must fit in. Builds without the `host` feature, so clients can read a config; its only I/O is
3//! reading the file and asking `systemctl` for a slice's `MemoryMax`. It never talks to Docker or GitHub.
4
5use std::collections::BTreeMap;
6use std::fmt;
7use std::path::{Path, PathBuf};
8use std::process::Command;
9use std::str::FromStr;
10use std::time::Duration;
11
12use serde::{Deserialize, Deserializer, de};
13
14use crate::api;
15
16/// A parsed `runnerd.toml`. `validate` checks it against the host's budget.
17#[derive(Debug, Deserialize)]
18#[serde(deny_unknown_fields)]
19pub struct Config {
20    /// The host's budget and isolation.
21    pub host: Host,
22    /// The caches runners share.
23    pub caches: Caches,
24    /// Runner profiles by name.
25    pub profiles: BTreeMap<String, Profile>,
26    /// The repositories to serve.
27    pub repositories: Vec<Repository>,
28    /// What is kept of past jobs.
29    #[serde(default)]
30    pub history: History,
31}
32
33/// `[host]`: where runner containers run and how they are isolated.
34#[derive(Debug, Deserialize)]
35#[serde(deny_unknown_fields)]
36pub struct Host {
37    /// The systemd slice every runner container is placed in; its `MemoryMax` is the budget.
38    pub slice: String,
39    /// How runner containers are kept from the host.
40    pub isolation: Isolation,
41    /// Must be set for `isolation = "unsafe-dind"`, so a copied config cannot turn it on silently.
42    #[serde(default)]
43    pub allow_unsafe: bool,
44    /// How often GitHub is asked for queued jobs.
45    #[serde(default = "default_poll_interval", deserialize_with = "duration")]
46    pub poll_interval: Duration,
47}
48
49fn default_poll_interval() -> Duration {
50    Duration::from_secs(15)
51}
52
53/// How runner containers are kept from the host: `isolation` under `[host]`.
54#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
55#[serde(rename_all = "kebab-case")]
56pub enum Isolation {
57    /// Sysbox: Docker inside an unprivileged container.
58    Sysbox,
59    /// Privileged Docker-in-Docker: a job can take over the host.
60    UnsafeDind,
61}
62
63/// `[caches]`: what runners share and keep between jobs.
64#[derive(Debug, Clone, Deserialize)]
65#[serde(deny_unknown_fields)]
66pub struct Caches {
67    /// The Docker Hub mirror.
68    pub mirror: Mirror,
69    /// The shared pnpm store.
70    pub pnpm: CacheLimit,
71    /// The shared Maven repository.
72    pub maven: CacheLimit,
73    /// When a profile has no job running and its slot 1 changed its layers by at least this much, slot 1's
74    /// layers become the profile's new snapshot.
75    #[serde(default = "default_reseed_at")]
76    pub reseed_at: ByteSize,
77}
78
79fn default_reseed_at() -> ByteSize {
80    ByteSize(2 << 30)
81}
82
83/// The Docker Hub pull-through mirror shared by all runners.
84#[derive(Debug, Clone, Deserialize)]
85#[serde(deny_unknown_fields)]
86pub struct Mirror {
87    /// Whether runners pull through the mirror.
88    pub enabled: bool,
89    /// The mirror container's memory limit.
90    pub memory: ByteSize,
91    /// The mirror container's CPUs.
92    pub cpus: u32,
93    /// How long content not pulled again is kept.
94    #[serde(deserialize_with = "duration")]
95    pub ttl: Duration,
96}
97
98/// A size limit on a shared cache.
99#[derive(Debug, Clone, Deserialize)]
100#[serde(deny_unknown_fields)]
101pub struct CacheLimit {
102    /// The cache is emptied above this size when no job runs.
103    pub max_size: ByteSize,
104}
105
106/// An image stage with its limits; every job of the profile gets one container with these limits.
107#[derive(Debug, Clone, Deserialize)]
108#[serde(deny_unknown_fields)]
109pub struct Profile {
110    /// The stage of `image/Dockerfile`; runners run the image `runnerd-runner:<image_stage>`.
111    pub image_stage: String,
112    /// CPUs per slot.
113    pub cpus: u32,
114    /// Memory per slot; the container's hard limit.
115    pub memory: ByteSize,
116    /// The most jobs of this profile running at once.
117    pub slots: u32,
118}
119
120/// A repository to serve: `[[repositories]]`.
121#[derive(Debug, Deserialize)]
122#[serde(deny_unknown_fields)]
123pub struct Repository {
124    /// `owner/name` on GitHub.
125    pub name: String,
126    /// The profile its jobs run in.
127    pub profile: String,
128    /// The labels its runners register with; a job runs on them when every label it asks for is one of these.
129    pub labels: Vec<String>,
130}
131
132/// `[history]`: the job history database and how long it keeps what. Every field has a default.
133#[derive(Debug, Clone, PartialEq, Eq, Deserialize)]
134#[serde(deny_unknown_fields, default)]
135pub struct History {
136    /// The SQLite database; default `$XDG_STATE_HOME/runnerd/runnerd.db`.
137    pub database: Option<PathBuf>,
138    /// How long CPU and memory samples are kept; a job's totals and peaks stay. Default 7 days.
139    #[serde(deserialize_with = "duration")]
140    pub samples: Duration,
141    /// How long a finished job's log is kept. Default 30 days.
142    #[serde(deserialize_with = "duration")]
143    pub logs: Duration,
144    /// How long events are kept for clients resuming with `Last-Event-ID`. Default 30 days.
145    #[serde(deserialize_with = "duration")]
146    pub events: Duration,
147    /// The most log stored per job; later lines are dropped. Default 50M.
148    pub log_max: ByteSize,
149}
150
151impl Default for History {
152    fn default() -> History {
153        History {
154            database: None,
155            samples: Duration::from_hours(7 * 24),
156            logs: Duration::from_hours(30 * 24),
157            events: Duration::from_hours(30 * 24),
158            log_max: ByteSize(50 << 20),
159        }
160    }
161}
162
163/// Why a config cannot be used.
164#[derive(Debug, thiserror::Error)]
165pub enum ConfigError {
166    /// The file cannot be read.
167    #[error("cannot read {}: {source}", path.display())]
168    Read {
169        /// The config file.
170        path: PathBuf,
171        /// Why reading it failed.
172        source: std::io::Error,
173    },
174    /// The file is not valid TOML or does not match the config's shape.
175    #[error(transparent)]
176    Parse(#[from] toml::de::Error),
177    /// A repository names a profile `[profiles]` does not define.
178    #[error(
179        "repository {repository} uses profile {profile:?}, which is not defined under [profiles]"
180    )]
181    UnknownProfile {
182        /// The repository's `owner/name`.
183        repository: String,
184        /// The profile it names.
185        profile: String,
186    },
187    /// A repository has no labels, or an empty one.
188    #[error("repository {repository} needs at least one non-empty label")]
189    MissingLabels {
190        /// The repository's `owner/name`.
191        repository: String,
192    },
193    /// `unsafe-dind` without `allow_unsafe = true`.
194    #[error(
195        "isolation \"unsafe-dind\" runs runner containers privileged, so a job can take over the host; \
196         set allow_unsafe = true under [host] to use it anyway"
197    )]
198    UnsafeNotAllowed,
199    /// A limit is 0.
200    #[error("{field} is 0; set it above 0")]
201    ZeroLimit {
202        /// The setting, such as `profiles.app.cpus`.
203        field: String,
204    },
205    /// The caps add up past `u64::MAX` bytes.
206    #[error("slots × memory over all profiles plus the mirror is too large to add up")]
207    BudgetOverflow,
208    /// The caps add up to more than the slice's `MemoryMax`.
209    #[error(
210        "slots × memory over all profiles plus the mirror is {total}, \
211         the slice {slice}'s MemoryMax is {memory_max}"
212    )]
213    OverBudget {
214        /// What every runner and the mirror may use at once.
215        total: ByteSize,
216        /// The slice runners run in.
217        slice: String,
218        /// The slice's `MemoryMax`.
219        memory_max: ByteSize,
220    },
221}
222
223/// What a valid config should still fix.
224#[derive(Debug, Clone, PartialEq, Eq)]
225pub enum Warning {
226    /// The slice has no unit file, so systemd creates it without limits.
227    UnlimitedSlice {
228        /// The slice runners run in.
229        slice: String,
230        /// What every runner and the mirror may use at once.
231        total: ByteSize,
232    },
233}
234
235impl fmt::Display for Warning {
236    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
237        match self {
238            Warning::UnlimitedSlice { slice, total } => write!(
239                f,
240                "the slice {slice} has no MemoryMax, so nothing holds runners to their {total} of caps; \
241                 install a unit file for it"
242            ),
243        }
244    }
245}
246
247impl Config {
248    /// Reads and parses a config file.
249    pub fn load(path: &Path) -> Result<Config, ConfigError> {
250        let text = std::fs::read_to_string(path).map_err(|source| ConfigError::Read {
251            path: path.to_owned(),
252            source,
253        })?;
254        Config::parse(&text)
255    }
256
257    /// Parses a config from TOML text.
258    pub fn parse(text: &str) -> Result<Config, ConfigError> {
259        Ok(toml::from_str(text)?)
260    }
261
262    /// Parses `text` and validates it against the `MemoryMax` that `memory_max` reads for the slice it names:
263    /// what `runnerd check-config` prints and `POST /v1/config/check` answers. Returns the config if it is valid.
264    pub fn check(
265        text: &str,
266        memory_max: impl FnOnce(&str) -> std::io::Result<Option<ByteSize>>,
267    ) -> (Option<Config>, api::ConfigCheck) {
268        let invalid = |message: String, memory_caps: Option<ByteSize>, max: Option<ByteSize>| {
269            api::ConfigCheck {
270                valid: false,
271                message,
272                warnings: Vec::new(),
273                memory_caps: memory_caps.map(|caps| caps.0),
274                memory_max: max.map(|max| max.0),
275            }
276        };
277        let config = match Config::parse(text) {
278            Ok(config) => config,
279            Err(error) => return (None, invalid(error.to_string(), None, None)),
280        };
281        let caps = config.memory_caps().ok();
282        let max = match memory_max(&config.host.slice) {
283            Ok(max) => max,
284            Err(error) => return (None, invalid(error.to_string(), caps, None)),
285        };
286        match config
287            .validate(max)
288            .and_then(|warnings| Ok((warnings, config.memory_caps()?)))
289        {
290            Ok((warnings, total)) => {
291                let check = api::ConfigCheck {
292                    valid: true,
293                    message: format!("valid, memory caps sum to {total}"),
294                    warnings: warnings.iter().map(ToString::to_string).collect(),
295                    memory_caps: Some(total.0),
296                    memory_max: max.map(|max| max.0),
297                };
298                (Some(config), check)
299            }
300            Err(error) => (None, invalid(error.to_string(), caps, max)),
301        }
302    }
303
304    /// The memory every runner and the mirror may use at once.
305    pub fn memory_caps(&self) -> Result<ByteSize, ConfigError> {
306        let mirror = if self.caches.mirror.enabled {
307            self.caches.mirror.memory.0
308        } else {
309            0
310        };
311        self.profiles
312            .values()
313            .try_fold(mirror, |total, p| {
314                p.memory
315                    .0
316                    .checked_mul(u64::from(p.slots))?
317                    .checked_add(total)
318            })
319            .map(ByteSize)
320            .ok_or(ConfigError::BudgetOverflow)
321    }
322
323    /// Checks the config against the slice's `MemoryMax` (`None` when the slice is unlimited).
324    pub fn validate(&self, memory_max: Option<ByteSize>) -> Result<Vec<Warning>, ConfigError> {
325        if self.host.isolation == Isolation::UnsafeDind && !self.host.allow_unsafe {
326            return Err(ConfigError::UnsafeNotAllowed);
327        }
328        let zero_limits = self.profiles.iter().flat_map(|(name, p)| {
329            [
330                (p.memory.0 == 0, format!("profiles.{name}.memory")),
331                (p.cpus == 0, format!("profiles.{name}.cpus")),
332                (p.slots == 0, format!("profiles.{name}.slots")),
333            ]
334        });
335        let mirror = &self.caches.mirror;
336        let mirror_zero_limits = [
337            (
338                mirror.enabled && mirror.memory.0 == 0,
339                "caches.mirror.memory".to_owned(),
340            ),
341            (
342                mirror.enabled && mirror.cpus == 0,
343                "caches.mirror.cpus".to_owned(),
344            ),
345            (
346                self.host.poll_interval.is_zero(),
347                "host.poll_interval".to_owned(),
348            ),
349        ];
350        if let Some((_, field)) = zero_limits
351            .chain(mirror_zero_limits)
352            .find(|(zero, _)| *zero)
353        {
354            return Err(ConfigError::ZeroLimit { field });
355        }
356        for repository in &self.repositories {
357            if !self.profiles.contains_key(&repository.profile) {
358                return Err(ConfigError::UnknownProfile {
359                    repository: repository.name.clone(),
360                    profile: repository.profile.clone(),
361                });
362            }
363            if repository.labels.is_empty() || repository.labels.iter().any(|l| l.trim().is_empty())
364            {
365                return Err(ConfigError::MissingLabels {
366                    repository: repository.name.clone(),
367                });
368            }
369        }
370        let total = self.memory_caps()?;
371        match memory_max {
372            Some(memory_max) if total > memory_max => Err(ConfigError::OverBudget {
373                total,
374                slice: self.host.slice.clone(),
375                memory_max,
376            }),
377            Some(_) => Ok(Vec::new()),
378            None => Ok(vec![Warning::UnlimitedSlice {
379                slice: self.host.slice.clone(),
380                total,
381            }]),
382        }
383    }
384}
385
386/// Reads a slice's `MemoryMax` from systemd; `None` when it is unlimited.
387pub fn slice_memory_max(slice: &str) -> std::io::Result<Option<ByteSize>> {
388    let output = Command::new("systemctl")
389        .args(["show", slice, "--property=MemoryMax", "--value"])
390        .output()?;
391    if !output.status.success() {
392        return Err(std::io::Error::other(format!(
393            "systemctl show {slice} failed: {}",
394            String::from_utf8_lossy(&output.stderr).trim()
395        )));
396    }
397    let value = String::from_utf8_lossy(&output.stdout);
398    parse_memory_max(value.trim()).ok_or_else(|| {
399        std::io::Error::other(format!(
400            "systemctl show {slice}: unexpected MemoryMax {value:?}"
401        ))
402    })
403}
404
405/// Parses a `MemoryMax` value as `systemctl show` prints it: bytes, or `infinity`.
406pub fn parse_memory_max(value: &str) -> Option<Option<ByteSize>> {
407    if value == "infinity" {
408        Some(None)
409    } else {
410        value.parse().ok().map(|bytes| Some(ByteSize(bytes)))
411    }
412}
413
414/// A size in bytes, written with a binary unit as Docker and systemd do: `512M`, `9G`, `32.5G`.
415#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
416pub struct ByteSize(pub u64);
417
418const UNITS: [(char, u64); 4] = [
419    ('T', 1 << 40),
420    ('G', 1 << 30),
421    ('M', 1 << 20),
422    ('K', 1 << 10),
423];
424
425impl FromStr for ByteSize {
426    type Err = String;
427
428    #[allow(
429        clippy::cast_precision_loss,
430        clippy::cast_possible_truncation,
431        clippy::cast_sign_loss,
432        reason = "sizes in a config are far below 2^52 bytes, and the result is checked against u64's range first"
433    )]
434    fn from_str(s: &str) -> Result<Self, Self::Err> {
435        let invalid = || {
436            format!(
437                "invalid size {s:?}, expected a number and a unit (K, M, G or T), such as \"9G\""
438            )
439        };
440        let unit = s.chars().last().ok_or_else(invalid)?.to_ascii_uppercase();
441        let (_, factor) = UNITS.iter().find(|(u, _)| *u == unit).ok_or_else(invalid)?;
442        let number = &s[..s.len() - 1];
443        if number.is_empty() || !number.chars().all(|c| c.is_ascii_digit() || c == '.') {
444            return Err(invalid());
445        }
446        let number: f64 = number.parse().map_err(|_| invalid())?;
447        let bytes = (number * *factor as f64).round();
448        // `u64::MAX as f64` is 2^64, the first value out of range.
449        if bytes >= u64::MAX as f64 {
450            return Err(format!("size {s:?} is too large"));
451        }
452        Ok(ByteSize(bytes as u64))
453    }
454}
455
456impl fmt::Display for ByteSize {
457    #[allow(
458        clippy::cast_precision_loss,
459        reason = "shown with two decimals, far coarser than f64's precision"
460    )]
461    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
462        let Some((unit, factor)) = UNITS.iter().find(|(_, factor)| self.0 >= *factor) else {
463            return write!(f, "{}B", self.0);
464        };
465        let number = format!("{:.2}", self.0 as f64 / *factor as f64);
466        let number = number.trim_end_matches('0').trim_end_matches('.');
467        write!(f, "{number}{unit}")
468    }
469}
470
471impl<'de> Deserialize<'de> for ByteSize {
472    fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
473        String::deserialize(deserializer)?
474            .parse()
475            .map_err(de::Error::custom)
476    }
477}
478
479/// A duration as Go and the registry write it: `30s`, `90m`, `168h`.
480fn duration<'de, D: Deserializer<'de>>(deserializer: D) -> Result<Duration, D::Error> {
481    let s = String::deserialize(deserializer)?;
482    let invalid = || {
483        de::Error::custom(format!(
484            "invalid duration {s:?}, expected a number and s, m or h, such as \"168h\""
485        ))
486    };
487    let (number, factor) = match s.chars().last() {
488        Some('s') => (&s[..s.len() - 1], 1),
489        Some('m') => (&s[..s.len() - 1], 60),
490        Some('h') => (&s[..s.len() - 1], 3600),
491        _ => return Err(invalid()),
492    };
493    let number: u64 = number.parse().map_err(|_| invalid())?;
494    number
495        .checked_mul(factor)
496        .map(Duration::from_secs)
497        .ok_or_else(invalid)
498}
499
500#[cfg(test)]
501mod tests;
runnerd_core/config.rs

runnerd_core/
config.rs
