use std::collections::BTreeMap;
use std::fmt;
use std::net::{Ipv4Addr, Ipv6Addr};
use std::path::{Path, PathBuf};
use std::process::Command;
use std::str::FromStr;
use std::time::Duration;
use serde::{Deserialize, Deserializer, de};
use crate::api;
mod budget;
pub use budget::{BudgetBlock, CheckBudget, Remainder, Verdict};
#[derive(Debug, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Config {
pub host: Host,
pub caches: Caches,
pub profiles: BTreeMap<String, Profile>,
pub repositories: Vec<Repository>,
#[serde(default)]
pub history: History,
#[serde(default)]
pub records: Records,
#[serde(default)]
pub sizing: Sizing,
#[serde(default)]
pub web: Web,
#[serde(default)]
pub notify: Notify,
}
#[derive(Debug, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Host {
pub slice: String,
pub isolation: Isolation,
#[serde(default)]
pub allow_unsafe: bool,
#[serde(default = "default_poll_interval", deserialize_with = "duration")]
pub poll_interval: Duration,
#[serde(default)]
pub admission: Admission,
#[serde(default)]
pub cpu_overcommit: Option<f64>,
}
impl Host {
fn check_cpu_overcommit(&self) -> Result<(), ConfigError> {
match self.cpu_overcommit {
Some(value) if !(value > 0.0 && value <= MAX_CPU_OVERCOMMIT) => {
Err(ConfigError::CpuOvercommit { value })
}
_ => Ok(()),
}
}
}
pub const MAX_CPU_OVERCOMMIT: f64 = 16.0;
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum Admission {
#[default]
Caps,
Measured,
}
impl Admission {
pub fn policy(self) -> api::AdmissionPolicy {
match self {
Admission::Caps => api::AdmissionPolicy::Caps,
Admission::Measured => api::AdmissionPolicy::Measured,
}
}
}
fn default_poll_interval() -> Duration {
Duration::from_secs(15)
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum Isolation {
Sysbox,
UnsafeDind,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Caches {
pub mirror: Mirror,
pub pnpm: CacheLimit,
pub maven: CacheLimit,
#[serde(default = "default_reseed_at")]
pub reseed_at: ByteSize,
#[serde(default = "default_snapshot_max")]
pub snapshot_max: ByteSize,
#[serde(default)]
pub build: BuildCache,
}
#[derive(Debug, Clone, PartialEq, Eq, Deserialize)]
#[serde(deny_unknown_fields, default)]
pub struct BuildCache {
pub enabled: bool,
pub max_size: ByteSize,
pub branches: Vec<String>,
pub directory: Option<PathBuf>,
}
impl BuildCache {
fn validate(&self) -> Result<(), ConfigError> {
if self.enabled && self.max_size.0 == 0 {
return Err(ConfigError::ZeroLimit {
field: "caches.build.max_size".to_owned(),
});
}
Ok(())
}
}
impl Default for BuildCache {
fn default() -> BuildCache {
BuildCache {
enabled: true,
max_size: ByteSize(10 << 30),
branches: Vec::new(),
directory: None,
}
}
}
fn default_reseed_at() -> ByteSize {
ByteSize(2 << 30)
}
fn default_snapshot_max() -> ByteSize {
ByteSize(10 << 30)
}
fn default_mirror_max_size() -> ByteSize {
ByteSize(10 << 30)
}
#[derive(Debug, Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Mirror {
pub enabled: bool,
pub memory: ByteSize,
pub cpus: u32,
#[serde(deserialize_with = "duration")]
pub ttl: Duration,
#[serde(default = "default_mirror_max_size")]
pub max_size: ByteSize,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct CacheLimit {
pub max_size: ByteSize,
}
pub const PUBLISHED_RUNNER: &str = "ghcr.io/ayagmar/runnerd-runner";
#[derive(Debug, Clone, Deserialize)]
#[serde(try_from = "ProfileToml")]
pub struct Profile {
pub image: Image,
pub cpus: u32,
pub memory: ByteSize,
pub slots: u32,
pub record_failures: bool,
pub min_memory: Option<ByteSize>,
pub max_memory: Option<ByteSize>,
pub max_slots: Option<u32>,
pub priced_as: Option<api::Sku>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Image {
Stage(String),
Pulled(String),
}
impl Image {
pub fn reference(&self) -> String {
match self {
Image::Stage(stage) => format!("runnerd-runner:{stage}"),
Image::Pulled(reference) => reference.clone(),
}
}
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct ProfileToml {
image_stage: Option<String>,
image: Option<String>,
cpus: u32,
memory: ByteSize,
slots: u32,
#[serde(default)]
record_failures: bool,
min_memory: Option<ByteSize>,
max_memory: Option<ByteSize>,
max_slots: Option<u32>,
priced_as: Option<api::Sku>,
}
impl TryFrom<ProfileToml> for Profile {
type Error = String;
fn try_from(profile: ProfileToml) -> Result<Profile, String> {
let image = match (profile.image_stage, profile.image) {
(Some(stage), None) if !stage.is_empty() => Image::Stage(stage),
(None, Some(reference)) if !reference.is_empty() => Image::Pulled(reference),
(Some(_), Some(_)) => {
return Err(
"set image_stage (a stage built from image/Dockerfile) or image (a reference to pull), \
not both"
.to_owned(),
);
}
_ => {
return Err(
"set image_stage (a stage built from image/Dockerfile) or image (a reference to pull)"
.to_owned(),
);
}
};
Ok(Profile {
image,
cpus: profile.cpus,
memory: profile.memory,
slots: profile.slots,
record_failures: profile.record_failures,
min_memory: profile.min_memory,
max_memory: profile.max_memory,
max_slots: profile.max_slots,
priced_as: profile.priced_as,
})
}
}
pub fn labels_match(wanted: &[String], labels: &[String]) -> bool {
!wanted.is_empty()
&& wanted.iter().all(|wanted| {
labels
.iter()
.any(|label| label.eq_ignore_ascii_case(wanted))
})
}
#[derive(Debug, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Repository {
pub name: String,
pub profile: String,
pub labels: Vec<String>,
#[serde(default = "default_warm")]
pub warm: u32,
#[serde(default = "default_true")]
pub build_cache: bool,
#[serde(default)]
pub build_cache_writers: BuildCacheWriters,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum BuildCacheWriters {
#[default]
DefaultBranch,
PullRequests,
}
fn default_true() -> bool {
true
}
fn default_warm() -> u32 {
1
}
#[derive(Debug, Clone, PartialEq, Eq, Deserialize)]
#[serde(deny_unknown_fields, default)]
pub struct History {
pub database: Option<PathBuf>,
#[serde(deserialize_with = "duration")]
pub samples: Duration,
#[serde(deserialize_with = "duration")]
pub logs: Duration,
#[serde(deserialize_with = "duration")]
pub events: Duration,
#[serde(deserialize_with = "duration")]
pub jobs: Duration,
pub log_max: ByteSize,
}
impl Default for History {
fn default() -> History {
History {
database: None,
samples: Duration::from_hours(7 * 24),
logs: Duration::from_hours(30 * 24),
events: Duration::from_hours(30 * 24),
jobs: Duration::from_hours(90 * 24),
log_max: ByteSize(50 << 20),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Deserialize)]
#[serde(deny_unknown_fields, default)]
pub struct Records {
pub max_total: ByteSize,
pub max_count: u32,
#[serde(deserialize_with = "duration")]
pub max_age: Duration,
pub min_free: ByteSize,
}
impl Default for Records {
fn default() -> Records {
Records {
max_total: ByteSize(20 << 30),
max_count: 10,
max_age: Duration::from_hours(7 * 24),
min_free: ByteSize(10 << 30),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Deserialize)]
#[serde(deny_unknown_fields, default)]
pub struct Sizing {
pub margin: u32,
pub min_jobs: u32,
pub auto: bool,
#[serde(deserialize_with = "duration")]
pub interval: Duration,
#[serde(deserialize_with = "duration")]
pub raise_interval: Duration,
pub max_step: u32,
}
impl Sizing {
pub fn raise_after(&self) -> Duration {
self.raise_interval.min(self.interval)
}
}
impl Default for Sizing {
fn default() -> Sizing {
Sizing {
margin: 15,
min_jobs: 10,
auto: false,
interval: Duration::from_hours(1),
raise_interval: Duration::from_hours(1),
max_step: 50,
}
}
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Deserialize)]
#[serde(deny_unknown_fields, default)]
pub struct Web {
pub origins: Vec<Origin>,
pub open: Option<Origin>,
}
impl Web {
fn validate(&self) -> Result<(), ConfigError> {
match &self.open {
Some(open) if !self.origins.contains(open) => {
Err(ConfigError::OpenNotListed { open: open.clone() })
}
_ => Ok(()),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Deserialize)]
#[serde(deny_unknown_fields, default)]
pub struct Notify {
pub digest_at: DayTime,
pub targets: Vec<NotifyTarget>,
}
impl Default for Notify {
fn default() -> Notify {
Notify {
digest_at: DayTime { minutes: 9 * 60 },
targets: Vec::new(),
}
}
}
impl Notify {
fn validate(&self) -> Result<(), ConfigError> {
let refuse = |target: &str, why: &'static str| {
Err(ConfigError::Notify {
target: target.to_owned(),
why,
})
};
for (index, target) in self.targets.iter().enumerate() {
let name = target.name.as_str();
if name.trim().is_empty() || name.chars().any(char::is_control) {
return refuse(name, "name is empty");
}
if self.targets[..index].iter().any(|other| other.name == name) {
return refuse(name, "another target has this name");
}
let url = target.url.expose();
if url == SECRET_SHOWN {
return refuse(
name,
"url is <secret>, which runnerd fills in only for a target of the same name that had one; write the url",
);
}
let Some(rest) = url
.strip_prefix("https://")
.or_else(|| url.strip_prefix("http://"))
else {
return refuse(name, "url does not start with https:// or http://");
};
let (host, path) = rest.split_once('/').unwrap_or((rest, ""));
if host.is_empty() || url.chars().any(|c| c.is_whitespace() || c.is_control()) {
return refuse(name, "url is not a URL");
}
if target.kind == NotifyKind::Ntfy
&& (path.is_empty() || path.contains(['/', '?', '#']))
{
return refuse(
name,
"url is not an ntfy topic's, such as https://ntfy.sh/<topic>",
);
}
for (header, value) in &target.headers {
if value.expose() == SECRET_SHOWN {
return refuse(
name,
"a header is <secret>, which runnerd fills in only for a target of the same name that had it; write its value",
);
}
let token = |c: char| c.is_ascii_alphanumeric() || "!#$%&'*+-.^_`|~".contains(c);
if header.is_empty() || !header.chars().all(token) {
return refuse(name, "a header's name is not a header name");
}
if value.expose().chars().any(char::is_control) {
return refuse(name, "a header's value has a control character");
}
}
if target.events.contains(&api::NotifyEvent::Test) {
return refuse(
name,
"events lists test, which only runnerctl notify test sends",
);
}
}
Ok(())
}
}
#[derive(Debug, Clone, PartialEq, Eq, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct NotifyTarget {
pub name: String,
pub kind: NotifyKind,
pub url: Secret,
#[serde(default)]
pub headers: BTreeMap<String, Secret>,
#[serde(default = "default_notify_events")]
pub events: Vec<api::NotifyEvent>,
}
fn default_notify_events() -> Vec<api::NotifyEvent> {
vec![
api::NotifyEvent::Failed,
api::NotifyEvent::Lost,
api::NotifyEvent::Runner,
api::NotifyEvent::Insights,
]
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum NotifyKind {
Webhook,
Ntfy,
Discord,
}
#[derive(Clone, PartialEq, Eq, Deserialize)]
#[serde(transparent)]
pub struct Secret(String);
impl Secret {
pub fn expose(&self) -> &str {
&self.0
}
}
impl From<&str> for Secret {
fn from(value: &str) -> Secret {
Secret(value.to_owned())
}
}
impl fmt::Debug for Secret {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str("<secret>")
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct DayTime {
pub minutes: u32,
}
impl FromStr for DayTime {
type Err = String;
fn from_str(text: &str) -> Result<DayTime, String> {
let invalid = || format!("invalid time of day {text:?}, expected HH:MM such as \"09:00\"");
let (hour, minute) = text.split_once(':').ok_or_else(invalid)?;
let digits = |part: &str| part.len() == 2 && part.chars().all(|c| c.is_ascii_digit());
if !digits(hour) || !digits(minute) {
return Err(invalid());
}
let (hour, minute): (u32, u32) = (
hour.parse().map_err(|_| invalid())?,
minute.parse().map_err(|_| invalid())?,
);
if hour > 23 || minute > 59 {
return Err(invalid());
}
Ok(DayTime {
minutes: hour * 60 + minute,
})
}
}
impl fmt::Display for DayTime {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{:02}:{:02}", self.minutes / 60, self.minutes % 60)
}
}
impl<'de> Deserialize<'de> for DayTime {
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
String::deserialize(deserializer)?
.parse()
.map_err(de::Error::custom)
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Origin(String);
impl Origin {
pub fn as_str(&self) -> &str {
&self.0
}
}
impl FromStr for Origin {
type Err = String;
fn from_str(s: &str) -> Result<Origin, String> {
let invalid = |why: &str| {
format!(
"invalid origin {s:?}: {why}; write the scheme, host and port only, such as \
\"https://host.example.ts.net:8443\""
)
};
let (scheme, rest) = s
.split_once("://")
.ok_or_else(|| invalid("it has no scheme"))?;
let scheme = scheme.to_ascii_lowercase();
if rest.contains(['/', '?', '#']) {
return Err(invalid("it has a path"));
}
if rest.contains('@') {
return Err(invalid("it names a user"));
}
let (host, port) = match rest.strip_prefix('[') {
Some(v6) => {
let (address, after) = v6
.split_once(']')
.ok_or_else(|| invalid("its IPv6 address has no closing ]"))?;
let port = match after {
"" => None,
_ => Some(after.strip_prefix(':').ok_or_else(|| invalid("its port"))?),
};
(format!("[{address}]"), port)
}
None => match rest.split_once(':') {
Some((host, port)) => (host.to_owned(), Some(port)),
None => (rest.to_owned(), None),
},
};
let host = host.to_ascii_lowercase();
let not_a_host = || invalid("its host is not a name or an address");
let (host, loopback) =
if let Some(v6) = host.strip_prefix('[').and_then(|v6| v6.strip_suffix(']')) {
let address: Ipv6Addr = v6.parse().map_err(|_| not_a_host())?;
(format!("[{}]", ipv6_origin(address)), address.is_loopback())
} else {
if host.is_empty()
|| !host
.chars()
.all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '.')
{
return Err(not_a_host());
}
let name = host.strip_suffix('.').unwrap_or(&host);
let last = name.rsplit('.').next().unwrap_or_default();
let numeric = !last.is_empty()
&& (last.bytes().all(|byte| byte.is_ascii_digit())
|| last
.strip_prefix("0x")
.is_some_and(|hex| hex.bytes().all(|byte| byte.is_ascii_hexdigit())));
if numeric {
let address: Ipv4Addr = name.parse().map_err(|_| {
invalid("its IPv4 address is not four decimal numbers, such as 192.0.2.1")
})?;
(address.to_string(), address.is_loopback())
} else {
let loopback = host == "localhost";
(host, loopback)
}
};
let port = port
.map(|port| {
Some(port)
.filter(|port| port.bytes().all(|byte| byte.is_ascii_digit()))
.and_then(|port| port.parse::<u16>().ok())
.filter(|port| *port != 0)
.ok_or_else(|| invalid("its port is not a number from 1 to 65535"))
})
.transpose()?;
let default_port = match scheme.as_str() {
"https" => 443,
"http" if loopback => 80,
"http" => {
return Err(invalid(
"http is for a loopback host only, since the session cookie would cross the network in \
the clear; use https",
));
}
_ => return Err(invalid("its scheme is not https")),
};
Ok(Origin(match port {
Some(port) if port != default_port => format!("{scheme}://{host}:{port}"),
_ => format!("{scheme}://{host}"),
}))
}
}
fn ipv6_origin(address: Ipv6Addr) -> String {
let pieces = address.segments();
let (mut zeros_at, mut zeros) = (0, 0);
let mut at = 0;
while at < pieces.len() {
let run = pieces[at..].iter().take_while(|piece| **piece == 0).count();
if run > zeros {
(zeros_at, zeros) = (at, run);
}
at += run.max(1);
}
let hex = |pieces: &[u16]| {
pieces
.iter()
.map(|piece| format!("{piece:x}"))
.collect::<Vec<_>>()
.join(":")
};
if zeros < 2 {
return hex(&pieces);
}
format!(
"{}::{}",
hex(&pieces[..zeros_at]),
hex(&pieces[zeros_at + zeros..])
)
}
impl fmt::Display for Origin {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(&self.0)
}
}
impl<'de> Deserialize<'de> for Origin {
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
String::deserialize(deserializer)?
.parse()
.map_err(de::Error::custom)
}
}
#[derive(Debug, thiserror::Error)]
pub enum ConfigError {
#[error("cannot read {}: {source}", path.display())]
Read {
path: PathBuf,
source: std::io::Error,
},
#[error(transparent)]
Parse(#[from] toml::de::Error),
#[error(
"repository {repository} uses profile {profile:?}, which is not defined under [profiles]; {}",
defined_profiles(.defined)
)]
UnknownProfile {
repository: String,
profile: String,
defined: Vec<String>,
},
#[error("repository {repository} needs at least one non-empty label")]
MissingLabels {
repository: String,
},
#[error(
"repository {repository} has two entries with profile {profile:?}; give each of its pools its own profile"
)]
PoolProfileTwice {
repository: String,
profile: String,
},
#[error(
"repository {repository}'s pools in {first} and {second} both have the label {label:?}, so a job asking for \
it could land in either; give each pool labels of its own, such as only {second:?}'s own label for the \
lighter one"
)]
PoolLabelsOverlap {
repository: String,
first: String,
second: String,
label: String,
},
#[error(
"repository {repository}'s entries set build_cache_writers differently; whose runs may write to the build \
cache is the repository's, so set the same in each"
)]
PoolWritersDiffer {
repository: String,
},
#[error(
"isolation \"unsafe-dind\" runs runner containers privileged, so a job can take over the host; \
set allow_unsafe = true under [host] to use it anyway"
)]
UnsafeNotAllowed,
#[error("{field} is 0; set it above 0")]
ZeroLimit {
field: String,
},
#[error(
"host.cpu_overcommit is {value}; set it above 0 and at most {MAX_CPU_OVERCOMMIT}, such as 1.0"
)]
CpuOvercommit {
value: f64,
},
#[error(
"web.open is {open}, which is not one of [web] origins; add it to origins, or remove open"
)]
OpenNotListed {
open: Origin,
},
#[error("profiles.{profile}: {why}")]
SizingBounds {
profile: String,
why: &'static str,
},
#[error("notify target {target:?}: {why}")]
Notify {
target: String,
why: &'static str,
},
#[error("slots × memory over all profiles plus the mirror is too large to add up")]
BudgetOverflow,
#[error(
"admission = \"measured\" admits runners against the slice {slice}'s MemoryMax, and it has none; install \
a unit file for it (`runnerd doctor` prints one)"
)]
MeasuredWithoutMemoryMax {
slice: String,
},
#[error(
"one runner of {profile} and the mirror take {needs}, the slice {slice}'s MemoryMax is {memory_max}; lower \
its `memory` or raise the slice's MemoryMax"
)]
ProfileOverBudget {
profile: String,
needs: ByteSize,
slice: String,
memory_max: ByteSize,
},
#[error(
"slots × memory over all profiles plus the mirror is {total}, \
the slice {slice}'s MemoryMax is {memory_max}; lower `slots` or `memory`, or raise the slice's \
MemoryMax (`runnerd doctor` prints its unit)"
)]
OverBudget {
total: ByteSize,
slice: String,
memory_max: ByteSize,
},
}
fn defined_profiles(defined: &[String]) -> String {
if defined.is_empty() {
"it defines none".to_owned()
} else {
format!("defined: {}", defined.join(", "))
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Warning {
UnlimitedSlice {
slice: String,
total: ByteSize,
},
}
impl fmt::Display for Warning {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Warning::UnlimitedSlice { slice, total } => write!(
f,
"the slice {slice} has no MemoryMax, so nothing holds runners to their {total} of caps; \
install a unit file for it"
),
}
}
}
impl Config {
pub fn load(path: &Path) -> Result<Config, ConfigError> {
let text = std::fs::read_to_string(path).map_err(|source| ConfigError::Read {
path: path.to_owned(),
source,
})?;
Config::parse(&text)
}
pub fn parse(text: &str) -> Result<Config, ConfigError> {
Ok(toml::from_str(text)?)
}
pub fn check(
text: &str,
memory_max: impl FnOnce(&str) -> std::io::Result<Option<ByteSize>>,
) -> (Option<Config>, api::ConfigCheck) {
let config = match Config::parse(text) {
Ok(config) => config,
Err(error) => return (None, budget::invalid(error.to_string(), None, None)),
};
let max = match memory_max(&config.host.slice) {
Ok(max) => max,
Err(error) => {
return (
None,
budget::invalid(error.to_string(), Some(&config), None),
);
}
};
let check = config.check_against(max);
if check.valid {
(Some(config), check)
} else {
(None, check)
}
}
pub fn mirror_memory(&self) -> u64 {
if self.caches.mirror.enabled {
self.caches.mirror.memory.0
} else {
0
}
}
pub fn memory_caps(&self) -> Result<ByteSize, ConfigError> {
let mirror = self.mirror_memory();
self.profiles
.values()
.try_fold(mirror, |total, p| {
p.memory
.0
.checked_mul(u64::from(p.slots))?
.checked_add(total)
})
.map(ByteSize)
.ok_or(ConfigError::BudgetOverflow)
}
pub fn memory_needed(&self) -> Result<ByteSize, ConfigError> {
match self.host.admission {
Admission::Caps => self.memory_caps(),
Admission::Measured => {
let largest = self
.profiles
.values()
.map(|p| p.memory.0)
.max()
.unwrap_or(0);
largest
.checked_add(self.mirror_memory())
.map(ByteSize)
.ok_or(ConfigError::BudgetOverflow)
}
}
}
fn check_sizing_bounds(&self) -> Result<(), ConfigError> {
for (name, p) in &self.profiles {
let memory = p.memory.0;
let field = if p
.min_memory
.zip(p.max_memory)
.is_some_and(|(min, max)| min.0 > max.0)
{
"min_memory is above its max_memory"
} else if p.min_memory.is_some_and(|min| memory < min.0) {
"memory is below its min_memory"
} else if p.max_memory.is_some_and(|max| memory > max.0) {
"memory is above its max_memory"
} else if p.max_slots.is_some_and(|max| p.slots > max) {
"slots are above its max_slots"
} else {
continue;
};
return Err(ConfigError::SizingBounds {
profile: name.clone(),
why: field,
});
}
Ok(())
}
fn check_pools(&self) -> Result<(), ConfigError> {
for (at, first) in self.repositories.iter().enumerate() {
for second in self.repositories[at + 1..]
.iter()
.filter(|second| second.name == first.name)
{
if second.profile == first.profile {
return Err(ConfigError::PoolProfileTwice {
repository: first.name.clone(),
profile: first.profile.clone(),
});
}
if let Some(label) = first.labels.iter().find(|label| {
second
.labels
.iter()
.any(|other| other.eq_ignore_ascii_case(label))
}) {
return Err(ConfigError::PoolLabelsOverlap {
repository: first.name.clone(),
first: first.profile.clone(),
second: second.profile.clone(),
label: label.clone(),
});
}
if second.build_cache_writers != first.build_cache_writers {
return Err(ConfigError::PoolWritersDiffer {
repository: first.name.clone(),
});
}
}
}
Ok(())
}
pub fn pool(&self, repository: &str, labels: &[String]) -> Option<&Repository> {
self.repositories
.iter()
.find(|pool| pool.name == repository && labels_match(labels, &pool.labels))
}
pub fn pool_in(&self, repository: &str, profile: &str) -> Option<&Repository> {
self.repositories
.iter()
.find(|pool| pool.name == repository && pool.profile == profile)
}
pub fn pricing(&self) -> api::Pricing {
api::Pricing {
profiles: self
.profiles
.iter()
.filter_map(|(name, profile)| Some((name.clone(), profile.priced_as?)))
.collect(),
default_sku: api::Sku::standard_for(std::env::consts::ARCH),
}
}
pub fn repository_names(&self) -> Vec<String> {
let mut names: Vec<String> = Vec::new();
for repository in &self.repositories {
if !names.contains(&repository.name) {
names.push(repository.name.clone());
}
}
names
}
pub fn validate(&self, memory_max: Option<ByteSize>) -> Result<Vec<Warning>, ConfigError> {
if self.host.isolation == Isolation::UnsafeDind && !self.host.allow_unsafe {
return Err(ConfigError::UnsafeNotAllowed);
}
let zero_limits = self.profiles.iter().flat_map(|(name, p)| {
[
(p.memory.0 == 0, format!("profiles.{name}.memory")),
(p.cpus == 0, format!("profiles.{name}.cpus")),
(p.slots == 0, format!("profiles.{name}.slots")),
(p.max_slots == Some(0), format!("profiles.{name}.max_slots")),
]
});
let mirror = &self.caches.mirror;
let mirror_zero_limits = [
(
mirror.enabled && mirror.memory.0 == 0,
"caches.mirror.memory".to_owned(),
),
(
mirror.enabled && mirror.cpus == 0,
"caches.mirror.cpus".to_owned(),
),
(
self.host.poll_interval.is_zero(),
"host.poll_interval".to_owned(),
),
(self.records.max_count == 0, "records.max_count".to_owned()),
(self.sizing.min_jobs == 0, "sizing.min_jobs".to_owned()),
(self.sizing.interval.is_zero(), "sizing.interval".to_owned()),
(
self.sizing.raise_interval.is_zero(),
"sizing.raise_interval".to_owned(),
),
(self.sizing.max_step == 0, "sizing.max_step".to_owned()),
(
self.records.max_total.0 == 0,
"records.max_total".to_owned(),
),
];
if let Some((_, field)) = zero_limits
.chain(mirror_zero_limits)
.find(|(zero, _)| *zero)
{
return Err(ConfigError::ZeroLimit { field });
}
self.check_sizing_bounds()?;
self.caches.build.validate()?;
self.host.check_cpu_overcommit()?;
self.web.validate()?;
self.notify.validate()?;
for repository in &self.repositories {
if !self.profiles.contains_key(&repository.profile) {
return Err(ConfigError::UnknownProfile {
repository: repository.name.clone(),
profile: repository.profile.clone(),
defined: self.profiles.keys().cloned().collect(),
});
}
if repository.labels.is_empty() || repository.labels.iter().any(|l| l.trim().is_empty())
{
return Err(ConfigError::MissingLabels {
repository: repository.name.clone(),
});
}
}
self.check_pools()?;
self.check_budget(memory_max)
}
fn check_budget(&self, memory_max: Option<ByteSize>) -> Result<Vec<Warning>, ConfigError> {
let total = self.memory_caps()?;
if self.host.admission == Admission::Measured {
let Some(memory_max) = memory_max else {
return Err(ConfigError::MeasuredWithoutMemoryMax {
slice: self.host.slice.clone(),
});
};
let mirror = self.mirror_memory();
if let Some((profile, needs)) = self
.profiles
.iter()
.map(|(name, profile)| (name, profile.memory.0.saturating_add(mirror)))
.find(|(_, needs)| *needs > memory_max.0)
{
return Err(ConfigError::ProfileOverBudget {
profile: profile.clone(),
needs: ByteSize(needs),
slice: self.host.slice.clone(),
memory_max,
});
}
let needed = self.memory_needed()?;
if needed > memory_max {
return Err(ConfigError::OverBudget {
total: needed,
slice: self.host.slice.clone(),
memory_max,
});
}
return Ok(Vec::new());
}
match memory_max {
Some(memory_max) if total > memory_max => Err(ConfigError::OverBudget {
total,
slice: self.host.slice.clone(),
memory_max,
}),
Some(_) => Ok(Vec::new()),
None => Ok(vec![Warning::UnlimitedSlice {
slice: self.host.slice.clone(),
total,
}]),
}
}
}
#[derive(Debug, thiserror::Error)]
pub enum ResizeError {
#[error(transparent)]
Parse(#[from] toml_edit::TomlError),
#[error("the config has no profile {0:?} under [profiles]")]
NoProfile(String),
}
pub fn resize_profile(
text: &str,
profile: &str,
resize: api::Resize,
) -> Result<String, ResizeError> {
let mut document: toml_edit::DocumentMut = text.parse()?;
let table = document
.get_mut("profiles")
.and_then(|profiles| profiles.get_mut(profile))
.and_then(toml_edit::Item::as_table_like_mut)
.ok_or_else(|| ResizeError::NoProfile(profile.to_owned()))?;
let values = [
(
"memory",
toml_edit::Value::from(ByteSize(resize.memory).to_string()),
),
("slots", toml_edit::Value::from(i64::from(resize.slots))),
];
let cpus = resize
.cpus
.map(|cpus| ("cpus", toml_edit::Value::from(i64::from(cpus))));
for (key, mut value) in values.into_iter().chain(cpus) {
if let Some(old) = table.get(key).and_then(toml_edit::Item::as_value) {
*value.decor_mut() = old.decor().clone();
}
table.insert(key, toml_edit::Item::Value(value));
}
Ok(document.to_string())
}
pub const SECRET_SHOWN: &str = "<secret>";
pub fn mask_secrets(text: &str) -> Option<String> {
let mut document: toml_edit::DocumentMut = text.parse().ok()?;
let mut masked = false;
for target in notify_targets(&mut document) {
for (_, value) in secrets_of(target) {
replace_value(value, toml_edit::Value::from(SECRET_SHOWN));
masked = true;
}
}
Some(if masked {
document.to_string()
} else {
text.to_owned()
})
}
pub fn keep_secrets(text: &str, stored: &str) -> String {
if !text.contains(SECRET_SHOWN) || Config::parse(text).is_err() {
return text.to_owned();
}
let (Ok(mut document), Ok(mut old)) = (
text.parse::<toml_edit::DocumentMut>(),
stored.parse::<toml_edit::DocumentMut>(),
) else {
return text.to_owned();
};
let mut kept: BTreeMap<String, Vec<(SecretField, toml_edit::Value)>> = BTreeMap::new();
for target in notify_targets(&mut old) {
let Some(name) = target_name(target) else {
continue;
};
let secrets = secrets_of(target)
.into_iter()
.map(|(field, value)| (field, value.clone()))
.collect();
kept.insert(name, secrets);
}
for target in notify_targets(&mut document) {
let Some(secrets) = target_name(target).and_then(|name| kept.get(&name)) else {
continue;
};
for (field, value) in secrets_of(target) {
if value.as_str() != Some(SECRET_SHOWN) {
continue;
}
if let Some((_, old)) = secrets.iter().find(|(kept, _)| *kept == field) {
replace_value(value, old.clone());
}
}
}
document.to_string()
}
#[derive(Debug, PartialEq, Eq)]
enum SecretField {
Url,
Header(String),
}
fn notify_targets(document: &mut toml_edit::DocumentMut) -> Vec<&mut dyn toml_edit::TableLike> {
match document
.get_mut("notify")
.and_then(|notify| notify.get_mut("targets"))
{
Some(toml_edit::Item::ArrayOfTables(tables)) => tables
.iter_mut()
.map(|table| table as &mut dyn toml_edit::TableLike)
.collect(),
Some(toml_edit::Item::Value(toml_edit::Value::Array(array))) => array
.iter_mut()
.filter_map(toml_edit::Value::as_inline_table_mut)
.map(|table| table as &mut dyn toml_edit::TableLike)
.collect(),
_ => Vec::new(),
}
}
fn target_name(target: &dyn toml_edit::TableLike) -> Option<String> {
target
.get("name")
.and_then(toml_edit::Item::as_str)
.map(str::to_owned)
}
fn secrets_of(target: &mut dyn toml_edit::TableLike) -> Vec<(SecretField, &mut toml_edit::Value)> {
let mut found = Vec::new();
for (key, item) in target.iter_mut() {
match key.get() {
"url" => found.extend(item.as_value_mut().map(|value| (SecretField::Url, value))),
"headers" => {
let Some(headers) = item.as_table_like_mut() else {
continue;
};
for (name, item) in headers.iter_mut() {
let field = SecretField::Header(name.get().to_owned());
found.extend(item.as_value_mut().map(|value| (field, value)));
}
}
_ => {}
}
}
found
}
fn replace_value(value: &mut toml_edit::Value, mut new: toml_edit::Value) {
*new.decor_mut() = value.decor().clone();
*value = new;
}
pub fn revision(text: &str) -> String {
let hash = text.bytes().fold(0xcbf2_9ce4_8422_2325_u64, |hash, byte| {
(hash ^ u64::from(byte)).wrapping_mul(0x0100_0000_01b3)
});
format!("{hash:016x}")
}
pub fn slice_memory_max(slice: &str) -> std::io::Result<Option<ByteSize>> {
let output = Command::new("systemctl")
.args(["show", slice, "--property=MemoryMax", "--value"])
.output()?;
if !output.status.success() {
return Err(std::io::Error::other(format!(
"systemctl show {slice} failed: {}",
String::from_utf8_lossy(&output.stderr).trim()
)));
}
let value = String::from_utf8_lossy(&output.stdout);
parse_memory_max(value.trim()).ok_or_else(|| {
std::io::Error::other(format!(
"systemctl show {slice}: unexpected MemoryMax {value:?}"
))
})
}
pub fn cpu_budget(millicores: u64, overcommit: f64) -> u64 {
#[expect(
clippy::cast_precision_loss,
clippy::cast_possible_truncation,
clippy::cast_sign_loss,
reason = "a host's millicores are far below 2^52, and the config check holds the factor above 0 and at \
most 16"
)]
let budget = (millicores as f64 * overcommit.max(0.0)).round() as u64;
budget
}
pub fn slice_cpus(slice: &str) -> std::io::Result<u64> {
let output = Command::new("systemctl")
.args([
"show",
slice,
"--property=EffectiveCPUs",
"--property=AllowedCPUs",
"--property=CPUQuotaPerSecUSec",
])
.output()?;
if !output.status.success() {
return Err(std::io::Error::other(format!(
"systemctl show {slice} failed: {}",
String::from_utf8_lossy(&output.stderr).trim()
)));
}
let host = std::thread::available_parallelism().map_or(1, std::num::NonZero::get) as u64;
parse_slice_cpus(&String::from_utf8_lossy(&output.stdout), host).ok_or_else(|| {
std::io::Error::other(format!(
"systemctl show {slice}: unexpected CPU properties {:?}",
String::from_utf8_lossy(&output.stdout).trim()
))
})
}
pub fn parse_slice_cpus(properties: &str, host: u64) -> Option<u64> {
let value = |name: &str| {
properties.lines().find_map(|line| {
line.strip_prefix(name)
.and_then(|rest| rest.strip_prefix('='))
.map(str::trim)
})
};
let set = [value("EffectiveCPUs"), value("AllowedCPUs")]
.into_iter()
.flatten()
.find(|cpus| !cpus.is_empty());
let cpus = match set {
Some(set) => cpu_count(set)?,
None => host,
};
let quota = match value("CPUQuotaPerSecUSec") {
None | Some("" | "infinity") => u64::MAX,
Some(quota) => quota_millicores(quota)?,
};
Some(cpus.saturating_mul(1000).min(quota))
}
fn cpu_count(set: &str) -> Option<u64> {
set.split([',', ' '])
.filter(|range| !range.is_empty())
.try_fold(0u64, |count, range| {
let (first, last) = range.split_once('-').unwrap_or((range, range));
let (first, last): (u64, u64) = (first.parse().ok()?, last.parse().ok()?);
Some(count + last.checked_sub(first)? + 1)
})
}
fn quota_millicores(quota: &str) -> Option<u64> {
const UNITS: [(&str, f64); 10] = [
("y", 31_557_600_000.0),
("M", 2_629_800_000.0),
("w", 604_800_000.0),
("d", 86_400_000.0),
("h", 3_600_000.0),
("min", 60_000.0),
("s", 1000.0),
("ms", 1.0),
("us", 0.001),
("µs", 0.001),
];
let millicores = quota.split_whitespace().try_fold(0.0, |total, part| {
let at = part.find(|c: char| !(c.is_ascii_digit() || c == '.'))?;
let (number, unit) = part.split_at(at);
let per_unit = UNITS
.iter()
.find_map(|(name, per_unit)| (*name == unit).then_some(*per_unit))?;
Some(total + number.parse::<f64>().ok()? * per_unit)
})?;
#[allow(
clippy::cast_possible_truncation,
clippy::cast_sign_loss,
reason = "a quota is a few thousand millicores per CPU, checked finite and not negative first"
)]
(!quota.trim().is_empty() && millicores.is_finite() && millicores >= 0.0)
.then(|| millicores.round() as u64)
}
pub fn parse_memory_max(value: &str) -> Option<Option<ByteSize>> {
if value == "infinity" {
Some(None)
} else {
value.parse().ok().map(|bytes| Some(ByteSize(bytes)))
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub struct ByteSize(pub u64);
const UNITS: [(char, u64); 4] = [
('T', 1 << 40),
('G', 1 << 30),
('M', 1 << 20),
('K', 1 << 10),
];
impl FromStr for ByteSize {
type Err = String;
#[allow(
clippy::cast_precision_loss,
clippy::cast_possible_truncation,
clippy::cast_sign_loss,
reason = "sizes in a config are far below 2^52 bytes, and the result is checked against u64's range first"
)]
fn from_str(s: &str) -> Result<Self, Self::Err> {
let invalid = || {
format!(
"invalid size {s:?}, expected a number and a unit (K, M, G or T), such as \"9G\""
)
};
let unit = s.chars().last().ok_or_else(invalid)?.to_ascii_uppercase();
let (_, factor) = UNITS.iter().find(|(u, _)| *u == unit).ok_or_else(invalid)?;
let number = &s[..s.len() - 1];
if number.is_empty() || !number.chars().all(|c| c.is_ascii_digit() || c == '.') {
return Err(invalid());
}
let number: f64 = number.parse().map_err(|_| invalid())?;
let bytes = (number * *factor as f64).round();
if bytes >= u64::MAX as f64 {
return Err(format!("size {s:?} is too large"));
}
Ok(ByteSize(bytes as u64))
}
}
impl fmt::Display for ByteSize {
#[allow(
clippy::cast_precision_loss,
reason = "shown with two decimals, far coarser than f64's precision"
)]
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let Some((unit, factor)) = UNITS.iter().find(|(_, factor)| self.0 >= *factor) else {
return write!(f, "{}B", self.0);
};
let number = format!("{:.2}", self.0 as f64 / *factor as f64);
let number = number.trim_end_matches('0').trim_end_matches('.');
write!(f, "{number}{unit}")
}
}
impl ByteSize {
#[allow(
clippy::cast_precision_loss,
reason = "shown with one decimal at most, far coarser than f64's precision"
)]
pub fn short(self) -> String {
let Some(index) = UNITS.iter().position(|(_, factor)| self.0 >= *factor) else {
return format!("{}B", self.0);
};
let (mut unit, factor) = UNITS[index];
let mut value = self.0 as f64 / factor as f64;
if value.round() >= 1024.0
&& let Some(larger) = index.checked_sub(1)
{
unit = UNITS[larger].0;
value /= 1024.0;
}
let tenths = (value * 10.0).round() / 10.0;
if tenths < 10.0 {
let number = format!("{tenths:.1}");
format!("{}{unit}", number.trim_end_matches(".0"))
} else {
format!("{:.0}{unit}", value.round())
}
}
}
impl<'de> Deserialize<'de> for ByteSize {
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
String::deserialize(deserializer)?
.parse()
.map_err(de::Error::custom)
}
}
pub fn parse_duration(s: &str) -> Result<Duration, String> {
let invalid = || {
format!(
"invalid duration {s:?}, expected a number and s, m, h or d, such as \"168h\" or \"7d\""
)
};
let (number, factor) = match s.chars().last() {
Some('s') => (&s[..s.len() - 1], 1),
Some('m') => (&s[..s.len() - 1], 60),
Some('h') => (&s[..s.len() - 1], 3600),
Some('d') => (&s[..s.len() - 1], 86400),
_ => return Err(invalid()),
};
let number: u64 = number.parse().map_err(|_| invalid())?;
number
.checked_mul(factor)
.map(Duration::from_secs)
.ok_or_else(invalid)
}
pub fn parse_age(text: &str) -> Result<Duration, String> {
let error = || "expected a number and s, m, h, d or w, such as 12h".to_owned();
let split = text.find(|c: char| !c.is_ascii_digit()).ok_or_else(error)?;
let (number, unit) = text.split_at(split);
let number: u64 = number.parse().map_err(|_| error())?;
let unit = match unit {
"s" => 1,
"m" => 60,
"h" => 3600,
"d" => 86_400,
"w" => 7 * 86_400,
_ => return Err(error()),
};
Ok(Duration::from_secs(number.saturating_mul(unit)))
}
fn duration<'de, D: Deserializer<'de>>(deserializer: D) -> Result<Duration, D::Error> {
parse_duration(&String::deserialize(deserializer)?).map_err(de::Error::custom)
}
#[cfg(test)]
mod tests;