use std::collections::{BTreeMap, BTreeSet, HashMap};
use std::time::Duration;
use bollard::models::{ContainerCreateBody, HostConfig, Mount, MountType};
use tokio::sync::{mpsc, watch};
use tokio::task::JoinHandle;
use tokio::time::Instant;
use tracing::{info, warn};
use super::control::ControlError;
use super::{Engine, Occupant};
use crate::admission::HIGH_WATER;
use crate::api;
use crate::config::ByteSize;
use crate::docker::{self, DockerApi, Tty, TtySize, cpu_shares, is_not_found};
use crate::github::GithubApi;
use crate::observe::capture;
use crate::record;
use crate::scheduler::SlotRef;
use crate::time;
#[derive(Debug)]
pub(super) struct Recording {
record: api::Record,
container: String,
estimate: u64,
stage: Stage,
}
#[derive(Debug)]
enum Stage {
Checking(JoinHandle<Result<Checked, String>>),
Committing(JoinHandle<Committed>),
}
#[derive(Debug)]
struct Committed {
bytes: Result<u64, String>,
left: bool,
}
impl Stage {
fn is_finished(&self) -> bool {
match self {
Stage::Checking(task) => task.is_finished(),
Stage::Committing(task) => task.is_finished(),
}
}
}
#[derive(Debug)]
struct Checked {
incoming: u64,
free: Option<u64>,
}
#[derive(Debug)]
pub(super) enum Undecided {
Docker(docker::Error),
Unread,
}
impl std::fmt::Display for Undecided {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Undecided::Docker(error) => error.fmt(f),
Undecided::Unread => f.write_str("its diagnostic files were not all read"),
}
}
}
#[derive(Debug)]
pub(super) struct RecordState {
list: Option<Vec<api::Record>>,
recordings: BTreeMap<u64, Recording>,
shells: BTreeMap<String, Shell>,
opened: u64,
status: watch::Sender<api::Records>,
done: mpsc::UnboundedSender<()>,
pub(super) finished: mpsc::UnboundedReceiver<()>,
}
impl RecordState {
pub(super) fn new() -> (RecordState, watch::Receiver<api::Records>) {
let (status, receiver) = watch::channel(api::Records::default());
let (done, finished) = mpsc::unbounded_channel();
let state = RecordState {
list: None,
recordings: BTreeMap::new(),
shells: BTreeMap::new(),
opened: 0,
status,
done,
finished,
};
(state, receiver)
}
fn is_listed(&self) -> bool {
self.list.is_some()
}
fn kept(&self) -> &[api::Record] {
self.list.as_deref().unwrap_or_default()
}
fn of_job(&self, job: u64) -> Option<&api::Record> {
self.kept().iter().find(|record| record.job == job)
}
fn set_listed(&mut self, records: Vec<api::Record>) {
self.list = Some(records);
}
fn keep(&mut self, record: api::Record) {
self.list.get_or_insert_default().push(record);
}
fn forget(&mut self, job: u64) -> u64 {
let Some(list) = self.list.as_mut() else {
return 0;
};
let Some(index) = list.iter().position(|record| record.job == job) else {
return 0;
};
list.remove(index).bytes
}
pub(super) fn is_recording(&self) -> bool {
!self.recordings.is_empty()
}
fn pending(&self) -> u64 {
self.recordings.values().map(|other| other.estimate).sum()
}
fn begin(&mut self, job: u64, recording: Recording) {
self.recordings.insert(job, recording);
}
fn finished_recordings(&self) -> Vec<u64> {
self.recordings
.iter()
.filter(|(_, recording)| recording.stage.is_finished())
.map(|(job, _)| *job)
.collect()
}
fn take_recording(&mut self, job: u64) -> Option<Recording> {
self.recordings.remove(&job)
}
pub(super) fn has_shells(&self) -> bool {
!self.shells.is_empty()
}
fn next_shell(&mut self) -> u64 {
self.opened += 1;
self.opened
}
fn shell_opened(&mut self, container: String, shell: Shell) {
self.shells.insert(container, shell);
}
fn shell_closed(&mut self, container: &str) {
self.shells.remove(container);
}
}
#[derive(Debug, Clone)]
pub(super) struct Shell {
job: u64,
pub(super) profile: String,
pub(super) memory: u64,
opened: Instant,
}
const MAX_SHELL: Duration = Duration::from_hours(8);
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum RecordSelection {
Job(u64),
All,
OlderThan(Duration),
}
const SHELL_SCRIPT: &str = r#"
. /usr/local/lib/runnerd-layers.sh && mount_layers > /dev/null 2>&1
install -d -o runner -g runner -m 700 "$XDG_RUNTIME_DIR" 2> /dev/null
cd "$1" 2> /dev/null || cd /home/runner
exec setpriv --reuid=runner --regid=runner --init-groups -- env HOME=/home/runner USER=runner LOGNAME=runner bash -l
"#;
const COMMIT_FACTOR: u64 = 2;
impl<G: GithubApi + 'static, D: DockerApi + 'static> Engine<G, D> {
fn records_failures(&self, profile: &str) -> bool {
self.config
.profiles
.get(profile)
.or_else(|| self.retired.get(profile))
.is_some_and(|profile| profile.record_failures)
}
pub(super) async fn begin_record(&mut self, slot: &SlotRef, occupant: &Occupant) -> bool {
let Some(job) = self.named_job(occupant) else {
return false;
};
if !self.records_failures(&slot.profile) {
return false;
}
if !self.records.is_listed() {
self.list_records().await;
if !self.records.is_listed() {
let why = "Docker did not list the records kept".to_owned();
self.skip_record(job, &occupant.repository, why);
return false;
}
}
let summary = self
.store
.job(job)
.await
.ok()
.flatten()
.map(|job| job.summary);
let record = api::Record {
job,
repository: occupant.repository.clone(),
profile: slot.profile.clone(),
workflow: summary
.as_ref()
.and_then(|summary| summary.workflow.clone()),
name: summary.map(|summary| summary.name),
created_at: time::now_ms() / 1000,
bytes: 0,
image: record::image(job),
};
info!(
"{}: keeping job {job}'s runner as {}",
occupant.repository, record.image
);
let docker = self.docker.clone();
let container = occupant.container.clone();
let done = self.records.done.clone();
let check = tokio::spawn(async move {
let checked = check(&*docker, &container).await;
let _ = done.send(());
checked
});
self.records.begin(
job,
Recording {
record,
container: occupant.container.clone(),
estimate: 0,
stage: Stage::Checking(check),
},
);
true
}
pub(super) async fn keep_left_record(
&mut self,
slot: &SlotRef,
occupant: &Occupant,
) -> Result<bool, Undecided> {
if !self.records_failures(&slot.profile)
|| occupant
.named_job()
.is_some_and(|job| self.records.of_job(job).is_some())
{
return Ok(false);
}
let failed = match self.docker.logs(&occupant.container).await {
Ok(log) => record::failed(&log),
Err(error) if docker::is_declined(&error) => false,
Err(error) => return Err(Undecided::Docker(error)),
};
if !failed {
return Ok(false);
}
let job = match occupant.named_job() {
Some(job) => Some(job),
None => self.left_job(&occupant.container).await?,
};
let Some(job) = job else {
warn!(
"{}: {} ran a job that failed, but names it nowhere: no record of it is kept",
occupant.repository, occupant.runner_name
);
return Ok(false);
};
if self.records.of_job(job).is_some() {
return Ok(false);
}
let mut occupant = occupant.clone();
occupant.engagement.named_by_worker_log(job);
Ok(self.begin_record(slot, &occupant).await)
}
async fn left_job(&self, container: &str) -> Result<Option<u64>, Undecided> {
let read = match self
.docker
.files(container, capture::DIAG, capture::WORKER_LOG)
.await
{
Ok(read) => read,
Err(error) if docker::is_declined(&error) => return Ok(None),
Err(error) => return Err(Undecided::Docker(error)),
};
match read
.files
.iter()
.find_map(|file| capture::worker_job(&file.contents))
{
Some(job) => Ok(Some(job)),
None if !read.complete => Err(Undecided::Unread),
None => Ok(None),
}
}
fn skip_record(&self, job: u64, repository: &str, why: String) {
warn!("{repository}: no record of job {job}: {why}");
self.store.record_skipped(job, why);
}
async fn make_room_and_commit(
&mut self,
record: api::Record,
container: String,
checked: Checked,
) {
let job = record.job;
let pending = self.records.pending();
let room = record::make_room(
self.records.kept(),
&self.shell_jobs(),
&self.config.records,
checked.incoming,
checked.free.map(|free| free.saturating_sub(pending)),
time::now_ms() / 1000,
);
let removed = match room {
Ok(removals) => {
let mut removed = Ok(());
for (old, why) in removals {
if let Err(error) = self.remove_record(old, why).await {
removed = Err(error.to_string());
break;
}
}
removed
}
Err(why) => Err(why),
};
if let Err(why) = removed {
self.skip_record(job, &record.repository, why);
self.remove_container(&container).await;
return;
}
let labels = labels(&record);
let docker = self.docker.clone();
let image = record.image.clone();
let done = self.records.done.clone();
let committed_container = container.clone();
let commit = tokio::spawn(async move {
let container = committed_container;
let bytes = docker
.commit(&container, &image, labels)
.await
.map_err(|error| error.to_string());
let left = match docker.remove(&container).await {
Err(error) if !is_not_found(&error) => {
warn!("cannot remove {container}: {error}");
true
}
_ => false,
};
let _ = done.send(());
Committed { bytes, left }
});
self.records.begin(
job,
Recording {
record,
container,
estimate: checked.incoming,
stage: Stage::Committing(commit),
},
);
}
async fn remove_container(&mut self, container: &str) {
if let Err(error) = self.docker.remove(container).await
&& !is_not_found(&error)
{
warn!("cannot remove {container}: {error}");
self.leftovers
.removing(container.to_owned(), container.to_owned());
}
}
pub(super) async fn collect_recordings(&mut self) {
while self.records.finished.try_recv().is_ok() {}
for job in self.records.finished_recordings() {
let Some(recording) = self.records.take_recording(job) else {
continue;
};
let Recording {
record,
container,
stage,
..
} = recording;
let task = match stage {
Stage::Checking(task) => {
let checked = task
.await
.map_err(|error| error.to_string())
.and_then(|checked| checked);
match checked {
Ok(checked) => self.make_room_and_commit(record, container, checked).await,
Err(why) => {
self.skip_record(job, &record.repository, why);
self.remove_container(&container).await;
}
}
continue;
}
Stage::Committing(task) => task,
};
let bytes = match task.await {
Ok(Committed { bytes, left }) => {
if left {
self.leftovers.removing(container.clone(), container);
}
bytes
}
Err(error) => Err(error.to_string()),
};
match bytes {
Ok(bytes) => self.keep_record(record, bytes).await,
Err(error) => self.skip_record(
job,
&record.repository,
format!("Docker did not commit it: {error}"),
),
}
}
}
async fn keep_record(&mut self, mut record: api::Record, bytes: u64) {
let job = record.job;
record.bytes = bytes;
self.name_from_history(&mut record).await;
info!(
"{}: kept job {job}'s runner as {} ({})",
record.repository,
record.image,
ByteSize(bytes)
);
let removals = record::make_room(
self.records.kept(),
&self.shell_jobs(),
&self.config.records,
bytes,
None,
record.created_at,
);
self.store.record_made(record.clone());
self.records.keep(record);
let removals = removals.unwrap_or_else(|_| vec![(job, api::RecordRemoval::Size)]);
for (old, why) in removals {
if let Err(error) = self.remove_record(old, why).await {
warn!("{error}");
}
}
}
async fn name_from_history(&self, record: &mut api::Record) {
if record.workflow.is_some() && record.name.is_some() {
return;
}
self.store.flush().await;
if let Ok(Some(job)) = self.store.job(record.job).await {
record.workflow = record.workflow.take().or(job.summary.workflow);
record.name.get_or_insert(job.summary.name);
}
}
pub(super) async fn keep_records(&mut self) {
if self.records.is_listed() {
self.expire_records().await;
} else {
self.list_records().await;
}
}
pub(super) async fn list_records(&mut self) {
match self.docker.images(record::LABEL).await {
Ok(images) => {
let mut records: Vec<api::Record> = images
.into_iter()
.filter_map(|image| {
let label = |name: &str| image.labels.get(name).cloned();
Some(api::Record {
job: label(record::LABEL)?.parse().ok()?,
repository: label("runnerd.repository").unwrap_or_default(),
profile: label("runnerd.profile").unwrap_or_default(),
workflow: label("runnerd.workflow"),
name: label("runnerd.name"),
created_at: label("runnerd.created-at")
.and_then(|at| at.parse().ok())
.unwrap_or(0),
bytes: image.bytes,
image: image.reference,
})
})
.collect();
records.sort_by_key(|record| (record.created_at, record.job));
for record in &mut records {
self.name_from_history(record).await;
let noted = self.store.job(record.job).await.ok().flatten();
if noted
.is_some_and(|job| job.record.is_none_or(|note| note.created_at.is_none()))
{
self.store.record_made(record.clone());
}
}
self.store.records_listed(
records.iter().map(|record| record.job).collect(),
time::now_ms(),
);
self.records.set_listed(records);
}
Err(error) => warn!("cannot list the records: {error}"),
}
}
pub(super) async fn expire_records(&mut self) {
let expired = record::expired(
self.records.kept(),
&self.shell_jobs(),
&self.config.records,
time::now_ms() / 1000,
);
for (job, why) in expired {
if let Err(error) = self.remove_record(job, why).await {
warn!("{error}");
}
}
}
async fn remove_record(
&mut self,
job: u64,
why: api::RecordRemoval,
) -> Result<u64, ControlError> {
let Some(image) = self.records.of_job(job).map(|record| record.image.clone()) else {
return Err(ControlError::NotFound(format!(
"job {job} has no record; `runnerctl records` lists them"
)));
};
self.docker.remove_image(&image).await.map_err(|error| {
ControlError::Failed(format!("cannot remove the record of job {job}: {error}"))
})?;
let bytes = self.records.forget(job);
info!("removed the record of job {job} ({why})");
self.store.record_removed(job, time::now_ms(), why);
Ok(bytes)
}
pub(super) async fn remove_records(
&mut self,
selection: &RecordSelection,
) -> Result<api::RemovedRecords, ControlError> {
let open = self.shell_jobs();
let now = time::now_ms() / 1000;
let jobs: Vec<u64> = match selection {
RecordSelection::Job(job) => {
if open.contains(job) {
return Err(ControlError::Conflict(format!(
"a shell is open in the record of job {job}; remove it once the shell exits"
)));
}
vec![*job]
}
RecordSelection::All => self
.records
.kept()
.iter()
.map(|record| record.job)
.collect(),
RecordSelection::OlderThan(age) => self
.records
.kept()
.iter()
.filter(|record| now.saturating_sub(record.created_at) > age.as_secs())
.map(|record| record.job)
.collect(),
};
let mut removed = api::RemovedRecords::default();
for job in jobs.into_iter().filter(|job| !open.contains(job)) {
removed.bytes += self.remove_record(job, api::RecordRemoval::User).await?;
removed.jobs.push(job);
}
Ok(removed)
}
fn shell_jobs(&self) -> BTreeSet<u64> {
self.records
.shells
.values()
.map(|shell| shell.job)
.collect()
}
pub(super) fn shell_profiles(&self) -> BTreeSet<&str> {
self.records
.shells
.values()
.map(|shell| shell.profile.as_str())
.collect()
}
pub(super) fn shell_memory(&self) -> u64 {
self.records.shells.values().map(|shell| shell.memory).sum()
}
pub(super) fn shell_budget(&self) -> Vec<api::ShellMemory> {
self.records
.shells
.values()
.map(|shell| api::ShellMemory {
job: shell.job,
memory: shell.memory,
})
.collect()
}
pub(super) fn publish_records(&self) {
let records = self.records_status();
self.records.status.send_if_modified(|current| {
let changed = *current != records;
*current = records;
changed
});
}
fn records_status(&self) -> api::Records {
let limits = &self.config.records;
api::Records {
records: self.records.kept().to_vec(),
limits: api::RecordLimits {
max_total: limits.max_total.0,
max_count: limits.max_count,
max_age: limits.max_age.as_secs(),
min_free: limits.min_free.0,
},
shells: self.shell_jobs().into_iter().collect(),
}
}
pub(super) async fn open_shell(
&mut self,
job: u64,
size: TtySize,
) -> Result<Tty, ControlError> {
let Some(record) = self.records.of_job(job).cloned() else {
return Err(ControlError::NotFound(format!(
"job {job} has no record; `runnerctl records` lists them"
)));
};
let profile = self.config.profiles.get(&record.profile);
let Some(profile) = profile.or_else(|| self.retired.get(&record.profile)) else {
return Err(ControlError::Invalid(format!(
"the record of job {job} ran in profile {}, which the config no longer has",
record.profile
)));
};
let (memory, cpus) = (profile.memory.0, profile.cpus);
if self.upkeep.reseed_runs(&record.profile) {
return Err(ControlError::Conflict(format!(
"{} is reseeding its snapshot, which a shell mounts; open it once the reseed is done",
record.profile
)));
}
if self.trim.as_ref().is_some_and(|trim| !trim.is_finished()) {
return Err(ControlError::Conflict(
"the shared caches, which a shell mounts, are being trimmed; open it once the trim is done"
.to_owned(),
));
}
if let Some(max) = self.memory_max {
let in_use = self.count(self.host_remembered()).pooled();
if in_use.saturating_add(memory) > max {
return Err(ControlError::Conflict(format!(
"the budget is full: {} of the slice's {} is counted, and a shell takes {} as a slot of {} \
does; open it once a job finishes",
ByteSize(in_use),
ByteSize(max),
ByteSize(memory),
record.profile
)));
}
}
if self.admitting(&mut [], None).held {
return Err(ControlError::Conflict(format!(
"the slice is at {HIGH_WATER}% of its MemoryMax, where measured admission starts nothing; open it \
once it falls under"
)));
}
let body = self.shell_body(&record, memory, cpus);
let name = format!(
"runnerd-shell-{job}-{}-{}",
time::now_ms() / 1000,
self.records.next_shell()
);
let tty = self
.docker
.shell(&name, body, size)
.await
.map_err(|error| ControlError::Failed(format!("cannot start a shell: {error}")))?;
info!("opened a shell in the record of job {job}");
self.records.shell_opened(
tty.container.clone(),
Shell {
job,
profile: record.profile,
memory,
opened: Instant::now(),
},
);
Ok(tty)
}
fn shell_body(&self, record: &api::Record, memory: u64, cpus: u32) -> ContainerCreateBody {
let workspace = record::workspace(&record.repository);
let limit = i64::try_from(memory).unwrap_or(i64::MAX);
let mut mounts = vec![Mount {
target: Some("/state".to_owned()),
typ: Some(MountType::VOLUME),
..Default::default()
}];
if self.config.profiles.contains_key(&record.profile) {
mounts.extend(
crate::cache::runner_mounts(&self.names, &record.profile, 1)
.into_iter()
.filter(|mount| mount.target.as_deref() != Some("/state"))
.map(|mount| Mount {
read_only: Some(true),
..mount
}),
);
}
let mut host_config = HostConfig {
cgroup_parent: Some(self.config.host.slice.clone()),
memory: Some(limit),
memory_swap: Some(limit),
nano_cpus: Some(i64::from(cpus) * 1_000_000_000),
cpu_shares: Some(cpu_shares(cpus)),
init: Some(true),
mounts: Some(mounts),
network_mode: Some(self.names.network()),
..Default::default()
};
self.isolation.configure(&mut host_config);
ContainerCreateBody {
image: Some(record.image.clone()),
entrypoint: Some(vec![
"bash".to_owned(),
"-c".to_owned(),
SHELL_SCRIPT.to_owned(),
]),
cmd: Some(vec!["shell".to_owned(), workspace]),
env: Some(vec!["TERM=xterm-256color".to_owned()]),
tty: Some(true),
open_stdin: Some(true),
stdin_once: Some(true),
attach_stdin: Some(true),
attach_stdout: Some(true),
attach_stderr: Some(true),
labels: Some(HashMap::from([(
record::SHELL_LABEL.to_owned(),
record.job.to_string(),
)])),
host_config: Some(host_config),
..Default::default()
}
}
pub(super) async fn close_shell(&mut self, container: &str) {
let Some(shell) = self.records.shells.get(container) else {
return;
};
let job = shell.job;
match self.docker.remove(container).await {
Err(error) if !is_not_found(&error) => {
warn!("cannot remove the shell in the record of job {job}: {error}");
}
_ => {
info!("closed the shell in the record of job {job}");
self.records.shell_closed(container);
}
}
}
pub(super) async fn collect_shells(&mut self) {
let shells: Vec<(String, Instant)> = self
.records
.shells
.iter()
.map(|(container, shell)| (container.clone(), shell.opened))
.collect();
for (container, opened) in shells {
if opened.elapsed() > MAX_SHELL {
info!("closing {container}: a shell is open for 8 hours at most");
self.close_shell(&container).await;
continue;
}
let ended = match self.docker.inspect(&container).await {
Ok(state) => !state.running,
Err(error) => is_not_found(&error),
};
if ended {
self.close_shell(&container).await;
}
}
}
pub(super) async fn close_shells(&mut self) {
let containers: Vec<String> = self.records.shells.keys().cloned().collect();
for container in containers {
self.close_shell(&container).await;
}
}
}
fn labels(record: &api::Record) -> BTreeMap<String, String> {
let mut labels = BTreeMap::from([
(record::LABEL.to_owned(), record.job.to_string()),
("runnerd.repository".to_owned(), record.repository.clone()),
("runnerd.profile".to_owned(), record.profile.clone()),
(
"runnerd.created-at".to_owned(),
record.created_at.to_string(),
),
]);
if let Some(workflow) = &record.workflow {
labels.insert("runnerd.workflow".to_owned(), workflow.clone());
}
if let Some(name) = &record.name {
labels.insert("runnerd.name".to_owned(), name.clone());
}
labels
}
async fn check(docker: &impl DockerApi, container: &str) -> Result<Checked, String> {
let changed = docker
.changes(container)
.await
.map_err(|error| format!("Docker did not list its files: {error}"))?;
let secrets = record::secrets_left(changed.iter().map(String::as_str));
if !secrets.is_empty() {
return Err(format!(
"its runner left files a record must not keep ({}): its image did not delete them; use the image of \
runnerd {}, or rebuild image_stage from its image/",
record::listed(&secrets),
env!("CARGO_PKG_VERSION")
));
}
let incoming = docker
.written(container)
.await
.map_err(|error| format!("Docker did not measure it: {error}"))?
.saturating_mul(COMMIT_FACTOR);
let free = match docker.free_space().await {
Ok(free) => Some(free),
Err(error) => {
warn!("cannot measure Docker's free disk: {error}");
None
}
};
Ok(Checked { incoming, free })
}