Skip to main content

runnerd_core/scheduler/engine/
records.rs

1//! Records of failed jobs (`crate::record`): keeps a failed job's stopped runner container as an image before it is
2//! removed, keeps the records within `[records]`, removes them on request, and runs the shells opened in them, whose
3//! memory counts against the budget like a slot's. It must not serve a shell's terminal: the daemon does, with the
4//! `Tty` `open_shell` returns, and closes it with `Control::close_shell`.
5
6use std::collections::{BTreeMap, BTreeSet, HashMap};
7use std::time::Duration;
8
9use bollard::models::{ContainerCreateBody, HostConfig, Mount, MountType};
10use tokio::sync::{mpsc, watch};
11use tokio::task::JoinHandle;
12use tokio::time::Instant;
13use tracing::{info, warn};
14
15use super::control::ControlError;
16use super::{Engine, Occupant};
17use crate::admission::HIGH_WATER;
18use crate::api;
19use crate::config::ByteSize;
20use crate::docker::{self, DockerApi, Tty, TtySize, cpu_shares, is_not_found};
21use crate::github::GithubApi;
22use crate::observe::capture;
23use crate::record;
24use crate::scheduler::SlotRef;
25use crate::time;
26
27/// A record being made: its container checked and measured, then committed and removed, each in a task of its own
28/// so the engine's loop does not wait for Docker's walk of the container's files.
29#[derive(Debug)]
30pub(super) struct Recording {
31    record: api::Record,
32    container: String,
33    /// The disk its commit is expected to take, once measured: what the records made after it count against
34    /// `min_free` until it is done.
35    estimate: u64,
36    stage: Stage,
37}
38
39#[derive(Debug)]
40enum Stage {
41    /// Its changes checked for secrets, and its size and Docker's free disk measured.
42    Checking(JoinHandle<Result<Checked, String>>),
43    /// The commit, then the container's removal; the record's size.
44    Committing(JoinHandle<Result<u64, String>>),
45}
46
47impl Stage {
48    fn is_finished(&self) -> bool {
49        match self {
50            Stage::Checking(task) => task.is_finished(),
51            Stage::Committing(task) => task.is_finished(),
52        }
53    }
54}
55
56/// What checking a container found: it may be kept, at about `incoming` bytes, with `free` bytes on Docker's disk
57/// when known.
58#[derive(Debug)]
59struct Checked {
60    incoming: u64,
61    free: Option<u64>,
62}
63
64/// Why recovery cannot yet tell whether a stopped runner container becomes its failed job's record.
65#[derive(Debug)]
66pub(super) enum Undecided {
67    /// Docker failed, and may not when asked again.
68    Docker(docker::Error),
69    /// Its diagnostic files were not all read, and those read name no job.
70    Unread,
71}
72
73impl std::fmt::Display for Undecided {
74    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
75        match self {
76            Undecided::Docker(error) => error.fmt(f),
77            Undecided::Unread => f.write_str("its diagnostic files were not all read"),
78        }
79    }
80}
81
82/// The records of failed jobs and the shells open in them.
83#[derive(Debug)]
84pub(super) struct RecordState {
85    /// Oldest first.
86    list: Vec<api::Record>,
87    /// Whether `list` is what Docker has: until Docker lists them, no record is made.
88    pub(super) listed: bool,
89    /// Records being made, by job.
90    pub(super) recordings: BTreeMap<u64, Recording>,
91    /// Shells open in records, by container ID.
92    pub(super) shells: BTreeMap<String, Shell>,
93    /// Shells opened by this engine, which tells their containers' names apart.
94    opened: u64,
95    /// What `Handle::records` shows.
96    status: watch::Sender<api::Records>,
97    /// A recording's tasks say here that they are done, so the loop moves them on without waiting for a tick.
98    done: mpsc::UnboundedSender<()>,
99    pub(super) finished: mpsc::UnboundedReceiver<()>,
100}
101
102impl RecordState {
103    /// No record known yet, and the receiver of what `publish_records` shows.
104    pub(super) fn new() -> (RecordState, watch::Receiver<api::Records>) {
105        let (status, receiver) = watch::channel(api::Records::default());
106        let (done, finished) = mpsc::unbounded_channel();
107        let state = RecordState {
108            list: Vec::new(),
109            listed: false,
110            recordings: BTreeMap::new(),
111            shells: BTreeMap::new(),
112            opened: 0,
113            status,
114            done,
115            finished,
116        };
117        (state, receiver)
118    }
119}
120
121/// A shell open in a record.
122#[derive(Debug, Clone)]
123pub(super) struct Shell {
124    job: u64,
125    /// The profile the record's job ran in, whose snapshot and shared caches the shell mounts.
126    pub(super) profile: String,
127    /// Its container's memory cap, which counts against the budget.
128    pub(super) memory: u64,
129    opened: Instant,
130}
131
132/// The longest a shell stays open: a forgotten one holds a slot's memory and keeps its record from going.
133const MAX_SHELL: Duration = Duration::from_hours(8);
134
135/// Which records a removal asks for.
136#[derive(Debug, Clone, PartialEq, Eq)]
137pub enum RecordSelection {
138    /// The record of one job.
139    Job(u64),
140    /// Every record without a shell open.
141    All,
142    /// The records older than this, without a shell open.
143    OlderThan(Duration),
144}
145
146/// What a shell container runs, as root: the profile's snapshot under the cache layers as a runner sees them, with
147/// the shell's own changes in its anonymous /state, then a login shell as `runner` in the directory `$1`.
148const SHELL_SCRIPT: &str = r#"
149. /usr/local/lib/runnerd-layers.sh && mount_layers > /dev/null 2>&1
150install -d -o runner -g runner -m 700 "$XDG_RUNTIME_DIR" 2> /dev/null
151cd "$1" 2> /dev/null || cd /home/runner
152exec setpriv --reuid=runner --regid=runner --init-groups -- env HOME=/home/runner USER=runner LOGNAME=runner bash -l
153"#;
154
155/// The disk Docker takes per byte a container's layer holds once committed: the containerd image store keeps the
156/// layer's archive besides its unpacked copy.
157const COMMIT_FACTOR: u64 = 2;
158
159impl<G: GithubApi + 'static, D: DockerApi + 'static> Engine<G, D> {
160    /// Whether the profile of a slot keeps its failed jobs' runners.
161    fn records_failures(&self, profile: &str) -> bool {
162        self.config
163            .profiles
164            .get(profile)
165            .or_else(|| self.retired.get(profile))
166            .is_some_and(|profile| profile.record_failures)
167    }
168
169    /// Starts keeping a stopped runner container whose job failed as the job's record: checks and measures it, makes
170    /// room within the limits, then commits and removes it, in the background. Returns whether it took the container
171    /// over; if not, the caller removes it.
172    pub(super) async fn begin_record(&mut self, slot: &SlotRef, occupant: &Occupant) -> bool {
173        // The capture's last report may have named the job after `occupant` was taken.
174        let job = self
175            .occupied()
176            .find(|(_, candidate)| candidate.runner_name == occupant.runner_name)
177            .and_then(|(_, candidate)| candidate.job)
178            .or(occupant.job);
179        let Some(job) = job else {
180            return false;
181        };
182        if !self.records_failures(&slot.profile) {
183            return false;
184        }
185        if !self.records.listed {
186            self.list_records().await;
187            if !self.records.listed {
188                let why = "Docker did not list the records kept".to_owned();
189                self.skip_record(job, &occupant.repository, why);
190                return false;
191            }
192        }
193        let summary = self
194            .store
195            .job(job)
196            .await
197            .ok()
198            .flatten()
199            .map(|job| job.summary);
200        let record = api::Record {
201            job,
202            repository: occupant.repository.clone(),
203            profile: slot.profile.clone(),
204            workflow: summary
205                .as_ref()
206                .and_then(|summary| summary.workflow.clone()),
207            name: summary.map(|summary| summary.name),
208            created_at: time::now_ms() / 1000,
209            bytes: 0,
210            image: record::image(job),
211        };
212        info!(
213            "{}: keeping job {job}'s runner as {}",
214            occupant.repository, record.image
215        );
216        let docker = self.docker.clone();
217        let container = occupant.container.clone();
218        let done = self.records.done.clone();
219        let check = tokio::spawn(async move {
220            let checked = check(&*docker, &container).await;
221            let _ = done.send(());
222            checked
223        });
224        self.records.recordings.insert(
225            job,
226            Recording {
227                record,
228                container: occupant.container.clone(),
229                estimate: 0,
230                stage: Stage::Checking(check),
231            },
232        );
233        true
234    }
235
236    /// Keeps a stopped runner container an earlier daemon left as its job's record, as `release` would have, if its
237    /// profile keeps failed jobs' runners, its log shows its job failed and no record of the job was made: the runner
238    /// ended while no daemon ran, or a daemon stopped before it collected it or finished the record. A job the history
239    /// never saw it take, as one an idle runner took while the daemon detached, is read from the runner's worker log.
240    /// Returns whether it took the container over; if not, the caller removes it. `Undecided` when Docker could not
241    /// tell, and may once asked again: the caller keeps the container to try again.
242    pub(super) async fn keep_left_record(
243        &mut self,
244        slot: &SlotRef,
245        occupant: &Occupant,
246    ) -> Result<bool, Undecided> {
247        let recorded =
248            |records: &RecordState, job| records.list.iter().any(|record| record.job == job);
249        if !self.records_failures(&slot.profile)
250            || occupant.job.is_some_and(|job| recorded(&self.records, job))
251        {
252            return Ok(false);
253        }
254        let failed = match self.docker.logs(&occupant.container).await {
255            Ok(log) => record::failed(&log),
256            Err(error) if docker::is_declined(&error) => false,
257            Err(error) => return Err(Undecided::Docker(error)),
258        };
259        if !failed {
260            return Ok(false);
261        }
262        let job = match occupant.job {
263            Some(job) => Some(job),
264            None => self.left_job(&occupant.container).await?,
265        };
266        let Some(job) = job else {
267            warn!(
268                "{}: {} ran a job that failed, but names it nowhere: no record of it is kept",
269                occupant.repository, occupant.runner_name
270            );
271            return Ok(false);
272        };
273        if recorded(&self.records, job) {
274            return Ok(false);
275        }
276        let occupant = Occupant {
277            job: Some(job),
278            ..occupant.clone()
279        };
280        Ok(self.begin_record(slot, &occupant).await)
281    }
282
283    /// The job a stopped runner container took, from its worker log's job message, read without starting it: the
284    /// capture reads the same while it runs. `None` when the runner wrote none, or Docker says it cannot read it.
285    async fn left_job(&self, container: &str) -> Result<Option<u64>, Undecided> {
286        let read = match self
287            .docker
288            .files(container, capture::DIAG, capture::WORKER_LOG)
289            .await
290        {
291            Ok(read) => read,
292            Err(error) if docker::is_declined(&error) => return Ok(None),
293            Err(error) => return Err(Undecided::Docker(error)),
294        };
295        match read
296            .files
297            .iter()
298            .find_map(|file| capture::worker_job(&file.contents))
299        {
300            Some(job) => Ok(Some(job)),
301            // A worker log the read did not reach may name it.
302            None if !read.complete => Err(Undecided::Unread),
303            None => Ok(None),
304        }
305    }
306
307    /// Notes in the history why a failed job has no record.
308    fn skip_record(&self, job: u64, repository: &str, why: String) {
309        warn!("{repository}: no record of job {job}: {why}");
310        self.store.record_skipped(job, why);
311    }
312
313    /// Makes room for a checked container within the limits, removing the records in its way, then commits it in
314    /// the background; or notes why it is not kept and removes it.
315    async fn make_room_and_commit(
316        &mut self,
317        record: api::Record,
318        container: String,
319        checked: Checked,
320    ) {
321        let job = record.job;
322        // Records being committed take their disk too.
323        let pending: u64 = self
324            .records
325            .recordings
326            .values()
327            .map(|other| other.estimate)
328            .sum();
329        let room = record::make_room(
330            &self.records.list,
331            &self.shell_jobs(),
332            &self.config.records,
333            checked.incoming,
334            checked.free.map(|free| free.saturating_sub(pending)),
335            time::now_ms() / 1000,
336        );
337        let removed = match room {
338            Ok(removals) => {
339                let mut removed = Ok(());
340                for (old, why) in removals {
341                    if let Err(error) = self.remove_record(old, why).await {
342                        removed = Err(error.to_string());
343                        break;
344                    }
345                }
346                removed
347            }
348            Err(why) => Err(why),
349        };
350        if let Err(why) = removed {
351            self.skip_record(job, &record.repository, why);
352            self.remove_container(&container).await;
353            return;
354        }
355        let labels = labels(&record);
356        let docker = self.docker.clone();
357        let image = record.image.clone();
358        let done = self.records.done.clone();
359        let committed_container = container.clone();
360        let commit = tokio::spawn(async move {
361            let container = committed_container;
362            let committed = docker.commit(&container, &image, labels).await;
363            if let Err(error) = docker.remove(&container).await
364                && !is_not_found(&error)
365            {
366                warn!("cannot remove {container}: {error}");
367            }
368            let _ = done.send(());
369            committed.map_err(|error| error.to_string())
370        });
371        self.records.recordings.insert(
372            job,
373            Recording {
374                record,
375                container,
376                estimate: checked.incoming,
377                stage: Stage::Committing(commit),
378            },
379        );
380    }
381
382    async fn remove_container(&self, container: &str) {
383        if let Err(error) = self.docker.remove(container).await
384            && !is_not_found(&error)
385        {
386            warn!("cannot remove {container}: {error}");
387        }
388    }
389
390    /// Moves the recordings whose task finished on: a checked container to its commit, a committed one into the
391    /// records, kept within their limits with its real size.
392    pub(super) async fn collect_recordings(&mut self) {
393        while self.records.finished.try_recv().is_ok() {}
394        let done: Vec<u64> = self
395            .records
396            .recordings
397            .iter()
398            .filter(|(_, recording)| recording.stage.is_finished())
399            .map(|(job, _)| *job)
400            .collect();
401        for job in done {
402            let Some(recording) = self.records.recordings.remove(&job) else {
403                continue;
404            };
405            let Recording {
406                record,
407                container,
408                stage,
409                ..
410            } = recording;
411            let task = match stage {
412                Stage::Checking(task) => {
413                    let checked = task
414                        .await
415                        .map_err(|error| error.to_string())
416                        .and_then(|checked| checked);
417                    match checked {
418                        Ok(checked) => self.make_room_and_commit(record, container, checked).await,
419                        Err(why) => {
420                            self.skip_record(job, &record.repository, why);
421                            self.remove_container(&container).await;
422                        }
423                    }
424                    continue;
425                }
426                Stage::Committing(task) => task,
427            };
428            match task
429                .await
430                .map_err(|error| error.to_string())
431                .and_then(|committed| committed)
432            {
433                Ok(bytes) => self.keep_record(record, bytes).await,
434                Err(error) => self.skip_record(
435                    job,
436                    &record.repository,
437                    format!("Docker did not commit it: {error}"),
438                ),
439            }
440        }
441    }
442
443    /// Adds a committed record, and removes the records past the limits with its real size.
444    async fn keep_record(&mut self, mut record: api::Record, bytes: u64) {
445        let job = record.job;
446        record.bytes = bytes;
447        self.name_from_history(&mut record).await;
448        info!(
449            "{}: kept job {job}'s runner as {} ({})",
450            record.repository,
451            record.image,
452            ByteSize(bytes)
453        );
454        let removals = record::make_room(
455            &self.records.list,
456            &self.shell_jobs(),
457            &self.config.records,
458            bytes,
459            None,
460            record.created_at,
461        );
462        self.store.record_made(record.clone());
463        self.records.list.push(record);
464        let removals = removals.unwrap_or_else(|_| vec![(job, api::RecordRemoval::Size)]);
465        for (old, why) in removals {
466            if let Err(error) = self.remove_record(old, why).await {
467                warn!("{error}");
468            }
469        }
470    }
471
472    /// Fills in a record's workflow and job name from the history, which may have learnt them from GitHub only after
473    /// the commit started. Reads after the writes sent before, such as those details: the store reads on its own
474    /// connection, which a busy writer leaves behind.
475    async fn name_from_history(&self, record: &mut api::Record) {
476        if record.workflow.is_some() && record.name.is_some() {
477            return;
478        }
479        self.store.flush().await;
480        if let Ok(Some(job)) = self.store.job(record.job).await {
481            record.workflow = record.workflow.take().or(job.summary.workflow);
482            record.name.get_or_insert(job.summary.name);
483        }
484    }
485
486    /// Lists the record images Docker has; until it answers, no record is made.
487    pub(super) async fn list_records(&mut self) {
488        match self.docker.images(record::LABEL).await {
489            Ok(images) => {
490                let mut records: Vec<api::Record> = images
491                    .into_iter()
492                    .filter_map(|image| {
493                        let label = |name: &str| image.labels.get(name).cloned();
494                        Some(api::Record {
495                            job: label(record::LABEL)?.parse().ok()?,
496                            repository: label("runnerd.repository").unwrap_or_default(),
497                            profile: label("runnerd.profile").unwrap_or_default(),
498                            workflow: label("runnerd.workflow"),
499                            name: label("runnerd.name"),
500                            created_at: label("runnerd.created-at")
501                                .and_then(|at| at.parse().ok())
502                                .unwrap_or(0),
503                            bytes: image.bytes,
504                            image: image.reference,
505                        })
506                    })
507                    .collect();
508                records.sort_by_key(|record| (record.created_at, record.job));
509                for record in &mut records {
510                    self.name_from_history(record).await;
511                    // Committed before an earlier daemon stopped, which did not get to note it.
512                    let noted = self.store.job(record.job).await.ok().flatten();
513                    if noted
514                        .is_some_and(|job| job.record.is_none_or(|note| note.created_at.is_none()))
515                    {
516                        self.store.record_made(record.clone());
517                    }
518                }
519                self.records.list = records;
520                self.records.listed = true;
521            }
522            Err(error) => warn!("cannot list the records: {error}"),
523        }
524    }
525
526    /// Removes the records past `max_age`.
527    pub(super) async fn expire_records(&mut self) {
528        let expired = record::expired(
529            &self.records.list,
530            &self.shell_jobs(),
531            &self.config.records,
532            time::now_ms() / 1000,
533        );
534        for (job, why) in expired {
535            if let Err(error) = self.remove_record(job, why).await {
536                warn!("{error}");
537            }
538        }
539    }
540
541    /// Removes one record's image, and its layers with it.
542    async fn remove_record(
543        &mut self,
544        job: u64,
545        why: api::RecordRemoval,
546    ) -> Result<u64, ControlError> {
547        let Some(index) = self
548            .records
549            .list
550            .iter()
551            .position(|record| record.job == job)
552        else {
553            return Err(ControlError::NotFound(format!(
554                "job {job} has no record; `runnerctl records` lists them"
555            )));
556        };
557        let image = self.records.list[index].image.clone();
558        self.docker.remove_image(&image).await.map_err(|error| {
559            ControlError::Failed(format!("cannot remove the record of job {job}: {error}"))
560        })?;
561        let removed = self.records.list.remove(index);
562        info!("removed the record of job {job} ({why})");
563        self.store.record_removed(job, time::now_ms(), why);
564        Ok(removed.bytes)
565    }
566
567    /// Removes the selected records, but those with a shell open: one asked for by its job is refused instead.
568    pub(super) async fn remove_records(
569        &mut self,
570        selection: &RecordSelection,
571    ) -> Result<api::RemovedRecords, ControlError> {
572        let open = self.shell_jobs();
573        let now = time::now_ms() / 1000;
574        let jobs: Vec<u64> = match selection {
575            RecordSelection::Job(job) => {
576                if open.contains(job) {
577                    return Err(ControlError::Conflict(format!(
578                        "a shell is open in the record of job {job}; remove it once the shell exits"
579                    )));
580                }
581                vec![*job]
582            }
583            RecordSelection::All => self.records.list.iter().map(|record| record.job).collect(),
584            RecordSelection::OlderThan(age) => self
585                .records
586                .list
587                .iter()
588                .filter(|record| now.saturating_sub(record.created_at) > age.as_secs())
589                .map(|record| record.job)
590                .collect(),
591        };
592        let mut removed = api::RemovedRecords::default();
593        for job in jobs.into_iter().filter(|job| !open.contains(job)) {
594            removed.bytes += self.remove_record(job, api::RecordRemoval::User).await?;
595            removed.jobs.push(job);
596        }
597        Ok(removed)
598    }
599
600    /// The jobs whose record has a shell open.
601    fn shell_jobs(&self) -> BTreeSet<u64> {
602        self.records
603            .shells
604            .values()
605            .map(|shell| shell.job)
606            .collect()
607    }
608
609    /// The profiles whose snapshot and shared caches a shell mounts.
610    pub(super) fn shell_profiles(&self) -> BTreeSet<&str> {
611        self.records
612            .shells
613            .values()
614            .map(|shell| shell.profile.as_str())
615            .collect()
616    }
617
618    /// The shells' part of the memory in use.
619    pub(super) fn shell_memory(&self) -> u64 {
620        self.records.shells.values().map(|shell| shell.memory).sum()
621    }
622
623    /// The shells' memory by job, for the published budget.
624    pub(super) fn shell_budget(&self) -> Vec<api::ShellMemory> {
625        self.records
626            .shells
627            .values()
628            .map(|shell| api::ShellMemory {
629                job: shell.job,
630                memory: shell.memory,
631            })
632            .collect()
633    }
634
635    /// Shows the records as they are now, if they changed.
636    pub(super) fn publish_records(&self) {
637        let records = self.records_status();
638        self.records.status.send_if_modified(|current| {
639            let changed = *current != records;
640            *current = records;
641            changed
642        });
643    }
644
645    /// Every record and the limits, as `GET /v1/records` shows them.
646    fn records_status(&self) -> api::Records {
647        let limits = &self.config.records;
648        api::Records {
649            records: self.records.list.clone(),
650            limits: api::RecordLimits {
651                max_total: limits.max_total.0,
652                max_count: limits.max_count,
653                max_age: limits.max_age.as_secs(),
654                min_free: limits.min_free.0,
655            },
656            shells: self.shell_jobs().into_iter().collect(),
657        }
658    }
659
660    /// Starts a shell in a job's record, with the record's profile's limits, if the budget has room for them as for
661    /// a slot of the profile.
662    pub(super) async fn open_shell(
663        &mut self,
664        job: u64,
665        size: TtySize,
666    ) -> Result<Tty, ControlError> {
667        let Some(record) = self
668            .records
669            .list
670            .iter()
671            .find(|record| record.job == job)
672            .cloned()
673        else {
674            return Err(ControlError::NotFound(format!(
675                "job {job} has no record; `runnerctl records` lists them"
676            )));
677        };
678        let profile = self.config.profiles.get(&record.profile);
679        let Some(profile) = profile.or_else(|| self.retired.get(&record.profile)) else {
680            return Err(ControlError::Invalid(format!(
681                "the record of job {job} ran in profile {}, which the config no longer has",
682                record.profile
683            )));
684        };
685        let (memory, cpus) = (profile.memory.0, profile.cpus);
686        // A reseed replaces the snapshot the shell's layers sit on.
687        if self.reseeds.contains_key(&record.profile) {
688            return Err(ControlError::Conflict(format!(
689                "{} is reseeding its snapshot, which a shell mounts; open it once the reseed is done",
690                record.profile
691            )));
692        }
693        // A trim removes the shared caches' volumes, which a shell mounts.
694        if self.trim.as_ref().is_some_and(|trim| !trim.is_finished()) {
695            return Err(ControlError::Conflict(
696                "the shared caches, which a shell mounts, are being trimmed; open it once the trim is done"
697                    .to_owned(),
698            ));
699        }
700        if let Some(max) = self.memory_max {
701            let in_use = self.memory_in_use();
702            if in_use.saturating_add(memory) > max {
703                return Err(ControlError::Conflict(format!(
704                    "the budget is full: {} of the slice's {} is held, and a shell takes {} as a slot of {} \
705                     does; open it once a job finishes",
706                    ByteSize(in_use),
707                    ByteSize(max),
708                    ByteSize(memory),
709                    record.profile
710                )));
711            }
712        }
713        // Under measured admission nothing starts at the high-water mark, a shell no more than a runner.
714        if self.admitting(&mut [], None).held {
715            return Err(ControlError::Conflict(format!(
716                "the slice is at {HIGH_WATER}% of its MemoryMax, where measured admission starts nothing; open it \
717                 once it falls under"
718            )));
719        }
720        let body = self.shell_body(&record, memory, cpus);
721        self.records.opened += 1;
722        let name = format!(
723            "runnerd-shell-{job}-{}-{}",
724            time::now_ms() / 1000,
725            self.records.opened
726        );
727        let tty = self
728            .docker
729            .shell(&name, body, size)
730            .await
731            .map_err(|error| ControlError::Failed(format!("cannot start a shell: {error}")))?;
732        info!("opened a shell in the record of job {job}");
733        self.records.shells.insert(
734            tty.container.clone(),
735            Shell {
736                job,
737                profile: record.profile,
738                memory,
739                opened: Instant::now(),
740            },
741        );
742        Ok(tty)
743    }
744
745    /// The container of a shell in `record`: the record's image under the profile's caps, its snapshot and shared
746    /// caches mounted read-only, and a terminal.
747    fn shell_body(&self, record: &api::Record, memory: u64, cpus: u32) -> ContainerCreateBody {
748        let workspace = record::workspace(&record.repository);
749        let limit = i64::try_from(memory).unwrap_or(i64::MAX);
750        let mut mounts = vec![Mount {
751            target: Some("/state".to_owned()),
752            typ: Some(MountType::VOLUME),
753            ..Default::default()
754        }];
755        if self.config.profiles.contains_key(&record.profile) {
756            // The snapshot and the shared caches as the job saw them, read-only: running jobs share them.
757            mounts.extend(
758                crate::cache::runner_mounts(&self.names, &record.profile, 1)
759                    .into_iter()
760                    .filter(|mount| mount.target.as_deref() != Some("/state"))
761                    .map(|mount| Mount {
762                        read_only: Some(true),
763                        ..mount
764                    }),
765            );
766        }
767        let mut host_config = HostConfig {
768            cgroup_parent: Some(self.config.host.slice.clone()),
769            memory: Some(limit),
770            memory_swap: Some(limit),
771            nano_cpus: Some(i64::from(cpus) * 1_000_000_000),
772            cpu_shares: Some(cpu_shares(cpus)),
773            init: Some(true),
774            mounts: Some(mounts),
775            network_mode: Some(self.names.network()),
776            ..Default::default()
777        };
778        self.isolation.configure(&mut host_config);
779        ContainerCreateBody {
780            image: Some(record.image.clone()),
781            entrypoint: Some(vec![
782                "bash".to_owned(),
783                "-c".to_owned(),
784                SHELL_SCRIPT.to_owned(),
785            ]),
786            cmd: Some(vec!["shell".to_owned(), workspace]),
787            env: Some(vec!["TERM=xterm-256color".to_owned()]),
788            tty: Some(true),
789            open_stdin: Some(true),
790            stdin_once: Some(true),
791            attach_stdin: Some(true),
792            attach_stdout: Some(true),
793            attach_stderr: Some(true),
794            labels: Some(HashMap::from([(
795                record::SHELL_LABEL.to_owned(),
796                record.job.to_string(),
797            )])),
798            host_config: Some(host_config),
799            ..Default::default()
800        }
801    }
802
803    /// Removes a shell's container and frees its memory. One Docker does not remove now goes at a later tick.
804    pub(super) async fn close_shell(&mut self, container: &str) {
805        let Some(shell) = self.records.shells.get(container) else {
806            return;
807        };
808        let job = shell.job;
809        match self.docker.remove(container).await {
810            Err(error) if !is_not_found(&error) => {
811                warn!("cannot remove the shell in the record of job {job}: {error}");
812            }
813            _ => {
814                info!("closed the shell in the record of job {job}");
815                self.records.shells.remove(container);
816            }
817        }
818    }
819
820    /// Removes the shells whose process exited, whatever became of their terminal, and those open longer than
821    /// `MAX_SHELL`.
822    pub(super) async fn collect_shells(&mut self) {
823        let shells: Vec<(String, Instant)> = self
824            .records
825            .shells
826            .iter()
827            .map(|(container, shell)| (container.clone(), shell.opened))
828            .collect();
829        for (container, opened) in shells {
830            if opened.elapsed() > MAX_SHELL {
831                info!("closing {container}: a shell is open for 8 hours at most");
832                self.close_shell(&container).await;
833                continue;
834            }
835            let ended = match self.docker.inspect(&container).await {
836                Ok(state) => !state.running,
837                Err(error) => is_not_found(&error),
838            };
839            if ended {
840                self.close_shell(&container).await;
841            }
842        }
843    }
844
845    /// Removes every shell: on shutdown, and those an earlier daemon left, which no terminal is attached to.
846    pub(super) async fn close_shells(&mut self) {
847        let containers: Vec<String> = self.records.shells.keys().cloned().collect();
848        for container in containers {
849            self.close_shell(&container).await;
850        }
851    }
852}
853
854/// The labels a record's image carries, which `list_records` reads back.
855fn labels(record: &api::Record) -> BTreeMap<String, String> {
856    let mut labels = BTreeMap::from([
857        (record::LABEL.to_owned(), record.job.to_string()),
858        ("runnerd.repository".to_owned(), record.repository.clone()),
859        ("runnerd.profile".to_owned(), record.profile.clone()),
860        (
861            "runnerd.created-at".to_owned(),
862            record.created_at.to_string(),
863        ),
864    ]);
865    if let Some(workflow) = &record.workflow {
866        labels.insert("runnerd.workflow".to_owned(), workflow.clone());
867    }
868    if let Some(name) = &record.name {
869        labels.insert("runnerd.name".to_owned(), name.clone());
870    }
871    labels
872}
873
874/// Checks that a stopped runner container may be kept, and measures it: refuses one whose changes still hold a
875/// secret, and estimates the disk its commit takes. Docker walks the container's files for both, so it runs in a
876/// task.
877async fn check(docker: &impl DockerApi, container: &str) -> Result<Checked, String> {
878    let changed = docker
879        .changes(container)
880        .await
881        .map_err(|error| format!("Docker did not list its files: {error}"))?;
882    let secrets = record::secrets_left(changed.iter().map(String::as_str));
883    if !secrets.is_empty() {
884        return Err(format!(
885            "its runner left files a record must not keep ({}): its image did not delete them; use the image of \
886             runnerd {}, or rebuild image_stage from its image/",
887            record::listed(&secrets),
888            env!("CARGO_PKG_VERSION")
889        ));
890    }
891    let incoming = docker
892        .written(container)
893        .await
894        .map_err(|error| format!("Docker did not measure it: {error}"))?
895        .saturating_mul(COMMIT_FACTOR);
896    let free = match docker.free_space().await {
897        Ok(free) => Some(free),
898        Err(error) => {
899            warn!("cannot measure Docker's free disk: {error}");
900            None
901        }
902    };
903    Ok(Checked { incoming, free })
904}