Skip to main content

runnerd_core/scheduler/engine/
records.rs

1//! Records of failed jobs (`crate::record`): keeps a failed job's stopped runner container as an image before it is
2//! removed, keeps the records within `[records]`, removes them on request, and runs the shells opened in them, whose
3//! memory counts against the budget like a slot's. It must not serve a shell's terminal: the daemon does, with the
4//! `Tty` `open_shell` returns, and closes it with `Control::close_shell`.
5
6use std::collections::{BTreeMap, BTreeSet, HashMap};
7use std::time::Duration;
8
9use bollard::models::{ContainerCreateBody, HostConfig, Mount, MountType};
10use tokio::sync::{mpsc, watch};
11use tokio::task::JoinHandle;
12use tokio::time::Instant;
13use tracing::{info, warn};
14
15use super::control::ControlError;
16use super::{Engine, Occupant};
17use crate::admission::HIGH_WATER;
18use crate::api;
19use crate::config::ByteSize;
20use crate::docker::{self, DockerApi, Tty, TtySize, cpu_shares, is_not_found};
21use crate::github::GithubApi;
22use crate::observe::capture;
23use crate::record;
24use crate::scheduler::SlotRef;
25use crate::time;
26
27/// A record being made: its container checked and measured, then committed and removed, each in a task of its own
28/// so the engine's loop does not wait for Docker's walk of the container's files.
29#[derive(Debug)]
30pub(super) struct Recording {
31    record: api::Record,
32    container: String,
33    /// The disk its commit is expected to take, once measured: what the records made after it count against
34    /// `min_free` until it is done.
35    estimate: u64,
36    stage: Stage,
37}
38
39#[derive(Debug)]
40enum Stage {
41    /// Its changes checked for secrets, and its size and Docker's free disk measured.
42    Checking(JoinHandle<Result<Checked, String>>),
43    /// The commit, then the container's removal.
44    Committing(JoinHandle<Committed>),
45}
46
47/// What a commit's task came to.
48#[derive(Debug)]
49struct Committed {
50    /// The record's size, or why Docker did not commit it.
51    bytes: Result<u64, String>,
52    /// Docker did not remove the container after it, so a tick has to.
53    left: bool,
54}
55
56impl Stage {
57    fn is_finished(&self) -> bool {
58        match self {
59            Stage::Checking(task) => task.is_finished(),
60            Stage::Committing(task) => task.is_finished(),
61        }
62    }
63}
64
65/// What checking a container found: it may be kept, at about `incoming` bytes, with `free` bytes on Docker's disk
66/// when known.
67#[derive(Debug)]
68struct Checked {
69    incoming: u64,
70    free: Option<u64>,
71}
72
73/// Why recovery cannot yet tell whether a stopped runner container becomes its failed job's record.
74#[derive(Debug)]
75pub(super) enum Undecided {
76    /// Docker failed, and may not when asked again.
77    Docker(docker::Error),
78    /// Its diagnostic files were not all read, and those read name no job.
79    Unread,
80}
81
82impl std::fmt::Display for Undecided {
83    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
84        match self {
85            Undecided::Docker(error) => error.fmt(f),
86            Undecided::Unread => f.write_str("its diagnostic files were not all read"),
87        }
88    }
89}
90
91/// The records of failed jobs and the shells open in them; only its methods write them.
92///
93/// A restart keeps none of it: the list is read again from Docker's images, a recording's container is found as a
94/// leftover and `keep_left_record` takes it up, and recovery removes the shells an earlier daemon left. A reload keeps
95/// all of it.
96#[derive(Debug)]
97pub(super) struct RecordState {
98    /// Oldest first; `None` until Docker lists them, and until then no record is made.
99    list: Option<Vec<api::Record>>,
100    /// Records being made, by job.
101    recordings: BTreeMap<u64, Recording>,
102    /// Shells open in records, by container ID.
103    shells: BTreeMap<String, Shell>,
104    /// Shells opened by this engine, which tells their containers' names apart.
105    opened: u64,
106    /// What `Handle::records` shows.
107    status: watch::Sender<api::Records>,
108    /// A recording's tasks say here that they are done, so the loop moves them on without waiting for a tick.
109    done: mpsc::UnboundedSender<()>,
110    pub(super) finished: mpsc::UnboundedReceiver<()>,
111}
112
113impl RecordState {
114    /// No record known yet, and the receiver of what `publish_records` shows.
115    pub(super) fn new() -> (RecordState, watch::Receiver<api::Records>) {
116        let (status, receiver) = watch::channel(api::Records::default());
117        let (done, finished) = mpsc::unbounded_channel();
118        let state = RecordState {
119            list: None,
120            recordings: BTreeMap::new(),
121            shells: BTreeMap::new(),
122            opened: 0,
123            status,
124            done,
125            finished,
126        };
127        (state, receiver)
128    }
129
130    /// Whether the list is what Docker has.
131    fn is_listed(&self) -> bool {
132        self.list.is_some()
133    }
134
135    /// The records known, oldest first: none before Docker listed them.
136    fn kept(&self) -> &[api::Record] {
137        self.list.as_deref().unwrap_or_default()
138    }
139
140    /// The record of `job`, if one is known.
141    fn of_job(&self, job: u64) -> Option<&api::Record> {
142        self.kept().iter().find(|record| record.job == job)
143    }
144
145    /// Takes what Docker listed as the records.
146    fn set_listed(&mut self, records: Vec<api::Record>) {
147        self.list = Some(records);
148    }
149
150    /// Adds a committed record. A record is only made once the list is known.
151    fn keep(&mut self, record: api::Record) {
152        self.list.get_or_insert_default().push(record);
153    }
154
155    /// Forgets `job`'s record, and returns the bytes it took.
156    fn forget(&mut self, job: u64) -> u64 {
157        let Some(list) = self.list.as_mut() else {
158            return 0;
159        };
160        let Some(index) = list.iter().position(|record| record.job == job) else {
161            return 0;
162        };
163        list.remove(index).bytes
164    }
165
166    /// Whether any record is being made.
167    pub(super) fn is_recording(&self) -> bool {
168        !self.recordings.is_empty()
169    }
170
171    /// The disk the records being made are expected to take.
172    fn pending(&self) -> u64 {
173        self.recordings.values().map(|other| other.estimate).sum()
174    }
175
176    /// Notes `job`'s record as being made, at this stage.
177    fn begin(&mut self, job: u64, recording: Recording) {
178        self.recordings.insert(job, recording);
179    }
180
181    /// The jobs whose recording's task finished.
182    fn finished_recordings(&self) -> Vec<u64> {
183        self.recordings
184            .iter()
185            .filter(|(_, recording)| recording.stage.is_finished())
186            .map(|(job, _)| *job)
187            .collect()
188    }
189
190    /// Takes `job`'s recording, to move it on.
191    fn take_recording(&mut self, job: u64) -> Option<Recording> {
192        self.recordings.remove(&job)
193    }
194
195    /// Whether a shell is open in any record.
196    pub(super) fn has_shells(&self) -> bool {
197        !self.shells.is_empty()
198    }
199
200    /// The number of the next shell, which tells its container's name apart.
201    fn next_shell(&mut self) -> u64 {
202        self.opened += 1;
203        self.opened
204    }
205
206    /// Notes a shell opened in `container`.
207    fn shell_opened(&mut self, container: String, shell: Shell) {
208        self.shells.insert(container, shell);
209    }
210
211    /// Forgets the shell of `container`.
212    fn shell_closed(&mut self, container: &str) {
213        self.shells.remove(container);
214    }
215}
216
217/// A shell open in a record.
218#[derive(Debug, Clone)]
219pub(super) struct Shell {
220    job: u64,
221    /// The profile the record's job ran in, whose snapshot and shared caches the shell mounts.
222    pub(super) profile: String,
223    /// Its container's memory cap, which counts against the budget.
224    pub(super) memory: u64,
225    opened: Instant,
226}
227
228/// The longest a shell stays open: a forgotten one holds a slot's memory and keeps its record from going.
229const MAX_SHELL: Duration = Duration::from_hours(8);
230
231/// Which records a removal asks for.
232#[derive(Debug, Clone, PartialEq, Eq)]
233pub enum RecordSelection {
234    /// The record of one job.
235    Job(u64),
236    /// Every record without a shell open.
237    All,
238    /// The records older than this, without a shell open.
239    OlderThan(Duration),
240}
241
242/// What a shell container runs, as root: the profile's snapshot under the cache layers as a runner sees them, with
243/// the shell's own changes in its anonymous /state, then a login shell as `runner` in the directory `$1`.
244const SHELL_SCRIPT: &str = r#"
245. /usr/local/lib/runnerd-layers.sh && mount_layers > /dev/null 2>&1
246install -d -o runner -g runner -m 700 "$XDG_RUNTIME_DIR" 2> /dev/null
247cd "$1" 2> /dev/null || cd /home/runner
248exec setpriv --reuid=runner --regid=runner --init-groups -- env HOME=/home/runner USER=runner LOGNAME=runner bash -l
249"#;
250
251/// The disk Docker takes per byte a container's layer holds once committed: the containerd image store keeps the
252/// layer's archive besides its unpacked copy.
253const COMMIT_FACTOR: u64 = 2;
254
255impl<G: GithubApi + 'static, D: DockerApi + 'static> Engine<G, D> {
256    /// Whether the profile of a slot keeps its failed jobs' runners.
257    fn records_failures(&self, profile: &str) -> bool {
258        self.config
259            .profiles
260            .get(profile)
261            .or_else(|| self.retired.get(profile))
262            .is_some_and(|profile| profile.record_failures)
263    }
264
265    /// Starts keeping a stopped runner container whose job failed as the job's record: checks and measures it, makes
266    /// room within the limits, then commits and removes it, in the background. Returns whether it took the container
267    /// over; if not, the caller removes it.
268    pub(super) async fn begin_record(&mut self, slot: &SlotRef, occupant: &Occupant) -> bool {
269        let Some(job) = self.named_job(occupant) else {
270            return false;
271        };
272        if !self.records_failures(&slot.profile) {
273            return false;
274        }
275        if !self.records.is_listed() {
276            self.list_records().await;
277            if !self.records.is_listed() {
278                let why = "Docker did not list the records kept".to_owned();
279                self.skip_record(job, &occupant.repository, why);
280                return false;
281            }
282        }
283        let summary = self
284            .store
285            .job(job)
286            .await
287            .ok()
288            .flatten()
289            .map(|job| job.summary);
290        let record = api::Record {
291            job,
292            repository: occupant.repository.clone(),
293            profile: slot.profile.clone(),
294            workflow: summary
295                .as_ref()
296                .and_then(|summary| summary.workflow.clone()),
297            name: summary.map(|summary| summary.name),
298            created_at: time::now_ms() / 1000,
299            bytes: 0,
300            image: record::image(job),
301        };
302        info!(
303            "{}: keeping job {job}'s runner as {}",
304            occupant.repository, record.image
305        );
306        let docker = self.docker.clone();
307        let container = occupant.container.clone();
308        let done = self.records.done.clone();
309        let check = tokio::spawn(async move {
310            let checked = check(&*docker, &container).await;
311            let _ = done.send(());
312            checked
313        });
314        self.records.begin(
315            job,
316            Recording {
317                record,
318                container: occupant.container.clone(),
319                estimate: 0,
320                stage: Stage::Checking(check),
321            },
322        );
323        true
324    }
325
326    /// Keeps a stopped runner container an earlier daemon left as its job's record, as `release` would have, if its
327    /// profile keeps failed jobs' runners, its log shows its job failed and no record of the job was made: the runner
328    /// ended while no daemon ran, or a daemon stopped before it collected it or finished the record. A job the history
329    /// never saw it take, as one an idle runner took while the daemon detached, is read from the runner's worker log.
330    /// Returns whether it took the container over; if not, the caller removes it. `Undecided` when Docker could not
331    /// tell, and may once asked again: the caller keeps the container to try again.
332    pub(super) async fn keep_left_record(
333        &mut self,
334        slot: &SlotRef,
335        occupant: &Occupant,
336    ) -> Result<bool, Undecided> {
337        if !self.records_failures(&slot.profile)
338            || occupant
339                .named_job()
340                .is_some_and(|job| self.records.of_job(job).is_some())
341        {
342            return Ok(false);
343        }
344        let failed = match self.docker.logs(&occupant.container).await {
345            Ok(log) => record::failed(&log),
346            Err(error) if docker::is_declined(&error) => false,
347            Err(error) => return Err(Undecided::Docker(error)),
348        };
349        if !failed {
350            return Ok(false);
351        }
352        let job = match occupant.named_job() {
353            Some(job) => Some(job),
354            None => self.left_job(&occupant.container).await?,
355        };
356        let Some(job) = job else {
357            warn!(
358                "{}: {} ran a job that failed, but names it nowhere: no record of it is kept",
359                occupant.repository, occupant.runner_name
360            );
361            return Ok(false);
362        };
363        if self.records.of_job(job).is_some() {
364            return Ok(false);
365        }
366        let mut occupant = occupant.clone();
367        occupant.engagement.named_by_worker_log(job);
368        Ok(self.begin_record(slot, &occupant).await)
369    }
370
371    /// The job a stopped runner container took, from its worker log's job message, read without starting it: the
372    /// capture reads the same while it runs. `None` when the runner wrote none, or Docker says it cannot read it.
373    async fn left_job(&self, container: &str) -> Result<Option<u64>, Undecided> {
374        let read = match self
375            .docker
376            .files(container, capture::DIAG, capture::WORKER_LOG)
377            .await
378        {
379            Ok(read) => read,
380            Err(error) if docker::is_declined(&error) => return Ok(None),
381            Err(error) => return Err(Undecided::Docker(error)),
382        };
383        match read
384            .files
385            .iter()
386            .find_map(|file| capture::worker_job(&file.contents))
387        {
388            Some(job) => Ok(Some(job)),
389            // A worker log the read did not reach may name it.
390            None if !read.complete => Err(Undecided::Unread),
391            None => Ok(None),
392        }
393    }
394
395    /// Notes in the history why a failed job has no record.
396    fn skip_record(&self, job: u64, repository: &str, why: String) {
397        warn!("{repository}: no record of job {job}: {why}");
398        self.store.record_skipped(job, why);
399    }
400
401    /// Makes room for a checked container within the limits, removing the records in its way, then commits it in
402    /// the background; or notes why it is not kept and removes it.
403    async fn make_room_and_commit(
404        &mut self,
405        record: api::Record,
406        container: String,
407        checked: Checked,
408    ) {
409        let job = record.job;
410        // Records being committed take their disk too.
411        let pending = self.records.pending();
412        let room = record::make_room(
413            self.records.kept(),
414            &self.shell_jobs(),
415            &self.config.records,
416            checked.incoming,
417            checked.free.map(|free| free.saturating_sub(pending)),
418            time::now_ms() / 1000,
419        );
420        let removed = match room {
421            Ok(removals) => {
422                let mut removed = Ok(());
423                for (old, why) in removals {
424                    if let Err(error) = self.remove_record(old, why).await {
425                        removed = Err(error.to_string());
426                        break;
427                    }
428                }
429                removed
430            }
431            Err(why) => Err(why),
432        };
433        if let Err(why) = removed {
434            self.skip_record(job, &record.repository, why);
435            self.remove_container(&container).await;
436            return;
437        }
438        let labels = labels(&record);
439        let docker = self.docker.clone();
440        let image = record.image.clone();
441        let done = self.records.done.clone();
442        let committed_container = container.clone();
443        let commit = tokio::spawn(async move {
444            let container = committed_container;
445            let bytes = docker
446                .commit(&container, &image, labels)
447                .await
448                .map_err(|error| error.to_string());
449            let left = match docker.remove(&container).await {
450                Err(error) if !is_not_found(&error) => {
451                    warn!("cannot remove {container}: {error}");
452                    true
453                }
454                _ => false,
455            };
456            let _ = done.send(());
457            Committed { bytes, left }
458        });
459        self.records.begin(
460            job,
461            Recording {
462                record,
463                container,
464                estimate: checked.incoming,
465                stage: Stage::Committing(commit),
466            },
467        );
468    }
469
470    /// Removes a container no record came of; each tick tries again if Docker does not.
471    async fn remove_container(&mut self, container: &str) {
472        if let Err(error) = self.docker.remove(container).await
473            && !is_not_found(&error)
474        {
475            warn!("cannot remove {container}: {error}");
476            self.leftovers
477                .removing(container.to_owned(), container.to_owned());
478        }
479    }
480
481    /// Moves the recordings whose task finished on: a checked container to its commit, a committed one into the
482    /// records, kept within their limits with its real size.
483    pub(super) async fn collect_recordings(&mut self) {
484        while self.records.finished.try_recv().is_ok() {}
485        for job in self.records.finished_recordings() {
486            let Some(recording) = self.records.take_recording(job) else {
487                continue;
488            };
489            let Recording {
490                record,
491                container,
492                stage,
493                ..
494            } = recording;
495            let task = match stage {
496                Stage::Checking(task) => {
497                    let checked = task
498                        .await
499                        .map_err(|error| error.to_string())
500                        .and_then(|checked| checked);
501                    match checked {
502                        Ok(checked) => self.make_room_and_commit(record, container, checked).await,
503                        Err(why) => {
504                            self.skip_record(job, &record.repository, why);
505                            self.remove_container(&container).await;
506                        }
507                    }
508                    continue;
509                }
510                Stage::Committing(task) => task,
511            };
512            let bytes = match task.await {
513                Ok(Committed { bytes, left }) => {
514                    if left {
515                        self.leftovers.removing(container.clone(), container);
516                    }
517                    bytes
518                }
519                Err(error) => Err(error.to_string()),
520            };
521            match bytes {
522                Ok(bytes) => self.keep_record(record, bytes).await,
523                Err(error) => self.skip_record(
524                    job,
525                    &record.repository,
526                    format!("Docker did not commit it: {error}"),
527                ),
528            }
529        }
530    }
531
532    /// Adds a committed record, and removes the records past the limits with its real size.
533    async fn keep_record(&mut self, mut record: api::Record, bytes: u64) {
534        let job = record.job;
535        record.bytes = bytes;
536        self.name_from_history(&mut record).await;
537        info!(
538            "{}: kept job {job}'s runner as {} ({})",
539            record.repository,
540            record.image,
541            ByteSize(bytes)
542        );
543        let removals = record::make_room(
544            self.records.kept(),
545            &self.shell_jobs(),
546            &self.config.records,
547            bytes,
548            None,
549            record.created_at,
550        );
551        self.store.record_made(record.clone());
552        self.records.keep(record);
553        let removals = removals.unwrap_or_else(|_| vec![(job, api::RecordRemoval::Size)]);
554        for (old, why) in removals {
555            if let Err(error) = self.remove_record(old, why).await {
556                warn!("{error}");
557            }
558        }
559    }
560
561    /// Fills in a record's workflow and job name from the history, which may have learnt them from GitHub only after
562    /// the commit started. Reads after the writes sent before, such as those details: the store reads on its own
563    /// connection, which a busy writer leaves behind.
564    async fn name_from_history(&self, record: &mut api::Record) {
565        if record.workflow.is_some() && record.name.is_some() {
566            return;
567        }
568        self.store.flush().await;
569        if let Ok(Some(job)) = self.store.job(record.job).await {
570            record.workflow = record.workflow.take().or(job.summary.workflow);
571            record.name.get_or_insert(job.summary.name);
572        }
573    }
574
575    /// A tick's care of the records: lists them until Docker answers, then removes those past `max_age`.
576    pub(super) async fn keep_records(&mut self) {
577        if self.records.is_listed() {
578            self.expire_records().await;
579        } else {
580            self.list_records().await;
581        }
582    }
583
584    /// Lists the record images Docker has; until it answers, no record is made.
585    pub(super) async fn list_records(&mut self) {
586        match self.docker.images(record::LABEL).await {
587            Ok(images) => {
588                let mut records: Vec<api::Record> = images
589                    .into_iter()
590                    .filter_map(|image| {
591                        let label = |name: &str| image.labels.get(name).cloned();
592                        Some(api::Record {
593                            job: label(record::LABEL)?.parse().ok()?,
594                            repository: label("runnerd.repository").unwrap_or_default(),
595                            profile: label("runnerd.profile").unwrap_or_default(),
596                            workflow: label("runnerd.workflow"),
597                            name: label("runnerd.name"),
598                            created_at: label("runnerd.created-at")
599                                .and_then(|at| at.parse().ok())
600                                .unwrap_or(0),
601                            bytes: image.bytes,
602                            image: image.reference,
603                        })
604                    })
605                    .collect();
606                records.sort_by_key(|record| (record.created_at, record.job));
607                for record in &mut records {
608                    self.name_from_history(record).await;
609                    // Committed before an earlier daemon stopped, which did not get to note it.
610                    let noted = self.store.job(record.job).await.ok().flatten();
611                    if noted
612                        .is_some_and(|job| job.record.is_none_or(|note| note.created_at.is_none()))
613                    {
614                        self.store.record_made(record.clone());
615                    }
616                }
617                // Only an inventory Docker answered says a noted record is gone.
618                self.store.records_listed(
619                    records.iter().map(|record| record.job).collect(),
620                    time::now_ms(),
621                );
622                self.records.set_listed(records);
623            }
624            Err(error) => warn!("cannot list the records: {error}"),
625        }
626    }
627
628    /// Removes the records past `max_age`.
629    pub(super) async fn expire_records(&mut self) {
630        let expired = record::expired(
631            self.records.kept(),
632            &self.shell_jobs(),
633            &self.config.records,
634            time::now_ms() / 1000,
635        );
636        for (job, why) in expired {
637            if let Err(error) = self.remove_record(job, why).await {
638                warn!("{error}");
639            }
640        }
641    }
642
643    /// Removes one record's image, and its layers with it.
644    async fn remove_record(
645        &mut self,
646        job: u64,
647        why: api::RecordRemoval,
648    ) -> Result<u64, ControlError> {
649        let Some(image) = self.records.of_job(job).map(|record| record.image.clone()) else {
650            return Err(ControlError::NotFound(format!(
651                "job {job} has no record; `runnerctl records` lists them"
652            )));
653        };
654        self.docker.remove_image(&image).await.map_err(|error| {
655            ControlError::Failed(format!("cannot remove the record of job {job}: {error}"))
656        })?;
657        let bytes = self.records.forget(job);
658        info!("removed the record of job {job} ({why})");
659        self.store.record_removed(job, time::now_ms(), why);
660        Ok(bytes)
661    }
662
663    /// Removes the selected records, but those with a shell open: one asked for by its job is refused instead.
664    pub(super) async fn remove_records(
665        &mut self,
666        selection: &RecordSelection,
667    ) -> Result<api::RemovedRecords, ControlError> {
668        let open = self.shell_jobs();
669        let now = time::now_ms() / 1000;
670        let jobs: Vec<u64> = match selection {
671            RecordSelection::Job(job) => {
672                if open.contains(job) {
673                    return Err(ControlError::Conflict(format!(
674                        "a shell is open in the record of job {job}; remove it once the shell exits"
675                    )));
676                }
677                vec![*job]
678            }
679            RecordSelection::All => self
680                .records
681                .kept()
682                .iter()
683                .map(|record| record.job)
684                .collect(),
685            RecordSelection::OlderThan(age) => self
686                .records
687                .kept()
688                .iter()
689                .filter(|record| now.saturating_sub(record.created_at) > age.as_secs())
690                .map(|record| record.job)
691                .collect(),
692        };
693        let mut removed = api::RemovedRecords::default();
694        for job in jobs.into_iter().filter(|job| !open.contains(job)) {
695            removed.bytes += self.remove_record(job, api::RecordRemoval::User).await?;
696            removed.jobs.push(job);
697        }
698        Ok(removed)
699    }
700
701    /// The jobs whose record has a shell open.
702    fn shell_jobs(&self) -> BTreeSet<u64> {
703        self.records
704            .shells
705            .values()
706            .map(|shell| shell.job)
707            .collect()
708    }
709
710    /// The profiles whose snapshot and shared caches a shell mounts.
711    pub(super) fn shell_profiles(&self) -> BTreeSet<&str> {
712        self.records
713            .shells
714            .values()
715            .map(|shell| shell.profile.as_str())
716            .collect()
717    }
718
719    /// The shells' part of the memory in use.
720    pub(super) fn shell_memory(&self) -> u64 {
721        self.records.shells.values().map(|shell| shell.memory).sum()
722    }
723
724    /// The shells' memory by job, for the published budget.
725    pub(super) fn shell_budget(&self) -> Vec<api::ShellMemory> {
726        self.records
727            .shells
728            .values()
729            .map(|shell| api::ShellMemory {
730                job: shell.job,
731                memory: shell.memory,
732            })
733            .collect()
734    }
735
736    /// Shows the records as they are now, if they changed.
737    pub(super) fn publish_records(&self) {
738        let records = self.records_status();
739        self.records.status.send_if_modified(|current| {
740            let changed = *current != records;
741            *current = records;
742            changed
743        });
744    }
745
746    /// Every record and the limits, as `GET /v1/records` shows them.
747    fn records_status(&self) -> api::Records {
748        let limits = &self.config.records;
749        api::Records {
750            records: self.records.kept().to_vec(),
751            limits: api::RecordLimits {
752                max_total: limits.max_total.0,
753                max_count: limits.max_count,
754                max_age: limits.max_age.as_secs(),
755                min_free: limits.min_free.0,
756            },
757            shells: self.shell_jobs().into_iter().collect(),
758        }
759    }
760
761    /// Starts a shell in a job's record, with the record's profile's limits, if the budget has room for them as for
762    /// a slot of the profile.
763    pub(super) async fn open_shell(
764        &mut self,
765        job: u64,
766        size: TtySize,
767    ) -> Result<Tty, ControlError> {
768        let Some(record) = self.records.of_job(job).cloned() else {
769            return Err(ControlError::NotFound(format!(
770                "job {job} has no record; `runnerctl records` lists them"
771            )));
772        };
773        let profile = self.config.profiles.get(&record.profile);
774        let Some(profile) = profile.or_else(|| self.retired.get(&record.profile)) else {
775            return Err(ControlError::Invalid(format!(
776                "the record of job {job} ran in profile {}, which the config no longer has",
777                record.profile
778            )));
779        };
780        let (memory, cpus) = (profile.memory.0, profile.cpus);
781        // A reseed replaces the snapshot the shell's layers sit on.
782        if self.upkeep.reseed_runs(&record.profile) {
783            return Err(ControlError::Conflict(format!(
784                "{} is reseeding its snapshot, which a shell mounts; open it once the reseed is done",
785                record.profile
786            )));
787        }
788        // A trim removes the shared caches' volumes, which a shell mounts.
789        if self.trim.as_ref().is_some_and(|trim| !trim.is_finished()) {
790            return Err(ControlError::Conflict(
791                "the shared caches, which a shell mounts, are being trimmed; open it once the trim is done"
792                    .to_owned(),
793            ));
794        }
795        // So does their emptying after a host crash that may have torn them (ADR 0141).
796        if self.unsynced.is_torn() {
797            return Err(ControlError::Conflict(
798                "the shared caches, which a shell mounts, are emptied after a host crash; open it once they are"
799                    .to_owned(),
800            ));
801        }
802        if let Some(max) = self.memory_max {
803            let in_use = self.count(self.host_remembered()).pooled();
804            if in_use.saturating_add(memory) > max {
805                return Err(ControlError::Conflict(format!(
806                    "the budget is full: {} of the slice's {} is counted, and a shell takes {} as a slot of {} \
807                     does; open it once a job finishes",
808                    ByteSize(in_use),
809                    ByteSize(max),
810                    ByteSize(memory),
811                    record.profile
812                )));
813            }
814        }
815        // Under measured admission nothing starts at the high-water mark, a shell no more than a runner.
816        if self.admitting(&mut [], None).held {
817            return Err(ControlError::Conflict(format!(
818                "the slice is at {HIGH_WATER}% of its MemoryMax, where measured admission starts nothing; open it \
819                 once it falls under"
820            )));
821        }
822        let body = self.shell_body(&record, memory, cpus);
823        let name = format!(
824            "runnerd-shell-{job}-{}-{}",
825            time::now_ms() / 1000,
826            self.records.next_shell()
827        );
828        let tty = self
829            .docker
830            .shell(&name, body, size)
831            .await
832            .map_err(|error| ControlError::Failed(format!("cannot start a shell: {error}")))?;
833        info!("opened a shell in the record of job {job}");
834        self.records.shell_opened(
835            tty.container.clone(),
836            Shell {
837                job,
838                profile: record.profile,
839                memory,
840                opened: Instant::now(),
841            },
842        );
843        Ok(tty)
844    }
845
846    /// The container of a shell in `record`: the record's image under the profile's caps, its snapshot and shared
847    /// caches mounted read-only, and a terminal.
848    fn shell_body(&self, record: &api::Record, memory: u64, cpus: u32) -> ContainerCreateBody {
849        let workspace = record::workspace(&record.repository);
850        let limit = i64::try_from(memory).unwrap_or(i64::MAX);
851        let mut mounts = vec![Mount {
852            target: Some("/state".to_owned()),
853            typ: Some(MountType::VOLUME),
854            ..Default::default()
855        }];
856        if self.config.profiles.contains_key(&record.profile) {
857            // The snapshot and the shared caches as the job saw them, read-only: running jobs share them.
858            mounts.extend(
859                crate::cache::runner_mounts(&self.names, &record.profile, 1)
860                    .into_iter()
861                    .filter(|mount| mount.target.as_deref() != Some("/state"))
862                    .map(|mount| Mount {
863                        read_only: Some(true),
864                        ..mount
865                    }),
866            );
867        }
868        let mut host_config = HostConfig {
869            cgroup_parent: Some(self.config.host.slice.clone()),
870            memory: Some(limit),
871            memory_swap: Some(limit),
872            nano_cpus: Some(i64::from(cpus) * 1_000_000_000),
873            cpu_shares: Some(cpu_shares(cpus)),
874            init: Some(true),
875            mounts: Some(mounts),
876            network_mode: Some(self.names.network()),
877            ..Default::default()
878        };
879        self.isolation.configure(&mut host_config);
880        ContainerCreateBody {
881            image: Some(record.image.clone()),
882            entrypoint: Some(vec![
883                "bash".to_owned(),
884                "-c".to_owned(),
885                SHELL_SCRIPT.to_owned(),
886            ]),
887            cmd: Some(vec!["shell".to_owned(), workspace]),
888            env: Some(vec!["TERM=xterm-256color".to_owned()]),
889            tty: Some(true),
890            open_stdin: Some(true),
891            stdin_once: Some(true),
892            attach_stdin: Some(true),
893            attach_stdout: Some(true),
894            attach_stderr: Some(true),
895            labels: Some(HashMap::from([(
896                record::SHELL_LABEL.to_owned(),
897                record.job.to_string(),
898            )])),
899            host_config: Some(host_config),
900            ..Default::default()
901        }
902    }
903
904    /// Removes a shell's container and frees its memory. One Docker does not remove now goes at a later tick.
905    pub(super) async fn close_shell(&mut self, container: &str) {
906        let Some(shell) = self.records.shells.get(container) else {
907            return;
908        };
909        let job = shell.job;
910        match self.docker.remove(container).await {
911            Err(error) if !is_not_found(&error) => {
912                warn!("cannot remove the shell in the record of job {job}: {error}");
913            }
914            _ => {
915                info!("closed the shell in the record of job {job}");
916                self.records.shell_closed(container);
917            }
918        }
919    }
920
921    /// Removes the shells whose process exited, whatever became of their terminal, and those open longer than
922    /// `MAX_SHELL`.
923    pub(super) async fn collect_shells(&mut self) {
924        let shells: Vec<(String, Instant)> = self
925            .records
926            .shells
927            .iter()
928            .map(|(container, shell)| (container.clone(), shell.opened))
929            .collect();
930        for (container, opened) in shells {
931            if opened.elapsed() > MAX_SHELL {
932                info!("closing {container}: a shell is open for 8 hours at most");
933                self.close_shell(&container).await;
934                continue;
935            }
936            let ended = match self.docker.inspect(&container).await {
937                Ok(state) => !state.running,
938                Err(error) => is_not_found(&error),
939            };
940            if ended {
941                self.close_shell(&container).await;
942            }
943        }
944    }
945
946    /// Removes every shell: on shutdown, and those an earlier daemon left, which no terminal is attached to.
947    pub(super) async fn close_shells(&mut self) {
948        let containers: Vec<String> = self.records.shells.keys().cloned().collect();
949        for container in containers {
950            self.close_shell(&container).await;
951        }
952    }
953}
954
955/// The labels a record's image carries, which `list_records` reads back.
956fn labels(record: &api::Record) -> BTreeMap<String, String> {
957    let mut labels = BTreeMap::from([
958        (record::LABEL.to_owned(), record.job.to_string()),
959        ("runnerd.repository".to_owned(), record.repository.clone()),
960        ("runnerd.profile".to_owned(), record.profile.clone()),
961        (
962            "runnerd.created-at".to_owned(),
963            record.created_at.to_string(),
964        ),
965    ]);
966    if let Some(workflow) = &record.workflow {
967        labels.insert("runnerd.workflow".to_owned(), workflow.clone());
968    }
969    if let Some(name) = &record.name {
970        labels.insert("runnerd.name".to_owned(), name.clone());
971    }
972    labels
973}
974
975/// Checks that a stopped runner container may be kept, and measures it: refuses one whose changes still hold a
976/// secret, and estimates the disk its commit takes. Docker walks the container's files for both, so it runs in a
977/// task.
978async fn check(docker: &impl DockerApi, container: &str) -> Result<Checked, String> {
979    let changed = docker
980        .changes(container)
981        .await
982        .map_err(|error| format!("Docker did not list its files: {error}"))?;
983    let secrets = record::secrets_left(changed.iter().map(String::as_str));
984    if !secrets.is_empty() {
985        return Err(format!(
986            "its runner left files a record must not keep ({}): its image did not delete them; use the image of \
987             runnerd {}, or rebuild image_stage from its image/",
988            record::listed(&secrets),
989            env!("CARGO_PKG_VERSION")
990        ));
991    }
992    let incoming = docker
993        .written(container)
994        .await
995        .map_err(|error| format!("Docker did not measure it: {error}"))?
996        .saturating_mul(COMMIT_FACTOR);
997    let free = match docker.free_space().await {
998        Ok(free) => Some(free),
999        Err(error) => {
1000            warn!("cannot measure Docker's free disk: {error}");
1001            None
1002        }
1003    };
1004    Ok(Checked { incoming, free })
1005}