Skip to main content

runnerd_core/scheduler/engine/
control.rs

1//! Control from the API: the `Control` handle the daemon's API holds (pause flags, which it records in the store so
2//! a restart keeps them, shutdown, a channel of `Command`s) and the engine's side of each command: pause and resume,
3//! of one repository or all, with or without cancelling their runs, reseed, cache clean, config replacement, the
4//! removal of records and the shells opened in them.
5//! Commands run between ticks or while a tick waits for GitHub, so they see the slot state a tick does (ADRs 0019,
6//! 0026). It must not serve HTTP or write the config file: the daemon does, through `Commit`.
7
8use std::collections::{BTreeMap, BTreeSet};
9use std::sync::Mutex;
10use std::time::Duration;
11
12use tokio::sync::{Notify, mpsc, oneshot};
13use tracing::{info, warn};
14
15use super::records::RecordSelection;
16use super::{Engine, Occupant};
17use crate::api;
18use crate::cache;
19use crate::config::{ByteSize, Config};
20use crate::docker::{DockerApi, Tty, TtySize};
21use crate::github::GithubApi;
22use crate::isolation;
23use crate::scheduler::SlotRef;
24use crate::sizing::auto;
25use crate::store::Store;
26use crate::sync::lock;
27use crate::time;
28
29/// How long the runner of a run a forced pause cancelled gets to report its job cancelled and exit, before it is
30/// removed: GitHub completes a cancelled run only once its runner reports it, which took 40s in a test.
31pub(super) const CANCEL_GRACE: Duration = Duration::from_secs(90);
32
33/// How the daemon is asked to stop.
34#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
35pub enum Shutdown {
36    /// Keep scheduling.
37    No,
38    /// Start nothing new, let running jobs finish.
39    Graceful,
40    /// Kill running jobs and delete their runners.
41    Force,
42}
43
44/// What a pause, a shutdown or a runner's removal does with the jobs runners are running.
45#[derive(Debug, Clone, Copy, PartialEq, Eq)]
46pub enum RunningJobs {
47    /// Leave them running: a pause refuses, a shutdown waits for them, a busy runner stays.
48    Keep,
49    /// Kill them: a pause cancels their runs, a shutdown and a removal kill their runners.
50    Kill,
51}
52
53/// What a new config may do to slots with a runner in them.
54#[derive(Debug, Clone, Copy, PartialEq, Eq)]
55pub enum RunningSlots {
56    /// A slot past a lowered `slots`, or of a removed profile, stays until its runner finishes.
57    Drain,
58    /// The config is refused while a runner is in a slot it does not have: what auto sizing asks for.
59    Keep,
60}
61
62/// Who writes a new config.
63#[derive(Debug, Clone, Copy, PartialEq, Eq)]
64pub enum Author {
65    /// The owner, through `PUT /v1/config`: a save that lowers `memory` under a raise of auto sizing's in the last
66    /// day is answered with a warning (`auto::undone_raises`).
67    Owner,
68    /// Auto sizing, whose own lowering is never a warning about its raise.
69    AutoSizing,
70}
71
72/// `Err` when `new` moves runnerd to another slice than `running` names: the sampler and the running containers stay
73/// in the slice runnerd started in, so the slice changes only at a restart.
74pub fn same_slice(running: &Config, new: &Config) -> Result<(), ControlError> {
75    if new.host.slice == running.host.slice {
76        return Ok(());
77    }
78    Err(ControlError::Invalid(format!(
79        "restart runnerd to change the slice: it runs in {}, and host.slice is {} in the new config",
80        running.host.slice, new.host.slice
81    )))
82}
83
84/// The repositories a pause or a resume applies to.
85#[derive(Debug, Clone, PartialEq, Eq)]
86pub enum Target {
87    /// One repository, `owner/name`.
88    Repository(String),
89    /// Every configured repository.
90    All,
91}
92
93/// Why the engine refused or failed a command from the API.
94#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
95pub enum ControlError {
96    /// No such repository, profile or cache.
97    #[error("{0}")]
98    NotFound(String),
99    /// Runners use what the command would change.
100    #[error("{message}")]
101    Busy {
102        /// Names the runners and their jobs.
103        message: String,
104        /// The runners in the way.
105        jobs: Vec<api::RunningJob>,
106    },
107    /// The command conflicts with one already in progress.
108    #[error("{0}")]
109    Conflict(String),
110    /// The command cannot apply to this target.
111    #[error("{0}")]
112    Invalid(String),
113    /// Docker or GitHub failed.
114    #[error("{0}")]
115    Failed(String),
116    /// A pause or resume holds in memory, but the job history could not record it: a restart undoes it.
117    #[error("{0}")]
118    NotRecorded(String),
119    /// The engine stopped.
120    #[error("runnerd is stopping")]
121    Stopped,
122}
123
124/// A command the engine carries out between ticks or while a tick waits for GitHub, with where its answer goes.
125#[derive(Debug)]
126pub enum Command {
127    /// Pause one repository or all; with `RunningJobs::Kill`, cancel their running jobs.
128    Pause {
129        /// What to pause.
130        target: Target,
131        /// Cancel running jobs, or refuse while there are any.
132        jobs: RunningJobs,
133        /// The answer.
134        reply: oneshot::Sender<Result<api::Paused, ControlError>>,
135    },
136    /// Resume one repository or all.
137    Resume {
138        /// What to resume.
139        target: Target,
140        /// The answer.
141        reply: oneshot::Sender<Result<(), ControlError>>,
142    },
143    /// Make slot 1's layers the profile's snapshot once its slots are free.
144    Reseed {
145        /// The profile.
146        profile: String,
147        /// The answer.
148        reply: oneshot::Sender<Result<(), ControlError>>,
149    },
150    /// Remove a slot's volume or a shared cache's, unless a runner uses it.
151    CleanCache {
152        /// The volume.
153        name: String,
154        /// The answer.
155        reply: oneshot::Sender<Result<(), ControlError>>,
156    },
157    /// Write a new, validated config to its file and run with it, with no tick in between.
158    ReplaceConfig {
159        /// The config.
160        config: Box<Config>,
161        /// The `MemoryMax` of the slice it names.
162        memory_max: Option<u64>,
163        /// The CPUs of the slice it names, in millicores, read when it sets `cpu_overcommit`.
164        cpus: Option<u64>,
165        /// Writes the file.
166        commit: Commit,
167        /// Whether it may leave runners in slots it does not have.
168        running: RunningSlots,
169        /// Who writes it.
170        author: Author,
171        /// Answered once the engine runs with it: with the commit's warning and what it undoes of auto sizing's
172        /// raises, or why it did not commit.
173        reply: oneshot::Sender<Result<Vec<String>, ControlError>>,
174    },
175    /// Remove records of failed jobs.
176    RemoveRecords {
177        /// Which.
178        selection: RecordSelection,
179        /// The answer.
180        reply: oneshot::Sender<Result<api::RemovedRecords, ControlError>>,
181    },
182    /// Start a shell in a job's record, if the budget has room for it.
183    OpenShell {
184        /// The job.
185        job: u64,
186        /// Its terminal's size.
187        size: TtySize,
188        /// The shell's terminal.
189        reply: oneshot::Sender<Result<Tty, ControlError>>,
190    },
191    /// Remove a shell's container: its terminal closed.
192    CloseShell {
193        /// The container's ID.
194        container: String,
195    },
196}
197
198/// What makes a new config durable, run by the engine before it runs with the config: `Ok` once the config is in
199/// place, with a warning if it may not survive a crash; `Err` with nothing changed.
200pub struct Commit(Box<dyn FnOnce() -> Result<Option<String>, String> + Send>);
201
202impl Commit {
203    /// A commit that runs `write`, off the engine's thread.
204    pub fn new(write: impl FnOnce() -> Result<Option<String>, String> + Send + 'static) -> Commit {
205        Commit(Box::new(write))
206    }
207
208    /// Runs it.
209    pub async fn run(self) -> Result<Option<String>, String> {
210        tokio::task::spawn_blocking(self.0)
211            .await
212            .unwrap_or_else(|join_error| Err(join_error.to_string()))
213    }
214}
215
216impl std::fmt::Debug for Commit {
217    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
218        f.write_str("Commit")
219    }
220}
221
222/// The engine's end of `Control`'s commands.
223#[derive(Debug)]
224pub struct Commands(mpsc::UnboundedReceiver<Command>);
225
226impl Commands {
227    /// The next command; `None` once `Control` is gone.
228    pub async fn recv(&mut self) -> Option<Command> {
229        self.0.recv().await
230    }
231}
232
233/// Commands from the API and signals to the loop.
234#[derive(Debug)]
235pub struct Control {
236    // Locked with `sync::lock`: the guarded data stays consistent, as every critical section is a single insert,
237    // remove or read.
238    repositories: Mutex<BTreeSet<String>>,
239    paused: Mutex<BTreeSet<String>>,
240    /// Where `paused` is recorded.
241    store: Store,
242    shutdown: Mutex<Shutdown>,
243    pub(super) wake: Notify,
244    commands: mpsc::UnboundedSender<Command>,
245}
246
247impl Control {
248    /// Controls for the configured repositories, of which those in `paused` start paused, and the commands they
249    /// send the engine. A paused repository the config no longer has is forgotten, in `store` too: one added back
250    /// starts active.
251    pub fn new(
252        repositories: impl IntoIterator<Item = String>,
253        paused: impl IntoIterator<Item = String>,
254        store: Store,
255    ) -> (Control, Commands) {
256        let repositories: BTreeSet<String> = repositories.into_iter().collect();
257        let paused: BTreeSet<String> = paused
258            .into_iter()
259            .filter(|repository| repositories.contains(repository))
260            .collect();
261        // Sent at once; a write that fails leaves rows the next start filters out the same way.
262        drop(store.paused(paused.iter().cloned().collect()));
263        let (commands, receiver) = mpsc::unbounded_channel();
264        let control = Control {
265            repositories: Mutex::new(repositories),
266            paused: Mutex::new(paused),
267            store,
268            shutdown: Mutex::new(Shutdown::No),
269            wake: Notify::new(),
270            commands,
271        };
272        (control, Commands(receiver))
273    }
274
275    /// Whether a shutdown was requested, and how.
276    pub fn mode(&self) -> Shutdown {
277        *lock(&self.shutdown)
278    }
279
280    /// Pauses or resumes one repository or all, whatever they run, and records it in the store before it answers,
281    /// so a restarted daemon starts with the same ones paused. `NotFound` for a repository the config does not have.
282    /// If the record fails, the change holds until a restart and the answer is `NotRecorded`. The status shows it
283    /// at the engine's next publish: the API pauses and resumes through `pause` and `resume`, which publish before
284    /// they answer.
285    pub async fn set_paused(&self, target: &Target, paused: bool) -> Result<(), ControlError> {
286        let recorded = {
287            let repositories = lock(&self.repositories);
288            let names: Vec<&String> = match target {
289                Target::Repository(repository) => {
290                    let Some(name) = repositories.get(repository) else {
291                        return Err(not_configured(repository, repositories.iter()));
292                    };
293                    vec![name]
294                }
295                Target::All => repositories.iter().collect(),
296            };
297            let mut set = lock(&self.paused);
298            for name in names {
299                if paused {
300                    set.insert(name.clone());
301                } else {
302                    set.remove(name);
303                }
304            }
305            // Sent under the lock, so the records change in the order the set does.
306            self.store.paused(set.iter().cloned().collect())
307        };
308        self.wake.notify_one();
309        if recorded.await {
310            return Ok(());
311        }
312        let name = match target {
313            Target::Repository(repository) => repository.as_str(),
314            Target::All => "every repository",
315        };
316        let (done, undone) = if paused {
317            ("paused", "resumes it")
318        } else {
319            ("resumed", "pauses it again")
320        };
321        let message = format!(
322            "{name} is {done} until runnerd restarts, but the job history cannot record it, so a restart {undone}"
323        );
324        warn!("{message}");
325        Err(ControlError::NotRecorded(message))
326    }
327
328    /// Replaces the configured repositories; the removed ones are no longer paused, in the store too.
329    fn set_repositories(&self, repositories: BTreeSet<String>) {
330        let mut configured = lock(&self.repositories);
331        let mut paused = lock(&self.paused);
332        let before = paused.len();
333        paused.retain(|repository| repositories.contains(repository));
334        if paused.len() != before {
335            // Not waited for: a restart forgets a repository the config does not have anyway.
336            drop(self.store.paused(paused.iter().cloned().collect()));
337        }
338        *configured = repositories;
339    }
340
341    /// Asks the loop to stop; a forced request overrides a graceful one, never the other way.
342    pub fn shutdown(&self, jobs: RunningJobs) {
343        let mode = match jobs {
344            RunningJobs::Keep => Shutdown::Graceful,
345            RunningJobs::Kill => Shutdown::Force,
346        };
347        let mut shutdown = lock(&self.shutdown);
348        *shutdown = (*shutdown).max(mode);
349        drop(shutdown);
350        self.wake.notify_one();
351    }
352
353    /// Whether the repository is paused.
354    pub fn is_paused(&self, repository: &str) -> bool {
355        lock(&self.paused).contains(repository)
356    }
357
358    async fn send<T>(
359        &self,
360        command: impl FnOnce(oneshot::Sender<T>) -> Command,
361    ) -> Result<T, ControlError> {
362        let (reply, answer) = oneshot::channel();
363        self.commands
364            .send(command(reply))
365            .map_err(|_| ControlError::Stopped)?;
366        answer.await.map_err(|_| ControlError::Stopped)
367    }
368
369    /// Pauses one repository or all. With `RunningJobs::Keep`, refuses while any of them runs a job, pausing none;
370    /// with `Kill`, cancels the runs of their running jobs on GitHub, whose runners are removed once they report it
371    /// or `CANCEL_GRACE` later, and removes their other runners at once.
372    pub async fn pause(
373        &self,
374        target: Target,
375        jobs: RunningJobs,
376    ) -> Result<api::Paused, ControlError> {
377        self.send(|reply| Command::Pause {
378            target,
379            jobs,
380            reply,
381        })
382        .await?
383    }
384
385    /// Resumes one repository or all, and publishes the status before it answers, so a client that reads the status
386    /// or the events after the answer sees them resumed. Fails as `set_paused` does.
387    pub async fn resume(&self, target: Target) -> Result<(), ControlError> {
388        self.send(|reply| Command::Resume { target, reply }).await?
389    }
390
391    /// Reseeds a profile as soon as its slots are free; none of them starts a runner until then.
392    pub async fn reseed(&self, profile: &str) -> Result<(), ControlError> {
393        self.send(|reply| Command::Reseed {
394            profile: profile.to_owned(),
395            reply,
396        })
397        .await?
398    }
399
400    /// Removes a slot's volume or a shared cache's, unless a runner uses it.
401    pub async fn clean_cache(&self, name: &str) -> Result<(), ControlError> {
402        self.send(|reply| Command::CleanCache {
403            name: name.to_owned(),
404            reply,
405        })
406        .await?
407    }
408
409    /// Removes the selected records of failed jobs and their images.
410    pub async fn remove_records(
411        &self,
412        selection: RecordSelection,
413    ) -> Result<api::RemovedRecords, ControlError> {
414        self.send(|reply| Command::RemoveRecords { selection, reply })
415            .await?
416    }
417
418    /// Starts a shell in a job's record and returns its terminal. The caller closes it with `close_shell` once the
419    /// terminal ends, from either side.
420    pub async fn open_shell(&self, job: u64, size: TtySize) -> Result<Tty, ControlError> {
421        self.send(|reply| Command::OpenShell { job, size, reply })
422            .await?
423    }
424
425    /// Removes a shell's container. Not waited for, so it can be sent while the terminal is dropped.
426    pub fn close_shell(&self, container: String) {
427        let _ = self.commands.send(Command::CloseShell { container });
428    }
429
430    /// Runs `commit` to write a validated config, and if it commits, runs with the config from then on, before the
431    /// next tick. If it does not, nothing changes and the answer is `Failed`. Running jobs keep running: under
432    /// `RunningSlots::Drain` removed profiles and slots go as their runners finish; under `Keep` a config without a
433    /// slot a runner is in is refused (`Conflict`) before `commit` runs. A config with another `host.slice` is refused
434    /// (`Invalid`) before `commit` runs: the slice takes effect when runnerd starts. The answer warns of what may not
435    /// survive a crash and, for `Author::Owner`, of each profile whose `memory` it lowers under a raise of auto
436    /// sizing's in the last day (`auto::undone_raises`), which are logged too.
437    pub async fn replace_config(
438        &self,
439        config: Config,
440        memory_max: Option<u64>,
441        cpus: Option<u64>,
442        commit: Commit,
443        running: RunningSlots,
444        author: Author,
445    ) -> Result<Vec<String>, ControlError> {
446        self.send(|reply| Command::ReplaceConfig {
447            config: Box::new(config),
448            memory_max,
449            cpus,
450            commit,
451            running,
452            author,
453            reply,
454        })
455        .await?
456    }
457}
458
459/// Names the repositories there are, so a mistyped name shows its fix.
460fn not_configured<'name>(
461    repository: &str,
462    configured: impl Iterator<Item = &'name String>,
463) -> ControlError {
464    let configured: Vec<&str> = configured.map(String::as_str).collect();
465    let configured = if configured.is_empty() {
466        "none is configured".to_owned()
467    } else {
468        format!("configured: {}", configured.join(", "))
469    };
470    ControlError::NotFound(format!(
471        "{repository} is not a configured repository; {configured}"
472    ))
473}
474
475/// `jobs` for people to read: `job 1 on runner-a, runner-b`.
476fn list(jobs: &[api::RunningJob]) -> String {
477    jobs.iter()
478        .map(ToString::to_string)
479        .collect::<Vec<_>>()
480        .join(", ")
481}
482
483impl<G: GithubApi + 'static, D: DockerApi + 'static> Engine<G, D> {
484    /// Carries out a command from the API.
485    pub(super) async fn command(&mut self, command: Command) {
486        match command {
487            Command::Pause {
488                target,
489                jobs,
490                reply,
491            } => {
492                let _ = reply.send(self.pause(&target, jobs).await);
493            }
494            Command::Resume { target, reply } => {
495                let _ = reply.send(self.control.set_paused(&target, false).await);
496            }
497            Command::Reseed { profile, reply } => {
498                let _ = reply.send(self.request_reseed(&profile));
499            }
500            Command::CleanCache { name, reply } => {
501                let _ = reply.send(self.clean_cache(&name).await);
502            }
503            Command::ReplaceConfig {
504                config,
505                memory_max,
506                cpus,
507                commit,
508                running,
509                author,
510                reply,
511            } => {
512                if let Err(refused) = same_slice(&self.config, &config) {
513                    let _ = reply.send(Err(refused));
514                    return;
515                }
516                if running == RunningSlots::Keep
517                    && let Some(occupied) = self.occupied_past(&config)
518                {
519                    let _ = reply.send(Err(ControlError::Conflict(occupied)));
520                    return;
521                }
522                let undone = match author {
523                    Author::Owner => self.undone_raises(&config).await,
524                    Author::AutoSizing => Vec::new(),
525                };
526                let answer = match commit.run().await {
527                    Ok(warning) => {
528                        for line in &undone {
529                            warn!("config replaced: {line}");
530                        }
531                        self.reload(*config, memory_max, cpus);
532                        Ok(warning.into_iter().chain(undone).collect())
533                    }
534                    Err(message) => Err(ControlError::Failed(message)),
535                };
536                let _ = reply.send(answer);
537            }
538            Command::RemoveRecords { selection, reply } => {
539                let _ = reply.send(self.remove_records(&selection).await);
540            }
541            Command::OpenShell { job, size, reply } => {
542                let opened = self.open_shell(job, size).await;
543                // A terminal nobody waits for any more is closed at once.
544                if let Err(Ok(tty)) = reply.send(opened) {
545                    self.close_shell(&tty.container).await;
546                }
547            }
548            Command::CloseShell { container } => self.close_shell(&container).await,
549        }
550    }
551
552    /// Pauses one repository or all. A runner of theirs that holds a job, or that GitHub reports busy, refuses it
553    /// with `RunningJobs::Keep`; with `Kill`, the runs of their jobs are cancelled, and their runners removed once
554    /// they exit or `CANCEL_GRACE` later; a runner whose run cannot be cancelled is removed at once.
555    pub(super) async fn pause(
556        &mut self,
557        target: &Target,
558        running: RunningJobs,
559    ) -> Result<api::Paused, ControlError> {
560        let repositories: BTreeSet<String> = match target {
561            Target::Repository(repository) if !self.repositories.contains_key(repository) => {
562                return Err(not_configured(repository, self.repositories.keys()));
563            }
564            Target::Repository(repository) => BTreeSet::from([repository.clone()]),
565            Target::All => self.repositories.keys().cloned().collect(),
566        };
567        self.take_found();
568        let runners: Vec<(SlotRef, Occupant)> = self
569            .occupied()
570            .filter(|(_, occupant)| repositories.contains(&occupant.repository))
571            .map(|(slot, occupant)| (slot, occupant.clone()))
572            .collect();
573        let jobs: Vec<api::RunningJob> = runners
574            .iter()
575            .filter(|(_, occupant)| occupant.busy || occupant.job.is_some())
576            .map(|(_, occupant)| occupant.running_job())
577            .collect();
578        if running == RunningJobs::Keep && !jobs.is_empty() {
579            let mut by_repository: BTreeMap<&str, Vec<api::RunningJob>> = BTreeMap::new();
580            for job in &jobs {
581                by_repository
582                    .entry(&job.repository)
583                    .or_default()
584                    .push(job.clone());
585            }
586            let runs: Vec<String> = by_repository
587                .iter()
588                .map(|(repository, jobs)| format!("{repository} runs {}", list(jobs)))
589                .collect();
590            // The jobs only: each client says how to force the pause in its own terms.
591            return Err(ControlError::Busy {
592                message: runs.join("; "),
593                jobs,
594            });
595        }
596        // Paused in memory even if not recorded: a forced pause still cancels the runs it was asked to.
597        let recorded = self.control.set_paused(target, true).await;
598        match target {
599            Target::Repository(repository) => info!("{repository}: paused"),
600            Target::All => info!("paused every repository"),
601        }
602        let mut stopped = Vec::new();
603        if running == RunningJobs::Keep {
604            return recorded.map(|()| api::Paused { stopped });
605        }
606        for (slot, occupant) in runners {
607            let repository = &occupant.repository;
608            let cancelled = if let Some(job) = occupant.job {
609                self.cancel(repository, job).await
610            } else {
611                if occupant.busy {
612                    warn!(
613                        "{repository}: {} runs a job its log has not named yet; GitHub fails it once the runner is gone",
614                        occupant.runner_name
615                    );
616                }
617                false
618            };
619            if cancelled {
620                // `collect_finished` removes it once it exits, `kill_cancelled` if it does not.
621                self.cancelling.insert(
622                    occupant.runner_name.clone(),
623                    tokio::time::Instant::now() + CANCEL_GRACE,
624                );
625            } else {
626                self.stop(&slot, RunningJobs::Kill).await;
627                if self
628                    .occupied()
629                    .any(|(_, candidate)| candidate.runner_name == occupant.runner_name)
630                {
631                    continue;
632                }
633            }
634            stopped.push(occupant.running_job());
635        }
636        match recorded {
637            Ok(()) => Ok(api::Paused { stopped }),
638            Err(ControlError::NotRecorded(message)) if !stopped.is_empty() => Err(
639                ControlError::NotRecorded(format!("{message}; stopped {}", list(&stopped))),
640            ),
641            Err(error) => Err(error),
642        }
643    }
644
645    /// Removes the runners of cancelled runs that did not exit within `CANCEL_GRACE`, every tick until they are gone.
646    pub(super) async fn kill_cancelled(&mut self) {
647        let now = tokio::time::Instant::now();
648        let running: BTreeSet<String> = self
649            .occupied()
650            .map(|(_, occupant)| occupant.runner_name.clone())
651            .collect();
652        self.cancelling.retain(|name, _| running.contains(name));
653        let overdue: Vec<(SlotRef, String)> = self
654            .occupied()
655            .filter(|(_, occupant)| {
656                self.cancelling
657                    .get(&occupant.runner_name)
658                    .is_some_and(|deadline| *deadline <= now)
659            })
660            .map(|(slot, occupant)| (slot, occupant.runner_name.clone()))
661            .collect();
662        for (slot, name) in overdue {
663            warn!(
664                "{name} did not exit {}s after its run was cancelled",
665                CANCEL_GRACE.as_secs()
666            );
667            // The deadline stays until the runner is gone, so a failed removal is tried again next tick.
668            self.stop(&slot, RunningJobs::Kill).await;
669        }
670    }
671
672    /// Cancels the run of a job; whether GitHub accepted it.
673    async fn cancel(&self, repository: &str, job: u64) -> bool {
674        let run = match self.github.job(repository, job).await {
675            Ok(details) => details.run_id,
676            Err(error) => {
677                warn!("{repository}: cannot read job {job} to cancel its run: {error}");
678                return false;
679            }
680        };
681        match self.github.cancel_run(repository, run).await {
682            Ok(()) => {
683                info!("{repository}: cancelled run {run} of job {job}");
684                true
685            }
686            Err(error) => {
687                warn!("{repository}: cannot cancel run {run}: {error}");
688                false
689            }
690        }
691    }
692
693    /// Asks for a profile's reseed: none of its slots starts a runner until they are free and it ran.
694    pub(super) fn request_reseed(&mut self, profile: &str) -> Result<(), ControlError> {
695        if !self.config.profiles.contains_key(profile) {
696            return Err(ControlError::NotFound(format!(
697                "{profile} is not a configured profile"
698            )));
699        }
700        if self.is_reseeding(profile) {
701            return Err(ControlError::Conflict(format!(
702                "{profile} is reseeding already"
703            )));
704        }
705        // A shell mounts the snapshot a reseed replaces, and may stay open for hours.
706        if self.shell_profiles().contains(profile) {
707            return Err(ControlError::Conflict(format!(
708                "a shell is open in a record of {profile}, which mounts its snapshot; reseed it once the shell exits"
709            )));
710        }
711        info!("{profile}: reseeding once its slots are free");
712        self.reseed_requested.insert(profile.to_owned());
713        Ok(())
714    }
715
716    /// Removes a slot's volume or a shared cache's, unless a job or a reseed uses it; the runners that use it and
717    /// wait for a job stop first. Snapshots are rebuilt by reseeding instead: slot layers sit on top of them.
718    pub(super) async fn clean_cache(&mut self, name: &str) -> Result<(), ControlError> {
719        let Some(volume) = cache::volumes(&self.names, &self.config)
720            .into_iter()
721            .find(|volume| volume.name == name)
722        else {
723            return Err(ControlError::NotFound(format!(
724                "{name} is not a cache volume of this config; `runnerctl caches` lists them"
725            )));
726        };
727        let using: Vec<SlotRef> = match &volume.kind {
728            api::CacheKind::Snapshot { profile } => {
729                return Err(ControlError::Invalid(format!(
730                    "{name} is {profile}'s snapshot, which every slot's layers sit on; reseed {profile} to rebuild it"
731                )));
732            }
733            api::CacheKind::Slot { profile, slot } => {
734                if self.reseeds.contains_key(profile) {
735                    return Err(ControlError::Conflict(format!(
736                        "{profile} is reseeding, which writes {name}"
737                    )));
738                }
739                self.occupied()
740                    .filter(|(at, _)| at.profile == *profile && at.index + 1 == *slot)
741                    .map(|(at, _)| at)
742                    .collect()
743            }
744            api::CacheKind::Pnpm
745            | api::CacheKind::Maven
746            | api::CacheKind::Mirror
747            | api::CacheKind::Build => self.occupied().map(|(at, _)| at).collect(),
748        };
749        let shared = matches!(volume.kind, api::CacheKind::Pnpm | api::CacheKind::Maven);
750        if shared && !self.records.shells.is_empty() {
751            return Err(ControlError::Conflict(format!(
752                "a shell open in a record mounts {name}; clean it once the shell exits"
753            )));
754        }
755        // Runners waiting for a job, warm ones too, give way: the next plan starts them again. One GitHub gave a
756        // job meanwhile stays, and holds the clean back.
757        for slot in &using {
758            if self
759                .occupant(slot)
760                .is_some_and(|occupant| !occupant.busy && occupant.job.is_none())
761            {
762                self.stop(slot, RunningJobs::Keep).await;
763            }
764        }
765        let users: Vec<api::RunningJob> = using
766            .iter()
767            .filter_map(|slot| self.occupant(slot))
768            .map(Occupant::running_job)
769            .collect();
770        if !users.is_empty() {
771            return Err(ControlError::Busy {
772                message: format!(
773                    "{name} is in use by {}; clean it once they finish",
774                    list(&users)
775                ),
776                jobs: users,
777            });
778        }
779        if volume.kind == api::CacheKind::Build {
780            let Some(cache) = self.build_cache.clone() else {
781                return Err(ControlError::Failed(
782                    "the build cache did not open when runnerd started; its log says why"
783                        .to_owned(),
784                ));
785            };
786            let trimmed = tokio::task::spawn_blocking(move || cache.clear())
787                .await
788                .map_err(|error| ControlError::Failed(format!("cannot empty {name}: {error}")))?;
789            info!(
790                "emptied {name}: {} entries, {}",
791                trimmed.before.entries,
792                ByteSize(trimmed.before.bytes)
793            );
794            self.trim_due = true;
795            return Ok(());
796        }
797        if volume.kind == api::CacheKind::Mirror && self.mirror_up {
798            cache::mirror_down(&*self.docker, &self.names)
799                .await
800                .map_err(|error| {
801                    ControlError::Failed(format!("cannot stop the mirror: {error}"))
802                })?;
803            self.mirror_up = false;
804        }
805        self.docker
806            .remove_volume(name)
807            .await
808            .map_err(|error| ControlError::Failed(format!("cannot remove {name}: {error}")))?;
809        info!("removed {name}");
810        // Measured again once idle.
811        self.trim_due = true;
812        Ok(())
813    }
814
815    /// What `config` undoes of auto sizing's `memory` raises in the last day; none when they cannot be read, which is
816    /// logged: the config runs either way.
817    async fn undone_raises(&self, config: &Config) -> Vec<String> {
818        let now = time::now_ms() / 1000;
819        let since = now.saturating_sub(auto::UNDONE_RAISES).saturating_mul(1000);
820        match self.store.sizing_changes(since).await {
821            Ok(changes) => auto::undone_raises(&self.config, config, &changes, now),
822            Err(store_error) => {
823                warn!(
824                    "cannot read auto sizing's changes to compare a new config with: {store_error}"
825                );
826                Vec::new()
827            }
828        }
829    }
830
831    /// The first slot with a runner in it that `config` does not have, for people to read.
832    fn occupied_past(&self, config: &Config) -> Option<String> {
833        self.slots.iter().find_map(|(name, slots)| {
834            let kept = config
835                .profiles
836                .get(name)
837                .map_or(0, |profile| profile.slots as usize);
838            slots
839                .iter()
840                .enumerate()
841                .skip(kept)
842                .find(|(_, slot)| slot.is_some())
843                .map(|(index, _)| {
844                    format!(
845                        "{name}'s slot {} has a runner, and the new config keeps {kept} slots",
846                        index + 1
847                    )
848                })
849        })
850    }
851
852    /// Runs with a new, validated config. Running jobs keep running: a removed profile, or slots past a lowered
853    /// number, stay until their runners finish, and no runner starts in a profile while it has as many as its
854    /// slots. Runners started from now on get the new settings.
855    /// `cpus` are the slice's, read when the config sets `cpu_overcommit`.
856    pub(super) fn reload(&mut self, config: Config, memory_max: Option<u64>, cpus: Option<u64>) {
857        let mut old = std::mem::replace(&mut self.config, config);
858        if let Some(cache) = &self.build_cache {
859            cache.set_max_size(self.config.caches.build.max_size.0);
860        }
861        for (name, profile) in &self.config.profiles {
862            let slots = self.slots.entry(name.clone()).or_default();
863            if slots.len() < profile.slots as usize {
864                slots.resize(profile.slots as usize, None);
865            }
866            self.retired.remove(name);
867        }
868        for (name, profile) in std::mem::take(&mut old.profiles) {
869            if !self.config.profiles.contains_key(&name) {
870                self.retired.insert(name, profile);
871            }
872        }
873        self.drop_closed_slots();
874        self.images.stale_due = true;
875        let names: BTreeSet<String> = self
876            .config
877            .repositories
878            .iter()
879            .map(|repository| repository.name.clone())
880            .collect();
881        let removed: Vec<String> = self
882            .repositories
883            .keys()
884            .filter(|name| !names.contains(*name))
885            .cloned()
886            .collect();
887        self.draining.extend(removed);
888        self.draining.retain(|name| !names.contains(name));
889        self.repositories.retain(|name, _| names.contains(name));
890        for name in &names {
891            self.repositories.entry(name.clone()).or_default();
892        }
893        self.control.set_repositories(names);
894        let profiles = &self.config.profiles;
895        self.reseed_requested
896            .retain(|name| profiles.contains_key(name));
897        self.oversized.retain(|name| profiles.contains_key(name));
898        self.isolation = isolation::backend(self.config.host.isolation);
899        self.memory_max = memory_max;
900        self.cpus = cpus;
901        self.caches
902            .send_replace(cache::volumes(&self.names, &self.config));
903        info!("reloaded the config");
904    }
905}