Skip to main content

runnerd_core/scheduler/engine/
control.rs

1//! Control from the API: the `Control` handle the daemon's API holds (pause flags, which it records in the store so
2//! a restart keeps them, shutdown, a channel of `Command`s) and the engine's side of each command: pause and resume,
3//! of one repository or all, with or without cancelling their runs, reseed, cache clean, config replacement, the
4//! removal of records and the shells opened in them.
5//! Commands run between ticks or while a tick waits for GitHub, so they see the slot state a tick does (ADRs 0019,
6//! 0026). It must not serve HTTP or write the config file: the daemon does, through `Commit`.
7
8use std::collections::{BTreeMap, BTreeSet};
9use std::sync::{Arc, Mutex};
10use std::time::Duration;
11
12use tokio::sync::{Notify, mpsc, oneshot, watch};
13use tracing::{info, warn};
14
15use super::records::RecordSelection;
16use super::stop::Stop;
17use super::{Engine, Occupant, Slot};
18use crate::actions::policy::{Allowed, Policies, Policy};
19use crate::api;
20use crate::cache;
21use crate::config::{ByteSize, Config};
22use crate::docker::{DockerApi, Tty, TtySize};
23use crate::github::GithubApi;
24use crate::isolation;
25use crate::scheduler::{SlotRef, StopReason};
26use crate::sizing::auto;
27use crate::store::Store;
28use crate::sync::lock;
29use crate::time;
30
31/// How long the runner of a run a forced pause cancelled gets to report its job cancelled and exit, before it is
32/// removed: GitHub completes a cancelled run only once its runner reports it, which took 40s in a test.
33pub(super) const CANCEL_GRACE: Duration = Duration::from_secs(90);
34
35/// How the daemon is asked to stop, from the least to the most urgent: a later request only ever makes the stop more
36/// urgent (ADR 0091).
37#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
38pub enum Shutdown {
39    /// Keep scheduling.
40    No,
41    /// Start nothing new, let running jobs finish, then remove every runner: `runnerd stop`.
42    Graceful,
43    /// Remove idle runners, leave busy ones and the mirror to the next start, which adopts them, and stop within
44    /// `STOP_DEADLINE`: a signal, as systemd sends on a stop.
45    Detach,
46    /// Kill running jobs and delete their runners, within `STOP_DEADLINE`: `runnerd stop --force`, a second signal.
47    Force,
48}
49
50impl Shutdown {
51    /// How a daemon stopping in this mode says it stops; `None` for `No`.
52    pub fn stopping(self) -> Option<api::Stopping> {
53        match self {
54            Shutdown::No => None,
55            Shutdown::Graceful => Some(api::Stopping::WaitsForJobs),
56            Shutdown::Detach => Some(api::Stopping::LeavesJobs),
57            Shutdown::Force => Some(api::Stopping::KillsJobs),
58        }
59    }
60}
61
62/// What a pause, a shutdown or a runner's removal does with the jobs runners are running.
63#[derive(Debug, Clone, Copy, PartialEq, Eq)]
64pub enum RunningJobs {
65    /// Leave them running: a pause refuses, a busy runner stays.
66    Keep,
67    /// Kill them: a pause cancels their runs, a removal kills their runners.
68    Kill,
69}
70
71/// What a new config may do to slots with a runner in them.
72#[derive(Debug, Clone, Copy, PartialEq, Eq)]
73pub enum RunningSlots {
74    /// A slot past a lowered `slots`, or of a removed profile, stays until its runner finishes.
75    Drain,
76    /// The config is refused while a runner is in a slot it does not have: what auto sizing asks for.
77    Keep,
78}
79
80/// Who writes a new config.
81#[derive(Debug, Clone, Copy, PartialEq, Eq)]
82pub enum Author {
83    /// The owner, through `PUT /v1/config`: a save that lowers `memory` under a raise of auto sizing's in the last
84    /// day is answered with a warning (`auto::undone_raises`).
85    Owner,
86    /// Auto sizing, whose own lowering is never a warning about its raise.
87    AutoSizing,
88}
89
90/// `Err` when `new` moves runnerd to another slice than `running` names: the sampler and the running containers stay
91/// in the slice runnerd started in, so the slice changes only at a restart.
92pub fn same_slice(running: &Config, new: &Config) -> Result<(), ControlError> {
93    if new.host.slice == running.host.slice {
94        return Ok(());
95    }
96    Err(ControlError::Invalid(format!(
97        "restart runnerd to change the slice: it runs in {}, and host.slice is {} in the new config",
98        running.host.slice, new.host.slice
99    )))
100}
101
102/// The repositories a pause or a resume applies to.
103#[derive(Debug, Clone, PartialEq, Eq)]
104pub enum Target {
105    /// One repository, `owner/name`.
106    Repository(String),
107    /// Every configured repository.
108    All,
109}
110
111/// Why the engine refused or failed a command from the API.
112#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
113pub enum ControlError {
114    /// No such repository, profile or cache.
115    #[error("{0}")]
116    NotFound(String),
117    /// Runners use what the command would change.
118    #[error("{message}")]
119    Busy {
120        /// Names the runners and their jobs.
121        message: String,
122        /// The runners in the way.
123        jobs: Vec<api::RunningJob>,
124    },
125    /// The command conflicts with one already in progress.
126    #[error("{0}")]
127    Conflict(String),
128    /// The command cannot apply to this target.
129    #[error("{0}")]
130    Invalid(String),
131    /// Docker or GitHub failed.
132    #[error("{0}")]
133    Failed(String),
134    /// A pause or resume holds in memory, but the job history could not record it: a restart undoes it.
135    #[error("{0}")]
136    NotRecorded(String),
137    /// The engine stopped.
138    #[error("runnerd is stopping")]
139    Stopped,
140}
141
142/// A command the engine carries out between ticks or while a tick waits for GitHub, with where its answer goes.
143#[derive(Debug)]
144pub enum Command {
145    /// Pause one repository or all; with `RunningJobs::Kill`, cancel their running jobs.
146    Pause {
147        /// What to pause.
148        target: Target,
149        /// Cancel running jobs, or refuse while there are any.
150        jobs: RunningJobs,
151        /// The answer.
152        reply: oneshot::Sender<Result<api::Paused, ControlError>>,
153    },
154    /// Resume one repository or all.
155    Resume {
156        /// What to resume.
157        target: Target,
158        /// The answer.
159        reply: oneshot::Sender<Result<(), ControlError>>,
160    },
161    /// Make slot 1's layers the profile's snapshot once its slots are free.
162    Reseed {
163        /// The profile.
164        profile: String,
165        /// The answer.
166        reply: oneshot::Sender<Result<(), ControlError>>,
167    },
168    /// Remove a slot's volume or a shared cache's, unless a runner uses it.
169    CleanCache {
170        /// The volume.
171        name: String,
172        /// The answer.
173        reply: oneshot::Sender<Result<(), ControlError>>,
174    },
175    /// Write a new, validated config to its file and run with it, with no tick in between.
176    ReplaceConfig {
177        /// The config.
178        config: Box<Config>,
179        /// The `MemoryMax` of the slice it names.
180        memory_max: Option<u64>,
181        /// The CPUs of the slice it names, in millicores, read when it sets `cpu_overcommit`.
182        cpus: Option<u64>,
183        /// Writes the file.
184        commit: Commit,
185        /// Whether it may leave runners in slots it does not have.
186        running: RunningSlots,
187        /// Who writes it.
188        author: Author,
189        /// Answered once the engine runs with it: with the commit's warning and what it undoes of auto sizing's
190        /// raises, or why it did not commit.
191        reply: oneshot::Sender<Result<Vec<String>, ControlError>>,
192    },
193    /// Remove records of failed jobs.
194    RemoveRecords {
195        /// Which.
196        selection: RecordSelection,
197        /// The answer.
198        reply: oneshot::Sender<Result<api::RemovedRecords, ControlError>>,
199    },
200    /// Start a shell in a job's record, if the budget has room for it.
201    OpenShell {
202        /// The job.
203        job: u64,
204        /// Its terminal's size.
205        size: TtySize,
206        /// The shell's terminal.
207        reply: oneshot::Sender<Result<Tty, ControlError>>,
208    },
209    /// Remove a shell's container: its terminal closed.
210    CloseShell {
211        /// The container's ID.
212        container: String,
213    },
214}
215
216/// What makes a new config durable, run by the engine before it runs with the config: `Ok` once the config is in
217/// place, with a warning if it may not survive a crash; `Err` with nothing changed.
218pub struct Commit(Box<dyn FnOnce() -> Result<Option<String>, String> + Send>);
219
220impl Commit {
221    /// A commit that runs `write`, off the engine's thread.
222    pub fn new(write: impl FnOnce() -> Result<Option<String>, String> + Send + 'static) -> Commit {
223        Commit(Box::new(write))
224    }
225
226    /// Runs it.
227    pub async fn run(self) -> Result<Option<String>, String> {
228        tokio::task::spawn_blocking(self.0)
229            .await
230            .unwrap_or_else(|join_error| Err(join_error.to_string()))
231    }
232}
233
234impl std::fmt::Debug for Commit {
235    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
236        f.write_str("Commit")
237    }
238}
239
240/// The engine's end of `Control`'s commands.
241#[derive(Debug)]
242pub struct Commands(mpsc::UnboundedReceiver<Command>);
243
244impl Commands {
245    /// The next command; `None` once `Control` is gone.
246    pub async fn recv(&mut self) -> Option<Command> {
247        self.0.recv().await
248    }
249}
250
251/// Commands from the API and signals to the loop.
252#[derive(Debug)]
253pub struct Control {
254    // Locked with `sync::lock`: the guarded data stays consistent, as every critical section is a single insert,
255    // remove or read.
256    repositories: Mutex<BTreeSet<String>>,
257    paused: Mutex<BTreeSet<String>>,
258    /// What the config lets the job actions do, which the engine sets at start and at each reload before it answers,
259    /// so the job actions check the config of the moment they are sent, as one snapshot (ADR 0134).
260    actions: Policies,
261    /// Where `paused` is recorded.
262    store: Store,
263    shutdown: watch::Sender<Stop>,
264    pub(super) wake: Notify,
265    commands: mpsc::UnboundedSender<Command>,
266}
267
268impl Control {
269    /// Controls for the configured repositories, of which those in `paused` start paused, and the commands they
270    /// send the engine. A paused repository the config no longer has is forgotten, in `store` too: one added back
271    /// starts active.
272    pub fn new(
273        repositories: impl IntoIterator<Item = String>,
274        paused: impl IntoIterator<Item = String>,
275        store: Store,
276    ) -> (Control, Commands) {
277        let repositories: BTreeSet<String> = repositories.into_iter().collect();
278        let paused: BTreeSet<String> = paused
279            .into_iter()
280            .filter(|repository| repositories.contains(repository))
281            .collect();
282        // Sent at once; a write that fails leaves rows the next start filters out the same way.
283        drop(store.paused(paused.iter().cloned().collect()));
284        let (commands, receiver) = mpsc::unbounded_channel();
285        let control = Control {
286            actions: Policies::new(Allowed {
287                setting: api::JobActions::On,
288                repositories: repositories.clone(),
289                // As `rerun_runner_faults` defaults: off.
290                reruns: BTreeSet::new(),
291            }),
292            repositories: Mutex::new(repositories),
293            paused: Mutex::new(paused),
294            store,
295            shutdown: watch::Sender::new(Stop::none()),
296            wake: Notify::new(),
297            commands,
298        };
299        (control, Commands(receiver))
300    }
301
302    /// Whether the daemon is stopping and how: as requested, or as a detach once Docker stopped answering the stop.
303    pub fn mode(&self) -> Shutdown {
304        self.shutdown.borrow().mode()
305    }
306
307    /// Whether Docker stopped answering the stop, which is then left to the next start.
308    pub(super) fn docker_gone(&self) -> bool {
309        self.shutdown.borrow().docker_gone()
310    }
311
312    /// Notes that Docker did not answer a call the stop needed, and returns whether that is news: not without a stop
313    /// under way, nor once noted. Its deadline starts, and a later `runnerd stop` answers that it leaves running jobs.
314    pub(super) fn note_docker_gone(&self) -> bool {
315        let news = self.shutdown.send_if_modified(Stop::note_docker_gone);
316        if news {
317            self.wake.notify_one();
318        }
319        news
320    }
321
322    /// Resolves once the stop has a deadline: it detaches or is forced.
323    pub(super) async fn hurried(&self) {
324        let mut mode = self.shutdown.subscribe();
325        // `self` holds the sender, so the wait ends only once the mode is reached.
326        let _ = mode.wait_for(|stop| stop.hurried()).await;
327    }
328
329    /// Pauses or resumes one repository or all, whatever they run, and records it in the store before it answers,
330    /// so a restarted daemon starts with the same ones paused. `NotFound` for a repository the config does not have.
331    /// If the record fails, the change holds until a restart and the answer is `NotRecorded`. The status shows it
332    /// at the engine's next publish: the API pauses and resumes through `pause` and `resume`, which publish before
333    /// they answer.
334    pub async fn set_paused(&self, target: &Target, paused: bool) -> Result<(), ControlError> {
335        let recorded = {
336            let repositories = lock(&self.repositories);
337            let names: Vec<&String> = match target {
338                Target::Repository(repository) => {
339                    let Some(name) = repositories.get(repository) else {
340                        return Err(not_configured(repository, repositories.iter()));
341                    };
342                    vec![name]
343                }
344                Target::All => repositories.iter().collect(),
345            };
346            let mut set = lock(&self.paused);
347            for name in names {
348                if paused {
349                    set.insert(name.clone());
350                } else {
351                    set.remove(name);
352                }
353            }
354            // Sent under the lock, so the records change in the order the set does.
355            self.store.paused(set.iter().cloned().collect())
356        };
357        self.wake.notify_one();
358        if recorded.await {
359            return Ok(());
360        }
361        let name = match target {
362            Target::Repository(repository) => repository.as_str(),
363            Target::All => "every repository",
364        };
365        let (done, undone) = if paused {
366            ("paused", "resumes it")
367        } else {
368            ("resumed", "pauses it again")
369        };
370        let message = format!(
371            "{name} is {done} until runnerd restarts, but the job history cannot record it, so a restart {undone}"
372        );
373        warn!("{message}");
374        Err(ControlError::NotRecorded(message))
375    }
376
377    /// Replaces the configured repositories; the removed ones are no longer paused, in the store too.
378    pub(crate) fn set_repositories(&self, repositories: BTreeSet<String>) {
379        let mut configured = lock(&self.repositories);
380        let mut paused = lock(&self.paused);
381        let before = paused.len();
382        paused.retain(|repository| repositories.contains(repository));
383        if paused.len() != before {
384            // Not waited for: a restart forgets a repository the config does not have anyway.
385            drop(self.store.paused(paused.iter().cloned().collect()));
386        }
387        *configured = repositories;
388    }
389
390    /// Asks the loop to stop; a more urgent request overrides a less urgent one, never the other way. Returns how it
391    /// stops now, which is what status says.
392    pub fn shutdown(&self, mode: Shutdown) -> Shutdown {
393        self.shutdown.send_if_modified(|stop| stop.ask(mode));
394        self.wake.notify_one();
395        self.mode()
396    }
397
398    /// Whether the repository is paused.
399    pub fn is_paused(&self, repository: &str) -> bool {
400        lock(&self.paused).contains(repository)
401    }
402
403    /// What the config of this moment lets the owner do with jobs (ADR 0134): one snapshot.
404    pub fn action_policy(&self) -> Arc<Policy> {
405        self.actions.now()
406    }
407
408    /// Replaces that snapshot whole, from `[github] job_actions`, the configured repositories and their
409    /// `rerun_runner_faults`: the engine does at start and at each reload.
410    pub fn set_action_policy(&self, allowed: Allowed) {
411        self.actions.replace(allowed);
412    }
413
414    /// Resolves once the job actions' snapshot is another than revision `revision`: a reload replaced it.
415    pub async fn policy_moved(&self, revision: u64) {
416        self.actions.moved(revision).await;
417    }
418
419    /// Resolves once a stop of any mode is asked for: what is sent to GitHub on the owner's behalf ends then.
420    pub async fn stop_asked(&self) {
421        let mut mode = self.shutdown.subscribe();
422        // `self` holds the sender, so the wait ends only once a stop is asked for.
423        let _ = mode.wait_for(|stop| stop.mode() != Shutdown::No).await;
424    }
425
426    async fn send<T>(
427        &self,
428        command: impl FnOnce(oneshot::Sender<T>) -> Command,
429    ) -> Result<T, ControlError> {
430        let (reply, answer) = oneshot::channel();
431        self.commands
432            .send(command(reply))
433            .map_err(|_| ControlError::Stopped)?;
434        answer.await.map_err(|_| ControlError::Stopped)
435    }
436
437    /// Pauses one repository or all. With `RunningJobs::Keep`, refuses while any of them runs a job, pausing none;
438    /// with `Kill`, cancels the runs of their running jobs on GitHub, whose runners are removed once they report it
439    /// or `CANCEL_GRACE` later, and removes their other runners at once.
440    pub async fn pause(
441        &self,
442        target: Target,
443        jobs: RunningJobs,
444    ) -> Result<api::Paused, ControlError> {
445        self.send(|reply| Command::Pause {
446            target,
447            jobs,
448            reply,
449        })
450        .await?
451    }
452
453    /// Resumes one repository or all, and publishes the status before it answers, so a client that reads the status
454    /// or the events after the answer sees them resumed. Fails as `set_paused` does.
455    pub async fn resume(&self, target: Target) -> Result<(), ControlError> {
456        self.send(|reply| Command::Resume { target, reply }).await?
457    }
458
459    /// Reseeds a profile as soon as its slots are free; none of them starts a runner until then.
460    pub async fn reseed(&self, profile: &str) -> Result<(), ControlError> {
461        self.send(|reply| Command::Reseed {
462            profile: profile.to_owned(),
463            reply,
464        })
465        .await?
466    }
467
468    /// Removes a slot's volume or a shared cache's, unless a runner uses it.
469    pub async fn clean_cache(&self, name: &str) -> Result<(), ControlError> {
470        self.send(|reply| Command::CleanCache {
471            name: name.to_owned(),
472            reply,
473        })
474        .await?
475    }
476
477    /// Removes the selected records of failed jobs and their images.
478    pub async fn remove_records(
479        &self,
480        selection: RecordSelection,
481    ) -> Result<api::RemovedRecords, ControlError> {
482        self.send(|reply| Command::RemoveRecords { selection, reply })
483            .await?
484    }
485
486    /// Starts a shell in a job's record and returns its terminal. The caller closes it with `close_shell` once the
487    /// terminal ends, from either side.
488    pub async fn open_shell(&self, job: u64, size: TtySize) -> Result<Tty, ControlError> {
489        self.send(|reply| Command::OpenShell { job, size, reply })
490            .await?
491    }
492
493    /// Removes a shell's container. Not waited for, so it can be sent while the terminal is dropped.
494    pub fn close_shell(&self, container: String) {
495        let _ = self.commands.send(Command::CloseShell { container });
496    }
497
498    /// Runs `commit` to write a validated config, and if it commits, runs with the config from then on, before the
499    /// next tick. If it does not, nothing changes and the answer is `Failed`. Running jobs keep running: under
500    /// `RunningSlots::Drain` removed profiles and slots go as their runners finish; under `Keep` a config without a
501    /// slot a runner is in is refused (`Conflict`) before `commit` runs. A config with another `host.slice` is refused
502    /// (`Invalid`) before `commit` runs: the slice takes effect when runnerd starts. The answer warns of what may not
503    /// survive a crash and, for `Author::Owner`, of each profile whose `memory` it lowers under a raise of auto
504    /// sizing's in the last day (`auto::undone_raises`), which are logged too.
505    pub async fn replace_config(
506        &self,
507        config: Config,
508        memory_max: Option<u64>,
509        cpus: Option<u64>,
510        commit: Commit,
511        running: RunningSlots,
512        author: Author,
513    ) -> Result<Vec<String>, ControlError> {
514        self.send(|reply| Command::ReplaceConfig {
515            config: Box::new(config),
516            memory_max,
517            cpus,
518            commit,
519            running,
520            author,
521            reply,
522        })
523        .await?
524    }
525}
526
527/// Names the repositories there are, so a mistyped name shows its fix.
528fn not_configured<'name>(
529    repository: &str,
530    configured: impl Iterator<Item = &'name String>,
531) -> ControlError {
532    let configured: Vec<&str> = configured.map(String::as_str).collect();
533    let configured = if configured.is_empty() {
534        "none is configured".to_owned()
535    } else {
536        format!("configured: {}", configured.join(", "))
537    };
538    ControlError::NotFound(format!(
539        "{repository} is not a configured repository; {configured}"
540    ))
541}
542
543/// `jobs` for people to read: `job 1 on runner-a, runner-b`.
544fn list(jobs: &[api::RunningJob]) -> String {
545    jobs.iter()
546        .map(ToString::to_string)
547        .collect::<Vec<_>>()
548        .join(", ")
549}
550
551impl<G: GithubApi + 'static, D: DockerApi + 'static> Engine<G, D> {
552    /// Carries out a command from the API.
553    pub(super) async fn command(&mut self, command: Command) {
554        match command {
555            Command::Pause {
556                target,
557                jobs,
558                reply,
559            } => {
560                let _ = reply.send(self.pause(&target, jobs).await);
561            }
562            Command::Resume { target, reply } => {
563                let _ = reply.send(self.control.set_paused(&target, false).await);
564            }
565            Command::Reseed { profile, reply } => {
566                let _ = reply.send(self.request_reseed(&profile));
567            }
568            Command::CleanCache { name, reply } => {
569                let _ = reply.send(self.clean_cache(&name).await);
570            }
571            Command::ReplaceConfig {
572                config,
573                memory_max,
574                cpus,
575                commit,
576                running,
577                author,
578                reply,
579            } => {
580                if let Err(refused) = same_slice(&self.config, &config) {
581                    let _ = reply.send(Err(refused));
582                    return;
583                }
584                if running == RunningSlots::Keep
585                    && let Some(occupied) = self.occupied_past(&config)
586                {
587                    let _ = reply.send(Err(ControlError::Conflict(occupied)));
588                    return;
589                }
590                let undone = match author {
591                    Author::Owner => self.undone_raises(&config).await,
592                    Author::AutoSizing => Vec::new(),
593                };
594                let answer = match commit.run().await {
595                    Ok(warning) => {
596                        for line in &undone {
597                            warn!("config replaced: {line}");
598                        }
599                        self.reload(*config, memory_max, cpus);
600                        Ok(warning.into_iter().chain(undone).collect())
601                    }
602                    Err(message) => Err(ControlError::Failed(message)),
603                };
604                let _ = reply.send(answer);
605            }
606            Command::RemoveRecords { selection, reply } => {
607                let _ = reply.send(self.remove_records(&selection).await);
608            }
609            Command::OpenShell { job, size, reply } => {
610                let opened = self.open_shell(job, size).await;
611                // A terminal nobody waits for any more is closed at once.
612                if let Err(Ok(tty)) = reply.send(opened) {
613                    self.close_shell(&tty.container).await;
614                }
615            }
616            Command::CloseShell { container } => self.close_shell(&container).await,
617        }
618    }
619
620    /// Pauses one repository or all. A runner of theirs that holds a job, or that GitHub reports busy, refuses it
621    /// with `RunningJobs::Keep`; with `Kill`, the runs of their jobs are cancelled, and their runners removed once
622    /// they exit or `CANCEL_GRACE` later; a runner whose run cannot be cancelled is removed at once.
623    pub(super) async fn pause(
624        &mut self,
625        target: &Target,
626        running: RunningJobs,
627    ) -> Result<api::Paused, ControlError> {
628        let repositories: BTreeSet<String> = match target {
629            Target::Repository(repository) if self.repositories.listed(repository).is_none() => {
630                return Err(not_configured(repository, self.repositories.listed_names()));
631            }
632            Target::Repository(repository) => BTreeSet::from([repository.clone()]),
633            Target::All => self.repositories.listed_names().cloned().collect(),
634        };
635        self.take_found();
636        let runners: Vec<(SlotRef, Occupant)> = self
637            .occupied()
638            .filter(|(_, occupant)| repositories.contains(&occupant.repository))
639            .map(|(slot, occupant)| (slot, occupant.clone()))
640            .collect();
641        let jobs: Vec<api::RunningJob> = runners
642            .iter()
643            .filter(|(_, occupant)| occupant.engaged())
644            .map(|(_, occupant)| occupant.running_job())
645            .collect();
646        if running == RunningJobs::Keep && !jobs.is_empty() {
647            let mut by_repository: BTreeMap<&str, Vec<api::RunningJob>> = BTreeMap::new();
648            for job in &jobs {
649                by_repository
650                    .entry(&job.repository)
651                    .or_default()
652                    .push(job.clone());
653            }
654            let runs: Vec<String> = by_repository
655                .iter()
656                .map(|(repository, jobs)| format!("{repository} runs {}", list(jobs)))
657                .collect();
658            // The jobs only: each client says how to force the pause in its own terms.
659            return Err(ControlError::Busy {
660                message: runs.join("; "),
661                jobs,
662            });
663        }
664        // Paused in memory even if not recorded: a forced pause still cancels the runs it was asked to.
665        let recorded = self.control.set_paused(target, true).await;
666        match target {
667            Target::Repository(repository) => info!("{repository}: paused"),
668            Target::All => info!("paused every repository"),
669        }
670        let mut stopped = Vec::new();
671        if running == RunningJobs::Keep {
672            return recorded.map(|()| api::Paused { stopped });
673        }
674        for (slot, occupant) in runners {
675            let repository = &occupant.repository;
676            let cancelled = if let Some(job) = occupant.named_job() {
677                self.cancel(repository, job).await
678            } else {
679                if occupant.listed_busy() {
680                    warn!(
681                        "{repository}: {} runs a job its log has not named yet; GitHub fails it once the runner is gone",
682                        occupant.runner_name
683                    );
684                }
685                false
686            };
687            if cancelled {
688                // `collect_finished` removes it once it exits, `kill_cancelled` if it does not.
689                if let Some(cancelled) = self
690                    .slots
691                    .values_mut()
692                    .flatten()
693                    .filter_map(Slot::occupant_mut)
694                    .find(|candidate| candidate.runner_name == occupant.runner_name)
695                {
696                    cancelled
697                        .engagement
698                        .cancelled(tokio::time::Instant::now() + CANCEL_GRACE);
699                }
700                // A restart keeps it.
701                self.store.runner_cancel_by(
702                    &occupant.runner_name,
703                    time::now_ms() + u64::try_from(CANCEL_GRACE.as_millis()).unwrap_or(u64::MAX),
704                );
705            } else {
706                self.kill(&slot).await;
707                if self
708                    .occupied()
709                    .any(|(_, candidate)| candidate.runner_name == occupant.runner_name)
710                {
711                    continue;
712                }
713            }
714            stopped.push(occupant.running_job());
715        }
716        match recorded {
717            Ok(()) => Ok(api::Paused { stopped }),
718            Err(ControlError::NotRecorded(message)) if !stopped.is_empty() => Err(
719                ControlError::NotRecorded(format!("{message}; stopped {}", list(&stopped))),
720            ),
721            Err(error) => Err(error),
722        }
723    }
724
725    /// Removes the runners of cancelled runs that did not exit within `CANCEL_GRACE`, every tick until they are gone.
726    pub(super) async fn kill_cancelled(&mut self) {
727        let now = tokio::time::Instant::now();
728        let overdue: Vec<(SlotRef, String)> = self
729            .occupied()
730            .filter(|(_, occupant)| occupant.engagement.cancel_overdue(now))
731            .map(|(slot, occupant)| (slot, occupant.runner_name.clone()))
732            .collect();
733        for (slot, name) in overdue {
734            warn!(
735                "{name} did not exit {}s after its run was cancelled",
736                CANCEL_GRACE.as_secs()
737            );
738            // The deadline stays until the runner is gone, so a failed removal is tried again next tick.
739            self.kill(&slot).await;
740        }
741    }
742
743    /// Cancels the run of a job; whether GitHub accepted it.
744    async fn cancel(&self, repository: &str, job: u64) -> bool {
745        let run = match self.github.job(repository, job).await {
746            Ok(details) => details.run_id,
747            Err(error) => {
748                warn!("{repository}: cannot read job {job} to cancel its run: {error}");
749                return false;
750            }
751        };
752        match self.github.cancel_run(repository, run).await {
753            Ok(()) => {
754                info!("{repository}: cancelled run {run} of job {job}");
755                true
756            }
757            Err(error) => {
758                warn!("{repository}: cannot cancel run {run}: {error}");
759                false
760            }
761        }
762    }
763
764    /// Asks for a profile's reseed: none of its slots starts a runner until they are free and it ran.
765    pub(super) fn request_reseed(&mut self, profile: &str) -> Result<(), ControlError> {
766        if !self.config.profiles.contains_key(profile) {
767            return Err(ControlError::NotFound(format!(
768                "{profile} is not a configured profile"
769            )));
770        }
771        if self.is_reseeding(profile) {
772            return Err(ControlError::Conflict(format!(
773                "{profile} is reseeding already"
774            )));
775        }
776        // A shell mounts the snapshot a reseed replaces, and may stay open for hours.
777        if self.shell_profiles().contains(profile) {
778            return Err(ControlError::Conflict(format!(
779                "a shell is open in a record of {profile}, which mounts its snapshot; reseed it once the shell exits"
780            )));
781        }
782        info!("{profile}: reseeding once its slots are free");
783        self.upkeep.request(profile);
784        Ok(())
785    }
786
787    /// Removes a slot's volume or a shared cache's, unless a job or a reseed uses it; the runners that use it and
788    /// wait for a job stop first. Snapshots are rebuilt by reseeding instead: slot layers sit on top of them.
789    pub(super) async fn clean_cache(&mut self, name: &str) -> Result<(), ControlError> {
790        let Some(volume) = cache::volumes(&self.names, &self.config)
791            .into_iter()
792            .find(|volume| volume.name == name)
793        else {
794            return Err(ControlError::NotFound(format!(
795                "{name} is not a cache volume of this config; `runnerctl caches` lists them"
796            )));
797        };
798        let using: Vec<SlotRef> = match &volume.kind {
799            api::CacheKind::Snapshot { profile } => {
800                return Err(ControlError::Invalid(format!(
801                    "{name} is {profile}'s snapshot, which every slot's layers sit on; reseed {profile} to rebuild it"
802                )));
803            }
804            api::CacheKind::Slot { profile, slot } => {
805                if self.upkeep.reseed_runs(profile) {
806                    return Err(ControlError::Conflict(format!(
807                        "{profile} is reseeding, which writes {name}"
808                    )));
809                }
810                self.occupied()
811                    .filter(|(at, _)| at.profile == *profile && at.index + 1 == *slot)
812                    .map(|(at, _)| at)
813                    .collect()
814            }
815            api::CacheKind::Pnpm
816            | api::CacheKind::Maven
817            | api::CacheKind::Mirror
818            | api::CacheKind::Build => self.occupied().map(|(at, _)| at).collect(),
819        };
820        let shared = matches!(volume.kind, api::CacheKind::Pnpm | api::CacheKind::Maven);
821        if shared && self.records.has_shells() {
822            return Err(ControlError::Conflict(format!(
823                "a shell open in a record mounts {name}; clean it once the shell exits"
824            )));
825        }
826        // Runners waiting for a job, warm ones too, give way: the next plan starts them again. One GitHub gave a
827        // job meanwhile stays, and holds the clean back.
828        let idle = using
829            .iter()
830            .filter(|slot| self.occupant(slot).is_some_and(Occupant::idle))
831            .map(|slot| (slot.clone(), StopReason::Clean))
832            .collect();
833        self.remove_idle(idle).await;
834        let users: Vec<api::RunningJob> = using
835            .iter()
836            .filter_map(|slot| self.occupant(slot))
837            .map(Occupant::running_job)
838            .collect();
839        if !users.is_empty() {
840            return Err(ControlError::Busy {
841                message: format!(
842                    "{name} is in use by {}; clean it once they finish",
843                    list(&users)
844                ),
845                jobs: users,
846            });
847        }
848        if volume.kind == api::CacheKind::Build {
849            let Some(cache) = self.build_cache.clone() else {
850                return Err(ControlError::Failed(
851                    "the build cache did not open when runnerd started; its log says why"
852                        .to_owned(),
853                ));
854            };
855            let trimmed = tokio::task::spawn_blocking(move || cache.clear())
856                .await
857                .map_err(|error| ControlError::Failed(format!("cannot empty {name}: {error}")))?;
858            info!(
859                "emptied {name}: {} entries, {}",
860                trimmed.before.entries,
861                ByteSize(trimmed.before.bytes)
862            );
863            self.trim_due = true;
864            return Ok(());
865        }
866        if volume.kind == api::CacheKind::Mirror && self.mirror.may_run() {
867            cache::mirror_down(&*self.docker, &self.names)
868                .await
869                .map_err(|error| {
870                    ControlError::Failed(format!("cannot stop the mirror: {error}"))
871                })?;
872            self.mirror.stopped();
873        }
874        self.docker
875            .remove_volume(name)
876            .await
877            .map_err(|error| ControlError::Failed(format!("cannot remove {name}: {error}")))?;
878        info!("removed {name}");
879        // Its layer no longer holds what a job cached.
880        if let api::CacheKind::Slot { profile, slot } = &volume.kind {
881            self.ran.remove(&SlotRef {
882                profile: profile.clone(),
883                index: slot.saturating_sub(1),
884            });
885        }
886        // Measured again once idle.
887        self.trim_due = true;
888        Ok(())
889    }
890
891    /// What `config` undoes of auto sizing's `memory` raises in the last day; none when they cannot be read, which is
892    /// logged: the config runs either way.
893    async fn undone_raises(&self, config: &Config) -> Vec<String> {
894        let now = time::now_ms() / 1000;
895        let since = now.saturating_sub(auto::UNDONE_RAISES).saturating_mul(1000);
896        match self.store.sizing_changes(since).await {
897            Ok(changes) => auto::undone_raises(&self.config, config, &changes, now),
898            Err(store_error) => {
899                warn!(
900                    "cannot read auto sizing's changes to compare a new config with: {store_error}"
901                );
902                Vec::new()
903            }
904        }
905    }
906
907    /// The first slot with a runner in it that `config` does not have, for people to read.
908    fn occupied_past(&self, config: &Config) -> Option<String> {
909        self.slots.iter().find_map(|(name, slots)| {
910            let kept = config
911                .profiles
912                .get(name)
913                .map_or(0, |profile| profile.slots as usize);
914            slots
915                .iter()
916                .enumerate()
917                .skip(kept)
918                .find(|(_, slot)| !slot.is_free())
919                .map(|(index, _)| {
920                    format!(
921                        "{name}'s slot {} has a runner, and the new config keeps {kept} slots",
922                        index + 1
923                    )
924                })
925        })
926    }
927
928    /// Runs with a new, validated config. Running jobs keep running: a removed profile, or slots past a lowered
929    /// number, stay until their runners finish, and no runner starts in a profile while it has as many as its
930    /// slots. Runners started from now on get the new settings.
931    /// `cpus` are the slice's, read when the config sets `cpu_overcommit`.
932    pub(super) fn reload(&mut self, config: Config, memory_max: Option<u64>, cpus: Option<u64>) {
933        let mut old = std::mem::replace(&mut self.config, config);
934        // First, so what the job actions send from now on, and what waits to send, answers to the new config.
935        self.control.set_action_policy(Allowed::of(&self.config));
936        // A pool gone keeps no hold a pool made there later would inherit, and one whose runners change starts again
937        // at once: what failed may be what changed.
938        for (name, state) in self.repositories.iter_listed_mut() {
939            let new = &self.config;
940            state.forget_health(|profile| same_runners(&old, new, name, profile));
941        }
942        if let Some(cache) = &self.build_cache {
943            cache.set_max_size(self.config.caches.build.max_size.0);
944        }
945        self.archives.reload(&self.config);
946        for (name, profile) in &self.config.profiles {
947            let slots = self.slots.entry(name.clone()).or_default();
948            if slots.len() < profile.slots as usize {
949                slots.resize_with(profile.slots as usize, Slot::default);
950            }
951            self.retired.remove(name);
952        }
953        for (name, profile) in std::mem::take(&mut old.profiles) {
954            if !self.config.profiles.contains_key(&name) {
955                self.retired.insert(name, profile);
956            }
957        }
958        self.drop_closed_slots();
959        self.images.stale_due = true;
960        let names: BTreeSet<String> = self
961            .config
962            .repositories
963            .iter()
964            .map(|repository| repository.name.clone())
965            .collect();
966        self.repositories.reload(&names);
967        // A pool gone, or moved to another profile, keeps no wait a pool made there later would inherit.
968        let config = &self.config;
969        for (name, state) in self.repositories.iter_listed_mut() {
970            state.forget_waits(|profile| config.pool_in(name, profile).is_some());
971        }
972        self.control.set_repositories(names);
973        let profiles = &self.config.profiles;
974        self.upkeep
975            .forget_removed(|name| profiles.contains_key(name));
976        self.prepare_failures.retain(|slot, _| {
977            profiles
978                .get(&slot.profile)
979                .is_some_and(|profile| slot.index < profile.slots as usize)
980        });
981        self.isolation = isolation::backend(self.config.host.isolation);
982        // The growth count's looks were taken by other rules, or none were: its brake begins again, as at a restart
983        // (ADR 0130). Its guard reads `MemoryMax` and the host's ceiling.
984        let pooled = |config: &Config| {
985            config.host.admission == crate::config::Admission::Measured && config.host.pooled_count
986        };
987        if pooled(&self.config)
988            && (!pooled(&old)
989                || memory_max != self.memory_max
990                || old.host.memory_ceiling != self.config.host.memory_ceiling)
991        {
992            self.measured.restart_growth(self.now());
993        }
994        self.memory_max = memory_max;
995        self.cpus = cpus;
996        self.caches
997            .send_replace(cache::volumes(&self.names, &self.config));
998        // Labels moved to another pool take their history with them: what each pool wants is learned again.
999        self.reread_warm();
1000        info!("reloaded the config");
1001    }
1002}
1003
1004/// Whether `repository`'s runners in `profile` start the same under `new` as under `old`: the pool's labels, and its
1005/// profile's image, memory and CPUs and the host's isolation. A pool either config lacks does not.
1006fn same_runners(old: &Config, new: &Config, repository: &str, profile: &str) -> bool {
1007    let labels = |pool: &crate::config::Repository| -> BTreeSet<String> {
1008        pool.labels
1009            .iter()
1010            .map(|label| label.to_ascii_lowercase())
1011            .collect()
1012    };
1013    let (Some(was), Some(is)) = (
1014        old.pool_in(repository, profile),
1015        new.pool_in(repository, profile),
1016    ) else {
1017        return false;
1018    };
1019    let (Some(was_made), Some(is_made)) = (old.profiles.get(profile), new.profiles.get(profile))
1020    else {
1021        return false;
1022    };
1023    labels(was) == labels(is)
1024        && was_made.image == is_made.image
1025        && was_made.memory.0 == is_made.memory.0
1026        && was_made.cpus == is_made.cpus
1027        && old.host.isolation == new.host.isolation
1028}