Skip to main content

runnerd_core/scheduler/engine/
control.rs

1//! Control from the API: the `Control` handle the daemon's API holds (pause flags, which it records in the store so
2//! a restart keeps them, shutdown, a channel of `Command`s) and the engine's side of each command: pause and resume,
3//! of one repository or all, with or without cancelling their runs, reseed, cache clean, config replacement, the
4//! removal of records and the shells opened in them.
5//! Commands run between ticks or while a tick waits for GitHub, so they see the slot state a tick does (ADRs 0019,
6//! 0026). It must not serve HTTP or write the config file: the daemon does, through `Commit`.
7
8use std::collections::{BTreeMap, BTreeSet};
9use std::sync::Mutex;
10use std::time::Duration;
11
12use tokio::sync::{Notify, mpsc, oneshot, watch};
13use tracing::{info, warn};
14
15use super::records::RecordSelection;
16use super::stop::Stop;
17use super::{Engine, Occupant, Slot};
18use crate::api;
19use crate::cache;
20use crate::config::{ByteSize, Config};
21use crate::docker::{DockerApi, Tty, TtySize};
22use crate::github::GithubApi;
23use crate::isolation;
24use crate::scheduler::{SlotRef, StopReason};
25use crate::sizing::auto;
26use crate::store::Store;
27use crate::sync::lock;
28use crate::time;
29
30/// How long the runner of a run a forced pause cancelled gets to report its job cancelled and exit, before it is
31/// removed: GitHub completes a cancelled run only once its runner reports it, which took 40s in a test.
32pub(super) const CANCEL_GRACE: Duration = Duration::from_secs(90);
33
34/// How the daemon is asked to stop, from the least to the most urgent: a later request only ever makes the stop more
35/// urgent (ADR 0091).
36#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
37pub enum Shutdown {
38    /// Keep scheduling.
39    No,
40    /// Start nothing new, let running jobs finish, then remove every runner: `runnerd stop`.
41    Graceful,
42    /// Remove idle runners, leave busy ones and the mirror to the next start, which adopts them, and stop within
43    /// `STOP_DEADLINE`: a signal, as systemd sends on a stop.
44    Detach,
45    /// Kill running jobs and delete their runners, within `STOP_DEADLINE`: `runnerd stop --force`, a second signal.
46    Force,
47}
48
49impl Shutdown {
50    /// How a daemon stopping in this mode says it stops; `None` for `No`.
51    pub fn stopping(self) -> Option<api::Stopping> {
52        match self {
53            Shutdown::No => None,
54            Shutdown::Graceful => Some(api::Stopping::WaitsForJobs),
55            Shutdown::Detach => Some(api::Stopping::LeavesJobs),
56            Shutdown::Force => Some(api::Stopping::KillsJobs),
57        }
58    }
59}
60
61/// What a pause, a shutdown or a runner's removal does with the jobs runners are running.
62#[derive(Debug, Clone, Copy, PartialEq, Eq)]
63pub enum RunningJobs {
64    /// Leave them running: a pause refuses, a busy runner stays.
65    Keep,
66    /// Kill them: a pause cancels their runs, a removal kills their runners.
67    Kill,
68}
69
70/// What a new config may do to slots with a runner in them.
71#[derive(Debug, Clone, Copy, PartialEq, Eq)]
72pub enum RunningSlots {
73    /// A slot past a lowered `slots`, or of a removed profile, stays until its runner finishes.
74    Drain,
75    /// The config is refused while a runner is in a slot it does not have: what auto sizing asks for.
76    Keep,
77}
78
79/// Who writes a new config.
80#[derive(Debug, Clone, Copy, PartialEq, Eq)]
81pub enum Author {
82    /// The owner, through `PUT /v1/config`: a save that lowers `memory` under a raise of auto sizing's in the last
83    /// day is answered with a warning (`auto::undone_raises`).
84    Owner,
85    /// Auto sizing, whose own lowering is never a warning about its raise.
86    AutoSizing,
87}
88
89/// `Err` when `new` moves runnerd to another slice than `running` names: the sampler and the running containers stay
90/// in the slice runnerd started in, so the slice changes only at a restart.
91pub fn same_slice(running: &Config, new: &Config) -> Result<(), ControlError> {
92    if new.host.slice == running.host.slice {
93        return Ok(());
94    }
95    Err(ControlError::Invalid(format!(
96        "restart runnerd to change the slice: it runs in {}, and host.slice is {} in the new config",
97        running.host.slice, new.host.slice
98    )))
99}
100
101/// The repositories a pause or a resume applies to.
102#[derive(Debug, Clone, PartialEq, Eq)]
103pub enum Target {
104    /// One repository, `owner/name`.
105    Repository(String),
106    /// Every configured repository.
107    All,
108}
109
110/// Why the engine refused or failed a command from the API.
111#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
112pub enum ControlError {
113    /// No such repository, profile or cache.
114    #[error("{0}")]
115    NotFound(String),
116    /// Runners use what the command would change.
117    #[error("{message}")]
118    Busy {
119        /// Names the runners and their jobs.
120        message: String,
121        /// The runners in the way.
122        jobs: Vec<api::RunningJob>,
123    },
124    /// The command conflicts with one already in progress.
125    #[error("{0}")]
126    Conflict(String),
127    /// The command cannot apply to this target.
128    #[error("{0}")]
129    Invalid(String),
130    /// Docker or GitHub failed.
131    #[error("{0}")]
132    Failed(String),
133    /// A pause or resume holds in memory, but the job history could not record it: a restart undoes it.
134    #[error("{0}")]
135    NotRecorded(String),
136    /// The engine stopped.
137    #[error("runnerd is stopping")]
138    Stopped,
139}
140
141/// A command the engine carries out between ticks or while a tick waits for GitHub, with where its answer goes.
142#[derive(Debug)]
143pub enum Command {
144    /// Pause one repository or all; with `RunningJobs::Kill`, cancel their running jobs.
145    Pause {
146        /// What to pause.
147        target: Target,
148        /// Cancel running jobs, or refuse while there are any.
149        jobs: RunningJobs,
150        /// The answer.
151        reply: oneshot::Sender<Result<api::Paused, ControlError>>,
152    },
153    /// Resume one repository or all.
154    Resume {
155        /// What to resume.
156        target: Target,
157        /// The answer.
158        reply: oneshot::Sender<Result<(), ControlError>>,
159    },
160    /// Make slot 1's layers the profile's snapshot once its slots are free.
161    Reseed {
162        /// The profile.
163        profile: String,
164        /// The answer.
165        reply: oneshot::Sender<Result<(), ControlError>>,
166    },
167    /// Remove a slot's volume or a shared cache's, unless a runner uses it.
168    CleanCache {
169        /// The volume.
170        name: String,
171        /// The answer.
172        reply: oneshot::Sender<Result<(), ControlError>>,
173    },
174    /// Write a new, validated config to its file and run with it, with no tick in between.
175    ReplaceConfig {
176        /// The config.
177        config: Box<Config>,
178        /// The `MemoryMax` of the slice it names.
179        memory_max: Option<u64>,
180        /// The CPUs of the slice it names, in millicores, read when it sets `cpu_overcommit`.
181        cpus: Option<u64>,
182        /// Writes the file.
183        commit: Commit,
184        /// Whether it may leave runners in slots it does not have.
185        running: RunningSlots,
186        /// Who writes it.
187        author: Author,
188        /// Answered once the engine runs with it: with the commit's warning and what it undoes of auto sizing's
189        /// raises, or why it did not commit.
190        reply: oneshot::Sender<Result<Vec<String>, ControlError>>,
191    },
192    /// Remove records of failed jobs.
193    RemoveRecords {
194        /// Which.
195        selection: RecordSelection,
196        /// The answer.
197        reply: oneshot::Sender<Result<api::RemovedRecords, ControlError>>,
198    },
199    /// Start a shell in a job's record, if the budget has room for it.
200    OpenShell {
201        /// The job.
202        job: u64,
203        /// Its terminal's size.
204        size: TtySize,
205        /// The shell's terminal.
206        reply: oneshot::Sender<Result<Tty, ControlError>>,
207    },
208    /// Remove a shell's container: its terminal closed.
209    CloseShell {
210        /// The container's ID.
211        container: String,
212    },
213}
214
215/// What makes a new config durable, run by the engine before it runs with the config: `Ok` once the config is in
216/// place, with a warning if it may not survive a crash; `Err` with nothing changed.
217pub struct Commit(Box<dyn FnOnce() -> Result<Option<String>, String> + Send>);
218
219impl Commit {
220    /// A commit that runs `write`, off the engine's thread.
221    pub fn new(write: impl FnOnce() -> Result<Option<String>, String> + Send + 'static) -> Commit {
222        Commit(Box::new(write))
223    }
224
225    /// Runs it.
226    pub async fn run(self) -> Result<Option<String>, String> {
227        tokio::task::spawn_blocking(self.0)
228            .await
229            .unwrap_or_else(|join_error| Err(join_error.to_string()))
230    }
231}
232
233impl std::fmt::Debug for Commit {
234    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
235        f.write_str("Commit")
236    }
237}
238
239/// The engine's end of `Control`'s commands.
240#[derive(Debug)]
241pub struct Commands(mpsc::UnboundedReceiver<Command>);
242
243impl Commands {
244    /// The next command; `None` once `Control` is gone.
245    pub async fn recv(&mut self) -> Option<Command> {
246        self.0.recv().await
247    }
248}
249
250/// Commands from the API and signals to the loop.
251#[derive(Debug)]
252pub struct Control {
253    // Locked with `sync::lock`: the guarded data stays consistent, as every critical section is a single insert,
254    // remove or read.
255    repositories: Mutex<BTreeSet<String>>,
256    paused: Mutex<BTreeSet<String>>,
257    /// Where `paused` is recorded.
258    store: Store,
259    shutdown: watch::Sender<Stop>,
260    pub(super) wake: Notify,
261    commands: mpsc::UnboundedSender<Command>,
262}
263
264impl Control {
265    /// Controls for the configured repositories, of which those in `paused` start paused, and the commands they
266    /// send the engine. A paused repository the config no longer has is forgotten, in `store` too: one added back
267    /// starts active.
268    pub fn new(
269        repositories: impl IntoIterator<Item = String>,
270        paused: impl IntoIterator<Item = String>,
271        store: Store,
272    ) -> (Control, Commands) {
273        let repositories: BTreeSet<String> = repositories.into_iter().collect();
274        let paused: BTreeSet<String> = paused
275            .into_iter()
276            .filter(|repository| repositories.contains(repository))
277            .collect();
278        // Sent at once; a write that fails leaves rows the next start filters out the same way.
279        drop(store.paused(paused.iter().cloned().collect()));
280        let (commands, receiver) = mpsc::unbounded_channel();
281        let control = Control {
282            repositories: Mutex::new(repositories),
283            paused: Mutex::new(paused),
284            store,
285            shutdown: watch::Sender::new(Stop::none()),
286            wake: Notify::new(),
287            commands,
288        };
289        (control, Commands(receiver))
290    }
291
292    /// Whether the daemon is stopping and how: as requested, or as a detach once Docker stopped answering the stop.
293    pub fn mode(&self) -> Shutdown {
294        self.shutdown.borrow().mode()
295    }
296
297    /// Whether Docker stopped answering the stop, which is then left to the next start.
298    pub(super) fn docker_gone(&self) -> bool {
299        self.shutdown.borrow().docker_gone()
300    }
301
302    /// Notes that Docker did not answer a call the stop needed, and returns whether that is news: not without a stop
303    /// under way, nor once noted. Its deadline starts, and a later `runnerd stop` answers that it leaves running jobs.
304    pub(super) fn note_docker_gone(&self) -> bool {
305        let news = self.shutdown.send_if_modified(Stop::note_docker_gone);
306        if news {
307            self.wake.notify_one();
308        }
309        news
310    }
311
312    /// Resolves once the stop has a deadline: it detaches or is forced.
313    pub(super) async fn hurried(&self) {
314        let mut mode = self.shutdown.subscribe();
315        // `self` holds the sender, so the wait ends only once the mode is reached.
316        let _ = mode.wait_for(|stop| stop.hurried()).await;
317    }
318
319    /// Pauses or resumes one repository or all, whatever they run, and records it in the store before it answers,
320    /// so a restarted daemon starts with the same ones paused. `NotFound` for a repository the config does not have.
321    /// If the record fails, the change holds until a restart and the answer is `NotRecorded`. The status shows it
322    /// at the engine's next publish: the API pauses and resumes through `pause` and `resume`, which publish before
323    /// they answer.
324    pub async fn set_paused(&self, target: &Target, paused: bool) -> Result<(), ControlError> {
325        let recorded = {
326            let repositories = lock(&self.repositories);
327            let names: Vec<&String> = match target {
328                Target::Repository(repository) => {
329                    let Some(name) = repositories.get(repository) else {
330                        return Err(not_configured(repository, repositories.iter()));
331                    };
332                    vec![name]
333                }
334                Target::All => repositories.iter().collect(),
335            };
336            let mut set = lock(&self.paused);
337            for name in names {
338                if paused {
339                    set.insert(name.clone());
340                } else {
341                    set.remove(name);
342                }
343            }
344            // Sent under the lock, so the records change in the order the set does.
345            self.store.paused(set.iter().cloned().collect())
346        };
347        self.wake.notify_one();
348        if recorded.await {
349            return Ok(());
350        }
351        let name = match target {
352            Target::Repository(repository) => repository.as_str(),
353            Target::All => "every repository",
354        };
355        let (done, undone) = if paused {
356            ("paused", "resumes it")
357        } else {
358            ("resumed", "pauses it again")
359        };
360        let message = format!(
361            "{name} is {done} until runnerd restarts, but the job history cannot record it, so a restart {undone}"
362        );
363        warn!("{message}");
364        Err(ControlError::NotRecorded(message))
365    }
366
367    /// Replaces the configured repositories; the removed ones are no longer paused, in the store too.
368    fn set_repositories(&self, repositories: BTreeSet<String>) {
369        let mut configured = lock(&self.repositories);
370        let mut paused = lock(&self.paused);
371        let before = paused.len();
372        paused.retain(|repository| repositories.contains(repository));
373        if paused.len() != before {
374            // Not waited for: a restart forgets a repository the config does not have anyway.
375            drop(self.store.paused(paused.iter().cloned().collect()));
376        }
377        *configured = repositories;
378    }
379
380    /// Asks the loop to stop; a more urgent request overrides a less urgent one, never the other way. Returns how it
381    /// stops now, which is what status says.
382    pub fn shutdown(&self, mode: Shutdown) -> Shutdown {
383        self.shutdown.send_if_modified(|stop| stop.ask(mode));
384        self.wake.notify_one();
385        self.mode()
386    }
387
388    /// Whether the repository is paused.
389    pub fn is_paused(&self, repository: &str) -> bool {
390        lock(&self.paused).contains(repository)
391    }
392
393    async fn send<T>(
394        &self,
395        command: impl FnOnce(oneshot::Sender<T>) -> Command,
396    ) -> Result<T, ControlError> {
397        let (reply, answer) = oneshot::channel();
398        self.commands
399            .send(command(reply))
400            .map_err(|_| ControlError::Stopped)?;
401        answer.await.map_err(|_| ControlError::Stopped)
402    }
403
404    /// Pauses one repository or all. With `RunningJobs::Keep`, refuses while any of them runs a job, pausing none;
405    /// with `Kill`, cancels the runs of their running jobs on GitHub, whose runners are removed once they report it
406    /// or `CANCEL_GRACE` later, and removes their other runners at once.
407    pub async fn pause(
408        &self,
409        target: Target,
410        jobs: RunningJobs,
411    ) -> Result<api::Paused, ControlError> {
412        self.send(|reply| Command::Pause {
413            target,
414            jobs,
415            reply,
416        })
417        .await?
418    }
419
420    /// Resumes one repository or all, and publishes the status before it answers, so a client that reads the status
421    /// or the events after the answer sees them resumed. Fails as `set_paused` does.
422    pub async fn resume(&self, target: Target) -> Result<(), ControlError> {
423        self.send(|reply| Command::Resume { target, reply }).await?
424    }
425
426    /// Reseeds a profile as soon as its slots are free; none of them starts a runner until then.
427    pub async fn reseed(&self, profile: &str) -> Result<(), ControlError> {
428        self.send(|reply| Command::Reseed {
429            profile: profile.to_owned(),
430            reply,
431        })
432        .await?
433    }
434
435    /// Removes a slot's volume or a shared cache's, unless a runner uses it.
436    pub async fn clean_cache(&self, name: &str) -> Result<(), ControlError> {
437        self.send(|reply| Command::CleanCache {
438            name: name.to_owned(),
439            reply,
440        })
441        .await?
442    }
443
444    /// Removes the selected records of failed jobs and their images.
445    pub async fn remove_records(
446        &self,
447        selection: RecordSelection,
448    ) -> Result<api::RemovedRecords, ControlError> {
449        self.send(|reply| Command::RemoveRecords { selection, reply })
450            .await?
451    }
452
453    /// Starts a shell in a job's record and returns its terminal. The caller closes it with `close_shell` once the
454    /// terminal ends, from either side.
455    pub async fn open_shell(&self, job: u64, size: TtySize) -> Result<Tty, ControlError> {
456        self.send(|reply| Command::OpenShell { job, size, reply })
457            .await?
458    }
459
460    /// Removes a shell's container. Not waited for, so it can be sent while the terminal is dropped.
461    pub fn close_shell(&self, container: String) {
462        let _ = self.commands.send(Command::CloseShell { container });
463    }
464
465    /// Runs `commit` to write a validated config, and if it commits, runs with the config from then on, before the
466    /// next tick. If it does not, nothing changes and the answer is `Failed`. Running jobs keep running: under
467    /// `RunningSlots::Drain` removed profiles and slots go as their runners finish; under `Keep` a config without a
468    /// slot a runner is in is refused (`Conflict`) before `commit` runs. A config with another `host.slice` is refused
469    /// (`Invalid`) before `commit` runs: the slice takes effect when runnerd starts. The answer warns of what may not
470    /// survive a crash and, for `Author::Owner`, of each profile whose `memory` it lowers under a raise of auto
471    /// sizing's in the last day (`auto::undone_raises`), which are logged too.
472    pub async fn replace_config(
473        &self,
474        config: Config,
475        memory_max: Option<u64>,
476        cpus: Option<u64>,
477        commit: Commit,
478        running: RunningSlots,
479        author: Author,
480    ) -> Result<Vec<String>, ControlError> {
481        self.send(|reply| Command::ReplaceConfig {
482            config: Box::new(config),
483            memory_max,
484            cpus,
485            commit,
486            running,
487            author,
488            reply,
489        })
490        .await?
491    }
492}
493
494/// Names the repositories there are, so a mistyped name shows its fix.
495fn not_configured<'name>(
496    repository: &str,
497    configured: impl Iterator<Item = &'name String>,
498) -> ControlError {
499    let configured: Vec<&str> = configured.map(String::as_str).collect();
500    let configured = if configured.is_empty() {
501        "none is configured".to_owned()
502    } else {
503        format!("configured: {}", configured.join(", "))
504    };
505    ControlError::NotFound(format!(
506        "{repository} is not a configured repository; {configured}"
507    ))
508}
509
510/// `jobs` for people to read: `job 1 on runner-a, runner-b`.
511fn list(jobs: &[api::RunningJob]) -> String {
512    jobs.iter()
513        .map(ToString::to_string)
514        .collect::<Vec<_>>()
515        .join(", ")
516}
517
518impl<G: GithubApi + 'static, D: DockerApi + 'static> Engine<G, D> {
519    /// Carries out a command from the API.
520    pub(super) async fn command(&mut self, command: Command) {
521        match command {
522            Command::Pause {
523                target,
524                jobs,
525                reply,
526            } => {
527                let _ = reply.send(self.pause(&target, jobs).await);
528            }
529            Command::Resume { target, reply } => {
530                let _ = reply.send(self.control.set_paused(&target, false).await);
531            }
532            Command::Reseed { profile, reply } => {
533                let _ = reply.send(self.request_reseed(&profile));
534            }
535            Command::CleanCache { name, reply } => {
536                let _ = reply.send(self.clean_cache(&name).await);
537            }
538            Command::ReplaceConfig {
539                config,
540                memory_max,
541                cpus,
542                commit,
543                running,
544                author,
545                reply,
546            } => {
547                if let Err(refused) = same_slice(&self.config, &config) {
548                    let _ = reply.send(Err(refused));
549                    return;
550                }
551                if running == RunningSlots::Keep
552                    && let Some(occupied) = self.occupied_past(&config)
553                {
554                    let _ = reply.send(Err(ControlError::Conflict(occupied)));
555                    return;
556                }
557                let undone = match author {
558                    Author::Owner => self.undone_raises(&config).await,
559                    Author::AutoSizing => Vec::new(),
560                };
561                let answer = match commit.run().await {
562                    Ok(warning) => {
563                        for line in &undone {
564                            warn!("config replaced: {line}");
565                        }
566                        self.reload(*config, memory_max, cpus);
567                        Ok(warning.into_iter().chain(undone).collect())
568                    }
569                    Err(message) => Err(ControlError::Failed(message)),
570                };
571                let _ = reply.send(answer);
572            }
573            Command::RemoveRecords { selection, reply } => {
574                let _ = reply.send(self.remove_records(&selection).await);
575            }
576            Command::OpenShell { job, size, reply } => {
577                let opened = self.open_shell(job, size).await;
578                // A terminal nobody waits for any more is closed at once.
579                if let Err(Ok(tty)) = reply.send(opened) {
580                    self.close_shell(&tty.container).await;
581                }
582            }
583            Command::CloseShell { container } => self.close_shell(&container).await,
584        }
585    }
586
587    /// Pauses one repository or all. A runner of theirs that holds a job, or that GitHub reports busy, refuses it
588    /// with `RunningJobs::Keep`; with `Kill`, the runs of their jobs are cancelled, and their runners removed once
589    /// they exit or `CANCEL_GRACE` later; a runner whose run cannot be cancelled is removed at once.
590    pub(super) async fn pause(
591        &mut self,
592        target: &Target,
593        running: RunningJobs,
594    ) -> Result<api::Paused, ControlError> {
595        let repositories: BTreeSet<String> = match target {
596            Target::Repository(repository) if self.repositories.listed(repository).is_none() => {
597                return Err(not_configured(repository, self.repositories.listed_names()));
598            }
599            Target::Repository(repository) => BTreeSet::from([repository.clone()]),
600            Target::All => self.repositories.listed_names().cloned().collect(),
601        };
602        self.take_found();
603        let runners: Vec<(SlotRef, Occupant)> = self
604            .occupied()
605            .filter(|(_, occupant)| repositories.contains(&occupant.repository))
606            .map(|(slot, occupant)| (slot, occupant.clone()))
607            .collect();
608        let jobs: Vec<api::RunningJob> = runners
609            .iter()
610            .filter(|(_, occupant)| occupant.engaged())
611            .map(|(_, occupant)| occupant.running_job())
612            .collect();
613        if running == RunningJobs::Keep && !jobs.is_empty() {
614            let mut by_repository: BTreeMap<&str, Vec<api::RunningJob>> = BTreeMap::new();
615            for job in &jobs {
616                by_repository
617                    .entry(&job.repository)
618                    .or_default()
619                    .push(job.clone());
620            }
621            let runs: Vec<String> = by_repository
622                .iter()
623                .map(|(repository, jobs)| format!("{repository} runs {}", list(jobs)))
624                .collect();
625            // The jobs only: each client says how to force the pause in its own terms.
626            return Err(ControlError::Busy {
627                message: runs.join("; "),
628                jobs,
629            });
630        }
631        // Paused in memory even if not recorded: a forced pause still cancels the runs it was asked to.
632        let recorded = self.control.set_paused(target, true).await;
633        match target {
634            Target::Repository(repository) => info!("{repository}: paused"),
635            Target::All => info!("paused every repository"),
636        }
637        let mut stopped = Vec::new();
638        if running == RunningJobs::Keep {
639            return recorded.map(|()| api::Paused { stopped });
640        }
641        for (slot, occupant) in runners {
642            let repository = &occupant.repository;
643            let cancelled = if let Some(job) = occupant.named_job() {
644                self.cancel(repository, job).await
645            } else {
646                if occupant.listed_busy() {
647                    warn!(
648                        "{repository}: {} runs a job its log has not named yet; GitHub fails it once the runner is gone",
649                        occupant.runner_name
650                    );
651                }
652                false
653            };
654            if cancelled {
655                // `collect_finished` removes it once it exits, `kill_cancelled` if it does not.
656                if let Some(cancelled) = self
657                    .slots
658                    .values_mut()
659                    .flatten()
660                    .filter_map(Slot::occupant_mut)
661                    .find(|candidate| candidate.runner_name == occupant.runner_name)
662                {
663                    cancelled
664                        .engagement
665                        .cancelled(tokio::time::Instant::now() + CANCEL_GRACE);
666                }
667                // A restart keeps it.
668                self.store.runner_cancel_by(
669                    &occupant.runner_name,
670                    time::now_ms() + u64::try_from(CANCEL_GRACE.as_millis()).unwrap_or(u64::MAX),
671                );
672            } else {
673                self.kill(&slot).await;
674                if self
675                    .occupied()
676                    .any(|(_, candidate)| candidate.runner_name == occupant.runner_name)
677                {
678                    continue;
679                }
680            }
681            stopped.push(occupant.running_job());
682        }
683        match recorded {
684            Ok(()) => Ok(api::Paused { stopped }),
685            Err(ControlError::NotRecorded(message)) if !stopped.is_empty() => Err(
686                ControlError::NotRecorded(format!("{message}; stopped {}", list(&stopped))),
687            ),
688            Err(error) => Err(error),
689        }
690    }
691
692    /// Removes the runners of cancelled runs that did not exit within `CANCEL_GRACE`, every tick until they are gone.
693    pub(super) async fn kill_cancelled(&mut self) {
694        let now = tokio::time::Instant::now();
695        let overdue: Vec<(SlotRef, String)> = self
696            .occupied()
697            .filter(|(_, occupant)| occupant.engagement.cancel_overdue(now))
698            .map(|(slot, occupant)| (slot, occupant.runner_name.clone()))
699            .collect();
700        for (slot, name) in overdue {
701            warn!(
702                "{name} did not exit {}s after its run was cancelled",
703                CANCEL_GRACE.as_secs()
704            );
705            // The deadline stays until the runner is gone, so a failed removal is tried again next tick.
706            self.kill(&slot).await;
707        }
708    }
709
710    /// Cancels the run of a job; whether GitHub accepted it.
711    async fn cancel(&self, repository: &str, job: u64) -> bool {
712        let run = match self.github.job(repository, job).await {
713            Ok(details) => details.run_id,
714            Err(error) => {
715                warn!("{repository}: cannot read job {job} to cancel its run: {error}");
716                return false;
717            }
718        };
719        match self.github.cancel_run(repository, run).await {
720            Ok(()) => {
721                info!("{repository}: cancelled run {run} of job {job}");
722                true
723            }
724            Err(error) => {
725                warn!("{repository}: cannot cancel run {run}: {error}");
726                false
727            }
728        }
729    }
730
731    /// Asks for a profile's reseed: none of its slots starts a runner until they are free and it ran.
732    pub(super) fn request_reseed(&mut self, profile: &str) -> Result<(), ControlError> {
733        if !self.config.profiles.contains_key(profile) {
734            return Err(ControlError::NotFound(format!(
735                "{profile} is not a configured profile"
736            )));
737        }
738        if self.is_reseeding(profile) {
739            return Err(ControlError::Conflict(format!(
740                "{profile} is reseeding already"
741            )));
742        }
743        // A shell mounts the snapshot a reseed replaces, and may stay open for hours.
744        if self.shell_profiles().contains(profile) {
745            return Err(ControlError::Conflict(format!(
746                "a shell is open in a record of {profile}, which mounts its snapshot; reseed it once the shell exits"
747            )));
748        }
749        info!("{profile}: reseeding once its slots are free");
750        self.upkeep.request(profile);
751        Ok(())
752    }
753
754    /// Removes a slot's volume or a shared cache's, unless a job or a reseed uses it; the runners that use it and
755    /// wait for a job stop first. Snapshots are rebuilt by reseeding instead: slot layers sit on top of them.
756    pub(super) async fn clean_cache(&mut self, name: &str) -> Result<(), ControlError> {
757        let Some(volume) = cache::volumes(&self.names, &self.config)
758            .into_iter()
759            .find(|volume| volume.name == name)
760        else {
761            return Err(ControlError::NotFound(format!(
762                "{name} is not a cache volume of this config; `runnerctl caches` lists them"
763            )));
764        };
765        let using: Vec<SlotRef> = match &volume.kind {
766            api::CacheKind::Snapshot { profile } => {
767                return Err(ControlError::Invalid(format!(
768                    "{name} is {profile}'s snapshot, which every slot's layers sit on; reseed {profile} to rebuild it"
769                )));
770            }
771            api::CacheKind::Slot { profile, slot } => {
772                if self.upkeep.reseed_runs(profile) {
773                    return Err(ControlError::Conflict(format!(
774                        "{profile} is reseeding, which writes {name}"
775                    )));
776                }
777                self.occupied()
778                    .filter(|(at, _)| at.profile == *profile && at.index + 1 == *slot)
779                    .map(|(at, _)| at)
780                    .collect()
781            }
782            api::CacheKind::Pnpm
783            | api::CacheKind::Maven
784            | api::CacheKind::Mirror
785            | api::CacheKind::Build => self.occupied().map(|(at, _)| at).collect(),
786        };
787        let shared = matches!(volume.kind, api::CacheKind::Pnpm | api::CacheKind::Maven);
788        if shared && self.records.has_shells() {
789            return Err(ControlError::Conflict(format!(
790                "a shell open in a record mounts {name}; clean it once the shell exits"
791            )));
792        }
793        // Runners waiting for a job, warm ones too, give way: the next plan starts them again. One GitHub gave a
794        // job meanwhile stays, and holds the clean back.
795        let idle = using
796            .iter()
797            .filter(|slot| self.occupant(slot).is_some_and(Occupant::idle))
798            .map(|slot| (slot.clone(), StopReason::Clean))
799            .collect();
800        self.remove_idle(idle).await;
801        let users: Vec<api::RunningJob> = using
802            .iter()
803            .filter_map(|slot| self.occupant(slot))
804            .map(Occupant::running_job)
805            .collect();
806        if !users.is_empty() {
807            return Err(ControlError::Busy {
808                message: format!(
809                    "{name} is in use by {}; clean it once they finish",
810                    list(&users)
811                ),
812                jobs: users,
813            });
814        }
815        if volume.kind == api::CacheKind::Build {
816            let Some(cache) = self.build_cache.clone() else {
817                return Err(ControlError::Failed(
818                    "the build cache did not open when runnerd started; its log says why"
819                        .to_owned(),
820                ));
821            };
822            let trimmed = tokio::task::spawn_blocking(move || cache.clear())
823                .await
824                .map_err(|error| ControlError::Failed(format!("cannot empty {name}: {error}")))?;
825            info!(
826                "emptied {name}: {} entries, {}",
827                trimmed.before.entries,
828                ByteSize(trimmed.before.bytes)
829            );
830            self.trim_due = true;
831            return Ok(());
832        }
833        if volume.kind == api::CacheKind::Mirror && self.mirror.may_run() {
834            cache::mirror_down(&*self.docker, &self.names)
835                .await
836                .map_err(|error| {
837                    ControlError::Failed(format!("cannot stop the mirror: {error}"))
838                })?;
839            self.mirror.stopped();
840        }
841        self.docker
842            .remove_volume(name)
843            .await
844            .map_err(|error| ControlError::Failed(format!("cannot remove {name}: {error}")))?;
845        info!("removed {name}");
846        // Its layer no longer holds what a job cached.
847        if let api::CacheKind::Slot { profile, slot } = &volume.kind {
848            self.ran.remove(&SlotRef {
849                profile: profile.clone(),
850                index: slot.saturating_sub(1),
851            });
852        }
853        // Measured again once idle.
854        self.trim_due = true;
855        Ok(())
856    }
857
858    /// What `config` undoes of auto sizing's `memory` raises in the last day; none when they cannot be read, which is
859    /// logged: the config runs either way.
860    async fn undone_raises(&self, config: &Config) -> Vec<String> {
861        let now = time::now_ms() / 1000;
862        let since = now.saturating_sub(auto::UNDONE_RAISES).saturating_mul(1000);
863        match self.store.sizing_changes(since).await {
864            Ok(changes) => auto::undone_raises(&self.config, config, &changes, now),
865            Err(store_error) => {
866                warn!(
867                    "cannot read auto sizing's changes to compare a new config with: {store_error}"
868                );
869                Vec::new()
870            }
871        }
872    }
873
874    /// The first slot with a runner in it that `config` does not have, for people to read.
875    fn occupied_past(&self, config: &Config) -> Option<String> {
876        self.slots.iter().find_map(|(name, slots)| {
877            let kept = config
878                .profiles
879                .get(name)
880                .map_or(0, |profile| profile.slots as usize);
881            slots
882                .iter()
883                .enumerate()
884                .skip(kept)
885                .find(|(_, slot)| !slot.is_free())
886                .map(|(index, _)| {
887                    format!(
888                        "{name}'s slot {} has a runner, and the new config keeps {kept} slots",
889                        index + 1
890                    )
891                })
892        })
893    }
894
895    /// Runs with a new, validated config. Running jobs keep running: a removed profile, or slots past a lowered
896    /// number, stay until their runners finish, and no runner starts in a profile while it has as many as its
897    /// slots. Runners started from now on get the new settings.
898    /// `cpus` are the slice's, read when the config sets `cpu_overcommit`.
899    pub(super) fn reload(&mut self, config: Config, memory_max: Option<u64>, cpus: Option<u64>) {
900        let mut old = std::mem::replace(&mut self.config, config);
901        // A pool gone keeps no hold a pool made there later would inherit, and one whose runners change starts again
902        // at once: what failed may be what changed.
903        for (name, state) in self.repositories.iter_listed_mut() {
904            let new = &self.config;
905            state.forget_health(|profile| same_runners(&old, new, name, profile));
906        }
907        if let Some(cache) = &self.build_cache {
908            cache.set_max_size(self.config.caches.build.max_size.0);
909        }
910        for (name, profile) in &self.config.profiles {
911            let slots = self.slots.entry(name.clone()).or_default();
912            if slots.len() < profile.slots as usize {
913                slots.resize_with(profile.slots as usize, Slot::default);
914            }
915            self.retired.remove(name);
916        }
917        for (name, profile) in std::mem::take(&mut old.profiles) {
918            if !self.config.profiles.contains_key(&name) {
919                self.retired.insert(name, profile);
920            }
921        }
922        self.drop_closed_slots();
923        self.images.stale_due = true;
924        let names: BTreeSet<String> = self
925            .config
926            .repositories
927            .iter()
928            .map(|repository| repository.name.clone())
929            .collect();
930        self.repositories.reload(&names);
931        // A pool gone, or moved to another profile, keeps no wait a pool made there later would inherit.
932        let config = &self.config;
933        for (name, state) in self.repositories.iter_listed_mut() {
934            state.forget_waits(|profile| config.pool_in(name, profile).is_some());
935        }
936        self.control.set_repositories(names);
937        let profiles = &self.config.profiles;
938        self.upkeep
939            .forget_removed(|name| profiles.contains_key(name));
940        self.prepare_failures.retain(|slot, _| {
941            profiles
942                .get(&slot.profile)
943                .is_some_and(|profile| slot.index < profile.slots as usize)
944        });
945        self.isolation = isolation::backend(self.config.host.isolation);
946        // The growth count's looks were taken by other rules, or none were: its brake begins again, as at a restart
947        // (ADR 0130). Its guard reads `MemoryMax` and the host's ceiling.
948        let pooled = |config: &Config| {
949            config.host.admission == crate::config::Admission::Measured && config.host.pooled_count
950        };
951        if pooled(&self.config)
952            && (!pooled(&old)
953                || memory_max != self.memory_max
954                || old.host.memory_ceiling != self.config.host.memory_ceiling)
955        {
956            self.measured.restart_growth(self.now());
957        }
958        self.memory_max = memory_max;
959        self.cpus = cpus;
960        self.caches
961            .send_replace(cache::volumes(&self.names, &self.config));
962        // Labels moved to another pool take their history with them: what each pool wants is learned again.
963        self.reread_warm();
964        info!("reloaded the config");
965    }
966}
967
968/// Whether `repository`'s runners in `profile` start the same under `new` as under `old`: the pool's labels, and its
969/// profile's image, memory and CPUs and the host's isolation. A pool either config lacks does not.
970fn same_runners(old: &Config, new: &Config, repository: &str, profile: &str) -> bool {
971    let labels = |pool: &crate::config::Repository| -> BTreeSet<String> {
972        pool.labels
973            .iter()
974            .map(|label| label.to_ascii_lowercase())
975            .collect()
976    };
977    let (Some(was), Some(is)) = (
978        old.pool_in(repository, profile),
979        new.pool_in(repository, profile),
980    ) else {
981        return false;
982    };
983    let (Some(was_made), Some(is_made)) = (old.profiles.get(profile), new.profiles.get(profile))
984    else {
985        return false;
986    };
987    labels(was) == labels(is)
988        && was_made.image == is_made.image
989        && was_made.memory.0 == is_made.memory.0
990        && was_made.cpus == is_made.cpus
991        && old.host.isolation == new.host.isolation
992}