Skip to main content

runnerd_core/scheduler/engine/
control.rs

1//! Control from the API: the `Control` handle the daemon's API holds (pause flags, which it records in the store so
2//! a restart keeps them, shutdown, a channel of `Command`s) and the engine's side of each command: pause and resume,
3//! of one repository or all, with or without cancelling their runs, reseed, cache clean, config replacement, the
4//! removal of records and the shells opened in them.
5//! Commands run between ticks or while a tick waits for GitHub, so they see the slot state a tick does (ADRs 0019,
6//! 0026). It must not serve HTTP or write the config file: the daemon does, through `Commit`.
7
8use std::collections::{BTreeMap, BTreeSet};
9use std::sync::{Arc, Mutex};
10use std::time::Duration;
11
12use tokio::sync::{Notify, mpsc, oneshot, watch};
13use tracing::{info, warn};
14
15use super::records::RecordSelection;
16use super::stop::Stop;
17use super::{Engine, Occupant, Slot};
18use crate::actions::policy::{Policies, Policy};
19use crate::api;
20use crate::cache;
21use crate::config::{ByteSize, Config};
22use crate::docker::{DockerApi, Tty, TtySize};
23use crate::github::GithubApi;
24use crate::isolation;
25use crate::scheduler::{SlotRef, StopReason};
26use crate::sizing::auto;
27use crate::store::Store;
28use crate::sync::lock;
29use crate::time;
30
31/// How long the runner of a run a forced pause cancelled gets to report its job cancelled and exit, before it is
32/// removed: GitHub completes a cancelled run only once its runner reports it, which took 40s in a test.
33pub(super) const CANCEL_GRACE: Duration = Duration::from_secs(90);
34
35/// How the daemon is asked to stop, from the least to the most urgent: a later request only ever makes the stop more
36/// urgent (ADR 0091).
37#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
38pub enum Shutdown {
39    /// Keep scheduling.
40    No,
41    /// Start nothing new, let running jobs finish, then remove every runner: `runnerd stop`.
42    Graceful,
43    /// Remove idle runners, leave busy ones and the mirror to the next start, which adopts them, and stop within
44    /// `STOP_DEADLINE`: a signal, as systemd sends on a stop.
45    Detach,
46    /// Kill running jobs and delete their runners, within `STOP_DEADLINE`: `runnerd stop --force`, a second signal.
47    Force,
48}
49
50impl Shutdown {
51    /// How a daemon stopping in this mode says it stops; `None` for `No`.
52    pub fn stopping(self) -> Option<api::Stopping> {
53        match self {
54            Shutdown::No => None,
55            Shutdown::Graceful => Some(api::Stopping::WaitsForJobs),
56            Shutdown::Detach => Some(api::Stopping::LeavesJobs),
57            Shutdown::Force => Some(api::Stopping::KillsJobs),
58        }
59    }
60}
61
62/// What a pause, a shutdown or a runner's removal does with the jobs runners are running.
63#[derive(Debug, Clone, Copy, PartialEq, Eq)]
64pub enum RunningJobs {
65    /// Leave them running: a pause refuses, a busy runner stays.
66    Keep,
67    /// Kill them: a pause cancels their runs, a removal kills their runners.
68    Kill,
69}
70
71/// What a new config may do to slots with a runner in them.
72#[derive(Debug, Clone, Copy, PartialEq, Eq)]
73pub enum RunningSlots {
74    /// A slot past a lowered `slots`, or of a removed profile, stays until its runner finishes.
75    Drain,
76    /// The config is refused while a runner is in a slot it does not have: what auto sizing asks for.
77    Keep,
78}
79
80/// Who writes a new config.
81#[derive(Debug, Clone, Copy, PartialEq, Eq)]
82pub enum Author {
83    /// The owner, through `PUT /v1/config`: a save that lowers `memory` under a raise of auto sizing's in the last
84    /// day is answered with a warning (`auto::undone_raises`).
85    Owner,
86    /// Auto sizing, whose own lowering is never a warning about its raise.
87    AutoSizing,
88}
89
90/// `Err` when `new` moves runnerd to another slice than `running` names: the sampler and the running containers stay
91/// in the slice runnerd started in, so the slice changes only at a restart.
92pub fn same_slice(running: &Config, new: &Config) -> Result<(), ControlError> {
93    if new.host.slice == running.host.slice {
94        return Ok(());
95    }
96    Err(ControlError::Invalid(format!(
97        "restart runnerd to change the slice: it runs in {}, and host.slice is {} in the new config",
98        running.host.slice, new.host.slice
99    )))
100}
101
102/// The repositories a pause or a resume applies to.
103#[derive(Debug, Clone, PartialEq, Eq)]
104pub enum Target {
105    /// One repository, `owner/name`.
106    Repository(String),
107    /// Every configured repository.
108    All,
109}
110
111/// Why the engine refused or failed a command from the API.
112#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
113pub enum ControlError {
114    /// No such repository, profile or cache.
115    #[error("{0}")]
116    NotFound(String),
117    /// Runners use what the command would change.
118    #[error("{message}")]
119    Busy {
120        /// Names the runners and their jobs.
121        message: String,
122        /// The runners in the way.
123        jobs: Vec<api::RunningJob>,
124    },
125    /// The command conflicts with one already in progress.
126    #[error("{0}")]
127    Conflict(String),
128    /// The command cannot apply to this target.
129    #[error("{0}")]
130    Invalid(String),
131    /// Docker or GitHub failed.
132    #[error("{0}")]
133    Failed(String),
134    /// A pause or resume holds in memory, but the job history could not record it: a restart undoes it.
135    #[error("{0}")]
136    NotRecorded(String),
137    /// The engine stopped.
138    #[error("runnerd is stopping")]
139    Stopped,
140}
141
142/// A command the engine carries out between ticks or while a tick waits for GitHub, with where its answer goes.
143#[derive(Debug)]
144pub enum Command {
145    /// Pause one repository or all; with `RunningJobs::Kill`, cancel their running jobs.
146    Pause {
147        /// What to pause.
148        target: Target,
149        /// Cancel running jobs, or refuse while there are any.
150        jobs: RunningJobs,
151        /// The answer.
152        reply: oneshot::Sender<Result<api::Paused, ControlError>>,
153    },
154    /// Resume one repository or all.
155    Resume {
156        /// What to resume.
157        target: Target,
158        /// The answer.
159        reply: oneshot::Sender<Result<(), ControlError>>,
160    },
161    /// Make slot 1's layers the profile's snapshot once its slots are free.
162    Reseed {
163        /// The profile.
164        profile: String,
165        /// The answer.
166        reply: oneshot::Sender<Result<(), ControlError>>,
167    },
168    /// Remove a slot's volume or a shared cache's, unless a runner uses it.
169    CleanCache {
170        /// The volume.
171        name: String,
172        /// The answer.
173        reply: oneshot::Sender<Result<(), ControlError>>,
174    },
175    /// Write a new, validated config to its file and run with it, with no tick in between.
176    ReplaceConfig {
177        /// The config.
178        config: Box<Config>,
179        /// The `MemoryMax` of the slice it names.
180        memory_max: Option<u64>,
181        /// The CPUs of the slice it names, in millicores, read when it sets `cpu_overcommit`.
182        cpus: Option<u64>,
183        /// Writes the file.
184        commit: Commit,
185        /// Whether it may leave runners in slots it does not have.
186        running: RunningSlots,
187        /// Who writes it.
188        author: Author,
189        /// Answered once the engine runs with it: with the commit's warning and what it undoes of auto sizing's
190        /// raises, or why it did not commit.
191        reply: oneshot::Sender<Result<Vec<String>, ControlError>>,
192    },
193    /// Remove records of failed jobs.
194    RemoveRecords {
195        /// Which.
196        selection: RecordSelection,
197        /// The answer.
198        reply: oneshot::Sender<Result<api::RemovedRecords, ControlError>>,
199    },
200    /// Start a shell in a job's record, if the budget has room for it.
201    OpenShell {
202        /// The job.
203        job: u64,
204        /// Its terminal's size.
205        size: TtySize,
206        /// The shell's terminal.
207        reply: oneshot::Sender<Result<Tty, ControlError>>,
208    },
209    /// Remove a shell's container: its terminal closed.
210    CloseShell {
211        /// The container's ID.
212        container: String,
213    },
214}
215
216/// What makes a new config durable, run by the engine before it runs with the config: `Ok` once the config is in
217/// place, with a warning if it may not survive a crash; `Err` with nothing changed.
218pub struct Commit(Box<dyn FnOnce() -> Result<Option<String>, String> + Send>);
219
220impl Commit {
221    /// A commit that runs `write`, off the engine's thread.
222    pub fn new(write: impl FnOnce() -> Result<Option<String>, String> + Send + 'static) -> Commit {
223        Commit(Box::new(write))
224    }
225
226    /// Runs it.
227    pub async fn run(self) -> Result<Option<String>, String> {
228        tokio::task::spawn_blocking(self.0)
229            .await
230            .unwrap_or_else(|join_error| Err(join_error.to_string()))
231    }
232}
233
234impl std::fmt::Debug for Commit {
235    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
236        f.write_str("Commit")
237    }
238}
239
240/// The engine's end of `Control`'s commands.
241#[derive(Debug)]
242pub struct Commands(mpsc::UnboundedReceiver<Command>);
243
244impl Commands {
245    /// The next command; `None` once `Control` is gone.
246    pub async fn recv(&mut self) -> Option<Command> {
247        self.0.recv().await
248    }
249}
250
251/// Commands from the API and signals to the loop.
252#[derive(Debug)]
253pub struct Control {
254    // Locked with `sync::lock`: the guarded data stays consistent, as every critical section is a single insert,
255    // remove or read.
256    repositories: Mutex<BTreeSet<String>>,
257    paused: Mutex<BTreeSet<String>>,
258    /// What the config lets the job actions do, which the engine sets at start and at each reload before it answers,
259    /// so the job actions check the config of the moment they are sent, as one snapshot (ADR 0134).
260    actions: Policies,
261    /// Where `paused` is recorded.
262    store: Store,
263    shutdown: watch::Sender<Stop>,
264    pub(super) wake: Notify,
265    commands: mpsc::UnboundedSender<Command>,
266}
267
268impl Control {
269    /// Controls for the configured repositories, of which those in `paused` start paused, and the commands they
270    /// send the engine. A paused repository the config no longer has is forgotten, in `store` too: one added back
271    /// starts active.
272    pub fn new(
273        repositories: impl IntoIterator<Item = String>,
274        paused: impl IntoIterator<Item = String>,
275        store: Store,
276    ) -> (Control, Commands) {
277        let repositories: BTreeSet<String> = repositories.into_iter().collect();
278        let paused: BTreeSet<String> = paused
279            .into_iter()
280            .filter(|repository| repositories.contains(repository))
281            .collect();
282        // Sent at once; a write that fails leaves rows the next start filters out the same way.
283        drop(store.paused(paused.iter().cloned().collect()));
284        let (commands, receiver) = mpsc::unbounded_channel();
285        let control = Control {
286            actions: Policies::new(api::JobActions::On, repositories.clone()),
287            repositories: Mutex::new(repositories),
288            paused: Mutex::new(paused),
289            store,
290            shutdown: watch::Sender::new(Stop::none()),
291            wake: Notify::new(),
292            commands,
293        };
294        (control, Commands(receiver))
295    }
296
297    /// Whether the daemon is stopping and how: as requested, or as a detach once Docker stopped answering the stop.
298    pub fn mode(&self) -> Shutdown {
299        self.shutdown.borrow().mode()
300    }
301
302    /// Whether Docker stopped answering the stop, which is then left to the next start.
303    pub(super) fn docker_gone(&self) -> bool {
304        self.shutdown.borrow().docker_gone()
305    }
306
307    /// Notes that Docker did not answer a call the stop needed, and returns whether that is news: not without a stop
308    /// under way, nor once noted. Its deadline starts, and a later `runnerd stop` answers that it leaves running jobs.
309    pub(super) fn note_docker_gone(&self) -> bool {
310        let news = self.shutdown.send_if_modified(Stop::note_docker_gone);
311        if news {
312            self.wake.notify_one();
313        }
314        news
315    }
316
317    /// Resolves once the stop has a deadline: it detaches or is forced.
318    pub(super) async fn hurried(&self) {
319        let mut mode = self.shutdown.subscribe();
320        // `self` holds the sender, so the wait ends only once the mode is reached.
321        let _ = mode.wait_for(|stop| stop.hurried()).await;
322    }
323
324    /// Pauses or resumes one repository or all, whatever they run, and records it in the store before it answers,
325    /// so a restarted daemon starts with the same ones paused. `NotFound` for a repository the config does not have.
326    /// If the record fails, the change holds until a restart and the answer is `NotRecorded`. The status shows it
327    /// at the engine's next publish: the API pauses and resumes through `pause` and `resume`, which publish before
328    /// they answer.
329    pub async fn set_paused(&self, target: &Target, paused: bool) -> Result<(), ControlError> {
330        let recorded = {
331            let repositories = lock(&self.repositories);
332            let names: Vec<&String> = match target {
333                Target::Repository(repository) => {
334                    let Some(name) = repositories.get(repository) else {
335                        return Err(not_configured(repository, repositories.iter()));
336                    };
337                    vec![name]
338                }
339                Target::All => repositories.iter().collect(),
340            };
341            let mut set = lock(&self.paused);
342            for name in names {
343                if paused {
344                    set.insert(name.clone());
345                } else {
346                    set.remove(name);
347                }
348            }
349            // Sent under the lock, so the records change in the order the set does.
350            self.store.paused(set.iter().cloned().collect())
351        };
352        self.wake.notify_one();
353        if recorded.await {
354            return Ok(());
355        }
356        let name = match target {
357            Target::Repository(repository) => repository.as_str(),
358            Target::All => "every repository",
359        };
360        let (done, undone) = if paused {
361            ("paused", "resumes it")
362        } else {
363            ("resumed", "pauses it again")
364        };
365        let message = format!(
366            "{name} is {done} until runnerd restarts, but the job history cannot record it, so a restart {undone}"
367        );
368        warn!("{message}");
369        Err(ControlError::NotRecorded(message))
370    }
371
372    /// Replaces the configured repositories; the removed ones are no longer paused, in the store too.
373    pub(crate) fn set_repositories(&self, repositories: BTreeSet<String>) {
374        let mut configured = lock(&self.repositories);
375        let mut paused = lock(&self.paused);
376        let before = paused.len();
377        paused.retain(|repository| repositories.contains(repository));
378        if paused.len() != before {
379            // Not waited for: a restart forgets a repository the config does not have anyway.
380            drop(self.store.paused(paused.iter().cloned().collect()));
381        }
382        *configured = repositories;
383    }
384
385    /// Asks the loop to stop; a more urgent request overrides a less urgent one, never the other way. Returns how it
386    /// stops now, which is what status says.
387    pub fn shutdown(&self, mode: Shutdown) -> Shutdown {
388        self.shutdown.send_if_modified(|stop| stop.ask(mode));
389        self.wake.notify_one();
390        self.mode()
391    }
392
393    /// Whether the repository is paused.
394    pub fn is_paused(&self, repository: &str) -> bool {
395        lock(&self.paused).contains(repository)
396    }
397
398    /// What the config of this moment lets the owner do with jobs (ADR 0134): one snapshot.
399    pub fn action_policy(&self) -> Arc<Policy> {
400        self.actions.now()
401    }
402
403    /// Replaces that snapshot whole, from `[github] job_actions` and the configured repositories: the engine does at
404    /// start and at each reload.
405    pub fn set_action_policy(&self, setting: api::JobActions, repositories: BTreeSet<String>) {
406        self.actions.replace(setting, repositories);
407    }
408
409    /// Resolves once a stop of any mode is asked for: what is sent to GitHub on the owner's behalf ends then.
410    pub async fn stop_asked(&self) {
411        let mut mode = self.shutdown.subscribe();
412        // `self` holds the sender, so the wait ends only once a stop is asked for.
413        let _ = mode.wait_for(|stop| stop.mode() != Shutdown::No).await;
414    }
415
416    async fn send<T>(
417        &self,
418        command: impl FnOnce(oneshot::Sender<T>) -> Command,
419    ) -> Result<T, ControlError> {
420        let (reply, answer) = oneshot::channel();
421        self.commands
422            .send(command(reply))
423            .map_err(|_| ControlError::Stopped)?;
424        answer.await.map_err(|_| ControlError::Stopped)
425    }
426
427    /// Pauses one repository or all. With `RunningJobs::Keep`, refuses while any of them runs a job, pausing none;
428    /// with `Kill`, cancels the runs of their running jobs on GitHub, whose runners are removed once they report it
429    /// or `CANCEL_GRACE` later, and removes their other runners at once.
430    pub async fn pause(
431        &self,
432        target: Target,
433        jobs: RunningJobs,
434    ) -> Result<api::Paused, ControlError> {
435        self.send(|reply| Command::Pause {
436            target,
437            jobs,
438            reply,
439        })
440        .await?
441    }
442
443    /// Resumes one repository or all, and publishes the status before it answers, so a client that reads the status
444    /// or the events after the answer sees them resumed. Fails as `set_paused` does.
445    pub async fn resume(&self, target: Target) -> Result<(), ControlError> {
446        self.send(|reply| Command::Resume { target, reply }).await?
447    }
448
449    /// Reseeds a profile as soon as its slots are free; none of them starts a runner until then.
450    pub async fn reseed(&self, profile: &str) -> Result<(), ControlError> {
451        self.send(|reply| Command::Reseed {
452            profile: profile.to_owned(),
453            reply,
454        })
455        .await?
456    }
457
458    /// Removes a slot's volume or a shared cache's, unless a runner uses it.
459    pub async fn clean_cache(&self, name: &str) -> Result<(), ControlError> {
460        self.send(|reply| Command::CleanCache {
461            name: name.to_owned(),
462            reply,
463        })
464        .await?
465    }
466
467    /// Removes the selected records of failed jobs and their images.
468    pub async fn remove_records(
469        &self,
470        selection: RecordSelection,
471    ) -> Result<api::RemovedRecords, ControlError> {
472        self.send(|reply| Command::RemoveRecords { selection, reply })
473            .await?
474    }
475
476    /// Starts a shell in a job's record and returns its terminal. The caller closes it with `close_shell` once the
477    /// terminal ends, from either side.
478    pub async fn open_shell(&self, job: u64, size: TtySize) -> Result<Tty, ControlError> {
479        self.send(|reply| Command::OpenShell { job, size, reply })
480            .await?
481    }
482
483    /// Removes a shell's container. Not waited for, so it can be sent while the terminal is dropped.
484    pub fn close_shell(&self, container: String) {
485        let _ = self.commands.send(Command::CloseShell { container });
486    }
487
488    /// Runs `commit` to write a validated config, and if it commits, runs with the config from then on, before the
489    /// next tick. If it does not, nothing changes and the answer is `Failed`. Running jobs keep running: under
490    /// `RunningSlots::Drain` removed profiles and slots go as their runners finish; under `Keep` a config without a
491    /// slot a runner is in is refused (`Conflict`) before `commit` runs. A config with another `host.slice` is refused
492    /// (`Invalid`) before `commit` runs: the slice takes effect when runnerd starts. The answer warns of what may not
493    /// survive a crash and, for `Author::Owner`, of each profile whose `memory` it lowers under a raise of auto
494    /// sizing's in the last day (`auto::undone_raises`), which are logged too.
495    pub async fn replace_config(
496        &self,
497        config: Config,
498        memory_max: Option<u64>,
499        cpus: Option<u64>,
500        commit: Commit,
501        running: RunningSlots,
502        author: Author,
503    ) -> Result<Vec<String>, ControlError> {
504        self.send(|reply| Command::ReplaceConfig {
505            config: Box::new(config),
506            memory_max,
507            cpus,
508            commit,
509            running,
510            author,
511            reply,
512        })
513        .await?
514    }
515}
516
517/// Names the repositories there are, so a mistyped name shows its fix.
518fn not_configured<'name>(
519    repository: &str,
520    configured: impl Iterator<Item = &'name String>,
521) -> ControlError {
522    let configured: Vec<&str> = configured.map(String::as_str).collect();
523    let configured = if configured.is_empty() {
524        "none is configured".to_owned()
525    } else {
526        format!("configured: {}", configured.join(", "))
527    };
528    ControlError::NotFound(format!(
529        "{repository} is not a configured repository; {configured}"
530    ))
531}
532
533/// `jobs` for people to read: `job 1 on runner-a, runner-b`.
534fn list(jobs: &[api::RunningJob]) -> String {
535    jobs.iter()
536        .map(ToString::to_string)
537        .collect::<Vec<_>>()
538        .join(", ")
539}
540
541impl<G: GithubApi + 'static, D: DockerApi + 'static> Engine<G, D> {
542    /// Carries out a command from the API.
543    pub(super) async fn command(&mut self, command: Command) {
544        match command {
545            Command::Pause {
546                target,
547                jobs,
548                reply,
549            } => {
550                let _ = reply.send(self.pause(&target, jobs).await);
551            }
552            Command::Resume { target, reply } => {
553                let _ = reply.send(self.control.set_paused(&target, false).await);
554            }
555            Command::Reseed { profile, reply } => {
556                let _ = reply.send(self.request_reseed(&profile));
557            }
558            Command::CleanCache { name, reply } => {
559                let _ = reply.send(self.clean_cache(&name).await);
560            }
561            Command::ReplaceConfig {
562                config,
563                memory_max,
564                cpus,
565                commit,
566                running,
567                author,
568                reply,
569            } => {
570                if let Err(refused) = same_slice(&self.config, &config) {
571                    let _ = reply.send(Err(refused));
572                    return;
573                }
574                if running == RunningSlots::Keep
575                    && let Some(occupied) = self.occupied_past(&config)
576                {
577                    let _ = reply.send(Err(ControlError::Conflict(occupied)));
578                    return;
579                }
580                let undone = match author {
581                    Author::Owner => self.undone_raises(&config).await,
582                    Author::AutoSizing => Vec::new(),
583                };
584                let answer = match commit.run().await {
585                    Ok(warning) => {
586                        for line in &undone {
587                            warn!("config replaced: {line}");
588                        }
589                        self.reload(*config, memory_max, cpus);
590                        Ok(warning.into_iter().chain(undone).collect())
591                    }
592                    Err(message) => Err(ControlError::Failed(message)),
593                };
594                let _ = reply.send(answer);
595            }
596            Command::RemoveRecords { selection, reply } => {
597                let _ = reply.send(self.remove_records(&selection).await);
598            }
599            Command::OpenShell { job, size, reply } => {
600                let opened = self.open_shell(job, size).await;
601                // A terminal nobody waits for any more is closed at once.
602                if let Err(Ok(tty)) = reply.send(opened) {
603                    self.close_shell(&tty.container).await;
604                }
605            }
606            Command::CloseShell { container } => self.close_shell(&container).await,
607        }
608    }
609
610    /// Pauses one repository or all. A runner of theirs that holds a job, or that GitHub reports busy, refuses it
611    /// with `RunningJobs::Keep`; with `Kill`, the runs of their jobs are cancelled, and their runners removed once
612    /// they exit or `CANCEL_GRACE` later; a runner whose run cannot be cancelled is removed at once.
613    pub(super) async fn pause(
614        &mut self,
615        target: &Target,
616        running: RunningJobs,
617    ) -> Result<api::Paused, ControlError> {
618        let repositories: BTreeSet<String> = match target {
619            Target::Repository(repository) if self.repositories.listed(repository).is_none() => {
620                return Err(not_configured(repository, self.repositories.listed_names()));
621            }
622            Target::Repository(repository) => BTreeSet::from([repository.clone()]),
623            Target::All => self.repositories.listed_names().cloned().collect(),
624        };
625        self.take_found();
626        let runners: Vec<(SlotRef, Occupant)> = self
627            .occupied()
628            .filter(|(_, occupant)| repositories.contains(&occupant.repository))
629            .map(|(slot, occupant)| (slot, occupant.clone()))
630            .collect();
631        let jobs: Vec<api::RunningJob> = runners
632            .iter()
633            .filter(|(_, occupant)| occupant.engaged())
634            .map(|(_, occupant)| occupant.running_job())
635            .collect();
636        if running == RunningJobs::Keep && !jobs.is_empty() {
637            let mut by_repository: BTreeMap<&str, Vec<api::RunningJob>> = BTreeMap::new();
638            for job in &jobs {
639                by_repository
640                    .entry(&job.repository)
641                    .or_default()
642                    .push(job.clone());
643            }
644            let runs: Vec<String> = by_repository
645                .iter()
646                .map(|(repository, jobs)| format!("{repository} runs {}", list(jobs)))
647                .collect();
648            // The jobs only: each client says how to force the pause in its own terms.
649            return Err(ControlError::Busy {
650                message: runs.join("; "),
651                jobs,
652            });
653        }
654        // Paused in memory even if not recorded: a forced pause still cancels the runs it was asked to.
655        let recorded = self.control.set_paused(target, true).await;
656        match target {
657            Target::Repository(repository) => info!("{repository}: paused"),
658            Target::All => info!("paused every repository"),
659        }
660        let mut stopped = Vec::new();
661        if running == RunningJobs::Keep {
662            return recorded.map(|()| api::Paused { stopped });
663        }
664        for (slot, occupant) in runners {
665            let repository = &occupant.repository;
666            let cancelled = if let Some(job) = occupant.named_job() {
667                self.cancel(repository, job).await
668            } else {
669                if occupant.listed_busy() {
670                    warn!(
671                        "{repository}: {} runs a job its log has not named yet; GitHub fails it once the runner is gone",
672                        occupant.runner_name
673                    );
674                }
675                false
676            };
677            if cancelled {
678                // `collect_finished` removes it once it exits, `kill_cancelled` if it does not.
679                if let Some(cancelled) = self
680                    .slots
681                    .values_mut()
682                    .flatten()
683                    .filter_map(Slot::occupant_mut)
684                    .find(|candidate| candidate.runner_name == occupant.runner_name)
685                {
686                    cancelled
687                        .engagement
688                        .cancelled(tokio::time::Instant::now() + CANCEL_GRACE);
689                }
690                // A restart keeps it.
691                self.store.runner_cancel_by(
692                    &occupant.runner_name,
693                    time::now_ms() + u64::try_from(CANCEL_GRACE.as_millis()).unwrap_or(u64::MAX),
694                );
695            } else {
696                self.kill(&slot).await;
697                if self
698                    .occupied()
699                    .any(|(_, candidate)| candidate.runner_name == occupant.runner_name)
700                {
701                    continue;
702                }
703            }
704            stopped.push(occupant.running_job());
705        }
706        match recorded {
707            Ok(()) => Ok(api::Paused { stopped }),
708            Err(ControlError::NotRecorded(message)) if !stopped.is_empty() => Err(
709                ControlError::NotRecorded(format!("{message}; stopped {}", list(&stopped))),
710            ),
711            Err(error) => Err(error),
712        }
713    }
714
715    /// Removes the runners of cancelled runs that did not exit within `CANCEL_GRACE`, every tick until they are gone.
716    pub(super) async fn kill_cancelled(&mut self) {
717        let now = tokio::time::Instant::now();
718        let overdue: Vec<(SlotRef, String)> = self
719            .occupied()
720            .filter(|(_, occupant)| occupant.engagement.cancel_overdue(now))
721            .map(|(slot, occupant)| (slot, occupant.runner_name.clone()))
722            .collect();
723        for (slot, name) in overdue {
724            warn!(
725                "{name} did not exit {}s after its run was cancelled",
726                CANCEL_GRACE.as_secs()
727            );
728            // The deadline stays until the runner is gone, so a failed removal is tried again next tick.
729            self.kill(&slot).await;
730        }
731    }
732
733    /// Cancels the run of a job; whether GitHub accepted it.
734    async fn cancel(&self, repository: &str, job: u64) -> bool {
735        let run = match self.github.job(repository, job).await {
736            Ok(details) => details.run_id,
737            Err(error) => {
738                warn!("{repository}: cannot read job {job} to cancel its run: {error}");
739                return false;
740            }
741        };
742        match self.github.cancel_run(repository, run).await {
743            Ok(()) => {
744                info!("{repository}: cancelled run {run} of job {job}");
745                true
746            }
747            Err(error) => {
748                warn!("{repository}: cannot cancel run {run}: {error}");
749                false
750            }
751        }
752    }
753
754    /// Asks for a profile's reseed: none of its slots starts a runner until they are free and it ran.
755    pub(super) fn request_reseed(&mut self, profile: &str) -> Result<(), ControlError> {
756        if !self.config.profiles.contains_key(profile) {
757            return Err(ControlError::NotFound(format!(
758                "{profile} is not a configured profile"
759            )));
760        }
761        if self.is_reseeding(profile) {
762            return Err(ControlError::Conflict(format!(
763                "{profile} is reseeding already"
764            )));
765        }
766        // A shell mounts the snapshot a reseed replaces, and may stay open for hours.
767        if self.shell_profiles().contains(profile) {
768            return Err(ControlError::Conflict(format!(
769                "a shell is open in a record of {profile}, which mounts its snapshot; reseed it once the shell exits"
770            )));
771        }
772        info!("{profile}: reseeding once its slots are free");
773        self.upkeep.request(profile);
774        Ok(())
775    }
776
777    /// Removes a slot's volume or a shared cache's, unless a job or a reseed uses it; the runners that use it and
778    /// wait for a job stop first. Snapshots are rebuilt by reseeding instead: slot layers sit on top of them.
779    pub(super) async fn clean_cache(&mut self, name: &str) -> Result<(), ControlError> {
780        let Some(volume) = cache::volumes(&self.names, &self.config)
781            .into_iter()
782            .find(|volume| volume.name == name)
783        else {
784            return Err(ControlError::NotFound(format!(
785                "{name} is not a cache volume of this config; `runnerctl caches` lists them"
786            )));
787        };
788        let using: Vec<SlotRef> = match &volume.kind {
789            api::CacheKind::Snapshot { profile } => {
790                return Err(ControlError::Invalid(format!(
791                    "{name} is {profile}'s snapshot, which every slot's layers sit on; reseed {profile} to rebuild it"
792                )));
793            }
794            api::CacheKind::Slot { profile, slot } => {
795                if self.upkeep.reseed_runs(profile) {
796                    return Err(ControlError::Conflict(format!(
797                        "{profile} is reseeding, which writes {name}"
798                    )));
799                }
800                self.occupied()
801                    .filter(|(at, _)| at.profile == *profile && at.index + 1 == *slot)
802                    .map(|(at, _)| at)
803                    .collect()
804            }
805            api::CacheKind::Pnpm
806            | api::CacheKind::Maven
807            | api::CacheKind::Mirror
808            | api::CacheKind::Build => self.occupied().map(|(at, _)| at).collect(),
809        };
810        let shared = matches!(volume.kind, api::CacheKind::Pnpm | api::CacheKind::Maven);
811        if shared && self.records.has_shells() {
812            return Err(ControlError::Conflict(format!(
813                "a shell open in a record mounts {name}; clean it once the shell exits"
814            )));
815        }
816        // Runners waiting for a job, warm ones too, give way: the next plan starts them again. One GitHub gave a
817        // job meanwhile stays, and holds the clean back.
818        let idle = using
819            .iter()
820            .filter(|slot| self.occupant(slot).is_some_and(Occupant::idle))
821            .map(|slot| (slot.clone(), StopReason::Clean))
822            .collect();
823        self.remove_idle(idle).await;
824        let users: Vec<api::RunningJob> = using
825            .iter()
826            .filter_map(|slot| self.occupant(slot))
827            .map(Occupant::running_job)
828            .collect();
829        if !users.is_empty() {
830            return Err(ControlError::Busy {
831                message: format!(
832                    "{name} is in use by {}; clean it once they finish",
833                    list(&users)
834                ),
835                jobs: users,
836            });
837        }
838        if volume.kind == api::CacheKind::Build {
839            let Some(cache) = self.build_cache.clone() else {
840                return Err(ControlError::Failed(
841                    "the build cache did not open when runnerd started; its log says why"
842                        .to_owned(),
843                ));
844            };
845            let trimmed = tokio::task::spawn_blocking(move || cache.clear())
846                .await
847                .map_err(|error| ControlError::Failed(format!("cannot empty {name}: {error}")))?;
848            info!(
849                "emptied {name}: {} entries, {}",
850                trimmed.before.entries,
851                ByteSize(trimmed.before.bytes)
852            );
853            self.trim_due = true;
854            return Ok(());
855        }
856        if volume.kind == api::CacheKind::Mirror && self.mirror.may_run() {
857            cache::mirror_down(&*self.docker, &self.names)
858                .await
859                .map_err(|error| {
860                    ControlError::Failed(format!("cannot stop the mirror: {error}"))
861                })?;
862            self.mirror.stopped();
863        }
864        self.docker
865            .remove_volume(name)
866            .await
867            .map_err(|error| ControlError::Failed(format!("cannot remove {name}: {error}")))?;
868        info!("removed {name}");
869        // Its layer no longer holds what a job cached.
870        if let api::CacheKind::Slot { profile, slot } = &volume.kind {
871            self.ran.remove(&SlotRef {
872                profile: profile.clone(),
873                index: slot.saturating_sub(1),
874            });
875        }
876        // Measured again once idle.
877        self.trim_due = true;
878        Ok(())
879    }
880
881    /// What `config` undoes of auto sizing's `memory` raises in the last day; none when they cannot be read, which is
882    /// logged: the config runs either way.
883    async fn undone_raises(&self, config: &Config) -> Vec<String> {
884        let now = time::now_ms() / 1000;
885        let since = now.saturating_sub(auto::UNDONE_RAISES).saturating_mul(1000);
886        match self.store.sizing_changes(since).await {
887            Ok(changes) => auto::undone_raises(&self.config, config, &changes, now),
888            Err(store_error) => {
889                warn!(
890                    "cannot read auto sizing's changes to compare a new config with: {store_error}"
891                );
892                Vec::new()
893            }
894        }
895    }
896
897    /// The first slot with a runner in it that `config` does not have, for people to read.
898    fn occupied_past(&self, config: &Config) -> Option<String> {
899        self.slots.iter().find_map(|(name, slots)| {
900            let kept = config
901                .profiles
902                .get(name)
903                .map_or(0, |profile| profile.slots as usize);
904            slots
905                .iter()
906                .enumerate()
907                .skip(kept)
908                .find(|(_, slot)| !slot.is_free())
909                .map(|(index, _)| {
910                    format!(
911                        "{name}'s slot {} has a runner, and the new config keeps {kept} slots",
912                        index + 1
913                    )
914                })
915        })
916    }
917
918    /// Runs with a new, validated config. Running jobs keep running: a removed profile, or slots past a lowered
919    /// number, stay until their runners finish, and no runner starts in a profile while it has as many as its
920    /// slots. Runners started from now on get the new settings.
921    /// `cpus` are the slice's, read when the config sets `cpu_overcommit`.
922    pub(super) fn reload(&mut self, config: Config, memory_max: Option<u64>, cpus: Option<u64>) {
923        let mut old = std::mem::replace(&mut self.config, config);
924        // A pool gone keeps no hold a pool made there later would inherit, and one whose runners change starts again
925        // at once: what failed may be what changed.
926        for (name, state) in self.repositories.iter_listed_mut() {
927            let new = &self.config;
928            state.forget_health(|profile| same_runners(&old, new, name, profile));
929        }
930        if let Some(cache) = &self.build_cache {
931            cache.set_max_size(self.config.caches.build.max_size.0);
932        }
933        for (name, profile) in &self.config.profiles {
934            let slots = self.slots.entry(name.clone()).or_default();
935            if slots.len() < profile.slots as usize {
936                slots.resize_with(profile.slots as usize, Slot::default);
937            }
938            self.retired.remove(name);
939        }
940        for (name, profile) in std::mem::take(&mut old.profiles) {
941            if !self.config.profiles.contains_key(&name) {
942                self.retired.insert(name, profile);
943            }
944        }
945        self.drop_closed_slots();
946        self.images.stale_due = true;
947        let names: BTreeSet<String> = self
948            .config
949            .repositories
950            .iter()
951            .map(|repository| repository.name.clone())
952            .collect();
953        self.repositories.reload(&names);
954        // A pool gone, or moved to another profile, keeps no wait a pool made there later would inherit.
955        let config = &self.config;
956        for (name, state) in self.repositories.iter_listed_mut() {
957            state.forget_waits(|profile| config.pool_in(name, profile).is_some());
958        }
959        self.control.set_action_policy(
960            api::JobActions::configured(self.config.github.job_actions),
961            names.clone(),
962        );
963        self.control.set_repositories(names);
964        let profiles = &self.config.profiles;
965        self.upkeep
966            .forget_removed(|name| profiles.contains_key(name));
967        self.prepare_failures.retain(|slot, _| {
968            profiles
969                .get(&slot.profile)
970                .is_some_and(|profile| slot.index < profile.slots as usize)
971        });
972        self.isolation = isolation::backend(self.config.host.isolation);
973        // The growth count's looks were taken by other rules, or none were: its brake begins again, as at a restart
974        // (ADR 0130). Its guard reads `MemoryMax` and the host's ceiling.
975        let pooled = |config: &Config| {
976            config.host.admission == crate::config::Admission::Measured && config.host.pooled_count
977        };
978        if pooled(&self.config)
979            && (!pooled(&old)
980                || memory_max != self.memory_max
981                || old.host.memory_ceiling != self.config.host.memory_ceiling)
982        {
983            self.measured.restart_growth(self.now());
984        }
985        self.memory_max = memory_max;
986        self.cpus = cpus;
987        self.caches
988            .send_replace(cache::volumes(&self.names, &self.config));
989        // Labels moved to another pool take their history with them: what each pool wants is learned again.
990        self.reread_warm();
991        info!("reloaded the config");
992    }
993}
994
995/// Whether `repository`'s runners in `profile` start the same under `new` as under `old`: the pool's labels, and its
996/// profile's image, memory and CPUs and the host's isolation. A pool either config lacks does not.
997fn same_runners(old: &Config, new: &Config, repository: &str, profile: &str) -> bool {
998    let labels = |pool: &crate::config::Repository| -> BTreeSet<String> {
999        pool.labels
1000            .iter()
1001            .map(|label| label.to_ascii_lowercase())
1002            .collect()
1003    };
1004    let (Some(was), Some(is)) = (
1005        old.pool_in(repository, profile),
1006        new.pool_in(repository, profile),
1007    ) else {
1008        return false;
1009    };
1010    let (Some(was_made), Some(is_made)) = (old.profiles.get(profile), new.profiles.get(profile))
1011    else {
1012        return false;
1013    };
1014    labels(was) == labels(is)
1015        && was_made.image == is_made.image
1016        && was_made.memory.0 == is_made.memory.0
1017        && was_made.cpus == is_made.cpus
1018        && old.host.isolation == new.host.isolation
1019}