Skip to main content

runnerd_core/scheduler/engine/
control.rs

1//! Control from the API: the `Control` handle the daemon's API holds (pause flags, which it records in the store so
2//! a restart keeps them, shutdown, a channel of `Command`s) and the engine's side of each command: pause and resume,
3//! of one repository or all, with or without cancelling their runs, reseed, cache clean, config replacement, the
4//! removal of records and the shells opened in them.
5//! Commands run between ticks or while a tick waits for GitHub, so they see the slot state a tick does (ADRs 0019,
6//! 0026). It must not serve HTTP or write the config file: the daemon does, through `Commit`.
7
8use std::collections::{BTreeMap, BTreeSet};
9use std::sync::Mutex;
10use std::time::Duration;
11
12use tokio::sync::{Notify, mpsc, oneshot, watch};
13use tracing::{info, warn};
14
15use super::records::RecordSelection;
16use super::{Engine, Occupant};
17use crate::api;
18use crate::cache;
19use crate::config::{ByteSize, Config};
20use crate::docker::{DockerApi, Tty, TtySize};
21use crate::github::GithubApi;
22use crate::isolation;
23use crate::scheduler::{SlotRef, StopReason};
24use crate::sizing::auto;
25use crate::store::Store;
26use crate::sync::lock;
27use crate::time;
28
29/// How long the runner of a run a forced pause cancelled gets to report its job cancelled and exit, before it is
30/// removed: GitHub completes a cancelled run only once its runner reports it, which took 40s in a test.
31pub(super) const CANCEL_GRACE: Duration = Duration::from_secs(90);
32
33/// How the daemon is asked to stop, from the least to the most urgent: a later request only ever makes the stop more
34/// urgent (ADR 0091).
35#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
36pub enum Shutdown {
37    /// Keep scheduling.
38    No,
39    /// Start nothing new, let running jobs finish, then remove every runner: `runnerd stop`.
40    Graceful,
41    /// Remove idle runners, leave busy ones and the mirror to the next start, which adopts them, and stop within
42    /// `STOP_DEADLINE`: a signal, as systemd sends on a stop.
43    Detach,
44    /// Kill running jobs and delete their runners, within `STOP_DEADLINE`: `runnerd stop --force`, a second signal.
45    Force,
46}
47
48impl Shutdown {
49    /// How a daemon stopping in this mode says it stops; `None` for `No`.
50    pub fn stopping(self) -> Option<api::Stopping> {
51        match self {
52            Shutdown::No => None,
53            Shutdown::Graceful => Some(api::Stopping::WaitsForJobs),
54            Shutdown::Detach => Some(api::Stopping::LeavesJobs),
55            Shutdown::Force => Some(api::Stopping::KillsJobs),
56        }
57    }
58}
59
60/// What a pause, a shutdown or a runner's removal does with the jobs runners are running.
61#[derive(Debug, Clone, Copy, PartialEq, Eq)]
62pub enum RunningJobs {
63    /// Leave them running: a pause refuses, a busy runner stays.
64    Keep,
65    /// Kill them: a pause cancels their runs, a removal kills their runners.
66    Kill,
67}
68
69/// What a new config may do to slots with a runner in them.
70#[derive(Debug, Clone, Copy, PartialEq, Eq)]
71pub enum RunningSlots {
72    /// A slot past a lowered `slots`, or of a removed profile, stays until its runner finishes.
73    Drain,
74    /// The config is refused while a runner is in a slot it does not have: what auto sizing asks for.
75    Keep,
76}
77
78/// Who writes a new config.
79#[derive(Debug, Clone, Copy, PartialEq, Eq)]
80pub enum Author {
81    /// The owner, through `PUT /v1/config`: a save that lowers `memory` under a raise of auto sizing's in the last
82    /// day is answered with a warning (`auto::undone_raises`).
83    Owner,
84    /// Auto sizing, whose own lowering is never a warning about its raise.
85    AutoSizing,
86}
87
88/// `Err` when `new` moves runnerd to another slice than `running` names: the sampler and the running containers stay
89/// in the slice runnerd started in, so the slice changes only at a restart.
90pub fn same_slice(running: &Config, new: &Config) -> Result<(), ControlError> {
91    if new.host.slice == running.host.slice {
92        return Ok(());
93    }
94    Err(ControlError::Invalid(format!(
95        "restart runnerd to change the slice: it runs in {}, and host.slice is {} in the new config",
96        running.host.slice, new.host.slice
97    )))
98}
99
100/// The repositories a pause or a resume applies to.
101#[derive(Debug, Clone, PartialEq, Eq)]
102pub enum Target {
103    /// One repository, `owner/name`.
104    Repository(String),
105    /// Every configured repository.
106    All,
107}
108
109/// Why the engine refused or failed a command from the API.
110#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
111pub enum ControlError {
112    /// No such repository, profile or cache.
113    #[error("{0}")]
114    NotFound(String),
115    /// Runners use what the command would change.
116    #[error("{message}")]
117    Busy {
118        /// Names the runners and their jobs.
119        message: String,
120        /// The runners in the way.
121        jobs: Vec<api::RunningJob>,
122    },
123    /// The command conflicts with one already in progress.
124    #[error("{0}")]
125    Conflict(String),
126    /// The command cannot apply to this target.
127    #[error("{0}")]
128    Invalid(String),
129    /// Docker or GitHub failed.
130    #[error("{0}")]
131    Failed(String),
132    /// A pause or resume holds in memory, but the job history could not record it: a restart undoes it.
133    #[error("{0}")]
134    NotRecorded(String),
135    /// The engine stopped.
136    #[error("runnerd is stopping")]
137    Stopped,
138}
139
140/// A command the engine carries out between ticks or while a tick waits for GitHub, with where its answer goes.
141#[derive(Debug)]
142pub enum Command {
143    /// Pause one repository or all; with `RunningJobs::Kill`, cancel their running jobs.
144    Pause {
145        /// What to pause.
146        target: Target,
147        /// Cancel running jobs, or refuse while there are any.
148        jobs: RunningJobs,
149        /// The answer.
150        reply: oneshot::Sender<Result<api::Paused, ControlError>>,
151    },
152    /// Resume one repository or all.
153    Resume {
154        /// What to resume.
155        target: Target,
156        /// The answer.
157        reply: oneshot::Sender<Result<(), ControlError>>,
158    },
159    /// Make slot 1's layers the profile's snapshot once its slots are free.
160    Reseed {
161        /// The profile.
162        profile: String,
163        /// The answer.
164        reply: oneshot::Sender<Result<(), ControlError>>,
165    },
166    /// Remove a slot's volume or a shared cache's, unless a runner uses it.
167    CleanCache {
168        /// The volume.
169        name: String,
170        /// The answer.
171        reply: oneshot::Sender<Result<(), ControlError>>,
172    },
173    /// Write a new, validated config to its file and run with it, with no tick in between.
174    ReplaceConfig {
175        /// The config.
176        config: Box<Config>,
177        /// The `MemoryMax` of the slice it names.
178        memory_max: Option<u64>,
179        /// The CPUs of the slice it names, in millicores, read when it sets `cpu_overcommit`.
180        cpus: Option<u64>,
181        /// Writes the file.
182        commit: Commit,
183        /// Whether it may leave runners in slots it does not have.
184        running: RunningSlots,
185        /// Who writes it.
186        author: Author,
187        /// Answered once the engine runs with it: with the commit's warning and what it undoes of auto sizing's
188        /// raises, or why it did not commit.
189        reply: oneshot::Sender<Result<Vec<String>, ControlError>>,
190    },
191    /// Remove records of failed jobs.
192    RemoveRecords {
193        /// Which.
194        selection: RecordSelection,
195        /// The answer.
196        reply: oneshot::Sender<Result<api::RemovedRecords, ControlError>>,
197    },
198    /// Start a shell in a job's record, if the budget has room for it.
199    OpenShell {
200        /// The job.
201        job: u64,
202        /// Its terminal's size.
203        size: TtySize,
204        /// The shell's terminal.
205        reply: oneshot::Sender<Result<Tty, ControlError>>,
206    },
207    /// Remove a shell's container: its terminal closed.
208    CloseShell {
209        /// The container's ID.
210        container: String,
211    },
212}
213
214/// What makes a new config durable, run by the engine before it runs with the config: `Ok` once the config is in
215/// place, with a warning if it may not survive a crash; `Err` with nothing changed.
216pub struct Commit(Box<dyn FnOnce() -> Result<Option<String>, String> + Send>);
217
218impl Commit {
219    /// A commit that runs `write`, off the engine's thread.
220    pub fn new(write: impl FnOnce() -> Result<Option<String>, String> + Send + 'static) -> Commit {
221        Commit(Box::new(write))
222    }
223
224    /// Runs it.
225    pub async fn run(self) -> Result<Option<String>, String> {
226        tokio::task::spawn_blocking(self.0)
227            .await
228            .unwrap_or_else(|join_error| Err(join_error.to_string()))
229    }
230}
231
232impl std::fmt::Debug for Commit {
233    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
234        f.write_str("Commit")
235    }
236}
237
238/// The engine's end of `Control`'s commands.
239#[derive(Debug)]
240pub struct Commands(mpsc::UnboundedReceiver<Command>);
241
242impl Commands {
243    /// The next command; `None` once `Control` is gone.
244    pub async fn recv(&mut self) -> Option<Command> {
245        self.0.recv().await
246    }
247}
248
249/// Commands from the API and signals to the loop.
250#[derive(Debug)]
251pub struct Control {
252    // Locked with `sync::lock`: the guarded data stays consistent, as every critical section is a single insert,
253    // remove or read.
254    repositories: Mutex<BTreeSet<String>>,
255    paused: Mutex<BTreeSet<String>>,
256    /// Where `paused` is recorded.
257    store: Store,
258    shutdown: watch::Sender<Shutdown>,
259    pub(super) wake: Notify,
260    commands: mpsc::UnboundedSender<Command>,
261}
262
263impl Control {
264    /// Controls for the configured repositories, of which those in `paused` start paused, and the commands they
265    /// send the engine. A paused repository the config no longer has is forgotten, in `store` too: one added back
266    /// starts active.
267    pub fn new(
268        repositories: impl IntoIterator<Item = String>,
269        paused: impl IntoIterator<Item = String>,
270        store: Store,
271    ) -> (Control, Commands) {
272        let repositories: BTreeSet<String> = repositories.into_iter().collect();
273        let paused: BTreeSet<String> = paused
274            .into_iter()
275            .filter(|repository| repositories.contains(repository))
276            .collect();
277        // Sent at once; a write that fails leaves rows the next start filters out the same way.
278        drop(store.paused(paused.iter().cloned().collect()));
279        let (commands, receiver) = mpsc::unbounded_channel();
280        let control = Control {
281            repositories: Mutex::new(repositories),
282            paused: Mutex::new(paused),
283            store,
284            shutdown: watch::Sender::new(Shutdown::No),
285            wake: Notify::new(),
286            commands,
287        };
288        (control, Commands(receiver))
289    }
290
291    /// Whether a shutdown was requested, and how.
292    pub fn mode(&self) -> Shutdown {
293        *self.shutdown.borrow()
294    }
295
296    /// Resolves once the stop has a deadline: it detaches or is forced.
297    pub(super) async fn hurried(&self) {
298        let mut mode = self.shutdown.subscribe();
299        // `self` holds the sender, so the wait ends only once the mode is reached.
300        let _ = mode.wait_for(|mode| *mode >= Shutdown::Detach).await;
301    }
302
303    /// Pauses or resumes one repository or all, whatever they run, and records it in the store before it answers,
304    /// so a restarted daemon starts with the same ones paused. `NotFound` for a repository the config does not have.
305    /// If the record fails, the change holds until a restart and the answer is `NotRecorded`. The status shows it
306    /// at the engine's next publish: the API pauses and resumes through `pause` and `resume`, which publish before
307    /// they answer.
308    pub async fn set_paused(&self, target: &Target, paused: bool) -> Result<(), ControlError> {
309        let recorded = {
310            let repositories = lock(&self.repositories);
311            let names: Vec<&String> = match target {
312                Target::Repository(repository) => {
313                    let Some(name) = repositories.get(repository) else {
314                        return Err(not_configured(repository, repositories.iter()));
315                    };
316                    vec![name]
317                }
318                Target::All => repositories.iter().collect(),
319            };
320            let mut set = lock(&self.paused);
321            for name in names {
322                if paused {
323                    set.insert(name.clone());
324                } else {
325                    set.remove(name);
326                }
327            }
328            // Sent under the lock, so the records change in the order the set does.
329            self.store.paused(set.iter().cloned().collect())
330        };
331        self.wake.notify_one();
332        if recorded.await {
333            return Ok(());
334        }
335        let name = match target {
336            Target::Repository(repository) => repository.as_str(),
337            Target::All => "every repository",
338        };
339        let (done, undone) = if paused {
340            ("paused", "resumes it")
341        } else {
342            ("resumed", "pauses it again")
343        };
344        let message = format!(
345            "{name} is {done} until runnerd restarts, but the job history cannot record it, so a restart {undone}"
346        );
347        warn!("{message}");
348        Err(ControlError::NotRecorded(message))
349    }
350
351    /// Replaces the configured repositories; the removed ones are no longer paused, in the store too.
352    fn set_repositories(&self, repositories: BTreeSet<String>) {
353        let mut configured = lock(&self.repositories);
354        let mut paused = lock(&self.paused);
355        let before = paused.len();
356        paused.retain(|repository| repositories.contains(repository));
357        if paused.len() != before {
358            // Not waited for: a restart forgets a repository the config does not have anyway.
359            drop(self.store.paused(paused.iter().cloned().collect()));
360        }
361        *configured = repositories;
362    }
363
364    /// Asks the loop to stop; a more urgent request overrides a less urgent one, never the other way. Returns how it
365    /// stops now.
366    pub fn shutdown(&self, mode: Shutdown) -> Shutdown {
367        self.shutdown.send_if_modified(|shutdown| {
368            let changed = mode > *shutdown;
369            *shutdown = (*shutdown).max(mode);
370            changed
371        });
372        self.wake.notify_one();
373        self.mode()
374    }
375
376    /// Whether the repository is paused.
377    pub fn is_paused(&self, repository: &str) -> bool {
378        lock(&self.paused).contains(repository)
379    }
380
381    async fn send<T>(
382        &self,
383        command: impl FnOnce(oneshot::Sender<T>) -> Command,
384    ) -> Result<T, ControlError> {
385        let (reply, answer) = oneshot::channel();
386        self.commands
387            .send(command(reply))
388            .map_err(|_| ControlError::Stopped)?;
389        answer.await.map_err(|_| ControlError::Stopped)
390    }
391
392    /// Pauses one repository or all. With `RunningJobs::Keep`, refuses while any of them runs a job, pausing none;
393    /// with `Kill`, cancels the runs of their running jobs on GitHub, whose runners are removed once they report it
394    /// or `CANCEL_GRACE` later, and removes their other runners at once.
395    pub async fn pause(
396        &self,
397        target: Target,
398        jobs: RunningJobs,
399    ) -> Result<api::Paused, ControlError> {
400        self.send(|reply| Command::Pause {
401            target,
402            jobs,
403            reply,
404        })
405        .await?
406    }
407
408    /// Resumes one repository or all, and publishes the status before it answers, so a client that reads the status
409    /// or the events after the answer sees them resumed. Fails as `set_paused` does.
410    pub async fn resume(&self, target: Target) -> Result<(), ControlError> {
411        self.send(|reply| Command::Resume { target, reply }).await?
412    }
413
414    /// Reseeds a profile as soon as its slots are free; none of them starts a runner until then.
415    pub async fn reseed(&self, profile: &str) -> Result<(), ControlError> {
416        self.send(|reply| Command::Reseed {
417            profile: profile.to_owned(),
418            reply,
419        })
420        .await?
421    }
422
423    /// Removes a slot's volume or a shared cache's, unless a runner uses it.
424    pub async fn clean_cache(&self, name: &str) -> Result<(), ControlError> {
425        self.send(|reply| Command::CleanCache {
426            name: name.to_owned(),
427            reply,
428        })
429        .await?
430    }
431
432    /// Removes the selected records of failed jobs and their images.
433    pub async fn remove_records(
434        &self,
435        selection: RecordSelection,
436    ) -> Result<api::RemovedRecords, ControlError> {
437        self.send(|reply| Command::RemoveRecords { selection, reply })
438            .await?
439    }
440
441    /// Starts a shell in a job's record and returns its terminal. The caller closes it with `close_shell` once the
442    /// terminal ends, from either side.
443    pub async fn open_shell(&self, job: u64, size: TtySize) -> Result<Tty, ControlError> {
444        self.send(|reply| Command::OpenShell { job, size, reply })
445            .await?
446    }
447
448    /// Removes a shell's container. Not waited for, so it can be sent while the terminal is dropped.
449    pub fn close_shell(&self, container: String) {
450        let _ = self.commands.send(Command::CloseShell { container });
451    }
452
453    /// Runs `commit` to write a validated config, and if it commits, runs with the config from then on, before the
454    /// next tick. If it does not, nothing changes and the answer is `Failed`. Running jobs keep running: under
455    /// `RunningSlots::Drain` removed profiles and slots go as their runners finish; under `Keep` a config without a
456    /// slot a runner is in is refused (`Conflict`) before `commit` runs. A config with another `host.slice` is refused
457    /// (`Invalid`) before `commit` runs: the slice takes effect when runnerd starts. The answer warns of what may not
458    /// survive a crash and, for `Author::Owner`, of each profile whose `memory` it lowers under a raise of auto
459    /// sizing's in the last day (`auto::undone_raises`), which are logged too.
460    pub async fn replace_config(
461        &self,
462        config: Config,
463        memory_max: Option<u64>,
464        cpus: Option<u64>,
465        commit: Commit,
466        running: RunningSlots,
467        author: Author,
468    ) -> Result<Vec<String>, ControlError> {
469        self.send(|reply| Command::ReplaceConfig {
470            config: Box::new(config),
471            memory_max,
472            cpus,
473            commit,
474            running,
475            author,
476            reply,
477        })
478        .await?
479    }
480}
481
482/// Names the repositories there are, so a mistyped name shows its fix.
483fn not_configured<'name>(
484    repository: &str,
485    configured: impl Iterator<Item = &'name String>,
486) -> ControlError {
487    let configured: Vec<&str> = configured.map(String::as_str).collect();
488    let configured = if configured.is_empty() {
489        "none is configured".to_owned()
490    } else {
491        format!("configured: {}", configured.join(", "))
492    };
493    ControlError::NotFound(format!(
494        "{repository} is not a configured repository; {configured}"
495    ))
496}
497
498/// `jobs` for people to read: `job 1 on runner-a, runner-b`.
499fn list(jobs: &[api::RunningJob]) -> String {
500    jobs.iter()
501        .map(ToString::to_string)
502        .collect::<Vec<_>>()
503        .join(", ")
504}
505
506impl<G: GithubApi + 'static, D: DockerApi + 'static> Engine<G, D> {
507    /// Carries out a command from the API.
508    pub(super) async fn command(&mut self, command: Command) {
509        match command {
510            Command::Pause {
511                target,
512                jobs,
513                reply,
514            } => {
515                let _ = reply.send(self.pause(&target, jobs).await);
516            }
517            Command::Resume { target, reply } => {
518                let _ = reply.send(self.control.set_paused(&target, false).await);
519            }
520            Command::Reseed { profile, reply } => {
521                let _ = reply.send(self.request_reseed(&profile));
522            }
523            Command::CleanCache { name, reply } => {
524                let _ = reply.send(self.clean_cache(&name).await);
525            }
526            Command::ReplaceConfig {
527                config,
528                memory_max,
529                cpus,
530                commit,
531                running,
532                author,
533                reply,
534            } => {
535                if let Err(refused) = same_slice(&self.config, &config) {
536                    let _ = reply.send(Err(refused));
537                    return;
538                }
539                if running == RunningSlots::Keep
540                    && let Some(occupied) = self.occupied_past(&config)
541                {
542                    let _ = reply.send(Err(ControlError::Conflict(occupied)));
543                    return;
544                }
545                let undone = match author {
546                    Author::Owner => self.undone_raises(&config).await,
547                    Author::AutoSizing => Vec::new(),
548                };
549                let answer = match commit.run().await {
550                    Ok(warning) => {
551                        for line in &undone {
552                            warn!("config replaced: {line}");
553                        }
554                        self.reload(*config, memory_max, cpus);
555                        Ok(warning.into_iter().chain(undone).collect())
556                    }
557                    Err(message) => Err(ControlError::Failed(message)),
558                };
559                let _ = reply.send(answer);
560            }
561            Command::RemoveRecords { selection, reply } => {
562                let _ = reply.send(self.remove_records(&selection).await);
563            }
564            Command::OpenShell { job, size, reply } => {
565                let opened = self.open_shell(job, size).await;
566                // A terminal nobody waits for any more is closed at once.
567                if let Err(Ok(tty)) = reply.send(opened) {
568                    self.close_shell(&tty.container).await;
569                }
570            }
571            Command::CloseShell { container } => self.close_shell(&container).await,
572        }
573    }
574
575    /// Pauses one repository or all. A runner of theirs that holds a job, or that GitHub reports busy, refuses it
576    /// with `RunningJobs::Keep`; with `Kill`, the runs of their jobs are cancelled, and their runners removed once
577    /// they exit or `CANCEL_GRACE` later; a runner whose run cannot be cancelled is removed at once.
578    pub(super) async fn pause(
579        &mut self,
580        target: &Target,
581        running: RunningJobs,
582    ) -> Result<api::Paused, ControlError> {
583        let repositories: BTreeSet<String> = match target {
584            Target::Repository(repository) if !self.repositories.contains_key(repository) => {
585                return Err(not_configured(repository, self.repositories.keys()));
586            }
587            Target::Repository(repository) => BTreeSet::from([repository.clone()]),
588            Target::All => self.repositories.keys().cloned().collect(),
589        };
590        self.take_found();
591        let runners: Vec<(SlotRef, Occupant)> = self
592            .occupied()
593            .filter(|(_, occupant)| repositories.contains(&occupant.repository))
594            .map(|(slot, occupant)| (slot, occupant.clone()))
595            .collect();
596        let jobs: Vec<api::RunningJob> = runners
597            .iter()
598            .filter(|(_, occupant)| occupant.busy || occupant.job.is_some())
599            .map(|(_, occupant)| occupant.running_job())
600            .collect();
601        if running == RunningJobs::Keep && !jobs.is_empty() {
602            let mut by_repository: BTreeMap<&str, Vec<api::RunningJob>> = BTreeMap::new();
603            for job in &jobs {
604                by_repository
605                    .entry(&job.repository)
606                    .or_default()
607                    .push(job.clone());
608            }
609            let runs: Vec<String> = by_repository
610                .iter()
611                .map(|(repository, jobs)| format!("{repository} runs {}", list(jobs)))
612                .collect();
613            // The jobs only: each client says how to force the pause in its own terms.
614            return Err(ControlError::Busy {
615                message: runs.join("; "),
616                jobs,
617            });
618        }
619        // Paused in memory even if not recorded: a forced pause still cancels the runs it was asked to.
620        let recorded = self.control.set_paused(target, true).await;
621        match target {
622            Target::Repository(repository) => info!("{repository}: paused"),
623            Target::All => info!("paused every repository"),
624        }
625        let mut stopped = Vec::new();
626        if running == RunningJobs::Keep {
627            return recorded.map(|()| api::Paused { stopped });
628        }
629        for (slot, occupant) in runners {
630            let repository = &occupant.repository;
631            let cancelled = if let Some(job) = occupant.job {
632                self.cancel(repository, job).await
633            } else {
634                if occupant.busy {
635                    warn!(
636                        "{repository}: {} runs a job its log has not named yet; GitHub fails it once the runner is gone",
637                        occupant.runner_name
638                    );
639                }
640                false
641            };
642            if cancelled {
643                // `collect_finished` removes it once it exits, `kill_cancelled` if it does not.
644                self.cancelling.insert(
645                    occupant.runner_name.clone(),
646                    tokio::time::Instant::now() + CANCEL_GRACE,
647                );
648            } else {
649                self.kill(&slot).await;
650                if self
651                    .occupied()
652                    .any(|(_, candidate)| candidate.runner_name == occupant.runner_name)
653                {
654                    continue;
655                }
656            }
657            stopped.push(occupant.running_job());
658        }
659        match recorded {
660            Ok(()) => Ok(api::Paused { stopped }),
661            Err(ControlError::NotRecorded(message)) if !stopped.is_empty() => Err(
662                ControlError::NotRecorded(format!("{message}; stopped {}", list(&stopped))),
663            ),
664            Err(error) => Err(error),
665        }
666    }
667
668    /// Removes the runners of cancelled runs that did not exit within `CANCEL_GRACE`, every tick until they are gone.
669    pub(super) async fn kill_cancelled(&mut self) {
670        let now = tokio::time::Instant::now();
671        let running: BTreeSet<String> = self
672            .occupied()
673            .map(|(_, occupant)| occupant.runner_name.clone())
674            .collect();
675        self.cancelling.retain(|name, _| running.contains(name));
676        let overdue: Vec<(SlotRef, String)> = self
677            .occupied()
678            .filter(|(_, occupant)| {
679                self.cancelling
680                    .get(&occupant.runner_name)
681                    .is_some_and(|deadline| *deadline <= now)
682            })
683            .map(|(slot, occupant)| (slot, occupant.runner_name.clone()))
684            .collect();
685        for (slot, name) in overdue {
686            warn!(
687                "{name} did not exit {}s after its run was cancelled",
688                CANCEL_GRACE.as_secs()
689            );
690            // The deadline stays until the runner is gone, so a failed removal is tried again next tick.
691            self.kill(&slot).await;
692        }
693    }
694
695    /// Cancels the run of a job; whether GitHub accepted it.
696    async fn cancel(&self, repository: &str, job: u64) -> bool {
697        let run = match self.github.job(repository, job).await {
698            Ok(details) => details.run_id,
699            Err(error) => {
700                warn!("{repository}: cannot read job {job} to cancel its run: {error}");
701                return false;
702            }
703        };
704        match self.github.cancel_run(repository, run).await {
705            Ok(()) => {
706                info!("{repository}: cancelled run {run} of job {job}");
707                true
708            }
709            Err(error) => {
710                warn!("{repository}: cannot cancel run {run}: {error}");
711                false
712            }
713        }
714    }
715
716    /// Asks for a profile's reseed: none of its slots starts a runner until they are free and it ran.
717    pub(super) fn request_reseed(&mut self, profile: &str) -> Result<(), ControlError> {
718        if !self.config.profiles.contains_key(profile) {
719            return Err(ControlError::NotFound(format!(
720                "{profile} is not a configured profile"
721            )));
722        }
723        if self.is_reseeding(profile) {
724            return Err(ControlError::Conflict(format!(
725                "{profile} is reseeding already"
726            )));
727        }
728        // A shell mounts the snapshot a reseed replaces, and may stay open for hours.
729        if self.shell_profiles().contains(profile) {
730            return Err(ControlError::Conflict(format!(
731                "a shell is open in a record of {profile}, which mounts its snapshot; reseed it once the shell exits"
732            )));
733        }
734        info!("{profile}: reseeding once its slots are free");
735        self.reseed_requested.insert(profile.to_owned());
736        Ok(())
737    }
738
739    /// Removes a slot's volume or a shared cache's, unless a job or a reseed uses it; the runners that use it and
740    /// wait for a job stop first. Snapshots are rebuilt by reseeding instead: slot layers sit on top of them.
741    pub(super) async fn clean_cache(&mut self, name: &str) -> Result<(), ControlError> {
742        let Some(volume) = cache::volumes(&self.names, &self.config)
743            .into_iter()
744            .find(|volume| volume.name == name)
745        else {
746            return Err(ControlError::NotFound(format!(
747                "{name} is not a cache volume of this config; `runnerctl caches` lists them"
748            )));
749        };
750        let using: Vec<SlotRef> = match &volume.kind {
751            api::CacheKind::Snapshot { profile } => {
752                return Err(ControlError::Invalid(format!(
753                    "{name} is {profile}'s snapshot, which every slot's layers sit on; reseed {profile} to rebuild it"
754                )));
755            }
756            api::CacheKind::Slot { profile, slot } => {
757                if self.reseeds.contains_key(profile) {
758                    return Err(ControlError::Conflict(format!(
759                        "{profile} is reseeding, which writes {name}"
760                    )));
761                }
762                self.occupied()
763                    .filter(|(at, _)| at.profile == *profile && at.index + 1 == *slot)
764                    .map(|(at, _)| at)
765                    .collect()
766            }
767            api::CacheKind::Pnpm
768            | api::CacheKind::Maven
769            | api::CacheKind::Mirror
770            | api::CacheKind::Build => self.occupied().map(|(at, _)| at).collect(),
771        };
772        let shared = matches!(volume.kind, api::CacheKind::Pnpm | api::CacheKind::Maven);
773        if shared && !self.records.shells.is_empty() {
774            return Err(ControlError::Conflict(format!(
775                "a shell open in a record mounts {name}; clean it once the shell exits"
776            )));
777        }
778        // Runners waiting for a job, warm ones too, give way: the next plan starts them again. One GitHub gave a
779        // job meanwhile stays, and holds the clean back.
780        let idle = using
781            .iter()
782            .filter(|slot| {
783                self.occupant(slot)
784                    .is_some_and(|occupant| !occupant.busy && occupant.job.is_none())
785            })
786            .map(|slot| (slot.clone(), StopReason::Clean))
787            .collect();
788        self.remove_idle(idle).await;
789        let users: Vec<api::RunningJob> = using
790            .iter()
791            .filter_map(|slot| self.occupant(slot))
792            .map(Occupant::running_job)
793            .collect();
794        if !users.is_empty() {
795            return Err(ControlError::Busy {
796                message: format!(
797                    "{name} is in use by {}; clean it once they finish",
798                    list(&users)
799                ),
800                jobs: users,
801            });
802        }
803        if volume.kind == api::CacheKind::Build {
804            let Some(cache) = self.build_cache.clone() else {
805                return Err(ControlError::Failed(
806                    "the build cache did not open when runnerd started; its log says why"
807                        .to_owned(),
808                ));
809            };
810            let trimmed = tokio::task::spawn_blocking(move || cache.clear())
811                .await
812                .map_err(|error| ControlError::Failed(format!("cannot empty {name}: {error}")))?;
813            info!(
814                "emptied {name}: {} entries, {}",
815                trimmed.before.entries,
816                ByteSize(trimmed.before.bytes)
817            );
818            self.trim_due = true;
819            return Ok(());
820        }
821        if volume.kind == api::CacheKind::Mirror && self.mirror_up {
822            cache::mirror_down(&*self.docker, &self.names)
823                .await
824                .map_err(|error| {
825                    ControlError::Failed(format!("cannot stop the mirror: {error}"))
826                })?;
827            self.mirror_up = false;
828        }
829        self.docker
830            .remove_volume(name)
831            .await
832            .map_err(|error| ControlError::Failed(format!("cannot remove {name}: {error}")))?;
833        info!("removed {name}");
834        // Its layer no longer holds what a job cached.
835        if let api::CacheKind::Slot { profile, slot } = &volume.kind {
836            self.ran.remove(&SlotRef {
837                profile: profile.clone(),
838                index: slot.saturating_sub(1),
839            });
840        }
841        // Measured again once idle.
842        self.trim_due = true;
843        Ok(())
844    }
845
846    /// What `config` undoes of auto sizing's `memory` raises in the last day; none when they cannot be read, which is
847    /// logged: the config runs either way.
848    async fn undone_raises(&self, config: &Config) -> Vec<String> {
849        let now = time::now_ms() / 1000;
850        let since = now.saturating_sub(auto::UNDONE_RAISES).saturating_mul(1000);
851        match self.store.sizing_changes(since).await {
852            Ok(changes) => auto::undone_raises(&self.config, config, &changes, now),
853            Err(store_error) => {
854                warn!(
855                    "cannot read auto sizing's changes to compare a new config with: {store_error}"
856                );
857                Vec::new()
858            }
859        }
860    }
861
862    /// The first slot with a runner in it that `config` does not have, for people to read.
863    fn occupied_past(&self, config: &Config) -> Option<String> {
864        self.slots.iter().find_map(|(name, slots)| {
865            let kept = config
866                .profiles
867                .get(name)
868                .map_or(0, |profile| profile.slots as usize);
869            slots
870                .iter()
871                .enumerate()
872                .skip(kept)
873                .find(|(_, slot)| slot.is_some())
874                .map(|(index, _)| {
875                    format!(
876                        "{name}'s slot {} has a runner, and the new config keeps {kept} slots",
877                        index + 1
878                    )
879                })
880        })
881    }
882
883    /// Runs with a new, validated config. Running jobs keep running: a removed profile, or slots past a lowered
884    /// number, stay until their runners finish, and no runner starts in a profile while it has as many as its
885    /// slots. Runners started from now on get the new settings.
886    /// `cpus` are the slice's, read when the config sets `cpu_overcommit`.
887    pub(super) fn reload(&mut self, config: Config, memory_max: Option<u64>, cpus: Option<u64>) {
888        let mut old = std::mem::replace(&mut self.config, config);
889        if let Some(cache) = &self.build_cache {
890            cache.set_max_size(self.config.caches.build.max_size.0);
891        }
892        for (name, profile) in &self.config.profiles {
893            let slots = self.slots.entry(name.clone()).or_default();
894            if slots.len() < profile.slots as usize {
895                slots.resize(profile.slots as usize, None);
896            }
897            self.retired.remove(name);
898        }
899        for (name, profile) in std::mem::take(&mut old.profiles) {
900            if !self.config.profiles.contains_key(&name) {
901                self.retired.insert(name, profile);
902            }
903        }
904        self.drop_closed_slots();
905        self.images.stale_due = true;
906        let names: BTreeSet<String> = self
907            .config
908            .repositories
909            .iter()
910            .map(|repository| repository.name.clone())
911            .collect();
912        let removed: Vec<String> = self
913            .repositories
914            .keys()
915            .filter(|name| !names.contains(*name))
916            .cloned()
917            .collect();
918        self.draining.extend(removed);
919        self.draining.retain(|name| !names.contains(name));
920        self.repositories.retain(|name, _| names.contains(name));
921        for name in &names {
922            self.repositories.entry(name.clone()).or_default();
923        }
924        // A pool gone, or moved to another profile, keeps no wait a pool made there later would inherit.
925        let config = &self.config;
926        for (name, state) in &mut self.repositories {
927            state
928                .waits
929                .retain(|profile, _| config.pool_in(name, profile).is_some());
930            state
931                .cpu_held
932                .retain(|profile| config.pool_in(name, profile).is_some());
933        }
934        self.control.set_repositories(names);
935        let profiles = &self.config.profiles;
936        self.reseed_requested
937            .retain(|name| profiles.contains_key(name));
938        self.oversized.retain(|name| profiles.contains_key(name));
939        self.measure_due.retain(|name| profiles.contains_key(name));
940        self.reseed_due.retain(|name| profiles.contains_key(name));
941        self.isolation = isolation::backend(self.config.host.isolation);
942        self.memory_max = memory_max;
943        self.cpus = cpus;
944        self.caches
945            .send_replace(cache::volumes(&self.names, &self.config));
946        info!("reloaded the config");
947    }
948}