1use std::collections::{BTreeMap, BTreeSet};
9use std::sync::Mutex;
10use std::time::Duration;
11
12use tokio::sync::{Notify, mpsc, oneshot};
13use tracing::{info, warn};
14
15use super::records::RecordSelection;
16use super::{Engine, Occupant};
17use crate::api;
18use crate::cache;
19use crate::config::{ByteSize, Config};
20use crate::docker::{DockerApi, Tty, TtySize};
21use crate::github::GithubApi;
22use crate::isolation;
23use crate::scheduler::SlotRef;
24use crate::sizing::auto;
25use crate::store::Store;
26use crate::sync::lock;
27use crate::time;
28
29pub(super) const CANCEL_GRACE: Duration = Duration::from_secs(90);
32
33#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
35pub enum Shutdown {
36 No,
38 Graceful,
40 Force,
42}
43
44#[derive(Debug, Clone, Copy, PartialEq, Eq)]
46pub enum RunningJobs {
47 Keep,
49 Kill,
51}
52
53#[derive(Debug, Clone, Copy, PartialEq, Eq)]
55pub enum RunningSlots {
56 Drain,
58 Keep,
60}
61
62#[derive(Debug, Clone, Copy, PartialEq, Eq)]
64pub enum Author {
65 Owner,
68 AutoSizing,
70}
71
72pub fn same_slice(running: &Config, new: &Config) -> Result<(), ControlError> {
75 if new.host.slice == running.host.slice {
76 return Ok(());
77 }
78 Err(ControlError::Invalid(format!(
79 "restart runnerd to change the slice: it runs in {}, and host.slice is {} in the new config",
80 running.host.slice, new.host.slice
81 )))
82}
83
84#[derive(Debug, Clone, PartialEq, Eq)]
86pub enum Target {
87 Repository(String),
89 All,
91}
92
93#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
95pub enum ControlError {
96 #[error("{0}")]
98 NotFound(String),
99 #[error("{message}")]
101 Busy {
102 message: String,
104 jobs: Vec<api::RunningJob>,
106 },
107 #[error("{0}")]
109 Conflict(String),
110 #[error("{0}")]
112 Invalid(String),
113 #[error("{0}")]
115 Failed(String),
116 #[error("{0}")]
118 NotRecorded(String),
119 #[error("runnerd is stopping")]
121 Stopped,
122}
123
124#[derive(Debug)]
126pub enum Command {
127 Pause {
129 target: Target,
131 jobs: RunningJobs,
133 reply: oneshot::Sender<Result<api::Paused, ControlError>>,
135 },
136 Resume {
138 target: Target,
140 reply: oneshot::Sender<Result<(), ControlError>>,
142 },
143 Reseed {
145 profile: String,
147 reply: oneshot::Sender<Result<(), ControlError>>,
149 },
150 CleanCache {
152 name: String,
154 reply: oneshot::Sender<Result<(), ControlError>>,
156 },
157 ReplaceConfig {
159 config: Box<Config>,
161 memory_max: Option<u64>,
163 cpus: Option<u64>,
165 commit: Commit,
167 running: RunningSlots,
169 author: Author,
171 reply: oneshot::Sender<Result<Vec<String>, ControlError>>,
174 },
175 RemoveRecords {
177 selection: RecordSelection,
179 reply: oneshot::Sender<Result<api::RemovedRecords, ControlError>>,
181 },
182 OpenShell {
184 job: u64,
186 size: TtySize,
188 reply: oneshot::Sender<Result<Tty, ControlError>>,
190 },
191 CloseShell {
193 container: String,
195 },
196}
197
198pub struct Commit(Box<dyn FnOnce() -> Result<Option<String>, String> + Send>);
201
202impl Commit {
203 pub fn new(write: impl FnOnce() -> Result<Option<String>, String> + Send + 'static) -> Commit {
205 Commit(Box::new(write))
206 }
207
208 pub async fn run(self) -> Result<Option<String>, String> {
210 tokio::task::spawn_blocking(self.0)
211 .await
212 .unwrap_or_else(|join_error| Err(join_error.to_string()))
213 }
214}
215
216impl std::fmt::Debug for Commit {
217 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
218 f.write_str("Commit")
219 }
220}
221
222#[derive(Debug)]
224pub struct Commands(mpsc::UnboundedReceiver<Command>);
225
226impl Commands {
227 pub async fn recv(&mut self) -> Option<Command> {
229 self.0.recv().await
230 }
231}
232
233#[derive(Debug)]
235pub struct Control {
236 repositories: Mutex<BTreeSet<String>>,
239 paused: Mutex<BTreeSet<String>>,
240 store: Store,
242 shutdown: Mutex<Shutdown>,
243 pub(super) wake: Notify,
244 commands: mpsc::UnboundedSender<Command>,
245}
246
247impl Control {
248 pub fn new(
252 repositories: impl IntoIterator<Item = String>,
253 paused: impl IntoIterator<Item = String>,
254 store: Store,
255 ) -> (Control, Commands) {
256 let repositories: BTreeSet<String> = repositories.into_iter().collect();
257 let paused: BTreeSet<String> = paused
258 .into_iter()
259 .filter(|repository| repositories.contains(repository))
260 .collect();
261 drop(store.paused(paused.iter().cloned().collect()));
263 let (commands, receiver) = mpsc::unbounded_channel();
264 let control = Control {
265 repositories: Mutex::new(repositories),
266 paused: Mutex::new(paused),
267 store,
268 shutdown: Mutex::new(Shutdown::No),
269 wake: Notify::new(),
270 commands,
271 };
272 (control, Commands(receiver))
273 }
274
275 pub fn mode(&self) -> Shutdown {
277 *lock(&self.shutdown)
278 }
279
280 pub async fn set_paused(&self, target: &Target, paused: bool) -> Result<(), ControlError> {
286 let recorded = {
287 let repositories = lock(&self.repositories);
288 let names: Vec<&String> = match target {
289 Target::Repository(repository) => {
290 let Some(name) = repositories.get(repository) else {
291 return Err(not_configured(repository, repositories.iter()));
292 };
293 vec![name]
294 }
295 Target::All => repositories.iter().collect(),
296 };
297 let mut set = lock(&self.paused);
298 for name in names {
299 if paused {
300 set.insert(name.clone());
301 } else {
302 set.remove(name);
303 }
304 }
305 self.store.paused(set.iter().cloned().collect())
307 };
308 self.wake.notify_one();
309 if recorded.await {
310 return Ok(());
311 }
312 let name = match target {
313 Target::Repository(repository) => repository.as_str(),
314 Target::All => "every repository",
315 };
316 let (done, undone) = if paused {
317 ("paused", "resumes it")
318 } else {
319 ("resumed", "pauses it again")
320 };
321 let message = format!(
322 "{name} is {done} until runnerd restarts, but the job history cannot record it, so a restart {undone}"
323 );
324 warn!("{message}");
325 Err(ControlError::NotRecorded(message))
326 }
327
328 fn set_repositories(&self, repositories: BTreeSet<String>) {
330 let mut configured = lock(&self.repositories);
331 let mut paused = lock(&self.paused);
332 let before = paused.len();
333 paused.retain(|repository| repositories.contains(repository));
334 if paused.len() != before {
335 drop(self.store.paused(paused.iter().cloned().collect()));
337 }
338 *configured = repositories;
339 }
340
341 pub fn shutdown(&self, jobs: RunningJobs) {
343 let mode = match jobs {
344 RunningJobs::Keep => Shutdown::Graceful,
345 RunningJobs::Kill => Shutdown::Force,
346 };
347 let mut shutdown = lock(&self.shutdown);
348 *shutdown = (*shutdown).max(mode);
349 drop(shutdown);
350 self.wake.notify_one();
351 }
352
353 pub fn is_paused(&self, repository: &str) -> bool {
355 lock(&self.paused).contains(repository)
356 }
357
358 async fn send<T>(
359 &self,
360 command: impl FnOnce(oneshot::Sender<T>) -> Command,
361 ) -> Result<T, ControlError> {
362 let (reply, answer) = oneshot::channel();
363 self.commands
364 .send(command(reply))
365 .map_err(|_| ControlError::Stopped)?;
366 answer.await.map_err(|_| ControlError::Stopped)
367 }
368
369 pub async fn pause(
373 &self,
374 target: Target,
375 jobs: RunningJobs,
376 ) -> Result<api::Paused, ControlError> {
377 self.send(|reply| Command::Pause {
378 target,
379 jobs,
380 reply,
381 })
382 .await?
383 }
384
385 pub async fn resume(&self, target: Target) -> Result<(), ControlError> {
388 self.send(|reply| Command::Resume { target, reply }).await?
389 }
390
391 pub async fn reseed(&self, profile: &str) -> Result<(), ControlError> {
393 self.send(|reply| Command::Reseed {
394 profile: profile.to_owned(),
395 reply,
396 })
397 .await?
398 }
399
400 pub async fn clean_cache(&self, name: &str) -> Result<(), ControlError> {
402 self.send(|reply| Command::CleanCache {
403 name: name.to_owned(),
404 reply,
405 })
406 .await?
407 }
408
409 pub async fn remove_records(
411 &self,
412 selection: RecordSelection,
413 ) -> Result<api::RemovedRecords, ControlError> {
414 self.send(|reply| Command::RemoveRecords { selection, reply })
415 .await?
416 }
417
418 pub async fn open_shell(&self, job: u64, size: TtySize) -> Result<Tty, ControlError> {
421 self.send(|reply| Command::OpenShell { job, size, reply })
422 .await?
423 }
424
425 pub fn close_shell(&self, container: String) {
427 let _ = self.commands.send(Command::CloseShell { container });
428 }
429
430 pub async fn replace_config(
438 &self,
439 config: Config,
440 memory_max: Option<u64>,
441 cpus: Option<u64>,
442 commit: Commit,
443 running: RunningSlots,
444 author: Author,
445 ) -> Result<Vec<String>, ControlError> {
446 self.send(|reply| Command::ReplaceConfig {
447 config: Box::new(config),
448 memory_max,
449 cpus,
450 commit,
451 running,
452 author,
453 reply,
454 })
455 .await?
456 }
457}
458
459fn not_configured<'name>(
461 repository: &str,
462 configured: impl Iterator<Item = &'name String>,
463) -> ControlError {
464 let configured: Vec<&str> = configured.map(String::as_str).collect();
465 let configured = if configured.is_empty() {
466 "none is configured".to_owned()
467 } else {
468 format!("configured: {}", configured.join(", "))
469 };
470 ControlError::NotFound(format!(
471 "{repository} is not a configured repository; {configured}"
472 ))
473}
474
475fn list(jobs: &[api::RunningJob]) -> String {
477 jobs.iter()
478 .map(ToString::to_string)
479 .collect::<Vec<_>>()
480 .join(", ")
481}
482
483impl<G: GithubApi + 'static, D: DockerApi + 'static> Engine<G, D> {
484 pub(super) async fn command(&mut self, command: Command) {
486 match command {
487 Command::Pause {
488 target,
489 jobs,
490 reply,
491 } => {
492 let _ = reply.send(self.pause(&target, jobs).await);
493 }
494 Command::Resume { target, reply } => {
495 let _ = reply.send(self.control.set_paused(&target, false).await);
496 }
497 Command::Reseed { profile, reply } => {
498 let _ = reply.send(self.request_reseed(&profile));
499 }
500 Command::CleanCache { name, reply } => {
501 let _ = reply.send(self.clean_cache(&name).await);
502 }
503 Command::ReplaceConfig {
504 config,
505 memory_max,
506 cpus,
507 commit,
508 running,
509 author,
510 reply,
511 } => {
512 if let Err(refused) = same_slice(&self.config, &config) {
513 let _ = reply.send(Err(refused));
514 return;
515 }
516 if running == RunningSlots::Keep
517 && let Some(occupied) = self.occupied_past(&config)
518 {
519 let _ = reply.send(Err(ControlError::Conflict(occupied)));
520 return;
521 }
522 let undone = match author {
523 Author::Owner => self.undone_raises(&config).await,
524 Author::AutoSizing => Vec::new(),
525 };
526 let answer = match commit.run().await {
527 Ok(warning) => {
528 for line in &undone {
529 warn!("config replaced: {line}");
530 }
531 self.reload(*config, memory_max, cpus);
532 Ok(warning.into_iter().chain(undone).collect())
533 }
534 Err(message) => Err(ControlError::Failed(message)),
535 };
536 let _ = reply.send(answer);
537 }
538 Command::RemoveRecords { selection, reply } => {
539 let _ = reply.send(self.remove_records(&selection).await);
540 }
541 Command::OpenShell { job, size, reply } => {
542 let opened = self.open_shell(job, size).await;
543 if let Err(Ok(tty)) = reply.send(opened) {
545 self.close_shell(&tty.container).await;
546 }
547 }
548 Command::CloseShell { container } => self.close_shell(&container).await,
549 }
550 }
551
552 pub(super) async fn pause(
556 &mut self,
557 target: &Target,
558 running: RunningJobs,
559 ) -> Result<api::Paused, ControlError> {
560 let repositories: BTreeSet<String> = match target {
561 Target::Repository(repository) if !self.repositories.contains_key(repository) => {
562 return Err(not_configured(repository, self.repositories.keys()));
563 }
564 Target::Repository(repository) => BTreeSet::from([repository.clone()]),
565 Target::All => self.repositories.keys().cloned().collect(),
566 };
567 self.take_found();
568 let runners: Vec<(SlotRef, Occupant)> = self
569 .occupied()
570 .filter(|(_, occupant)| repositories.contains(&occupant.repository))
571 .map(|(slot, occupant)| (slot, occupant.clone()))
572 .collect();
573 let jobs: Vec<api::RunningJob> = runners
574 .iter()
575 .filter(|(_, occupant)| occupant.busy || occupant.job.is_some())
576 .map(|(_, occupant)| occupant.running_job())
577 .collect();
578 if running == RunningJobs::Keep && !jobs.is_empty() {
579 let mut by_repository: BTreeMap<&str, Vec<api::RunningJob>> = BTreeMap::new();
580 for job in &jobs {
581 by_repository
582 .entry(&job.repository)
583 .or_default()
584 .push(job.clone());
585 }
586 let runs: Vec<String> = by_repository
587 .iter()
588 .map(|(repository, jobs)| format!("{repository} runs {}", list(jobs)))
589 .collect();
590 return Err(ControlError::Busy {
592 message: runs.join("; "),
593 jobs,
594 });
595 }
596 let recorded = self.control.set_paused(target, true).await;
598 match target {
599 Target::Repository(repository) => info!("{repository}: paused"),
600 Target::All => info!("paused every repository"),
601 }
602 let mut stopped = Vec::new();
603 if running == RunningJobs::Keep {
604 return recorded.map(|()| api::Paused { stopped });
605 }
606 for (slot, occupant) in runners {
607 let repository = &occupant.repository;
608 let cancelled = if let Some(job) = occupant.job {
609 self.cancel(repository, job).await
610 } else {
611 if occupant.busy {
612 warn!(
613 "{repository}: {} runs a job its log has not named yet; GitHub fails it once the runner is gone",
614 occupant.runner_name
615 );
616 }
617 false
618 };
619 if cancelled {
620 self.cancelling.insert(
622 occupant.runner_name.clone(),
623 tokio::time::Instant::now() + CANCEL_GRACE,
624 );
625 } else {
626 self.stop(&slot, RunningJobs::Kill).await;
627 if self
628 .occupied()
629 .any(|(_, candidate)| candidate.runner_name == occupant.runner_name)
630 {
631 continue;
632 }
633 }
634 stopped.push(occupant.running_job());
635 }
636 match recorded {
637 Ok(()) => Ok(api::Paused { stopped }),
638 Err(ControlError::NotRecorded(message)) if !stopped.is_empty() => Err(
639 ControlError::NotRecorded(format!("{message}; stopped {}", list(&stopped))),
640 ),
641 Err(error) => Err(error),
642 }
643 }
644
645 pub(super) async fn kill_cancelled(&mut self) {
647 let now = tokio::time::Instant::now();
648 let running: BTreeSet<String> = self
649 .occupied()
650 .map(|(_, occupant)| occupant.runner_name.clone())
651 .collect();
652 self.cancelling.retain(|name, _| running.contains(name));
653 let overdue: Vec<(SlotRef, String)> = self
654 .occupied()
655 .filter(|(_, occupant)| {
656 self.cancelling
657 .get(&occupant.runner_name)
658 .is_some_and(|deadline| *deadline <= now)
659 })
660 .map(|(slot, occupant)| (slot, occupant.runner_name.clone()))
661 .collect();
662 for (slot, name) in overdue {
663 warn!(
664 "{name} did not exit {}s after its run was cancelled",
665 CANCEL_GRACE.as_secs()
666 );
667 self.stop(&slot, RunningJobs::Kill).await;
669 }
670 }
671
672 async fn cancel(&self, repository: &str, job: u64) -> bool {
674 let run = match self.github.job(repository, job).await {
675 Ok(details) => details.run_id,
676 Err(error) => {
677 warn!("{repository}: cannot read job {job} to cancel its run: {error}");
678 return false;
679 }
680 };
681 match self.github.cancel_run(repository, run).await {
682 Ok(()) => {
683 info!("{repository}: cancelled run {run} of job {job}");
684 true
685 }
686 Err(error) => {
687 warn!("{repository}: cannot cancel run {run}: {error}");
688 false
689 }
690 }
691 }
692
693 pub(super) fn request_reseed(&mut self, profile: &str) -> Result<(), ControlError> {
695 if !self.config.profiles.contains_key(profile) {
696 return Err(ControlError::NotFound(format!(
697 "{profile} is not a configured profile"
698 )));
699 }
700 if self.is_reseeding(profile) {
701 return Err(ControlError::Conflict(format!(
702 "{profile} is reseeding already"
703 )));
704 }
705 if self.shell_profiles().contains(profile) {
707 return Err(ControlError::Conflict(format!(
708 "a shell is open in a record of {profile}, which mounts its snapshot; reseed it once the shell exits"
709 )));
710 }
711 info!("{profile}: reseeding once its slots are free");
712 self.reseed_requested.insert(profile.to_owned());
713 Ok(())
714 }
715
716 pub(super) async fn clean_cache(&mut self, name: &str) -> Result<(), ControlError> {
719 let Some(volume) = cache::volumes(&self.names, &self.config)
720 .into_iter()
721 .find(|volume| volume.name == name)
722 else {
723 return Err(ControlError::NotFound(format!(
724 "{name} is not a cache volume of this config; `runnerctl caches` lists them"
725 )));
726 };
727 let using: Vec<SlotRef> = match &volume.kind {
728 api::CacheKind::Snapshot { profile } => {
729 return Err(ControlError::Invalid(format!(
730 "{name} is {profile}'s snapshot, which every slot's layers sit on; reseed {profile} to rebuild it"
731 )));
732 }
733 api::CacheKind::Slot { profile, slot } => {
734 if self.reseeds.contains_key(profile) {
735 return Err(ControlError::Conflict(format!(
736 "{profile} is reseeding, which writes {name}"
737 )));
738 }
739 self.occupied()
740 .filter(|(at, _)| at.profile == *profile && at.index + 1 == *slot)
741 .map(|(at, _)| at)
742 .collect()
743 }
744 api::CacheKind::Pnpm
745 | api::CacheKind::Maven
746 | api::CacheKind::Mirror
747 | api::CacheKind::Build => self.occupied().map(|(at, _)| at).collect(),
748 };
749 let shared = matches!(volume.kind, api::CacheKind::Pnpm | api::CacheKind::Maven);
750 if shared && !self.records.shells.is_empty() {
751 return Err(ControlError::Conflict(format!(
752 "a shell open in a record mounts {name}; clean it once the shell exits"
753 )));
754 }
755 for slot in &using {
758 if self
759 .occupant(slot)
760 .is_some_and(|occupant| !occupant.busy && occupant.job.is_none())
761 {
762 self.stop(slot, RunningJobs::Keep).await;
763 }
764 }
765 let users: Vec<api::RunningJob> = using
766 .iter()
767 .filter_map(|slot| self.occupant(slot))
768 .map(Occupant::running_job)
769 .collect();
770 if !users.is_empty() {
771 return Err(ControlError::Busy {
772 message: format!(
773 "{name} is in use by {}; clean it once they finish",
774 list(&users)
775 ),
776 jobs: users,
777 });
778 }
779 if volume.kind == api::CacheKind::Build {
780 let Some(cache) = self.build_cache.clone() else {
781 return Err(ControlError::Failed(
782 "the build cache did not open when runnerd started; its log says why"
783 .to_owned(),
784 ));
785 };
786 let trimmed = tokio::task::spawn_blocking(move || cache.clear())
787 .await
788 .map_err(|error| ControlError::Failed(format!("cannot empty {name}: {error}")))?;
789 info!(
790 "emptied {name}: {} entries, {}",
791 trimmed.before.entries,
792 ByteSize(trimmed.before.bytes)
793 );
794 self.trim_due = true;
795 return Ok(());
796 }
797 if volume.kind == api::CacheKind::Mirror && self.mirror_up {
798 cache::mirror_down(&*self.docker, &self.names)
799 .await
800 .map_err(|error| {
801 ControlError::Failed(format!("cannot stop the mirror: {error}"))
802 })?;
803 self.mirror_up = false;
804 }
805 self.docker
806 .remove_volume(name)
807 .await
808 .map_err(|error| ControlError::Failed(format!("cannot remove {name}: {error}")))?;
809 info!("removed {name}");
810 self.trim_due = true;
812 Ok(())
813 }
814
815 async fn undone_raises(&self, config: &Config) -> Vec<String> {
818 let now = time::now_ms() / 1000;
819 let since = now.saturating_sub(auto::UNDONE_RAISES).saturating_mul(1000);
820 match self.store.sizing_changes(since).await {
821 Ok(changes) => auto::undone_raises(&self.config, config, &changes, now),
822 Err(store_error) => {
823 warn!(
824 "cannot read auto sizing's changes to compare a new config with: {store_error}"
825 );
826 Vec::new()
827 }
828 }
829 }
830
831 fn occupied_past(&self, config: &Config) -> Option<String> {
833 self.slots.iter().find_map(|(name, slots)| {
834 let kept = config
835 .profiles
836 .get(name)
837 .map_or(0, |profile| profile.slots as usize);
838 slots
839 .iter()
840 .enumerate()
841 .skip(kept)
842 .find(|(_, slot)| slot.is_some())
843 .map(|(index, _)| {
844 format!(
845 "{name}'s slot {} has a runner, and the new config keeps {kept} slots",
846 index + 1
847 )
848 })
849 })
850 }
851
852 pub(super) fn reload(&mut self, config: Config, memory_max: Option<u64>, cpus: Option<u64>) {
857 let mut old = std::mem::replace(&mut self.config, config);
858 if let Some(cache) = &self.build_cache {
859 cache.set_max_size(self.config.caches.build.max_size.0);
860 }
861 for (name, profile) in &self.config.profiles {
862 let slots = self.slots.entry(name.clone()).or_default();
863 if slots.len() < profile.slots as usize {
864 slots.resize(profile.slots as usize, None);
865 }
866 self.retired.remove(name);
867 }
868 for (name, profile) in std::mem::take(&mut old.profiles) {
869 if !self.config.profiles.contains_key(&name) {
870 self.retired.insert(name, profile);
871 }
872 }
873 self.drop_closed_slots();
874 self.images.stale_due = true;
875 let names: BTreeSet<String> = self
876 .config
877 .repositories
878 .iter()
879 .map(|repository| repository.name.clone())
880 .collect();
881 let removed: Vec<String> = self
882 .repositories
883 .keys()
884 .filter(|name| !names.contains(*name))
885 .cloned()
886 .collect();
887 self.draining.extend(removed);
888 self.draining.retain(|name| !names.contains(name));
889 self.repositories.retain(|name, _| names.contains(name));
890 for name in &names {
891 self.repositories.entry(name.clone()).or_default();
892 }
893 self.control.set_repositories(names);
894 let profiles = &self.config.profiles;
895 self.reseed_requested
896 .retain(|name| profiles.contains_key(name));
897 self.oversized.retain(|name| profiles.contains_key(name));
898 self.isolation = isolation::backend(self.config.host.isolation);
899 self.memory_max = memory_max;
900 self.cpus = cpus;
901 self.caches
902 .send_replace(cache::volumes(&self.names, &self.config));
903 info!("reloaded the config");
904 }
905}