1use std::collections::{BTreeMap, HashMap};
7
8use bollard::models::{
9 ContainerCreateBody, EndpointSettings, HostConfig, Mount, MountType, NetworkingConfig,
10};
11
12use crate::api;
13use crate::config::{self, ByteSize};
14use crate::docker::{DockerApi, Error, RunnerSpec, is_not_found};
15use crate::isolation::Isolation;
16
17pub const MIRROR_IMAGE: &str =
19 "registry:3.1.2@sha256:01cd312e67026f06f5d89c3e133a4a9adc7cb9eeb9b1194e0b2954dfd770efd2";
20
21const MIRROR_ALIAS: &str = "mirror";
23
24const MIRROR_OOM_SCORE_ADJ: i64 = -500;
26
27#[derive(Debug, Clone)]
29pub struct Names {
30 prefix: String,
31}
32
33impl Names {
34 pub fn new(prefix: &str) -> Names {
36 Names {
37 prefix: prefix.to_owned(),
38 }
39 }
40
41 pub fn network(&self) -> String {
43 self.prefix.clone()
44 }
45
46 pub fn base(&self, profile: &str) -> String {
48 format!("{}-{profile}-base", self.prefix)
49 }
50
51 pub fn slot(&self, profile: &str, slot: usize) -> String {
53 format!("{}-{profile}-slot-{slot}", self.prefix)
54 }
55
56 pub fn pnpm(&self) -> String {
58 format!("{}-pnpm", self.prefix)
59 }
60
61 pub fn maven(&self) -> String {
63 format!("{}-maven", self.prefix)
64 }
65
66 pub fn slot_of(&self, profile: &str, volume: &str) -> Option<usize> {
68 volume
69 .strip_prefix(&format!("{}-{profile}-slot-", self.prefix))?
70 .parse()
71 .ok()
72 }
73
74 pub fn is_slot(&self, volume: &str) -> bool {
77 volume
78 .strip_prefix(&format!("{}-", self.prefix))
79 .and_then(|rest| rest.rsplit_once("-slot-"))
80 .is_some_and(|(profile, slot)| {
81 !profile.is_empty()
82 && !slot.is_empty()
83 && slot.bytes().all(|byte| byte.is_ascii_digit())
84 })
85 }
86
87 pub fn volume_prefix(&self) -> String {
89 format!("{}-", self.prefix)
90 }
91
92 pub fn start_over(&self) -> String {
94 format!("{}-start-over", self.prefix)
95 }
96
97 pub fn seed(&self, profile: &str) -> String {
99 format!("{}-{profile}-seed", self.prefix)
100 }
101
102 pub fn measure(&self, profile: &str) -> String {
104 format!("{}-{profile}-measure", self.prefix)
105 }
106
107 pub fn mirror(&self) -> String {
109 format!("{}-mirror", self.prefix)
110 }
111
112 pub fn build_cache(&self) -> String {
115 format!("{}-build-cache", self.prefix)
116 }
117}
118
119fn volume(source: String, target: &str, read_only: bool) -> Mount {
120 Mount {
121 target: Some(target.to_owned()),
122 source: Some(source),
123 typ: Some(MountType::VOLUME),
124 read_only: Some(read_only),
125 ..Default::default()
126 }
127}
128
129pub fn runner_mounts(names: &Names, profile: &str, slot: usize) -> Vec<Mount> {
131 vec![
132 volume(names.base(profile), "/base", true),
133 volume(names.slot(profile, slot), "/state", false),
134 volume(names.pnpm(), "/shared/pnpm", false),
135 volume(names.maven(), "/shared/maven", false),
136 ]
137}
138
139pub fn mirror_body(names: &Names, mirror: &config::Mirror, slice: &str) -> ContainerCreateBody {
142 let memory = i64::try_from(mirror.memory.0).unwrap_or(i64::MAX);
143 ContainerCreateBody {
144 image: Some(MIRROR_IMAGE.to_owned()),
145 env: Some(vec![
146 "REGISTRY_PROXY_REMOTEURL=https://registry-1.docker.io".to_owned(),
147 format!("REGISTRY_PROXY_TTL={}s", mirror.ttl.as_secs()),
148 "REGISTRY_STORAGE_DELETE_ENABLED=true".to_owned(),
150 ]),
151 labels: Some(HashMap::from([(
152 "runnerd.mirror".to_owned(),
153 String::new(),
154 )])),
155 host_config: Some(HostConfig {
156 cgroup_parent: Some(slice.to_owned()),
157 memory: Some(memory),
158 memory_swap: Some(memory),
159 nano_cpus: Some(i64::from(mirror.cpus) * 1_000_000_000),
160 oom_score_adj: Some(MIRROR_OOM_SCORE_ADJ),
163 mounts: Some(vec![volume(names.mirror(), "/var/lib/registry", false)]),
164 network_mode: Some(names.network()),
165 ..Default::default()
166 }),
167 networking_config: Some(NetworkingConfig {
168 endpoints_config: Some(HashMap::from([(
169 names.network(),
170 EndpointSettings {
171 aliases: Some(vec![MIRROR_ALIAS.to_owned()]),
172 ..Default::default()
173 },
174 )])),
175 }),
176 ..Default::default()
177 }
178}
179
180pub async fn prepare(
182 docker: &impl DockerApi,
183 names: &Names,
184 profile: &str,
185 slot: usize,
186) -> Result<(), Error> {
187 docker.ensure_network(&names.network()).await?;
188 for name in [
189 names.base(profile),
190 names.slot(profile, slot),
191 names.pnpm(),
192 names.maven(),
193 ] {
194 docker.ensure_volume(&name).await?;
195 }
196 Ok(())
197}
198
199pub async fn mirror_up(
201 docker: &impl DockerApi,
202 names: &Names,
203 mirror: &config::Mirror,
204 slice: &str,
205) -> Result<(), Error> {
206 match docker.inspect(&names.mirror()).await {
207 Ok(state) if state.running => return Ok(()),
208 Ok(_) => docker.remove(&names.mirror()).await?,
210 Err(error) if is_not_found(&error) => {}
211 Err(error) => return Err(error),
212 }
213 docker.ensure_network(&names.network()).await?;
214 if !docker.image_present(MIRROR_IMAGE).await? {
215 docker.pull(MIRROR_IMAGE).await?;
216 }
217 docker
218 .run(&names.mirror(), mirror_body(names, mirror, slice))
219 .await?;
220 Ok(())
221}
222
223pub async fn mirror_down(docker: &impl DockerApi, names: &Names) -> Result<(), Error> {
225 match docker.remove(&names.mirror()).await {
226 Err(error) if is_not_found(&error) => Ok(()),
227 other => other,
228 }
229}
230
231#[derive(Debug, Clone, Copy, PartialEq, Eq)]
233pub enum Mirror {
234 Up,
236 Down,
238}
239
240#[derive(Debug, Clone, PartialEq, Eq)]
242pub struct Trimmed {
243 pub sizes: BTreeMap<String, u64>,
245 pub emptied: Vec<Emptied>,
247 pub closed: Vec<Emptied>,
249 pub refused: Vec<Refused>,
252 pub free: Option<u64>,
254}
255
256#[derive(Debug, Clone, PartialEq, Eq)]
258pub struct Refused {
259 pub name: String,
261 pub error: String,
263}
264
265#[derive(Debug, Clone, PartialEq, Eq)]
267pub struct Emptied {
268 pub name: String,
270 pub size: ByteSize,
272}
273
274pub async fn trim_shared(
279 docker: &impl DockerApi,
280 names: &Names,
281 caches: &config::Caches,
282 mirror: Mirror,
283 slots: &BTreeMap<String, usize>,
284) -> Result<Trimmed, Error> {
285 let sizes = docker.volume_sizes().await?;
286 let free = docker.free_space().await.ok();
287 let mut emptied = Vec::new();
288 let mut refused = Vec::new();
289 let mut limits = vec![
290 (names.pnpm(), caches.pnpm.max_size),
291 (names.maven(), caches.maven.max_size),
292 ];
293 if mirror == Mirror::Down {
294 limits.push((names.mirror(), caches.mirror.max_size));
295 }
296 for (name, limit) in limits {
297 if let Some(&size) = sizes.get(&name)
298 && size > limit.0
299 {
300 if let Err(error) = docker.remove_volume(&name).await {
302 refused.push(Refused {
303 name,
304 error: error.to_string(),
305 });
306 continue;
307 }
308 emptied.push(Emptied {
309 name,
310 size: ByteSize(size),
311 });
312 }
313 }
314 let mut closed = Vec::new();
315 for (name, &size) in &sizes {
316 let past = slots.iter().any(|(profile, &count)| {
317 names
318 .slot_of(profile, name)
319 .is_some_and(|slot| slot > count)
320 });
321 if past {
322 if let Err(error) = docker.remove_volume(name).await {
324 refused.push(Refused {
325 name: name.clone(),
326 error: error.to_string(),
327 });
328 continue;
329 }
330 closed.push(Emptied {
331 name: name.clone(),
332 size: ByteSize(size),
333 });
334 }
335 }
336 Ok(Trimmed {
337 sizes,
338 emptied,
339 closed,
340 refused,
341 free,
342 })
343}
344
345pub fn docker_disk(
348 names: &Names,
349 config: &config::Config,
350 free: u64,
351 sizes: &BTreeMap<String, u64>,
352) -> u64 {
353 let held: u64 = config
354 .profiles
355 .iter()
356 .flat_map(|(profile, settings)| {
357 (1..=settings.slots as usize).map(move |slot| names.slot(profile, slot))
358 })
359 .filter_map(|name| sizes.get(&name))
360 .sum();
361 free.saturating_add(held) / DOCKER_DISK_SHARE
362}
363
364pub fn docker_budget_gib(disk: u64, slots: u64) -> u64 {
367 ((disk / slots.max(1)) >> 30).clamp(1, DOCKER_SLOT_MAX_GIB)
368}
369
370const DOCKER_DISK_SHARE: u64 = 4;
372
373const DOCKER_SLOT_MAX_GIB: u64 = 20;
375
376#[derive(Debug, Default)]
378pub struct Cleared {
379 pub removed: Vec<String>,
381 pub refused: Vec<Refusal>,
383}
384
385#[derive(Debug)]
387pub struct Refusal {
388 pub volume: String,
390 pub why: String,
392 pub cause: Kept,
394}
395
396#[derive(Debug, Clone, Copy, PartialEq, Eq)]
398pub enum Kept {
399 Runs,
401 NotRunnerds,
403 Docker,
405}
406
407impl Refusal {
408 pub fn held(&self) -> bool {
410 self.cause != Kept::Docker
411 }
412}
413
414pub async fn empty_volumes(docker: &impl DockerApi, volumes: &[String]) -> Cleared {
418 let mut emptied = Cleared::default();
419 for volume in volumes {
420 match empty_volume(docker, volume).await {
421 Ok(()) => emptied.removed.push(volume.clone()),
422 Err(refusal) => emptied.refused.push(refusal),
423 }
424 }
425 emptied
426}
427
428async fn empty_volume(docker: &impl DockerApi, volume: &str) -> Result<(), Refusal> {
430 let refused = |why: String, cause: Kept| Refusal {
431 volume: volume.to_owned(),
432 why,
433 cause,
434 };
435 let failed = |error: Error| refused(error.to_string(), Kept::Docker);
436 for holder in docker.holders(volume).await.map_err(failed)? {
437 let name = if holder.name.is_empty() {
438 &holder.id
439 } else {
440 &holder.name
441 };
442 if holder.running {
443 return Err(refused(
444 format!("container {name} mounts it and runs"),
445 Kept::Runs,
446 ));
447 }
448 if !holder
449 .labels
450 .keys()
451 .any(|label| label.starts_with("runnerd."))
452 {
453 return Err(refused(
454 format!("container {name}, which is not runnerd's, mounts it"),
455 Kept::NotRunnerds,
456 ));
457 }
458 match docker.remove(&holder.id).await {
459 Err(error) if !is_not_found(&error) => {
460 return Err(refused(
461 format!("container {name}, which mounts it, cannot be removed: {error}"),
462 Kept::Docker,
463 ));
464 }
465 _ => {}
466 }
467 }
468 docker.remove_volume(volume).await.map_err(failed)
469}
470
471pub fn volumes(names: &Names, config: &config::Config) -> Vec<api::CacheVolume> {
473 let volume = |name: String, kind: api::CacheKind, limit: Option<u64>| api::CacheVolume {
474 name,
475 kind,
476 bytes: 0,
477 limit,
478 };
479 let mut volumes = Vec::new();
480 for (profile, settings) in &config.profiles {
481 volumes.push(volume(
482 names.base(profile),
483 api::CacheKind::Snapshot {
484 profile: profile.clone(),
485 },
486 Some(config.caches.snapshot_max.0),
487 ));
488 for slot in 1..=settings.slots as usize {
489 volumes.push(volume(
490 names.slot(profile, slot),
491 api::CacheKind::Slot {
492 profile: profile.clone(),
493 slot,
494 },
495 None,
496 ));
497 }
498 }
499 volumes.push(volume(
500 names.pnpm(),
501 api::CacheKind::Pnpm,
502 Some(config.caches.pnpm.max_size.0),
503 ));
504 volumes.push(volume(
505 names.maven(),
506 api::CacheKind::Maven,
507 Some(config.caches.maven.max_size.0),
508 ));
509 if config.caches.mirror.enabled {
510 volumes.push(volume(
511 names.mirror(),
512 api::CacheKind::Mirror,
513 Some(config.caches.mirror.max_size.0),
514 ));
515 }
516 if config.caches.build.enabled {
517 volumes.push(volume(
518 names.build_cache(),
519 api::CacheKind::Build,
520 Some(config.caches.build.max_size.0),
521 ));
522 }
523 volumes.sort_by(|a, b| a.name.cmp(&b.name));
524 volumes
525}
526
527#[derive(Debug)]
529pub struct Reseed<'profile> {
530 pub names: &'profile Names,
532 pub profile: &'profile str,
534 pub image: &'profile str,
536 pub slots: usize,
538 pub slice: &'profile str,
540 pub cpus: u32,
542 pub memory: ByteSize,
544 pub min_changes: ByteSize,
546 pub max_size: ByteSize,
548 pub isolation: &'profile dyn Isolation,
550}
551
552pub async fn reseed(docker: &impl DockerApi, reseed: &Reseed<'_>) -> Result<String, Error> {
557 let names = reseed.names;
558 let mut mounts = vec![volume(names.base(reseed.profile), "/base", false)];
559 mounts.extend((1..=reseed.slots).map(|slot| {
560 volume(
561 names.slot(reseed.profile, slot),
562 &format!("/workers/{slot}"),
563 false,
564 )
565 }));
566 for mount in &mounts {
567 if let Some(source) = &mount.source {
568 docker.ensure_volume(source).await?;
569 }
570 }
571 let name = names.seed(reseed.profile);
572 let spec = RunnerSpec {
573 name: name.clone(),
574 image: reseed.image.to_owned(),
575 profile: reseed.profile.to_owned(),
576 slice: reseed.slice.to_owned(),
577 cpus: reseed.cpus,
578 memory: reseed.memory,
579 env: Vec::new(),
580 mounts,
581 network: Some("none".to_owned()),
582 labels: BTreeMap::from([("runnerd.seed".to_owned(), String::new())]),
583 entrypoint: Some(vec![
584 "/usr/local/sbin/runnerd-seed".to_owned(),
585 reseed.min_changes.0.to_string(),
586 reseed.max_size.0.to_string(),
587 ]),
588 jit_config: None,
589 };
590 match docker.remove(&name).await {
592 Err(error) if !is_not_found(&error) => return Err(error),
593 _ => {}
594 }
595 let id = docker.start(&spec, reseed.isolation).await?;
596 finish_reseed(docker, &id).await
597}
598
599#[derive(Debug)]
602pub struct StartOver<'volume> {
603 pub names: &'volume Names,
605 pub volume: &'volume str,
607 pub image: &'volume str,
609 pub slice: &'volume str,
611 pub isolation: &'volume dyn Isolation,
613}
614
615const START_OVER_MEMORY: ByteSize = ByteSize(256 << 20);
617
618pub async fn start_layers_over(
624 docker: &impl DockerApi,
625 start: &StartOver<'_>,
626) -> Result<(), Error> {
627 let name = start.names.start_over();
628 match docker.remove(&name).await {
629 Err(error) if !is_not_found(&error) => return Err(error),
630 _ => {}
631 }
632 let spec = RunnerSpec {
633 name: name.clone(),
634 image: start.image.to_owned(),
635 profile: String::new(),
636 slice: start.slice.to_owned(),
637 cpus: 1,
638 memory: START_OVER_MEMORY,
639 env: Vec::new(),
640 mounts: vec![volume(start.volume.to_owned(), "/state", false)],
641 network: Some("none".to_owned()),
642 labels: BTreeMap::from([("runnerd.seed".to_owned(), String::new())]),
643 entrypoint: Some(
644 [
645 "find",
646 "/state",
647 "-mindepth",
648 "1",
649 "-maxdepth",
650 "1",
651 "!",
652 "-name",
653 "docker",
654 "-exec",
655 "rm",
656 "-rf",
657 "{}",
658 "+",
659 ]
660 .map(str::to_owned)
661 .to_vec(),
662 ),
663 jit_config: None,
664 };
665 let id = docker.start(&spec, start.isolation).await?;
666 finish_reseed(docker, &id).await.map(|_| ())
667}
668
669#[derive(Debug)]
672pub struct Measure<'profile> {
673 pub names: &'profile Names,
675 pub profile: &'profile str,
677 pub image: &'profile str,
679 pub slice: &'profile str,
681 pub isolation: &'profile dyn Isolation,
683}
684
685#[derive(Clone, Copy, Debug, PartialEq, Eq)]
687pub enum Measured {
688 Changed(u64),
690 Unfinished,
692}
693
694const MEASURE_MEMORY: ByteSize = ByteSize(256 << 20);
696
697pub async fn measure(docker: &impl DockerApi, measure: &Measure<'_>) -> Result<Measured, Error> {
701 let spec = measure_spec(measure);
702 match docker.remove(&spec.name).await {
703 Err(error) if !is_not_found(&error) => return Err(error),
704 _ => {}
705 }
706 let id = docker.start(&spec, measure.isolation).await?;
707 let output = finish_reseed(docker, &id).await?;
708 measured(&output).ok_or_else(|| Error::DockerContainerWaitError {
709 error: format!("measuring printed {:?}", output.trim()),
710 code: 0,
711 })
712}
713
714pub fn measure_spec(measure: &Measure<'_>) -> RunnerSpec {
716 let names = measure.names;
717 RunnerSpec {
718 name: names.measure(measure.profile),
719 image: measure.image.to_owned(),
720 profile: measure.profile.to_owned(),
721 slice: measure.slice.to_owned(),
722 cpus: 1,
723 memory: MEASURE_MEMORY,
724 env: Vec::new(),
725 mounts: vec![
726 volume(names.base(measure.profile), "/base", true),
727 volume(names.slot(measure.profile, 1), "/workers/1", true),
728 ],
729 network: Some("none".to_owned()),
730 labels: BTreeMap::from([("runnerd.seed".to_owned(), String::new())]),
732 entrypoint: Some(vec![
733 "/usr/local/sbin/runnerd-seed".to_owned(),
734 "measure".to_owned(),
735 ]),
736 jit_config: None,
737 }
738}
739
740fn measured(output: &str) -> Option<Measured> {
743 output.lines().rev().map(str::trim).find_map(|line| {
744 if line == "the last reseed did not finish" {
745 return Some(Measured::Unfinished);
746 }
747 line.strip_prefix("slot 1 changed ")?
748 .strip_suffix(" bytes")?
749 .parse()
750 .ok()
751 .map(Measured::Changed)
752 })
753}
754
755pub fn seeded(output: &str) -> bool {
758 !output.trim_start().starts_with("not seeded")
759}
760
761pub async fn finish_reseed(docker: &impl DockerApi, container: &str) -> Result<String, Error> {
764 let result = async {
765 let code = docker.wait(container).await?;
766 let output = docker.logs(container).await?;
767 if code == 0 {
768 Ok(output)
769 } else {
770 Err(Error::DockerContainerWaitError {
771 error: format!("seeding exited {code}: {}", output.trim()),
772 code,
773 })
774 }
775 }
776 .await;
777 let removed = docker.remove(container).await;
778 let output = result?;
779 removed?;
780 Ok(output)
781}
782
783#[cfg(test)]
784mod tests {
785 use std::path::Path;
786 use std::process::Command;
787
788 use super::docker_budget_gib;
789 use crate::testdir::TestDir;
790
791 #[allow(
793 clippy::unwrap_used,
794 reason = "a test helper: a script that does not run fails the test"
795 )]
796 fn with_layers(script: &str, args: &[&Path]) -> String {
797 let layers = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../image/layers.sh");
798 let output = Command::new("bash")
799 .arg("-euc")
800 .arg(format!(". \"$0\"; {script}"))
801 .arg(layers)
802 .args(args)
803 .output()
804 .unwrap();
805 assert!(
806 output.status.success(),
807 "the script exits 0: {}",
808 String::from_utf8_lossy(&output.stderr)
809 );
810 String::from_utf8(output.stdout).unwrap()
811 }
812
813 #[test]
814 #[allow(
815 clippy::unwrap_used,
816 reason = "a test: a file it cannot write fails it"
817 )]
818 fn no_credential_a_job_leaves_in_the_cache_layer_reaches_the_slots_next_job() {
819 let listed = with_layers("printf '%s\\n' \"${CACHE_SECRETS[@]}\"", &[]);
820 let secrets: Vec<&str> = listed.lines().collect();
821 for credential in [
822 "cargo/credentials.toml",
823 "gradle/gradle.properties",
824 "gradle/init.d",
825 "m2/settings.xml",
826 "m2/settings-security.xml",
827 "huggingface/token",
828 ] {
829 assert!(
830 secrets.contains(&credential),
831 "{credential} is in CACHE_SECRETS: {secrets:?}"
832 );
833 }
834 let dir = TestDir::new("cache-secrets");
835 let upper = dir.0.join("upper");
836 let write = |path: &str| {
837 let path = upper.join(path);
838 std::fs::create_dir_all(path.parent().unwrap()).unwrap();
839 std::fs::write(path, "token").unwrap();
840 };
841 for secret in &secrets {
843 write(&format!("files/{secret}"));
844 write(&format!("dirs/{secret}/held"));
845 }
846 let kept = [
847 "npm/_cacache/index",
848 "m2/wrapper/dists/maven.zip",
849 "gradle/caches/jars",
850 ];
851 for path in kept {
852 write(&format!("files/{path}"));
853 }
854 with_layers(
855 "drop_cache_secrets \"$1/files\"; drop_cache_secrets \"$1/dirs\"",
856 &[&upper],
857 );
858 for secret in &secrets {
859 for copy in ["files", "dirs"] {
860 let path = upper.join(copy).join(secret);
861 assert!(!path.exists(), "{} is gone", path.display());
862 }
863 }
864 for path in kept {
865 assert!(upper.join("files").join(path).exists(), "{path} stays");
866 }
867 }
868
869 #[test]
870 #[allow(
871 clippy::unwrap_used,
872 reason = "a test: a file it cannot write fails it"
873 )]
874 fn what_slot_1_changed_counts_its_layers_and_not_its_docker_engines_data() {
875 let dir = TestDir::new("layer-changes");
876 let state = dir.0.join("state");
877 let write = |path: &str, bytes: usize| {
878 let path = state.join(path);
879 std::fs::create_dir_all(path.parent().unwrap()).unwrap();
880 std::fs::write(path, vec![0u8; bytes]).unwrap();
881 };
882 let changes = || -> u64 {
883 with_layers("layer_changes \"$1\"", &[&state])
884 .trim()
885 .parse()
886 .unwrap()
887 };
888 write("cache/upper/npm/index", 1);
889 let before = changes();
890 write("docker/overlay2/layer", 1 << 20);
892 write("tools/work/scratch", 1 << 20);
894 assert_eq!(changes(), before);
895 write("tools/upper/node/bin", 100_000);
896 write("android/upper/ndk/lib", 200_000);
897 let after = changes();
898 assert!(
899 after >= before + 300_000 && after < before + 400_000,
900 "the tools and android layers count: {before} then {after}"
901 );
902 }
903
904 #[test]
905 fn a_slots_docker_budget_is_its_share_between_1_and_20_gib() {
906 assert_eq!(docker_budget_gib(100 << 30, 10), 10);
907 assert_eq!(
908 docker_budget_gib(1 << 40, 4),
909 20,
910 "a large disk still stops at 20 GiB"
911 );
912 assert_eq!(
913 docker_budget_gib(1 << 30, 13),
914 1,
915 "a full disk still keeps 1 GiB"
916 );
917 assert_eq!(docker_budget_gib(1 << 30, 0), 1);
918 }
919}