1use std::collections::{BTreeMap, HashMap};
7
8use bollard::models::{
9 ContainerCreateBody, EndpointSettings, HostConfig, Mount, MountType, NetworkingConfig,
10};
11
12use crate::api;
13use crate::config::{self, ByteSize};
14use crate::docker::{DockerApi, Error, RunnerSpec, is_not_found};
15use crate::isolation::Isolation;
16
17pub const MIRROR_IMAGE: &str =
19 "registry:3.1.2@sha256:01cd312e67026f06f5d89c3e133a4a9adc7cb9eeb9b1194e0b2954dfd770efd2";
20
21const MIRROR_ALIAS: &str = "mirror";
23
24const MIRROR_OOM_SCORE_ADJ: i64 = -500;
26
27#[derive(Debug, Clone)]
29pub struct Names {
30 prefix: String,
31}
32
33impl Names {
34 pub fn new(prefix: &str) -> Names {
36 Names {
37 prefix: prefix.to_owned(),
38 }
39 }
40
41 pub fn network(&self) -> String {
43 self.prefix.clone()
44 }
45
46 pub fn base(&self, profile: &str) -> String {
48 format!("{}-{profile}-base", self.prefix)
49 }
50
51 pub fn slot(&self, profile: &str, slot: usize) -> String {
53 format!("{}-{profile}-slot-{slot}", self.prefix)
54 }
55
56 pub fn pnpm(&self) -> String {
58 format!("{}-pnpm", self.prefix)
59 }
60
61 pub fn maven(&self) -> String {
63 format!("{}-maven", self.prefix)
64 }
65
66 pub fn slot_of(&self, profile: &str, volume: &str) -> Option<usize> {
68 volume
69 .strip_prefix(&format!("{}-{profile}-slot-", self.prefix))?
70 .parse()
71 .ok()
72 }
73
74 pub fn seed(&self, profile: &str) -> String {
76 format!("{}-{profile}-seed", self.prefix)
77 }
78
79 pub fn measure(&self, profile: &str) -> String {
81 format!("{}-{profile}-measure", self.prefix)
82 }
83
84 pub fn mirror(&self) -> String {
86 format!("{}-mirror", self.prefix)
87 }
88
89 pub fn build_cache(&self) -> String {
92 format!("{}-build-cache", self.prefix)
93 }
94}
95
96fn volume(source: String, target: &str, read_only: bool) -> Mount {
97 Mount {
98 target: Some(target.to_owned()),
99 source: Some(source),
100 typ: Some(MountType::VOLUME),
101 read_only: Some(read_only),
102 ..Default::default()
103 }
104}
105
106pub fn runner_mounts(names: &Names, profile: &str, slot: usize) -> Vec<Mount> {
108 vec![
109 volume(names.base(profile), "/base", true),
110 volume(names.slot(profile, slot), "/state", false),
111 volume(names.pnpm(), "/shared/pnpm", false),
112 volume(names.maven(), "/shared/maven", false),
113 ]
114}
115
116pub fn mirror_body(names: &Names, mirror: &config::Mirror, slice: &str) -> ContainerCreateBody {
119 let memory = i64::try_from(mirror.memory.0).unwrap_or(i64::MAX);
120 ContainerCreateBody {
121 image: Some(MIRROR_IMAGE.to_owned()),
122 env: Some(vec![
123 "REGISTRY_PROXY_REMOTEURL=https://registry-1.docker.io".to_owned(),
124 format!("REGISTRY_PROXY_TTL={}s", mirror.ttl.as_secs()),
125 "REGISTRY_STORAGE_DELETE_ENABLED=true".to_owned(),
127 ]),
128 labels: Some(HashMap::from([(
129 "runnerd.mirror".to_owned(),
130 String::new(),
131 )])),
132 host_config: Some(HostConfig {
133 cgroup_parent: Some(slice.to_owned()),
134 memory: Some(memory),
135 memory_swap: Some(memory),
136 nano_cpus: Some(i64::from(mirror.cpus) * 1_000_000_000),
137 oom_score_adj: Some(MIRROR_OOM_SCORE_ADJ),
140 mounts: Some(vec![volume(names.mirror(), "/var/lib/registry", false)]),
141 network_mode: Some(names.network()),
142 ..Default::default()
143 }),
144 networking_config: Some(NetworkingConfig {
145 endpoints_config: Some(HashMap::from([(
146 names.network(),
147 EndpointSettings {
148 aliases: Some(vec![MIRROR_ALIAS.to_owned()]),
149 ..Default::default()
150 },
151 )])),
152 }),
153 ..Default::default()
154 }
155}
156
157pub async fn prepare(
159 docker: &impl DockerApi,
160 names: &Names,
161 profile: &str,
162 slot: usize,
163) -> Result<(), Error> {
164 docker.ensure_network(&names.network()).await?;
165 for name in [
166 names.base(profile),
167 names.slot(profile, slot),
168 names.pnpm(),
169 names.maven(),
170 ] {
171 docker.ensure_volume(&name).await?;
172 }
173 Ok(())
174}
175
176pub async fn mirror_up(
178 docker: &impl DockerApi,
179 names: &Names,
180 mirror: &config::Mirror,
181 slice: &str,
182) -> Result<(), Error> {
183 match docker.inspect(&names.mirror()).await {
184 Ok(state) if state.running => return Ok(()),
185 Ok(_) => docker.remove(&names.mirror()).await?,
187 Err(error) if is_not_found(&error) => {}
188 Err(error) => return Err(error),
189 }
190 docker.ensure_network(&names.network()).await?;
191 if !docker.image_present(MIRROR_IMAGE).await? {
192 docker.pull(MIRROR_IMAGE).await?;
193 }
194 docker
195 .run(&names.mirror(), mirror_body(names, mirror, slice))
196 .await?;
197 Ok(())
198}
199
200pub async fn mirror_down(docker: &impl DockerApi, names: &Names) -> Result<(), Error> {
202 match docker.remove(&names.mirror()).await {
203 Err(error) if is_not_found(&error) => Ok(()),
204 other => other,
205 }
206}
207
208#[derive(Debug, Clone, Copy, PartialEq, Eq)]
210pub enum Mirror {
211 Up,
213 Down,
215}
216
217#[derive(Debug, Clone, PartialEq, Eq)]
219pub struct Trimmed {
220 pub sizes: BTreeMap<String, u64>,
222 pub emptied: Vec<Emptied>,
224 pub closed: Vec<Emptied>,
226 pub refused: Vec<Refused>,
229 pub free: Option<u64>,
231}
232
233#[derive(Debug, Clone, PartialEq, Eq)]
235pub struct Refused {
236 pub name: String,
238 pub error: String,
240}
241
242#[derive(Debug, Clone, PartialEq, Eq)]
244pub struct Emptied {
245 pub name: String,
247 pub size: ByteSize,
249}
250
251pub async fn trim_shared(
256 docker: &impl DockerApi,
257 names: &Names,
258 caches: &config::Caches,
259 mirror: Mirror,
260 slots: &BTreeMap<String, usize>,
261) -> Result<Trimmed, Error> {
262 let sizes = docker.volume_sizes().await?;
263 let free = docker.free_space().await.ok();
264 let mut emptied = Vec::new();
265 let mut refused = Vec::new();
266 let mut limits = vec![
267 (names.pnpm(), caches.pnpm.max_size),
268 (names.maven(), caches.maven.max_size),
269 ];
270 if mirror == Mirror::Down {
271 limits.push((names.mirror(), caches.mirror.max_size));
272 }
273 for (name, limit) in limits {
274 if let Some(&size) = sizes.get(&name)
275 && size > limit.0
276 {
277 if let Err(error) = docker.remove_volume(&name).await {
279 refused.push(Refused {
280 name,
281 error: error.to_string(),
282 });
283 continue;
284 }
285 emptied.push(Emptied {
286 name,
287 size: ByteSize(size),
288 });
289 }
290 }
291 let mut closed = Vec::new();
292 for (name, &size) in &sizes {
293 let past = slots.iter().any(|(profile, &count)| {
294 names
295 .slot_of(profile, name)
296 .is_some_and(|slot| slot > count)
297 });
298 if past {
299 if let Err(error) = docker.remove_volume(name).await {
301 refused.push(Refused {
302 name: name.clone(),
303 error: error.to_string(),
304 });
305 continue;
306 }
307 closed.push(Emptied {
308 name: name.clone(),
309 size: ByteSize(size),
310 });
311 }
312 }
313 Ok(Trimmed {
314 sizes,
315 emptied,
316 closed,
317 refused,
318 free,
319 })
320}
321
322pub fn docker_disk(
325 names: &Names,
326 config: &config::Config,
327 free: u64,
328 sizes: &BTreeMap<String, u64>,
329) -> u64 {
330 let held: u64 = config
331 .profiles
332 .iter()
333 .flat_map(|(profile, settings)| {
334 (1..=settings.slots as usize).map(move |slot| names.slot(profile, slot))
335 })
336 .filter_map(|name| sizes.get(&name))
337 .sum();
338 free.saturating_add(held) / DOCKER_DISK_SHARE
339}
340
341pub fn docker_budget_gib(disk: u64, slots: u64) -> u64 {
344 ((disk / slots.max(1)) >> 30).clamp(1, DOCKER_SLOT_MAX_GIB)
345}
346
347const DOCKER_DISK_SHARE: u64 = 4;
349
350const DOCKER_SLOT_MAX_GIB: u64 = 20;
352
353#[derive(Debug, Default)]
355pub struct Cleared {
356 pub removed: Vec<String>,
358 pub refused: Vec<Refusal>,
360}
361
362#[derive(Debug)]
364pub struct Refusal {
365 pub volume: String,
367 pub why: String,
369 pub held: bool,
372}
373
374pub async fn empty_volumes(docker: &impl DockerApi, volumes: &[String]) -> Cleared {
378 let mut emptied = Cleared::default();
379 for volume in volumes {
380 match empty_volume(docker, volume).await {
381 Ok(()) => emptied.removed.push(volume.clone()),
382 Err(refusal) => emptied.refused.push(refusal),
383 }
384 }
385 emptied
386}
387
388async fn empty_volume(docker: &impl DockerApi, volume: &str) -> Result<(), Refusal> {
390 let refused = |why: String, held: bool| Refusal {
391 volume: volume.to_owned(),
392 why,
393 held,
394 };
395 let failed = |error: Error| refused(error.to_string(), false);
396 for holder in docker.holders(volume).await.map_err(failed)? {
397 let name = if holder.name.is_empty() {
398 &holder.id
399 } else {
400 &holder.name
401 };
402 if holder.running {
403 return Err(refused(
404 format!("container {name} mounts it and runs"),
405 true,
406 ));
407 }
408 if !holder
409 .labels
410 .keys()
411 .any(|label| label.starts_with("runnerd."))
412 {
413 return Err(refused(
414 format!("container {name}, which is not runnerd's, mounts it"),
415 true,
416 ));
417 }
418 match docker.remove(&holder.id).await {
419 Err(error) if !is_not_found(&error) => {
420 return Err(refused(
421 format!("container {name}, which mounts it, cannot be removed: {error}"),
422 false,
423 ));
424 }
425 _ => {}
426 }
427 }
428 docker.remove_volume(volume).await.map_err(failed)
429}
430
431pub fn volumes(names: &Names, config: &config::Config) -> Vec<api::CacheVolume> {
433 let volume = |name: String, kind: api::CacheKind, limit: Option<u64>| api::CacheVolume {
434 name,
435 kind,
436 bytes: 0,
437 limit,
438 };
439 let mut volumes = Vec::new();
440 for (profile, settings) in &config.profiles {
441 volumes.push(volume(
442 names.base(profile),
443 api::CacheKind::Snapshot {
444 profile: profile.clone(),
445 },
446 Some(config.caches.snapshot_max.0),
447 ));
448 for slot in 1..=settings.slots as usize {
449 volumes.push(volume(
450 names.slot(profile, slot),
451 api::CacheKind::Slot {
452 profile: profile.clone(),
453 slot,
454 },
455 None,
456 ));
457 }
458 }
459 volumes.push(volume(
460 names.pnpm(),
461 api::CacheKind::Pnpm,
462 Some(config.caches.pnpm.max_size.0),
463 ));
464 volumes.push(volume(
465 names.maven(),
466 api::CacheKind::Maven,
467 Some(config.caches.maven.max_size.0),
468 ));
469 if config.caches.mirror.enabled {
470 volumes.push(volume(
471 names.mirror(),
472 api::CacheKind::Mirror,
473 Some(config.caches.mirror.max_size.0),
474 ));
475 }
476 if config.caches.build.enabled {
477 volumes.push(volume(
478 names.build_cache(),
479 api::CacheKind::Build,
480 Some(config.caches.build.max_size.0),
481 ));
482 }
483 volumes.sort_by(|a, b| a.name.cmp(&b.name));
484 volumes
485}
486
487#[derive(Debug)]
489pub struct Reseed<'profile> {
490 pub names: &'profile Names,
492 pub profile: &'profile str,
494 pub image: &'profile str,
496 pub slots: usize,
498 pub slice: &'profile str,
500 pub cpus: u32,
502 pub memory: ByteSize,
504 pub min_changes: ByteSize,
506 pub max_size: ByteSize,
508 pub isolation: &'profile dyn Isolation,
510}
511
512pub async fn reseed(docker: &impl DockerApi, reseed: &Reseed<'_>) -> Result<String, Error> {
517 let names = reseed.names;
518 let mut mounts = vec![volume(names.base(reseed.profile), "/base", false)];
519 mounts.extend((1..=reseed.slots).map(|slot| {
520 volume(
521 names.slot(reseed.profile, slot),
522 &format!("/workers/{slot}"),
523 false,
524 )
525 }));
526 for mount in &mounts {
527 if let Some(source) = &mount.source {
528 docker.ensure_volume(source).await?;
529 }
530 }
531 let name = names.seed(reseed.profile);
532 let spec = RunnerSpec {
533 name: name.clone(),
534 image: reseed.image.to_owned(),
535 profile: reseed.profile.to_owned(),
536 slice: reseed.slice.to_owned(),
537 cpus: reseed.cpus,
538 memory: reseed.memory,
539 env: Vec::new(),
540 mounts,
541 network: Some("none".to_owned()),
542 labels: BTreeMap::from([("runnerd.seed".to_owned(), String::new())]),
543 entrypoint: Some(vec![
544 "/usr/local/sbin/runnerd-seed".to_owned(),
545 reseed.min_changes.0.to_string(),
546 reseed.max_size.0.to_string(),
547 ]),
548 jit_config: None,
549 };
550 match docker.remove(&name).await {
552 Err(error) if !is_not_found(&error) => return Err(error),
553 _ => {}
554 }
555 let id = docker.start(&spec, reseed.isolation).await?;
556 finish_reseed(docker, &id).await
557}
558
559#[derive(Debug)]
562pub struct Measure<'profile> {
563 pub names: &'profile Names,
565 pub profile: &'profile str,
567 pub image: &'profile str,
569 pub slice: &'profile str,
571 pub isolation: &'profile dyn Isolation,
573}
574
575#[derive(Clone, Copy, Debug, PartialEq, Eq)]
577pub enum Measured {
578 Changed(u64),
580 Unfinished,
582}
583
584const MEASURE_MEMORY: ByteSize = ByteSize(256 << 20);
586
587pub async fn measure(docker: &impl DockerApi, measure: &Measure<'_>) -> Result<Measured, Error> {
591 let spec = measure_spec(measure);
592 match docker.remove(&spec.name).await {
593 Err(error) if !is_not_found(&error) => return Err(error),
594 _ => {}
595 }
596 let id = docker.start(&spec, measure.isolation).await?;
597 let output = finish_reseed(docker, &id).await?;
598 measured(&output).ok_or_else(|| Error::DockerContainerWaitError {
599 error: format!("measuring printed {:?}", output.trim()),
600 code: 0,
601 })
602}
603
604pub fn measure_spec(measure: &Measure<'_>) -> RunnerSpec {
606 let names = measure.names;
607 RunnerSpec {
608 name: names.measure(measure.profile),
609 image: measure.image.to_owned(),
610 profile: measure.profile.to_owned(),
611 slice: measure.slice.to_owned(),
612 cpus: 1,
613 memory: MEASURE_MEMORY,
614 env: Vec::new(),
615 mounts: vec![
616 volume(names.base(measure.profile), "/base", true),
617 volume(names.slot(measure.profile, 1), "/workers/1", true),
618 ],
619 network: Some("none".to_owned()),
620 labels: BTreeMap::from([("runnerd.seed".to_owned(), String::new())]),
622 entrypoint: Some(vec![
623 "/usr/local/sbin/runnerd-seed".to_owned(),
624 "measure".to_owned(),
625 ]),
626 jit_config: None,
627 }
628}
629
630fn measured(output: &str) -> Option<Measured> {
633 output.lines().rev().map(str::trim).find_map(|line| {
634 if line == "the last reseed did not finish" {
635 return Some(Measured::Unfinished);
636 }
637 line.strip_prefix("slot 1 changed ")?
638 .strip_suffix(" bytes")?
639 .parse()
640 .ok()
641 .map(Measured::Changed)
642 })
643}
644
645pub fn seeded(output: &str) -> bool {
648 !output.trim_start().starts_with("not seeded")
649}
650
651pub async fn finish_reseed(docker: &impl DockerApi, container: &str) -> Result<String, Error> {
654 let result = async {
655 let code = docker.wait(container).await?;
656 let output = docker.logs(container).await?;
657 if code == 0 {
658 Ok(output)
659 } else {
660 Err(Error::DockerContainerWaitError {
661 error: format!("seeding exited {code}: {}", output.trim()),
662 code,
663 })
664 }
665 }
666 .await;
667 let removed = docker.remove(container).await;
668 let output = result?;
669 removed?;
670 Ok(output)
671}
672
673#[cfg(test)]
674mod tests {
675 use std::path::Path;
676 use std::process::Command;
677
678 use super::docker_budget_gib;
679 use crate::testdir::TestDir;
680
681 #[allow(
683 clippy::unwrap_used,
684 reason = "a test helper: a script that does not run fails the test"
685 )]
686 fn with_layers(script: &str, args: &[&Path]) -> String {
687 let layers = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../image/layers.sh");
688 let output = Command::new("bash")
689 .arg("-euc")
690 .arg(format!(". \"$0\"; {script}"))
691 .arg(layers)
692 .args(args)
693 .output()
694 .unwrap();
695 assert!(
696 output.status.success(),
697 "the script exits 0: {}",
698 String::from_utf8_lossy(&output.stderr)
699 );
700 String::from_utf8(output.stdout).unwrap()
701 }
702
703 #[test]
704 #[allow(
705 clippy::unwrap_used,
706 reason = "a test: a file it cannot write fails it"
707 )]
708 fn no_credential_a_job_leaves_in_the_cache_layer_reaches_the_slots_next_job() {
709 let listed = with_layers("printf '%s\\n' \"${CACHE_SECRETS[@]}\"", &[]);
710 let secrets: Vec<&str> = listed.lines().collect();
711 for credential in [
712 "cargo/credentials.toml",
713 "gradle/gradle.properties",
714 "gradle/init.d",
715 "m2/settings.xml",
716 "m2/settings-security.xml",
717 "huggingface/token",
718 ] {
719 assert!(
720 secrets.contains(&credential),
721 "{credential} is in CACHE_SECRETS: {secrets:?}"
722 );
723 }
724 let dir = TestDir::new("cache-secrets");
725 let upper = dir.0.join("upper");
726 let write = |path: &str| {
727 let path = upper.join(path);
728 std::fs::create_dir_all(path.parent().unwrap()).unwrap();
729 std::fs::write(path, "token").unwrap();
730 };
731 for secret in &secrets {
733 write(&format!("files/{secret}"));
734 write(&format!("dirs/{secret}/held"));
735 }
736 let kept = [
737 "npm/_cacache/index",
738 "m2/wrapper/dists/maven.zip",
739 "gradle/caches/jars",
740 ];
741 for path in kept {
742 write(&format!("files/{path}"));
743 }
744 with_layers(
745 "drop_cache_secrets \"$1/files\"; drop_cache_secrets \"$1/dirs\"",
746 &[&upper],
747 );
748 for secret in &secrets {
749 for copy in ["files", "dirs"] {
750 let path = upper.join(copy).join(secret);
751 assert!(!path.exists(), "{} is gone", path.display());
752 }
753 }
754 for path in kept {
755 assert!(upper.join("files").join(path).exists(), "{path} stays");
756 }
757 }
758
759 #[test]
760 #[allow(
761 clippy::unwrap_used,
762 reason = "a test: a file it cannot write fails it"
763 )]
764 fn what_slot_1_changed_counts_its_layers_and_not_its_docker_engines_data() {
765 let dir = TestDir::new("layer-changes");
766 let state = dir.0.join("state");
767 let write = |path: &str, bytes: usize| {
768 let path = state.join(path);
769 std::fs::create_dir_all(path.parent().unwrap()).unwrap();
770 std::fs::write(path, vec![0u8; bytes]).unwrap();
771 };
772 let changes = || -> u64 {
773 with_layers("layer_changes \"$1\"", &[&state])
774 .trim()
775 .parse()
776 .unwrap()
777 };
778 write("cache/upper/npm/index", 1);
779 let before = changes();
780 write("docker/overlay2/layer", 1 << 20);
782 write("tools/work/scratch", 1 << 20);
784 assert_eq!(changes(), before);
785 write("tools/upper/node/bin", 100_000);
786 write("android/upper/ndk/lib", 200_000);
787 let after = changes();
788 assert!(
789 after >= before + 300_000 && after < before + 400_000,
790 "the tools and android layers count: {before} then {after}"
791 );
792 }
793
794 #[test]
795 fn a_slots_docker_budget_is_its_share_between_1_and_20_gib() {
796 assert_eq!(docker_budget_gib(100 << 30, 10), 10);
797 assert_eq!(
798 docker_budget_gib(1 << 40, 4),
799 20,
800 "a large disk still stops at 20 GiB"
801 );
802 assert_eq!(
803 docker_budget_gib(1 << 30, 13),
804 1,
805 "a full disk still keeps 1 GiB"
806 );
807 assert_eq!(docker_budget_gib(1 << 30, 0), 1);
808 }
809}