1use std::collections::{BTreeMap, HashMap};
7
8use bollard::models::{
9 ContainerCreateBody, EndpointSettings, HostConfig, Mount, MountType, NetworkingConfig,
10};
11
12use crate::api;
13use crate::config::{self, ByteSize};
14use crate::docker::{DockerApi, Error, RunnerSpec, is_not_found};
15use crate::isolation::Isolation;
16
17pub const MIRROR_IMAGE: &str =
19 "registry:3.1.2@sha256:01cd312e67026f06f5d89c3e133a4a9adc7cb9eeb9b1194e0b2954dfd770efd2";
20
21const MIRROR_ALIAS: &str = "mirror";
23
24const MIRROR_OOM_SCORE_ADJ: i64 = -500;
26
27#[derive(Debug, Clone)]
29pub struct Names {
30 prefix: String,
31}
32
33impl Names {
34 pub fn new(prefix: &str) -> Names {
36 Names {
37 prefix: prefix.to_owned(),
38 }
39 }
40
41 pub fn network(&self) -> String {
43 self.prefix.clone()
44 }
45
46 pub fn base(&self, profile: &str) -> String {
48 format!("{}-{profile}-base", self.prefix)
49 }
50
51 pub fn slot(&self, profile: &str, slot: usize) -> String {
53 format!("{}-{profile}-slot-{slot}", self.prefix)
54 }
55
56 pub fn pnpm(&self) -> String {
58 format!("{}-pnpm", self.prefix)
59 }
60
61 pub fn maven(&self) -> String {
63 format!("{}-maven", self.prefix)
64 }
65
66 pub fn slot_of(&self, profile: &str, volume: &str) -> Option<usize> {
68 volume
69 .strip_prefix(&format!("{}-{profile}-slot-", self.prefix))?
70 .parse()
71 .ok()
72 }
73
74 pub fn seed(&self, profile: &str) -> String {
76 format!("{}-{profile}-seed", self.prefix)
77 }
78
79 pub fn measure(&self, profile: &str) -> String {
81 format!("{}-{profile}-measure", self.prefix)
82 }
83
84 pub fn mirror(&self) -> String {
86 format!("{}-mirror", self.prefix)
87 }
88
89 pub fn build_cache(&self) -> String {
92 format!("{}-build-cache", self.prefix)
93 }
94}
95
96fn volume(source: String, target: &str, read_only: bool) -> Mount {
97 Mount {
98 target: Some(target.to_owned()),
99 source: Some(source),
100 typ: Some(MountType::VOLUME),
101 read_only: Some(read_only),
102 ..Default::default()
103 }
104}
105
106pub fn runner_mounts(names: &Names, profile: &str, slot: usize) -> Vec<Mount> {
108 vec![
109 volume(names.base(profile), "/base", true),
110 volume(names.slot(profile, slot), "/state", false),
111 volume(names.pnpm(), "/shared/pnpm", false),
112 volume(names.maven(), "/shared/maven", false),
113 ]
114}
115
116pub fn mirror_body(names: &Names, mirror: &config::Mirror, slice: &str) -> ContainerCreateBody {
119 let memory = i64::try_from(mirror.memory.0).unwrap_or(i64::MAX);
120 ContainerCreateBody {
121 image: Some(MIRROR_IMAGE.to_owned()),
122 env: Some(vec![
123 "REGISTRY_PROXY_REMOTEURL=https://registry-1.docker.io".to_owned(),
124 format!("REGISTRY_PROXY_TTL={}s", mirror.ttl.as_secs()),
125 "REGISTRY_STORAGE_DELETE_ENABLED=true".to_owned(),
127 ]),
128 labels: Some(HashMap::from([(
129 "runnerd.mirror".to_owned(),
130 String::new(),
131 )])),
132 host_config: Some(HostConfig {
133 cgroup_parent: Some(slice.to_owned()),
134 memory: Some(memory),
135 memory_swap: Some(memory),
136 nano_cpus: Some(i64::from(mirror.cpus) * 1_000_000_000),
137 oom_score_adj: Some(MIRROR_OOM_SCORE_ADJ),
140 mounts: Some(vec![volume(names.mirror(), "/var/lib/registry", false)]),
141 network_mode: Some(names.network()),
142 ..Default::default()
143 }),
144 networking_config: Some(NetworkingConfig {
145 endpoints_config: Some(HashMap::from([(
146 names.network(),
147 EndpointSettings {
148 aliases: Some(vec![MIRROR_ALIAS.to_owned()]),
149 ..Default::default()
150 },
151 )])),
152 }),
153 ..Default::default()
154 }
155}
156
157pub async fn prepare(
159 docker: &impl DockerApi,
160 names: &Names,
161 profile: &str,
162 slot: usize,
163) -> Result<(), Error> {
164 docker.ensure_network(&names.network()).await?;
165 for name in [
166 names.base(profile),
167 names.slot(profile, slot),
168 names.pnpm(),
169 names.maven(),
170 ] {
171 docker.ensure_volume(&name).await?;
172 }
173 Ok(())
174}
175
176pub async fn mirror_up(
178 docker: &impl DockerApi,
179 names: &Names,
180 mirror: &config::Mirror,
181 slice: &str,
182) -> Result<(), Error> {
183 match docker.inspect(&names.mirror()).await {
184 Ok(state) if state.running => return Ok(()),
185 Ok(_) => docker.remove(&names.mirror()).await?,
187 Err(error) if is_not_found(&error) => {}
188 Err(error) => return Err(error),
189 }
190 docker.ensure_network(&names.network()).await?;
191 if !docker.image_present(MIRROR_IMAGE).await? {
192 docker.pull(MIRROR_IMAGE).await?;
193 }
194 docker
195 .run(&names.mirror(), mirror_body(names, mirror, slice))
196 .await?;
197 Ok(())
198}
199
200pub async fn mirror_down(docker: &impl DockerApi, names: &Names) -> Result<(), Error> {
202 match docker.remove(&names.mirror()).await {
203 Err(error) if is_not_found(&error) => Ok(()),
204 other => other,
205 }
206}
207
208#[derive(Debug, Clone, Copy, PartialEq, Eq)]
210pub enum Mirror {
211 Up,
213 Down,
215}
216
217#[derive(Debug, Clone, PartialEq, Eq)]
219pub struct Trimmed {
220 pub sizes: BTreeMap<String, u64>,
222 pub emptied: Vec<Emptied>,
224 pub closed: Vec<Emptied>,
226 pub refused: Vec<Refused>,
229 pub free: Option<u64>,
231}
232
233#[derive(Debug, Clone, PartialEq, Eq)]
235pub struct Refused {
236 pub name: String,
238 pub error: String,
240}
241
242#[derive(Debug, Clone, PartialEq, Eq)]
244pub struct Emptied {
245 pub name: String,
247 pub size: ByteSize,
249}
250
251pub async fn trim_shared(
256 docker: &impl DockerApi,
257 names: &Names,
258 caches: &config::Caches,
259 mirror: Mirror,
260 slots: &BTreeMap<String, usize>,
261) -> Result<Trimmed, Error> {
262 let sizes = docker.volume_sizes().await?;
263 let free = docker.free_space().await.ok();
264 let mut emptied = Vec::new();
265 let mut refused = Vec::new();
266 let mut limits = vec![
267 (names.pnpm(), caches.pnpm.max_size),
268 (names.maven(), caches.maven.max_size),
269 ];
270 if mirror == Mirror::Down {
271 limits.push((names.mirror(), caches.mirror.max_size));
272 }
273 for (name, limit) in limits {
274 if let Some(&size) = sizes.get(&name)
275 && size > limit.0
276 {
277 if let Err(error) = docker.remove_volume(&name).await {
279 refused.push(Refused {
280 name,
281 error: error.to_string(),
282 });
283 continue;
284 }
285 emptied.push(Emptied {
286 name,
287 size: ByteSize(size),
288 });
289 }
290 }
291 let mut closed = Vec::new();
292 for (name, &size) in &sizes {
293 let past = slots.iter().any(|(profile, &count)| {
294 names
295 .slot_of(profile, name)
296 .is_some_and(|slot| slot > count)
297 });
298 if past {
299 if let Err(error) = docker.remove_volume(name).await {
301 refused.push(Refused {
302 name: name.clone(),
303 error: error.to_string(),
304 });
305 continue;
306 }
307 closed.push(Emptied {
308 name: name.clone(),
309 size: ByteSize(size),
310 });
311 }
312 }
313 Ok(Trimmed {
314 sizes,
315 emptied,
316 closed,
317 refused,
318 free,
319 })
320}
321
322pub fn docker_disk(
325 names: &Names,
326 config: &config::Config,
327 free: u64,
328 sizes: &BTreeMap<String, u64>,
329) -> u64 {
330 let held: u64 = config
331 .profiles
332 .iter()
333 .flat_map(|(profile, settings)| {
334 (1..=settings.slots as usize).map(move |slot| names.slot(profile, slot))
335 })
336 .filter_map(|name| sizes.get(&name))
337 .sum();
338 free.saturating_add(held) / DOCKER_DISK_SHARE
339}
340
341pub fn docker_budget_gib(disk: u64, slots: u64) -> u64 {
344 ((disk / slots.max(1)) >> 30).clamp(1, DOCKER_SLOT_MAX_GIB)
345}
346
347const DOCKER_DISK_SHARE: u64 = 4;
349
350const DOCKER_SLOT_MAX_GIB: u64 = 20;
352
353pub fn volumes(names: &Names, config: &config::Config) -> Vec<api::CacheVolume> {
355 let volume = |name: String, kind: api::CacheKind, limit: Option<u64>| api::CacheVolume {
356 name,
357 kind,
358 bytes: 0,
359 limit,
360 };
361 let mut volumes = Vec::new();
362 for (profile, settings) in &config.profiles {
363 volumes.push(volume(
364 names.base(profile),
365 api::CacheKind::Snapshot {
366 profile: profile.clone(),
367 },
368 Some(config.caches.snapshot_max.0),
369 ));
370 for slot in 1..=settings.slots as usize {
371 volumes.push(volume(
372 names.slot(profile, slot),
373 api::CacheKind::Slot {
374 profile: profile.clone(),
375 slot,
376 },
377 None,
378 ));
379 }
380 }
381 volumes.push(volume(
382 names.pnpm(),
383 api::CacheKind::Pnpm,
384 Some(config.caches.pnpm.max_size.0),
385 ));
386 volumes.push(volume(
387 names.maven(),
388 api::CacheKind::Maven,
389 Some(config.caches.maven.max_size.0),
390 ));
391 if config.caches.mirror.enabled {
392 volumes.push(volume(
393 names.mirror(),
394 api::CacheKind::Mirror,
395 Some(config.caches.mirror.max_size.0),
396 ));
397 }
398 if config.caches.build.enabled {
399 volumes.push(volume(
400 names.build_cache(),
401 api::CacheKind::Build,
402 Some(config.caches.build.max_size.0),
403 ));
404 }
405 volumes.sort_by(|a, b| a.name.cmp(&b.name));
406 volumes
407}
408
409#[derive(Debug)]
411pub struct Reseed<'profile> {
412 pub names: &'profile Names,
414 pub profile: &'profile str,
416 pub image: &'profile str,
418 pub slots: usize,
420 pub slice: &'profile str,
422 pub cpus: u32,
424 pub memory: ByteSize,
426 pub min_changes: ByteSize,
428 pub max_size: ByteSize,
430 pub isolation: &'profile dyn Isolation,
432}
433
434pub async fn reseed(docker: &impl DockerApi, reseed: &Reseed<'_>) -> Result<String, Error> {
439 let names = reseed.names;
440 let mut mounts = vec![volume(names.base(reseed.profile), "/base", false)];
441 mounts.extend((1..=reseed.slots).map(|slot| {
442 volume(
443 names.slot(reseed.profile, slot),
444 &format!("/workers/{slot}"),
445 false,
446 )
447 }));
448 for mount in &mounts {
449 if let Some(source) = &mount.source {
450 docker.ensure_volume(source).await?;
451 }
452 }
453 let name = names.seed(reseed.profile);
454 let spec = RunnerSpec {
455 name: name.clone(),
456 image: reseed.image.to_owned(),
457 profile: reseed.profile.to_owned(),
458 slice: reseed.slice.to_owned(),
459 cpus: reseed.cpus,
460 memory: reseed.memory,
461 env: Vec::new(),
462 mounts,
463 network: Some("none".to_owned()),
464 labels: BTreeMap::from([("runnerd.seed".to_owned(), String::new())]),
465 entrypoint: Some(vec![
466 "/usr/local/sbin/runnerd-seed".to_owned(),
467 reseed.min_changes.0.to_string(),
468 reseed.max_size.0.to_string(),
469 ]),
470 jit_config: None,
471 };
472 match docker.remove(&name).await {
474 Err(error) if !is_not_found(&error) => return Err(error),
475 _ => {}
476 }
477 let id = docker.start(&spec, reseed.isolation).await?;
478 finish_reseed(docker, &id).await
479}
480
481#[derive(Debug)]
484pub struct Measure<'profile> {
485 pub names: &'profile Names,
487 pub profile: &'profile str,
489 pub image: &'profile str,
491 pub slice: &'profile str,
493 pub isolation: &'profile dyn Isolation,
495}
496
497#[derive(Clone, Copy, Debug, PartialEq, Eq)]
499pub enum Measured {
500 Changed(u64),
502 Unfinished,
504}
505
506const MEASURE_MEMORY: ByteSize = ByteSize(256 << 20);
508
509pub async fn measure(docker: &impl DockerApi, measure: &Measure<'_>) -> Result<Measured, Error> {
513 let spec = measure_spec(measure);
514 match docker.remove(&spec.name).await {
515 Err(error) if !is_not_found(&error) => return Err(error),
516 _ => {}
517 }
518 let id = docker.start(&spec, measure.isolation).await?;
519 let output = finish_reseed(docker, &id).await?;
520 measured(&output).ok_or_else(|| Error::DockerContainerWaitError {
521 error: format!("measuring printed {:?}", output.trim()),
522 code: 0,
523 })
524}
525
526pub fn measure_spec(measure: &Measure<'_>) -> RunnerSpec {
528 let names = measure.names;
529 RunnerSpec {
530 name: names.measure(measure.profile),
531 image: measure.image.to_owned(),
532 profile: measure.profile.to_owned(),
533 slice: measure.slice.to_owned(),
534 cpus: 1,
535 memory: MEASURE_MEMORY,
536 env: Vec::new(),
537 mounts: vec![
538 volume(names.base(measure.profile), "/base", true),
539 volume(names.slot(measure.profile, 1), "/workers/1", true),
540 ],
541 network: Some("none".to_owned()),
542 labels: BTreeMap::from([("runnerd.seed".to_owned(), String::new())]),
544 entrypoint: Some(vec![
545 "/usr/local/sbin/runnerd-seed".to_owned(),
546 "measure".to_owned(),
547 ]),
548 jit_config: None,
549 }
550}
551
552fn measured(output: &str) -> Option<Measured> {
555 output.lines().rev().map(str::trim).find_map(|line| {
556 if line == "the last reseed did not finish" {
557 return Some(Measured::Unfinished);
558 }
559 line.strip_prefix("slot 1 changed ")?
560 .strip_suffix(" bytes")?
561 .parse()
562 .ok()
563 .map(Measured::Changed)
564 })
565}
566
567pub fn seeded(output: &str) -> bool {
570 !output.trim_start().starts_with("not seeded")
571}
572
573pub async fn finish_reseed(docker: &impl DockerApi, container: &str) -> Result<String, Error> {
576 let result = async {
577 let code = docker.wait(container).await?;
578 let output = docker.logs(container).await?;
579 if code == 0 {
580 Ok(output)
581 } else {
582 Err(Error::DockerContainerWaitError {
583 error: format!("seeding exited {code}: {}", output.trim()),
584 code,
585 })
586 }
587 }
588 .await;
589 let removed = docker.remove(container).await;
590 let output = result?;
591 removed?;
592 Ok(output)
593}
594
595#[cfg(test)]
596mod tests {
597 use std::path::Path;
598 use std::process::Command;
599
600 use super::docker_budget_gib;
601 use crate::testdir::TestDir;
602
603 #[allow(
605 clippy::unwrap_used,
606 reason = "a test helper: a script that does not run fails the test"
607 )]
608 fn with_layers(script: &str, args: &[&Path]) -> String {
609 let layers = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../image/layers.sh");
610 let output = Command::new("bash")
611 .arg("-euc")
612 .arg(format!(". \"$0\"; {script}"))
613 .arg(layers)
614 .args(args)
615 .output()
616 .unwrap();
617 assert!(
618 output.status.success(),
619 "the script exits 0: {}",
620 String::from_utf8_lossy(&output.stderr)
621 );
622 String::from_utf8(output.stdout).unwrap()
623 }
624
625 #[test]
626 #[allow(
627 clippy::unwrap_used,
628 reason = "a test: a file it cannot write fails it"
629 )]
630 fn no_credential_a_job_leaves_in_the_cache_layer_reaches_the_slots_next_job() {
631 let listed = with_layers("printf '%s\\n' \"${CACHE_SECRETS[@]}\"", &[]);
632 let secrets: Vec<&str> = listed.lines().collect();
633 for credential in [
634 "cargo/credentials.toml",
635 "gradle/gradle.properties",
636 "gradle/init.d",
637 "m2/settings.xml",
638 "m2/settings-security.xml",
639 "huggingface/token",
640 ] {
641 assert!(
642 secrets.contains(&credential),
643 "{credential} is in CACHE_SECRETS: {secrets:?}"
644 );
645 }
646 let dir = TestDir::new("cache-secrets");
647 let upper = dir.0.join("upper");
648 let write = |path: &str| {
649 let path = upper.join(path);
650 std::fs::create_dir_all(path.parent().unwrap()).unwrap();
651 std::fs::write(path, "token").unwrap();
652 };
653 for secret in &secrets {
655 write(&format!("files/{secret}"));
656 write(&format!("dirs/{secret}/held"));
657 }
658 let kept = [
659 "npm/_cacache/index",
660 "m2/wrapper/dists/maven.zip",
661 "gradle/caches/jars",
662 ];
663 for path in kept {
664 write(&format!("files/{path}"));
665 }
666 with_layers(
667 "drop_cache_secrets \"$1/files\"; drop_cache_secrets \"$1/dirs\"",
668 &[&upper],
669 );
670 for secret in &secrets {
671 for copy in ["files", "dirs"] {
672 let path = upper.join(copy).join(secret);
673 assert!(!path.exists(), "{} is gone", path.display());
674 }
675 }
676 for path in kept {
677 assert!(upper.join("files").join(path).exists(), "{path} stays");
678 }
679 }
680
681 #[test]
682 #[allow(
683 clippy::unwrap_used,
684 reason = "a test: a file it cannot write fails it"
685 )]
686 fn what_slot_1_changed_counts_its_layers_and_not_its_docker_engines_data() {
687 let dir = TestDir::new("layer-changes");
688 let state = dir.0.join("state");
689 let write = |path: &str, bytes: usize| {
690 let path = state.join(path);
691 std::fs::create_dir_all(path.parent().unwrap()).unwrap();
692 std::fs::write(path, vec![0u8; bytes]).unwrap();
693 };
694 let changes = || -> u64 {
695 with_layers("layer_changes \"$1\"", &[&state])
696 .trim()
697 .parse()
698 .unwrap()
699 };
700 write("cache/upper/npm/index", 1);
701 let before = changes();
702 write("docker/overlay2/layer", 1 << 20);
704 write("tools/work/scratch", 1 << 20);
706 assert_eq!(changes(), before);
707 write("tools/upper/node/bin", 100_000);
708 write("android/upper/ndk/lib", 200_000);
709 let after = changes();
710 assert!(
711 after >= before + 300_000 && after < before + 400_000,
712 "the tools and android layers count: {before} then {after}"
713 );
714 }
715
716 #[test]
717 fn a_slots_docker_budget_is_its_share_between_1_and_20_gib() {
718 assert_eq!(docker_budget_gib(100 << 30, 10), 10);
719 assert_eq!(
720 docker_budget_gib(1 << 40, 4),
721 20,
722 "a large disk still stops at 20 GiB"
723 );
724 assert_eq!(
725 docker_budget_gib(1 << 30, 13),
726 1,
727 "a full disk still keeps 1 GiB"
728 );
729 assert_eq!(docker_budget_gib(1 << 30, 0), 1);
730 }
731}