Skip to main content

runnerd_core/
cache.rs

1//! The warm caches runner containers share and keep between jobs: each profile's read-only snapshot, each slot's
2//! volume (its layers' changes and its Docker Engine's data), the shared pnpm store and Maven repository, and the
3//! Docker Hub mirror on the runners' network. Owns their names, mounts and lifecycle; it never decides when a
4//! runner starts, and it never touches a volume a running container uses.
5
6use std::collections::{BTreeMap, HashMap};
7
8use bollard::models::{
9    ContainerCreateBody, EndpointSettings, HostConfig, Mount, MountType, NetworkingConfig,
10};
11
12use crate::api;
13use crate::config::{self, ByteSize};
14use crate::docker::{DockerApi, Error, RunnerSpec, is_not_found};
15use crate::isolation::Isolation;
16
17/// The Docker Hub pull-through cache (`registry` 3.1.2, multi-platform index).
18pub const MIRROR_IMAGE: &str =
19    "registry:3.1.2@sha256:01cd312e67026f06f5d89c3e133a4a9adc7cb9eeb9b1194e0b2954dfd770efd2";
20
21/// The host name runner containers reach the mirror at (`image/daemon.json`).
22const MIRROR_ALIAS: &str = "mirror";
23
24/// The mirror's `oom_score_adj`: the kernel picks it after the runners' processes, which keep the default 0.
25const MIRROR_OOM_SCORE_ADJ: i64 = -500;
26
27/// The names of everything runnerd creates in Docker, under one prefix: `runnerd`, or another for tests.
28#[derive(Debug, Clone)]
29pub struct Names {
30    prefix: String,
31}
32
33impl Names {
34    /// Names under `prefix`.
35    pub fn new(prefix: &str) -> Names {
36        Names {
37            prefix: prefix.to_owned(),
38        }
39    }
40
41    /// The network runners and the mirror share.
42    pub fn network(&self) -> String {
43        self.prefix.clone()
44    }
45
46    /// The profile's read-only snapshot.
47    pub fn base(&self, profile: &str) -> String {
48        format!("{}-{profile}-base", self.prefix)
49    }
50
51    /// Slot `slot` (1-based) of the profile: its layers' changes and its Docker Engine's data.
52    pub fn slot(&self, profile: &str, slot: usize) -> String {
53        format!("{}-{profile}-slot-{slot}", self.prefix)
54    }
55
56    /// The pnpm store all slots share.
57    pub fn pnpm(&self) -> String {
58        format!("{}-pnpm", self.prefix)
59    }
60
61    /// The Maven repository all slots share.
62    pub fn maven(&self) -> String {
63        format!("{}-maven", self.prefix)
64    }
65
66    /// The slot (1-based) of the profile whose volume `volume` is, if it is one of its slots'.
67    pub fn slot_of(&self, profile: &str, volume: &str) -> Option<usize> {
68        volume
69            .strip_prefix(&format!("{}-{profile}-slot-", self.prefix))?
70            .parse()
71            .ok()
72    }
73
74    /// Whether `volume` is a slot's volume of any profile, configured now or not: `<prefix>-<profile>-slot-<N>`.
75    /// A prefix that is another's start, as a test's `runnerd-test-…` is `runnerd`'s, matches the other's too.
76    pub fn is_slot(&self, volume: &str) -> bool {
77        volume
78            .strip_prefix(&format!("{}-", self.prefix))
79            .and_then(|rest| rest.rsplit_once("-slot-"))
80            .is_some_and(|(profile, slot)| {
81                !profile.is_empty()
82                    && !slot.is_empty()
83                    && slot.bytes().all(|byte| byte.is_ascii_digit())
84            })
85    }
86
87    /// The prefix every volume of runnerd's starts with, with its dash.
88    pub fn volume_prefix(&self) -> String {
89        format!("{}-", self.prefix)
90    }
91
92    /// The container that starts a slot's layers over after a host crash, when its volume could not be emptied.
93    pub fn start_over(&self) -> String {
94        format!("{}-start-over", self.prefix)
95    }
96
97    /// The container that rebuilds the profile's snapshot.
98    pub fn seed(&self, profile: &str) -> String {
99        format!("{}-{profile}-seed", self.prefix)
100    }
101
102    /// The container that measures what the profile's slot 1 changed, before its upkeep (`measure`).
103    pub fn measure(&self, profile: &str) -> String {
104        format!("{}-{profile}-measure", self.prefix)
105    }
106
107    /// The mirror's container and its storage volume.
108    pub fn mirror(&self) -> String {
109        format!("{}-mirror", self.prefix)
110    }
111
112    /// What `GET /v1/caches` and `runnerctl caches clean` call the shared build cache: a directory of the daemon's,
113    /// not a Docker volume.
114    pub fn build_cache(&self) -> String {
115        format!("{}-build-cache", self.prefix)
116    }
117}
118
119fn volume(source: String, target: &str, read_only: bool) -> Mount {
120    Mount {
121        target: Some(target.to_owned()),
122        source: Some(source),
123        typ: Some(MountType::VOLUME),
124        read_only: Some(read_only),
125        ..Default::default()
126    }
127}
128
129/// What a runner container in slot `slot` (1-based) of the profile mounts (paths from `image/Dockerfile`).
130pub fn runner_mounts(names: &Names, profile: &str, slot: usize) -> Vec<Mount> {
131    vec![
132        volume(names.base(profile), "/base", true),
133        volume(names.slot(profile, slot), "/state", false),
134        volume(names.pnpm(), "/shared/pnpm", false),
135        volume(names.maven(), "/shared/maven", false),
136    ]
137}
138
139/// The mirror container: Docker Hub through a proxy registry, which refuses pushes, so a job can only make it
140/// fetch what Docker Hub serves. Content not pulled again within the TTL expires. No published port.
141pub fn mirror_body(names: &Names, mirror: &config::Mirror, slice: &str) -> ContainerCreateBody {
142    let memory = i64::try_from(mirror.memory.0).unwrap_or(i64::MAX);
143    ContainerCreateBody {
144        image: Some(MIRROR_IMAGE.to_owned()),
145        env: Some(vec![
146            "REGISTRY_PROXY_REMOTEURL=https://registry-1.docker.io".to_owned(),
147            format!("REGISTRY_PROXY_TTL={}s", mirror.ttl.as_secs()),
148            // Expiry deletes content.
149            "REGISTRY_STORAGE_DELETE_ENABLED=true".to_owned(),
150        ]),
151        labels: Some(HashMap::from([(
152            "runnerd.mirror".to_owned(),
153            String::new(),
154        )])),
155        host_config: Some(HostConfig {
156            cgroup_parent: Some(slice.to_owned()),
157            memory: Some(memory),
158            memory_swap: Some(memory),
159            nano_cpus: Some(i64::from(mirror.cpus) * 1_000_000_000),
160            // When the slice runs out of memory, which measured admission allows, the kernel kills a job's process
161            // before the mirror every runner pulls through.
162            oom_score_adj: Some(MIRROR_OOM_SCORE_ADJ),
163            mounts: Some(vec![volume(names.mirror(), "/var/lib/registry", false)]),
164            network_mode: Some(names.network()),
165            ..Default::default()
166        }),
167        networking_config: Some(NetworkingConfig {
168            endpoints_config: Some(HashMap::from([(
169                names.network(),
170                EndpointSettings {
171                    aliases: Some(vec![MIRROR_ALIAS.to_owned()]),
172                    ..Default::default()
173                },
174            )])),
175        }),
176        ..Default::default()
177    }
178}
179
180/// Creates what a profile's runner in `slot` (1-based) mounts and joins, if missing.
181pub async fn prepare(
182    docker: &impl DockerApi,
183    names: &Names,
184    profile: &str,
185    slot: usize,
186) -> Result<(), Error> {
187    docker.ensure_network(&names.network()).await?;
188    for name in [
189        names.base(profile),
190        names.slot(profile, slot),
191        names.pnpm(),
192        names.maven(),
193    ] {
194        docker.ensure_volume(&name).await?;
195    }
196    Ok(())
197}
198
199/// Starts the mirror unless it runs.
200pub async fn mirror_up(
201    docker: &impl DockerApi,
202    names: &Names,
203    mirror: &config::Mirror,
204    slice: &str,
205) -> Result<(), Error> {
206    match docker.inspect(&names.mirror()).await {
207        Ok(state) if state.running => return Ok(()),
208        // Stopped on its own: started again from scratch, its storage is the volume.
209        Ok(_) => docker.remove(&names.mirror()).await?,
210        Err(error) if is_not_found(&error) => {}
211        Err(error) => return Err(error),
212    }
213    docker.ensure_network(&names.network()).await?;
214    if !docker.image_present(MIRROR_IMAGE).await? {
215        docker.pull(MIRROR_IMAGE).await?;
216    }
217    docker
218        .run(&names.mirror(), mirror_body(names, mirror, slice))
219        .await?;
220    Ok(())
221}
222
223/// Stops the mirror; its storage stays.
224pub async fn mirror_down(docker: &impl DockerApi, names: &Names) -> Result<(), Error> {
225    match docker.remove(&names.mirror()).await {
226        Err(error) if is_not_found(&error) => Ok(()),
227        other => other,
228    }
229}
230
231/// Whether the mirror's container is down, so its storage can be removed.
232#[derive(Debug, Clone, Copy, PartialEq, Eq)]
233pub enum Mirror {
234    /// It runs, or may: an earlier daemon may have left it up.
235    Up,
236    /// It is stopped.
237    Down,
238}
239
240/// What `trim_shared` measured and emptied.
241#[derive(Debug, Clone, PartialEq, Eq)]
242pub struct Trimmed {
243    /// Every volume's size in bytes, by name, as measured before emptying any.
244    pub sizes: BTreeMap<String, u64>,
245    /// The shared caches it emptied.
246    pub emptied: Vec<Emptied>,
247    /// The slot volumes past their profile's slot count it removed.
248    pub closed: Vec<Emptied>,
249    /// The volumes it was to empty or remove that Docker refused to, as one a stopped container still uses: tried
250    /// again at the next check.
251    pub refused: Vec<Refused>,
252    /// Docker's free disk in bytes, as measured before emptying any; `None` when it could not be measured.
253    pub free: Option<u64>,
254}
255
256/// A volume `trim_shared` was to remove that Docker refused to.
257#[derive(Debug, Clone, PartialEq, Eq)]
258pub struct Refused {
259    /// The volume.
260    pub name: String,
261    /// What Docker said.
262    pub error: String,
263}
264
265/// A shared cache `trim_shared` emptied.
266#[derive(Debug, Clone, PartialEq, Eq)]
267pub struct Emptied {
268    /// The volume.
269    pub name: String,
270    /// How large it had grown.
271    pub size: ByteSize,
272}
273
274/// Empties the shared caches that outgrew their limit, the mirror's storage among them once the mirror is down
275/// (`Mirror::Down`), and removes the volumes of slots past the count `slots` gives their profile, as a lowered count
276/// leaves them: each holds a slot's layers and Docker store, which no job prunes again (docs/adr/0054). Call only
277/// while no runner runs, and leave out of `slots` a profile whose reseed runs, which mounts its slots' volumes.
278pub async fn trim_shared(
279    docker: &impl DockerApi,
280    names: &Names,
281    caches: &config::Caches,
282    mirror: Mirror,
283    slots: &BTreeMap<String, usize>,
284) -> Result<Trimmed, Error> {
285    let sizes = docker.volume_sizes().await?;
286    let free = docker.free_space().await.ok();
287    let mut emptied = Vec::new();
288    let mut refused = Vec::new();
289    let mut limits = vec![
290        (names.pnpm(), caches.pnpm.max_size),
291        (names.maven(), caches.maven.max_size),
292    ];
293    if mirror == Mirror::Down {
294        limits.push((names.mirror(), caches.mirror.max_size));
295    }
296    for (name, limit) in limits {
297        if let Some(&size) = sizes.get(&name)
298            && size > limit.0
299        {
300            // One Docker refuses stays to the next check; the others are still trimmed, and the sizes kept.
301            if let Err(error) = docker.remove_volume(&name).await {
302                refused.push(Refused {
303                    name,
304                    error: error.to_string(),
305                });
306                continue;
307            }
308            emptied.push(Emptied {
309                name,
310                size: ByteSize(size),
311            });
312        }
313    }
314    let mut closed = Vec::new();
315    for (name, &size) in &sizes {
316        let past = slots.iter().any(|(profile, &count)| {
317            names
318                .slot_of(profile, name)
319                .is_some_and(|slot| slot > count)
320        });
321        if past {
322            // As a stopped container kept as a failed job's record still uses it.
323            if let Err(error) = docker.remove_volume(name).await {
324                refused.push(Refused {
325                    name: name.clone(),
326                    error: error.to_string(),
327                });
328                continue;
329            }
330            closed.push(Emptied {
331                name: name.clone(),
332                size: ByteSize(size),
333            });
334        }
335    }
336    Ok(Trimmed {
337        sizes,
338        emptied,
339        closed,
340        refused,
341        free,
342    })
343}
344
345/// What the slots' Docker Engines may keep together, in bytes: a quarter of Docker's disk not otherwise used, which
346/// is what is free plus what the slot volumes already hold, measured together (docs/adr/0054).
347pub fn docker_disk(
348    names: &Names,
349    config: &config::Config,
350    free: u64,
351    sizes: &BTreeMap<String, u64>,
352) -> u64 {
353    let held: u64 = config
354        .profiles
355        .iter()
356        .flat_map(|(profile, settings)| {
357            (1..=settings.slots as usize).map(move |slot| names.slot(profile, slot))
358        })
359        .filter_map(|name| sizes.get(&name))
360        .sum();
361    free.saturating_add(held) / DOCKER_DISK_SHARE
362}
363
364/// One slot's share of `disk` among `slots`, in whole GiB as `image/layers.sh` takes it: at least 1, at most
365/// `DOCKER_SLOT_MAX_GIB`.
366pub fn docker_budget_gib(disk: u64, slots: u64) -> u64 {
367    ((disk / slots.max(1)) >> 30).clamp(1, DOCKER_SLOT_MAX_GIB)
368}
369
370/// The slots' Docker Engines together keep at most one part in this of Docker's disk not otherwise used.
371const DOCKER_DISK_SHARE: u64 = 4;
372
373/// The most one slot's Docker Engine keeps, in GiB: the image's own default, for a daemon that sets none.
374const DOCKER_SLOT_MAX_GIB: u64 = 20;
375
376/// What `empty_volumes` came to.
377#[derive(Debug, Default)]
378pub struct Cleared {
379    /// The volumes it removed.
380    pub removed: Vec<String>,
381    /// The volumes it left, each with why.
382    pub refused: Vec<Refusal>,
383}
384
385/// A volume `empty_volumes` left.
386#[derive(Debug)]
387pub struct Refusal {
388    /// The volume's name.
389    pub volume: String,
390    /// Why, as the log says it.
391    pub why: String,
392    /// What kept it.
393    pub cause: Kept,
394}
395
396/// What kept a volume `empty_volumes` did not remove.
397#[derive(Debug, Clone, Copy, PartialEq, Eq)]
398pub enum Kept {
399    /// A running container mounts it.
400    Runs,
401    /// A stopped container that is not runnerd's mounts it.
402    NotRunnerds,
403    /// Docker's error, such as an outage, a 5xx or a timeout.
404    Docker,
405}
406
407impl Refusal {
408    /// Whether a container mounts it that `empty_volumes` leaves: a running one, or one that is not runnerd's.
409    pub fn held(&self) -> bool {
410        self.cause != Kept::Docker
411    }
412}
413
414/// Removes each of `volumes` that exists, after the containers of runnerd's that mount it and no longer run, such as
415/// a runner or a seed a crash stopped: Docker refuses to remove a volume a container mounts, even a stopped one. It
416/// leaves a volume a running container mounts, or a container that is not runnerd's, and removes no such container.
417pub async fn empty_volumes(docker: &impl DockerApi, volumes: &[String]) -> Cleared {
418    let mut emptied = Cleared::default();
419    for volume in volumes {
420        match empty_volume(docker, volume).await {
421            Ok(()) => emptied.removed.push(volume.clone()),
422            Err(refusal) => emptied.refused.push(refusal),
423        }
424    }
425    emptied
426}
427
428/// Removes `volume`, if it exists, once nothing of runnerd's mounts it.
429async fn empty_volume(docker: &impl DockerApi, volume: &str) -> Result<(), Refusal> {
430    let refused = |why: String, cause: Kept| Refusal {
431        volume: volume.to_owned(),
432        why,
433        cause,
434    };
435    let failed = |error: Error| refused(error.to_string(), Kept::Docker);
436    for holder in docker.holders(volume).await.map_err(failed)? {
437        let name = if holder.name.is_empty() {
438            &holder.id
439        } else {
440            &holder.name
441        };
442        if holder.running {
443            return Err(refused(
444                format!("container {name} mounts it and runs"),
445                Kept::Runs,
446            ));
447        }
448        if !holder
449            .labels
450            .keys()
451            .any(|label| label.starts_with("runnerd."))
452        {
453            return Err(refused(
454                format!("container {name}, which is not runnerd's, mounts it"),
455                Kept::NotRunnerds,
456            ));
457        }
458        match docker.remove(&holder.id).await {
459            Err(error) if !is_not_found(&error) => {
460                return Err(refused(
461                    format!("container {name}, which mounts it, cannot be removed: {error}"),
462                    Kept::Docker,
463                ));
464            }
465            _ => {}
466        }
467    }
468    docker.remove_volume(volume).await.map_err(failed)
469}
470
471/// Every cache volume the config's profiles use, what it holds and its limit, with size 0.
472pub fn volumes(names: &Names, config: &config::Config) -> Vec<api::CacheVolume> {
473    let volume = |name: String, kind: api::CacheKind, limit: Option<u64>| api::CacheVolume {
474        name,
475        kind,
476        bytes: 0,
477        limit,
478    };
479    let mut volumes = Vec::new();
480    for (profile, settings) in &config.profiles {
481        volumes.push(volume(
482            names.base(profile),
483            api::CacheKind::Snapshot {
484                profile: profile.clone(),
485            },
486            Some(config.caches.snapshot_max.0),
487        ));
488        for slot in 1..=settings.slots as usize {
489            volumes.push(volume(
490                names.slot(profile, slot),
491                api::CacheKind::Slot {
492                    profile: profile.clone(),
493                    slot,
494                },
495                None,
496            ));
497        }
498    }
499    volumes.push(volume(
500        names.pnpm(),
501        api::CacheKind::Pnpm,
502        Some(config.caches.pnpm.max_size.0),
503    ));
504    volumes.push(volume(
505        names.maven(),
506        api::CacheKind::Maven,
507        Some(config.caches.maven.max_size.0),
508    ));
509    if config.caches.mirror.enabled {
510        volumes.push(volume(
511            names.mirror(),
512            api::CacheKind::Mirror,
513            Some(config.caches.mirror.max_size.0),
514        ));
515    }
516    if config.caches.build.enabled {
517        volumes.push(volume(
518            names.build_cache(),
519            api::CacheKind::Build,
520            Some(config.caches.build.max_size.0),
521        ));
522    }
523    volumes.sort_by(|a, b| a.name.cmp(&b.name));
524    volumes
525}
526
527/// What `reseed` runs on: the profile's image and limits, its slots and the isolation backend.
528#[derive(Debug)]
529pub struct Reseed<'profile> {
530    /// Where the profile's volumes are.
531    pub names: &'profile Names,
532    /// The profile to reseed.
533    pub profile: &'profile str,
534    /// The profile's runner image, which holds the seed script.
535    pub image: &'profile str,
536    /// How many slots the profile has; each slot's layer changes are dropped.
537    pub slots: usize,
538    /// The systemd slice the seed container runs in.
539    pub slice: &'profile str,
540    /// The profile's CPUs.
541    pub cpus: u32,
542    /// The profile's memory.
543    pub memory: ByteSize,
544    /// Slot 1's layers must have changed at least this much (`caches.reseed_at`).
545    pub min_changes: ByteSize,
546    /// A snapshot that would outgrow this starts over (`caches.snapshot_max`).
547    pub max_size: ByteSize,
548    /// How the seed container is isolated, as runners are.
549    pub isolation: &'profile dyn Isolation,
550}
551
552/// Makes slot 1's layers the profile's new snapshot and drops every slot's own layer changes (`image/seed.sh`),
553/// if slot 1 changed at least `min_changes`. A snapshot that would outgrow `max_size` starts over from the image,
554/// as every slot's changes go too (docs/adr/0065); after a reseed that did not finish, it starts over whatever the
555/// limits (docs/adr/0066). Call only while none of the profile's slots runs. Returns the seed container's output.
556pub async fn reseed(docker: &impl DockerApi, reseed: &Reseed<'_>) -> Result<String, Error> {
557    let names = reseed.names;
558    let mut mounts = vec![volume(names.base(reseed.profile), "/base", false)];
559    mounts.extend((1..=reseed.slots).map(|slot| {
560        volume(
561            names.slot(reseed.profile, slot),
562            &format!("/workers/{slot}"),
563            false,
564        )
565    }));
566    for mount in &mounts {
567        if let Some(source) = &mount.source {
568            docker.ensure_volume(source).await?;
569        }
570    }
571    let name = names.seed(reseed.profile);
572    let spec = RunnerSpec {
573        name: name.clone(),
574        image: reseed.image.to_owned(),
575        profile: reseed.profile.to_owned(),
576        slice: reseed.slice.to_owned(),
577        cpus: reseed.cpus,
578        memory: reseed.memory,
579        env: Vec::new(),
580        mounts,
581        network: Some("none".to_owned()),
582        labels: BTreeMap::from([("runnerd.seed".to_owned(), String::new())]),
583        entrypoint: Some(vec![
584            "/usr/local/sbin/runnerd-seed".to_owned(),
585            reseed.min_changes.0.to_string(),
586            reseed.max_size.0.to_string(),
587        ]),
588        jit_config: None,
589    };
590    // A seed container left by a daemon that died while seeding.
591    match docker.remove(&name).await {
592        Err(error) if !is_not_found(&error) => return Err(error),
593        _ => {}
594    }
595    let id = docker.start(&spec, reseed.isolation).await?;
596    finish_reseed(docker, &id).await
597}
598
599/// What `start_layers_over` runs on: a slot's volume a host crash may have torn and the guard could not empty, and
600/// the image, slice and isolation of a runner.
601#[derive(Debug)]
602pub struct StartOver<'volume> {
603    /// Where runnerd's volumes are.
604    pub names: &'volume Names,
605    /// The slot's volume.
606    pub volume: &'volume str,
607    /// A runner image, for its shell: any profile's will do.
608    pub image: &'volume str,
609    /// The systemd slice the container runs in.
610    pub slice: &'volume str,
611    /// How the container is isolated, as runners are.
612    pub isolation: &'volume dyn Isolation,
613}
614
615/// The memory the container of `start_layers_over` may use: it only deletes.
616const START_OVER_MEMORY: ByteSize = ByteSize(256 << 20);
617
618/// Deletes everything in a slot's volume but its Docker Engine's store (`image/layers.sh`'s `DOCKER_DATA_ROOT`), in
619/// a container that mounts it: its layers' changes and work directories, and the record of the snapshot they were
620/// made against. The slot's next runner starts them over from the snapshot, and a reseed finds nothing of slot 1's
621/// to copy (ADR 0142). For a volume the guard could not remove, as another container holds it: Docker lets a second
622/// container mount it. A container an earlier daemon left is removed first.
623pub async fn start_layers_over(
624    docker: &impl DockerApi,
625    start: &StartOver<'_>,
626) -> Result<(), Error> {
627    let name = start.names.start_over();
628    match docker.remove(&name).await {
629        Err(error) if !is_not_found(&error) => return Err(error),
630        _ => {}
631    }
632    let spec = RunnerSpec {
633        name: name.clone(),
634        image: start.image.to_owned(),
635        profile: String::new(),
636        slice: start.slice.to_owned(),
637        cpus: 1,
638        memory: START_OVER_MEMORY,
639        env: Vec::new(),
640        mounts: vec![volume(start.volume.to_owned(), "/state", false)],
641        network: Some("none".to_owned()),
642        labels: BTreeMap::from([("runnerd.seed".to_owned(), String::new())]),
643        entrypoint: Some(
644            [
645                "find",
646                "/state",
647                "-mindepth",
648                "1",
649                "-maxdepth",
650                "1",
651                "!",
652                "-name",
653                "docker",
654                "-exec",
655                "rm",
656                "-rf",
657                "{}",
658                "+",
659            ]
660            .map(str::to_owned)
661            .to_vec(),
662        ),
663        jit_config: None,
664    };
665    let id = docker.start(&spec, start.isolation).await?;
666    finish_reseed(docker, &id).await.map(|_| ())
667}
668
669/// What a measure container (`image/seed.sh measure`) runs on: the profile's image, its slot 1 and the snapshot,
670/// both read-only, and the isolation backend.
671#[derive(Debug)]
672pub struct Measure<'profile> {
673    /// Where the profile's volumes are.
674    pub names: &'profile Names,
675    /// The profile whose slot 1 is measured.
676    pub profile: &'profile str,
677    /// The profile's runner image, which holds the seed script.
678    pub image: &'profile str,
679    /// The systemd slice the measure container runs in.
680    pub slice: &'profile str,
681    /// How the measure container is isolated, as runners are.
682    pub isolation: &'profile dyn Isolation,
683}
684
685/// What a measure found: what a reseed would weigh against `caches.reseed_at`.
686#[derive(Clone, Copy, Debug, PartialEq, Eq)]
687pub enum Measured {
688    /// Slot 1's changes to the layers hold this many bytes, its Docker Engine's data aside.
689    Changed(u64),
690    /// The profile's last reseed did not finish (docs/adr/0066): the next one starts over, whatever slot 1 changed.
691    Unfinished,
692}
693
694/// The memory a measure container may use: it only adds up file sizes.
695const MEASURE_MEMORY: ByteSize = ByteSize(256 << 20);
696
697/// Measures what the profile's slot 1 changed (`image/seed.sh measure`), in a container that mounts slot 1's
698/// volume and the snapshot read-only, so it may run while the profile's runners do. A measure container an earlier
699/// daemon left is removed first.
700pub async fn measure(docker: &impl DockerApi, measure: &Measure<'_>) -> Result<Measured, Error> {
701    let spec = measure_spec(measure);
702    match docker.remove(&spec.name).await {
703        Err(error) if !is_not_found(&error) => return Err(error),
704        _ => {}
705    }
706    let id = docker.start(&spec, measure.isolation).await?;
707    let output = finish_reseed(docker, &id).await?;
708    measured(&output).ok_or_else(|| Error::DockerContainerWaitError {
709        error: format!("measuring printed {:?}", output.trim()),
710        code: 0,
711    })
712}
713
714/// The container `measure` runs: `seed.sh measure` on slot 1's volume and the snapshot, both read-only.
715pub fn measure_spec(measure: &Measure<'_>) -> RunnerSpec {
716    let names = measure.names;
717    RunnerSpec {
718        name: names.measure(measure.profile),
719        image: measure.image.to_owned(),
720        profile: measure.profile.to_owned(),
721        slice: measure.slice.to_owned(),
722        cpus: 1,
723        memory: MEASURE_MEMORY,
724        env: Vec::new(),
725        mounts: vec![
726            volume(names.base(measure.profile), "/base", true),
727            volume(names.slot(measure.profile, 1), "/workers/1", true),
728        ],
729        network: Some("none".to_owned()),
730        // As a seed container's: a restart finds it beside them (`seed`), and removes it.
731        labels: BTreeMap::from([("runnerd.seed".to_owned(), String::new())]),
732        entrypoint: Some(vec![
733            "/usr/local/sbin/runnerd-seed".to_owned(),
734            "measure".to_owned(),
735        ]),
736        jit_config: None,
737    }
738}
739
740/// What the output of `image/seed.sh measure` says, in its last line that says it: a warning before it, such as
741/// one of `du` about a file a job deleted while it counted, leaves the number as advisory as it is anyway.
742fn measured(output: &str) -> Option<Measured> {
743    output.lines().rev().map(str::trim).find_map(|line| {
744        if line == "the last reseed did not finish" {
745            return Some(Measured::Unfinished);
746        }
747        line.strip_prefix("slot 1 changed ")?
748            .strip_suffix(" bytes")?
749            .parse()
750            .ok()
751            .map(Measured::Changed)
752    })
753}
754
755/// Whether the output of a reseed (`image/seed.sh`) says it replaced the snapshot, and so dropped every slot's own
756/// layer changes, rather than left them because slot 1 changed too little.
757pub fn seeded(output: &str) -> bool {
758    !output.trim_start().starts_with("not seeded")
759}
760
761/// Waits for a seed or measure container to exit, removes it, and returns its output. Also takes over a seed
762/// container an earlier daemon left running: stopping it between its `rm` and `mv` would lose that snapshot layer.
763pub async fn finish_reseed(docker: &impl DockerApi, container: &str) -> Result<String, Error> {
764    let result = async {
765        let code = docker.wait(container).await?;
766        let output = docker.logs(container).await?;
767        if code == 0 {
768            Ok(output)
769        } else {
770            Err(Error::DockerContainerWaitError {
771                error: format!("seeding exited {code}: {}", output.trim()),
772                code,
773            })
774        }
775    }
776    .await;
777    let removed = docker.remove(container).await;
778    let output = result?;
779    removed?;
780    Ok(output)
781}
782
783#[cfg(test)]
784mod tests {
785    use std::path::Path;
786    use std::process::Command;
787
788    use super::docker_budget_gib;
789    use crate::testdir::TestDir;
790
791    /// Runs `script` in bash with `image/layers.sh` sourced and `args` as its arguments; returns what it printed.
792    #[allow(
793        clippy::unwrap_used,
794        reason = "a test helper: a script that does not run fails the test"
795    )]
796    fn with_layers(script: &str, args: &[&Path]) -> String {
797        let layers = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../image/layers.sh");
798        let output = Command::new("bash")
799            .arg("-euc")
800            .arg(format!(". \"$0\"; {script}"))
801            .arg(layers)
802            .args(args)
803            .output()
804            .unwrap();
805        assert!(
806            output.status.success(),
807            "the script exits 0: {}",
808            String::from_utf8_lossy(&output.stderr)
809        );
810        String::from_utf8(output.stdout).unwrap()
811    }
812
813    #[test]
814    #[allow(
815        clippy::unwrap_used,
816        reason = "a test: a file it cannot write fails it"
817    )]
818    fn no_credential_a_job_leaves_in_the_cache_layer_reaches_the_slots_next_job() {
819        let listed = with_layers("printf '%s\\n' \"${CACHE_SECRETS[@]}\"", &[]);
820        let secrets: Vec<&str> = listed.lines().collect();
821        for credential in [
822            "cargo/credentials.toml",
823            "gradle/gradle.properties",
824            "gradle/init.d",
825            "m2/settings.xml",
826            "m2/settings-security.xml",
827            "huggingface/token",
828        ] {
829            assert!(
830                secrets.contains(&credential),
831                "{credential} is in CACHE_SECRETS: {secrets:?}"
832            );
833        }
834        let dir = TestDir::new("cache-secrets");
835        let upper = dir.0.join("upper");
836        let write = |path: &str| {
837            let path = upper.join(path);
838            std::fs::create_dir_all(path.parent().unwrap()).unwrap();
839            std::fs::write(path, "token").unwrap();
840        };
841        // Each as a file, and as a directory a job made there, as Gradle's init.d is.
842        for secret in &secrets {
843            write(&format!("files/{secret}"));
844            write(&format!("dirs/{secret}/held"));
845        }
846        let kept = [
847            "npm/_cacache/index",
848            "m2/wrapper/dists/maven.zip",
849            "gradle/caches/jars",
850        ];
851        for path in kept {
852            write(&format!("files/{path}"));
853        }
854        with_layers(
855            "drop_cache_secrets \"$1/files\"; drop_cache_secrets \"$1/dirs\"",
856            &[&upper],
857        );
858        for secret in &secrets {
859            for copy in ["files", "dirs"] {
860                let path = upper.join(copy).join(secret);
861                assert!(!path.exists(), "{} is gone", path.display());
862            }
863        }
864        for path in kept {
865            assert!(upper.join("files").join(path).exists(), "{path} stays");
866        }
867    }
868
869    #[test]
870    #[allow(
871        clippy::unwrap_used,
872        reason = "a test: a file it cannot write fails it"
873    )]
874    fn what_slot_1_changed_counts_its_layers_and_not_its_docker_engines_data() {
875        let dir = TestDir::new("layer-changes");
876        let state = dir.0.join("state");
877        let write = |path: &str, bytes: usize| {
878            let path = state.join(path);
879            std::fs::create_dir_all(path.parent().unwrap()).unwrap();
880            std::fs::write(path, vec![0u8; bytes]).unwrap();
881        };
882        let changes = || -> u64 {
883            with_layers("layer_changes \"$1\"", &[&state])
884                .trim()
885                .parse()
886                .unwrap()
887        };
888        write("cache/upper/npm/index", 1);
889        let before = changes();
890        // A job's images in the slot's Docker Engine: a reseed leaves them, so they must not reach `reseed_at`.
891        write("docker/overlay2/layer", 1 << 20);
892        // Layer work directories hold nothing a reseed copies.
893        write("tools/work/scratch", 1 << 20);
894        assert_eq!(changes(), before);
895        write("tools/upper/node/bin", 100_000);
896        write("android/upper/ndk/lib", 200_000);
897        let after = changes();
898        assert!(
899            after >= before + 300_000 && after < before + 400_000,
900            "the tools and android layers count: {before} then {after}"
901        );
902    }
903
904    #[test]
905    fn a_slots_docker_budget_is_its_share_between_1_and_20_gib() {
906        assert_eq!(docker_budget_gib(100 << 30, 10), 10);
907        assert_eq!(
908            docker_budget_gib(1 << 40, 4),
909            20,
910            "a large disk still stops at 20 GiB"
911        );
912        assert_eq!(
913            docker_budget_gib(1 << 30, 13),
914            1,
915            "a full disk still keeps 1 GiB"
916        );
917        assert_eq!(docker_budget_gib(1 << 30, 0), 1);
918    }
919}