Skip to main content

runnerd_core/
cache.rs

1//! The warm caches runner containers share and keep between jobs: each profile's read-only snapshot, each slot's
2//! volume (its layers' changes and its Docker Engine's data), the shared pnpm store and Maven repository, and the
3//! Docker Hub mirror on the runners' network. Owns their names, mounts and lifecycle; it never decides when a
4//! runner starts, and it never touches a volume a running container uses.
5
6use std::collections::{BTreeMap, HashMap};
7
8use bollard::models::{
9    ContainerCreateBody, EndpointSettings, HostConfig, Mount, MountType, NetworkingConfig,
10};
11
12use crate::api;
13use crate::config::{self, ByteSize};
14use crate::docker::{DockerApi, Error, RunnerSpec, is_not_found};
15use crate::isolation::Isolation;
16
17pub mod mirror;
18
19/// The prefix of the names of what the daemon makes in Docker (`Names`).
20pub const PREFIX: &str = "runnerd";
21
22/// The Docker Hub pull-through cache (`registry` 3.1.2, multi-platform index).
23pub const MIRROR_IMAGE: &str =
24    "registry:3.1.2@sha256:01cd312e67026f06f5d89c3e133a4a9adc7cb9eeb9b1194e0b2954dfd770efd2";
25
26/// The host name runner containers reach the mirror at (`image/daemon.json`).
27const MIRROR_ALIAS: &str = "mirror";
28
29/// The mirror's `oom_score_adj`: the kernel picks it after the runners' processes, which keep the default 0.
30const MIRROR_OOM_SCORE_ADJ: i64 = -500;
31
32/// The names of everything runnerd creates in Docker, under one prefix: `runnerd`, or another for tests.
33#[derive(Debug, Clone)]
34pub struct Names {
35    prefix: String,
36}
37
38impl Names {
39    /// Names under `prefix`.
40    pub fn new(prefix: &str) -> Names {
41        Names {
42            prefix: prefix.to_owned(),
43        }
44    }
45
46    /// The network runners and the mirror share.
47    pub fn network(&self) -> String {
48        self.prefix.clone()
49    }
50
51    /// The profile's read-only snapshot.
52    pub fn base(&self, profile: &str) -> String {
53        format!("{}-{profile}-base", self.prefix)
54    }
55
56    /// Slot `slot` (1-based) of the profile: its layers' changes and its Docker Engine's data.
57    pub fn slot(&self, profile: &str, slot: usize) -> String {
58        format!("{}-{profile}-slot-{slot}", self.prefix)
59    }
60
61    /// The pnpm store all slots share.
62    pub fn pnpm(&self) -> String {
63        format!("{}-pnpm", self.prefix)
64    }
65
66    /// The Maven repository all slots share.
67    pub fn maven(&self) -> String {
68        format!("{}-maven", self.prefix)
69    }
70
71    /// The slot (1-based) of the profile whose volume `volume` is, if it is one of its slots'.
72    pub fn slot_of(&self, profile: &str, volume: &str) -> Option<usize> {
73        volume
74            .strip_prefix(&format!("{}-{profile}-slot-", self.prefix))?
75            .parse()
76            .ok()
77    }
78
79    /// Whether `volume` is a slot's volume of any profile, configured now or not: `<prefix>-<profile>-slot-<N>`.
80    /// A prefix that is another's start, as a test's `runnerd-test-…` is `runnerd`'s, matches the other's too.
81    pub fn is_slot(&self, volume: &str) -> bool {
82        volume
83            .strip_prefix(&format!("{}-", self.prefix))
84            .and_then(|rest| rest.rsplit_once("-slot-"))
85            .is_some_and(|(profile, slot)| {
86                !profile.is_empty()
87                    && !slot.is_empty()
88                    && slot.bytes().all(|byte| byte.is_ascii_digit())
89            })
90    }
91
92    /// The prefix every volume of runnerd's starts with, with its dash.
93    pub fn volume_prefix(&self) -> String {
94        format!("{}-", self.prefix)
95    }
96
97    /// The container that starts a slot's layers over after a host crash, when its volume could not be emptied.
98    pub fn start_over(&self) -> String {
99        format!("{}-start-over", self.prefix)
100    }
101
102    /// The container that rebuilds the profile's snapshot.
103    pub fn seed(&self, profile: &str) -> String {
104        format!("{}-{profile}-seed", self.prefix)
105    }
106
107    /// The container that measures what the profile's slot 1 changed, before its upkeep (`measure`).
108    pub fn measure(&self, profile: &str) -> String {
109        format!("{}-{profile}-measure", self.prefix)
110    }
111
112    /// The mirror's container and its storage volume.
113    pub fn mirror(&self) -> String {
114        format!("{}-mirror", self.prefix)
115    }
116
117    /// The container that repairs the mirror's storage before the mirror starts.
118    pub fn mirror_repair(&self) -> String {
119        format!("{}-mirror-repair", self.prefix)
120    }
121
122    /// What `GET /v1/caches` and `runnerctl caches clean` call the shared build cache: a directory of the daemon's,
123    /// not a Docker volume.
124    pub fn build_cache(&self) -> String {
125        format!("{}-build-cache", self.prefix)
126    }
127}
128
129fn volume(source: String, target: &str, read_only: bool) -> Mount {
130    Mount {
131        target: Some(target.to_owned()),
132        source: Some(source),
133        typ: Some(MountType::VOLUME),
134        read_only: Some(read_only),
135        ..Default::default()
136    }
137}
138
139/// What a runner container in slot `slot` (1-based) of the profile mounts (paths from `image/Dockerfile`).
140pub fn runner_mounts(names: &Names, profile: &str, slot: usize) -> Vec<Mount> {
141    vec![
142        volume(names.base(profile), "/base", true),
143        volume(names.slot(profile, slot), "/state", false),
144        volume(names.pnpm(), "/shared/pnpm", false),
145        volume(names.maven(), "/shared/maven", false),
146    ]
147}
148
149/// The mirror container: Docker Hub through a proxy registry, which refuses pushes, so a job can only make it
150/// fetch what Docker Hub serves. Content not pulled again within the TTL expires. No published port. It keeps no
151/// blob descriptor cache: the proxy's expiry of one repository's copy of a blob deletes the blob other repositories
152/// still link to, and the cache would go on answering their pulls with no data (docs/adr/0144).
153pub fn mirror_body(names: &Names, mirror: &config::Mirror, slice: &str) -> ContainerCreateBody {
154    let memory = i64::try_from(mirror.memory.0).unwrap_or(i64::MAX);
155    ContainerCreateBody {
156        image: Some(MIRROR_IMAGE.to_owned()),
157        env: Some(vec![
158            "REGISTRY_PROXY_REMOTEURL=https://registry-1.docker.io".to_owned(),
159            format!("REGISTRY_PROXY_TTL={}s", mirror.ttl.as_secs()),
160            // Expiry deletes content.
161            "REGISTRY_STORAGE_DELETE_ENABLED=true".to_owned(),
162            format!("{}=", mirror::DESCRIPTOR_CACHE_ENV),
163        ]),
164        labels: Some(HashMap::from([(
165            "runnerd.mirror".to_owned(),
166            String::new(),
167        )])),
168        host_config: Some(HostConfig {
169            cgroup_parent: Some(slice.to_owned()),
170            memory: Some(memory),
171            memory_swap: Some(memory),
172            nano_cpus: Some(i64::from(mirror.cpus) * 1_000_000_000),
173            // When the slice runs out of memory, which measured admission allows, the kernel kills a job's process
174            // before the mirror every runner pulls through.
175            oom_score_adj: Some(MIRROR_OOM_SCORE_ADJ),
176            mounts: Some(vec![volume(names.mirror(), "/var/lib/registry", false)]),
177            network_mode: Some(names.network()),
178            ..Default::default()
179        }),
180        networking_config: Some(NetworkingConfig {
181            endpoints_config: Some(HashMap::from([(
182                names.network(),
183                EndpointSettings {
184                    aliases: Some(vec![MIRROR_ALIAS.to_owned()]),
185                    ..Default::default()
186                },
187            )])),
188        }),
189        ..Default::default()
190    }
191}
192
193/// Creates what a profile's runner in `slot` (1-based) mounts and joins, if missing.
194pub async fn prepare(
195    docker: &impl DockerApi,
196    names: &Names,
197    profile: &str,
198    slot: usize,
199) -> Result<(), Error> {
200    docker.ensure_network(&names.network()).await?;
201    for name in [
202        names.base(profile),
203        names.slot(profile, slot),
204        names.pnpm(),
205        names.maven(),
206    ] {
207        docker.ensure_volume(&name).await?;
208    }
209    Ok(())
210}
211
212/// Starts the mirror unless it runs, after removing the links in its storage whose blob is gone (`mirror::Repair`),
213/// which a failed repair does not hold back.
214pub async fn mirror_up(
215    docker: &impl DockerApi,
216    names: &Names,
217    mirror: &config::Mirror,
218    slice: &str,
219) -> Result<mirror::MirrorUp, Error> {
220    match docker.inspect(&names.mirror()).await {
221        Ok(state) if state.running => return Ok(mirror::MirrorUp::Running),
222        // Stopped on its own: started again from scratch, its storage is the volume.
223        Ok(_) => docker.remove(&names.mirror()).await?,
224        Err(error) if is_not_found(&error) => {}
225        Err(error) => return Err(error),
226    }
227    docker.ensure_network(&names.network()).await?;
228    if !docker.image_present(MIRROR_IMAGE).await? {
229        docker.pull(MIRROR_IMAGE).await?;
230    }
231    let repair = mirror::repair(docker, names, slice).await;
232    docker
233        .run(&names.mirror(), mirror_body(names, mirror, slice))
234        .await?;
235    Ok(mirror::MirrorUp::Started(repair))
236}
237
238/// Stops the mirror, and removes a repair container a killed daemon left on its storage, which would keep the
239/// volume from being removed; its storage stays.
240pub async fn mirror_down(docker: &impl DockerApi, names: &Names) -> Result<(), Error> {
241    for name in [names.mirror(), names.mirror_repair()] {
242        match docker.remove(&name).await {
243            Err(error) if !is_not_found(&error) => return Err(error),
244            _ => {}
245        }
246    }
247    Ok(())
248}
249
250/// Whether the mirror's container is down, so its storage can be removed.
251#[derive(Debug, Clone, Copy, PartialEq, Eq)]
252pub enum Mirror {
253    /// It runs, or may: an earlier daemon may have left it up.
254    Up,
255    /// It is stopped.
256    Down,
257}
258
259/// What `trim_shared` measured and emptied.
260#[derive(Debug, Clone, PartialEq, Eq)]
261pub struct Trimmed {
262    /// Every volume's size in bytes, by name, as measured before emptying any.
263    pub sizes: BTreeMap<String, u64>,
264    /// The shared caches it emptied.
265    pub emptied: Vec<Emptied>,
266    /// The slot volumes past their profile's slot count it removed.
267    pub closed: Vec<Emptied>,
268    /// The volumes it was to empty or remove that Docker refused to, as one a stopped container still uses: tried
269    /// again at the next check.
270    pub refused: Vec<Refused>,
271    /// Docker's free disk in bytes, as measured before emptying any; `None` when it could not be measured.
272    pub free: Option<u64>,
273}
274
275/// A volume `trim_shared` was to remove that Docker refused to.
276#[derive(Debug, Clone, PartialEq, Eq)]
277pub struct Refused {
278    /// The volume.
279    pub name: String,
280    /// What Docker said.
281    pub error: String,
282}
283
284/// A shared cache `trim_shared` emptied.
285#[derive(Debug, Clone, PartialEq, Eq)]
286pub struct Emptied {
287    /// The volume.
288    pub name: String,
289    /// How large it had grown.
290    pub size: ByteSize,
291}
292
293/// Empties the shared caches that outgrew their limit, the mirror's storage among them once the mirror is down
294/// (`Mirror::Down`), and removes the volumes of slots past the count `slots` gives their profile, as a lowered count
295/// leaves them: each holds a slot's layers and Docker store, which no job prunes again (docs/adr/0054). Call only
296/// while no runner runs, and leave out of `slots` a profile whose reseed runs, which mounts its slots' volumes.
297pub async fn trim_shared(
298    docker: &impl DockerApi,
299    names: &Names,
300    caches: &config::Caches,
301    mirror: Mirror,
302    slots: &BTreeMap<String, usize>,
303) -> Result<Trimmed, Error> {
304    let sizes = docker.volume_sizes().await?;
305    let free = docker.free_space().await.ok();
306    let mut emptied = Vec::new();
307    let mut refused = Vec::new();
308    let mut limits = vec![
309        (names.pnpm(), caches.pnpm.max_size),
310        (names.maven(), caches.maven.max_size),
311    ];
312    if mirror == Mirror::Down {
313        limits.push((names.mirror(), caches.mirror.max_size));
314    }
315    for (name, limit) in limits {
316        if let Some(&size) = sizes.get(&name)
317            && size > limit.0
318        {
319            // One Docker refuses stays to the next check; the others are still trimmed, and the sizes kept.
320            if let Err(error) = docker.remove_volume(&name).await {
321                refused.push(Refused {
322                    name,
323                    error: error.to_string(),
324                });
325                continue;
326            }
327            emptied.push(Emptied {
328                name,
329                size: ByteSize(size),
330            });
331        }
332    }
333    let mut closed = Vec::new();
334    for (name, &size) in &sizes {
335        let past = slots.iter().any(|(profile, &count)| {
336            names
337                .slot_of(profile, name)
338                .is_some_and(|slot| slot > count)
339        });
340        if past {
341            // As a stopped container kept as a failed job's record still uses it.
342            if let Err(error) = docker.remove_volume(name).await {
343                refused.push(Refused {
344                    name: name.clone(),
345                    error: error.to_string(),
346                });
347                continue;
348            }
349            closed.push(Emptied {
350                name: name.clone(),
351                size: ByteSize(size),
352            });
353        }
354    }
355    Ok(Trimmed {
356        sizes,
357        emptied,
358        closed,
359        refused,
360        free,
361    })
362}
363
364/// What the slots' Docker Engines may keep together, in bytes: a quarter of Docker's disk not otherwise used, which
365/// is what is free plus what the slot volumes already hold, measured together (docs/adr/0054).
366pub fn docker_disk(
367    names: &Names,
368    config: &config::Config,
369    free: u64,
370    sizes: &BTreeMap<String, u64>,
371) -> u64 {
372    let held: u64 = config
373        .profiles
374        .iter()
375        .flat_map(|(profile, settings)| {
376            (1..=settings.slots as usize).map(move |slot| names.slot(profile, slot))
377        })
378        .filter_map(|name| sizes.get(&name))
379        .sum();
380    free.saturating_add(held) / DOCKER_DISK_SHARE
381}
382
383/// One slot's share of `disk` among `slots`, in whole GiB as `image/layers.sh` takes it: at least 1, at most
384/// `DOCKER_SLOT_MAX_GIB`.
385pub fn docker_budget_gib(disk: u64, slots: u64) -> u64 {
386    ((disk / slots.max(1)) >> 30).clamp(1, DOCKER_SLOT_MAX_GIB)
387}
388
389/// The slots' Docker Engines together keep at most one part in this of Docker's disk not otherwise used.
390const DOCKER_DISK_SHARE: u64 = 4;
391
392/// The most one slot's Docker Engine keeps, in GiB: the image's own default, for a daemon that sets none.
393const DOCKER_SLOT_MAX_GIB: u64 = 20;
394
395/// What `empty_volumes` came to.
396#[derive(Debug, Default)]
397pub struct Cleared {
398    /// The volumes it removed.
399    pub removed: Vec<String>,
400    /// The volumes it left, each with why.
401    pub refused: Vec<Refusal>,
402}
403
404/// A volume `empty_volumes` left.
405#[derive(Debug)]
406pub struct Refusal {
407    /// The volume's name.
408    pub volume: String,
409    /// Why, as the log says it.
410    pub why: String,
411    /// What kept it.
412    pub cause: Kept,
413}
414
415/// What kept a volume `empty_volumes` did not remove.
416#[derive(Debug, Clone, Copy, PartialEq, Eq)]
417pub enum Kept {
418    /// A running container mounts it.
419    Runs,
420    /// A stopped container that is not runnerd's mounts it.
421    NotRunnerds,
422    /// Docker's error, such as an outage, a 5xx or a timeout.
423    Docker,
424}
425
426impl Refusal {
427    /// Whether a container mounts it that `empty_volumes` leaves: a running one, or one that is not runnerd's.
428    pub fn held(&self) -> bool {
429        self.cause != Kept::Docker
430    }
431}
432
433/// Removes each of `volumes` that exists, after the containers of runnerd's that mount it and no longer run, such as
434/// a runner or a seed a crash stopped: Docker refuses to remove a volume a container mounts, even a stopped one. It
435/// leaves a volume a running container mounts, or a container that is not runnerd's, and removes no such container.
436pub async fn empty_volumes(docker: &impl DockerApi, volumes: &[String]) -> Cleared {
437    let mut emptied = Cleared::default();
438    for volume in volumes {
439        match empty_volume(docker, volume).await {
440            Ok(()) => emptied.removed.push(volume.clone()),
441            Err(refusal) => emptied.refused.push(refusal),
442        }
443    }
444    emptied
445}
446
447/// Removes `volume`, if it exists, once nothing of runnerd's mounts it.
448async fn empty_volume(docker: &impl DockerApi, volume: &str) -> Result<(), Refusal> {
449    let refused = |why: String, cause: Kept| Refusal {
450        volume: volume.to_owned(),
451        why,
452        cause,
453    };
454    let failed = |error: Error| refused(error.to_string(), Kept::Docker);
455    for holder in docker.holders(volume).await.map_err(failed)? {
456        let name = if holder.name.is_empty() {
457            &holder.id
458        } else {
459            &holder.name
460        };
461        if holder.running {
462            return Err(refused(
463                format!("container {name} mounts it and runs"),
464                Kept::Runs,
465            ));
466        }
467        if !holder
468            .labels
469            .keys()
470            .any(|label| label.starts_with("runnerd."))
471        {
472            return Err(refused(
473                format!("container {name}, which is not runnerd's, mounts it"),
474                Kept::NotRunnerds,
475            ));
476        }
477        match docker.remove(&holder.id).await {
478            Err(error) if !is_not_found(&error) => {
479                return Err(refused(
480                    format!("container {name}, which mounts it, cannot be removed: {error}"),
481                    Kept::Docker,
482                ));
483            }
484            _ => {}
485        }
486    }
487    docker.remove_volume(volume).await.map_err(failed)
488}
489
490/// Every cache volume the config's profiles use, what it holds and its limit, with size 0.
491pub fn volumes(names: &Names, config: &config::Config) -> Vec<api::CacheVolume> {
492    let volume = |name: String, kind: api::CacheKind, limit: Option<u64>| api::CacheVolume {
493        name,
494        kind,
495        bytes: 0,
496        limit,
497    };
498    let mut volumes = Vec::new();
499    for (profile, settings) in &config.profiles {
500        volumes.push(volume(
501            names.base(profile),
502            api::CacheKind::Snapshot {
503                profile: profile.clone(),
504            },
505            Some(config.caches.snapshot_max.0),
506        ));
507        for slot in 1..=settings.slots as usize {
508            volumes.push(volume(
509                names.slot(profile, slot),
510                api::CacheKind::Slot {
511                    profile: profile.clone(),
512                    slot,
513                },
514                None,
515            ));
516        }
517    }
518    volumes.push(volume(
519        names.pnpm(),
520        api::CacheKind::Pnpm,
521        Some(config.caches.pnpm.max_size.0),
522    ));
523    volumes.push(volume(
524        names.maven(),
525        api::CacheKind::Maven,
526        Some(config.caches.maven.max_size.0),
527    ));
528    if config.caches.mirror.enabled {
529        volumes.push(volume(
530            names.mirror(),
531            api::CacheKind::Mirror,
532            Some(config.caches.mirror.max_size.0),
533        ));
534    }
535    if config.caches.build.enabled {
536        volumes.push(volume(
537            names.build_cache(),
538            api::CacheKind::Build,
539            Some(config.caches.build.max_size.0),
540        ));
541    }
542    volumes.sort_by(|a, b| a.name.cmp(&b.name));
543    volumes
544}
545
546/// What `reseed` runs on: the profile's image and limits, its slots and the isolation backend.
547#[derive(Debug)]
548pub struct Reseed<'profile> {
549    /// Where the profile's volumes are.
550    pub names: &'profile Names,
551    /// The profile to reseed.
552    pub profile: &'profile str,
553    /// The profile's runner image, which holds the seed script.
554    pub image: &'profile str,
555    /// How many slots the profile has; each slot's layer changes are dropped.
556    pub slots: usize,
557    /// The systemd slice the seed container runs in.
558    pub slice: &'profile str,
559    /// The profile's CPUs.
560    pub cpus: u32,
561    /// The profile's memory.
562    pub memory: ByteSize,
563    /// Slot 1's layers must have changed at least this much (`caches.reseed_at`).
564    pub min_changes: ByteSize,
565    /// A snapshot that would outgrow this starts over (`caches.snapshot_max`).
566    pub max_size: ByteSize,
567    /// How the seed container is isolated, as runners are.
568    pub isolation: &'profile dyn Isolation,
569}
570
571/// Makes slot 1's layers the profile's new snapshot and drops every slot's own layer changes (`image/seed.sh`),
572/// if slot 1 changed at least `min_changes`. A snapshot that would outgrow `max_size` starts over from the image,
573/// as every slot's changes go too (docs/adr/0065); after a reseed that did not finish, it starts over whatever the
574/// limits (docs/adr/0066). Call only while none of the profile's slots runs. Returns the seed container's output.
575pub async fn reseed(docker: &impl DockerApi, reseed: &Reseed<'_>) -> Result<String, Error> {
576    let names = reseed.names;
577    let mut mounts = vec![volume(names.base(reseed.profile), "/base", false)];
578    mounts.extend((1..=reseed.slots).map(|slot| {
579        volume(
580            names.slot(reseed.profile, slot),
581            &format!("/workers/{slot}"),
582            false,
583        )
584    }));
585    for mount in &mounts {
586        if let Some(source) = &mount.source {
587            docker.ensure_volume(source).await?;
588        }
589    }
590    let name = names.seed(reseed.profile);
591    let spec = RunnerSpec {
592        name: name.clone(),
593        image: reseed.image.to_owned(),
594        profile: reseed.profile.to_owned(),
595        slice: reseed.slice.to_owned(),
596        cpus: reseed.cpus,
597        memory: reseed.memory,
598        env: Vec::new(),
599        mounts,
600        network: Some("none".to_owned()),
601        labels: BTreeMap::from([("runnerd.seed".to_owned(), String::new())]),
602        entrypoint: Some(vec![
603            "/usr/local/sbin/runnerd-seed".to_owned(),
604            reseed.min_changes.0.to_string(),
605            reseed.max_size.0.to_string(),
606        ]),
607        jit_config: None,
608    };
609    // A seed container left by a daemon that died while seeding.
610    match docker.remove(&name).await {
611        Err(error) if !is_not_found(&error) => return Err(error),
612        _ => {}
613    }
614    let id = docker.start(&spec, reseed.isolation).await?;
615    finish_reseed(docker, &id).await
616}
617
618/// What `start_layers_over` runs on: a slot's volume a host crash may have torn and the guard could not empty, and
619/// the image, slice and isolation of a runner.
620#[derive(Debug)]
621pub struct StartOver<'volume> {
622    /// Where runnerd's volumes are.
623    pub names: &'volume Names,
624    /// The slot's volume.
625    pub volume: &'volume str,
626    /// A runner image, for its shell: any profile's will do.
627    pub image: &'volume str,
628    /// The systemd slice the container runs in.
629    pub slice: &'volume str,
630    /// How the container is isolated, as runners are.
631    pub isolation: &'volume dyn Isolation,
632}
633
634/// The memory the container of `start_layers_over` may use: it only deletes.
635const START_OVER_MEMORY: ByteSize = ByteSize(256 << 20);
636
637/// Deletes everything in a slot's volume but its Docker Engine's store (`image/layers.sh`'s `DOCKER_DATA_ROOT`), in
638/// a container that mounts it: its layers' changes and work directories, and the record of the snapshot they were
639/// made against. The slot's next runner starts them over from the snapshot, and a reseed finds nothing of slot 1's
640/// to copy (ADR 0142). For a volume the guard could not remove, as another container holds it: Docker lets a second
641/// container mount it. A container an earlier daemon left is removed first.
642pub async fn start_layers_over(
643    docker: &impl DockerApi,
644    start: &StartOver<'_>,
645) -> Result<(), Error> {
646    let name = start.names.start_over();
647    match docker.remove(&name).await {
648        Err(error) if !is_not_found(&error) => return Err(error),
649        _ => {}
650    }
651    let spec = RunnerSpec {
652        name: name.clone(),
653        image: start.image.to_owned(),
654        profile: String::new(),
655        slice: start.slice.to_owned(),
656        cpus: 1,
657        memory: START_OVER_MEMORY,
658        env: Vec::new(),
659        mounts: vec![volume(start.volume.to_owned(), "/state", false)],
660        network: Some("none".to_owned()),
661        labels: BTreeMap::from([("runnerd.seed".to_owned(), String::new())]),
662        entrypoint: Some(
663            [
664                "find",
665                "/state",
666                "-mindepth",
667                "1",
668                "-maxdepth",
669                "1",
670                "!",
671                "-name",
672                "docker",
673                "-exec",
674                "rm",
675                "-rf",
676                "{}",
677                "+",
678            ]
679            .map(str::to_owned)
680            .to_vec(),
681        ),
682        jit_config: None,
683    };
684    let id = docker.start(&spec, start.isolation).await?;
685    finish_reseed(docker, &id).await.map(|_| ())
686}
687
688/// What a measure container (`image/seed.sh measure`) runs on: the profile's image, its slot 1 and the snapshot,
689/// both read-only, and the isolation backend.
690#[derive(Debug)]
691pub struct Measure<'profile> {
692    /// Where the profile's volumes are.
693    pub names: &'profile Names,
694    /// The profile whose slot 1 is measured.
695    pub profile: &'profile str,
696    /// The profile's runner image, which holds the seed script.
697    pub image: &'profile str,
698    /// The systemd slice the measure container runs in.
699    pub slice: &'profile str,
700    /// How the measure container is isolated, as runners are.
701    pub isolation: &'profile dyn Isolation,
702}
703
704/// What a measure found: what a reseed would weigh against `caches.reseed_at`.
705#[derive(Clone, Copy, Debug, PartialEq, Eq)]
706pub enum Measured {
707    /// Slot 1's changes to the layers hold this many bytes, its Docker Engine's data aside.
708    Changed(u64),
709    /// The profile's last reseed did not finish (docs/adr/0066): the next one starts over, whatever slot 1 changed.
710    Unfinished,
711}
712
713/// The memory a measure container may use: it only adds up file sizes.
714const MEASURE_MEMORY: ByteSize = ByteSize(256 << 20);
715
716/// Measures what the profile's slot 1 changed (`image/seed.sh measure`), in a container that mounts slot 1's
717/// volume and the snapshot read-only, so it may run while the profile's runners do. A measure container an earlier
718/// daemon left is removed first.
719pub async fn measure(docker: &impl DockerApi, measure: &Measure<'_>) -> Result<Measured, Error> {
720    let spec = measure_spec(measure);
721    match docker.remove(&spec.name).await {
722        Err(error) if !is_not_found(&error) => return Err(error),
723        _ => {}
724    }
725    let id = docker.start(&spec, measure.isolation).await?;
726    let output = finish_reseed(docker, &id).await?;
727    measured(&output).ok_or_else(|| Error::DockerContainerWaitError {
728        error: format!("measuring printed {:?}", output.trim()),
729        code: 0,
730    })
731}
732
733/// The container `measure` runs: `seed.sh measure` on slot 1's volume and the snapshot, both read-only.
734pub fn measure_spec(measure: &Measure<'_>) -> RunnerSpec {
735    let names = measure.names;
736    RunnerSpec {
737        name: names.measure(measure.profile),
738        image: measure.image.to_owned(),
739        profile: measure.profile.to_owned(),
740        slice: measure.slice.to_owned(),
741        cpus: 1,
742        memory: MEASURE_MEMORY,
743        env: Vec::new(),
744        mounts: vec![
745            volume(names.base(measure.profile), "/base", true),
746            volume(names.slot(measure.profile, 1), "/workers/1", true),
747        ],
748        network: Some("none".to_owned()),
749        // As a seed container's: a restart finds it beside them (`seed`), and removes it.
750        labels: BTreeMap::from([("runnerd.seed".to_owned(), String::new())]),
751        entrypoint: Some(vec![
752            "/usr/local/sbin/runnerd-seed".to_owned(),
753            "measure".to_owned(),
754        ]),
755        jit_config: None,
756    }
757}
758
759/// What the output of `image/seed.sh measure` says, in its last line that says it: a warning before it, such as
760/// one of `du` about a file a job deleted while it counted, leaves the number as advisory as it is anyway.
761fn measured(output: &str) -> Option<Measured> {
762    output.lines().rev().map(str::trim).find_map(|line| {
763        if line == "the last reseed did not finish" {
764            return Some(Measured::Unfinished);
765        }
766        line.strip_prefix("slot 1 changed ")?
767            .strip_suffix(" bytes")?
768            .parse()
769            .ok()
770            .map(Measured::Changed)
771    })
772}
773
774/// Whether the output of a reseed (`image/seed.sh`) says it replaced the snapshot, and so dropped every slot's own
775/// layer changes, rather than left them because slot 1 changed too little.
776pub fn seeded(output: &str) -> bool {
777    !output.trim_start().starts_with("not seeded")
778}
779
780/// Waits for a seed or measure container to exit, removes it, and returns its output. Also takes over a seed
781/// container an earlier daemon left running: stopping it between its `rm` and `mv` would lose that snapshot layer.
782pub async fn finish_reseed(docker: &impl DockerApi, container: &str) -> Result<String, Error> {
783    let result = async {
784        let code = docker.wait(container).await?;
785        let output = docker.logs(container).await?;
786        if code == 0 {
787            Ok(output)
788        } else {
789            Err(Error::DockerContainerWaitError {
790                error: format!("seeding exited {code}: {}", output.trim()),
791                code,
792            })
793        }
794    }
795    .await;
796    let removed = docker.remove(container).await;
797    let output = result?;
798    removed?;
799    Ok(output)
800}
801
802#[cfg(test)]
803mod tests {
804    use std::path::Path;
805    use std::process::Command;
806
807    use super::docker_budget_gib;
808    use crate::testdir::TestDir;
809
810    /// Runs `script` in bash with `image/layers.sh` sourced and `args` as its arguments; returns what it printed.
811    #[allow(
812        clippy::unwrap_used,
813        reason = "a test helper: a script that does not run fails the test"
814    )]
815    fn with_layers(script: &str, args: &[&Path]) -> String {
816        let layers = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../image/layers.sh");
817        let output = Command::new("bash")
818            .arg("-euc")
819            .arg(format!(". \"$0\"; {script}"))
820            .arg(layers)
821            .args(args)
822            .output()
823            .unwrap();
824        assert!(
825            output.status.success(),
826            "the script exits 0: {}",
827            String::from_utf8_lossy(&output.stderr)
828        );
829        String::from_utf8(output.stdout).unwrap()
830    }
831
832    #[test]
833    #[allow(
834        clippy::unwrap_used,
835        reason = "a test: a file it cannot write fails it"
836    )]
837    fn no_credential_a_job_leaves_in_the_cache_layer_reaches_the_slots_next_job() {
838        let listed = with_layers("printf '%s\\n' \"${CACHE_SECRETS[@]}\"", &[]);
839        let secrets: Vec<&str> = listed.lines().collect();
840        for credential in [
841            "cargo/credentials.toml",
842            "gradle/gradle.properties",
843            "gradle/init.d",
844            "m2/settings.xml",
845            "m2/settings-security.xml",
846            "huggingface/token",
847        ] {
848            assert!(
849                secrets.contains(&credential),
850                "{credential} is in CACHE_SECRETS: {secrets:?}"
851            );
852        }
853        let dir = TestDir::new("cache-secrets");
854        let upper = dir.0.join("upper");
855        let write = |path: &str| {
856            let path = upper.join(path);
857            std::fs::create_dir_all(path.parent().unwrap()).unwrap();
858            std::fs::write(path, "token").unwrap();
859        };
860        // Each as a file, and as a directory a job made there, as Gradle's init.d is.
861        for secret in &secrets {
862            write(&format!("files/{secret}"));
863            write(&format!("dirs/{secret}/held"));
864        }
865        let kept = [
866            "npm/_cacache/index",
867            "m2/wrapper/dists/maven.zip",
868            "gradle/caches/jars",
869        ];
870        for path in kept {
871            write(&format!("files/{path}"));
872        }
873        with_layers(
874            "drop_cache_secrets \"$1/files\"; drop_cache_secrets \"$1/dirs\"",
875            &[&upper],
876        );
877        for secret in &secrets {
878            for copy in ["files", "dirs"] {
879                let path = upper.join(copy).join(secret);
880                assert!(!path.exists(), "{} is gone", path.display());
881            }
882        }
883        for path in kept {
884            assert!(upper.join("files").join(path).exists(), "{path} stays");
885        }
886    }
887
888    #[test]
889    #[allow(
890        clippy::unwrap_used,
891        reason = "a test: a file it cannot write fails it"
892    )]
893    fn what_slot_1_changed_counts_its_layers_and_not_its_docker_engines_data() {
894        let dir = TestDir::new("layer-changes");
895        let state = dir.0.join("state");
896        let write = |path: &str, bytes: usize| {
897            let path = state.join(path);
898            std::fs::create_dir_all(path.parent().unwrap()).unwrap();
899            std::fs::write(path, vec![0u8; bytes]).unwrap();
900        };
901        let changes = || -> u64 {
902            with_layers("layer_changes \"$1\"", &[&state])
903                .trim()
904                .parse()
905                .unwrap()
906        };
907        write("cache/upper/npm/index", 1);
908        let before = changes();
909        // A job's images in the slot's Docker Engine: a reseed leaves them, so they must not reach `reseed_at`.
910        write("docker/overlay2/layer", 1 << 20);
911        // Layer work directories hold nothing a reseed copies.
912        write("tools/work/scratch", 1 << 20);
913        assert_eq!(changes(), before);
914        write("tools/upper/node/bin", 100_000);
915        write("android/upper/ndk/lib", 200_000);
916        let after = changes();
917        assert!(
918            after >= before + 300_000 && after < before + 400_000,
919            "the tools and android layers count: {before} then {after}"
920        );
921    }
922
923    #[test]
924    fn a_slots_docker_budget_is_its_share_between_1_and_20_gib() {
925        assert_eq!(docker_budget_gib(100 << 30, 10), 10);
926        assert_eq!(
927            docker_budget_gib(1 << 40, 4),
928            20,
929            "a large disk still stops at 20 GiB"
930        );
931        assert_eq!(
932            docker_budget_gib(1 << 30, 13),
933            1,
934            "a full disk still keeps 1 GiB"
935        );
936        assert_eq!(docker_budget_gib(1 << 30, 0), 1);
937    }
938}