Skip to main content

runnerd_core/
cache.rs

1//! The warm caches runner containers share and keep between jobs: each profile's read-only snapshot, each slot's
2//! volume (its layers' changes and its Docker Engine's data), the shared pnpm store and Maven repository, and the
3//! Docker Hub mirror on the runners' network. Owns their names, mounts and lifecycle; it never decides when a
4//! runner starts, and it never touches a volume a running container uses.
5
6use std::collections::{BTreeMap, HashMap};
7
8use bollard::models::{
9    ContainerCreateBody, EndpointSettings, HostConfig, Mount, MountType, NetworkingConfig,
10};
11
12use crate::api;
13use crate::config::{self, ByteSize};
14use crate::docker::{DockerApi, Error, RunnerSpec, is_not_found};
15use crate::isolation::Isolation;
16
17/// The Docker Hub pull-through cache (`registry` 3.1.2, multi-platform index).
18pub const MIRROR_IMAGE: &str =
19    "registry:3.1.2@sha256:01cd312e67026f06f5d89c3e133a4a9adc7cb9eeb9b1194e0b2954dfd770efd2";
20
21/// The host name runner containers reach the mirror at (`image/daemon.json`).
22const MIRROR_ALIAS: &str = "mirror";
23
24/// The mirror's `oom_score_adj`: the kernel picks it after the runners' processes, which keep the default 0.
25const MIRROR_OOM_SCORE_ADJ: i64 = -500;
26
27/// The names of everything runnerd creates in Docker, under one prefix: `runnerd`, or another for tests.
28#[derive(Debug, Clone)]
29pub struct Names {
30    prefix: String,
31}
32
33impl Names {
34    /// Names under `prefix`.
35    pub fn new(prefix: &str) -> Names {
36        Names {
37            prefix: prefix.to_owned(),
38        }
39    }
40
41    /// The network runners and the mirror share.
42    pub fn network(&self) -> String {
43        self.prefix.clone()
44    }
45
46    /// The profile's read-only snapshot.
47    pub fn base(&self, profile: &str) -> String {
48        format!("{}-{profile}-base", self.prefix)
49    }
50
51    /// Slot `slot` (1-based) of the profile: its layers' changes and its Docker Engine's data.
52    pub fn slot(&self, profile: &str, slot: usize) -> String {
53        format!("{}-{profile}-slot-{slot}", self.prefix)
54    }
55
56    /// The pnpm store all slots share.
57    pub fn pnpm(&self) -> String {
58        format!("{}-pnpm", self.prefix)
59    }
60
61    /// The Maven repository all slots share.
62    pub fn maven(&self) -> String {
63        format!("{}-maven", self.prefix)
64    }
65
66    /// The slot (1-based) of the profile whose volume `volume` is, if it is one of its slots'.
67    pub fn slot_of(&self, profile: &str, volume: &str) -> Option<usize> {
68        volume
69            .strip_prefix(&format!("{}-{profile}-slot-", self.prefix))?
70            .parse()
71            .ok()
72    }
73
74    /// The container that rebuilds the profile's snapshot.
75    pub fn seed(&self, profile: &str) -> String {
76        format!("{}-{profile}-seed", self.prefix)
77    }
78
79    /// The container that measures what the profile's slot 1 changed, before its upkeep (`measure`).
80    pub fn measure(&self, profile: &str) -> String {
81        format!("{}-{profile}-measure", self.prefix)
82    }
83
84    /// The mirror's container and its storage volume.
85    pub fn mirror(&self) -> String {
86        format!("{}-mirror", self.prefix)
87    }
88
89    /// What `GET /v1/caches` and `runnerctl caches clean` call the shared build cache: a directory of the daemon's,
90    /// not a Docker volume.
91    pub fn build_cache(&self) -> String {
92        format!("{}-build-cache", self.prefix)
93    }
94}
95
96fn volume(source: String, target: &str, read_only: bool) -> Mount {
97    Mount {
98        target: Some(target.to_owned()),
99        source: Some(source),
100        typ: Some(MountType::VOLUME),
101        read_only: Some(read_only),
102        ..Default::default()
103    }
104}
105
106/// What a runner container in slot `slot` (1-based) of the profile mounts (paths from `image/Dockerfile`).
107pub fn runner_mounts(names: &Names, profile: &str, slot: usize) -> Vec<Mount> {
108    vec![
109        volume(names.base(profile), "/base", true),
110        volume(names.slot(profile, slot), "/state", false),
111        volume(names.pnpm(), "/shared/pnpm", false),
112        volume(names.maven(), "/shared/maven", false),
113    ]
114}
115
116/// The mirror container: Docker Hub through a proxy registry, which refuses pushes, so a job can only make it
117/// fetch what Docker Hub serves. Content not pulled again within the TTL expires. No published port.
118pub fn mirror_body(names: &Names, mirror: &config::Mirror, slice: &str) -> ContainerCreateBody {
119    let memory = i64::try_from(mirror.memory.0).unwrap_or(i64::MAX);
120    ContainerCreateBody {
121        image: Some(MIRROR_IMAGE.to_owned()),
122        env: Some(vec![
123            "REGISTRY_PROXY_REMOTEURL=https://registry-1.docker.io".to_owned(),
124            format!("REGISTRY_PROXY_TTL={}s", mirror.ttl.as_secs()),
125            // Expiry deletes content.
126            "REGISTRY_STORAGE_DELETE_ENABLED=true".to_owned(),
127        ]),
128        labels: Some(HashMap::from([(
129            "runnerd.mirror".to_owned(),
130            String::new(),
131        )])),
132        host_config: Some(HostConfig {
133            cgroup_parent: Some(slice.to_owned()),
134            memory: Some(memory),
135            memory_swap: Some(memory),
136            nano_cpus: Some(i64::from(mirror.cpus) * 1_000_000_000),
137            // When the slice runs out of memory, which measured admission allows, the kernel kills a job's process
138            // before the mirror every runner pulls through.
139            oom_score_adj: Some(MIRROR_OOM_SCORE_ADJ),
140            mounts: Some(vec![volume(names.mirror(), "/var/lib/registry", false)]),
141            network_mode: Some(names.network()),
142            ..Default::default()
143        }),
144        networking_config: Some(NetworkingConfig {
145            endpoints_config: Some(HashMap::from([(
146                names.network(),
147                EndpointSettings {
148                    aliases: Some(vec![MIRROR_ALIAS.to_owned()]),
149                    ..Default::default()
150                },
151            )])),
152        }),
153        ..Default::default()
154    }
155}
156
157/// Creates what a profile's runner in `slot` (1-based) mounts and joins, if missing.
158pub async fn prepare(
159    docker: &impl DockerApi,
160    names: &Names,
161    profile: &str,
162    slot: usize,
163) -> Result<(), Error> {
164    docker.ensure_network(&names.network()).await?;
165    for name in [
166        names.base(profile),
167        names.slot(profile, slot),
168        names.pnpm(),
169        names.maven(),
170    ] {
171        docker.ensure_volume(&name).await?;
172    }
173    Ok(())
174}
175
176/// Starts the mirror unless it runs.
177pub async fn mirror_up(
178    docker: &impl DockerApi,
179    names: &Names,
180    mirror: &config::Mirror,
181    slice: &str,
182) -> Result<(), Error> {
183    match docker.inspect(&names.mirror()).await {
184        Ok(state) if state.running => return Ok(()),
185        // Stopped on its own: started again from scratch, its storage is the volume.
186        Ok(_) => docker.remove(&names.mirror()).await?,
187        Err(error) if is_not_found(&error) => {}
188        Err(error) => return Err(error),
189    }
190    docker.ensure_network(&names.network()).await?;
191    if !docker.image_present(MIRROR_IMAGE).await? {
192        docker.pull(MIRROR_IMAGE).await?;
193    }
194    docker
195        .run(&names.mirror(), mirror_body(names, mirror, slice))
196        .await?;
197    Ok(())
198}
199
200/// Stops the mirror; its storage stays.
201pub async fn mirror_down(docker: &impl DockerApi, names: &Names) -> Result<(), Error> {
202    match docker.remove(&names.mirror()).await {
203        Err(error) if is_not_found(&error) => Ok(()),
204        other => other,
205    }
206}
207
208/// Whether the mirror's container is down, so its storage can be removed.
209#[derive(Debug, Clone, Copy, PartialEq, Eq)]
210pub enum Mirror {
211    /// It runs, or may: an earlier daemon may have left it up.
212    Up,
213    /// It is stopped.
214    Down,
215}
216
217/// What `trim_shared` measured and emptied.
218#[derive(Debug, Clone, PartialEq, Eq)]
219pub struct Trimmed {
220    /// Every volume's size in bytes, by name, as measured before emptying any.
221    pub sizes: BTreeMap<String, u64>,
222    /// The shared caches it emptied.
223    pub emptied: Vec<Emptied>,
224    /// The slot volumes past their profile's slot count it removed.
225    pub closed: Vec<Emptied>,
226    /// The volumes it was to empty or remove that Docker refused to, as one a stopped container still uses: tried
227    /// again at the next check.
228    pub refused: Vec<Refused>,
229    /// Docker's free disk in bytes, as measured before emptying any; `None` when it could not be measured.
230    pub free: Option<u64>,
231}
232
233/// A volume `trim_shared` was to remove that Docker refused to.
234#[derive(Debug, Clone, PartialEq, Eq)]
235pub struct Refused {
236    /// The volume.
237    pub name: String,
238    /// What Docker said.
239    pub error: String,
240}
241
242/// A shared cache `trim_shared` emptied.
243#[derive(Debug, Clone, PartialEq, Eq)]
244pub struct Emptied {
245    /// The volume.
246    pub name: String,
247    /// How large it had grown.
248    pub size: ByteSize,
249}
250
251/// Empties the shared caches that outgrew their limit, the mirror's storage among them once the mirror is down
252/// (`Mirror::Down`), and removes the volumes of slots past the count `slots` gives their profile, as a lowered count
253/// leaves them: each holds a slot's layers and Docker store, which no job prunes again (docs/adr/0054). Call only
254/// while no runner runs, and leave out of `slots` a profile whose reseed runs, which mounts its slots' volumes.
255pub async fn trim_shared(
256    docker: &impl DockerApi,
257    names: &Names,
258    caches: &config::Caches,
259    mirror: Mirror,
260    slots: &BTreeMap<String, usize>,
261) -> Result<Trimmed, Error> {
262    let sizes = docker.volume_sizes().await?;
263    let free = docker.free_space().await.ok();
264    let mut emptied = Vec::new();
265    let mut refused = Vec::new();
266    let mut limits = vec![
267        (names.pnpm(), caches.pnpm.max_size),
268        (names.maven(), caches.maven.max_size),
269    ];
270    if mirror == Mirror::Down {
271        limits.push((names.mirror(), caches.mirror.max_size));
272    }
273    for (name, limit) in limits {
274        if let Some(&size) = sizes.get(&name)
275            && size > limit.0
276        {
277            // One Docker refuses stays to the next check; the others are still trimmed, and the sizes kept.
278            if let Err(error) = docker.remove_volume(&name).await {
279                refused.push(Refused {
280                    name,
281                    error: error.to_string(),
282                });
283                continue;
284            }
285            emptied.push(Emptied {
286                name,
287                size: ByteSize(size),
288            });
289        }
290    }
291    let mut closed = Vec::new();
292    for (name, &size) in &sizes {
293        let past = slots.iter().any(|(profile, &count)| {
294            names
295                .slot_of(profile, name)
296                .is_some_and(|slot| slot > count)
297        });
298        if past {
299            // As a stopped container kept as a failed job's record still uses it.
300            if let Err(error) = docker.remove_volume(name).await {
301                refused.push(Refused {
302                    name: name.clone(),
303                    error: error.to_string(),
304                });
305                continue;
306            }
307            closed.push(Emptied {
308                name: name.clone(),
309                size: ByteSize(size),
310            });
311        }
312    }
313    Ok(Trimmed {
314        sizes,
315        emptied,
316        closed,
317        refused,
318        free,
319    })
320}
321
322/// What the slots' Docker Engines may keep together, in bytes: a quarter of Docker's disk not otherwise used, which
323/// is what is free plus what the slot volumes already hold, measured together (docs/adr/0054).
324pub fn docker_disk(
325    names: &Names,
326    config: &config::Config,
327    free: u64,
328    sizes: &BTreeMap<String, u64>,
329) -> u64 {
330    let held: u64 = config
331        .profiles
332        .iter()
333        .flat_map(|(profile, settings)| {
334            (1..=settings.slots as usize).map(move |slot| names.slot(profile, slot))
335        })
336        .filter_map(|name| sizes.get(&name))
337        .sum();
338    free.saturating_add(held) / DOCKER_DISK_SHARE
339}
340
341/// One slot's share of `disk` among `slots`, in whole GiB as `image/layers.sh` takes it: at least 1, at most
342/// `DOCKER_SLOT_MAX_GIB`.
343pub fn docker_budget_gib(disk: u64, slots: u64) -> u64 {
344    ((disk / slots.max(1)) >> 30).clamp(1, DOCKER_SLOT_MAX_GIB)
345}
346
347/// The slots' Docker Engines together keep at most one part in this of Docker's disk not otherwise used.
348const DOCKER_DISK_SHARE: u64 = 4;
349
350/// The most one slot's Docker Engine keeps, in GiB: the image's own default, for a daemon that sets none.
351const DOCKER_SLOT_MAX_GIB: u64 = 20;
352
353/// Every cache volume the config's profiles use, what it holds and its limit, with size 0.
354pub fn volumes(names: &Names, config: &config::Config) -> Vec<api::CacheVolume> {
355    let volume = |name: String, kind: api::CacheKind, limit: Option<u64>| api::CacheVolume {
356        name,
357        kind,
358        bytes: 0,
359        limit,
360    };
361    let mut volumes = Vec::new();
362    for (profile, settings) in &config.profiles {
363        volumes.push(volume(
364            names.base(profile),
365            api::CacheKind::Snapshot {
366                profile: profile.clone(),
367            },
368            Some(config.caches.snapshot_max.0),
369        ));
370        for slot in 1..=settings.slots as usize {
371            volumes.push(volume(
372                names.slot(profile, slot),
373                api::CacheKind::Slot {
374                    profile: profile.clone(),
375                    slot,
376                },
377                None,
378            ));
379        }
380    }
381    volumes.push(volume(
382        names.pnpm(),
383        api::CacheKind::Pnpm,
384        Some(config.caches.pnpm.max_size.0),
385    ));
386    volumes.push(volume(
387        names.maven(),
388        api::CacheKind::Maven,
389        Some(config.caches.maven.max_size.0),
390    ));
391    if config.caches.mirror.enabled {
392        volumes.push(volume(
393            names.mirror(),
394            api::CacheKind::Mirror,
395            Some(config.caches.mirror.max_size.0),
396        ));
397    }
398    if config.caches.build.enabled {
399        volumes.push(volume(
400            names.build_cache(),
401            api::CacheKind::Build,
402            Some(config.caches.build.max_size.0),
403        ));
404    }
405    volumes.sort_by(|a, b| a.name.cmp(&b.name));
406    volumes
407}
408
409/// What `reseed` runs on: the profile's image and limits, its slots and the isolation backend.
410#[derive(Debug)]
411pub struct Reseed<'profile> {
412    /// Where the profile's volumes are.
413    pub names: &'profile Names,
414    /// The profile to reseed.
415    pub profile: &'profile str,
416    /// The profile's runner image, which holds the seed script.
417    pub image: &'profile str,
418    /// How many slots the profile has; each slot's layer changes are dropped.
419    pub slots: usize,
420    /// The systemd slice the seed container runs in.
421    pub slice: &'profile str,
422    /// The profile's CPUs.
423    pub cpus: u32,
424    /// The profile's memory.
425    pub memory: ByteSize,
426    /// Slot 1's layers must have changed at least this much (`caches.reseed_at`).
427    pub min_changes: ByteSize,
428    /// A snapshot that would outgrow this starts over (`caches.snapshot_max`).
429    pub max_size: ByteSize,
430    /// How the seed container is isolated, as runners are.
431    pub isolation: &'profile dyn Isolation,
432}
433
434/// Makes slot 1's layers the profile's new snapshot and drops every slot's own layer changes (`image/seed.sh`),
435/// if slot 1 changed at least `min_changes`. A snapshot that would outgrow `max_size` starts over from the image,
436/// as every slot's changes go too (docs/adr/0065); after a reseed that did not finish, it starts over whatever the
437/// limits (docs/adr/0066). Call only while none of the profile's slots runs. Returns the seed container's output.
438pub async fn reseed(docker: &impl DockerApi, reseed: &Reseed<'_>) -> Result<String, Error> {
439    let names = reseed.names;
440    let mut mounts = vec![volume(names.base(reseed.profile), "/base", false)];
441    mounts.extend((1..=reseed.slots).map(|slot| {
442        volume(
443            names.slot(reseed.profile, slot),
444            &format!("/workers/{slot}"),
445            false,
446        )
447    }));
448    for mount in &mounts {
449        if let Some(source) = &mount.source {
450            docker.ensure_volume(source).await?;
451        }
452    }
453    let name = names.seed(reseed.profile);
454    let spec = RunnerSpec {
455        name: name.clone(),
456        image: reseed.image.to_owned(),
457        profile: reseed.profile.to_owned(),
458        slice: reseed.slice.to_owned(),
459        cpus: reseed.cpus,
460        memory: reseed.memory,
461        env: Vec::new(),
462        mounts,
463        network: Some("none".to_owned()),
464        labels: BTreeMap::from([("runnerd.seed".to_owned(), String::new())]),
465        entrypoint: Some(vec![
466            "/usr/local/sbin/runnerd-seed".to_owned(),
467            reseed.min_changes.0.to_string(),
468            reseed.max_size.0.to_string(),
469        ]),
470        jit_config: None,
471    };
472    // A seed container left by a daemon that died while seeding.
473    match docker.remove(&name).await {
474        Err(error) if !is_not_found(&error) => return Err(error),
475        _ => {}
476    }
477    let id = docker.start(&spec, reseed.isolation).await?;
478    finish_reseed(docker, &id).await
479}
480
481/// What a measure container (`image/seed.sh measure`) runs on: the profile's image, its slot 1 and the snapshot,
482/// both read-only, and the isolation backend.
483#[derive(Debug)]
484pub struct Measure<'profile> {
485    /// Where the profile's volumes are.
486    pub names: &'profile Names,
487    /// The profile whose slot 1 is measured.
488    pub profile: &'profile str,
489    /// The profile's runner image, which holds the seed script.
490    pub image: &'profile str,
491    /// The systemd slice the measure container runs in.
492    pub slice: &'profile str,
493    /// How the measure container is isolated, as runners are.
494    pub isolation: &'profile dyn Isolation,
495}
496
497/// What a measure found: what a reseed would weigh against `caches.reseed_at`.
498#[derive(Clone, Copy, Debug, PartialEq, Eq)]
499pub enum Measured {
500    /// Slot 1's changes to the layers hold this many bytes, its Docker Engine's data aside.
501    Changed(u64),
502    /// The profile's last reseed did not finish (docs/adr/0066): the next one starts over, whatever slot 1 changed.
503    Unfinished,
504}
505
506/// The memory a measure container may use: it only adds up file sizes.
507const MEASURE_MEMORY: ByteSize = ByteSize(256 << 20);
508
509/// Measures what the profile's slot 1 changed (`image/seed.sh measure`), in a container that mounts slot 1's
510/// volume and the snapshot read-only, so it may run while the profile's runners do. A measure container an earlier
511/// daemon left is removed first.
512pub async fn measure(docker: &impl DockerApi, measure: &Measure<'_>) -> Result<Measured, Error> {
513    let spec = measure_spec(measure);
514    match docker.remove(&spec.name).await {
515        Err(error) if !is_not_found(&error) => return Err(error),
516        _ => {}
517    }
518    let id = docker.start(&spec, measure.isolation).await?;
519    let output = finish_reseed(docker, &id).await?;
520    measured(&output).ok_or_else(|| Error::DockerContainerWaitError {
521        error: format!("measuring printed {:?}", output.trim()),
522        code: 0,
523    })
524}
525
526/// The container `measure` runs: `seed.sh measure` on slot 1's volume and the snapshot, both read-only.
527pub fn measure_spec(measure: &Measure<'_>) -> RunnerSpec {
528    let names = measure.names;
529    RunnerSpec {
530        name: names.measure(measure.profile),
531        image: measure.image.to_owned(),
532        profile: measure.profile.to_owned(),
533        slice: measure.slice.to_owned(),
534        cpus: 1,
535        memory: MEASURE_MEMORY,
536        env: Vec::new(),
537        mounts: vec![
538            volume(names.base(measure.profile), "/base", true),
539            volume(names.slot(measure.profile, 1), "/workers/1", true),
540        ],
541        network: Some("none".to_owned()),
542        // As a seed container's: a restart finds it beside them (`seed`), and removes it.
543        labels: BTreeMap::from([("runnerd.seed".to_owned(), String::new())]),
544        entrypoint: Some(vec![
545            "/usr/local/sbin/runnerd-seed".to_owned(),
546            "measure".to_owned(),
547        ]),
548        jit_config: None,
549    }
550}
551
552/// What the output of `image/seed.sh measure` says, in its last line that says it: a warning before it, such as
553/// one of `du` about a file a job deleted while it counted, leaves the number as advisory as it is anyway.
554fn measured(output: &str) -> Option<Measured> {
555    output.lines().rev().map(str::trim).find_map(|line| {
556        if line == "the last reseed did not finish" {
557            return Some(Measured::Unfinished);
558        }
559        line.strip_prefix("slot 1 changed ")?
560            .strip_suffix(" bytes")?
561            .parse()
562            .ok()
563            .map(Measured::Changed)
564    })
565}
566
567/// Whether the output of a reseed (`image/seed.sh`) says it replaced the snapshot, and so dropped every slot's own
568/// layer changes, rather than left them because slot 1 changed too little.
569pub fn seeded(output: &str) -> bool {
570    !output.trim_start().starts_with("not seeded")
571}
572
573/// Waits for a seed or measure container to exit, removes it, and returns its output. Also takes over a seed
574/// container an earlier daemon left running: stopping it between its `rm` and `mv` would lose that snapshot layer.
575pub async fn finish_reseed(docker: &impl DockerApi, container: &str) -> Result<String, Error> {
576    let result = async {
577        let code = docker.wait(container).await?;
578        let output = docker.logs(container).await?;
579        if code == 0 {
580            Ok(output)
581        } else {
582            Err(Error::DockerContainerWaitError {
583                error: format!("seeding exited {code}: {}", output.trim()),
584                code,
585            })
586        }
587    }
588    .await;
589    let removed = docker.remove(container).await;
590    let output = result?;
591    removed?;
592    Ok(output)
593}
594
595#[cfg(test)]
596mod tests {
597    use std::path::Path;
598    use std::process::Command;
599
600    use super::docker_budget_gib;
601    use crate::testdir::TestDir;
602
603    /// Runs `script` in bash with `image/layers.sh` sourced and `args` as its arguments; returns what it printed.
604    #[allow(
605        clippy::unwrap_used,
606        reason = "a test helper: a script that does not run fails the test"
607    )]
608    fn with_layers(script: &str, args: &[&Path]) -> String {
609        let layers = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../image/layers.sh");
610        let output = Command::new("bash")
611            .arg("-euc")
612            .arg(format!(". \"$0\"; {script}"))
613            .arg(layers)
614            .args(args)
615            .output()
616            .unwrap();
617        assert!(
618            output.status.success(),
619            "the script exits 0: {}",
620            String::from_utf8_lossy(&output.stderr)
621        );
622        String::from_utf8(output.stdout).unwrap()
623    }
624
625    #[test]
626    #[allow(
627        clippy::unwrap_used,
628        reason = "a test: a file it cannot write fails it"
629    )]
630    fn no_credential_a_job_leaves_in_the_cache_layer_reaches_the_slots_next_job() {
631        let listed = with_layers("printf '%s\\n' \"${CACHE_SECRETS[@]}\"", &[]);
632        let secrets: Vec<&str> = listed.lines().collect();
633        for credential in [
634            "cargo/credentials.toml",
635            "gradle/gradle.properties",
636            "gradle/init.d",
637            "m2/settings.xml",
638            "m2/settings-security.xml",
639            "huggingface/token",
640        ] {
641            assert!(
642                secrets.contains(&credential),
643                "{credential} is in CACHE_SECRETS: {secrets:?}"
644            );
645        }
646        let dir = TestDir::new("cache-secrets");
647        let upper = dir.0.join("upper");
648        let write = |path: &str| {
649            let path = upper.join(path);
650            std::fs::create_dir_all(path.parent().unwrap()).unwrap();
651            std::fs::write(path, "token").unwrap();
652        };
653        // Each as a file, and as a directory a job made there, as Gradle's init.d is.
654        for secret in &secrets {
655            write(&format!("files/{secret}"));
656            write(&format!("dirs/{secret}/held"));
657        }
658        let kept = [
659            "npm/_cacache/index",
660            "m2/wrapper/dists/maven.zip",
661            "gradle/caches/jars",
662        ];
663        for path in kept {
664            write(&format!("files/{path}"));
665        }
666        with_layers(
667            "drop_cache_secrets \"$1/files\"; drop_cache_secrets \"$1/dirs\"",
668            &[&upper],
669        );
670        for secret in &secrets {
671            for copy in ["files", "dirs"] {
672                let path = upper.join(copy).join(secret);
673                assert!(!path.exists(), "{} is gone", path.display());
674            }
675        }
676        for path in kept {
677            assert!(upper.join("files").join(path).exists(), "{path} stays");
678        }
679    }
680
681    #[test]
682    #[allow(
683        clippy::unwrap_used,
684        reason = "a test: a file it cannot write fails it"
685    )]
686    fn what_slot_1_changed_counts_its_layers_and_not_its_docker_engines_data() {
687        let dir = TestDir::new("layer-changes");
688        let state = dir.0.join("state");
689        let write = |path: &str, bytes: usize| {
690            let path = state.join(path);
691            std::fs::create_dir_all(path.parent().unwrap()).unwrap();
692            std::fs::write(path, vec![0u8; bytes]).unwrap();
693        };
694        let changes = || -> u64 {
695            with_layers("layer_changes \"$1\"", &[&state])
696                .trim()
697                .parse()
698                .unwrap()
699        };
700        write("cache/upper/npm/index", 1);
701        let before = changes();
702        // A job's images in the slot's Docker Engine: a reseed leaves them, so they must not reach `reseed_at`.
703        write("docker/overlay2/layer", 1 << 20);
704        // Layer work directories hold nothing a reseed copies.
705        write("tools/work/scratch", 1 << 20);
706        assert_eq!(changes(), before);
707        write("tools/upper/node/bin", 100_000);
708        write("android/upper/ndk/lib", 200_000);
709        let after = changes();
710        assert!(
711            after >= before + 300_000 && after < before + 400_000,
712            "the tools and android layers count: {before} then {after}"
713        );
714    }
715
716    #[test]
717    fn a_slots_docker_budget_is_its_share_between_1_and_20_gib() {
718        assert_eq!(docker_budget_gib(100 << 30, 10), 10);
719        assert_eq!(
720            docker_budget_gib(1 << 40, 4),
721            20,
722            "a large disk still stops at 20 GiB"
723        );
724        assert_eq!(
725            docker_budget_gib(1 << 30, 13),
726            1,
727            "a full disk still keeps 1 GiB"
728        );
729        assert_eq!(docker_budget_gib(1 << 30, 0), 1);
730    }
731}