Skip to main content

runnerd_core/
cache.rs

1//! The warm caches runner containers share and keep between jobs: each profile's read-only snapshot, each slot's
2//! volume (its layers' changes and its Docker Engine's data), the shared pnpm store and Maven repository, and the
3//! Docker Hub mirror on the runners' network. Owns their names, mounts and lifecycle; it never decides when a
4//! runner starts, and it never touches a volume a running container uses.
5
6use std::collections::{BTreeMap, HashMap};
7use std::time::Duration;
8
9use bollard::models::{
10    ContainerCreateBody, EndpointSettings, HostConfig, Mount, MountType, NetworkingConfig,
11};
12use tracing::warn;
13
14use crate::api;
15use crate::config::{self, ByteSize};
16use crate::docker::{DockerApi, Error, RunnerSpec, is_not_found};
17use crate::isolation::Isolation;
18
19pub mod mirror;
20
21/// The prefix of the names of what the daemon makes in Docker (`Names`).
22pub const PREFIX: &str = "runnerd";
23
24/// The Docker Hub pull-through cache (`registry` 3.1.2, multi-platform index).
25pub const MIRROR_IMAGE: &str =
26    "registry:3.1.2@sha256:01cd312e67026f06f5d89c3e133a4a9adc7cb9eeb9b1194e0b2954dfd770efd2";
27
28/// The host name runner containers reach the mirror at (`image/daemon.json`).
29const MIRROR_ALIAS: &str = "mirror";
30
31/// The mirror's `oom_score_adj`: the kernel picks it after the runners' processes, which keep the default 0.
32const MIRROR_OOM_SCORE_ADJ: i64 = -500;
33
34/// The names of everything runnerd creates in Docker, under one prefix: `runnerd`, or another for tests.
35#[derive(Debug, Clone)]
36pub struct Names {
37    prefix: String,
38}
39
40impl Names {
41    /// Names under `prefix`.
42    pub fn new(prefix: &str) -> Names {
43        Names {
44            prefix: prefix.to_owned(),
45        }
46    }
47
48    /// The network runners and the mirror share.
49    pub fn network(&self) -> String {
50        self.prefix.clone()
51    }
52
53    /// The profile's read-only snapshot.
54    pub fn base(&self, profile: &str) -> String {
55        format!("{}-{profile}-base", self.prefix)
56    }
57
58    /// Slot `slot` (1-based) of the profile: its layers' changes and its Docker Engine's data.
59    pub fn slot(&self, profile: &str, slot: usize) -> String {
60        format!("{}-{profile}-slot-{slot}", self.prefix)
61    }
62
63    /// The pnpm store all slots share.
64    pub fn pnpm(&self) -> String {
65        format!("{}-pnpm", self.prefix)
66    }
67
68    /// The Maven repository all slots share.
69    pub fn maven(&self) -> String {
70        format!("{}-maven", self.prefix)
71    }
72
73    /// The slot (1-based) of the profile whose volume `volume` is, if it is one of its slots'.
74    pub fn slot_of(&self, profile: &str, volume: &str) -> Option<usize> {
75        volume
76            .strip_prefix(&format!("{}-{profile}-slot-", self.prefix))?
77            .parse()
78            .ok()
79    }
80
81    /// Whether `volume` is a slot's volume of any profile, configured now or not: `<prefix>-<profile>-slot-<N>`.
82    /// A prefix that is another's start, as a test's `runnerd-test-…` is `runnerd`'s, matches the other's too.
83    pub fn is_slot(&self, volume: &str) -> bool {
84        volume
85            .strip_prefix(&format!("{}-", self.prefix))
86            .and_then(|rest| rest.rsplit_once("-slot-"))
87            .is_some_and(|(profile, slot)| {
88                !profile.is_empty()
89                    && !slot.is_empty()
90                    && slot.bytes().all(|byte| byte.is_ascii_digit())
91            })
92    }
93
94    /// The prefix every volume of runnerd's starts with, with its dash.
95    pub fn volume_prefix(&self) -> String {
96        format!("{}-", self.prefix)
97    }
98
99    /// The container that starts a slot's layers over after a host crash, when its volume could not be emptied.
100    pub fn start_over(&self) -> String {
101        format!("{}-start-over", self.prefix)
102    }
103
104    /// The container that rebuilds the profile's snapshot.
105    pub fn seed(&self, profile: &str) -> String {
106        format!("{}-{profile}-seed", self.prefix)
107    }
108
109    /// The container that measures what the profile's slot 1 changed, before its upkeep (`measure`).
110    pub fn measure(&self, profile: &str) -> String {
111        format!("{}-{profile}-measure", self.prefix)
112    }
113
114    /// The mirror's container and its storage volume.
115    pub fn mirror(&self) -> String {
116        format!("{}-mirror", self.prefix)
117    }
118
119    /// The container that repairs the mirror's storage before the mirror starts.
120    pub fn mirror_repair(&self) -> String {
121        format!("{}-mirror-repair", self.prefix)
122    }
123
124    /// What `GET /v1/caches` and `runnerctl caches clean` call the shared build cache: a directory of the daemon's,
125    /// not a Docker volume.
126    pub fn build_cache(&self) -> String {
127        format!("{}-build-cache", self.prefix)
128    }
129}
130
131fn volume(source: String, target: &str, read_only: bool) -> Mount {
132    Mount {
133        target: Some(target.to_owned()),
134        source: Some(source),
135        typ: Some(MountType::VOLUME),
136        read_only: Some(read_only),
137        ..Default::default()
138    }
139}
140
141/// What a runner container in slot `slot` (1-based) of the profile mounts (paths from `image/Dockerfile`).
142pub fn runner_mounts(names: &Names, profile: &str, slot: usize) -> Vec<Mount> {
143    vec![
144        volume(names.base(profile), "/base", true),
145        volume(names.slot(profile, slot), "/state", false),
146        volume(names.pnpm(), "/shared/pnpm", false),
147        volume(names.maven(), "/shared/maven", false),
148    ]
149}
150
151/// The mirror container: Docker Hub through a proxy registry, which refuses pushes, so a job can only make it
152/// fetch what Docker Hub serves. Content not pulled again within the TTL expires. No published port. It keeps no
153/// blob descriptor cache: the proxy's expiry of one repository's copy of a blob deletes the blob other repositories
154/// still link to, and the cache would go on answering their pulls with no data (docs/adr/0144).
155pub fn mirror_body(names: &Names, mirror: &config::Mirror, slice: &str) -> ContainerCreateBody {
156    let memory = i64::try_from(mirror.memory.0).unwrap_or(i64::MAX);
157    ContainerCreateBody {
158        image: Some(MIRROR_IMAGE.to_owned()),
159        env: Some(vec![
160            "REGISTRY_PROXY_REMOTEURL=https://registry-1.docker.io".to_owned(),
161            format!("REGISTRY_PROXY_TTL={}s", mirror.ttl.as_secs()),
162            // Expiry deletes content.
163            "REGISTRY_STORAGE_DELETE_ENABLED=true".to_owned(),
164            format!("{}=", mirror::DESCRIPTOR_CACHE_ENV),
165        ]),
166        labels: Some(HashMap::from([(
167            "runnerd.mirror".to_owned(),
168            String::new(),
169        )])),
170        host_config: Some(HostConfig {
171            cgroup_parent: Some(slice.to_owned()),
172            memory: Some(memory),
173            memory_swap: Some(memory),
174            nano_cpus: Some(i64::from(mirror.cpus) * 1_000_000_000),
175            // When the slice runs out of memory, which measured admission allows, the kernel kills a job's process
176            // before the mirror every runner pulls through.
177            oom_score_adj: Some(MIRROR_OOM_SCORE_ADJ),
178            mounts: Some(vec![volume(names.mirror(), "/var/lib/registry", false)]),
179            network_mode: Some(names.network()),
180            ..Default::default()
181        }),
182        networking_config: Some(NetworkingConfig {
183            endpoints_config: Some(HashMap::from([(
184                names.network(),
185                EndpointSettings {
186                    aliases: Some(vec![MIRROR_ALIAS.to_owned()]),
187                    ..Default::default()
188                },
189            )])),
190        }),
191        ..Default::default()
192    }
193}
194
195/// Creates what a profile's runner in `slot` (1-based) mounts and joins, if missing.
196pub async fn prepare(
197    docker: &impl DockerApi,
198    names: &Names,
199    profile: &str,
200    slot: usize,
201) -> Result<(), Error> {
202    docker.ensure_network(&names.network()).await?;
203    for name in [
204        names.base(profile),
205        names.slot(profile, slot),
206        names.pnpm(),
207        names.maven(),
208    ] {
209        docker.ensure_volume(&name).await?;
210    }
211    Ok(())
212}
213
214/// Starts the mirror unless it runs, after removing the links in its storage whose blob is gone (`mirror::Repair`),
215/// which a failed repair does not hold back.
216pub async fn mirror_up(
217    docker: &impl DockerApi,
218    names: &Names,
219    mirror: &config::Mirror,
220    slice: &str,
221) -> Result<mirror::MirrorUp, Error> {
222    match docker.inspect(&names.mirror()).await {
223        Ok(state) if state.running => return Ok(mirror::MirrorUp::Running),
224        // Stopped on its own: started again from scratch, its storage is the volume.
225        Ok(_) => docker.remove(&names.mirror()).await?,
226        Err(error) if is_not_found(&error) => {}
227        Err(error) => return Err(error),
228    }
229    docker.ensure_network(&names.network()).await?;
230    if !docker.image_present(MIRROR_IMAGE).await? {
231        docker.pull(MIRROR_IMAGE).await?;
232    }
233    let repair = mirror::repair(docker, names, slice).await;
234    docker
235        .run(&names.mirror(), mirror_body(names, mirror, slice))
236        .await?;
237    Ok(mirror::MirrorUp::Started(repair))
238}
239
240/// Stops the mirror, and removes a repair container a killed daemon left on its storage, which would keep the
241/// volume from being removed; its storage stays.
242pub async fn mirror_down(docker: &impl DockerApi, names: &Names) -> Result<(), Error> {
243    for name in [names.mirror(), names.mirror_repair()] {
244        match docker.remove(&name).await {
245            Err(error) if !is_not_found(&error) => return Err(error),
246            _ => {}
247        }
248    }
249    Ok(())
250}
251
252/// Whether the mirror's container is down, so its storage can be removed.
253#[derive(Debug, Clone, Copy, PartialEq, Eq)]
254pub enum Mirror {
255    /// It runs, or may: an earlier daemon may have left it up.
256    Up,
257    /// It is stopped.
258    Down,
259}
260
261/// What `trim_shared` measured and emptied.
262#[derive(Debug, Clone, PartialEq, Eq)]
263pub struct Trimmed {
264    /// Every volume's size in bytes, by name, as measured before emptying any.
265    pub sizes: BTreeMap<String, u64>,
266    /// The shared caches it emptied.
267    pub emptied: Vec<Emptied>,
268    /// The slot volumes past their profile's slot count it removed.
269    pub closed: Vec<Emptied>,
270    /// The volumes it was to empty or remove that Docker refused to, as one a stopped container still uses: tried
271    /// again at the next check.
272    pub refused: Vec<Refused>,
273    /// Docker's free disk in bytes, as measured before emptying any; `None` when it could not be measured.
274    pub free: Option<u64>,
275}
276
277/// A volume `trim_shared` was to remove that Docker refused to.
278#[derive(Debug, Clone, PartialEq, Eq)]
279pub struct Refused {
280    /// The volume.
281    pub name: String,
282    /// What Docker said.
283    pub error: String,
284}
285
286/// A shared cache `trim_shared` emptied.
287#[derive(Debug, Clone, PartialEq, Eq)]
288pub struct Emptied {
289    /// The volume.
290    pub name: String,
291    /// How large it had grown.
292    pub size: ByteSize,
293}
294
295/// Empties the shared caches that outgrew their limit, the mirror's storage among them once the mirror is down
296/// (`Mirror::Down`), and removes the volumes of slots past the count `slots` gives their profile, as a lowered count
297/// leaves them: each holds a slot's layers and Docker store, which no job prunes again (docs/adr/0054). Call only
298/// while no runner runs, and leave out of `slots` a profile whose reseed runs, which mounts its slots' volumes.
299pub async fn trim_shared(
300    docker: &impl DockerApi,
301    names: &Names,
302    caches: &config::Caches,
303    mirror: Mirror,
304    slots: &BTreeMap<String, usize>,
305) -> Result<Trimmed, Error> {
306    let sizes = docker.volume_sizes().await?;
307    let free = docker.free_space().await.ok();
308    let mut emptied = Vec::new();
309    let mut refused = Vec::new();
310    let mut limits = vec![
311        (names.pnpm(), caches.pnpm.max_size),
312        (names.maven(), caches.maven.max_size),
313    ];
314    if mirror == Mirror::Down {
315        limits.push((names.mirror(), caches.mirror.max_size));
316    }
317    for (name, limit) in limits {
318        if let Some(&size) = sizes.get(&name)
319            && size > limit.0
320        {
321            // One Docker refuses stays to the next check; the others are still trimmed, and the sizes kept.
322            if let Err(error) = docker.remove_volume(&name).await {
323                refused.push(Refused {
324                    name,
325                    error: error.to_string(),
326                });
327                continue;
328            }
329            emptied.push(Emptied {
330                name,
331                size: ByteSize(size),
332            });
333        }
334    }
335    let mut closed = Vec::new();
336    for (name, &size) in &sizes {
337        let past = slots.iter().any(|(profile, &count)| {
338            names
339                .slot_of(profile, name)
340                .is_some_and(|slot| slot > count)
341        });
342        if past {
343            // As a stopped container kept as a failed job's record still uses it.
344            if let Err(error) = docker.remove_volume(name).await {
345                refused.push(Refused {
346                    name: name.clone(),
347                    error: error.to_string(),
348                });
349                continue;
350            }
351            closed.push(Emptied {
352                name: name.clone(),
353                size: ByteSize(size),
354            });
355        }
356    }
357    Ok(Trimmed {
358        sizes,
359        emptied,
360        closed,
361        refused,
362        free,
363    })
364}
365
366/// What the slots' Docker Engines may keep together, in bytes: a quarter of Docker's disk not otherwise used, which
367/// is what is free plus what the slot volumes already hold, measured together (docs/adr/0054).
368pub fn docker_disk(
369    names: &Names,
370    config: &config::Config,
371    free: u64,
372    sizes: &BTreeMap<String, u64>,
373) -> u64 {
374    let held: u64 = config
375        .profiles
376        .iter()
377        .flat_map(|(profile, settings)| {
378            (1..=settings.slots as usize).map(move |slot| names.slot(profile, slot))
379        })
380        .filter_map(|name| sizes.get(&name))
381        .sum();
382    free.saturating_add(held) / DOCKER_DISK_SHARE
383}
384
385/// One slot's share of `disk` among `slots`, in whole GiB as `image/layers.sh` takes it: at least 1, at most
386/// `DOCKER_SLOT_MAX_GIB`.
387pub fn docker_budget_gib(disk: u64, slots: u64) -> u64 {
388    ((disk / slots.max(1)) >> 30).clamp(1, DOCKER_SLOT_MAX_GIB)
389}
390
391/// The slots' Docker Engines together keep at most one part in this of Docker's disk not otherwise used.
392const DOCKER_DISK_SHARE: u64 = 4;
393
394/// The most one slot's Docker Engine keeps, in GiB: the image's own default, for a daemon that sets none.
395const DOCKER_SLOT_MAX_GIB: u64 = 20;
396
397/// What `empty_volumes` came to.
398#[derive(Debug, Default)]
399pub struct Cleared {
400    /// The volumes it removed.
401    pub removed: Vec<String>,
402    /// The volumes it left, each with why.
403    pub refused: Vec<Refusal>,
404}
405
406/// A volume `empty_volumes` left.
407#[derive(Debug)]
408pub struct Refusal {
409    /// The volume's name.
410    pub volume: String,
411    /// Why, as the log says it.
412    pub why: String,
413    /// What kept it.
414    pub cause: Kept,
415}
416
417/// What kept a volume `empty_volumes` did not remove.
418#[derive(Debug, Clone, Copy, PartialEq, Eq)]
419pub enum Kept {
420    /// A running container mounts it.
421    Runs,
422    /// A stopped container that is not runnerd's mounts it.
423    NotRunnerds,
424    /// Docker's error, such as an outage, a 5xx or a timeout.
425    Docker,
426}
427
428impl Refusal {
429    /// Whether a container mounts it that `empty_volumes` leaves: a running one, or one that is not runnerd's.
430    pub fn held(&self) -> bool {
431        self.cause != Kept::Docker
432    }
433}
434
435/// Removes each of `volumes` that exists, after the containers of runnerd's that mount it and no longer run, such as
436/// a runner or a seed a crash stopped: Docker refuses to remove a volume a container mounts, even a stopped one. It
437/// leaves a volume a running container mounts, or a container that is not runnerd's, and removes no such container.
438pub async fn empty_volumes(docker: &impl DockerApi, volumes: &[String]) -> Cleared {
439    let mut emptied = Cleared::default();
440    for volume in volumes {
441        match empty_volume(docker, volume).await {
442            Ok(()) => emptied.removed.push(volume.clone()),
443            Err(refusal) => emptied.refused.push(refusal),
444        }
445    }
446    emptied
447}
448
449/// Removes `volume`, if it exists, once nothing of runnerd's mounts it.
450async fn empty_volume(docker: &impl DockerApi, volume: &str) -> Result<(), Refusal> {
451    let refused = |why: String, cause: Kept| Refusal {
452        volume: volume.to_owned(),
453        why,
454        cause,
455    };
456    let failed = |error: Error| refused(error.to_string(), Kept::Docker);
457    for holder in docker.holders(volume).await.map_err(failed)? {
458        let name = if holder.name.is_empty() {
459            &holder.id
460        } else {
461            &holder.name
462        };
463        if holder.running {
464            return Err(refused(
465                format!("container {name} mounts it and runs"),
466                Kept::Runs,
467            ));
468        }
469        if !holder
470            .labels
471            .keys()
472            .any(|label| label.starts_with("runnerd."))
473        {
474            return Err(refused(
475                format!("container {name}, which is not runnerd's, mounts it"),
476                Kept::NotRunnerds,
477            ));
478        }
479        match docker.remove(&holder.id).await {
480            Err(error) if !is_not_found(&error) => {
481                return Err(refused(
482                    format!("container {name}, which mounts it, cannot be removed: {error}"),
483                    Kept::Docker,
484                ));
485            }
486            _ => {}
487        }
488    }
489    docker.remove_volume(volume).await.map_err(failed)
490}
491
492/// Every cache volume the config's profiles use, what it holds and its limit, with size 0.
493pub fn volumes(names: &Names, config: &config::Config) -> Vec<api::CacheVolume> {
494    let volume = |name: String, kind: api::CacheKind, limit: Option<u64>| api::CacheVolume {
495        name,
496        kind,
497        bytes: 0,
498        limit,
499    };
500    let mut volumes = Vec::new();
501    for (profile, settings) in &config.profiles {
502        volumes.push(volume(
503            names.base(profile),
504            api::CacheKind::Snapshot {
505                profile: profile.clone(),
506            },
507            Some(config.caches.snapshot_max.0),
508        ));
509        for slot in 1..=settings.slots as usize {
510            volumes.push(volume(
511                names.slot(profile, slot),
512                api::CacheKind::Slot {
513                    profile: profile.clone(),
514                    slot,
515                },
516                None,
517            ));
518        }
519    }
520    volumes.push(volume(
521        names.pnpm(),
522        api::CacheKind::Pnpm,
523        Some(config.caches.pnpm.max_size.0),
524    ));
525    volumes.push(volume(
526        names.maven(),
527        api::CacheKind::Maven,
528        Some(config.caches.maven.max_size.0),
529    ));
530    if config.caches.mirror.enabled {
531        volumes.push(volume(
532            names.mirror(),
533            api::CacheKind::Mirror,
534            Some(config.caches.mirror.max_size.0),
535        ));
536    }
537    if config.caches.build.enabled {
538        volumes.push(volume(
539            names.build_cache(),
540            api::CacheKind::Build,
541            Some(config.caches.build.max_size.0),
542        ));
543    }
544    volumes.sort_by(|a, b| a.name.cmp(&b.name));
545    volumes
546}
547
548/// What `reseed` runs on: the profile's image and limits, its slots and the isolation backend.
549#[derive(Debug)]
550pub struct Reseed<'profile> {
551    /// Where the profile's volumes are.
552    pub names: &'profile Names,
553    /// The profile to reseed.
554    pub profile: &'profile str,
555    /// The profile's runner image, which holds the seed script.
556    pub image: &'profile str,
557    /// How many slots the profile has; each slot's layer changes are dropped.
558    pub slots: usize,
559    /// The systemd slice the seed container runs in.
560    pub slice: &'profile str,
561    /// The profile's CPUs.
562    pub cpus: u32,
563    /// The profile's memory.
564    pub memory: ByteSize,
565    /// Slot 1's layers must have changed at least this much (`caches.reseed_at`).
566    pub min_changes: ByteSize,
567    /// A snapshot that would outgrow this starts over (`caches.snapshot_max`).
568    pub max_size: ByteSize,
569    /// How the seed container is isolated, as runners are.
570    pub isolation: &'profile dyn Isolation,
571}
572
573/// Makes slot 1's layers the profile's new snapshot and drops every slot's own layer changes (`image/seed.sh`),
574/// if slot 1 changed at least `min_changes`. A snapshot that would outgrow `max_size` starts its tools layer, then
575/// its Android layer, over from the image until the rest fits, and every layer when that is not enough, each with
576/// every slot's changes to it (docs/adr/0065, docs/adr/0152); after a reseed that did not finish, it starts over
577/// whatever the limits (docs/adr/0066). Call only while none of the profile's slots runs. Returns the seed container's output.
578pub async fn reseed(docker: &impl DockerApi, reseed: &Reseed<'_>) -> Result<String, Error> {
579    let names = reseed.names;
580    let mut mounts = vec![volume(names.base(reseed.profile), "/base", false)];
581    mounts.extend((1..=reseed.slots).map(|slot| {
582        volume(
583            names.slot(reseed.profile, slot),
584            &format!("/workers/{slot}"),
585            false,
586        )
587    }));
588    for mount in &mounts {
589        if let Some(source) = &mount.source {
590            docker.ensure_volume(source).await?;
591        }
592    }
593    let name = names.seed(reseed.profile);
594    let spec = RunnerSpec {
595        name: name.clone(),
596        image: reseed.image.to_owned(),
597        profile: reseed.profile.to_owned(),
598        slice: reseed.slice.to_owned(),
599        cpus: reseed.cpus,
600        memory: reseed.memory,
601        env: Vec::new(),
602        mounts,
603        network: Some("none".to_owned()),
604        labels: BTreeMap::from([("runnerd.seed".to_owned(), String::new())]),
605        entrypoint: Some(vec![
606            "/usr/local/sbin/runnerd-seed".to_owned(),
607            reseed.min_changes.0.to_string(),
608            reseed.max_size.0.to_string(),
609        ]),
610        jit_config: None,
611    };
612    // A seed container left by a daemon that died while seeding.
613    match docker.remove(&name).await {
614        Err(error) if !is_not_found(&error) => return Err(error),
615        _ => {}
616    }
617    let id = match docker.start(&spec, reseed.isolation).await {
618        Ok(id) => id,
619        // A start whose answer was lost may have left the container running: it ends only once Docker says so.
620        Err(failed) => match failed.container {
621            Some(container) if failed.may_have_run => {
622                let _ = finish_reseed(docker, &container).await;
623                return Err(failed.error);
624            }
625            _ => return Err(failed.error),
626        },
627    };
628    finish_reseed(docker, &id).await
629}
630
631/// What `start_layers_over` runs on: a slot's volume a host crash may have torn and the guard could not empty, and
632/// the image, slice and isolation of a runner.
633#[derive(Debug)]
634pub struct StartOver<'volume> {
635    /// Where runnerd's volumes are.
636    pub names: &'volume Names,
637    /// The slot's volume.
638    pub volume: &'volume str,
639    /// A runner image, for its shell: any profile's will do.
640    pub image: &'volume str,
641    /// The systemd slice the container runs in.
642    pub slice: &'volume str,
643    /// How the container is isolated, as runners are.
644    pub isolation: &'volume dyn Isolation,
645}
646
647/// The memory the container of `start_layers_over` may use: it only deletes.
648const START_OVER_MEMORY: ByteSize = ByteSize(256 << 20);
649
650/// Deletes everything in a slot's volume but its Docker Engine's store (`image/layers.sh`'s `DOCKER_DATA_ROOT`), in
651/// a container that mounts it: its layers' changes and work directories, and the record of the snapshot they were
652/// made against. The slot's next runner starts them over from the snapshot, and a reseed finds nothing of slot 1's
653/// to copy (ADR 0142). For a volume the guard could not remove, as another container holds it: Docker lets a second
654/// container mount it. A container an earlier daemon left is removed first.
655pub async fn start_layers_over(
656    docker: &impl DockerApi,
657    start: &StartOver<'_>,
658) -> Result<(), Error> {
659    let name = start.names.start_over();
660    match docker.remove(&name).await {
661        Err(error) if !is_not_found(&error) => return Err(error),
662        _ => {}
663    }
664    let spec = RunnerSpec {
665        name: name.clone(),
666        image: start.image.to_owned(),
667        profile: String::new(),
668        slice: start.slice.to_owned(),
669        cpus: 1,
670        memory: START_OVER_MEMORY,
671        env: Vec::new(),
672        mounts: vec![volume(start.volume.to_owned(), "/state", false)],
673        network: Some("none".to_owned()),
674        labels: BTreeMap::from([("runnerd.seed".to_owned(), String::new())]),
675        entrypoint: Some(
676            [
677                "find",
678                "/state",
679                "-mindepth",
680                "1",
681                "-maxdepth",
682                "1",
683                "!",
684                "-name",
685                "docker",
686                "-exec",
687                "rm",
688                "-rf",
689                "{}",
690                "+",
691            ]
692            .map(str::to_owned)
693            .to_vec(),
694        ),
695        jit_config: None,
696    };
697    let id = docker.start(&spec, start.isolation).await?;
698    // Once: the tick awaits it. A container Docker cannot say ended is an error, so the caches stay torn, and the
699    // next try removes it before it starts another.
700    match finish_once(docker, &id).await {
701        Finished::Ended(result) => result.map(|_| ()),
702        Finished::Unconfirmed(error) => Err(error),
703    }
704}
705
706/// What a measure container (`image/seed.sh measure`) runs on: the profile's image, its slot 1 and the snapshot,
707/// both read-only, and the isolation backend.
708#[derive(Debug)]
709pub struct Measure<'profile> {
710    /// Where the profile's volumes are.
711    pub names: &'profile Names,
712    /// The profile whose slot 1 is measured.
713    pub profile: &'profile str,
714    /// The profile's runner image, which holds the seed script.
715    pub image: &'profile str,
716    /// The systemd slice the measure container runs in.
717    pub slice: &'profile str,
718    /// How the measure container is isolated, as runners are.
719    pub isolation: &'profile dyn Isolation,
720}
721
722/// What a measure found: what a reseed would weigh against `caches.reseed_at`.
723#[derive(Clone, Copy, Debug, PartialEq, Eq)]
724pub enum Measured {
725    /// Slot 1's changes to the layers hold this many bytes, its Docker Engine's data aside.
726    Changed(u64),
727    /// The profile's last reseed did not finish (docs/adr/0066): the next one starts over, whatever slot 1 changed.
728    Unfinished,
729}
730
731/// The memory a measure container may use: it only adds up file sizes.
732const MEASURE_MEMORY: ByteSize = ByteSize(256 << 20);
733
734/// Measures what the profile's slot 1 changed (`image/seed.sh measure`), in a container that mounts slot 1's
735/// volume and the snapshot read-only, so it may run while the profile's runners do. A measure container an earlier
736/// daemon left is removed first.
737pub async fn measure(docker: &impl DockerApi, measure: &Measure<'_>) -> Result<Measured, Error> {
738    let spec = measure_spec(measure);
739    match docker.remove(&spec.name).await {
740        Err(error) if !is_not_found(&error) => return Err(error),
741        _ => {}
742    }
743    let id = docker.start(&spec, measure.isolation).await?;
744    let output = finish_reseed(docker, &id).await?;
745    measured(&output).ok_or_else(|| Error::DockerContainerWaitError {
746        error: format!("measuring printed {:?}", output.trim()),
747        code: 0,
748    })
749}
750
751/// The container `measure` runs: `seed.sh measure` on slot 1's volume and the snapshot, both read-only.
752pub fn measure_spec(measure: &Measure<'_>) -> RunnerSpec {
753    let names = measure.names;
754    RunnerSpec {
755        name: names.measure(measure.profile),
756        image: measure.image.to_owned(),
757        profile: measure.profile.to_owned(),
758        slice: measure.slice.to_owned(),
759        cpus: 1,
760        memory: MEASURE_MEMORY,
761        env: Vec::new(),
762        mounts: vec![
763            volume(names.base(measure.profile), "/base", true),
764            volume(names.slot(measure.profile, 1), "/workers/1", true),
765        ],
766        network: Some("none".to_owned()),
767        // As a seed container's: a restart finds it beside them (`seed`), and removes it.
768        labels: BTreeMap::from([("runnerd.seed".to_owned(), String::new())]),
769        entrypoint: Some(vec![
770            "/usr/local/sbin/runnerd-seed".to_owned(),
771            "measure".to_owned(),
772        ]),
773        jit_config: None,
774    }
775}
776
777/// What the output of `image/seed.sh measure` says, in its last line that says it: a warning before it, such as
778/// one of `du` about a file a job deleted while it counted, leaves the number as advisory as it is anyway.
779fn measured(output: &str) -> Option<Measured> {
780    output.lines().rev().map(str::trim).find_map(|line| {
781        if line == "the last reseed did not finish" {
782            return Some(Measured::Unfinished);
783        }
784        line.strip_prefix("slot 1 changed ")?
785            .strip_suffix(" bytes")?
786            .parse()
787            .ok()
788            .map(Measured::Changed)
789    })
790}
791
792/// Whether the output of a reseed (`image/seed.sh`) says it replaced the snapshot, and so dropped every slot's own
793/// layer changes, rather than left them because slot 1 changed too little.
794pub fn seeded(output: &str) -> bool {
795    !output.trim_start().starts_with("not seeded")
796}
797
798/// How long `finish_reseed` waits before it asks Docker again about a container it could not tell had ended.
799const ASK_AGAIN: Duration = Duration::from_secs(10);
800
801/// Waits for a seed or measure container to exit, removes it, and returns its output, in a background task. Also takes
802/// over a seed container an earlier daemon left running: stopping it between its `rm` and `mv` would lose that
803/// snapshot layer. It returns only once Docker said the container exited or is gone: one whose wait and removal both
804/// failed, as when the Engine's API went away, may still be rewriting the snapshot or a slot's layers, so it asks
805/// again every `ASK_AGAIN`, and the profile reseeds until then (docs/adr/0152). Never awaited in the tick.
806pub async fn finish_reseed(docker: &impl DockerApi, container: &str) -> Result<String, Error> {
807    loop {
808        match finish_once(docker, container).await {
809            Finished::Ended(result) => return result,
810            Finished::Unconfirmed(error) => {
811                warn!(
812                    "cannot tell whether {container} ended, so what it writes stays held: asking again in {}s: \
813                     {error}",
814                    ASK_AGAIN.as_secs()
815                );
816                tokio::time::sleep(ASK_AGAIN).await;
817            }
818        }
819    }
820}
821
822/// What one attempt of `finish_reseed` found.
823enum Finished {
824    /// The container exited or is gone: its output, or why there is none.
825    Ended(Result<String, Error>),
826    /// Docker said neither: why its removal failed.
827    Unconfirmed(Error),
828}
829
830/// Waits for the container once and removes it.
831async fn finish_once(docker: &impl DockerApi, container: &str) -> Finished {
832    let waited = docker.wait(container).await;
833    let exited = waited.is_ok();
834    let result = async {
835        let code = waited?;
836        let output = docker.logs(container).await?;
837        if code == 0 {
838            Ok(output)
839        } else {
840            Err(Error::DockerContainerWaitError {
841                error: format!("seeding exited {code}: {}", output.trim()),
842                code,
843            })
844        }
845    }
846    .await;
847    match docker.remove(container).await {
848        Err(error) if is_not_found(&error) => Finished::Ended(result),
849        Err(error) if !exited => Finished::Unconfirmed(error),
850        Err(error) => Finished::Ended(result.and(Err(error))),
851        Ok(()) => Finished::Ended(result),
852    }
853}
854
855#[cfg(test)]
856mod tests {
857    use std::path::Path;
858    use std::process::Command;
859
860    use super::docker_budget_gib;
861    use crate::testdir::TestDir;
862
863    /// Runs `script` in bash with `image/layers.sh` sourced and `args` as its arguments; returns what it printed.
864    #[allow(
865        clippy::unwrap_used,
866        reason = "a test helper: a script that does not run fails the test"
867    )]
868    fn with_layers(script: &str, args: &[&Path]) -> String {
869        let layers = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../image/layers.sh");
870        let output = Command::new("bash")
871            .arg("-euc")
872            .arg(format!(". \"$0\"; {script}"))
873            .arg(layers)
874            .args(args)
875            .output()
876            .unwrap();
877        assert!(
878            output.status.success(),
879            "the script exits 0: {}",
880            String::from_utf8_lossy(&output.stderr)
881        );
882        String::from_utf8(output.stdout).unwrap()
883    }
884
885    #[test]
886    #[allow(
887        clippy::unwrap_used,
888        reason = "a test: a file it cannot write fails it"
889    )]
890    fn no_credential_a_job_leaves_in_the_cache_layer_reaches_the_slots_next_job() {
891        let listed = with_layers("printf '%s\\n' \"${CACHE_SECRETS[@]}\"", &[]);
892        let secrets: Vec<&str> = listed.lines().collect();
893        for credential in [
894            "cargo/credentials.toml",
895            "gradle/gradle.properties",
896            "gradle/init.d",
897            "m2/settings.xml",
898            "m2/settings-security.xml",
899            "huggingface/token",
900        ] {
901            assert!(
902                secrets.contains(&credential),
903                "{credential} is in CACHE_SECRETS: {secrets:?}"
904            );
905        }
906        let dir = TestDir::new("cache-secrets");
907        let upper = dir.0.join("upper");
908        let write = |path: &str| {
909            let path = upper.join(path);
910            std::fs::create_dir_all(path.parent().unwrap()).unwrap();
911            std::fs::write(path, "token").unwrap();
912        };
913        // Each as a file, and as a directory a job made there, as Gradle's init.d is.
914        for secret in &secrets {
915            write(&format!("files/{secret}"));
916            write(&format!("dirs/{secret}/held"));
917        }
918        let kept = [
919            "npm/_cacache/index",
920            "m2/wrapper/dists/maven.zip",
921            "gradle/caches/jars",
922        ];
923        for path in kept {
924            write(&format!("files/{path}"));
925        }
926        with_layers(
927            "drop_cache_secrets \"$1/files\"; drop_cache_secrets \"$1/dirs\"",
928            &[&upper],
929        );
930        for secret in &secrets {
931            for copy in ["files", "dirs"] {
932                let path = upper.join(copy).join(secret);
933                assert!(!path.exists(), "{} is gone", path.display());
934            }
935        }
936        for path in kept {
937            assert!(upper.join("files").join(path).exists(), "{path} stays");
938        }
939    }
940
941    #[test]
942    #[allow(
943        clippy::unwrap_used,
944        reason = "a test: a file it cannot write fails it"
945    )]
946    fn what_slot_1_changed_counts_its_layers_and_not_its_docker_engines_data() {
947        let dir = TestDir::new("layer-changes");
948        let state = dir.0.join("state");
949        let write = |path: &str, bytes: usize| {
950            let path = state.join(path);
951            std::fs::create_dir_all(path.parent().unwrap()).unwrap();
952            std::fs::write(path, vec![0u8; bytes]).unwrap();
953        };
954        let changes = || -> u64 {
955            with_layers("layer_changes \"$1\"", &[&state])
956                .trim()
957                .parse()
958                .unwrap()
959        };
960        write("cache/upper/npm/index", 1);
961        let before = changes();
962        // A job's images in the slot's Docker Engine: a reseed leaves them, so they must not reach `reseed_at`.
963        write("docker/overlay2/layer", 1 << 20);
964        // Layer work directories hold nothing a reseed copies.
965        write("tools/work/scratch", 1 << 20);
966        assert_eq!(changes(), before);
967        write("tools/upper/node/bin", 100_000);
968        write("android/upper/ndk/lib", 200_000);
969        let after = changes();
970        assert!(
971            after >= before + 300_000 && after < before + 400_000,
972            "the tools and android layers count: {before} then {after}"
973        );
974    }
975
976    #[test]
977    fn a_slots_docker_budget_is_its_share_between_1_and_20_gib() {
978        assert_eq!(docker_budget_gib(100 << 30, 10), 10);
979        assert_eq!(
980            docker_budget_gib(1 << 40, 4),
981            20,
982            "a large disk still stops at 20 GiB"
983        );
984        assert_eq!(
985            docker_budget_gib(1 << 30, 13),
986            1,
987            "a full disk still keeps 1 GiB"
988        );
989        assert_eq!(docker_budget_gib(1 << 30, 0), 1);
990    }
991}