Skip to main content

runnerd_core/cache/
mirror.rs

1//! The Docker Hub mirror's storage: finding the links in it whose blob is gone or empty, removing them before the
2//! mirror starts, and the doctor's check of a running mirror (docs/adr/0144). It reaches the storage only through a
3//! container of the mirror's image, never the host's view of the volume; it writes only while no mirror runs, and
4//! only reads a running one's.
5
6use std::collections::HashMap;
7use std::time::Duration;
8
9use bollard::models::{ContainerCreateBody, HostConfig};
10use futures_util::TryStreamExt as _;
11
12use super::{MIRROR_IMAGE, Names, volume};
13use crate::api::{self, Check, CheckStatus};
14use crate::docker::{DockerApi, Error, is_not_found};
15
16/// What reads the mirror's links, in the image's busybox `sh`: `repair` removes those whose blob is gone, `check` only
17/// lists them.
18const LINKS_SCRIPT: &str = include_str!("mirror_links.sh");
19
20/// The registry's setting for its blob descriptor cache. runnerd sets it empty, which turns the cache off: the
21/// image's config keeps it in memory, where it goes on saying a blob is stored after the proxy deleted it.
22pub(super) const DESCRIPTOR_CACHE_ENV: &str = "REGISTRY_STORAGE_CACHE_BLOBDESCRIPTOR";
23
24/// How long the repair before a start may take. The engine waits for it, so it is short: a few thousand links take
25/// about a second and 100,000 about 8 seconds. Past it the mirror starts as it is, which only an empty blob a crash
26/// left can hurt, since the mirror keeps no blob descriptor cache.
27const REPAIR_DEADLINE: Duration = Duration::from_secs(15);
28
29/// How many paths the doctor's fix removes by name; past it, the fix has runnerd repair the storage.
30const FIX_PATHS: usize = 8;
31
32/// The memory the repair container may use: it only reads small files and removes some.
33const REPAIR_MEMORY: i64 = 64 << 20;
34
35/// The mirror's storage as its container sees it.
36const V2: &str = "/var/lib/registry/docker/registry/v2";
37
38/// How many of the links removed or found the log line and the doctor name.
39const NAMED: usize = 3;
40
41/// What is wrong with a link in the mirror's storage.
42#[derive(Debug, Clone, Copy, PartialEq, Eq)]
43pub enum Fault {
44    /// Its blob's data is missing: the registry fetches the blob again from Docker Hub, unless its blob descriptor
45    /// cache still holds it, and then answers with no data.
46    Gone,
47    /// Its blob's data is empty: the registry answers with no data, and keeps the empty file when it fetches the
48    /// blob again. The data goes with the link.
49    Empty,
50    /// It holds no digest.
51    Garbled,
52}
53
54impl Fault {
55    /// Whether the mirror answers a pull of it with no data: always for an empty blob, and for a missing one while
56    /// the mirror keeps the blob descriptor cache.
57    fn serves_nothing(self, cached: bool) -> bool {
58        match self {
59            Fault::Empty => true,
60            Fault::Gone => cached,
61            Fault::Garbled => false,
62        }
63    }
64}
65
66/// A link in the mirror's storage whose blob is gone or empty, or that holds no digest.
67#[derive(Debug, Clone, PartialEq, Eq)]
68pub struct Dangling {
69    /// What is wrong with it.
70    pub fault: Fault,
71    /// The repository the link is in, such as `library/caddy`.
72    pub repository: String,
73    /// The digest the link holds, such as `sha256:881f…`; `None` when it holds none.
74    pub digest: Option<String>,
75    /// The link's path under the storage's `docker/registry/v2`.
76    pub link: String,
77}
78
79impl Dangling {
80    /// What a repair removes for it, as paths in the mirror's container: the link, and an empty blob's data.
81    fn removes(&self) -> Vec<String> {
82        let mut paths = vec![format!("{V2}/{}", self.link)];
83        if let (Fault::Empty, Some((algorithm, hex))) = (
84            self.fault,
85            self.digest
86                .as_deref()
87                .and_then(|digest| digest.split_once(':')),
88        ) && let Some(prefix) = hex.get(..2)
89        {
90            paths.push(format!("{V2}/blobs/{algorithm}/{prefix}/{hex}/data"));
91        }
92        paths
93    }
94}
95
96impl std::fmt::Display for Dangling {
97    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
98        match &self.digest {
99            Some(digest) => write!(f, "{}@{digest}", self.repository)?,
100            None => write!(f, "{} ({})", self.repository, self.link)?,
101        }
102        if self.fault == Fault::Empty {
103            f.write_str(" (empty)")?;
104        }
105        Ok(())
106    }
107}
108
109/// What a read of the mirror's links found.
110#[derive(Debug, Clone, PartialEq, Eq)]
111pub struct Links {
112    /// How many links it read.
113    pub count: usize,
114    /// Those whose blob is gone or empty, in the order it read them.
115    pub dangling: Vec<Dangling>,
116}
117
118impl Links {
119    /// What `mirror_links.sh` printed; `None` when it was cut off before its last line.
120    fn parse(output: &str) -> Option<Links> {
121        let mut dangling = Vec::new();
122        let mut count = None;
123        for line in output.lines() {
124            let fault = match line.split_once(' ') {
125                Some(("gone", rest)) => Some((Fault::Gone, rest)),
126                Some(("empty", rest)) => Some((Fault::Empty, rest)),
127                Some(("garbled", rest)) => Some((Fault::Garbled, rest)),
128                _ => None,
129            };
130            if let Some((fault, found)) = fault {
131                let Some((digest, link)) = found.split_once(' ') else {
132                    continue;
133                };
134                dangling.push(Dangling {
135                    fault,
136                    repository: repository_of(link),
137                    digest: (digest != "-").then(|| digest.to_owned()),
138                    link: link.to_owned(),
139                });
140            } else if let Some(read) = line.strip_prefix("links ") {
141                count = read.trim().parse().ok();
142            }
143        }
144        Some(Links {
145            count: count?,
146            dangling,
147        })
148    }
149}
150
151/// The first few of `dangling`, and how many more.
152fn named(dangling: &[&Dangling]) -> String {
153    let mut named: Vec<String> = dangling
154        .iter()
155        .take(NAMED)
156        .map(ToString::to_string)
157        .collect();
158    if dangling.len() > NAMED {
159        named.push(format!("{} more", dangling.len() - NAMED));
160    }
161    named.join(", ")
162}
163
164/// The repository a link is in: its path between `repositories/` and `/_layers/` or `/_manifests/`.
165fn repository_of(link: &str) -> String {
166    let path = link.strip_prefix("repositories/").unwrap_or(link);
167    ["/_layers/", "/_manifests/"]
168        .iter()
169        .find_map(|marker| path.find(marker).map(|at| &path[..at]))
170        .unwrap_or(path)
171        .to_owned()
172}
173
174/// What the repair before a start came to. It never keeps the mirror from starting.
175#[derive(Debug, Clone, PartialEq, Eq)]
176pub enum Repair {
177    /// Every link leads to a stored blob, or there are none, as in a new volume.
178    Clean,
179    /// It removed these links.
180    Removed(Links),
181    /// It could not read or remove them all, with why; those left are found again at the next start.
182    Failed(String),
183}
184
185impl Repair {
186    /// Logs `line`: a failure as a warning.
187    pub fn log(&self) {
188        match (self, self.line()) {
189            (Repair::Failed(_), Some(line)) => tracing::warn!("{line}"),
190            (_, Some(line)) => tracing::info!("{line}"),
191            (_, None) => {}
192        }
193    }
194
195    /// The line the daemon logs for it; none when there was nothing to repair.
196    pub fn line(&self) -> Option<String> {
197        match self {
198            Repair::Clean => None,
199            Repair::Removed(links) => Some(format!(
200                "removed {} of the mirror's {} links whose blob is gone, before starting it: {}",
201                links.dangling.len(),
202                links.count,
203                named(&links.dangling.iter().collect::<Vec<_>>())
204            )),
205            Repair::Failed(why) => Some(format!(
206                "cannot repair the mirror's storage, starting it as it is: {why}"
207            )),
208        }
209    }
210}
211
212/// What `mirror_up` did.
213#[derive(Debug, Clone, PartialEq, Eq)]
214pub enum MirrorUp {
215    /// It ran already: its storage is left as it is.
216    Running,
217    /// It started, after the repair of its storage.
218    Started(Repair),
219}
220
221/// Removes the mirror's links whose blob is gone or empty, in a container of the mirror's image that mounts its
222/// storage. Call only while no mirror runs. A repair container an earlier daemon left, as a restart while it ran
223/// leaves it, is removed first; the repair reads every link again, so one cut off is done over.
224pub async fn repair(docker: &impl DockerApi, names: &Names, slice: &str) -> Repair {
225    let name = names.mirror_repair();
226    let repaired = tokio::time::timeout(REPAIR_DEADLINE, async {
227        remove_left(docker, &name).await?;
228        docker.run(&name, repair_body(names, slice)).await?;
229        let code = docker.wait(&name).await?;
230        let output = docker.logs(&name).await?;
231        Ok::<_, Error>((code, output))
232    })
233    .await;
234    // Also when the deadline cut it off: no repair runs beside the mirror.
235    let removed = remove_left(docker, &name).await;
236    let (code, output) = match repaired {
237        Err(_) => {
238            return Repair::Failed(format!(
239                "it did not finish within {}",
240                api::duration(REPAIR_DEADLINE.as_secs())
241            ));
242        }
243        Ok(Err(error)) => return Repair::Failed(error.to_string()),
244        Ok(Ok(done)) => done,
245    };
246    if let Err(error) = removed {
247        return Repair::Failed(format!("cannot remove its container: {error}"));
248    }
249    match Links::parse(&output) {
250        Some(links) if code == 0 && links.dangling.is_empty() => Repair::Clean,
251        Some(links) if code == 0 => Repair::Removed(links),
252        _ => Repair::Failed(failure(code, &output)),
253    }
254}
255
256/// Why a repair that exited `code` failed: how many faulty links it printed and its last other line, such as an error
257/// `rm` wrote, never its whole output, which names every link.
258fn failure(code: i64, output: &str) -> String {
259    let is_fault = |line: &str| {
260        ["gone ", "empty ", "garbled "]
261            .iter()
262            .any(|kind| line.starts_with(kind))
263    };
264    let faults = output.lines().filter(|line| is_fault(line)).count();
265    let said = output
266        .lines()
267        .rev()
268        .map(str::trim)
269        .find(|line| !line.is_empty() && !is_fault(line) && !line.starts_with("links "));
270    match said {
271        Some(said) => format!("it exited {code} after {faults} faulty links: {said}"),
272        None => format!("it exited {code} after {faults} faulty links"),
273    }
274}
275
276/// Removes the container `name` if there is one.
277async fn remove_left(docker: &impl DockerApi, name: &str) -> Result<(), Error> {
278    match docker.remove(name).await {
279        Err(error) if !is_not_found(&error) => Err(error),
280        _ => Ok(()),
281    }
282}
283
284/// The repair container: `mirror_links.sh repair` on the mirror's storage, without a network.
285pub(super) fn repair_body(names: &Names, slice: &str) -> ContainerCreateBody {
286    ContainerCreateBody {
287        image: Some(MIRROR_IMAGE.to_owned()),
288        entrypoint: Some(script("repair")),
289        labels: Some(HashMap::from([(
290            "runnerd.mirror-repair".to_owned(),
291            String::new(),
292        )])),
293        host_config: Some(HostConfig {
294            cgroup_parent: Some(slice.to_owned()),
295            memory: Some(REPAIR_MEMORY),
296            memory_swap: Some(REPAIR_MEMORY),
297            nano_cpus: Some(1_000_000_000),
298            mounts: Some(vec![volume(names.mirror(), "/var/lib/registry", false)]),
299            network_mode: Some("none".to_owned()),
300            ..Default::default()
301        }),
302        ..Default::default()
303    }
304}
305
306/// `mirror_links.sh` in `mode`, as a command for the image's `sh`.
307fn script(mode: &str) -> Vec<String> {
308    ["/bin/sh", "-c", LINKS_SCRIPT, "mirror-links", mode]
309        .map(str::to_owned)
310        .to_vec()
311}
312
313/// What the doctor found of the mirror's storage.
314#[derive(Debug, Clone, PartialEq, Eq)]
315pub enum Found {
316    /// The mirror is not running: runnerd repairs its storage before it starts it.
317    NotRunning,
318    /// It runs, and its links read so.
319    Read {
320        /// Its links.
321        links: Links,
322        /// Whether it keeps the blob descriptor cache, as a mirror an older runnerd started does.
323        cached: bool,
324    },
325    /// It could not be read, with why.
326    Unread(String),
327}
328
329/// Reads a running mirror's links, without changing them, for the doctor.
330pub async fn check(docker: &impl DockerApi, names: &Names) -> Found {
331    match docker.inspect(&names.mirror()).await {
332        Ok(state) if state.running => {}
333        Ok(_) => return Found::NotRunning,
334        Err(error) if is_not_found(&error) => return Found::NotRunning,
335        Err(error) => return Found::Unread(error.to_string()),
336    }
337    let output = match docker.exec(&names.mirror(), script("check")).await {
338        Ok(output) => output.try_concat().await,
339        Err(error) => Err(error),
340    };
341    let output = match output {
342        Ok(output) => String::from_utf8_lossy(&output).into_owned(),
343        Err(error) => return Found::Unread(error.to_string()),
344    };
345    let Some(links) = Links::parse(&output) else {
346        return Found::Unread(format!("reading its links printed {:?}", output.trim()));
347    };
348    // `cache` with nothing after it is runnerd's setting.
349    let cached = !output.lines().any(|line| line == "cache ");
350    Found::Read { links, cached }
351}
352
353/// The doctor's `mirror` check: it fails while a link answers a pull with no data, which an empty blob does, and a
354/// missing one while the mirror keeps the blob descriptor cache. A mirror that keeps that cache warns; a missing blob
355/// without it is only noted, as the mirror fetches it again.
356pub fn mirror_check(names: &Names, found: &Found) -> Check {
357    let check = |status, detail: String, fix: Option<String>| Check {
358        name: "mirror".to_owned(),
359        status,
360        detail,
361        fix,
362    };
363    let (links, cached) = match found {
364        Found::NotRunning => {
365            return check(
366                CheckStatus::Info,
367                "not running; runnerd removes the links whose blob is gone or empty from its storage before it \
368                 starts it"
369                    .to_owned(),
370                None,
371            );
372        }
373        Found::Unread(why) => {
374            return check(
375                CheckStatus::Skip,
376                format!("cannot read its storage: {why}"),
377                None,
378            );
379        }
380        Found::Read { links, cached } => (links, *cached),
381    };
382    let all: Vec<&Dangling> = links.dangling.iter().collect();
383    let broken: Vec<&Dangling> = links
384        .dangling
385        .iter()
386        .filter(|dangling| dangling.fault.serves_nothing(cached))
387        .collect();
388    let cache = "it keeps the blob descriptor cache, as a mirror an older runnerd started does, so once one \
389                 repository's copy of a layer expires, pulls of that layer from another get no data; runnerd starts \
390                 it without that cache when its jobs have finished";
391    if !broken.is_empty() {
392        let removes: Vec<String> = all.iter().flat_map(|dangling| dangling.removes()).collect();
393        let fix = if removes.len() > FIX_PATHS {
394            format!(
395                "remove the mirror; jobs pull from Docker Hub directly until runnerd starts it again for the first \
396                 job after none ran, with its storage repaired and without the blob descriptor cache:\n\
397                 docker rm -f {}",
398                names.mirror()
399            )
400        } else {
401            let restart = if cached {
402                format!(
403                    "\ndocker restart {}   # empties the blob descriptor cache",
404                    names.mirror()
405                )
406            } else {
407                String::new()
408            };
409            format!(
410                "remove them, and the mirror fetches each again from Docker Hub:\ndocker exec {} rm -f {}{restart}",
411                names.mirror(),
412                removes.join(" ")
413            )
414        };
415        return check(
416            CheckStatus::Fail,
417            format!(
418                "{} of its {} links answer a pull with no data: {}; jobs that pull them fail with \
419                 \"short read: expected … bytes but got 0\"",
420                broken.len(),
421                links.count,
422                named(&broken)
423            ),
424            Some(fix),
425        );
426    }
427    if cached {
428        return check(CheckStatus::Warn, cache.to_owned(), None);
429    }
430    if all.is_empty() {
431        return check(
432            CheckStatus::Pass,
433            format!("each of its {} links leads to a stored blob", links.count),
434            None,
435        );
436    }
437    check(
438        CheckStatus::Info,
439        format!(
440            "{} of its {} links lead to a blob that is gone: {}; it fetches each again from Docker Hub when pulled, \
441             and runnerd removes them before it next starts it",
442            all.len(),
443            links.count,
444            named(&all)
445        ),
446        None,
447    )
448}
449
450#[cfg(test)]
451mod tests;