1use std::collections::HashMap;
7use std::time::Duration;
8
9use bollard::models::{ContainerCreateBody, HostConfig};
10use futures_util::TryStreamExt as _;
11
12use super::{MIRROR_IMAGE, Names, volume};
13use crate::api::{self, Check, CheckStatus};
14use crate::docker::{DockerApi, Error, is_not_found};
15
16const LINKS_SCRIPT: &str = include_str!("mirror_links.sh");
19
20pub(super) const DESCRIPTOR_CACHE_ENV: &str = "REGISTRY_STORAGE_CACHE_BLOBDESCRIPTOR";
23
24const REPAIR_DEADLINE: Duration = Duration::from_secs(15);
28
29const FIX_PATHS: usize = 8;
31
32const REPAIR_MEMORY: i64 = 64 << 20;
34
35const V2: &str = "/var/lib/registry/docker/registry/v2";
37
38const NAMED: usize = 3;
40
41#[derive(Debug, Clone, Copy, PartialEq, Eq)]
43pub enum Fault {
44 Gone,
47 Empty,
50 Garbled,
52}
53
54impl Fault {
55 fn serves_nothing(self, cached: bool) -> bool {
58 match self {
59 Fault::Empty => true,
60 Fault::Gone => cached,
61 Fault::Garbled => false,
62 }
63 }
64}
65
66#[derive(Debug, Clone, PartialEq, Eq)]
68pub struct Dangling {
69 pub fault: Fault,
71 pub repository: String,
73 pub digest: Option<String>,
75 pub link: String,
77}
78
79impl Dangling {
80 fn removes(&self) -> Vec<String> {
82 let mut paths = vec![format!("{V2}/{}", self.link)];
83 if let (Fault::Empty, Some((algorithm, hex))) = (
84 self.fault,
85 self.digest
86 .as_deref()
87 .and_then(|digest| digest.split_once(':')),
88 ) && let Some(prefix) = hex.get(..2)
89 {
90 paths.push(format!("{V2}/blobs/{algorithm}/{prefix}/{hex}/data"));
91 }
92 paths
93 }
94}
95
96impl std::fmt::Display for Dangling {
97 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
98 match &self.digest {
99 Some(digest) => write!(f, "{}@{digest}", self.repository)?,
100 None => write!(f, "{} ({})", self.repository, self.link)?,
101 }
102 if self.fault == Fault::Empty {
103 f.write_str(" (empty)")?;
104 }
105 Ok(())
106 }
107}
108
109#[derive(Debug, Clone, PartialEq, Eq)]
111pub struct Links {
112 pub count: usize,
114 pub dangling: Vec<Dangling>,
116}
117
118impl Links {
119 fn parse(output: &str) -> Option<Links> {
121 let mut dangling = Vec::new();
122 let mut count = None;
123 for line in output.lines() {
124 let fault = match line.split_once(' ') {
125 Some(("gone", rest)) => Some((Fault::Gone, rest)),
126 Some(("empty", rest)) => Some((Fault::Empty, rest)),
127 Some(("garbled", rest)) => Some((Fault::Garbled, rest)),
128 _ => None,
129 };
130 if let Some((fault, found)) = fault {
131 let Some((digest, link)) = found.split_once(' ') else {
132 continue;
133 };
134 dangling.push(Dangling {
135 fault,
136 repository: repository_of(link),
137 digest: (digest != "-").then(|| digest.to_owned()),
138 link: link.to_owned(),
139 });
140 } else if let Some(read) = line.strip_prefix("links ") {
141 count = read.trim().parse().ok();
142 }
143 }
144 Some(Links {
145 count: count?,
146 dangling,
147 })
148 }
149}
150
151fn named(dangling: &[&Dangling]) -> String {
153 let mut named: Vec<String> = dangling
154 .iter()
155 .take(NAMED)
156 .map(ToString::to_string)
157 .collect();
158 if dangling.len() > NAMED {
159 named.push(format!("{} more", dangling.len() - NAMED));
160 }
161 named.join(", ")
162}
163
164fn repository_of(link: &str) -> String {
166 let path = link.strip_prefix("repositories/").unwrap_or(link);
167 ["/_layers/", "/_manifests/"]
168 .iter()
169 .find_map(|marker| path.find(marker).map(|at| &path[..at]))
170 .unwrap_or(path)
171 .to_owned()
172}
173
174#[derive(Debug, Clone, PartialEq, Eq)]
176pub enum Repair {
177 Clean,
179 Removed(Links),
181 Failed(String),
183}
184
185impl Repair {
186 pub fn log(&self) {
188 match (self, self.line()) {
189 (Repair::Failed(_), Some(line)) => tracing::warn!("{line}"),
190 (_, Some(line)) => tracing::info!("{line}"),
191 (_, None) => {}
192 }
193 }
194
195 pub fn line(&self) -> Option<String> {
197 match self {
198 Repair::Clean => None,
199 Repair::Removed(links) => Some(format!(
200 "removed {} of the mirror's {} links whose blob is gone, before starting it: {}",
201 links.dangling.len(),
202 links.count,
203 named(&links.dangling.iter().collect::<Vec<_>>())
204 )),
205 Repair::Failed(why) => Some(format!(
206 "cannot repair the mirror's storage, starting it as it is: {why}"
207 )),
208 }
209 }
210}
211
212#[derive(Debug, Clone, PartialEq, Eq)]
214pub enum MirrorUp {
215 Running,
217 Started(Repair),
219}
220
221pub async fn repair(docker: &impl DockerApi, names: &Names, slice: &str) -> Repair {
225 let name = names.mirror_repair();
226 let repaired = tokio::time::timeout(REPAIR_DEADLINE, async {
227 remove_left(docker, &name).await?;
228 docker.run(&name, repair_body(names, slice)).await?;
229 let code = docker.wait(&name).await?;
230 let output = docker.logs(&name).await?;
231 Ok::<_, Error>((code, output))
232 })
233 .await;
234 let removed = remove_left(docker, &name).await;
236 let (code, output) = match repaired {
237 Err(_) => {
238 return Repair::Failed(format!(
239 "it did not finish within {}",
240 api::duration(REPAIR_DEADLINE.as_secs())
241 ));
242 }
243 Ok(Err(error)) => return Repair::Failed(error.to_string()),
244 Ok(Ok(done)) => done,
245 };
246 if let Err(error) = removed {
247 return Repair::Failed(format!("cannot remove its container: {error}"));
248 }
249 match Links::parse(&output) {
250 Some(links) if code == 0 && links.dangling.is_empty() => Repair::Clean,
251 Some(links) if code == 0 => Repair::Removed(links),
252 _ => Repair::Failed(failure(code, &output)),
253 }
254}
255
256fn failure(code: i64, output: &str) -> String {
259 let is_fault = |line: &str| {
260 ["gone ", "empty ", "garbled "]
261 .iter()
262 .any(|kind| line.starts_with(kind))
263 };
264 let faults = output.lines().filter(|line| is_fault(line)).count();
265 let said = output
266 .lines()
267 .rev()
268 .map(str::trim)
269 .find(|line| !line.is_empty() && !is_fault(line) && !line.starts_with("links "));
270 match said {
271 Some(said) => format!("it exited {code} after {faults} faulty links: {said}"),
272 None => format!("it exited {code} after {faults} faulty links"),
273 }
274}
275
276async fn remove_left(docker: &impl DockerApi, name: &str) -> Result<(), Error> {
278 match docker.remove(name).await {
279 Err(error) if !is_not_found(&error) => Err(error),
280 _ => Ok(()),
281 }
282}
283
284pub(super) fn repair_body(names: &Names, slice: &str) -> ContainerCreateBody {
286 ContainerCreateBody {
287 image: Some(MIRROR_IMAGE.to_owned()),
288 entrypoint: Some(script("repair")),
289 labels: Some(HashMap::from([(
290 "runnerd.mirror-repair".to_owned(),
291 String::new(),
292 )])),
293 host_config: Some(HostConfig {
294 cgroup_parent: Some(slice.to_owned()),
295 memory: Some(REPAIR_MEMORY),
296 memory_swap: Some(REPAIR_MEMORY),
297 nano_cpus: Some(1_000_000_000),
298 mounts: Some(vec![volume(names.mirror(), "/var/lib/registry", false)]),
299 network_mode: Some("none".to_owned()),
300 ..Default::default()
301 }),
302 ..Default::default()
303 }
304}
305
306fn script(mode: &str) -> Vec<String> {
308 ["/bin/sh", "-c", LINKS_SCRIPT, "mirror-links", mode]
309 .map(str::to_owned)
310 .to_vec()
311}
312
313#[derive(Debug, Clone, PartialEq, Eq)]
315pub enum Found {
316 NotRunning,
318 Read {
320 links: Links,
322 cached: bool,
324 },
325 Unread(String),
327}
328
329pub async fn check(docker: &impl DockerApi, names: &Names) -> Found {
331 match docker.inspect(&names.mirror()).await {
332 Ok(state) if state.running => {}
333 Ok(_) => return Found::NotRunning,
334 Err(error) if is_not_found(&error) => return Found::NotRunning,
335 Err(error) => return Found::Unread(error.to_string()),
336 }
337 let output = match docker.exec(&names.mirror(), script("check")).await {
338 Ok(output) => output.try_concat().await,
339 Err(error) => Err(error),
340 };
341 let output = match output {
342 Ok(output) => String::from_utf8_lossy(&output).into_owned(),
343 Err(error) => return Found::Unread(error.to_string()),
344 };
345 let Some(links) = Links::parse(&output) else {
346 return Found::Unread(format!("reading its links printed {:?}", output.trim()));
347 };
348 let cached = !output.lines().any(|line| line == "cache ");
350 Found::Read { links, cached }
351}
352
353pub fn mirror_check(names: &Names, found: &Found) -> Check {
357 let check = |status, detail: String, fix: Option<String>| Check {
358 name: "mirror".to_owned(),
359 status,
360 detail,
361 fix,
362 };
363 let (links, cached) = match found {
364 Found::NotRunning => {
365 return check(
366 CheckStatus::Info,
367 "not running; runnerd removes the links whose blob is gone or empty from its storage before it \
368 starts it"
369 .to_owned(),
370 None,
371 );
372 }
373 Found::Unread(why) => {
374 return check(
375 CheckStatus::Skip,
376 format!("cannot read its storage: {why}"),
377 None,
378 );
379 }
380 Found::Read { links, cached } => (links, *cached),
381 };
382 let all: Vec<&Dangling> = links.dangling.iter().collect();
383 let broken: Vec<&Dangling> = links
384 .dangling
385 .iter()
386 .filter(|dangling| dangling.fault.serves_nothing(cached))
387 .collect();
388 let cache = "it keeps the blob descriptor cache, as a mirror an older runnerd started does, so once one \
389 repository's copy of a layer expires, pulls of that layer from another get no data; runnerd starts \
390 it without that cache when its jobs have finished";
391 if !broken.is_empty() {
392 let removes: Vec<String> = all.iter().flat_map(|dangling| dangling.removes()).collect();
393 let fix = if removes.len() > FIX_PATHS {
394 format!(
395 "remove the mirror; jobs pull from Docker Hub directly until runnerd starts it again for the first \
396 job after none ran, with its storage repaired and without the blob descriptor cache:\n\
397 docker rm -f {}",
398 names.mirror()
399 )
400 } else {
401 let restart = if cached {
402 format!(
403 "\ndocker restart {} # empties the blob descriptor cache",
404 names.mirror()
405 )
406 } else {
407 String::new()
408 };
409 format!(
410 "remove them, and the mirror fetches each again from Docker Hub:\ndocker exec {} rm -f {}{restart}",
411 names.mirror(),
412 removes.join(" ")
413 )
414 };
415 return check(
416 CheckStatus::Fail,
417 format!(
418 "{} of its {} links answer a pull with no data: {}; jobs that pull them fail with \
419 \"short read: expected … bytes but got 0\"",
420 broken.len(),
421 links.count,
422 named(&broken)
423 ),
424 Some(fix),
425 );
426 }
427 if cached {
428 return check(CheckStatus::Warn, cache.to_owned(), None);
429 }
430 if all.is_empty() {
431 return check(
432 CheckStatus::Pass,
433 format!("each of its {} links leads to a stored blob", links.count),
434 None,
435 );
436 }
437 check(
438 CheckStatus::Info,
439 format!(
440 "{} of its {} links lead to a blob that is gone: {}; it fetches each again from Docker Hub when pulled, \
441 and runnerd removes them before it next starts it",
442 all.len(),
443 links.count,
444 named(&all)
445 ),
446 None,
447 )
448}
449
450#[cfg(test)]
451mod tests;