use std::collections::HashMap;
use std::time::Duration;
use bollard::models::{ContainerCreateBody, HostConfig};
use futures_util::TryStreamExt as _;
use super::{MIRROR_IMAGE, Names, volume};
use crate::api::{self, Check, CheckStatus};
use crate::docker::{DockerApi, Error, is_not_found};
const LINKS_SCRIPT: &str = include_str!("mirror_links.sh");
pub(super) const DESCRIPTOR_CACHE_ENV: &str = "REGISTRY_STORAGE_CACHE_BLOBDESCRIPTOR";
const REPAIR_DEADLINE: Duration = Duration::from_secs(15);
const FIX_PATHS: usize = 8;
const REPAIR_MEMORY: i64 = 64 << 20;
const V2: &str = "/var/lib/registry/docker/registry/v2";
const NAMED: usize = 3;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Fault {
Gone,
Empty,
Garbled,
}
impl Fault {
fn serves_nothing(self, cached: bool) -> bool {
match self {
Fault::Empty => true,
Fault::Gone => cached,
Fault::Garbled => false,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Dangling {
pub fault: Fault,
pub repository: String,
pub digest: Option<String>,
pub link: String,
}
impl Dangling {
fn removes(&self) -> Vec<String> {
let mut paths = vec![format!("{V2}/{}", self.link)];
if let (Fault::Empty, Some((algorithm, hex))) = (
self.fault,
self.digest
.as_deref()
.and_then(|digest| digest.split_once(':')),
) && let Some(prefix) = hex.get(..2)
{
paths.push(format!("{V2}/blobs/{algorithm}/{prefix}/{hex}/data"));
}
paths
}
}
impl std::fmt::Display for Dangling {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match &self.digest {
Some(digest) => write!(f, "{}@{digest}", self.repository)?,
None => write!(f, "{} ({})", self.repository, self.link)?,
}
if self.fault == Fault::Empty {
f.write_str(" (empty)")?;
}
Ok(())
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Links {
pub count: usize,
pub dangling: Vec<Dangling>,
}
impl Links {
fn parse(output: &str) -> Option<Links> {
let mut dangling = Vec::new();
let mut count = None;
for line in output.lines() {
let fault = match line.split_once(' ') {
Some(("gone", rest)) => Some((Fault::Gone, rest)),
Some(("empty", rest)) => Some((Fault::Empty, rest)),
Some(("garbled", rest)) => Some((Fault::Garbled, rest)),
_ => None,
};
if let Some((fault, found)) = fault {
let Some((digest, link)) = found.split_once(' ') else {
continue;
};
dangling.push(Dangling {
fault,
repository: repository_of(link),
digest: (digest != "-").then(|| digest.to_owned()),
link: link.to_owned(),
});
} else if let Some(read) = line.strip_prefix("links ") {
count = read.trim().parse().ok();
}
}
Some(Links {
count: count?,
dangling,
})
}
}
fn named(dangling: &[&Dangling]) -> String {
let mut named: Vec<String> = dangling
.iter()
.take(NAMED)
.map(ToString::to_string)
.collect();
if dangling.len() > NAMED {
named.push(format!("{} more", dangling.len() - NAMED));
}
named.join(", ")
}
fn repository_of(link: &str) -> String {
let path = link.strip_prefix("repositories/").unwrap_or(link);
["/_layers/", "/_manifests/"]
.iter()
.find_map(|marker| path.find(marker).map(|at| &path[..at]))
.unwrap_or(path)
.to_owned()
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Repair {
Clean,
Removed(Links),
Failed(String),
}
impl Repair {
pub fn log(&self) {
match (self, self.line()) {
(Repair::Failed(_), Some(line)) => tracing::warn!("{line}"),
(_, Some(line)) => tracing::info!("{line}"),
(_, None) => {}
}
}
pub fn line(&self) -> Option<String> {
match self {
Repair::Clean => None,
Repair::Removed(links) => Some(format!(
"removed {} of the mirror's {} links whose blob is gone, before starting it: {}",
links.dangling.len(),
links.count,
named(&links.dangling.iter().collect::<Vec<_>>())
)),
Repair::Failed(why) => Some(format!(
"cannot repair the mirror's storage, starting it as it is: {why}"
)),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum MirrorUp {
Running,
Started(Repair),
}
pub async fn repair(docker: &impl DockerApi, names: &Names, slice: &str) -> Repair {
let name = names.mirror_repair();
let repaired = tokio::time::timeout(REPAIR_DEADLINE, async {
remove_left(docker, &name).await?;
docker.run(&name, repair_body(names, slice)).await?;
let code = docker.wait(&name).await?;
let output = docker.logs(&name).await?;
Ok::<_, Error>((code, output))
})
.await;
let removed = remove_left(docker, &name).await;
let (code, output) = match repaired {
Err(_) => {
return Repair::Failed(format!(
"it did not finish within {}",
api::duration(REPAIR_DEADLINE.as_secs())
));
}
Ok(Err(error)) => return Repair::Failed(error.to_string()),
Ok(Ok(done)) => done,
};
if let Err(error) = removed {
return Repair::Failed(format!("cannot remove its container: {error}"));
}
match Links::parse(&output) {
Some(links) if code == 0 && links.dangling.is_empty() => Repair::Clean,
Some(links) if code == 0 => Repair::Removed(links),
_ => Repair::Failed(failure(code, &output)),
}
}
fn failure(code: i64, output: &str) -> String {
let is_fault = |line: &str| {
["gone ", "empty ", "garbled "]
.iter()
.any(|kind| line.starts_with(kind))
};
let faults = output.lines().filter(|line| is_fault(line)).count();
let said = output
.lines()
.rev()
.map(str::trim)
.find(|line| !line.is_empty() && !is_fault(line) && !line.starts_with("links "));
match said {
Some(said) => format!("it exited {code} after {faults} faulty links: {said}"),
None => format!("it exited {code} after {faults} faulty links"),
}
}
async fn remove_left(docker: &impl DockerApi, name: &str) -> Result<(), Error> {
match docker.remove(name).await {
Err(error) if !is_not_found(&error) => Err(error),
_ => Ok(()),
}
}
pub(super) fn repair_body(names: &Names, slice: &str) -> ContainerCreateBody {
ContainerCreateBody {
image: Some(MIRROR_IMAGE.to_owned()),
entrypoint: Some(script("repair")),
labels: Some(HashMap::from([(
"runnerd.mirror-repair".to_owned(),
String::new(),
)])),
host_config: Some(HostConfig {
cgroup_parent: Some(slice.to_owned()),
memory: Some(REPAIR_MEMORY),
memory_swap: Some(REPAIR_MEMORY),
nano_cpus: Some(1_000_000_000),
mounts: Some(vec![volume(names.mirror(), "/var/lib/registry", false)]),
network_mode: Some("none".to_owned()),
..Default::default()
}),
..Default::default()
}
}
fn script(mode: &str) -> Vec<String> {
["/bin/sh", "-c", LINKS_SCRIPT, "mirror-links", mode]
.map(str::to_owned)
.to_vec()
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Found {
NotRunning,
Read {
links: Links,
cached: bool,
},
Unread(String),
}
pub async fn check(docker: &impl DockerApi, names: &Names) -> Found {
match docker.inspect(&names.mirror()).await {
Ok(state) if state.running => {}
Ok(_) => return Found::NotRunning,
Err(error) if is_not_found(&error) => return Found::NotRunning,
Err(error) => return Found::Unread(error.to_string()),
}
let output = match docker.exec(&names.mirror(), script("check")).await {
Ok(output) => output.try_concat().await,
Err(error) => Err(error),
};
let output = match output {
Ok(output) => String::from_utf8_lossy(&output).into_owned(),
Err(error) => return Found::Unread(error.to_string()),
};
let Some(links) = Links::parse(&output) else {
return Found::Unread(format!("reading its links printed {:?}", output.trim()));
};
let cached = !output.lines().any(|line| line == "cache ");
Found::Read { links, cached }
}
pub fn mirror_check(names: &Names, found: &Found) -> Check {
let check = |status, detail: String, fix: Option<String>| Check {
name: "mirror".to_owned(),
status,
detail,
fix,
};
let (links, cached) = match found {
Found::NotRunning => {
return check(
CheckStatus::Info,
"not running; runnerd removes the links whose blob is gone or empty from its storage before it \
starts it"
.to_owned(),
None,
);
}
Found::Unread(why) => {
return check(
CheckStatus::Skip,
format!("cannot read its storage: {why}"),
None,
);
}
Found::Read { links, cached } => (links, *cached),
};
let all: Vec<&Dangling> = links.dangling.iter().collect();
let broken: Vec<&Dangling> = links
.dangling
.iter()
.filter(|dangling| dangling.fault.serves_nothing(cached))
.collect();
let cache = "it keeps the blob descriptor cache, as a mirror an older runnerd started does, so once one \
repository's copy of a layer expires, pulls of that layer from another get no data; runnerd starts \
it without that cache when its jobs have finished";
if !broken.is_empty() {
let removes: Vec<String> = all.iter().flat_map(|dangling| dangling.removes()).collect();
let fix = if removes.len() > FIX_PATHS {
format!(
"remove the mirror; jobs pull from Docker Hub directly until runnerd starts it again for the first \
job after none ran, with its storage repaired and without the blob descriptor cache:\n\
docker rm -f {}",
names.mirror()
)
} else {
let restart = if cached {
format!(
"\ndocker restart {} # empties the blob descriptor cache",
names.mirror()
)
} else {
String::new()
};
format!(
"remove them, and the mirror fetches each again from Docker Hub:\ndocker exec {} rm -f {}{restart}",
names.mirror(),
removes.join(" ")
)
};
return check(
CheckStatus::Fail,
format!(
"{} of its {} links answer a pull with no data: {}; jobs that pull them fail with \
\"short read: expected … bytes but got 0\"",
broken.len(),
links.count,
named(&broken)
),
Some(fix),
);
}
if cached {
return check(CheckStatus::Warn, cache.to_owned(), None);
}
if all.is_empty() {
return check(
CheckStatus::Pass,
format!("each of its {} links leads to a stored blob", links.count),
None,
);
}
check(
CheckStatus::Info,
format!(
"{} of its {} links lead to a blob that is gone: {}; it fetches each again from Docker Hub when pulled, \
and runnerd removes them before it next starts it",
all.len(),
links.count,
named(&all)
),
None,
)
}
#[cfg(test)]
mod tests;