mod build;
mod inject;
pub use build::{ROOTFS_CAPACITY_GRANULARITY, RootfsSource, RootfsSpec};
use std::collections::BTreeSet;
use std::io::Read;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use anyhow::{Context, Result, bail};
use arcbox_ext4::constants::file_mode;
use arcbox_ext4::{FormatOptions, Formatter};
use arcbox_snapshot::snapshot_cow::BlockTools;
use tokio::process::Command;
use uuid::Uuid;
use crate::error::ComputerError;
const DEFAULT_ROOTFS_SIZE: u64 = 512 * 1024 * 1024;
const TEMPLATE_ROOTFS_SIZE: u64 = 512 * 1024 * 1024;
pub const VM_AGENT_PATH: &str = "/sbin/vm-agent";
#[derive(Debug, Clone)]
pub struct RootfsPaths {
pub vm_agent: PathBuf,
pub cache_dir: PathBuf,
pub busybox: PathBuf,
}
pub struct RootfsBuilder {
paths: RootfsPaths,
#[cfg_attr(
not(target_os = "linux"),
allow(
dead_code,
reason = "the injection loop mount is Linux-only; other platforms never attach"
)
)]
block_tools: Arc<dyn BlockTools>,
}
fn build_lock() -> &'static tokio::sync::Mutex<()> {
static LOCK: std::sync::OnceLock<tokio::sync::Mutex<()>> = std::sync::OnceLock::new();
LOCK.get_or_init(|| tokio::sync::Mutex::new(()))
}
impl RootfsBuilder {
pub fn new(paths: RootfsPaths, block_tools: Arc<dyn BlockTools>) -> Self {
Self { paths, block_tools }
}
pub fn paths(&self) -> &RootfsPaths {
&self.paths
}
}
fn rootfs_err(error: anyhow::Error) -> ComputerError {
ComputerError::Rootfs(format!("{error:#}"))
}
pub fn has_ext4_magic(path: &Path) -> bool {
use std::io::{Seek, SeekFrom};
let Ok(mut file) = std::fs::File::open(path) else {
return false;
};
let mut magic = [0u8; 2];
file.seek(SeekFrom::Start(0x438)).is_ok()
&& file.read_exact(&mut magic).is_ok()
&& magic == [0x53, 0xEF]
}
const OCI_LAYOUT_MARKER: &str = "oci-layout";
fn is_oci_layout(dir: &Path) -> bool {
dir.join(OCI_LAYOUT_MARKER).is_file()
}
impl RootfsBuilder {
pub async fn convert_layer_to_rootfs(
&self,
layer_path: &str,
pinned: &BTreeSet<PathBuf>,
) -> crate::error::Result<String> {
self.convert_layer(layer_path, pinned)
.await
.map_err(rootfs_err)
}
async fn convert_layer(&self, layer_path: &str, pinned: &BTreeSet<PathBuf>) -> Result<String> {
if !Path::new(layer_path).exists() {
bail!("layer path not found: {layer_path}");
}
let layer_key = path_hash(layer_path);
let agent_key = self.vm_agent_key().await?;
let cache_dir = &self.paths.cache_dir;
let ext4_path = cache_dir
.join(format!("rootfs-{layer_key}-{agent_key}.ext4"))
.to_string_lossy()
.into_owned();
if Path::new(&ext4_path).exists() && has_ext4_magic(Path::new(&ext4_path)) {
tracing::info!(path = %ext4_path, "using cached rootfs");
return Ok(ext4_path);
}
tracing::info!(layer = %layer_path, ext4 = %ext4_path, "converting image layer to ext4");
self.write_and_publish(
RootfsSource::Directory(PathBuf::from(layer_path)),
Path::new(&ext4_path),
TEMPLATE_ROOTFS_SIZE,
)
.await?;
sweep_superseded(cache_dir, &layer_key, &agent_key, pinned).await;
tracing::info!(path = %ext4_path, "rootfs ready");
Ok(ext4_path)
}
async fn vm_agent_key(&self) -> Result<String> {
use std::hash::Hasher;
let bytes = tokio::fs::read(&self.paths.vm_agent)
.await
.with_context(|| format!("failed to read {}", self.paths.vm_agent.display()))?;
let mut hasher = std::collections::hash_map::DefaultHasher::new();
hasher.write(&bytes);
Ok(format!("{:016x}", hasher.finish()))
}
}
async fn sweep_superseded(
cache_dir: &Path,
layer_key: &str,
agent_key: &str,
pinned: &BTreeSet<PathBuf>,
) {
let Ok(mut entries) = tokio::fs::read_dir(cache_dir).await else {
return;
};
let mut removed = 0usize;
while let Ok(Some(entry)) = entries.next_entry().await {
let name = entry.file_name().to_string_lossy().into_owned();
if !is_superseded_image(&name, layer_key, agent_key) {
continue;
}
let path = entry.path();
if pinned.contains(&path) {
tracing::debug!(path = %path.display(), "keeping superseded rootfs: pinned");
continue;
}
if tokio::fs::remove_file(&path).await.is_ok() {
removed += 1;
}
}
if removed > 0 {
tracing::info!(removed, layer = %layer_key, "removed superseded rootfs images");
}
}
fn is_superseded_image(name: &str, layer_key: &str, agent_key: &str) -> bool {
let Some(keys) = name
.strip_prefix("rootfs-")
.and_then(|rest| rest.strip_suffix(".ext4"))
else {
return false;
};
let Some((layer, agent)) = keys.split_once('-') else {
return keys == layer_key;
};
layer == layer_key && agent != agent_key
}
impl RootfsBuilder {
pub async fn sweep_stale_tmp(&self) {
let Ok(mut entries) = tokio::fs::read_dir(&self.paths.cache_dir).await else {
return;
};
let mut removed = 0usize;
while let Ok(Some(entry)) = entries.next_entry().await {
if entry.file_name().to_string_lossy().ends_with(".ext4.tmp")
&& tokio::fs::remove_file(entry.path()).await.is_ok()
{
removed += 1;
}
}
if removed > 0 {
tracing::info!(removed, "swept stale rootfs build artifacts");
}
}
pub async fn ensure_default_rootfs(&self, path: &str) -> crate::error::Result<()> {
self.ensure_default(path).await.map_err(rootfs_err)
}
async fn ensure_default(&self, path: &str) -> Result<()> {
let image = Path::new(path);
if self.is_default_rootfs_fresh(image) {
return Ok(());
}
let _build = build_lock().lock().await;
if self.is_default_rootfs_fresh(image) {
return Ok(());
}
if !self.paths.vm_agent.exists() {
bail!(
"vm-agent not found at {}; it is staged by the host daemon next to arcbox-agent",
self.paths.vm_agent.display()
);
}
let applets = self.busybox_applets().await?;
let parent = image
.parent()
.with_context(|| format!("default rootfs path has no parent: {path}"))?;
tokio::fs::create_dir_all(parent)
.await
.context("failed to create default rootfs dir")?;
let tmp = parent.join(format!(".default-{}.ext4.tmp", Uuid::new_v4()));
let spec = DefaultRootfsSpec {
busybox: self.paths.busybox.clone(),
vm_agent: self.paths.vm_agent.clone(),
applets,
size: DEFAULT_ROOTFS_SIZE,
};
tracing::info!(path, "building default sandbox rootfs");
{
let tmp = tmp.clone();
tokio::task::spawn_blocking(move || build_default_rootfs(&spec, &tmp))
.await
.context("default rootfs build task panicked")??;
}
tokio::fs::rename(&tmp, image)
.await
.context("failed to move default rootfs into place")?;
tracing::info!(path, "default sandbox rootfs ready");
Ok(())
}
fn is_default_rootfs_fresh(&self, image: &Path) -> bool {
is_fresh_against(image, &[&self.paths.busybox, &self.paths.vm_agent])
}
async fn busybox_applets(&self) -> Result<Vec<String>> {
let output = Command::new(&self.paths.busybox)
.arg("--list")
.output()
.await
.with_context(|| format!("failed to run {} --list", self.paths.busybox.display()))?;
if !output.status.success() {
bail!("busybox --list failed with {}", output.status);
}
Ok(String::from_utf8_lossy(&output.stdout)
.lines()
.map(str::trim)
.filter(|l| !l.is_empty())
.map(str::to_owned)
.collect())
}
}
fn is_fresh_against(image: &Path, sources: &[&Path]) -> bool {
if !has_ext4_magic(image) {
return false;
}
let Ok(image_mtime) = image.metadata().and_then(|m| m.modified()) else {
return false;
};
for source in sources {
if let Ok(mtime) = std::fs::metadata(source).and_then(|m| m.modified())
&& mtime > image_mtime
{
return false;
}
}
true
}
struct DefaultRootfsSpec {
busybox: std::path::PathBuf,
vm_agent: std::path::PathBuf,
applets: Vec<String>,
size: u64,
}
fn build_default_rootfs(spec: &DefaultRootfsSpec, out: &Path) -> Result<()> {
const DIR: u16 = file_mode::S_IFDIR | 0o755;
const EXE: u16 = file_mode::S_IFREG | 0o755;
const LNK: u16 = file_mode::S_IFLNK | 0o777;
let mut fmt = Formatter::with_options(out, FormatOptions::new(spec.size).label("arcbox-sbx"))
.context("failed to create ext4 formatter")?;
for dir in [
"/bin", "/sbin", "/dev", "/proc", "/sys", "/run", "/etc", "/root", "/var",
] {
fmt.create(dir, DIR, None, None, None, None, None, None)
.with_context(|| format!("mkdir {dir}"))?;
}
fmt.create(
"/tmp",
file_mode::S_IFDIR | file_mode::S_ISVTX | 0o777,
None,
None,
None,
None,
None,
None,
)
.context("mkdir /tmp")?;
let mut busybox = std::fs::File::open(&spec.busybox)
.with_context(|| format!("failed to open {}", spec.busybox.display()))?;
fmt.create(
"/bin/busybox",
EXE,
None,
None,
Some(&mut busybox),
None,
None,
None,
)
.context("write /bin/busybox")?;
for applet in &spec.applets {
if applet == "busybox" || applet.contains('/') {
continue;
}
let path = format!("/bin/{applet}");
fmt.create(&path, LNK, Some("busybox"), None, None, None, None, None)
.with_context(|| format!("symlink {path}"))?;
}
let mut vm_agent = std::fs::File::open(&spec.vm_agent)
.with_context(|| format!("failed to open {}", spec.vm_agent.display()))?;
fmt.create(
VM_AGENT_PATH,
EXE,
None,
None,
Some(&mut vm_agent),
None,
None,
None,
)
.context("write /sbin/vm-agent")?;
fmt.create(
"/sbin/init",
LNK,
Some("vm-agent"),
None,
None,
None,
None,
None,
)
.context("symlink /sbin/init")?;
let passwd = "root:x:0:0:root:/root:/bin/sh\n";
fmt.create(
"/etc/passwd",
file_mode::S_IFREG | 0o644,
None,
None,
Some(&mut passwd.as_bytes()),
None,
None,
None,
)
.context("write /etc/passwd")?;
let group = "root:x:0:\n";
fmt.create(
"/etc/group",
file_mode::S_IFREG | 0o644,
None,
None,
Some(&mut group.as_bytes()),
None,
None,
None,
)
.context("write /etc/group")?;
fmt.create(
"/etc/resolv.conf",
LNK,
Some("../run/resolv.conf"),
None,
None,
None,
None,
None,
)
.context("symlink /etc/resolv.conf")?;
fmt.close().context("failed to finalize ext4 image")?;
Ok(())
}
fn path_hash(path: &str) -> String {
use std::hash::{Hash, Hasher};
let mut hasher = std::collections::hash_map::DefaultHasher::new();
path.hash(&mut hasher);
format!("{:016x}", hasher.finish())
}
#[cfg(test)]
mod tests {
use super::*;
pub(super) fn isolated_log_test(name: &str) -> bool {
const CHILD_TEST: &str = "ARCBOX_ROOTFS_LOG_TEST";
if std::env::var(CHILD_TEST).as_deref() == Ok(name) {
return true;
}
let output = std::process::Command::new(std::env::current_exe().unwrap())
.args(["--exact", name, "--nocapture"])
.env(CHILD_TEST, name)
.output()
.unwrap();
let stdout = String::from_utf8_lossy(&output.stdout);
assert!(
output.status.success() && stdout.contains("1 passed; 0 failed; 0 ignored"),
"isolated log test {name} failed:\n{stdout}\n{}",
String::from_utf8_lossy(&output.stderr)
);
false
}
#[test]
fn test_has_ext4_magic_nonexistent() {
assert!(!has_ext4_magic(Path::new("/nonexistent")));
}
#[test]
fn is_superseded_image_matches_only_the_same_layer_with_another_agent() {
assert!(is_superseded_image("rootfs-aaaa-0000.ext4", "aaaa", "1111"));
assert!(!is_superseded_image(
"rootfs-aaaa-1111.ext4",
"aaaa",
"1111"
));
assert!(!is_superseded_image(
"rootfs-bbbb-0000.ext4",
"aaaa",
"1111"
));
assert!(is_superseded_image("rootfs-aaaa.ext4", "aaaa", "1111"));
assert!(!is_superseded_image("rootfs-bbbb.ext4", "aaaa", "1111"));
assert!(!is_superseded_image(
".rootfs-aaaa-1111.ext4.tmp",
"aaaa",
"1111"
));
assert!(!is_superseded_image("default.ext4", "aaaa", "1111"));
assert!(!is_superseded_image(
"rootfs-aaaa-1111.ext4.bak",
"aaaa",
"1111"
));
}
#[test]
fn is_oci_layout_detects_the_layout_marker() {
let dir = tempfile::TempDir::new().unwrap();
std::fs::create_dir(dir.path().join("diff")).unwrap();
assert!(!is_oci_layout(dir.path()));
std::fs::write(dir.path().join("manifest.json"), "[]").unwrap();
std::fs::write(
dir.path().join(OCI_LAYOUT_MARKER),
r#"{"imageLayoutVersion":"1.0.0"}"#,
)
.unwrap();
assert!(is_oci_layout(dir.path()));
}
#[tokio::test]
async fn sweep_superseded_keeps_images_a_snapshot_still_needs() {
let dir = tempfile::tempdir().unwrap();
let image = |name: &str| dir.path().join(name);
for name in [
"rootfs-aaaa-0000.ext4",
"rootfs-aaaa.ext4",
"rootfs-aaaa-1111.ext4",
"rootfs-bbbb-0000.ext4",
] {
std::fs::write(image(name), b"x").unwrap();
}
let pinned = BTreeSet::from([image("rootfs-aaaa.ext4")]);
sweep_superseded(dir.path(), "aaaa", "1111", &pinned).await;
assert!(!image("rootfs-aaaa-0000.ext4").exists());
assert!(image("rootfs-aaaa.ext4").exists());
assert!(image("rootfs-aaaa-1111.ext4").exists());
assert!(image("rootfs-bbbb-0000.ext4").exists());
}
#[test]
fn test_path_hash_deterministic() {
let a = path_hash("/var/lib/docker/overlay2/abc123");
let b = path_hash("/var/lib/docker/overlay2/abc123");
assert_eq!(a, b);
assert_eq!(a.len(), 16);
}
#[test]
fn test_path_hash_different() {
let a = path_hash("/var/lib/docker/overlay2/abc123");
let b = path_hash("/var/lib/docker/overlay2/def456");
assert_ne!(a, b);
}
#[test]
fn default_rootfs_builds_and_contains_init_chain() {
let dir = tempfile::tempdir().unwrap();
let busybox = dir.path().join("busybox");
let vm_agent = dir.path().join("vm-agent");
std::fs::write(&busybox, b"#!busybox-stub").unwrap();
std::fs::write(&vm_agent, b"#!vm-agent-stub").unwrap();
let out = dir.path().join("rootfs.ext4");
let spec = DefaultRootfsSpec {
busybox,
vm_agent,
applets: vec!["sh".into(), "ls".into(), "busybox".into()],
size: 8 * 1024 * 1024,
};
build_default_rootfs(&spec, &out).unwrap();
assert!(has_ext4_magic(&out));
let reader = arcbox_ext4::Reader::new(&out).unwrap();
for path in [
"/bin/busybox",
"/sbin/vm-agent",
"/sbin/init",
"/bin/sh",
"/etc/passwd",
] {
assert!(
reader.tree().lookup(Path::new(path)).is_some(),
"missing {path}"
);
}
assert!(
reader
.tree()
.lookup(Path::new("/bin/nonexistent"))
.is_none()
);
}
#[test]
fn existing_image_is_fresh_when_build_sources_are_absent() {
let dir = tempfile::tempdir().unwrap();
let image = dir.path().join("rootfs.ext4");
let spec = DefaultRootfsSpec {
busybox: dir.path().join("busybox"),
vm_agent: dir.path().join("vm-agent"),
applets: vec!["sh".into()],
size: 8 * 1024 * 1024,
};
std::fs::write(&spec.busybox, b"stub").unwrap();
std::fs::write(&spec.vm_agent, b"stub").unwrap();
build_default_rootfs(&spec, &image).unwrap();
assert!(has_ext4_magic(&image));
let absent = dir.path().join("absent");
assert!(
is_fresh_against(&image, &[&absent, &absent]),
"a valid image with absent build sources must be reused, not rebuilt"
);
assert!(!is_fresh_against(
&dir.path().join("missing.ext4"),
&[&absent]
));
std::thread::sleep(std::time::Duration::from_millis(20));
let newer = dir.path().join("newer-agent");
std::fs::write(&newer, b"stub").unwrap();
assert!(!is_fresh_against(&image, &[&newer]));
}
}