use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::process::Command;
use std::sync::Mutex;
use tokio::sync::Mutex as AsyncMutex;
use tracing::{debug, info, warn};
use crate::error::{Result, VmmError};
const BUSYBOX: &str = "/bin/busybox";
const DMSETUP_CANDIDATES: &[&str] = &["/arcbox/bin/dmsetup", "/usr/sbin/dmsetup", "/sbin/dmsetup"];
const SNAPSHOT_CHUNK_SECTORS: u64 = 8;
const DM_NAME_PREFIX: &str = "arcbox-snap-";
const DM_NAME_MAX_LEN: usize = 127;
const TEMPLATE_LOOP_DIR: &str = ".template-loops";
fn validate_dm_name_suffix(sandbox_id: &str) -> Result<()> {
if sandbox_id.is_empty() {
return Err(VmmError::DeviceMapper("empty sandbox id".into()));
}
if DM_NAME_PREFIX.len() + sandbox_id.len() > DM_NAME_MAX_LEN {
return Err(VmmError::DeviceMapper(format!(
"sandbox id too long for dm-name (max {} chars after prefix)",
DM_NAME_MAX_LEN - DM_NAME_PREFIX.len()
)));
}
if let Some(bad) = sandbox_id
.chars()
.find(|c| !(c.is_ascii_alphanumeric() || matches!(c, '_' | '+' | '.' | '-')))
{
return Err(VmmError::DeviceMapper(format!(
"sandbox id contains character {bad:?} not allowed in dm-name"
)));
}
Ok(())
}
struct TemplateEntry {
loop_device: String,
sectors: u64,
refcount: usize,
}
#[derive(Debug)]
pub struct CowHandle {
pub dm_name: String,
pub dm_device: String,
pub cow_loop: String,
pub cow_file: PathBuf,
pub template_path: PathBuf,
}
pub struct CowManager {
templates: Mutex<HashMap<PathBuf, TemplateEntry>>,
losetup_lock: AsyncMutex<()>,
cow_dir: PathBuf,
dmsetup_bin: Option<String>,
}
impl CowManager {
pub fn new(data_dir: &str) -> std::io::Result<Self> {
let cow_dir = PathBuf::from(data_dir).join("cow");
std::fs::create_dir_all(&cow_dir)?;
let dmsetup_bin = DMSETUP_CANDIDATES
.iter()
.find(|p| Path::new(p).exists())
.map(|s| (*s).to_string());
if dmsetup_bin.is_none() {
warn!("dmsetup not found; dm-snapshot CoW will be unavailable");
}
let mgr = Self {
templates: Mutex::new(HashMap::new()),
losetup_lock: AsyncMutex::new(()),
cow_dir,
dmsetup_bin,
};
mgr.cleanup_stale_sync();
Ok(mgr)
}
pub async fn setup(&self, sandbox_id: &str, rootfs_path: &str) -> Result<CowHandle> {
validate_dm_name_suffix(sandbox_id)?;
let dmsetup = self
.dmsetup_bin
.as_deref()
.ok_or_else(|| VmmError::DeviceMapper("dmsetup binary not found".into()))?;
let template = PathBuf::from(rootfs_path);
let (template_loop, sectors) = 'acquire: {
if let Some(cached) = {
let mut templates = self.templates.lock().unwrap();
templates.get_mut(&template).map(|entry| {
entry.refcount += 1;
debug!(
template = %rootfs_path,
loop_dev = %entry.loop_device,
refcount = entry.refcount,
"reusing template loop device"
);
(entry.loop_device.clone(), entry.sectors)
})
} {
break 'acquire cached;
}
let _losetup_guard = self.losetup_lock.lock().await;
if let Some(cached) = {
let mut templates = self.templates.lock().unwrap();
templates.get_mut(&template).map(|entry| {
entry.refcount += 1;
debug!(
template = %rootfs_path,
loop_dev = %entry.loop_device,
refcount = entry.refcount,
"reusing template loop device (after lock)"
);
(entry.loop_device.clone(), entry.sectors)
})
} {
break 'acquire cached;
}
let loop_dev = losetup_attach(BUSYBOX, Path::new(rootfs_path), true).await?;
let sectors = blockdev_getsz(BUSYBOX, &loop_dev).await.inspect_err(|_| {
let ld = loop_dev.clone();
tokio::spawn(async move {
let _ = losetup_detach(BUSYBOX, &ld).await;
});
})?;
self.write_template_marker(&loop_dev, &template);
debug!(
template = %rootfs_path,
loop_dev = %loop_dev,
sectors,
"attached new template loop device"
);
{
let mut templates = self.templates.lock().unwrap();
templates.insert(
template.clone(),
TemplateEntry {
loop_device: loop_dev.clone(),
sectors,
refcount: 1,
},
);
}
(loop_dev, sectors)
};
let cow_file = self.cow_dir.join(format!("arcbox-cow-{sandbox_id}.img"));
let cow_size = sectors * 512;
if let Err(e) = create_sparse_file(&cow_file, cow_size).await {
self.release_template(&template);
return Err(e);
}
let cow_loop_result = {
let losetup_guard = self.losetup_lock.lock().await;
let result = losetup_attach(BUSYBOX, &cow_file, false).await;
drop(losetup_guard);
result
};
let cow_loop = match cow_loop_result {
Ok(dev) => dev,
Err(e) => {
let _ = std::fs::remove_file(&cow_file);
self.release_template(&template);
return Err(e);
}
};
let dm_name = format!("{DM_NAME_PREFIX}{sandbox_id}");
let table =
format!("0 {sectors} snapshot {template_loop} {cow_loop} P {SNAPSHOT_CHUNK_SECTORS}");
if let Err(e) = dmsetup_create(dmsetup, &dm_name, &table).await {
let _ = losetup_detach(BUSYBOX, &cow_loop).await;
let _ = std::fs::remove_file(&cow_file);
self.release_template(&template);
return Err(e);
}
let dm_device = format!("/dev/mapper/{dm_name}");
info!(
sandbox_id,
dm_device = %dm_device,
cow_file = %cow_file.display(),
"dm-snapshot created"
);
Ok(CowHandle {
dm_name,
dm_device,
cow_loop,
cow_file,
template_path: template,
})
}
pub async fn teardown(&self, handle: &CowHandle) {
let dmsetup = self.dmsetup_bin.as_deref().unwrap_or("dmsetup");
let dm_removed = dmsetup_remove(dmsetup, &handle.dm_name).await.is_ok();
if !dm_removed {
warn!(dm = %handle.dm_name, "failed to remove dm device");
}
let loop_detached = losetup_detach(BUSYBOX, &handle.cow_loop).await.is_ok();
if !loop_detached {
warn!(loop_dev = %handle.cow_loop, "failed to detach cow loop");
}
if dm_removed && loop_detached {
if let Err(e) = std::fs::remove_file(&handle.cow_file) {
warn!(file = %handle.cow_file.display(), error = %e, "failed to remove cow file");
}
} else {
warn!(
file = %handle.cow_file.display(),
"skipping cow file removal — backing resources not fully released"
);
}
self.release_template(&handle.template_path);
info!(sandbox = %handle.dm_name, "dm-snapshot teardown complete");
}
fn cleanup_stale_sync(&self) {
let dmsetup = match self.dmsetup_bin.as_deref() {
Some(bin) => bin,
None => return,
};
if let Ok(output) = Command::new(dmsetup)
.args(["ls", "--target", "snapshot"])
.output()
{
let stdout = String::from_utf8_lossy(&output.stdout);
for line in stdout.lines() {
if let Some(name) = line.split_whitespace().next()
&& name.starts_with(DM_NAME_PREFIX)
{
debug!(dm = %name, "removing stale dm-snapshot");
let _ = Command::new(dmsetup).args(["remove", name]).output();
}
}
}
if let Ok(entries) = std::fs::read_dir(&self.cow_dir) {
for entry in entries.flatten() {
let path = entry.path();
if path
.file_name()
.and_then(|n| n.to_str())
.is_some_and(|n| n.starts_with("arcbox-cow-"))
{
if let Ok(output) = Command::new(BUSYBOX)
.args(["losetup", "-j", path.to_str().unwrap_or("")])
.output()
{
let stdout = String::from_utf8_lossy(&output.stdout);
for line in stdout.lines() {
if let Some(dev) = line.split(':').next() {
let _ = Command::new(BUSYBOX)
.args(["losetup", "-d", dev.trim()])
.output();
}
}
}
debug!(file = %path.display(), "removing stale cow file");
let _ = std::fs::remove_file(&path);
}
}
}
self.cleanup_stale_template_markers();
}
fn template_marker_path(&self, loop_dev: &str) -> Option<PathBuf> {
let basename = Path::new(loop_dev).file_name()?;
Some(self.cow_dir.join(TEMPLATE_LOOP_DIR).join(basename))
}
fn write_template_marker(&self, loop_dev: &str, template_path: &Path) {
let Some(marker) = self.template_marker_path(loop_dev) else {
warn!(
loop_dev,
"skipping template marker: unparseable loop device"
);
return;
};
if let Some(parent) = marker.parent()
&& let Err(e) = std::fs::create_dir_all(parent)
{
warn!(error = %e, "failed to create template-loops dir");
return;
}
if let Err(e) = std::fs::write(&marker, template_path.to_string_lossy().as_bytes()) {
warn!(loop_dev, error = %e, "failed to write template-loop marker");
}
}
fn cleanup_stale_template_markers(&self) {
let dir = self.cow_dir.join(TEMPLATE_LOOP_DIR);
let Ok(entries) = std::fs::read_dir(&dir) else {
return;
};
for entry in entries.flatten() {
let marker_path = entry.path();
let Some(loop_basename) = marker_path.file_name().and_then(|n| n.to_str()) else {
continue;
};
let dev = format!("/dev/{loop_basename}");
let expected_backing = std::fs::read_to_string(&marker_path)
.ok()
.map(|s| s.trim().to_string())
.unwrap_or_default();
let actual_backing =
std::fs::read_to_string(format!("/sys/block/{loop_basename}/loop/backing_file"))
.ok()
.map(|s| s.trim().to_string());
if !expected_backing.is_empty()
&& actual_backing.as_deref() == Some(expected_backing.as_str())
{
debug!(dev = %dev, "detaching stale template loop");
let _ = Command::new(BUSYBOX).args(["losetup", "-d", &dev]).output();
} else {
debug!(
dev = %dev,
expected = %expected_backing,
actual = ?actual_backing,
"skipping stale template loop: backing mismatch"
);
}
let _ = std::fs::remove_file(&marker_path);
}
}
fn release_template(&self, template_path: &Path) {
let mut templates = self.templates.lock().unwrap();
let should_detach = if let Some(entry) = templates.get_mut(template_path) {
entry.refcount = entry.refcount.saturating_sub(1);
if entry.refcount == 0 {
Some(entry.loop_device.clone())
} else {
None
}
} else {
None
};
if let Some(loop_dev) = should_detach {
templates.remove(template_path);
drop(templates);
let marker = self.template_marker_path(&loop_dev);
tokio::spawn(async move {
if let Err(e) = losetup_detach(BUSYBOX, &loop_dev).await {
warn!(loop_dev = %loop_dev, error = %e, "failed to detach template loop");
return;
}
if let Some(marker) = marker {
let _ = std::fs::remove_file(&marker);
}
});
}
}
}
async fn run_cmd(mut cmd: Command) -> Result<std::process::Output> {
tokio::task::spawn_blocking(move || cmd.output())
.await
.map_err(|e| VmmError::DeviceMapper(format!("spawn_blocking join: {e}")))?
.map_err(|e| VmmError::DeviceMapper(format!("command spawn: {e}")))
}
async fn losetup_attach(bin: &str, path: &Path, read_only: bool) -> Result<String> {
let path_str = path
.to_str()
.ok_or_else(|| VmmError::DeviceMapper("non-UTF-8 path".into()))?;
let mut cmd = Command::new(bin);
if read_only {
cmd.args(["losetup", "-r", "-f", "--show", path_str]);
} else {
cmd.args(["losetup", "-f", "--show", path_str]);
}
let output = run_cmd(cmd).await?;
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
return Err(VmmError::DeviceMapper(format!(
"losetup attach {}: {stderr}",
path.display()
)));
}
let dev = String::from_utf8_lossy(&output.stdout).trim().to_string();
if dev.is_empty() {
return Err(VmmError::DeviceMapper(
"losetup --show returned empty device path".into(),
));
}
Ok(dev)
}
async fn losetup_detach(bin: &str, dev: &str) -> Result<()> {
let mut cmd = Command::new(bin);
cmd.args(["losetup", "-d", dev]);
let output = run_cmd(cmd).await?;
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
return Err(VmmError::DeviceMapper(format!(
"losetup -d {dev}: {stderr}"
)));
}
Ok(())
}
async fn blockdev_getsz(bin: &str, dev: &str) -> Result<u64> {
let mut cmd = Command::new(bin);
cmd.args(["blockdev", "--getsz", dev]);
let output = run_cmd(cmd).await?;
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
return Err(VmmError::DeviceMapper(format!(
"blockdev --getsz {dev}: {stderr}"
)));
}
String::from_utf8_lossy(&output.stdout)
.trim()
.parse::<u64>()
.map_err(|e| VmmError::DeviceMapper(format!("blockdev parse: {e}")))
}
async fn dmsetup_create(bin: &str, name: &str, table: &str) -> Result<()> {
let mut cmd = Command::new(bin);
cmd.args(["create", name, "--table", table]);
let output = run_cmd(cmd).await?;
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
return Err(VmmError::DeviceMapper(format!(
"dmsetup create {name}: {stderr}"
)));
}
Ok(())
}
async fn dmsetup_remove(bin: &str, name: &str) -> Result<()> {
let mut cmd = Command::new(bin);
cmd.args(["remove", name]);
let output = run_cmd(cmd).await?;
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
return Err(VmmError::DeviceMapper(format!(
"dmsetup remove {name}: {stderr}"
)));
}
Ok(())
}
async fn create_sparse_file(path: &Path, size: u64) -> Result<()> {
let path = path.to_path_buf();
tokio::task::spawn_blocking(move || {
let file = std::fs::File::create(&path)
.map_err(|e| VmmError::DeviceMapper(format!("create cow file: {e}")))?;
file.set_len(size)
.map_err(|e| VmmError::DeviceMapper(format!("truncate cow file: {e}")))?;
Ok(())
})
.await
.map_err(|e| VmmError::DeviceMapper(format!("spawn_blocking join: {e}")))?
}
pub async fn device_major_minor(path: &str) -> Result<(u32, u32)> {
let mut cmd = Command::new(BUSYBOX);
cmd.args(["stat", "-c", "%t %T", path]);
let output = run_cmd(cmd).await?;
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
return Err(VmmError::DeviceMapper(format!("stat {path}: {stderr}")));
}
let stdout = String::from_utf8_lossy(&output.stdout);
let parts: Vec<&str> = stdout.split_whitespace().collect();
if parts.len() != 2 {
return Err(VmmError::DeviceMapper(format!(
"unexpected stat output for {path}: {stdout}"
)));
}
let major = u32::from_str_radix(parts[0], 16)
.map_err(|e| VmmError::DeviceMapper(format!("parse major: {e}")))?;
let minor = u32::from_str_radix(parts[1], 16)
.map_err(|e| VmmError::DeviceMapper(format!("parse minor: {e}")))?;
Ok((major, minor))
}
pub async fn mknod_blkdev(node_path: &Path, major: u32, minor: u32) -> Result<()> {
let path_str = node_path
.to_str()
.ok_or_else(|| VmmError::DeviceMapper("non-UTF-8 node path".into()))?;
let mut cmd = Command::new(BUSYBOX);
cmd.args([
"mknod",
path_str,
"b",
&major.to_string(),
&minor.to_string(),
]);
let output = run_cmd(cmd).await?;
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
return Err(VmmError::DeviceMapper(format!(
"mknod {path_str}: {stderr}"
)));
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_dm_name_format() {
let name = format!("{DM_NAME_PREFIX}test-sandbox-123");
assert_eq!(name, "arcbox-snap-test-sandbox-123");
}
#[test]
fn validate_dm_name_suffix_accepts_uuid_and_basic_ids() {
validate_dm_name_suffix("550e8400-e29b-41d4-a716-446655440000").unwrap();
validate_dm_name_suffix("sandbox_1").unwrap();
validate_dm_name_suffix("a.b+c-d").unwrap();
}
#[test]
fn validate_dm_name_suffix_rejects_invalid_chars() {
assert!(validate_dm_name_suffix("").is_err());
assert!(validate_dm_name_suffix("has space").is_err());
assert!(validate_dm_name_suffix("with/slash").is_err());
assert!(validate_dm_name_suffix("with:colon").is_err());
assert!(validate_dm_name_suffix(&"x".repeat(DM_NAME_MAX_LEN)).is_err());
}
#[test]
fn test_cow_file_path() {
let cow_dir = PathBuf::from("/var/lib/firecracker-vmm/cow");
let path = cow_dir.join(format!("arcbox-cow-{}.img", "sandbox-1"));
assert_eq!(
path,
PathBuf::from("/var/lib/firecracker-vmm/cow/arcbox-cow-sandbox-1.img")
);
}
#[test]
fn test_snapshot_table_format() {
let sectors = 2097152_u64; let table =
format!("0 {sectors} snapshot /dev/loop0 /dev/loop1 P {SNAPSHOT_CHUNK_SECTORS}");
assert_eq!(table, "0 2097152 snapshot /dev/loop0 /dev/loop1 P 8");
}
#[test]
fn template_marker_round_trip() {
let tmp = tempfile::tempdir().unwrap();
let mgr = CowManager {
templates: Mutex::new(HashMap::new()),
losetup_lock: AsyncMutex::new(()),
cow_dir: tmp.path().to_path_buf(),
dmsetup_bin: None,
};
let template = PathBuf::from("/var/lib/arcbox/rootfs.ext4");
mgr.write_template_marker("/dev/loop7", &template);
let marker = mgr.template_marker_path("/dev/loop7").unwrap();
assert_eq!(marker.file_name().unwrap(), "loop7");
assert!(marker.exists());
let content = std::fs::read_to_string(&marker).unwrap();
assert_eq!(content, template.to_string_lossy());
}
#[tokio::test]
async fn test_release_template_refcount() {
let mgr = CowManager {
templates: Mutex::new(HashMap::new()),
losetup_lock: AsyncMutex::new(()),
cow_dir: PathBuf::from("/tmp"),
dmsetup_bin: None,
};
let path = PathBuf::from("/tmp/template.ext4");
{
let mut t = mgr.templates.lock().unwrap();
t.insert(
path.clone(),
TemplateEntry {
loop_device: "/dev/loop99".into(),
sectors: 1024,
refcount: 2,
},
);
}
mgr.release_template(&path);
{
let t = mgr.templates.lock().unwrap();
assert_eq!(t.get(&path).unwrap().refcount, 1);
}
mgr.release_template(&path);
{
let t = mgr.templates.lock().unwrap();
assert!(!t.contains_key(&path));
}
}
}