arcbox-agent 0.9.0

//! Btrfs data volume detection, format, and bind-mount setup.
//!
//! On first boot the data device is formatted as Btrfs with dedicated
//! subvolumes for container runtimes, Kubernetes, and sandboxes, each
//! bind-mounted to its canonical path. The fsync-hot boltdb metadata is NOT
//! kept here — it lives on the ext4 metadata volume (`metadata_volume.rs`);
//! btrfs holds the bulk, compression-friendly data.

use std::io::{Read as _, Seek as _, SeekFrom};
use std::os::unix::fs::PermissionsExt;
use std::path::Path;

use arcbox_constants::paths::{
    CNI_DATA_MOUNT_POINT, CONTAINERD_DATA_MOUNT_POINT, DOCKER_DATA_MOUNT_POINT,
    K3S_DATA_MOUNT_POINT, KUBELET_DATA_MOUNT_POINT,
};

use super::cmdline::docker_data_device;
use crate::config::SANDBOX_DATA_DIR;

/// Offset of the `total_bytes` field in the Btrfs superblock (superblock at
/// 64 KiB + 0x70 within it).
const BTRFS_TOTAL_BYTES_OFFSET: u64 = 0x10070;

/// Temporary mount point for the raw Btrfs device before subvolume bind mounts.
///
/// Must live on a writable filesystem. `/run` is tmpfs (set up in PID1 init),
/// while EROFS root is read-only and cannot host dynamic mountpoints.
pub(super) const BTRFS_TEMP_MOUNT: &str = "/run/arcbox/data";
const LEGACY_SANDBOX_RUNTIME_DIR: &str = "/var/lib/arcbox/sandboxes";

const DATA_SUBVOLUME_MOUNTS: [(&str, &str); 6] = [
    ("@docker", DOCKER_DATA_MOUNT_POINT),
    ("@containerd", CONTAINERD_DATA_MOUNT_POINT),
    ("@k3s", K3S_DATA_MOUNT_POINT),
    ("@kubelet", KUBELET_DATA_MOUNT_POINT),
    ("@cni", CNI_DATA_MOUNT_POINT),
    ("@sandboxes", SANDBOX_DATA_DIR),
];

static DATA_MOUNT_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());

fn data_mounts_ready(mut is_mounted: impl FnMut(&str) -> bool) -> bool {
    is_mounted(BTRFS_TEMP_MOUNT)
        && DATA_SUBVOLUME_MOUNTS
            .iter()
            .all(|(_, target)| is_mounted(target))
}

/// Reads the device size recorded in the Btrfs superblock.
fn btrfs_total_bytes(device: &str) -> std::io::Result<u64> {
    let mut file = std::fs::File::open(device)?;
    file.seek(SeekFrom::Start(BTRFS_TOTAL_BYTES_OFFSET))?;
    let mut buf = [0_u8; 8];
    file.read_exact(&mut buf)?;
    Ok(u64::from_le_bytes(buf))
}

/// Reads a block device's size in bytes by seeking to its end.
fn block_device_size(device: &str) -> std::io::Result<u64> {
    let mut file = std::fs::File::open(device)?;
    file.seek(SeekFrom::End(0))
}

/// Fails loudly when the block device is smaller than the Btrfs filesystem it
/// carries. Btrfs refuses to mount (`open_ctree failed`) in that case, so a
/// bare mount error would otherwise repeat every retry with no explanation.
///
/// The mismatch happens when the VM engine is switched to a backend that
/// exposes a smaller disk than the one the filesystem was grown against (the
/// VZ→HV capacity bug). The data is intact; switching the engine back recovers.
fn check_device_fits_filesystem(device: &str) -> Result<(), String> {
    let fs_bytes = btrfs_total_bytes(device)
        .map_err(|error| format!("read {device} filesystem capacity: {error}"))?;
    let dev_bytes = block_device_size(device)
        .map_err(|error| format!("read {device} block device capacity: {error}"))?;
    if dev_bytes < fs_bytes {
        return Err(format!(
            "data disk is {dev_bytes} bytes but its Btrfs filesystem was grown to \
             {fs_bytes} bytes; the block device shrank (VM engine switched to a \
             backend exposing a smaller disk). Switch the engine back to VZ to \
             recover — the data is intact."
        ));
    }
    Ok(())
}

/// Consumes explicit new-volume authority or verifies the existing Btrfs UUID.
fn ensure_btrfs_format(device: &str) -> Result<String, String> {
    super::storage_volume::ensure_filesystem(arcbox_storage::VolumeRole::Data, device, |uuid| {
        let status = std::process::Command::new("/sbin/mkfs.btrfs")
            .args(["-f", "-U", &uuid.to_string(), device])
            .status()
            .map_err(|error| format!("execute mkfs.btrfs: {error}"))?;
        if !status.success() {
            return Err(format!(
                "mkfs.btrfs failed on {device} (exit={})",
                status.code().unwrap_or(-1)
            ));
        }
        Ok(())
    })?;
    Ok(format!("verified Btrfs identity on {device}"))
}

/// Mounts the data volume (Btrfs), creates subvolumes, and bind-mounts them.
///
/// Layout after this function returns:
/// - `/run/arcbox/data` — raw Btrfs mount (internal, not used by daemons)
/// - `/var/lib/docker` — bind mount of `@docker` subvolume
/// - `/var/lib/containerd` — bind mount of `@containerd` subvolume
/// - `/var/lib/rancher/k3s` — bind mount of `@k3s` subvolume
/// - `/var/lib/kubelet` — bind mount of `@kubelet` subvolume
/// - `/var/lib/cni` — bind mount of `@cni` subvolume
/// - `/var/lib/arcbox/sandbox` — bind mount of `@sandboxes` subvolume
///
/// Returns `Ok(notes)` on success or `Err(reason)` if the data volume
/// could not be set up. Callers must abort service startup on error —
/// running a stateful workload without persistent storage is unsafe.
fn secure_sandbox_data_dir() -> Result<(), String> {
    std::fs::set_permissions(SANDBOX_DATA_DIR, std::fs::Permissions::from_mode(0o700))
        .map_err(|error| format!("secure {SANDBOX_DATA_DIR}: {error}"))
}

pub(super) fn ensure_data_mount() -> Result<String, String> {
    let _guard = DATA_MOUNT_LOCK
        .lock()
        .map_err(|_| "data volume setup lock poisoned".to_string())?;

    // Already fully set up?
    if data_mounts_ready(crate::mount::is_mounted) {
        secure_sandbox_data_dir()?;
        return Ok("data subvolumes already mounted".to_string());
    }

    let device = docker_data_device();

    // Wait for the VirtIO block device to appear. The kernel may need a
    // moment to probe and register the device after boot. The 5 s budget is
    // a heuristic; if it expires the underlying kernel probe is stuck and
    // raising it would just mask the symptom.
    {
        let mut attempts = 0;
        while !Path::new(&device).exists() {
            attempts += 1;
            if attempts > 50 {
                return Err(format!("data device {} not available after 5 s", device));
            }
            std::thread::sleep(std::time::Duration::from_millis(100));
        }
        if attempts > 0 {
            tracing::info!(device, attempts, "waited for data device");
        }
    }

    // Step 1: Format if not Btrfs.
    let note = ensure_btrfs_format(&device)?;
    tracing::info!("{}", note);

    // Step 1.5: Fail loudly if the device is smaller than its filesystem — Btrfs
    // would otherwise reject the mount below with an opaque error on every retry.
    check_device_fits_filesystem(&device)?;

    // Step 2: Mount raw Btrfs to temporary writable mount point.
    if !crate::mount::is_mounted(BTRFS_TEMP_MOUNT) {
        if let Err(e) = std::fs::create_dir_all(BTRFS_TEMP_MOUNT) {
            return Err(format!("failed to create {}: {}", BTRFS_TEMP_MOUNT, e));
        }
        match std::process::Command::new("/bin/busybox")
            .args([
                "mount",
                "-t",
                "btrfs",
                "-o",
                "compress=zstd:3,discard=async",
                &device,
                BTRFS_TEMP_MOUNT,
            ])
            .status()
        {
            Ok(s) if s.success() => {}
            Ok(s) => {
                return Err(format!(
                    "mount -t btrfs {} {} failed (exit={})",
                    device,
                    BTRFS_TEMP_MOUNT,
                    s.code().unwrap_or(-1)
                ));
            }
            Err(e) => return Err(format!("mount exec failed: {}", e)),
        }
    }

    let mut notes = Vec::new();

    // Step 2.5: Grow the Btrfs filesystem to fill the (possibly resized)
    // block device. The host sparse image may have grown since the last
    // boot (e.g. 64 GiB → 8 TiB upgrade). `BTRFS_IOC_RESIZE` with "max"
    // is a no-op when the FS already fills the device, so this is safe to
    // run unconditionally. Failures are surfaced as a note rather than
    // aborting the mount — running on the old capacity is preferable to
    // refusing to start.
    match btrfs_resize_max(BTRFS_TEMP_MOUNT) {
        Ok(note) => notes.push(note),
        Err(e) => {
            tracing::error!(error = %e, "btrfs resize max failed");
            notes.push(format!("resize failed: {}", e));
        }
    }

    // Step 3: Create subvolumes if missing.
    for (subvol, _) in DATA_SUBVOLUME_MOUNTS {
        let subvol_path = format!("{}/{}", BTRFS_TEMP_MOUNT, subvol);
        if Path::new(&subvol_path).exists() {
            continue;
        }
        // EROFS only includes mkfs.btrfs, not full btrfs-progs. Use the
        // BTRFS_IOC_SUBVOL_CREATE ioctl directly to create subvolumes.
        if let Err(e) = btrfs_create_subvolume(&subvol_path) {
            return Err(format!("failed to create subvolume {}: {}", subvol, e));
        }
    }

    // Step 4: Bind mount subvolumes to final paths.
    for (subvol, target) in DATA_SUBVOLUME_MOUNTS {
        if crate::mount::is_mounted(target) {
            continue;
        }
        if let Err(e) = std::fs::create_dir_all(target) {
            return Err(format!("failed to create {}: {}", target, e));
        }
        let opts = format!(
            "compress=zstd:1,discard=async,noatime,space_cache=v2,subvol={}",
            subvol
        );
        match std::process::Command::new("/bin/busybox")
            .args(["mount", "-t", "btrfs", "-o", &opts, &device, target])
            .status()
        {
            Ok(s) if s.success() => {
                notes.push(format!("mounted {} -> {}", subvol, target));
            }
            Ok(s) => {
                return Err(format!(
                    "mount subvol={} {} failed (exit={})",
                    subvol,
                    target,
                    s.code().unwrap_or(-1)
                ));
            }
            Err(e) => return Err(format!("mount exec failed: {}", e)),
        }
    }
    secure_sandbox_data_dir()?;

    if notes.is_empty() {
        Ok("data subvolumes already mounted".to_string())
    } else {
        Ok(notes.join("; "))
    }
}

pub(super) fn ensure_no_legacy_live_sandboxes() -> Result<(), String> {
    let entries = match std::fs::read_dir(LEGACY_SANDBOX_RUNTIME_DIR) {
        Ok(entries) => entries,
        Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()),
        Err(error) => {
            return Err(format!("inspect legacy sandbox runtime directory: {error}"));
        }
    };
    for entry in entries {
        let entry = entry.map_err(|error| format!("inspect legacy sandbox runtime: {error}"))?;
        if entry
            .file_type()
            .map_err(|error| format!("inspect legacy sandbox runtime entry: {error}"))?
            .is_dir()
        {
            return Err(
                "legacy live Sandbox state requires a guest restart before enabling persistent Sandbox storage"
                    .into(),
            );
        }
    }
    Ok(())
}

// BTRFS_IOC_SUBVOL_CREATE = _IOW(0x94, 14, struct btrfs_ioctl_vol_args)
// struct btrfs_ioctl_vol_args { __s64 fd; char name[4088]; }  total = 4096 bytes
//
// nix::ioctl_write_ptr! computes the request number portably (handles
// c_int on musl vs c_ulong on glibc).
nix::ioctl_write_ptr!(btrfs_ioc_subvol_create, 0x94, 14, [u8; 4096]);

// BTRFS_IOC_RESIZE = _IOW(0x94, 3, struct btrfs_ioctl_vol_args)
nix::ioctl_write_ptr!(btrfs_ioc_resize, 0x94, 3, [u8; 4096]);

/// Builds the `BTRFS_IOC_RESIZE` argument buffer.
///
/// Layout: `__s64 devid` (8 bytes) followed by a null-terminated name string
/// (up to 4088 bytes). For `BTRFS_IOC_RESIZE` the default device is `devid=1`
/// and the size token is e.g. `"max"`.
fn build_resize_args(devid: i64, size: &str) -> Result<[u8; 4096], String> {
    let size_bytes = size.as_bytes();
    if size_bytes.len() >= 4088 {
        return Err("resize size token too long".to_string());
    }
    let mut args = [0u8; 4096];
    args[0..8].copy_from_slice(&devid.to_le_bytes());
    args[8..8 + size_bytes.len()].copy_from_slice(size_bytes);
    Ok(args)
}

/// Grows the Btrfs filesystem on `mount_point` to fill the underlying
/// block device. Uses `BTRFS_IOC_RESIZE` with `"max"` — this is a no-op
/// when the FS already occupies the full device.
fn btrfs_resize_max(mount_point: &str) -> Result<String, String> {
    use std::os::unix::io::AsRawFd;

    let dir = std::fs::File::open(mount_point)
        .map_err(|e| format!("open {} for resize: {}", mount_point, e))?;
    let args = build_resize_args(1, "max")?;

    // SAFETY: valid fd from File::open, args buffer matches kernel struct layout.
    unsafe { btrfs_ioc_resize(dir.as_raw_fd(), &raw const args) }
        .map_err(|e| format!("BTRFS_IOC_RESIZE max on {}: {}", mount_point, e))?;

    tracing::info!(mount_point, "btrfs resize max succeeded");
    Ok(format!("resized {} to device max", mount_point))
}

/// Creates a Btrfs subvolume using the `BTRFS_IOC_SUBVOL_CREATE` ioctl.
///
/// This avoids needing the full `btrfs-progs` CLI in the EROFS rootfs.
fn btrfs_create_subvolume(path: &str) -> Result<(), String> {
    use std::os::unix::io::AsRawFd;

    let parent = Path::new(path)
        .parent()
        .ok_or_else(|| "no parent directory".to_string())?;
    let name = Path::new(path)
        .file_name()
        .ok_or_else(|| "no subvolume name".to_string())?
        .to_str()
        .ok_or_else(|| "invalid subvolume name".to_string())?;

    let parent_dir =
        std::fs::File::open(parent).map_err(|e| format!("open {}: {}", parent.display(), e))?;

    let mut args = [0u8; 4096];
    // First 8 bytes: fd field (unused for SUBVOL_CREATE, set to 0).
    // Bytes 8..4096: null-terminated name.
    let name_bytes = name.as_bytes();
    if name_bytes.len() >= 4088 {
        return Err("subvolume name too long".to_string());
    }
    args[8..8 + name_bytes.len()].copy_from_slice(name_bytes);

    // SAFETY: valid fd from File::open, args buffer is 4096 bytes matching
    // the kernel struct btrfs_ioctl_vol_args layout.
    unsafe { btrfs_ioc_subvol_create(parent_dir.as_raw_fd(), &raw const args) }
        .map_err(|e| format!("BTRFS_IOC_SUBVOL_CREATE: {}", e))?;

    tracing::info!("created Btrfs subvolume {}", path);
    Ok(())
}

#[cfg(test)]
mod tests {
    use super::{build_resize_args, data_mounts_ready};
    use crate::config::SANDBOX_DATA_DIR;

    #[test]
    fn resize_args_layout_matches_kernel_struct() {
        let args = build_resize_args(1, "max").unwrap();
        // devid=1 as little-endian i64 in the first 8 bytes.
        assert_eq!(&args[0..8], &1i64.to_le_bytes());
        // "max" then a NUL terminator from the zero-initialized buffer.
        assert_eq!(&args[8..11], b"max");
        assert_eq!(args[11], 0);
        // Remainder must stay zero.
        assert!(args[12..].iter().all(|b| *b == 0));
    }

    #[test]
    fn resize_args_rejects_oversized_token() {
        let huge = "x".repeat(4088);
        assert!(build_resize_args(1, &huge).is_err());
    }

    #[test]
    fn sandbox_mount_is_required_for_data_readiness() {
        assert!(!data_mounts_ready(|target| target != SANDBOX_DATA_DIR));
        assert!(data_mounts_ready(|_| true));
    }
}