zlayer-overlayd 0.13.0

//! `zlayer-overlayd` — the standalone `ZLayer` overlay daemon binary.
//!
//! Binds the IPC endpoint ([`transport::serve`]), wraps an [`OverlaydServer`]
//! in an `Arc<Mutex<_>>`, and runs each accepted connection's request loop
//! against it. A `Shutdown` request drops the overlay adapter and exits the
//! process gracefully.
//!
//! On Windows, when launched by the Service Control Manager with `--service`,
//! control is handed to the SCM dispatcher in [`mod@service`] which registers a
//! control handler (Stop/Shutdown → graceful exit), reports `Running`/`Stopped`,
//! and drives [`run_overlayd`] on a runtime it builds itself. The plain
//! foreground path uses the same [`run_overlayd`] with a `Ctrl-C`-wired
//! shutdown so dev runs exit cleanly too.

use std::path::PathBuf;
use std::sync::Arc;

use clap::Parser;
use tokio::sync::Mutex;
use zlayer_overlayd::server::OverlaydServer;
use zlayer_overlayd::transport;
use zlayer_paths::ZLayerDirs;
use zlayer_types::overlayd::{OverlaydFrame, OverlaydRequest};

#[cfg(windows)]
mod service;

/// Standalone `ZLayer` overlay daemon.
#[derive(Debug, Parser)]
#[command(name = "zlayer-overlayd", about, version)]
struct Args {
    /// Root data directory (HCN markers, IPAM state, UAPI sockets live under it).
    #[arg(long, value_name = "PATH")]
    data_dir: Option<PathBuf>,

    /// IPC endpoint to listen on (Unix socket path, or `\\.\pipe\...` on
    /// Windows). Defaults to a data-dir-aware path.
    #[arg(long, value_name = "PATH")]
    socket: Option<PathBuf>,

    /// Run under the Windows Service Control Manager (SCM). Set automatically
    /// by `zlayer daemon install` when registering the overlayd service; not
    /// meant to be passed by hand. Windows-only; ignored elsewhere.
    #[arg(long, hide = true)]
    #[cfg_attr(not(windows), allow(dead_code))]
    service: bool,

    /// Print the deterministic source build-id (stamped at compile time from the
    /// overlayd source files) and exit. The install/sync change-gate runs this on
    /// the staged and installed binaries to decide whether the overlayd service
    /// needs re-registering — it's stable across non-reproducible rebuilds of
    /// byte-identical source, unlike the binary's content hash. Not meant to be
    /// passed by hand.
    #[arg(long, hide = true)]
    build_id: bool,
}

fn main() -> anyhow::Result<()> {
    let args = Args::parse();

    // Source build-id probe: print the compile-time-stamped id and exit. The
    // change-gate shells out to this, so it must stay side-effect-free and run
    // before any logging init or privileged setup.
    if args.build_id {
        println!("{}", env!("ZLAYER_OVERLAYD_BUILD_ID"));
        return Ok(());
    }

    // Shared logging: console + a rotated file sink at ~/.zlayer/logs/overlayd.log
    // (plus OTLP forwarding when the OTEL_* env is set). Hold the guards for the
    // whole process so the async file writer keeps flushing. A failed init must
    // not stop the daemon from coming up, so fall back to a bare stderr subscriber.
    let _log_guards = match zlayer_observability::init_common_logging(
        "overlayd",
        zlayer_observability::CommonLoggingOptions::default(),
    ) {
        Ok(guards) => Some(guards),
        Err(e) => {
            eprintln!("overlayd: shared logging init failed ({e}); falling back to stderr");
            tracing_subscriber::fmt()
                .with_env_filter(
                    tracing_subscriber::EnvFilter::try_from_default_env()
                        .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")),
                )
                .init();
            None
        }
    };

    let data_dir = args.data_dir.unwrap_or_else(ZLayerDirs::default_data_dir);
    let socket = args
        .socket
        .unwrap_or_else(|| PathBuf::from(ZLayerDirs::default_overlayd_socket_path_for(&data_dir)));

    tracing::info!(
        data_dir = %data_dir.display(),
        socket = %socket.display(),
        "starting zlayer-overlayd"
    );

    // Windows SCM mode: hand control to the dispatcher, which owns its own
    // tokio runtime and drives `run_overlayd` for us. Blocks until SCM stops
    // the service.
    #[cfg(windows)]
    if args.service {
        return service::run_as_overlayd_service(data_dir, socket);
    }

    // Foreground / dev mode: build a runtime and drive `run_overlayd` directly,
    // wiring Ctrl-C (and SIGTERM on Unix) to the external shutdown so the
    // process exits cleanly when interrupted.
    let runtime = tokio::runtime::Builder::new_multi_thread()
        .enable_all()
        .thread_name("zlayer-overlayd-worker")
        .build()?;

    runtime.block_on(async move {
        let (external_tx, external_rx) = tokio::sync::oneshot::channel::<()>();
        tokio::spawn(async move {
            // Wait for the FIRST termination signal — Ctrl-C (SIGINT) in dev, or
            // SIGTERM from `systemctl stop` — then fire the external shutdown so
            // `run_overlayd` can run teardown before exiting. Without the SIGTERM
            // arm, `systemctl stop` (default SIGTERM) would kill the process with
            // no teardown and leave host network state behind until a reboot.
            #[cfg(unix)]
            {
                use tokio::signal::unix::{signal, SignalKind};
                match signal(SignalKind::terminate()) {
                    Ok(mut sigterm) => {
                        tokio::select! {
                            r = tokio::signal::ctrl_c() => {
                                if r.is_ok() {
                                    tracing::info!(
                                        "overlayd: Ctrl-C (SIGINT) received; signaling shutdown"
                                    );
                                }
                            }
                            _ = sigterm.recv() => {
                                tracing::info!("overlayd: SIGTERM received; signaling shutdown");
                            }
                        }
                    }
                    Err(e) => {
                        // Couldn't install the SIGTERM handler; still honor Ctrl-C.
                        tracing::warn!(error = %e, "overlayd: failed to install SIGTERM handler");
                        if tokio::signal::ctrl_c().await.is_ok() {
                            tracing::info!("overlayd: Ctrl-C received; signaling shutdown");
                        }
                    }
                }
            }
            #[cfg(not(unix))]
            if tokio::signal::ctrl_c().await.is_ok() {
                tracing::info!("overlayd: Ctrl-C received; signaling shutdown");
            }
            let _ = external_tx.send(());
        });
        run_overlayd(data_dir, socket, external_rx).await
    })
}

/// Run the overlayd serve loop until either an in-band `Shutdown` IPC request
/// is observed, the `external_shutdown` oneshot fires, or the accept loop
/// returns. Shared by the foreground path and the Windows SCM dispatcher.
///
/// `external_shutdown` is fired by the caller's signal handler (Ctrl-C in
/// dev, an SCM Stop/Shutdown control under the service) and joins the same
/// `select!` as the existing in-band shutdown, so both routes tear the
/// adapter down gracefully.
///
/// # Errors
///
/// Returns an error if binding the IPC endpoint or the accept loop fails.
async fn run_overlayd(
    data_dir: PathBuf,
    socket: PathBuf,
    external_shutdown: tokio::sync::oneshot::Receiver<()>,
) -> anyhow::Result<()> {
    // Scope the overlay's UAPI sockets to this data dir so a test daemon's
    // sockets don't collide with a host-wide install.
    let dirs = ZLayerDirs::new(data_dir.clone());
    let uapi_dir = dirs.wireguard();

    // Self-heal the boringtun UAPI socket directory before any overlay setup.
    // boringtun hardcodes its control socket under this dir and, on a restricted
    // root overlayd (no CAP_DAC_OVERRIDE), cannot delete a stale `zl-*-g.sock`
    // left in it if the dir is owned by a non-root uid — which permanently wedges
    // the global overlay with `bind(): Address already in use`. See
    // `normalize_uapi_sock_dir` for the full rationale.
    #[cfg(not(windows))]
    normalize_uapi_sock_dir(&uapi_dir);

    let server = OverlaydServer::new(data_dir).with_uapi_sock_dir(uapi_dir);
    let server = Arc::new(Mutex::new(server));

    // A oneshot fired when a connection handler observes a Shutdown request, so
    // `serve`'s accept loop can be aborted and the process exit gracefully.
    let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel::<()>();
    let shutdown_tx = Arc::new(Mutex::new(Some(shutdown_tx)));

    let serve = {
        let server = Arc::clone(&server);
        let shutdown_tx = Arc::clone(&shutdown_tx);
        transport::serve(&socket, move |mut conn| {
            let server = Arc::clone(&server);
            let shutdown_tx = Arc::clone(&shutdown_tx);
            async move {
                loop {
                    let frame = match conn.recv().await {
                        Ok(f) => f,
                        Err(zlayer_overlayd::OverlaydError::Closed) => break,
                        Err(e) => {
                            tracing::warn!(error = %e, "overlayd: recv failed; closing connection");
                            break;
                        }
                    };
                    let OverlaydFrame::Request { id, request } = frame else {
                        tracing::warn!("overlayd: received non-Request frame; ignoring");
                        continue;
                    };

                    let (response, shutdown) = {
                        let mut guard = server.lock().await;
                        let response = guard.handle(request).await;
                        (response, guard.shutdown_requested())
                    };

                    if let Err(e) = conn.send(&OverlaydFrame::Response { id, response }).await {
                        tracing::warn!(error = %e, "overlayd: send failed; closing connection");
                        break;
                    }

                    if shutdown {
                        tracing::info!("overlayd: shutdown requested; signaling exit");
                        if let Some(tx) = shutdown_tx.lock().await.take() {
                            let _ = tx.send(());
                        }
                        break;
                    }
                }
            }
        })
    };

    tokio::select! {
        res = serve => {
            res?;
        }
        _ = shutdown_rx => {
            // A connection handler observed an in-band `Shutdown` request. Its
            // `handle()` already ran `teardown_global_overlay()` (it set
            // `shutdown_requested`), so no additional teardown is needed here.
            tracing::info!("overlayd: exiting after graceful shutdown (in-band request)");
        }
        _ = external_shutdown => {
            // Ctrl-C / SIGTERM. Nothing has torn the overlay down yet, so run the
            // same graceful teardown the in-band `Shutdown` request performs by
            // routing a `Shutdown` through the existing dispatch path. This reverts
            // global overlay + host network state (forwarding sysctls, overlay
            // iptables chains) so a `systemctl stop` recovers connectivity without
            // a reboot. Guard on `shutdown_requested()` so teardown runs exactly
            // once even if an in-band `Shutdown` raced in just before the signal.
            let mut guard = server.lock().await;
            if guard.shutdown_requested() {
                tracing::info!(
                    "overlayd: exiting after graceful shutdown \
                     (external signal; teardown already performed)"
                );
            } else {
                tracing::info!(
                    "overlayd: external signal received; running global overlay teardown"
                );
                let _ = guard.handle(OverlaydRequest::Shutdown).await;
                tracing::info!("overlayd: exiting after graceful shutdown (external signal)");
            }
        }
    }

    Ok(())
}

/// Remove every stale boringtun UAPI control socket (`zl-*.sock`) from `dir`.
///
/// On a fresh overlayd boot every such socket is orphaned — the process that
/// created it is gone — so removing it is always safe and clears both the
/// stable-name global `-g` socket and any historical per-service leftovers. The
/// caller must already own `dir` (or hold `CAP_DAC_OVERRIDE`); see
/// [`normalize_uapi_sock_dir`], which guarantees that before calling this.
#[cfg(not(windows))]
fn sweep_stale_uapi_sockets(dir: &std::path::Path) {
    let Ok(entries) = std::fs::read_dir(dir) else {
        return;
    };
    for entry in entries.flatten() {
        let name = entry.file_name();
        let name = name.to_string_lossy();
        if name.starts_with("zl-") && name.ends_with(".sock") {
            let _ = std::fs::remove_file(entry.path());
        }
    }
}

/// Normalize the boringtun UAPI socket directory so overlayd always operates in
/// a **root-owned** directory free of stale sockets.
///
/// boringtun hardcodes its UAPI control socket under `/var/run/wireguard/`
/// (`SOCK_DIR`, see `boringtun::device::api`) and ignores any configured path.
/// If that directory ends up owned by a non-root uid (e.g. created by a rootless
/// run), a capability-restricted root overlayd — which holds `CAP_NET_ADMIN` but
/// **not** `CAP_DAC_OVERRIDE` — cannot `unlink()` the stale `zl-*-g.sock` left by
/// a prior instance: deleting a file needs write on the *parent* directory, which
/// the kernel denies on a foreign-owned dir without `CAP_DAC_OVERRIDE`. boringtun's
/// own pre-bind `remove_file` then also fails, `bind()` returns `EADDRINUSE`, and
/// the global overlay never comes up → the node's overlay IP stays `None` →
/// internal service endpoints fall back to binding `127.0.0.1` (and lose overlay
/// DNS). With the now-stable per-host instance id the global interface name is
/// constant across restarts, so the same un-removable socket collides on *every*
/// boot — a permanent wedge.
///
/// This runs once at startup, before any `SetupGlobalOverlay`, and is
/// **capability-free**: when the dir is foreign-owned we `rename` it aside within
/// the same parent (`/run`), which needs only write on the parent — held by uid 0
/// as its owner, no `CAP_DAC_OVERRIDE` required — then recreate it root-owned.
#[cfg(not(windows))]
fn normalize_uapi_sock_dir(dir: &std::path::Path) {
    use std::os::unix::fs::MetadataExt;
    use std::os::unix::fs::PermissionsExt;

    // pid + a process-local counter keep any move-aside name unique without a
    // wall-clock read.
    static ASIDE_SEQ: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(0);

    // Rootless: the daemon entered a userns + private mount namespace and bound a
    // fresh tmpfs over `/var/run/wireguard` (see `bin/zlayer/src/rootless.rs`
    // `redirect_wireguard_sock_dir`). That tmpfs is owned by uid 0 *inside the
    // userns* and already empty, so the move-aside path below is both unnecessary
    // and impossible (a `rename` of the tmpfs mountpoint within `/run` fails). Keep
    // only the idempotent create_dir_all + stale-socket sweep.
    if std::env::var_os("ZLAYER_ROOTLESS").is_some() {
        if let Err(e) = std::fs::create_dir_all(dir) {
            tracing::warn!(dir = %dir.display(), error = %e, "failed to create UAPI socket dir (rootless)");
            return;
        }
        let _ = std::fs::set_permissions(dir, std::fs::Permissions::from_mode(0o700));
        sweep_stale_uapi_sockets(dir);
        return;
    }

    // If the directory exists but is owned by a non-root uid, move it aside so we
    // can recreate it root-owned. Renaming an entry within its parent updates only
    // the parent's directory block (the moved dir's `..` is unchanged), so it needs
    // write on the parent alone — available to uid 0 without CAP_DAC_OVERRIDE.
    if let Ok(meta) = std::fs::metadata(dir) {
        if meta.uid() != 0 {
            if let Some(parent) = dir.parent() {
                let name = dir.file_name().map_or_else(
                    || "wireguard".to_string(),
                    |n| n.to_string_lossy().into_owned(),
                );
                let seq = ASIDE_SEQ.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
                let aside = parent.join(format!(".{name}.stale-{}-{seq}", std::process::id()));
                match std::fs::rename(dir, &aside) {
                    Ok(()) => tracing::warn!(
                        dir = %dir.display(),
                        uid = meta.uid(),
                        moved_to = %aside.display(),
                        "UAPI socket dir was not root-owned; moved aside and recreating root-owned"
                    ),
                    Err(e) => tracing::warn!(
                        dir = %dir.display(),
                        uid = meta.uid(),
                        error = %e,
                        "UAPI socket dir is not root-owned and could not be moved aside; \
                         overlay setup may fail to clear a stale socket"
                    ),
                }
            }
        }
    }

    // Ensure the dir exists (recreated root-owned if we just moved one aside) and
    // is private to root.
    if let Err(e) = std::fs::create_dir_all(dir) {
        tracing::warn!(dir = %dir.display(), error = %e, "failed to create UAPI socket dir");
        return;
    }
    let _ = std::fs::set_permissions(dir, std::fs::Permissions::from_mode(0o700));

    // Now that we own the dir, sweep any stale boringtun control sockets so the
    // global overlay can bind cleanly.
    sweep_stale_uapi_sockets(dir);
}

#[cfg(all(test, not(windows)))]
mod tests {
    use super::*;
    use std::fs;
    use std::os::unix::ffi::OsStrExt;
    use std::os::unix::fs::MetadataExt;

    fn unique_tmp(tag: &str) -> std::path::PathBuf {
        static SEQ: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(0);
        let seq = SEQ.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
        std::env::temp_dir().join(format!(
            "zlayer-overlayd-uapitest-{tag}-{}-{seq}",
            std::process::id()
        ))
    }

    /// The stale-socket sweep removes `zl-*.sock` entries and leaves everything
    /// else alone. No root required.
    #[test]
    fn sweep_removes_only_zl_sockets() {
        let dir = unique_tmp("sweep");
        fs::create_dir_all(&dir).unwrap();
        let stale_global = dir.join("zl-124aeda7be-g.sock");
        let stale_service = dir.join("zl-81c6bc17c7-s.sock");
        let keep = dir.join("keep.txt");
        let keep_sock = dir.join("other.sock"); // not a zl- socket
        for p in [&stale_global, &stale_service, &keep, &keep_sock] {
            fs::write(p, b"x").unwrap();
        }

        sweep_stale_uapi_sockets(&dir);

        assert!(
            !stale_global.exists(),
            "stale global socket should be swept"
        );
        assert!(
            !stale_service.exists(),
            "stale service socket should be swept"
        );
        assert!(keep.exists(), "non-socket file must be preserved");
        assert!(
            keep_sock.exists(),
            "a .sock without the zl- prefix must be preserved"
        );

        let _ = fs::remove_dir_all(&dir);
    }

    /// On a directory already owned by root (the normal case), `normalize` keeps
    /// the directory in place and just sweeps stale sockets. No root required:
    /// when the test runs as root the dir we create is root-owned and the
    /// in-place path is exercised; when run as non-root the dir is moved aside and
    /// recreated, but the post-condition (dir exists, no stale `zl-*.sock`) holds
    /// in both cases — which is exactly the contract callers depend on.
    #[test]
    fn normalize_leaves_usable_root_owned_dir_without_stale_sockets() {
        let dir = unique_tmp("normalize");
        fs::create_dir_all(&dir).unwrap();
        let stale = dir.join("zl-124aeda7be-g.sock");
        fs::write(&stale, b"x").unwrap();

        normalize_uapi_sock_dir(&dir);

        assert!(dir.exists(), "uapi dir must exist after normalize");
        assert!(
            !dir.join("zl-124aeda7be-g.sock").exists(),
            "stale socket must not survive in the live dir after normalize"
        );

        let _ = fs::remove_dir_all(&dir);
        reap_aside_siblings(&dir);
    }

    /// Best-effort cleanup of any `.<name>.stale-*` sibling left when normalize
    /// took the move-aside path (non-root test runs).
    fn reap_aside_siblings(dir: &std::path::Path) {
        if let Some(parent) = dir.parent() {
            if let Some(name) = dir.file_name().map(|n| n.to_string_lossy().into_owned()) {
                let prefix = format!(".{name}.stale-");
                if let Ok(entries) = fs::read_dir(parent) {
                    for entry in entries.flatten() {
                        if entry.file_name().to_string_lossy().starts_with(&prefix) {
                            let _ = fs::remove_dir_all(entry.path());
                        }
                    }
                }
            }
        }
    }

    /// Root-gated: reproduces the exact production failure — a UAPI dir owned by a
    /// non-root uid with a root-owned stale socket inside. `normalize` must move it
    /// aside and hand back a root-owned, empty dir so the next `bind()` succeeds.
    /// Skipped (passes vacuously) when not run as root so `--workspace` stays green
    /// for non-root CI; exercised under the root-capable host run.
    #[test]
    #[allow(unsafe_code)] // libc::geteuid / libc::chown for the root-gated setup
    fn normalize_reclaims_foreign_owned_dir_root_only() {
        // SAFETY: `geteuid` is a pure read of the caller's effective uid.
        let is_root = unsafe { libc::geteuid() } == 0;
        if !is_root {
            eprintln!("skipping normalize_reclaims_foreign_owned_dir_root_only: requires root");
            return;
        }

        let dir = unique_tmp("foreign");
        fs::create_dir_all(&dir).unwrap();
        let stale = dir.join("zl-124aeda7be-g.sock");
        fs::write(&stale, b"x").unwrap();

        // chown the dir to a non-root uid (1 = daemon/bin, present on every Linux
        // box) to mimic the `/var/run/wireguard` owned-by-1000 production state.
        let c_dir = std::ffi::CString::new(dir.as_os_str().as_bytes()).unwrap();
        // SAFETY: chown on a path we just created; uid 1 / gid 1 are well-known.
        let rc = unsafe { libc::chown(c_dir.as_ptr(), 1, 1) };
        assert_eq!(rc, 0, "test setup: chown to uid 1 must succeed as root");
        assert_ne!(
            fs::metadata(&dir).unwrap().uid(),
            0,
            "test setup: dir should now be foreign-owned"
        );

        normalize_uapi_sock_dir(&dir);

        let meta = fs::metadata(&dir).expect("uapi dir must exist after normalize");
        assert_eq!(
            meta.uid(),
            0,
            "normalize must hand back a root-owned uapi dir"
        );
        assert!(
            !dir.join("zl-124aeda7be-g.sock").exists(),
            "the stale socket must be gone from the reclaimed dir"
        );

        let _ = fs::remove_dir_all(&dir);
        reap_aside_siblings(&dir);
    }
}