Skip to main content

zlayer_builder/backend/buildah_sidecar/
lifecycle.rs

1//! Sidecar process lifecycle: spawn, handshake, mTLS dialing, teardown.
2//!
3//! The lifecycle manager keeps at most one live `zlayer-buildd` instance
4//! alive per `BuildahSidecarBackend`. On the first call to
5//! [`SidecarLifecycle::ensure`] we either:
6//!
7//! * dial a pre-existing remote sidecar named by `SidecarConfig::addr`,
8//!   or
9//! * discover the local `zlayer-buildd` binary, spawn it bound to
10//!   `127.0.0.1:0`, read its `LISTENING host:port` handshake from
11//!   stdout, and dial the reported address.
12//!
13//! The connected [`Channel`] is stored alongside the spawned `Child`
14//! (when applicable). The `Child` is held in an `Arc<ChildHolder>` so
15//! its `Drop` impl is what actually tears the process down (SIGTERM,
16//! 5s grace, then SIGKILL). Cloning a [`LiveSidecar`] therefore shares
17//! the same process — teardown happens when the last clone is dropped.
18
19use std::io::{BufRead, BufReader};
20use std::path::PathBuf;
21use std::process::{Child, Command, Stdio};
22use std::sync::Arc;
23use std::time::{Duration, Instant};
24
25use tokio::sync::Mutex;
26use tonic::transport::{Certificate, Channel, ClientTlsConfig, Endpoint, Identity};
27
28use crate::backend::buildah_sidecar::discover::{self, Discovery};
29use crate::backend::buildah_sidecar::proto::build_service_client::BuildServiceClient;
30use crate::backend::buildah_sidecar::tls::{ensure_tls_material, set_dir_mode_0700, TlsMaterial};
31use crate::error::{BuildError, Result};
32
33/// Prefix the sidecar prints to stdout exactly once on startup.
34const HANDSHAKE_PREFIX: &str = "LISTENING ";
35
36/// How long to wait for `LISTENING host:port` after spawning the sidecar.
37const SPAWN_HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(15);
38
39/// How long to wait for SIGTERM to take effect before escalating to SIGKILL.
40const SIGTERM_GRACE: Duration = Duration::from_secs(5);
41
42/// Holds the `Child` so its `Drop` impl performs graceful teardown.
43///
44/// Kept private to this module — outside callers interact with
45/// [`LiveSidecar`], which wraps this holder in an `Arc` so multiple
46/// clones share one process and only the last drop terminates it.
47#[derive(Debug)]
48struct ChildHolder {
49    child: std::sync::Mutex<Option<Child>>,
50}
51
52impl ChildHolder {
53    fn new(child: Child) -> Self {
54        Self {
55            child: std::sync::Mutex::new(Some(child)),
56        }
57    }
58}
59
60impl Drop for ChildHolder {
61    fn drop(&mut self) {
62        // Grab the inner Child. If it was already taken by manual
63        // shutdown, nothing to do.
64        let mut guard = match self.child.lock() {
65            Ok(g) => g,
66            Err(p) => p.into_inner(),
67        };
68        let Some(mut child) = guard.take() else {
69            return;
70        };
71
72        // Try graceful shutdown via SIGTERM on Unix.
73        #[cfg(unix)]
74        {
75            use nix::sys::signal::{kill, Signal};
76            use nix::unistd::Pid;
77
78            // Child::id() returns u32 (the PID). Real PIDs always fit
79            // in a non-negative i32 (PID_MAX is well below i32::MAX),
80            // so an overflow here would indicate either kernel
81            // misbehavior or a wrap that already invalidates the
82            // signal target — fall through to try_wait / kill in that
83            // case instead of panicking.
84            if let Ok(raw) = i32::try_from(child.id()) {
85                let pid = Pid::from_raw(raw);
86                let _ = kill(pid, Signal::SIGTERM);
87            }
88        }
89
90        // On Windows there is no SIGTERM; fall straight through to
91        // try_wait + kill below.
92        let deadline = Instant::now() + SIGTERM_GRACE;
93        loop {
94            match child.try_wait() {
95                Ok(Some(_)) => return,
96                Ok(None) => {
97                    if Instant::now() >= deadline {
98                        let _ = child.kill();
99                        let _ = child.wait();
100                        return;
101                    }
102                    std::thread::sleep(Duration::from_millis(50));
103                }
104                Err(_) => {
105                    let _ = child.kill();
106                    let _ = child.wait();
107                    return;
108                }
109            }
110        }
111    }
112}
113
114/// A live `zlayer-buildd` plus the gRPC channel pointing at it.
115///
116/// The held `Arc<ChildHolder>` is the lifetime anchor for the spawned
117/// process; when all `LiveSidecar` clones are dropped the underlying
118/// `Child` is SIGTERM'd (5s grace) and then SIGKILL'd. For
119/// remote-sidecar mode (`SidecarConfig::addr = Some(_)`) the child
120/// field is `None` because the operator owns lifecycle.
121#[derive(Debug, Clone)]
122pub struct LiveSidecar {
123    /// The `host:port` we connected to. For locally-spawned sidecars
124    /// this is always `127.0.0.1:<auto-port>`.
125    pub addr: String,
126    /// Paths to the mTLS material in use.
127    pub tls: TlsMaterial,
128    /// The binary we spawned (empty `PathBuf` when `addr` was supplied
129    /// and we did not spawn anything).
130    pub binary: PathBuf,
131    /// Connected channel ready to issue RPCs.
132    pub channel: Channel,
133    /// Holds the spawned child (when present); `Drop` performs
134    /// teardown. `None` when we dialed a remote sidecar.
135    _child: Option<Arc<ChildHolder>>,
136}
137
138impl LiveSidecar {
139    /// Build a fresh `BuildServiceClient` over the shared channel.
140    #[must_use]
141    pub fn client(&self) -> BuildServiceClient<Channel> {
142        BuildServiceClient::new(self.channel.clone())
143    }
144}
145
146/// Lifecycle manager owned by `BuildahSidecarBackend`.
147///
148/// Holds at most one `LiveSidecar` and lazily produces it on the first
149/// call to [`Self::ensure`]. Cheap to clone (the inner `Arc` shares the
150/// underlying state), so the backend can pass `&self` references into
151/// async build pipelines without ceremony.
152#[derive(Debug)]
153pub struct SidecarLifecycle {
154    config: Arc<zlayer_types::builder::SidecarConfig>,
155    state: Mutex<Option<LiveSidecar>>,
156}
157
158impl SidecarLifecycle {
159    /// Construct a manager bound to `config`. No filesystem or network
160    /// I/O happens here.
161    #[must_use]
162    pub fn new(config: Arc<zlayer_types::builder::SidecarConfig>) -> Self {
163        Self {
164            config,
165            state: Mutex::new(None),
166        }
167    }
168
169    /// Return the cached [`LiveSidecar`], spawning + dialing on first
170    /// call.
171    ///
172    /// # Errors
173    ///
174    /// Returns whatever the underlying spawn / handshake / dial flow
175    /// produced — typically [`BuildError::NotSupported`] when the
176    /// binary is missing or the handshake times out.
177    pub async fn ensure(&self) -> Result<LiveSidecar> {
178        // Refresh the canonical on-disk binary from the bundle BEFORE we
179        // consult the cache or discovery. An upgraded `zlayer` ships a new
180        // `zlayer-buildd` next to itself; without this a stale copy left in
181        // `${ZLAYER_DATA_DIR}/bin` would keep being spawned (and an already
182        // cached `LiveSidecar` would keep running the old process). When the
183        // refresh actually recopies the binary we drop any cached connection
184        // so the next spawn picks up the new code.
185        let refreshed = refresh_canonical_binary();
186
187        let mut guard = self.state.lock().await;
188
189        if refreshed {
190            // The binary on disk changed under a running daemon: tear down the
191            // cached live sidecar so we respawn the new binary below.
192            *guard = None;
193        }
194
195        if let Some(existing) = guard.as_ref() {
196            return Ok(existing.clone());
197        }
198
199        let live = self.spawn_and_dial().await?;
200        *guard = Some(live.clone());
201        Ok(live)
202    }
203
204    /// Drop the cached [`LiveSidecar`] so the next call to
205    /// [`Self::ensure`] performs a fresh spawn + dial. Tears down the
206    /// previous child via its `Drop` impl once the last outstanding
207    /// clone is released.
208    pub async fn drop_connection(&self) {
209        let mut guard = self.state.lock().await;
210        *guard = None;
211    }
212
213    #[allow(clippy::too_many_lines)]
214    async fn spawn_and_dial(&self) -> Result<LiveSidecar> {
215        let tls_dir = self.config.tls_dir.clone().unwrap_or_else(default_tls_dir);
216
217        let tls = ensure_tls_material(&tls_dir)?;
218
219        let storage = prepare_storage_spec(&self.config)?;
220
221        // Remote-sidecar branch: caller pre-configured a reachable
222        // address, so we never spawn.
223        if let Some(addr) = self.config.addr.clone() {
224            let channel = dial_mtls(&addr, &tls).await?;
225            return Ok(LiveSidecar {
226                addr,
227                tls,
228                binary: PathBuf::new(),
229                channel,
230                _child: None,
231            });
232        }
233
234        // Local spawn branch.
235        let Discovery { binary, tried } = discover::discover_default()?;
236        tracing::info!(?binary, ?tried, "spawning zlayer-buildd");
237
238        let mut cmd = Command::new(&binary);
239        cmd.arg("--bind").arg("127.0.0.1:0");
240        cmd.arg("--tls-ca").arg(&tls.ca_pem);
241        cmd.arg("--tls-cert").arg(&tls.cert_pem);
242        cmd.arg("--tls-key").arg(&tls.key_pem);
243        cmd.arg("--idle-secs")
244            .arg(self.config.idle_secs.to_string());
245        cmd.arg("--storage-root").arg(&storage.graph_root);
246        cmd.arg("--storage-runroot").arg(&storage.run_root);
247        cmd.arg("--storage-driver").arg(&storage.driver);
248        cmd.stdin(Stdio::null());
249        cmd.stdout(Stdio::piped());
250        cmd.stderr(Stdio::piped());
251
252        let mut child = cmd.spawn().map_err(|e| BuildError::NotSupported {
253            operation: format!("spawning zlayer-buildd at {}: {e}", binary.display()),
254        })?;
255
256        // Drain the sidecar's stderr to OUR stderr (prefixed) so its structured
257        // logs — including the real error behind a failed build — are visible
258        // instead of being silently discarded into an unread pipe. Without this
259        // a buildd build failure surfaced only as a terse gRPC status, and the
260        // backend's silent fallback to buildah-cli hid that buildd ran at all.
261        if let Some(child_stderr) = child.stderr.take() {
262            std::thread::spawn(move || {
263                let reader = BufReader::new(child_stderr);
264                for line in reader.lines().map_while(std::result::Result::ok) {
265                    eprintln!("[zlayer-buildd] {line}");
266                }
267            });
268        }
269
270        let stdout = child
271            .stdout
272            .take()
273            .ok_or_else(|| BuildError::NotSupported {
274                operation: "zlayer-buildd: missing stdout pipe".into(),
275            })?;
276
277        // Read the handshake line on a dedicated blocking thread. We
278        // can't park the async runtime on a synchronous
279        // `read_line` against a `std::process::Child`'s stdout, and
280        // the runtime-free thread keeps the code portable across
281        // tokio current_thread / multi_thread schedulers.
282        let (tx, rx) = std::sync::mpsc::channel::<Result<String>>();
283        std::thread::spawn(move || {
284            let mut reader = BufReader::new(stdout);
285            let mut line = String::new();
286            let send_result = match reader.read_line(&mut line) {
287                Ok(0) => tx.send(Err(BuildError::NotSupported {
288                    operation: "zlayer-buildd exited before printing LISTENING".into(),
289                })),
290                Ok(_) => {
291                    let trimmed = line
292                        .trim_end_matches('\n')
293                        .trim_end_matches('\r')
294                        .to_string();
295                    if let Some(addr) = trimmed.strip_prefix(HANDSHAKE_PREFIX) {
296                        tx.send(Ok(addr.to_string()))
297                    } else {
298                        tx.send(Err(BuildError::NotSupported {
299                            operation: format!("zlayer-buildd handshake malformed: {trimmed:?}"),
300                        }))
301                    }
302                }
303                Err(e) => tx.send(Err(BuildError::NotSupported {
304                    operation: format!("reading zlayer-buildd stdout: {e}"),
305                })),
306            };
307            // If the receiver has already gone away (e.g. timeout
308            // killed the child), there's nothing we can do.
309            let _ = send_result;
310        });
311
312        let addr_string = match rx.recv_timeout(SPAWN_HANDSHAKE_TIMEOUT) {
313            Ok(Ok(addr)) => addr,
314            Ok(Err(e)) => {
315                let _ = child.kill();
316                let _ = child.wait();
317                return Err(e);
318            }
319            Err(_) => {
320                let _ = child.kill();
321                let _ = child.wait();
322                return Err(BuildError::NotSupported {
323                    operation: format!(
324                        "zlayer-buildd did not print LISTENING within {SPAWN_HANDSHAKE_TIMEOUT:?}"
325                    ),
326                });
327            }
328        };
329
330        let channel = match dial_mtls(&addr_string, &tls).await {
331            Ok(ch) => ch,
332            Err(e) => {
333                // Dial failed — make sure we don't leak the orphaned
334                // child since we haven't yet wrapped it in the
335                // `ChildHolder` Drop guard.
336                let _ = child.kill();
337                let _ = child.wait();
338                return Err(e);
339            }
340        };
341
342        Ok(LiveSidecar {
343            addr: addr_string,
344            tls,
345            binary,
346            channel,
347            _child: Some(Arc::new(ChildHolder::new(child))),
348        })
349    }
350}
351
352/// Refresh the canonical `${ZLAYER_DATA_DIR}/bin/zlayer-buildd` from the
353/// bundle that ships next to the running `zlayer` executable, BEFORE
354/// discovery selects a binary to spawn.
355///
356/// Returns `true` only when the on-disk binary was actually (re)copied —
357/// i.e. it was missing or its content differed from the bundle. That is the
358/// signal the caller uses to drop a cached, already-running sidecar so the
359/// new binary takes effect.
360///
361/// Refresh failure is non-fatal: a dev build has no bundled sidecar next to
362/// `zlayer`, and `ZLAYER_BUILDD_BIN` overrides skip install entirely. In
363/// either case we log a warning and let discovery proceed against whatever
364/// already exists, exactly as before this refresh was wired in.
365fn refresh_canonical_binary() -> bool {
366    // The install dir MUST match the location discovery prefers
367    // (`discover.rs` step 2: `${ZLAYER_DATA_DIR}/bin`). Reuse discovery's own
368    // data-dir resolver so the two can never drift.
369    let Some(data_dir) = discover::default_data_dir() else {
370        tracing::debug!("zlayer-buildd refresh: no data dir resolved; skipping");
371        return false;
372    };
373    let install_dir = data_dir.join("bin");
374
375    #[cfg(unix)]
376    {
377        use crate::buildah::{ensure_buildd_sidecar, SidecarInstallOutcome};
378        match ensure_buildd_sidecar(&install_dir) {
379            // A recopy happened: the binary was missing or differed from the
380            // bundle. This is the only outcome that requires a respawn.
381            Ok(SidecarInstallOutcome::CopiedFromBundle { source, dest }) => {
382                tracing::info!(
383                    ?source,
384                    ?dest,
385                    "refreshed canonical zlayer-buildd from bundle"
386                );
387                true
388            }
389            Ok(other) => {
390                tracing::debug!(outcome = ?other, "zlayer-buildd already current");
391                false
392            }
393            Err(e) => {
394                // Non-fatal: missing bundle (dev build) or other install
395                // failure must not break the build — discovery still runs.
396                tracing::warn!(error = %e, "zlayer-buildd refresh failed; proceeding to discovery");
397                false
398            }
399        }
400    }
401
402    #[cfg(not(unix))]
403    {
404        let _ = install_dir;
405        false
406    }
407}
408
409/// Shared resolution for the sidecar's per-user data root.
410///
411/// Both the TLS material directory and the containers/storage tree hang
412/// off this path, so they share a single source of truth via
413/// `ZLayerDirs::buildd()` (which honors `ZLAYER_DATA_DIR` overrides).
414fn default_buildd_dir() -> PathBuf {
415    // The sidecar build runs ROOTLESS as the invoking user, so its
416    // containers/storage tree + mTLS material must live somewhere that user
417    // owns and can chmod. When a system daemon is installed, `zlayer` exports
418    // `ZLAYER_DATA_DIR=/var/lib/zlayer` process-wide (`main.rs`), whose `buildd`
419    // subdir the daemon created `root:root 0700` — a non-root `zlayer build`
420    // then EPERMs trying to chmod it, the sidecar never starts, and the backend
421    // silently falls back to the buildah-cli shellout. So pin the per-user home
422    // location (`~/.zlayer/buildd`) when not root, regardless of the system
423    // data dir. Root daemons keep the system path.
424    if zlayer_paths::is_root() {
425        zlayer_paths::ZLayerDirs::system_default().buildd()
426    } else {
427        let home = std::env::var_os("HOME")
428            .map(PathBuf::from)
429            .filter(|p| !p.as_os_str().is_empty())
430            .unwrap_or_else(std::env::temp_dir);
431        home.join(".zlayer").join("buildd")
432    }
433}
434
435fn default_tls_dir() -> PathBuf {
436    default_buildd_dir()
437}
438
439/// Per-user containers/storage paths handed to `zlayer-buildd`.
440///
441/// Storing graph + run under `${ZLAYER_DATA_DIR}/buildd/storage/` keeps
442/// the layout rootless-safe: nothing in `/var/lib/containers` or
443/// `/run/containers` is touched, so non-root users on a host with a
444/// root-only `/etc/containers/storage.conf` still build cleanly.
445#[derive(Debug, Clone, PartialEq, Eq)]
446struct StorageSpec {
447    graph_root: PathBuf,
448    run_root: PathBuf,
449    driver: String,
450}
451
452/// Resolve `StorageSpec`, pre-create graph + run dirs, and tighten perms.
453///
454/// containers/storage only creates the leaf directory itself, not parent
455/// paths, so we own the parent-dir contract. 0700 keeps any other local
456/// user from peering into the per-user image store.
457fn prepare_storage_spec(config: &zlayer_types::builder::SidecarConfig) -> Result<StorageSpec> {
458    let spec = resolve_storage_spec(config);
459    std::fs::create_dir_all(&spec.graph_root).map_err(BuildError::from)?;
460    std::fs::create_dir_all(&spec.run_root).map_err(BuildError::from)?;
461    set_dir_mode_0700(&spec.graph_root)?;
462    set_dir_mode_0700(&spec.run_root)?;
463    Ok(spec)
464}
465
466fn resolve_storage_spec(config: &zlayer_types::builder::SidecarConfig) -> StorageSpec {
467    let storage_base = default_buildd_dir().join("storage");
468
469    StorageSpec {
470        graph_root: config
471            .storage_graph_root
472            .clone()
473            .unwrap_or_else(|| storage_base.join("graph")),
474        run_root: config
475            .storage_run_root
476            .clone()
477            .unwrap_or_else(|| storage_base.join("run")),
478        driver: config
479            .storage_driver
480            .clone()
481            .unwrap_or_else(|| "vfs".to_string()),
482    }
483}
484
485async fn dial_mtls(addr: &str, tls: &TlsMaterial) -> Result<Channel> {
486    let ca = std::fs::read(&tls.ca_pem).map_err(BuildError::from)?;
487    let cert = std::fs::read(&tls.cert_pem).map_err(BuildError::from)?;
488    let key = std::fs::read(&tls.key_pem).map_err(BuildError::from)?;
489
490    let identity = Identity::from_pem(&cert, &key);
491    let ca_root = Certificate::from_pem(&ca);
492
493    let tls_config = ClientTlsConfig::new()
494        .ca_certificate(ca_root)
495        .identity(identity)
496        .domain_name("zlayer-buildd");
497
498    let uri = format!("https://{addr}")
499        .parse::<tonic::transport::Uri>()
500        .map_err(|e| BuildError::NotSupported {
501            operation: format!("invalid sidecar address {addr:?}: {e}"),
502        })?;
503
504    let endpoint = Endpoint::from(uri)
505        .tls_config(tls_config)
506        .map_err(|e| BuildError::NotSupported {
507            operation: format!("sidecar TLS config: {e}"),
508        })?
509        .connect_timeout(Duration::from_secs(10))
510        .timeout(Duration::from_secs(600));
511
512    endpoint
513        .connect()
514        .await
515        .map_err(|e| BuildError::NotSupported {
516            operation: format!("dialing sidecar at {addr}: {e}"),
517        })
518}
519
520#[cfg(test)]
521#[allow(unsafe_code)]
522mod tests {
523    use super::*;
524    use std::sync::Arc;
525
526    // The env-lock guard *must* be held across the `.await` below so
527    // that no other test races us on `PATH`/`ZLAYER_BUILDD_BIN`/
528    // `ZLAYER_DATA_DIR` while we have them clobbered. The blocking
529    // mutex is the right tool — switching to an async mutex would
530    // make every other env-mutating sync test (in discover.rs etc.)
531    // unusable.
532    #[tokio::test]
533    #[allow(clippy::await_holding_lock)]
534    async fn ensure_fails_cleanly_when_binary_missing() {
535        // No env override, restricted PATH → discover_default must
536        // fail, and ensure() must surface that as an error rather
537        // than hanging.
538        let _g = crate::TEST_ENV_LOCK
539            .lock()
540            .unwrap_or_else(std::sync::PoisonError::into_inner);
541
542        let prev_path = std::env::var_os("PATH");
543        let prev_buildd_bin = std::env::var_os("ZLAYER_BUILDD_BIN");
544        let prev_data_dir = std::env::var_os("ZLAYER_DATA_DIR");
545
546        let tmp = tempfile::tempdir().unwrap();
547        // SAFETY: env mutation serialized by `TEST_ENV_LOCK`.
548        unsafe {
549            std::env::remove_var("ZLAYER_BUILDD_BIN");
550            std::env::set_var("PATH", "/nonexistent-zlayer-test-dir");
551            std::env::set_var("ZLAYER_DATA_DIR", tmp.path());
552        }
553
554        let cfg = Arc::new(zlayer_types::builder::SidecarConfig {
555            addr: None,
556            tls_dir: Some(tmp.path().to_path_buf()),
557            idle_secs: 30,
558            ..Default::default()
559        });
560        let lifecycle = SidecarLifecycle::new(cfg);
561        let result = lifecycle.ensure().await;
562
563        // SAFETY: env mutation serialized by `TEST_ENV_LOCK`.
564        unsafe {
565            match prev_path {
566                Some(v) => std::env::set_var("PATH", v),
567                None => std::env::remove_var("PATH"),
568            }
569            match prev_buildd_bin {
570                Some(v) => std::env::set_var("ZLAYER_BUILDD_BIN", v),
571                None => std::env::remove_var("ZLAYER_BUILDD_BIN"),
572            }
573            match prev_data_dir {
574                Some(v) => std::env::set_var("ZLAYER_DATA_DIR", v),
575                None => std::env::remove_var("ZLAYER_DATA_DIR"),
576            }
577        }
578
579        let err = result.expect_err("ensure() should fail when binary cannot be discovered");
580        let msg = err.to_string();
581        assert!(
582            msg.contains("zlayer-buildd") || msg.contains("not found"),
583            "error should mention the missing binary: {msg}"
584        );
585    }
586
587    /// End-to-end spawn smoke. Builds the Go sidecar once with
588    /// `make build` in `bin/zlayer-buildd`, then sets
589    /// `ZLAYER_BUILDD_BIN` and runs this test with `--ignored`.
590    #[tokio::test]
591    #[ignore = "requires zlayer-buildd binary; gate with ZLAYER_BUILDD_BIN"]
592    #[allow(clippy::await_holding_lock)]
593    async fn spawn_smoke() {
594        let _g = crate::TEST_ENV_LOCK
595            .lock()
596            .unwrap_or_else(std::sync::PoisonError::into_inner);
597        let tmp = tempfile::tempdir().unwrap();
598        // Scope storage to the tempdir too so the smoke test doesn't
599        // pollute the real per-user `${data_dir}/buildd/storage` tree.
600        let cfg = Arc::new(zlayer_types::builder::SidecarConfig {
601            addr: None,
602            tls_dir: Some(tmp.path().to_path_buf()),
603            idle_secs: 30,
604            storage_graph_root: Some(tmp.path().join("storage").join("graph")),
605            storage_run_root: Some(tmp.path().join("storage").join("run")),
606            storage_driver: Some("vfs".into()),
607            context_mount: None,
608        });
609        let lifecycle = SidecarLifecycle::new(cfg);
610        let live = lifecycle
611            .ensure()
612            .await
613            .expect("sidecar should spawn and handshake");
614        assert!(
615            live.addr.starts_with("127.0.0.1:"),
616            "expected loopback addr, got {}",
617            live.addr
618        );
619        // Build a client; we don't issue an RPC because the proto
620        // server-side handlers may not exist yet — the dial alone
621        // proves we got past the handshake and TLS negotiation.
622        let _client = live.client();
623    }
624
625    #[test]
626    fn storage_spec_defaults_to_buildd_storage() {
627        let cfg = zlayer_types::builder::SidecarConfig::default();
628        let spec = resolve_storage_spec(&cfg);
629        assert_eq!(spec.driver, "vfs");
630        assert!(
631            spec.graph_root.ends_with("graph"),
632            "expected graph leaf, got {:?}",
633            spec.graph_root
634        );
635        assert!(
636            spec.run_root.ends_with("run"),
637            "expected run leaf, got {:?}",
638            spec.run_root
639        );
640        assert!(
641            spec.graph_root.parent().unwrap().ends_with("storage"),
642            "expected storage parent, got {:?}",
643            spec.graph_root.parent()
644        );
645        assert!(
646            spec.run_root.parent().unwrap().ends_with("storage"),
647            "expected storage parent, got {:?}",
648            spec.run_root.parent()
649        );
650    }
651
652    #[test]
653    fn storage_spec_honors_override() {
654        let cfg = zlayer_types::builder::SidecarConfig {
655            storage_graph_root: Some(PathBuf::from("/tmp/test-graph")),
656            storage_run_root: Some(PathBuf::from("/tmp/test-run")),
657            storage_driver: Some("overlay".into()),
658            ..Default::default()
659        };
660        let spec = resolve_storage_spec(&cfg);
661        assert_eq!(spec.graph_root, PathBuf::from("/tmp/test-graph"));
662        assert_eq!(spec.run_root, PathBuf::from("/tmp/test-run"));
663        assert_eq!(spec.driver, "overlay");
664    }
665}