zlayer_builder/backend/buildah_sidecar/
lifecycle.rs1use std::io::{BufRead, BufReader};
20use std::path::PathBuf;
21use std::process::{Child, Command, Stdio};
22use std::sync::Arc;
23use std::time::{Duration, Instant};
24
25use tokio::sync::Mutex;
26use tonic::transport::{Certificate, Channel, ClientTlsConfig, Endpoint, Identity};
27
28use crate::backend::buildah_sidecar::discover::{self, Discovery};
29use crate::backend::buildah_sidecar::proto::build_service_client::BuildServiceClient;
30use crate::backend::buildah_sidecar::tls::{ensure_tls_material, set_dir_mode_0700, TlsMaterial};
31use crate::error::{BuildError, Result};
32
33const HANDSHAKE_PREFIX: &str = "LISTENING ";
35
36const SPAWN_HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(15);
38
39const SIGTERM_GRACE: Duration = Duration::from_secs(5);
41
42#[derive(Debug)]
48struct ChildHolder {
49 child: std::sync::Mutex<Option<Child>>,
50}
51
52impl ChildHolder {
53 fn new(child: Child) -> Self {
54 Self {
55 child: std::sync::Mutex::new(Some(child)),
56 }
57 }
58}
59
60impl Drop for ChildHolder {
61 fn drop(&mut self) {
62 let mut guard = match self.child.lock() {
65 Ok(g) => g,
66 Err(p) => p.into_inner(),
67 };
68 let Some(mut child) = guard.take() else {
69 return;
70 };
71
72 #[cfg(unix)]
74 {
75 use nix::sys::signal::{kill, Signal};
76 use nix::unistd::Pid;
77
78 if let Ok(raw) = i32::try_from(child.id()) {
85 let pid = Pid::from_raw(raw);
86 let _ = kill(pid, Signal::SIGTERM);
87 }
88 }
89
90 let deadline = Instant::now() + SIGTERM_GRACE;
93 loop {
94 match child.try_wait() {
95 Ok(Some(_)) => return,
96 Ok(None) => {
97 if Instant::now() >= deadline {
98 let _ = child.kill();
99 let _ = child.wait();
100 return;
101 }
102 std::thread::sleep(Duration::from_millis(50));
103 }
104 Err(_) => {
105 let _ = child.kill();
106 let _ = child.wait();
107 return;
108 }
109 }
110 }
111 }
112}
113
114#[derive(Debug, Clone)]
122pub struct LiveSidecar {
123 pub addr: String,
126 pub tls: TlsMaterial,
128 pub binary: PathBuf,
131 pub channel: Channel,
133 _child: Option<Arc<ChildHolder>>,
136}
137
138impl LiveSidecar {
139 #[must_use]
141 pub fn client(&self) -> BuildServiceClient<Channel> {
142 BuildServiceClient::new(self.channel.clone())
143 }
144}
145
146#[derive(Debug)]
153pub struct SidecarLifecycle {
154 config: Arc<zlayer_types::builder::SidecarConfig>,
155 state: Mutex<Option<LiveSidecar>>,
156}
157
158impl SidecarLifecycle {
159 #[must_use]
162 pub fn new(config: Arc<zlayer_types::builder::SidecarConfig>) -> Self {
163 Self {
164 config,
165 state: Mutex::new(None),
166 }
167 }
168
169 pub async fn ensure(&self) -> Result<LiveSidecar> {
178 let refreshed = refresh_canonical_binary();
186
187 let mut guard = self.state.lock().await;
188
189 if refreshed {
190 *guard = None;
193 }
194
195 if let Some(existing) = guard.as_ref() {
196 return Ok(existing.clone());
197 }
198
199 let live = self.spawn_and_dial().await?;
200 *guard = Some(live.clone());
201 Ok(live)
202 }
203
204 pub async fn drop_connection(&self) {
209 let mut guard = self.state.lock().await;
210 *guard = None;
211 }
212
213 #[allow(clippy::too_many_lines)]
214 async fn spawn_and_dial(&self) -> Result<LiveSidecar> {
215 let tls_dir = self.config.tls_dir.clone().unwrap_or_else(default_tls_dir);
216
217 let tls = ensure_tls_material(&tls_dir)?;
218
219 let storage = prepare_storage_spec(&self.config)?;
220
221 if let Some(addr) = self.config.addr.clone() {
224 let channel = dial_mtls(&addr, &tls).await?;
225 return Ok(LiveSidecar {
226 addr,
227 tls,
228 binary: PathBuf::new(),
229 channel,
230 _child: None,
231 });
232 }
233
234 let Discovery { binary, tried } = discover::discover_default()?;
236 tracing::info!(?binary, ?tried, "spawning zlayer-buildd");
237
238 let mut cmd = Command::new(&binary);
239 cmd.arg("--bind").arg("127.0.0.1:0");
240 cmd.arg("--tls-ca").arg(&tls.ca_pem);
241 cmd.arg("--tls-cert").arg(&tls.cert_pem);
242 cmd.arg("--tls-key").arg(&tls.key_pem);
243 cmd.arg("--idle-secs")
244 .arg(self.config.idle_secs.to_string());
245 cmd.arg("--storage-root").arg(&storage.graph_root);
246 cmd.arg("--storage-runroot").arg(&storage.run_root);
247 cmd.arg("--storage-driver").arg(&storage.driver);
248 cmd.stdin(Stdio::null());
249 cmd.stdout(Stdio::piped());
250 cmd.stderr(Stdio::piped());
251
252 let mut child = cmd.spawn().map_err(|e| BuildError::NotSupported {
253 operation: format!("spawning zlayer-buildd at {}: {e}", binary.display()),
254 })?;
255
256 if let Some(child_stderr) = child.stderr.take() {
262 std::thread::spawn(move || {
263 let reader = BufReader::new(child_stderr);
264 for line in reader.lines().map_while(std::result::Result::ok) {
265 eprintln!("[zlayer-buildd] {line}");
266 }
267 });
268 }
269
270 let stdout = child
271 .stdout
272 .take()
273 .ok_or_else(|| BuildError::NotSupported {
274 operation: "zlayer-buildd: missing stdout pipe".into(),
275 })?;
276
277 let (tx, rx) = std::sync::mpsc::channel::<Result<String>>();
283 std::thread::spawn(move || {
284 let mut reader = BufReader::new(stdout);
285 let mut line = String::new();
286 let send_result = match reader.read_line(&mut line) {
287 Ok(0) => tx.send(Err(BuildError::NotSupported {
288 operation: "zlayer-buildd exited before printing LISTENING".into(),
289 })),
290 Ok(_) => {
291 let trimmed = line
292 .trim_end_matches('\n')
293 .trim_end_matches('\r')
294 .to_string();
295 if let Some(addr) = trimmed.strip_prefix(HANDSHAKE_PREFIX) {
296 tx.send(Ok(addr.to_string()))
297 } else {
298 tx.send(Err(BuildError::NotSupported {
299 operation: format!("zlayer-buildd handshake malformed: {trimmed:?}"),
300 }))
301 }
302 }
303 Err(e) => tx.send(Err(BuildError::NotSupported {
304 operation: format!("reading zlayer-buildd stdout: {e}"),
305 })),
306 };
307 let _ = send_result;
310 });
311
312 let addr_string = match rx.recv_timeout(SPAWN_HANDSHAKE_TIMEOUT) {
313 Ok(Ok(addr)) => addr,
314 Ok(Err(e)) => {
315 let _ = child.kill();
316 let _ = child.wait();
317 return Err(e);
318 }
319 Err(_) => {
320 let _ = child.kill();
321 let _ = child.wait();
322 return Err(BuildError::NotSupported {
323 operation: format!(
324 "zlayer-buildd did not print LISTENING within {SPAWN_HANDSHAKE_TIMEOUT:?}"
325 ),
326 });
327 }
328 };
329
330 let channel = match dial_mtls(&addr_string, &tls).await {
331 Ok(ch) => ch,
332 Err(e) => {
333 let _ = child.kill();
337 let _ = child.wait();
338 return Err(e);
339 }
340 };
341
342 Ok(LiveSidecar {
343 addr: addr_string,
344 tls,
345 binary,
346 channel,
347 _child: Some(Arc::new(ChildHolder::new(child))),
348 })
349 }
350}
351
352fn refresh_canonical_binary() -> bool {
366 let Some(data_dir) = discover::default_data_dir() else {
370 tracing::debug!("zlayer-buildd refresh: no data dir resolved; skipping");
371 return false;
372 };
373 let install_dir = data_dir.join("bin");
374
375 #[cfg(unix)]
376 {
377 use crate::buildah::{ensure_buildd_sidecar, SidecarInstallOutcome};
378 match ensure_buildd_sidecar(&install_dir) {
379 Ok(SidecarInstallOutcome::CopiedFromBundle { source, dest }) => {
382 tracing::info!(
383 ?source,
384 ?dest,
385 "refreshed canonical zlayer-buildd from bundle"
386 );
387 true
388 }
389 Ok(other) => {
390 tracing::debug!(outcome = ?other, "zlayer-buildd already current");
391 false
392 }
393 Err(e) => {
394 tracing::warn!(error = %e, "zlayer-buildd refresh failed; proceeding to discovery");
397 false
398 }
399 }
400 }
401
402 #[cfg(not(unix))]
403 {
404 let _ = install_dir;
405 false
406 }
407}
408
409fn default_buildd_dir() -> PathBuf {
415 if zlayer_paths::is_root() {
425 zlayer_paths::ZLayerDirs::system_default().buildd()
426 } else {
427 let home = std::env::var_os("HOME")
428 .map(PathBuf::from)
429 .filter(|p| !p.as_os_str().is_empty())
430 .unwrap_or_else(std::env::temp_dir);
431 home.join(".zlayer").join("buildd")
432 }
433}
434
435fn default_tls_dir() -> PathBuf {
436 default_buildd_dir()
437}
438
439#[derive(Debug, Clone, PartialEq, Eq)]
446struct StorageSpec {
447 graph_root: PathBuf,
448 run_root: PathBuf,
449 driver: String,
450}
451
452fn prepare_storage_spec(config: &zlayer_types::builder::SidecarConfig) -> Result<StorageSpec> {
458 let spec = resolve_storage_spec(config);
459 std::fs::create_dir_all(&spec.graph_root).map_err(BuildError::from)?;
460 std::fs::create_dir_all(&spec.run_root).map_err(BuildError::from)?;
461 set_dir_mode_0700(&spec.graph_root)?;
462 set_dir_mode_0700(&spec.run_root)?;
463 Ok(spec)
464}
465
466fn resolve_storage_spec(config: &zlayer_types::builder::SidecarConfig) -> StorageSpec {
467 let storage_base = default_buildd_dir().join("storage");
468
469 StorageSpec {
470 graph_root: config
471 .storage_graph_root
472 .clone()
473 .unwrap_or_else(|| storage_base.join("graph")),
474 run_root: config
475 .storage_run_root
476 .clone()
477 .unwrap_or_else(|| storage_base.join("run")),
478 driver: config
479 .storage_driver
480 .clone()
481 .unwrap_or_else(|| "vfs".to_string()),
482 }
483}
484
485async fn dial_mtls(addr: &str, tls: &TlsMaterial) -> Result<Channel> {
486 let ca = std::fs::read(&tls.ca_pem).map_err(BuildError::from)?;
487 let cert = std::fs::read(&tls.cert_pem).map_err(BuildError::from)?;
488 let key = std::fs::read(&tls.key_pem).map_err(BuildError::from)?;
489
490 let identity = Identity::from_pem(&cert, &key);
491 let ca_root = Certificate::from_pem(&ca);
492
493 let tls_config = ClientTlsConfig::new()
494 .ca_certificate(ca_root)
495 .identity(identity)
496 .domain_name("zlayer-buildd");
497
498 let uri = format!("https://{addr}")
499 .parse::<tonic::transport::Uri>()
500 .map_err(|e| BuildError::NotSupported {
501 operation: format!("invalid sidecar address {addr:?}: {e}"),
502 })?;
503
504 let endpoint = Endpoint::from(uri)
505 .tls_config(tls_config)
506 .map_err(|e| BuildError::NotSupported {
507 operation: format!("sidecar TLS config: {e}"),
508 })?
509 .connect_timeout(Duration::from_secs(10))
510 .timeout(Duration::from_secs(600));
511
512 endpoint
513 .connect()
514 .await
515 .map_err(|e| BuildError::NotSupported {
516 operation: format!("dialing sidecar at {addr}: {e}"),
517 })
518}
519
520#[cfg(test)]
521#[allow(unsafe_code)]
522mod tests {
523 use super::*;
524 use std::sync::Arc;
525
526 #[tokio::test]
533 #[allow(clippy::await_holding_lock)]
534 async fn ensure_fails_cleanly_when_binary_missing() {
535 let _g = crate::TEST_ENV_LOCK
539 .lock()
540 .unwrap_or_else(std::sync::PoisonError::into_inner);
541
542 let prev_path = std::env::var_os("PATH");
543 let prev_buildd_bin = std::env::var_os("ZLAYER_BUILDD_BIN");
544 let prev_data_dir = std::env::var_os("ZLAYER_DATA_DIR");
545
546 let tmp = tempfile::tempdir().unwrap();
547 unsafe {
549 std::env::remove_var("ZLAYER_BUILDD_BIN");
550 std::env::set_var("PATH", "/nonexistent-zlayer-test-dir");
551 std::env::set_var("ZLAYER_DATA_DIR", tmp.path());
552 }
553
554 let cfg = Arc::new(zlayer_types::builder::SidecarConfig {
555 addr: None,
556 tls_dir: Some(tmp.path().to_path_buf()),
557 idle_secs: 30,
558 ..Default::default()
559 });
560 let lifecycle = SidecarLifecycle::new(cfg);
561 let result = lifecycle.ensure().await;
562
563 unsafe {
565 match prev_path {
566 Some(v) => std::env::set_var("PATH", v),
567 None => std::env::remove_var("PATH"),
568 }
569 match prev_buildd_bin {
570 Some(v) => std::env::set_var("ZLAYER_BUILDD_BIN", v),
571 None => std::env::remove_var("ZLAYER_BUILDD_BIN"),
572 }
573 match prev_data_dir {
574 Some(v) => std::env::set_var("ZLAYER_DATA_DIR", v),
575 None => std::env::remove_var("ZLAYER_DATA_DIR"),
576 }
577 }
578
579 let err = result.expect_err("ensure() should fail when binary cannot be discovered");
580 let msg = err.to_string();
581 assert!(
582 msg.contains("zlayer-buildd") || msg.contains("not found"),
583 "error should mention the missing binary: {msg}"
584 );
585 }
586
587 #[tokio::test]
591 #[ignore = "requires zlayer-buildd binary; gate with ZLAYER_BUILDD_BIN"]
592 #[allow(clippy::await_holding_lock)]
593 async fn spawn_smoke() {
594 let _g = crate::TEST_ENV_LOCK
595 .lock()
596 .unwrap_or_else(std::sync::PoisonError::into_inner);
597 let tmp = tempfile::tempdir().unwrap();
598 let cfg = Arc::new(zlayer_types::builder::SidecarConfig {
601 addr: None,
602 tls_dir: Some(tmp.path().to_path_buf()),
603 idle_secs: 30,
604 storage_graph_root: Some(tmp.path().join("storage").join("graph")),
605 storage_run_root: Some(tmp.path().join("storage").join("run")),
606 storage_driver: Some("vfs".into()),
607 context_mount: None,
608 });
609 let lifecycle = SidecarLifecycle::new(cfg);
610 let live = lifecycle
611 .ensure()
612 .await
613 .expect("sidecar should spawn and handshake");
614 assert!(
615 live.addr.starts_with("127.0.0.1:"),
616 "expected loopback addr, got {}",
617 live.addr
618 );
619 let _client = live.client();
623 }
624
625 #[test]
626 fn storage_spec_defaults_to_buildd_storage() {
627 let cfg = zlayer_types::builder::SidecarConfig::default();
628 let spec = resolve_storage_spec(&cfg);
629 assert_eq!(spec.driver, "vfs");
630 assert!(
631 spec.graph_root.ends_with("graph"),
632 "expected graph leaf, got {:?}",
633 spec.graph_root
634 );
635 assert!(
636 spec.run_root.ends_with("run"),
637 "expected run leaf, got {:?}",
638 spec.run_root
639 );
640 assert!(
641 spec.graph_root.parent().unwrap().ends_with("storage"),
642 "expected storage parent, got {:?}",
643 spec.graph_root.parent()
644 );
645 assert!(
646 spec.run_root.parent().unwrap().ends_with("storage"),
647 "expected storage parent, got {:?}",
648 spec.run_root.parent()
649 );
650 }
651
652 #[test]
653 fn storage_spec_honors_override() {
654 let cfg = zlayer_types::builder::SidecarConfig {
655 storage_graph_root: Some(PathBuf::from("/tmp/test-graph")),
656 storage_run_root: Some(PathBuf::from("/tmp/test-run")),
657 storage_driver: Some("overlay".into()),
658 ..Default::default()
659 };
660 let spec = resolve_storage_spec(&cfg);
661 assert_eq!(spec.graph_root, PathBuf::from("/tmp/test-graph"));
662 assert_eq!(spec.run_root, PathBuf::from("/tmp/test-run"));
663 assert_eq!(spec.driver, "overlay");
664 }
665}