1use crate::audit::{audit, audit_error, AuditEventType};
2use crate::container::{
3 ContainerConfig, ContainerState, ContainerStateManager, ContainerStateParams, OciStatus,
4 RootfsMode, ServiceMode, WorkspaceMode,
5};
6use crate::error::{NucleusError, Result, StateTransition};
7use crate::filesystem::{
8 audit_mounts, bind_mount_host_paths, bind_mount_rootfs, copy_workspace_in, create_dev_nodes,
9 create_minimal_fs, mask_proc_paths, mount_gpu_passthrough, mount_home_tmpfs, mount_procfs,
10 mount_provider_configs, mount_secrets_inmemory, mount_volumes, mount_workspace,
11 overlay_mount_rootfs, resolve_gpu_devices, snapshot_context_dir, switch_root,
12 sync_workspace_out, validate_production_rootfs_path, verify_context_manifest,
13 verify_rootfs_attestation, FilesystemState, GpuDeviceSet, LazyContextPopulator, TmpfsMount,
14};
15use crate::isolation::{NamespaceManager, UserNamespaceMapper};
16use crate::network::{BridgeDriver, BridgeNetwork, NatBackend, NetworkMode, UserspaceNetwork};
17use crate::resources::{Cgroup, ResourceStats};
18use crate::security::{
19 CapabilityManager, GVisorRuntime, LandlockManager, OciContainerState, OciHooks,
20 SeccompDenyLogger, SeccompManager, SeccompTraceReader, SecurityState,
21};
22use crate::telemetry::{CleanupEvent, ContainerControlEvent, EventSink, ExitStatusEvent};
23use caps::{CapSet, Capability, CapsHashSet};
24use nix::sys::signal::{kill, Signal};
25use nix::sys::signal::{pthread_sigmask, SigSet, SigmaskHow};
26use nix::sys::stat::Mode;
27use nix::sys::wait::{waitpid, WaitStatus};
28use nix::unistd::{
29 chown, fork, pipe, read, setresgid, setresuid, write, ForkResult, Gid, Pid, Uid,
30};
31use std::net::{SocketAddr, SocketAddrV4, TcpStream};
32use std::os::fd::OwnedFd;
33use std::os::unix::fs::PermissionsExt;
34use std::path::{Path, PathBuf};
35use std::sync::atomic::{AtomicBool, Ordering};
36use std::sync::Arc;
37use std::thread::JoinHandle;
38use std::time::Duration;
39use tempfile::Builder;
40use tracing::{debug, error, info, info_span, warn};
41
42use super::console::{NativeConsoleRelay, NativePty};
43
44pub struct Container {
53 pub(super) config: ContainerConfig,
54 pub(super) runsc_path: Option<String>,
57 pub(super) event_sink: Option<EventSink>,
58}
59
60pub struct CreatedContainer {
64 pub(super) config: ContainerConfig,
65 pub(super) state_mgr: ContainerStateManager,
66 pub(super) state: ContainerState,
67 pub(super) child: Pid,
68 pub(super) cgroup_opt: Option<Cgroup>,
69 pub(super) network_driver: Option<BridgeDriver>,
70 pub(super) trace_reader: Option<SeccompTraceReader>,
71 pub(super) deny_logger: Option<SeccompDenyLogger>,
72 pub(super) exec_fifo_path: Option<PathBuf>,
73 pub(super) native_console_master: Option<OwnedFd>,
74 pub(super) event_sink: Option<EventSink>,
75 pub(super) _lifecycle_span: tracing::Span,
76}
77
78const CREDENTIAL_BROKER_CONNECT_TIMEOUT: Duration = Duration::from_millis(750);
79
80impl Container {
81 pub fn new(config: ContainerConfig) -> Self {
82 Self {
83 config,
84 runsc_path: None,
85 event_sink: None,
86 }
87 }
88
89 pub fn with_event_sink(mut self, event_sink: Option<EventSink>) -> Self {
90 self.event_sink = event_sink;
91 self
92 }
93
94 fn probe_credential_broker(broker: &crate::network::CredentialBrokerConfig) -> Result<()> {
95 let addr = SocketAddr::V4(SocketAddrV4::new(broker.broker_ip, broker.broker_port));
96 TcpStream::connect_timeout(&addr, CREDENTIAL_BROKER_CONNECT_TIMEOUT)
97 .map(|_| ())
98 .map_err(|e| {
99 NucleusError::NetworkError(format!(
100 "Credential broker {} was not reachable within {:?}: {}",
101 addr, CREDENTIAL_BROKER_CONNECT_TIMEOUT, e
102 ))
103 })
104 }
105
106 fn validate_credential_broker_resolved_nat(
107 config: &ContainerConfig,
108 host_is_root: bool,
109 ) -> Result<()> {
110 if config.credential_broker.is_none() {
111 return Ok(());
112 }
113 let NetworkMode::Bridge(ref bridge_config) = config.network else {
114 return Ok(());
115 };
116
117 let backend =
118 bridge_config.selected_nat_backend(host_is_root, config.user_ns_config.is_some());
119 if backend == NatBackend::Userspace {
120 return Err(NucleusError::ConfigError(
121 "Credential broker egress requires the kernel NAT backend; \
122 --nat-backend auto resolves to userspace for rootless/native containers"
123 .to_string(),
124 ));
125 }
126
127 Ok(())
128 }
129
130 pub fn run(&self) -> Result<i32> {
132 self.create_internal(false)?.start()
133 }
134
135 pub fn create(&self) -> Result<CreatedContainer> {
139 self.create_internal(true)
140 }
141
142 fn sanitize_fds() {
148 const CLOSE_RANGE_CLOEXEC: libc::c_uint = 4;
151 let ret =
153 unsafe { libc::syscall(libc::SYS_close_range, 3u32, u32::MAX, CLOSE_RANGE_CLOEXEC) };
154 if ret == 0 {
155 return;
156 }
157 if let Ok(entries) = std::fs::read_dir("/proc/self/fd") {
161 let fds: Vec<i32> = entries
162 .flatten()
163 .filter_map(|entry| entry.file_name().into_string().ok())
164 .filter_map(|s| s.parse::<i32>().ok())
165 .filter(|&fd| fd > 2)
166 .collect();
167 for fd in fds {
168 unsafe { libc::fcntl(fd, libc::F_SETFD, libc::FD_CLOEXEC) };
169 }
170 }
171 }
172
173 pub(crate) fn close_nonstdio_fds() {
178 let ret = unsafe { libc::syscall(libc::SYS_close_range, 3u32, u32::MAX, 0u32) };
180 if ret == 0 {
181 return;
182 }
183
184 if let Ok(entries) = std::fs::read_dir("/proc/self/fd") {
185 let fds: Vec<i32> = entries
186 .flatten()
187 .filter_map(|entry| entry.file_name().into_string().ok())
188 .filter_map(|s| s.parse::<i32>().ok())
189 .filter(|&fd| fd > 2)
190 .collect();
191 for fd in fds {
192 unsafe { libc::close(fd) };
193 }
194 return;
195 }
196
197 let max_fd = match unsafe { libc::sysconf(libc::_SC_OPEN_MAX) } {
198 n if n > 3 && n <= i32::MAX as libc::c_long => n as i32,
199 _ => 1024,
200 };
201 for fd in 3..max_fd {
202 unsafe { libc::close(fd) };
203 }
204 }
205
206 pub(crate) fn assert_single_threaded_for_fork(context: &str) -> Result<()> {
207 let thread_count = std::fs::read_to_string("/proc/self/status")
208 .ok()
209 .and_then(|s| {
210 s.lines()
211 .find(|line| line.starts_with("Threads:"))
212 .and_then(|line| line.split_whitespace().nth(1))
213 .and_then(|count| count.parse::<u32>().ok())
214 });
215
216 if thread_count == Some(1) {
217 return Ok(());
218 }
219
220 Err(NucleusError::ExecError(format!(
221 "{} requires a single-threaded process before fork, found {:?} threads",
222 context, thread_count
223 )))
224 }
225
226 fn gvisor_needs_precreated_userns(config: &ContainerConfig, host_is_root: bool) -> bool {
227 config.use_gvisor
228 && !host_is_root
229 && config.user_ns_config.is_some()
230 && matches!(
231 config.network,
232 NetworkMode::Bridge(_) | NetworkMode::GVisorHost
233 )
234 }
235
236 fn prepare_runtime_base_override(
237 config: &ContainerConfig,
238 host_is_root: bool,
239 needs_external_userns_mapping: bool,
240 ) -> Result<Option<PathBuf>> {
241 if !needs_external_userns_mapping {
242 return Ok(None);
243 }
244
245 if !host_is_root {
246 return Ok(Some(
247 dirs::runtime_dir()
248 .map(|d| d.join("nucleus"))
249 .unwrap_or_else(std::env::temp_dir),
250 ));
251 }
252
253 let user_config = config.user_ns_config.as_ref().ok_or_else(|| {
254 NucleusError::ExecError("Missing user namespace configuration".to_string())
255 })?;
256 let host_uid =
257 Self::mapped_host_id_for_container_id(&user_config.uid_mappings, 0, "uid mappings")?;
258 let host_gid =
259 Self::mapped_host_id_for_container_id(&user_config.gid_mappings, 0, "gid mappings")?;
260
261 let root = PathBuf::from("/run/nucleus");
262 Self::ensure_runtime_parent_dir(&root)?;
263
264 let runtime_root = root.join("runtime");
265 Self::ensure_runtime_parent_dir(&runtime_root)?;
266
267 let base = runtime_root.join(&config.id);
268 std::fs::create_dir_all(&base).map_err(|e| {
269 NucleusError::FilesystemError(format!(
270 "Failed to create user namespace runtime base {:?}: {}",
271 base, e
272 ))
273 })?;
274 chown(
275 &base,
276 Some(Uid::from_raw(host_uid)),
277 Some(Gid::from_raw(host_gid)),
278 )
279 .map_err(|e| {
280 NucleusError::FilesystemError(format!(
281 "Failed to chown user namespace runtime base {:?} to {}:{}: {}",
282 base, host_uid, host_gid, e
283 ))
284 })?;
285 std::fs::set_permissions(&base, std::fs::Permissions::from_mode(0o700)).map_err(|e| {
286 NucleusError::FilesystemError(format!(
287 "Failed to secure user namespace runtime base {:?}: {}",
288 base, e
289 ))
290 })?;
291
292 Ok(Some(base))
293 }
294
295 fn ensure_runtime_parent_dir(path: &std::path::Path) -> Result<()> {
296 std::fs::create_dir_all(path).map_err(|e| {
297 NucleusError::FilesystemError(format!(
298 "Failed to create runtime parent dir {:?}: {}",
299 path, e
300 ))
301 })?;
302 std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o711)).map_err(|e| {
303 NucleusError::FilesystemError(format!(
304 "Failed to secure runtime parent dir {:?}: {}",
305 path, e
306 ))
307 })?;
308 Ok(())
309 }
310
311 fn prepare_workspace_staging(
312 config: &mut ContainerConfig,
313 runtime_base_override: Option<&Path>,
314 host_is_root: bool,
315 needs_external_userns_mapping: bool,
316 ) -> Result<()> {
317 if config.workspace.mode != WorkspaceMode::CopyInOut {
318 return Ok(());
319 }
320
321 let source = config.workspace.host_path.clone().ok_or_else(|| {
322 NucleusError::ConfigError(
323 "--workspace-mode copy-in-out requires --workspace".to_string(),
324 )
325 })?;
326 let parent = runtime_base_override
327 .map(Path::to_path_buf)
328 .unwrap_or_else(|| {
329 dirs::runtime_dir()
330 .map(|d| d.join("nucleus"))
331 .unwrap_or_else(std::env::temp_dir)
332 });
333 std::fs::create_dir_all(&parent).map_err(|e| {
334 NucleusError::FilesystemError(format!(
335 "Failed to create workspace staging parent {:?}: {}",
336 parent, e
337 ))
338 })?;
339
340 let staging = parent.join(format!("workspace-{}", config.id));
341 if staging.exists() {
342 return Err(NucleusError::FilesystemError(format!(
343 "Workspace staging directory already exists: {}",
344 staging.display()
345 )));
346 }
347 std::fs::create_dir(&staging).map_err(|e| {
348 NucleusError::FilesystemError(format!(
349 "Failed to create workspace staging directory {:?}: {}",
350 staging, e
351 ))
352 })?;
353 std::fs::set_permissions(&staging, std::fs::Permissions::from_mode(0o700)).map_err(
354 |e| {
355 NucleusError::FilesystemError(format!(
356 "Failed to secure workspace staging directory {:?}: {}",
357 staging, e
358 ))
359 },
360 )?;
361
362 copy_workspace_in(&source, &staging)?;
363
364 if host_is_root && needs_external_userns_mapping {
365 let user_config = config.user_ns_config.as_ref().ok_or_else(|| {
366 NucleusError::ExecError("Missing user namespace configuration".to_string())
367 })?;
368 let host_uid = Self::mapped_host_id_for_container_id(
369 &user_config.uid_mappings,
370 0,
371 "uid mappings",
372 )?;
373 let host_gid = Self::mapped_host_id_for_container_id(
374 &user_config.gid_mappings,
375 0,
376 "gid mappings",
377 )?;
378 Self::chown_tree_no_symlinks(&staging, host_uid, host_gid)?;
379 }
380
381 config.workspace.staging_path = Some(staging);
382 Ok(())
383 }
384
385 fn prepare_rootfs_overlay(
386 config: &mut ContainerConfig,
387 state_mgr: &ContainerStateManager,
388 host_is_root: bool,
389 needs_external_userns_mapping: bool,
390 ) -> Result<()> {
391 if config.rootfs_mode != RootfsMode::Overlay {
392 return Ok(());
393 }
394 if config.rootfs_path.is_none() {
395 return Err(NucleusError::ConfigError(
396 "--rootfs-mode overlay requires a rootfs path".to_string(),
397 ));
398 }
399 if config.rootfs_overlay.is_some() {
400 return Ok(());
401 }
402
403 let overlay_dir = state_mgr.rootfs_overlay_dir_path(&config.id)?;
404 if overlay_dir.exists() {
405 return Err(NucleusError::FilesystemError(format!(
406 "Rootfs overlay directory already exists: {}",
407 overlay_dir.display()
408 )));
409 }
410 std::fs::create_dir(&overlay_dir).map_err(|e| {
411 NucleusError::FilesystemError(format!(
412 "Failed to create rootfs overlay directory {:?}: {}",
413 overlay_dir, e
414 ))
415 })?;
416 std::fs::set_permissions(&overlay_dir, std::fs::Permissions::from_mode(0o700)).map_err(
417 |e| {
418 NucleusError::FilesystemError(format!(
419 "Failed to secure rootfs overlay directory {:?}: {}",
420 overlay_dir, e
421 ))
422 },
423 )?;
424
425 let upperdir = overlay_dir.join("upper");
426 let workdir = overlay_dir.join("work");
427 std::fs::create_dir(&upperdir).map_err(|e| {
428 NucleusError::FilesystemError(format!(
429 "Failed to create rootfs overlay upperdir {:?}: {}",
430 upperdir, e
431 ))
432 })?;
433 std::fs::create_dir(&workdir).map_err(|e| {
434 NucleusError::FilesystemError(format!(
435 "Failed to create rootfs overlay workdir {:?}: {}",
436 workdir, e
437 ))
438 })?;
439
440 if host_is_root && needs_external_userns_mapping {
441 let user_config = config.user_ns_config.as_ref().ok_or_else(|| {
442 NucleusError::ExecError("Missing user namespace configuration".to_string())
443 })?;
444 let host_uid = Self::mapped_host_id_for_container_id(
445 &user_config.uid_mappings,
446 0,
447 "uid mappings",
448 )?;
449 let host_gid = Self::mapped_host_id_for_container_id(
450 &user_config.gid_mappings,
451 0,
452 "gid mappings",
453 )?;
454 Self::chown_tree_no_symlinks(&overlay_dir, host_uid, host_gid)?;
455 }
456
457 config.rootfs_overlay = Some(crate::container::RootfsOverlayConfig { upperdir, workdir });
458 Ok(())
459 }
460
461 fn chown_tree_no_symlinks(path: &Path, uid: u32, gid: u32) -> Result<()> {
462 let metadata = std::fs::symlink_metadata(path).map_err(|e| {
463 NucleusError::FilesystemError(format!(
464 "Failed to stat workspace staging path {:?}: {}",
465 path, e
466 ))
467 })?;
468 if metadata.file_type().is_symlink() {
469 return Ok(());
470 }
471
472 chown(path, Some(Uid::from_raw(uid)), Some(Gid::from_raw(gid))).map_err(|e| {
473 NucleusError::FilesystemError(format!(
474 "Failed to chown workspace staging path {:?} to {}:{}: {}",
475 path, uid, gid, e
476 ))
477 })?;
478
479 if metadata.is_dir() {
480 for entry in std::fs::read_dir(path).map_err(|e| {
481 NucleusError::FilesystemError(format!(
482 "Failed to read workspace staging dir {:?}: {}",
483 path, e
484 ))
485 })? {
486 let entry = entry.map_err(|e| {
487 NucleusError::FilesystemError(format!(
488 "Failed to read workspace staging entry in {:?}: {}",
489 path, e
490 ))
491 })?;
492 Self::chown_tree_no_symlinks(&entry.path(), uid, gid)?;
493 }
494 }
495
496 Ok(())
497 }
498
499 fn sync_workspace_copy_in_out(config: &ContainerConfig) -> Result<()> {
500 if config.workspace.mode != WorkspaceMode::CopyInOut {
501 return Ok(());
502 }
503 let Some(staging) = config.workspace.staging_path.as_ref() else {
504 return Err(NucleusError::FilesystemError(
505 "copy-in-out workspace missing staging path".to_string(),
506 ));
507 };
508 let Some(host_path) = config.workspace.host_path.as_ref() else {
509 return Err(NucleusError::FilesystemError(
510 "copy-in-out workspace missing host path".to_string(),
511 ));
512 };
513
514 sync_workspace_out(staging, host_path)
515 }
516
517 fn cleanup_workspace_staging(config: &ContainerConfig) {
518 if config.workspace.mode != WorkspaceMode::CopyInOut {
519 return;
520 }
521 if let Some(staging) = config.workspace.staging_path.as_ref() {
522 if let Err(e) = std::fs::remove_dir_all(staging) {
523 warn!(
524 "Failed to remove workspace staging directory {:?}: {}",
525 staging, e
526 );
527 }
528 }
529 }
530
531 fn mapped_host_id_for_container_id(
532 mappings: &[crate::isolation::IdMapping],
533 container_id: u32,
534 label: &str,
535 ) -> Result<u32> {
536 for mapping in mappings {
537 let end = mapping
538 .container_id
539 .checked_add(mapping.count)
540 .ok_or_else(|| {
541 NucleusError::ConfigError(format!(
542 "{} overflow for container id {}",
543 label, container_id
544 ))
545 })?;
546 if container_id >= mapping.container_id && container_id < end {
547 return mapping
548 .host_id
549 .checked_add(container_id - mapping.container_id)
550 .ok_or_else(|| {
551 NucleusError::ConfigError(format!(
552 "{} host id overflow for container id {}",
553 label, container_id
554 ))
555 });
556 }
557 }
558
559 Err(NucleusError::ConfigError(format!(
560 "{} do not map container id {}",
561 label, container_id
562 )))
563 }
564
565 fn create_internal(&self, defer_exec_until_start: bool) -> Result<CreatedContainer> {
566 let lifecycle_span = info_span!(
567 "container.lifecycle",
568 container.id = %self.config.id,
569 container.name = %self.config.name,
570 runtime = if self.config.use_gvisor { "gvisor" } else { "native" }
571 );
572 let _enter = lifecycle_span.enter();
573
574 info!(
575 "Creating container: {} (ID: {})",
576 self.config.name, self.config.id
577 );
578 audit(
579 &self.config.id,
580 &self.config.name,
581 AuditEventType::ContainerStart,
582 format!(
583 "command={:?} mode={:?} runtime={}",
584 crate::audit::redact_command(&self.config.command),
585 self.config.service_mode,
586 if self.config.use_gvisor {
587 "gvisor"
588 } else {
589 "native"
590 }
591 ),
592 );
593
594 let is_root = nix::unistd::Uid::effective().is_root();
596 let mut config = self.config.clone();
597
598 if !is_root && config.user_ns_config.is_none() {
599 info!("Not running as root, automatically enabling rootless mode");
600 config.namespaces.user = true;
601 let workload_uid = if config.process_identity.uid != 0 {
605 Some(config.process_identity.uid)
606 } else {
607 None
608 };
609 config.user_ns_config =
610 Some(crate::isolation::UserNamespaceConfig::for_unprivileged_rootless(
611 workload_uid,
612 ));
613 }
614
615 if is_root && !config.namespaces.user {
619 if config.service_mode.requires_user_namespace_mapping() {
620 info!(
621 "Running as root in {}: enabling user namespace with UID remapping",
622 config.service_mode.label()
623 );
624 config.namespaces.user = true;
625 config.user_ns_config =
626 Some(crate::isolation::UserNamespaceConfig::root_remapped());
627 } else {
628 warn!(
629 "Running as root WITHOUT user namespace isolation. \
630 Container processes will run as real host UID 0. \
631 Use --user-ns, strict-agent mode, or production mode for UID remapping."
632 );
633 }
634 }
635
636 config.validate_strict_agent_mode()?;
638 config.validate_production_mode()?;
639 if config.service_mode == ServiceMode::Production {
640 let rootfs_path = config.rootfs_path.as_ref().ok_or_else(|| {
641 NucleusError::ConfigError(
642 "Production mode requires explicit --rootfs path (no host bind mounts)"
643 .to_string(),
644 )
645 })?;
646 config.rootfs_path = Some(validate_production_rootfs_path(rootfs_path)?);
647 }
648 Self::assert_kernel_lockdown(&config)?;
649
650 Self::apply_network_mode_guards(&mut config, is_root)?;
651 Self::apply_trust_level_guards(&mut config)?;
652 config.validate_runtime_support()?;
653 Self::validate_credential_broker_resolved_nat(&config, is_root)?;
654
655 if let NetworkMode::Bridge(ref bridge_config) = config.network {
656 let backend =
657 bridge_config.selected_nat_backend(is_root, config.user_ns_config.is_some());
658 if backend == NatBackend::Kernel && !is_root {
659 return Err(NucleusError::NetworkError(
660 "Kernel bridge networking requires root. Use --nat-backend userspace or leave the default auto selection for rootless/native containers."
661 .to_string(),
662 ));
663 }
664 }
665
666 let state_mgr = ContainerStateManager::new_with_root(config.state_root.clone())?;
668
669 if let Ok(all_states) = state_mgr.list_states() {
671 if all_states.iter().any(|s| s.name == config.name) {
672 return Err(NucleusError::ConfigError(format!(
673 "A container named '{}' already exists; use a different --name, \
674 or remove the stale state with 'nucleus delete'",
675 config.name
676 )));
677 }
678 }
679
680 let exec_fifo = if defer_exec_until_start {
684 let exec_fifo = state_mgr.exec_fifo_path(&config.id)?;
685 nix::unistd::mkfifo(&exec_fifo, Mode::S_IRUSR | Mode::S_IWUSR).map_err(|e| {
686 NucleusError::ExecError(format!(
687 "Failed to create exec FIFO {:?}: {}",
688 exec_fifo, e
689 ))
690 })?;
691 Some(exec_fifo)
692 } else {
693 None
694 };
695
696 let cgroup_name = format!("nucleus-{}", config.id);
698 let mut cgroup_opt = match Cgroup::create(&cgroup_name) {
699 Ok(mut cgroup) => {
700 match cgroup.set_limits(&config.limits) {
702 Ok(_) => {
703 info!("Created cgroup with resource limits");
704 Some(cgroup)
705 }
706 Err(e) => {
707 if config.service_mode.requires_cgroup_enforcement() {
708 let _ = cgroup.cleanup();
709 return Err(NucleusError::CgroupError(format!(
710 "{} requires cgroup resource enforcement, but \
711 applying limits failed: {}",
712 config.service_mode.label(),
713 e
714 )));
715 }
716 warn!("Failed to set cgroup limits: {}", e);
717 let _ = cgroup.cleanup();
718 None
719 }
720 }
721 }
722 Err(e) => {
723 if config.service_mode.requires_cgroup_enforcement() {
724 return Err(NucleusError::CgroupError(format!(
725 "{} requires cgroup resource enforcement, but \
726 cgroup creation failed: {}",
727 config.service_mode.label(),
728 e
729 )));
730 }
731
732 if config.user_ns_config.is_some() {
733 if config.limits.memory_bytes.is_some()
734 || config.limits.cpu_quota_us.is_some()
735 || config.limits.pids_max.is_some()
736 {
737 warn!(
738 "Running in rootless mode: requested resource limits cannot be \
739 enforced – cgroup creation requires root ({})",
740 e
741 );
742 } else {
743 debug!("Running in rootless mode without cgroup resource limits");
744 }
745 } else {
746 warn!(
747 "Failed to create cgroup (running without resource limits): {}",
748 e
749 );
750 }
751 None
752 }
753 };
754
755 if let Some(gpu_config) = config.gpu.as_ref() {
761 match resolve_gpu_devices(gpu_config) {
762 Ok(Some(set)) => {
763 if let Some(ref cgroup) = cgroup_opt {
764 let specs = set.device_specs();
765 if let Err(e) = cgroup.install_gpu_device_allowlist(
766 &specs,
767 config.terminal,
768 true,
769 ) {
770 warn!("Failed to install GPU cgroup device allowlist: {}", e);
771 }
772 }
773 }
774 Ok(None) => {
775 warn!(
776 "--gpu requested but no GPU devices were discovered; continuing without GPU"
777 );
778 }
779 Err(e) => return Err(e),
780 }
781 }
782
783 let runsc_path = if config.use_gvisor {
785 Some(GVisorRuntime::resolve_path().map_err(|e| {
786 NucleusError::GVisorError(format!("Failed to resolve runsc path: {}", e))
787 })?)
788 } else {
789 None
790 };
791 let gvisor_needs_precreated_userns = Self::gvisor_needs_precreated_userns(&config, is_root);
792 let needs_external_userns_mapping = config.user_ns_config.is_some()
793 && (!config.use_gvisor || gvisor_needs_precreated_userns);
794 let runtime_base_override =
795 Self::prepare_runtime_base_override(&config, is_root, needs_external_userns_mapping)?;
796 Self::prepare_rootfs_overlay(
797 &mut config,
798 &state_mgr,
799 is_root,
800 needs_external_userns_mapping,
801 )?;
802 Self::prepare_workspace_staging(
803 &mut config,
804 runtime_base_override.as_deref(),
805 is_root,
806 needs_external_userns_mapping,
807 )?;
808
809 let event_sink = self.event_sink.clone();
810 let mut native_pty = if config.terminal && !config.use_gvisor {
811 Some(NativePty::open(config.console_size)?)
812 } else {
813 None
814 };
815
816 let (ready_read, ready_write) = pipe().map_err(|e| {
818 NucleusError::ExecError(format!("Failed to create namespace sync pipe: {}", e))
819 })?;
820 let userns_sync = if needs_external_userns_mapping {
821 let (request_read, request_write) = pipe().map_err(|e| {
822 NucleusError::ExecError(format!(
823 "Failed to create user namespace request pipe: {}",
824 e
825 ))
826 })?;
827 let (ack_read, ack_write) = pipe().map_err(|e| {
828 NucleusError::ExecError(format!("Failed to create user namespace ack pipe: {}", e))
829 })?;
830 Some((request_read, request_write, ack_read, ack_write))
831 } else {
832 None
833 };
834 let (parent_setup_read, parent_setup_write) = pipe().map_err(|e| {
835 NucleusError::ExecError(format!("Failed to create parent setup sync pipe: {}", e))
836 })?;
837
838 Self::assert_single_threaded_for_fork("container create fork")?;
843 match unsafe { fork() }? {
846 ForkResult::Parent { child } => {
847 drop(ready_write);
848 drop(parent_setup_read);
849 let (userns_request_read, userns_ack_write) =
850 if let Some((request_read, request_write, ack_read, ack_write)) = userns_sync {
851 drop(request_write);
852 drop(ack_read);
853 (Some(request_read), Some(ack_write))
854 } else {
855 (None, None)
856 };
857 let mut native_console_master = native_pty.as_mut().and_then(|pty| {
858 drop(pty.slave.take());
859 pty.master.take()
860 });
861 info!("Forked child process: {}", child);
862
863 let mut target_pid_for_cleanup: Option<u32> = None;
868 let parent_setup = || -> Result<CreatedContainer> {
869 if needs_external_userns_mapping {
870 let user_config = config.user_ns_config.as_ref().ok_or_else(|| {
871 NucleusError::ExecError(
872 "Missing user namespace configuration in parent".to_string(),
873 )
874 })?;
875 let request_read = userns_request_read.as_ref().ok_or_else(|| {
876 NucleusError::ExecError(
877 "Missing user namespace request pipe in parent".to_string(),
878 )
879 })?;
880 let ack_write = userns_ack_write.as_ref().ok_or_else(|| {
881 NucleusError::ExecError(
882 "Missing user namespace ack pipe in parent".to_string(),
883 )
884 })?;
885
886 Self::wait_for_sync_byte(
887 request_read,
888 &format!(
889 "Child {} exited before requesting user namespace mappings",
890 child
891 ),
892 "Failed waiting for child user namespace request",
893 )?;
894 UserNamespaceMapper::new(user_config.clone())
895 .write_mappings_for_pid(child.as_raw() as u32)?;
896 Self::send_sync_byte(
897 ack_write,
898 "Failed to notify child that user namespace mappings are ready",
899 )?;
900 }
901
902 let target_pid = Self::wait_for_namespace_ready(&ready_read, child)?;
903 target_pid_for_cleanup = Some(target_pid);
904
905 if !config.use_gvisor {
906 if let Some(ref socket_path) = config.console_socket {
907 let master = native_console_master.take().ok_or_else(|| {
908 NucleusError::ExecError(
909 "Terminal mode requested but no native PTY master is available"
910 .to_string(),
911 )
912 })?;
913 NativePty::send_master_to_console_socket(socket_path, &master)?;
914 info!(
915 "Sent PTY master for container {} to console socket {}",
916 config.id,
917 socket_path.display()
918 );
919 }
920 }
921
922 let cgroup_path = cgroup_opt
923 .as_ref()
924 .map(|cgroup| cgroup.path().display().to_string());
925 let cpu_millicores = config
926 .limits
927 .cpu_quota_us
928 .map(|quota| quota.saturating_mul(1000) / config.limits.cpu_period_us);
929 let mut state = ContainerState::new(ContainerStateParams {
930 id: config.id.clone(),
931 name: config.name.clone(),
932 pid: target_pid,
933 command: config.command.clone(),
934 memory_limit: config.limits.memory_bytes,
935 cpu_limit: cpu_millicores,
936 using_gvisor: config.use_gvisor,
937 rootless: config.user_ns_config.is_some(),
938 cgroup_path,
939 process_uid: config.process_identity.uid,
940 process_gid: config.process_identity.gid,
941 additional_gids: config.process_identity.additional_gids.clone(),
942 });
943 state.environment = config.environment.iter().cloned().collect();
944 state.workdir = config.workdir.display().to_string();
945 state.config_hash = config.config_hash;
946 state.bundle_path =
947 config.rootfs_path.as_ref().map(|p| p.display().to_string());
948 state.rootfs_path =
949 config.rootfs_path.as_ref().map(|p| p.display().to_string());
950 state.rootfs_mode = config.rootfs_mode;
951 if let Some(overlay) = config.rootfs_overlay.as_ref() {
952 state.rootfs_upperdir = Some(overlay.upperdir.display().to_string());
953 state.rootfs_workdir = Some(overlay.workdir.display().to_string());
954 }
955
956 let mut network_driver: Option<BridgeDriver> = None;
957 let trace_reader = Self::maybe_start_seccomp_trace_reader(&config, target_pid)?;
958
959 state.status = OciStatus::Created;
961 state_mgr.save_state(&state)?;
962
963 if let Some(ref pid_path) = config.pid_file {
965 std::fs::write(pid_path, target_pid.to_string()).map_err(|e| {
966 NucleusError::ConfigError(format!(
967 "Failed to write pid-file '{}': {}",
968 pid_path.display(),
969 e
970 ))
971 })?;
972 info!("Wrote PID {} to {}", target_pid, pid_path.display());
973 }
974
975 if let Some(ref mut cgroup) = cgroup_opt {
976 cgroup.attach_process(target_pid)?;
977 }
978
979 let deny_logger = Self::maybe_start_seccomp_deny_logger(
980 &config,
981 target_pid,
982 cgroup_opt.as_ref().map(|cgroup| cgroup.path()),
983 )?;
984
985 if let NetworkMode::Bridge(ref bridge_config) = config.network {
986 match BridgeDriver::setup_with_id(
987 target_pid,
988 bridge_config,
989 &config.id,
990 is_root,
991 config.user_ns_config.is_some(),
992 ) {
993 Ok(net) => {
994 if let Some(ref egress) = config.egress_policy {
995 if let Err(e) = net.apply_egress_policy(
996 target_pid,
997 egress,
998 config.user_ns_config.is_some(),
999 ) {
1000 if config.service_mode.enforces_strict_isolation() {
1001 return Err(NucleusError::NetworkError(format!(
1002 "Failed to apply egress policy: {}",
1003 e
1004 )));
1005 }
1006 warn!("Failed to apply egress policy: {}", e);
1007 }
1008 }
1009 if let Some(ref broker) = config.credential_broker {
1010 Self::probe_credential_broker(broker)?;
1011 }
1012 network_driver = Some(net);
1013 }
1014 Err(e) => {
1015 if config.service_mode.enforces_strict_isolation() {
1016 return Err(e);
1017 }
1018 warn!("Failed to set up bridge networking: {}", e);
1019 }
1020 }
1021 }
1022
1023 Self::send_sync_byte(
1024 &parent_setup_write,
1025 "Failed to notify child that parent setup is complete",
1026 )?;
1027
1028 info!(
1029 "Container {} created (child pid {}), waiting for start",
1030 config.id, target_pid
1031 );
1032
1033 Ok(CreatedContainer {
1034 config,
1035 state_mgr,
1036 state,
1037 child,
1038 cgroup_opt,
1039 network_driver,
1040 trace_reader,
1041 deny_logger,
1042 exec_fifo_path: exec_fifo,
1043 native_console_master,
1044 event_sink,
1045 _lifecycle_span: lifecycle_span.clone(),
1046 })
1047 };
1048
1049 parent_setup().inspect_err(|_| {
1050 if let Some(target_pid) = target_pid_for_cleanup {
1051 let _ = kill(Pid::from_raw(target_pid as i32), Signal::SIGKILL);
1052 }
1053 let _ = kill(child, Signal::SIGKILL);
1054 let _ = waitpid(child, None);
1055 })
1056 }
1057 ForkResult::Child => {
1058 drop(ready_read);
1059 drop(parent_setup_write);
1060 let (userns_request_write, userns_ack_read) =
1061 if let Some((request_read, request_write, ack_read, ack_write)) = userns_sync {
1062 drop(request_read);
1063 drop(ack_write);
1064 (Some(request_write), Some(ack_read))
1065 } else {
1066 (None, None)
1067 };
1068 let native_console_slave = native_pty.as_mut().and_then(|pty| {
1069 drop(pty.master.take());
1070 pty.slave.take()
1071 });
1072 Self::sanitize_fds();
1074 let temp_container = Container {
1075 config,
1076 runsc_path,
1077 event_sink: None,
1078 };
1079 match temp_container.setup_and_exec(
1080 Some(ready_write),
1081 userns_request_write,
1082 userns_ack_read,
1083 Some(parent_setup_read),
1084 exec_fifo,
1085 runtime_base_override,
1086 native_console_slave,
1087 ) {
1088 Ok(_) => unreachable!(),
1089 Err(e) => {
1090 error!("Container setup failed: {}", e);
1091 std::process::exit(1);
1092 }
1093 }
1094 }
1095 }
1096 }
1097
1098 pub fn trigger_start(container_id: &str, state_root: Option<PathBuf>) -> Result<()> {
1101 let state_mgr = ContainerStateManager::new_with_root(state_root)?;
1102 let fifo_path = state_mgr.exec_fifo_path(container_id)?;
1103 if !fifo_path.exists() {
1104 return Err(NucleusError::ConfigError(format!(
1105 "No exec FIFO found for container {}; is it in 'created' state?",
1106 container_id
1107 )));
1108 }
1109
1110 let file = std::fs::File::open(&fifo_path)
1112 .map_err(|e| NucleusError::ExecError(format!("Failed to open exec FIFO: {}", e)))?;
1113 let mut buf = [0u8; 1];
1114 std::io::Read::read(&mut &file, &mut buf)
1115 .map_err(|e| NucleusError::ExecError(format!("Failed to read exec FIFO: {}", e)))?;
1116 drop(file);
1117
1118 let _ = std::fs::remove_file(&fifo_path);
1119
1120 let mut state = state_mgr.resolve_container(container_id)?;
1122 state.status = OciStatus::Running;
1123 state_mgr.save_state(&state)?;
1124
1125 Ok(())
1126 }
1127
1128 #[allow(clippy::too_many_arguments)]
1133 fn setup_and_exec(
1134 &self,
1135 ready_pipe: Option<OwnedFd>,
1136 userns_request_pipe: Option<OwnedFd>,
1137 userns_ack_pipe: Option<OwnedFd>,
1138 parent_setup_pipe: Option<OwnedFd>,
1139 exec_fifo: Option<PathBuf>,
1140 runtime_base_override: Option<PathBuf>,
1141 native_console_slave: Option<OwnedFd>,
1142 ) -> Result<()> {
1143 let is_rootless = self.config.user_ns_config.is_some();
1144 let allow_degraded_security = Self::allow_degraded_security(&self.config);
1145 let context_manifest = if self.config.verify_context_integrity {
1146 self.config
1147 .context_dir
1148 .as_ref()
1149 .map(|dir| snapshot_context_dir(dir))
1150 .transpose()?
1151 } else {
1152 None
1153 };
1154
1155 let mut fs_state = FilesystemState::Unmounted;
1157 let mut sec_state = SecurityState::Privileged;
1158
1159 if self.config.use_gvisor {
1164 let gvisor_precreated_userns =
1165 if Self::gvisor_needs_precreated_userns(&self.config, Uid::effective().is_root()) {
1166 self.prepare_gvisor_handoff_namespace(
1167 userns_request_pipe.as_ref(),
1168 userns_ack_pipe.as_ref(),
1169 )?
1170 } else {
1171 false
1172 };
1173
1174 if let Some(fd) = ready_pipe {
1175 Self::notify_namespace_ready(&fd, std::process::id())?;
1176 }
1177 if let Some(fd) = parent_setup_pipe.as_ref() {
1178 Self::wait_for_sync_byte(
1179 fd,
1180 "Parent closed setup pipe before signalling gVisor child",
1181 "Failed waiting for parent setup acknowledgement",
1182 )?;
1183 }
1184 return self.setup_and_exec_gvisor(gvisor_precreated_userns);
1185 }
1186
1187 let mut namespace_mgr = NamespaceManager::new(self.config.namespaces.clone());
1189 namespace_mgr.unshare_namespaces()?;
1190 if self.config.user_ns_config.is_some() {
1191 let request_fd = userns_request_pipe.as_ref().ok_or_else(|| {
1192 NucleusError::ExecError(
1193 "Missing user namespace request pipe in container child".to_string(),
1194 )
1195 })?;
1196 let ack_fd = userns_ack_pipe.as_ref().ok_or_else(|| {
1197 NucleusError::ExecError(
1198 "Missing user namespace acknowledgement pipe in container child".to_string(),
1199 )
1200 })?;
1201
1202 Self::send_sync_byte(
1203 request_fd,
1204 "Failed to request user namespace mappings from parent",
1205 )?;
1206 Self::wait_for_sync_byte(
1207 ack_fd,
1208 "Parent closed user namespace ack pipe before mappings were written",
1209 "Failed waiting for parent to finish user namespace mappings",
1210 )?;
1211 Self::become_userns_root_for_setup()?;
1212 }
1213
1214 if self.config.namespaces.pid {
1217 Self::assert_single_threaded_for_fork("PID namespace init fork")?;
1218 match unsafe { fork() }? {
1219 ForkResult::Parent { child } => {
1220 if let Some(fd) = ready_pipe {
1221 Self::notify_namespace_ready(&fd, child.as_raw() as u32)?;
1222 }
1223 std::process::exit(Self::wait_for_pid_namespace_child(child));
1224 }
1225 ForkResult::Child => {
1226 if let Some(fd) = parent_setup_pipe.as_ref() {
1227 Self::wait_for_sync_byte(
1228 fd,
1229 "Parent closed setup pipe before signalling PID 1 child",
1230 "Failed waiting for parent setup acknowledgement",
1231 )?;
1232 }
1233 }
1235 }
1236 } else {
1237 if let Some(fd) = ready_pipe {
1238 Self::notify_namespace_ready(&fd, std::process::id())?;
1239 }
1240 if let Some(fd) = parent_setup_pipe.as_ref() {
1241 Self::wait_for_sync_byte(
1242 fd,
1243 "Parent closed setup pipe before signalling container child",
1244 "Failed waiting for parent setup acknowledgement",
1245 )?;
1246 }
1247 }
1248
1249 namespace_mgr.enter()?;
1251
1252 self.enforce_no_new_privs()?;
1256 audit(
1257 &self.config.id,
1258 &self.config.name,
1259 AuditEventType::NoNewPrivsSet,
1260 "prctl(PR_SET_NO_NEW_PRIVS, 1) applied (early, before mounts)",
1261 );
1262
1263 if let Some(hostname) = &self.config.hostname {
1265 namespace_mgr.set_hostname(hostname)?;
1266 }
1267
1268 let runtime_base = if let Some(path) = runtime_base_override {
1273 path
1274 } else if nix::unistd::Uid::effective().is_root() {
1275 PathBuf::from("/run/nucleus")
1276 } else {
1277 dirs::runtime_dir()
1278 .map(|d| d.join("nucleus"))
1279 .unwrap_or_else(std::env::temp_dir)
1280 };
1281 let _ = std::fs::create_dir_all(&runtime_base);
1282 let runtime_dir = Builder::new()
1283 .prefix("nucleus-runtime-")
1284 .tempdir_in(&runtime_base)
1285 .map_err(|e| {
1286 NucleusError::FilesystemError(format!("Failed to create runtime dir: {}", e))
1287 })?;
1288 let container_root = runtime_dir.path().to_path_buf();
1289 let mut tmpfs = TmpfsMount::new(&container_root, Some(1024 * 1024 * 1024)); tmpfs.mount()?;
1291 fs_state = fs_state.transition(FilesystemState::Mounted)?;
1292
1293 let rootfs_overlay_mounted = if self.config.rootfs_mode == RootfsMode::Overlay {
1294 let rootfs_path = self.config.rootfs_path.as_ref().ok_or_else(|| {
1295 NucleusError::ConfigError(
1296 "--rootfs-mode overlay requires a rootfs path".to_string(),
1297 )
1298 })?;
1299 if self.config.verify_rootfs_attestation {
1300 verify_rootfs_attestation(rootfs_path)?;
1301 }
1302 let overlay = self.config.rootfs_overlay.as_ref().ok_or_else(|| {
1303 NucleusError::FilesystemError(
1304 "rootfs overlay mode missing prepared upper/work dirs".to_string(),
1305 )
1306 })?;
1307 overlay_mount_rootfs(
1308 &container_root,
1309 rootfs_path,
1310 &overlay.upperdir,
1311 &overlay.workdir,
1312 )?;
1313 true
1314 } else {
1315 false
1316 };
1317
1318 create_minimal_fs(&container_root)?;
1320
1321 let dev_path = container_root.join("dev");
1323 create_dev_nodes(&dev_path, self.config.terminal)?;
1324
1325 let gpu_device_set: Option<GpuDeviceSet> = match self.config.gpu.as_ref() {
1329 Some(gpu_config) => match resolve_gpu_devices(gpu_config) {
1330 Ok(Some(set)) => Some(set),
1331 Ok(None) => {
1332 warn!(
1333 "--gpu requested but no GPU devices discovered in child; skipping mounts"
1334 );
1335 None
1336 }
1337 Err(e) => {
1338 return Err(NucleusError::FilesystemError(format!(
1339 "GPU device resolution failed: {}",
1340 e
1341 )))
1342 }
1343 },
1344 None => None,
1345 };
1346 if let Some(ref set) = gpu_device_set {
1347 let gpu_config = self.config.gpu.as_ref().expect("gpu config present");
1348 mount_gpu_passthrough(
1349 &container_root,
1350 set,
1351 gpu_config,
1352 &self.config.process_identity,
1353 )?;
1354 }
1355
1356 let shm_path = dev_path.join("shm");
1359 std::fs::create_dir_all(&shm_path).map_err(|e| {
1360 NucleusError::FilesystemError(format!("Failed to create /dev/shm: {}", e))
1361 })?;
1362 nix::mount::mount(
1363 Some("shm"),
1364 &shm_path,
1365 Some("tmpfs"),
1366 nix::mount::MsFlags::MS_NOSUID
1367 | nix::mount::MsFlags::MS_NODEV
1368 | nix::mount::MsFlags::MS_NOEXEC,
1369 Some("mode=1777,size=64m"),
1370 )
1371 .map_err(|e| {
1372 NucleusError::FilesystemError(format!("Failed to mount tmpfs on /dev/shm: {}", e))
1373 })?;
1374 debug!("Mounted tmpfs on /dev/shm");
1375
1376 if let Some(context_dir) = &self.config.context_dir {
1379 let context_dest = container_root.join("context");
1380 LazyContextPopulator::populate(&self.config.context_mode, context_dir, &context_dest)?;
1381 if let Some(expected) = &context_manifest {
1382 verify_context_manifest(expected, &context_dest)?;
1383 }
1384 }
1385 fs_state = fs_state.transition(FilesystemState::Populated)?;
1386
1387 if rootfs_overlay_mounted {
1389 debug!("Rootfs already mounted through overlayfs");
1390 } else if let Some(ref rootfs_path) = self.config.rootfs_path {
1391 let rootfs_path = if self.config.service_mode == ServiceMode::Production {
1392 validate_production_rootfs_path(rootfs_path)?
1393 } else {
1394 rootfs_path.clone()
1395 };
1396 if self.config.verify_rootfs_attestation {
1397 verify_rootfs_attestation(&rootfs_path)?;
1398 }
1399 bind_mount_rootfs(&container_root, &rootfs_path)?;
1400 } else {
1401 bind_mount_host_paths(&container_root, is_rootless)?;
1402 }
1403
1404 mount_home_tmpfs(
1406 &container_root,
1407 &self.config.home,
1408 &self.config.process_identity,
1409 )?;
1410
1411 mount_provider_configs(
1413 &container_root,
1414 &self.config.home,
1415 &self.config.provider_configs,
1416 )?;
1417
1418 mount_workspace(&container_root, &self.config.workspace)?;
1420
1421 mount_volumes(&container_root, &self.config.volumes)?;
1423
1424 if let NetworkMode::Bridge(ref bridge_config) = self.config.network {
1428 let bridge_dns = if bridge_config.selected_nat_backend(!is_rootless, is_rootless)
1429 == NatBackend::Userspace
1430 && bridge_config.dns.is_empty()
1431 {
1432 vec![UserspaceNetwork::default_dns_server(&bridge_config.subnet)?]
1433 } else {
1434 bridge_config.dns.clone()
1435 };
1436 if self.config.rootfs_path.is_some() {
1437 BridgeNetwork::bind_mount_resolv_conf(&container_root, &bridge_dns)?;
1438 } else {
1439 BridgeNetwork::write_resolv_conf(&container_root, &bridge_dns)?;
1440 }
1441 }
1442
1443 mount_secrets_inmemory(
1445 &container_root,
1446 &self.config.secrets,
1447 &self.config.process_identity,
1448 )?;
1449
1450 let proc_path = container_root.join("proc");
1452 let production_mode = self.config.service_mode == ServiceMode::Production;
1453 let hide_pids = production_mode;
1454 let procfs_best_effort = is_rootless && !production_mode;
1455 mount_procfs(
1456 &proc_path,
1457 procfs_best_effort,
1458 self.config.proc_readonly,
1459 hide_pids,
1460 )?;
1461
1462 mask_proc_paths(
1465 &proc_path,
1466 self.config.service_mode == ServiceMode::Production,
1467 )?;
1468
1469 if let Some(ref hooks) = self.config.hooks {
1471 if !hooks.create_runtime.is_empty() {
1472 let hook_state = OciContainerState {
1473 oci_version: "1.0.2".to_string(),
1474 id: self.config.id.clone(),
1475 status: OciStatus::Creating,
1476 pid: std::process::id(),
1477 bundle: String::new(),
1478 };
1479 OciHooks::run_hooks(&hooks.create_runtime, &hook_state, "createRuntime")?;
1480 }
1481 }
1482
1483 switch_root(&container_root, self.config.allow_chroot_fallback)?;
1486 fs_state = fs_state.transition(FilesystemState::Pivoted)?;
1487 debug!("Filesystem state: {:?}", fs_state);
1488
1489 audit_mounts(self.config.service_mode == ServiceMode::Production)?;
1491 audit(
1492 &self.config.id,
1493 &self.config.name,
1494 AuditEventType::MountAuditPassed,
1495 "all mount flags verified",
1496 );
1497
1498 if let Some(ref hooks) = self.config.hooks {
1500 if !hooks.create_container.is_empty() {
1501 let hook_state = OciContainerState {
1502 oci_version: "1.0.2".to_string(),
1503 id: self.config.id.clone(),
1504 status: OciStatus::Created,
1505 pid: std::process::id(),
1506 bundle: String::new(),
1507 };
1508 OciHooks::run_hooks(&hooks.create_container, &hook_state, "createContainer")?;
1509 }
1510 }
1511
1512 let mut cap_mgr = CapabilityManager::new();
1522 if let Some(ref policy_path) = self.config.caps_policy {
1523 let policy: crate::security::CapsPolicy = crate::security::load_toml_policy(
1524 policy_path,
1525 self.config.caps_policy_sha256.as_deref(),
1526 )?;
1527 if self.config.service_mode == ServiceMode::Production {
1529 policy.validate_production()?;
1530 }
1531 policy.apply(&mut cap_mgr)?;
1532 Self::apply_process_identity_to_current_process(
1534 &self.config.process_identity,
1535 self.config.user_ns_config.is_some(),
1536 )?;
1537 audit(
1538 &self.config.id,
1539 &self.config.name,
1540 AuditEventType::CapabilitiesDropped,
1541 format!("capability policy applied from {:?}", policy_path),
1542 );
1543 } else {
1544 cap_mgr.drop_bounding_set()?;
1546
1547 Self::apply_process_identity_to_current_process(
1551 &self.config.process_identity,
1552 self.config.user_ns_config.is_some(),
1553 )?;
1554
1555 cap_mgr.finalize_drop()?;
1558
1559 audit(
1560 &self.config.id,
1561 &self.config.name,
1562 AuditEventType::CapabilitiesDropped,
1563 "all capabilities dropped including bounding set",
1564 );
1565 }
1566 sec_state = sec_state.transition(SecurityState::CapabilitiesDropped)?;
1567
1568 {
1573 let is_production = self.config.service_mode == ServiceMode::Production;
1574
1575 if let Some(nproc_limit) = self.config.limits.pids_max {
1576 let rlim_nproc = libc::rlimit {
1577 rlim_cur: nproc_limit,
1578 rlim_max: nproc_limit,
1579 };
1580 if unsafe { libc::setrlimit(libc::RLIMIT_NPROC, &rlim_nproc) } != 0 {
1582 let err = std::io::Error::last_os_error();
1583 if is_production {
1584 return Err(NucleusError::SeccompError(format!(
1585 "Failed to set RLIMIT_NPROC to {} in production mode: {}",
1586 nproc_limit, err
1587 )));
1588 }
1589 warn!("Failed to set RLIMIT_NPROC to {}: {}", nproc_limit, err);
1590 }
1591 }
1592
1593 let rlim_nofile = libc::rlimit {
1594 rlim_cur: 1024,
1595 rlim_max: 1024,
1596 };
1597 if unsafe { libc::setrlimit(libc::RLIMIT_NOFILE, &rlim_nofile) } != 0 {
1599 let err = std::io::Error::last_os_error();
1600 if is_production {
1601 return Err(NucleusError::SeccompError(format!(
1602 "Failed to set RLIMIT_NOFILE to 1024 in production mode: {}",
1603 err
1604 )));
1605 }
1606 warn!("Failed to set RLIMIT_NOFILE to 1024: {}", err);
1607 }
1608
1609 let memlock_limit: u64 = self.config.limits.memlock_bytes.unwrap_or(64 * 1024);
1614 let rlim_memlock = libc::rlimit {
1615 rlim_cur: memlock_limit,
1616 rlim_max: memlock_limit,
1617 };
1618 if unsafe { libc::setrlimit(libc::RLIMIT_MEMLOCK, &rlim_memlock) } != 0 {
1620 let err = std::io::Error::last_os_error();
1621 if is_production {
1622 return Err(NucleusError::SeccompError(format!(
1623 "Failed to set RLIMIT_MEMLOCK to {} in production mode: {}",
1624 memlock_limit, err
1625 )));
1626 }
1627 warn!("Failed to set RLIMIT_MEMLOCK to {}: {}", memlock_limit, err);
1628 }
1629 }
1630
1631 CapabilityManager::verify_no_namespace_caps(
1635 self.config.service_mode == ServiceMode::Production,
1636 )?;
1637
1638 use crate::container::config::SeccompMode;
1641 let mut seccomp_mgr = SeccompManager::new();
1642 let allow_network = !matches!(self.config.network, NetworkMode::None);
1643 let seccomp_applied = match self.config.seccomp_mode {
1644 SeccompMode::Trace => {
1645 audit(
1646 &self.config.id,
1647 &self.config.name,
1648 AuditEventType::SeccompApplied,
1649 "seccomp trace mode: allow-all + LOG",
1650 );
1651 seccomp_mgr.apply_trace_filter()?
1652 }
1653 SeccompMode::Enforce => {
1654 if let Some(ref profile_path) = self.config.seccomp_profile {
1655 audit(
1656 &self.config.id,
1657 &self.config.name,
1658 AuditEventType::SeccompProfileLoaded,
1659 format!("path={:?}", profile_path),
1660 );
1661 seccomp_mgr.apply_profile_from_file(
1662 profile_path,
1663 self.config.seccomp_profile_sha256.as_deref(),
1664 self.config.seccomp_log_denied,
1665 )?
1666 } else {
1667 let gpu_mode = self.config.gpu.is_some();
1668 seccomp_mgr.apply_filter_with_options(
1669 allow_network,
1670 allow_degraded_security,
1671 self.config.seccomp_log_denied,
1672 &self.config.seccomp_allow_syscalls,
1673 gpu_mode,
1674 )?
1675 }
1676 }
1677 };
1678 if seccomp_applied {
1679 sec_state = sec_state.transition(SecurityState::SeccompApplied)?;
1680 audit(
1681 &self.config.id,
1682 &self.config.name,
1683 AuditEventType::SeccompApplied,
1684 format!("network={}", allow_network),
1685 );
1686 } else if !allow_degraded_security {
1687 return Err(NucleusError::SeccompError(
1688 "Seccomp filter is required but was not enforced".to_string(),
1689 ));
1690 } else {
1691 warn!("Seccomp not enforced; container is running with degraded hardening");
1692 }
1693
1694 let landlock_applied = if let Some(ref policy_path) = self.config.landlock_policy {
1696 let policy: crate::security::LandlockPolicy = crate::security::load_toml_policy(
1697 policy_path,
1698 self.config.landlock_policy_sha256.as_deref(),
1699 )?;
1700 if self.config.service_mode == ServiceMode::Production {
1702 policy.validate_production()?;
1703 }
1704 policy.apply(allow_degraded_security)?
1705 } else {
1706 let mut landlock_mgr = LandlockManager::new();
1707 landlock_mgr.assert_minimum_abi(self.config.service_mode == ServiceMode::Production)?;
1708 landlock_mgr.set_workspace_access(
1709 self.config.workspace.is_read_only(),
1710 self.config.workspace.allow_execute,
1711 );
1712 landlock_mgr.add_rw_path(&self.config.home.to_string_lossy());
1713 for provider_config in &self.config.provider_configs {
1715 let provider_dest = crate::filesystem::normalize_provider_config_destination(
1716 &self.config.home,
1717 &provider_config.dest,
1718 )?;
1719 if provider_config.read_only {
1720 landlock_mgr.add_ro_path(&provider_dest.to_string_lossy());
1721 } else {
1722 landlock_mgr.add_rw_path(&provider_dest.to_string_lossy());
1723 }
1724 }
1725 for vol in &self.config.volumes {
1726 if vol.read_only {
1727 landlock_mgr.add_ro_path(&vol.dest.to_string_lossy());
1728 } else {
1729 landlock_mgr.add_rw_path(&vol.dest.to_string_lossy());
1730 }
1731 }
1732 if let Some(ref set) = gpu_device_set {
1735 for node in &set.nodes {
1736 landlock_mgr.add_rw_path(&node.to_string_lossy());
1737 }
1738 }
1739 landlock_mgr.apply_container_policy_with_mode(allow_degraded_security)?
1740 };
1741 if seccomp_applied && landlock_applied {
1742 sec_state = sec_state.transition(SecurityState::LandlockApplied)?;
1743 if self.config.seccomp_mode == SeccompMode::Trace {
1744 warn!("Security state NOT locked: seccomp in trace mode (allow-all)");
1745 } else {
1746 sec_state = sec_state.transition(SecurityState::Locked)?;
1747 }
1748 audit(
1749 &self.config.id,
1750 &self.config.name,
1751 AuditEventType::LandlockApplied,
1752 if self.config.seccomp_mode == SeccompMode::Trace {
1753 "landlock applied, but seccomp in trace mode - not locked".to_string()
1754 } else {
1755 "security state locked: all hardening layers active".to_string()
1756 },
1757 );
1758 } else if !allow_degraded_security {
1759 return Err(NucleusError::LandlockError(
1760 "Landlock policy is required but was not enforced".to_string(),
1761 ));
1762 } else {
1763 warn!("Security state not locked; one or more hardening controls are inactive");
1764 }
1765 debug!("Security state: {:?}", sec_state);
1766
1767 if let Some(ref fifo_path) = exec_fifo {
1771 debug!("Waiting on exec FIFO {:?} for start signal", fifo_path);
1772 let file = std::fs::OpenOptions::new()
1773 .write(true)
1774 .open(fifo_path)
1775 .map_err(|e| {
1776 NucleusError::ExecError(format!("Failed to open exec FIFO for writing: {}", e))
1777 })?;
1778 std::io::Write::write_all(&mut &file, &[0u8]).map_err(|e| {
1779 NucleusError::ExecError(format!("Failed to write exec FIFO sync byte: {}", e))
1780 })?;
1781 drop(file);
1782 debug!("Exec FIFO released, proceeding to exec");
1783 }
1784
1785 if let Some(ref hooks) = self.config.hooks {
1787 if !hooks.start_container.is_empty() {
1788 let hook_state = OciContainerState {
1789 oci_version: "1.0.2".to_string(),
1790 id: self.config.id.clone(),
1791 status: OciStatus::Running,
1792 pid: std::process::id(),
1793 bundle: String::new(),
1794 };
1795 OciHooks::run_hooks(&hooks.start_container, &hook_state, "startContainer")?;
1796 }
1797 }
1798
1799 if let Some(slave) = native_console_slave {
1800 NativePty::configure_child_terminal(slave)?;
1801 }
1802
1803 if self.config.service_mode == ServiceMode::Production && self.config.namespaces.pid {
1806 return self.run_as_init();
1807 }
1808
1809 self.exec_command()?;
1811
1812 Ok(())
1814 }
1815
1816 pub(super) fn setup_signal_forwarding_static(
1821 child: Pid,
1822 ) -> Result<(Arc<AtomicBool>, JoinHandle<()>)> {
1823 let mut set = SigSet::empty();
1824 for signal in [
1825 Signal::SIGTERM,
1826 Signal::SIGINT,
1827 Signal::SIGHUP,
1828 Signal::SIGQUIT,
1829 Signal::SIGUSR1,
1830 Signal::SIGUSR2,
1831 Signal::SIGWINCH,
1832 ] {
1833 set.add(signal);
1834 }
1835
1836 let unblock_set = set;
1837 pthread_sigmask(SigmaskHow::SIG_BLOCK, Some(&unblock_set), None).map_err(|e| {
1838 NucleusError::ExecError(format!("Failed to block forwarded signals: {}", e))
1839 })?;
1840
1841 let stop = Arc::new(AtomicBool::new(false));
1842 let stop_clone = stop.clone();
1843 let handle = std::thread::Builder::new()
1844 .name("sig-forward".to_string())
1845 .spawn(move || {
1846 loop {
1848 if let Ok(signal) = unblock_set.wait() {
1849 if stop_clone.load(Ordering::Acquire) {
1853 break;
1854 }
1855 let _ = kill(child, signal);
1856 }
1857 }
1858 })
1859 .map_err(|e| {
1860 let mut restore = SigSet::empty();
1863 for signal in [
1864 Signal::SIGTERM,
1865 Signal::SIGINT,
1866 Signal::SIGHUP,
1867 Signal::SIGQUIT,
1868 Signal::SIGUSR1,
1869 Signal::SIGUSR2,
1870 Signal::SIGWINCH,
1871 ] {
1872 restore.add(signal);
1873 }
1874 let _ = pthread_sigmask(SigmaskHow::SIG_UNBLOCK, Some(&restore), None);
1875 NucleusError::ExecError(format!("Failed to spawn signal thread: {}", e))
1876 })?;
1877
1878 info!("Signal forwarding configured");
1879 Ok((stop, handle))
1880 }
1881
1882 pub(super) fn wait_for_child_static(child: Pid) -> Result<i32> {
1884 loop {
1885 match waitpid(child, None) {
1886 Ok(WaitStatus::Exited(_, code)) => {
1887 return Ok(code);
1888 }
1889 Ok(WaitStatus::Signaled(_, signal, _)) => {
1890 info!("Child killed by signal: {:?}", signal);
1891 return Ok(128 + signal as i32);
1892 }
1893 Err(nix::errno::Errno::EINTR) => {
1894 continue;
1895 }
1896 Err(e) => {
1897 return Err(NucleusError::ExecError(format!(
1898 "Failed to wait for child: {}",
1899 e
1900 )));
1901 }
1902 _ => {
1903 continue;
1904 }
1905 }
1906 }
1907 }
1908
1909 fn wait_for_namespace_ready(ready_read: &OwnedFd, child: Pid) -> Result<u32> {
1910 let mut pid_buf = [0u8; 4];
1911 loop {
1912 match read(ready_read, &mut pid_buf) {
1913 Err(nix::errno::Errno::EINTR) => continue,
1914 Ok(4) => return Ok(u32::from_ne_bytes(pid_buf)),
1915 Ok(0) => {
1916 return Err(NucleusError::ExecError(format!(
1917 "Child {} exited before namespace initialization",
1918 child
1919 )))
1920 }
1921 Ok(_) => {
1922 return Err(NucleusError::ExecError(
1923 "Invalid namespace sync payload from child".to_string(),
1924 ))
1925 }
1926 Err(e) => {
1927 return Err(NucleusError::ExecError(format!(
1928 "Failed waiting for child namespace setup: {}",
1929 e
1930 )))
1931 }
1932 }
1933 }
1934 }
1935
1936 fn notify_namespace_ready(fd: &OwnedFd, pid: u32) -> Result<()> {
1937 let payload = pid.to_ne_bytes();
1938 let mut written = 0;
1939 while written < payload.len() {
1940 let n = write(fd, &payload[written..]).map_err(|e| {
1941 NucleusError::ExecError(format!("Failed to notify namespace readiness: {}", e))
1942 })?;
1943 if n == 0 {
1944 return Err(NucleusError::ExecError(
1945 "Failed to notify namespace readiness: short write".to_string(),
1946 ));
1947 }
1948 written += n;
1949 }
1950 Ok(())
1951 }
1952
1953 fn send_sync_byte(fd: &OwnedFd, error_context: &str) -> Result<()> {
1954 let mut written = 0;
1955 let payload = [1u8];
1956 while written < payload.len() {
1957 let n = write(fd, &payload[written..])
1958 .map_err(|e| NucleusError::ExecError(format!("{}: {}", error_context, e)))?;
1959 if n == 0 {
1960 return Err(NucleusError::ExecError(format!(
1961 "{}: short write",
1962 error_context
1963 )));
1964 }
1965 written += n;
1966 }
1967 Ok(())
1968 }
1969
1970 fn wait_for_sync_byte(fd: &OwnedFd, eof_context: &str, error_context: &str) -> Result<()> {
1971 let mut payload = [0u8; 1];
1972 loop {
1973 match read(fd, &mut payload) {
1974 Err(nix::errno::Errno::EINTR) => continue,
1975 Ok(1) => return Ok(()),
1976 Ok(0) => return Err(NucleusError::ExecError(eof_context.to_string())),
1977 Ok(_) => {
1978 return Err(NucleusError::ExecError(format!(
1979 "{}: invalid sync payload",
1980 error_context
1981 )))
1982 }
1983 Err(e) => return Err(NucleusError::ExecError(format!("{}: {}", error_context, e))),
1984 }
1985 }
1986 }
1987
1988 fn become_userns_root_for_setup() -> Result<()> {
1989 setresgid(Gid::from_raw(0), Gid::from_raw(0), Gid::from_raw(0)).map_err(|e| {
1990 NucleusError::NamespaceError(format!(
1991 "Failed to become gid 0 inside mapped user namespace: {}",
1992 e
1993 ))
1994 })?;
1995 setresuid(Uid::from_raw(0), Uid::from_raw(0), Uid::from_raw(0)).map_err(|e| {
1996 NucleusError::NamespaceError(format!(
1997 "Failed to become uid 0 inside mapped user namespace: {}",
1998 e
1999 ))
2000 })?;
2001 Self::restore_gvisor_handoff_capabilities()?;
2002 debug!("Switched setup process to uid/gid 0 inside mapped user namespace");
2003 Ok(())
2004 }
2005
2006 fn gvisor_handoff_capabilities() -> CapsHashSet {
2007 [
2008 Capability::CAP_CHOWN,
2009 Capability::CAP_DAC_OVERRIDE,
2010 Capability::CAP_DAC_READ_SEARCH,
2011 Capability::CAP_FOWNER,
2012 Capability::CAP_FSETID,
2013 Capability::CAP_SYS_CHROOT,
2014 Capability::CAP_SYS_PTRACE,
2015 Capability::CAP_SETUID,
2016 Capability::CAP_SETGID,
2017 Capability::CAP_SYS_ADMIN,
2018 Capability::CAP_SETPCAP,
2019 ]
2020 .into_iter()
2021 .collect()
2022 }
2023
2024 fn restore_gvisor_handoff_capabilities() -> Result<()> {
2025 let caps = Self::gvisor_handoff_capabilities();
2026 let permitted = caps::read(None, CapSet::Permitted).map_err(|e| {
2027 NucleusError::CapabilityError(format!(
2028 "Failed to read gVisor handoff permitted capabilities: {}",
2029 e
2030 ))
2031 })?;
2032 let missing: Vec<_> = caps.difference(&permitted).copied().collect();
2033 if !missing.is_empty() {
2034 return Err(NucleusError::CapabilityError(format!(
2035 "Missing gVisor handoff permitted capabilities after entering mapped user namespace: {:?}",
2036 missing
2037 )));
2038 }
2039 let extra_permitted: Vec<_> = permitted.difference(&caps).copied().collect();
2040 let bounding = caps::read(None, CapSet::Bounding).map_err(|e| {
2041 NucleusError::CapabilityError(format!(
2042 "Failed to read gVisor handoff bounding capabilities: {}",
2043 e
2044 ))
2045 })?;
2046 let extra_bounding: Vec<_> = bounding.difference(&caps).copied().collect();
2047 caps::clear(None, CapSet::Ambient).map_err(|e| {
2048 NucleusError::CapabilityError(format!(
2049 "Failed to clear gVisor handoff ambient capabilities: {}",
2050 e
2051 ))
2052 })?;
2053 caps::set(None, CapSet::Effective, &caps).map_err(|e| {
2057 NucleusError::CapabilityError(format!(
2058 "Failed to set gVisor handoff effective capabilities: {}",
2059 e
2060 ))
2061 })?;
2062 caps::set(None, CapSet::Inheritable, &caps).map_err(|e| {
2063 NucleusError::CapabilityError(format!(
2064 "Failed to set gVisor handoff inheritable capabilities: {}",
2065 e
2066 ))
2067 })?;
2068 for cap in &extra_bounding {
2069 caps::drop(None, CapSet::Bounding, *cap).map_err(|e| {
2070 NucleusError::CapabilityError(format!(
2071 "Failed to drop extra gVisor handoff bounding capability {cap:?}: {e}"
2072 ))
2073 })?;
2074 }
2075 for cap in &extra_permitted {
2076 caps::drop(None, CapSet::Permitted, *cap).map_err(|e| {
2077 NucleusError::CapabilityError(format!(
2078 "Failed to drop extra gVisor handoff permitted capability {cap:?}: {e}"
2079 ))
2080 })?;
2081 }
2082 caps::set(None, CapSet::Ambient, &caps).map_err(|e| {
2083 NucleusError::CapabilityError(format!(
2084 "Failed to set gVisor handoff ambient capabilities: {}",
2085 e
2086 ))
2087 })?;
2088 debug!(
2089 ?caps,
2090 ?extra_permitted,
2091 ?extra_bounding,
2092 "Restored gVisor handoff capabilities inside mapped user namespace"
2093 );
2094 Ok(())
2095 }
2096
2097 fn prepare_gvisor_handoff_namespace(
2098 &self,
2099 userns_request_pipe: Option<&OwnedFd>,
2100 userns_ack_pipe: Option<&OwnedFd>,
2101 ) -> Result<bool> {
2102 let mut precreated_userns = false;
2103 if self.config.user_ns_config.is_some() && !Uid::effective().is_root() {
2104 nix::sched::unshare(nix::sched::CloneFlags::CLONE_NEWUSER).map_err(|e| {
2105 NucleusError::NamespaceError(format!(
2106 "Failed to unshare gVisor handoff user namespace: {}",
2107 e
2108 ))
2109 })?;
2110
2111 let request_fd = userns_request_pipe.ok_or_else(|| {
2112 NucleusError::ExecError(
2113 "Missing user namespace request pipe in gVisor handoff child".to_string(),
2114 )
2115 })?;
2116 let ack_fd = userns_ack_pipe.ok_or_else(|| {
2117 NucleusError::ExecError(
2118 "Missing user namespace acknowledgement pipe in gVisor handoff child"
2119 .to_string(),
2120 )
2121 })?;
2122
2123 Self::send_sync_byte(
2124 request_fd,
2125 "Failed to request gVisor handoff user namespace mappings from parent",
2126 )?;
2127 Self::wait_for_sync_byte(
2128 ack_fd,
2129 "Parent closed user namespace ack pipe before gVisor handoff mappings were written",
2130 "Failed waiting for parent to finish gVisor handoff user namespace mappings",
2131 )?;
2132 Self::become_userns_root_for_setup()?;
2133 precreated_userns = true;
2134 }
2135
2136 if matches!(self.config.network, NetworkMode::Bridge(_)) {
2137 nix::sched::unshare(nix::sched::CloneFlags::CLONE_NEWNET).map_err(|e| {
2138 NucleusError::NamespaceError(format!(
2139 "Failed to unshare gVisor bridge network namespace: {}",
2140 e
2141 ))
2142 })?;
2143 }
2144 Ok(precreated_userns)
2145 }
2146
2147 fn wait_for_pid_namespace_child(child: Pid) -> i32 {
2148 loop {
2149 match waitpid(child, None) {
2150 Ok(WaitStatus::Exited(_, code)) => return code,
2151 Ok(WaitStatus::Signaled(_, signal, _)) => return 128 + signal as i32,
2152 Err(nix::errno::Errno::EINTR) => continue,
2153 Err(_) => return 1,
2154 _ => continue,
2155 }
2156 }
2157 }
2158}
2159
2160impl CreatedContainer {
2161 fn emit_control_event(&self, event: ContainerControlEvent) {
2162 if let Some(sink) = &self.event_sink {
2163 if let Err(e) = sink.emit(&event) {
2164 warn!("Failed to emit control event: {}", e);
2165 }
2166 }
2167 }
2168
2169 fn collect_resource_stats(
2170 cgroup_path: Option<&str>,
2171 ) -> (Option<ResourceStats>, Option<String>) {
2172 match cgroup_path {
2173 Some(path) => match ResourceStats::from_cgroup(path) {
2174 Ok(stats) => (Some(stats), None),
2175 Err(e) => (None, Some(e.to_string())),
2176 },
2177 None => (None, None),
2178 }
2179 }
2180
2181 pub fn start(mut self) -> Result<i32> {
2184 let config = &self.config;
2185 let _enter = self._lifecycle_span.enter();
2186
2187 if let Some(exec_fifo_path) = &self.exec_fifo_path {
2190 let file = std::fs::File::open(exec_fifo_path).map_err(|e| {
2191 NucleusError::ExecError(format!("Failed to open exec FIFO for reading: {}", e))
2192 })?;
2193 let mut buf = [0u8; 1];
2194 let read = std::io::Read::read(&mut &file, &mut buf).map_err(|e| {
2195 NucleusError::ExecError(format!("Failed to read exec FIFO sync byte: {}", e))
2196 })?;
2197 if read != 1 {
2198 return Err(NucleusError::ExecError(
2199 "Exec FIFO closed before start signal was delivered".to_string(),
2200 ));
2201 }
2202 let _ = std::fs::remove_file(exec_fifo_path);
2203 }
2204
2205 let target_pid = self.state.pid;
2207 let child = self.child;
2208 self.state.status = OciStatus::Running;
2209 self.state_mgr.save_state(&self.state)?;
2210 self.emit_control_event(ContainerControlEvent::started(
2211 config,
2212 target_pid,
2213 self.state.cgroup_path.clone(),
2214 ));
2215
2216 let (sig_stop, sig_handle) =
2217 Container::setup_signal_forwarding_static(Pid::from_raw(target_pid as i32))?;
2218
2219 let mut sig_guard = SignalThreadGuard {
2221 stop: Some(sig_stop),
2222 handle: Some(sig_handle),
2223 };
2224
2225 let mut console_relay = if config.terminal && config.console_socket.is_none() {
2226 self.native_console_master
2227 .take()
2228 .map(NativeConsoleRelay::start)
2229 .transpose()?
2230 } else {
2231 None
2232 };
2233
2234 if let Some(ref probe) = config.readiness_probe {
2236 let notify_socket = if config.sd_notify {
2237 std::env::var("NOTIFY_SOCKET").ok()
2238 } else {
2239 None
2240 };
2241 Container::run_readiness_probe(
2242 target_pid,
2243 &config.name,
2244 probe,
2245 config.user_ns_config.is_some(),
2246 config.use_gvisor,
2247 &config.process_identity,
2248 notify_socket.as_deref(),
2249 )?;
2250 }
2251
2252 let cancel_flag = Arc::new(AtomicBool::new(false));
2254 let health_handle = if let Some(ref hc) = config.health_check {
2255 if !hc.command.is_empty() {
2256 let hc = hc.clone();
2257 let pid = target_pid;
2258 let container_name = config.name.clone();
2259 let rootless = config.user_ns_config.is_some();
2260 let using_gvisor = config.use_gvisor;
2261 let process_identity = config.process_identity.clone();
2262 let cancel = cancel_flag.clone();
2263 Some(std::thread::spawn(move || {
2264 Container::health_check_loop(
2265 pid,
2266 &container_name,
2267 rootless,
2268 using_gvisor,
2269 &hc,
2270 &process_identity,
2271 &cancel,
2272 );
2273 }))
2274 } else {
2275 None
2276 }
2277 } else {
2278 None
2279 };
2280
2281 let mut health_guard = HealthThreadGuard {
2283 cancel: Some(cancel_flag),
2284 handle: health_handle,
2285 };
2286
2287 if let Some(ref hooks) = config.hooks {
2289 if !hooks.poststart.is_empty() {
2290 let hook_state = OciContainerState {
2291 oci_version: "1.0.2".to_string(),
2292 id: config.id.clone(),
2293 status: OciStatus::Running,
2294 pid: target_pid,
2295 bundle: String::new(),
2296 };
2297 OciHooks::run_hooks(&hooks.poststart, &hook_state, "poststart")?;
2298 }
2299 }
2300
2301 let mut child_waited = false;
2302 let mut run_result: Result<i32> = (|| {
2303 let exit_code = Container::wait_for_child_static(child)?;
2304
2305 self.state.status = OciStatus::Stopped;
2307 let _ = self.state_mgr.save_state(&self.state);
2308
2309 child_waited = true;
2310 Ok(exit_code)
2311 })();
2312
2313 if let Some(relay) = console_relay.as_mut() {
2316 relay.stop();
2317 }
2318 health_guard.stop();
2319 sig_guard.stop();
2320
2321 if let Some(ref hooks) = config.hooks {
2323 if !hooks.poststop.is_empty() {
2324 let hook_state = OciContainerState {
2325 oci_version: "1.0.2".to_string(),
2326 id: config.id.clone(),
2327 status: OciStatus::Stopped,
2328 pid: 0,
2329 bundle: String::new(),
2330 };
2331 OciHooks::run_hooks_best_effort(&hooks.poststop, &hook_state, "poststop");
2332 }
2333 }
2334
2335 let cgroup_path_for_summary = self.state.cgroup_path.clone();
2336 let (resource_stats, resource_stats_error) =
2337 Self::collect_resource_stats(cgroup_path_for_summary.as_deref());
2338 let mut cleanup_errors = Vec::new();
2339
2340 let workspace_synced = match Container::sync_workspace_copy_in_out(config) {
2341 Ok(()) => true,
2342 Err(e) => {
2343 let msg = format!("workspace sync failed: {}", e);
2344 warn!("{}", msg);
2345 cleanup_errors.push(msg);
2346 if run_result.is_ok() {
2347 run_result = Err(e);
2348 }
2349 false
2350 }
2351 };
2352 if workspace_synced {
2353 Container::cleanup_workspace_staging(config);
2354 }
2355
2356 if let Some(net) = self.network_driver.take() {
2357 if let Err(e) = net.cleanup() {
2358 let msg = format!("network cleanup failed: {}", e);
2359 warn!("{}", msg);
2360 cleanup_errors.push(msg);
2361 }
2362 }
2363
2364 if !child_waited {
2365 let _ = kill(child, Signal::SIGKILL);
2366 let _ = waitpid(child, None);
2367 }
2368
2369 if let Some(reader) = self.trace_reader.take() {
2370 reader.stop_and_flush();
2371 }
2372
2373 if let Some(logger) = self.deny_logger.take() {
2374 logger.stop();
2375 }
2376
2377 if let Some(cgroup) = self.cgroup_opt.take() {
2378 if let Err(e) = cgroup.cleanup() {
2379 let msg = format!("cgroup cleanup failed: {}", e);
2380 warn!("{}", msg);
2381 cleanup_errors.push(msg);
2382 }
2383 }
2384
2385 if config.use_gvisor {
2386 if let Err(e) = Container::cleanup_gvisor_artifacts(&config.id) {
2387 let msg = format!("gVisor artifact cleanup failed for {}: {}", config.id, e);
2388 warn!("{}", msg);
2389 cleanup_errors.push(msg);
2390 }
2391 }
2392
2393 if let Err(e) = self.state_mgr.delete_state(&config.id) {
2394 let msg = format!("state cleanup failed for {}: {}", config.id, e);
2395 warn!("{}", msg);
2396 cleanup_errors.push(msg);
2397 }
2398
2399 let exit_status_event = match &run_result {
2400 Ok(exit_code) => ExitStatusEvent::code(*exit_code),
2401 Err(e) => ExitStatusEvent::error(e.to_string()),
2402 };
2403 self.emit_control_event(ContainerControlEvent::summary(
2404 config,
2405 target_pid,
2406 cgroup_path_for_summary,
2407 exit_status_event,
2408 resource_stats,
2409 resource_stats_error,
2410 CleanupEvent::from_errors(cleanup_errors),
2411 ));
2412
2413 match run_result {
2414 Ok(exit_code) => {
2415 audit(
2416 &config.id,
2417 &config.name,
2418 AuditEventType::ContainerStop,
2419 format!("exit_code={}", exit_code),
2420 );
2421 info!(
2422 "Container {} ({}) exited with code {}",
2423 config.name, config.id, exit_code
2424 );
2425 Ok(exit_code)
2426 }
2427 Err(e) => {
2428 audit_error(
2429 &config.id,
2430 &config.name,
2431 AuditEventType::ContainerStop,
2432 format!("error={}", e),
2433 );
2434 Err(e)
2435 }
2436 }
2437 }
2438}
2439
2440struct SignalThreadGuard {
2442 stop: Option<Arc<AtomicBool>>,
2443 handle: Option<JoinHandle<()>>,
2444}
2445
2446impl SignalThreadGuard {
2447 fn stop(&mut self) {
2448 if let Some(flag) = self.stop.take() {
2449 flag.store(true, Ordering::Release);
2450 if let Some(handle) = self.handle.as_ref() {
2451 super::signals::wake_sigwait_thread(handle, Signal::SIGUSR1);
2452 }
2453 }
2454 if let Some(handle) = self.handle.take() {
2455 let _ = handle.join();
2456 }
2457 }
2458}
2459
2460impl Drop for SignalThreadGuard {
2461 fn drop(&mut self) {
2462 self.stop();
2463 }
2464}
2465
2466struct HealthThreadGuard {
2468 cancel: Option<Arc<AtomicBool>>,
2469 handle: Option<JoinHandle<()>>,
2470}
2471
2472impl HealthThreadGuard {
2473 fn stop(&mut self) {
2474 if let Some(flag) = self.cancel.take() {
2475 flag.store(true, Ordering::Relaxed);
2476 }
2477 if let Some(handle) = self.handle.take() {
2478 let _ = handle.join();
2479 }
2480 }
2481}
2482
2483impl Drop for HealthThreadGuard {
2484 fn drop(&mut self) {
2485 self.stop();
2486 }
2487}
2488
2489#[cfg(test)]
2490mod tests {
2491 use super::*;
2492 use crate::container::KernelLockdownMode;
2493 use crate::network::NetworkMode;
2494 use std::ffi::OsString;
2495 use std::sync::{Mutex, MutexGuard};
2496
2497 static ENV_LOCK: Mutex<()> = Mutex::new(());
2498
2499 struct EnvLock {
2500 _guard: MutexGuard<'static, ()>,
2501 }
2502
2503 impl EnvLock {
2504 fn acquire() -> Self {
2505 Self {
2506 _guard: ENV_LOCK.lock().unwrap(),
2507 }
2508 }
2509 }
2510
2511 struct EnvVarGuard {
2512 key: &'static str,
2513 previous: Option<OsString>,
2514 }
2515
2516 impl EnvVarGuard {
2517 fn set(key: &'static str, value: impl AsRef<std::ffi::OsStr>) -> Self {
2518 let previous = std::env::var_os(key);
2519 std::env::set_var(key, value);
2520 Self { key, previous }
2521 }
2522
2523 fn remove(key: &'static str) -> Self {
2524 let previous = std::env::var_os(key);
2525 std::env::remove_var(key);
2526 Self { key, previous }
2527 }
2528 }
2529
2530 impl Drop for EnvVarGuard {
2531 fn drop(&mut self) {
2532 match &self.previous {
2533 Some(value) => std::env::set_var(self.key, value),
2534 None => std::env::remove_var(self.key),
2535 }
2536 }
2537 }
2538
2539 fn extract_fn_body<'a>(source: &'a str, fn_signature: &str) -> &'a str {
2540 let fn_start = source
2541 .find(fn_signature)
2542 .unwrap_or_else(|| panic!("function '{}' not found in source", fn_signature));
2543 let after = &source[fn_start..];
2544 let open = after
2545 .find('{')
2546 .unwrap_or_else(|| panic!("no opening brace found for '{}'", fn_signature));
2547 let mut depth = 0u32;
2548 let mut end = open;
2549 for (i, ch) in after[open..].char_indices() {
2550 match ch {
2551 '{' => depth += 1,
2552 '}' => {
2553 depth -= 1;
2554 if depth == 0 {
2555 end = open + i + 1;
2556 break;
2557 }
2558 }
2559 _ => {}
2560 }
2561 }
2562 &after[..end]
2563 }
2564
2565 #[test]
2566 fn test_container_config() {
2567 let config = ContainerConfig::try_new(None, vec!["/bin/sh".to_string()]).unwrap();
2568 assert!(!config.id.is_empty());
2569 assert_eq!(config.command, vec!["/bin/sh"]);
2570 assert!(config.use_gvisor);
2571 }
2572
2573 #[test]
2574 fn test_run_uses_immediate_start_path_with_parent_setup_gate() {
2575 let source = include_str!("runtime.rs");
2576 let fn_start = source.find("pub fn run(&self) -> Result<i32>").unwrap();
2577 let after = &source[fn_start..];
2578 let open = after.find('{').unwrap();
2579 let mut depth = 0u32;
2580 let mut fn_end = open;
2581 for (i, ch) in after[open..].char_indices() {
2582 match ch {
2583 '{' => depth += 1,
2584 '}' => {
2585 depth -= 1;
2586 if depth == 0 {
2587 fn_end = open + i + 1;
2588 break;
2589 }
2590 }
2591 _ => {}
2592 }
2593 }
2594 let run_body = &after[..fn_end];
2595 assert!(
2596 run_body.contains("create_internal(false)"),
2597 "run() must bypass deferred exec FIFO startup to avoid cross-root deadlocks"
2598 );
2599 assert!(
2600 !run_body.contains("self.create()?.start()"),
2601 "run() must not route through create()+start()"
2602 );
2603
2604 let create_body = extract_fn_body(source, "fn create_internal");
2605 assert!(
2606 create_body.contains("parent_setup_write"),
2607 "immediate run() must still use a parent setup gate before child setup proceeds"
2608 );
2609 }
2610
2611 #[test]
2612 fn test_container_config_with_name() {
2613 let config =
2614 ContainerConfig::try_new(Some("mycontainer".to_string()), vec!["/bin/sh".to_string()])
2615 .unwrap();
2616 assert_eq!(config.name, "mycontainer");
2617 assert!(!config.id.is_empty());
2618 assert_ne!(config.id, config.name);
2619 }
2620
2621 #[test]
2622 fn test_allow_degraded_security_requires_explicit_config() {
2623 let strict = ContainerConfig::try_new(None, vec!["/bin/sh".to_string()]).unwrap();
2624 assert!(!Container::allow_degraded_security(&strict));
2625
2626 let relaxed = strict.clone().with_allow_degraded_security(true);
2627 assert!(Container::allow_degraded_security(&relaxed));
2628 }
2629
2630 #[test]
2631 fn test_env_var_cannot_force_degraded_security_without_explicit_opt_in() {
2632 let prev = std::env::var_os("NUCLEUS_ALLOW_DEGRADED_SECURITY");
2633 std::env::set_var("NUCLEUS_ALLOW_DEGRADED_SECURITY", "1");
2634
2635 let strict = ContainerConfig::try_new(None, vec!["/bin/sh".to_string()]).unwrap();
2636 assert!(!Container::allow_degraded_security(&strict));
2637
2638 let explicit = strict.with_allow_degraded_security(true);
2639 assert!(Container::allow_degraded_security(&explicit));
2640
2641 match prev {
2642 Some(v) => std::env::set_var("NUCLEUS_ALLOW_DEGRADED_SECURITY", v),
2643 None => std::env::remove_var("NUCLEUS_ALLOW_DEGRADED_SECURITY"),
2644 }
2645 }
2646
2647 #[test]
2648 fn test_host_network_requires_explicit_opt_in() {
2649 let mut config = ContainerConfig::try_new(None, vec!["/bin/sh".to_string()])
2650 .unwrap()
2651 .with_gvisor(false)
2652 .with_network(NetworkMode::Host)
2653 .with_allow_host_network(false);
2654 let err = Container::apply_network_mode_guards(&mut config, true).unwrap_err();
2655 assert!(matches!(err, NucleusError::NetworkError(_)));
2656 }
2657
2658 #[test]
2659 fn test_host_network_opt_in_disables_net_namespace() {
2660 let mut config = ContainerConfig::try_new(None, vec!["/bin/sh".to_string()])
2661 .unwrap()
2662 .with_gvisor(false)
2663 .with_network(NetworkMode::Host)
2664 .with_allow_host_network(true);
2665 assert!(config.namespaces.net);
2666 Container::apply_network_mode_guards(&mut config, true).unwrap();
2667 assert!(!config.namespaces.net);
2668 }
2669
2670 #[test]
2671 fn test_host_network_with_gvisor_requires_gvisor_host_mode() {
2672 let mut config = ContainerConfig::try_new(None, vec!["/bin/sh".to_string()])
2673 .unwrap()
2674 .with_network(NetworkMode::Host)
2675 .with_allow_host_network(true);
2676 let err = Container::apply_network_mode_guards(&mut config, true).unwrap_err();
2677 assert!(matches!(err, NucleusError::NetworkError(_)));
2678 assert!(err.to_string().contains("gvisor-host"));
2679 }
2680
2681 #[test]
2682 fn test_gvisor_host_requires_explicit_opt_in() {
2683 let mut config = ContainerConfig::try_new(None, vec!["/bin/sh".to_string()])
2684 .unwrap()
2685 .with_network(NetworkMode::GVisorHost)
2686 .with_allow_host_network(false);
2687 let err = Container::apply_network_mode_guards(&mut config, true).unwrap_err();
2688 assert!(matches!(err, NucleusError::NetworkError(_)));
2689 assert!(err.to_string().contains("--allow-host-network"));
2690 }
2691
2692 #[test]
2693 fn test_gvisor_host_preserves_namespace_config_for_oci_setup() {
2694 let mut config = ContainerConfig::try_new(None, vec!["/bin/sh".to_string()])
2695 .unwrap()
2696 .with_network(NetworkMode::GVisorHost)
2697 .with_allow_host_network(true);
2698 assert!(config.namespaces.net);
2699 Container::apply_network_mode_guards(&mut config, true).unwrap();
2700 assert!(config.namespaces.net);
2701 }
2702
2703 #[test]
2704 fn test_non_host_network_does_not_require_host_opt_in() {
2705 let mut config = ContainerConfig::try_new(None, vec!["/bin/sh".to_string()])
2706 .unwrap()
2707 .with_network(NetworkMode::None)
2708 .with_allow_host_network(false);
2709 assert!(config.namespaces.net);
2710 Container::apply_network_mode_guards(&mut config, true).unwrap();
2711 assert!(config.namespaces.net);
2712 }
2713
2714 #[test]
2715 fn test_credential_broker_auto_nat_rejects_rootless_userspace_resolution() {
2716 let broker =
2717 crate::network::CredentialBrokerConfig::parse_endpoint("10.0.42.1:8080").unwrap();
2718 let mut config = ContainerConfig::try_new(None, vec!["/bin/true".to_string()])
2719 .unwrap()
2720 .with_network(NetworkMode::Bridge(crate::network::BridgeConfig::default()))
2721 .with_credential_broker(broker.clone())
2722 .with_egress_policy(broker.egress_policy())
2723 .with_rootless();
2724
2725 let err = Container::validate_credential_broker_resolved_nat(&config, false).unwrap_err();
2726 assert!(err.to_string().contains("auto resolves to userspace"));
2727
2728 config.user_ns_config = None;
2729 config.namespaces.user = false;
2730 assert!(Container::validate_credential_broker_resolved_nat(&config, true).is_ok());
2731 }
2732
2733 #[test]
2734 fn test_parse_kernel_lockdown_mode() {
2735 assert_eq!(
2736 Container::parse_active_lockdown_mode("none [integrity] confidentiality"),
2737 Some(KernelLockdownMode::Integrity)
2738 );
2739 assert_eq!(
2740 Container::parse_active_lockdown_mode("none integrity [confidentiality]"),
2741 Some(KernelLockdownMode::Confidentiality)
2742 );
2743 assert_eq!(
2744 Container::parse_active_lockdown_mode("[none] integrity"),
2745 None
2746 );
2747 }
2748
2749 #[test]
2750 fn test_stage_gvisor_secret_files_rewrites_sources_under_stage_dir() {
2751 let temp = tempfile::TempDir::new().unwrap();
2752 let source = temp.path().join("source-secret");
2753 std::fs::write(&source, "supersecret").unwrap();
2754
2755 let staged = Container::stage_gvisor_secret_files(
2756 &temp.path().join("stage"),
2757 &[crate::container::SecretMount {
2758 source: source.clone(),
2759 dest: std::path::PathBuf::from("/etc/app/secret.txt"),
2760 mode: 0o400,
2761 }],
2762 &crate::container::ProcessIdentity::root(),
2763 )
2764 .unwrap();
2765
2766 assert_eq!(staged.len(), 1);
2767 assert!(staged[0].source.starts_with(temp.path().join("stage")));
2768 assert_eq!(
2769 std::fs::read_to_string(&staged[0].source).unwrap(),
2770 "supersecret"
2771 );
2772 }
2773
2774 #[test]
2775 fn test_stage_gvisor_secret_files_rejects_symlink_source() {
2776 use std::os::unix::fs::symlink;
2777
2778 let temp = tempfile::TempDir::new().unwrap();
2779 let source = temp.path().join("source-secret");
2780 let link = temp.path().join("source-link");
2781 std::fs::write(&source, "supersecret").unwrap();
2782 symlink(&source, &link).unwrap();
2783
2784 let err = Container::stage_gvisor_secret_files(
2785 &temp.path().join("stage"),
2786 &[crate::container::SecretMount {
2787 source: link,
2788 dest: std::path::PathBuf::from("/etc/app/secret.txt"),
2789 mode: 0o400,
2790 }],
2791 &crate::container::ProcessIdentity::root(),
2792 )
2793 .unwrap_err();
2794
2795 assert!(
2796 err.to_string().contains("O_NOFOLLOW"),
2797 "gVisor secret staging must reject symlink sources"
2798 );
2799 }
2800
2801 #[test]
2802 fn test_native_runtime_uses_inmemory_secrets_for_all_modes() {
2803 let source = include_str!("runtime.rs");
2804 let fn_body = extract_fn_body(source, "fn setup_and_exec");
2805 assert!(
2806 fn_body.contains("mount_secrets_inmemory("),
2807 "setup_and_exec must use in-memory secret mounting"
2808 );
2809 assert!(
2810 !fn_body.contains("mount_secrets(&"),
2811 "setup_and_exec must not bind-mount secrets from the host"
2812 );
2813 }
2814
2815 #[test]
2816 fn test_overlay_rootfs_does_not_register_writable_landlock_root() {
2817 let source = include_str!("runtime.rs");
2818 let fn_body = extract_fn_body(source, "fn setup_and_exec");
2819 assert!(
2820 !fn_body.contains("landlock_mgr.add_rw_exec_path(\"/\")"),
2821 "overlay rootfs mode must not grant broad read/write/execute Landlock access to /"
2822 );
2823 }
2824
2825 #[test]
2826 fn test_overlay_rootfs_drops_all_capabilities() {
2827 let source = include_str!("runtime.rs");
2828 let fn_body = extract_fn_body(source, "fn setup_and_exec");
2829 assert!(
2830 fn_body.contains("cap_mgr.drop_bounding_set()?")
2831 && fn_body.contains("cap_mgr.finalize_drop()?")
2832 && !fn_body.contains("drop_bounding_set_except")
2833 && !fn_body.contains("finalize_drop_except"),
2834 "overlay rootfs mode must use the default drop-all capability path"
2835 );
2836 }
2837
2838 #[test]
2839 fn test_native_production_procfs_mount_is_not_rootless_best_effort() {
2840 let source = include_str!("runtime.rs");
2841 let fn_body = extract_fn_body(source, "fn setup_and_exec");
2842
2843 assert!(
2844 fn_body.contains(
2845 "let production_mode = self.config.service_mode == ServiceMode::Production;"
2846 ),
2847 "setup_and_exec must derive an explicit production-mode guard for procfs hardening"
2848 );
2849 assert!(
2850 fn_body.contains("let procfs_best_effort = is_rootless && !production_mode;"),
2851 "rootless best-effort procfs fallback must be disabled in production mode"
2852 );
2853 assert!(
2854 fn_body.contains(
2855 "mount_procfs(\n &proc_path,\n procfs_best_effort,"
2856 ),
2857 "mount_procfs must receive the production-aware best-effort flag"
2858 );
2859 }
2860
2861 #[test]
2862 fn test_gvisor_uses_inmemory_secret_staging_for_all_modes() {
2863 let source = include_str!("gvisor_setup.rs");
2864 let fn_body = extract_fn_body(source, "fn setup_and_exec_gvisor_oci");
2865 assert!(
2866 fn_body.contains("with_inmemory_secret_mounts"),
2867 "gVisor setup must use the tmpfs-backed secret staging path"
2868 );
2869 assert!(
2870 !fn_body.contains("with_secret_mounts"),
2871 "gVisor setup must not bind-mount host secret paths"
2872 );
2873 }
2874
2875 #[test]
2876 fn test_gvisor_precreated_userns_skips_nested_oci_userns() {
2877 let source = include_str!("gvisor_setup.rs");
2878 let fn_body = extract_fn_body(source, "fn setup_and_exec_gvisor_oci");
2879 let precreated_check = fn_body.find("if precreated_userns").unwrap();
2880 let remove_oci_userns = fn_body.find("without_user_namespace").unwrap();
2881 let oci_userns = fn_body.find("with_rootless_user_namespace").unwrap();
2882 assert!(
2883 precreated_check < remove_oci_userns && remove_oci_userns < oci_userns,
2884 "pre-created rootless gVisor userns must remove nested OCI user namespace setup"
2885 );
2886 }
2887
2888 #[test]
2889 fn test_gvisor_precreated_userns_disables_oci_no_new_privileges() {
2890 let source = include_str!("gvisor_setup.rs");
2891 let fn_body = extract_fn_body(source, "fn setup_and_exec_gvisor_oci");
2892 assert!(
2893 fn_body.contains("if precreated_userns")
2894 && fn_body.contains("with_no_new_privileges(false)"),
2895 "pre-created rootless gVisor userns must not pass OCI noNewPrivileges to runsc"
2896 );
2897 }
2898
2899 #[test]
2900 fn test_gvisor_rootless_passes_runsc_rootless() {
2901 let source = include_str!("gvisor_setup.rs");
2902 let fn_body = extract_fn_body(source, "fn setup_and_exec_gvisor_oci");
2903 assert!(
2904 fn_body.contains("let runsc_rootless = rootless_gvisor"),
2905 "rootless gVisor must tell runsc to keep rootless supervisor semantics"
2906 );
2907 }
2908
2909 #[test]
2910 fn test_gvisor_rootless_preserves_configured_platform() {
2911 let source = include_str!("gvisor_setup.rs");
2912 let fn_body = extract_fn_body(source, "fn setup_and_exec_gvisor_oci");
2913 assert!(
2914 fn_body.contains("let platform = self.config.gvisor_platform;")
2915 && fn_body.contains("platform,"),
2916 "rootless gVisor must preserve the configured runsc platform"
2917 );
2918 assert!(
2919 !fn_body.contains("GVisorPlatform::Ptrace"),
2920 "rootless gVisor must not silently rewrite systrap to ptrace"
2921 );
2922 }
2923
2924 #[test]
2925 fn test_gvisor_precreated_userns_keeps_store_runsc_binary() {
2926 let source = include_str!("gvisor_setup.rs");
2927 let fn_body = extract_fn_body(source, "fn setup_and_exec_gvisor_oci");
2928 assert!(
2929 fn_body.contains("let stage_runsc_binary = false")
2930 && fn_body.contains("stage_runsc_binary,"),
2931 "pre-created rootless gVisor userns must keep runsc on its validated store path"
2932 );
2933 }
2934
2935 #[test]
2936 fn test_gvisor_root_remap_keeps_production_supervisor_exec_policy() {
2937 let source = include_str!("gvisor_setup.rs");
2938 let fn_body = extract_fn_body(source, "fn setup_and_exec_gvisor_oci");
2939 let helper = extract_fn_body(source, "fn require_gvisor_supervisor_exec_policy");
2940 let policy = fn_body.find("let require_supervisor_exec_policy").unwrap();
2941 assert!(
2942 fn_body[policy..].contains(
2943 "require_gvisor_supervisor_exec_policy(self.config.service_mode, precreated_userns)"
2944 ),
2945 "gVisor setup must use the explicit supervisor policy predicate"
2946 );
2947 assert!(
2948 helper.contains("ServiceMode::Production")
2949 && helper.contains("!precreated_userns")
2950 && !helper.contains("rootless_gvisor")
2951 && !helper.contains("user_ns_config"),
2952 "root-remapped user namespace config must not disable the production exec policy"
2953 );
2954 }
2955
2956 #[test]
2957 fn test_gvisor_bridge_rootless_requests_external_userns_mapping() {
2958 let source = include_str!("runtime.rs");
2959 let create_body = extract_fn_body(source, "fn create_internal");
2960 let helper = extract_fn_body(source, "fn gvisor_needs_precreated_userns");
2961 assert!(
2962 create_body.contains("let gvisor_needs_precreated_userns"),
2963 "gVisor bridge rootless setup must request parent-written userns mappings"
2964 );
2965 assert!(
2966 create_body.contains("!config.use_gvisor || gvisor_needs_precreated_userns"),
2967 "external mapping request must be routed through the gVisor pre-created userns predicate"
2968 );
2969 assert!(
2970 helper.contains("NetworkMode::Bridge(_)"),
2971 "external mapping request must include gVisor bridge networking"
2972 );
2973 }
2974
2975 #[test]
2976 fn test_gvisor_host_rootless_requests_external_userns_mapping() {
2977 let source = include_str!("runtime.rs");
2978 let helper = extract_fn_body(source, "fn gvisor_needs_precreated_userns");
2979 assert!(
2980 helper.contains("NetworkMode::GVisorHost"),
2981 "gVisor host mode must use the pre-created userns handoff so runsc hostinet starts rootless"
2982 );
2983 assert!(
2984 helper.contains("!host_is_root") && helper.contains("config.user_ns_config.is_some()"),
2985 "gVisor host handoff must stay scoped to rootless user namespace launches"
2986 );
2987 }
2988
2989 #[test]
2990 fn test_gvisor_handoff_namespace_creates_userns_before_bridge_netns() {
2991 let source = include_str!("runtime.rs");
2992 let fn_body = extract_fn_body(source, "fn prepare_gvisor_handoff_namespace");
2993 let userns = fn_body.find("CLONE_NEWUSER").unwrap();
2994 let request = fn_body.find("send_sync_byte").unwrap();
2995 let become_root = fn_body.find("become_userns_root_for_setup").unwrap();
2996 let bridge_only = fn_body
2997 .find("if matches!(self.config.network, NetworkMode::Bridge(_))")
2998 .unwrap();
2999 let netns = fn_body.find("CLONE_NEWNET").unwrap();
3000 assert!(
3001 userns < request
3002 && request < become_root
3003 && become_root < bridge_only
3004 && bridge_only < netns,
3005 "rootless gVisor handoff must map userns before creating the bridge netns"
3006 );
3007 assert!(
3008 !fn_body.contains("NetworkMode::GVisorHost"),
3009 "gVisor host handoff must not create an OCI network namespace before runsc hostinet"
3010 );
3011 }
3012
3013 #[test]
3014 fn test_gvisor_handoff_userns_preserves_caps_for_runsc_exec() {
3015 let source = include_str!("runtime.rs");
3016 let fn_body = extract_fn_body(source, "fn become_userns_root_for_setup");
3017 let setuid = fn_body.find("setresuid").unwrap();
3018 let restore = fn_body.find("restore_gvisor_handoff_capabilities").unwrap();
3019 assert!(
3020 setuid < restore,
3021 "gVisor setup must restore exec-time capabilities after switching to uid 0 in the mapped namespace"
3022 );
3023 assert!(
3024 !fn_body.contains("set_keepcaps"),
3025 "gVisor setup must let the uid 0 switch grant namespace capabilities instead of preserving the pre-switch empty set"
3026 );
3027 }
3028
3029 #[test]
3030 fn test_gvisor_handoff_narrows_permitted_caps() {
3031 let source = include_str!("runtime.rs");
3032 let fn_body = extract_fn_body(source, "fn restore_gvisor_handoff_capabilities");
3033 let read_permitted = fn_body.find("caps::read(None, CapSet::Permitted)").unwrap();
3034 let missing_check = fn_body.find("if !missing.is_empty()").unwrap();
3035 let set_effective = fn_body.find("caps::set(None, CapSet::Effective").unwrap();
3036 let drop_permitted = fn_body.find("caps::drop(None, CapSet::Permitted").unwrap();
3037 assert!(
3038 read_permitted < missing_check && missing_check < set_effective,
3039 "gVisor handoff must verify existing permitted capabilities"
3040 );
3041 assert!(
3042 fn_body.contains("permitted.difference(&caps)")
3043 && set_effective < drop_permitted
3044 && !fn_body.contains("caps::set(None, CapSet::Permitted"),
3045 "gVisor handoff must drop extra permitted capabilities without trying to raise permitted"
3046 );
3047 }
3048
3049 #[test]
3050 fn test_gvisor_handoff_narrows_bounding_caps() {
3051 let source = include_str!("runtime.rs");
3052 let fn_body = extract_fn_body(source, "fn restore_gvisor_handoff_capabilities");
3053 let set_effective = fn_body.find("caps::set(None, CapSet::Effective").unwrap();
3054 let drop_bounding = fn_body.find("caps::drop(None, CapSet::Bounding").unwrap();
3055 let drop_permitted = fn_body.find("caps::drop(None, CapSet::Permitted").unwrap();
3056 assert!(
3057 fn_body.contains("caps::read(None, CapSet::Bounding)")
3058 && fn_body.contains("bounding.difference(&caps)")
3059 && set_effective < drop_bounding
3060 && drop_bounding < drop_permitted,
3061 "gVisor handoff must drop extra bounding capabilities before exec"
3062 );
3063 }
3064
3065 #[test]
3066 fn test_gvisor_handoff_caps_are_bounded_to_runsc_startup() {
3067 let caps = Container::gvisor_handoff_capabilities();
3068 for cap in [
3069 Capability::CAP_CHOWN,
3070 Capability::CAP_DAC_OVERRIDE,
3071 Capability::CAP_DAC_READ_SEARCH,
3072 Capability::CAP_FOWNER,
3073 Capability::CAP_FSETID,
3074 Capability::CAP_SYS_CHROOT,
3075 Capability::CAP_SYS_PTRACE,
3076 Capability::CAP_SETUID,
3077 Capability::CAP_SETGID,
3078 Capability::CAP_SYS_ADMIN,
3079 Capability::CAP_SETPCAP,
3080 ] {
3081 assert!(
3082 caps.contains(&cap),
3083 "gVisor handoff caps must include {cap:?}"
3084 );
3085 }
3086 assert_eq!(
3087 caps.len(),
3088 11,
3089 "gVisor handoff caps must stay aligned with the frontend systemd bounding set"
3090 );
3091 assert!(
3092 !caps.contains(&Capability::CAP_NET_ADMIN) && !caps.contains(&Capability::CAP_MKNOD),
3093 "gVisor handoff must not grow into broader host-style privilege"
3094 );
3095 }
3096
3097 #[test]
3098 fn test_native_fork_sites_assert_single_threaded() {
3099 let runtime_source = include_str!("runtime.rs");
3100 let create_body = extract_fn_body(runtime_source, "fn create_internal");
3101 assert!(
3102 create_body.contains("assert_single_threaded_for_fork(\"container create fork\")"),
3103 "create_internal must assert single-threaded before fork"
3104 );
3105
3106 let setup_body = extract_fn_body(runtime_source, "fn setup_and_exec");
3107 assert!(
3108 setup_body.contains("assert_single_threaded_for_fork(\"PID namespace init fork\")"),
3109 "PID namespace setup must assert single-threaded before fork"
3110 );
3111
3112 let exec_source = include_str!("exec.rs");
3113 let init_body = extract_fn_body(exec_source, "fn run_as_init");
3114 assert!(
3115 init_body.contains("assert_single_threaded_for_fork(\"init supervisor fork\")"),
3116 "run_as_init must assert single-threaded before fork"
3117 );
3118 }
3119
3120 #[test]
3121 fn test_parent_setup_gate_released_after_network_policy() {
3122 let source = include_str!("runtime.rs");
3123 let create_body = extract_fn_body(source, "fn create_internal");
3124
3125 let cgroup_attach = create_body.find("cgroup.attach_process").unwrap();
3126 let deny_logger = create_body.find("maybe_start_seccomp_deny_logger").unwrap();
3127 let bridge_setup = create_body.find("BridgeDriver::setup_with_id").unwrap();
3128 let egress_policy = create_body.find("net.apply_egress_policy").unwrap();
3129 let broker_probe = create_body.find("probe_credential_broker").unwrap();
3130 let release = create_body
3131 .find("Failed to notify child that parent setup is complete")
3132 .unwrap();
3133 let created = create_body.find("Ok(CreatedContainer").unwrap();
3134
3135 assert!(
3136 cgroup_attach < bridge_setup,
3137 "parent setup gate must not release before cgroup attachment"
3138 );
3139 assert!(
3140 cgroup_attach < deny_logger && deny_logger < bridge_setup,
3141 "seccomp deny logger must start after cgroup attachment and before workload release"
3142 );
3143 assert!(
3144 create_body.contains("cgroup_opt.as_ref().map(|cgroup| cgroup.path())"),
3145 "seccomp deny logger must receive the container cgroup scope"
3146 );
3147 assert!(
3148 bridge_setup < egress_policy && egress_policy < broker_probe && broker_probe < release,
3149 "parent setup gate must not release before bridge, egress policy, and broker probe setup"
3150 );
3151 assert!(
3152 release < created,
3153 "create_internal must release the child only after all fallible parent setup succeeds"
3154 );
3155 assert!(
3156 !create_body.contains("cgroup attachment is complete"),
3157 "child setup gate must not be released immediately after cgroup attachment"
3158 );
3159 }
3160
3161 #[test]
3162 fn test_child_waits_for_parent_setup_before_exec_paths() {
3163 let source = include_str!("runtime.rs");
3164 let setup_body = extract_fn_body(source, "fn setup_and_exec");
3165
3166 let gvisor_wait = setup_body
3167 .find("Parent closed setup pipe before signalling gVisor child")
3168 .unwrap();
3169 let gvisor_exec = setup_body.find("setup_and_exec_gvisor").unwrap();
3170 assert!(
3171 gvisor_wait < gvisor_exec,
3172 "gVisor path must wait for parent setup before execing runsc"
3173 );
3174
3175 let pid1_wait = setup_body
3176 .find("Parent closed setup pipe before signalling PID 1 child")
3177 .unwrap();
3178 let namespace_enter = setup_body.find("namespace_mgr.enter()?").unwrap();
3179 assert!(
3180 pid1_wait < namespace_enter,
3181 "PID namespace child must wait for parent setup before container setup continues"
3182 );
3183
3184 let direct_wait = setup_body
3185 .find("Parent closed setup pipe before signalling container child")
3186 .unwrap();
3187 assert!(
3188 direct_wait < namespace_enter,
3189 "non-PID namespace child must wait for parent setup before container setup continues"
3190 );
3191 }
3192
3193 #[test]
3194 fn test_parent_setup_failure_kills_reported_target_pid() {
3195 let source = include_str!("runtime.rs");
3196 let create_body = extract_fn_body(source, "fn create_internal");
3197
3198 let record_target = create_body
3199 .find("target_pid_for_cleanup = Some(target_pid)")
3200 .unwrap();
3201 let kill_target = create_body
3202 .find("kill(Pid::from_raw(target_pid as i32), Signal::SIGKILL)")
3203 .unwrap();
3204 let kill_intermediate = create_body.find("kill(child, Signal::SIGKILL)").unwrap();
3205
3206 assert!(
3207 record_target < kill_target,
3208 "parent setup cleanup must remember the reported target PID"
3209 );
3210 assert!(
3211 kill_target < kill_intermediate,
3212 "cleanup must kill the target PID before reaping the intermediate fork"
3213 );
3214 }
3215
3216 #[test]
3217 fn test_run_as_init_keeps_identity_drop_in_workload_child_path() {
3218 let source = include_str!("exec.rs");
3219 let fn_body = extract_fn_body(source, "fn run_as_init");
3220 assert!(
3221 !fn_body.contains("Self::apply_process_identity_to_current_process("),
3222 "run_as_init must not drop identity before the supervisor fork"
3223 );
3224 assert!(
3225 fn_body.contains("self.exec_command()?"),
3226 "workload child must still route through exec_command for identity application"
3227 );
3228 }
3229
3230 #[test]
3231 fn test_run_as_init_parent_closes_nonstdio_fds() {
3232 let runtime_source = include_str!("runtime.rs");
3233 let close_body = extract_fn_body(runtime_source, "fn close_nonstdio_fds");
3234 assert!(
3235 close_body.contains("libc::SYS_close_range, 3u32, u32::MAX, 0u32")
3236 && close_body.contains("libc::close(fd)"),
3237 "close_nonstdio_fds must immediately close descriptors, not only mark them CLOEXEC"
3238 );
3239 assert!(
3240 !close_body.contains("CLOSE_RANGE_CLOEXEC")
3241 && !close_body.contains("FD_CLOEXEC")
3242 && !close_body.contains("F_SETFD"),
3243 "PID 1 supervisor cleanup must not rely on close-on-exec"
3244 );
3245
3246 let exec_source = include_str!("exec.rs");
3247 let init_body = extract_fn_body(exec_source, "fn run_as_init");
3248 let parent = init_body.find("ForkResult::Parent").unwrap();
3249 let close = init_body.find("Self::close_nonstdio_fds()").unwrap();
3250 let signal_setup = init_body.find("let mut sigset = SigSet::empty()").unwrap();
3251 let child = init_body.find("ForkResult::Child").unwrap();
3252 assert!(
3253 parent < close && close < signal_setup && close < child,
3254 "run_as_init must close inherited FDs in the non-execing PID 1 parent before supervisor work"
3255 );
3256 }
3257
3258 #[test]
3259 fn test_signal_thread_shutdown_uses_thread_directed_wakeup() {
3260 let runtime_source = include_str!("runtime.rs");
3261 let exec_source = include_str!("exec.rs");
3262 let signal_helper_source = include_str!("signals.rs");
3263 let process_directed_wakeup = ["kill(Pid::this()", ", Signal::SIGUSR1)"].concat();
3264
3265 assert!(
3266 !runtime_source.contains(&process_directed_wakeup),
3267 "CreatedContainer signal-thread shutdown must not send process-directed SIGUSR1"
3268 );
3269 assert!(
3270 !exec_source.contains(&process_directed_wakeup),
3271 "init supervisor signal-thread shutdown must not send process-directed SIGUSR1"
3272 );
3273 assert!(
3274 signal_helper_source.contains("libc::pthread_kill"),
3275 "signal-thread shutdown must wake the sigwait owner with a thread-directed signal"
3276 );
3277 }
3278
3279 #[test]
3280 fn test_cleanup_gvisor_artifacts_removes_artifact_dir() {
3281 let _env_lock = EnvLock::acquire();
3282 let temp = tempfile::TempDir::new().unwrap();
3283 let _artifact_base = EnvVarGuard::set(
3284 "NUCLEUS_GVISOR_ARTIFACT_BASE",
3285 temp.path().join("gvisor-artifacts"),
3286 );
3287 let artifact_dir = Container::gvisor_artifact_dir("cleanup-test");
3288 std::fs::create_dir_all(&artifact_dir).unwrap();
3289 std::fs::write(artifact_dir.join("config.json"), "{}").unwrap();
3290
3291 Container::cleanup_gvisor_artifacts("cleanup-test").unwrap();
3292 assert!(!artifact_dir.exists());
3293 }
3294
3295 #[test]
3296 fn test_gvisor_artifact_base_prefers_xdg_runtime_dir() {
3297 let _env_lock = EnvLock::acquire();
3298 let temp = tempfile::TempDir::new().unwrap();
3299 let _artifact_override = EnvVarGuard::remove("NUCLEUS_GVISOR_ARTIFACT_BASE");
3300 let _runtime = EnvVarGuard::set("XDG_RUNTIME_DIR", temp.path());
3301
3302 assert_eq!(
3303 Container::gvisor_artifact_dir("xdg-test"),
3304 temp.path().join("nucleus-gvisor").join("xdg-test")
3305 );
3306 }
3307
3308 #[test]
3309 fn test_health_check_loop_supports_cancellation() {
3310 let source = include_str!("health.rs");
3314 let fn_start = source.find("fn health_check_loop").unwrap();
3315 let fn_body = &source[fn_start..fn_start + 2500];
3316 assert!(
3317 fn_body.contains("AtomicBool") && fn_body.contains("cancel"),
3318 "health_check_loop must accept an AtomicBool cancellation flag"
3319 );
3320 assert!(
3322 fn_body.contains("cancellable_sleep") || fn_body.contains("cancel.load"),
3323 "health_check_loop must check cancellation during sleep intervals"
3324 );
3325 }
3326
3327 #[test]
3328 fn test_runtime_probes_do_not_spawn_host_nsenter() {
3329 let source = include_str!("health.rs");
3331
3332 let readiness_start = source.find("fn run_readiness_probe").unwrap();
3333 let readiness_body = &source[readiness_start..readiness_start + 2500];
3334 assert!(
3335 !readiness_body.contains("Command::new(&nsenter_bin)"),
3336 "readiness probes must not execute via host nsenter"
3337 );
3338
3339 let health_start = source.find("fn health_check_loop").unwrap();
3340 let health_body = &source[health_start..health_start + 2200];
3341 assert!(
3342 !health_body.contains("Command::new(&nsenter_bin)"),
3343 "health checks must not execute via host nsenter"
3344 );
3345 }
3346
3347 #[test]
3348 fn test_oci_mount_strip_prefix_no_expect() {
3349 let source = include_str!("gvisor_setup.rs");
3352 let fn_start = source.find("fn prepare_oci_mountpoints").unwrap();
3353 let fn_body = &source[fn_start..fn_start + 600];
3354 assert!(
3355 !fn_body.contains(".expect("),
3356 "prepare_oci_mountpoints must not use expect() – return Err instead"
3357 );
3358 }
3359
3360 #[test]
3361 fn test_notify_namespace_ready_validates_write_length() {
3362 let source = include_str!("runtime.rs");
3364 let fn_start = source.find("fn notify_namespace_ready").unwrap();
3365 let fn_body = &source[fn_start..fn_start + 500];
3366 assert!(
3368 fn_body.contains("written")
3369 || fn_body.contains("4")
3370 || fn_body.contains("payload.len()"),
3371 "notify_namespace_ready must validate complete write of all 4 bytes"
3372 );
3373 }
3374
3375 #[test]
3376 fn test_rlimit_failures_fatal_in_production() {
3377 let source = include_str!("runtime.rs");
3379 let rlimit_start = source.find("12b. RLIMIT backstop").unwrap();
3380 let rlimit_section = &source[rlimit_start..rlimit_start + 2000];
3381 assert!(
3382 rlimit_section.contains("is_production") && rlimit_section.contains("return Err"),
3383 "RLIMIT failures must return Err in production mode"
3384 );
3385 }
3386
3387 #[test]
3388 fn test_tcp_readiness_probe_uses_portable_check() {
3389 let source = include_str!("health.rs");
3392 let probe_fn = source.find("TcpPort(port)").unwrap();
3393 let probe_body = &source[probe_fn..probe_fn + 500];
3394 assert!(
3395 !probe_body.contains("/dev/tcp"),
3396 "TCP readiness probe must not use /dev/tcp (bash-specific, fails on dash/ash)"
3397 );
3398 }
3399}