1use std::collections::{HashMap, HashSet};
80use std::net::{IpAddr, Ipv4Addr};
81use std::time::Duration;
82
83use bollard::container::{
84 Config, CreateContainerOptions, RemoveContainerOptions, StartContainerOptions,
85 StopContainerOptions,
86};
87use bollard::image::CreateImageOptions;
88use bollard::models::{
89 ContainerInspectResponse, HostConfig, PortBinding, RestartPolicy as DockerRestartPolicy,
90 RestartPolicyNameEnum,
91};
92use bollard::network::{CreateNetworkOptions, ListNetworksOptions};
93use bollard::Docker;
94use futures_util::StreamExt;
95
96use crate::cgroup;
97use crate::error::SubstrateError;
98use crate::mount::{self, VolumeMount};
99use crate::principal::ContainerRef;
100
101pub const LABEL_MANAGED: &str = "substrate.lifecycle.managed";
105
106#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
111pub enum RestartPolicy {
112 #[default]
114 Never,
115 Always,
117 UnlessStopped,
119 OnFailure { max_retries: i64 },
121}
122
123impl RestartPolicy {
124 fn into_docker(self) -> DockerRestartPolicy {
125 match self {
126 RestartPolicy::Never => DockerRestartPolicy {
127 name: Some(RestartPolicyNameEnum::NO),
128 maximum_retry_count: None,
129 },
130 RestartPolicy::Always => DockerRestartPolicy {
131 name: Some(RestartPolicyNameEnum::ALWAYS),
132 maximum_retry_count: None,
133 },
134 RestartPolicy::UnlessStopped => DockerRestartPolicy {
135 name: Some(RestartPolicyNameEnum::UNLESS_STOPPED),
136 maximum_retry_count: None,
137 },
138 RestartPolicy::OnFailure { max_retries } => DockerRestartPolicy {
139 name: Some(RestartPolicyNameEnum::ON_FAILURE),
140 maximum_retry_count: Some(max_retries),
141 },
142 }
143 }
144}
145
146#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
148pub enum Protocol {
149 #[default]
150 Tcp,
151 Udp,
152}
153
154impl Protocol {
155 fn as_str(self) -> &'static str {
156 match self {
157 Protocol::Tcp => "tcp",
158 Protocol::Udp => "udp",
159 }
160 }
161}
162
163#[derive(Debug, Clone, PartialEq, Eq)]
171pub struct PortMapping {
172 pub container_port: u16,
173 pub host_port: Option<u16>,
175 pub host_ip: IpAddr,
176 pub protocol: Protocol,
177}
178
179impl PortMapping {
180 pub fn tcp(container_port: u16) -> Self {
183 Self {
184 container_port,
185 host_port: None,
186 host_ip: IpAddr::V4(Ipv4Addr::LOCALHOST),
187 protocol: Protocol::Tcp,
188 }
189 }
190
191 pub fn udp(container_port: u16) -> Self {
193 Self {
194 protocol: Protocol::Udp,
195 ..Self::tcp(container_port)
196 }
197 }
198
199 pub fn with_host_port(mut self, host_port: u16) -> Self {
200 self.host_port = Some(host_port);
201 self
202 }
203
204 pub fn with_host_ip(mut self, host_ip: IpAddr) -> Self {
207 self.host_ip = host_ip;
208 self
209 }
210
211 fn docker_key(&self) -> String {
213 format!("{}/{}", self.container_port, self.protocol.as_str())
214 }
215
216 fn validate(&self) -> Result<(), SubstrateError> {
217 if self.container_port == 0 {
218 return Err(SubstrateError::InvalidSpec(
219 "port mapping has container_port 0".to_string(),
220 ));
221 }
222 if self.host_port == Some(0) {
223 return Err(SubstrateError::InvalidSpec(format!(
224 "port mapping for {} has host_port 0; use None to let dockerd allocate",
225 self.docker_key()
226 )));
227 }
228 Ok(())
229 }
230}
231
232#[derive(Clone)]
239#[non_exhaustive]
240pub struct ContainerSpec {
241 pub name: String,
243 pub image: String,
244 pub env: HashMap<String, String>,
247 pub cmd: Option<Vec<String>>,
250 pub labels: HashMap<String, String>,
251 pub restart_policy: RestartPolicy,
252 pub ports: Vec<PortMapping>,
254 pub volumes: Vec<VolumeMount>,
256 pub network: Option<String>,
261}
262
263impl std::fmt::Debug for ContainerSpec {
264 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
265 let mut env_keys: Vec<&String> = self.env.keys().collect();
266 env_keys.sort();
267 f.debug_struct("ContainerSpec")
268 .field("name", &self.name)
269 .field("image", &self.image)
270 .field("env_keys", &env_keys)
271 .field("cmd", &self.cmd)
272 .field("labels", &self.labels)
273 .field("restart_policy", &self.restart_policy)
274 .field("ports", &self.ports)
275 .field("volumes", &self.volumes)
276 .field("network", &self.network)
277 .finish()
278 }
279}
280
281impl ContainerSpec {
282 pub fn new(name: impl Into<String>, image: impl Into<String>) -> Self {
283 Self {
284 name: name.into(),
285 image: image.into(),
286 env: HashMap::new(),
287 cmd: None,
288 labels: HashMap::new(),
289 restart_policy: RestartPolicy::default(),
290 ports: Vec::new(),
291 volumes: Vec::new(),
292 network: None,
293 }
294 }
295
296 pub fn with_env(mut self, env: HashMap<String, String>) -> Self {
297 self.env = env;
298 self
299 }
300
301 pub fn with_cmd(mut self, cmd: Vec<String>) -> Self {
302 self.cmd = Some(cmd);
303 self
304 }
305
306 pub fn with_labels(mut self, labels: HashMap<String, String>) -> Self {
307 self.labels = labels;
308 self
309 }
310
311 pub fn with_restart_policy(mut self, policy: RestartPolicy) -> Self {
312 self.restart_policy = policy;
313 self
314 }
315
316 pub fn with_port(mut self, port: PortMapping) -> Self {
318 self.ports.push(port);
319 self
320 }
321
322 pub fn with_ports(mut self, ports: Vec<PortMapping>) -> Self {
324 self.ports = ports;
325 self
326 }
327
328 pub fn with_volume(mut self, volume: VolumeMount) -> Self {
330 self.volumes.push(volume);
331 self
332 }
333
334 pub fn with_volumes(mut self, volumes: Vec<VolumeMount>) -> Self {
336 self.volumes = volumes;
337 self
338 }
339
340 pub fn with_network(mut self, name: impl Into<String>) -> Self {
342 self.network = Some(name.into());
343 self
344 }
345
346 pub fn validate(&self) -> Result<(), SubstrateError> {
350 let mut host_binds = HashSet::new();
351 for port in &self.ports {
352 port.validate()?;
353 if let Some(host_port) = port.host_port {
354 if !host_binds.insert((port.host_ip, host_port, port.protocol)) {
355 return Err(SubstrateError::InvalidSpec(format!(
356 "host {}:{host_port}/{} is published more than once",
357 port.host_ip,
358 port.protocol.as_str()
359 )));
360 }
361 }
362 }
363
364 let mut targets = HashSet::new();
365 for volume in &self.volumes {
366 volume.validate()?;
367 if !targets.insert(volume.target.as_str()) {
368 return Err(SubstrateError::InvalidSpec(format!(
369 "container path {:?} has more than one volume mounted",
370 volume.target
371 )));
372 }
373 }
374
375 if let Some(network) = &self.network {
376 if network.is_empty() || network == "host" || network.starts_with("container:") {
381 return Err(SubstrateError::InvalidSpec(format!(
382 "network {network:?} is not supported: use a bridge name, `none`, or a \
383 user-defined network"
384 )));
385 }
386 }
387 Ok(())
388 }
389
390 fn to_docker_config(&self) -> Config<String> {
393 let mut env: Vec<String> = self.env.iter().map(|(k, v)| format!("{k}={v}")).collect();
394 env.sort();
395
396 let mut exposed_ports: HashMap<String, HashMap<(), ()>> = HashMap::new();
397 let mut port_bindings: HashMap<String, Option<Vec<PortBinding>>> = HashMap::new();
398 for port in &self.ports {
399 let key = port.docker_key();
400 exposed_ports.insert(key.clone(), HashMap::new());
401 port_bindings
402 .entry(key)
403 .or_insert_with(|| Some(Vec::new()))
404 .as_mut()
405 .expect("just inserted Some")
406 .push(PortBinding {
407 host_ip: Some(port.host_ip.to_string()),
408 host_port: port.host_port.map(|p| p.to_string()),
409 });
410 }
411
412 let host_config = HostConfig {
413 restart_policy: Some(self.restart_policy.into_docker()),
414 port_bindings: (!port_bindings.is_empty()).then_some(port_bindings),
415 mounts: (!self.volumes.is_empty())
416 .then(|| self.volumes.iter().map(VolumeMount::to_docker).collect()),
417 network_mode: self.network.clone(),
418 ..Default::default()
419 };
420
421 Config {
422 image: Some(self.image.clone()),
423 cmd: self.cmd.clone(),
424 env: Some(env),
425 labels: Some(self.labels.clone()),
426 exposed_ports: (!exposed_ports.is_empty()).then_some(exposed_ports),
427 host_config: Some(host_config),
428 ..Default::default()
429 }
430 }
431}
432
433pub struct Lifecycle {
437 docker: Docker,
438}
439
440impl Lifecycle {
441 pub async fn connect() -> Result<Self, SubstrateError> {
446 let docker = Docker::connect_with_local_defaults()?;
447 docker.ping().await?;
448 Ok(Self { docker })
449 }
450
451 pub async fn pull_image(&self, image: &str) -> Result<(), SubstrateError> {
455 let (from_image, tag) = split_image_ref(image);
456 let options = CreateImageOptions {
457 from_image,
458 tag,
459 ..Default::default()
460 };
461 let mut stream = self.docker.create_image(Some(options), None, None);
462 while let Some(item) = stream.next().await {
463 item?;
464 }
465 Ok(())
466 }
467
468 pub async fn run(&self, spec: &ContainerSpec) -> Result<String, SubstrateError> {
473 spec.validate()?;
474 let config = spec.to_docker_config();
475
476 let options = CreateContainerOptions {
477 name: spec.name.clone(),
478 platform: None,
479 };
480
481 let created = self.docker.create_container(Some(options), config).await?;
482
483 if let Err(e) = self
484 .docker
485 .start_container(&created.id, None::<StartContainerOptions<String>>)
486 .await
487 {
488 let _ = self.remove(&created.id, true).await;
489 return Err(e.into());
490 }
491
492 Ok(created.id)
493 }
494
495 pub async fn stop(&self, container_id: &str, timeout: Duration) -> Result<(), SubstrateError> {
498 let options = StopContainerOptions {
499 t: timeout.as_secs() as i64,
500 };
501 self.docker
502 .stop_container(container_id, Some(options))
503 .await?;
504 Ok(())
505 }
506
507 pub async fn remove(&self, container_id: &str, force: bool) -> Result<(), SubstrateError> {
510 let options = RemoveContainerOptions {
511 force,
512 ..Default::default()
513 };
514 match self
515 .docker
516 .remove_container(container_id, Some(options))
517 .await
518 {
519 Ok(()) => Ok(()),
520 Err(e) if is_not_found(&e) => Ok(()),
521 Err(e) => Err(e.into()),
522 }
523 }
524
525 pub async fn stop_and_remove(
530 &self,
531 container_id: &str,
532 timeout: Duration,
533 ) -> Result<(), SubstrateError> {
534 if let Err(e) = self.stop(container_id, timeout).await {
535 if !matches!(&e, SubstrateError::Docker(inner) if is_not_found(inner)) {
536 tracing::warn!(container_id, error = %e, "stop failed (continuing to remove)");
537 }
538 }
539 self.remove(container_id, true).await
540 }
541
542 pub async fn wait_until_running(
546 &self,
547 container_id: &str,
548 timeout: Duration,
549 ) -> Result<(), SubstrateError> {
550 const CONFIRM_GRACE: Duration = Duration::from_millis(500);
551 let deadline = tokio::time::Instant::now() + timeout;
552 loop {
553 let (running, exited, exit_code) = self.inspect_run_state(container_id).await?;
554 if exited {
555 return Err(SubstrateError::Resolve(format!(
556 "container {container_id} exited immediately (exit code {exit_code})"
557 )));
558 }
559 if running {
560 tokio::time::sleep(CONFIRM_GRACE).await;
565 let (still_running, exited_after, exit_code_after) =
566 self.inspect_run_state(container_id).await?;
567 if exited_after {
568 return Err(SubstrateError::Resolve(format!(
569 "container {container_id} exited shortly after starting (exit code {exit_code_after})"
570 )));
571 }
572 if still_running {
573 return Ok(());
574 }
575 }
579 if tokio::time::Instant::now() >= deadline {
580 return Err(SubstrateError::Timeout(format!(
581 "container {container_id} did not reach (and stay at) Running within {timeout:?}"
582 )));
583 }
584 tokio::time::sleep(Duration::from_millis(200)).await;
585 }
586 }
587
588 pub async fn ensure_network(&self, name: &str) -> Result<(), SubstrateError> {
594 let mut filters = HashMap::new();
595 filters.insert("name".to_string(), vec![name.to_string()]);
596 let existing = self
597 .docker
598 .list_networks(Some(ListNetworksOptions { filters }))
599 .await?;
600 if existing.iter().any(|n| n.name.as_deref() == Some(name)) {
601 return Ok(());
602 }
603 let options = CreateNetworkOptions {
604 name: name.to_string(),
605 driver: "bridge".to_string(),
606 ..Default::default()
607 };
608 match self.docker.create_network(options).await {
609 Ok(_) => Ok(()),
610 Err(e) if is_conflict(&e) => Ok(()),
611 Err(e) => Err(e.into()),
612 }
613 }
614
615 pub async fn published_host_port(
620 &self,
621 container_id: &str,
622 container_port: u16,
623 protocol: Protocol,
624 ) -> Result<Option<u16>, SubstrateError> {
625 let info = self.inspect(container_id).await?;
626 let key = format!("{container_port}/{}", protocol.as_str());
627 Ok(info
628 .network_settings
629 .and_then(|n| n.ports)
630 .and_then(|mut ports| ports.remove(&key))
631 .flatten()
632 .and_then(|bindings| {
633 bindings
634 .into_iter()
635 .find_map(|b| b.host_port.and_then(|p| p.parse::<u16>().ok()))
636 }))
637 }
638
639 pub async fn inspect(
642 &self,
643 container_id: &str,
644 ) -> Result<ContainerInspectResponse, SubstrateError> {
645 Ok(self.docker.inspect_container(container_id, None).await?)
646 }
647
648 async fn inspect_run_state(
651 &self,
652 container_id: &str,
653 ) -> Result<(bool, bool, i64), SubstrateError> {
654 let info = self.inspect(container_id).await?;
655 let running = info.state.as_ref().and_then(|s| s.running).unwrap_or(false);
656 let exited = info
657 .state
658 .as_ref()
659 .and_then(|s| s.status)
660 .map(|s| matches!(s, bollard::models::ContainerStateStatusEnum::EXITED))
661 .unwrap_or(false);
662 let exit_code = info.state.as_ref().and_then(|s| s.exit_code).unwrap_or(-1);
663 Ok((running, exited, exit_code))
664 }
665
666 pub async fn container_ref(&self, container_id: &str) -> Result<ContainerRef, SubstrateError> {
682 let info = self.inspect(container_id).await?;
683 let pid = info
684 .state
685 .as_ref()
686 .and_then(|s| s.pid)
687 .filter(|&p| p > 0)
688 .ok_or_else(|| {
689 SubstrateError::Resolve(format!(
690 "container {container_id} has no live pid (is it running?)"
691 ))
692 })? as u32;
693 let uid = read_uid_from_proc_status(pid)?;
694
695 let mut container_ref = ContainerRef::new(container_id.to_string(), uid);
696
697 match cgroup::resolve_cgroup_path(pid) {
698 Ok(path) => container_ref = container_ref.with_cgroup(path),
699 Err(e) => {
700 tracing::warn!(container_id, pid, error = %e, "could not resolve cgroup path");
701 }
702 }
703
704 match mount::resolve_mount_namespace(pid) {
705 Ok(ns) => container_ref = container_ref.with_mount_ns(ns),
706 Err(e) => {
707 tracing::warn!(
708 container_id, pid, error = %e,
709 "could not resolve mount namespace (needs root/CAP_SYS_PTRACE to read \
710 another user's /proc/<pid>/ns/mnt — see mount::resolve_mount_namespace's doc)"
711 );
712 }
713 }
714
715 Ok(container_ref)
716 }
717}
718
719fn read_uid_from_proc_status(pid: u32) -> Result<u32, SubstrateError> {
725 let raw = std::fs::read_to_string(format!("/proc/{pid}/status"))?;
726 let line = raw
727 .lines()
728 .find_map(|line| line.strip_prefix("Uid:"))
729 .ok_or_else(|| SubstrateError::Resolve(format!("no Uid line in /proc/{pid}/status")))?;
730 let real_uid = line.split_whitespace().next().ok_or_else(|| {
731 SubstrateError::Resolve(format!("malformed Uid line in /proc/{pid}/status"))
732 })?;
733 real_uid
734 .parse::<u32>()
735 .map_err(|_| SubstrateError::Resolve(format!("non-numeric uid in /proc/{pid}/status")))
736}
737
738fn is_conflict(e: &bollard::errors::Error) -> bool {
739 matches!(e, bollard::errors::Error::DockerResponseServerError { status_code, .. } if *status_code == 409)
740}
741
742fn is_not_found(e: &bollard::errors::Error) -> bool {
743 matches!(e, bollard::errors::Error::DockerResponseServerError { status_code, .. } if *status_code == 404)
744}
745
746fn split_image_ref(image: &str) -> (String, String) {
752 let last_slash = image.rfind('/').unwrap_or(0);
753 match image[last_slash..].rfind(':') {
754 Some(rel_idx) => {
755 let idx = last_slash + rel_idx;
756 (image[..idx].to_string(), image[idx + 1..].to_string())
757 }
758 None => (image.to_string(), "latest".to_string()),
759 }
760}
761
762#[cfg(test)]
763mod tests {
764 use super::*;
765
766 #[test]
767 fn split_image_ref_handles_a_plain_tag() {
768 assert_eq!(
769 split_image_ref("alpine:3"),
770 ("alpine".to_string(), "3".to_string())
771 );
772 }
773
774 #[test]
775 fn split_image_ref_defaults_to_latest() {
776 assert_eq!(
777 split_image_ref("alpine"),
778 ("alpine".to_string(), "latest".to_string())
779 );
780 }
781
782 #[test]
783 fn split_image_ref_handles_a_registry_host_with_a_port() {
784 assert_eq!(
785 split_image_ref("registry.example.com:5000/substrate-kit-test:latest"),
786 (
787 "registry.example.com:5000/substrate-kit-test".to_string(),
788 "latest".to_string()
789 )
790 );
791 }
792
793 #[test]
794 fn restart_policy_never_maps_to_dockers_no_policy() {
795 let policy = RestartPolicy::Never.into_docker();
796 assert_eq!(policy.name, Some(RestartPolicyNameEnum::NO));
797 assert_eq!(policy.maximum_retry_count, None);
798 }
799
800 #[test]
801 fn restart_policy_on_failure_carries_its_retry_count() {
802 let policy = RestartPolicy::OnFailure { max_retries: 3 }.into_docker();
803 assert_eq!(policy.name, Some(RestartPolicyNameEnum::ON_FAILURE));
804 assert_eq!(policy.maximum_retry_count, Some(3));
805 }
806
807 #[test]
808 fn container_spec_builders_set_expected_fields() {
809 let mut env = HashMap::new();
810 env.insert("FOO".to_string(), "bar".to_string());
811 let spec = ContainerSpec::new("test-name", "alpine:3")
812 .with_env(env.clone())
813 .with_cmd(vec!["sleep".to_string(), "1".to_string()])
814 .with_restart_policy(RestartPolicy::Always);
815
816 assert_eq!(spec.name, "test-name");
817 assert_eq!(spec.image, "alpine:3");
818 assert_eq!(spec.env, env);
819 assert_eq!(spec.cmd, Some(vec!["sleep".to_string(), "1".to_string()]));
820 assert_eq!(spec.restart_policy, RestartPolicy::Always);
821 }
822
823 #[test]
824 fn a_spec_with_no_networking_or_volumes_produces_the_pre_clus55_config() {
825 let cfg = ContainerSpec::new("n", "alpine:3").to_docker_config();
826 let hc = cfg.host_config.unwrap();
827 assert!(cfg.exposed_ports.is_none());
828 assert!(hc.port_bindings.is_none());
829 assert!(hc.mounts.is_none());
830 assert!(hc.network_mode.is_none());
831 }
832
833 #[test]
834 fn ports_default_to_loopback_and_map_to_docker_port_bindings() {
835 let spec = ContainerSpec::new("n", "alpine:3")
836 .with_port(PortMapping::tcp(8096).with_host_port(8096))
837 .with_port(PortMapping::udp(5353));
838 let cfg = spec.to_docker_config();
839 let exposed = cfg.exposed_ports.unwrap();
840 assert!(exposed.contains_key("8096/tcp") && exposed.contains_key("5353/udp"));
841 let bindings = cfg.host_config.unwrap().port_bindings.unwrap();
842 let tcp = bindings["8096/tcp"].as_ref().unwrap();
843 assert_eq!(tcp.len(), 1);
844 assert_eq!(tcp[0].host_ip.as_deref(), Some("127.0.0.1"));
845 assert_eq!(tcp[0].host_port.as_deref(), Some("8096"));
846 let udp = bindings["5353/udp"].as_ref().unwrap();
847 assert_eq!(
848 udp[0].host_port, None,
849 "None must leave allocation to dockerd"
850 );
851 }
852
853 #[test]
854 fn the_same_container_port_on_two_interfaces_yields_two_bindings() {
855 let spec = ContainerSpec::new("n", "alpine:3")
856 .with_port(PortMapping::tcp(80).with_host_port(8080))
857 .with_port(
858 PortMapping::tcp(80)
859 .with_host_port(8080)
860 .with_host_ip("::1".parse().unwrap()),
861 );
862 spec.validate().unwrap();
863 let bindings = spec
864 .to_docker_config()
865 .host_config
866 .unwrap()
867 .port_bindings
868 .unwrap();
869 assert_eq!(bindings["80/tcp"].as_ref().unwrap().len(), 2);
870 }
871
872 #[test]
873 fn network_and_volumes_reach_the_host_config() {
874 let spec = ContainerSpec::new("n", "alpine:3")
875 .with_network("service-net")
876 .with_volume(VolumeMount::named("cfg", "/config"));
877 let hc = spec.to_docker_config().host_config.unwrap();
878 assert_eq!(hc.network_mode.as_deref(), Some("service-net"));
879 let mounts = hc.mounts.unwrap();
880 assert_eq!(mounts.len(), 1);
881 assert_eq!(mounts[0].source.as_deref(), Some("cfg"));
882 }
883
884 #[test]
885 fn validate_rejects_bad_ports_volumes_and_networks() {
886 let bad: Vec<ContainerSpec> = vec![
887 ContainerSpec::new("n", "i").with_port(PortMapping::tcp(0)),
888 ContainerSpec::new("n", "i").with_port(PortMapping::tcp(80).with_host_port(0)),
889 ContainerSpec::new("n", "i")
890 .with_port(PortMapping::tcp(80).with_host_port(8080))
891 .with_port(PortMapping::tcp(81).with_host_port(8080)),
892 ContainerSpec::new("n", "i")
893 .with_volume(VolumeMount::named("a", "/d"))
894 .with_volume(VolumeMount::named("b", "/d")),
895 ContainerSpec::new("n", "i").with_volume(VolumeMount::named("/etc", "/d")),
896 ContainerSpec::new("n", "i").with_network(""),
897 ContainerSpec::new("n", "i").with_network("host"),
898 ContainerSpec::new("n", "i").with_network("container:abc"),
899 ];
900 for (i, spec) in bad.iter().enumerate() {
901 assert!(
902 matches!(spec.validate(), Err(SubstrateError::InvalidSpec(_))),
903 "case {i} should be rejected"
904 );
905 }
906 }
907
908 #[test]
909 fn same_host_port_on_different_protocols_is_allowed() {
910 ContainerSpec::new("n", "i")
911 .with_port(PortMapping::tcp(53).with_host_port(5353))
912 .with_port(PortMapping::udp(53).with_host_port(5353))
913 .validate()
914 .unwrap();
915 }
916
917 #[test]
918 fn debug_output_never_contains_env_values() {
919 let mut env = HashMap::new();
920 env.insert("API_TOKEN".to_string(), "s3cr3t-value".to_string());
921 let spec = ContainerSpec::new("n", "alpine:3").with_env(env);
922 let rendered = format!("{spec:?}");
923 assert!(rendered.contains("API_TOKEN"));
924 assert!(!rendered.contains("s3cr3t-value"), "{rendered}");
925 }
926}
927
928#[cfg(test)]
955mod live {
956 use super::*;
957
958 async fn require_docker() -> Lifecycle {
962 match Lifecycle::connect().await {
963 Ok(lifecycle) => lifecycle,
964 Err(e) => panic!("these live tests need a reachable Docker daemon: {e}"),
965 }
966 }
967
968 fn test_name(prefix: &str) -> String {
969 format!(
970 "{prefix}-{}",
971 std::time::SystemTime::now()
972 .duration_since(std::time::UNIX_EPOCH)
973 .unwrap()
974 .as_millis()
975 )
976 }
977
978 fn managed_labels() -> HashMap<String, String> {
979 let mut labels = HashMap::new();
980 labels.insert(LABEL_MANAGED.to_string(), "true".to_string());
981 labels
982 }
983
984 #[tokio::test]
990 #[ignore = "needs a reachable Docker daemon; run with `cargo test -- --ignored`"]
991 #[cfg(target_os = "linux")]
992 async fn create_start_health_container_ref_stop_round_trips_for_real() {
993 let lifecycle = require_docker().await;
994 let image = "alpine:3";
995 lifecycle.pull_image(image).await.expect("pull_image");
996
997 let name = test_name("substrate-lifecycle-live");
998 let spec = ContainerSpec::new(&name, image)
999 .with_cmd(vec!["sleep".to_string(), "300".to_string()])
1000 .with_labels(managed_labels());
1001
1002 let container_id = lifecycle.run(&spec).await.expect("run");
1003
1004 lifecycle
1005 .wait_until_running(&container_id, Duration::from_secs(15))
1006 .await
1007 .expect("wait_until_running");
1008
1009 let container_ref = lifecycle
1010 .container_ref(&container_id)
1011 .await
1012 .expect("container_ref");
1013
1014 assert_eq!(container_ref.container_id, container_id);
1015 assert_eq!(container_ref.uid, 0);
1018
1019 let cgroup_path = container_ref
1020 .cgroup
1021 .as_deref()
1022 .expect("cgroup path must resolve — /proc/<pid>/cgroup is world-readable");
1023 assert!(
1024 cgroup_path.starts_with(cgroup::CGROUP_V2_MOUNT),
1025 "cgroup path: {cgroup_path}"
1026 );
1027
1028 if let Some(mount_ns) = &container_ref.mount_ns {
1029 assert!(
1030 mount_ns.starts_with("mnt:["),
1031 "mount namespace handle: {mount_ns}"
1032 );
1033 } else {
1034 eprintln!(
1035 "NOTE: mount_ns did not resolve (expected without root/CAP_SYS_PTRACE against \
1036 a root-owned container — see mount::resolve_mount_namespace's doc)"
1037 );
1038 }
1039
1040 lifecycle
1041 .stop_and_remove(&container_id, Duration::from_secs(5))
1042 .await
1043 .expect("stop_and_remove");
1044 }
1045
1046 #[tokio::test]
1050 #[ignore = "needs a reachable Docker daemon; run with `cargo test -- --ignored`"]
1051 async fn wait_until_running_fails_fast_for_a_container_that_exits_immediately() {
1052 let lifecycle = require_docker().await;
1053 let image = "alpine:3";
1054 lifecycle.pull_image(image).await.expect("pull_image");
1055
1056 let name = test_name("substrate-lifecycle-live-exit");
1057 let spec = ContainerSpec::new(&name, image)
1058 .with_cmd(vec![
1059 "sh".to_string(),
1060 "-c".to_string(),
1061 "exit 0".to_string(),
1062 ])
1063 .with_labels(managed_labels());
1064
1065 let container_id = lifecycle.run(&spec).await.expect("run");
1066
1067 let result = tokio::time::timeout(
1068 Duration::from_secs(10),
1069 lifecycle.wait_until_running(&container_id, Duration::from_secs(30)),
1070 )
1071 .await
1072 .expect("wait_until_running must return well before its own 30s timeout, not hang");
1073
1074 assert!(
1075 result.is_err(),
1076 "an immediately-exited container must be reported as a failure"
1077 );
1078
1079 lifecycle
1080 .stop_and_remove(&container_id, Duration::from_secs(5))
1081 .await
1082 .expect("stop_and_remove");
1083 }
1084
1085 #[tokio::test]
1090 #[ignore = "needs a reachable Docker daemon; run with `cargo test -- --ignored`"]
1091 async fn restart_policy_is_applied_and_visible_in_dockerds_own_inspect() {
1092 let lifecycle = require_docker().await;
1093 let image = "alpine:3";
1094 lifecycle.pull_image(image).await.expect("pull_image");
1095
1096 let name = test_name("substrate-lifecycle-live-restart-policy");
1097 let spec = ContainerSpec::new(&name, image)
1098 .with_cmd(vec!["sleep".to_string(), "300".to_string()])
1099 .with_labels(managed_labels())
1100 .with_restart_policy(RestartPolicy::OnFailure { max_retries: 2 });
1101
1102 let container_id = lifecycle.run(&spec).await.expect("run");
1103
1104 let info = lifecycle.inspect(&container_id).await.expect("inspect");
1105 let restart_policy = info
1106 .host_config
1107 .as_ref()
1108 .and_then(|hc| hc.restart_policy.as_ref())
1109 .expect("dockerd must echo back a restart policy");
1110 assert_eq!(restart_policy.name, Some(RestartPolicyNameEnum::ON_FAILURE));
1111 assert_eq!(restart_policy.maximum_retry_count, Some(2));
1112
1113 lifecycle
1114 .stop_and_remove(&container_id, Duration::from_secs(5))
1115 .await
1116 .expect("stop_and_remove");
1117 }
1118
1119 #[tokio::test]
1127 #[ignore = "needs a reachable Docker daemon; run with `cargo test -- --ignored`"]
1128 #[cfg(target_os = "linux")]
1129 async fn mount_namespace_round_trips_when_privileged() {
1130 let lifecycle = require_docker().await;
1131 let image = "alpine:3";
1132 lifecycle.pull_image(image).await.expect("pull_image");
1133
1134 let name = test_name("substrate-lifecycle-live-mount-ns");
1135 let spec = ContainerSpec::new(&name, image)
1136 .with_cmd(vec!["sleep".to_string(), "300".to_string()])
1137 .with_labels(managed_labels());
1138
1139 let container_id = lifecycle.run(&spec).await.expect("run");
1140 lifecycle
1141 .wait_until_running(&container_id, Duration::from_secs(15))
1142 .await
1143 .expect("wait_until_running");
1144
1145 let container_ref = lifecycle
1146 .container_ref(&container_id)
1147 .await
1148 .expect("container_ref");
1149
1150 let require_root_ns = std::env::var_os("SUBSTRATE_REQUIRE_ROOT_NS").is_some();
1151 match (&container_ref.mount_ns, require_root_ns) {
1152 (Some(ns), _) => assert!(ns.starts_with("mnt:["), "mount namespace handle: {ns}"),
1153 (None, true) => panic!(
1154 "SUBSTRATE_REQUIRE_ROOT_NS set but mount_ns did not resolve — \
1155 run as root/with CAP_SYS_PTRACE"
1156 ),
1157 (None, false) => eprintln!(
1158 "SKIPPED (assertion only): mount_ns did not resolve without root/CAP_SYS_PTRACE; \
1159 set SUBSTRATE_REQUIRE_ROOT_NS=1 under sudo to hard-fail and prove the privileged path"
1160 ),
1161 }
1162
1163 lifecycle
1164 .stop_and_remove(&container_id, Duration::from_secs(5))
1165 .await
1166 .expect("stop_and_remove");
1167 }
1168
1169 async fn wait_exit_code(lifecycle: &Lifecycle, id: &str) -> i64 {
1173 for _ in 0..100 {
1174 let info = lifecycle.inspect(id).await.expect("inspect");
1175 if let Some(state) = info.state {
1176 if state.running != Some(true) && state.status.is_some() {
1177 use bollard::models::ContainerStateStatusEnum as S;
1178 if matches!(state.status, Some(S::EXITED) | Some(S::DEAD)) {
1179 return state.exit_code.unwrap_or(-1);
1180 }
1181 }
1182 }
1183 tokio::time::sleep(Duration::from_millis(200)).await;
1184 }
1185 panic!("container {id} did not exit in time");
1186 }
1187
1188 #[tokio::test]
1191 #[ignore = "needs a reachable Docker daemon; run with `cargo test -- --ignored`"]
1192 async fn ephemeral_loopback_port_is_published_and_readable() {
1193 let lifecycle = require_docker().await;
1194 let image = "alpine:3";
1195 lifecycle.pull_image(image).await.expect("pull_image");
1196
1197 let name = test_name("substrate-lifecycle-live-port");
1198 let spec = ContainerSpec::new(&name, image)
1199 .with_cmd(vec!["sleep".to_string(), "300".to_string()])
1200 .with_labels(managed_labels())
1201 .with_port(PortMapping::tcp(8080));
1202 let id = lifecycle.run(&spec).await.expect("run");
1203
1204 let host_port = lifecycle
1205 .published_host_port(&id, 8080, Protocol::Tcp)
1206 .await
1207 .expect("published_host_port")
1208 .expect("dockerd must have allocated a host port");
1209 assert!(host_port > 0);
1210
1211 let info = lifecycle.inspect(&id).await.expect("inspect");
1212 let binding = info
1213 .host_config
1214 .and_then(|hc| hc.port_bindings)
1215 .and_then(|mut pb| pb.remove("8080/tcp"))
1216 .flatten()
1217 .expect("binding echoed back");
1218 assert_eq!(binding[0].host_ip.as_deref(), Some("127.0.0.1"));
1219
1220 assert_eq!(
1221 lifecycle
1222 .published_host_port(&id, 9999, Protocol::Tcp)
1223 .await
1224 .unwrap(),
1225 None
1226 );
1227
1228 lifecycle
1229 .stop_and_remove(&id, Duration::from_secs(5))
1230 .await
1231 .expect("stop_and_remove");
1232 }
1233
1234 #[tokio::test]
1237 #[ignore = "needs a reachable Docker daemon; run with `cargo test -- --ignored`"]
1238 async fn pinned_one_to_one_port_accepts_a_tcp_connection() {
1239 let lifecycle = require_docker().await;
1240 let image = "alpine:3";
1241 lifecycle.pull_image(image).await.expect("pull_image");
1242
1243 let free = std::net::TcpListener::bind("127.0.0.1:0")
1245 .unwrap()
1246 .local_addr()
1247 .unwrap()
1248 .port();
1249
1250 let name = test_name("substrate-lifecycle-live-pinned");
1251 let spec = ContainerSpec::new(&name, image)
1252 .with_cmd(vec![
1253 "sh".to_string(),
1254 "-c".to_string(),
1255 format!("nc -lk -p {free} -e /bin/cat"),
1256 ])
1257 .with_labels(managed_labels())
1258 .with_port(PortMapping::tcp(free).with_host_port(free));
1259 let id = lifecycle.run(&spec).await.expect("run");
1260 lifecycle
1261 .wait_until_running(&id, Duration::from_secs(15))
1262 .await
1263 .expect("wait_until_running");
1264
1265 assert_eq!(
1266 lifecycle
1267 .published_host_port(&id, free, Protocol::Tcp)
1268 .await
1269 .unwrap(),
1270 Some(free)
1271 );
1272 let mut connected = false;
1275 for _ in 0..25 {
1276 if tokio::net::TcpStream::connect(("127.0.0.1", free))
1277 .await
1278 .is_ok()
1279 {
1280 connected = true;
1281 break;
1282 }
1283 tokio::time::sleep(Duration::from_millis(200)).await;
1284 }
1285 assert!(
1286 connected,
1287 "could not connect to published loopback port {free}"
1288 );
1289
1290 lifecycle
1291 .stop_and_remove(&id, Duration::from_secs(5))
1292 .await
1293 .expect("stop_and_remove");
1294 }
1295
1296 #[tokio::test]
1299 #[ignore = "needs a reachable Docker daemon; run with `cargo test -- --ignored`"]
1300 async fn containers_on_a_user_defined_network_resolve_each_other_by_name() {
1301 let lifecycle = require_docker().await;
1302 let image = "alpine:3";
1303 lifecycle.pull_image(image).await.expect("pull_image");
1304
1305 let net = test_name("substrate-live-net");
1306 lifecycle
1307 .ensure_network(&net)
1308 .await
1309 .expect("ensure_network");
1310 lifecycle
1311 .ensure_network(&net)
1312 .await
1313 .expect("ensure_network must be idempotent");
1314
1315 let server_name = test_name("substrate-live-net-server");
1316 let server = lifecycle
1317 .run(
1318 &ContainerSpec::new(&server_name, image)
1319 .with_cmd(vec!["sleep".to_string(), "300".to_string()])
1320 .with_labels(managed_labels())
1321 .with_network(&net),
1322 )
1323 .await
1324 .expect("run server");
1325 lifecycle
1326 .wait_until_running(&server, Duration::from_secs(15))
1327 .await
1328 .expect("server running");
1329
1330 let client_name = test_name("substrate-live-net-client");
1331 let client = lifecycle
1332 .run(
1333 &ContainerSpec::new(&client_name, image)
1334 .with_cmd(vec![
1335 "sh".to_string(),
1336 "-c".to_string(),
1337 format!("getent hosts {server_name}"),
1338 ])
1339 .with_labels(managed_labels())
1340 .with_network(&net),
1341 )
1342 .await
1343 .expect("run client");
1344 let code = wait_exit_code(&lifecycle, &client).await;
1345 assert_eq!(code, 0, "client could not resolve the server by name");
1346
1347 let info = lifecycle.inspect(&server).await.expect("inspect");
1348 let networks = info.network_settings.and_then(|n| n.networks).unwrap();
1349 assert!(networks.contains_key(&net), "server not attached to {net}");
1350
1351 for id in [&client, &server] {
1352 lifecycle
1353 .stop_and_remove(id, Duration::from_secs(5))
1354 .await
1355 .expect("stop_and_remove");
1356 }
1357 lifecycle
1358 .docker
1359 .remove_network(&net)
1360 .await
1361 .expect("remove_network");
1362 }
1363
1364 #[tokio::test]
1368 #[ignore = "needs a reachable Docker daemon; run with `cargo test -- --ignored`"]
1369 async fn named_volume_persists_across_containers_and_read_only_is_enforced() {
1370 let lifecycle = require_docker().await;
1371 let image = "alpine:3";
1372 lifecycle.pull_image(image).await.expect("pull_image");
1373
1374 let vol = test_name("substrate-live-vol");
1375
1376 let run_sh = |label: &'static str, script: &'static str, ro: bool| {
1377 let lifecycle = &lifecycle;
1378 let vol = vol.clone();
1379 async move {
1380 let mount = VolumeMount::named(&vol, "/data");
1381 let mount = if ro { mount.read_only() } else { mount };
1382 let spec = ContainerSpec::new(test_name(label), image)
1383 .with_cmd(vec!["sh".to_string(), "-c".to_string(), script.to_string()])
1384 .with_labels(managed_labels())
1385 .with_volume(mount);
1386 let id = lifecycle.run(&spec).await.expect("run");
1387 let code = wait_exit_code(lifecycle, &id).await;
1388 let info = lifecycle.inspect(&id).await.expect("inspect");
1389 let mounts = info.mounts.unwrap_or_default();
1390 lifecycle.remove(&id, true).await.expect("remove");
1391 (code, mounts)
1392 }
1393 };
1394
1395 let (code, mounts) = run_sh("substrate-live-vol-w", "echo hello > /data/f", false).await;
1396 assert_eq!(code, 0);
1397 assert_eq!(mounts.len(), 1);
1398 assert_eq!(
1399 mounts[0].typ,
1400 Some(bollard::models::MountPointTypeEnum::VOLUME)
1401 );
1402 assert_eq!(mounts[0].name.as_deref(), Some(vol.as_str()));
1403
1404 let (code, _) = run_sh("substrate-live-vol-r", "grep -q hello /data/f", false).await;
1405 assert_eq!(code, 0, "data written by the first container must persist");
1406
1407 let (code, mounts) = run_sh("substrate-live-vol-ro", "echo x > /data/g", true).await;
1408 assert_ne!(code, 0, "write to a read-only volume must fail");
1409 assert_eq!(mounts[0].rw, Some(false));
1410
1411 lifecycle
1412 .docker
1413 .remove_volume(&vol, None)
1414 .await
1415 .expect("remove_volume");
1416 }
1417
1418 #[tokio::test]
1420 #[ignore = "needs a reachable Docker daemon; run with `cargo test -- --ignored`"]
1421 async fn an_invalid_spec_is_rejected_without_creating_a_container() {
1422 let lifecycle = require_docker().await;
1423 let name = test_name("substrate-lifecycle-live-invalid");
1424 let spec =
1425 ContainerSpec::new(&name, "alpine:3").with_volume(VolumeMount::named("/etc", "/d"));
1426 let err = lifecycle.run(&spec).await.expect_err("must be rejected");
1427 assert!(matches!(err, SubstrateError::InvalidSpec(_)));
1428 assert!(
1429 lifecycle.inspect(&name).await.is_err(),
1430 "no container may have been created"
1431 );
1432 }
1433}