Skip to main content

substrate_kit/
lifecycle.rs

1//! `lifecycle` — Container / process lifecycle over dockerd — create,
2//! start, stop, health, restart policy.
3//!
4//! This is the piece `principal.rs`'s own module doc flags as deferred:
5//! "\[r\]esolving a real `ContainerRef` from a running container is
6//! `lifecycle`'s job, in a later slice." [`Lifecycle::container_ref`] is
7//! that job — it inspects a running container over the real dockerd
8//! Engine API, resolves its uid from `/proc`, and composes `cgroup`'s and
9//! `mount`'s own resolvers to fill in `ContainerRef::cgroup`/`mount_ns`,
10//! so `principal::embody` can be called with real data end-to-end.
11//!
12//! **Scope boundary (hard constraint, same as every substrate-kit
13//! module):** this is pure OS mechanics over dockerd. It never constructs
14//! a `ServicePrincipal`, never decides which principal a container
15//! belongs to, and never makes an authorization decision — see
16//! `principal.rs`'s module doc for the full hard-constraint statement.
17//!
18//! ## What "health" means here, deliberately, and what it doesn't
19//!
20//! [`Lifecycle::wait_until_running`] confirms a container reaches, and
21//! *stays at*, Docker's own `State.Running: true` — a single "is it
22//! Running" read is a TOCTOU race against a container whose process
23//! crashes moments after starting, so this re-checks after a short grace
24//! period before declaring success. This does **not** prove the service
25//! *inside* the container is ready to take traffic (no HTTP readiness
26//! probe, no Docker `HEALTHCHECK` wiring) — that is a real, separate
27//! concern for whatever calls this crate, not this slice's job. This
28//! keeps health at the layer dockerd itself reports.
29//!
30//! ## Restart-policy handling
31//!
32//! [`RestartPolicy`] is a thin, OS-mechanics-only wrapper over dockerd's
33//! own restart-policy enum (`bollard::models::RestartPolicyNameEnum`) —
34//! it carries no meaning beyond "what dockerd should do when this
35//! container's process exits," the same as `docker run --restart`.
36//!
37//! ## Networking and volumes
38//!
39//! [`ContainerSpec`] carries three additive, opt-in fields:
40//!
41//! - [`ContainerSpec::ports`] — [`PortMapping`]s published to the host.
42//!   **Loopback-only by default** (`127.0.0.1`), so a service is never reachable
43//!   from the network by accident; exposing a port on another interface is an explicit
44//!   [`PortMapping::with_host_ip`].
45//! - [`ContainerSpec::network`] — attach to a user-defined bridge network
46//!   (so containers resolve each other by name via dockerd's embedded DNS).
47//!   [`Lifecycle::ensure_network`] creates it idempotently.
48//! - [`ContainerSpec::volumes`] — named Docker volumes
49//!   ([`crate::mount::VolumeMount`]), optionally plugin-backed. No host bind
50//!   mounts; see that type's doc for why.
51//!
52//! All three default to empty/`None`, which produces exactly the container
53//! this module created before.
54//!
55//! ## Environment and credentials
56//!
57//! `ContainerSpec::env` is **not a credential channel.** Anything in it is
58//! readable by `docker inspect` and shows up in dockerd's own state. The
59//! rule is that no credential travels in an environment variable or a
60//! unit file. This crate therefore does not accept, read or log
61//! credentials: it never reads `std::env`, and `ContainerSpec`'s `Debug`
62//! impl prints env *keys only*, so a `tracing::debug!(?spec)` cannot leak a
63//! value. A credential-delivery seam (secret files mounted read-only,
64//! sourced from `$CREDENTIALS_DIRECTORY` by the caller) and registry-pull
65//! credentials are designed but **not implemented** here; both
66//! need a security review first.
67//!
68//! ## Deliberately out of scope, not oversights
69//!
70//! - **No registry-credential handling** in [`Lifecycle::pull_image`] —
71//!   every image this crate's own tests pull is public
72//!   (`docker.io` short names); a private-registry credential seam is
73//!   real, deferred follow-up.
74//! - **No Docker-socket hardening.** This talks to the raw local
75//!   dockerd socket — `socket.rs`'s job is AF_UNIX hardening for
76//!   *services running on* the substrate, not this crate's own dockerd
77//!   client connection.
78
79use std::collections::{HashMap, HashSet};
80use std::net::{IpAddr, Ipv4Addr};
81use std::time::Duration;
82
83use bollard::container::{
84    Config, CreateContainerOptions, RemoveContainerOptions, StartContainerOptions,
85    StopContainerOptions,
86};
87use bollard::image::CreateImageOptions;
88use bollard::models::{
89    ContainerInspectResponse, HostConfig, PortBinding, RestartPolicy as DockerRestartPolicy,
90    RestartPolicyNameEnum,
91};
92use bollard::network::{CreateNetworkOptions, ListNetworksOptions};
93use bollard::Docker;
94use futures_util::StreamExt;
95
96use crate::cgroup;
97use crate::error::SubstrateError;
98use crate::mount::{self, VolumeMount};
99use crate::principal::ContainerRef;
100
101/// Every container this module creates carries this label, purely so a
102/// caller (or this crate's own tests) can find/clean up what it started
103/// without keeping its own in-memory bookkeeping.
104pub const LABEL_MANAGED: &str = "substrate.lifecycle.managed";
105
106/// dockerd's own restart-policy semantics, wrapped only so callers don't
107/// need to reach into `bollard`'s model types directly. Carries no
108/// meaning beyond what dockerd itself gives `--restart` — see this
109/// module's doc.
110#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
111pub enum RestartPolicy {
112    /// dockerd's default: never automatically restart.
113    #[default]
114    Never,
115    /// Always restart, regardless of exit status.
116    Always,
117    /// Restart unless a human/operator explicitly stopped the container.
118    UnlessStopped,
119    /// Restart only on a non-zero exit, up to `max_retries` times.
120    OnFailure { max_retries: i64 },
121}
122
123impl RestartPolicy {
124    fn into_docker(self) -> DockerRestartPolicy {
125        match self {
126            RestartPolicy::Never => DockerRestartPolicy {
127                name: Some(RestartPolicyNameEnum::NO),
128                maximum_retry_count: None,
129            },
130            RestartPolicy::Always => DockerRestartPolicy {
131                name: Some(RestartPolicyNameEnum::ALWAYS),
132                maximum_retry_count: None,
133            },
134            RestartPolicy::UnlessStopped => DockerRestartPolicy {
135                name: Some(RestartPolicyNameEnum::UNLESS_STOPPED),
136                maximum_retry_count: None,
137            },
138            RestartPolicy::OnFailure { max_retries } => DockerRestartPolicy {
139                name: Some(RestartPolicyNameEnum::ON_FAILURE),
140                maximum_retry_count: Some(max_retries),
141            },
142        }
143    }
144}
145
146/// Transport protocol of a published port.
147#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
148pub enum Protocol {
149    #[default]
150    Tcp,
151    Udp,
152}
153
154impl Protocol {
155    fn as_str(self) -> &'static str {
156        match self {
157            Protocol::Tcp => "tcp",
158            Protocol::Udp => "udp",
159        }
160    }
161}
162
163/// One container port published to the host (`docker run -p`).
164///
165/// **Safe by default:** bound to loopback (`127.0.0.1`). The host port is
166/// `None` (dockerd picks a free ephemeral port; read it back with
167/// [`Lifecycle::published_host_port`]) unless [`PortMapping::with_host_port`]
168/// pins it. A 1:1 mapping is
169/// `PortMapping::tcp(p).with_host_port(p)`.
170#[derive(Debug, Clone, PartialEq, Eq)]
171pub struct PortMapping {
172    pub container_port: u16,
173    /// `None` lets dockerd allocate a free host port.
174    pub host_port: Option<u16>,
175    pub host_ip: IpAddr,
176    pub protocol: Protocol,
177}
178
179impl PortMapping {
180    /// Publish TCP `container_port` on loopback with a dockerd-allocated
181    /// host port.
182    pub fn tcp(container_port: u16) -> Self {
183        Self {
184            container_port,
185            host_port: None,
186            host_ip: IpAddr::V4(Ipv4Addr::LOCALHOST),
187            protocol: Protocol::Tcp,
188        }
189    }
190
191    /// As [`PortMapping::tcp`] but UDP.
192    pub fn udp(container_port: u16) -> Self {
193        Self {
194            protocol: Protocol::Udp,
195            ..Self::tcp(container_port)
196        }
197    }
198
199    pub fn with_host_port(mut self, host_port: u16) -> Self {
200        self.host_port = Some(host_port);
201        self
202    }
203
204    /// Bind on a specific host interface. `0.0.0.0` exposes the port to the
205    /// LAN — an explicit, reviewable choice, never the default.
206    pub fn with_host_ip(mut self, host_ip: IpAddr) -> Self {
207        self.host_ip = host_ip;
208        self
209    }
210
211    /// Docker's `port/proto` key, e.g. `8096/tcp`.
212    fn docker_key(&self) -> String {
213        format!("{}/{}", self.container_port, self.protocol.as_str())
214    }
215
216    fn validate(&self) -> Result<(), SubstrateError> {
217        if self.container_port == 0 {
218            return Err(SubstrateError::InvalidSpec(
219                "port mapping has container_port 0".to_string(),
220            ));
221        }
222        if self.host_port == Some(0) {
223            return Err(SubstrateError::InvalidSpec(format!(
224                "port mapping for {} has host_port 0; use None to let dockerd allocate",
225                self.docker_key()
226            )));
227        }
228        Ok(())
229    }
230}
231
232/// What to run.
233///
234/// `#[non_exhaustive]`: build with [`ContainerSpec::new`] and the `with_*`
235/// builders so fields can keep being added without breaking callers.
236/// `Debug` is hand-written and prints `env` **keys only** (see the module
237/// doc on credentials).
238#[derive(Clone)]
239#[non_exhaustive]
240pub struct ContainerSpec {
241    /// Docker container name — must be unique per host.
242    pub name: String,
243    pub image: String,
244    /// Environment variables. **Not a credential channel**: values are
245    /// visible to `docker inspect`. See the module doc.
246    pub env: HashMap<String, String>,
247    /// Override the image's own `ENTRYPOINT`/`CMD`. `None` leaves the
248    /// image's own default command untouched.
249    pub cmd: Option<Vec<String>>,
250    pub labels: HashMap<String, String>,
251    pub restart_policy: RestartPolicy,
252    /// Ports to publish to the host. Empty publishes nothing.
253    pub ports: Vec<PortMapping>,
254    /// Named volumes to attach. Empty attaches nothing.
255    pub volumes: Vec<VolumeMount>,
256    /// A user-defined Docker network to attach to instead of the default
257    /// `bridge` (`HostConfig.NetworkMode`); containers on the same
258    /// user-defined network resolve each other by container name. `None`
259    /// keeps dockerd's default. Create it with [`Lifecycle::ensure_network`].
260    pub network: Option<String>,
261}
262
263impl std::fmt::Debug for ContainerSpec {
264    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
265        let mut env_keys: Vec<&String> = self.env.keys().collect();
266        env_keys.sort();
267        f.debug_struct("ContainerSpec")
268            .field("name", &self.name)
269            .field("image", &self.image)
270            .field("env_keys", &env_keys)
271            .field("cmd", &self.cmd)
272            .field("labels", &self.labels)
273            .field("restart_policy", &self.restart_policy)
274            .field("ports", &self.ports)
275            .field("volumes", &self.volumes)
276            .field("network", &self.network)
277            .finish()
278    }
279}
280
281impl ContainerSpec {
282    pub fn new(name: impl Into<String>, image: impl Into<String>) -> Self {
283        Self {
284            name: name.into(),
285            image: image.into(),
286            env: HashMap::new(),
287            cmd: None,
288            labels: HashMap::new(),
289            restart_policy: RestartPolicy::default(),
290            ports: Vec::new(),
291            volumes: Vec::new(),
292            network: None,
293        }
294    }
295
296    pub fn with_env(mut self, env: HashMap<String, String>) -> Self {
297        self.env = env;
298        self
299    }
300
301    pub fn with_cmd(mut self, cmd: Vec<String>) -> Self {
302        self.cmd = Some(cmd);
303        self
304    }
305
306    pub fn with_labels(mut self, labels: HashMap<String, String>) -> Self {
307        self.labels = labels;
308        self
309    }
310
311    pub fn with_restart_policy(mut self, policy: RestartPolicy) -> Self {
312        self.restart_policy = policy;
313        self
314    }
315
316    /// Publish one more port.
317    pub fn with_port(mut self, port: PortMapping) -> Self {
318        self.ports.push(port);
319        self
320    }
321
322    /// Replace the published ports.
323    pub fn with_ports(mut self, ports: Vec<PortMapping>) -> Self {
324        self.ports = ports;
325        self
326    }
327
328    /// Attach one more named volume.
329    pub fn with_volume(mut self, volume: VolumeMount) -> Self {
330        self.volumes.push(volume);
331        self
332    }
333
334    /// Replace the attached volumes.
335    pub fn with_volumes(mut self, volumes: Vec<VolumeMount>) -> Self {
336        self.volumes = volumes;
337        self
338    }
339
340    /// Attach to the user-defined network `name`.
341    pub fn with_network(mut self, name: impl Into<String>) -> Self {
342        self.network = Some(name.into());
343        self
344    }
345
346    /// Reject specs dockerd would refuse or that would silently mean
347    /// something else, before any Docker call. Called by
348    /// [`Lifecycle::run`]; public so a caller can fail fast earlier.
349    pub fn validate(&self) -> Result<(), SubstrateError> {
350        let mut host_binds = HashSet::new();
351        for port in &self.ports {
352            port.validate()?;
353            if let Some(host_port) = port.host_port {
354                if !host_binds.insert((port.host_ip, host_port, port.protocol)) {
355                    return Err(SubstrateError::InvalidSpec(format!(
356                        "host {}:{host_port}/{} is published more than once",
357                        port.host_ip,
358                        port.protocol.as_str()
359                    )));
360                }
361            }
362        }
363
364        let mut targets = HashSet::new();
365        for volume in &self.volumes {
366            volume.validate()?;
367            if !targets.insert(volume.target.as_str()) {
368                return Err(SubstrateError::InvalidSpec(format!(
369                    "container path {:?} has more than one volume mounted",
370                    volume.target
371                )));
372            }
373        }
374
375        if let Some(network) = &self.network {
376            // `host` and `container:<id>` share another namespace and would
377            // silently bypass the isolation this substrate exists to give;
378            // not supported by this slice (`bridge`, `none` and user-defined
379            // names are).
380            if network.is_empty() || network == "host" || network.starts_with("container:") {
381                return Err(SubstrateError::InvalidSpec(format!(
382                    "network {network:?} is not supported: use a bridge name, `none`, or a \
383                     user-defined network"
384                )));
385            }
386        }
387        Ok(())
388    }
389
390    /// The dockerd create request this spec produces. Pure (no I/O), so the
391    /// mapping is unit-testable without a daemon.
392    fn to_docker_config(&self) -> Config<String> {
393        let mut env: Vec<String> = self.env.iter().map(|(k, v)| format!("{k}={v}")).collect();
394        env.sort();
395
396        let mut exposed_ports: HashMap<String, HashMap<(), ()>> = HashMap::new();
397        let mut port_bindings: HashMap<String, Option<Vec<PortBinding>>> = HashMap::new();
398        for port in &self.ports {
399            let key = port.docker_key();
400            exposed_ports.insert(key.clone(), HashMap::new());
401            port_bindings
402                .entry(key)
403                .or_insert_with(|| Some(Vec::new()))
404                .as_mut()
405                .expect("just inserted Some")
406                .push(PortBinding {
407                    host_ip: Some(port.host_ip.to_string()),
408                    host_port: port.host_port.map(|p| p.to_string()),
409                });
410        }
411
412        let host_config = HostConfig {
413            restart_policy: Some(self.restart_policy.into_docker()),
414            port_bindings: (!port_bindings.is_empty()).then_some(port_bindings),
415            mounts: (!self.volumes.is_empty())
416                .then(|| self.volumes.iter().map(VolumeMount::to_docker).collect()),
417            network_mode: self.network.clone(),
418            ..Default::default()
419        };
420
421        Config {
422            image: Some(self.image.clone()),
423            cmd: self.cmd.clone(),
424            env: Some(env),
425            labels: Some(self.labels.clone()),
426            exposed_ports: (!exposed_ports.is_empty()).then_some(exposed_ports),
427            host_config: Some(host_config),
428            ..Default::default()
429        }
430    }
431}
432
433/// A connected handle to the local dockerd Engine API. Never a remote/TCP
434/// Docker endpoint — this always runs containers on the same host the
435/// substrate itself is running on, like any other service on the box.
436pub struct Lifecycle {
437    docker: Docker,
438}
439
440impl Lifecycle {
441    /// Connect to the local Docker daemon (the Unix socket on Linux —
442    /// `bollard`'s own platform default) and confirm it's actually
443    /// reachable with a cheap `ping`, rather than surfacing a confusing
444    /// failure on the first real call.
445    pub async fn connect() -> Result<Self, SubstrateError> {
446        let docker = Docker::connect_with_local_defaults()?;
447        docker.ping().await?;
448        Ok(Self { docker })
449    }
450
451    /// Pull `image`, driving the streamed pull response to completion.
452    /// Docker's pull API is a no-op-but-still-valid stream when the image
453    /// is already cached, so no special-casing is needed for that case.
454    pub async fn pull_image(&self, image: &str) -> Result<(), SubstrateError> {
455        let (from_image, tag) = split_image_ref(image);
456        let options = CreateImageOptions {
457            from_image,
458            tag,
459            ..Default::default()
460        };
461        let mut stream = self.docker.create_image(Some(options), None, None);
462        while let Some(item) = stream.next().await {
463            item?;
464        }
465        Ok(())
466    }
467
468    /// Create and start a container per `spec`. On any failure after the
469    /// container was created, best-effort removes it — never leaves a
470    /// created-but-never-started container behind for a caller's retry to
471    /// trip over on a name conflict.
472    pub async fn run(&self, spec: &ContainerSpec) -> Result<String, SubstrateError> {
473        spec.validate()?;
474        let config = spec.to_docker_config();
475
476        let options = CreateContainerOptions {
477            name: spec.name.clone(),
478            platform: None,
479        };
480
481        let created = self.docker.create_container(Some(options), config).await?;
482
483        if let Err(e) = self
484            .docker
485            .start_container(&created.id, None::<StartContainerOptions<String>>)
486            .await
487        {
488            let _ = self.remove(&created.id, true).await;
489            return Err(e.into());
490        }
491
492        Ok(created.id)
493    }
494
495    /// Stop a container, giving it `timeout` to shut down cleanly before
496    /// dockerd kills it.
497    pub async fn stop(&self, container_id: &str, timeout: Duration) -> Result<(), SubstrateError> {
498        let options = StopContainerOptions {
499            t: timeout.as_secs() as i64,
500        };
501        self.docker
502            .stop_container(container_id, Some(options))
503            .await?;
504        Ok(())
505    }
506
507    /// Remove a container. `force` kills it first if still running
508    /// (`docker rm -f` semantics) rather than requiring a separate `stop`.
509    pub async fn remove(&self, container_id: &str, force: bool) -> Result<(), SubstrateError> {
510        let options = RemoveContainerOptions {
511            force,
512            ..Default::default()
513        };
514        match self
515            .docker
516            .remove_container(container_id, Some(options))
517            .await
518        {
519            Ok(()) => Ok(()),
520            Err(e) if is_not_found(&e) => Ok(()),
521            Err(e) => Err(e.into()),
522        }
523    }
524
525    /// Stop then remove — the common cleanup composition, idempotent
526    /// against "already gone" (a not-found from either step is not an
527    /// error: a caller cleaning up after a partially-failed run should
528    /// not itself fail because there was nothing left to clean up).
529    pub async fn stop_and_remove(
530        &self,
531        container_id: &str,
532        timeout: Duration,
533    ) -> Result<(), SubstrateError> {
534        if let Err(e) = self.stop(container_id, timeout).await {
535            if !matches!(&e, SubstrateError::Docker(inner) if is_not_found(inner)) {
536                tracing::warn!(container_id, error = %e, "stop failed (continuing to remove)");
537            }
538        }
539        self.remove(container_id, true).await
540    }
541
542    /// Poll until the container's `State.Running` is true and *stays*
543    /// true past a short grace period, or `timeout` elapses. See this
544    /// module's doc for exactly what this does and doesn't prove.
545    pub async fn wait_until_running(
546        &self,
547        container_id: &str,
548        timeout: Duration,
549    ) -> Result<(), SubstrateError> {
550        const CONFIRM_GRACE: Duration = Duration::from_millis(500);
551        let deadline = tokio::time::Instant::now() + timeout;
552        loop {
553            let (running, exited, exit_code) = self.inspect_run_state(container_id).await?;
554            if exited {
555                return Err(SubstrateError::Resolve(format!(
556                    "container {container_id} exited immediately (exit code {exit_code})"
557                )));
558            }
559            if running {
560                // A single "Running: true" read is a TOCTOU race against a
561                // process that crashes within the first tens of
562                // milliseconds — re-confirm after a short grace period
563                // before declaring success (see this module's doc).
564                tokio::time::sleep(CONFIRM_GRACE).await;
565                let (still_running, exited_after, exit_code_after) =
566                    self.inspect_run_state(container_id).await?;
567                if exited_after {
568                    return Err(SubstrateError::Resolve(format!(
569                        "container {container_id} exited shortly after starting (exit code {exit_code_after})"
570                    )));
571                }
572                if still_running {
573                    return Ok(());
574                }
575                // Neither running nor exited (e.g. transiently
576                // restarting) — keep polling rather than declaring either
577                // outcome on an ambiguous read.
578            }
579            if tokio::time::Instant::now() >= deadline {
580                return Err(SubstrateError::Timeout(format!(
581                    "container {container_id} did not reach (and stay at) Running within {timeout:?}"
582                )));
583            }
584            tokio::time::sleep(Duration::from_millis(200)).await;
585        }
586    }
587
588    /// Idempotently make sure the user-defined bridge network `name`
589    /// exists, for [`ContainerSpec::network`]. List-then-create, with a
590    /// create that loses a race to another caller (409) treated as success.
591    /// dockerd's `name` filter is a substring match, so the exact name is
592    /// re-checked here.
593    pub async fn ensure_network(&self, name: &str) -> Result<(), SubstrateError> {
594        let mut filters = HashMap::new();
595        filters.insert("name".to_string(), vec![name.to_string()]);
596        let existing = self
597            .docker
598            .list_networks(Some(ListNetworksOptions { filters }))
599            .await?;
600        if existing.iter().any(|n| n.name.as_deref() == Some(name)) {
601            return Ok(());
602        }
603        let options = CreateNetworkOptions {
604            name: name.to_string(),
605            driver: "bridge".to_string(),
606            ..Default::default()
607        };
608        match self.docker.create_network(options).await {
609            Ok(_) => Ok(()),
610            Err(e) if is_conflict(&e) => Ok(()),
611            Err(e) => Err(e.into()),
612        }
613    }
614
615    /// The host port dockerd bound for `container_port`/`protocol` on a
616    /// running container, if any — how a caller learns the port when the
617    /// [`PortMapping`] left `host_port` as `None`. Returns the first
618    /// binding when the port is published on several interfaces.
619    pub async fn published_host_port(
620        &self,
621        container_id: &str,
622        container_port: u16,
623        protocol: Protocol,
624    ) -> Result<Option<u16>, SubstrateError> {
625        let info = self.inspect(container_id).await?;
626        let key = format!("{container_port}/{}", protocol.as_str());
627        Ok(info
628            .network_settings
629            .and_then(|n| n.ports)
630            .and_then(|mut ports| ports.remove(&key))
631            .flatten()
632            .and_then(|bindings| {
633                bindings
634                    .into_iter()
635                    .find_map(|b| b.host_port.and_then(|p| p.parse::<u16>().ok()))
636            }))
637    }
638
639    /// Raw `inspect_container` passthrough, for callers that need more
640    /// than this module's own convenience accessors.
641    pub async fn inspect(
642        &self,
643        container_id: &str,
644    ) -> Result<ContainerInspectResponse, SubstrateError> {
645        Ok(self.docker.inspect_container(container_id, None).await?)
646    }
647
648    /// `(running, exited, exit_code)` from a fresh `inspect_container`
649    /// call — `exit_code` is only meaningful when `exited` is true.
650    async fn inspect_run_state(
651        &self,
652        container_id: &str,
653    ) -> Result<(bool, bool, i64), SubstrateError> {
654        let info = self.inspect(container_id).await?;
655        let running = info.state.as_ref().and_then(|s| s.running).unwrap_or(false);
656        let exited = info
657            .state
658            .as_ref()
659            .and_then(|s| s.status)
660            .map(|s| matches!(s, bollard::models::ContainerStateStatusEnum::EXITED))
661            .unwrap_or(false);
662        let exit_code = info.state.as_ref().and_then(|s| s.exit_code).unwrap_or(-1);
663        Ok((running, exited, exit_code))
664    }
665
666    /// Resolve a [`ContainerRef`] for a currently-running container — the
667    /// concrete slice `principal.rs`'s own doc flags as deferred to
668    /// `lifecycle`. Resolves `container_id`/`uid` from the container's
669    /// real host pid via dockerd's own inspect + `/proc`, then composes
670    /// `cgroup::resolve_cgroup_path` and `mount::resolve_mount_namespace`
671    /// to fill in the optional facts.
672    ///
673    /// `cgroup`/`mount_ns` resolution is best-effort: a failure there
674    /// (e.g. the real, documented permission boundary
675    /// `mount::resolve_mount_namespace` hits against a root-owned
676    /// container when called by a non-root caller) is logged and leaves
677    /// that field `None`, rather than failing the whole `ContainerRef` —
678    /// they were already `Option`s for exactly this reason. A failure to
679    /// resolve the container's pid/uid at all *is* fatal: those two
680    /// fields are not optional on `ContainerRef`.
681    pub async fn container_ref(&self, container_id: &str) -> Result<ContainerRef, SubstrateError> {
682        let info = self.inspect(container_id).await?;
683        let pid = info
684            .state
685            .as_ref()
686            .and_then(|s| s.pid)
687            .filter(|&p| p > 0)
688            .ok_or_else(|| {
689                SubstrateError::Resolve(format!(
690                    "container {container_id} has no live pid (is it running?)"
691                ))
692            })? as u32;
693        let uid = read_uid_from_proc_status(pid)?;
694
695        let mut container_ref = ContainerRef::new(container_id.to_string(), uid);
696
697        match cgroup::resolve_cgroup_path(pid) {
698            Ok(path) => container_ref = container_ref.with_cgroup(path),
699            Err(e) => {
700                tracing::warn!(container_id, pid, error = %e, "could not resolve cgroup path");
701            }
702        }
703
704        match mount::resolve_mount_namespace(pid) {
705            Ok(ns) => container_ref = container_ref.with_mount_ns(ns),
706            Err(e) => {
707                tracing::warn!(
708                    container_id, pid, error = %e,
709                    "could not resolve mount namespace (needs root/CAP_SYS_PTRACE to read \
710                     another user's /proc/<pid>/ns/mnt — see mount::resolve_mount_namespace's doc)"
711                );
712            }
713        }
714
715        Ok(container_ref)
716    }
717}
718
719/// Read the real uid a running process is executing as, from
720/// `/proc/<pid>/status`'s `Uid:` line (the first of its four
721/// space-separated fields — real uid). World-readable, same as
722/// `cgroup::resolve_cgroup_path`'s `/proc/<pid>/cgroup` read — no
723/// elevated privilege needed even for a root-owned container process.
724fn read_uid_from_proc_status(pid: u32) -> Result<u32, SubstrateError> {
725    let raw = std::fs::read_to_string(format!("/proc/{pid}/status"))?;
726    let line = raw
727        .lines()
728        .find_map(|line| line.strip_prefix("Uid:"))
729        .ok_or_else(|| SubstrateError::Resolve(format!("no Uid line in /proc/{pid}/status")))?;
730    let real_uid = line.split_whitespace().next().ok_or_else(|| {
731        SubstrateError::Resolve(format!("malformed Uid line in /proc/{pid}/status"))
732    })?;
733    real_uid
734        .parse::<u32>()
735        .map_err(|_| SubstrateError::Resolve(format!("non-numeric uid in /proc/{pid}/status")))
736}
737
738fn is_conflict(e: &bollard::errors::Error) -> bool {
739    matches!(e, bollard::errors::Error::DockerResponseServerError { status_code, .. } if *status_code == 409)
740}
741
742fn is_not_found(e: &bollard::errors::Error) -> bool {
743    matches!(e, bollard::errors::Error::DockerResponseServerError { status_code, .. } if *status_code == 404)
744}
745
746/// Split `repo[:tag]` into `(repo, tag)`, defaulting the tag to `latest` —
747/// `create_image`'s `from_image`/`tag` fields want these separated. Only
748/// the *last* `:` after the *last* `/` is treated as the tag separator, so
749/// a registry host with an explicit port (`host:5000/repo`) isn't
750/// misparsed — the same disambiguation rule `docker` itself uses.
751fn split_image_ref(image: &str) -> (String, String) {
752    let last_slash = image.rfind('/').unwrap_or(0);
753    match image[last_slash..].rfind(':') {
754        Some(rel_idx) => {
755            let idx = last_slash + rel_idx;
756            (image[..idx].to_string(), image[idx + 1..].to_string())
757        }
758        None => (image.to_string(), "latest".to_string()),
759    }
760}
761
762#[cfg(test)]
763mod tests {
764    use super::*;
765
766    #[test]
767    fn split_image_ref_handles_a_plain_tag() {
768        assert_eq!(
769            split_image_ref("alpine:3"),
770            ("alpine".to_string(), "3".to_string())
771        );
772    }
773
774    #[test]
775    fn split_image_ref_defaults_to_latest() {
776        assert_eq!(
777            split_image_ref("alpine"),
778            ("alpine".to_string(), "latest".to_string())
779        );
780    }
781
782    #[test]
783    fn split_image_ref_handles_a_registry_host_with_a_port() {
784        assert_eq!(
785            split_image_ref("registry.example.com:5000/substrate-kit-test:latest"),
786            (
787                "registry.example.com:5000/substrate-kit-test".to_string(),
788                "latest".to_string()
789            )
790        );
791    }
792
793    #[test]
794    fn restart_policy_never_maps_to_dockers_no_policy() {
795        let policy = RestartPolicy::Never.into_docker();
796        assert_eq!(policy.name, Some(RestartPolicyNameEnum::NO));
797        assert_eq!(policy.maximum_retry_count, None);
798    }
799
800    #[test]
801    fn restart_policy_on_failure_carries_its_retry_count() {
802        let policy = RestartPolicy::OnFailure { max_retries: 3 }.into_docker();
803        assert_eq!(policy.name, Some(RestartPolicyNameEnum::ON_FAILURE));
804        assert_eq!(policy.maximum_retry_count, Some(3));
805    }
806
807    #[test]
808    fn container_spec_builders_set_expected_fields() {
809        let mut env = HashMap::new();
810        env.insert("FOO".to_string(), "bar".to_string());
811        let spec = ContainerSpec::new("test-name", "alpine:3")
812            .with_env(env.clone())
813            .with_cmd(vec!["sleep".to_string(), "1".to_string()])
814            .with_restart_policy(RestartPolicy::Always);
815
816        assert_eq!(spec.name, "test-name");
817        assert_eq!(spec.image, "alpine:3");
818        assert_eq!(spec.env, env);
819        assert_eq!(spec.cmd, Some(vec!["sleep".to_string(), "1".to_string()]));
820        assert_eq!(spec.restart_policy, RestartPolicy::Always);
821    }
822
823    #[test]
824    fn a_spec_with_no_networking_or_volumes_produces_the_pre_clus55_config() {
825        let cfg = ContainerSpec::new("n", "alpine:3").to_docker_config();
826        let hc = cfg.host_config.unwrap();
827        assert!(cfg.exposed_ports.is_none());
828        assert!(hc.port_bindings.is_none());
829        assert!(hc.mounts.is_none());
830        assert!(hc.network_mode.is_none());
831    }
832
833    #[test]
834    fn ports_default_to_loopback_and_map_to_docker_port_bindings() {
835        let spec = ContainerSpec::new("n", "alpine:3")
836            .with_port(PortMapping::tcp(8096).with_host_port(8096))
837            .with_port(PortMapping::udp(5353));
838        let cfg = spec.to_docker_config();
839        let exposed = cfg.exposed_ports.unwrap();
840        assert!(exposed.contains_key("8096/tcp") && exposed.contains_key("5353/udp"));
841        let bindings = cfg.host_config.unwrap().port_bindings.unwrap();
842        let tcp = bindings["8096/tcp"].as_ref().unwrap();
843        assert_eq!(tcp.len(), 1);
844        assert_eq!(tcp[0].host_ip.as_deref(), Some("127.0.0.1"));
845        assert_eq!(tcp[0].host_port.as_deref(), Some("8096"));
846        let udp = bindings["5353/udp"].as_ref().unwrap();
847        assert_eq!(
848            udp[0].host_port, None,
849            "None must leave allocation to dockerd"
850        );
851    }
852
853    #[test]
854    fn the_same_container_port_on_two_interfaces_yields_two_bindings() {
855        let spec = ContainerSpec::new("n", "alpine:3")
856            .with_port(PortMapping::tcp(80).with_host_port(8080))
857            .with_port(
858                PortMapping::tcp(80)
859                    .with_host_port(8080)
860                    .with_host_ip("::1".parse().unwrap()),
861            );
862        spec.validate().unwrap();
863        let bindings = spec
864            .to_docker_config()
865            .host_config
866            .unwrap()
867            .port_bindings
868            .unwrap();
869        assert_eq!(bindings["80/tcp"].as_ref().unwrap().len(), 2);
870    }
871
872    #[test]
873    fn network_and_volumes_reach_the_host_config() {
874        let spec = ContainerSpec::new("n", "alpine:3")
875            .with_network("service-net")
876            .with_volume(VolumeMount::named("cfg", "/config"));
877        let hc = spec.to_docker_config().host_config.unwrap();
878        assert_eq!(hc.network_mode.as_deref(), Some("service-net"));
879        let mounts = hc.mounts.unwrap();
880        assert_eq!(mounts.len(), 1);
881        assert_eq!(mounts[0].source.as_deref(), Some("cfg"));
882    }
883
884    #[test]
885    fn validate_rejects_bad_ports_volumes_and_networks() {
886        let bad: Vec<ContainerSpec> = vec![
887            ContainerSpec::new("n", "i").with_port(PortMapping::tcp(0)),
888            ContainerSpec::new("n", "i").with_port(PortMapping::tcp(80).with_host_port(0)),
889            ContainerSpec::new("n", "i")
890                .with_port(PortMapping::tcp(80).with_host_port(8080))
891                .with_port(PortMapping::tcp(81).with_host_port(8080)),
892            ContainerSpec::new("n", "i")
893                .with_volume(VolumeMount::named("a", "/d"))
894                .with_volume(VolumeMount::named("b", "/d")),
895            ContainerSpec::new("n", "i").with_volume(VolumeMount::named("/etc", "/d")),
896            ContainerSpec::new("n", "i").with_network(""),
897            ContainerSpec::new("n", "i").with_network("host"),
898            ContainerSpec::new("n", "i").with_network("container:abc"),
899        ];
900        for (i, spec) in bad.iter().enumerate() {
901            assert!(
902                matches!(spec.validate(), Err(SubstrateError::InvalidSpec(_))),
903                "case {i} should be rejected"
904            );
905        }
906    }
907
908    #[test]
909    fn same_host_port_on_different_protocols_is_allowed() {
910        ContainerSpec::new("n", "i")
911            .with_port(PortMapping::tcp(53).with_host_port(5353))
912            .with_port(PortMapping::udp(53).with_host_port(5353))
913            .validate()
914            .unwrap();
915    }
916
917    #[test]
918    fn debug_output_never_contains_env_values() {
919        let mut env = HashMap::new();
920        env.insert("API_TOKEN".to_string(), "s3cr3t-value".to_string());
921        let spec = ContainerSpec::new("n", "alpine:3").with_env(env);
922        let rendered = format!("{spec:?}");
923        assert!(rendered.contains("API_TOKEN"));
924        assert!(!rendered.contains("s3cr3t-value"), "{rendered}");
925    }
926}
927
928/// Live tests against a real dockerd: no mocks, because a mocked dockerd
929/// defeats the point. They are `#[ignore]`d by default, so a plain
930/// `cargo test` (and the public CI, whose macOS and Windows runners have no
931/// usable Docker) never depends on a daemon.
932///
933/// Run them with: `cargo test lifecycle::live -- --ignored --test-threads=1`
934/// on a Linux host with a reachable Docker daemon (the current user must be
935/// in the `docker` group or otherwise able to reach the socket). An
936/// unreachable daemon fails the test rather than skipping it. The tests that
937/// resolve a `ContainerRef` read the host `/proc` and cgroup v2, so they are
938/// compiled on Linux only; the others only need the Docker Engine API.
939///
940/// **What these tests can and can't prove about `container_ref` without
941/// root:** `cgroup::resolve_cgroup_path` reads a world-readable `/proc`
942/// file, so it resolves for real regardless of privilege.
943/// `mount::resolve_mount_namespace` reads `/proc/<pid>/ns/mnt`, which
944/// requires the same uid as the target process or `CAP_SYS_PTRACE` — a
945/// non-root test run resolving a root-owned dockerd container's mount
946/// namespace will hit that real permission boundary and get `None` back
947/// (logged, not silently dropped — see `Lifecycle::container_ref`'s doc).
948/// `mount_namespace_round_trips_when_privileged` asserts the full
949/// success path and is itself gated behind `SUBSTRATE_REQUIRE_ROOT_NS`,
950/// separate from the Docker requirement — set it (as root, e.g. `sudo
951/// -E`) to prove real end-to-end resolution; otherwise this suite still
952/// runs and asserts the graceful, correctly-typed permission-denied
953/// failure instead, which is itself real, exercised behavior, not a skip.
954#[cfg(test)]
955mod live {
956    use super::*;
957
958    /// Connect to the local dockerd, or fail the test. These tests are
959    /// `#[ignore]`d, so running one is an explicit request for a real
960    /// daemon: an unreachable one is a failure, never a silent pass.
961    async fn require_docker() -> Lifecycle {
962        match Lifecycle::connect().await {
963            Ok(lifecycle) => lifecycle,
964            Err(e) => panic!("these live tests need a reachable Docker daemon: {e}"),
965        }
966    }
967
968    fn test_name(prefix: &str) -> String {
969        format!(
970            "{prefix}-{}",
971            std::time::SystemTime::now()
972                .duration_since(std::time::UNIX_EPOCH)
973                .unwrap()
974                .as_millis()
975        )
976    }
977
978    fn managed_labels() -> HashMap<String, String> {
979        let mut labels = HashMap::new();
980        labels.insert(LABEL_MANAGED.to_string(), "true".to_string());
981        labels
982    }
983
984    /// The full round trip a real substrate consumer drives: pull a small
985    /// public image, create+start a long-lived container, confirm it
986    /// reaches (and stays at) Running, resolve its `ContainerRef` for
987    /// real (cgroup path definitely, mount namespace best-effort per this
988    /// module's doc), then stop+remove it.
989    #[tokio::test]
990    #[ignore = "needs a reachable Docker daemon; run with `cargo test -- --ignored`"]
991    #[cfg(target_os = "linux")]
992    async fn create_start_health_container_ref_stop_round_trips_for_real() {
993        let lifecycle = require_docker().await;
994        let image = "alpine:3";
995        lifecycle.pull_image(image).await.expect("pull_image");
996
997        let name = test_name("substrate-lifecycle-live");
998        let spec = ContainerSpec::new(&name, image)
999            .with_cmd(vec!["sleep".to_string(), "300".to_string()])
1000            .with_labels(managed_labels());
1001
1002        let container_id = lifecycle.run(&spec).await.expect("run");
1003
1004        lifecycle
1005            .wait_until_running(&container_id, Duration::from_secs(15))
1006            .await
1007            .expect("wait_until_running");
1008
1009        let container_ref = lifecycle
1010            .container_ref(&container_id)
1011            .await
1012            .expect("container_ref");
1013
1014        assert_eq!(container_ref.container_id, container_id);
1015        // alpine's default entrypoint runs as root — a real, resolved
1016        // fact from /proc, not a fixture default.
1017        assert_eq!(container_ref.uid, 0);
1018
1019        let cgroup_path = container_ref
1020            .cgroup
1021            .as_deref()
1022            .expect("cgroup path must resolve — /proc/<pid>/cgroup is world-readable");
1023        assert!(
1024            cgroup_path.starts_with(cgroup::CGROUP_V2_MOUNT),
1025            "cgroup path: {cgroup_path}"
1026        );
1027
1028        if let Some(mount_ns) = &container_ref.mount_ns {
1029            assert!(
1030                mount_ns.starts_with("mnt:["),
1031                "mount namespace handle: {mount_ns}"
1032            );
1033        } else {
1034            eprintln!(
1035                "NOTE: mount_ns did not resolve (expected without root/CAP_SYS_PTRACE against \
1036                 a root-owned container — see mount::resolve_mount_namespace's doc)"
1037            );
1038        }
1039
1040        lifecycle
1041            .stop_and_remove(&container_id, Duration::from_secs(5))
1042            .await
1043            .expect("stop_and_remove");
1044    }
1045
1046    /// A container whose entrypoint exits immediately must be reported as
1047    /// a real failure by `wait_until_running`, not hang until the
1048    /// timeout.
1049    #[tokio::test]
1050    #[ignore = "needs a reachable Docker daemon; run with `cargo test -- --ignored`"]
1051    async fn wait_until_running_fails_fast_for_a_container_that_exits_immediately() {
1052        let lifecycle = require_docker().await;
1053        let image = "alpine:3";
1054        lifecycle.pull_image(image).await.expect("pull_image");
1055
1056        let name = test_name("substrate-lifecycle-live-exit");
1057        let spec = ContainerSpec::new(&name, image)
1058            .with_cmd(vec![
1059                "sh".to_string(),
1060                "-c".to_string(),
1061                "exit 0".to_string(),
1062            ])
1063            .with_labels(managed_labels());
1064
1065        let container_id = lifecycle.run(&spec).await.expect("run");
1066
1067        let result = tokio::time::timeout(
1068            Duration::from_secs(10),
1069            lifecycle.wait_until_running(&container_id, Duration::from_secs(30)),
1070        )
1071        .await
1072        .expect("wait_until_running must return well before its own 30s timeout, not hang");
1073
1074        assert!(
1075            result.is_err(),
1076            "an immediately-exited container must be reported as a failure"
1077        );
1078
1079        lifecycle
1080            .stop_and_remove(&container_id, Duration::from_secs(5))
1081            .await
1082            .expect("stop_and_remove");
1083    }
1084
1085    /// Restart-policy handling actually reaches dockerd: create a
1086    /// container with `OnFailure { max_retries: 2 }` and confirm dockerd's
1087    /// own inspect reflects it back, rather than trusting our own
1088    /// in-process `into_docker()` mapping alone.
1089    #[tokio::test]
1090    #[ignore = "needs a reachable Docker daemon; run with `cargo test -- --ignored`"]
1091    async fn restart_policy_is_applied_and_visible_in_dockerds_own_inspect() {
1092        let lifecycle = require_docker().await;
1093        let image = "alpine:3";
1094        lifecycle.pull_image(image).await.expect("pull_image");
1095
1096        let name = test_name("substrate-lifecycle-live-restart-policy");
1097        let spec = ContainerSpec::new(&name, image)
1098            .with_cmd(vec!["sleep".to_string(), "300".to_string()])
1099            .with_labels(managed_labels())
1100            .with_restart_policy(RestartPolicy::OnFailure { max_retries: 2 });
1101
1102        let container_id = lifecycle.run(&spec).await.expect("run");
1103
1104        let info = lifecycle.inspect(&container_id).await.expect("inspect");
1105        let restart_policy = info
1106            .host_config
1107            .as_ref()
1108            .and_then(|hc| hc.restart_policy.as_ref())
1109            .expect("dockerd must echo back a restart policy");
1110        assert_eq!(restart_policy.name, Some(RestartPolicyNameEnum::ON_FAILURE));
1111        assert_eq!(restart_policy.maximum_retry_count, Some(2));
1112
1113        lifecycle
1114            .stop_and_remove(&container_id, Duration::from_secs(5))
1115            .await
1116            .expect("stop_and_remove");
1117    }
1118
1119    /// Asserts the *privileged* success path of mount-namespace
1120    /// resolution end-to-end — gated separately from
1121    /// the Docker requirement because it needs a second, independent
1122    /// precondition (root/`CAP_SYS_PTRACE`), not just a reachable daemon.
1123    /// Without `SUBSTRATE_REQUIRE_ROOT_NS` set, this still runs but only
1124    /// asserts the graceful permission-denied path — real, typed,
1125    /// exercised behavior either way, never a silent no-op.
1126    #[tokio::test]
1127    #[ignore = "needs a reachable Docker daemon; run with `cargo test -- --ignored`"]
1128    #[cfg(target_os = "linux")]
1129    async fn mount_namespace_round_trips_when_privileged() {
1130        let lifecycle = require_docker().await;
1131        let image = "alpine:3";
1132        lifecycle.pull_image(image).await.expect("pull_image");
1133
1134        let name = test_name("substrate-lifecycle-live-mount-ns");
1135        let spec = ContainerSpec::new(&name, image)
1136            .with_cmd(vec!["sleep".to_string(), "300".to_string()])
1137            .with_labels(managed_labels());
1138
1139        let container_id = lifecycle.run(&spec).await.expect("run");
1140        lifecycle
1141            .wait_until_running(&container_id, Duration::from_secs(15))
1142            .await
1143            .expect("wait_until_running");
1144
1145        let container_ref = lifecycle
1146            .container_ref(&container_id)
1147            .await
1148            .expect("container_ref");
1149
1150        let require_root_ns = std::env::var_os("SUBSTRATE_REQUIRE_ROOT_NS").is_some();
1151        match (&container_ref.mount_ns, require_root_ns) {
1152            (Some(ns), _) => assert!(ns.starts_with("mnt:["), "mount namespace handle: {ns}"),
1153            (None, true) => panic!(
1154                "SUBSTRATE_REQUIRE_ROOT_NS set but mount_ns did not resolve — \
1155                 run as root/with CAP_SYS_PTRACE"
1156            ),
1157            (None, false) => eprintln!(
1158                "SKIPPED (assertion only): mount_ns did not resolve without root/CAP_SYS_PTRACE; \
1159                 set SUBSTRATE_REQUIRE_ROOT_NS=1 under sudo to hard-fail and prove the privileged path"
1160            ),
1161        }
1162
1163        lifecycle
1164            .stop_and_remove(&container_id, Duration::from_secs(5))
1165            .await
1166            .expect("stop_and_remove");
1167    }
1168
1169    // ---- network / port / volume live tests ----------------------
1170
1171    /// Poll inspect until the container has exited; returns its exit code.
1172    async fn wait_exit_code(lifecycle: &Lifecycle, id: &str) -> i64 {
1173        for _ in 0..100 {
1174            let info = lifecycle.inspect(id).await.expect("inspect");
1175            if let Some(state) = info.state {
1176                if state.running != Some(true) && state.status.is_some() {
1177                    use bollard::models::ContainerStateStatusEnum as S;
1178                    if matches!(state.status, Some(S::EXITED) | Some(S::DEAD)) {
1179                        return state.exit_code.unwrap_or(-1);
1180                    }
1181                }
1182            }
1183            tokio::time::sleep(Duration::from_millis(200)).await;
1184        }
1185        panic!("container {id} did not exit in time");
1186    }
1187
1188    /// A dockerd-allocated, loopback-only publish round trips: the port is
1189    /// bound on 127.0.0.1 only, and `published_host_port` reads it back.
1190    #[tokio::test]
1191    #[ignore = "needs a reachable Docker daemon; run with `cargo test -- --ignored`"]
1192    async fn ephemeral_loopback_port_is_published_and_readable() {
1193        let lifecycle = require_docker().await;
1194        let image = "alpine:3";
1195        lifecycle.pull_image(image).await.expect("pull_image");
1196
1197        let name = test_name("substrate-lifecycle-live-port");
1198        let spec = ContainerSpec::new(&name, image)
1199            .with_cmd(vec!["sleep".to_string(), "300".to_string()])
1200            .with_labels(managed_labels())
1201            .with_port(PortMapping::tcp(8080));
1202        let id = lifecycle.run(&spec).await.expect("run");
1203
1204        let host_port = lifecycle
1205            .published_host_port(&id, 8080, Protocol::Tcp)
1206            .await
1207            .expect("published_host_port")
1208            .expect("dockerd must have allocated a host port");
1209        assert!(host_port > 0);
1210
1211        let info = lifecycle.inspect(&id).await.expect("inspect");
1212        let binding = info
1213            .host_config
1214            .and_then(|hc| hc.port_bindings)
1215            .and_then(|mut pb| pb.remove("8080/tcp"))
1216            .flatten()
1217            .expect("binding echoed back");
1218        assert_eq!(binding[0].host_ip.as_deref(), Some("127.0.0.1"));
1219
1220        assert_eq!(
1221            lifecycle
1222                .published_host_port(&id, 9999, Protocol::Tcp)
1223                .await
1224                .unwrap(),
1225            None
1226        );
1227
1228        lifecycle
1229            .stop_and_remove(&id, Duration::from_secs(5))
1230            .await
1231            .expect("stop_and_remove");
1232    }
1233
1234    /// A pinned 1:1 host port really accepts a
1235    /// TCP connection on loopback, end to end through dockerd's proxy.
1236    #[tokio::test]
1237    #[ignore = "needs a reachable Docker daemon; run with `cargo test -- --ignored`"]
1238    async fn pinned_one_to_one_port_accepts_a_tcp_connection() {
1239        let lifecycle = require_docker().await;
1240        let image = "alpine:3";
1241        lifecycle.pull_image(image).await.expect("pull_image");
1242
1243        // Reserve a free port, release it, then ask dockerd for it.
1244        let free = std::net::TcpListener::bind("127.0.0.1:0")
1245            .unwrap()
1246            .local_addr()
1247            .unwrap()
1248            .port();
1249
1250        let name = test_name("substrate-lifecycle-live-pinned");
1251        let spec = ContainerSpec::new(&name, image)
1252            .with_cmd(vec![
1253                "sh".to_string(),
1254                "-c".to_string(),
1255                format!("nc -lk -p {free} -e /bin/cat"),
1256            ])
1257            .with_labels(managed_labels())
1258            .with_port(PortMapping::tcp(free).with_host_port(free));
1259        let id = lifecycle.run(&spec).await.expect("run");
1260        lifecycle
1261            .wait_until_running(&id, Duration::from_secs(15))
1262            .await
1263            .expect("wait_until_running");
1264
1265        assert_eq!(
1266            lifecycle
1267                .published_host_port(&id, free, Protocol::Tcp)
1268                .await
1269                .unwrap(),
1270            Some(free)
1271        );
1272        // dockerd's proxy accepts the connection as soon as the port is
1273        // bound, independent of the process behind it.
1274        let mut connected = false;
1275        for _ in 0..25 {
1276            if tokio::net::TcpStream::connect(("127.0.0.1", free))
1277                .await
1278                .is_ok()
1279            {
1280                connected = true;
1281                break;
1282            }
1283            tokio::time::sleep(Duration::from_millis(200)).await;
1284        }
1285        assert!(
1286            connected,
1287            "could not connect to published loopback port {free}"
1288        );
1289
1290        lifecycle
1291            .stop_and_remove(&id, Duration::from_secs(5))
1292            .await
1293            .expect("stop_and_remove");
1294    }
1295
1296    /// `ensure_network` is idempotent, and two containers on it resolve
1297    /// each other by container name.
1298    #[tokio::test]
1299    #[ignore = "needs a reachable Docker daemon; run with `cargo test -- --ignored`"]
1300    async fn containers_on_a_user_defined_network_resolve_each_other_by_name() {
1301        let lifecycle = require_docker().await;
1302        let image = "alpine:3";
1303        lifecycle.pull_image(image).await.expect("pull_image");
1304
1305        let net = test_name("substrate-live-net");
1306        lifecycle
1307            .ensure_network(&net)
1308            .await
1309            .expect("ensure_network");
1310        lifecycle
1311            .ensure_network(&net)
1312            .await
1313            .expect("ensure_network must be idempotent");
1314
1315        let server_name = test_name("substrate-live-net-server");
1316        let server = lifecycle
1317            .run(
1318                &ContainerSpec::new(&server_name, image)
1319                    .with_cmd(vec!["sleep".to_string(), "300".to_string()])
1320                    .with_labels(managed_labels())
1321                    .with_network(&net),
1322            )
1323            .await
1324            .expect("run server");
1325        lifecycle
1326            .wait_until_running(&server, Duration::from_secs(15))
1327            .await
1328            .expect("server running");
1329
1330        let client_name = test_name("substrate-live-net-client");
1331        let client = lifecycle
1332            .run(
1333                &ContainerSpec::new(&client_name, image)
1334                    .with_cmd(vec![
1335                        "sh".to_string(),
1336                        "-c".to_string(),
1337                        format!("getent hosts {server_name}"),
1338                    ])
1339                    .with_labels(managed_labels())
1340                    .with_network(&net),
1341            )
1342            .await
1343            .expect("run client");
1344        let code = wait_exit_code(&lifecycle, &client).await;
1345        assert_eq!(code, 0, "client could not resolve the server by name");
1346
1347        let info = lifecycle.inspect(&server).await.expect("inspect");
1348        let networks = info.network_settings.and_then(|n| n.networks).unwrap();
1349        assert!(networks.contains_key(&net), "server not attached to {net}");
1350
1351        for id in [&client, &server] {
1352            lifecycle
1353                .stop_and_remove(id, Duration::from_secs(5))
1354                .await
1355                .expect("stop_and_remove");
1356        }
1357        lifecycle
1358            .docker
1359            .remove_network(&net)
1360            .await
1361            .expect("remove_network");
1362    }
1363
1364    /// A named volume persists across containers, a read-only mount
1365    /// refuses writes, and dockerd (not just our mapping) reports a
1366    /// `volume`-type mount, never a bind.
1367    #[tokio::test]
1368    #[ignore = "needs a reachable Docker daemon; run with `cargo test -- --ignored`"]
1369    async fn named_volume_persists_across_containers_and_read_only_is_enforced() {
1370        let lifecycle = require_docker().await;
1371        let image = "alpine:3";
1372        lifecycle.pull_image(image).await.expect("pull_image");
1373
1374        let vol = test_name("substrate-live-vol");
1375
1376        let run_sh = |label: &'static str, script: &'static str, ro: bool| {
1377            let lifecycle = &lifecycle;
1378            let vol = vol.clone();
1379            async move {
1380                let mount = VolumeMount::named(&vol, "/data");
1381                let mount = if ro { mount.read_only() } else { mount };
1382                let spec = ContainerSpec::new(test_name(label), image)
1383                    .with_cmd(vec!["sh".to_string(), "-c".to_string(), script.to_string()])
1384                    .with_labels(managed_labels())
1385                    .with_volume(mount);
1386                let id = lifecycle.run(&spec).await.expect("run");
1387                let code = wait_exit_code(lifecycle, &id).await;
1388                let info = lifecycle.inspect(&id).await.expect("inspect");
1389                let mounts = info.mounts.unwrap_or_default();
1390                lifecycle.remove(&id, true).await.expect("remove");
1391                (code, mounts)
1392            }
1393        };
1394
1395        let (code, mounts) = run_sh("substrate-live-vol-w", "echo hello > /data/f", false).await;
1396        assert_eq!(code, 0);
1397        assert_eq!(mounts.len(), 1);
1398        assert_eq!(
1399            mounts[0].typ,
1400            Some(bollard::models::MountPointTypeEnum::VOLUME)
1401        );
1402        assert_eq!(mounts[0].name.as_deref(), Some(vol.as_str()));
1403
1404        let (code, _) = run_sh("substrate-live-vol-r", "grep -q hello /data/f", false).await;
1405        assert_eq!(code, 0, "data written by the first container must persist");
1406
1407        let (code, mounts) = run_sh("substrate-live-vol-ro", "echo x > /data/g", true).await;
1408        assert_ne!(code, 0, "write to a read-only volume must fail");
1409        assert_eq!(mounts[0].rw, Some(false));
1410
1411        lifecycle
1412            .docker
1413            .remove_volume(&vol, None)
1414            .await
1415            .expect("remove_volume");
1416    }
1417
1418    /// A rejected spec fails before any dockerd call: nothing is created.
1419    #[tokio::test]
1420    #[ignore = "needs a reachable Docker daemon; run with `cargo test -- --ignored`"]
1421    async fn an_invalid_spec_is_rejected_without_creating_a_container() {
1422        let lifecycle = require_docker().await;
1423        let name = test_name("substrate-lifecycle-live-invalid");
1424        let spec =
1425            ContainerSpec::new(&name, "alpine:3").with_volume(VolumeMount::named("/etc", "/d"));
1426        let err = lifecycle.run(&spec).await.expect_err("must be rejected");
1427        assert!(matches!(err, SubstrateError::InvalidSpec(_)));
1428        assert!(
1429            lifecycle.inspect(&name).await.is_err(),
1430            "no container may have been created"
1431        );
1432    }
1433}