Skip to main content

runnerd_core/
docker.rs

1//! Runner containers through the Docker Engine API: the connection, container, volume and image requests, and the
2//! `DockerApi` trait the engine and the caches call. It decides nothing about scheduling, and a JIT config only
3//! ever reaches a container as a root-only file, never the create request or a log line.
4
5use std::collections::{BTreeMap, BTreeSet, HashMap};
6use std::fmt;
7use std::time::{Duration, Instant};
8
9use bollard::exec::{CreateExecOptions, StartExecResults};
10use std::pin::Pin;
11
12use bollard::models::{
13    ChangeType, ContainerConfig, ContainerCreateBody, CreateImageInfo, HostConfig, Mount,
14    NetworkCreateRequest, SystemInfo, VolumeCreateRequest,
15};
16use bollard::query_parameters::{
17    AttachContainerOptionsBuilder, CommitContainerOptionsBuilder, CreateContainerOptionsBuilder,
18    CreateImageOptionsBuilder, DataUsageOptionsBuilder, DownloadFromContainerOptionsBuilder,
19    InspectContainerOptionsBuilder, ListContainersOptionsBuilder, ListImagesOptionsBuilder,
20    LogsOptionsBuilder, RemoveContainerOptionsBuilder, RemoveImageOptionsBuilder,
21    RemoveVolumeOptionsBuilder, ResizeContainerTTYOptionsBuilder, UploadToContainerOptionsBuilder,
22    WaitContainerOptionsBuilder,
23};
24use futures_util::stream::BoxStream;
25use futures_util::{StreamExt, TryStreamExt};
26use tracing::info;
27
28use crate::config::ByteSize;
29use crate::isolation::Isolation;
30
31pub use bollard::errors::Error;
32
33/// One runner container: a profile's image stage with the profile's limits.
34#[derive(Debug, Clone)]
35pub struct RunnerSpec {
36    /// The container's name; for a runner, also the runner's.
37    pub name: String,
38    /// The image, such as `runnerd-runner:base`.
39    pub image: String,
40    /// The profile, put on the `runnerd.profile` label.
41    pub profile: String,
42    /// The systemd slice the container is placed in.
43    pub slice: String,
44    /// CPUs the container may use.
45    pub cpus: u32,
46    /// Memory the container may use, swap included.
47    pub memory: ByteSize,
48    /// `NAME=value` environment variables.
49    pub env: Vec<String>,
50    /// Volumes to mount.
51    pub mounts: Vec<Mount>,
52    /// The network the container joins; `None` is Docker's default bridge.
53    pub network: Option<String>,
54    /// Labels on top of `runnerd.profile` and `runnerd.isolation`.
55    pub labels: BTreeMap<String, String>,
56    /// Replaces the image's entrypoint; `None` runs the runner.
57    pub entrypoint: Option<Vec<String>>,
58    /// The runner's just-in-time config, which the entrypoint reads from `JIT_CONFIG_PATH`.
59    pub jit_config: Option<JitConfig>,
60}
61
62/// Where `start` puts the just-in-time config in the container: a root-only file, never the create request,
63/// which Docker stores and clients may log.
64pub const JIT_CONFIG_PATH: &str = "/run/runnerd/jitconfig";
65
66/// A runner's just-in-time config from GitHub: credentials, never printed.
67#[derive(Clone)]
68pub struct JitConfig(pub String);
69
70impl fmt::Debug for JitConfig {
71    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
72        f.write_str("JitConfig(<redacted>)")
73    }
74}
75
76/// What `inspect` reports about a runner container.
77#[derive(Debug, Clone, PartialEq, Eq)]
78pub struct RunnerState {
79    /// Whether the container runs.
80    pub running: bool,
81    /// Its exit code, once it stopped.
82    pub exit_code: Option<i64>,
83    /// Whether the kernel killed a process in it for memory, as Docker saw.
84    pub oom_killed: bool,
85    /// True under `unsafe-dind`: a job in the container can take over the host.
86    pub privileged: bool,
87    /// The runtime Docker runs it with, such as `sysbox-runc`.
88    pub runtime: Option<String>,
89    /// Its main process on the host, while it runs.
90    pub pid: Option<u32>,
91    /// Its address on its first network.
92    pub address: Option<String>,
93    /// When it started, in milliseconds since the epoch.
94    pub started_at: Option<u64>,
95    /// Its memory cap in bytes; `None` when it has none.
96    pub memory: Option<u64>,
97    /// Its CPU limit in millicores; `None` when it has none.
98    pub cpus: Option<u64>,
99}
100
101/// What a command run in a container prints on stdout, as it prints it.
102pub type Output = BoxStream<'static, Result<Vec<u8>, Error>>;
103
104/// An image `images` found.
105#[derive(Debug, Clone, PartialEq, Eq)]
106pub struct Image {
107    /// Its first tag, such as `runnerd-record:42`; its ID when it has none.
108    pub reference: String,
109    /// Its labels.
110    pub labels: HashMap<String, String>,
111    /// The disk it takes beyond its parent image: what removing it frees.
112    pub bytes: u64,
113}
114
115/// An image `tagged_images` found.
116#[derive(Debug, Clone, PartialEq, Eq)]
117pub struct TaggedImage {
118    /// Its ID, such as `sha256:224e…`.
119    pub id: String,
120    /// Its tags, such as `ghcr.io/ayagmar/runnerd-runner:node-0.0.27`.
121    pub tags: Vec<String>,
122    /// The registry digests it was pulled by, such as `ghcr.io/ayagmar/runnerd-runner@sha256:9e79…`.
123    pub digests: Vec<String>,
124}
125
126/// A file `files` read from a container.
127#[derive(Debug, Clone, PartialEq, Eq)]
128pub struct File {
129    /// Its name in the directory read, such as `Worker_20261031-120000-utc.log`.
130    pub name: String,
131    /// What it holds, as text: invalid UTF-8 is replaced. Cut short past `FILES_LIMIT`.
132    pub contents: String,
133}
134
135/// What `files` read of a directory.
136#[derive(Debug, Clone, Default, PartialEq, Eq)]
137pub struct Files {
138    /// The files read, in the archive's order.
139    pub files: Vec<File>,
140    /// Whether they are all there and whole: the archive ended as a tar does, and none was cut at `FILES_LIMIT`. A
141    /// file missing from an incomplete read may be in the directory.
142    pub complete: bool,
143}
144
145/// The most of the files `files` keeps, together. What it skips, such as a runner's job log pages below `_diag`,
146/// does not count: it is read through and dropped.
147pub(crate) const FILES_LIMIT: usize = 16 << 20;
148
149/// A container's terminal, attached once it was created and before it started, so none of its output is missed.
150pub struct Tty {
151    /// The container's ID.
152    pub container: String,
153    /// What the terminal shows, as it shows it; it ends when the container's process exits.
154    pub output: BoxStream<'static, Result<Vec<u8>, Error>>,
155    /// What is typed into it.
156    pub input: Pin<Box<dyn tokio::io::AsyncWrite + Send>>,
157}
158
159impl fmt::Debug for Tty {
160    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
161        f.debug_struct("Tty")
162            .field("container", &self.container)
163            .finish_non_exhaustive()
164    }
165}
166
167/// A terminal's size.
168#[derive(Debug, Clone, Copy, PartialEq, Eq)]
169pub struct TtySize {
170    /// Columns.
171    pub cols: u16,
172    /// Rows.
173    pub rows: u16,
174}
175
176/// The CPU shares of a runner's container or a record's shell with `cpus` CPUs: 1024 a CPU, within the 2 to 262144
177/// Docker takes, which it writes as the container's `cpu.weight`. They sit in the slice beside each other, so when the
178/// slice's CPUs are all in use they share them in proportion to their `cpus`, not evenly; a container with work still
179/// gets at most its `cpus` (`nano_cpus`). The mirror keeps Docker's default weight, which every job's pulls go
180/// through, and programs outside the slice share the host's CPUs with the slice by its own weight (ADR 0079).
181pub fn cpu_shares(cpus: u32) -> i64 {
182    (i64::from(cpus) * 1024).clamp(2, 262_144)
183}
184
185/// The create-container request for a runner container under an isolation backend.
186pub fn create_body(spec: &RunnerSpec, isolation: &dyn Isolation) -> ContainerCreateBody {
187    // Clamped to 8 EiB, the most Docker takes.
188    let memory = i64::try_from(spec.memory.0).unwrap_or(i64::MAX);
189    let mut host_config = HostConfig {
190        cgroup_parent: Some(spec.slice.clone()),
191        memory: Some(memory),
192        // Swap equal to memory: the container gets no swap on top of its memory.
193        memory_swap: Some(memory),
194        nano_cpus: Some(i64::from(spec.cpus) * 1_000_000_000),
195        cpu_shares: Some(cpu_shares(spec.cpus)),
196        init: Some(true),
197        mounts: Some(spec.mounts.clone()),
198        network_mode: spec.network.clone(),
199        ..Default::default()
200    };
201    isolation.configure(&mut host_config);
202    let mut labels: HashMap<String, String> = spec.labels.clone().into_iter().collect();
203    labels.insert("runnerd.profile".to_owned(), spec.profile.clone());
204    labels.insert("runnerd.isolation".to_owned(), isolation.name().to_owned());
205    ContainerCreateBody {
206        image: Some(spec.image.clone()),
207        env: Some(spec.env.clone()),
208        entrypoint: spec.entrypoint.clone(),
209        labels: Some(labels),
210        host_config: Some(host_config),
211        ..Default::default()
212    }
213}
214
215/// A container `list` found.
216#[derive(Debug, Clone, PartialEq, Eq)]
217pub struct Container {
218    /// The Docker container ID.
219    pub id: String,
220    /// Without Docker's leading `/`.
221    pub name: String,
222    /// Whether it runs.
223    pub running: bool,
224    /// Its labels.
225    pub labels: HashMap<String, String>,
226}
227
228/// A runner start that failed, with the container it may have left: Docker may have made, or even started, it before
229/// the request failed, as when its answer is lost.
230#[derive(Debug)]
231pub struct StartError {
232    /// The container the start left, by ID, or by name when Docker could not say whether one exists; `None` when
233    /// none does. It may run.
234    pub container: Option<String>,
235    /// Why the start failed.
236    pub error: Error,
237}
238
239impl fmt::Display for StartError {
240    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
241        self.error.fmt(f)
242    }
243}
244
245impl std::error::Error for StartError {
246    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
247        Some(&self.error)
248    }
249}
250
251/// For callers that go on whether or not a container was left.
252impl From<StartError> for Error {
253    fn from(failed: StartError) -> Error {
254        failed.error
255    }
256}
257
258/// Whether Docker answered a request with a refusal, so it did nothing: a 4xx answer. Any other error may come after
259/// Docker carried the request out.
260fn refused(error: &Error) -> bool {
261    matches!(
262        error,
263        Error::DockerResponseServerError {
264            status_code: 400..=499,
265            ..
266        }
267    )
268}
269
270/// Whether a Docker error says the Engine did not answer at all: its socket is gone, refuses connections or closed
271/// the connection, as while the machine shuts down. Not an answer the Engine gave, nor a timeout, which a busy Engine
272/// also gives.
273pub fn is_unreachable(error: &Error) -> bool {
274    matches!(
275        error,
276        Error::HyperLegacyError { .. }
277            | Error::HyperResponseError { .. }
278            | Error::IOError { .. }
279            | Error::SocketNotFoundError(_)
280    )
281}
282
283/// Whether Docker answered that it cannot or will not do what was asked: a 4xx, or 501 for what it lacks, such as
284/// the log of a container whose log driver keeps none. Asking again gets the same answer; any other error may not.
285pub fn is_declined(error: &Error) -> bool {
286    matches!(
287        error,
288        Error::DockerResponseServerError {
289            status_code: 400..=499 | 501,
290            ..
291        }
292    )
293}
294
295/// Whether a Docker error says the container, volume or image does not exist.
296pub fn is_not_found(error: &Error) -> bool {
297    matches!(
298        error,
299        Error::DockerResponseServerError {
300            status_code: 404,
301            ..
302        }
303    )
304}
305
306/// The Docker Engine calls the scheduler and the caches make. `Docker` is the real one; the engine's tests use a
307/// fake.
308pub trait DockerApi: Send + Sync {
309    /// Creates and starts a runner container; returns its ID. A start that fails removes what it made, and says
310    /// which container is left when it cannot.
311    fn start(
312        &self,
313        spec: &RunnerSpec,
314        isolation: &dyn Isolation,
315    ) -> impl Future<Output = Result<String, StartError>> + Send;
316    /// A container's state and isolation.
317    fn inspect(&self, container: &str) -> impl Future<Output = Result<RunnerState, Error>> + Send;
318    /// Removes a runner container, stopping it first if it runs.
319    fn remove(&self, container: &str) -> impl Future<Output = Result<(), Error>> + Send;
320    /// Creates and starts a container that is not a runner, such as the mirror; returns its ID.
321    fn run(
322        &self,
323        name: &str,
324        body: ContainerCreateBody,
325    ) -> impl Future<Output = Result<String, Error>> + Send;
326    /// Every container, running or not, that has the label.
327    fn list(&self, label: &str) -> impl Future<Output = Result<Vec<Container>, Error>> + Send;
328    /// Waits for a container to exit; returns its exit code.
329    fn wait(&self, container: &str) -> impl Future<Output = Result<i64, Error>> + Send;
330    /// A container's output so far, stdout and stderr interleaved.
331    fn logs(&self, container: &str) -> impl Future<Output = Result<String, Error>> + Send;
332    /// A container's output since `since` (seconds since the epoch), each line prefixed with Docker's RFC 3339
333    /// timestamp and a space.
334    fn logs_since(
335        &self,
336        container: &str,
337        since: u64,
338    ) -> impl Future<Output = Result<String, Error>> + Send;
339    /// Runs a command as root in a running container; its stdout streams until it exits or the container stops.
340    /// Its stderr is dropped.
341    fn exec(
342        &self,
343        container: &str,
344        command: Vec<String>,
345    ) -> impl Future<Output = Result<Output, Error>> + Send;
346    /// Whether Docker has an image.
347    fn image_present(&self, image: &str) -> impl Future<Output = Result<bool, Error>> + Send;
348    /// Pulls an image, logging its progress every few seconds.
349    fn pull(&self, image: &str) -> impl Future<Output = Result<(), Error>> + Send;
350    /// Creates a bridge network unless it exists.
351    fn ensure_network(&self, name: &str) -> impl Future<Output = Result<(), Error>> + Send;
352    /// Creates a volume unless it exists.
353    fn ensure_volume(&self, name: &str) -> impl Future<Output = Result<(), Error>> + Send;
354    /// Removes a volume; a volume that does not exist is not an error.
355    fn remove_volume(&self, name: &str) -> impl Future<Output = Result<(), Error>> + Send;
356    /// The disk size of every volume Docker knows, by name. Slow: Docker measures each volume.
357    fn volume_sizes(&self) -> impl Future<Output = Result<BTreeMap<String, u64>, Error>> + Send;
358    /// The paths a container's own layer added or changed, not those it deleted.
359    fn changes(&self, container: &str) -> impl Future<Output = Result<Vec<String>, Error>> + Send;
360    /// The bytes a container's own layer holds.
361    fn written(&self, container: &str) -> impl Future<Output = Result<u64, Error>> + Send;
362    /// The files directly in a container's `directory` whose names start with `prefix`, not those in the
363    /// directories below it, read from its filesystem whether it runs or stopped, without starting it.
364    fn files(
365        &self,
366        container: &str,
367        directory: &str,
368        prefix: &str,
369    ) -> impl Future<Output = Result<Files, Error>> + Send;
370    /// Commits a stopped container's own layer as the image `reference` (`repository:tag`) with `labels`; returns
371    /// the disk the image takes beyond its parent.
372    fn commit(
373        &self,
374        container: &str,
375        reference: &str,
376        labels: BTreeMap<String, String>,
377    ) -> impl Future<Output = Result<u64, Error>> + Send;
378    /// Every image with the label.
379    fn images(&self, label: &str) -> impl Future<Output = Result<Vec<Image>, Error>> + Send;
380    /// The ID of the image a reference (tag, digest or ID) names; `None` when Docker does not have it.
381    fn image_id(
382        &self,
383        reference: &str,
384    ) -> impl Future<Output = Result<Option<String>, Error>> + Send;
385    /// Every image Docker has a tag for.
386    fn tagged_images(&self) -> impl Future<Output = Result<Vec<TaggedImage>, Error>> + Send;
387    /// The image ID of every container, running or stopped.
388    fn container_images(&self) -> impl Future<Output = Result<BTreeSet<String>, Error>> + Send;
389    /// Removes an image and the layers only it used; one that does not exist is not an error.
390    fn remove_image(&self, reference: &str) -> impl Future<Output = Result<(), Error>> + Send;
391    /// The free disk where Docker keeps its images and containers.
392    fn free_space(&self) -> impl Future<Output = Result<u64, Error>> + Send;
393    /// Creates a container with a terminal of `size`, attaches to it and starts it. The container's stdin closes
394    /// when `Tty::input` is dropped.
395    fn shell(
396        &self,
397        name: &str,
398        body: ContainerCreateBody,
399        size: TtySize,
400    ) -> impl Future<Output = Result<Tty, Error>> + Send;
401}
402
403/// The local Docker Engine.
404#[derive(Debug)]
405pub struct Docker {
406    api: bollard::Docker,
407}
408
409impl Docker {
410    /// Removes the container a failed start `made`, if any; the start's `error` is the one worth reporting. The
411    /// container stays in the answer unless its removal is confirmed.
412    async fn clean_up(&self, made: Option<String>, error: Error) -> StartError {
413        let container = match made {
414            Some(container) => match self.remove(&container).await {
415                Err(failed) if !is_not_found(&failed) => Some(container),
416                _ => None,
417            },
418            None => None,
419        };
420        StartError { container, error }
421    }
422
423    /// Connects to the Docker Engine at `DOCKER_HOST`, or its local socket. Only `unix://` hosts are supported.
424    pub fn connect() -> Result<Docker, Error> {
425        connect_to(std::env::var("DOCKER_HOST").ok().as_deref())
426    }
427
428    /// The Engine's system information.
429    pub async fn info(&self) -> Result<SystemInfo, Error> {
430        self.api.info().await
431    }
432
433    /// An image's labels; `None` when Docker does not have it.
434    pub async fn image_labels(
435        &self,
436        image: &str,
437    ) -> Result<Option<HashMap<String, String>>, Error> {
438        match self.api.inspect_image(image).await {
439            Ok(inspect) => Ok(Some(
440                inspect
441                    .config
442                    .and_then(|config| config.labels)
443                    .unwrap_or_default(),
444            )),
445            Err(error) if is_not_found(&error) => Ok(None),
446            Err(error) => Err(error),
447        }
448    }
449
450    /// Asks the image's registry for its manifest, without pulling it, and returns the platforms it is built for,
451    /// such as `linux/amd64`. An error means `pull` could not get it.
452    pub async fn registry_platforms(&self, image: &str) -> Result<Vec<String>, Error> {
453        let inspect = self.api.inspect_registry_image(image, None).await?;
454        Ok(inspect
455            .platforms
456            .into_iter()
457            .filter_map(|platform| Some(format!("{}/{}", platform.os?, platform.architecture?)))
458            // Attestation manifests in an index carry the platform unknown/unknown.
459            .filter(|platform| platform != "unknown/unknown")
460            .collect())
461    }
462}
463
464impl DockerApi for Docker {
465    async fn start(
466        &self,
467        spec: &RunnerSpec,
468        isolation: &dyn Isolation,
469    ) -> Result<String, StartError> {
470        let options = CreateContainerOptionsBuilder::new()
471            .name(&spec.name)
472            .build();
473        let id = match self
474            .api
475            .create_container(Some(options), create_body(spec, isolation))
476            .await
477        {
478            Ok(created) => created.id,
479            Err(error) if refused(&error) => {
480                return Err(StartError {
481                    container: None,
482                    error,
483                });
484            }
485            // Docker may have made it and lost the answer: its name, unique to this start, finds it.
486            Err(error) => {
487                let made = match self.api.inspect_container(&spec.name, None).await {
488                    Ok(found) => Some(found.id.unwrap_or_else(|| spec.name.clone())),
489                    Err(missing) if is_not_found(&missing) => None,
490                    Err(_) => Some(spec.name.clone()),
491                };
492                return Err(self.clean_up(made, error).await);
493            }
494        };
495        let started = async {
496            if let Some(jit_config) = &spec.jit_config {
497                let options = UploadToContainerOptionsBuilder::new().path("/").build();
498                let tar = jit_config_tar(jit_config)?;
499                // As a stream: bollard traces a full body's bytes, but a stream only as "(stream)".
500                let tar = futures_util::stream::once(std::future::ready(tar.into()));
501                self.api
502                    .upload_to_container(&id, Some(options), bollard::body_stream(tar))
503                    .await?;
504            }
505            self.api.start_container(&id, None).await
506        };
507        if let Err(error) = started.await {
508            return Err(self.clean_up(Some(id), error).await);
509        }
510        Ok(id)
511    }
512
513    async fn inspect(&self, container: &str) -> Result<RunnerState, Error> {
514        let response = self.api.inspect_container(container, None).await?;
515        let state = response.state.unwrap_or_default();
516        let host_config = response.host_config.unwrap_or_default();
517        let address = response
518            .network_settings
519            .and_then(|settings| settings.networks)
520            .and_then(|networks| {
521                networks
522                    .into_values()
523                    .find_map(|network| network.ip_address.filter(|address| !address.is_empty()))
524            });
525        Ok(RunnerState {
526            running: state.running.unwrap_or(false),
527            exit_code: state.exit_code,
528            oom_killed: state.oom_killed.unwrap_or(false),
529            privileged: host_config.privileged.unwrap_or(false),
530            runtime: host_config.runtime,
531            pid: state
532                .pid
533                .and_then(|pid| u32::try_from(pid).ok())
534                .filter(|pid| *pid > 0),
535            address,
536            started_at: state.started_at.as_deref().and_then(crate::time::parse_ms),
537            memory: host_config
538                .memory
539                .and_then(|memory| u64::try_from(memory).ok())
540                .filter(|memory| *memory > 0),
541            cpus: host_config
542                .nano_cpus
543                .and_then(|nano| u64::try_from(nano / 1_000_000).ok())
544                .filter(|cpus| *cpus > 0),
545        })
546    }
547
548    async fn remove(&self, container: &str) -> Result<(), Error> {
549        // With its anonymous volumes, such as a shell's /state.
550        let options = RemoveContainerOptionsBuilder::new()
551            .force(true)
552            .v(true)
553            .build();
554        self.api.remove_container(container, Some(options)).await
555    }
556
557    async fn run(&self, name: &str, body: ContainerCreateBody) -> Result<String, Error> {
558        let options = CreateContainerOptionsBuilder::new().name(name).build();
559        let id = self.api.create_container(Some(options), body).await?.id;
560        if let Err(error) = self.api.start_container(&id, None).await {
561            let _ = self.remove(&id).await;
562            return Err(error);
563        }
564        Ok(id)
565    }
566
567    async fn list(&self, label: &str) -> Result<Vec<Container>, Error> {
568        let options = ListContainersOptionsBuilder::new()
569            .all(true)
570            .filters(&HashMap::from([("label", vec![label])]))
571            .build();
572        let containers = self.api.list_containers(Some(options)).await?;
573        Ok(containers
574            .into_iter()
575            .map(|container| Container {
576                id: container.id.unwrap_or_default(),
577                name: container
578                    .names
579                    .and_then(|names| names.into_iter().next())
580                    .unwrap_or_default()
581                    .trim_start_matches('/')
582                    .to_owned(),
583                running: container.state
584                    == Some(bollard::models::ContainerSummaryStateEnum::RUNNING),
585                labels: container.labels.unwrap_or_default(),
586            })
587            .collect())
588    }
589
590    async fn wait(&self, container: &str) -> Result<i64, Error> {
591        let options = WaitContainerOptionsBuilder::new()
592            .condition("not-running")
593            .build();
594        let mut responses = self.api.wait_container(container, Some(options));
595        match responses.next().await {
596            Some(Ok(response)) => Ok(response.status_code),
597            // Bollard reports a non-zero exit as an error.
598            Some(Err(Error::DockerContainerWaitError { code, .. })) => Ok(code),
599            Some(Err(error)) => Err(error),
600            None => Ok(0),
601        }
602    }
603
604    async fn logs(&self, container: &str) -> Result<String, Error> {
605        let options = LogsOptionsBuilder::new().stdout(true).stderr(true).build();
606        let chunks: Vec<_> = self
607            .api
608            .logs(container, Some(options))
609            .try_collect()
610            .await?;
611        Ok(chunks.iter().map(ToString::to_string).collect())
612    }
613
614    async fn logs_since(&self, container: &str, since: u64) -> Result<String, Error> {
615        let options = LogsOptionsBuilder::new()
616            .stdout(true)
617            .stderr(true)
618            .timestamps(true)
619            .since(i32::try_from(since).unwrap_or(i32::MAX))
620            .build();
621        let chunks: Vec<_> = self
622            .api
623            .logs(container, Some(options))
624            .try_collect()
625            .await?;
626        Ok(chunks.iter().map(ToString::to_string).collect())
627    }
628
629    async fn exec(&self, container: &str, command: Vec<String>) -> Result<Output, Error> {
630        let options = CreateExecOptions {
631            cmd: Some(command),
632            user: Some("root".to_owned()),
633            attach_stdout: Some(true),
634            attach_stderr: Some(true),
635            ..Default::default()
636        };
637        let exec = self.api.create_exec(container, options).await?;
638        match self.api.start_exec(&exec.id, None).await? {
639            StartExecResults::Attached { output, .. } => Ok(output
640                .try_filter_map(|output| {
641                    std::future::ready(Ok(match output {
642                        bollard::container::LogOutput::StdOut { message } => Some(message.to_vec()),
643                        _ => None,
644                    }))
645                })
646                .boxed()),
647            StartExecResults::Detached => Ok(futures_util::stream::empty().boxed()),
648        }
649    }
650
651    async fn image_present(&self, image: &str) -> Result<bool, Error> {
652        Ok(self.image_labels(image).await?.is_some())
653    }
654
655    async fn pull(&self, image: &str) -> Result<(), Error> {
656        let options = CreateImageOptionsBuilder::new().from_image(image).build();
657        let mut stream = self.api.create_image(Some(options), None, None);
658        let mut progress = PullProgress::default();
659        let mut logged = Instant::now();
660        info!("pulling {image}");
661        while let Some(update) = stream.try_next().await? {
662            progress.update(&update);
663            if logged.elapsed() >= PULL_LOG_INTERVAL {
664                logged = Instant::now();
665                info!("pulling {image}: {progress}");
666            }
667        }
668        info!("pulled {image}");
669        Ok(())
670    }
671
672    async fn ensure_network(&self, name: &str) -> Result<(), Error> {
673        match self.api.inspect_network(name, None).await {
674            Err(error) if is_not_found(&error) => {
675                let request = NetworkCreateRequest {
676                    name: name.to_owned(),
677                    driver: Some("bridge".to_owned()),
678                    ..Default::default()
679                };
680                self.api.create_network(request).await.map(|_| ())
681            }
682            other => other.map(|_| ()),
683        }
684    }
685
686    async fn ensure_volume(&self, name: &str) -> Result<(), Error> {
687        // Creating a volume that exists is a no-op in Docker.
688        let request = VolumeCreateRequest {
689            name: Some(name.to_owned()),
690            ..Default::default()
691        };
692        self.api.create_volume(request).await.map(|_| ())
693    }
694
695    async fn remove_volume(&self, name: &str) -> Result<(), Error> {
696        let options = RemoveVolumeOptionsBuilder::new().build();
697        match self.api.remove_volume(name, Some(options)).await {
698            Err(error) if is_not_found(&error) => Ok(()),
699            other => other,
700        }
701    }
702
703    async fn volume_sizes(&self) -> Result<BTreeMap<String, u64>, Error> {
704        // Without a type filter: bollard cannot encode one in the query.
705        let options = DataUsageOptionsBuilder::new().verbose(true).build();
706        let usage = self.api.df(Some(options)).await?;
707        // API 1.53 leaves the items untyped; they are volumes as `GET /volumes` describes them.
708        Ok(usage
709            .volume_usage
710            .and_then(|volumes| volumes.items)
711            .unwrap_or_default()
712            .into_iter()
713            .filter_map(|volume| {
714                let name = volume.get("Name")?.as_str()?.to_owned();
715                let size = volume.get("UsageData")?.get("Size")?.as_u64()?;
716                Some((name, size))
717            })
718            .collect())
719    }
720
721    async fn changes(&self, container: &str) -> Result<Vec<String>, Error> {
722        Ok(self
723            .api
724            .container_changes(container)
725            .await?
726            .unwrap_or_default()
727            .into_iter()
728            .filter(|change| change.kind != ChangeType::_2)
729            .map(|change| change.path)
730            .collect())
731    }
732
733    async fn written(&self, container: &str) -> Result<u64, Error> {
734        let options = InspectContainerOptionsBuilder::new().size(true).build();
735        let response = self.api.inspect_container(container, Some(options)).await?;
736        Ok(response
737            .size_rw
738            .and_then(|bytes| u64::try_from(bytes).ok())
739            .unwrap_or(0))
740    }
741
742    async fn files(&self, container: &str, directory: &str, prefix: &str) -> Result<Files, Error> {
743        let options = DownloadFromContainerOptionsBuilder::new()
744            .path(directory)
745            .build();
746        let mut archive = self.api.download_from_container(container, Some(options));
747        // The archive is read as it arrives, by a blocking reader, so what it skips is never held.
748        let (chunks, received) = tokio::sync::mpsc::channel(4);
749        let prefix = prefix.to_owned();
750        let reader = tokio::task::spawn_blocking(move || {
751            files_in(
752                Chunks {
753                    received,
754                    chunk: Vec::new(),
755                    at: 0,
756                },
757                &prefix,
758            )
759        });
760        let mut failed = None;
761        while let Some(chunk) = archive.next().await {
762            match chunk {
763                // Fails once the reader is done, at the archive's end or `FILES_LIMIT`.
764                Ok(chunk) => {
765                    if chunks.send(chunk.to_vec()).await.is_err() {
766                        break;
767                    }
768                }
769                Err(error) => {
770                    failed = Some(error);
771                    break;
772                }
773            }
774        }
775        drop(chunks);
776        let files = match reader.await {
777            Ok(files) => files,
778            Err(error) => match error.try_into_panic() {
779                Ok(panic) => std::panic::resume_unwind(panic),
780                // Only while the runtime shuts down.
781                Err(_) => Files::default(),
782            },
783        };
784        match failed {
785            Some(error) => Err(error),
786            None => Ok(files),
787        }
788    }
789
790    async fn commit(
791        &self,
792        container: &str,
793        reference: &str,
794        labels: BTreeMap<String, String>,
795    ) -> Result<u64, Error> {
796        let (repository, tag) = reference.rsplit_once(':').unwrap_or((reference, "latest"));
797        let options = CommitContainerOptionsBuilder::new()
798            .container(container)
799            .repo(repository)
800            .tag(tag)
801            .pause(false)
802            .build();
803        let config = ContainerConfig {
804            labels: Some(labels.into_iter().collect()),
805            ..Default::default()
806        };
807        // The image the container runs is the record's parent.
808        let parent = match self.api.inspect_container(container, None).await?.image {
809            Some(parent) => self.api.inspect_image(&parent).await?.size.unwrap_or(0),
810            None => 0,
811        };
812        let id = self.api.commit_container(options, config).await?.id;
813        let size = self.api.inspect_image(&id).await?.size.unwrap_or(0);
814        Ok(u64::try_from(size - parent).unwrap_or(0))
815    }
816
817    async fn images(&self, label: &str) -> Result<Vec<Image>, Error> {
818        let options = ListImagesOptionsBuilder::new()
819            .filters(&HashMap::from([("label", vec![label])]))
820            .build();
821        let images = self.api.list_images(Some(options)).await?;
822        let mut parents: HashMap<String, i64> = HashMap::new();
823        let mut found = Vec::new();
824        for image in images {
825            let parent = if image.parent_id.is_empty() {
826                0
827            } else if let Some(size) = parents.get(&image.parent_id) {
828                *size
829            } else {
830                let size = self
831                    .api
832                    .inspect_image(&image.parent_id)
833                    .await
834                    .ok()
835                    .and_then(|parent| parent.size)
836                    .unwrap_or(0);
837                parents.insert(image.parent_id.clone(), size);
838                size
839            };
840            found.push(Image {
841                reference: image.repo_tags.into_iter().next().unwrap_or(image.id),
842                labels: image.labels,
843                bytes: u64::try_from(image.size - parent).unwrap_or(0),
844            });
845        }
846        Ok(found)
847    }
848
849    async fn image_id(&self, reference: &str) -> Result<Option<String>, Error> {
850        match self.api.inspect_image(reference).await {
851            Ok(inspect) => Ok(inspect.id),
852            Err(error) if is_not_found(&error) => Ok(None),
853            Err(error) => Err(error),
854        }
855    }
856
857    async fn tagged_images(&self) -> Result<Vec<TaggedImage>, Error> {
858        let images = self
859            .api
860            .list_images(Some(ListImagesOptionsBuilder::new().build()))
861            .await?;
862        Ok(images
863            .into_iter()
864            .map(|image| TaggedImage {
865                id: image.id,
866                tags: image
867                    .repo_tags
868                    .into_iter()
869                    .filter(|tag| tag != "<none>:<none>")
870                    .collect(),
871                digests: image
872                    .repo_digests
873                    .into_iter()
874                    .filter(|digest| digest != "<none>@<none>")
875                    .collect(),
876            })
877            .filter(|image| !image.tags.is_empty())
878            .collect())
879    }
880
881    async fn container_images(&self) -> Result<BTreeSet<String>, Error> {
882        let options = ListContainersOptionsBuilder::new().all(true).build();
883        let containers = self.api.list_containers(Some(options)).await?;
884        Ok(containers
885            .into_iter()
886            .filter_map(|container| container.image_id)
887            .collect())
888    }
889
890    async fn remove_image(&self, reference: &str) -> Result<(), Error> {
891        let options = RemoveImageOptionsBuilder::new().build();
892        match self.api.remove_image(reference, Some(options), None).await {
893            Err(error) if is_not_found(&error) => Ok(()),
894            other => other.map(|_| ()),
895        }
896    }
897
898    async fn free_space(&self) -> Result<u64, Error> {
899        let root = self.api.info().await?.docker_root_dir.unwrap_or_default();
900        // Only the directories above Docker's root need to be searchable: the daemon's user cannot read the root.
901        let stat = rustix::fs::statvfs(root.as_str()).map_err(|error| Error::IOError {
902            err: std::io::Error::other(format!("cannot measure the disk of {root}: {error}")),
903        })?;
904        Ok(stat.f_bavail.saturating_mul(stat.f_frsize))
905    }
906
907    async fn shell(
908        &self,
909        name: &str,
910        body: ContainerCreateBody,
911        size: TtySize,
912    ) -> Result<Tty, Error> {
913        let options = CreateContainerOptionsBuilder::new().name(name).build();
914        let id = self.api.create_container(Some(options), body).await?.id;
915        let attached = async {
916            let options = AttachContainerOptionsBuilder::new()
917                .stream(true)
918                .stdin(true)
919                .stdout(true)
920                .stderr(true)
921                .build();
922            let attached = self.api.attach_container(&id, Some(options)).await?;
923            self.api.start_container(&id, None).await?;
924            let resize = ResizeContainerTTYOptionsBuilder::new()
925                .w(i32::from(size.cols))
926                .h(i32::from(size.rows))
927                .build();
928            self.api.resize_container_tty(&id, resize).await?;
929            Ok::<_, Error>(attached)
930        };
931        match attached.await {
932            Ok(attached) => Ok(Tty {
933                container: id,
934                output: attached
935                    .output
936                    .map_ok(|output| output.into_bytes().to_vec())
937                    .boxed(),
938                input: attached.input,
939            }),
940            Err(error) => {
941                let _ = self.remove(&id).await;
942                Err(error)
943            }
944        }
945    }
946}
947
948/// How often `pull` logs its progress.
949const PULL_LOG_INTERVAL: Duration = Duration::from_secs(10);
950
951/// A pull's layers so far, from the statuses Docker streams per layer.
952#[derive(Debug, Default)]
953struct PullProgress {
954    layers: BTreeMap<String, Layer>,
955}
956
957#[derive(Debug, Default)]
958struct Layer {
959    downloaded: u64,
960    size: u64,
961    done: bool,
962}
963
964impl PullProgress {
965    fn update(&mut self, update: &CreateImageInfo) {
966        let (Some(id), Some(status)) = (&update.id, &update.status) else {
967            return;
968        };
969        let bytes = |value: Option<i64>| value.and_then(|value| u64::try_from(value).ok());
970        let detail = update.progress_detail.as_ref();
971        match status.as_str() {
972            "Pulling fs layer" | "Waiting" => {
973                self.layers.entry(id.clone()).or_default();
974            }
975            "Downloading" => {
976                let layer = self.layers.entry(id.clone()).or_default();
977                layer.downloaded =
978                    bytes(detail.and_then(|detail| detail.current)).unwrap_or(layer.downloaded);
979                layer.size = bytes(detail.and_then(|detail| detail.total)).unwrap_or(layer.size);
980            }
981            "Download complete" => {
982                if let Some(layer) = self.layers.get_mut(id) {
983                    layer.downloaded = layer.size;
984                }
985            }
986            "Pull complete" | "Already exists" => {
987                let layer = self.layers.entry(id.clone()).or_default();
988                layer.downloaded = layer.size;
989                layer.done = true;
990            }
991            _ => {}
992        }
993    }
994}
995
996impl fmt::Display for PullProgress {
997    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
998        let done = self.layers.values().filter(|layer| layer.done).count();
999        let downloaded: u64 = self.layers.values().map(|layer| layer.downloaded).sum();
1000        let size: u64 = self.layers.values().map(|layer| layer.size).sum();
1001        write!(
1002            f,
1003            "{done} of {} layers done, {} of {} downloaded",
1004            self.layers.len(),
1005            ByteSize(downloaded),
1006            ByteSize(size)
1007        )
1008    }
1009}
1010
1011fn connect_to(docker_host: Option<&str>) -> Result<Docker, Error> {
1012    let api = match docker_host.filter(|host| !host.is_empty()) {
1013        None => bollard::Docker::connect_with_socket_defaults()?,
1014        Some(host) if host.starts_with("unix://") => {
1015            bollard::Docker::connect_with_socket(host, 120, bollard::API_DEFAULT_VERSION)?
1016        }
1017        // Bollard would quietly fall back to the local socket: a different Engine than the one asked for.
1018        Some(host) => {
1019            return Err(Error::UnsupportedURISchemeError {
1020                uri: host.to_owned(),
1021            });
1022        }
1023    };
1024    Ok(Docker { api })
1025}
1026
1027/// A tar archive, to extract at `/`, holding the config as a root-only file at `JIT_CONFIG_PATH`.
1028/// The chunks of an archive as they arrive, for a blocking reader: it ends when no more come.
1029struct Chunks {
1030    received: tokio::sync::mpsc::Receiver<Vec<u8>>,
1031    chunk: Vec<u8>,
1032    at: usize,
1033}
1034
1035impl std::io::Read for Chunks {
1036    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
1037        while self.at == self.chunk.len() {
1038            match self.received.blocking_recv() {
1039                Some(chunk) => {
1040                    self.chunk = chunk;
1041                    self.at = 0;
1042                }
1043                None => return Ok(0),
1044            }
1045        }
1046        let n = buf.len().min(self.chunk.len() - self.at);
1047        buf[..n].copy_from_slice(&self.chunk[self.at..self.at + n]);
1048        self.at += n;
1049        Ok(n)
1050    }
1051}
1052
1053/// A reader that notes whether its last read found the end: `tar` reads past a zero block, the archive's end, no
1054/// further, and takes running out between entries as the end too.
1055struct Ends<R> {
1056    inner: R,
1057    ran_out: bool,
1058}
1059
1060impl<R: std::io::Read> std::io::Read for Ends<R> {
1061    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
1062        let n = self.inner.read(buf)?;
1063        self.ran_out = n == 0 && !buf.is_empty();
1064        Ok(n)
1065    }
1066}
1067
1068/// The regular files at the top of a directory's `tar` whose names start with `prefix`. Docker roots the archive at
1069/// the directory's own name. Every other entry is read through and dropped, so only the files kept count against
1070/// `FILES_LIMIT`. Complete only once the archive's end marker is read: one that breaks off, between entries too, or
1071/// a file cut at the limit is not; what was read is kept.
1072pub(crate) fn files_in(tar: impl std::io::Read, prefix: &str) -> Files {
1073    let mut tar = Ends {
1074        inner: tar,
1075        ran_out: false,
1076    };
1077    let mut read = Files::default();
1078    let ended = files_until_end(&mut tar, prefix, &mut read.files);
1079    read.complete = ended && !tar.ran_out;
1080    read
1081}
1082
1083/// Reads `tar` into `files` as `files_in` says; whether it got to the archive's end, which `files_in` checks was its
1084/// marker.
1085fn files_until_end(tar: impl std::io::Read, prefix: &str, files: &mut Vec<File>) -> bool {
1086    let mut archive = tar::Archive::new(tar);
1087    let Ok(entries) = archive.entries() else {
1088        return false;
1089    };
1090    let mut kept = 0;
1091    for entry in entries {
1092        let Ok(mut entry) = entry else {
1093            return false;
1094        };
1095        let name = match entry.path() {
1096            Ok(path) if path.components().count() == 2 => path
1097                .file_name()
1098                .map(|name| name.to_string_lossy().into_owned()),
1099            _ => None,
1100        };
1101        let Some(name) =
1102            name.filter(|name| name.starts_with(prefix) && entry.header().entry_type().is_file())
1103        else {
1104            continue;
1105        };
1106        let room = FILES_LIMIT - kept;
1107        let mut contents = Vec::new();
1108        // One byte past the room tells a file cut at the limit from one that fits it exactly.
1109        let whole = std::io::Read::read_to_end(
1110            &mut std::io::Read::take(&mut entry, room as u64 + 1),
1111            &mut contents,
1112        )
1113        .is_ok()
1114            && contents.len() <= room;
1115        contents.truncate(room);
1116        kept += contents.len();
1117        files.push(File {
1118            name,
1119            contents: String::from_utf8_lossy(&contents).into_owned(),
1120        });
1121        if !whole {
1122            return false;
1123        }
1124    }
1125    true
1126}
1127
1128fn jit_config_tar(jit_config: &JitConfig) -> std::io::Result<Vec<u8>> {
1129    let mut tar = tar::Builder::new(Vec::new());
1130    let path = JIT_CONFIG_PATH.trim_start_matches('/');
1131    let mut dir = tar::Header::new_gnu();
1132    dir.set_entry_type(tar::EntryType::Directory);
1133    dir.set_mode(0o700);
1134    dir.set_size(0);
1135    tar.append_data(
1136        &mut dir,
1137        path.rsplit_once('/').map_or("", |(dir, _)| dir),
1138        std::io::empty(),
1139    )?;
1140    let mut file = tar::Header::new_gnu();
1141    file.set_mode(0o400);
1142    file.set_size(jit_config.0.len() as u64);
1143    tar.append_data(&mut file, path, jit_config.0.as_bytes())?;
1144    tar.into_inner()
1145}
1146
1147#[cfg(test)]
1148pub(crate) mod tests;