Skip to main content

runnerd_core/
docker.rs

1//! Runner containers through the Docker Engine API: the connection, container, volume and image requests, and the
2//! `DockerApi` trait the engine and the caches call. It decides nothing about scheduling, and a JIT config only
3//! ever reaches a container as a root-only file, never the create request or a log line.
4
5use std::collections::{BTreeMap, BTreeSet, HashMap};
6use std::fmt;
7use std::time::{Duration, Instant};
8
9use bollard::exec::{CreateExecOptions, StartExecResults};
10use std::pin::Pin;
11
12use bollard::models::{
13    ChangeType, ContainerConfig, ContainerCreateBody, CreateImageInfo, HostConfig, Mount,
14    NetworkCreateRequest, SystemInfo, VolumeCreateRequest,
15};
16use bollard::query_parameters::{
17    AttachContainerOptionsBuilder, CommitContainerOptionsBuilder, CreateContainerOptionsBuilder,
18    CreateImageOptionsBuilder, DataUsageOptionsBuilder, DownloadFromContainerOptionsBuilder,
19    EventsOptionsBuilder, InspectContainerOptionsBuilder, ListContainersOptionsBuilder,
20    ListImagesOptionsBuilder, LogsOptionsBuilder, RemoveContainerOptionsBuilder,
21    RemoveImageOptionsBuilder, RemoveVolumeOptionsBuilder, ResizeContainerTTYOptionsBuilder,
22    UploadToContainerOptionsBuilder, WaitContainerOptionsBuilder,
23};
24use futures_util::stream::BoxStream;
25use futures_util::{StreamExt, TryStreamExt};
26use tracing::info;
27
28use crate::config::ByteSize;
29use crate::isolation::Isolation;
30
31pub use bollard::errors::Error;
32
33/// One runner container: a profile's image stage with the profile's limits.
34#[derive(Debug, Clone)]
35pub struct RunnerSpec {
36    /// The container's name; for a runner, also the runner's.
37    pub name: String,
38    /// The image, such as `runnerd-runner:base`.
39    pub image: String,
40    /// The profile, put on the `runnerd.profile` label.
41    pub profile: String,
42    /// The systemd slice the container is placed in.
43    pub slice: String,
44    /// CPUs the container may use.
45    pub cpus: u32,
46    /// Memory the container may use, swap included.
47    pub memory: ByteSize,
48    /// `NAME=value` environment variables.
49    pub env: Vec<String>,
50    /// Volumes to mount.
51    pub mounts: Vec<Mount>,
52    /// The network the container joins; `None` is Docker's default bridge.
53    pub network: Option<String>,
54    /// Labels on top of `runnerd.profile` and `runnerd.isolation`.
55    pub labels: BTreeMap<String, String>,
56    /// Replaces the image's entrypoint; `None` runs the runner.
57    pub entrypoint: Option<Vec<String>>,
58    /// The runner's just-in-time config, which the entrypoint reads from `JIT_CONFIG_PATH`.
59    pub jit_config: Option<JitConfig>,
60}
61
62/// Where `start` puts the just-in-time config in the container: a root-only file, never the create request,
63/// which Docker stores and clients may log.
64pub const JIT_CONFIG_PATH: &str = "/run/runnerd/jitconfig";
65
66/// A runner's just-in-time config from GitHub: credentials, never printed.
67#[derive(Clone)]
68pub struct JitConfig(pub String);
69
70impl fmt::Debug for JitConfig {
71    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
72        f.write_str("JitConfig(<redacted>)")
73    }
74}
75
76/// What `inspect` reports about a runner container.
77#[derive(Debug, Clone, PartialEq, Eq)]
78pub struct RunnerState {
79    /// Whether the container runs.
80    pub running: bool,
81    /// Its exit code, once it stopped.
82    pub exit_code: Option<i64>,
83    /// Whether the kernel killed a process in it for memory, as Docker saw.
84    pub oom_killed: bool,
85    /// True under `unsafe-dind`: a job in the container can take over the host.
86    pub privileged: bool,
87    /// The runtime Docker runs it with, such as `sysbox-runc`.
88    pub runtime: Option<String>,
89    /// Its main process on the host, while it runs.
90    pub pid: Option<u32>,
91    /// Its address on its first network.
92    pub address: Option<String>,
93    /// When it started, in milliseconds since the epoch.
94    pub started_at: Option<u64>,
95    /// When it stopped, in milliseconds since the epoch; `None` while it runs.
96    pub finished_at: Option<u64>,
97    /// Its memory cap in bytes; `None` when it has none.
98    pub memory: Option<u64>,
99    /// Its CPU limit in millicores; `None` when it has none.
100    pub cpus: Option<u64>,
101}
102
103/// What a command run in a container prints on stdout, as it prints it.
104pub type Output = BoxStream<'static, Result<Vec<u8>, Error>>;
105
106/// An image `images` found.
107#[derive(Debug, Clone, PartialEq, Eq)]
108pub struct Image {
109    /// Its first tag, such as `runnerd-record:42`; its ID when it has none.
110    pub reference: String,
111    /// Its labels.
112    pub labels: HashMap<String, String>,
113    /// The disk it takes beyond its parent image: what removing it frees.
114    pub bytes: u64,
115}
116
117/// An image `tagged_images` found.
118#[derive(Debug, Clone, PartialEq, Eq)]
119pub struct TaggedImage {
120    /// Its ID, such as `sha256:224e…`.
121    pub id: String,
122    /// Its tags, such as `ghcr.io/ayagmar/runnerd-runner:node-0.1.0`.
123    pub tags: Vec<String>,
124    /// The registry digests it was pulled by, such as `ghcr.io/ayagmar/runnerd-runner@sha256:9e79…`.
125    pub digests: Vec<String>,
126}
127
128/// A file `files` read from a container.
129#[derive(Debug, Clone, PartialEq, Eq)]
130pub struct File {
131    /// Its name in the directory read, such as `Worker_20261031-120000-utc.log`.
132    pub name: String,
133    /// What it holds, as text: invalid UTF-8 is replaced. Cut short past `FILES_LIMIT`.
134    pub contents: String,
135}
136
137/// What `files` read of a directory.
138#[derive(Debug, Clone, Default, PartialEq, Eq)]
139pub struct Files {
140    /// The files read, in the archive's order.
141    pub files: Vec<File>,
142    /// Whether they are all there and whole: the archive ended as a tar does, and none was cut at `FILES_LIMIT`. A
143    /// file missing from an incomplete read may be in the directory.
144    pub complete: bool,
145}
146
147/// The most of the files `files` keeps, together. What it skips, such as a runner's job log pages below `_diag`,
148/// does not count: it is read through and dropped.
149pub(crate) const FILES_LIMIT: usize = 16 << 20;
150
151/// A container's terminal, attached once it was created and before it started, so none of its output is missed.
152pub struct Tty {
153    /// The container's ID.
154    pub container: String,
155    /// What the terminal shows, as it shows it; it ends when the container's process exits.
156    pub output: BoxStream<'static, Result<Vec<u8>, Error>>,
157    /// What is typed into it.
158    pub input: Pin<Box<dyn tokio::io::AsyncWrite + Send>>,
159}
160
161impl fmt::Debug for Tty {
162    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
163        f.debug_struct("Tty")
164            .field("container", &self.container)
165            .finish_non_exhaustive()
166    }
167}
168
169/// A terminal's size.
170#[derive(Debug, Clone, Copy, PartialEq, Eq)]
171pub struct TtySize {
172    /// Columns.
173    pub cols: u16,
174    /// Rows.
175    pub rows: u16,
176}
177
178/// The CPU shares of a runner's container or a record's shell with `cpus` CPUs: 1024 a CPU, within the 2 to 262144
179/// Docker takes, which it writes as the container's `cpu.weight`. They sit in the slice beside each other, so when the
180/// slice's CPUs are all in use they share them in proportion to their `cpus`, not evenly; a container with work still
181/// gets at most its `cpus` (`nano_cpus`). The mirror keeps Docker's default weight, which every job's pulls go
182/// through, and programs outside the slice share the host's CPUs with the slice by its own weight (ADR 0079).
183pub fn cpu_shares(cpus: u32) -> i64 {
184    (i64::from(cpus) * 1024).clamp(2, 262_144)
185}
186
187/// The create-container request for a runner container under an isolation backend.
188pub fn create_body(spec: &RunnerSpec, isolation: &dyn Isolation) -> ContainerCreateBody {
189    // Clamped to 8 EiB, the most Docker takes.
190    let memory = i64::try_from(spec.memory.0).unwrap_or(i64::MAX);
191    let mut host_config = HostConfig {
192        cgroup_parent: Some(spec.slice.clone()),
193        memory: Some(memory),
194        // Swap equal to memory: the container gets no swap on top of its memory.
195        memory_swap: Some(memory),
196        nano_cpus: Some(i64::from(spec.cpus) * 1_000_000_000),
197        cpu_shares: Some(cpu_shares(spec.cpus)),
198        init: Some(true),
199        mounts: Some(spec.mounts.clone()),
200        network_mode: spec.network.clone(),
201        ..Default::default()
202    };
203    isolation.configure(&mut host_config);
204    let mut labels: HashMap<String, String> = spec.labels.clone().into_iter().collect();
205    labels.insert("runnerd.profile".to_owned(), spec.profile.clone());
206    labels.insert("runnerd.isolation".to_owned(), isolation.name().to_owned());
207    ContainerCreateBody {
208        image: Some(spec.image.clone()),
209        env: Some(spec.env.clone()),
210        entrypoint: spec.entrypoint.clone(),
211        labels: Some(labels),
212        host_config: Some(host_config),
213        ..Default::default()
214    }
215}
216
217/// A container `list` found.
218#[derive(Debug, Clone, PartialEq, Eq)]
219pub struct Container {
220    /// The Docker container ID.
221    pub id: String,
222    /// Without Docker's leading `/`.
223    pub name: String,
224    /// Whether it runs.
225    pub running: bool,
226    /// Its labels.
227    pub labels: HashMap<String, String>,
228}
229
230/// A runner start that failed, with the container it may have left: Docker may have made, or even started, it before
231/// the request failed, as when its answer is lost.
232#[derive(Debug)]
233pub struct StartError {
234    /// The container the start left, by ID, or by name when Docker could not say whether one exists; `None` when
235    /// none does. It may run.
236    pub container: Option<String>,
237    /// Whether Docker may have started the container before the start failed: its runner may have taken a job,
238    /// even if the container is gone now.
239    pub may_have_run: bool,
240    /// Why the start failed.
241    pub error: Error,
242}
243
244impl fmt::Display for StartError {
245    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
246        self.error.fmt(f)
247    }
248}
249
250impl std::error::Error for StartError {
251    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
252        Some(&self.error)
253    }
254}
255
256/// For callers that go on whether or not a container was left.
257impl From<StartError> for Error {
258    fn from(failed: StartError) -> Error {
259        failed.error
260    }
261}
262
263/// Whether Docker answered a request with a refusal, so it did nothing: a 4xx answer. Any other error may come after
264/// Docker carried the request out.
265fn refused(error: &Error) -> bool {
266    matches!(
267        error,
268        Error::DockerResponseServerError {
269            status_code: 400..=499,
270            ..
271        }
272    )
273}
274
275/// Whether a Docker error says the Engine did not answer at all: its socket is gone, refuses connections or closed
276/// the connection, as while the machine shuts down. Not an answer the Engine gave, nor a timeout, which a busy Engine
277/// also gives.
278pub fn is_unreachable(error: &Error) -> bool {
279    matches!(
280        error,
281        Error::HyperLegacyError { .. }
282            | Error::HyperResponseError { .. }
283            | Error::IOError { .. }
284            | Error::SocketNotFoundError(_)
285    )
286}
287
288/// Whether Docker answered that it cannot or will not do what was asked: a 4xx, or 501 for what it lacks, such as
289/// the log of a container whose log driver keeps none. Asking again gets the same answer; any other error may not.
290pub fn is_declined(error: &Error) -> bool {
291    matches!(
292        error,
293        Error::DockerResponseServerError {
294            status_code: 400..=499 | 501,
295            ..
296        }
297    )
298}
299
300/// Whether a Docker error says the container, volume or image does not exist.
301pub fn is_not_found(error: &Error) -> bool {
302    matches!(
303        error,
304        Error::DockerResponseServerError {
305            status_code: 404,
306            ..
307        }
308    )
309}
310
311/// Whether Docker refused because it is removing the container already, as after a `docker rm -f`: it says so only in
312/// the words of its 409, which a running container's refusal shares. It may still be stopping.
313pub fn is_being_removed(error: &Error) -> bool {
314    matches!(
315        error,
316        Error::DockerResponseServerError { status_code: 409, message }
317            if message.contains("is already in progress") || message.contains("marked for removal")
318    )
319}
320
321/// Whether Docker refused an exec because the container stopped, as one that ended while it was adopted: its log
322/// is read once it is released.
323pub fn is_not_running(error: &Error) -> bool {
324    matches!(
325        error,
326        Error::DockerResponseServerError { status_code: 409, message } if message.ends_with(" is not running")
327    )
328}
329
330/// Whether a container's log is out of reach because it is gone or going. A removal under way can still fail, so this
331/// says nothing of whether the container will go.
332pub fn is_log_gone(error: &Error) -> bool {
333    is_not_found(error) || is_being_removed(error)
334}
335
336/// The Docker Engine calls the scheduler and the caches make. `Docker` is the real one; the engine's tests use a
337/// fake.
338pub trait DockerApi: Send + Sync {
339    /// Creates and starts a runner container; returns its ID. A start that fails removes what it made, and says
340    /// which container is left when it cannot.
341    fn start(
342        &self,
343        spec: &RunnerSpec,
344        isolation: &dyn Isolation,
345    ) -> impl Future<Output = Result<String, StartError>> + Send;
346    /// A container's state and isolation.
347    fn inspect(&self, container: &str) -> impl Future<Output = Result<RunnerState, Error>> + Send;
348    /// Removes a runner container, stopping it first if it runs.
349    fn remove(&self, container: &str) -> impl Future<Output = Result<(), Error>> + Send;
350    /// Creates and starts a container that is not a runner, such as the mirror; returns its ID.
351    fn run(
352        &self,
353        name: &str,
354        body: ContainerCreateBody,
355    ) -> impl Future<Output = Result<String, Error>> + Send;
356    /// Every container, running or not, that has the label.
357    fn list(&self, label: &str) -> impl Future<Output = Result<Vec<Container>, Error>> + Send;
358    /// The IDs of runner containers (labelled `runnerd.job`) as each stops, from Docker's events. A hint: Docker sends
359    /// an event once at most, none while the stream is down, and keeps none to replay. The stream fails or ends with
360    /// the connection, or when Docker restarts.
361    fn exits(&self) -> BoxStream<'static, Result<String, Error>>;
362    /// Waits for a container to exit; returns its exit code.
363    fn wait(&self, container: &str) -> impl Future<Output = Result<i64, Error>> + Send;
364    /// A container's output so far, stdout and stderr interleaved.
365    fn logs(&self, container: &str) -> impl Future<Output = Result<String, Error>> + Send;
366    /// A container's output since `since` (seconds since the epoch), each line prefixed with Docker's RFC 3339
367    /// timestamp and a space.
368    fn logs_since(
369        &self,
370        container: &str,
371        since: u64,
372    ) -> impl Future<Output = Result<String, Error>> + Send;
373    /// Runs a command as root in a running container; its stdout streams until it exits or the container stops.
374    /// Its stderr is dropped.
375    fn exec(
376        &self,
377        container: &str,
378        command: Vec<String>,
379    ) -> impl Future<Output = Result<Output, Error>> + Send;
380    /// Whether Docker has an image.
381    fn image_present(&self, image: &str) -> impl Future<Output = Result<bool, Error>> + Send;
382    /// Pulls an image, logging its progress every few seconds.
383    fn pull(&self, image: &str) -> impl Future<Output = Result<(), Error>> + Send;
384    /// Creates a bridge network unless it exists.
385    fn ensure_network(&self, name: &str) -> impl Future<Output = Result<(), Error>> + Send;
386    /// Creates a volume unless it exists.
387    fn ensure_volume(&self, name: &str) -> impl Future<Output = Result<(), Error>> + Send;
388    /// Removes a volume; a volume that does not exist is not an error.
389    fn remove_volume(&self, name: &str) -> impl Future<Output = Result<(), Error>> + Send;
390    /// The disk size of every volume Docker knows, by name. Slow: Docker measures each volume.
391    fn volume_sizes(&self) -> impl Future<Output = Result<BTreeMap<String, u64>, Error>> + Send;
392    /// The paths a container's own layer added or changed, not those it deleted.
393    fn changes(&self, container: &str) -> impl Future<Output = Result<Vec<String>, Error>> + Send;
394    /// The bytes a container's own layer holds.
395    fn written(&self, container: &str) -> impl Future<Output = Result<u64, Error>> + Send;
396    /// The files directly in a container's `directory` whose names start with `prefix`, not those in the
397    /// directories below it, read from its filesystem whether it runs or stopped, without starting it.
398    fn files(
399        &self,
400        container: &str,
401        directory: &str,
402        prefix: &str,
403    ) -> impl Future<Output = Result<Files, Error>> + Send;
404    /// Commits a stopped container's own layer as the image `reference` (`repository:tag`) with `labels`; returns
405    /// the disk the image takes beyond its parent.
406    fn commit(
407        &self,
408        container: &str,
409        reference: &str,
410        labels: BTreeMap<String, String>,
411    ) -> impl Future<Output = Result<u64, Error>> + Send;
412    /// Every image with the label.
413    fn images(&self, label: &str) -> impl Future<Output = Result<Vec<Image>, Error>> + Send;
414    /// The ID of the image a reference (tag, digest or ID) names; `None` when Docker does not have it.
415    fn image_id(
416        &self,
417        reference: &str,
418    ) -> impl Future<Output = Result<Option<String>, Error>> + Send;
419    /// Every image Docker has a tag for.
420    fn tagged_images(&self) -> impl Future<Output = Result<Vec<TaggedImage>, Error>> + Send;
421    /// The image ID of every container, running or stopped.
422    fn container_images(&self) -> impl Future<Output = Result<BTreeSet<String>, Error>> + Send;
423    /// Removes an image and the layers only it used; one that does not exist is not an error.
424    fn remove_image(&self, reference: &str) -> impl Future<Output = Result<(), Error>> + Send;
425    /// The free disk where Docker keeps its images and containers.
426    fn free_space(&self) -> impl Future<Output = Result<u64, Error>> + Send;
427    /// Creates a container with a terminal of `size`, attaches to it and starts it. The container's stdin closes
428    /// when `Tty::input` is dropped.
429    fn shell(
430        &self,
431        name: &str,
432        body: ContainerCreateBody,
433        size: TtySize,
434    ) -> impl Future<Output = Result<Tty, Error>> + Send;
435}
436
437/// The local Docker Engine.
438#[derive(Debug)]
439pub struct Docker {
440    api: bollard::Docker,
441}
442
443impl Docker {
444    /// Removes the container a failed start `made`, if any; the start's `error` is the one worth reporting. The
445    /// container stays in the answer unless its removal is confirmed.
446    async fn clean_up(&self, made: Option<String>, error: Error, may_have_run: bool) -> StartError {
447        let container = match made {
448            Some(container) => match self.remove(&container).await {
449                Err(failed) if !is_not_found(&failed) => Some(container),
450                _ => None,
451            },
452            None => None,
453        };
454        StartError {
455            container,
456            may_have_run,
457            error,
458        }
459    }
460
461    /// Connects to the Docker Engine at `DOCKER_HOST`, or its local socket. Only `unix://` hosts are supported.
462    pub fn connect() -> Result<Docker, Error> {
463        connect_to(std::env::var("DOCKER_HOST").ok().as_deref())
464    }
465
466    /// The Engine's system information.
467    pub async fn info(&self) -> Result<SystemInfo, Error> {
468        self.api.info().await
469    }
470
471    /// An image's labels; `None` when Docker does not have it.
472    pub async fn image_labels(
473        &self,
474        image: &str,
475    ) -> Result<Option<HashMap<String, String>>, Error> {
476        match self.api.inspect_image(image).await {
477            Ok(inspect) => Ok(Some(
478                inspect
479                    .config
480                    .and_then(|config| config.labels)
481                    .unwrap_or_default(),
482            )),
483            Err(error) if is_not_found(&error) => Ok(None),
484            Err(error) => Err(error),
485        }
486    }
487
488    /// Asks the image's registry for its manifest, without pulling it, and returns the platforms it is built for,
489    /// such as `linux/amd64`. An error means `pull` could not get it.
490    pub async fn registry_platforms(&self, image: &str) -> Result<Vec<String>, Error> {
491        let inspect = self.api.inspect_registry_image(image, None).await?;
492        Ok(inspect
493            .platforms
494            .into_iter()
495            .filter_map(|platform| Some(format!("{}/{}", platform.os?, platform.architecture?)))
496            // Attestation manifests in an index carry the platform unknown/unknown.
497            .filter(|platform| platform != "unknown/unknown")
498            .collect())
499    }
500}
501
502impl DockerApi for Docker {
503    async fn start(
504        &self,
505        spec: &RunnerSpec,
506        isolation: &dyn Isolation,
507    ) -> Result<String, StartError> {
508        let options = CreateContainerOptionsBuilder::new()
509            .name(&spec.name)
510            .build();
511        let id = match self
512            .api
513            .create_container(Some(options), create_body(spec, isolation))
514            .await
515        {
516            Ok(created) => created.id,
517            Err(error) if refused(&error) => {
518                return Err(StartError {
519                    container: None,
520                    may_have_run: false,
521                    error,
522                });
523            }
524            // Docker may have made it and lost the answer: its name, unique to this start, finds it.
525            Err(error) => {
526                let made = match self.api.inspect_container(&spec.name, None).await {
527                    Ok(found) => Some(found.id.unwrap_or_else(|| spec.name.clone())),
528                    Err(missing) if is_not_found(&missing) => None,
529                    Err(_) => Some(spec.name.clone()),
530                };
531                // Made at most, never started.
532                return Err(self.clean_up(made, error, false).await);
533            }
534        };
535        let uploaded = async {
536            if let Some(jit_config) = &spec.jit_config {
537                let options = UploadToContainerOptionsBuilder::new().path("/").build();
538                let tar = jit_config_tar(jit_config)?;
539                // As a stream: bollard traces a full body's bytes, but a stream only as "(stream)".
540                let tar = futures_util::stream::once(std::future::ready(tar.into()));
541                self.api
542                    .upload_to_container(&id, Some(options), bollard::body_stream(tar))
543                    .await?;
544            }
545            Ok::<(), Error>(())
546        };
547        if let Err(error) = uploaded.await {
548            return Err(self.clean_up(Some(id), error, false).await);
549        }
550        // A start whose answer is lost may have started it.
551        if let Err(error) = self.api.start_container(&id, None).await {
552            return Err(self.clean_up(Some(id), error, true).await);
553        }
554        Ok(id)
555    }
556
557    async fn inspect(&self, container: &str) -> Result<RunnerState, Error> {
558        let response = self.api.inspect_container(container, None).await?;
559        let state = response.state.unwrap_or_default();
560        let host_config = response.host_config.unwrap_or_default();
561        let address = response
562            .network_settings
563            .and_then(|settings| settings.networks)
564            .and_then(|networks| {
565                networks
566                    .into_values()
567                    .find_map(|network| network.ip_address.filter(|address| !address.is_empty()))
568            });
569        Ok(RunnerState {
570            running: state.running.unwrap_or(false),
571            exit_code: state.exit_code,
572            oom_killed: state.oom_killed.unwrap_or(false),
573            privileged: host_config.privileged.unwrap_or(false),
574            runtime: host_config.runtime,
575            pid: state
576                .pid
577                .and_then(|pid| u32::try_from(pid).ok())
578                .filter(|pid| *pid > 0),
579            address,
580            started_at: state.started_at.as_deref().and_then(crate::time::parse_ms),
581            // Docker says 0001-01-01 for one that never stopped, which reads as none.
582            finished_at: state.finished_at.as_deref().and_then(crate::time::parse_ms),
583            memory: host_config
584                .memory
585                .and_then(|memory| u64::try_from(memory).ok())
586                .filter(|memory| *memory > 0),
587            cpus: host_config
588                .nano_cpus
589                .and_then(|nano| u64::try_from(nano / 1_000_000).ok())
590                .filter(|cpus| *cpus > 0),
591        })
592    }
593
594    async fn remove(&self, container: &str) -> Result<(), Error> {
595        // With its anonymous volumes, such as a shell's /state.
596        let options = RemoveContainerOptionsBuilder::new()
597            .force(true)
598            .v(true)
599            .build();
600        self.api.remove_container(container, Some(options)).await
601    }
602
603    async fn run(&self, name: &str, body: ContainerCreateBody) -> Result<String, Error> {
604        let options = CreateContainerOptionsBuilder::new().name(name).build();
605        let id = self.api.create_container(Some(options), body).await?.id;
606        if let Err(error) = self.api.start_container(&id, None).await {
607            let _ = self.remove(&id).await;
608            return Err(error);
609        }
610        Ok(id)
611    }
612
613    async fn list(&self, label: &str) -> Result<Vec<Container>, Error> {
614        let options = ListContainersOptionsBuilder::new()
615            .all(true)
616            .filters(&HashMap::from([("label", vec![label])]))
617            .build();
618        let containers = self.api.list_containers(Some(options)).await?;
619        Ok(containers
620            .into_iter()
621            .map(|container| Container {
622                id: container.id.unwrap_or_default(),
623                name: container
624                    .names
625                    .and_then(|names| names.into_iter().next())
626                    .unwrap_or_default()
627                    .trim_start_matches('/')
628                    .to_owned(),
629                running: container.state
630                    == Some(bollard::models::ContainerSummaryStateEnum::RUNNING),
631                labels: container.labels.unwrap_or_default(),
632            })
633            .collect())
634    }
635
636    fn exits(&self) -> BoxStream<'static, Result<String, Error>> {
637        let filters = HashMap::from([
638            ("type", vec!["container"]),
639            ("event", vec!["die"]),
640            ("label", vec!["runnerd.job"]),
641        ]);
642        let options = EventsOptionsBuilder::new().filters(&filters).build();
643        self.api
644            .events(Some(options))
645            .try_filter_map(|event| std::future::ready(Ok(event.actor.and_then(|actor| actor.id))))
646            .boxed()
647    }
648
649    async fn wait(&self, container: &str) -> Result<i64, Error> {
650        let options = WaitContainerOptionsBuilder::new()
651            .condition("not-running")
652            .build();
653        let mut responses = self.api.wait_container(container, Some(options));
654        match responses.next().await {
655            Some(Ok(response)) => Ok(response.status_code),
656            // Bollard reports a non-zero exit as an error.
657            Some(Err(Error::DockerContainerWaitError { code, .. })) => Ok(code),
658            Some(Err(error)) => Err(error),
659            None => Ok(0),
660        }
661    }
662
663    async fn logs(&self, container: &str) -> Result<String, Error> {
664        let options = LogsOptionsBuilder::new().stdout(true).stderr(true).build();
665        let chunks: Vec<_> = self
666            .api
667            .logs(container, Some(options))
668            .try_collect()
669            .await?;
670        Ok(chunks.iter().map(ToString::to_string).collect())
671    }
672
673    async fn logs_since(&self, container: &str, since: u64) -> Result<String, Error> {
674        let options = LogsOptionsBuilder::new()
675            .stdout(true)
676            .stderr(true)
677            .timestamps(true)
678            .since(i32::try_from(since).unwrap_or(i32::MAX))
679            .build();
680        let chunks: Vec<_> = self
681            .api
682            .logs(container, Some(options))
683            .try_collect()
684            .await?;
685        Ok(chunks.iter().map(ToString::to_string).collect())
686    }
687
688    async fn exec(&self, container: &str, command: Vec<String>) -> Result<Output, Error> {
689        let options = CreateExecOptions {
690            cmd: Some(command),
691            user: Some("root".to_owned()),
692            attach_stdout: Some(true),
693            attach_stderr: Some(true),
694            ..Default::default()
695        };
696        let exec = self.api.create_exec(container, options).await?;
697        match self.api.start_exec(&exec.id, None).await? {
698            StartExecResults::Attached { output, .. } => Ok(output
699                .try_filter_map(|output| {
700                    std::future::ready(Ok(match output {
701                        bollard::container::LogOutput::StdOut { message } => Some(message.to_vec()),
702                        _ => None,
703                    }))
704                })
705                .boxed()),
706            StartExecResults::Detached => Ok(futures_util::stream::empty().boxed()),
707        }
708    }
709
710    async fn image_present(&self, image: &str) -> Result<bool, Error> {
711        Ok(self.image_labels(image).await?.is_some())
712    }
713
714    async fn pull(&self, image: &str) -> Result<(), Error> {
715        let options = CreateImageOptionsBuilder::new().from_image(image).build();
716        let mut stream = self.api.create_image(Some(options), None, None);
717        let mut progress = PullProgress::default();
718        let mut logged = Instant::now();
719        info!("pulling {image}");
720        while let Some(update) = stream.try_next().await? {
721            progress.update(&update);
722            if logged.elapsed() >= PULL_LOG_INTERVAL {
723                logged = Instant::now();
724                info!("pulling {image}: {progress}");
725            }
726        }
727        info!("pulled {image}");
728        Ok(())
729    }
730
731    async fn ensure_network(&self, name: &str) -> Result<(), Error> {
732        match self.api.inspect_network(name, None).await {
733            Err(error) if is_not_found(&error) => {
734                let request = NetworkCreateRequest {
735                    name: name.to_owned(),
736                    driver: Some("bridge".to_owned()),
737                    ..Default::default()
738                };
739                self.api.create_network(request).await.map(|_| ())
740            }
741            other => other.map(|_| ()),
742        }
743    }
744
745    async fn ensure_volume(&self, name: &str) -> Result<(), Error> {
746        // Creating a volume that exists is a no-op in Docker.
747        let request = VolumeCreateRequest {
748            name: Some(name.to_owned()),
749            ..Default::default()
750        };
751        self.api.create_volume(request).await.map(|_| ())
752    }
753
754    async fn remove_volume(&self, name: &str) -> Result<(), Error> {
755        let options = RemoveVolumeOptionsBuilder::new().build();
756        match self.api.remove_volume(name, Some(options)).await {
757            Err(error) if is_not_found(&error) => Ok(()),
758            other => other,
759        }
760    }
761
762    async fn volume_sizes(&self) -> Result<BTreeMap<String, u64>, Error> {
763        // Without a type filter: bollard cannot encode one in the query.
764        let options = DataUsageOptionsBuilder::new().verbose(true).build();
765        let usage = self.api.df(Some(options)).await?;
766        // API 1.53 leaves the items untyped; they are volumes as `GET /volumes` describes them.
767        Ok(usage
768            .volume_usage
769            .and_then(|volumes| volumes.items)
770            .unwrap_or_default()
771            .into_iter()
772            .filter_map(|volume| {
773                let name = volume.get("Name")?.as_str()?.to_owned();
774                let size = volume.get("UsageData")?.get("Size")?.as_u64()?;
775                Some((name, size))
776            })
777            .collect())
778    }
779
780    async fn changes(&self, container: &str) -> Result<Vec<String>, Error> {
781        Ok(self
782            .api
783            .container_changes(container)
784            .await?
785            .unwrap_or_default()
786            .into_iter()
787            .filter(|change| change.kind != ChangeType::_2)
788            .map(|change| change.path)
789            .collect())
790    }
791
792    async fn written(&self, container: &str) -> Result<u64, Error> {
793        let options = InspectContainerOptionsBuilder::new().size(true).build();
794        let response = self.api.inspect_container(container, Some(options)).await?;
795        Ok(response
796            .size_rw
797            .and_then(|bytes| u64::try_from(bytes).ok())
798            .unwrap_or(0))
799    }
800
801    async fn files(&self, container: &str, directory: &str, prefix: &str) -> Result<Files, Error> {
802        let options = DownloadFromContainerOptionsBuilder::new()
803            .path(directory)
804            .build();
805        let mut archive = self.api.download_from_container(container, Some(options));
806        // The archive is read as it arrives, by a blocking reader, so what it skips is never held.
807        let (chunks, received) = tokio::sync::mpsc::channel(4);
808        let prefix = prefix.to_owned();
809        let reader = tokio::task::spawn_blocking(move || {
810            files_in(
811                Chunks {
812                    received,
813                    chunk: Vec::new(),
814                    at: 0,
815                },
816                &prefix,
817            )
818        });
819        let mut failed = None;
820        while let Some(chunk) = archive.next().await {
821            match chunk {
822                // Fails once the reader is done, at the archive's end or `FILES_LIMIT`.
823                Ok(chunk) => {
824                    if chunks.send(chunk.to_vec()).await.is_err() {
825                        break;
826                    }
827                }
828                Err(error) => {
829                    failed = Some(error);
830                    break;
831                }
832            }
833        }
834        drop(chunks);
835        let files = match reader.await {
836            Ok(files) => files,
837            Err(error) => match error.try_into_panic() {
838                Ok(panic) => std::panic::resume_unwind(panic),
839                // Only while the runtime shuts down.
840                Err(_) => Files::default(),
841            },
842        };
843        match failed {
844            Some(error) => Err(error),
845            None => Ok(files),
846        }
847    }
848
849    async fn commit(
850        &self,
851        container: &str,
852        reference: &str,
853        labels: BTreeMap<String, String>,
854    ) -> Result<u64, Error> {
855        let (repository, tag) = reference.rsplit_once(':').unwrap_or((reference, "latest"));
856        let options = CommitContainerOptionsBuilder::new()
857            .container(container)
858            .repo(repository)
859            .tag(tag)
860            .pause(false)
861            .build();
862        let config = ContainerConfig {
863            labels: Some(labels.into_iter().collect()),
864            ..Default::default()
865        };
866        // The image the container runs is the record's parent.
867        let parent = match self.api.inspect_container(container, None).await?.image {
868            Some(parent) => self.api.inspect_image(&parent).await?.size.unwrap_or(0),
869            None => 0,
870        };
871        let id = self.api.commit_container(options, config).await?.id;
872        let size = self.api.inspect_image(&id).await?.size.unwrap_or(0);
873        Ok(u64::try_from(size - parent).unwrap_or(0))
874    }
875
876    async fn images(&self, label: &str) -> Result<Vec<Image>, Error> {
877        let options = ListImagesOptionsBuilder::new()
878            .filters(&HashMap::from([("label", vec![label])]))
879            .build();
880        let images = self.api.list_images(Some(options)).await?;
881        let mut parents: HashMap<String, i64> = HashMap::new();
882        let mut found = Vec::new();
883        for image in images {
884            let parent = if image.parent_id.is_empty() {
885                0
886            } else if let Some(size) = parents.get(&image.parent_id) {
887                *size
888            } else {
889                let size = self
890                    .api
891                    .inspect_image(&image.parent_id)
892                    .await
893                    .ok()
894                    .and_then(|parent| parent.size)
895                    .unwrap_or(0);
896                parents.insert(image.parent_id.clone(), size);
897                size
898            };
899            found.push(Image {
900                reference: image.repo_tags.into_iter().next().unwrap_or(image.id),
901                labels: image.labels,
902                bytes: u64::try_from(image.size - parent).unwrap_or(0),
903            });
904        }
905        Ok(found)
906    }
907
908    async fn image_id(&self, reference: &str) -> Result<Option<String>, Error> {
909        match self.api.inspect_image(reference).await {
910            Ok(inspect) => Ok(inspect.id),
911            Err(error) if is_not_found(&error) => Ok(None),
912            Err(error) => Err(error),
913        }
914    }
915
916    async fn tagged_images(&self) -> Result<Vec<TaggedImage>, Error> {
917        let images = self
918            .api
919            .list_images(Some(ListImagesOptionsBuilder::new().build()))
920            .await?;
921        Ok(images
922            .into_iter()
923            .map(|image| TaggedImage {
924                id: image.id,
925                tags: image
926                    .repo_tags
927                    .into_iter()
928                    .filter(|tag| tag != "<none>:<none>")
929                    .collect(),
930                digests: image
931                    .repo_digests
932                    .into_iter()
933                    .filter(|digest| digest != "<none>@<none>")
934                    .collect(),
935            })
936            .filter(|image| !image.tags.is_empty())
937            .collect())
938    }
939
940    async fn container_images(&self) -> Result<BTreeSet<String>, Error> {
941        let options = ListContainersOptionsBuilder::new().all(true).build();
942        let containers = self.api.list_containers(Some(options)).await?;
943        Ok(containers
944            .into_iter()
945            .filter_map(|container| container.image_id)
946            .collect())
947    }
948
949    async fn remove_image(&self, reference: &str) -> Result<(), Error> {
950        let options = RemoveImageOptionsBuilder::new().build();
951        match self.api.remove_image(reference, Some(options), None).await {
952            Err(error) if is_not_found(&error) => Ok(()),
953            other => other.map(|_| ()),
954        }
955    }
956
957    async fn free_space(&self) -> Result<u64, Error> {
958        let root = self.api.info().await?.docker_root_dir.unwrap_or_default();
959        // Only the directories above Docker's root need to be searchable: the daemon's user cannot read the root.
960        let stat = rustix::fs::statvfs(root.as_str()).map_err(|error| Error::IOError {
961            err: std::io::Error::other(format!("cannot measure the disk of {root}: {error}")),
962        })?;
963        Ok(stat.f_bavail.saturating_mul(stat.f_frsize))
964    }
965
966    async fn shell(
967        &self,
968        name: &str,
969        body: ContainerCreateBody,
970        size: TtySize,
971    ) -> Result<Tty, Error> {
972        let options = CreateContainerOptionsBuilder::new().name(name).build();
973        let id = self.api.create_container(Some(options), body).await?.id;
974        let attached = async {
975            let options = AttachContainerOptionsBuilder::new()
976                .stream(true)
977                .stdin(true)
978                .stdout(true)
979                .stderr(true)
980                .build();
981            let attached = self.api.attach_container(&id, Some(options)).await?;
982            self.api.start_container(&id, None).await?;
983            let resize = ResizeContainerTTYOptionsBuilder::new()
984                .w(i32::from(size.cols))
985                .h(i32::from(size.rows))
986                .build();
987            self.api.resize_container_tty(&id, resize).await?;
988            Ok::<_, Error>(attached)
989        };
990        match attached.await {
991            Ok(attached) => Ok(Tty {
992                container: id,
993                output: attached
994                    .output
995                    .map_ok(|output| output.into_bytes().to_vec())
996                    .boxed(),
997                input: attached.input,
998            }),
999            Err(error) => {
1000                let _ = self.remove(&id).await;
1001                Err(error)
1002            }
1003        }
1004    }
1005}
1006
1007/// How often `pull` logs its progress.
1008const PULL_LOG_INTERVAL: Duration = Duration::from_secs(10);
1009
1010/// A pull's layers so far, from the statuses Docker streams per layer.
1011#[derive(Debug, Default)]
1012struct PullProgress {
1013    layers: BTreeMap<String, Layer>,
1014}
1015
1016#[derive(Debug, Default)]
1017struct Layer {
1018    downloaded: u64,
1019    size: u64,
1020    done: bool,
1021}
1022
1023impl PullProgress {
1024    fn update(&mut self, update: &CreateImageInfo) {
1025        let (Some(id), Some(status)) = (&update.id, &update.status) else {
1026            return;
1027        };
1028        let bytes = |value: Option<i64>| value.and_then(|value| u64::try_from(value).ok());
1029        let detail = update.progress_detail.as_ref();
1030        match status.as_str() {
1031            "Pulling fs layer" | "Waiting" => {
1032                self.layers.entry(id.clone()).or_default();
1033            }
1034            "Downloading" => {
1035                let layer = self.layers.entry(id.clone()).or_default();
1036                layer.downloaded =
1037                    bytes(detail.and_then(|detail| detail.current)).unwrap_or(layer.downloaded);
1038                layer.size = bytes(detail.and_then(|detail| detail.total)).unwrap_or(layer.size);
1039            }
1040            "Download complete" => {
1041                if let Some(layer) = self.layers.get_mut(id) {
1042                    layer.downloaded = layer.size;
1043                }
1044            }
1045            "Pull complete" | "Already exists" => {
1046                let layer = self.layers.entry(id.clone()).or_default();
1047                layer.downloaded = layer.size;
1048                layer.done = true;
1049            }
1050            _ => {}
1051        }
1052    }
1053}
1054
1055impl fmt::Display for PullProgress {
1056    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1057        let done = self.layers.values().filter(|layer| layer.done).count();
1058        let downloaded: u64 = self.layers.values().map(|layer| layer.downloaded).sum();
1059        let size: u64 = self.layers.values().map(|layer| layer.size).sum();
1060        write!(
1061            f,
1062            "{done} of {} layers done, {} of {} downloaded",
1063            self.layers.len(),
1064            ByteSize(downloaded),
1065            ByteSize(size)
1066        )
1067    }
1068}
1069
1070fn connect_to(docker_host: Option<&str>) -> Result<Docker, Error> {
1071    let api = match docker_host.filter(|host| !host.is_empty()) {
1072        None => bollard::Docker::connect_with_socket_defaults()?,
1073        Some(host) if host.starts_with("unix://") => {
1074            bollard::Docker::connect_with_socket(host, 120, bollard::API_DEFAULT_VERSION)?
1075        }
1076        // Bollard would quietly fall back to the local socket: a different Engine than the one asked for.
1077        Some(host) => {
1078            return Err(Error::UnsupportedURISchemeError {
1079                uri: host.to_owned(),
1080            });
1081        }
1082    };
1083    Ok(Docker { api })
1084}
1085
1086/// A tar archive, to extract at `/`, holding the config as a root-only file at `JIT_CONFIG_PATH`.
1087/// The chunks of an archive as they arrive, for a blocking reader: it ends when no more come.
1088struct Chunks {
1089    received: tokio::sync::mpsc::Receiver<Vec<u8>>,
1090    chunk: Vec<u8>,
1091    at: usize,
1092}
1093
1094impl std::io::Read for Chunks {
1095    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
1096        while self.at == self.chunk.len() {
1097            match self.received.blocking_recv() {
1098                Some(chunk) => {
1099                    self.chunk = chunk;
1100                    self.at = 0;
1101                }
1102                None => return Ok(0),
1103            }
1104        }
1105        let n = buf.len().min(self.chunk.len() - self.at);
1106        buf[..n].copy_from_slice(&self.chunk[self.at..self.at + n]);
1107        self.at += n;
1108        Ok(n)
1109    }
1110}
1111
1112/// A reader that notes whether its last read found the end: `tar` reads past a zero block, the archive's end, no
1113/// further, and takes running out between entries as the end too.
1114struct Ends<R> {
1115    inner: R,
1116    ran_out: bool,
1117}
1118
1119impl<R: std::io::Read> std::io::Read for Ends<R> {
1120    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
1121        let n = self.inner.read(buf)?;
1122        self.ran_out = n == 0 && !buf.is_empty();
1123        Ok(n)
1124    }
1125}
1126
1127/// The regular files at the top of a directory's `tar` whose names start with `prefix`. Docker roots the archive at
1128/// the directory's own name. Every other entry is read through and dropped, so only the files kept count against
1129/// `FILES_LIMIT`. Complete only once the archive's end marker is read: one that breaks off, between entries too, or
1130/// a file cut at the limit is not; what was read is kept.
1131pub(crate) fn files_in(tar: impl std::io::Read, prefix: &str) -> Files {
1132    let mut tar = Ends {
1133        inner: tar,
1134        ran_out: false,
1135    };
1136    let mut read = Files::default();
1137    let ended = files_until_end(&mut tar, prefix, &mut read.files);
1138    read.complete = ended && !tar.ran_out;
1139    read
1140}
1141
1142/// Reads `tar` into `files` as `files_in` says; whether it got to the archive's end, which `files_in` checks was its
1143/// marker.
1144fn files_until_end(tar: impl std::io::Read, prefix: &str, files: &mut Vec<File>) -> bool {
1145    let mut archive = tar::Archive::new(tar);
1146    let Ok(entries) = archive.entries() else {
1147        return false;
1148    };
1149    let mut kept = 0;
1150    for entry in entries {
1151        let Ok(mut entry) = entry else {
1152            return false;
1153        };
1154        let name = match entry.path() {
1155            Ok(path) if path.components().count() == 2 => path
1156                .file_name()
1157                .map(|name| name.to_string_lossy().into_owned()),
1158            _ => None,
1159        };
1160        let Some(name) =
1161            name.filter(|name| name.starts_with(prefix) && entry.header().entry_type().is_file())
1162        else {
1163            continue;
1164        };
1165        let room = FILES_LIMIT - kept;
1166        let mut contents = Vec::new();
1167        // One byte past the room tells a file cut at the limit from one that fits it exactly.
1168        let whole = std::io::Read::read_to_end(
1169            &mut std::io::Read::take(&mut entry, room as u64 + 1),
1170            &mut contents,
1171        )
1172        .is_ok()
1173            && contents.len() <= room;
1174        contents.truncate(room);
1175        kept += contents.len();
1176        files.push(File {
1177            name,
1178            contents: String::from_utf8_lossy(&contents).into_owned(),
1179        });
1180        if !whole {
1181            return false;
1182        }
1183    }
1184    true
1185}
1186
1187fn jit_config_tar(jit_config: &JitConfig) -> std::io::Result<Vec<u8>> {
1188    let mut tar = tar::Builder::new(Vec::new());
1189    let path = JIT_CONFIG_PATH.trim_start_matches('/');
1190    let mut dir = tar::Header::new_gnu();
1191    dir.set_entry_type(tar::EntryType::Directory);
1192    dir.set_mode(0o700);
1193    dir.set_size(0);
1194    tar.append_data(
1195        &mut dir,
1196        path.rsplit_once('/').map_or("", |(dir, _)| dir),
1197        std::io::empty(),
1198    )?;
1199    let mut file = tar::Header::new_gnu();
1200    file.set_mode(0o400);
1201    file.set_size(jit_config.0.len() as u64);
1202    tar.append_data(&mut file, path, jit_config.0.as_bytes())?;
1203    tar.into_inner()
1204}
1205
1206#[cfg(test)]
1207pub(crate) mod tests;