Skip to main content

runnerd_core/
docker.rs

1//! Runner containers through the Docker Engine API: the connection, container, volume and image requests, and the
2//! `DockerApi` trait the engine and the caches call. It decides nothing about scheduling, and a JIT config only
3//! ever reaches a container as a root-only file, never the create request or a log line.
4
5use std::collections::{BTreeMap, BTreeSet, HashMap};
6use std::fmt;
7use std::time::{Duration, Instant};
8
9use bollard::exec::{CreateExecOptions, StartExecResults};
10use std::pin::Pin;
11
12use bollard::models::{
13    ChangeType, ContainerConfig, ContainerCreateBody, CreateImageInfo, HostConfig, Mount,
14    NetworkCreateRequest, SystemInfo, VolumeCreateRequest,
15};
16use bollard::query_parameters::{
17    AttachContainerOptionsBuilder, CommitContainerOptionsBuilder, CreateContainerOptionsBuilder,
18    CreateImageOptionsBuilder, DataUsageOptionsBuilder, DownloadFromContainerOptionsBuilder,
19    EventsOptionsBuilder, InspectContainerOptionsBuilder, ListContainersOptionsBuilder,
20    ListImagesOptionsBuilder, LogsOptionsBuilder, RemoveContainerOptionsBuilder,
21    RemoveImageOptionsBuilder, RemoveVolumeOptionsBuilder, ResizeContainerTTYOptionsBuilder,
22    UploadToContainerOptionsBuilder, WaitContainerOptionsBuilder,
23};
24use futures_util::stream::BoxStream;
25use futures_util::{StreamExt, TryStreamExt};
26use tracing::info;
27
28use crate::config::ByteSize;
29use crate::isolation::Isolation;
30
31pub use bollard::errors::Error;
32
33/// One runner container: a profile's image stage with the profile's limits.
34#[derive(Debug, Clone)]
35pub struct RunnerSpec {
36    /// The container's name; for a runner, also the runner's.
37    pub name: String,
38    /// The image, such as `runnerd-runner:base`.
39    pub image: String,
40    /// The profile, put on the `runnerd.profile` label.
41    pub profile: String,
42    /// The systemd slice the container is placed in.
43    pub slice: String,
44    /// CPUs the container may use.
45    pub cpus: u32,
46    /// Memory the container may use, swap included.
47    pub memory: ByteSize,
48    /// `NAME=value` environment variables.
49    pub env: Vec<String>,
50    /// Volumes to mount.
51    pub mounts: Vec<Mount>,
52    /// The network the container joins; `None` is Docker's default bridge.
53    pub network: Option<String>,
54    /// Labels on top of `runnerd.profile` and `runnerd.isolation`.
55    pub labels: BTreeMap<String, String>,
56    /// Replaces the image's entrypoint; `None` runs the runner.
57    pub entrypoint: Option<Vec<String>>,
58    /// The runner's just-in-time config, which the entrypoint reads from `JIT_CONFIG_PATH`.
59    pub jit_config: Option<JitConfig>,
60}
61
62/// Where `start` puts the just-in-time config in the container: a root-only file, never the create request,
63/// which Docker stores and clients may log.
64pub const JIT_CONFIG_PATH: &str = "/run/runnerd/jitconfig";
65
66/// What a runner container's environment holds to have its image answer the runner's sync calls without syncing
67/// (`[host] skip_syncs`, ADR 0141). runnerd sets it only for an image that carries `SKIP_SYNCS_LABEL`; an image
68/// without `runnerd-skip-syncs` would ignore it and sync.
69pub const SKIP_SYNCS: &str = "RUNNERD_SKIP_SYNCS=1";
70
71/// The label of a runner image whose entrypoint runs `runnerd-skip-syncs` (`image/Dockerfile`), with the version of
72/// what it answers to: `1`, `SKIP_SYNCS`. Runner images before it lack it, and their runners keep their syncs.
73pub const SKIP_SYNCS_LABEL: &str = "runnerd.skip-syncs";
74
75/// Whether an image's labels say it answers `SKIP_SYNCS`.
76pub fn skips_syncs<S: std::hash::BuildHasher>(labels: &HashMap<String, String, S>) -> bool {
77    labels
78        .get(SKIP_SYNCS_LABEL)
79        .is_some_and(|version| version == "1")
80}
81
82/// A runner's just-in-time config from GitHub: credentials, never printed.
83#[derive(Clone)]
84pub struct JitConfig(pub String);
85
86impl fmt::Debug for JitConfig {
87    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
88        f.write_str("JitConfig(<redacted>)")
89    }
90}
91
92/// What `inspect` reports about a runner container.
93#[derive(Debug, Clone, PartialEq, Eq)]
94pub struct RunnerState {
95    /// Whether the container runs.
96    pub running: bool,
97    /// Its exit code, once it stopped.
98    pub exit_code: Option<i64>,
99    /// Whether the kernel killed a process in it for memory, as Docker saw.
100    pub oom_killed: bool,
101    /// True under `unsafe-dind`: a job in the container can take over the host.
102    pub privileged: bool,
103    /// The runtime Docker runs it with, such as `sysbox-runc`.
104    pub runtime: Option<String>,
105    /// Its main process on the host, while it runs.
106    pub pid: Option<u32>,
107    /// Its address on its first network.
108    pub address: Option<String>,
109    /// When it started, in milliseconds since the epoch.
110    pub started_at: Option<u64>,
111    /// When it stopped, in milliseconds since the epoch; `None` while it runs.
112    pub finished_at: Option<u64>,
113    /// Its memory cap in bytes; `None` when it has none.
114    pub memory: Option<u64>,
115    /// Its CPU limit in millicores; `None` when it has none.
116    pub cpus: Option<u64>,
117}
118
119/// What a command run in a container prints on stdout, as it prints it.
120pub type Output = BoxStream<'static, Result<Vec<u8>, Error>>;
121
122/// An image `images` found.
123#[derive(Debug, Clone, PartialEq, Eq)]
124pub struct Image {
125    /// Its first tag, such as `runnerd-record:42`; its ID when it has none.
126    pub reference: String,
127    /// Its labels.
128    pub labels: HashMap<String, String>,
129    /// The disk it takes beyond its parent image: what removing it frees.
130    pub bytes: u64,
131}
132
133/// An image `tagged_images` found.
134#[derive(Debug, Clone, PartialEq, Eq)]
135pub struct TaggedImage {
136    /// Its ID, such as `sha256:224e…`.
137    pub id: String,
138    /// Its tags, such as `ghcr.io/ayagmar/runnerd-runner:node-0.1.2`.
139    pub tags: Vec<String>,
140    /// The registry digests it was pulled by, such as `ghcr.io/ayagmar/runnerd-runner@sha256:9e79…`.
141    pub digests: Vec<String>,
142}
143
144/// A file `files` read from a container.
145#[derive(Debug, Clone, PartialEq, Eq)]
146pub struct File {
147    /// Its name in the directory read, such as `Worker_20261031-120000-utc.log`.
148    pub name: String,
149    /// What it holds, as text: invalid UTF-8 is replaced. Cut short past `FILES_LIMIT`.
150    pub contents: String,
151}
152
153/// What `files` read of a directory.
154#[derive(Debug, Clone, Default, PartialEq, Eq)]
155pub struct Files {
156    /// The files read, in the archive's order.
157    pub files: Vec<File>,
158    /// Whether they are all there and whole: the archive ended as a tar does, and none was cut at `FILES_LIMIT`. A
159    /// file missing from an incomplete read may be in the directory.
160    pub complete: bool,
161}
162
163/// The most of the files `files` keeps, together. What it skips, such as a runner's job log pages below `_diag`,
164/// does not count: it is read through and dropped.
165pub(crate) const FILES_LIMIT: usize = 16 << 20;
166
167/// A container's terminal, attached once it was created and before it started, so none of its output is missed.
168pub struct Tty {
169    /// The container's ID.
170    pub container: String,
171    /// What the terminal shows, as it shows it; it ends when the container's process exits.
172    pub output: BoxStream<'static, Result<Vec<u8>, Error>>,
173    /// What is typed into it.
174    pub input: Pin<Box<dyn tokio::io::AsyncWrite + Send>>,
175}
176
177impl fmt::Debug for Tty {
178    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
179        f.debug_struct("Tty")
180            .field("container", &self.container)
181            .finish_non_exhaustive()
182    }
183}
184
185/// A terminal's size.
186#[derive(Debug, Clone, Copy, PartialEq, Eq)]
187pub struct TtySize {
188    /// Columns.
189    pub cols: u16,
190    /// Rows.
191    pub rows: u16,
192}
193
194/// The CPU shares of a runner's container or a record's shell with `cpus` CPUs: 1024 a CPU, within the 2 to 262144
195/// Docker takes, which it writes as the container's `cpu.weight`. They sit in the slice beside each other, so when the
196/// slice's CPUs are all in use they share them in proportion to their `cpus`, not evenly; a container with work still
197/// gets at most its `cpus` (`nano_cpus`). The mirror keeps Docker's default weight, which every job's pulls go
198/// through, and programs outside the slice share the host's CPUs with the slice by its own weight (ADR 0079).
199pub fn cpu_shares(cpus: u32) -> i64 {
200    (i64::from(cpus) * 1024).clamp(2, 262_144)
201}
202
203/// The create-container request for a runner container under an isolation backend.
204pub fn create_body(spec: &RunnerSpec, isolation: &dyn Isolation) -> ContainerCreateBody {
205    // Clamped to 8 EiB, the most Docker takes.
206    let memory = i64::try_from(spec.memory.0).unwrap_or(i64::MAX);
207    let mut host_config = HostConfig {
208        cgroup_parent: Some(spec.slice.clone()),
209        memory: Some(memory),
210        // Swap equal to memory: the container gets no swap on top of its memory.
211        memory_swap: Some(memory),
212        nano_cpus: Some(i64::from(spec.cpus) * 1_000_000_000),
213        cpu_shares: Some(cpu_shares(spec.cpus)),
214        init: Some(true),
215        mounts: Some(spec.mounts.clone()),
216        network_mode: spec.network.clone(),
217        ..Default::default()
218    };
219    isolation.configure(&mut host_config);
220    let mut labels: HashMap<String, String> = spec.labels.clone().into_iter().collect();
221    labels.insert("runnerd.profile".to_owned(), spec.profile.clone());
222    labels.insert("runnerd.isolation".to_owned(), isolation.name().to_owned());
223    ContainerCreateBody {
224        image: Some(spec.image.clone()),
225        env: Some(spec.env.clone()),
226        entrypoint: spec.entrypoint.clone(),
227        labels: Some(labels),
228        host_config: Some(host_config),
229        ..Default::default()
230    }
231}
232
233/// A container `list` found.
234#[derive(Debug, Clone, PartialEq, Eq)]
235pub struct Container {
236    /// The Docker container ID.
237    pub id: String,
238    /// Without Docker's leading `/`.
239    pub name: String,
240    /// Whether it runs.
241    pub running: bool,
242    /// Its labels.
243    pub labels: HashMap<String, String>,
244}
245
246/// A runner start that failed, with the container it may have left: Docker may have made, or even started, it before
247/// the request failed, as when its answer is lost.
248#[derive(Debug)]
249pub struct StartError {
250    /// The container the start left, by ID, or by name when Docker could not say whether one exists; `None` when
251    /// none does. It may run.
252    pub container: Option<String>,
253    /// Whether Docker may have started the container before the start failed: its runner may have taken a job,
254    /// even if the container is gone now.
255    pub may_have_run: bool,
256    /// Why the start failed.
257    pub error: Error,
258}
259
260impl fmt::Display for StartError {
261    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
262        self.error.fmt(f)
263    }
264}
265
266impl std::error::Error for StartError {
267    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
268        Some(&self.error)
269    }
270}
271
272/// For callers that go on whether or not a container was left.
273impl From<StartError> for Error {
274    fn from(failed: StartError) -> Error {
275        failed.error
276    }
277}
278
279/// Whether Docker answered a request with a refusal, so it did nothing: a 4xx answer. Any other error may come after
280/// Docker carried the request out.
281fn refused(error: &Error) -> bool {
282    matches!(
283        error,
284        Error::DockerResponseServerError {
285            status_code: 400..=499,
286            ..
287        }
288    )
289}
290
291/// Whether a Docker error says the Engine did not answer at all: its socket is gone, refuses connections or closed
292/// the connection, as while the machine shuts down. Not an answer the Engine gave, nor a timeout, which a busy Engine
293/// also gives.
294pub fn is_unreachable(error: &Error) -> bool {
295    matches!(
296        error,
297        Error::HyperLegacyError { .. }
298            | Error::HyperResponseError { .. }
299            | Error::IOError { .. }
300            | Error::SocketNotFoundError(_)
301    )
302}
303
304/// Whether Docker answered that it cannot or will not do what was asked: a 4xx, or 501 for what it lacks, such as
305/// the log of a container whose log driver keeps none. Asking again gets the same answer; any other error may not.
306pub fn is_declined(error: &Error) -> bool {
307    matches!(
308        error,
309        Error::DockerResponseServerError {
310            status_code: 400..=499 | 501,
311            ..
312        }
313    )
314}
315
316/// Whether a Docker error says the container, volume or image does not exist.
317pub fn is_not_found(error: &Error) -> bool {
318    matches!(
319        error,
320        Error::DockerResponseServerError {
321            status_code: 404,
322            ..
323        }
324    )
325}
326
327/// Whether Docker refused because it is removing the container already, as after a `docker rm -f`: it says so only in
328/// the words of its 409, which a running container's refusal shares. It may still be stopping.
329pub fn is_being_removed(error: &Error) -> bool {
330    matches!(
331        error,
332        Error::DockerResponseServerError { status_code: 409, message }
333            if message.contains("is already in progress") || message.contains("marked for removal")
334    )
335}
336
337/// Whether Docker refused an exec because the container stopped, as one that ended while it was adopted: its log
338/// is read once it is released.
339pub fn is_not_running(error: &Error) -> bool {
340    matches!(
341        error,
342        Error::DockerResponseServerError { status_code: 409, message } if message.ends_with(" is not running")
343    )
344}
345
346/// Whether a container's log is out of reach because it is gone or going. A removal under way can still fail, so this
347/// says nothing of whether the container will go.
348pub fn is_log_gone(error: &Error) -> bool {
349    is_not_found(error) || is_being_removed(error)
350}
351
352/// The Docker Engine calls the scheduler and the caches make. `Docker` is the real one; the engine's tests use a
353/// fake.
354pub trait DockerApi: Send + Sync {
355    /// Creates and starts a runner container; returns its ID. A start that fails removes what it made, and says
356    /// which container is left when it cannot.
357    fn start(
358        &self,
359        spec: &RunnerSpec,
360        isolation: &dyn Isolation,
361    ) -> impl Future<Output = Result<String, StartError>> + Send;
362    /// A container's state and isolation.
363    fn inspect(&self, container: &str) -> impl Future<Output = Result<RunnerState, Error>> + Send;
364    /// Removes a runner container, stopping it first if it runs.
365    fn remove(&self, container: &str) -> impl Future<Output = Result<(), Error>> + Send;
366    /// Creates and starts a container that is not a runner, such as the mirror; returns its ID.
367    fn run(
368        &self,
369        name: &str,
370        body: ContainerCreateBody,
371    ) -> impl Future<Output = Result<String, Error>> + Send;
372    /// Every container, running or not, that has the label.
373    fn list(&self, label: &str) -> impl Future<Output = Result<Vec<Container>, Error>> + Send;
374    /// The IDs of runner containers (labelled `runnerd.job`) as each stops, from Docker's events. A hint: Docker sends
375    /// an event once at most, none while the stream is down, and keeps none to replay. The stream fails or ends with
376    /// the connection, or when Docker restarts.
377    fn exits(&self) -> BoxStream<'static, Result<String, Error>>;
378    /// Waits for a container to exit; returns its exit code.
379    fn wait(&self, container: &str) -> impl Future<Output = Result<i64, Error>> + Send;
380    /// A container's output so far, stdout and stderr interleaved.
381    fn logs(&self, container: &str) -> impl Future<Output = Result<String, Error>> + Send;
382    /// A container's output since `since` (seconds since the epoch), each line prefixed with Docker's RFC 3339
383    /// timestamp and a space.
384    fn logs_since(
385        &self,
386        container: &str,
387        since: u64,
388    ) -> impl Future<Output = Result<String, Error>> + Send;
389    /// Runs a command as root in a running container; its stdout streams until it exits or the container stops.
390    /// Its stderr is dropped.
391    fn exec(
392        &self,
393        container: &str,
394        command: Vec<String>,
395    ) -> impl Future<Output = Result<Output, Error>> + Send;
396    /// Whether Docker has an image.
397    fn image_present(&self, image: &str) -> impl Future<Output = Result<bool, Error>> + Send;
398    /// Whether Docker has an image and it answers `SKIP_SYNCS`, as its labels say (`skips_syncs`).
399    fn image_skips_syncs(&self, image: &str) -> impl Future<Output = Result<bool, Error>> + Send;
400    /// Pulls an image, logging its progress every few seconds.
401    fn pull(&self, image: &str) -> impl Future<Output = Result<(), Error>> + Send;
402    /// Creates a bridge network unless it exists.
403    fn ensure_network(&self, name: &str) -> impl Future<Output = Result<(), Error>> + Send;
404    /// Creates a volume unless it exists.
405    fn ensure_volume(&self, name: &str) -> impl Future<Output = Result<(), Error>> + Send;
406    /// Removes a volume; a volume that does not exist is not an error. Docker refuses one a container mounts, even
407    /// a stopped one.
408    fn remove_volume(&self, name: &str) -> impl Future<Output = Result<(), Error>> + Send;
409    /// Every container, running or not, that mounts the volume.
410    fn holders(&self, volume: &str) -> impl Future<Output = Result<Vec<Container>, Error>> + Send;
411    /// The disk size of every volume Docker knows, by name. Slow: Docker measures each volume.
412    fn volume_sizes(&self) -> impl Future<Output = Result<BTreeMap<String, u64>, Error>> + Send;
413    /// The paths a container's own layer added or changed, not those it deleted.
414    fn changes(&self, container: &str) -> impl Future<Output = Result<Vec<String>, Error>> + Send;
415    /// The bytes a container's own layer holds.
416    fn written(&self, container: &str) -> impl Future<Output = Result<u64, Error>> + Send;
417    /// The files directly in a container's `directory` whose names start with `prefix`, not those in the
418    /// directories below it, read from its filesystem whether it runs or stopped, without starting it.
419    fn files(
420        &self,
421        container: &str,
422        directory: &str,
423        prefix: &str,
424    ) -> impl Future<Output = Result<Files, Error>> + Send;
425    /// Commits a stopped container's own layer as the image `reference` (`repository:tag`) with `labels`; returns
426    /// the disk the image takes beyond its parent.
427    fn commit(
428        &self,
429        container: &str,
430        reference: &str,
431        labels: BTreeMap<String, String>,
432    ) -> impl Future<Output = Result<u64, Error>> + Send;
433    /// Every image with the label.
434    fn images(&self, label: &str) -> impl Future<Output = Result<Vec<Image>, Error>> + Send;
435    /// The ID of the image a reference (tag, digest or ID) names; `None` when Docker does not have it.
436    fn image_id(
437        &self,
438        reference: &str,
439    ) -> impl Future<Output = Result<Option<String>, Error>> + Send;
440    /// Every image Docker has a tag for.
441    fn tagged_images(&self) -> impl Future<Output = Result<Vec<TaggedImage>, Error>> + Send;
442    /// The image ID of every container, running or stopped.
443    fn container_images(&self) -> impl Future<Output = Result<BTreeSet<String>, Error>> + Send;
444    /// Removes an image and the layers only it used; one that does not exist is not an error.
445    fn remove_image(&self, reference: &str) -> impl Future<Output = Result<(), Error>> + Send;
446    /// The free disk where Docker keeps its images and containers.
447    fn free_space(&self) -> impl Future<Output = Result<u64, Error>> + Send;
448    /// Creates a container with a terminal of `size`, attaches to it and starts it. The container's stdin closes
449    /// when `Tty::input` is dropped.
450    fn shell(
451        &self,
452        name: &str,
453        body: ContainerCreateBody,
454        size: TtySize,
455    ) -> impl Future<Output = Result<Tty, Error>> + Send;
456}
457
458/// The local Docker Engine.
459#[derive(Debug)]
460pub struct Docker {
461    api: bollard::Docker,
462}
463
464impl Docker {
465    /// Removes the container a failed start `made`, if any; the start's `error` is the one worth reporting. The
466    /// container stays in the answer unless its removal is confirmed.
467    async fn clean_up(&self, made: Option<String>, error: Error, may_have_run: bool) -> StartError {
468        let container = match made {
469            Some(container) => match self.remove(&container).await {
470                Err(failed) if !is_not_found(&failed) => Some(container),
471                _ => None,
472            },
473            None => None,
474        };
475        StartError {
476            container,
477            may_have_run,
478            error,
479        }
480    }
481
482    /// Connects to the Docker Engine at `DOCKER_HOST`, or its local socket. Only `unix://` hosts are supported.
483    pub fn connect() -> Result<Docker, Error> {
484        connect_to(std::env::var("DOCKER_HOST").ok().as_deref())
485    }
486
487    /// The Engine's system information.
488    pub async fn info(&self) -> Result<SystemInfo, Error> {
489        self.api.info().await
490    }
491
492    /// An image's labels; `None` when Docker does not have it.
493    pub async fn image_labels(
494        &self,
495        image: &str,
496    ) -> Result<Option<HashMap<String, String>>, Error> {
497        match self.api.inspect_image(image).await {
498            Ok(inspect) => Ok(Some(
499                inspect
500                    .config
501                    .and_then(|config| config.labels)
502                    .unwrap_or_default(),
503            )),
504            Err(error) if is_not_found(&error) => Ok(None),
505            Err(error) => Err(error),
506        }
507    }
508
509    /// Every container, running or not, that `filter` matches with `value`, as `docker ps --all --filter` does.
510    async fn containers(&self, filter: &str, value: &str) -> Result<Vec<Container>, Error> {
511        let options = ListContainersOptionsBuilder::new()
512            .all(true)
513            .filters(&HashMap::from([(filter, vec![value])]))
514            .build();
515        let containers = self.api.list_containers(Some(options)).await?;
516
517        Ok(containers
518            .into_iter()
519            .map(|container| Container {
520                id: container.id.unwrap_or_default(),
521                name: container
522                    .names
523                    .and_then(|names| names.into_iter().next())
524                    .unwrap_or_default()
525                    .trim_start_matches('/')
526                    .to_owned(),
527                running: container.state
528                    == Some(bollard::models::ContainerSummaryStateEnum::RUNNING),
529                labels: container.labels.unwrap_or_default(),
530            })
531            .collect())
532    }
533
534    /// Asks the image's registry for its manifest, without pulling it, and returns the platforms it is built for,
535    /// such as `linux/amd64`. An error means `pull` could not get it.
536    pub async fn registry_platforms(&self, image: &str) -> Result<Vec<String>, Error> {
537        let inspect = self.api.inspect_registry_image(image, None).await?;
538        Ok(inspect
539            .platforms
540            .into_iter()
541            .filter_map(|platform| Some(format!("{}/{}", platform.os?, platform.architecture?)))
542            // Attestation manifests in an index carry the platform unknown/unknown.
543            .filter(|platform| platform != "unknown/unknown")
544            .collect())
545    }
546}
547
548impl DockerApi for Docker {
549    async fn start(
550        &self,
551        spec: &RunnerSpec,
552        isolation: &dyn Isolation,
553    ) -> Result<String, StartError> {
554        let options = CreateContainerOptionsBuilder::new()
555            .name(&spec.name)
556            .build();
557        let id = match self
558            .api
559            .create_container(Some(options), create_body(spec, isolation))
560            .await
561        {
562            Ok(created) => created.id,
563            Err(error) if refused(&error) => {
564                return Err(StartError {
565                    container: None,
566                    may_have_run: false,
567                    error,
568                });
569            }
570            // Docker may have made it and lost the answer: its name, unique to this start, finds it.
571            Err(error) => {
572                let made = match self.api.inspect_container(&spec.name, None).await {
573                    Ok(found) => Some(found.id.unwrap_or_else(|| spec.name.clone())),
574                    Err(missing) if is_not_found(&missing) => None,
575                    Err(_) => Some(spec.name.clone()),
576                };
577                // Made at most, never started.
578                return Err(self.clean_up(made, error, false).await);
579            }
580        };
581        let uploaded = async {
582            if let Some(jit_config) = &spec.jit_config {
583                let options = UploadToContainerOptionsBuilder::new().path("/").build();
584                let tar = jit_config_tar(jit_config)?;
585                // As a stream: bollard traces a full body's bytes, but a stream only as "(stream)".
586                let tar = futures_util::stream::once(std::future::ready(tar.into()));
587                self.api
588                    .upload_to_container(&id, Some(options), bollard::body_stream(tar))
589                    .await?;
590            }
591            Ok::<(), Error>(())
592        };
593        if let Err(error) = uploaded.await {
594            return Err(self.clean_up(Some(id), error, false).await);
595        }
596        // A start whose answer is lost may have started it.
597        if let Err(error) = self.api.start_container(&id, None).await {
598            return Err(self.clean_up(Some(id), error, true).await);
599        }
600        Ok(id)
601    }
602
603    async fn inspect(&self, container: &str) -> Result<RunnerState, Error> {
604        let response = self.api.inspect_container(container, None).await?;
605        let state = response.state.unwrap_or_default();
606        let host_config = response.host_config.unwrap_or_default();
607        let address = response
608            .network_settings
609            .and_then(|settings| settings.networks)
610            .and_then(|networks| {
611                networks
612                    .into_values()
613                    .find_map(|network| network.ip_address.filter(|address| !address.is_empty()))
614            });
615        Ok(RunnerState {
616            running: state.running.unwrap_or(false),
617            exit_code: state.exit_code,
618            oom_killed: state.oom_killed.unwrap_or(false),
619            privileged: host_config.privileged.unwrap_or(false),
620            runtime: host_config.runtime,
621            pid: state
622                .pid
623                .and_then(|pid| u32::try_from(pid).ok())
624                .filter(|pid| *pid > 0),
625            address,
626            started_at: state.started_at.as_deref().and_then(crate::time::parse_ms),
627            // Docker says 0001-01-01 for one that never stopped, which reads as none.
628            finished_at: state.finished_at.as_deref().and_then(crate::time::parse_ms),
629            memory: host_config
630                .memory
631                .and_then(|memory| u64::try_from(memory).ok())
632                .filter(|memory| *memory > 0),
633            cpus: host_config
634                .nano_cpus
635                .and_then(|nano| u64::try_from(nano / 1_000_000).ok())
636                .filter(|cpus| *cpus > 0),
637        })
638    }
639
640    async fn remove(&self, container: &str) -> Result<(), Error> {
641        // With its anonymous volumes, such as a shell's /state.
642        let options = RemoveContainerOptionsBuilder::new()
643            .force(true)
644            .v(true)
645            .build();
646        self.api.remove_container(container, Some(options)).await
647    }
648
649    async fn run(&self, name: &str, body: ContainerCreateBody) -> Result<String, Error> {
650        let options = CreateContainerOptionsBuilder::new().name(name).build();
651        let id = self.api.create_container(Some(options), body).await?.id;
652        if let Err(error) = self.api.start_container(&id, None).await {
653            let _ = self.remove(&id).await;
654            return Err(error);
655        }
656        Ok(id)
657    }
658
659    async fn list(&self, label: &str) -> Result<Vec<Container>, Error> {
660        self.containers("label", label).await
661    }
662
663    async fn holders(&self, volume: &str) -> Result<Vec<Container>, Error> {
664        self.containers("volume", volume).await
665    }
666
667    fn exits(&self) -> BoxStream<'static, Result<String, Error>> {
668        let filters = HashMap::from([
669            ("type", vec!["container"]),
670            ("event", vec!["die"]),
671            ("label", vec!["runnerd.job"]),
672        ]);
673        let options = EventsOptionsBuilder::new().filters(&filters).build();
674        self.api
675            .events(Some(options))
676            .try_filter_map(|event| std::future::ready(Ok(event.actor.and_then(|actor| actor.id))))
677            .boxed()
678    }
679
680    async fn wait(&self, container: &str) -> Result<i64, Error> {
681        let options = WaitContainerOptionsBuilder::new()
682            .condition("not-running")
683            .build();
684        let mut responses = self.api.wait_container(container, Some(options));
685        match responses.next().await {
686            Some(Ok(response)) => Ok(response.status_code),
687            // Bollard reports a non-zero exit as an error.
688            Some(Err(Error::DockerContainerWaitError { code, .. })) => Ok(code),
689            Some(Err(error)) => Err(error),
690            None => Ok(0),
691        }
692    }
693
694    async fn logs(&self, container: &str) -> Result<String, Error> {
695        let options = LogsOptionsBuilder::new().stdout(true).stderr(true).build();
696        let chunks: Vec<_> = self
697            .api
698            .logs(container, Some(options))
699            .try_collect()
700            .await?;
701        Ok(chunks.iter().map(ToString::to_string).collect())
702    }
703
704    async fn logs_since(&self, container: &str, since: u64) -> Result<String, Error> {
705        let options = LogsOptionsBuilder::new()
706            .stdout(true)
707            .stderr(true)
708            .timestamps(true)
709            .since(i32::try_from(since).unwrap_or(i32::MAX))
710            .build();
711        let chunks: Vec<_> = self
712            .api
713            .logs(container, Some(options))
714            .try_collect()
715            .await?;
716        Ok(chunks.iter().map(ToString::to_string).collect())
717    }
718
719    async fn exec(&self, container: &str, command: Vec<String>) -> Result<Output, Error> {
720        let options = CreateExecOptions {
721            cmd: Some(command),
722            user: Some("root".to_owned()),
723            attach_stdout: Some(true),
724            attach_stderr: Some(true),
725            ..Default::default()
726        };
727        let exec = self.api.create_exec(container, options).await?;
728        match self.api.start_exec(&exec.id, None).await? {
729            StartExecResults::Attached { output, .. } => Ok(output
730                .try_filter_map(|output| {
731                    std::future::ready(Ok(match output {
732                        bollard::container::LogOutput::StdOut { message } => Some(message.to_vec()),
733                        _ => None,
734                    }))
735                })
736                .boxed()),
737            StartExecResults::Detached => Ok(futures_util::stream::empty().boxed()),
738        }
739    }
740
741    async fn image_present(&self, image: &str) -> Result<bool, Error> {
742        Ok(self.image_labels(image).await?.is_some())
743    }
744
745    async fn image_skips_syncs(&self, image: &str) -> Result<bool, Error> {
746        Ok(self
747            .image_labels(image)
748            .await?
749            .is_some_and(|labels| skips_syncs(&labels)))
750    }
751
752    async fn pull(&self, image: &str) -> Result<(), Error> {
753        let options = CreateImageOptionsBuilder::new().from_image(image).build();
754        let mut stream = self.api.create_image(Some(options), None, None);
755        let mut progress = PullProgress::default();
756        let mut logged = Instant::now();
757        info!("pulling {image}");
758        while let Some(update) = stream.try_next().await? {
759            progress.update(&update);
760            if logged.elapsed() >= PULL_LOG_INTERVAL {
761                logged = Instant::now();
762                info!("pulling {image}: {progress}");
763            }
764        }
765        info!("pulled {image}");
766        Ok(())
767    }
768
769    async fn ensure_network(&self, name: &str) -> Result<(), Error> {
770        match self.api.inspect_network(name, None).await {
771            Err(error) if is_not_found(&error) => {
772                let request = NetworkCreateRequest {
773                    name: name.to_owned(),
774                    driver: Some("bridge".to_owned()),
775                    ..Default::default()
776                };
777                self.api.create_network(request).await.map(|_| ())
778            }
779            other => other.map(|_| ()),
780        }
781    }
782
783    async fn ensure_volume(&self, name: &str) -> Result<(), Error> {
784        // Creating a volume that exists is a no-op in Docker.
785        let request = VolumeCreateRequest {
786            name: Some(name.to_owned()),
787            ..Default::default()
788        };
789        self.api.create_volume(request).await.map(|_| ())
790    }
791
792    async fn remove_volume(&self, name: &str) -> Result<(), Error> {
793        let options = RemoveVolumeOptionsBuilder::new().build();
794        match self.api.remove_volume(name, Some(options)).await {
795            Err(error) if is_not_found(&error) => Ok(()),
796            other => other,
797        }
798    }
799
800    async fn volume_sizes(&self) -> Result<BTreeMap<String, u64>, Error> {
801        // Without a type filter: bollard cannot encode one in the query.
802        let options = DataUsageOptionsBuilder::new().verbose(true).build();
803        let usage = self.api.df(Some(options)).await?;
804        // API 1.53 leaves the items untyped; they are volumes as `GET /volumes` describes them.
805        Ok(usage
806            .volume_usage
807            .and_then(|volumes| volumes.items)
808            .unwrap_or_default()
809            .into_iter()
810            .filter_map(|volume| {
811                let name = volume.get("Name")?.as_str()?.to_owned();
812                let size = volume.get("UsageData")?.get("Size")?.as_u64()?;
813                Some((name, size))
814            })
815            .collect())
816    }
817
818    async fn changes(&self, container: &str) -> Result<Vec<String>, Error> {
819        Ok(self
820            .api
821            .container_changes(container)
822            .await?
823            .unwrap_or_default()
824            .into_iter()
825            .filter(|change| change.kind != ChangeType::_2)
826            .map(|change| change.path)
827            .collect())
828    }
829
830    async fn written(&self, container: &str) -> Result<u64, Error> {
831        let options = InspectContainerOptionsBuilder::new().size(true).build();
832        let response = self.api.inspect_container(container, Some(options)).await?;
833        Ok(response
834            .size_rw
835            .and_then(|bytes| u64::try_from(bytes).ok())
836            .unwrap_or(0))
837    }
838
839    async fn files(&self, container: &str, directory: &str, prefix: &str) -> Result<Files, Error> {
840        let options = DownloadFromContainerOptionsBuilder::new()
841            .path(directory)
842            .build();
843        let mut archive = self.api.download_from_container(container, Some(options));
844        // The archive is read as it arrives, by a blocking reader, so what it skips is never held.
845        let (chunks, received) = tokio::sync::mpsc::channel(4);
846        let prefix = prefix.to_owned();
847        let reader = tokio::task::spawn_blocking(move || {
848            files_in(
849                Chunks {
850                    received,
851                    chunk: Vec::new(),
852                    at: 0,
853                },
854                &prefix,
855            )
856        });
857        let mut failed = None;
858        while let Some(chunk) = archive.next().await {
859            match chunk {
860                // Fails once the reader is done, at the archive's end or `FILES_LIMIT`.
861                Ok(chunk) => {
862                    if chunks.send(chunk.to_vec()).await.is_err() {
863                        break;
864                    }
865                }
866                Err(error) => {
867                    failed = Some(error);
868                    break;
869                }
870            }
871        }
872        drop(chunks);
873        let files = match reader.await {
874            Ok(files) => files,
875            Err(error) => match error.try_into_panic() {
876                Ok(panic) => std::panic::resume_unwind(panic),
877                // Only while the runtime shuts down.
878                Err(_) => Files::default(),
879            },
880        };
881        match failed {
882            Some(error) => Err(error),
883            None => Ok(files),
884        }
885    }
886
887    async fn commit(
888        &self,
889        container: &str,
890        reference: &str,
891        labels: BTreeMap<String, String>,
892    ) -> Result<u64, Error> {
893        let (repository, tag) = reference.rsplit_once(':').unwrap_or((reference, "latest"));
894        let options = CommitContainerOptionsBuilder::new()
895            .container(container)
896            .repo(repository)
897            .tag(tag)
898            .pause(false)
899            .build();
900        let config = ContainerConfig {
901            labels: Some(labels.into_iter().collect()),
902            ..Default::default()
903        };
904        // The image the container runs is the record's parent.
905        let parent = match self.api.inspect_container(container, None).await?.image {
906            Some(parent) => self.api.inspect_image(&parent).await?.size.unwrap_or(0),
907            None => 0,
908        };
909        let id = self.api.commit_container(options, config).await?.id;
910        let size = self.api.inspect_image(&id).await?.size.unwrap_or(0);
911        Ok(u64::try_from(size - parent).unwrap_or(0))
912    }
913
914    async fn images(&self, label: &str) -> Result<Vec<Image>, Error> {
915        let options = ListImagesOptionsBuilder::new()
916            .filters(&HashMap::from([("label", vec![label])]))
917            .build();
918        let images = self.api.list_images(Some(options)).await?;
919        let mut parents: HashMap<String, i64> = HashMap::new();
920        let mut found = Vec::new();
921        for image in images {
922            let parent = if image.parent_id.is_empty() {
923                0
924            } else if let Some(size) = parents.get(&image.parent_id) {
925                *size
926            } else {
927                let size = self
928                    .api
929                    .inspect_image(&image.parent_id)
930                    .await
931                    .ok()
932                    .and_then(|parent| parent.size)
933                    .unwrap_or(0);
934                parents.insert(image.parent_id.clone(), size);
935                size
936            };
937            found.push(Image {
938                reference: image.repo_tags.into_iter().next().unwrap_or(image.id),
939                labels: image.labels,
940                bytes: u64::try_from(image.size - parent).unwrap_or(0),
941            });
942        }
943        Ok(found)
944    }
945
946    async fn image_id(&self, reference: &str) -> Result<Option<String>, Error> {
947        match self.api.inspect_image(reference).await {
948            Ok(inspect) => Ok(inspect.id),
949            Err(error) if is_not_found(&error) => Ok(None),
950            Err(error) => Err(error),
951        }
952    }
953
954    async fn tagged_images(&self) -> Result<Vec<TaggedImage>, Error> {
955        let images = self
956            .api
957            .list_images(Some(ListImagesOptionsBuilder::new().build()))
958            .await?;
959        Ok(images
960            .into_iter()
961            .map(|image| TaggedImage {
962                id: image.id,
963                tags: image
964                    .repo_tags
965                    .into_iter()
966                    .filter(|tag| tag != "<none>:<none>")
967                    .collect(),
968                digests: image
969                    .repo_digests
970                    .into_iter()
971                    .filter(|digest| digest != "<none>@<none>")
972                    .collect(),
973            })
974            .filter(|image| !image.tags.is_empty())
975            .collect())
976    }
977
978    async fn container_images(&self) -> Result<BTreeSet<String>, Error> {
979        let options = ListContainersOptionsBuilder::new().all(true).build();
980        let containers = self.api.list_containers(Some(options)).await?;
981        Ok(containers
982            .into_iter()
983            .filter_map(|container| container.image_id)
984            .collect())
985    }
986
987    async fn remove_image(&self, reference: &str) -> Result<(), Error> {
988        let options = RemoveImageOptionsBuilder::new().build();
989        match self.api.remove_image(reference, Some(options), None).await {
990            Err(error) if is_not_found(&error) => Ok(()),
991            other => other.map(|_| ()),
992        }
993    }
994
995    async fn free_space(&self) -> Result<u64, Error> {
996        let root = self.api.info().await?.docker_root_dir.unwrap_or_default();
997        // Only the directories above Docker's root need to be searchable: the daemon's user cannot read the root.
998        let stat = rustix::fs::statvfs(root.as_str()).map_err(|error| Error::IOError {
999            err: std::io::Error::other(format!("cannot measure the disk of {root}: {error}")),
1000        })?;
1001        Ok(stat.f_bavail.saturating_mul(stat.f_frsize))
1002    }
1003
1004    async fn shell(
1005        &self,
1006        name: &str,
1007        body: ContainerCreateBody,
1008        size: TtySize,
1009    ) -> Result<Tty, Error> {
1010        let options = CreateContainerOptionsBuilder::new().name(name).build();
1011        let id = self.api.create_container(Some(options), body).await?.id;
1012        let attached = async {
1013            let options = AttachContainerOptionsBuilder::new()
1014                .stream(true)
1015                .stdin(true)
1016                .stdout(true)
1017                .stderr(true)
1018                .build();
1019            let attached = self.api.attach_container(&id, Some(options)).await?;
1020            self.api.start_container(&id, None).await?;
1021            let resize = ResizeContainerTTYOptionsBuilder::new()
1022                .w(i32::from(size.cols))
1023                .h(i32::from(size.rows))
1024                .build();
1025            self.api.resize_container_tty(&id, resize).await?;
1026            Ok::<_, Error>(attached)
1027        };
1028        match attached.await {
1029            Ok(attached) => Ok(Tty {
1030                container: id,
1031                output: attached
1032                    .output
1033                    .map_ok(|output| output.into_bytes().to_vec())
1034                    .boxed(),
1035                input: attached.input,
1036            }),
1037            Err(error) => {
1038                let _ = self.remove(&id).await;
1039                Err(error)
1040            }
1041        }
1042    }
1043}
1044
1045/// How often `pull` logs its progress.
1046const PULL_LOG_INTERVAL: Duration = Duration::from_secs(10);
1047
1048/// A pull's layers so far, from the statuses Docker streams per layer.
1049#[derive(Debug, Default)]
1050struct PullProgress {
1051    layers: BTreeMap<String, Layer>,
1052}
1053
1054#[derive(Debug, Default)]
1055struct Layer {
1056    downloaded: u64,
1057    size: u64,
1058    done: bool,
1059}
1060
1061impl PullProgress {
1062    fn update(&mut self, update: &CreateImageInfo) {
1063        let (Some(id), Some(status)) = (&update.id, &update.status) else {
1064            return;
1065        };
1066        let bytes = |value: Option<i64>| value.and_then(|value| u64::try_from(value).ok());
1067        let detail = update.progress_detail.as_ref();
1068        match status.as_str() {
1069            "Pulling fs layer" | "Waiting" => {
1070                self.layers.entry(id.clone()).or_default();
1071            }
1072            "Downloading" => {
1073                let layer = self.layers.entry(id.clone()).or_default();
1074                layer.downloaded =
1075                    bytes(detail.and_then(|detail| detail.current)).unwrap_or(layer.downloaded);
1076                layer.size = bytes(detail.and_then(|detail| detail.total)).unwrap_or(layer.size);
1077            }
1078            "Download complete" => {
1079                if let Some(layer) = self.layers.get_mut(id) {
1080                    layer.downloaded = layer.size;
1081                }
1082            }
1083            "Pull complete" | "Already exists" => {
1084                let layer = self.layers.entry(id.clone()).or_default();
1085                layer.downloaded = layer.size;
1086                layer.done = true;
1087            }
1088            _ => {}
1089        }
1090    }
1091}
1092
1093impl fmt::Display for PullProgress {
1094    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1095        let done = self.layers.values().filter(|layer| layer.done).count();
1096        let downloaded: u64 = self.layers.values().map(|layer| layer.downloaded).sum();
1097        let size: u64 = self.layers.values().map(|layer| layer.size).sum();
1098        write!(
1099            f,
1100            "{done} of {} layers done, {} of {} downloaded",
1101            self.layers.len(),
1102            ByteSize(downloaded),
1103            ByteSize(size)
1104        )
1105    }
1106}
1107
1108fn connect_to(docker_host: Option<&str>) -> Result<Docker, Error> {
1109    let api = match docker_host.filter(|host| !host.is_empty()) {
1110        None => bollard::Docker::connect_with_socket_defaults()?,
1111        Some(host) if host.starts_with("unix://") => {
1112            bollard::Docker::connect_with_socket(host, 120, bollard::API_DEFAULT_VERSION)?
1113        }
1114        // Bollard would quietly fall back to the local socket: a different Engine than the one asked for.
1115        Some(host) => {
1116            return Err(Error::UnsupportedURISchemeError {
1117                uri: host.to_owned(),
1118            });
1119        }
1120    };
1121    Ok(Docker { api })
1122}
1123
1124/// A tar archive, to extract at `/`, holding the config as a root-only file at `JIT_CONFIG_PATH`.
1125/// The chunks of an archive as they arrive, for a blocking reader: it ends when no more come.
1126struct Chunks {
1127    received: tokio::sync::mpsc::Receiver<Vec<u8>>,
1128    chunk: Vec<u8>,
1129    at: usize,
1130}
1131
1132impl std::io::Read for Chunks {
1133    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
1134        while self.at == self.chunk.len() {
1135            match self.received.blocking_recv() {
1136                Some(chunk) => {
1137                    self.chunk = chunk;
1138                    self.at = 0;
1139                }
1140                None => return Ok(0),
1141            }
1142        }
1143        let n = buf.len().min(self.chunk.len() - self.at);
1144        buf[..n].copy_from_slice(&self.chunk[self.at..self.at + n]);
1145        self.at += n;
1146        Ok(n)
1147    }
1148}
1149
1150/// A reader that notes whether its last read found the end: `tar` reads past a zero block, the archive's end, no
1151/// further, and takes running out between entries as the end too.
1152struct Ends<R> {
1153    inner: R,
1154    ran_out: bool,
1155}
1156
1157impl<R: std::io::Read> std::io::Read for Ends<R> {
1158    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
1159        let n = self.inner.read(buf)?;
1160        self.ran_out = n == 0 && !buf.is_empty();
1161        Ok(n)
1162    }
1163}
1164
1165/// The regular files at the top of a directory's `tar` whose names start with `prefix`. Docker roots the archive at
1166/// the directory's own name. Every other entry is read through and dropped, so only the files kept count against
1167/// `FILES_LIMIT`. Complete only once the archive's end marker is read: one that breaks off, between entries too, or
1168/// a file cut at the limit is not; what was read is kept.
1169pub(crate) fn files_in(tar: impl std::io::Read, prefix: &str) -> Files {
1170    let mut tar = Ends {
1171        inner: tar,
1172        ran_out: false,
1173    };
1174    let mut read = Files::default();
1175    let ended = files_until_end(&mut tar, prefix, &mut read.files);
1176    read.complete = ended && !tar.ran_out;
1177    read
1178}
1179
1180/// Reads `tar` into `files` as `files_in` says; whether it got to the archive's end, which `files_in` checks was its
1181/// marker.
1182fn files_until_end(tar: impl std::io::Read, prefix: &str, files: &mut Vec<File>) -> bool {
1183    let mut archive = tar::Archive::new(tar);
1184    let Ok(entries) = archive.entries() else {
1185        return false;
1186    };
1187    let mut kept = 0;
1188    for entry in entries {
1189        let Ok(mut entry) = entry else {
1190            return false;
1191        };
1192        let name = match entry.path() {
1193            Ok(path) if path.components().count() == 2 => path
1194                .file_name()
1195                .map(|name| name.to_string_lossy().into_owned()),
1196            _ => None,
1197        };
1198        let Some(name) =
1199            name.filter(|name| name.starts_with(prefix) && entry.header().entry_type().is_file())
1200        else {
1201            continue;
1202        };
1203        let room = FILES_LIMIT - kept;
1204        let mut contents = Vec::new();
1205        // One byte past the room tells a file cut at the limit from one that fits it exactly.
1206        let whole = std::io::Read::read_to_end(
1207            &mut std::io::Read::take(&mut entry, room as u64 + 1),
1208            &mut contents,
1209        )
1210        .is_ok()
1211            && contents.len() <= room;
1212        contents.truncate(room);
1213        kept += contents.len();
1214        files.push(File {
1215            name,
1216            contents: String::from_utf8_lossy(&contents).into_owned(),
1217        });
1218        if !whole {
1219            return false;
1220        }
1221    }
1222    true
1223}
1224
1225fn jit_config_tar(jit_config: &JitConfig) -> std::io::Result<Vec<u8>> {
1226    let mut tar = tar::Builder::new(Vec::new());
1227    let path = JIT_CONFIG_PATH.trim_start_matches('/');
1228    let mut dir = tar::Header::new_gnu();
1229    dir.set_entry_type(tar::EntryType::Directory);
1230    dir.set_mode(0o700);
1231    dir.set_size(0);
1232    tar.append_data(
1233        &mut dir,
1234        path.rsplit_once('/').map_or("", |(dir, _)| dir),
1235        std::io::empty(),
1236    )?;
1237    let mut file = tar::Header::new_gnu();
1238    file.set_mode(0o400);
1239    file.set_size(jit_config.0.len() as u64);
1240    tar.append_data(&mut file, path, jit_config.0.as_bytes())?;
1241    tar.into_inner()
1242}
1243
1244#[cfg(test)]
1245pub(crate) mod tests;