1use std::collections::{BTreeMap, BTreeSet, HashMap};
6use std::fmt;
7use std::time::{Duration, Instant};
8
9use bollard::exec::{CreateExecOptions, StartExecResults};
10use std::pin::Pin;
11
12use bollard::models::{
13 ChangeType, ContainerConfig, ContainerCreateBody, CreateImageInfo, HostConfig, Mount,
14 NetworkCreateRequest, SystemInfo, VolumeCreateRequest,
15};
16use bollard::query_parameters::{
17 AttachContainerOptionsBuilder, CommitContainerOptionsBuilder, CreateContainerOptionsBuilder,
18 CreateImageOptionsBuilder, DataUsageOptionsBuilder, DownloadFromContainerOptionsBuilder,
19 EventsOptionsBuilder, InspectContainerOptionsBuilder, ListContainersOptionsBuilder,
20 ListImagesOptionsBuilder, ListVolumesOptionsBuilder, LogsOptionsBuilder,
21 RemoveContainerOptionsBuilder, RemoveImageOptionsBuilder, RemoveVolumeOptionsBuilder,
22 ResizeContainerTTYOptionsBuilder, UploadToContainerOptionsBuilder, WaitContainerOptionsBuilder,
23};
24use futures_util::stream::BoxStream;
25use futures_util::{StreamExt, TryStreamExt};
26use tracing::info;
27
28use crate::config::ByteSize;
29use crate::isolation::Isolation;
30
31pub use bollard::errors::Error;
32
33#[derive(Debug, Clone)]
35pub struct RunnerSpec {
36 pub name: String,
38 pub image: String,
40 pub profile: String,
42 pub slice: String,
44 pub cpus: u32,
46 pub memory: ByteSize,
48 pub env: Vec<String>,
50 pub mounts: Vec<Mount>,
52 pub network: Option<String>,
54 pub labels: BTreeMap<String, String>,
56 pub entrypoint: Option<Vec<String>>,
58 pub jit_config: Option<JitConfig>,
60}
61
62pub const JIT_CONFIG_PATH: &str = "/run/runnerd/jitconfig";
65
66pub const SKIP_SYNCS: &str = "RUNNERD_SKIP_SYNCS=1";
70
71pub const SKIP_SYNCS_LABEL: &str = "runnerd.skip-syncs";
74
75pub const VOLATILE_LAYERS_LABEL: &str = "runnerd.volatile-layers";
80
81pub const PROBE_LABEL: &str = "runnerd.runner-probe";
86
87pub fn probe<S: std::hash::BuildHasher>(
92 labels: &HashMap<String, String, S>,
93 exit_code: Option<i64>,
94) -> Option<crate::api::RunnerProbe> {
95 use crate::api::RunnerProbe;
96 if labels.get(PROBE_LABEL).map(String::as_str) != Some("1") {
97 return None;
98 }
99 match exit_code? {
100 200 => Some(RunnerProbe::Clean),
101 201 => Some(RunnerProbe::Mount),
102 202 => Some(RunnerProbe::Procfs),
103 203 => Some(RunnerProbe::MountAndProcfs),
104 204 => Some(RunnerProbe::TimedOut),
105 205 => Some(RunnerProbe::Broken),
106 _ => None,
107 }
108}
109
110pub fn skips_syncs<S: std::hash::BuildHasher>(labels: &HashMap<String, String, S>) -> bool {
112 labels
113 .get(SKIP_SYNCS_LABEL)
114 .is_some_and(|version| version == "1")
115}
116
117pub fn image_syncs<S: std::hash::BuildHasher>(
120 labels: &HashMap<String, String, S>,
121) -> crate::config::ImageSyncs {
122 use crate::config::ImageSyncs;
123 if !skips_syncs(labels) {
124 ImageSyncs::Syncs
125 } else if labels.contains_key(VOLATILE_LAYERS_LABEL) {
126 ImageSyncs::Skips
127 } else {
128 ImageSyncs::SkipsCalls
129 }
130}
131
132#[derive(Clone)]
134pub struct JitConfig(pub String);
135
136impl fmt::Debug for JitConfig {
137 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
138 f.write_str("JitConfig(<redacted>)")
139 }
140}
141
142#[derive(Debug, Clone, PartialEq, Eq)]
144pub struct RunnerState {
145 pub running: bool,
147 pub exit_code: Option<i64>,
149 pub oom_killed: bool,
151 pub probe: Option<crate::api::RunnerProbe>,
154 pub privileged: bool,
156 pub runtime: Option<String>,
158 pub pid: Option<u32>,
160 pub address: Option<String>,
162 pub started_at: Option<u64>,
164 pub finished_at: Option<u64>,
166 pub memory: Option<u64>,
168 pub cpus: Option<u64>,
170}
171
172pub type Output = BoxStream<'static, Result<Vec<u8>, Error>>;
174
175#[derive(Debug, Clone, PartialEq, Eq)]
177pub struct Image {
178 pub reference: String,
180 pub labels: HashMap<String, String>,
182 pub bytes: u64,
184}
185
186#[derive(Debug, Clone, PartialEq, Eq)]
188pub struct TaggedImage {
189 pub id: String,
191 pub tags: Vec<String>,
193 pub digests: Vec<String>,
195}
196
197#[derive(Debug, Clone, PartialEq, Eq)]
199pub struct File {
200 pub name: String,
202 pub contents: String,
204}
205
206#[derive(Debug, Clone, Default, PartialEq, Eq)]
208pub struct Files {
209 pub files: Vec<File>,
211 pub complete: bool,
214}
215
216pub(crate) const FILES_LIMIT: usize = 16 << 20;
219
220pub struct Tty {
222 pub container: String,
224 pub output: BoxStream<'static, Result<Vec<u8>, Error>>,
226 pub input: Pin<Box<dyn tokio::io::AsyncWrite + Send>>,
228}
229
230impl fmt::Debug for Tty {
231 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
232 f.debug_struct("Tty")
233 .field("container", &self.container)
234 .finish_non_exhaustive()
235 }
236}
237
238#[derive(Debug, Clone, Copy, PartialEq, Eq)]
240pub struct TtySize {
241 pub cols: u16,
243 pub rows: u16,
245}
246
247pub fn cpu_shares(cpus: u32) -> i64 {
253 (i64::from(cpus) * 1024).clamp(2, 262_144)
254}
255
256pub fn create_body(spec: &RunnerSpec, isolation: &dyn Isolation) -> ContainerCreateBody {
258 let memory = i64::try_from(spec.memory.0).unwrap_or(i64::MAX);
260 let mut host_config = HostConfig {
261 cgroup_parent: Some(spec.slice.clone()),
262 memory: Some(memory),
263 memory_swap: Some(memory),
265 nano_cpus: Some(i64::from(spec.cpus) * 1_000_000_000),
266 cpu_shares: Some(cpu_shares(spec.cpus)),
267 init: Some(true),
268 mounts: Some(spec.mounts.clone()),
269 network_mode: spec.network.clone(),
270 ..Default::default()
271 };
272 isolation.configure(&mut host_config);
273 let mut labels: HashMap<String, String> = spec.labels.clone().into_iter().collect();
274 labels.insert("runnerd.profile".to_owned(), spec.profile.clone());
275 labels.insert("runnerd.isolation".to_owned(), isolation.name().to_owned());
276 ContainerCreateBody {
277 image: Some(spec.image.clone()),
278 env: Some(spec.env.clone()),
279 entrypoint: spec.entrypoint.clone(),
280 labels: Some(labels),
281 host_config: Some(host_config),
282 ..Default::default()
283 }
284}
285
286#[derive(Debug, Clone, PartialEq, Eq)]
288pub struct Container {
289 pub id: String,
291 pub name: String,
293 pub running: bool,
295 pub labels: HashMap<String, String>,
297}
298
299#[derive(Debug)]
302pub struct StartError {
303 pub container: Option<String>,
306 pub may_have_run: bool,
309 pub error: Error,
311}
312
313impl fmt::Display for StartError {
314 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
315 self.error.fmt(f)
316 }
317}
318
319impl std::error::Error for StartError {
320 fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
321 Some(&self.error)
322 }
323}
324
325impl From<StartError> for Error {
327 fn from(failed: StartError) -> Error {
328 failed.error
329 }
330}
331
332fn refused(error: &Error) -> bool {
335 matches!(
336 error,
337 Error::DockerResponseServerError {
338 status_code: 400..=499,
339 ..
340 }
341 )
342}
343
344pub fn is_unreachable(error: &Error) -> bool {
348 matches!(
349 error,
350 Error::HyperLegacyError { .. }
351 | Error::HyperResponseError { .. }
352 | Error::IOError { .. }
353 | Error::SocketNotFoundError(_)
354 )
355}
356
357pub fn is_declined(error: &Error) -> bool {
360 matches!(
361 error,
362 Error::DockerResponseServerError {
363 status_code: 400..=499 | 501,
364 ..
365 }
366 )
367}
368
369pub fn is_not_found(error: &Error) -> bool {
371 matches!(
372 error,
373 Error::DockerResponseServerError {
374 status_code: 404,
375 ..
376 }
377 )
378}
379
380pub fn is_being_removed(error: &Error) -> bool {
383 matches!(
384 error,
385 Error::DockerResponseServerError { status_code: 409, message }
386 if message.contains("is already in progress") || message.contains("marked for removal")
387 )
388}
389
390pub fn is_not_running(error: &Error) -> bool {
393 matches!(
394 error,
395 Error::DockerResponseServerError { status_code: 409, message } if message.ends_with(" is not running")
396 )
397}
398
399pub fn is_log_gone(error: &Error) -> bool {
402 is_not_found(error) || is_being_removed(error)
403}
404
405pub trait DockerApi: Send + Sync {
408 fn start(
411 &self,
412 spec: &RunnerSpec,
413 isolation: &dyn Isolation,
414 ) -> impl Future<Output = Result<String, StartError>> + Send;
415 fn inspect(&self, container: &str) -> impl Future<Output = Result<RunnerState, Error>> + Send;
417 fn remove(&self, container: &str) -> impl Future<Output = Result<(), Error>> + Send;
419 fn run(
421 &self,
422 name: &str,
423 body: ContainerCreateBody,
424 ) -> impl Future<Output = Result<String, Error>> + Send;
425 fn list(&self, label: &str) -> impl Future<Output = Result<Vec<Container>, Error>> + Send;
427 fn exits(&self) -> BoxStream<'static, Result<String, Error>>;
431 fn wait(&self, container: &str) -> impl Future<Output = Result<i64, Error>> + Send;
433 fn logs(&self, container: &str) -> impl Future<Output = Result<String, Error>> + Send;
435 fn logs_since(
438 &self,
439 container: &str,
440 since: u64,
441 ) -> impl Future<Output = Result<String, Error>> + Send;
442 fn exec(
445 &self,
446 container: &str,
447 command: Vec<String>,
448 ) -> impl Future<Output = Result<Output, Error>> + Send;
449 fn image_present(&self, image: &str) -> impl Future<Output = Result<bool, Error>> + Send;
451 fn image_skips_syncs(&self, image: &str) -> impl Future<Output = Result<bool, Error>> + Send;
453 fn pull(&self, image: &str) -> impl Future<Output = Result<(), Error>> + Send;
455 fn ensure_network(&self, name: &str) -> impl Future<Output = Result<(), Error>> + Send;
457 fn ensure_volume(&self, name: &str) -> impl Future<Output = Result<(), Error>> + Send;
459 fn remove_volume(&self, name: &str) -> impl Future<Output = Result<(), Error>> + Send;
462 fn holders(&self, volume: &str) -> impl Future<Output = Result<Vec<Container>, Error>> + Send;
464 fn volume_sizes(&self) -> impl Future<Output = Result<BTreeMap<String, u64>, Error>> + Send;
466 fn volumes(&self, prefix: &str) -> impl Future<Output = Result<Vec<String>, Error>> + Send;
468 fn changes(&self, container: &str) -> impl Future<Output = Result<Vec<String>, Error>> + Send;
470 fn written(&self, container: &str) -> impl Future<Output = Result<u64, Error>> + Send;
472 fn files(
475 &self,
476 container: &str,
477 directory: &str,
478 prefix: &str,
479 ) -> impl Future<Output = Result<Files, Error>> + Send;
480 fn commit(
483 &self,
484 container: &str,
485 reference: &str,
486 labels: BTreeMap<String, String>,
487 ) -> impl Future<Output = Result<u64, Error>> + Send;
488 fn images(&self, label: &str) -> impl Future<Output = Result<Vec<Image>, Error>> + Send;
490 fn image_id(
492 &self,
493 reference: &str,
494 ) -> impl Future<Output = Result<Option<String>, Error>> + Send;
495 fn tagged_images(&self) -> impl Future<Output = Result<Vec<TaggedImage>, Error>> + Send;
497 fn container_images(&self) -> impl Future<Output = Result<BTreeSet<String>, Error>> + Send;
499 fn remove_image(&self, reference: &str) -> impl Future<Output = Result<(), Error>> + Send;
501 fn free_space(&self) -> impl Future<Output = Result<u64, Error>> + Send;
504 fn root_dir(&self) -> impl Future<Output = Result<String, Error>> + Send;
506 fn shell(
509 &self,
510 name: &str,
511 body: ContainerCreateBody,
512 size: TtySize,
513 ) -> impl Future<Output = Result<Tty, Error>> + Send;
514}
515
516#[derive(Debug)]
518pub struct Docker {
519 api: bollard::Docker,
520}
521
522impl Docker {
523 async fn clean_up(&self, made: Option<String>, error: Error, may_have_run: bool) -> StartError {
526 let container = match made {
527 Some(container) => match self.remove(&container).await {
528 Err(failed) if !is_not_found(&failed) => Some(container),
529 _ => None,
530 },
531 None => None,
532 };
533 StartError {
534 container,
535 may_have_run,
536 error,
537 }
538 }
539
540 pub fn connect() -> Result<Docker, Error> {
542 connect_to(std::env::var("DOCKER_HOST").ok().as_deref())
543 }
544
545 pub async fn info(&self) -> Result<SystemInfo, Error> {
547 self.api.info().await
548 }
549
550 pub async fn image_labels(
552 &self,
553 image: &str,
554 ) -> Result<Option<HashMap<String, String>>, Error> {
555 match self.api.inspect_image(image).await {
556 Ok(inspect) => Ok(Some(
557 inspect
558 .config
559 .and_then(|config| config.labels)
560 .unwrap_or_default(),
561 )),
562 Err(error) if is_not_found(&error) => Ok(None),
563 Err(error) => Err(error),
564 }
565 }
566
567 async fn containers(&self, filter: &str, value: &str) -> Result<Vec<Container>, Error> {
569 let options = ListContainersOptionsBuilder::new()
570 .all(true)
571 .filters(&HashMap::from([(filter, vec![value])]))
572 .build();
573 let containers = self.api.list_containers(Some(options)).await?;
574
575 Ok(containers
576 .into_iter()
577 .map(|container| Container {
578 id: container.id.unwrap_or_default(),
579 name: container
580 .names
581 .and_then(|names| names.into_iter().next())
582 .unwrap_or_default()
583 .trim_start_matches('/')
584 .to_owned(),
585 running: container.state
586 == Some(bollard::models::ContainerSummaryStateEnum::RUNNING),
587 labels: container.labels.unwrap_or_default(),
588 })
589 .collect())
590 }
591
592 pub async fn registry_platforms(&self, image: &str) -> Result<Vec<String>, Error> {
595 let inspect = self.api.inspect_registry_image(image, None).await?;
596 Ok(inspect
597 .platforms
598 .into_iter()
599 .filter_map(|platform| Some(format!("{}/{}", platform.os?, platform.architecture?)))
600 .filter(|platform| platform != "unknown/unknown")
602 .collect())
603 }
604}
605
606impl DockerApi for Docker {
607 async fn start(
608 &self,
609 spec: &RunnerSpec,
610 isolation: &dyn Isolation,
611 ) -> Result<String, StartError> {
612 let options = CreateContainerOptionsBuilder::new()
613 .name(&spec.name)
614 .build();
615 let id = match self
616 .api
617 .create_container(Some(options), create_body(spec, isolation))
618 .await
619 {
620 Ok(created) => created.id,
621 Err(error) if refused(&error) => {
622 return Err(StartError {
623 container: None,
624 may_have_run: false,
625 error,
626 });
627 }
628 Err(error) => {
630 let made = match self.api.inspect_container(&spec.name, None).await {
631 Ok(found) => Some(found.id.unwrap_or_else(|| spec.name.clone())),
632 Err(missing) if is_not_found(&missing) => None,
633 Err(_) => Some(spec.name.clone()),
634 };
635 return Err(self.clean_up(made, error, false).await);
637 }
638 };
639 let uploaded = async {
640 if let Some(jit_config) = &spec.jit_config {
641 let options = UploadToContainerOptionsBuilder::new().path("/").build();
642 let tar = jit_config_tar(jit_config)?;
643 let tar = futures_util::stream::once(std::future::ready(tar.into()));
645 self.api
646 .upload_to_container(&id, Some(options), bollard::body_stream(tar))
647 .await?;
648 }
649 Ok::<(), Error>(())
650 };
651 if let Err(error) = uploaded.await {
652 return Err(self.clean_up(Some(id), error, false).await);
653 }
654 if let Err(error) = self.api.start_container(&id, None).await {
656 return Err(self.clean_up(Some(id), error, true).await);
657 }
658 Ok(id)
659 }
660
661 async fn inspect(&self, container: &str) -> Result<RunnerState, Error> {
662 let response = self.api.inspect_container(container, None).await?;
663 let state = response.state.unwrap_or_default();
664 let host_config = response.host_config.unwrap_or_default();
665 let running = state.running.unwrap_or(false);
666 let labels = response
667 .config
668 .and_then(|config| config.labels)
669 .unwrap_or_default();
670 let address = response
671 .network_settings
672 .and_then(|settings| settings.networks)
673 .and_then(|networks| {
674 networks
675 .into_values()
676 .find_map(|network| network.ip_address.filter(|address| !address.is_empty()))
677 });
678 Ok(RunnerState {
679 running,
680 exit_code: state.exit_code,
681 oom_killed: state.oom_killed.unwrap_or(false),
682 probe: if running {
683 None
684 } else {
685 probe(&labels, state.exit_code)
686 },
687 privileged: host_config.privileged.unwrap_or(false),
688 runtime: host_config.runtime,
689 pid: state
690 .pid
691 .and_then(|pid| u32::try_from(pid).ok())
692 .filter(|pid| *pid > 0),
693 address,
694 started_at: state.started_at.as_deref().and_then(crate::time::parse_ms),
695 finished_at: state.finished_at.as_deref().and_then(crate::time::parse_ms),
697 memory: host_config
698 .memory
699 .and_then(|memory| u64::try_from(memory).ok())
700 .filter(|memory| *memory > 0),
701 cpus: host_config
702 .nano_cpus
703 .and_then(|nano| u64::try_from(nano / 1_000_000).ok())
704 .filter(|cpus| *cpus > 0),
705 })
706 }
707
708 async fn remove(&self, container: &str) -> Result<(), Error> {
709 let options = RemoveContainerOptionsBuilder::new()
711 .force(true)
712 .v(true)
713 .build();
714 self.api.remove_container(container, Some(options)).await
715 }
716
717 async fn run(&self, name: &str, body: ContainerCreateBody) -> Result<String, Error> {
718 let options = CreateContainerOptionsBuilder::new().name(name).build();
719 let id = self.api.create_container(Some(options), body).await?.id;
720 if let Err(error) = self.api.start_container(&id, None).await {
721 let _ = self.remove(&id).await;
722 return Err(error);
723 }
724 Ok(id)
725 }
726
727 async fn list(&self, label: &str) -> Result<Vec<Container>, Error> {
728 self.containers("label", label).await
729 }
730
731 async fn holders(&self, volume: &str) -> Result<Vec<Container>, Error> {
732 self.containers("volume", volume).await
733 }
734
735 fn exits(&self) -> BoxStream<'static, Result<String, Error>> {
736 let filters = HashMap::from([
737 ("type", vec!["container"]),
738 ("event", vec!["die"]),
739 ("label", vec!["runnerd.job"]),
740 ]);
741 let options = EventsOptionsBuilder::new().filters(&filters).build();
742 self.api
743 .events(Some(options))
744 .try_filter_map(|event| std::future::ready(Ok(event.actor.and_then(|actor| actor.id))))
745 .boxed()
746 }
747
748 async fn wait(&self, container: &str) -> Result<i64, Error> {
749 let options = WaitContainerOptionsBuilder::new()
750 .condition("not-running")
751 .build();
752 let mut responses = self.api.wait_container(container, Some(options));
753 match responses.next().await {
754 Some(Ok(response)) => Ok(response.status_code),
755 Some(Err(Error::DockerContainerWaitError { code, .. })) => Ok(code),
757 Some(Err(error)) => Err(error),
758 None => Ok(0),
759 }
760 }
761
762 async fn logs(&self, container: &str) -> Result<String, Error> {
763 let options = LogsOptionsBuilder::new().stdout(true).stderr(true).build();
764 let chunks: Vec<_> = self
765 .api
766 .logs(container, Some(options))
767 .try_collect()
768 .await?;
769 Ok(chunks.iter().map(ToString::to_string).collect())
770 }
771
772 async fn logs_since(&self, container: &str, since: u64) -> Result<String, Error> {
773 let options = LogsOptionsBuilder::new()
774 .stdout(true)
775 .stderr(true)
776 .timestamps(true)
777 .since(i32::try_from(since).unwrap_or(i32::MAX))
778 .build();
779 let chunks: Vec<_> = self
780 .api
781 .logs(container, Some(options))
782 .try_collect()
783 .await?;
784 Ok(chunks.iter().map(ToString::to_string).collect())
785 }
786
787 async fn exec(&self, container: &str, command: Vec<String>) -> Result<Output, Error> {
788 let options = CreateExecOptions {
789 cmd: Some(command),
790 user: Some("root".to_owned()),
791 attach_stdout: Some(true),
792 attach_stderr: Some(true),
793 ..Default::default()
794 };
795 let exec = self.api.create_exec(container, options).await?;
796 match self.api.start_exec(&exec.id, None).await? {
797 StartExecResults::Attached { output, .. } => Ok(output
798 .try_filter_map(|output| {
799 std::future::ready(Ok(match output {
800 bollard::container::LogOutput::StdOut { message } => Some(message.to_vec()),
801 _ => None,
802 }))
803 })
804 .boxed()),
805 StartExecResults::Detached => Ok(futures_util::stream::empty().boxed()),
806 }
807 }
808
809 async fn image_present(&self, image: &str) -> Result<bool, Error> {
810 Ok(self.image_labels(image).await?.is_some())
811 }
812
813 async fn image_skips_syncs(&self, image: &str) -> Result<bool, Error> {
814 Ok(self
815 .image_labels(image)
816 .await?
817 .is_some_and(|labels| skips_syncs(&labels)))
818 }
819
820 async fn pull(&self, image: &str) -> Result<(), Error> {
821 let options = CreateImageOptionsBuilder::new().from_image(image).build();
822 let mut stream = self.api.create_image(Some(options), None, None);
823 let mut progress = PullProgress::default();
824 let mut logged = Instant::now();
825 info!("pulling {image}");
826 while let Some(update) = stream.try_next().await? {
827 progress.update(&update);
828 if logged.elapsed() >= PULL_LOG_INTERVAL {
829 logged = Instant::now();
830 info!("pulling {image}: {progress}");
831 }
832 }
833 info!("pulled {image}");
834 Ok(())
835 }
836
837 async fn ensure_network(&self, name: &str) -> Result<(), Error> {
838 match self.api.inspect_network(name, None).await {
839 Err(error) if is_not_found(&error) => {
840 let request = NetworkCreateRequest {
841 name: name.to_owned(),
842 driver: Some("bridge".to_owned()),
843 ..Default::default()
844 };
845 self.api.create_network(request).await.map(|_| ())
846 }
847 other => other.map(|_| ()),
848 }
849 }
850
851 async fn ensure_volume(&self, name: &str) -> Result<(), Error> {
852 let request = VolumeCreateRequest {
854 name: Some(name.to_owned()),
855 ..Default::default()
856 };
857 self.api.create_volume(request).await.map(|_| ())
858 }
859
860 async fn remove_volume(&self, name: &str) -> Result<(), Error> {
861 let options = RemoveVolumeOptionsBuilder::new().build();
862 match self.api.remove_volume(name, Some(options)).await {
863 Err(error) if is_not_found(&error) => Ok(()),
864 other => other,
865 }
866 }
867
868 async fn volumes(&self, prefix: &str) -> Result<Vec<String>, Error> {
869 let options = ListVolumesOptionsBuilder::new()
871 .filters(&HashMap::from([("name", vec![prefix])]))
872 .build();
873 Ok(self
874 .api
875 .list_volumes(Some(options))
876 .await?
877 .volumes
878 .unwrap_or_default()
879 .into_iter()
880 .map(|volume| volume.name)
881 .filter(|name| name.starts_with(prefix))
882 .collect())
883 }
884
885 async fn volume_sizes(&self) -> Result<BTreeMap<String, u64>, Error> {
886 let options = DataUsageOptionsBuilder::new().verbose(true).build();
888 let usage = self.api.df(Some(options)).await?;
889 Ok(usage
891 .volume_usage
892 .and_then(|volumes| volumes.items)
893 .unwrap_or_default()
894 .into_iter()
895 .filter_map(|volume| {
896 let name = volume.get("Name")?.as_str()?.to_owned();
897 let size = volume.get("UsageData")?.get("Size")?.as_u64()?;
898 Some((name, size))
899 })
900 .collect())
901 }
902
903 async fn changes(&self, container: &str) -> Result<Vec<String>, Error> {
904 Ok(self
905 .api
906 .container_changes(container)
907 .await?
908 .unwrap_or_default()
909 .into_iter()
910 .filter(|change| change.kind != ChangeType::_2)
911 .map(|change| change.path)
912 .collect())
913 }
914
915 async fn written(&self, container: &str) -> Result<u64, Error> {
916 let options = InspectContainerOptionsBuilder::new().size(true).build();
917 let response = self.api.inspect_container(container, Some(options)).await?;
918 Ok(response
919 .size_rw
920 .and_then(|bytes| u64::try_from(bytes).ok())
921 .unwrap_or(0))
922 }
923
924 async fn files(&self, container: &str, directory: &str, prefix: &str) -> Result<Files, Error> {
925 let options = DownloadFromContainerOptionsBuilder::new()
926 .path(directory)
927 .build();
928 let mut archive = self.api.download_from_container(container, Some(options));
929 let (chunks, received) = tokio::sync::mpsc::channel(4);
931 let prefix = prefix.to_owned();
932 let reader = tokio::task::spawn_blocking(move || {
933 files_in(
934 Chunks {
935 received,
936 chunk: Vec::new(),
937 at: 0,
938 },
939 &prefix,
940 )
941 });
942 let mut failed = None;
943 while let Some(chunk) = archive.next().await {
944 match chunk {
945 Ok(chunk) => {
947 if chunks.send(chunk.to_vec()).await.is_err() {
948 break;
949 }
950 }
951 Err(error) => {
952 failed = Some(error);
953 break;
954 }
955 }
956 }
957 drop(chunks);
958 let files = match reader.await {
959 Ok(files) => files,
960 Err(error) => match error.try_into_panic() {
961 Ok(panic) => std::panic::resume_unwind(panic),
962 Err(_) => Files::default(),
964 },
965 };
966 match failed {
967 Some(error) => Err(error),
968 None => Ok(files),
969 }
970 }
971
972 async fn commit(
973 &self,
974 container: &str,
975 reference: &str,
976 labels: BTreeMap<String, String>,
977 ) -> Result<u64, Error> {
978 let (repository, tag) = reference.rsplit_once(':').unwrap_or((reference, "latest"));
979 let options = CommitContainerOptionsBuilder::new()
980 .container(container)
981 .repo(repository)
982 .tag(tag)
983 .pause(false)
984 .build();
985 let config = ContainerConfig {
986 labels: Some(labels.into_iter().collect()),
987 ..Default::default()
988 };
989 let parent = match self.api.inspect_container(container, None).await?.image {
991 Some(parent) => self.api.inspect_image(&parent).await?.size.unwrap_or(0),
992 None => 0,
993 };
994 let id = self.api.commit_container(options, config).await?.id;
995 let size = self.api.inspect_image(&id).await?.size.unwrap_or(0);
996 Ok(u64::try_from(size - parent).unwrap_or(0))
997 }
998
999 async fn images(&self, label: &str) -> Result<Vec<Image>, Error> {
1000 let options = ListImagesOptionsBuilder::new()
1001 .filters(&HashMap::from([("label", vec![label])]))
1002 .build();
1003 let images = self.api.list_images(Some(options)).await?;
1004 let mut parents: HashMap<String, i64> = HashMap::new();
1005 let mut found = Vec::new();
1006 for image in images {
1007 let parent = if image.parent_id.is_empty() {
1008 0
1009 } else if let Some(size) = parents.get(&image.parent_id) {
1010 *size
1011 } else {
1012 let size = self
1013 .api
1014 .inspect_image(&image.parent_id)
1015 .await
1016 .ok()
1017 .and_then(|parent| parent.size)
1018 .unwrap_or(0);
1019 parents.insert(image.parent_id.clone(), size);
1020 size
1021 };
1022 found.push(Image {
1023 reference: image.repo_tags.into_iter().next().unwrap_or(image.id),
1024 labels: image.labels,
1025 bytes: u64::try_from(image.size - parent).unwrap_or(0),
1026 });
1027 }
1028 Ok(found)
1029 }
1030
1031 async fn image_id(&self, reference: &str) -> Result<Option<String>, Error> {
1032 match self.api.inspect_image(reference).await {
1033 Ok(inspect) => Ok(inspect.id),
1034 Err(error) if is_not_found(&error) => Ok(None),
1035 Err(error) => Err(error),
1036 }
1037 }
1038
1039 async fn tagged_images(&self) -> Result<Vec<TaggedImage>, Error> {
1040 let images = self
1041 .api
1042 .list_images(Some(ListImagesOptionsBuilder::new().build()))
1043 .await?;
1044 Ok(images
1045 .into_iter()
1046 .map(|image| TaggedImage {
1047 id: image.id,
1048 tags: image
1049 .repo_tags
1050 .into_iter()
1051 .filter(|tag| tag != "<none>:<none>")
1052 .collect(),
1053 digests: image
1054 .repo_digests
1055 .into_iter()
1056 .filter(|digest| digest != "<none>@<none>")
1057 .collect(),
1058 })
1059 .filter(|image| !image.tags.is_empty())
1060 .collect())
1061 }
1062
1063 async fn container_images(&self) -> Result<BTreeSet<String>, Error> {
1064 let options = ListContainersOptionsBuilder::new().all(true).build();
1065 let containers = self.api.list_containers(Some(options)).await?;
1066 Ok(containers
1067 .into_iter()
1068 .filter_map(|container| container.image_id)
1069 .collect())
1070 }
1071
1072 async fn remove_image(&self, reference: &str) -> Result<(), Error> {
1073 let options = RemoveImageOptionsBuilder::new().build();
1074 match self.api.remove_image(reference, Some(options), None).await {
1075 Err(error) if is_not_found(&error) => Ok(()),
1076 other => other.map(|_| ()),
1077 }
1078 }
1079
1080 async fn root_dir(&self) -> Result<String, Error> {
1081 Ok(self.api.info().await?.docker_root_dir.unwrap_or_default())
1082 }
1083
1084 async fn free_space(&self) -> Result<u64, Error> {
1085 let root = self.api.info().await?.docker_root_dir.unwrap_or_default();
1086 let statfs = |path: &std::path::Path| {
1088 let stat = rustix::fs::statvfs(path).map_err(std::io::Error::from)?;
1089 Ok(stat.f_bavail.saturating_mul(stat.f_frsize))
1090 };
1091 let paths = crate::observe::metrics::Paths::default();
1092 crate::observe::disk::free_space(&paths, std::path::Path::new(&root), statfs).map_err(
1093 |error| Error::IOError {
1094 err: std::io::Error::other(format!("cannot measure the disk of {root}: {error}")),
1095 },
1096 )
1097 }
1098
1099 async fn shell(
1100 &self,
1101 name: &str,
1102 body: ContainerCreateBody,
1103 size: TtySize,
1104 ) -> Result<Tty, Error> {
1105 let options = CreateContainerOptionsBuilder::new().name(name).build();
1106 let id = self.api.create_container(Some(options), body).await?.id;
1107 let attached = async {
1108 let options = AttachContainerOptionsBuilder::new()
1109 .stream(true)
1110 .stdin(true)
1111 .stdout(true)
1112 .stderr(true)
1113 .build();
1114 let attached = self.api.attach_container(&id, Some(options)).await?;
1115 self.api.start_container(&id, None).await?;
1116 let resize = ResizeContainerTTYOptionsBuilder::new()
1117 .w(i32::from(size.cols))
1118 .h(i32::from(size.rows))
1119 .build();
1120 self.api.resize_container_tty(&id, resize).await?;
1121 Ok::<_, Error>(attached)
1122 };
1123 match attached.await {
1124 Ok(attached) => Ok(Tty {
1125 container: id,
1126 output: attached
1127 .output
1128 .map_ok(|output| output.into_bytes().to_vec())
1129 .boxed(),
1130 input: attached.input,
1131 }),
1132 Err(error) => {
1133 let _ = self.remove(&id).await;
1134 Err(error)
1135 }
1136 }
1137 }
1138}
1139
1140const PULL_LOG_INTERVAL: Duration = Duration::from_secs(10);
1142
1143#[derive(Debug, Default)]
1145struct PullProgress {
1146 layers: BTreeMap<String, Layer>,
1147}
1148
1149#[derive(Debug, Default)]
1150struct Layer {
1151 downloaded: u64,
1152 size: u64,
1153 done: bool,
1154}
1155
1156impl PullProgress {
1157 fn update(&mut self, update: &CreateImageInfo) {
1158 let (Some(id), Some(status)) = (&update.id, &update.status) else {
1159 return;
1160 };
1161 let bytes = |value: Option<i64>| value.and_then(|value| u64::try_from(value).ok());
1162 let detail = update.progress_detail.as_ref();
1163 match status.as_str() {
1164 "Pulling fs layer" | "Waiting" => {
1165 self.layers.entry(id.clone()).or_default();
1166 }
1167 "Downloading" => {
1168 let layer = self.layers.entry(id.clone()).or_default();
1169 layer.downloaded =
1170 bytes(detail.and_then(|detail| detail.current)).unwrap_or(layer.downloaded);
1171 layer.size = bytes(detail.and_then(|detail| detail.total)).unwrap_or(layer.size);
1172 }
1173 "Download complete" => {
1174 if let Some(layer) = self.layers.get_mut(id) {
1175 layer.downloaded = layer.size;
1176 }
1177 }
1178 "Pull complete" | "Already exists" => {
1179 let layer = self.layers.entry(id.clone()).or_default();
1180 layer.downloaded = layer.size;
1181 layer.done = true;
1182 }
1183 _ => {}
1184 }
1185 }
1186}
1187
1188impl fmt::Display for PullProgress {
1189 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1190 let done = self.layers.values().filter(|layer| layer.done).count();
1191 let downloaded: u64 = self.layers.values().map(|layer| layer.downloaded).sum();
1192 let size: u64 = self.layers.values().map(|layer| layer.size).sum();
1193 write!(
1194 f,
1195 "{done} of {} layers done, {} of {} downloaded",
1196 self.layers.len(),
1197 ByteSize(downloaded),
1198 ByteSize(size)
1199 )
1200 }
1201}
1202
1203fn connect_to(docker_host: Option<&str>) -> Result<Docker, Error> {
1204 let api = match docker_host.filter(|host| !host.is_empty()) {
1205 None => bollard::Docker::connect_with_socket_defaults()?,
1206 Some(host) if host.starts_with("unix://") => {
1207 bollard::Docker::connect_with_socket(host, 120, bollard::API_DEFAULT_VERSION)?
1208 }
1209 Some(host) => {
1211 return Err(Error::UnsupportedURISchemeError {
1212 uri: host.to_owned(),
1213 });
1214 }
1215 };
1216 Ok(Docker { api })
1217}
1218
1219struct Chunks {
1222 received: tokio::sync::mpsc::Receiver<Vec<u8>>,
1223 chunk: Vec<u8>,
1224 at: usize,
1225}
1226
1227impl std::io::Read for Chunks {
1228 fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
1229 while self.at == self.chunk.len() {
1230 match self.received.blocking_recv() {
1231 Some(chunk) => {
1232 self.chunk = chunk;
1233 self.at = 0;
1234 }
1235 None => return Ok(0),
1236 }
1237 }
1238 let n = buf.len().min(self.chunk.len() - self.at);
1239 buf[..n].copy_from_slice(&self.chunk[self.at..self.at + n]);
1240 self.at += n;
1241 Ok(n)
1242 }
1243}
1244
1245struct Ends<R> {
1248 inner: R,
1249 ran_out: bool,
1250}
1251
1252impl<R: std::io::Read> std::io::Read for Ends<R> {
1253 fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
1254 let n = self.inner.read(buf)?;
1255 self.ran_out = n == 0 && !buf.is_empty();
1256 Ok(n)
1257 }
1258}
1259
1260pub(crate) fn files_in(tar: impl std::io::Read, prefix: &str) -> Files {
1265 let mut tar = Ends {
1266 inner: tar,
1267 ran_out: false,
1268 };
1269 let mut read = Files::default();
1270 let ended = files_until_end(&mut tar, prefix, &mut read.files);
1271 read.complete = ended && !tar.ran_out;
1272 read
1273}
1274
1275fn files_until_end(tar: impl std::io::Read, prefix: &str, files: &mut Vec<File>) -> bool {
1278 let mut archive = tar::Archive::new(tar);
1279 let Ok(entries) = archive.entries() else {
1280 return false;
1281 };
1282 let mut kept = 0;
1283 for entry in entries {
1284 let Ok(mut entry) = entry else {
1285 return false;
1286 };
1287 let name = match entry.path() {
1288 Ok(path) if path.components().count() == 2 => path
1289 .file_name()
1290 .map(|name| name.to_string_lossy().into_owned()),
1291 _ => None,
1292 };
1293 let Some(name) =
1294 name.filter(|name| name.starts_with(prefix) && entry.header().entry_type().is_file())
1295 else {
1296 continue;
1297 };
1298 let room = FILES_LIMIT - kept;
1299 let mut contents = Vec::new();
1300 let whole = std::io::Read::read_to_end(
1302 &mut std::io::Read::take(&mut entry, room as u64 + 1),
1303 &mut contents,
1304 )
1305 .is_ok()
1306 && contents.len() <= room;
1307 contents.truncate(room);
1308 kept += contents.len();
1309 files.push(File {
1310 name,
1311 contents: String::from_utf8_lossy(&contents).into_owned(),
1312 });
1313 if !whole {
1314 return false;
1315 }
1316 }
1317 true
1318}
1319
1320fn jit_config_tar(jit_config: &JitConfig) -> std::io::Result<Vec<u8>> {
1321 let mut tar = tar::Builder::new(Vec::new());
1322 let path = JIT_CONFIG_PATH.trim_start_matches('/');
1323 let mut dir = tar::Header::new_gnu();
1324 dir.set_entry_type(tar::EntryType::Directory);
1325 dir.set_mode(0o700);
1326 dir.set_size(0);
1327 tar.append_data(
1328 &mut dir,
1329 path.rsplit_once('/').map_or("", |(dir, _)| dir),
1330 std::io::empty(),
1331 )?;
1332 let mut file = tar::Header::new_gnu();
1333 file.set_mode(0o400);
1334 file.set_size(jit_config.0.len() as u64);
1335 tar.append_data(&mut file, path, jit_config.0.as_bytes())?;
1336 tar.into_inner()
1337}
1338
1339#[cfg(test)]
1340pub(crate) mod tests;