1use std::collections::{BTreeMap, BTreeSet, HashMap};
7use std::time::Duration;
8
9use bollard::models::{ContainerCreateBody, HostConfig, Mount, MountType};
10use tokio::sync::{mpsc, watch};
11use tokio::task::JoinHandle;
12use tokio::time::Instant;
13use tracing::{info, warn};
14
15use super::control::ControlError;
16use super::{Engine, Occupant};
17use crate::admission::HIGH_WATER;
18use crate::api;
19use crate::config::ByteSize;
20use crate::docker::{self, DockerApi, Tty, TtySize, cpu_shares, is_not_found};
21use crate::github::GithubApi;
22use crate::observe::capture;
23use crate::record;
24use crate::scheduler::SlotRef;
25use crate::time;
26
27#[derive(Debug)]
30pub(super) struct Recording {
31 record: api::Record,
32 container: String,
33 estimate: u64,
36 stage: Stage,
37}
38
39#[derive(Debug)]
40enum Stage {
41 Checking(JoinHandle<Result<Checked, String>>),
43 Committing(JoinHandle<Result<u64, String>>),
45}
46
47impl Stage {
48 fn is_finished(&self) -> bool {
49 match self {
50 Stage::Checking(task) => task.is_finished(),
51 Stage::Committing(task) => task.is_finished(),
52 }
53 }
54}
55
56#[derive(Debug)]
59struct Checked {
60 incoming: u64,
61 free: Option<u64>,
62}
63
64#[derive(Debug)]
66pub(super) enum Undecided {
67 Docker(docker::Error),
69 Unread,
71}
72
73impl std::fmt::Display for Undecided {
74 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
75 match self {
76 Undecided::Docker(error) => error.fmt(f),
77 Undecided::Unread => f.write_str("its diagnostic files were not all read"),
78 }
79 }
80}
81
82#[derive(Debug)]
84pub(super) struct RecordState {
85 list: Vec<api::Record>,
87 pub(super) listed: bool,
89 pub(super) recordings: BTreeMap<u64, Recording>,
91 pub(super) shells: BTreeMap<String, Shell>,
93 opened: u64,
95 status: watch::Sender<api::Records>,
97 done: mpsc::UnboundedSender<()>,
99 pub(super) finished: mpsc::UnboundedReceiver<()>,
100}
101
102impl RecordState {
103 pub(super) fn new() -> (RecordState, watch::Receiver<api::Records>) {
105 let (status, receiver) = watch::channel(api::Records::default());
106 let (done, finished) = mpsc::unbounded_channel();
107 let state = RecordState {
108 list: Vec::new(),
109 listed: false,
110 recordings: BTreeMap::new(),
111 shells: BTreeMap::new(),
112 opened: 0,
113 status,
114 done,
115 finished,
116 };
117 (state, receiver)
118 }
119}
120
121#[derive(Debug, Clone)]
123pub(super) struct Shell {
124 job: u64,
125 pub(super) profile: String,
127 pub(super) memory: u64,
129 opened: Instant,
130}
131
132const MAX_SHELL: Duration = Duration::from_hours(8);
134
135#[derive(Debug, Clone, PartialEq, Eq)]
137pub enum RecordSelection {
138 Job(u64),
140 All,
142 OlderThan(Duration),
144}
145
146const SHELL_SCRIPT: &str = r#"
149. /usr/local/lib/runnerd-layers.sh && mount_layers > /dev/null 2>&1
150install -d -o runner -g runner -m 700 "$XDG_RUNTIME_DIR" 2> /dev/null
151cd "$1" 2> /dev/null || cd /home/runner
152exec setpriv --reuid=runner --regid=runner --init-groups -- env HOME=/home/runner USER=runner LOGNAME=runner bash -l
153"#;
154
155const COMMIT_FACTOR: u64 = 2;
158
159impl<G: GithubApi + 'static, D: DockerApi + 'static> Engine<G, D> {
160 fn records_failures(&self, profile: &str) -> bool {
162 self.config
163 .profiles
164 .get(profile)
165 .or_else(|| self.retired.get(profile))
166 .is_some_and(|profile| profile.record_failures)
167 }
168
169 pub(super) async fn begin_record(&mut self, slot: &SlotRef, occupant: &Occupant) -> bool {
173 let job = self
175 .occupied()
176 .find(|(_, candidate)| candidate.runner_name == occupant.runner_name)
177 .and_then(|(_, candidate)| candidate.job)
178 .or(occupant.job);
179 let Some(job) = job else {
180 return false;
181 };
182 if !self.records_failures(&slot.profile) {
183 return false;
184 }
185 if !self.records.listed {
186 self.list_records().await;
187 if !self.records.listed {
188 let why = "Docker did not list the records kept".to_owned();
189 self.skip_record(job, &occupant.repository, why);
190 return false;
191 }
192 }
193 let summary = self
194 .store
195 .job(job)
196 .await
197 .ok()
198 .flatten()
199 .map(|job| job.summary);
200 let record = api::Record {
201 job,
202 repository: occupant.repository.clone(),
203 profile: slot.profile.clone(),
204 workflow: summary
205 .as_ref()
206 .and_then(|summary| summary.workflow.clone()),
207 name: summary.map(|summary| summary.name),
208 created_at: time::now_ms() / 1000,
209 bytes: 0,
210 image: record::image(job),
211 };
212 info!(
213 "{}: keeping job {job}'s runner as {}",
214 occupant.repository, record.image
215 );
216 let docker = self.docker.clone();
217 let container = occupant.container.clone();
218 let done = self.records.done.clone();
219 let check = tokio::spawn(async move {
220 let checked = check(&*docker, &container).await;
221 let _ = done.send(());
222 checked
223 });
224 self.records.recordings.insert(
225 job,
226 Recording {
227 record,
228 container: occupant.container.clone(),
229 estimate: 0,
230 stage: Stage::Checking(check),
231 },
232 );
233 true
234 }
235
236 pub(super) async fn keep_left_record(
243 &mut self,
244 slot: &SlotRef,
245 occupant: &Occupant,
246 ) -> Result<bool, Undecided> {
247 let recorded =
248 |records: &RecordState, job| records.list.iter().any(|record| record.job == job);
249 if !self.records_failures(&slot.profile)
250 || occupant.job.is_some_and(|job| recorded(&self.records, job))
251 {
252 return Ok(false);
253 }
254 let failed = match self.docker.logs(&occupant.container).await {
255 Ok(log) => record::failed(&log),
256 Err(error) if docker::is_declined(&error) => false,
257 Err(error) => return Err(Undecided::Docker(error)),
258 };
259 if !failed {
260 return Ok(false);
261 }
262 let job = match occupant.job {
263 Some(job) => Some(job),
264 None => self.left_job(&occupant.container).await?,
265 };
266 let Some(job) = job else {
267 warn!(
268 "{}: {} ran a job that failed, but names it nowhere: no record of it is kept",
269 occupant.repository, occupant.runner_name
270 );
271 return Ok(false);
272 };
273 if recorded(&self.records, job) {
274 return Ok(false);
275 }
276 let occupant = Occupant {
277 job: Some(job),
278 ..occupant.clone()
279 };
280 Ok(self.begin_record(slot, &occupant).await)
281 }
282
283 async fn left_job(&self, container: &str) -> Result<Option<u64>, Undecided> {
286 let read = match self
287 .docker
288 .files(container, capture::DIAG, capture::WORKER_LOG)
289 .await
290 {
291 Ok(read) => read,
292 Err(error) if docker::is_declined(&error) => return Ok(None),
293 Err(error) => return Err(Undecided::Docker(error)),
294 };
295 match read
296 .files
297 .iter()
298 .find_map(|file| capture::worker_job(&file.contents))
299 {
300 Some(job) => Ok(Some(job)),
301 None if !read.complete => Err(Undecided::Unread),
303 None => Ok(None),
304 }
305 }
306
307 fn skip_record(&self, job: u64, repository: &str, why: String) {
309 warn!("{repository}: no record of job {job}: {why}");
310 self.store.record_skipped(job, why);
311 }
312
313 async fn make_room_and_commit(
316 &mut self,
317 record: api::Record,
318 container: String,
319 checked: Checked,
320 ) {
321 let job = record.job;
322 let pending: u64 = self
324 .records
325 .recordings
326 .values()
327 .map(|other| other.estimate)
328 .sum();
329 let room = record::make_room(
330 &self.records.list,
331 &self.shell_jobs(),
332 &self.config.records,
333 checked.incoming,
334 checked.free.map(|free| free.saturating_sub(pending)),
335 time::now_ms() / 1000,
336 );
337 let removed = match room {
338 Ok(removals) => {
339 let mut removed = Ok(());
340 for (old, why) in removals {
341 if let Err(error) = self.remove_record(old, why).await {
342 removed = Err(error.to_string());
343 break;
344 }
345 }
346 removed
347 }
348 Err(why) => Err(why),
349 };
350 if let Err(why) = removed {
351 self.skip_record(job, &record.repository, why);
352 self.remove_container(&container).await;
353 return;
354 }
355 let labels = labels(&record);
356 let docker = self.docker.clone();
357 let image = record.image.clone();
358 let done = self.records.done.clone();
359 let committed_container = container.clone();
360 let commit = tokio::spawn(async move {
361 let container = committed_container;
362 let committed = docker.commit(&container, &image, labels).await;
363 if let Err(error) = docker.remove(&container).await
364 && !is_not_found(&error)
365 {
366 warn!("cannot remove {container}: {error}");
367 }
368 let _ = done.send(());
369 committed.map_err(|error| error.to_string())
370 });
371 self.records.recordings.insert(
372 job,
373 Recording {
374 record,
375 container,
376 estimate: checked.incoming,
377 stage: Stage::Committing(commit),
378 },
379 );
380 }
381
382 async fn remove_container(&self, container: &str) {
383 if let Err(error) = self.docker.remove(container).await
384 && !is_not_found(&error)
385 {
386 warn!("cannot remove {container}: {error}");
387 }
388 }
389
390 pub(super) async fn collect_recordings(&mut self) {
393 while self.records.finished.try_recv().is_ok() {}
394 let done: Vec<u64> = self
395 .records
396 .recordings
397 .iter()
398 .filter(|(_, recording)| recording.stage.is_finished())
399 .map(|(job, _)| *job)
400 .collect();
401 for job in done {
402 let Some(recording) = self.records.recordings.remove(&job) else {
403 continue;
404 };
405 let Recording {
406 record,
407 container,
408 stage,
409 ..
410 } = recording;
411 let task = match stage {
412 Stage::Checking(task) => {
413 let checked = task
414 .await
415 .map_err(|error| error.to_string())
416 .and_then(|checked| checked);
417 match checked {
418 Ok(checked) => self.make_room_and_commit(record, container, checked).await,
419 Err(why) => {
420 self.skip_record(job, &record.repository, why);
421 self.remove_container(&container).await;
422 }
423 }
424 continue;
425 }
426 Stage::Committing(task) => task,
427 };
428 match task
429 .await
430 .map_err(|error| error.to_string())
431 .and_then(|committed| committed)
432 {
433 Ok(bytes) => self.keep_record(record, bytes).await,
434 Err(error) => self.skip_record(
435 job,
436 &record.repository,
437 format!("Docker did not commit it: {error}"),
438 ),
439 }
440 }
441 }
442
443 async fn keep_record(&mut self, mut record: api::Record, bytes: u64) {
445 let job = record.job;
446 record.bytes = bytes;
447 self.name_from_history(&mut record).await;
448 info!(
449 "{}: kept job {job}'s runner as {} ({})",
450 record.repository,
451 record.image,
452 ByteSize(bytes)
453 );
454 let removals = record::make_room(
455 &self.records.list,
456 &self.shell_jobs(),
457 &self.config.records,
458 bytes,
459 None,
460 record.created_at,
461 );
462 self.store.record_made(record.clone());
463 self.records.list.push(record);
464 let removals = removals.unwrap_or_else(|_| vec![(job, api::RecordRemoval::Size)]);
465 for (old, why) in removals {
466 if let Err(error) = self.remove_record(old, why).await {
467 warn!("{error}");
468 }
469 }
470 }
471
472 async fn name_from_history(&self, record: &mut api::Record) {
476 if record.workflow.is_some() && record.name.is_some() {
477 return;
478 }
479 self.store.flush().await;
480 if let Ok(Some(job)) = self.store.job(record.job).await {
481 record.workflow = record.workflow.take().or(job.summary.workflow);
482 record.name.get_or_insert(job.summary.name);
483 }
484 }
485
486 pub(super) async fn list_records(&mut self) {
488 match self.docker.images(record::LABEL).await {
489 Ok(images) => {
490 let mut records: Vec<api::Record> = images
491 .into_iter()
492 .filter_map(|image| {
493 let label = |name: &str| image.labels.get(name).cloned();
494 Some(api::Record {
495 job: label(record::LABEL)?.parse().ok()?,
496 repository: label("runnerd.repository").unwrap_or_default(),
497 profile: label("runnerd.profile").unwrap_or_default(),
498 workflow: label("runnerd.workflow"),
499 name: label("runnerd.name"),
500 created_at: label("runnerd.created-at")
501 .and_then(|at| at.parse().ok())
502 .unwrap_or(0),
503 bytes: image.bytes,
504 image: image.reference,
505 })
506 })
507 .collect();
508 records.sort_by_key(|record| (record.created_at, record.job));
509 for record in &mut records {
510 self.name_from_history(record).await;
511 let noted = self.store.job(record.job).await.ok().flatten();
513 if noted
514 .is_some_and(|job| job.record.is_none_or(|note| note.created_at.is_none()))
515 {
516 self.store.record_made(record.clone());
517 }
518 }
519 self.records.list = records;
520 self.records.listed = true;
521 }
522 Err(error) => warn!("cannot list the records: {error}"),
523 }
524 }
525
526 pub(super) async fn expire_records(&mut self) {
528 let expired = record::expired(
529 &self.records.list,
530 &self.shell_jobs(),
531 &self.config.records,
532 time::now_ms() / 1000,
533 );
534 for (job, why) in expired {
535 if let Err(error) = self.remove_record(job, why).await {
536 warn!("{error}");
537 }
538 }
539 }
540
541 async fn remove_record(
543 &mut self,
544 job: u64,
545 why: api::RecordRemoval,
546 ) -> Result<u64, ControlError> {
547 let Some(index) = self
548 .records
549 .list
550 .iter()
551 .position(|record| record.job == job)
552 else {
553 return Err(ControlError::NotFound(format!(
554 "job {job} has no record; `runnerctl records` lists them"
555 )));
556 };
557 let image = self.records.list[index].image.clone();
558 self.docker.remove_image(&image).await.map_err(|error| {
559 ControlError::Failed(format!("cannot remove the record of job {job}: {error}"))
560 })?;
561 let removed = self.records.list.remove(index);
562 info!("removed the record of job {job} ({why})");
563 self.store.record_removed(job, time::now_ms(), why);
564 Ok(removed.bytes)
565 }
566
567 pub(super) async fn remove_records(
569 &mut self,
570 selection: &RecordSelection,
571 ) -> Result<api::RemovedRecords, ControlError> {
572 let open = self.shell_jobs();
573 let now = time::now_ms() / 1000;
574 let jobs: Vec<u64> = match selection {
575 RecordSelection::Job(job) => {
576 if open.contains(job) {
577 return Err(ControlError::Conflict(format!(
578 "a shell is open in the record of job {job}; remove it once the shell exits"
579 )));
580 }
581 vec![*job]
582 }
583 RecordSelection::All => self.records.list.iter().map(|record| record.job).collect(),
584 RecordSelection::OlderThan(age) => self
585 .records
586 .list
587 .iter()
588 .filter(|record| now.saturating_sub(record.created_at) > age.as_secs())
589 .map(|record| record.job)
590 .collect(),
591 };
592 let mut removed = api::RemovedRecords::default();
593 for job in jobs.into_iter().filter(|job| !open.contains(job)) {
594 removed.bytes += self.remove_record(job, api::RecordRemoval::User).await?;
595 removed.jobs.push(job);
596 }
597 Ok(removed)
598 }
599
600 fn shell_jobs(&self) -> BTreeSet<u64> {
602 self.records
603 .shells
604 .values()
605 .map(|shell| shell.job)
606 .collect()
607 }
608
609 pub(super) fn shell_profiles(&self) -> BTreeSet<&str> {
611 self.records
612 .shells
613 .values()
614 .map(|shell| shell.profile.as_str())
615 .collect()
616 }
617
618 pub(super) fn shell_memory(&self) -> u64 {
620 self.records.shells.values().map(|shell| shell.memory).sum()
621 }
622
623 pub(super) fn shell_budget(&self) -> Vec<api::ShellMemory> {
625 self.records
626 .shells
627 .values()
628 .map(|shell| api::ShellMemory {
629 job: shell.job,
630 memory: shell.memory,
631 })
632 .collect()
633 }
634
635 pub(super) fn publish_records(&self) {
637 let records = self.records_status();
638 self.records.status.send_if_modified(|current| {
639 let changed = *current != records;
640 *current = records;
641 changed
642 });
643 }
644
645 fn records_status(&self) -> api::Records {
647 let limits = &self.config.records;
648 api::Records {
649 records: self.records.list.clone(),
650 limits: api::RecordLimits {
651 max_total: limits.max_total.0,
652 max_count: limits.max_count,
653 max_age: limits.max_age.as_secs(),
654 min_free: limits.min_free.0,
655 },
656 shells: self.shell_jobs().into_iter().collect(),
657 }
658 }
659
660 pub(super) async fn open_shell(
663 &mut self,
664 job: u64,
665 size: TtySize,
666 ) -> Result<Tty, ControlError> {
667 let Some(record) = self
668 .records
669 .list
670 .iter()
671 .find(|record| record.job == job)
672 .cloned()
673 else {
674 return Err(ControlError::NotFound(format!(
675 "job {job} has no record; `runnerctl records` lists them"
676 )));
677 };
678 let profile = self.config.profiles.get(&record.profile);
679 let Some(profile) = profile.or_else(|| self.retired.get(&record.profile)) else {
680 return Err(ControlError::Invalid(format!(
681 "the record of job {job} ran in profile {}, which the config no longer has",
682 record.profile
683 )));
684 };
685 let (memory, cpus) = (profile.memory.0, profile.cpus);
686 if self.reseeds.contains_key(&record.profile) {
688 return Err(ControlError::Conflict(format!(
689 "{} is reseeding its snapshot, which a shell mounts; open it once the reseed is done",
690 record.profile
691 )));
692 }
693 if self.trim.as_ref().is_some_and(|trim| !trim.is_finished()) {
695 return Err(ControlError::Conflict(
696 "the shared caches, which a shell mounts, are being trimmed; open it once the trim is done"
697 .to_owned(),
698 ));
699 }
700 if let Some(max) = self.memory_max {
701 let in_use = self.memory_in_use();
702 if in_use.saturating_add(memory) > max {
703 return Err(ControlError::Conflict(format!(
704 "the budget is full: {} of the slice's {} is held, and a shell takes {} as a slot of {} \
705 does; open it once a job finishes",
706 ByteSize(in_use),
707 ByteSize(max),
708 ByteSize(memory),
709 record.profile
710 )));
711 }
712 }
713 if self.admitting(&mut [], None).held {
715 return Err(ControlError::Conflict(format!(
716 "the slice is at {HIGH_WATER}% of its MemoryMax, where measured admission starts nothing; open it \
717 once it falls under"
718 )));
719 }
720 let body = self.shell_body(&record, memory, cpus);
721 self.records.opened += 1;
722 let name = format!(
723 "runnerd-shell-{job}-{}-{}",
724 time::now_ms() / 1000,
725 self.records.opened
726 );
727 let tty = self
728 .docker
729 .shell(&name, body, size)
730 .await
731 .map_err(|error| ControlError::Failed(format!("cannot start a shell: {error}")))?;
732 info!("opened a shell in the record of job {job}");
733 self.records.shells.insert(
734 tty.container.clone(),
735 Shell {
736 job,
737 profile: record.profile,
738 memory,
739 opened: Instant::now(),
740 },
741 );
742 Ok(tty)
743 }
744
745 fn shell_body(&self, record: &api::Record, memory: u64, cpus: u32) -> ContainerCreateBody {
748 let workspace = record::workspace(&record.repository);
749 let limit = i64::try_from(memory).unwrap_or(i64::MAX);
750 let mut mounts = vec![Mount {
751 target: Some("/state".to_owned()),
752 typ: Some(MountType::VOLUME),
753 ..Default::default()
754 }];
755 if self.config.profiles.contains_key(&record.profile) {
756 mounts.extend(
758 crate::cache::runner_mounts(&self.names, &record.profile, 1)
759 .into_iter()
760 .filter(|mount| mount.target.as_deref() != Some("/state"))
761 .map(|mount| Mount {
762 read_only: Some(true),
763 ..mount
764 }),
765 );
766 }
767 let mut host_config = HostConfig {
768 cgroup_parent: Some(self.config.host.slice.clone()),
769 memory: Some(limit),
770 memory_swap: Some(limit),
771 nano_cpus: Some(i64::from(cpus) * 1_000_000_000),
772 cpu_shares: Some(cpu_shares(cpus)),
773 init: Some(true),
774 mounts: Some(mounts),
775 network_mode: Some(self.names.network()),
776 ..Default::default()
777 };
778 self.isolation.configure(&mut host_config);
779 ContainerCreateBody {
780 image: Some(record.image.clone()),
781 entrypoint: Some(vec![
782 "bash".to_owned(),
783 "-c".to_owned(),
784 SHELL_SCRIPT.to_owned(),
785 ]),
786 cmd: Some(vec!["shell".to_owned(), workspace]),
787 env: Some(vec!["TERM=xterm-256color".to_owned()]),
788 tty: Some(true),
789 open_stdin: Some(true),
790 stdin_once: Some(true),
791 attach_stdin: Some(true),
792 attach_stdout: Some(true),
793 attach_stderr: Some(true),
794 labels: Some(HashMap::from([(
795 record::SHELL_LABEL.to_owned(),
796 record.job.to_string(),
797 )])),
798 host_config: Some(host_config),
799 ..Default::default()
800 }
801 }
802
803 pub(super) async fn close_shell(&mut self, container: &str) {
805 let Some(shell) = self.records.shells.get(container) else {
806 return;
807 };
808 let job = shell.job;
809 match self.docker.remove(container).await {
810 Err(error) if !is_not_found(&error) => {
811 warn!("cannot remove the shell in the record of job {job}: {error}");
812 }
813 _ => {
814 info!("closed the shell in the record of job {job}");
815 self.records.shells.remove(container);
816 }
817 }
818 }
819
820 pub(super) async fn collect_shells(&mut self) {
823 let shells: Vec<(String, Instant)> = self
824 .records
825 .shells
826 .iter()
827 .map(|(container, shell)| (container.clone(), shell.opened))
828 .collect();
829 for (container, opened) in shells {
830 if opened.elapsed() > MAX_SHELL {
831 info!("closing {container}: a shell is open for 8 hours at most");
832 self.close_shell(&container).await;
833 continue;
834 }
835 let ended = match self.docker.inspect(&container).await {
836 Ok(state) => !state.running,
837 Err(error) => is_not_found(&error),
838 };
839 if ended {
840 self.close_shell(&container).await;
841 }
842 }
843 }
844
845 pub(super) async fn close_shells(&mut self) {
847 let containers: Vec<String> = self.records.shells.keys().cloned().collect();
848 for container in containers {
849 self.close_shell(&container).await;
850 }
851 }
852}
853
854fn labels(record: &api::Record) -> BTreeMap<String, String> {
856 let mut labels = BTreeMap::from([
857 (record::LABEL.to_owned(), record.job.to_string()),
858 ("runnerd.repository".to_owned(), record.repository.clone()),
859 ("runnerd.profile".to_owned(), record.profile.clone()),
860 (
861 "runnerd.created-at".to_owned(),
862 record.created_at.to_string(),
863 ),
864 ]);
865 if let Some(workflow) = &record.workflow {
866 labels.insert("runnerd.workflow".to_owned(), workflow.clone());
867 }
868 if let Some(name) = &record.name {
869 labels.insert("runnerd.name".to_owned(), name.clone());
870 }
871 labels
872}
873
874async fn check(docker: &impl DockerApi, container: &str) -> Result<Checked, String> {
878 let changed = docker
879 .changes(container)
880 .await
881 .map_err(|error| format!("Docker did not list its files: {error}"))?;
882 let secrets = record::secrets_left(changed.iter().map(String::as_str));
883 if !secrets.is_empty() {
884 return Err(format!(
885 "its runner left files a record must not keep ({}): its image did not delete them; use the image of \
886 runnerd {}, or rebuild image_stage from its image/",
887 record::listed(&secrets),
888 env!("CARGO_PKG_VERSION")
889 ));
890 }
891 let incoming = docker
892 .written(container)
893 .await
894 .map_err(|error| format!("Docker did not measure it: {error}"))?
895 .saturating_mul(COMMIT_FACTOR);
896 let free = match docker.free_space().await {
897 Ok(free) => Some(free),
898 Err(error) => {
899 warn!("cannot measure Docker's free disk: {error}");
900 None
901 }
902 };
903 Ok(Checked { incoming, free })
904}