1use std::collections::{BTreeMap, BTreeSet, HashMap};
7use std::time::Duration;
8
9use bollard::models::{ContainerCreateBody, HostConfig, Mount, MountType};
10use tokio::sync::{mpsc, watch};
11use tokio::task::JoinHandle;
12use tokio::time::Instant;
13use tracing::{info, warn};
14
15use super::control::ControlError;
16use super::{Engine, Occupant};
17use crate::admission::HIGH_WATER;
18use crate::api;
19use crate::config::ByteSize;
20use crate::docker::{self, DockerApi, Tty, TtySize, cpu_shares, is_not_found};
21use crate::github::GithubApi;
22use crate::observe::capture;
23use crate::record;
24use crate::scheduler::SlotRef;
25use crate::time;
26
27#[derive(Debug)]
30pub(super) struct Recording {
31 record: api::Record,
32 container: String,
33 estimate: u64,
36 stage: Stage,
37}
38
39#[derive(Debug)]
40enum Stage {
41 Checking(JoinHandle<Result<Checked, String>>),
43 Committing(JoinHandle<Committed>),
45}
46
47#[derive(Debug)]
49struct Committed {
50 bytes: Result<u64, String>,
52 left: bool,
54}
55
56impl Stage {
57 fn is_finished(&self) -> bool {
58 match self {
59 Stage::Checking(task) => task.is_finished(),
60 Stage::Committing(task) => task.is_finished(),
61 }
62 }
63}
64
65#[derive(Debug)]
68struct Checked {
69 incoming: u64,
70 free: Option<u64>,
71}
72
73#[derive(Debug)]
75pub(super) enum Undecided {
76 Docker(docker::Error),
78 Unread,
80}
81
82impl std::fmt::Display for Undecided {
83 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
84 match self {
85 Undecided::Docker(error) => error.fmt(f),
86 Undecided::Unread => f.write_str("its diagnostic files were not all read"),
87 }
88 }
89}
90
91#[derive(Debug)]
97pub(super) struct RecordState {
98 list: Option<Vec<api::Record>>,
100 recordings: BTreeMap<u64, Recording>,
102 shells: BTreeMap<String, Shell>,
104 opened: u64,
106 status: watch::Sender<api::Records>,
108 done: mpsc::UnboundedSender<()>,
110 pub(super) finished: mpsc::UnboundedReceiver<()>,
111}
112
113impl RecordState {
114 pub(super) fn new() -> (RecordState, watch::Receiver<api::Records>) {
116 let (status, receiver) = watch::channel(api::Records::default());
117 let (done, finished) = mpsc::unbounded_channel();
118 let state = RecordState {
119 list: None,
120 recordings: BTreeMap::new(),
121 shells: BTreeMap::new(),
122 opened: 0,
123 status,
124 done,
125 finished,
126 };
127 (state, receiver)
128 }
129
130 fn is_listed(&self) -> bool {
132 self.list.is_some()
133 }
134
135 fn kept(&self) -> &[api::Record] {
137 self.list.as_deref().unwrap_or_default()
138 }
139
140 fn of_job(&self, job: u64) -> Option<&api::Record> {
142 self.kept().iter().find(|record| record.job == job)
143 }
144
145 fn set_listed(&mut self, records: Vec<api::Record>) {
147 self.list = Some(records);
148 }
149
150 fn keep(&mut self, record: api::Record) {
152 self.list.get_or_insert_default().push(record);
153 }
154
155 fn forget(&mut self, job: u64) -> u64 {
157 let Some(list) = self.list.as_mut() else {
158 return 0;
159 };
160 let Some(index) = list.iter().position(|record| record.job == job) else {
161 return 0;
162 };
163 list.remove(index).bytes
164 }
165
166 pub(super) fn is_recording(&self) -> bool {
168 !self.recordings.is_empty()
169 }
170
171 fn pending(&self) -> u64 {
173 self.recordings.values().map(|other| other.estimate).sum()
174 }
175
176 fn begin(&mut self, job: u64, recording: Recording) {
178 self.recordings.insert(job, recording);
179 }
180
181 fn finished_recordings(&self) -> Vec<u64> {
183 self.recordings
184 .iter()
185 .filter(|(_, recording)| recording.stage.is_finished())
186 .map(|(job, _)| *job)
187 .collect()
188 }
189
190 fn take_recording(&mut self, job: u64) -> Option<Recording> {
192 self.recordings.remove(&job)
193 }
194
195 pub(super) fn has_shells(&self) -> bool {
197 !self.shells.is_empty()
198 }
199
200 fn next_shell(&mut self) -> u64 {
202 self.opened += 1;
203 self.opened
204 }
205
206 fn shell_opened(&mut self, container: String, shell: Shell) {
208 self.shells.insert(container, shell);
209 }
210
211 fn shell_closed(&mut self, container: &str) {
213 self.shells.remove(container);
214 }
215}
216
217#[derive(Debug, Clone)]
219pub(super) struct Shell {
220 job: u64,
221 pub(super) profile: String,
223 pub(super) memory: u64,
225 opened: Instant,
226}
227
228const MAX_SHELL: Duration = Duration::from_hours(8);
230
231#[derive(Debug, Clone, PartialEq, Eq)]
233pub enum RecordSelection {
234 Job(u64),
236 All,
238 OlderThan(Duration),
240}
241
242const SHELL_SCRIPT: &str = r#"
245. /usr/local/lib/runnerd-layers.sh && mount_layers > /dev/null 2>&1
246install -d -o runner -g runner -m 700 "$XDG_RUNTIME_DIR" 2> /dev/null
247cd "$1" 2> /dev/null || cd /home/runner
248exec setpriv --reuid=runner --regid=runner --init-groups -- env HOME=/home/runner USER=runner LOGNAME=runner bash -l
249"#;
250
251const COMMIT_FACTOR: u64 = 2;
254
255impl<G: GithubApi + 'static, D: DockerApi + 'static> Engine<G, D> {
256 fn records_failures(&self, profile: &str) -> bool {
258 self.config
259 .profiles
260 .get(profile)
261 .or_else(|| self.retired.get(profile))
262 .is_some_and(|profile| profile.record_failures)
263 }
264
265 pub(super) async fn begin_record(&mut self, slot: &SlotRef, occupant: &Occupant) -> bool {
269 let Some(job) = self.named_job(occupant) else {
270 return false;
271 };
272 if !self.records_failures(&slot.profile) {
273 return false;
274 }
275 if !self.records.is_listed() {
276 self.list_records().await;
277 if !self.records.is_listed() {
278 let why = "Docker did not list the records kept".to_owned();
279 self.skip_record(job, &occupant.repository, why);
280 return false;
281 }
282 }
283 let summary = self
284 .store
285 .job(job)
286 .await
287 .ok()
288 .flatten()
289 .map(|job| job.summary);
290 let record = api::Record {
291 job,
292 repository: occupant.repository.clone(),
293 profile: slot.profile.clone(),
294 workflow: summary
295 .as_ref()
296 .and_then(|summary| summary.workflow.clone()),
297 name: summary.map(|summary| summary.name),
298 created_at: time::now_ms() / 1000,
299 bytes: 0,
300 image: record::image(job),
301 };
302 info!(
303 "{}: keeping job {job}'s runner as {}",
304 occupant.repository, record.image
305 );
306 let docker = self.docker.clone();
307 let container = occupant.container.clone();
308 let done = self.records.done.clone();
309 let check = tokio::spawn(async move {
310 let checked = check(&*docker, &container).await;
311 let _ = done.send(());
312 checked
313 });
314 self.records.begin(
315 job,
316 Recording {
317 record,
318 container: occupant.container.clone(),
319 estimate: 0,
320 stage: Stage::Checking(check),
321 },
322 );
323 true
324 }
325
326 pub(super) async fn keep_left_record(
333 &mut self,
334 slot: &SlotRef,
335 occupant: &Occupant,
336 ) -> Result<bool, Undecided> {
337 if !self.records_failures(&slot.profile)
338 || occupant
339 .named_job()
340 .is_some_and(|job| self.records.of_job(job).is_some())
341 {
342 return Ok(false);
343 }
344 let failed = match self.docker.logs(&occupant.container).await {
345 Ok(log) => record::failed(&log),
346 Err(error) if docker::is_declined(&error) => false,
347 Err(error) => return Err(Undecided::Docker(error)),
348 };
349 if !failed {
350 return Ok(false);
351 }
352 let job = match occupant.named_job() {
353 Some(job) => Some(job),
354 None => self.left_job(&occupant.container).await?,
355 };
356 let Some(job) = job else {
357 warn!(
358 "{}: {} ran a job that failed, but names it nowhere: no record of it is kept",
359 occupant.repository, occupant.runner_name
360 );
361 return Ok(false);
362 };
363 if self.records.of_job(job).is_some() {
364 return Ok(false);
365 }
366 let mut occupant = occupant.clone();
367 occupant.engagement.named_by_worker_log(job);
368 Ok(self.begin_record(slot, &occupant).await)
369 }
370
371 async fn left_job(&self, container: &str) -> Result<Option<u64>, Undecided> {
374 let read = match self
375 .docker
376 .files(container, capture::DIAG, capture::WORKER_LOG)
377 .await
378 {
379 Ok(read) => read,
380 Err(error) if docker::is_declined(&error) => return Ok(None),
381 Err(error) => return Err(Undecided::Docker(error)),
382 };
383 match read
384 .files
385 .iter()
386 .find_map(|file| capture::worker_job(&file.contents))
387 {
388 Some(job) => Ok(Some(job)),
389 None if !read.complete => Err(Undecided::Unread),
391 None => Ok(None),
392 }
393 }
394
395 fn skip_record(&self, job: u64, repository: &str, why: String) {
397 warn!("{repository}: no record of job {job}: {why}");
398 self.store.record_skipped(job, why);
399 }
400
401 async fn make_room_and_commit(
404 &mut self,
405 record: api::Record,
406 container: String,
407 checked: Checked,
408 ) {
409 let job = record.job;
410 let pending = self.records.pending();
412 let room = record::make_room(
413 self.records.kept(),
414 &self.shell_jobs(),
415 &self.config.records,
416 checked.incoming,
417 checked.free.map(|free| free.saturating_sub(pending)),
418 time::now_ms() / 1000,
419 );
420 let removed = match room {
421 Ok(removals) => {
422 let mut removed = Ok(());
423 for (old, why) in removals {
424 if let Err(error) = self.remove_record(old, why).await {
425 removed = Err(error.to_string());
426 break;
427 }
428 }
429 removed
430 }
431 Err(why) => Err(why),
432 };
433 if let Err(why) = removed {
434 self.skip_record(job, &record.repository, why);
435 self.remove_container(&container).await;
436 return;
437 }
438 let labels = labels(&record);
439 let docker = self.docker.clone();
440 let image = record.image.clone();
441 let done = self.records.done.clone();
442 let committed_container = container.clone();
443 let commit = tokio::spawn(async move {
444 let container = committed_container;
445 let bytes = docker
446 .commit(&container, &image, labels)
447 .await
448 .map_err(|error| error.to_string());
449 let left = match docker.remove(&container).await {
450 Err(error) if !is_not_found(&error) => {
451 warn!("cannot remove {container}: {error}");
452 true
453 }
454 _ => false,
455 };
456 let _ = done.send(());
457 Committed { bytes, left }
458 });
459 self.records.begin(
460 job,
461 Recording {
462 record,
463 container,
464 estimate: checked.incoming,
465 stage: Stage::Committing(commit),
466 },
467 );
468 }
469
470 async fn remove_container(&mut self, container: &str) {
472 if let Err(error) = self.docker.remove(container).await
473 && !is_not_found(&error)
474 {
475 warn!("cannot remove {container}: {error}");
476 self.leftovers
477 .removing(container.to_owned(), container.to_owned());
478 }
479 }
480
481 pub(super) async fn collect_recordings(&mut self) {
484 while self.records.finished.try_recv().is_ok() {}
485 for job in self.records.finished_recordings() {
486 let Some(recording) = self.records.take_recording(job) else {
487 continue;
488 };
489 let Recording {
490 record,
491 container,
492 stage,
493 ..
494 } = recording;
495 let task = match stage {
496 Stage::Checking(task) => {
497 let checked = task
498 .await
499 .map_err(|error| error.to_string())
500 .and_then(|checked| checked);
501 match checked {
502 Ok(checked) => self.make_room_and_commit(record, container, checked).await,
503 Err(why) => {
504 self.skip_record(job, &record.repository, why);
505 self.remove_container(&container).await;
506 }
507 }
508 continue;
509 }
510 Stage::Committing(task) => task,
511 };
512 let bytes = match task.await {
513 Ok(Committed { bytes, left }) => {
514 if left {
515 self.leftovers.removing(container.clone(), container);
516 }
517 bytes
518 }
519 Err(error) => Err(error.to_string()),
520 };
521 match bytes {
522 Ok(bytes) => self.keep_record(record, bytes).await,
523 Err(error) => self.skip_record(
524 job,
525 &record.repository,
526 format!("Docker did not commit it: {error}"),
527 ),
528 }
529 }
530 }
531
532 async fn keep_record(&mut self, mut record: api::Record, bytes: u64) {
534 let job = record.job;
535 record.bytes = bytes;
536 self.name_from_history(&mut record).await;
537 info!(
538 "{}: kept job {job}'s runner as {} ({})",
539 record.repository,
540 record.image,
541 ByteSize(bytes)
542 );
543 let removals = record::make_room(
544 self.records.kept(),
545 &self.shell_jobs(),
546 &self.config.records,
547 bytes,
548 None,
549 record.created_at,
550 );
551 self.store.record_made(record.clone());
552 self.records.keep(record);
553 let removals = removals.unwrap_or_else(|_| vec![(job, api::RecordRemoval::Size)]);
554 for (old, why) in removals {
555 if let Err(error) = self.remove_record(old, why).await {
556 warn!("{error}");
557 }
558 }
559 }
560
561 async fn name_from_history(&self, record: &mut api::Record) {
565 if record.workflow.is_some() && record.name.is_some() {
566 return;
567 }
568 self.store.flush().await;
569 if let Ok(Some(job)) = self.store.job(record.job).await {
570 record.workflow = record.workflow.take().or(job.summary.workflow);
571 record.name.get_or_insert(job.summary.name);
572 }
573 }
574
575 pub(super) async fn keep_records(&mut self) {
577 if self.records.is_listed() {
578 self.expire_records().await;
579 } else {
580 self.list_records().await;
581 }
582 }
583
584 pub(super) async fn list_records(&mut self) {
586 match self.docker.images(record::LABEL).await {
587 Ok(images) => {
588 let mut records: Vec<api::Record> = images
589 .into_iter()
590 .filter_map(|image| {
591 let label = |name: &str| image.labels.get(name).cloned();
592 Some(api::Record {
593 job: label(record::LABEL)?.parse().ok()?,
594 repository: label("runnerd.repository").unwrap_or_default(),
595 profile: label("runnerd.profile").unwrap_or_default(),
596 workflow: label("runnerd.workflow"),
597 name: label("runnerd.name"),
598 created_at: label("runnerd.created-at")
599 .and_then(|at| at.parse().ok())
600 .unwrap_or(0),
601 bytes: image.bytes,
602 image: image.reference,
603 })
604 })
605 .collect();
606 records.sort_by_key(|record| (record.created_at, record.job));
607 for record in &mut records {
608 self.name_from_history(record).await;
609 let noted = self.store.job(record.job).await.ok().flatten();
611 if noted
612 .is_some_and(|job| job.record.is_none_or(|note| note.created_at.is_none()))
613 {
614 self.store.record_made(record.clone());
615 }
616 }
617 self.store.records_listed(
619 records.iter().map(|record| record.job).collect(),
620 time::now_ms(),
621 );
622 self.records.set_listed(records);
623 }
624 Err(error) => warn!("cannot list the records: {error}"),
625 }
626 }
627
628 pub(super) async fn expire_records(&mut self) {
630 let expired = record::expired(
631 self.records.kept(),
632 &self.shell_jobs(),
633 &self.config.records,
634 time::now_ms() / 1000,
635 );
636 for (job, why) in expired {
637 if let Err(error) = self.remove_record(job, why).await {
638 warn!("{error}");
639 }
640 }
641 }
642
643 async fn remove_record(
645 &mut self,
646 job: u64,
647 why: api::RecordRemoval,
648 ) -> Result<u64, ControlError> {
649 let Some(image) = self.records.of_job(job).map(|record| record.image.clone()) else {
650 return Err(ControlError::NotFound(format!(
651 "job {job} has no record; `runnerctl records` lists them"
652 )));
653 };
654 self.docker.remove_image(&image).await.map_err(|error| {
655 ControlError::Failed(format!("cannot remove the record of job {job}: {error}"))
656 })?;
657 let bytes = self.records.forget(job);
658 info!("removed the record of job {job} ({why})");
659 self.store.record_removed(job, time::now_ms(), why);
660 Ok(bytes)
661 }
662
663 pub(super) async fn remove_records(
665 &mut self,
666 selection: &RecordSelection,
667 ) -> Result<api::RemovedRecords, ControlError> {
668 let open = self.shell_jobs();
669 let now = time::now_ms() / 1000;
670 let jobs: Vec<u64> = match selection {
671 RecordSelection::Job(job) => {
672 if open.contains(job) {
673 return Err(ControlError::Conflict(format!(
674 "a shell is open in the record of job {job}; remove it once the shell exits"
675 )));
676 }
677 vec![*job]
678 }
679 RecordSelection::All => self
680 .records
681 .kept()
682 .iter()
683 .map(|record| record.job)
684 .collect(),
685 RecordSelection::OlderThan(age) => self
686 .records
687 .kept()
688 .iter()
689 .filter(|record| now.saturating_sub(record.created_at) > age.as_secs())
690 .map(|record| record.job)
691 .collect(),
692 };
693 let mut removed = api::RemovedRecords::default();
694 for job in jobs.into_iter().filter(|job| !open.contains(job)) {
695 removed.bytes += self.remove_record(job, api::RecordRemoval::User).await?;
696 removed.jobs.push(job);
697 }
698 Ok(removed)
699 }
700
701 fn shell_jobs(&self) -> BTreeSet<u64> {
703 self.records
704 .shells
705 .values()
706 .map(|shell| shell.job)
707 .collect()
708 }
709
710 pub(super) fn shell_profiles(&self) -> BTreeSet<&str> {
712 self.records
713 .shells
714 .values()
715 .map(|shell| shell.profile.as_str())
716 .collect()
717 }
718
719 pub(super) fn shell_memory(&self) -> u64 {
721 self.records.shells.values().map(|shell| shell.memory).sum()
722 }
723
724 pub(super) fn shell_budget(&self) -> Vec<api::ShellMemory> {
726 self.records
727 .shells
728 .values()
729 .map(|shell| api::ShellMemory {
730 job: shell.job,
731 memory: shell.memory,
732 })
733 .collect()
734 }
735
736 pub(super) fn publish_records(&self) {
738 let records = self.records_status();
739 self.records.status.send_if_modified(|current| {
740 let changed = *current != records;
741 *current = records;
742 changed
743 });
744 }
745
746 fn records_status(&self) -> api::Records {
748 let limits = &self.config.records;
749 api::Records {
750 records: self.records.kept().to_vec(),
751 limits: api::RecordLimits {
752 max_total: limits.max_total.0,
753 max_count: limits.max_count,
754 max_age: limits.max_age.as_secs(),
755 min_free: limits.min_free.0,
756 },
757 shells: self.shell_jobs().into_iter().collect(),
758 }
759 }
760
761 pub(super) async fn open_shell(
764 &mut self,
765 job: u64,
766 size: TtySize,
767 ) -> Result<Tty, ControlError> {
768 let Some(record) = self.records.of_job(job).cloned() else {
769 return Err(ControlError::NotFound(format!(
770 "job {job} has no record; `runnerctl records` lists them"
771 )));
772 };
773 let profile = self.config.profiles.get(&record.profile);
774 let Some(profile) = profile.or_else(|| self.retired.get(&record.profile)) else {
775 return Err(ControlError::Invalid(format!(
776 "the record of job {job} ran in profile {}, which the config no longer has",
777 record.profile
778 )));
779 };
780 let (memory, cpus) = (profile.memory.0, profile.cpus);
781 if self.upkeep.reseed_runs(&record.profile) {
783 return Err(ControlError::Conflict(format!(
784 "{} is reseeding its snapshot, which a shell mounts; open it once the reseed is done",
785 record.profile
786 )));
787 }
788 if self.trim.as_ref().is_some_and(|trim| !trim.is_finished()) {
790 return Err(ControlError::Conflict(
791 "the shared caches, which a shell mounts, are being trimmed; open it once the trim is done"
792 .to_owned(),
793 ));
794 }
795 if self.unsynced.is_torn() {
797 return Err(ControlError::Conflict(
798 "the shared caches, which a shell mounts, are emptied after a host crash; open it once they are"
799 .to_owned(),
800 ));
801 }
802 if let Some(max) = self.memory_max {
803 let in_use = self.count(self.host_remembered()).pooled();
804 if in_use.saturating_add(memory) > max {
805 return Err(ControlError::Conflict(format!(
806 "the budget is full: {} of the slice's {} is counted, and a shell takes {} as a slot of {} \
807 does; open it once a job finishes",
808 ByteSize(in_use),
809 ByteSize(max),
810 ByteSize(memory),
811 record.profile
812 )));
813 }
814 }
815 if self.admitting(&mut [], None).held {
817 return Err(ControlError::Conflict(format!(
818 "the slice is at {HIGH_WATER}% of its MemoryMax, where measured admission starts nothing; open it \
819 once it falls under"
820 )));
821 }
822 let body = self.shell_body(&record, memory, cpus);
823 let name = format!(
824 "runnerd-shell-{job}-{}-{}",
825 time::now_ms() / 1000,
826 self.records.next_shell()
827 );
828 let tty = self
829 .docker
830 .shell(&name, body, size)
831 .await
832 .map_err(|error| ControlError::Failed(format!("cannot start a shell: {error}")))?;
833 info!("opened a shell in the record of job {job}");
834 self.records.shell_opened(
835 tty.container.clone(),
836 Shell {
837 job,
838 profile: record.profile,
839 memory,
840 opened: Instant::now(),
841 },
842 );
843 Ok(tty)
844 }
845
846 fn shell_body(&self, record: &api::Record, memory: u64, cpus: u32) -> ContainerCreateBody {
849 let workspace = record::workspace(&record.repository);
850 let limit = i64::try_from(memory).unwrap_or(i64::MAX);
851 let mut mounts = vec![Mount {
852 target: Some("/state".to_owned()),
853 typ: Some(MountType::VOLUME),
854 ..Default::default()
855 }];
856 if self.config.profiles.contains_key(&record.profile) {
857 mounts.extend(
859 crate::cache::runner_mounts(&self.names, &record.profile, 1)
860 .into_iter()
861 .filter(|mount| mount.target.as_deref() != Some("/state"))
862 .map(|mount| Mount {
863 read_only: Some(true),
864 ..mount
865 }),
866 );
867 }
868 let mut host_config = HostConfig {
869 cgroup_parent: Some(self.config.host.slice.clone()),
870 memory: Some(limit),
871 memory_swap: Some(limit),
872 nano_cpus: Some(i64::from(cpus) * 1_000_000_000),
873 cpu_shares: Some(cpu_shares(cpus)),
874 init: Some(true),
875 mounts: Some(mounts),
876 network_mode: Some(self.names.network()),
877 ..Default::default()
878 };
879 self.isolation.configure(&mut host_config);
880 ContainerCreateBody {
881 image: Some(record.image.clone()),
882 entrypoint: Some(vec![
883 "bash".to_owned(),
884 "-c".to_owned(),
885 SHELL_SCRIPT.to_owned(),
886 ]),
887 cmd: Some(vec!["shell".to_owned(), workspace]),
888 env: Some(vec!["TERM=xterm-256color".to_owned()]),
889 tty: Some(true),
890 open_stdin: Some(true),
891 stdin_once: Some(true),
892 attach_stdin: Some(true),
893 attach_stdout: Some(true),
894 attach_stderr: Some(true),
895 labels: Some(HashMap::from([(
896 record::SHELL_LABEL.to_owned(),
897 record.job.to_string(),
898 )])),
899 host_config: Some(host_config),
900 ..Default::default()
901 }
902 }
903
904 pub(super) async fn close_shell(&mut self, container: &str) {
906 let Some(shell) = self.records.shells.get(container) else {
907 return;
908 };
909 let job = shell.job;
910 match self.docker.remove(container).await {
911 Err(error) if !is_not_found(&error) => {
912 warn!("cannot remove the shell in the record of job {job}: {error}");
913 }
914 _ => {
915 info!("closed the shell in the record of job {job}");
916 self.records.shell_closed(container);
917 }
918 }
919 }
920
921 pub(super) async fn collect_shells(&mut self) {
924 let shells: Vec<(String, Instant)> = self
925 .records
926 .shells
927 .iter()
928 .map(|(container, shell)| (container.clone(), shell.opened))
929 .collect();
930 for (container, opened) in shells {
931 if opened.elapsed() > MAX_SHELL {
932 info!("closing {container}: a shell is open for 8 hours at most");
933 self.close_shell(&container).await;
934 continue;
935 }
936 let ended = match self.docker.inspect(&container).await {
937 Ok(state) => !state.running,
938 Err(error) => is_not_found(&error),
939 };
940 if ended {
941 self.close_shell(&container).await;
942 }
943 }
944 }
945
946 pub(super) async fn close_shells(&mut self) {
948 let containers: Vec<String> = self.records.shells.keys().cloned().collect();
949 for container in containers {
950 self.close_shell(&container).await;
951 }
952 }
953}
954
955fn labels(record: &api::Record) -> BTreeMap<String, String> {
957 let mut labels = BTreeMap::from([
958 (record::LABEL.to_owned(), record.job.to_string()),
959 ("runnerd.repository".to_owned(), record.repository.clone()),
960 ("runnerd.profile".to_owned(), record.profile.clone()),
961 (
962 "runnerd.created-at".to_owned(),
963 record.created_at.to_string(),
964 ),
965 ]);
966 if let Some(workflow) = &record.workflow {
967 labels.insert("runnerd.workflow".to_owned(), workflow.clone());
968 }
969 if let Some(name) = &record.name {
970 labels.insert("runnerd.name".to_owned(), name.clone());
971 }
972 labels
973}
974
975async fn check(docker: &impl DockerApi, container: &str) -> Result<Checked, String> {
979 let changed = docker
980 .changes(container)
981 .await
982 .map_err(|error| format!("Docker did not list its files: {error}"))?;
983 let secrets = record::secrets_left(changed.iter().map(String::as_str));
984 if !secrets.is_empty() {
985 return Err(format!(
986 "its runner left files a record must not keep ({}): its image did not delete them; use the image of \
987 runnerd {}, or rebuild image_stage from its image/",
988 record::listed(&secrets),
989 env!("CARGO_PKG_VERSION")
990 ));
991 }
992 let incoming = docker
993 .written(container)
994 .await
995 .map_err(|error| format!("Docker did not measure it: {error}"))?
996 .saturating_mul(COMMIT_FACTOR);
997 let free = match docker.free_space().await {
998 Ok(free) => Some(free),
999 Err(error) => {
1000 warn!("cannot measure Docker's free disk: {error}");
1001 None
1002 }
1003 };
1004 Ok(Checked { incoming, free })
1005}