1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
//! Cache upkeep: measures what slot 1 changed, reseeds idle profiles, stops the mirror, empties the shared caches and the mirror's storage over
//! their limit and trims the build cache to its own when no runner runs, and counts what the mirror served each
//! runner. Measures, reseeds and cache checks run as background tasks the tick collects; the build cache's trim runs before the
//! tick goes on, as no runner may start during it. After a host crash that may have torn what runners wrote without
//! syncing, it empties the slots' volumes and the shared pnpm and Maven stores before any runner starts (ADR 0141).
//! It must not start or stop runners.
use std::collections::{BTreeMap, BTreeSet};
use std::time::{Duration, SystemTime};
use tokio::task::JoinHandle;
use tracing::{info, warn};
use super::unsynced::{AfterRefusal, REFUSALS};
use super::upkeep::Due;
use super::{Engine, Meanwhile, Occupant, Shutdown, Slot};
use crate::admission::host;
use crate::api;
use crate::buildcache::BuildCache;
use crate::cache::{self, Emptied, Measured, Refused, Trimmed};
use crate::config::ByteSize;
use crate::docker::{DockerApi, is_not_found};
use crate::github::GithubApi;
use crate::time;
/// How long after the shared caches' last check the next stops warm runners to run (ADR 0050). Without warm runners
/// the check runs whenever no runner runs; warm runners can keep one running all day.
const CHECK_EVERY: Duration = Duration::from_secs(3600);
/// How long after a profile's last measure of slot 1 the next may start (ADR 0094). A profile whose slot 1 runs a
/// job a minute would otherwise run a measure container as often, most of them to find what the last one found.
pub(super) const MEASURE_EVERY: Duration = Duration::from_secs(600);
impl<G: GithubApi + 'static, D: DockerApi + 'static> Engine<G, D> {
/// Counts what the mirror served each runner since the last count.
pub(super) async fn count_mirror_bytes(&mut self) {
if !self.config.caches.mirror.enabled || !self.mirror.may_run() {
return;
}
let log = match self
.docker
.logs_since(&self.names.mirror(), self.mirror_until.at / 1_000_000_000)
.await
{
Ok(log) => log,
Err(error) if is_not_found(&error) => return,
Err(error) => {
warn!("cannot read the mirror's log: {error}");
return;
}
};
let (bytes, until) = self.leases.attribute(&log, self.mirror_until);
// Past these lines only once they are stored: if not, the next count reads them again.
if until != self.mirror_until && self.store.mirror_bytes(bytes, until).await {
self.mirror_until = until;
self.leases.forget_before(until.at / 1_000_000);
}
}
/// Measures slot 1 of the profiles whose slot 1 ran a job, and reseeds the idle profiles due to reseed, once Docker
/// has their image; with no runner running at all, stops the mirror and, unless shutting down, empties the shared
/// caches over their limit. None of it while caches a crash may have torn wait to be emptied.
pub(super) async fn idle_caches(&mut self, mode: Shutdown, tick_began: SystemTime) {
// Caches a crash may have torn are emptied first: a reseed would copy slot 1's layers into the snapshot, and
// a measure or a trim would mount the volumes being removed.
if self.unsynced.is_torn() {
return;
}
if mode == Shutdown::No {
self.start_measures(tick_began);
let due: Vec<String> = self
.upkeep
.due_to_reseed()
.filter(|name| {
self.slots
.get(*name)
.is_some_and(|slots| slots.iter().all(Slot::is_free))
})
// A shell mounts the snapshot a reseed replaces.
.filter(|name| !self.shell_profiles().contains(name.as_str()))
// The seed container runs the profile's image: a reseed waits for its pull, still due.
.filter(|name| {
!self
.config
.profiles
.get(*name)
.is_some_and(|profile| self.awaits_image(&profile.image))
})
.cloned()
.collect();
for name in due {
// The seed container runs at its profile's cap: a reseed waits for room in the slice and on the
// host, still due, as a runner does.
if !self.fits_seed(&name) {
continue;
}
let forced = self.upkeep.take_due(&name).is_some_and(Due::forces);
self.spawn_reseed(&name, forced);
}
}
// A shell mounts the shared caches.
if self.occupied().next().is_some() || self.records.has_shells() {
return;
}
if self.mirror.may_run() {
match cache::mirror_down(&*self.docker, &self.names).await {
Ok(()) => self.mirror.stopped(),
Err(error) => warn!("cannot stop the mirror: {error}"),
}
}
// On a shutdown, left to the next start, which checks them first.
if mode == Shutdown::No && self.trim_due && self.trim.is_none() {
self.trim_due = false;
self.checked_at = SystemTime::now();
let docker = self.docker.clone();
let names = self.names.clone();
let caches = self.config.caches.clone();
let mirror = self.mirror.storage();
// A reseed mounts its profile's slot volumes.
let slots: BTreeMap<String, usize> = self
.config
.profiles
.iter()
.filter(|(name, _)| !self.upkeep.reseed_runs(name))
.map(|(name, profile)| (name.clone(), profile.slots as usize))
.collect();
// Here, not in the background task: runners start only in the tick, so none starts while it removes
// entries. Commands are answered meanwhile; none starts a runner.
let build = match self.build_cache.clone() {
Some(build) => {
self.serving(trim_build_cache(build), Meanwhile::Commands)
.await
}
None => None,
};
self.trim = Some(tokio::spawn(async move {
let mut trimmed =
cache::trim_shared(&*docker, &names, &caches, mirror, &slots).await?;
if let Some(bytes) = build {
trimmed.sizes.insert(names.build_cache(), bytes);
}
Ok(trimmed)
}));
}
}
/// Measures slot 1 of each profile whose slot 1 ran a job since its last measure (`cache::measure`), once
/// `MEASURE_EVERY` passed since that one, when slot 1 runs no job and Docker has the profile's image. It holds
/// no slot and stops no runner: its container reads slot 1 and the snapshot read-only, and a job slot 1 takes
/// meanwhile only skews the number, which the reseed weighs again. Not for a profile already due to reseed, or
/// reseeding, nor in the tick that started slot 1's runner (`tick_began`), whose container is still mounting
/// the layers and may drop some.
fn start_measures(&mut self, tick_began: SystemTime) {
let now = tokio::time::Instant::now();
let due: Vec<String> = self
.upkeep
.due_to_measure(now)
// Mid-job, it would measure what the job is still changing.
.filter(|name| {
self.slots
.get(*name)
.and_then(|slots| slots.first())
.is_some_and(|slot| {
slot.occupant().is_none_or(|occupant| {
occupant.idle() && occupant.started_at < tick_began
})
})
})
.filter(|name| {
self.config
.profiles
.get(*name)
.is_some_and(|profile| !self.awaits_image(&profile.image))
})
.cloned()
.collect();
for name in due {
let Some(profile) = self.config.profiles.get(&name) else {
continue;
};
let docker = self.docker.clone();
let names = self.names.clone();
let image = profile.image.reference();
let slice = self.config.host.slice.clone();
let isolation = self.isolation;
let profile_name = name.clone();
let task = tokio::spawn(async move {
cache::measure(
&*docker,
&cache::Measure {
names: &names,
profile: &profile_name,
image: &image,
slice: &slice,
isolation,
},
)
.await
});
self.upkeep.measure_started(&name, task, now);
}
}
/// Takes in the measures of slot 1 that ended: a profile whose slot 1 changed `reseed_at` or more, whose last
/// reseed did not finish, or whose measure failed, is due to reseed, which its upkeep then stops its runners for.
/// One that changed less keeps them: a reseed would change nothing.
pub(super) async fn collect_measures(&mut self) {
let reseed_at = self.config.caches.reseed_at;
for (name, task) in self.upkeep.finished_measures() {
let measured = match task.await {
Ok(measured) => measured.map_err(|error| error.to_string()),
Err(error) => Err(error.to_string()),
};
// A reload removed the profile meanwhile.
if !self.config.profiles.contains_key(&name) {
continue;
}
match measured {
Ok(Measured::Changed(bytes)) if bytes < reseed_at.0 => {
info!(
"{name}: slot 1 changed {}, under reseed_at ({reseed_at}): no reseed",
ByteSize(bytes)
);
continue;
}
Ok(Measured::Changed(bytes)) => info!(
"{name}: slot 1 changed {}, reseed_at ({reseed_at}) or more: reseeding once its runners are idle",
ByteSize(bytes)
),
Ok(Measured::Unfinished) => info!(
"{name}: its last reseed did not finish: reseeding once its runners are idle"
),
Err(error) => warn!(
"{name}: cannot measure what slot 1 changed, so its runners stop for a reseed that does: {error}"
),
}
self.upkeep.mark(&name, Due::Changed);
}
}
/// Whether `profile`'s seed container, at its profile's cap, fits beside what else runs: in the slice's
/// `MemoryMax` and in what the host leaves the slice (ADR 0055), and not while measured admission holds the
/// slice at its high-water mark. The profile's own runners, which stop before it starts, and the room its
/// waiting seed holds count as free. The host's limit is `waiting_seed`'s, without the larger margin of a short
/// host: the room it holds keeps other jobs waiting for the host and so the host short.
pub(super) fn fits_seed(&self, profile: &str) -> bool {
let memory = self.config.profiles.get(profile).map_or(0, |p| p.memory.0);
let runners: u64 = self
.slots
.get(profile)
.into_iter()
.flatten()
.filter_map(Slot::occupant)
.map(|occupant| self.takes(occupant))
.sum();
let own = runners.saturating_add(self.waiting_seed(profile).unwrap_or(0));
// As a plan reads them: `MemoryMax` the pooled count and the host what it counts (ADR 0099, 0108), or plans
// would keep spending room a reseed waits for.
let count = self.count(self.host_remembered());
let in_use = count.pooled().saturating_sub(own);
self.memory_max
.is_none_or(|max| max.saturating_sub(in_use) >= memory)
&& self.host_budget().is_none_or(|host| {
let in_use = count.hosted().saturating_sub(own);
host::left(host::limit(&host.memory, false), in_use, host.stalled)
>= i64::try_from(memory).unwrap_or(i64::MAX)
})
&& !self.admitting(&mut [], None).held
}
/// The room a requested reseed holds while it waits for it once its profile drained: its profile's cap, so
/// later starts leave that room and the seed starts as runners finish, rather than hold the profile's jobs for
/// as long as other profiles' take what frees. `None` until then, while a shell holds it back, once it runs, and
/// while the cap would not fit even with no runner running: the room would then hold every start for a seed that
/// cannot start, which instead waits as a due one does.
fn waiting_seed(&self, profile: &str) -> Option<u64> {
let drained = self
.slots
.get(profile)
.is_some_and(|slots| slots.iter().all(Slot::is_free));
let waits = self.upkeep.requested_reseed_waits(profile)
&& drained
&& !self.shell_profiles().contains(profile);
let memory = self.config.profiles.get(profile)?.memory.0;
// Not `memory_in_use`, which counts this room. The host's limit as the first runner into an empty slice
// sees it, without the larger margin of a short or stalling host, which passes.
let others = self.upkeep.seeds_memory() + self.shell_memory() + self.mirror_memory_in_use();
let fits = self
.memory_max
.is_none_or(|max| max.saturating_sub(others) >= memory)
&& self
.host_memory()
.is_none_or(|host| host::limit(&host, false).saturating_sub(others) >= memory);
(waits && fits).then_some(memory)
}
/// The memory each seed container holds of the budget, by profile: a running one's cap as it started, and the
/// room a requested reseed holds while it waits (`waiting_seed`).
pub(super) fn seed_memory(&self) -> BTreeMap<String, u64> {
let mut seeds = self.upkeep.seed_memories();
for profile in self.upkeep.requested() {
if let Some(memory) = self.waiting_seed(profile) {
seeds.entry(profile.clone()).or_insert(memory);
}
}
seeds
}
/// Rebuilds a profile's snapshot in the background; its slots start nothing until it is done. A `requested`
/// reseed, or that of a snapshot over its limit, runs however little slot 1 changed.
fn spawn_reseed(&mut self, name: &str, requested: bool) {
let Some(profile) = self.config.profiles.get(name) else {
return;
};
let docker = self.docker.clone();
let names = self.names.clone();
let image = profile.image.reference();
let slice = self.config.host.slice.clone();
let (slots, cpus, memory) = (profile.slots as usize, profile.cpus, profile.memory);
let min_changes = if requested {
ByteSize(0)
} else {
self.config.caches.reseed_at
};
let max_size = self.config.caches.snapshot_max;
let isolation = self.isolation;
let profile_name = name.to_owned();
let task = tokio::spawn(async move {
cache::reseed(
&*docker,
&cache::Reseed {
names: &names,
profile: &profile_name,
image: &image,
slots,
slice: &slice,
cpus,
memory,
min_changes,
max_size,
isolation,
},
)
.await
});
self.upkeep.reseed_started(name, task, memory.0);
}
pub(super) async fn collect_trim(&mut self) {
if !self.trim.as_ref().is_some_and(JoinHandle::is_finished) {
return;
}
let Some(task) = self.trim.take() else {
return;
};
match task.await {
Ok(Ok(Trimmed {
sizes,
emptied,
closed,
refused,
free,
})) => {
self.mark_oversized(&sizes);
if let Some(free) = free {
self.docker_disk.send_replace(Some(cache::docker_disk(
&self.names,
&self.config,
free,
&sizes,
)));
}
let ours = self.names.network();
self.store.cache_sizes(
time::now_ms(),
sizes
.into_iter()
.filter(|(name, _)| name.starts_with(&format!("{ours}-")))
.collect(),
);
for Emptied { name, size } in emptied {
info!("emptied {name}, which had grown to {size}");
}
for Emptied { name, size } in closed {
info!("removed {name} ({size}): its profile has fewer slots now");
}
for Refused { name, error } in refused {
warn!("cannot remove {name}, tried again at the next check: {error}");
}
}
Ok(Err(error)) => warn!("cannot check the shared caches: {error}"),
Err(error) => warn!("checking the shared caches stopped: {error}"),
}
}
/// Starts measuring what the slots' Docker Engines may keep (`cache::docker_disk`), for the runners that start
/// before the first check of the caches, which waits for no runner to run: after a restart that adopted some,
/// that can be long. Docker measures every volume, which takes minutes on a host whose slots hold Docker stores,
/// so it runs in the background; until it ends, each image keeps its own default.
pub(super) fn measure_docker_disk(&mut self) {
let docker = self.docker.clone();
self.disk = Some(tokio::spawn(async move {
Ok(Disk {
free: docker.free_space().await?,
sizes: docker.volume_sizes().await?,
})
}));
}
/// Takes in the measure `measure_docker_disk` started, once it ended. A check of the caches since measured
/// later, so it wins.
pub(super) async fn collect_disk(&mut self) {
if !self.disk.as_ref().is_some_and(JoinHandle::is_finished) {
return;
}
let Some(task) = self.disk.take() else {
return;
};
match task.await {
Ok(Ok(Disk { free, sizes })) => {
if self.docker_disk.borrow().is_none() {
self.docker_disk.send_replace(Some(cache::docker_disk(
&self.names,
&self.config,
free,
&sizes,
)));
}
}
Ok(Err(error)) => warn!(
"cannot measure Docker's disk, so runners keep their image's Docker budget: {error}"
),
Err(error) => warn!("measuring Docker's disk stopped: {error}"),
}
}
/// Makes the profiles whose snapshot `sizes` measured over `snapshot_max` due for a reseed, which then cuts the
/// snapshot back, by what the image's seed script drops (docs/adr/0041, docs/adr/0152): slot 1 may never change
/// enough to reach `reseed_at`. A reseed running now, or one that finished after the check started, may have
/// replaced the snapshot after it was measured.
fn mark_oversized(&mut self, sizes: &BTreeMap<String, u64>) {
let max = self.config.caches.snapshot_max;
for profile in self.config.profiles.keys() {
let Some(&size) = sizes.get(&self.names.base(profile)) else {
continue;
};
if size > max.0
&& self
.upkeep
.snapshot_unchanged_since(profile, self.checked_at)
{
info!(
"{profile}: the snapshot holds {}, over snapshot_max ({max}): it is cut back at the next reseed, once \
the profile is idle",
ByteSize(size)
);
self.upkeep.mark(profile, Due::Oversized);
}
}
}
pub(super) async fn collect_reseeds(&mut self) {
for (name, task) in self.upkeep.finished_reseeds() {
let (output, error) = match task.await {
Ok(Ok(output)) => {
info!("{name}: {}", output.trim().replace('\n', "; "));
(output, None)
}
Ok(Err(error)) => {
warn!("{name}: reseeding failed: {error}");
(String::new(), Some(error.to_string()))
}
Err(error) => {
warn!("{name}: reseeding stopped: {error}");
(String::new(), Some(error.to_string()))
}
};
// A reseed that made a snapshot dropped every slot's layer, so none holds a job's caches; one that failed
// may have dropped some.
if error.is_some() || cache::seeded(&output) {
self.ran.retain(|slot, _| slot.profile != name);
}
self.store.event(api::Event::CacheReseeded {
profile: name,
output: output.trim().to_owned(),
error,
});
}
self.drop_closed_slots();
}
/// Whether upkeep of `profile` waits only for its runners, all idle: its reseed, or the shared caches' check
/// (`CHECK_EVERY` after the last, or running), which waits for every profile's runners. Its warm runners then
/// stop and none starts.
pub(super) fn upkeep_waits_for_warm(&self, profile: &str) -> bool {
let idle = |slots: &Vec<Slot>| slots.iter().filter_map(Slot::occupant).all(Occupant::idle);
// A shell holds back the reseed until it closes, so runners keep serving meanwhile. So does a due reseed
// with no room for its seed even once they stop: it is not asked for, and may wait long.
let reseed = self
.upkeep
.due(profile)
.is_some_and(|due| due == Due::Requested || self.fits_seed(profile))
&& !self.shell_profiles().contains(profile)
&& self.slots.get(profile).is_some_and(idle);
// Until the check ends: a runner started meanwhile mounts the volumes it may remove.
let check = (self.trim.is_some()
|| self.trim_due
&& self
.checked_at
.elapsed()
.is_ok_and(|age| age >= CHECK_EVERY))
&& !self.records.has_shells()
&& self.slots.values().all(idle);
reseed || check
}
/// Empties what runners write and later jobs read, once a host crash may have torn it (ADR 0141): every slot's
/// volume Docker has under runnerd's prefix, the config's and those of a lowered slot count or a removed profile
/// alike (ADR 0142), layers and Docker store, and the pnpm store and Maven repository every slot shares, each after
/// the stopped containers of runnerd's that mount it (`cache::empty_volumes`). Not the snapshots, which only a
/// reseed writes, with its syncs, nor the mirror or the build cache, which keep theirs, nor a slot's volume outside
/// the config that a running container mounts, which another daemon's runner or the owner's started in this boot.
/// It waits for the records being committed, whose containers mount the slots' volumes and go once committed. A
/// volume Docker does not remove keeps them owed, and no runner starts, until a later tick removes it, or
/// `unsynced::REFUSALS` ticks found a container it leaves on one: then runners start on what the caches hold, once
/// the layers of each slot's volume left are started over (`cache::start_layers_over`), which a slot's next runner
/// would otherwise mount, and a reseed copy, as they are. Docker's own errors, an outage, a 5xx or a timeout, count
/// nothing and are retried until it removes them.
pub(super) async fn empty_torn_caches(&mut self) {
if self.records.is_recording() {
return;
}
let listed = match self.docker.volumes(&self.names.volume_prefix()).await {
Ok(listed) => listed,
Err(error) => {
warn!(
"cannot list the volumes a host crash may have torn, so no runner starts until Docker does: \
{error}"
);
return;
}
};
let configured: BTreeSet<String> = cache::volumes(&self.names, &self.config)
.into_iter()
.filter(|volume| {
matches!(
volume.kind,
api::CacheKind::Slot { .. } | api::CacheKind::Pnpm | api::CacheKind::Maven
)
})
.map(|volume| volume.name)
.collect();
let volumes: Vec<String> = configured
.iter()
.cloned()
.chain(
listed
.into_iter()
.filter(|volume| self.names.is_slot(volume)),
)
.collect::<BTreeSet<_>>()
.into_iter()
.collect();
let cache::Cleared { removed, refused } =
cache::empty_volumes(&*self.docker, &volumes).await;
// Each slot's layers no longer hold what a job cached.
self.ran.clear();
self.trim_due = true;
// A slot's volume the config does not have, which a running container mounts, is another daemon's whose
// prefix starts with this one's, or the owner's: started in this boot, it is that runner's to answer for.
let (others, refused): (Vec<_>, Vec<_>) = refused.into_iter().partition(|refusal| {
refusal.cause == cache::Kept::Runs && !configured.contains(&refusal.volume)
});
if !others.is_empty() {
info!(
"left what a host crash may have torn in {}, which this config does not have and a running container \
mounts",
others
.iter()
.map(|refusal| format!("{} ({})", refusal.volume, refusal.why))
.collect::<Vec<_>>()
.join(", ")
);
}
if !refused.is_empty() {
let why = refused
.iter()
.map(|refusal| format!("{}: {}", refusal.volume, refusal.why))
.collect::<Vec<_>>()
.join("; ");
// Only a container left on a volume is the owner's doing; Docker's own errors are retried for good.
if !refused.iter().any(cache::Refusal::held) {
warn!(
"cannot empty what a host crash may have torn, so no runner starts until Docker removes it: {why}"
);
return;
}
let held = refused
.iter()
.filter(|refusal| refusal.held())
.map(|refusal| refusal.volume.clone())
.collect();
match self.unsynced.refused(held) {
AfterRefusal::Again { tries } => warn!(
"cannot empty what a host crash may have torn, so no runner starts until it is (try {tries} of \
{REFUSALS}): {why}"
),
AfterRefusal::GiveUp => self.give_up(&refused, &why).await,
}
return;
}
if let Err(error) = self.unsynced.emptied() {
warn!("emptied the caches a host crash may have torn, but cannot note it: {error}");
return;
}
info!(
"the host booted again with what runners wrote unsynced; emptied {} before any runner starts",
if removed.is_empty() {
"nothing, as no cache volume was left".to_owned()
} else {
removed.join(", ")
}
);
}
/// Gives up on the torn volumes Docker would not remove, as `refused` says and `why` words it, once the layers of
/// each slot's volume among them are started over: until then the caches stay torn.
async fn give_up(&mut self, refused: &[cache::Refusal], why: &str) {
let slots: Vec<&str> = refused
.iter()
.map(|refusal| refusal.volume.as_str())
.filter(|volume| self.names.is_slot(volume))
.collect();
if let Err(error) = self.start_layers_over(&slots).await {
warn!(
"gave up emptying what a host crash may have torn, but cannot start the layers of {} over, so no \
runner starts: {error}; {why}",
slots.join(", ")
);
return;
}
match self.unsynced.gave_up() {
Ok(()) => warn!(
"gave up emptying what a host crash may have torn after {REFUSALS} tries: runners start on what {} \
hold{}; remove them once nothing mounts them, with docker volume rm: {why}",
refused
.iter()
.map(|refusal| refusal.volume.as_str())
.collect::<Vec<_>>()
.join(", "),
if slots.is_empty() {
String::new()
} else {
format!(", the layers of {} started over", slots.join(", "))
}
),
Err(error) => warn!(
"gave up emptying what a host crash may have torn, but cannot note it, so no runner starts: {error}; \
{why}"
),
}
}
/// Starts the layers in each of the slots' `volumes` over (`cache::start_layers_over`), in a container of the
/// image of the volume's profile, or any configured one's for a profile the config no longer has.
async fn start_layers_over(&self, volumes: &[&str]) -> Result<(), String> {
for volume in volumes {
let profile = self
.config
.profiles
.iter()
.find(|(name, _)| self.names.slot_of(name, volume).is_some())
.or_else(|| self.config.profiles.iter().next())
.map(|(_, profile)| profile)
.ok_or_else(|| "no profile's runner image to run it with".to_owned())?;
cache::start_layers_over(
&*self.docker,
&cache::StartOver {
names: &self.names,
volume,
image: &profile.image.reference(),
slice: &self.config.host.slice,
isolation: self.isolation,
},
)
.await
.map_err(|error| format!("{volume}: {error}"))?;
}
Ok(())
}
/// Whether a profile's snapshot is being rebuilt, or will be once its slots are free.
pub(super) fn is_reseeding(&self, profile: &str) -> bool {
self.upkeep.reseed_runs_or_is_requested(profile)
}
}
/// Trims the build cache to its limit off the engine's loop; returns its bytes after, or `None` if the trim stopped.
async fn trim_build_cache(build: BuildCache) -> Option<u64> {
match tokio::task::spawn_blocking(move || build.trim()).await {
Ok(build) => {
if build.after != build.before {
info!(
"trimmed the build cache from {} to {}, its least recently used entries first",
ByteSize(build.before.bytes),
ByteSize(build.after.bytes)
);
}
Some(build.after.bytes)
}
Err(error) => {
warn!("trimming the build cache stopped: {error}");
None
}
}
}
/// Docker's free disk and the volumes' sizes, as `measure_docker_disk` read them.
#[derive(Debug)]
pub(super) struct Disk {
free: u64,
sizes: BTreeMap<String, u64>,
}