Skip to main content

runnerd_core/
archives.rs

1//! The action archives runnerd keeps for its runners (ADR 0147): it reads which actions each served repository's
2//! default branch pins by commit, downloads each public one's archive from codeload without a credential, verifies it,
3//! and keeps it where every runner reads it, read-only, as `ACTIONS_RUNNER_ACTION_ARCHIVE_CACHE`, so a job copies it
4//! instead of downloading it. An archive it does not hold, a job downloads itself, as without it.
5//!
6//! One task owns the store (`run`), off the scheduling path: it refreshes at start and every `REFRESH`, reading a
7//! repository's tree and files only when its default branch moved, and publishes the view runners mount only after a
8//! check found every kept file good. It removes an archive only after a refresh that knew every pin whole. It never
9//! fetches a private action, never takes what to fetch from a job's log, and never sets
10//! `ACTIONS_RUNNER_SYMLINK_CACHED_ACTIONS`. The token stays in `github::Github` and is never sent for an archive.
11
12mod pins;
13mod state;
14mod store;
15mod verify;
16
17use std::collections::{BTreeMap, BTreeSet, HashMap};
18use std::future::Future;
19use std::path::{Path, PathBuf};
20use std::sync::atomic::{AtomicU64, Ordering};
21use std::sync::{Arc, Mutex};
22use std::time::{Duration, SystemTime};
23
24use bollard::models::{Mount, MountType};
25use futures_util::StreamExt;
26use tokio::sync::watch;
27use tracing::{info, warn};
28
29use self::pins::{MAX_DEPTH, Pin};
30use self::state::{Refresher, Wanted};
31use self::store::{Action, ArchiveStore, Key, Staged};
32use crate::api;
33use crate::config::Config;
34use crate::github::{self, Error};
35use crate::sync::lock;
36
37/// Where a runner container finds the archives.
38pub const MOUNT: &str = "/opt/runnerd/actions";
39
40/// The label of a runner container that mounts the archives: what a restart counts it by.
41pub const LABEL: &str = "runnerd.action-archives";
42
43/// The runner's setting that points it at the archives: `ACTIONS_RUNNER_ACTION_ARCHIVE_CACHE`.
44pub const ENV: &str = "ACTIONS_RUNNER_ACTION_ARCHIVE_CACHE";
45
46/// How often the store refreshes: each served repository is asked for its default branch and that branch's head this
47/// often, two requests each while it is unchanged, and its tree and files only when the head moved.
48const REFRESH: Duration = Duration::from_secs(600);
49
50/// What one refresh may spend.
51#[derive(Debug, Clone, Copy, PartialEq, Eq)]
52pub(crate) struct Limits {
53    /// The largest archive downloaded: far above any action's.
54    pub(crate) archive: u64,
55    /// The most bytes one refresh downloads, failed and refused downloads included; what is left waits for the next.
56    pub(crate) bytes: u64,
57    /// The most requests one refresh sends to GitHub's API; what is left waits for the next.
58    pub(crate) requests: u32,
59    /// The most bytes the store holds: a download that could pass it waits until retention makes room.
60    pub(crate) store: u64,
61}
62
63/// The limits runnerd refreshes by.
64const LIMITS: Limits = Limits {
65    archive: 256 << 20,
66    bytes: 2 << 30,
67    requests: 1000,
68    store: 8 << 30,
69};
70
71/// What a refresh leaves of GitHub's rate limit: below it, it asks nothing more, as runners' registrations and the
72/// scheduler's reads come first.
73const RESERVE: u64 = 1000;
74
75/// How many archives download at once.
76const DOWNLOADS: usize = 3;
77
78/// How many times an archive's download is tried in one refresh, and the wait before each retry.
79const ATTEMPTS: [Duration; 3] = [
80    Duration::ZERO,
81    Duration::from_secs(5),
82    Duration::from_secs(20),
83];
84
85/// The most files of a repository's tree read for pins, and the largest read.
86const PIN_FILES: usize = 300;
87const PIN_FILE_BYTES: u64 = 1 << 20;
88
89/// A repository as the archives need it from `GET /repos/{owner}/{repo}`.
90#[derive(Debug, Clone, PartialEq, Eq)]
91pub struct ActionRepository {
92    /// `owner/repo` as GitHub names it, case and all.
93    pub full_name: String,
94    /// Whether it is public: only a public action's archive is kept.
95    pub public: bool,
96    /// Its default branch.
97    pub default_branch: String,
98}
99
100/// A repository's tree at a commit, as `GET /repos/{owner}/{repo}/git/trees/{sha}?recursive=1` gives it.
101#[derive(Debug, Clone, PartialEq, Eq)]
102pub struct Tree {
103    /// Whether GitHub cut it short: its files are then not all there.
104    pub truncated: bool,
105    /// Its files.
106    pub files: Vec<TreeFile>,
107}
108
109/// A file of a [`Tree`].
110#[derive(Debug, Clone, PartialEq, Eq)]
111pub struct TreeFile {
112    /// Its path from the repository's root.
113    pub path: String,
114    /// Its blob's SHA.
115    pub blob: String,
116    /// Its bytes.
117    pub size: u64,
118}
119
120/// What the archives read from GitHub. `github::Github` is the real one; the token never leaves it.
121pub trait Source: Send + Sync {
122    /// A repository's canonical name, visibility and default branch.
123    fn action_repository(
124        &self,
125        repository: &str,
126    ) -> impl Future<Output = Result<ActionRepository, Error>> + Send;
127    /// The commit `branch` of `repository` is at.
128    fn branch_head(
129        &self,
130        repository: &str,
131        branch: &str,
132    ) -> impl Future<Output = Result<String, Error>> + Send;
133    /// The repository's tree at `commit`, every file of it.
134    fn tree(
135        &self,
136        repository: &str,
137        commit: &str,
138    ) -> impl Future<Output = Result<Tree, Error>> + Send;
139    /// A file's bytes by its blob's SHA, at most `limit` of them.
140    fn blob(
141        &self,
142        repository: &str,
143        blob: &str,
144        limit: u64,
145    ) -> impl Future<Output = Result<Vec<u8>, Error>> + Send;
146    /// Downloads the archive of `repository` at `commit` anonymously (codeload.github.com's `/{owner}/{repo}/tar.gz/
147    /// {sha}`) into the new file `to`, at most `limit` bytes, and returns its size. Never with a token: a repository
148    /// that is not public is not there. Adds every byte of the body it receives to `received`, the one past `limit`
149    /// and a failed download's included.
150    fn tarball(
151        &self,
152        repository: &str,
153        commit: &str,
154        to: &Path,
155        limit: u64,
156        received: &AtomicU64,
157    ) -> impl Future<Output = Result<u64, Error>> + Send;
158    /// The requests left of GitHub's rate limit before it resets, as last read; `None` when no answer said.
159    fn spare_requests(&self) -> Option<u64>;
160}
161
162/// What the store goes by, from the config: replaced whole at each reload.
163#[derive(Debug, Clone, PartialEq, Eq)]
164pub struct Settings {
165    /// `[caches.actions] enabled`.
166    pub enabled: bool,
167    /// `[caches.actions] keep`.
168    pub keep: Duration,
169    /// The served repositories, each once.
170    pub repositories: Vec<String>,
171}
172
173impl Settings {
174    /// The settings of `config`.
175    pub fn of(config: &Config) -> Settings {
176        Settings {
177            enabled: config.caches.actions.enabled,
178            keep: config.caches.actions.keep,
179            repositories: config.repository_names(),
180        }
181    }
182}
183
184/// The engine's side of the store: where it is, the settings it hands the task, what the task last published (what
185/// clients show, and what runners get), the runners a restart adopted with the archives mounted, and the
186/// repositories jobs resolved as immutable action packages. The task publishes the runners' view only after the store
187/// opened and a check found every file under `archives/` good, and withdraws it before every check and while the
188/// archives are off; a runner reads it once as it starts. Nothing survives a restart: a new task checks before it
189/// publishes, and the store keeps the packages.
190#[derive(Debug)]
191pub struct Link {
192    /// The store's directory; `None` when the daemon could not name one.
193    directory: Option<PathBuf>,
194    settings: watch::Sender<Settings>,
195    status: watch::Sender<Option<api::ActionArchives>>,
196    /// What a runner gets while the archives are checked good; `None` otherwise.
197    ready: watch::Sender<Option<RunnerArchives>>,
198    /// Runners a restart adopted whose container mounts the archives, by name, until each is gone: they read a view
199    /// no check of this daemon validated when they started. `None` until recovery is done, which the first check
200    /// waits for.
201    readers: watch::Sender<Option<BTreeSet<String>>>,
202    /// Repositories jobs resolved as immutable action packages, as their logs named them.
203    packages: Packages,
204}
205
206/// Where a job's log tells the archives task of an immutable action package (`record::immutable_packages`).
207#[derive(Debug, Clone)]
208pub struct Packages(watch::Sender<BTreeSet<String>>);
209
210impl Packages {
211    /// Notes the packages `lines`, stored lines of a job's log, show a runner downloading.
212    pub fn read(&self, lines: &[String]) {
213        let found = crate::record::immutable_packages(lines);
214        if found.is_empty() {
215            return;
216        }
217        self.0.send_if_modified(|known| {
218            let before = known.len();
219            known.extend(found.iter().map(|name| name.to_ascii_lowercase()));
220            known.len() != before
221        });
222    }
223}
224
225impl Link {
226    /// The link to a store at `directory`, going by `config`.
227    pub fn new(directory: Option<PathBuf>, config: &Config) -> Link {
228        Link {
229            directory,
230            settings: watch::Sender::new(Settings::of(config)),
231            status: watch::Sender::new(None),
232            ready: watch::Sender::new(None),
233            readers: watch::Sender::new(None),
234            packages: Packages(watch::Sender::new(BTreeSet::new())),
235        }
236    }
237
238    /// Starts the task that owns the store, unless there is no directory. Dropping what it returns stops it.
239    pub fn start<S: Source + 'static>(&self, source: Arc<S>) -> Option<Running> {
240        let directory = self.directory.clone()?;
241        let task = tokio::spawn(run(
242            source,
243            directory,
244            self.settings.subscribe(),
245            Published {
246                status: self.status.clone(),
247                ready: self.ready.clone(),
248                readers: self.readers.subscribe(),
249                packages: self.packages.0.subscribe(),
250            },
251        ));
252        Some(Running(Some(task)))
253    }
254
255    /// Hands the task `config`'s settings, after a reload. Turned off, runners get no archives from then on; turned
256    /// on again, not before the task checked them.
257    pub fn reload(&self, config: &Config) {
258        if !config.caches.actions.enabled {
259            self.ready.send_replace(None);
260        }
261        self.settings.send_if_modified(|settings| {
262            let new = Settings::of(config);
263            let changed = *settings != new;
264            *settings = new;
265            changed
266        });
267    }
268
269    /// Notes that a restart adopted `runner`, if its container's `labels` say it mounts the archives.
270    pub fn adopted<S: std::hash::BuildHasher>(
271        &self,
272        runner: &str,
273        labels: &HashMap<String, String, S>,
274    ) {
275        if !labels.contains_key(LABEL) {
276            return;
277        }
278        self.readers.send_if_modified(|readers| {
279            readers
280                .get_or_insert_with(BTreeSet::new)
281                .insert(runner.to_owned())
282        });
283    }
284
285    /// Notes that recovery adopted every runner it will: the first check, which waits for this, then knows every
286    /// runner that reads the archives unchecked.
287    pub fn recovered(&self) {
288        self.readers.send_if_modified(|readers| {
289            let first = readers.is_none();
290            readers.get_or_insert_with(BTreeSet::new);
291            first
292        });
293    }
294
295    /// Notes that `runner` is gone.
296    pub fn released(&self, runner: &str) {
297        self.readers.send_if_modified(|readers| {
298            readers
299                .as_mut()
300                .is_some_and(|readers| readers.remove(runner))
301        });
302    }
303
304    /// Where jobs' logs tell of immutable action packages.
305    pub fn packages(&self) -> Packages {
306        self.packages.clone()
307    }
308
309    /// What clients show of the store: `None` while it is off, or has not opened and no adopted runner reads it.
310    pub fn status(&self) -> Option<api::ActionArchives> {
311        if !self.settings.borrow().enabled {
312            return None;
313        }
314        let readers = self.readers.borrow().clone().unwrap_or_default();
315        match self.status.borrow().clone() {
316            Some(mut status) => {
317                // A runner that was exposed and is gone is no longer anyone's concern.
318                status.failures.retain(|failure| {
319                    failure.kind != api::ArchiveFailureKind::Exposed
320                        || readers.contains(&failure.subject)
321                });
322                Some(status)
323            }
324            // Before the first check, every adopted reader read a view no check validated.
325            None if !readers.is_empty() => Some(api::ActionArchives {
326                unchecked: readers.len() as u64,
327                ..api::ActionArchives::default()
328            }),
329            None => None,
330        }
331    }
332
333    /// The runners' view as it changes. A receiver sees only the latest value: a withdrawal the task replaces before
334    /// the receiver looks is missed, so a test that must see one holds the task inside its check.
335    #[cfg(test)]
336    pub(crate) fn ready_changes(&self) -> watch::Receiver<Option<RunnerArchives>> {
337        self.ready.subscribe()
338    }
339
340    /// What a runner container mounts and its environment gets, read once as it starts: the archives' directory,
341    /// read-only, at [`MOUNT`], and [`ENV`] naming it, while the store is on and its last check found it good.
342    pub fn runner(&self) -> Option<RunnerArchives> {
343        if !self.settings.borrow().enabled {
344            return None;
345        }
346        self.ready.borrow().clone()
347    }
348}
349
350impl RunnerArchives {
351    /// What a runner gets of the archives at `archives`.
352    fn of(archives: &Path) -> RunnerArchives {
353        RunnerArchives {
354            mount: Mount {
355                target: Some(MOUNT.to_owned()),
356                source: Some(archives.to_string_lossy().into_owned()),
357                typ: Some(MountType::BIND),
358                read_only: Some(true),
359                ..Default::default()
360            },
361            env: format!("{ENV}={MOUNT}"),
362        }
363    }
364}
365
366/// What the task publishes to its `Link`, and what it hears from it.
367struct Published {
368    status: watch::Sender<Option<api::ActionArchives>>,
369    ready: watch::Sender<Option<RunnerArchives>>,
370    readers: watch::Receiver<Option<BTreeSet<String>>>,
371    packages: watch::Receiver<BTreeSet<String>>,
372}
373
374/// What a runner container gets of the archives.
375#[derive(Debug, Clone, PartialEq)]
376pub struct RunnerArchives {
377    /// The archives' directory, read-only.
378    pub mount: Mount,
379    /// `ACTIONS_RUNNER_ACTION_ARCHIVE_CACHE=/opt/runnerd/actions`.
380    pub env: String,
381}
382
383/// The running task; dropping it stops the task. A download stopped midway leaves a file in `staging/`, which the
384/// next start removes. The store's lock is held until the last blocking hash, check or write it started ends, as each
385/// holds the store.
386#[derive(Debug)]
387pub struct Running(Option<tokio::task::JoinHandle<()>>);
388
389impl Running {
390    /// Stops the task and waits until it has.
391    pub async fn stop(mut self) {
392        if let Some(task) = self.0.take() {
393            task.abort();
394            let _ = task.await;
395        }
396    }
397}
398
399impl Drop for Running {
400    fn drop(&mut self) {
401        if let Some(task) = &self.0 {
402            task.abort();
403        }
404    }
405}
406
407/// Keeps `bytes` in the store at `directory` as the archive of `repository` at `sha`, as a refresh left it: the next
408/// check finds it matching, and the next refresh asks GitHub whether its repository is still public.
409#[cfg(test)]
410pub(crate) fn keep(
411    directory: &Path,
412    repository: &str,
413    sha: &str,
414    bytes: &[u8],
415) -> std::io::Result<()> {
416    let mut store = ArchiveStore::open(directory)?;
417    let file = store.staging_file()?;
418    std::fs::write(&file, bytes)?;
419    let (sha256, size) = verify::hash_file(&file)?;
420    let key = Key {
421        repository: repository.to_owned(),
422        sha: sha.to_owned(),
423    };
424    let staged = Staged {
425        file,
426        sha256,
427        size,
428        uses: BTreeMap::new(),
429        unread: BTreeSet::new(),
430        actions: BTreeSet::new(),
431    };
432    store.publish(key, staged, unix_now())
433}
434
435fn unix_now() -> u64 {
436    SystemTime::now()
437        .duration_since(SystemTime::UNIX_EPOCH)
438        .unwrap_or_default()
439        .as_secs()
440}
441
442/// Owns the store at `directory`: refreshes at start, every `REFRESH` and whenever the settings change, and withdraws
443/// what jobs resolved as packages as soon as a log names one. Each refresh withdraws the runners' view, opens the
444/// store if it is not open yet, checks it, withdraws what is no longer public or is a package, and publishes the view
445/// again only if the check found it good; then it downloads, and publishes what clients show.
446async fn run<S: Source + 'static>(
447    source: Arc<S>,
448    directory: PathBuf,
449    mut settings: watch::Receiver<Settings>,
450    mut published: Published,
451) {
452    let mut archives = Archives::new(directory.clone());
453    let view = RunnerArchives::of(&directory.join(store::ARCHIVES));
454    // The first check names the runners recovery adopted, so it waits for recovery.
455    if published.readers.wait_for(Option::is_some).await.is_err() {
456        return;
457    }
458    let mut first = true;
459    loop {
460        let now = settings.borrow_and_update().clone();
461        if now.enabled {
462            // A start while the check runs gets nothing, rather than what the check may yet quarantine.
463            published.ready.send_replace(None);
464            let packages = published.packages.borrow_and_update().clone();
465            github::REQUESTS
466                .scope(std::cell::Cell::new(0), async {
467                    let mut round = archives.round();
468                    let checked = archives.check(unix_now()).await;
469                    if first {
470                        let readers = published.readers.borrow().clone().unwrap_or_default();
471                        archives.exposed(&readers, checked, unix_now());
472                        first = false;
473                    }
474                    archives.packages(&packages, unix_now()).await;
475                    archives.vet(&*source, &mut round, unix_now()).await;
476                    published
477                        .ready
478                        .send_replace(checked.good().then(|| view.clone()));
479                    published.status.send_replace(Some(archives.status()));
480                    archives
481                        .update(&*source, &mut round, &now, unix_now())
482                        .await;
483                })
484                .await;
485            published.status.send_replace(Some(archives.status()));
486        } else {
487            published.ready.send_replace(None);
488        }
489        loop {
490            tokio::select! {
491                () = tokio::time::sleep(REFRESH) => break,
492                changed = settings.changed() => if changed.is_err() { return } else { break },
493                changed = published.packages.changed() => {
494                    if changed.is_err() {
495                        return;
496                    }
497                    let packages = published.packages.borrow_and_update().clone();
498                    if settings.borrow().enabled {
499                        archives.packages(&packages, unix_now()).await;
500                        published.status.send_replace(Some(archives.status()));
501                    }
502                }
503            }
504        }
505    }
506}
507
508/// The store and what the task knows besides it: what a refresh works on.
509#[derive(Debug)]
510pub(crate) struct Archives {
511    /// The store's directory.
512    directory: PathBuf,
513    /// Shared with the blocking tasks that hash, verify and write; only this task's refresh uses them, one at a time.
514    /// `None` until it opens: each refresh tries again.
515    store: Option<Arc<Mutex<ArchiveStore>>>,
516    refresher: Refresher,
517    limits: Limits,
518}
519
520/// What one refresh spends and has asked: its meter and the names GitHub gave.
521#[derive(Debug)]
522pub(crate) struct Round {
523    meter: Meter,
524    names: Names,
525}
526
527/// What a check found, as the task acts on it.
528#[derive(Debug, Clone, Copy, PartialEq, Eq)]
529pub(crate) enum Checked {
530    /// Every file is good, and none was moved.
531    Good,
532    /// Every file is good now, after the check moved some to quarantine.
533    Moved,
534    /// The store did not open, or a file could not be read or moved: what runners would read is not known good.
535    Bad,
536}
537
538impl Checked {
539    /// Whether runners may read the archives.
540    pub(crate) fn good(self) -> bool {
541        self != Checked::Bad
542    }
543}
544
545impl Archives {
546    /// The archives kept at `directory`, opened at the first check.
547    pub(crate) fn new(directory: PathBuf) -> Archives {
548        Archives {
549            directory,
550            store: None,
551            refresher: Refresher::default(),
552            limits: LIMITS,
553        }
554    }
555
556    /// These archives, refreshed within `limits`.
557    #[cfg(test)]
558    pub(crate) fn with_limits(mut self, limits: Limits) -> Archives {
559        self.limits = limits;
560        self
561    }
562
563    /// A new refresh's round.
564    pub(crate) fn round(&self) -> Round {
565        Round {
566            meter: Meter::new(self.limits),
567            names: Names::default(),
568        }
569    }
570
571    /// What clients show.
572    pub(crate) fn status(&self) -> api::ActionArchives {
573        let (entries, bytes) = self
574            .store
575            .as_ref()
576            .map_or((0, 0), |store| lock(store).size());
577        self.refresher.status(entries, bytes)
578    }
579
580    /// One refresh at `now`, as the task runs it, without the runners' view.
581    #[cfg(test)]
582    pub(crate) async fn refresh<S: Source>(&mut self, source: &S, settings: &Settings, now: u64) {
583        github::REQUESTS
584            .scope(std::cell::Cell::new(0), async {
585                let mut round = self.round();
586                self.check(now).await;
587                self.vet(source, &mut round, now).await;
588                self.update(source, &mut round, settings, now).await;
589            })
590            .await;
591    }
592
593    /// One update at `now`, without the check before it.
594    #[cfg(test)]
595    pub(crate) async fn refill<S: Source>(&mut self, source: &S, settings: &Settings, now: u64) {
596        github::REQUESTS
597            .scope(std::cell::Cell::new(0), async {
598                let mut round = self.round();
599                self.update(source, &mut round, settings, now).await;
600            })
601            .await;
602    }
603
604    /// The store's directory, as failures of the store name it.
605    fn subject(&self) -> String {
606        self.directory.display().to_string()
607    }
608
609    /// Notes what the restart's first check means for `readers`, the runners it adopted with the archives mounted:
610    /// when it moved or could not settle a file, each may have read it, and is named until it is gone.
611    pub(crate) fn exposed(&mut self, readers: &BTreeSet<String>, checked: Checked, now: u64) {
612        if readers.is_empty() {
613            return;
614        }
615        if checked == Checked::Good {
616            info!(
617                "{} runners a restart adopted read the action archives; the first check found every one good",
618                readers.len()
619            );
620            return;
621        }
622        for reader in readers {
623            warn!(
624                "{reader} kept running across the restart with the action archives mounted, and the first check found \
625                 one changed or could not read one: its job may have read it"
626            );
627            self.refresher
628                .fail(reader, api::ArchiveFailureKind::Exposed, now);
629        }
630    }
631
632    /// Withdraws every archive of `packages`, repositories jobs resolved as immutable action packages, and keeps them
633    /// out from then on.
634    pub(crate) async fn packages(&mut self, packages: &BTreeSet<String>, now: u64) {
635        let Some(store) = self.store.clone() else {
636            return;
637        };
638        for package in packages {
639            let noted = {
640                let store = store.clone();
641                let package = package.clone();
642                tokio::task::spawn_blocking(move || lock(&store).packaged(&package)).await
643            };
644            match noted {
645                Ok(Ok(withdrawn)) => {
646                    for key in &withdrawn.removed {
647                        info!(
648                            "action archive {} removed: a job resolved its repository as an immutable package",
649                            key.label()
650                        );
651                    }
652                    if withdrawn.new {
653                        self.refresher
654                            .fail(package, api::ArchiveFailureKind::Package, now);
655                    }
656                }
657                Ok(Err(error)) => {
658                    warn!("cannot remove the action archives of the package {package}: {error}");
659                    self.refresher
660                        .fail(&self.subject(), api::ArchiveFailureKind::Store, now);
661                }
662                Err(_) => {}
663            }
664        }
665    }
666
667    /// Reads each served repository's pins, from its tree only when its default branch moved, asks GitHub which are
668    /// public every time, downloads and verifies what is pinned and missing, following the pins of each archive as
669    /// deep as a runner nests actions, and removes what no repository pinned for `keep`. A repository whose pins are
670    /// not all known keeps every archive it last reached; an action that is not supported (not public, a package, an
671    /// archive that failed its check, no action at its path) is known, and kept out. Nothing while the store is not
672    /// open.
673    pub(crate) async fn update<S: Source>(
674        &mut self,
675        source: &S,
676        round: &mut Round,
677        settings: &Settings,
678        now: u64,
679    ) {
680        let Some(store) = self.store.clone() else {
681            return;
682        };
683        let mut directs: BTreeMap<String, Option<Direct>> = BTreeMap::new();
684        for repository in &settings.repositories {
685            let direct = self.read_pins(source, round, &store, repository, now).await;
686            directs.insert(repository.clone(), direct);
687        }
688        // Each archive is tried once a refresh: one that failed waits for the next.
689        let mut tried: BTreeSet<Key> = BTreeSet::new();
690        // Those the refresh's budget or the store's cap left for later: missing, but not for want of reading.
691        let mut deferred: BTreeSet<Key> = BTreeSet::new();
692        // Each round downloads what the last made known, one level deeper: level 0, the repositories' own pins, to
693        // level `MAX_DEPTH`.
694        for _ in 0..=MAX_DEPTH {
695            let mut wanted = BTreeSet::new();
696            for direct in directs.values().flatten() {
697                let reach = closure(source, round, &store, &direct.wanted).await;
698                wanted.extend(reach.missing);
699            }
700            let missing: Vec<Key> = wanted.difference(&tried).cloned().collect();
701            if missing.is_empty() {
702                break;
703            }
704            tried.extend(missing.iter().cloned());
705            deferred.extend(self.fetch_all(source, round, &store, missing, now).await);
706        }
707        let mut whole = BTreeMap::new();
708        let mut pinned = BTreeSet::new();
709        let mut unsupported = BTreeMap::new();
710        let mut refused = BTreeSet::new();
711        for (repository, direct) in &directs {
712            let (complete, waiting) = match direct {
713                Some(direct) => {
714                    let reach = closure(source, round, &store, &direct.wanted).await;
715                    unsupported.extend(reach.unsupported);
716                    refused.extend(reach.refused);
717                    pinned.extend(reach.keys.iter().cloned());
718                    let read = direct.known && reach.answered;
719                    let complete = read && reach.missing.is_empty();
720                    let waiting = read && reach.missing.is_subset(&deferred);
721                    if complete {
722                        whole.insert(repository.clone(), reach.keys);
723                    }
724                    (complete, waiting)
725                }
726                None => (false, false),
727            };
728            if complete || waiting {
729                self.refresher
730                    .succeed_kind(repository, api::ArchiveFailureKind::Held);
731            }
732            if !complete {
733                // What it reached when all its pins were last known stays: what was not read may pin it.
734                pinned.extend(lock(&store).reached(repository));
735                // Only waiting for downloads is no news: the next refresh goes on with them.
736                if !waiting {
737                    info!(
738                        "{repository}: not all its pins are known; the archives it last reached are kept"
739                    );
740                    self.refresher
741                        .fail(repository, api::ArchiveFailureKind::Held, now);
742                }
743            }
744        }
745        for (subject, kind) in &unsupported {
746            self.refresher.fail(subject, *kind, now);
747        }
748        let retained = {
749            let store = store.clone();
750            let keep = settings.keep;
751            let served = settings.repositories.clone();
752            let pinned = pinned.clone();
753            tokio::task::spawn_blocking(move || {
754                let mut store = lock(&store);
755                store.record(whole, &served)?;
756                store.retain(&pinned, keep, now)
757            })
758            .await
759        };
760        match retained {
761            Ok(Ok(removed)) => {
762                for key in removed {
763                    info!(
764                        "action archive {} removed: no repository pinned it for {}",
765                        key.label(),
766                        api::duration(settings.keep.as_secs())
767                    );
768                }
769            }
770            Ok(Err(error)) => {
771                warn!("cannot remove action archives no longer pinned: {error}");
772                self.refresher
773                    .fail(&self.subject(), api::ArchiveFailureKind::Store, now);
774            }
775            Err(_) => {}
776        }
777        let mut subjects: BTreeSet<String> = pinned.iter().map(Key::label).collect();
778        subjects.extend(unsupported.into_keys());
779        subjects.extend(refused);
780        self.refresher
781            .finish(&settings.repositories, &subjects, &self.subject(), now);
782    }
783
784    /// Opens the store if it is not open, and checks every kept archive against the index: what changed is
785    /// quarantined, and fetched again by the next update.
786    pub(crate) async fn check(&mut self, now: u64) -> Checked {
787        let subject = self.subject();
788        let store = match self.open().await {
789            Ok(store) => store,
790            Err(error) => {
791                warn!("no action archives: cannot open {subject}: {error}");
792                self.refresher
793                    .fail(&subject, api::ArchiveFailureKind::Store, now);
794                return Checked::Bad;
795            }
796        };
797        // What jobs resolved as packages stays reported, a restart included.
798        for package in lock(&store).packages() {
799            self.refresher
800                .fail(&package, api::ArchiveFailureKind::Package, now);
801        }
802        let checked = match tokio::task::spawn_blocking(move || lock(&store).check(now)).await {
803            Ok(Ok(checked)) => checked,
804            Ok(Err(error)) => {
805                warn!("cannot check the action archives: {error}");
806                self.refresher
807                    .fail(&subject, api::ArchiveFailureKind::Store, now);
808                return Checked::Bad;
809            }
810            Err(_) => return Checked::Bad,
811        };
812        for key in &checked.changed {
813            self.refresher
814                .fail(&key.label(), api::ArchiveFailureKind::Quarantined, now);
815        }
816        for stranger in &checked.strangers {
817            warn!("action archive {stranger} is not in the index; moved to quarantine");
818        }
819        for error in &checked.errors {
820            warn!("cannot check the action archive {error}; runners get no archives until it is");
821        }
822        if !checked.clean() {
823            self.refresher
824                .fail(&subject, api::ArchiveFailureKind::Store, now);
825            return Checked::Bad;
826        }
827        self.refresher.succeed(&subject);
828        if checked.changed.is_empty() && checked.strangers.is_empty() {
829            Checked::Good
830        } else {
831            Checked::Moved
832        }
833    }
834
835    /// Removes every kept archive whose repository GitHub now says is not public, or not there, before runners get
836    /// the view: anything a runner reads is what anyone may download, whether or not a repository still pins it. A
837    /// repository GitHub does not answer for is left as it is.
838    pub(crate) async fn vet<S: Source>(&mut self, source: &S, round: &mut Round, now: u64) {
839        let Some(store) = self.store.clone() else {
840            return;
841        };
842        let repositories = lock(&store).repositories();
843        let mut hidden = Vec::new();
844        for repository in repositories {
845            if let Ok(None) = round
846                .names
847                .public(source, &round.meter, &store, &repository)
848                .await
849            {
850                hidden.push(repository);
851            }
852        }
853        if hidden.is_empty() {
854            return;
855        }
856        let withdrawn = {
857            let store = store.clone();
858            tokio::task::spawn_blocking(move || lock(&store).withdraw(&hidden)).await
859        };
860        match withdrawn {
861            Ok(Ok(keys)) => {
862                for key in keys {
863                    info!(
864                        "action archive {} removed: its repository is not public",
865                        key.label()
866                    );
867                }
868            }
869            Ok(Err(error)) => {
870                warn!("cannot remove action archives no longer public: {error}");
871                self.refresher
872                    .fail(&self.subject(), api::ArchiveFailureKind::Store, now);
873            }
874            Err(_) => {}
875        }
876    }
877
878    /// The store, opened now if it is not yet.
879    async fn open(&mut self) -> std::io::Result<Arc<Mutex<ArchiveStore>>> {
880        if let Some(store) = &self.store {
881            return Ok(store.clone());
882        }
883        let directory = self.directory.clone();
884        let store = tokio::task::spawn_blocking(move || ArchiveStore::open(&directory))
885            .await
886            .map_err(std::io::Error::other)??;
887        let store = Arc::new(Mutex::new(store));
888        self.store = Some(store.clone());
889        Ok(store)
890    }
891
892    /// The pins of `repository` this refresh goes by, each asked of GitHub for its name and visibility: its last
893    /// scan's when its default branch has not moved, read from its tree when it has, and the last scan's, not known
894    /// whole, when it cannot be read. `None` when it cannot be read and never was.
895    async fn read_pins<S: Source>(
896        &mut self,
897        source: &S,
898        round: &mut Round,
899        store: &Arc<Mutex<ArchiveStore>>,
900        repository: &str,
901        now: u64,
902    ) -> Option<Direct> {
903        let last = lock(store).scan(repository).cloned();
904        let (scan, known) = match scan_of(source, &round.meter, repository, last.as_ref()).await {
905            Ok(scan) => {
906                self.refresher.succeed(repository);
907                if last.as_ref() != Some(&scan) {
908                    let saved = {
909                        let store = store.clone();
910                        let repository = repository.to_owned();
911                        let scan = scan.clone();
912                        tokio::task::spawn_blocking(move || lock(&store).scanned(&repository, scan))
913                            .await
914                    };
915                    if let Ok(Err(error)) = saved {
916                        warn!("cannot keep {repository}'s pins: {error}");
917                        self.refresher
918                            .fail(&self.subject(), api::ArchiveFailureKind::Store, now);
919                    }
920                }
921                (scan, true)
922            }
923            Err(error) => {
924                warn!(
925                    "{repository}: cannot read the actions it pins; keeping the last read and every archive they \
926                     reached: {}",
927                    error.brief()
928                );
929                self.refresher
930                    .fail(repository, api::ArchiveFailureKind::Pins, now);
931                (last?, false)
932            }
933        };
934        let mut direct = Direct {
935            wanted: BTreeSet::new(),
936            known,
937        };
938        for pin in scan.pins {
939            match round
940                .names
941                .public(source, &round.meter, store, &pin.repository)
942                .await
943            {
944                Ok(Some(name)) => {
945                    direct.wanted.insert(Wanted {
946                        key: Key {
947                            repository: name,
948                            sha: pin.sha,
949                        },
950                        path: pin.path,
951                    });
952                }
953                Ok(None) => {}
954                Err(error) => {
955                    warn!(
956                        "cannot read {}, which {repository} pins: {}",
957                        pin.repository,
958                        error.brief()
959                    );
960                    direct.known = false;
961                }
962            }
963        }
964        Some(direct)
965    }
966
967    /// Downloads, verifies and publishes each of `missing`, a few at a time. Returns those left for a later refresh,
968    /// by its budget or the store's cap.
969    async fn fetch_all<S: Source>(
970        &mut self,
971        source: &S,
972        round: &Round,
973        store: &Arc<Mutex<ArchiveStore>>,
974        missing: Vec<Key>,
975        now: u64,
976    ) -> BTreeSet<Key> {
977        let mut deferred = BTreeSet::new();
978        let meter = &round.meter;
979        let results: Vec<(Key, Result<u64, Fetch>)> = futures_util::stream::iter(missing)
980            .map(|key| async move {
981                let result = fetch(source, meter, store, &key, now).await;
982                (key, result)
983            })
984            .buffer_unordered(DOWNLOADS)
985            .collect()
986            .await;
987        for (key, result) in results {
988            let label = key.label();
989            match result {
990                Ok(_) => self.refresher.succeed(&label),
991                Err(Fetch::Deferred) => {
992                    deferred.insert(key);
993                }
994                Err(Fetch::Full) => {
995                    deferred.insert(key);
996                    warn!(
997                        "the action archives are at their size cap; {label} waits for room, and jobs download it \
998                         themselves"
999                    );
1000                    self.refresher
1001                        .fail(&self.subject(), api::ArchiveFailureKind::Full, now);
1002                }
1003                Err(Fetch::Download(error)) => {
1004                    warn!(
1005                        "cannot download the action archive {label}; jobs download it themselves: {}",
1006                        error.brief()
1007                    );
1008                    self.refresher
1009                        .fail(&label, api::ArchiveFailureKind::Download, now);
1010                }
1011                Err(Fetch::Gone) => {
1012                    warn!(
1013                        "the action archive {label} is not there to anyone; it is not asked for again while pinned"
1014                    );
1015                    self.refusal(store, key, now).await;
1016                    self.refresher
1017                        .fail(&label, api::ArchiveFailureKind::Download, now);
1018                }
1019                Err(Fetch::Rejected(why)) => {
1020                    warn!(
1021                        "the action archive {label} {why}; not kept, jobs download it themselves"
1022                    );
1023                    self.refusal(store, key, now).await;
1024                    self.refresher
1025                        .fail(&label, api::ArchiveFailureKind::Rejected, now);
1026                }
1027                Err(Fetch::Store(error)) => {
1028                    warn!("cannot keep the action archive {label}: {error}");
1029                    self.refresher
1030                        .fail(&self.subject(), api::ArchiveFailureKind::Store, now);
1031                }
1032            }
1033        }
1034        deferred
1035    }
1036
1037    /// Notes that `key`'s archive is not one to download again.
1038    async fn refusal(&mut self, store: &Arc<Mutex<ArchiveStore>>, key: Key, now: u64) {
1039        let store = store.clone();
1040        if let Ok(Err(error)) = tokio::task::spawn_blocking(move || lock(&store).refuse(key)).await
1041        {
1042            warn!("cannot keep a refused action archive's note: {error}");
1043            self.refresher
1044                .fail(&self.subject(), api::ArchiveFailureKind::Store, now);
1045        }
1046    }
1047}
1048
1049/// A served repository's own pins, as one refresh goes by them.
1050#[derive(Debug)]
1051struct Direct {
1052    /// Those public, by GitHub's names.
1053    wanted: BTreeSet<Wanted>,
1054    /// Whether they are known whole: its files read in full at its head, and every repository they name answered.
1055    known: bool,
1056}
1057
1058/// `repository`'s pins at its default branch's head: `last`'s when the head has not moved, read whole from its tree
1059/// when it has. A tree GitHub cut short, more files than are read, and a file too large or not YAML each fail it: what
1060/// was not read may pin anything.
1061async fn scan_of<S: Source>(
1062    source: &S,
1063    meter: &Meter,
1064    repository: &str,
1065    last: Option<&store::Scan>,
1066) -> Result<store::Scan, PinsError> {
1067    meter.ask(source)?;
1068    let details = source.action_repository(repository).await?;
1069    meter.ask(source)?;
1070    let head = source
1071        .branch_head(repository, &details.default_branch)
1072        .await?;
1073    if let Some(last) = last.filter(|last| last.head == head) {
1074        return Ok(last.clone());
1075    }
1076    meter.ask(source)?;
1077    let tree = source.tree(repository, &head).await?;
1078    if tree.truncated {
1079        return Err(PinsError::Truncated);
1080    }
1081    let files: Vec<&TreeFile> = tree
1082        .files
1083        .iter()
1084        .filter(|file| pins::read_for_pins(&file.path))
1085        .collect();
1086    if files.len() > PIN_FILES {
1087        return Err(PinsError::Files(files.len()));
1088    }
1089    let mut pins: BTreeSet<Pin> = BTreeSet::new();
1090    for file in files {
1091        if file.size > PIN_FILE_BYTES {
1092            return Err(PinsError::Large(file.path.clone()));
1093        }
1094        meter.ask(source)?;
1095        let bytes = source.blob(repository, &file.blob, PIN_FILE_BYTES).await?;
1096        let found = pins::file_pins(&file.path, &bytes).map_err(|why| PinsError::Unreadable {
1097            path: file.path.clone(),
1098            why,
1099        })?;
1100        pins.extend(found);
1101    }
1102    Ok(store::Scan { head, pins })
1103}
1104
1105/// What a repository's pins reach now, through the pins of the archives held: level 0 is its own, and the pins of
1106/// each level's archives are the next, to level `MAX_DEPTH`, the deepest a runner nests.
1107#[derive(Debug, Default)]
1108struct Reach {
1109    /// Every archive reached: held, to download, or refused.
1110    keys: BTreeSet<Key>,
1111    /// Those not held, to download: while any is left, what it pins is not known.
1112    missing: BTreeSet<Key>,
1113    /// Actions reached that are not supported, by subject: known, and kept out.
1114    unsupported: BTreeMap<String, api::ArchiveFailureKind>,
1115    /// Archives reached that failed their check or are not there to anyone, by label: known, and not downloaded
1116    /// again.
1117    refused: BTreeSet<String>,
1118    /// Whether every repository a nested pin names answered.
1119    answered: bool,
1120}
1121
1122/// What `direct` reaches now (`Reach`).
1123async fn closure<S: Source>(
1124    source: &S,
1125    round: &mut Round,
1126    store: &Arc<Mutex<ArchiveStore>>,
1127    direct: &BTreeSet<Wanted>,
1128) -> Reach {
1129    let mut reach = Reach {
1130        answered: true,
1131        ..Reach::default()
1132    };
1133    let mut seen: BTreeSet<Wanted> = BTreeSet::new();
1134    let mut level: BTreeSet<Wanted> = direct.clone();
1135    for depth in 0..=MAX_DEPTH {
1136        let mut next = BTreeSet::new();
1137        for wanted in &level {
1138            if !seen.insert(wanted.clone()) {
1139                continue;
1140            }
1141            let (refused, action) = {
1142                let store = lock(store);
1143                (
1144                    store.refused(&wanted.key),
1145                    store.action(&wanted.key, &wanted.path),
1146                )
1147            };
1148            let subject = wanted.subject();
1149            let nested = match action {
1150                // Refused before: the runner downloads it itself, and runnerd does not again.
1151                Action::Missing if refused => {
1152                    // Still reached: its refusal stays while it is.
1153                    reach.keys.insert(wanted.key.clone());
1154                    reach.refused.insert(wanted.key.label());
1155                    continue;
1156                }
1157                Action::Missing => {
1158                    reach.keys.insert(wanted.key.clone());
1159                    reach.missing.insert(wanted.key.clone());
1160                    continue;
1161                }
1162                Action::NoFile => {
1163                    reach.keys.insert(wanted.key.clone());
1164                    reach
1165                        .unsupported
1166                        .insert(subject, api::ArchiveFailureKind::NotAction);
1167                    continue;
1168                }
1169                Action::Unread => {
1170                    reach.keys.insert(wanted.key.clone());
1171                    reach
1172                        .unsupported
1173                        .insert(subject, api::ArchiveFailureKind::Unreadable);
1174                    continue;
1175                }
1176                Action::Pins(nested) => {
1177                    reach.keys.insert(wanted.key.clone());
1178                    nested
1179                }
1180            };
1181            // A runner nests no deeper: what the deepest level pins, it never runs.
1182            if depth == MAX_DEPTH {
1183                continue;
1184            }
1185            for pin in nested {
1186                match round
1187                    .names
1188                    .public(source, &round.meter, store, &pin.repository)
1189                    .await
1190                {
1191                    Ok(Some(name)) => {
1192                        next.insert(Wanted {
1193                            key: Key {
1194                                repository: name,
1195                                sha: pin.sha,
1196                            },
1197                            path: pin.path,
1198                        });
1199                    }
1200                    Ok(None) => {}
1201                    Err(error) => {
1202                        reach.answered = false;
1203                        warn!(
1204                            "cannot read {}, which {} pins: {}",
1205                            pin.repository,
1206                            wanted.key.label(),
1207                            error.brief()
1208                        );
1209                    }
1210                }
1211            }
1212        }
1213        if next.is_empty() {
1214            break;
1215        }
1216        level = next;
1217    }
1218    reach
1219}
1220
1221/// Why an archive is not kept.
1222#[derive(Debug)]
1223enum Fetch {
1224    Download(Error),
1225    /// Not there to anyone: codeload does not send it anonymously.
1226    Gone,
1227    Rejected(String),
1228    Store(std::io::Error),
1229    /// Not tried, or not again: the refresh's bytes are spent, or GitHub's rate limit is.
1230    Deferred,
1231    /// Not tried: the store is at its size cap.
1232    Full,
1233}
1234
1235/// What one refresh may still spend: requests to GitHub's API, as sent (`github::REQUESTS`, a request sent again
1236/// with a new token included), and bytes received. Each download reserves the largest archive's bytes before it starts
1237/// and gives back what it did not receive once it ends, whether it succeeded or not, so the downloads in flight
1238/// together never pass what is left, nor, with what the store holds, its cap.
1239#[derive(Debug)]
1240struct Meter {
1241    limits: Limits,
1242    bytes: Arc<Mutex<Bytes>>,
1243}
1244
1245/// The bytes a refresh has left to receive, and those downloads in flight may still add to the store.
1246#[derive(Debug)]
1247struct Bytes {
1248    left: u64,
1249    in_flight: u64,
1250}
1251
1252/// Why a download may not start.
1253enum Refused {
1254    /// The refresh's bytes are spent.
1255    Spent,
1256    /// The store, with what is in flight, would pass its cap.
1257    Full,
1258}
1259
1260/// A download's place in the store's cap, given back when it ends. Once its archive is published, it is given back
1261/// with the store still locked, so a download reserving with the store locked counts the archive once: in the store's
1262/// size or in flight.
1263struct InFlight {
1264    bytes: Arc<Mutex<Bytes>>,
1265    archive: u64,
1266}
1267
1268impl Drop for InFlight {
1269    fn drop(&mut self) {
1270        let mut bytes = lock(&self.bytes);
1271        bytes.in_flight = bytes.in_flight.saturating_sub(self.archive);
1272    }
1273}
1274
1275impl Meter {
1276    fn new(limits: Limits) -> Meter {
1277        Meter {
1278            limits,
1279            bytes: Arc::new(Mutex::new(Bytes {
1280                left: limits.bytes,
1281                in_flight: 0,
1282            })),
1283        }
1284    }
1285
1286    /// Lets one more request go to GitHub's API, if the refresh and the rate limit have one to spare.
1287    fn ask<S: Source>(&self, source: &S) -> Result<(), Error> {
1288        if source.spare_requests().is_some_and(|spare| spare < RESERVE) {
1289            return Err(Error::Held(
1290                "what is left of GitHub's rate limit is kept for runners",
1291            ));
1292        }
1293        // A request may be sent twice, once more with a new token after a 401.
1294        if github::requests_sent() + 2 > u64::from(self.limits.requests) {
1295            return Err(Error::Held("this refresh's requests to GitHub are spent"));
1296        }
1297        Ok(())
1298    }
1299
1300    /// Reserves a download's bytes, if the refresh has them left and `store` has room for them beside the downloads in
1301    /// flight. The place in the cap is held until what it returns is dropped.
1302    fn reserve(&self, store: &ArchiveStore) -> Result<InFlight, Refused> {
1303        let held = store.size().1;
1304        let mut bytes = lock(&self.bytes);
1305        if held + bytes.in_flight + self.limits.archive > self.limits.store {
1306            return Err(Refused::Full);
1307        }
1308        if bytes.left < self.limits.archive {
1309            return Err(Refused::Spent);
1310        }
1311        bytes.left -= self.limits.archive;
1312        bytes.in_flight += self.limits.archive;
1313        Ok(InFlight {
1314            bytes: self.bytes.clone(),
1315            archive: self.limits.archive,
1316        })
1317    }
1318
1319    /// Gives back what a download reserved and did not receive.
1320    fn spend(&self, received: u64) {
1321        lock(&self.bytes).left += self.limits.archive - received.min(self.limits.archive);
1322    }
1323}
1324
1325/// Downloads `key`'s archive into `staging/`, a few attempts, verifies it whole, and publishes it. Whatever fails, its
1326/// staged file goes.
1327async fn fetch<S: Source>(
1328    source: &S,
1329    meter: &Meter,
1330    store: &Arc<Mutex<ArchiveStore>>,
1331    key: &Key,
1332    now: u64,
1333) -> Result<u64, Fetch> {
1334    let file = lock(store).staging_file().map_err(Fetch::Store)?;
1335    let mut last = None;
1336    // Held until the archive is published or given up on.
1337    let mut place = None;
1338    for wait in ATTEMPTS {
1339        tokio::time::sleep(wait).await;
1340        drop(place.take());
1341        // With the store locked: an archive another download publishes is in the store's size or in flight, not both.
1342        let reserved = meter.reserve(&lock(store));
1343        place = Some(match reserved {
1344            Ok(place) => place,
1345            Err(Refused::Full) => return Err(Fetch::Full),
1346            Err(Refused::Spent) => return Err(Fetch::Deferred),
1347        });
1348        let received = AtomicU64::new(0);
1349        let result = source
1350            .tarball(
1351                &key.repository,
1352                &key.sha,
1353                &file,
1354                meter.limits.archive,
1355                &received,
1356            )
1357            .await;
1358        meter.spend(received.load(Ordering::Relaxed));
1359        match result {
1360            Ok(_) => {
1361                last = None;
1362                break;
1363            }
1364            Err(error) => {
1365                let _ = tokio::fs::remove_file(&file).await;
1366                if matches!(error, Error::RateLimited { .. } | Error::Held(_)) {
1367                    return Err(Fetch::Deferred);
1368                }
1369                // Asking again gets the same answer.
1370                if error.is_gone() {
1371                    return Err(Fetch::Gone);
1372                }
1373                last = Some(error);
1374            }
1375        }
1376    }
1377    if let Some(error) = last {
1378        return Err(Fetch::Download(error));
1379    }
1380    let published = {
1381        let store = store.clone();
1382        let key = key.clone();
1383        let file = file.clone();
1384        tokio::task::spawn_blocking(move || -> Result<u64, Fetch> {
1385            let verified = verify::verify(&file, &key.sha, verify::BUDGET)
1386                .map_err(|rejected| Fetch::Rejected(format!("failed its check: {rejected}")))?;
1387            let size = verified.size;
1388            let mut uses: BTreeMap<String, Vec<Pin>> = BTreeMap::new();
1389            let mut unread = BTreeSet::new();
1390            let actions: BTreeSet<String> = verified.actions.keys().cloned().collect();
1391            for (dir, action) in &verified.actions {
1392                match action.bytes.as_deref().map(pins::action_file_pins) {
1393                    Some(Ok(found)) if found.is_empty() => {}
1394                    Some(Ok(found)) => {
1395                        uses.insert(dir.clone(), found);
1396                    }
1397                    Some(Err(_)) | None => {
1398                        unread.insert(dir.clone());
1399                    }
1400                }
1401            }
1402            let staged = Staged {
1403                file: file.clone(),
1404                sha256: verified.sha256,
1405                size,
1406                uses,
1407                unread,
1408                actions,
1409            };
1410            let mut store = lock(&store);
1411            store.publish(key, staged, now).map_err(Fetch::Store)?;
1412            // Its bytes are in the store's size now: given back before another download can read that size.
1413            drop(place);
1414            drop(store);
1415            Ok(size)
1416        })
1417        .await
1418    };
1419    let result = match published {
1420        Ok(result) => result,
1421        Err(error) => Err(Fetch::Store(std::io::Error::other(error))),
1422    };
1423    if result.is_err() {
1424        let _ = tokio::fs::remove_file(&file).await;
1425    }
1426    result
1427}
1428
1429/// Why a repository's pins could not be read.
1430#[derive(Debug, thiserror::Error)]
1431enum PinsError {
1432    #[error(transparent)]
1433    Github(#[from] Error),
1434    #[error("GitHub cut its tree short")]
1435    Truncated,
1436    #[error("it has {0} workflows and action files, more than are read")]
1437    Files(usize),
1438    #[error("{0} is larger than is read")]
1439    Large(String),
1440    #[error("{path}: {why}")]
1441    Unreadable { path: String, why: pins::Unreadable },
1442}
1443
1444impl PinsError {
1445    /// What failed, as a log line about another's repository says it (`Error::brief`).
1446    fn brief(&self) -> String {
1447        match self {
1448            PinsError::Github(error) => error.brief(),
1449            PinsError::Truncated
1450            | PinsError::Files(_)
1451            | PinsError::Large(_)
1452            | PinsError::Unreadable { .. } => self.to_string(),
1453        }
1454    }
1455}
1456
1457/// The canonical names of the repositories pins name, as asked in one refresh: `None` for one that is not public, not
1458/// there, or a package.
1459#[derive(Debug, Default)]
1460struct Names(HashMap<String, Option<String>>);
1461
1462impl Names {
1463    /// `repository`'s canonical name, if it is public, no job resolved it as a package, and its name is one a file
1464    /// may be named by.
1465    async fn public<S: Source>(
1466        &mut self,
1467        source: &S,
1468        meter: &Meter,
1469        store: &Arc<Mutex<ArchiveStore>>,
1470        repository: &str,
1471    ) -> Result<Option<String>, Error> {
1472        let asked = repository.to_ascii_lowercase();
1473        if let Some(name) = self.0.get(&asked) {
1474            return Ok(name.clone());
1475        }
1476        if lock(store).package(&asked) {
1477            self.0.insert(asked, None);
1478            return Ok(None);
1479        }
1480        meter.ask(source)?;
1481        let name = match source.action_repository(repository).await {
1482            Ok(found) => (found.public
1483                && pins::repository(&found.full_name)
1484                && !lock(store).package(&found.full_name))
1485            .then_some(found.full_name),
1486            Err(error) if error.is_gone() => None,
1487            Err(error) => return Err(error),
1488        };
1489        self.0.insert(asked, name.clone());
1490        Ok(name)
1491    }
1492}
1493
1494#[cfg(test)]
1495mod tests;