Skip to main content

runnerd_core/
buildcache.rs

1//! The build cache every runner shares (ADR 0052, 0053, 0083): the entries of Gradle's HTTP build cache, and of the
2//! other tools the runner image points at it, Turborepo's and Nx's among them, kept per repository in a directory of
3//! the daemon's and served to each runner on a Unix socket of its own, which the runner image forwards to
4//! `127.0.0.1`. Owns the entries, the size limit, the sockets, and what each runner was served and stored. Which
5//! socket a request came in on is who sent it; no job can claim another's. It never decides whether a job may write:
6//! the engine tells it (`Writes`), from what GitHub says of the job. It never logs a request's headers, which may
7//! carry a tool's token. Where the config turns it on, the same socket serves GitHub Actions' cache (`actions`, ADR
8//! 0082) from the same entries' directory and limits.
9
10pub(crate) mod actions;
11mod decision;
12mod serve;
13mod turbo;
14
15use std::collections::{BTreeSet, HashMap};
16use std::io;
17use std::os::unix::fs::PermissionsExt;
18use std::path::{Path, PathBuf};
19use std::sync::{Arc, Mutex};
20
21use tokio::net::UnixListener;
22use tokio::sync::watch;
23use tokio::task::{JoinHandle, JoinSet};
24use tracing::warn;
25
26use crate::api::BuildCacheCounts;
27use crate::config::BuildCacheWriters;
28use crate::github::{PullRequest, RunTrigger};
29use crate::sync::lock;
30
31/// Where a runner container mounts its socket's directory, read-only (`image/entrypoint.sh`).
32pub const MOUNT: &str = "/run/runnerd-build-cache";
33
34/// The socket's name in that directory.
35pub const SOCKET: &str = "cache.sock";
36
37/// A file in that directory when the runner's socket serves GitHub Actions' cache too: the image forwards it for
38/// node actions and BuildKit only then (`image/build-cache.sh`).
39pub const ACTIONS_MARKER: &str = "actions";
40
41/// The directory of the entries, in the build cache's directory: nothing else there is counted or trimmed.
42const ENTRIES: &str = "entries";
43
44/// Where uploads are written before they become entries, in the build cache's directory; emptied at start.
45const UPLOADS: &str = "uploads";
46
47/// What an entry or upload counts for at least against the size limit, in bytes: its file's inode and directory
48/// entry cost the disk and the index something even when it is empty, so the limit also bounds how many there are.
49const MIN_CHARGE: u64 = 4 << 10;
50
51/// The uploads one runner may hold while its writes are undecided; past them, its uploads are refused.
52const MAX_HELD: usize = 10_000;
53
54/// The connections one runner's socket serves at once; more wait to be accepted. Above the connection pools of the
55/// tools it serves (Bazel opens 100).
56const MAX_CONNECTIONS: usize = 256;
57
58/// What `bytes` of an entry or upload count for against the size limit.
59fn charge(bytes: u64) -> u64 {
60    bytes.max(MIN_CHARGE)
61}
62
63/// Whether a runner's job may write entries.
64#[derive(Debug, Clone, Copy, PartialEq, Eq)]
65pub enum Writes {
66    /// Not known yet: what it sends is held until it is.
67    Undecided,
68    /// GitHub says it runs a push, scheduled or manually started run of a branch that may write, or, where the
69    /// repository lets them, a pull request of its own branches that people opened.
70    Allowed,
71    /// Anything else: another pull request, another branch, or a job GitHub does not place on this runner.
72    Refused,
73}
74
75/// Whether a runner's socket serves GitHub Actions' cache besides the build cache's tools (ADR 0082).
76#[derive(Debug, Clone, Copy, PartialEq, Eq)]
77pub enum ActionsCache {
78    /// It does: the image points the job's `actions/cache`, `setup-*` caches and BuildKit's `type=gha` at it.
79    Served,
80    /// It does not: the job uses GitHub's cache service.
81    Off,
82}
83
84/// The scopes of a runner's GitHub Actions cache entries, as GitHub's: refs, `refs/heads/<branch>` or
85/// `refs/pull/<number>/merge`, from what GitHub said of its job (ADR 0082).
86#[derive(Debug, Clone, Default, PartialEq, Eq)]
87pub struct Scopes {
88    /// Its run's own ref, where its entries go and what it reads first when its writes are `Allowed`: its branch's
89    /// for a branch's run, its pull request's for a pull request's. `None` when it has none, as a run GitHub ties
90    /// to several pull requests.
91    pub own: Option<String>,
92    /// What it reads next when its writes are `Allowed`: the branch its pull request merges into.
93    pub base: Option<String>,
94    /// Its repository's default branch, whose entries every job of the repository reads last.
95    pub default: Option<String>,
96}
97
98/// The events whose runs may write: they run code that reached a branch, never a pull request's.
99const WRITING_EVENTS: [&str; 3] = ["push", "schedule", "workflow_dispatch"];
100
101/// The event of a pull request's run that `BuildCacheWriters::PullRequests` lets write. Never
102/// `pull_request_target`, which runs with the repository's secrets whatever the pull request's head is.
103pub const PULL_REQUEST_EVENT: &str = "pull_request";
104
105/// Whether `runner`'s job may write, from what GitHub says: the runner it placed the job on (`job_runner`, which
106/// a job cannot forge as it can its runner's log), and its run's trigger. A push, scheduled or manually started run
107/// of `repository` itself on one of `branches` may. With `writers` at `PullRequests`, so may a `pull_request` run of
108/// one of the repository's own branches that people started, once GitHub also says each of its pull requests was
109/// opened by one (`opened_by_user`). `Undecided` while GitHub places the job on no runner.
110pub fn writes(
111    runner: &str,
112    job_runner: Option<&str>,
113    repository: &str,
114    trigger: &RunTrigger,
115    branches: &[String],
116    writers: BuildCacheWriters,
117) -> Writes {
118    match job_runner {
119        None => Writes::Undecided,
120        Some(placed) if placed != runner => Writes::Refused,
121        Some(_) if !same_repository(trigger.head_repository.as_deref(), repository) => {
122            Writes::Refused
123        }
124        Some(_) => {
125            let Some(branch) = &trigger.head_branch else {
126                return Writes::Refused;
127            };
128            let event = trigger.event.as_str();
129            let branch_run = WRITING_EVENTS.contains(&event) && branches.contains(branch);
130            let pull_request = writers == BuildCacheWriters::PullRequests
131                && event == PULL_REQUEST_EVENT
132                && trigger.by_user
133                && !trigger.pull_requests.is_empty();
134            if branch_run || pull_request {
135                Writes::Allowed
136            } else {
137                Writes::Refused
138            }
139        }
140    }
141}
142
143/// Whether a pull request `trigger` runs lets it write: a person opened it, from `trigger`'s head branch in
144/// `repository` itself. A bot's, such as Renovate's or Dependabot's, only reads.
145pub fn opened_by_user(pull: &PullRequest, repository: &str, trigger: &RunTrigger) -> bool {
146    pull.by_user
147        && same_repository(pull.head_repository.as_deref(), repository)
148        && trigger.head_branch.as_ref() == Some(&pull.head_branch)
149}
150
151/// Whether GitHub's `head` names `repository`, as GitHub does whatever the case.
152fn same_repository(head: Option<&str>, repository: &str) -> bool {
153    head.is_some_and(|head| head.eq_ignore_ascii_case(repository))
154}
155
156/// The build cache's size now.
157#[derive(Debug, Clone, Copy, PartialEq, Eq)]
158pub struct Size {
159    /// The entries' bytes.
160    pub bytes: u64,
161    /// How many entries there are.
162    pub entries: u64,
163}
164
165/// What `trim` removed.
166#[derive(Debug, Clone, Copy, PartialEq, Eq)]
167pub struct Trimmed {
168    /// The size before.
169    pub before: Size,
170    /// The size after.
171    pub after: Size,
172}
173
174/// The shared build cache. Cloning shares it.
175#[derive(Debug, Clone)]
176pub struct BuildCache {
177    inner: Arc<Inner>,
178}
179
180#[derive(Debug)]
181struct Inner {
182    /// The entries, under `<owner>/<name>/<tool>/<key>`.
183    root: PathBuf,
184    /// The uploads not yet entries.
185    uploads: PathBuf,
186    /// One directory per runner, holding its socket.
187    sockets: PathBuf,
188    /// Where requests for GitHub's results service it does not answer go.
189    results: actions::Results,
190    /// Where turbo's requests with a workflow's own token go.
191    vercel: turbo::Vercel,
192    /// Holds block lists' joins where a test says.
193    #[cfg(test)]
194    join_gate: Gate,
195    // A panic midway leaves at worst an entry counted that is gone, or a file not counted, which `open` counts at
196    // the next start.
197    state: Mutex<State>,
198}
199
200#[derive(Debug, Default)]
201struct State {
202    max_size: u64,
203    entries: HashMap<PathBuf, Entry>,
204    /// The entries' bytes.
205    bytes: u64,
206    /// What the entries count for against `max_size` (`charge`).
207    charged: u64,
208    runners: HashMap<String, Runner>,
209    next_upload: u64,
210    /// What the uploads not yet entries, being received or held, count for (`charge`): at most `max_size`, so jobs
211    /// whose writes are undecided cannot fill the disk.
212    uploading: u64,
213    /// Each repository's default branch's ref, as the last decision said: what an undecided job's GitHub Actions
214    /// cache lookups read.
215    default_branches: HashMap<String, String>,
216    /// When the last GitHub Actions cache entry was created, in ms: each is created later than any before, those of
217    /// earlier runs included, so its creation time names it.
218    last_created: u64,
219    /// The GitHub Actions cache entries of `entries`, by the directory of their scope and version.
220    actions: HashMap<PathBuf, BTreeSet<PathBuf>>,
221}
222
223impl State {
224    /// Indexes `path` among the GitHub Actions cache entries, if it is one.
225    fn index(&mut self, path: &Path) {
226        if let Some(scope) = actions::scope_of(path) {
227            self.actions
228                .entry(scope)
229                .or_default()
230                .insert(path.to_owned());
231        }
232    }
233
234    /// Takes `path` out of the GitHub Actions cache entries' index.
235    fn unindex(&mut self, path: &Path) {
236        if let Some(scope) = actions::scope_of(path)
237            && let Some(paths) = self.actions.get_mut(&scope)
238        {
239            paths.remove(path);
240            if paths.is_empty() {
241                self.actions.remove(&scope);
242            }
243        }
244    }
245}
246
247#[derive(Debug, Clone, Copy)]
248struct Entry {
249    bytes: u64,
250    /// When it was last written or served, in ms: the least recently used go first.
251    used: u64,
252}
253
254#[derive(Debug)]
255struct Runner {
256    repository: String,
257    /// Whether its job may write, the uploads waiting for that, and its GitHub Actions cache scopes once it is known.
258    writes: decision::WriteDecision,
259    counts: BuildCacheCounts,
260    /// What its uploads not yet entries, being received or held, count for: at most a quarter of `max_size` while
261    /// its writes are undecided, so one job cannot take the room another's uploads need.
262    uploading: u64,
263    dir: PathBuf,
264    serving: JoinHandle<()>,
265    /// Whether it serves GitHub Actions' cache.
266    actions: ActionsCache,
267    /// Changes when its writes are decided, for the uploads it is receiving.
268    decided: watch::Sender<()>,
269    /// Its GitHub Actions cache saves not yet entries, by the number in their upload URL: being uploaded, or, while
270    /// its writes are undecided, finalized and waiting for them. At most `actions::MAX_SAVES` of both.
271    saves: HashMap<u64, actions::Save>,
272    /// The entries its GitHub Actions cache lookups found, by their creation time, the number in their download URL.
273    found: HashMap<u64, PathBuf>,
274}
275
276/// An upload written to a file of its own, not an entry yet.
277#[derive(Debug)]
278struct Upload {
279    file: PathBuf,
280    entry: PathBuf,
281    bytes: u64,
282    /// The uploads' room it took (`reserve`).
283    room: u64,
284}
285
286impl BuildCache {
287    /// Opens the build cache in `directory`, counting the entries an earlier run left, with each runner's socket
288    /// under `sockets`.
289    pub fn open(directory: &Path, sockets: &Path, max_size: u64) -> io::Result<BuildCache> {
290        std::fs::create_dir_all(directory)?;
291        let (root, uploads) = (directory.join(ENTRIES), directory.join(UPLOADS));
292        match std::fs::remove_dir_all(&uploads) {
293            Err(error) if error.kind() != io::ErrorKind::NotFound => return Err(error),
294            _ => {}
295        }
296        private_dir(&root)?;
297        private_dir(&uploads)?;
298        private_dir(sockets)?;
299        let mut entries = HashMap::new();
300        count_entries(&root, &root, &mut entries)?;
301        let bytes = entries.values().map(|entry: &Entry| entry.bytes).sum();
302        let charged = entries.values().map(|entry| charge(entry.bytes)).sum();
303        let mut state = State {
304            max_size,
305            bytes,
306            charged,
307            // An upload's number, in its URL, is one no earlier run gave, so a job's upload sent again after a restart
308            // reaches no save made since: the count passes the clock only at a million uploads a second.
309            next_upload: std::time::SystemTime::now()
310                .duration_since(std::time::UNIX_EPOCH)
311                .map_or(0, |since| u64::try_from(since.as_micros()).unwrap_or(0)),
312            ..State::default()
313        };
314        for path in entries.keys() {
315            state.index(path);
316            state.last_created = state
317                .last_created
318                .max(actions::created_of(path).unwrap_or(0));
319        }
320        state.entries = entries;
321        Ok(BuildCache {
322            inner: Arc::new(Inner {
323                root,
324                uploads,
325                sockets: sockets.to_owned(),
326                results: actions::Results::new(),
327                vercel: turbo::Vercel::new(),
328                #[cfg(test)]
329                join_gate: Gate::default(),
330                state: Mutex::new(state),
331            }),
332        })
333    }
334
335    /// Where the entries are: `entries` in the build cache's directory.
336    pub fn directory(&self) -> &Path {
337        &self.inner.root
338    }
339
340    /// Sends the requests for GitHub's results service it does not answer to `origin` instead of GitHub's.
341    #[cfg(test)]
342    pub(crate) fn set_results_origin(&self, origin: &str) {
343        self.inner.results.set_origin(origin);
344    }
345
346    /// Sends turbo's requests with a workflow's own token to `origin` instead of Vercel's API.
347    #[cfg(test)]
348    pub(crate) fn set_vercel_origin(&self, origin: &str) {
349        self.inner.vercel.set_origin(origin);
350    }
351
352    /// Sets the size `trim` brings it under.
353    pub fn set_max_size(&self, max_size: u64) {
354        lock(&self.inner.state).max_size = max_size;
355    }
356
357    /// Opens `runner`'s socket for a job of `repository` and returns the directory its container mounts at `MOUNT`,
358    /// with `ACTIONS_MARKER` in it when it serves GitHub Actions' cache too. A runner already open, as after a
359    /// restart, gets its socket back in the directory its container mounts.
360    pub fn attach(
361        &self,
362        runner: &str,
363        repository: &str,
364        actions: ActionsCache,
365    ) -> io::Result<PathBuf> {
366        if !valid_repository(repository) {
367            return Err(io::Error::new(
368                io::ErrorKind::InvalidInput,
369                format!("{repository} is not a repository name"),
370            ));
371        }
372        let dir = self.inner.sockets.join(socket_dir(runner));
373        std::fs::create_dir_all(&dir)?;
374        // The container's root, which forwards to it, is not the daemon's user under sysbox.
375        std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o755))?;
376        let marker = dir.join(ACTIONS_MARKER);
377        match actions {
378            ActionsCache::Served => {
379                std::fs::write(&marker, b"")?;
380                std::fs::set_permissions(&marker, std::fs::Permissions::from_mode(0o644))?;
381            }
382            ActionsCache::Off => match std::fs::remove_file(&marker) {
383                Err(error) if error.kind() != io::ErrorKind::NotFound => return Err(error),
384                _ => {}
385            },
386        }
387        let socket = dir.join(SOCKET);
388        match std::fs::remove_file(&socket) {
389            Err(error) if error.kind() != io::ErrorKind::NotFound => return Err(error),
390            _ => {}
391        }
392        let listener = UnixListener::bind(&socket)?;
393        std::fs::set_permissions(&socket, std::fs::Permissions::from_mode(0o666))?;
394        let serving = tokio::spawn(accept(self.inner.clone(), runner.to_owned(), listener));
395        let old = lock(&self.inner.state).runners.insert(
396            runner.to_owned(),
397            Runner {
398                repository: repository.to_owned(),
399                writes: decision::WriteDecision::undecided(),
400                counts: BuildCacheCounts::default(),
401                uploading: 0,
402                dir: dir.clone(),
403                serving,
404                actions,
405                decided: watch::Sender::new(()),
406                saves: HashMap::new(),
407                found: HashMap::new(),
408            },
409        );
410        if let Some(mut old) = old {
411            old.serving.abort();
412            self.inner.remove_uploads(None, old.writes.held());
413            self.inner.remove_saves(None, &mut old.saves);
414        }
415        Ok(dir)
416    }
417
418    /// Removes the socket directories of runners not in `keep`, which an earlier run left.
419    pub fn forget_others(&self, keep: &[&str]) {
420        let keep: Vec<String> = keep.iter().map(|runner| socket_dir(runner)).collect();
421        let Ok(dirs) = std::fs::read_dir(&self.inner.sockets) else {
422            return;
423        };
424        for dir in dirs.flatten() {
425            if !keep
426                .iter()
427                .any(|name| *name == dir.file_name().to_string_lossy())
428            {
429                let _ = std::fs::remove_dir_all(dir.path());
430            }
431        }
432    }
433
434    /// The runners open now.
435    #[cfg(test)]
436    pub(crate) fn attached(&self) -> Vec<String> {
437        lock(&self.inner.state).runners.keys().cloned().collect()
438    }
439
440    /// Whether `runner` is open and its writes undecided.
441    pub fn undecided(&self, runner: &str) -> bool {
442        lock(&self.inner.state)
443            .runners
444            .get(runner)
445            .is_some_and(|runner| runner.writes.is_undecided())
446    }
447
448    /// Decides whether `runner`'s job may write, once, with the scopes of its GitHub Actions cache entries: what it
449    /// sent meanwhile becomes entries or is dropped.
450    pub fn decide(&self, runner: &str, writes: Writes, scopes: Scopes) {
451        let (held, dropped) = {
452            let mut state = lock(&self.inner.state);
453            let Some(open) = state.runners.get_mut(runner) else {
454                return;
455            };
456            let default = scopes.default.clone();
457            let Some(held) = open.writes.decide(writes, scopes) else {
458                return;
459            };
460            let repository = open.repository.clone();
461            if let Some(default) = default {
462                state.default_branches.insert(repository, default);
463            }
464            // Under the same lock as the decision: no save is finalized between the two.
465            let dropped = actions::decided(&self.inner, &mut state, runner);
466            if let Some(open) = state.runners.get(runner) {
467                open.decided.send_replace(());
468            }
469            (held, dropped)
470        };
471        self.inner.remove_uploads(Some(runner), &dropped);
472        match writes {
473            Writes::Allowed => {
474                for upload in held {
475                    self.inner.store(runner, upload);
476                }
477            }
478            Writes::Refused | Writes::Undecided => {
479                if let Some(open) = lock(&self.inner.state).runners.get_mut(runner) {
480                    open.counts.refused += held.len() as u64;
481                }
482                self.inner.remove_uploads(Some(runner), &held);
483            }
484        }
485    }
486
487    /// Closes the socket of `runner` if its writes are undecided and it holds uploads or finalized GitHub Actions
488    /// cache saves, and says whether it did: those stay until `decide` and `detach`. Saves its job did not finalize,
489    /// which it no longer can, go now.
490    pub fn close(&self, runner: &str) -> bool {
491        let (dir, mut unfinished) = {
492            let mut state = lock(&self.inner.state);
493            let Some(open) = state.runners.get_mut(runner) else {
494                return false;
495            };
496            if !open.writes.is_undecided()
497                || open.writes.held().is_empty() && actions::finalized(&open.saves) == 0
498            {
499                return false;
500            }
501            open.serving.abort();
502            (open.dir.clone(), actions::unfinished(&mut open.saves))
503        };
504        self.inner.remove_saves(Some(runner), &mut unfinished);
505        if let Err(error) = std::fs::remove_dir_all(&dir)
506            && error.kind() != io::ErrorKind::NotFound
507        {
508            warn!("cannot remove {}: {error}", dir.display());
509        }
510        true
511    }
512
513    /// Closes `runner`'s socket and returns what it was served and stored, and whether it could write. What it sent
514    /// while undecided is dropped.
515    pub fn detach(&self, runner: &str) -> Option<BuildCacheCounts> {
516        let Some(mut open) = lock(&self.inner.state).runners.remove(runner) else {
517            // One a restart kept the directory of without opening it again, as with the cache turned off since.
518            let dir = self.inner.sockets.join(socket_dir(runner));
519            if let Err(error) = std::fs::remove_dir_all(&dir)
520                && error.kind() != io::ErrorKind::NotFound
521            {
522                warn!("cannot remove {}: {error}", dir.display());
523            }
524            return None;
525        };
526        open.serving.abort();
527        // Its finalized saves are left only while its writes are undecided: `decide` stores or drops them.
528        let dropped = open.writes.held().len() as u64 + actions::finalized(&open.saves);
529        if open.writes.is_undecided() && dropped > 0 {
530            warn!(
531                "{runner}: dropped {dropped} build cache uploads its job sent: GitHub never said which runner ran the job"
532            );
533        }
534        self.inner.remove_uploads(None, open.writes.held());
535        self.inner.remove_saves(None, &mut open.saves);
536        open.counts.refused += dropped;
537        open.counts.writer = open.writes.current() == Writes::Allowed;
538        if let Err(error) = std::fs::remove_dir_all(&open.dir)
539            && error.kind() != io::ErrorKind::NotFound
540        {
541            warn!("cannot remove {}: {error}", open.dir.display());
542        }
543        Some(open.counts)
544    }
545
546    /// The size of the entries now.
547    pub fn size(&self) -> Size {
548        let state = lock(&self.inner.state);
549        Size {
550            bytes: state.bytes,
551            entries: state.entries.len() as u64,
552        }
553    }
554
555    /// Removes the entries used least recently until the rest fit in the size limit, each counted at least
556    /// `MIN_CHARGE`, with the directories that leaves empty under a tool's. Blocks on the file system: call it off the
557    /// engine's loop, and only while no runner runs, so no job finds an entry gone midway.
558    pub fn trim(&self) -> Trimmed {
559        let limit = lock(&self.inner.state).max_size;
560        self.trim_to(limit)
561    }
562
563    /// Removes every entry; call it only while no runner runs, as `trim`.
564    pub fn clear(&self) -> Trimmed {
565        self.trim_to(0)
566    }
567
568    fn trim_to(&self, limit: u64) -> Trimmed {
569        let (before, victims) = {
570            let mut state = lock(&self.inner.state);
571            let before = Size {
572                bytes: state.bytes,
573                entries: state.entries.len() as u64,
574            };
575            let mut by_use: Vec<(PathBuf, Entry)> = state
576                .entries
577                .iter()
578                .map(|(path, entry)| (path.clone(), *entry))
579                .collect();
580            by_use.sort_by_key(|(path, entry)| (entry.used, path.clone()));
581            let mut victims = Vec::new();
582            for (path, entry) in by_use {
583                if state.charged <= limit && (limit > 0 || state.entries.is_empty()) {
584                    break;
585                }
586                state.entries.remove(&path);
587                state.unindex(&path);
588                state.bytes -= entry.bytes;
589                state.charged -= charge(entry.bytes);
590                victims.push(path);
591            }
592            (before, victims)
593        };
594        for path in victims {
595            // Under the lock, and only while the index still lacks it: an upload `store` renamed to the same path
596            // since is an entry again, and stays.
597            let state = lock(&self.inner.state);
598            if state.entries.contains_key(&path) {
599                continue;
600            }
601            match std::fs::remove_file(self.inner.root.join(&path)) {
602                Err(error) if error.kind() != io::ErrorKind::NotFound => {
603                    warn!(
604                        "cannot remove build cache entry {}: {error}",
605                        path.display()
606                    );
607                }
608                _ => self.inner.remove_empty_parents(&path),
609            }
610            drop(state);
611        }
612        Trimmed {
613            before,
614            after: self.size(),
615        }
616    }
617}
618
619impl Inner {
620    /// Makes an upload of `runner`'s, whose job may write, an entry; refuses it when the entries already take twice
621    /// the size limit, which only a busy hour between two trims reaches. Renames under the lock: quick, on the
622    /// daemon's own file system.
623    fn store(&self, runner: &str, upload: Upload) {
624        // One lock from the check to the index: two uploads of one entry renamed in one order and indexed in the
625        // other would leave the index counting the size of the file replaced.
626        self.store_locked(&mut lock(&self.state), runner, upload);
627    }
628
629    /// `store`, holding the lock; says whether the upload became an entry.
630    fn store_locked(&self, state: &mut State, runner: &str, upload: Upload) -> bool {
631        let replaced = state
632            .entries
633            .get(&upload.entry)
634            .map_or(0, |entry| charge(entry.bytes));
635        let fits =
636            state.charged - replaced + charge(upload.bytes) <= state.max_size.saturating_mul(2);
637        let path = self.root.join(&upload.entry);
638        let stored = fits
639            && path
640                .parent()
641                .is_none_or(|parent| std::fs::create_dir_all(parent).is_ok())
642            && std::fs::rename(&upload.file, &path).is_ok();
643        Self::give_back(state, Some(runner), upload.room);
644        if stored {
645            let entry = Entry {
646                bytes: upload.bytes,
647                used: crate::time::now_ms(),
648            };
649            state.index(&upload.entry);
650            if let Some(old) = state.entries.insert(upload.entry, entry) {
651                state.bytes -= old.bytes;
652                state.charged -= charge(old.bytes);
653            }
654            state.bytes += upload.bytes;
655            state.charged += charge(upload.bytes);
656        } else {
657            let _ = std::fs::remove_file(&upload.file);
658        }
659        if let Some(open) = state.runners.get_mut(runner) {
660            if stored {
661                open.counts.written_bytes += upload.bytes;
662            } else {
663                open.counts.refused += 1;
664            }
665        }
666        stored
667    }
668
669    /// Takes `bytes` more of the uploads' room for `runner`, unless they would take more than `max_size`, its writes
670    /// are refused, or, while they are undecided, its own would take more than a quarter of it. `max_size` is the one the upload started with: a
671    /// lower one a reload set since holds back the uploads that start after it.
672    fn reserve(&self, runner: &str, bytes: u64, max_size: u64) -> bool {
673        let mut state = lock(&self.state);
674        let total = state.uploading;
675        let Some(open) = state.runners.get_mut(runner) else {
676            return false;
677        };
678        let fits = total + bytes <= max_size
679            && match open.writes.current() {
680                Writes::Undecided => open.uploading + bytes <= max_size / 4,
681                Writes::Allowed => true,
682                Writes::Refused => false,
683            };
684        if fits {
685            open.uploading += bytes;
686            state.uploading += bytes;
687        }
688        fits
689    }
690
691    /// Gives back what `reserve` took for an upload of `runner`'s, if it is still open, that became an entry or was
692    /// removed.
693    fn release(&self, runner: Option<&str>, bytes: u64) {
694        Self::give_back(&mut lock(&self.state), runner, bytes);
695    }
696
697    fn give_back(state: &mut State, runner: Option<&str>, bytes: u64) {
698        state.uploading = state.uploading.saturating_sub(bytes);
699        if let Some(open) = runner.and_then(|runner| state.runners.get_mut(runner)) {
700            open.uploading = open.uploading.saturating_sub(bytes);
701        }
702    }
703
704    /// Removes uploads of `runner`'s, or of a runner no longer open, and gives back their room.
705    fn remove_uploads(&self, runner: Option<&str>, uploads: &[Upload]) {
706        for upload in uploads {
707            let _ = std::fs::remove_file(&upload.file);
708            self.release(runner, upload.room);
709        }
710    }
711
712    /// Removes the GitHub Actions cache saves of `runner`'s, or of a runner no longer open, that did not become
713    /// entries, with their blocks, and gives back their room.
714    fn remove_saves(&self, runner: Option<&str>, saves: &mut HashMap<u64, actions::Save>) {
715        for (_, save) in saves.drain() {
716            self.remove_uploads(runner, &save.into_uploads());
717        }
718    }
719
720    /// Removes the directories above the removed entry `entry` that it leaves empty, up to its tool's
721    /// (`<owner>/<name>/<tool>`). Call it holding the lock, as `store` makes them under it.
722    fn remove_empty_parents(&self, entry: &Path) {
723        let mut dir = entry.parent();
724        while let Some(parent) = dir
725            && parent.components().count() > 3
726            && std::fs::remove_dir(self.root.join(parent)).is_ok()
727        {
728            dir = parent.parent();
729        }
730    }
731
732    /// Marks an entry used now, in the index and, so the order survives a restart, in its file's time.
733    fn touch(&self, entry: &Path) {
734        if let Some(used) = lock(&self.state).entries.get_mut(entry) {
735            used.used = crate::time::now_ms();
736        }
737        let path = self.root.join(entry);
738        tokio::task::spawn_blocking(move || {
739            if let Ok(file) = std::fs::File::open(&path) {
740                let _ = file.set_modified(std::time::SystemTime::now());
741            }
742        });
743    }
744}
745
746/// Serves `runner`'s connections, at most `MAX_CONNECTIONS` at once, until it is detached, which aborts this task and
747/// with it every connection.
748async fn accept(inner: Arc<Inner>, runner: String, listener: UnixListener) {
749    let mut connections = JoinSet::new();
750    loop {
751        let full = connections.len() >= MAX_CONNECTIONS;
752        tokio::select! {
753            accepted = listener.accept(), if !full => match accepted {
754                Ok((stream, _)) => {
755                    connections.spawn(serve::connection(inner.clone(), runner.clone(), stream));
756                }
757                Err(error) => {
758                    warn!("{runner}: the build cache's socket failed: {error}");
759                    return;
760                }
761            },
762            Some(_) = connections.join_next() => {}
763        }
764    }
765}
766
767/// The directory of a runner's socket: a hash of its name, as a socket's path must stay under 108 bytes.
768fn socket_dir(runner: &str) -> String {
769    // FNV-1a, as `HostIdentity` hashes.
770    let hash = runner
771        .bytes()
772        .fold(0xcbf2_9ce4_8422_2325_u64, |hash, byte| {
773            (hash ^ u64::from(byte)).wrapping_mul(0x0000_0100_0000_01b3)
774        });
775    format!("{hash:016x}")
776}
777
778/// Whether `name` is a path segment of an entry: letters, digits, `.`, `_` and `-`, not `.` or `..`.
779fn valid_segment(name: &str) -> bool {
780    !name.is_empty()
781        && name.len() <= 200
782        && name != "."
783        && name != ".."
784        && name
785            .bytes()
786            .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-'))
787}
788
789/// Whether `repository` is `owner/name` with no other segment.
790fn valid_repository(repository: &str) -> bool {
791    repository.split_once('/').is_some_and(|(owner, name)| {
792        valid_segment(owner) && valid_segment(name) && !owner.starts_with('.')
793    })
794}
795
796fn private_dir(path: &Path) -> io::Result<()> {
797    std::fs::create_dir_all(path)?;
798    std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o700))
799}
800
801/// Counts the entries under `dir`, by their path under `root`, last used when last modified.
802fn count_entries(root: &Path, dir: &Path, entries: &mut HashMap<PathBuf, Entry>) -> io::Result<()> {
803    for item in std::fs::read_dir(dir)? {
804        let item = item?;
805        let path = item.path();
806        let metadata = item.metadata()?;
807        if metadata.is_dir() {
808            count_entries(root, &path, entries)?;
809        } else if metadata.is_file()
810            && let Ok(relative) = path.strip_prefix(root)
811        {
812            entries.insert(
813                relative.to_owned(),
814                Entry {
815                    bytes: metadata.len(),
816                    used: metadata.modified().map_or(0, crate::time::ms),
817                },
818            );
819        }
820    }
821    Ok(())
822}
823
824#[cfg(test)]
825mod tests;
826
827/// Where joins wait while a test holds them, to remove a runner midway.
828#[cfg(test)]
829#[derive(Debug, Default)]
830pub(crate) struct Gate {
831    /// Whether joins are held, and how many wait.
832    state: Mutex<(bool, usize)>,
833    changed: std::sync::Condvar,
834}
835
836#[cfg(test)]
837impl Gate {
838    pub(crate) fn hold(&self) {
839        lock(&self.state).0 = true;
840    }
841
842    pub(crate) fn release(&self) {
843        lock(&self.state).0 = false;
844        self.changed.notify_all();
845    }
846
847    pub(crate) fn waiting(&self) -> usize {
848        lock(&self.state).1
849    }
850
851    /// Waits while joins are held.
852    fn pass(&self) {
853        let mut state = lock(&self.state);
854        state.1 += 1;
855        while state.0 {
856            state = self
857                .changed
858                .wait(state)
859                .unwrap_or_else(std::sync::PoisonError::into_inner);
860        }
861        state.1 -= 1;
862    }
863}
864
865/// Sends one request to a runner's socket in `dir`, as a tool in its container would through the forwarder, and
866/// returns the response's status and body.
867#[cfg(test)]
868#[allow(
869    clippy::unwrap_used,
870    reason = "a test helper: a socket that does not answer fails the test"
871)]
872pub(crate) async fn request(dir: &Path, method: &str, path: &str, body: &[u8]) -> (u16, Vec<u8>) {
873    use tokio::io::{AsyncReadExt, AsyncWriteExt};
874    let mut stream = tokio::net::UnixStream::connect(dir.join(SOCKET))
875        .await
876        .unwrap();
877    let head = format!(
878        "{method} {path} HTTP/1.1\r\nHost: 127.0.0.1\r\nAuthorization: Basic dG9rZW46c2VjcmV0\r\n\
879         Content-Length: {}\r\nConnection: close\r\n\r\n",
880        body.len()
881    );
882    stream.write_all(head.as_bytes()).await.unwrap();
883    stream.write_all(body).await.unwrap();
884    let mut response = Vec::new();
885    stream.read_to_end(&mut response).await.unwrap();
886    let split = response
887        .windows(4)
888        .position(|window| window == b"\r\n\r\n")
889        .unwrap();
890    let status = String::from_utf8_lossy(&response[9..12]).parse().unwrap();
891    (status, response[split + 4..].to_vec())
892}