Skip to main content

runnerd_core/
buildcache.rs

1//! The build cache every runner shares (ADR 0052, 0053, 0083): the entries of Gradle's HTTP build cache, and of the
2//! other tools the runner image points at it, Turborepo's and Nx's among them, kept per repository in a directory of
3//! the daemon's and served to each runner on a Unix socket of its own, which the runner image forwards to
4//! `127.0.0.1`. Owns the entries, the size limit, the sockets, and what each runner was served and stored. Which
5//! socket a request came in on is who sent it; no job can claim another's. It never decides whether a job may write:
6//! the engine tells it (`Writes`), from what GitHub says of the job. It never logs a request's headers, which may
7//! carry a tool's token. Where the config turns it on, the same socket serves GitHub Actions' cache (`actions`, ADR
8//! 0082) from the same entries' directory and limits.
9
10pub(crate) mod actions;
11mod serve;
12mod turbo;
13
14use std::collections::{BTreeSet, HashMap};
15use std::io;
16use std::os::unix::fs::PermissionsExt;
17use std::path::{Path, PathBuf};
18use std::sync::{Arc, Mutex};
19
20use tokio::net::UnixListener;
21use tokio::sync::watch;
22use tokio::task::{JoinHandle, JoinSet};
23use tracing::warn;
24
25use crate::api::BuildCacheCounts;
26use crate::config::BuildCacheWriters;
27use crate::github::{PullRequest, RunTrigger};
28use crate::sync::lock;
29
30/// Where a runner container mounts its socket's directory, read-only (`image/entrypoint.sh`).
31pub const MOUNT: &str = "/run/runnerd-build-cache";
32
33/// The socket's name in that directory.
34pub const SOCKET: &str = "cache.sock";
35
36/// A file in that directory when the runner's socket serves GitHub Actions' cache too: the image forwards it for
37/// node actions and BuildKit only then (`image/build-cache.sh`).
38pub const ACTIONS_MARKER: &str = "actions";
39
40/// The directory of the entries, in the build cache's directory: nothing else there is counted or trimmed.
41const ENTRIES: &str = "entries";
42
43/// Where uploads are written before they become entries, in the build cache's directory; emptied at start.
44const UPLOADS: &str = "uploads";
45
46/// What an entry or upload counts for at least against the size limit, in bytes: its file's inode and directory
47/// entry cost the disk and the index something even when it is empty, so the limit also bounds how many there are.
48const MIN_CHARGE: u64 = 4 << 10;
49
50/// The uploads one runner may hold while its writes are undecided; past them, its uploads are refused.
51const MAX_HELD: usize = 10_000;
52
53/// The connections one runner's socket serves at once; more wait to be accepted. Above the connection pools of the
54/// tools it serves (Bazel opens 100).
55const MAX_CONNECTIONS: usize = 256;
56
57/// What `bytes` of an entry or upload count for against the size limit.
58fn charge(bytes: u64) -> u64 {
59    bytes.max(MIN_CHARGE)
60}
61
62/// Whether a runner's job may write entries.
63#[derive(Debug, Clone, Copy, PartialEq, Eq)]
64pub enum Writes {
65    /// Not known yet: what it sends is held until it is.
66    Undecided,
67    /// GitHub says it runs a push, scheduled or manually started run of a branch that may write, or, where the
68    /// repository lets them, a pull request of its own branches that people opened.
69    Allowed,
70    /// Anything else: another pull request, another branch, or a job GitHub does not place on this runner.
71    Refused,
72}
73
74/// Whether a runner's socket serves GitHub Actions' cache besides the build cache's tools (ADR 0082).
75#[derive(Debug, Clone, Copy, PartialEq, Eq)]
76pub enum ActionsCache {
77    /// It does: the image points the job's `actions/cache`, `setup-*` caches and BuildKit's `type=gha` at it.
78    Served,
79    /// It does not: the job uses GitHub's cache service.
80    Off,
81}
82
83/// The scopes of a runner's GitHub Actions cache entries, as GitHub's: refs, `refs/heads/<branch>` or
84/// `refs/pull/<number>/merge`, from what GitHub said of its job (ADR 0082).
85#[derive(Debug, Clone, Default, PartialEq, Eq)]
86pub struct Scopes {
87    /// Its run's own ref, where its entries go and what it reads first when its writes are `Allowed`: its branch's
88    /// for a branch's run, its pull request's for a pull request's. `None` when it has none, as a run GitHub ties
89    /// to several pull requests.
90    pub own: Option<String>,
91    /// What it reads next when its writes are `Allowed`: the branch its pull request merges into.
92    pub base: Option<String>,
93    /// Its repository's default branch, whose entries every job of the repository reads last.
94    pub default: Option<String>,
95}
96
97/// The events whose runs may write: they run code that reached a branch, never a pull request's.
98const WRITING_EVENTS: [&str; 3] = ["push", "schedule", "workflow_dispatch"];
99
100/// The event of a pull request's run that `BuildCacheWriters::PullRequests` lets write. Never
101/// `pull_request_target`, which runs with the repository's secrets whatever the pull request's head is.
102pub const PULL_REQUEST_EVENT: &str = "pull_request";
103
104/// Whether `runner`'s job may write, from what GitHub says: the runner it placed the job on (`job_runner`, which
105/// a job cannot forge as it can its runner's log), and its run's trigger. A push, scheduled or manually started run
106/// of `repository` itself on one of `branches` may. With `writers` at `PullRequests`, so may a `pull_request` run of
107/// one of the repository's own branches that people started, once GitHub also says each of its pull requests was
108/// opened by one (`opened_by_user`). `Undecided` while GitHub places the job on no runner.
109pub fn writes(
110    runner: &str,
111    job_runner: Option<&str>,
112    repository: &str,
113    trigger: &RunTrigger,
114    branches: &[String],
115    writers: BuildCacheWriters,
116) -> Writes {
117    match job_runner {
118        None => Writes::Undecided,
119        Some(placed) if placed != runner => Writes::Refused,
120        Some(_) if !same_repository(trigger.head_repository.as_deref(), repository) => {
121            Writes::Refused
122        }
123        Some(_) => {
124            let Some(branch) = &trigger.head_branch else {
125                return Writes::Refused;
126            };
127            let event = trigger.event.as_str();
128            let branch_run = WRITING_EVENTS.contains(&event) && branches.contains(branch);
129            let pull_request = writers == BuildCacheWriters::PullRequests
130                && event == PULL_REQUEST_EVENT
131                && trigger.by_user
132                && !trigger.pull_requests.is_empty();
133            if branch_run || pull_request {
134                Writes::Allowed
135            } else {
136                Writes::Refused
137            }
138        }
139    }
140}
141
142/// Whether a pull request `trigger` runs lets it write: a person opened it, from `trigger`'s head branch in
143/// `repository` itself. A bot's, such as Renovate's or Dependabot's, only reads.
144pub fn opened_by_user(pull: &PullRequest, repository: &str, trigger: &RunTrigger) -> bool {
145    pull.by_user
146        && same_repository(pull.head_repository.as_deref(), repository)
147        && trigger.head_branch.as_ref() == Some(&pull.head_branch)
148}
149
150/// Whether GitHub's `head` names `repository`, as GitHub does whatever the case.
151fn same_repository(head: Option<&str>, repository: &str) -> bool {
152    head.is_some_and(|head| head.eq_ignore_ascii_case(repository))
153}
154
155/// The build cache's size now.
156#[derive(Debug, Clone, Copy, PartialEq, Eq)]
157pub struct Size {
158    /// The entries' bytes.
159    pub bytes: u64,
160    /// How many entries there are.
161    pub entries: u64,
162}
163
164/// What `trim` removed.
165#[derive(Debug, Clone, Copy, PartialEq, Eq)]
166pub struct Trimmed {
167    /// The size before.
168    pub before: Size,
169    /// The size after.
170    pub after: Size,
171}
172
173/// The shared build cache. Cloning shares it.
174#[derive(Debug, Clone)]
175pub struct BuildCache {
176    inner: Arc<Inner>,
177}
178
179#[derive(Debug)]
180struct Inner {
181    /// The entries, under `<owner>/<name>/<tool>/<key>`.
182    root: PathBuf,
183    /// The uploads not yet entries.
184    uploads: PathBuf,
185    /// One directory per runner, holding its socket.
186    sockets: PathBuf,
187    /// Where requests for GitHub's results service it does not answer go.
188    results: actions::Results,
189    /// Where turbo's requests with a workflow's own token go.
190    vercel: turbo::Vercel,
191    /// Holds block lists' joins where a test says.
192    #[cfg(test)]
193    join_gate: Gate,
194    // A panic midway leaves at worst an entry counted that is gone, or a file not counted, which `open` counts at
195    // the next start.
196    state: Mutex<State>,
197}
198
199#[derive(Debug, Default)]
200struct State {
201    max_size: u64,
202    entries: HashMap<PathBuf, Entry>,
203    /// The entries' bytes.
204    bytes: u64,
205    /// What the entries count for against `max_size` (`charge`).
206    charged: u64,
207    runners: HashMap<String, Runner>,
208    next_upload: u64,
209    /// What the uploads not yet entries, being received or held, count for (`charge`): at most `max_size`, so jobs
210    /// whose writes are undecided cannot fill the disk.
211    uploading: u64,
212    /// Each repository's default branch's ref, as the last decision said: what an undecided job's GitHub Actions
213    /// cache lookups read.
214    default_branches: HashMap<String, String>,
215    /// When the last GitHub Actions cache entry was created, in ms: each is created later than any before, those of
216    /// earlier runs included, so its creation time names it.
217    last_created: u64,
218    /// The GitHub Actions cache entries of `entries`, by the directory of their scope and version.
219    actions: HashMap<PathBuf, BTreeSet<PathBuf>>,
220}
221
222impl State {
223    /// Indexes `path` among the GitHub Actions cache entries, if it is one.
224    fn index(&mut self, path: &Path) {
225        if let Some(scope) = actions::scope_of(path) {
226            self.actions
227                .entry(scope)
228                .or_default()
229                .insert(path.to_owned());
230        }
231    }
232
233    /// Takes `path` out of the GitHub Actions cache entries' index.
234    fn unindex(&mut self, path: &Path) {
235        if let Some(scope) = actions::scope_of(path)
236            && let Some(paths) = self.actions.get_mut(&scope)
237        {
238            paths.remove(path);
239            if paths.is_empty() {
240                self.actions.remove(&scope);
241            }
242        }
243    }
244}
245
246#[derive(Debug, Clone, Copy)]
247struct Entry {
248    bytes: u64,
249    /// When it was last written or served, in ms: the least recently used go first.
250    used: u64,
251}
252
253#[derive(Debug)]
254struct Runner {
255    repository: String,
256    writes: Writes,
257    counts: BuildCacheCounts,
258    /// Uploads waiting for `Writes` to be decided, at most `MAX_HELD`.
259    held: Vec<Upload>,
260    /// What its uploads not yet entries, being received or held, count for: at most a quarter of `max_size` while
261    /// its writes are undecided, so one job cannot take the room another's uploads need.
262    uploading: u64,
263    dir: PathBuf,
264    serving: JoinHandle<()>,
265    /// Whether it serves GitHub Actions' cache.
266    actions: ActionsCache,
267    /// Its GitHub Actions cache scopes, once its writes are decided.
268    scopes: Scopes,
269    /// Changes when its writes are decided, for the uploads it is receiving.
270    decided: watch::Sender<()>,
271    /// Its GitHub Actions cache saves not yet entries, by the number in their upload URL: being uploaded, or, while
272    /// its writes are undecided, finalized and waiting for them. At most `actions::MAX_SAVES` of both.
273    saves: HashMap<u64, actions::Save>,
274    /// The entries its GitHub Actions cache lookups found, by their creation time, the number in their download URL.
275    found: HashMap<u64, PathBuf>,
276}
277
278/// An upload written to a file of its own, not an entry yet.
279#[derive(Debug)]
280struct Upload {
281    file: PathBuf,
282    entry: PathBuf,
283    bytes: u64,
284    /// The uploads' room it took (`reserve`).
285    room: u64,
286}
287
288impl BuildCache {
289    /// Opens the build cache in `directory`, counting the entries an earlier run left, with each runner's socket
290    /// under `sockets`.
291    pub fn open(directory: &Path, sockets: &Path, max_size: u64) -> io::Result<BuildCache> {
292        std::fs::create_dir_all(directory)?;
293        let (root, uploads) = (directory.join(ENTRIES), directory.join(UPLOADS));
294        match std::fs::remove_dir_all(&uploads) {
295            Err(error) if error.kind() != io::ErrorKind::NotFound => return Err(error),
296            _ => {}
297        }
298        private_dir(&root)?;
299        private_dir(&uploads)?;
300        private_dir(sockets)?;
301        let mut entries = HashMap::new();
302        count_entries(&root, &root, &mut entries)?;
303        let bytes = entries.values().map(|entry: &Entry| entry.bytes).sum();
304        let charged = entries.values().map(|entry| charge(entry.bytes)).sum();
305        let mut state = State {
306            max_size,
307            bytes,
308            charged,
309            // An upload's number, in its URL, is one no earlier run gave, so a job's upload sent again after a restart
310            // reaches no save made since: the count passes the clock only at a million uploads a second.
311            next_upload: std::time::SystemTime::now()
312                .duration_since(std::time::UNIX_EPOCH)
313                .map_or(0, |since| u64::try_from(since.as_micros()).unwrap_or(0)),
314            ..State::default()
315        };
316        for path in entries.keys() {
317            state.index(path);
318            state.last_created = state
319                .last_created
320                .max(actions::created_of(path).unwrap_or(0));
321        }
322        state.entries = entries;
323        Ok(BuildCache {
324            inner: Arc::new(Inner {
325                root,
326                uploads,
327                sockets: sockets.to_owned(),
328                results: actions::Results::new(),
329                vercel: turbo::Vercel::new(),
330                #[cfg(test)]
331                join_gate: Gate::default(),
332                state: Mutex::new(state),
333            }),
334        })
335    }
336
337    /// Where the entries are: `entries` in the build cache's directory.
338    pub fn directory(&self) -> &Path {
339        &self.inner.root
340    }
341
342    /// Sends the requests for GitHub's results service it does not answer to `origin` instead of GitHub's.
343    #[cfg(test)]
344    pub(crate) fn set_results_origin(&self, origin: &str) {
345        self.inner.results.set_origin(origin);
346    }
347
348    /// Sends turbo's requests with a workflow's own token to `origin` instead of Vercel's API.
349    #[cfg(test)]
350    pub(crate) fn set_vercel_origin(&self, origin: &str) {
351        self.inner.vercel.set_origin(origin);
352    }
353
354    /// Sets the size `trim` brings it under.
355    pub fn set_max_size(&self, max_size: u64) {
356        lock(&self.inner.state).max_size = max_size;
357    }
358
359    /// Opens `runner`'s socket for a job of `repository` and returns the directory its container mounts at `MOUNT`,
360    /// with `ACTIONS_MARKER` in it when it serves GitHub Actions' cache too. A runner already open, as after a
361    /// restart, gets its socket back in the directory its container mounts.
362    pub fn attach(
363        &self,
364        runner: &str,
365        repository: &str,
366        actions: ActionsCache,
367    ) -> io::Result<PathBuf> {
368        if !valid_repository(repository) {
369            return Err(io::Error::new(
370                io::ErrorKind::InvalidInput,
371                format!("{repository} is not a repository name"),
372            ));
373        }
374        let dir = self.inner.sockets.join(socket_dir(runner));
375        std::fs::create_dir_all(&dir)?;
376        // The container's root, which forwards to it, is not the daemon's user under sysbox.
377        std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o755))?;
378        let marker = dir.join(ACTIONS_MARKER);
379        match actions {
380            ActionsCache::Served => {
381                std::fs::write(&marker, b"")?;
382                std::fs::set_permissions(&marker, std::fs::Permissions::from_mode(0o644))?;
383            }
384            ActionsCache::Off => match std::fs::remove_file(&marker) {
385                Err(error) if error.kind() != io::ErrorKind::NotFound => return Err(error),
386                _ => {}
387            },
388        }
389        let socket = dir.join(SOCKET);
390        match std::fs::remove_file(&socket) {
391            Err(error) if error.kind() != io::ErrorKind::NotFound => return Err(error),
392            _ => {}
393        }
394        let listener = UnixListener::bind(&socket)?;
395        std::fs::set_permissions(&socket, std::fs::Permissions::from_mode(0o666))?;
396        let serving = tokio::spawn(accept(self.inner.clone(), runner.to_owned(), listener));
397        let old = lock(&self.inner.state).runners.insert(
398            runner.to_owned(),
399            Runner {
400                repository: repository.to_owned(),
401                writes: Writes::Undecided,
402                counts: BuildCacheCounts::default(),
403                held: Vec::new(),
404                uploading: 0,
405                dir: dir.clone(),
406                serving,
407                actions,
408                scopes: Scopes::default(),
409                decided: watch::Sender::new(()),
410                saves: HashMap::new(),
411                found: HashMap::new(),
412            },
413        );
414        if let Some(mut old) = old {
415            old.serving.abort();
416            self.inner.remove_uploads(None, &old.held);
417            self.inner.remove_saves(None, &mut old.saves);
418        }
419        Ok(dir)
420    }
421
422    /// Removes the socket directories of runners not in `keep`, which an earlier run left.
423    pub fn forget_others(&self, keep: &[&str]) {
424        let keep: Vec<String> = keep.iter().map(|runner| socket_dir(runner)).collect();
425        let Ok(dirs) = std::fs::read_dir(&self.inner.sockets) else {
426            return;
427        };
428        for dir in dirs.flatten() {
429            if !keep
430                .iter()
431                .any(|name| *name == dir.file_name().to_string_lossy())
432            {
433                let _ = std::fs::remove_dir_all(dir.path());
434            }
435        }
436    }
437
438    /// Whether `runner` is open and its writes undecided.
439    pub fn undecided(&self, runner: &str) -> bool {
440        lock(&self.inner.state)
441            .runners
442            .get(runner)
443            .is_some_and(|runner| runner.writes == Writes::Undecided)
444    }
445
446    /// Decides whether `runner`'s job may write, once, with the scopes of its GitHub Actions cache entries: what it
447    /// sent meanwhile becomes entries or is dropped.
448    pub fn decide(&self, runner: &str, writes: Writes, scopes: Scopes) {
449        let (held, dropped) = {
450            let mut state = lock(&self.inner.state);
451            let Some(open) = state.runners.get_mut(runner) else {
452                return;
453            };
454            if open.writes != Writes::Undecided || writes == Writes::Undecided {
455                return;
456            }
457            open.writes = writes;
458            let held = std::mem::take(&mut open.held);
459            let repository = open.repository.clone();
460            if let Some(default) = &scopes.default {
461                state.default_branches.insert(repository, default.clone());
462            }
463            if let Some(open) = state.runners.get_mut(runner) {
464                open.scopes = scopes;
465            }
466            // Under the same lock as the decision: no save is finalized between the two.
467            let dropped = actions::decided(&self.inner, &mut state, runner);
468            if let Some(open) = state.runners.get(runner) {
469                open.decided.send_replace(());
470            }
471            (held, dropped)
472        };
473        self.inner.remove_uploads(Some(runner), &dropped);
474        match writes {
475            Writes::Allowed => {
476                for upload in held {
477                    self.inner.store(runner, upload);
478                }
479            }
480            Writes::Refused | Writes::Undecided => {
481                if let Some(open) = lock(&self.inner.state).runners.get_mut(runner) {
482                    open.counts.refused += held.len() as u64;
483                }
484                self.inner.remove_uploads(Some(runner), &held);
485            }
486        }
487    }
488
489    /// Closes the socket of `runner` if its writes are undecided and it holds uploads or finalized GitHub Actions
490    /// cache saves, and says whether it did: those stay until `decide` and `detach`. Saves its job did not finalize,
491    /// which it no longer can, go now.
492    pub fn close(&self, runner: &str) -> bool {
493        let (dir, mut unfinished) = {
494            let mut state = lock(&self.inner.state);
495            let Some(open) = state.runners.get_mut(runner) else {
496                return false;
497            };
498            if open.writes != Writes::Undecided
499                || open.held.is_empty() && actions::finalized(&open.saves) == 0
500            {
501                return false;
502            }
503            open.serving.abort();
504            (open.dir.clone(), actions::unfinished(&mut open.saves))
505        };
506        self.inner.remove_saves(Some(runner), &mut unfinished);
507        if let Err(error) = std::fs::remove_dir_all(&dir)
508            && error.kind() != io::ErrorKind::NotFound
509        {
510            warn!("cannot remove {}: {error}", dir.display());
511        }
512        true
513    }
514
515    /// Closes `runner`'s socket and returns what it was served and stored, and whether it could write. What it sent
516    /// while undecided is dropped.
517    pub fn detach(&self, runner: &str) -> Option<BuildCacheCounts> {
518        let Some(mut open) = lock(&self.inner.state).runners.remove(runner) else {
519            // One a restart kept the directory of without opening it again, as with the cache turned off since.
520            let dir = self.inner.sockets.join(socket_dir(runner));
521            if let Err(error) = std::fs::remove_dir_all(&dir)
522                && error.kind() != io::ErrorKind::NotFound
523            {
524                warn!("cannot remove {}: {error}", dir.display());
525            }
526            return None;
527        };
528        open.serving.abort();
529        // Its finalized saves are left only while its writes are undecided: `decide` stores or drops them.
530        let dropped = open.held.len() as u64 + actions::finalized(&open.saves);
531        if open.writes == Writes::Undecided && dropped > 0 {
532            warn!(
533                "{runner}: dropped {dropped} build cache uploads its job sent: GitHub never said which runner ran the job"
534            );
535        }
536        self.inner.remove_uploads(None, &open.held);
537        self.inner.remove_saves(None, &mut open.saves);
538        open.counts.refused += dropped;
539        open.counts.writer = open.writes == Writes::Allowed;
540        if let Err(error) = std::fs::remove_dir_all(&open.dir)
541            && error.kind() != io::ErrorKind::NotFound
542        {
543            warn!("cannot remove {}: {error}", open.dir.display());
544        }
545        Some(open.counts)
546    }
547
548    /// The size of the entries now.
549    pub fn size(&self) -> Size {
550        let state = lock(&self.inner.state);
551        Size {
552            bytes: state.bytes,
553            entries: state.entries.len() as u64,
554        }
555    }
556
557    /// Removes the entries used least recently until the rest fit in the size limit, each counted at least
558    /// `MIN_CHARGE`, with the directories that leaves empty under a tool's. Blocks on the file system: call it off the
559    /// engine's loop, and only while no runner runs, so no job finds an entry gone midway.
560    pub fn trim(&self) -> Trimmed {
561        let limit = lock(&self.inner.state).max_size;
562        self.trim_to(limit)
563    }
564
565    /// Removes every entry; call it only while no runner runs, as `trim`.
566    pub fn clear(&self) -> Trimmed {
567        self.trim_to(0)
568    }
569
570    fn trim_to(&self, limit: u64) -> Trimmed {
571        let (before, victims) = {
572            let mut state = lock(&self.inner.state);
573            let before = Size {
574                bytes: state.bytes,
575                entries: state.entries.len() as u64,
576            };
577            let mut by_use: Vec<(PathBuf, Entry)> = state
578                .entries
579                .iter()
580                .map(|(path, entry)| (path.clone(), *entry))
581                .collect();
582            by_use.sort_by_key(|(path, entry)| (entry.used, path.clone()));
583            let mut victims = Vec::new();
584            for (path, entry) in by_use {
585                if state.charged <= limit && (limit > 0 || state.entries.is_empty()) {
586                    break;
587                }
588                state.entries.remove(&path);
589                state.unindex(&path);
590                state.bytes -= entry.bytes;
591                state.charged -= charge(entry.bytes);
592                victims.push(path);
593            }
594            (before, victims)
595        };
596        for path in victims {
597            // Under the lock, and only while the index still lacks it: an upload `store` renamed to the same path
598            // since is an entry again, and stays.
599            let state = lock(&self.inner.state);
600            if state.entries.contains_key(&path) {
601                continue;
602            }
603            match std::fs::remove_file(self.inner.root.join(&path)) {
604                Err(error) if error.kind() != io::ErrorKind::NotFound => {
605                    warn!(
606                        "cannot remove build cache entry {}: {error}",
607                        path.display()
608                    );
609                }
610                _ => self.inner.remove_empty_parents(&path),
611            }
612            drop(state);
613        }
614        Trimmed {
615            before,
616            after: self.size(),
617        }
618    }
619}
620
621impl Inner {
622    /// Makes an upload of `runner`'s, whose job may write, an entry; refuses it when the entries already take twice
623    /// the size limit, which only a busy hour between two trims reaches. Renames under the lock: quick, on the
624    /// daemon's own file system.
625    fn store(&self, runner: &str, upload: Upload) {
626        // One lock from the check to the index: two uploads of one entry renamed in one order and indexed in the
627        // other would leave the index counting the size of the file replaced.
628        self.store_locked(&mut lock(&self.state), runner, upload);
629    }
630
631    /// `store`, holding the lock; says whether the upload became an entry.
632    fn store_locked(&self, state: &mut State, runner: &str, upload: Upload) -> bool {
633        let replaced = state
634            .entries
635            .get(&upload.entry)
636            .map_or(0, |entry| charge(entry.bytes));
637        let fits =
638            state.charged - replaced + charge(upload.bytes) <= state.max_size.saturating_mul(2);
639        let path = self.root.join(&upload.entry);
640        let stored = fits
641            && path
642                .parent()
643                .is_none_or(|parent| std::fs::create_dir_all(parent).is_ok())
644            && std::fs::rename(&upload.file, &path).is_ok();
645        Self::give_back(state, Some(runner), upload.room);
646        if stored {
647            let entry = Entry {
648                bytes: upload.bytes,
649                used: crate::time::now_ms(),
650            };
651            state.index(&upload.entry);
652            if let Some(old) = state.entries.insert(upload.entry, entry) {
653                state.bytes -= old.bytes;
654                state.charged -= charge(old.bytes);
655            }
656            state.bytes += upload.bytes;
657            state.charged += charge(upload.bytes);
658        } else {
659            let _ = std::fs::remove_file(&upload.file);
660        }
661        if let Some(open) = state.runners.get_mut(runner) {
662            if stored {
663                open.counts.written_bytes += upload.bytes;
664            } else {
665                open.counts.refused += 1;
666            }
667        }
668        stored
669    }
670
671    /// Takes `bytes` more of the uploads' room for `runner`, unless they would take more than `max_size`, its writes
672    /// are refused, or, while they are undecided, its own would take more than a quarter of it. `max_size` is the one the upload started with: a
673    /// lower one a reload set since holds back the uploads that start after it.
674    fn reserve(&self, runner: &str, bytes: u64, max_size: u64) -> bool {
675        let mut state = lock(&self.state);
676        let total = state.uploading;
677        let Some(open) = state.runners.get_mut(runner) else {
678            return false;
679        };
680        let fits = total + bytes <= max_size
681            && match open.writes {
682                Writes::Undecided => open.uploading + bytes <= max_size / 4,
683                Writes::Allowed => true,
684                Writes::Refused => false,
685            };
686        if fits {
687            open.uploading += bytes;
688            state.uploading += bytes;
689        }
690        fits
691    }
692
693    /// Gives back what `reserve` took for an upload of `runner`'s, if it is still open, that became an entry or was
694    /// removed.
695    fn release(&self, runner: Option<&str>, bytes: u64) {
696        Self::give_back(&mut lock(&self.state), runner, bytes);
697    }
698
699    fn give_back(state: &mut State, runner: Option<&str>, bytes: u64) {
700        state.uploading = state.uploading.saturating_sub(bytes);
701        if let Some(open) = runner.and_then(|runner| state.runners.get_mut(runner)) {
702            open.uploading = open.uploading.saturating_sub(bytes);
703        }
704    }
705
706    /// Removes uploads of `runner`'s, or of a runner no longer open, and gives back their room.
707    fn remove_uploads(&self, runner: Option<&str>, uploads: &[Upload]) {
708        for upload in uploads {
709            let _ = std::fs::remove_file(&upload.file);
710            self.release(runner, upload.room);
711        }
712    }
713
714    /// Removes the GitHub Actions cache saves of `runner`'s, or of a runner no longer open, that did not become
715    /// entries, with their blocks, and gives back their room.
716    fn remove_saves(&self, runner: Option<&str>, saves: &mut HashMap<u64, actions::Save>) {
717        for (_, save) in saves.drain() {
718            self.remove_uploads(runner, &save.into_uploads());
719        }
720    }
721
722    /// Removes the directories above the removed entry `entry` that it leaves empty, up to its tool's
723    /// (`<owner>/<name>/<tool>`). Call it holding the lock, as `store` makes them under it.
724    fn remove_empty_parents(&self, entry: &Path) {
725        let mut dir = entry.parent();
726        while let Some(parent) = dir
727            && parent.components().count() > 3
728            && std::fs::remove_dir(self.root.join(parent)).is_ok()
729        {
730            dir = parent.parent();
731        }
732    }
733
734    /// Marks an entry used now, in the index and, so the order survives a restart, in its file's time.
735    fn touch(&self, entry: &Path) {
736        if let Some(used) = lock(&self.state).entries.get_mut(entry) {
737            used.used = crate::time::now_ms();
738        }
739        let path = self.root.join(entry);
740        tokio::task::spawn_blocking(move || {
741            if let Ok(file) = std::fs::File::open(&path) {
742                let _ = file.set_modified(std::time::SystemTime::now());
743            }
744        });
745    }
746}
747
748/// Serves `runner`'s connections, at most `MAX_CONNECTIONS` at once, until it is detached, which aborts this task and
749/// with it every connection.
750async fn accept(inner: Arc<Inner>, runner: String, listener: UnixListener) {
751    let mut connections = JoinSet::new();
752    loop {
753        let full = connections.len() >= MAX_CONNECTIONS;
754        tokio::select! {
755            accepted = listener.accept(), if !full => match accepted {
756                Ok((stream, _)) => {
757                    connections.spawn(serve::connection(inner.clone(), runner.clone(), stream));
758                }
759                Err(error) => {
760                    warn!("{runner}: the build cache's socket failed: {error}");
761                    return;
762                }
763            },
764            Some(_) = connections.join_next() => {}
765        }
766    }
767}
768
769/// The directory of a runner's socket: a hash of its name, as a socket's path must stay under 108 bytes.
770fn socket_dir(runner: &str) -> String {
771    // FNV-1a, as `HostIdentity` hashes.
772    let hash = runner
773        .bytes()
774        .fold(0xcbf2_9ce4_8422_2325_u64, |hash, byte| {
775            (hash ^ u64::from(byte)).wrapping_mul(0x0000_0100_0000_01b3)
776        });
777    format!("{hash:016x}")
778}
779
780/// Whether `name` is a path segment of an entry: letters, digits, `.`, `_` and `-`, not `.` or `..`.
781fn valid_segment(name: &str) -> bool {
782    !name.is_empty()
783        && name.len() <= 200
784        && name != "."
785        && name != ".."
786        && name
787            .bytes()
788            .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-'))
789}
790
791/// Whether `repository` is `owner/name` with no other segment.
792fn valid_repository(repository: &str) -> bool {
793    repository.split_once('/').is_some_and(|(owner, name)| {
794        valid_segment(owner) && valid_segment(name) && !owner.starts_with('.')
795    })
796}
797
798fn private_dir(path: &Path) -> io::Result<()> {
799    std::fs::create_dir_all(path)?;
800    std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o700))
801}
802
803/// Counts the entries under `dir`, by their path under `root`, last used when last modified.
804fn count_entries(root: &Path, dir: &Path, entries: &mut HashMap<PathBuf, Entry>) -> io::Result<()> {
805    for item in std::fs::read_dir(dir)? {
806        let item = item?;
807        let path = item.path();
808        let metadata = item.metadata()?;
809        if metadata.is_dir() {
810            count_entries(root, &path, entries)?;
811        } else if metadata.is_file()
812            && let Ok(relative) = path.strip_prefix(root)
813        {
814            entries.insert(
815                relative.to_owned(),
816                Entry {
817                    bytes: metadata.len(),
818                    used: metadata.modified().map_or(0, crate::time::ms),
819                },
820            );
821        }
822    }
823    Ok(())
824}
825
826#[cfg(test)]
827mod tests;
828
829/// Where joins wait while a test holds them, to remove a runner midway.
830#[cfg(test)]
831#[derive(Debug, Default)]
832pub(crate) struct Gate {
833    /// Whether joins are held, and how many wait.
834    state: Mutex<(bool, usize)>,
835    changed: std::sync::Condvar,
836}
837
838#[cfg(test)]
839impl Gate {
840    pub(crate) fn hold(&self) {
841        lock(&self.state).0 = true;
842    }
843
844    pub(crate) fn release(&self) {
845        lock(&self.state).0 = false;
846        self.changed.notify_all();
847    }
848
849    pub(crate) fn waiting(&self) -> usize {
850        lock(&self.state).1
851    }
852
853    /// Waits while joins are held.
854    fn pass(&self) {
855        let mut state = lock(&self.state);
856        state.1 += 1;
857        while state.0 {
858            state = self
859                .changed
860                .wait(state)
861                .unwrap_or_else(std::sync::PoisonError::into_inner);
862        }
863        state.1 -= 1;
864    }
865}
866
867/// Sends one request to a runner's socket in `dir`, as a tool in its container would through the forwarder, and
868/// returns the response's status and body.
869#[cfg(test)]
870#[allow(
871    clippy::unwrap_used,
872    reason = "a test helper: a socket that does not answer fails the test"
873)]
874pub(crate) async fn request(dir: &Path, method: &str, path: &str, body: &[u8]) -> (u16, Vec<u8>) {
875    use tokio::io::{AsyncReadExt, AsyncWriteExt};
876    let mut stream = tokio::net::UnixStream::connect(dir.join(SOCKET))
877        .await
878        .unwrap();
879    let head = format!(
880        "{method} {path} HTTP/1.1\r\nHost: 127.0.0.1\r\nAuthorization: Basic dG9rZW46c2VjcmV0\r\n\
881         Content-Length: {}\r\nConnection: close\r\n\r\n",
882        body.len()
883    );
884    stream.write_all(head.as_bytes()).await.unwrap();
885    stream.write_all(body).await.unwrap();
886    let mut response = Vec::new();
887    stream.read_to_end(&mut response).await.unwrap();
888    let split = response
889        .windows(4)
890        .position(|window| window == b"\r\n\r\n")
891        .unwrap();
892    let status = String::from_utf8_lossy(&response[9..12]).parse().unwrap();
893    (status, response[split + 4..].to_vec())
894}