1pub(crate) mod actions;
11mod decision;
12mod serve;
13mod turbo;
14
15use std::collections::{BTreeSet, HashMap};
16use std::io;
17use std::os::unix::fs::PermissionsExt;
18use std::path::{Path, PathBuf};
19use std::sync::{Arc, Mutex};
20
21use tokio::net::UnixListener;
22use tokio::sync::watch;
23use tokio::task::{JoinHandle, JoinSet};
24use tracing::warn;
25
26use crate::api::BuildCacheCounts;
27use crate::config::BuildCacheWriters;
28use crate::github::{PullRequest, RunTrigger};
29use crate::sync::lock;
30
31pub const MOUNT: &str = "/run/runnerd-build-cache";
33
34pub const SOCKET: &str = "cache.sock";
36
37pub const ACTIONS_MARKER: &str = "actions";
40
41const ENTRIES: &str = "entries";
43
44const UPLOADS: &str = "uploads";
46
47const MIN_CHARGE: u64 = 4 << 10;
50
51const MAX_HELD: usize = 10_000;
53
54const MAX_CONNECTIONS: usize = 256;
57
58fn charge(bytes: u64) -> u64 {
60 bytes.max(MIN_CHARGE)
61}
62
63#[derive(Debug, Clone, Copy, PartialEq, Eq)]
65pub enum Writes {
66 Undecided,
68 Allowed,
71 Refused,
73}
74
75#[derive(Debug, Clone, Copy, PartialEq, Eq)]
77pub enum ActionsCache {
78 Served,
80 Off,
82}
83
84#[derive(Debug, Clone, Default, PartialEq, Eq)]
87pub struct Scopes {
88 pub own: Option<String>,
92 pub base: Option<String>,
94 pub default: Option<String>,
96}
97
98const WRITING_EVENTS: [&str; 3] = ["push", "schedule", "workflow_dispatch"];
100
101pub const PULL_REQUEST_EVENT: &str = "pull_request";
104
105pub fn writes(
111 runner: &str,
112 job_runner: Option<&str>,
113 repository: &str,
114 trigger: &RunTrigger,
115 branches: &[String],
116 writers: BuildCacheWriters,
117) -> Writes {
118 match job_runner {
119 None => Writes::Undecided,
120 Some(placed) if placed != runner => Writes::Refused,
121 Some(_) if !same_repository(trigger.head_repository.as_deref(), repository) => {
122 Writes::Refused
123 }
124 Some(_) => {
125 let Some(branch) = &trigger.head_branch else {
126 return Writes::Refused;
127 };
128 let event = trigger.event.as_str();
129 let branch_run = WRITING_EVENTS.contains(&event) && branches.contains(branch);
130 let pull_request = writers == BuildCacheWriters::PullRequests
131 && event == PULL_REQUEST_EVENT
132 && trigger.by_user
133 && !trigger.pull_requests.is_empty();
134 if branch_run || pull_request {
135 Writes::Allowed
136 } else {
137 Writes::Refused
138 }
139 }
140 }
141}
142
143pub fn opened_by_user(pull: &PullRequest, repository: &str, trigger: &RunTrigger) -> bool {
146 pull.by_user
147 && same_repository(pull.head_repository.as_deref(), repository)
148 && trigger.head_branch.as_ref() == Some(&pull.head_branch)
149}
150
151fn same_repository(head: Option<&str>, repository: &str) -> bool {
153 head.is_some_and(|head| head.eq_ignore_ascii_case(repository))
154}
155
156#[derive(Debug, Clone, Copy, PartialEq, Eq)]
158pub struct Size {
159 pub bytes: u64,
161 pub entries: u64,
163}
164
165#[derive(Debug, Clone, Copy, PartialEq, Eq)]
167pub struct Trimmed {
168 pub before: Size,
170 pub after: Size,
172}
173
174#[derive(Debug, Clone)]
176pub struct BuildCache {
177 inner: Arc<Inner>,
178}
179
180#[derive(Debug)]
181struct Inner {
182 root: PathBuf,
184 uploads: PathBuf,
186 sockets: PathBuf,
188 results: actions::Results,
190 vercel: turbo::Vercel,
192 #[cfg(test)]
194 join_gate: Gate,
195 state: Mutex<State>,
198}
199
200#[derive(Debug, Default)]
201struct State {
202 max_size: u64,
203 entries: HashMap<PathBuf, Entry>,
204 bytes: u64,
206 charged: u64,
208 runners: HashMap<String, Runner>,
209 next_upload: u64,
210 uploading: u64,
213 default_branches: HashMap<String, String>,
216 last_created: u64,
219 actions: HashMap<PathBuf, BTreeSet<PathBuf>>,
221}
222
223impl State {
224 fn index(&mut self, path: &Path) {
226 if let Some(scope) = actions::scope_of(path) {
227 self.actions
228 .entry(scope)
229 .or_default()
230 .insert(path.to_owned());
231 }
232 }
233
234 fn unindex(&mut self, path: &Path) {
236 if let Some(scope) = actions::scope_of(path)
237 && let Some(paths) = self.actions.get_mut(&scope)
238 {
239 paths.remove(path);
240 if paths.is_empty() {
241 self.actions.remove(&scope);
242 }
243 }
244 }
245}
246
247#[derive(Debug, Clone, Copy)]
248struct Entry {
249 bytes: u64,
250 used: u64,
252}
253
254#[derive(Debug)]
255struct Runner {
256 repository: String,
257 writes: decision::WriteDecision,
259 counts: BuildCacheCounts,
260 uploading: u64,
263 dir: PathBuf,
264 serving: JoinHandle<()>,
265 actions: ActionsCache,
267 decided: watch::Sender<()>,
269 saves: HashMap<u64, actions::Save>,
272 found: HashMap<u64, PathBuf>,
274}
275
276#[derive(Debug)]
278struct Upload {
279 file: PathBuf,
280 entry: PathBuf,
281 bytes: u64,
282 room: u64,
284}
285
286impl BuildCache {
287 pub fn open(directory: &Path, sockets: &Path, max_size: u64) -> io::Result<BuildCache> {
290 std::fs::create_dir_all(directory)?;
291 let (root, uploads) = (directory.join(ENTRIES), directory.join(UPLOADS));
292 match std::fs::remove_dir_all(&uploads) {
293 Err(error) if error.kind() != io::ErrorKind::NotFound => return Err(error),
294 _ => {}
295 }
296 private_dir(&root)?;
297 private_dir(&uploads)?;
298 private_dir(sockets)?;
299 let mut entries = HashMap::new();
300 count_entries(&root, &root, &mut entries)?;
301 let bytes = entries.values().map(|entry: &Entry| entry.bytes).sum();
302 let charged = entries.values().map(|entry| charge(entry.bytes)).sum();
303 let mut state = State {
304 max_size,
305 bytes,
306 charged,
307 next_upload: std::time::SystemTime::now()
310 .duration_since(std::time::UNIX_EPOCH)
311 .map_or(0, |since| u64::try_from(since.as_micros()).unwrap_or(0)),
312 ..State::default()
313 };
314 for path in entries.keys() {
315 state.index(path);
316 state.last_created = state
317 .last_created
318 .max(actions::created_of(path).unwrap_or(0));
319 }
320 state.entries = entries;
321 Ok(BuildCache {
322 inner: Arc::new(Inner {
323 root,
324 uploads,
325 sockets: sockets.to_owned(),
326 results: actions::Results::new(),
327 vercel: turbo::Vercel::new(),
328 #[cfg(test)]
329 join_gate: Gate::default(),
330 state: Mutex::new(state),
331 }),
332 })
333 }
334
335 pub fn directory(&self) -> &Path {
337 &self.inner.root
338 }
339
340 #[cfg(test)]
342 pub(crate) fn set_results_origin(&self, origin: &str) {
343 self.inner.results.set_origin(origin);
344 }
345
346 #[cfg(test)]
348 pub(crate) fn set_vercel_origin(&self, origin: &str) {
349 self.inner.vercel.set_origin(origin);
350 }
351
352 pub fn set_max_size(&self, max_size: u64) {
354 lock(&self.inner.state).max_size = max_size;
355 }
356
357 pub fn attach(
361 &self,
362 runner: &str,
363 repository: &str,
364 actions: ActionsCache,
365 ) -> io::Result<PathBuf> {
366 if !valid_repository(repository) {
367 return Err(io::Error::new(
368 io::ErrorKind::InvalidInput,
369 format!("{repository} is not a repository name"),
370 ));
371 }
372 let dir = self.inner.sockets.join(socket_dir(runner));
373 std::fs::create_dir_all(&dir)?;
374 std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o755))?;
376 let marker = dir.join(ACTIONS_MARKER);
377 match actions {
378 ActionsCache::Served => {
379 std::fs::write(&marker, b"")?;
380 std::fs::set_permissions(&marker, std::fs::Permissions::from_mode(0o644))?;
381 }
382 ActionsCache::Off => match std::fs::remove_file(&marker) {
383 Err(error) if error.kind() != io::ErrorKind::NotFound => return Err(error),
384 _ => {}
385 },
386 }
387 let socket = dir.join(SOCKET);
388 match std::fs::remove_file(&socket) {
389 Err(error) if error.kind() != io::ErrorKind::NotFound => return Err(error),
390 _ => {}
391 }
392 let listener = UnixListener::bind(&socket)?;
393 std::fs::set_permissions(&socket, std::fs::Permissions::from_mode(0o666))?;
394 let serving = tokio::spawn(accept(self.inner.clone(), runner.to_owned(), listener));
395 let old = lock(&self.inner.state).runners.insert(
396 runner.to_owned(),
397 Runner {
398 repository: repository.to_owned(),
399 writes: decision::WriteDecision::undecided(),
400 counts: BuildCacheCounts::default(),
401 uploading: 0,
402 dir: dir.clone(),
403 serving,
404 actions,
405 decided: watch::Sender::new(()),
406 saves: HashMap::new(),
407 found: HashMap::new(),
408 },
409 );
410 if let Some(mut old) = old {
411 old.serving.abort();
412 self.inner.remove_uploads(None, old.writes.held());
413 self.inner.remove_saves(None, &mut old.saves);
414 }
415 Ok(dir)
416 }
417
418 pub fn forget_others(&self, keep: &[&str]) {
420 let keep: Vec<String> = keep.iter().map(|runner| socket_dir(runner)).collect();
421 let Ok(dirs) = std::fs::read_dir(&self.inner.sockets) else {
422 return;
423 };
424 for dir in dirs.flatten() {
425 if !keep
426 .iter()
427 .any(|name| *name == dir.file_name().to_string_lossy())
428 {
429 let _ = std::fs::remove_dir_all(dir.path());
430 }
431 }
432 }
433
434 #[cfg(test)]
436 pub(crate) fn attached(&self) -> Vec<String> {
437 lock(&self.inner.state).runners.keys().cloned().collect()
438 }
439
440 pub fn undecided(&self, runner: &str) -> bool {
442 lock(&self.inner.state)
443 .runners
444 .get(runner)
445 .is_some_and(|runner| runner.writes.is_undecided())
446 }
447
448 pub fn decide(&self, runner: &str, writes: Writes, scopes: Scopes) {
451 let (held, dropped) = {
452 let mut state = lock(&self.inner.state);
453 let Some(open) = state.runners.get_mut(runner) else {
454 return;
455 };
456 let default = scopes.default.clone();
457 let Some(held) = open.writes.decide(writes, scopes) else {
458 return;
459 };
460 let repository = open.repository.clone();
461 if let Some(default) = default {
462 state.default_branches.insert(repository, default);
463 }
464 let dropped = actions::decided(&self.inner, &mut state, runner);
466 if let Some(open) = state.runners.get(runner) {
467 open.decided.send_replace(());
468 }
469 (held, dropped)
470 };
471 self.inner.remove_uploads(Some(runner), &dropped);
472 match writes {
473 Writes::Allowed => {
474 for upload in held {
475 self.inner.store(runner, upload);
476 }
477 }
478 Writes::Refused | Writes::Undecided => {
479 if let Some(open) = lock(&self.inner.state).runners.get_mut(runner) {
480 open.counts.refused += held.len() as u64;
481 }
482 self.inner.remove_uploads(Some(runner), &held);
483 }
484 }
485 }
486
487 pub fn close(&self, runner: &str) -> bool {
491 let (dir, mut unfinished) = {
492 let mut state = lock(&self.inner.state);
493 let Some(open) = state.runners.get_mut(runner) else {
494 return false;
495 };
496 if !open.writes.is_undecided()
497 || open.writes.held().is_empty() && actions::finalized(&open.saves) == 0
498 {
499 return false;
500 }
501 open.serving.abort();
502 (open.dir.clone(), actions::unfinished(&mut open.saves))
503 };
504 self.inner.remove_saves(Some(runner), &mut unfinished);
505 if let Err(error) = std::fs::remove_dir_all(&dir)
506 && error.kind() != io::ErrorKind::NotFound
507 {
508 warn!("cannot remove {}: {error}", dir.display());
509 }
510 true
511 }
512
513 pub fn detach(&self, runner: &str) -> Option<BuildCacheCounts> {
516 let Some(mut open) = lock(&self.inner.state).runners.remove(runner) else {
517 let dir = self.inner.sockets.join(socket_dir(runner));
519 if let Err(error) = std::fs::remove_dir_all(&dir)
520 && error.kind() != io::ErrorKind::NotFound
521 {
522 warn!("cannot remove {}: {error}", dir.display());
523 }
524 return None;
525 };
526 open.serving.abort();
527 let dropped = open.writes.held().len() as u64 + actions::finalized(&open.saves);
529 if open.writes.is_undecided() && dropped > 0 {
530 warn!(
531 "{runner}: dropped {dropped} build cache uploads its job sent: GitHub never said which runner ran the job"
532 );
533 }
534 self.inner.remove_uploads(None, open.writes.held());
535 self.inner.remove_saves(None, &mut open.saves);
536 open.counts.refused += dropped;
537 open.counts.writer = open.writes.current() == Writes::Allowed;
538 if let Err(error) = std::fs::remove_dir_all(&open.dir)
539 && error.kind() != io::ErrorKind::NotFound
540 {
541 warn!("cannot remove {}: {error}", open.dir.display());
542 }
543 Some(open.counts)
544 }
545
546 pub fn size(&self) -> Size {
548 let state = lock(&self.inner.state);
549 Size {
550 bytes: state.bytes,
551 entries: state.entries.len() as u64,
552 }
553 }
554
555 pub fn trim(&self) -> Trimmed {
559 let limit = lock(&self.inner.state).max_size;
560 self.trim_to(limit)
561 }
562
563 pub fn clear(&self) -> Trimmed {
565 self.trim_to(0)
566 }
567
568 fn trim_to(&self, limit: u64) -> Trimmed {
569 let (before, victims) = {
570 let mut state = lock(&self.inner.state);
571 let before = Size {
572 bytes: state.bytes,
573 entries: state.entries.len() as u64,
574 };
575 let mut by_use: Vec<(PathBuf, Entry)> = state
576 .entries
577 .iter()
578 .map(|(path, entry)| (path.clone(), *entry))
579 .collect();
580 by_use.sort_by_key(|(path, entry)| (entry.used, path.clone()));
581 let mut victims = Vec::new();
582 for (path, entry) in by_use {
583 if state.charged <= limit && (limit > 0 || state.entries.is_empty()) {
584 break;
585 }
586 state.entries.remove(&path);
587 state.unindex(&path);
588 state.bytes -= entry.bytes;
589 state.charged -= charge(entry.bytes);
590 victims.push(path);
591 }
592 (before, victims)
593 };
594 for path in victims {
595 let state = lock(&self.inner.state);
598 if state.entries.contains_key(&path) {
599 continue;
600 }
601 match std::fs::remove_file(self.inner.root.join(&path)) {
602 Err(error) if error.kind() != io::ErrorKind::NotFound => {
603 warn!(
604 "cannot remove build cache entry {}: {error}",
605 path.display()
606 );
607 }
608 _ => self.inner.remove_empty_parents(&path),
609 }
610 drop(state);
611 }
612 Trimmed {
613 before,
614 after: self.size(),
615 }
616 }
617}
618
619impl Inner {
620 fn store(&self, runner: &str, upload: Upload) {
624 self.store_locked(&mut lock(&self.state), runner, upload);
627 }
628
629 fn store_locked(&self, state: &mut State, runner: &str, upload: Upload) -> bool {
631 let replaced = state
632 .entries
633 .get(&upload.entry)
634 .map_or(0, |entry| charge(entry.bytes));
635 let fits =
636 state.charged - replaced + charge(upload.bytes) <= state.max_size.saturating_mul(2);
637 let path = self.root.join(&upload.entry);
638 let stored = fits
639 && path
640 .parent()
641 .is_none_or(|parent| std::fs::create_dir_all(parent).is_ok())
642 && std::fs::rename(&upload.file, &path).is_ok();
643 Self::give_back(state, Some(runner), upload.room);
644 if stored {
645 let entry = Entry {
646 bytes: upload.bytes,
647 used: crate::time::now_ms(),
648 };
649 state.index(&upload.entry);
650 if let Some(old) = state.entries.insert(upload.entry, entry) {
651 state.bytes -= old.bytes;
652 state.charged -= charge(old.bytes);
653 }
654 state.bytes += upload.bytes;
655 state.charged += charge(upload.bytes);
656 } else {
657 let _ = std::fs::remove_file(&upload.file);
658 }
659 if let Some(open) = state.runners.get_mut(runner) {
660 if stored {
661 open.counts.written_bytes += upload.bytes;
662 } else {
663 open.counts.refused += 1;
664 }
665 }
666 stored
667 }
668
669 fn reserve(&self, runner: &str, bytes: u64, max_size: u64) -> bool {
673 let mut state = lock(&self.state);
674 let total = state.uploading;
675 let Some(open) = state.runners.get_mut(runner) else {
676 return false;
677 };
678 let fits = total + bytes <= max_size
679 && match open.writes.current() {
680 Writes::Undecided => open.uploading + bytes <= max_size / 4,
681 Writes::Allowed => true,
682 Writes::Refused => false,
683 };
684 if fits {
685 open.uploading += bytes;
686 state.uploading += bytes;
687 }
688 fits
689 }
690
691 fn release(&self, runner: Option<&str>, bytes: u64) {
694 Self::give_back(&mut lock(&self.state), runner, bytes);
695 }
696
697 fn give_back(state: &mut State, runner: Option<&str>, bytes: u64) {
698 state.uploading = state.uploading.saturating_sub(bytes);
699 if let Some(open) = runner.and_then(|runner| state.runners.get_mut(runner)) {
700 open.uploading = open.uploading.saturating_sub(bytes);
701 }
702 }
703
704 fn remove_uploads(&self, runner: Option<&str>, uploads: &[Upload]) {
706 for upload in uploads {
707 let _ = std::fs::remove_file(&upload.file);
708 self.release(runner, upload.room);
709 }
710 }
711
712 fn remove_saves(&self, runner: Option<&str>, saves: &mut HashMap<u64, actions::Save>) {
715 for (_, save) in saves.drain() {
716 self.remove_uploads(runner, &save.into_uploads());
717 }
718 }
719
720 fn remove_empty_parents(&self, entry: &Path) {
723 let mut dir = entry.parent();
724 while let Some(parent) = dir
725 && parent.components().count() > 3
726 && std::fs::remove_dir(self.root.join(parent)).is_ok()
727 {
728 dir = parent.parent();
729 }
730 }
731
732 fn touch(&self, entry: &Path) {
734 if let Some(used) = lock(&self.state).entries.get_mut(entry) {
735 used.used = crate::time::now_ms();
736 }
737 let path = self.root.join(entry);
738 tokio::task::spawn_blocking(move || {
739 if let Ok(file) = std::fs::File::open(&path) {
740 let _ = file.set_modified(std::time::SystemTime::now());
741 }
742 });
743 }
744}
745
746async fn accept(inner: Arc<Inner>, runner: String, listener: UnixListener) {
749 let mut connections = JoinSet::new();
750 loop {
751 let full = connections.len() >= MAX_CONNECTIONS;
752 tokio::select! {
753 accepted = listener.accept(), if !full => match accepted {
754 Ok((stream, _)) => {
755 connections.spawn(serve::connection(inner.clone(), runner.clone(), stream));
756 }
757 Err(error) => {
758 warn!("{runner}: the build cache's socket failed: {error}");
759 return;
760 }
761 },
762 Some(_) = connections.join_next() => {}
763 }
764 }
765}
766
767fn socket_dir(runner: &str) -> String {
769 let hash = runner
771 .bytes()
772 .fold(0xcbf2_9ce4_8422_2325_u64, |hash, byte| {
773 (hash ^ u64::from(byte)).wrapping_mul(0x0000_0100_0000_01b3)
774 });
775 format!("{hash:016x}")
776}
777
778fn valid_segment(name: &str) -> bool {
780 !name.is_empty()
781 && name.len() <= 200
782 && name != "."
783 && name != ".."
784 && name
785 .bytes()
786 .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-'))
787}
788
789fn valid_repository(repository: &str) -> bool {
791 repository.split_once('/').is_some_and(|(owner, name)| {
792 valid_segment(owner) && valid_segment(name) && !owner.starts_with('.')
793 })
794}
795
796fn private_dir(path: &Path) -> io::Result<()> {
797 std::fs::create_dir_all(path)?;
798 std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o700))
799}
800
801fn count_entries(root: &Path, dir: &Path, entries: &mut HashMap<PathBuf, Entry>) -> io::Result<()> {
803 for item in std::fs::read_dir(dir)? {
804 let item = item?;
805 let path = item.path();
806 let metadata = item.metadata()?;
807 if metadata.is_dir() {
808 count_entries(root, &path, entries)?;
809 } else if metadata.is_file()
810 && let Ok(relative) = path.strip_prefix(root)
811 {
812 entries.insert(
813 relative.to_owned(),
814 Entry {
815 bytes: metadata.len(),
816 used: metadata.modified().map_or(0, crate::time::ms),
817 },
818 );
819 }
820 }
821 Ok(())
822}
823
824#[cfg(test)]
825mod tests;
826
827#[cfg(test)]
829#[derive(Debug, Default)]
830pub(crate) struct Gate {
831 state: Mutex<(bool, usize)>,
833 changed: std::sync::Condvar,
834}
835
836#[cfg(test)]
837impl Gate {
838 pub(crate) fn hold(&self) {
839 lock(&self.state).0 = true;
840 }
841
842 pub(crate) fn release(&self) {
843 lock(&self.state).0 = false;
844 self.changed.notify_all();
845 }
846
847 pub(crate) fn waiting(&self) -> usize {
848 lock(&self.state).1
849 }
850
851 fn pass(&self) {
853 let mut state = lock(&self.state);
854 state.1 += 1;
855 while state.0 {
856 state = self
857 .changed
858 .wait(state)
859 .unwrap_or_else(std::sync::PoisonError::into_inner);
860 }
861 state.1 -= 1;
862 }
863}
864
865#[cfg(test)]
868#[allow(
869 clippy::unwrap_used,
870 reason = "a test helper: a socket that does not answer fails the test"
871)]
872pub(crate) async fn request(dir: &Path, method: &str, path: &str, body: &[u8]) -> (u16, Vec<u8>) {
873 use tokio::io::{AsyncReadExt, AsyncWriteExt};
874 let mut stream = tokio::net::UnixStream::connect(dir.join(SOCKET))
875 .await
876 .unwrap();
877 let head = format!(
878 "{method} {path} HTTP/1.1\r\nHost: 127.0.0.1\r\nAuthorization: Basic dG9rZW46c2VjcmV0\r\n\
879 Content-Length: {}\r\nConnection: close\r\n\r\n",
880 body.len()
881 );
882 stream.write_all(head.as_bytes()).await.unwrap();
883 stream.write_all(body).await.unwrap();
884 let mut response = Vec::new();
885 stream.read_to_end(&mut response).await.unwrap();
886 let split = response
887 .windows(4)
888 .position(|window| window == b"\r\n\r\n")
889 .unwrap();
890 let status = String::from_utf8_lossy(&response[9..12]).parse().unwrap();
891 (status, response[split + 4..].to_vec())
892}