1pub(crate) mod actions;
11mod serve;
12mod turbo;
13
14use std::collections::{BTreeSet, HashMap};
15use std::io;
16use std::os::unix::fs::PermissionsExt;
17use std::path::{Path, PathBuf};
18use std::sync::{Arc, Mutex};
19
20use tokio::net::UnixListener;
21use tokio::sync::watch;
22use tokio::task::{JoinHandle, JoinSet};
23use tracing::warn;
24
25use crate::api::BuildCacheCounts;
26use crate::config::BuildCacheWriters;
27use crate::github::{PullRequest, RunTrigger};
28use crate::sync::lock;
29
30pub const MOUNT: &str = "/run/runnerd-build-cache";
32
33pub const SOCKET: &str = "cache.sock";
35
36pub const ACTIONS_MARKER: &str = "actions";
39
40const ENTRIES: &str = "entries";
42
43const UPLOADS: &str = "uploads";
45
46const MIN_CHARGE: u64 = 4 << 10;
49
50const MAX_HELD: usize = 10_000;
52
53const MAX_CONNECTIONS: usize = 256;
56
57fn charge(bytes: u64) -> u64 {
59 bytes.max(MIN_CHARGE)
60}
61
62#[derive(Debug, Clone, Copy, PartialEq, Eq)]
64pub enum Writes {
65 Undecided,
67 Allowed,
70 Refused,
72}
73
74#[derive(Debug, Clone, Copy, PartialEq, Eq)]
76pub enum ActionsCache {
77 Served,
79 Off,
81}
82
83#[derive(Debug, Clone, Default, PartialEq, Eq)]
86pub struct Scopes {
87 pub own: Option<String>,
91 pub base: Option<String>,
93 pub default: Option<String>,
95}
96
97const WRITING_EVENTS: [&str; 3] = ["push", "schedule", "workflow_dispatch"];
99
100pub const PULL_REQUEST_EVENT: &str = "pull_request";
103
104pub fn writes(
110 runner: &str,
111 job_runner: Option<&str>,
112 repository: &str,
113 trigger: &RunTrigger,
114 branches: &[String],
115 writers: BuildCacheWriters,
116) -> Writes {
117 match job_runner {
118 None => Writes::Undecided,
119 Some(placed) if placed != runner => Writes::Refused,
120 Some(_) if !same_repository(trigger.head_repository.as_deref(), repository) => {
121 Writes::Refused
122 }
123 Some(_) => {
124 let Some(branch) = &trigger.head_branch else {
125 return Writes::Refused;
126 };
127 let event = trigger.event.as_str();
128 let branch_run = WRITING_EVENTS.contains(&event) && branches.contains(branch);
129 let pull_request = writers == BuildCacheWriters::PullRequests
130 && event == PULL_REQUEST_EVENT
131 && trigger.by_user
132 && !trigger.pull_requests.is_empty();
133 if branch_run || pull_request {
134 Writes::Allowed
135 } else {
136 Writes::Refused
137 }
138 }
139 }
140}
141
142pub fn opened_by_user(pull: &PullRequest, repository: &str, trigger: &RunTrigger) -> bool {
145 pull.by_user
146 && same_repository(pull.head_repository.as_deref(), repository)
147 && trigger.head_branch.as_ref() == Some(&pull.head_branch)
148}
149
150fn same_repository(head: Option<&str>, repository: &str) -> bool {
152 head.is_some_and(|head| head.eq_ignore_ascii_case(repository))
153}
154
155#[derive(Debug, Clone, Copy, PartialEq, Eq)]
157pub struct Size {
158 pub bytes: u64,
160 pub entries: u64,
162}
163
164#[derive(Debug, Clone, Copy, PartialEq, Eq)]
166pub struct Trimmed {
167 pub before: Size,
169 pub after: Size,
171}
172
173#[derive(Debug, Clone)]
175pub struct BuildCache {
176 inner: Arc<Inner>,
177}
178
179#[derive(Debug)]
180struct Inner {
181 root: PathBuf,
183 uploads: PathBuf,
185 sockets: PathBuf,
187 results: actions::Results,
189 vercel: turbo::Vercel,
191 #[cfg(test)]
193 join_gate: Gate,
194 state: Mutex<State>,
197}
198
199#[derive(Debug, Default)]
200struct State {
201 max_size: u64,
202 entries: HashMap<PathBuf, Entry>,
203 bytes: u64,
205 charged: u64,
207 runners: HashMap<String, Runner>,
208 next_upload: u64,
209 uploading: u64,
212 default_branches: HashMap<String, String>,
215 last_created: u64,
218 actions: HashMap<PathBuf, BTreeSet<PathBuf>>,
220}
221
222impl State {
223 fn index(&mut self, path: &Path) {
225 if let Some(scope) = actions::scope_of(path) {
226 self.actions
227 .entry(scope)
228 .or_default()
229 .insert(path.to_owned());
230 }
231 }
232
233 fn unindex(&mut self, path: &Path) {
235 if let Some(scope) = actions::scope_of(path)
236 && let Some(paths) = self.actions.get_mut(&scope)
237 {
238 paths.remove(path);
239 if paths.is_empty() {
240 self.actions.remove(&scope);
241 }
242 }
243 }
244}
245
246#[derive(Debug, Clone, Copy)]
247struct Entry {
248 bytes: u64,
249 used: u64,
251}
252
253#[derive(Debug)]
254struct Runner {
255 repository: String,
256 writes: Writes,
257 counts: BuildCacheCounts,
258 held: Vec<Upload>,
260 uploading: u64,
263 dir: PathBuf,
264 serving: JoinHandle<()>,
265 actions: ActionsCache,
267 scopes: Scopes,
269 decided: watch::Sender<()>,
271 saves: HashMap<u64, actions::Save>,
274 found: HashMap<u64, PathBuf>,
276}
277
278#[derive(Debug)]
280struct Upload {
281 file: PathBuf,
282 entry: PathBuf,
283 bytes: u64,
284 room: u64,
286}
287
288impl BuildCache {
289 pub fn open(directory: &Path, sockets: &Path, max_size: u64) -> io::Result<BuildCache> {
292 std::fs::create_dir_all(directory)?;
293 let (root, uploads) = (directory.join(ENTRIES), directory.join(UPLOADS));
294 match std::fs::remove_dir_all(&uploads) {
295 Err(error) if error.kind() != io::ErrorKind::NotFound => return Err(error),
296 _ => {}
297 }
298 private_dir(&root)?;
299 private_dir(&uploads)?;
300 private_dir(sockets)?;
301 let mut entries = HashMap::new();
302 count_entries(&root, &root, &mut entries)?;
303 let bytes = entries.values().map(|entry: &Entry| entry.bytes).sum();
304 let charged = entries.values().map(|entry| charge(entry.bytes)).sum();
305 let mut state = State {
306 max_size,
307 bytes,
308 charged,
309 next_upload: std::time::SystemTime::now()
312 .duration_since(std::time::UNIX_EPOCH)
313 .map_or(0, |since| u64::try_from(since.as_micros()).unwrap_or(0)),
314 ..State::default()
315 };
316 for path in entries.keys() {
317 state.index(path);
318 state.last_created = state
319 .last_created
320 .max(actions::created_of(path).unwrap_or(0));
321 }
322 state.entries = entries;
323 Ok(BuildCache {
324 inner: Arc::new(Inner {
325 root,
326 uploads,
327 sockets: sockets.to_owned(),
328 results: actions::Results::new(),
329 vercel: turbo::Vercel::new(),
330 #[cfg(test)]
331 join_gate: Gate::default(),
332 state: Mutex::new(state),
333 }),
334 })
335 }
336
337 pub fn directory(&self) -> &Path {
339 &self.inner.root
340 }
341
342 #[cfg(test)]
344 pub(crate) fn set_results_origin(&self, origin: &str) {
345 self.inner.results.set_origin(origin);
346 }
347
348 #[cfg(test)]
350 pub(crate) fn set_vercel_origin(&self, origin: &str) {
351 self.inner.vercel.set_origin(origin);
352 }
353
354 pub fn set_max_size(&self, max_size: u64) {
356 lock(&self.inner.state).max_size = max_size;
357 }
358
359 pub fn attach(
363 &self,
364 runner: &str,
365 repository: &str,
366 actions: ActionsCache,
367 ) -> io::Result<PathBuf> {
368 if !valid_repository(repository) {
369 return Err(io::Error::new(
370 io::ErrorKind::InvalidInput,
371 format!("{repository} is not a repository name"),
372 ));
373 }
374 let dir = self.inner.sockets.join(socket_dir(runner));
375 std::fs::create_dir_all(&dir)?;
376 std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o755))?;
378 let marker = dir.join(ACTIONS_MARKER);
379 match actions {
380 ActionsCache::Served => {
381 std::fs::write(&marker, b"")?;
382 std::fs::set_permissions(&marker, std::fs::Permissions::from_mode(0o644))?;
383 }
384 ActionsCache::Off => match std::fs::remove_file(&marker) {
385 Err(error) if error.kind() != io::ErrorKind::NotFound => return Err(error),
386 _ => {}
387 },
388 }
389 let socket = dir.join(SOCKET);
390 match std::fs::remove_file(&socket) {
391 Err(error) if error.kind() != io::ErrorKind::NotFound => return Err(error),
392 _ => {}
393 }
394 let listener = UnixListener::bind(&socket)?;
395 std::fs::set_permissions(&socket, std::fs::Permissions::from_mode(0o666))?;
396 let serving = tokio::spawn(accept(self.inner.clone(), runner.to_owned(), listener));
397 let old = lock(&self.inner.state).runners.insert(
398 runner.to_owned(),
399 Runner {
400 repository: repository.to_owned(),
401 writes: Writes::Undecided,
402 counts: BuildCacheCounts::default(),
403 held: Vec::new(),
404 uploading: 0,
405 dir: dir.clone(),
406 serving,
407 actions,
408 scopes: Scopes::default(),
409 decided: watch::Sender::new(()),
410 saves: HashMap::new(),
411 found: HashMap::new(),
412 },
413 );
414 if let Some(mut old) = old {
415 old.serving.abort();
416 self.inner.remove_uploads(None, &old.held);
417 self.inner.remove_saves(None, &mut old.saves);
418 }
419 Ok(dir)
420 }
421
422 pub fn forget_others(&self, keep: &[&str]) {
424 let keep: Vec<String> = keep.iter().map(|runner| socket_dir(runner)).collect();
425 let Ok(dirs) = std::fs::read_dir(&self.inner.sockets) else {
426 return;
427 };
428 for dir in dirs.flatten() {
429 if !keep
430 .iter()
431 .any(|name| *name == dir.file_name().to_string_lossy())
432 {
433 let _ = std::fs::remove_dir_all(dir.path());
434 }
435 }
436 }
437
438 pub fn undecided(&self, runner: &str) -> bool {
440 lock(&self.inner.state)
441 .runners
442 .get(runner)
443 .is_some_and(|runner| runner.writes == Writes::Undecided)
444 }
445
446 pub fn decide(&self, runner: &str, writes: Writes, scopes: Scopes) {
449 let (held, dropped) = {
450 let mut state = lock(&self.inner.state);
451 let Some(open) = state.runners.get_mut(runner) else {
452 return;
453 };
454 if open.writes != Writes::Undecided || writes == Writes::Undecided {
455 return;
456 }
457 open.writes = writes;
458 let held = std::mem::take(&mut open.held);
459 let repository = open.repository.clone();
460 if let Some(default) = &scopes.default {
461 state.default_branches.insert(repository, default.clone());
462 }
463 if let Some(open) = state.runners.get_mut(runner) {
464 open.scopes = scopes;
465 }
466 let dropped = actions::decided(&self.inner, &mut state, runner);
468 if let Some(open) = state.runners.get(runner) {
469 open.decided.send_replace(());
470 }
471 (held, dropped)
472 };
473 self.inner.remove_uploads(Some(runner), &dropped);
474 match writes {
475 Writes::Allowed => {
476 for upload in held {
477 self.inner.store(runner, upload);
478 }
479 }
480 Writes::Refused | Writes::Undecided => {
481 if let Some(open) = lock(&self.inner.state).runners.get_mut(runner) {
482 open.counts.refused += held.len() as u64;
483 }
484 self.inner.remove_uploads(Some(runner), &held);
485 }
486 }
487 }
488
489 pub fn close(&self, runner: &str) -> bool {
493 let (dir, mut unfinished) = {
494 let mut state = lock(&self.inner.state);
495 let Some(open) = state.runners.get_mut(runner) else {
496 return false;
497 };
498 if open.writes != Writes::Undecided
499 || open.held.is_empty() && actions::finalized(&open.saves) == 0
500 {
501 return false;
502 }
503 open.serving.abort();
504 (open.dir.clone(), actions::unfinished(&mut open.saves))
505 };
506 self.inner.remove_saves(Some(runner), &mut unfinished);
507 if let Err(error) = std::fs::remove_dir_all(&dir)
508 && error.kind() != io::ErrorKind::NotFound
509 {
510 warn!("cannot remove {}: {error}", dir.display());
511 }
512 true
513 }
514
515 pub fn detach(&self, runner: &str) -> Option<BuildCacheCounts> {
518 let Some(mut open) = lock(&self.inner.state).runners.remove(runner) else {
519 let dir = self.inner.sockets.join(socket_dir(runner));
521 if let Err(error) = std::fs::remove_dir_all(&dir)
522 && error.kind() != io::ErrorKind::NotFound
523 {
524 warn!("cannot remove {}: {error}", dir.display());
525 }
526 return None;
527 };
528 open.serving.abort();
529 let dropped = open.held.len() as u64 + actions::finalized(&open.saves);
531 if open.writes == Writes::Undecided && dropped > 0 {
532 warn!(
533 "{runner}: dropped {dropped} build cache uploads its job sent: GitHub never said which runner ran the job"
534 );
535 }
536 self.inner.remove_uploads(None, &open.held);
537 self.inner.remove_saves(None, &mut open.saves);
538 open.counts.refused += dropped;
539 open.counts.writer = open.writes == Writes::Allowed;
540 if let Err(error) = std::fs::remove_dir_all(&open.dir)
541 && error.kind() != io::ErrorKind::NotFound
542 {
543 warn!("cannot remove {}: {error}", open.dir.display());
544 }
545 Some(open.counts)
546 }
547
548 pub fn size(&self) -> Size {
550 let state = lock(&self.inner.state);
551 Size {
552 bytes: state.bytes,
553 entries: state.entries.len() as u64,
554 }
555 }
556
557 pub fn trim(&self) -> Trimmed {
561 let limit = lock(&self.inner.state).max_size;
562 self.trim_to(limit)
563 }
564
565 pub fn clear(&self) -> Trimmed {
567 self.trim_to(0)
568 }
569
570 fn trim_to(&self, limit: u64) -> Trimmed {
571 let (before, victims) = {
572 let mut state = lock(&self.inner.state);
573 let before = Size {
574 bytes: state.bytes,
575 entries: state.entries.len() as u64,
576 };
577 let mut by_use: Vec<(PathBuf, Entry)> = state
578 .entries
579 .iter()
580 .map(|(path, entry)| (path.clone(), *entry))
581 .collect();
582 by_use.sort_by_key(|(path, entry)| (entry.used, path.clone()));
583 let mut victims = Vec::new();
584 for (path, entry) in by_use {
585 if state.charged <= limit && (limit > 0 || state.entries.is_empty()) {
586 break;
587 }
588 state.entries.remove(&path);
589 state.unindex(&path);
590 state.bytes -= entry.bytes;
591 state.charged -= charge(entry.bytes);
592 victims.push(path);
593 }
594 (before, victims)
595 };
596 for path in victims {
597 let state = lock(&self.inner.state);
600 if state.entries.contains_key(&path) {
601 continue;
602 }
603 match std::fs::remove_file(self.inner.root.join(&path)) {
604 Err(error) if error.kind() != io::ErrorKind::NotFound => {
605 warn!(
606 "cannot remove build cache entry {}: {error}",
607 path.display()
608 );
609 }
610 _ => self.inner.remove_empty_parents(&path),
611 }
612 drop(state);
613 }
614 Trimmed {
615 before,
616 after: self.size(),
617 }
618 }
619}
620
621impl Inner {
622 fn store(&self, runner: &str, upload: Upload) {
626 self.store_locked(&mut lock(&self.state), runner, upload);
629 }
630
631 fn store_locked(&self, state: &mut State, runner: &str, upload: Upload) -> bool {
633 let replaced = state
634 .entries
635 .get(&upload.entry)
636 .map_or(0, |entry| charge(entry.bytes));
637 let fits =
638 state.charged - replaced + charge(upload.bytes) <= state.max_size.saturating_mul(2);
639 let path = self.root.join(&upload.entry);
640 let stored = fits
641 && path
642 .parent()
643 .is_none_or(|parent| std::fs::create_dir_all(parent).is_ok())
644 && std::fs::rename(&upload.file, &path).is_ok();
645 Self::give_back(state, Some(runner), upload.room);
646 if stored {
647 let entry = Entry {
648 bytes: upload.bytes,
649 used: crate::time::now_ms(),
650 };
651 state.index(&upload.entry);
652 if let Some(old) = state.entries.insert(upload.entry, entry) {
653 state.bytes -= old.bytes;
654 state.charged -= charge(old.bytes);
655 }
656 state.bytes += upload.bytes;
657 state.charged += charge(upload.bytes);
658 } else {
659 let _ = std::fs::remove_file(&upload.file);
660 }
661 if let Some(open) = state.runners.get_mut(runner) {
662 if stored {
663 open.counts.written_bytes += upload.bytes;
664 } else {
665 open.counts.refused += 1;
666 }
667 }
668 stored
669 }
670
671 fn reserve(&self, runner: &str, bytes: u64, max_size: u64) -> bool {
675 let mut state = lock(&self.state);
676 let total = state.uploading;
677 let Some(open) = state.runners.get_mut(runner) else {
678 return false;
679 };
680 let fits = total + bytes <= max_size
681 && match open.writes {
682 Writes::Undecided => open.uploading + bytes <= max_size / 4,
683 Writes::Allowed => true,
684 Writes::Refused => false,
685 };
686 if fits {
687 open.uploading += bytes;
688 state.uploading += bytes;
689 }
690 fits
691 }
692
693 fn release(&self, runner: Option<&str>, bytes: u64) {
696 Self::give_back(&mut lock(&self.state), runner, bytes);
697 }
698
699 fn give_back(state: &mut State, runner: Option<&str>, bytes: u64) {
700 state.uploading = state.uploading.saturating_sub(bytes);
701 if let Some(open) = runner.and_then(|runner| state.runners.get_mut(runner)) {
702 open.uploading = open.uploading.saturating_sub(bytes);
703 }
704 }
705
706 fn remove_uploads(&self, runner: Option<&str>, uploads: &[Upload]) {
708 for upload in uploads {
709 let _ = std::fs::remove_file(&upload.file);
710 self.release(runner, upload.room);
711 }
712 }
713
714 fn remove_saves(&self, runner: Option<&str>, saves: &mut HashMap<u64, actions::Save>) {
717 for (_, save) in saves.drain() {
718 self.remove_uploads(runner, &save.into_uploads());
719 }
720 }
721
722 fn remove_empty_parents(&self, entry: &Path) {
725 let mut dir = entry.parent();
726 while let Some(parent) = dir
727 && parent.components().count() > 3
728 && std::fs::remove_dir(self.root.join(parent)).is_ok()
729 {
730 dir = parent.parent();
731 }
732 }
733
734 fn touch(&self, entry: &Path) {
736 if let Some(used) = lock(&self.state).entries.get_mut(entry) {
737 used.used = crate::time::now_ms();
738 }
739 let path = self.root.join(entry);
740 tokio::task::spawn_blocking(move || {
741 if let Ok(file) = std::fs::File::open(&path) {
742 let _ = file.set_modified(std::time::SystemTime::now());
743 }
744 });
745 }
746}
747
748async fn accept(inner: Arc<Inner>, runner: String, listener: UnixListener) {
751 let mut connections = JoinSet::new();
752 loop {
753 let full = connections.len() >= MAX_CONNECTIONS;
754 tokio::select! {
755 accepted = listener.accept(), if !full => match accepted {
756 Ok((stream, _)) => {
757 connections.spawn(serve::connection(inner.clone(), runner.clone(), stream));
758 }
759 Err(error) => {
760 warn!("{runner}: the build cache's socket failed: {error}");
761 return;
762 }
763 },
764 Some(_) = connections.join_next() => {}
765 }
766 }
767}
768
769fn socket_dir(runner: &str) -> String {
771 let hash = runner
773 .bytes()
774 .fold(0xcbf2_9ce4_8422_2325_u64, |hash, byte| {
775 (hash ^ u64::from(byte)).wrapping_mul(0x0000_0100_0000_01b3)
776 });
777 format!("{hash:016x}")
778}
779
780fn valid_segment(name: &str) -> bool {
782 !name.is_empty()
783 && name.len() <= 200
784 && name != "."
785 && name != ".."
786 && name
787 .bytes()
788 .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-'))
789}
790
791fn valid_repository(repository: &str) -> bool {
793 repository.split_once('/').is_some_and(|(owner, name)| {
794 valid_segment(owner) && valid_segment(name) && !owner.starts_with('.')
795 })
796}
797
798fn private_dir(path: &Path) -> io::Result<()> {
799 std::fs::create_dir_all(path)?;
800 std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o700))
801}
802
803fn count_entries(root: &Path, dir: &Path, entries: &mut HashMap<PathBuf, Entry>) -> io::Result<()> {
805 for item in std::fs::read_dir(dir)? {
806 let item = item?;
807 let path = item.path();
808 let metadata = item.metadata()?;
809 if metadata.is_dir() {
810 count_entries(root, &path, entries)?;
811 } else if metadata.is_file()
812 && let Ok(relative) = path.strip_prefix(root)
813 {
814 entries.insert(
815 relative.to_owned(),
816 Entry {
817 bytes: metadata.len(),
818 used: metadata.modified().map_or(0, crate::time::ms),
819 },
820 );
821 }
822 }
823 Ok(())
824}
825
826#[cfg(test)]
827mod tests;
828
829#[cfg(test)]
831#[derive(Debug, Default)]
832pub(crate) struct Gate {
833 state: Mutex<(bool, usize)>,
835 changed: std::sync::Condvar,
836}
837
838#[cfg(test)]
839impl Gate {
840 pub(crate) fn hold(&self) {
841 lock(&self.state).0 = true;
842 }
843
844 pub(crate) fn release(&self) {
845 lock(&self.state).0 = false;
846 self.changed.notify_all();
847 }
848
849 pub(crate) fn waiting(&self) -> usize {
850 lock(&self.state).1
851 }
852
853 fn pass(&self) {
855 let mut state = lock(&self.state);
856 state.1 += 1;
857 while state.0 {
858 state = self
859 .changed
860 .wait(state)
861 .unwrap_or_else(std::sync::PoisonError::into_inner);
862 }
863 state.1 -= 1;
864 }
865}
866
867#[cfg(test)]
870#[allow(
871 clippy::unwrap_used,
872 reason = "a test helper: a socket that does not answer fails the test"
873)]
874pub(crate) async fn request(dir: &Path, method: &str, path: &str, body: &[u8]) -> (u16, Vec<u8>) {
875 use tokio::io::{AsyncReadExt, AsyncWriteExt};
876 let mut stream = tokio::net::UnixStream::connect(dir.join(SOCKET))
877 .await
878 .unwrap();
879 let head = format!(
880 "{method} {path} HTTP/1.1\r\nHost: 127.0.0.1\r\nAuthorization: Basic dG9rZW46c2VjcmV0\r\n\
881 Content-Length: {}\r\nConnection: close\r\n\r\n",
882 body.len()
883 );
884 stream.write_all(head.as_bytes()).await.unwrap();
885 stream.write_all(body).await.unwrap();
886 let mut response = Vec::new();
887 stream.read_to_end(&mut response).await.unwrap();
888 let split = response
889 .windows(4)
890 .position(|window| window == b"\r\n\r\n")
891 .unwrap();
892 let status = String::from_utf8_lossy(&response[9..12]).parse().unwrap();
893 (status, response[split + 4..].to_vec())
894}