1pub(crate) mod actions;
11mod serve;
12mod turbo;
13
14use std::collections::{BTreeSet, HashMap};
15use std::io;
16use std::os::unix::fs::PermissionsExt;
17use std::path::{Path, PathBuf};
18use std::sync::{Arc, Mutex};
19
20use tokio::net::UnixListener;
21use tokio::sync::Notify;
22use tokio::task::{JoinHandle, JoinSet};
23use tracing::warn;
24
25use crate::api::BuildCacheCounts;
26use crate::config::BuildCacheWriters;
27use crate::github::{PullRequest, RunTrigger};
28use crate::sync::lock;
29
30pub const MOUNT: &str = "/run/runnerd-build-cache";
32
33pub const SOCKET: &str = "cache.sock";
35
36pub const ACTIONS_MARKER: &str = "actions";
39
40const ENTRIES: &str = "entries";
42
43const UPLOADS: &str = "uploads";
45
46const MIN_CHARGE: u64 = 4 << 10;
49
50const MAX_HELD: usize = 10_000;
52
53const MAX_CONNECTIONS: usize = 256;
56
57fn charge(bytes: u64) -> u64 {
59 bytes.max(MIN_CHARGE)
60}
61
62#[derive(Debug, Clone, Copy, PartialEq, Eq)]
64pub enum Writes {
65 Undecided,
67 Allowed,
70 Refused,
72}
73
74#[derive(Debug, Clone, Copy, PartialEq, Eq)]
76pub enum ActionsCache {
77 Served,
79 Off,
81}
82
83#[derive(Debug, Clone, Default, PartialEq, Eq)]
86pub struct Scopes {
87 pub own: Option<String>,
91 pub base: Option<String>,
93 pub default: Option<String>,
95}
96
97const WRITING_EVENTS: [&str; 3] = ["push", "schedule", "workflow_dispatch"];
99
100pub const PULL_REQUEST_EVENT: &str = "pull_request";
103
104pub fn writes(
110 runner: &str,
111 job_runner: Option<&str>,
112 repository: &str,
113 trigger: &RunTrigger,
114 branches: &[String],
115 writers: BuildCacheWriters,
116) -> Writes {
117 match job_runner {
118 None => Writes::Undecided,
119 Some(placed) if placed != runner => Writes::Refused,
120 Some(_) if !same_repository(trigger.head_repository.as_deref(), repository) => {
121 Writes::Refused
122 }
123 Some(_) => {
124 let Some(branch) = &trigger.head_branch else {
125 return Writes::Refused;
126 };
127 let event = trigger.event.as_str();
128 let branch_run = WRITING_EVENTS.contains(&event) && branches.contains(branch);
129 let pull_request = writers == BuildCacheWriters::PullRequests
130 && event == PULL_REQUEST_EVENT
131 && trigger.by_user
132 && !trigger.pull_requests.is_empty();
133 if branch_run || pull_request {
134 Writes::Allowed
135 } else {
136 Writes::Refused
137 }
138 }
139 }
140}
141
142pub fn opened_by_user(pull: &PullRequest, repository: &str, trigger: &RunTrigger) -> bool {
145 pull.by_user
146 && same_repository(pull.head_repository.as_deref(), repository)
147 && trigger.head_branch.as_ref() == Some(&pull.head_branch)
148}
149
150fn same_repository(head: Option<&str>, repository: &str) -> bool {
152 head.is_some_and(|head| head.eq_ignore_ascii_case(repository))
153}
154
155#[derive(Debug, Clone, Copy, PartialEq, Eq)]
157pub struct Size {
158 pub bytes: u64,
160 pub entries: u64,
162}
163
164#[derive(Debug, Clone, Copy, PartialEq, Eq)]
166pub struct Trimmed {
167 pub before: Size,
169 pub after: Size,
171}
172
173#[derive(Debug, Clone)]
175pub struct BuildCache {
176 inner: Arc<Inner>,
177}
178
179#[derive(Debug)]
180struct Inner {
181 root: PathBuf,
183 uploads: PathBuf,
185 sockets: PathBuf,
187 results: actions::Results,
189 vercel: turbo::Vercel,
191 state: Mutex<State>,
194}
195
196#[derive(Debug, Default)]
197struct State {
198 max_size: u64,
199 entries: HashMap<PathBuf, Entry>,
200 bytes: u64,
202 charged: u64,
204 runners: HashMap<String, Runner>,
205 next_upload: u64,
206 uploading: u64,
209 default_branches: HashMap<String, String>,
212 last_created: u64,
215 actions: HashMap<PathBuf, BTreeSet<PathBuf>>,
217}
218
219impl State {
220 fn index(&mut self, path: &Path) {
222 if let Some(scope) = actions::scope_of(path) {
223 self.actions
224 .entry(scope)
225 .or_default()
226 .insert(path.to_owned());
227 }
228 }
229
230 fn unindex(&mut self, path: &Path) {
232 if let Some(scope) = actions::scope_of(path)
233 && let Some(paths) = self.actions.get_mut(&scope)
234 {
235 paths.remove(path);
236 if paths.is_empty() {
237 self.actions.remove(&scope);
238 }
239 }
240 }
241}
242
243#[derive(Debug, Clone, Copy)]
244struct Entry {
245 bytes: u64,
246 used: u64,
248}
249
250#[derive(Debug)]
251struct Runner {
252 repository: String,
253 writes: Writes,
254 counts: BuildCacheCounts,
255 held: Vec<Upload>,
257 uploading: u64,
260 dir: PathBuf,
261 serving: JoinHandle<()>,
262 actions: ActionsCache,
264 scopes: Scopes,
266 decided: Arc<Notify>,
268 saves: HashMap<u64, actions::Save>,
270 found: HashMap<u64, PathBuf>,
272}
273
274#[derive(Debug)]
276struct Upload {
277 file: PathBuf,
278 entry: PathBuf,
279 bytes: u64,
280 room: u64,
282}
283
284impl BuildCache {
285 pub fn open(directory: &Path, sockets: &Path, max_size: u64) -> io::Result<BuildCache> {
288 std::fs::create_dir_all(directory)?;
289 let (root, uploads) = (directory.join(ENTRIES), directory.join(UPLOADS));
290 match std::fs::remove_dir_all(&uploads) {
291 Err(error) if error.kind() != io::ErrorKind::NotFound => return Err(error),
292 _ => {}
293 }
294 private_dir(&root)?;
295 private_dir(&uploads)?;
296 private_dir(sockets)?;
297 let mut entries = HashMap::new();
298 count_entries(&root, &root, &mut entries)?;
299 let bytes = entries.values().map(|entry: &Entry| entry.bytes).sum();
300 let charged = entries.values().map(|entry| charge(entry.bytes)).sum();
301 let mut state = State {
302 max_size,
303 bytes,
304 charged,
305 ..State::default()
306 };
307 for path in entries.keys() {
308 state.index(path);
309 state.last_created = state
310 .last_created
311 .max(actions::created_of(path).unwrap_or(0));
312 }
313 state.entries = entries;
314 Ok(BuildCache {
315 inner: Arc::new(Inner {
316 root,
317 uploads,
318 sockets: sockets.to_owned(),
319 results: actions::Results::new(),
320 vercel: turbo::Vercel::new(),
321 state: Mutex::new(state),
322 }),
323 })
324 }
325
326 pub fn directory(&self) -> &Path {
328 &self.inner.root
329 }
330
331 #[cfg(test)]
333 pub(crate) fn set_results_origin(&self, origin: &str) {
334 self.inner.results.set_origin(origin);
335 }
336
337 #[cfg(test)]
339 pub(crate) fn set_vercel_origin(&self, origin: &str) {
340 self.inner.vercel.set_origin(origin);
341 }
342
343 pub fn set_max_size(&self, max_size: u64) {
345 lock(&self.inner.state).max_size = max_size;
346 }
347
348 pub fn attach(
352 &self,
353 runner: &str,
354 repository: &str,
355 actions: ActionsCache,
356 ) -> io::Result<PathBuf> {
357 if !valid_repository(repository) {
358 return Err(io::Error::new(
359 io::ErrorKind::InvalidInput,
360 format!("{repository} is not a repository name"),
361 ));
362 }
363 let dir = self.inner.sockets.join(socket_dir(runner));
364 std::fs::create_dir_all(&dir)?;
365 std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o755))?;
367 let marker = dir.join(ACTIONS_MARKER);
368 match actions {
369 ActionsCache::Served => {
370 std::fs::write(&marker, b"")?;
371 std::fs::set_permissions(&marker, std::fs::Permissions::from_mode(0o644))?;
372 }
373 ActionsCache::Off => match std::fs::remove_file(&marker) {
374 Err(error) if error.kind() != io::ErrorKind::NotFound => return Err(error),
375 _ => {}
376 },
377 }
378 let socket = dir.join(SOCKET);
379 match std::fs::remove_file(&socket) {
380 Err(error) if error.kind() != io::ErrorKind::NotFound => return Err(error),
381 _ => {}
382 }
383 let listener = UnixListener::bind(&socket)?;
384 std::fs::set_permissions(&socket, std::fs::Permissions::from_mode(0o666))?;
385 let serving = tokio::spawn(accept(self.inner.clone(), runner.to_owned(), listener));
386 let old = lock(&self.inner.state).runners.insert(
387 runner.to_owned(),
388 Runner {
389 repository: repository.to_owned(),
390 writes: Writes::Undecided,
391 counts: BuildCacheCounts::default(),
392 held: Vec::new(),
393 uploading: 0,
394 dir: dir.clone(),
395 serving,
396 actions,
397 scopes: Scopes::default(),
398 decided: Arc::new(Notify::new()),
399 saves: HashMap::new(),
400 found: HashMap::new(),
401 },
402 );
403 if let Some(mut old) = old {
404 old.serving.abort();
405 self.inner.remove_uploads(None, &old.held);
406 self.inner.remove_saves(None, &mut old.saves);
407 }
408 Ok(dir)
409 }
410
411 pub fn forget_others(&self, keep: &[&str]) {
413 let keep: Vec<String> = keep.iter().map(|runner| socket_dir(runner)).collect();
414 let Ok(dirs) = std::fs::read_dir(&self.inner.sockets) else {
415 return;
416 };
417 for dir in dirs.flatten() {
418 if !keep
419 .iter()
420 .any(|name| *name == dir.file_name().to_string_lossy())
421 {
422 let _ = std::fs::remove_dir_all(dir.path());
423 }
424 }
425 }
426
427 pub fn undecided(&self, runner: &str) -> bool {
429 lock(&self.inner.state)
430 .runners
431 .get(runner)
432 .is_some_and(|runner| runner.writes == Writes::Undecided)
433 }
434
435 pub fn decide(&self, runner: &str, writes: Writes, scopes: Scopes) {
438 let held = {
439 let mut state = lock(&self.inner.state);
440 let Some(open) = state.runners.get_mut(runner) else {
441 return;
442 };
443 if open.writes != Writes::Undecided || writes == Writes::Undecided {
444 return;
445 }
446 open.writes = writes;
447 open.decided.notify_waiters();
448 let held = std::mem::take(&mut open.held);
449 let repository = open.repository.clone();
450 if let Some(default) = &scopes.default {
451 state.default_branches.insert(repository, default.clone());
452 }
453 if let Some(open) = state.runners.get_mut(runner) {
454 open.scopes = scopes;
455 }
456 held
457 };
458 match writes {
459 Writes::Allowed => {
460 for upload in held {
461 self.inner.store(runner, upload);
462 }
463 }
464 Writes::Refused | Writes::Undecided => {
465 if let Some(open) = lock(&self.inner.state).runners.get_mut(runner) {
466 open.counts.refused += held.len() as u64;
467 }
468 self.inner.remove_uploads(Some(runner), &held);
469 }
470 }
471 }
472
473 pub fn close(&self, runner: &str) -> bool {
476 let dir = {
477 let state = lock(&self.inner.state);
478 let Some(open) = state.runners.get(runner) else {
479 return false;
480 };
481 if open.writes != Writes::Undecided || open.held.is_empty() {
482 return false;
483 }
484 open.serving.abort();
485 open.dir.clone()
486 };
487 if let Err(error) = std::fs::remove_dir_all(&dir)
488 && error.kind() != io::ErrorKind::NotFound
489 {
490 warn!("cannot remove {}: {error}", dir.display());
491 }
492 true
493 }
494
495 pub fn detach(&self, runner: &str) -> Option<BuildCacheCounts> {
498 let Some(mut open) = lock(&self.inner.state).runners.remove(runner) else {
499 let dir = self.inner.sockets.join(socket_dir(runner));
501 if let Err(error) = std::fs::remove_dir_all(&dir)
502 && error.kind() != io::ErrorKind::NotFound
503 {
504 warn!("cannot remove {}: {error}", dir.display());
505 }
506 return None;
507 };
508 open.serving.abort();
509 self.inner.remove_uploads(None, &open.held);
510 self.inner.remove_saves(None, &mut open.saves);
511 open.counts.refused += open.held.len() as u64;
512 open.counts.writer = open.writes == Writes::Allowed;
513 if let Err(error) = std::fs::remove_dir_all(&open.dir)
514 && error.kind() != io::ErrorKind::NotFound
515 {
516 warn!("cannot remove {}: {error}", open.dir.display());
517 }
518 Some(open.counts)
519 }
520
521 pub fn size(&self) -> Size {
523 let state = lock(&self.inner.state);
524 Size {
525 bytes: state.bytes,
526 entries: state.entries.len() as u64,
527 }
528 }
529
530 pub fn trim(&self) -> Trimmed {
534 let limit = lock(&self.inner.state).max_size;
535 self.trim_to(limit)
536 }
537
538 pub fn clear(&self) -> Trimmed {
540 self.trim_to(0)
541 }
542
543 fn trim_to(&self, limit: u64) -> Trimmed {
544 let (before, victims) = {
545 let mut state = lock(&self.inner.state);
546 let before = Size {
547 bytes: state.bytes,
548 entries: state.entries.len() as u64,
549 };
550 let mut by_use: Vec<(PathBuf, Entry)> = state
551 .entries
552 .iter()
553 .map(|(path, entry)| (path.clone(), *entry))
554 .collect();
555 by_use.sort_by_key(|(path, entry)| (entry.used, path.clone()));
556 let mut victims = Vec::new();
557 for (path, entry) in by_use {
558 if state.charged <= limit && (limit > 0 || state.entries.is_empty()) {
559 break;
560 }
561 state.entries.remove(&path);
562 state.unindex(&path);
563 state.bytes -= entry.bytes;
564 state.charged -= charge(entry.bytes);
565 victims.push(path);
566 }
567 (before, victims)
568 };
569 for path in victims {
570 let state = lock(&self.inner.state);
573 if state.entries.contains_key(&path) {
574 continue;
575 }
576 match std::fs::remove_file(self.inner.root.join(&path)) {
577 Err(error) if error.kind() != io::ErrorKind::NotFound => {
578 warn!(
579 "cannot remove build cache entry {}: {error}",
580 path.display()
581 );
582 }
583 _ => self.inner.remove_empty_parents(&path),
584 }
585 drop(state);
586 }
587 Trimmed {
588 before,
589 after: self.size(),
590 }
591 }
592}
593
594impl Inner {
595 fn store(&self, runner: &str, upload: Upload) {
599 let mut state = lock(&self.state);
602 let replaced = state
603 .entries
604 .get(&upload.entry)
605 .map_or(0, |entry| charge(entry.bytes));
606 let fits =
607 state.charged - replaced + charge(upload.bytes) <= state.max_size.saturating_mul(2);
608 let path = self.root.join(&upload.entry);
609 let stored = fits
610 && path
611 .parent()
612 .is_none_or(|parent| std::fs::create_dir_all(parent).is_ok())
613 && std::fs::rename(&upload.file, &path).is_ok();
614 Self::give_back(&mut state, Some(runner), upload.room);
615 if stored {
616 let entry = Entry {
617 bytes: upload.bytes,
618 used: crate::time::now_ms(),
619 };
620 state.index(&upload.entry);
621 if let Some(old) = state.entries.insert(upload.entry, entry) {
622 state.bytes -= old.bytes;
623 state.charged -= charge(old.bytes);
624 }
625 state.bytes += upload.bytes;
626 state.charged += charge(upload.bytes);
627 } else {
628 let _ = std::fs::remove_file(&upload.file);
629 }
630 if let Some(open) = state.runners.get_mut(runner) {
631 if stored {
632 open.counts.written_bytes += upload.bytes;
633 } else {
634 open.counts.refused += 1;
635 }
636 }
637 }
638
639 fn reserve(&self, runner: &str, bytes: u64, max_size: u64) -> bool {
643 let mut state = lock(&self.state);
644 let total = state.uploading;
645 let Some(open) = state.runners.get_mut(runner) else {
646 return false;
647 };
648 let fits = total + bytes <= max_size
649 && (open.writes != Writes::Undecided || open.uploading + bytes <= max_size / 4);
650 if fits {
651 open.uploading += bytes;
652 state.uploading += bytes;
653 }
654 fits
655 }
656
657 fn release(&self, runner: Option<&str>, bytes: u64) {
660 Self::give_back(&mut lock(&self.state), runner, bytes);
661 }
662
663 fn give_back(state: &mut State, runner: Option<&str>, bytes: u64) {
664 state.uploading = state.uploading.saturating_sub(bytes);
665 if let Some(open) = runner.and_then(|runner| state.runners.get_mut(runner)) {
666 open.uploading = open.uploading.saturating_sub(bytes);
667 }
668 }
669
670 fn remove_uploads(&self, runner: Option<&str>, uploads: &[Upload]) {
672 for upload in uploads {
673 let _ = std::fs::remove_file(&upload.file);
674 self.release(runner, upload.room);
675 }
676 }
677
678 fn remove_saves(&self, runner: Option<&str>, saves: &mut HashMap<u64, actions::Save>) {
681 for (_, save) in saves.drain() {
682 self.remove_uploads(runner, &save.into_uploads());
683 }
684 }
685
686 fn remove_empty_parents(&self, entry: &Path) {
689 let mut dir = entry.parent();
690 while let Some(parent) = dir
691 && parent.components().count() > 3
692 && std::fs::remove_dir(self.root.join(parent)).is_ok()
693 {
694 dir = parent.parent();
695 }
696 }
697
698 fn touch(&self, entry: &Path) {
700 if let Some(used) = lock(&self.state).entries.get_mut(entry) {
701 used.used = crate::time::now_ms();
702 }
703 let path = self.root.join(entry);
704 tokio::task::spawn_blocking(move || {
705 if let Ok(file) = std::fs::File::open(&path) {
706 let _ = file.set_modified(std::time::SystemTime::now());
707 }
708 });
709 }
710}
711
712async fn accept(inner: Arc<Inner>, runner: String, listener: UnixListener) {
715 let mut connections = JoinSet::new();
716 loop {
717 let full = connections.len() >= MAX_CONNECTIONS;
718 tokio::select! {
719 accepted = listener.accept(), if !full => match accepted {
720 Ok((stream, _)) => {
721 connections.spawn(serve::connection(inner.clone(), runner.clone(), stream));
722 }
723 Err(error) => {
724 warn!("{runner}: the build cache's socket failed: {error}");
725 return;
726 }
727 },
728 Some(_) = connections.join_next() => {}
729 }
730 }
731}
732
733fn socket_dir(runner: &str) -> String {
735 let hash = runner
737 .bytes()
738 .fold(0xcbf2_9ce4_8422_2325_u64, |hash, byte| {
739 (hash ^ u64::from(byte)).wrapping_mul(0x0000_0100_0000_01b3)
740 });
741 format!("{hash:016x}")
742}
743
744fn valid_segment(name: &str) -> bool {
746 !name.is_empty()
747 && name.len() <= 200
748 && name != "."
749 && name != ".."
750 && name
751 .bytes()
752 .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-'))
753}
754
755fn valid_repository(repository: &str) -> bool {
757 repository.split_once('/').is_some_and(|(owner, name)| {
758 valid_segment(owner) && valid_segment(name) && !owner.starts_with('.')
759 })
760}
761
762fn private_dir(path: &Path) -> io::Result<()> {
763 std::fs::create_dir_all(path)?;
764 std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o700))
765}
766
767fn count_entries(root: &Path, dir: &Path, entries: &mut HashMap<PathBuf, Entry>) -> io::Result<()> {
769 for item in std::fs::read_dir(dir)? {
770 let item = item?;
771 let path = item.path();
772 let metadata = item.metadata()?;
773 if metadata.is_dir() {
774 count_entries(root, &path, entries)?;
775 } else if metadata.is_file()
776 && let Ok(relative) = path.strip_prefix(root)
777 {
778 entries.insert(
779 relative.to_owned(),
780 Entry {
781 bytes: metadata.len(),
782 used: metadata.modified().map_or(0, crate::time::ms),
783 },
784 );
785 }
786 }
787 Ok(())
788}
789
790#[cfg(test)]
791mod tests;
792
793#[cfg(test)]
796#[allow(
797 clippy::unwrap_used,
798 reason = "a test helper: a socket that does not answer fails the test"
799)]
800pub(crate) async fn request(dir: &Path, method: &str, path: &str, body: &[u8]) -> (u16, Vec<u8>) {
801 use tokio::io::{AsyncReadExt, AsyncWriteExt};
802 let mut stream = tokio::net::UnixStream::connect(dir.join(SOCKET))
803 .await
804 .unwrap();
805 let head = format!(
806 "{method} {path} HTTP/1.1\r\nHost: 127.0.0.1\r\nAuthorization: Basic dG9rZW46c2VjcmV0\r\n\
807 Content-Length: {}\r\nConnection: close\r\n\r\n",
808 body.len()
809 );
810 stream.write_all(head.as_bytes()).await.unwrap();
811 stream.write_all(body).await.unwrap();
812 let mut response = Vec::new();
813 stream.read_to_end(&mut response).await.unwrap();
814 let split = response
815 .windows(4)
816 .position(|window| window == b"\r\n\r\n")
817 .unwrap();
818 let status = String::from_utf8_lossy(&response[9..12]).parse().unwrap();
819 (status, response[split + 4..].to_vec())
820}