1mod serve;
9
10use std::collections::HashMap;
11use std::io;
12use std::os::unix::fs::PermissionsExt;
13use std::path::{Path, PathBuf};
14use std::sync::{Arc, Mutex};
15
16use tokio::net::UnixListener;
17use tokio::task::{JoinHandle, JoinSet};
18use tracing::warn;
19
20use crate::api::BuildCacheCounts;
21use crate::config::BuildCacheWriters;
22use crate::github::{PullRequest, RunTrigger};
23use crate::sync::lock;
24
25pub const MOUNT: &str = "/run/runnerd-build-cache";
27
28pub const SOCKET: &str = "cache.sock";
30
31const ENTRIES: &str = "entries";
33
34const UPLOADS: &str = "uploads";
36
37const MIN_CHARGE: u64 = 4 << 10;
40
41const MAX_HELD: usize = 10_000;
43
44const MAX_CONNECTIONS: usize = 256;
47
48fn charge(bytes: u64) -> u64 {
50 bytes.max(MIN_CHARGE)
51}
52
53#[derive(Debug, Clone, Copy, PartialEq, Eq)]
55pub enum Writes {
56 Undecided,
58 Allowed,
61 Refused,
63}
64
65const WRITING_EVENTS: [&str; 3] = ["push", "schedule", "workflow_dispatch"];
67
68pub const PULL_REQUEST_EVENT: &str = "pull_request";
71
72pub fn writes(
78 runner: &str,
79 job_runner: Option<&str>,
80 repository: &str,
81 trigger: &RunTrigger,
82 branches: &[String],
83 writers: BuildCacheWriters,
84) -> Writes {
85 match job_runner {
86 None => Writes::Undecided,
87 Some(placed) if placed != runner => Writes::Refused,
88 Some(_) if !same_repository(trigger.head_repository.as_deref(), repository) => {
89 Writes::Refused
90 }
91 Some(_) => {
92 let Some(branch) = &trigger.head_branch else {
93 return Writes::Refused;
94 };
95 let event = trigger.event.as_str();
96 let branch_run = WRITING_EVENTS.contains(&event) && branches.contains(branch);
97 let pull_request = writers == BuildCacheWriters::PullRequests
98 && event == PULL_REQUEST_EVENT
99 && trigger.by_user
100 && !trigger.pull_requests.is_empty();
101 if branch_run || pull_request {
102 Writes::Allowed
103 } else {
104 Writes::Refused
105 }
106 }
107 }
108}
109
110pub fn opened_by_user(pull: &PullRequest, repository: &str, trigger: &RunTrigger) -> bool {
113 pull.by_user
114 && same_repository(pull.head_repository.as_deref(), repository)
115 && trigger.head_branch.as_ref() == Some(&pull.head_branch)
116}
117
118fn same_repository(head: Option<&str>, repository: &str) -> bool {
120 head.is_some_and(|head| head.eq_ignore_ascii_case(repository))
121}
122
123#[derive(Debug, Clone, Copy, PartialEq, Eq)]
125pub struct Size {
126 pub bytes: u64,
128 pub entries: u64,
130}
131
132#[derive(Debug, Clone, Copy, PartialEq, Eq)]
134pub struct Trimmed {
135 pub before: Size,
137 pub after: Size,
139}
140
141#[derive(Debug, Clone)]
143pub struct BuildCache {
144 inner: Arc<Inner>,
145}
146
147#[derive(Debug)]
148struct Inner {
149 root: PathBuf,
151 uploads: PathBuf,
153 sockets: PathBuf,
155 state: Mutex<State>,
158}
159
160#[derive(Debug, Default)]
161struct State {
162 max_size: u64,
163 entries: HashMap<PathBuf, Entry>,
164 bytes: u64,
166 charged: u64,
168 runners: HashMap<String, Runner>,
169 next_upload: u64,
170 uploading: u64,
173}
174
175#[derive(Debug, Clone, Copy)]
176struct Entry {
177 bytes: u64,
178 used: u64,
180}
181
182#[derive(Debug)]
183struct Runner {
184 repository: String,
185 writes: Writes,
186 counts: BuildCacheCounts,
187 held: Vec<Upload>,
189 uploading: u64,
192 dir: PathBuf,
193 serving: JoinHandle<()>,
194}
195
196#[derive(Debug)]
198struct Upload {
199 file: PathBuf,
200 entry: PathBuf,
201 bytes: u64,
202 room: u64,
204}
205
206impl BuildCache {
207 pub fn open(directory: &Path, sockets: &Path, max_size: u64) -> io::Result<BuildCache> {
210 std::fs::create_dir_all(directory)?;
211 let (root, uploads) = (directory.join(ENTRIES), directory.join(UPLOADS));
212 match std::fs::remove_dir_all(&uploads) {
213 Err(error) if error.kind() != io::ErrorKind::NotFound => return Err(error),
214 _ => {}
215 }
216 private_dir(&root)?;
217 private_dir(&uploads)?;
218 private_dir(sockets)?;
219 let mut entries = HashMap::new();
220 count_entries(&root, &root, &mut entries)?;
221 let bytes = entries.values().map(|entry: &Entry| entry.bytes).sum();
222 let charged = entries.values().map(|entry| charge(entry.bytes)).sum();
223 Ok(BuildCache {
224 inner: Arc::new(Inner {
225 root,
226 uploads,
227 sockets: sockets.to_owned(),
228 state: Mutex::new(State {
229 max_size,
230 entries,
231 bytes,
232 charged,
233 ..State::default()
234 }),
235 }),
236 })
237 }
238
239 pub fn directory(&self) -> &Path {
241 &self.inner.root
242 }
243
244 pub fn set_max_size(&self, max_size: u64) {
246 lock(&self.inner.state).max_size = max_size;
247 }
248
249 pub fn attach(&self, runner: &str, repository: &str) -> io::Result<PathBuf> {
252 if !valid_repository(repository) {
253 return Err(io::Error::new(
254 io::ErrorKind::InvalidInput,
255 format!("{repository} is not a repository name"),
256 ));
257 }
258 let dir = self.inner.sockets.join(socket_dir(runner));
259 std::fs::create_dir_all(&dir)?;
260 std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o755))?;
262 let socket = dir.join(SOCKET);
263 match std::fs::remove_file(&socket) {
264 Err(error) if error.kind() != io::ErrorKind::NotFound => return Err(error),
265 _ => {}
266 }
267 let listener = UnixListener::bind(&socket)?;
268 std::fs::set_permissions(&socket, std::fs::Permissions::from_mode(0o666))?;
269 let serving = tokio::spawn(accept(self.inner.clone(), runner.to_owned(), listener));
270 let old = lock(&self.inner.state).runners.insert(
271 runner.to_owned(),
272 Runner {
273 repository: repository.to_owned(),
274 writes: Writes::Undecided,
275 counts: BuildCacheCounts::default(),
276 held: Vec::new(),
277 uploading: 0,
278 dir: dir.clone(),
279 serving,
280 },
281 );
282 if let Some(old) = old {
283 old.serving.abort();
284 self.inner.remove_uploads(None, &old.held);
285 }
286 Ok(dir)
287 }
288
289 pub fn forget_others(&self, keep: &[&str]) {
291 let keep: Vec<String> = keep.iter().map(|runner| socket_dir(runner)).collect();
292 let Ok(dirs) = std::fs::read_dir(&self.inner.sockets) else {
293 return;
294 };
295 for dir in dirs.flatten() {
296 if !keep
297 .iter()
298 .any(|name| *name == dir.file_name().to_string_lossy())
299 {
300 let _ = std::fs::remove_dir_all(dir.path());
301 }
302 }
303 }
304
305 pub fn undecided(&self, runner: &str) -> bool {
307 lock(&self.inner.state)
308 .runners
309 .get(runner)
310 .is_some_and(|runner| runner.writes == Writes::Undecided)
311 }
312
313 pub fn decide(&self, runner: &str, writes: Writes) {
315 let held = {
316 let mut state = lock(&self.inner.state);
317 let Some(open) = state.runners.get_mut(runner) else {
318 return;
319 };
320 if open.writes != Writes::Undecided || writes == Writes::Undecided {
321 return;
322 }
323 open.writes = writes;
324 std::mem::take(&mut open.held)
325 };
326 match writes {
327 Writes::Allowed => {
328 for upload in held {
329 self.inner.store(runner, upload);
330 }
331 }
332 Writes::Refused | Writes::Undecided => {
333 if let Some(open) = lock(&self.inner.state).runners.get_mut(runner) {
334 open.counts.refused += held.len() as u64;
335 }
336 self.inner.remove_uploads(Some(runner), &held);
337 }
338 }
339 }
340
341 pub fn close(&self, runner: &str) -> bool {
344 let dir = {
345 let state = lock(&self.inner.state);
346 let Some(open) = state.runners.get(runner) else {
347 return false;
348 };
349 if open.writes != Writes::Undecided || open.held.is_empty() {
350 return false;
351 }
352 open.serving.abort();
353 open.dir.clone()
354 };
355 if let Err(error) = std::fs::remove_dir_all(&dir)
356 && error.kind() != io::ErrorKind::NotFound
357 {
358 warn!("cannot remove {}: {error}", dir.display());
359 }
360 true
361 }
362
363 pub fn detach(&self, runner: &str) -> Option<BuildCacheCounts> {
366 let Some(mut open) = lock(&self.inner.state).runners.remove(runner) else {
367 let dir = self.inner.sockets.join(socket_dir(runner));
369 if let Err(error) = std::fs::remove_dir_all(&dir)
370 && error.kind() != io::ErrorKind::NotFound
371 {
372 warn!("cannot remove {}: {error}", dir.display());
373 }
374 return None;
375 };
376 open.serving.abort();
377 self.inner.remove_uploads(None, &open.held);
378 open.counts.refused += open.held.len() as u64;
379 open.counts.writer = open.writes == Writes::Allowed;
380 if let Err(error) = std::fs::remove_dir_all(&open.dir)
381 && error.kind() != io::ErrorKind::NotFound
382 {
383 warn!("cannot remove {}: {error}", open.dir.display());
384 }
385 Some(open.counts)
386 }
387
388 pub fn size(&self) -> Size {
390 let state = lock(&self.inner.state);
391 Size {
392 bytes: state.bytes,
393 entries: state.entries.len() as u64,
394 }
395 }
396
397 pub fn trim(&self) -> Trimmed {
401 let limit = lock(&self.inner.state).max_size;
402 self.trim_to(limit)
403 }
404
405 pub fn clear(&self) -> Trimmed {
407 self.trim_to(0)
408 }
409
410 fn trim_to(&self, limit: u64) -> Trimmed {
411 let (before, victims) = {
412 let mut state = lock(&self.inner.state);
413 let before = Size {
414 bytes: state.bytes,
415 entries: state.entries.len() as u64,
416 };
417 let mut by_use: Vec<(PathBuf, Entry)> = state
418 .entries
419 .iter()
420 .map(|(path, entry)| (path.clone(), *entry))
421 .collect();
422 by_use.sort_by_key(|(path, entry)| (entry.used, path.clone()));
423 let mut victims = Vec::new();
424 for (path, entry) in by_use {
425 if state.charged <= limit && (limit > 0 || state.entries.is_empty()) {
426 break;
427 }
428 state.entries.remove(&path);
429 state.bytes -= entry.bytes;
430 state.charged -= charge(entry.bytes);
431 victims.push(path);
432 }
433 (before, victims)
434 };
435 for path in victims {
436 let state = lock(&self.inner.state);
439 if state.entries.contains_key(&path) {
440 continue;
441 }
442 match std::fs::remove_file(self.inner.root.join(&path)) {
443 Err(error) if error.kind() != io::ErrorKind::NotFound => {
444 warn!(
445 "cannot remove build cache entry {}: {error}",
446 path.display()
447 );
448 }
449 _ => self.inner.remove_empty_parents(&path),
450 }
451 drop(state);
452 }
453 Trimmed {
454 before,
455 after: self.size(),
456 }
457 }
458}
459
460impl Inner {
461 fn store(&self, runner: &str, upload: Upload) {
465 let mut state = lock(&self.state);
468 let replaced = state
469 .entries
470 .get(&upload.entry)
471 .map_or(0, |entry| charge(entry.bytes));
472 let fits =
473 state.charged - replaced + charge(upload.bytes) <= state.max_size.saturating_mul(2);
474 let path = self.root.join(&upload.entry);
475 let stored = fits
476 && path
477 .parent()
478 .is_none_or(|parent| std::fs::create_dir_all(parent).is_ok())
479 && std::fs::rename(&upload.file, &path).is_ok();
480 Self::give_back(&mut state, Some(runner), upload.room);
481 if stored {
482 let entry = Entry {
483 bytes: upload.bytes,
484 used: crate::time::now_ms(),
485 };
486 if let Some(old) = state.entries.insert(upload.entry, entry) {
487 state.bytes -= old.bytes;
488 state.charged -= charge(old.bytes);
489 }
490 state.bytes += upload.bytes;
491 state.charged += charge(upload.bytes);
492 } else {
493 let _ = std::fs::remove_file(&upload.file);
494 }
495 if let Some(open) = state.runners.get_mut(runner) {
496 if stored {
497 open.counts.written_bytes += upload.bytes;
498 } else {
499 open.counts.refused += 1;
500 }
501 }
502 }
503
504 fn reserve(&self, runner: &str, bytes: u64, max_size: u64) -> bool {
508 let mut state = lock(&self.state);
509 let total = state.uploading;
510 let Some(open) = state.runners.get_mut(runner) else {
511 return false;
512 };
513 let fits = total + bytes <= max_size
514 && (open.writes != Writes::Undecided || open.uploading + bytes <= max_size / 4);
515 if fits {
516 open.uploading += bytes;
517 state.uploading += bytes;
518 }
519 fits
520 }
521
522 fn release(&self, runner: Option<&str>, bytes: u64) {
525 Self::give_back(&mut lock(&self.state), runner, bytes);
526 }
527
528 fn give_back(state: &mut State, runner: Option<&str>, bytes: u64) {
529 state.uploading = state.uploading.saturating_sub(bytes);
530 if let Some(open) = runner.and_then(|runner| state.runners.get_mut(runner)) {
531 open.uploading = open.uploading.saturating_sub(bytes);
532 }
533 }
534
535 fn remove_uploads(&self, runner: Option<&str>, uploads: &[Upload]) {
537 for upload in uploads {
538 let _ = std::fs::remove_file(&upload.file);
539 self.release(runner, upload.room);
540 }
541 }
542
543 fn remove_empty_parents(&self, entry: &Path) {
546 let mut dir = entry.parent();
547 while let Some(parent) = dir
548 && parent.components().count() > 3
549 && std::fs::remove_dir(self.root.join(parent)).is_ok()
550 {
551 dir = parent.parent();
552 }
553 }
554
555 fn touch(&self, entry: &Path) {
557 if let Some(used) = lock(&self.state).entries.get_mut(entry) {
558 used.used = crate::time::now_ms();
559 }
560 let path = self.root.join(entry);
561 tokio::task::spawn_blocking(move || {
562 if let Ok(file) = std::fs::File::open(&path) {
563 let _ = file.set_modified(std::time::SystemTime::now());
564 }
565 });
566 }
567}
568
569async fn accept(inner: Arc<Inner>, runner: String, listener: UnixListener) {
572 let mut connections = JoinSet::new();
573 loop {
574 let full = connections.len() >= MAX_CONNECTIONS;
575 tokio::select! {
576 accepted = listener.accept(), if !full => match accepted {
577 Ok((stream, _)) => {
578 connections.spawn(serve::connection(inner.clone(), runner.clone(), stream));
579 }
580 Err(error) => {
581 warn!("{runner}: the build cache's socket failed: {error}");
582 return;
583 }
584 },
585 Some(_) = connections.join_next() => {}
586 }
587 }
588}
589
590fn socket_dir(runner: &str) -> String {
592 let hash = runner
594 .bytes()
595 .fold(0xcbf2_9ce4_8422_2325_u64, |hash, byte| {
596 (hash ^ u64::from(byte)).wrapping_mul(0x0000_0100_0000_01b3)
597 });
598 format!("{hash:016x}")
599}
600
601fn valid_segment(name: &str) -> bool {
603 !name.is_empty()
604 && name.len() <= 200
605 && name != "."
606 && name != ".."
607 && name
608 .bytes()
609 .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-'))
610}
611
612fn valid_repository(repository: &str) -> bool {
614 repository.split_once('/').is_some_and(|(owner, name)| {
615 valid_segment(owner) && valid_segment(name) && !owner.starts_with('.')
616 })
617}
618
619fn private_dir(path: &Path) -> io::Result<()> {
620 std::fs::create_dir_all(path)?;
621 std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o700))
622}
623
624fn count_entries(root: &Path, dir: &Path, entries: &mut HashMap<PathBuf, Entry>) -> io::Result<()> {
626 for item in std::fs::read_dir(dir)? {
627 let item = item?;
628 let path = item.path();
629 let metadata = item.metadata()?;
630 if metadata.is_dir() {
631 count_entries(root, &path, entries)?;
632 } else if metadata.is_file()
633 && let Ok(relative) = path.strip_prefix(root)
634 {
635 entries.insert(
636 relative.to_owned(),
637 Entry {
638 bytes: metadata.len(),
639 used: metadata.modified().map_or(0, crate::time::ms),
640 },
641 );
642 }
643 }
644 Ok(())
645}
646
647#[cfg(test)]
648mod tests;
649
650#[cfg(test)]
653#[allow(
654 clippy::unwrap_used,
655 reason = "a test helper: a socket that does not answer fails the test"
656)]
657pub(crate) async fn request(dir: &Path, method: &str, path: &str, body: &[u8]) -> (u16, Vec<u8>) {
658 use tokio::io::{AsyncReadExt, AsyncWriteExt};
659 let mut stream = tokio::net::UnixStream::connect(dir.join(SOCKET))
660 .await
661 .unwrap();
662 let head = format!(
663 "{method} {path} HTTP/1.1\r\nHost: 127.0.0.1\r\nAuthorization: Basic dG9rZW46c2VjcmV0\r\n\
664 Content-Length: {}\r\nConnection: close\r\n\r\n",
665 body.len()
666 );
667 stream.write_all(head.as_bytes()).await.unwrap();
668 stream.write_all(body).await.unwrap();
669 let mut response = Vec::new();
670 stream.read_to_end(&mut response).await.unwrap();
671 let split = response
672 .windows(4)
673 .position(|window| window == b"\r\n\r\n")
674 .unwrap();
675 let status = String::from_utf8_lossy(&response[9..12]).parse().unwrap();
676 (status, response[split + 4..].to_vec())
677}