use std::collections::BTreeMap;
use std::sync::Arc;
use std::time::{Duration, SystemTime};
use bollard::models::{Mount, MountType};
use futures_util::StreamExt;
use tracing::warn;
use super::tick::CONCURRENT_REQUESTS;
use super::{Engine, Meanwhile, Occupant};
use crate::buildcache::{self, ActionsCache, Scopes, Writes};
use crate::cache;
use crate::config::BuildCacheWriters;
use crate::docker::DockerApi;
use crate::github::{self, GithubApi, JobDetails};
const DEFAULT_BRANCH_FOR: Duration = Duration::from_secs(3600);
pub(super) const ENDED_WAIT: Duration = Duration::from_secs(600);
#[derive(Debug, Clone)]
pub(super) struct DefaultBranch {
branch: String,
pub(super) read_at: SystemTime,
}
#[derive(Debug)]
pub(super) struct EndedUploads {
pub(super) repository: String,
pub(super) job: u64,
pub(super) ended_at: SystemTime,
}
#[derive(Debug)]
pub(super) struct Undecided {
runner: String,
repository: String,
writers: BuildCacheWriters,
job: JobDetails,
}
struct Answer {
runner: String,
repository: String,
writes: Option<Writes>,
scopes: Scoped,
default_branch: Option<String>,
}
struct Decision {
writes: Writes,
scopes: Scoped,
}
#[derive(Default)]
struct Scoped {
own: Option<String>,
base: Option<String>,
}
impl<G: GithubApi + 'static, D: DockerApi + 'static> Engine<G, D> {
pub(super) fn runner_mounts(
&self,
profile: &str,
slot: usize,
runner: &str,
repository: &str,
archives: Option<&crate::archives::RunnerArchives>,
) -> Vec<Mount> {
let mut mounts = cache::runner_mounts(&self.names, profile, slot);
mounts.extend(self.attach_build_cache(runner, repository, profile));
mounts.extend(archives.map(|archives| archives.mount.clone()));
mounts
}
pub(super) fn close_build_cache(&self, runner: &str) {
if let Some(cache) = &self.build_cache {
cache.detach(runner);
}
}
pub(super) fn attach_build_cache(
&self,
runner: &str,
repository: &str,
profile: &str,
) -> Option<Mount> {
let cache = self.build_cache.as_ref()?;
let wanted = self.config.caches.build.enabled
&& self
.config
.pool_in(repository, profile)
.is_some_and(|pool| pool.build_cache);
if !wanted {
return None;
}
let actions = if self.config.caches.build.actions_cache {
ActionsCache::Served
} else {
ActionsCache::Off
};
match cache.attach(runner, repository, actions) {
Ok(dir) => Some(Mount {
target: Some(buildcache::MOUNT.to_owned()),
source: Some(dir.to_string_lossy().into_owned()),
typ: Some(MountType::BIND),
read_only: Some(true),
..Default::default()
}),
Err(error) => {
warn!("{runner}: cannot open the build cache: {error}");
None
}
}
}
pub(super) fn detach_build_cache(&mut self, occupant: &Occupant, job: Option<u64>) {
let Some(cache) = &self.build_cache else {
return;
};
if let Some(job) = job
&& cache.close(&occupant.runner_name)
{
self.ended_uploads.insert(
occupant.runner_name.clone(),
EndedUploads {
repository: occupant.repository.clone(),
job,
ended_at: SystemTime::now(),
},
);
return;
}
if let Some(counts) = cache.detach(&occupant.runner_name) {
self.store.build_cache(&occupant.runner_name, counts);
}
}
pub(super) fn detach_ended(&mut self) {
let Some(cache) = &self.build_cache else {
self.ended_uploads.clear();
return;
};
let done: Vec<String> = self
.ended_uploads
.iter()
.filter(|(runner, ended)| {
!cache.undecided(runner)
|| !ended
.ended_at
.elapsed()
.is_ok_and(|waited| waited < ENDED_WAIT)
})
.map(|(runner, _)| runner.clone())
.collect();
for runner in done {
self.ended_uploads.remove(&runner);
if let Some(counts) = self
.build_cache
.as_ref()
.and_then(|cache| cache.detach(&runner))
{
self.store.build_cache(&runner, counts);
}
}
}
pub(super) fn undecided(&self, described: &[(String, u64, JobDetails)]) -> Vec<Undecided> {
let Some(cache) = &self.build_cache else {
return Vec::new();
};
let ended = self
.ended_uploads
.iter()
.map(|(runner, ended)| (runner.as_str(), Some(ended.job)));
self.occupied()
.map(|(_, occupant)| (occupant.runner_name.as_str(), occupant.named_job()))
.chain(ended)
.filter(|(runner, _)| cache.undecided(runner))
.filter_map(|(runner, job)| {
let (repository, _, details) =
described.iter().find(|(_, id, _)| job == Some(*id))?;
let writers = self
.config
.repositories
.iter()
.find(|candidate| candidate.name == *repository)
.map(|candidate| candidate.build_cache_writers)
.unwrap_or_default();
Some(Undecided {
runner: runner.to_owned(),
repository: repository.clone(),
writers,
job: details.clone(),
})
})
.collect()
}
pub(super) async fn decide_writes(&mut self, undecided: Vec<Undecided>) {
if undecided.is_empty() {
return;
}
let branches = self.config.caches.build.branches.clone();
let known = self.known_default_branches();
let github = self.github.clone();
let answers = self
.serving(
read_answers(github, undecided, branches, known),
Meanwhile::Commands,
)
.await;
self.take_answers(answers);
}
fn known_default_branches(&self) -> BTreeMap<String, String> {
self.default_branches
.iter()
.filter(|(_, known)| {
known
.read_at
.elapsed()
.is_ok_and(|age| age < DEFAULT_BRANCH_FOR)
})
.map(|(repository, known)| (repository.clone(), known.branch.clone()))
.collect()
}
fn take_answers(&mut self, answers: Vec<Answer>) {
for answer in answers {
if let Some(branch) = &answer.default_branch {
self.default_branches.insert(
answer.repository.clone(),
DefaultBranch {
branch: branch.clone(),
read_at: SystemTime::now(),
},
);
}
if let (Some(writes), Some(cache)) = (answer.writes, &self.build_cache) {
let scopes = Scopes {
own: answer.scopes.own,
base: answer.scopes.base,
default: answer.default_branch.map(|branch| branch_ref(&branch)),
};
cache.decide(&answer.runner, writes, scopes);
}
}
}
}
async fn read_answers<G: GithubApi>(
github: Arc<G>,
undecided: Vec<Undecided>,
branches: Vec<String>,
known: BTreeMap<String, String>,
) -> Vec<Answer> {
let (github, branches, known) = (&*github, &branches, &known);
futures_util::stream::iter(undecided)
.map(|undecided| async move {
let default_branch = match known.get(&undecided.repository) {
Some(branch) => Ok(branch.clone()),
None => github
.repository(&undecided.repository)
.await
.map(|details| details.default_branch),
};
let decision = decide(github, &undecided, branches, default_branch.as_ref()).await;
let (writes, scopes) = match decision {
Some(decision) => (Some(decision.writes), decision.scopes),
None => (None, Scoped::default()),
};
Answer {
runner: undecided.runner,
repository: undecided.repository,
writes,
scopes,
default_branch: default_branch.ok(),
}
})
.buffered(CONCURRENT_REQUESTS)
.collect()
.await
}
async fn decide<G: GithubApi>(
github: &G,
undecided: &Undecided,
branches: &[String],
default_branch: Result<&String, &github::Error>,
) -> Option<Decision> {
let decision = |writes| Decision {
writes,
scopes: Scoped::default(),
};
let repository = &undecided.repository;
let trigger = match github.run_trigger(repository, undecided.job.run_id).await {
Ok(trigger) => trigger,
Err(error) => {
warn!(
"{repository}: cannot read run {}: {error}",
undecided.job.run_id
);
return None;
}
};
let default_branch = match default_branch {
Ok(branch) => branch,
Err(error) => {
warn!("{repository}: cannot read its default branch: {error}");
return None;
}
};
let mut writing = vec![default_branch.clone()];
writing.extend(branches.iter().cloned());
let writes = buildcache::writes(
&undecided.runner,
undecided.job.runner_name.as_deref(),
repository,
&trigger,
&writing,
undecided.writers,
);
let (Writes::Allowed, Some(branch)) = (writes, &trigger.head_branch) else {
return (writes != Writes::Undecided).then(|| decision(writes));
};
let mut scopes = Scoped {
own: Some(branch_ref(branch)),
base: None,
};
if trigger.event == buildcache::PULL_REQUEST_EVENT {
scopes.own = None;
for &number in &trigger.pull_requests {
match github.pull_request(repository, number).await {
Ok(pull) if buildcache::opened_by_user(&pull, repository, &trigger) => {
if let [_] = trigger.pull_requests.as_slice() {
scopes = Scoped {
own: Some(format!("refs/pull/{number}/merge")),
base: Some(branch_ref(&pull.base_branch)),
};
}
}
Ok(_) => return Some(decision(Writes::Refused)),
Err(error) => {
warn!("{repository}: cannot read pull request #{number}: {error}");
return None;
}
}
}
}
match github
.on_branch(repository, branch, &trigger.head_sha)
.await
{
Ok(true) => Some(Decision {
writes: Writes::Allowed,
scopes,
}),
Ok(false) => Some(decision(Writes::Refused)),
Err(error) => {
warn!(
"{repository}: cannot compare {branch} with run {}: {error}",
undecided.job.run_id
);
None
}
}
}
fn branch_ref(branch: &str) -> String {
format!("refs/heads/{branch}")
}