use crate::build::{BuildLogger, PrevGitSource};
use dora_message::{DataflowId, SessionId, common::LogLevel};
use eyre::{ContextCompat, WrapErr, bail};
use git2::FetchOptions;
use itertools::Itertools;
use std::{
collections::{BTreeMap, BTreeSet},
path::{Path, PathBuf},
sync::{
Arc, Mutex,
atomic::{AtomicU64, Ordering},
},
};
use url::Url;
#[derive(Default)]
pub struct GitManager {
pub clones_in_use: BTreeMap<PathBuf, BTreeSet<DataflowId>>,
prepared_builds: BTreeMap<SessionId, PreparedBuild>,
clones_in_progress: Arc<Mutex<BTreeMap<PathBuf, usize>>>,
}
struct InProgressClaim {
dir: PathBuf,
claims: Arc<Mutex<BTreeMap<PathBuf, usize>>>,
}
impl Drop for InProgressClaim {
fn drop(&mut self) {
let mut claims = lock_in_progress(&self.claims);
if let Some(count) = claims.get_mut(&self.dir) {
*count -= 1;
if *count == 0 {
claims.remove(&self.dir);
}
}
}
}
fn lock_in_progress(
claims: &Mutex<BTreeMap<PathBuf, usize>>,
) -> std::sync::MutexGuard<'_, BTreeMap<PathBuf, usize>> {
claims
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
}
#[derive(Default)]
struct PreparedBuild {
planned_clone_dirs: BTreeSet<PathBuf>,
}
impl GitManager {
pub fn choose_clone_dir(
&mut self,
session_id: SessionId,
repo: String,
commit_hash: String,
prev_git: Option<PrevGitSource>,
target_dir: &Path,
) -> eyre::Result<GitFolder> {
let repo_url = Url::parse(&repo).context("failed to parse git repository URL")?;
let clone_dir = Self::clone_dir_path(target_dir, &repo_url, &commit_hash)?;
let prev_commit_hash = prev_git
.as_ref()
.filter(|p| p.git_source.repo == repo)
.map(|p| &p.git_source.commit_hash);
if let Some(using) = self.clones_in_use.get(&clone_dir)
&& !using.is_empty()
{
eyre::bail!(
"the build directory is still in use by the following \
dataflows, please stop them before rebuilding: {}",
using.iter().join(", ")
)
}
let reuse = if self.clone_dir_ready(session_id, &clone_dir) {
let in_progress = lock_in_progress(&self.clones_in_progress).contains_key(&clone_dir);
ReuseOptions::Reuse {
dir: clone_dir.clone(),
verify_commit: (!in_progress).then_some(commit_hash),
}
} else if let Some(previous_commit_hash) = prev_commit_hash {
let prev_clone_dir = Self::clone_dir_path(target_dir, &repo_url, previous_commit_hash)?;
if prev_clone_dir.exists() {
let still_needed = prev_git
.map(|g| g.still_needed_for_this_build)
.unwrap_or(false);
let used_by_others = self
.clones_in_use
.get(&prev_clone_dir)
.map(|ids| !ids.is_empty())
.unwrap_or(false);
if still_needed || used_by_others {
ReuseOptions::CopyAndFetch {
from: prev_clone_dir,
target_dir: clone_dir.clone(),
commit_hash,
}
} else {
ReuseOptions::RenameAndFetch {
from: prev_clone_dir,
target_dir: clone_dir.clone(),
commit_hash,
}
}
} else {
ReuseOptions::NewClone {
target_dir: clone_dir.clone(),
repo_url,
commit_hash,
}
}
} else {
ReuseOptions::NewClone {
target_dir: clone_dir.clone(),
repo_url,
commit_hash,
}
};
self.register_ready_clone_dir(session_id, clone_dir.clone());
let claim = matches!(
reuse,
ReuseOptions::NewClone { .. }
| ReuseOptions::CopyAndFetch { .. }
| ReuseOptions::RenameAndFetch { .. }
)
.then(|| {
*lock_in_progress(&self.clones_in_progress)
.entry(clone_dir.clone())
.or_insert(0) += 1;
InProgressClaim {
dir: clone_dir,
claims: self.clones_in_progress.clone(),
}
});
Ok(GitFolder {
reuse,
_claim: claim,
})
}
pub fn clone_dir_ready(&self, session_id: SessionId, dir: &Path) -> bool {
self.prepared_builds
.get(&session_id)
.map(|p| p.planned_clone_dirs.contains(dir))
.unwrap_or(false)
|| dir.exists()
}
pub fn register_ready_clone_dir(&mut self, session_id: SessionId, dir: PathBuf) -> bool {
self.prepared_builds
.entry(session_id)
.or_default()
.planned_clone_dirs
.insert(dir)
}
fn clone_dir_path(
base_dir: &Path,
repo_url: &Url,
commit_hash: &String,
) -> eyre::Result<PathBuf> {
let host = repo_url.host_str().unwrap_or("localhost");
let mut path = base_dir.join(sanitize_dir_component(host));
path.extend(
repo_url
.path_segments()
.context("no path in git URL")?
.map(sanitize_dir_component),
);
let path = path.join(commit_hash);
Ok(dunce::simplified(&path).to_owned())
}
pub fn clear_planned_builds(&mut self, session_id: SessionId) {
self.prepared_builds.remove(&session_id);
}
}
pub struct GitFolder {
reuse: ReuseOptions,
_claim: Option<InProgressClaim>,
}
impl GitFolder {
pub async fn prepare(self, logger: &mut impl BuildLogger) -> eyre::Result<PathBuf> {
let GitFolder { reuse, _claim } = self;
tracing::info!("reuse: {reuse:?}");
let clone_dir = match reuse {
ReuseOptions::NewClone {
target_dir,
repo_url,
commit_hash,
} => {
logger
.log_message(
LogLevel::Info,
format!(
"cloning {repo_url}#{commit_hash} into {}",
target_dir.display()
),
)
.await;
let tmp_dir = partial_clone_path(&target_dir);
let clone_target = tmp_dir.clone();
let checkout_result = match tokio::task::spawn_blocking(move || {
let repository = clone_into(repo_url.clone(), &clone_target)
.with_context(|| format!("failed to clone git repo from `{repo_url}`"))?;
checkout_tree(&repository, &commit_hash)
.with_context(|| format!("failed to checkout commit `{commit_hash}`"))
})
.await
{
Ok(result) => result,
Err(join_err) => {
Err(eyre::Report::new(join_err)).context("git clone/checkout task panicked")
}
};
match checkout_result {
Ok(()) => promote_clone(logger, &tmp_dir, &target_dir).await?,
Err(err) => {
logger
.log_message(LogLevel::Error, format!("{err:?}"))
.await;
cleanup_failed_clone(logger, &tmp_dir).await;
bail!(err)
}
}
}
ReuseOptions::CopyAndFetch {
from,
target_dir,
commit_hash,
} => {
let tmp_dir = partial_clone_path(&target_dir);
let from_clone = from.clone();
let to = tmp_dir.clone();
let result: eyre::Result<()> = async {
tokio::task::spawn_blocking(move || {
std::fs::create_dir_all(&to)
.context("failed to create directory for copying git repo")?;
fs_extra::dir::copy(
&from_clone,
&to,
&fs_extra::dir::CopyOptions::new().content_only(true),
)
.with_context(|| {
format!(
"failed to copy repo clone from `{}` to `{}`",
from_clone.display(),
to.display()
)
})
})
.await??;
logger
.log_message(
LogLevel::Info,
format!("fetching changes after copying {}", from.display()),
)
.await;
let repository = fetch_changes(&tmp_dir, None).await?;
checkout_tree(&repository, &commit_hash)?;
Ok(())
}
.await;
match result {
Ok(()) => promote_clone(logger, &tmp_dir, &target_dir).await?,
Err(err) => {
cleanup_failed_clone(logger, &tmp_dir).await;
bail!(err)
}
}
}
ReuseOptions::RenameAndFetch {
from,
target_dir,
commit_hash,
} => {
let tmp_dir = partial_clone_path(&target_dir);
tokio::fs::rename(&from, &tmp_dir)
.await
.context("failed to rename repo clone")?;
logger
.log_message(
LogLevel::Info,
format!("fetching changes after renaming {}", from.display()),
)
.await;
let result: eyre::Result<()> = async {
let repository = fetch_changes(&tmp_dir, None).await?;
checkout_tree(&repository, &commit_hash)?;
Ok(())
}
.await;
match result {
Ok(()) => promote_clone(logger, &tmp_dir, &target_dir).await?,
Err(err) => {
cleanup_failed_clone(logger, &tmp_dir).await;
bail!(err)
}
}
}
ReuseOptions::Reuse { dir, verify_commit } => {
if let Some(commit_hash) = verify_commit
&& dir.exists()
&& is_full_commit_hash(&commit_hash)
{
let repo_dir = dir.clone();
let head = tokio::task::spawn_blocking(move || -> eyre::Result<String> {
let repo =
git2::Repository::open(&repo_dir).context("failed to open git repo")?;
let id = repo
.head()
.context("failed to read HEAD")?
.peel_to_commit()
.context("failed to resolve HEAD commit")?
.id()
.to_string();
Ok(id)
})
.await
.context("HEAD read task panicked")?;
match head {
Ok(h) if h.eq_ignore_ascii_case(&commit_hash) => {}
Ok(h) => {
cleanup_failed_clone(logger, &dir).await;
bail!(
"clone dir {} is not on the requested commit {commit_hash} \
(found {h}); I removed it, please rebuild",
dir.display()
);
}
Err(err) => bail!(
"couldn't verify clone dir {} is on commit {commit_hash}: {err:?}; \
leaving it in place in case another build is writing it, \
please retry",
dir.display()
),
}
}
logger
.log_message(
LogLevel::Info,
format!("reusing up-to-date {}", dir.display()),
)
.await;
dir
}
};
Ok(clone_dir)
}
}
#[derive(Debug)]
enum ReuseOptions {
NewClone {
target_dir: PathBuf,
repo_url: Url,
commit_hash: String,
},
Reuse {
dir: PathBuf,
verify_commit: Option<String>,
},
CopyAndFetch {
from: PathBuf,
target_dir: PathBuf,
commit_hash: String,
},
RenameAndFetch {
from: PathBuf,
target_dir: PathBuf,
commit_hash: String,
},
}
async fn cleanup_failed_clone(logger: &mut impl BuildLogger, dir: &Path) {
match tokio::fs::remove_dir_all(dir).await {
Ok(()) => {}
Err(err) if err.kind() == std::io::ErrorKind::NotFound => {}
Err(err) => {
logger
.log_message(
LogLevel::Error,
format!(
"couldn't remove clone dir after a failed build: {}",
err.kind()
),
)
.await;
}
}
}
fn partial_clone_path(target: &Path) -> PathBuf {
static COUNTER: AtomicU64 = AtomicU64::new(0);
let n = COUNTER.fetch_add(1, Ordering::Relaxed);
let pid = std::process::id();
let name = target
.file_name()
.map(|n| n.to_string_lossy())
.unwrap_or_default();
target.with_file_name(format!(".{name}.partial-{pid}-{n}"))
}
async fn promote_clone(
logger: &mut impl BuildLogger,
tmp: &Path,
target: &Path,
) -> eyre::Result<PathBuf> {
match tokio::fs::rename(tmp, target).await {
Ok(()) => Ok(target.to_owned()),
Err(_) if target.exists() => {
cleanup_failed_clone(logger, tmp).await;
Ok(target.to_owned())
}
Err(err) => {
logger
.log_message(LogLevel::Error, format!("{err:?}"))
.await;
cleanup_failed_clone(logger, tmp).await;
bail!(
"failed to move finished clone from {} into {}: {err}",
tmp.display(),
target.display()
)
}
}
}
fn is_full_commit_hash(s: &str) -> bool {
matches!(s.len(), 40 | 64) && s.bytes().all(|b| b.is_ascii_hexdigit())
}
fn sanitize_dir_component(component: &str) -> String {
component
.chars()
.map(|c| {
if c.is_control() || matches!(c, '<' | '>' | ':' | '"' | '|' | '?' | '*' | '\\' | '/') {
'_'
} else {
c
}
})
.collect()
}
fn clone_into(repo_addr: Url, clone_dir: &Path) -> eyre::Result<git2::Repository> {
if let Some(parent) = clone_dir.parent() {
std::fs::create_dir_all(parent)
.context("failed to create parent directory for git clone")?;
}
let clone_dir = clone_dir.to_owned();
let mut builder = git2::build::RepoBuilder::new();
let mut fetch_options = git2::FetchOptions::new();
fetch_options.download_tags(git2::AutotagOption::All);
builder.fetch_options(fetch_options);
builder
.clone(repo_addr.as_str(), &clone_dir)
.context("failed to clone repo")
}
async fn fetch_changes(
repo_dir: &Path,
refname: Option<String>,
) -> Result<git2::Repository, eyre::Error> {
let repo_dir = repo_dir.to_owned();
let fetch_changes = tokio::task::spawn_blocking(move || {
let repository = git2::Repository::open(&repo_dir).context("failed to open git repo")?;
{
let mut remote = repository
.find_remote("origin")
.context("failed to find remote `origin` in repo")?;
remote
.connect(git2::Direction::Fetch)
.context("failed to connect to remote")?;
let default_branch = remote
.default_branch()
.context("failed to get default branch for remote")?;
let fetch = match &refname {
Some(refname) => refname,
None => default_branch
.as_str()
.context("failed to read default branch as string")?,
};
let mut fetch_options = FetchOptions::new();
fetch_options.download_tags(git2::AutotagOption::All);
remote
.fetch(&[&fetch], Some(&mut fetch_options), None)
.context("failed to fetch from git repo")?;
}
Result::<_, eyre::Error>::Ok(repository)
});
let repository = fetch_changes.await??;
Ok(repository)
}
fn checkout_tree(repository: &git2::Repository, commit_hash: &str) -> eyre::Result<()> {
if commit_hash.contains("..")
|| commit_hash.contains(':')
|| commit_hash.contains('^')
|| commit_hash.contains('~')
|| commit_hash.contains('@')
|| commit_hash.contains('{')
|| commit_hash.contains('}')
{
eyre::bail!(
"invalid commit reference '{commit_hash}': rev-spec expressions are not allowed"
);
}
let (object, reference) = repository
.revparse_ext(commit_hash)
.context("failed to parse ref")?;
repository
.checkout_tree(&object, None)
.context("failed to checkout ref")?;
match reference {
Some(reference) => repository
.set_head(reference.name().context("failed to get reference_name")?)
.context("failed to set head")?,
None => repository
.set_head_detached(object.id())
.context("failed to set detached head")?,
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use dora_message::common::LogLevelOrStdout;
struct TestLogger;
impl BuildLogger for TestLogger {
type Clone = TestLogger;
async fn log_message(
&mut self,
_level: impl Into<LogLevelOrStdout> + Send,
_message: impl Into<String> + Send,
) {
}
async fn try_clone(&self) -> eyre::Result<Self::Clone> {
Ok(TestLogger)
}
}
fn init_repo_with_commit(path: &Path) -> String {
let repo = git2::Repository::init(path).unwrap();
std::fs::write(path.join("file.txt"), b"A").unwrap();
let mut index = repo.index().unwrap();
index.add_path(Path::new("file.txt")).unwrap();
index.write().unwrap();
let tree_id = index.write_tree().unwrap();
let tree = repo.find_tree(tree_id).unwrap();
let sig = git2::Signature::now("t", "t@t").unwrap();
repo.commit(Some("HEAD"), &sig, &sig, "A", &tree, &[])
.unwrap()
.to_string()
}
fn file_url(path: &Path) -> Url {
Url::from_file_path(path).expect("test repo path must be absolute")
}
#[test]
fn clone_dir_path_keeps_a_file_url_drive_letter_out_of_the_on_disk_path() {
let url = Url::parse("file:///C:/Users/runner/repo").unwrap();
let dir = GitManager::clone_dir_path(Path::new("base"), &url, &"a".repeat(40)).unwrap();
for component in dir.components() {
let name = component.as_os_str().to_string_lossy();
assert!(
!name.contains(':'),
"component `{name}` keeps a colon Windows rejects, in {}",
dir.display()
);
}
}
#[tokio::test]
async fn rename_and_fetch_removes_dir_when_fetch_fails() {
let base = tempfile::tempdir().unwrap();
let from = base.path().join("from");
let target = base.path().join("target");
init_repo_with_commit(&from);
let folder = GitFolder {
reuse: ReuseOptions::RenameAndFetch {
from,
target_dir: target.clone(),
commit_hash: "deadbeef".repeat(5),
},
_claim: None,
};
assert!(folder.prepare(&mut TestLogger).await.is_err());
assert!(
!target.exists(),
"a failed rename+fetch must not leave the dir behind"
);
}
#[tokio::test]
async fn copy_and_fetch_removes_dir_when_fetch_fails() {
let base = tempfile::tempdir().unwrap();
let from = base.path().join("from");
let target = base.path().join("target");
init_repo_with_commit(&from);
let folder = GitFolder {
reuse: ReuseOptions::CopyAndFetch {
from,
target_dir: target.clone(),
commit_hash: "deadbeef".repeat(5),
},
_claim: None,
};
assert!(folder.prepare(&mut TestLogger).await.is_err());
assert!(
!target.exists(),
"a failed copy+fetch must not leave the dir behind"
);
}
#[tokio::test]
async fn reuse_bails_and_removes_dir_on_head_mismatch() {
let base = tempfile::tempdir().unwrap();
let dir = base.path().join("clone");
init_repo_with_commit(&dir);
let folder = GitFolder {
reuse: ReuseOptions::Reuse {
dir: dir.clone(),
verify_commit: Some("0".repeat(40)),
},
_claim: None,
};
assert!(folder.prepare(&mut TestLogger).await.is_err());
assert!(
!dir.exists(),
"a clone on the wrong commit must be removed so the next build re-clones"
);
}
#[tokio::test]
async fn reuse_ok_when_head_matches() {
let base = tempfile::tempdir().unwrap();
let dir = base.path().join("clone");
let oid = init_repo_with_commit(&dir);
let folder = GitFolder {
reuse: ReuseOptions::Reuse {
dir: dir.clone(),
verify_commit: Some(oid),
},
_claim: None,
};
assert_eq!(folder.prepare(&mut TestLogger).await.unwrap(), dir);
assert!(dir.exists());
}
#[tokio::test]
async fn reuse_leaves_the_dir_alone_when_head_wont_resolve() {
let base = tempfile::tempdir().unwrap();
let dir = base.path().join("clone");
std::fs::create_dir_all(&dir).unwrap();
std::fs::write(dir.join("partial.txt"), b"mid-clone").unwrap();
let folder = GitFolder {
reuse: ReuseOptions::Reuse {
dir: dir.clone(),
verify_commit: Some("0".repeat(40)),
},
_claim: None,
};
assert!(folder.prepare(&mut TestLogger).await.is_err());
assert!(
dir.exists(),
"an unresolvable HEAD must not delete the dir, another build may be writing it"
);
}
#[tokio::test]
async fn reuse_skips_verification_for_branch_ref() {
let base = tempfile::tempdir().unwrap();
let dir = base.path().join("clone");
init_repo_with_commit(&dir);
let folder = GitFolder {
reuse: ReuseOptions::Reuse {
dir: dir.clone(),
verify_commit: Some("main".into()),
},
_claim: None,
};
assert!(folder.prepare(&mut TestLogger).await.is_ok());
}
#[tokio::test]
async fn reuse_does_not_verify_a_dir_a_concurrent_session_is_cloning_into() {
let repo_dir = tempfile::tempdir().unwrap();
let repo_path = repo_dir.path().join("repo");
let commit = init_repo_with_commit(&repo_path);
let repo_url = file_url(&repo_path);
let repo_url_str = repo_url.to_string();
let target_dir = tempfile::tempdir().unwrap();
let mut manager = GitManager::default();
let session_a = SessionId::generate();
let folder_a = manager
.choose_clone_dir(
session_a,
repo_url_str.clone(),
commit.clone(),
None,
target_dir.path(),
)
.unwrap();
assert!(matches!(folder_a.reuse, ReuseOptions::NewClone { .. }));
let clone_dir = GitManager::clone_dir_path(target_dir.path(), &repo_url, &commit).unwrap();
std::fs::create_dir_all(&clone_dir).unwrap();
let session_b = SessionId::generate();
let folder_b = manager
.choose_clone_dir(session_b, repo_url_str, commit, None, target_dir.path())
.unwrap();
assert!(
matches!(
&folder_b.reuse,
ReuseOptions::Reuse {
verify_commit: None,
..
}
),
"a dir a concurrent session is still cloning into must be reused without verification"
);
drop(folder_a);
}
#[tokio::test]
async fn a_dir_stays_protected_while_any_overlapping_claim_is_alive() {
let repo_dir = tempfile::tempdir().unwrap();
let repo_path = repo_dir.path().join("repo");
let commit = init_repo_with_commit(&repo_path);
let repo_url = file_url(&repo_path);
let repo_url_str = repo_url.to_string();
let target_dir = tempfile::tempdir().unwrap();
let mut manager = GitManager::default();
let folder_a = manager
.choose_clone_dir(
SessionId::generate(),
repo_url_str.clone(),
commit.clone(),
None,
target_dir.path(),
)
.unwrap();
let folder_b = manager
.choose_clone_dir(
SessionId::generate(),
repo_url_str.clone(),
commit.clone(),
None,
target_dir.path(),
)
.unwrap();
assert!(matches!(folder_a.reuse, ReuseOptions::NewClone { .. }));
assert!(matches!(folder_b.reuse, ReuseOptions::NewClone { .. }));
let clone_dir = GitManager::clone_dir_path(target_dir.path(), &repo_url, &commit).unwrap();
std::fs::create_dir_all(&clone_dir).unwrap();
drop(folder_a);
let folder_c = manager
.choose_clone_dir(
SessionId::generate(),
repo_url_str.clone(),
commit.clone(),
None,
target_dir.path(),
)
.unwrap();
assert!(
matches!(
&folder_c.reuse,
ReuseOptions::Reuse {
verify_commit: None,
..
}
),
"dropping one of two overlapping claims must not expose the dir to verification"
);
drop(folder_b);
drop(folder_c);
let folder_d = manager
.choose_clone_dir(
SessionId::generate(),
repo_url_str,
commit,
None,
target_dir.path(),
)
.unwrap();
assert!(
matches!(
&folder_d.reuse,
ReuseOptions::Reuse {
verify_commit: Some(_),
..
}
),
"once every claim is gone the dir must be verified again"
);
}
#[tokio::test]
async fn reuse_still_verifies_a_dir_left_by_a_different_finished_session() {
let repo_dir = tempfile::tempdir().unwrap();
let repo_path = repo_dir.path().join("repo");
let old_commit = init_repo_with_commit(&repo_path);
let repo_url = file_url(&repo_path).to_string();
let target_dir = tempfile::tempdir().unwrap();
let mut manager = GitManager::default();
let session_a = SessionId::generate();
let folder_a = manager
.choose_clone_dir(
session_a,
repo_url.clone(),
old_commit.clone(),
None,
target_dir.path(),
)
.unwrap();
let clone_dir = folder_a.prepare(&mut TestLogger).await.unwrap();
std::fs::write(clone_dir.join("file.txt"), b"B").unwrap();
{
let repo = git2::Repository::open(&clone_dir).unwrap();
let parent = repo.head().unwrap().peel_to_commit().unwrap();
let mut index = repo.index().unwrap();
index.add_path(Path::new("file.txt")).unwrap();
index.write().unwrap();
let tree_id = index.write_tree().unwrap();
let tree = repo.find_tree(tree_id).unwrap();
let sig = git2::Signature::now("t", "t@t").unwrap();
repo.commit(Some("HEAD"), &sig, &sig, "B", &tree, &[&parent])
.unwrap();
}
let session_b = SessionId::generate();
let folder_b = manager
.choose_clone_dir(session_b, repo_url, old_commit, None, target_dir.path())
.unwrap();
assert!(
matches!(
&folder_b.reuse,
ReuseOptions::Reuse {
verify_commit: Some(_),
..
}
),
"a dir left by a different, finished session must still be verified, not silently reused"
);
assert!(folder_b.prepare(&mut TestLogger).await.is_err());
assert!(
!clone_dir.exists(),
"the stale wrong-commit clone must be removed"
);
}
#[tokio::test]
async fn reuse_without_verify_leaves_a_sibling_clone_alone() {
let base = tempfile::tempdir().unwrap();
let dir = base.path().join("clone");
init_repo_with_commit(&dir);
let folder = GitFolder {
reuse: ReuseOptions::Reuse {
dir: dir.clone(),
verify_commit: None,
},
_claim: None,
};
assert_eq!(folder.prepare(&mut TestLogger).await.unwrap(), dir);
assert!(dir.exists(), "a sibling-owned clone must never be deleted");
}
fn repo_with_two_commits() -> (tempfile::TempDir, git2::Repository) {
let dir = tempfile::tempdir().unwrap();
let repository = git2::Repository::init(dir.path()).unwrap();
let signature = git2::Signature::now("test", "test@dora.rs").unwrap();
{
let tree_id = repository.index().unwrap().write_tree().unwrap();
let tree = repository.find_tree(tree_id).unwrap();
let first = repository
.commit(Some("HEAD"), &signature, &signature, "first", &tree, &[])
.unwrap();
let first_commit = repository.find_commit(first).unwrap();
repository
.commit(
Some("HEAD"),
&signature,
&signature,
"second",
&tree,
&[&first_commit],
)
.unwrap();
}
(dir, repository)
}
#[test]
fn rejects_rev_spec_navigation_operators() {
let (_dir, repository) = repo_with_two_commits();
for resolvable in ["HEAD~1", "HEAD^"] {
assert!(
repository.revparse_ext(resolvable).is_ok(),
"test setup: `{resolvable}` should resolve in a two-commit repo"
);
}
for commit_hash in [
"HEAD~1",
"main~1",
"HEAD@{1}",
"main@{upstream}",
"@",
"HEAD^",
"main..HEAD",
"HEAD:foo",
] {
let err = checkout_tree(&repository, commit_hash).unwrap_err();
assert!(
format!("{err:#}").contains("rev-spec expressions are not allowed"),
"expected `{commit_hash}` to be rejected as a rev-spec, got: {err:#}"
);
}
}
#[test]
fn accepts_branch_name_and_commit_hash() {
let (_dir, repository) = repo_with_two_commits();
let head = repository.head().unwrap();
let branch_name = head.shorthand().unwrap().to_string();
let head_commit = head.peel_to_commit().unwrap().id();
checkout_tree(&repository, &branch_name).unwrap();
checkout_tree(&repository, &head_commit.to_string()).unwrap();
}
fn partial_prefix(target: &Path) -> String {
format!(
".{}.partial-",
target.file_name().unwrap().to_string_lossy()
)
}
fn has_partial_leftover(target: &Path) -> bool {
let prefix = partial_prefix(target);
std::fs::read_dir(target.parent().unwrap())
.unwrap()
.filter_map(|e| e.ok())
.any(|e| e.file_name().to_string_lossy().starts_with(&prefix))
}
fn clone_from_origin(origin: &Path, dest: &Path) {
git2::Repository::clone(file_url(origin).as_str(), dest).unwrap();
}
fn head_commit(dir: &Path) -> String {
git2::Repository::open(dir)
.unwrap()
.head()
.unwrap()
.peel_to_commit()
.unwrap()
.id()
.to_string()
}
#[test]
fn partial_clone_path_is_a_unique_sibling() {
let target = Path::new("/base/localhost/org/repo").join("a".repeat(40));
let p1 = partial_clone_path(&target);
let p2 = partial_clone_path(&target);
assert_ne!(p1, target);
assert_ne!(p1, p2, "each call must produce a distinct temp path");
assert_eq!(p1.parent(), target.parent(), "temp must be a sibling");
let name = p1.file_name().unwrap().to_string_lossy();
assert!(name.starts_with(&partial_prefix(&target)));
}
#[tokio::test]
async fn promote_clone_moves_temp_into_absent_target() {
let base = tempfile::tempdir().unwrap();
let target = base.path().join("clone");
let tmp = partial_clone_path(&target);
std::fs::create_dir_all(&tmp).unwrap();
std::fs::write(tmp.join("f"), b"data").unwrap();
let out = promote_clone(&mut TestLogger, &tmp, &target).await.unwrap();
assert_eq!(out, target);
assert!(!tmp.exists(), "temp must be consumed by the rename");
assert_eq!(std::fs::read(target.join("f")).unwrap(), b"data");
}
#[tokio::test]
async fn promote_clone_reuses_winner_and_drops_temp_on_race() {
let base = tempfile::tempdir().unwrap();
let target = base.path().join("clone");
let tmp = partial_clone_path(&target);
std::fs::create_dir_all(&tmp).unwrap();
std::fs::write(tmp.join("mine"), b"mine").unwrap();
std::fs::create_dir_all(&target).unwrap();
std::fs::write(target.join("winner"), b"winner").unwrap();
let out = promote_clone(&mut TestLogger, &tmp, &target).await.unwrap();
assert_eq!(out, target);
assert!(!tmp.exists(), "our redundant temp must be dropped");
assert!(
target.join("winner").exists() && !target.join("mine").exists(),
"the winner's clone must be reused untouched"
);
}
#[tokio::test]
async fn new_clone_promotes_temp_into_target_and_cleans_up() {
let repo_dir = tempfile::tempdir().unwrap();
let repo_path = repo_dir.path().join("repo");
let commit = init_repo_with_commit(&repo_path);
let repo_url = file_url(&repo_path);
let base = tempfile::tempdir().unwrap();
let target = base.path().join("localhost").join(&commit);
let folder = GitFolder {
reuse: ReuseOptions::NewClone {
target_dir: target.clone(),
repo_url,
commit_hash: commit.clone(),
},
_claim: None,
};
let out = folder.prepare(&mut TestLogger).await.unwrap();
assert_eq!(out, target);
assert_eq!(head_commit(&target), commit);
assert!(
!has_partial_leftover(&target),
"temp dir must be gone after promotion"
);
}
#[tokio::test]
async fn new_clone_failure_leaves_no_target_dir() {
let base = tempfile::tempdir().unwrap();
let target = base.path().join("localhost").join("a".repeat(40));
std::fs::create_dir_all(target.parent().unwrap()).unwrap();
let missing = base.path().join("does-not-exist");
let repo_url = file_url(&missing);
let folder = GitFolder {
reuse: ReuseOptions::NewClone {
target_dir: target.clone(),
repo_url,
commit_hash: "a".repeat(40),
},
_claim: None,
};
assert!(folder.prepare(&mut TestLogger).await.is_err());
assert!(
!target.exists(),
"a failed clone must never leave a dir at the target path"
);
assert!(!has_partial_leftover(&target), "temp must be cleaned up");
}
#[tokio::test]
async fn copy_and_fetch_promotes_into_target() {
let base = tempfile::tempdir().unwrap();
let origin = base.path().join("origin");
let commit = init_repo_with_commit(&origin);
let from = base.path().join("localhost").join("prev");
clone_from_origin(&origin, &from);
let target = base.path().join("localhost").join(&commit);
let folder = GitFolder {
reuse: ReuseOptions::CopyAndFetch {
from: from.clone(),
target_dir: target.clone(),
commit_hash: commit.clone(),
},
_claim: None,
};
let out = folder.prepare(&mut TestLogger).await.unwrap();
assert_eq!(out, target);
assert_eq!(head_commit(&target), commit);
assert!(from.exists(), "the source clone must be left in place");
assert!(!has_partial_leftover(&target), "temp must be gone");
}
#[tokio::test]
async fn copy_and_fetch_failure_leaves_no_target_dir() {
let base = tempfile::tempdir().unwrap();
let from = base.path().join("localhost").join("prev");
init_repo_with_commit(&from);
let target = base.path().join("localhost").join("a".repeat(40));
let folder = GitFolder {
reuse: ReuseOptions::CopyAndFetch {
from,
target_dir: target.clone(),
commit_hash: "deadbeef".repeat(5),
},
_claim: None,
};
assert!(folder.prepare(&mut TestLogger).await.is_err());
assert!(!target.exists(), "a failed copy+fetch must leave no target");
assert!(!has_partial_leftover(&target), "temp must be cleaned up");
}
#[tokio::test]
async fn rename_and_fetch_promotes_into_target() {
let base = tempfile::tempdir().unwrap();
let origin = base.path().join("origin");
let commit = init_repo_with_commit(&origin);
let from = base.path().join("localhost").join("prev");
clone_from_origin(&origin, &from);
let target = base.path().join("localhost").join(&commit);
let folder = GitFolder {
reuse: ReuseOptions::RenameAndFetch {
from: from.clone(),
target_dir: target.clone(),
commit_hash: commit.clone(),
},
_claim: None,
};
let out = folder.prepare(&mut TestLogger).await.unwrap();
assert_eq!(out, target);
assert_eq!(head_commit(&target), commit);
assert!(!from.exists(), "the source clone is consumed by the rename");
assert!(!has_partial_leftover(&target), "temp must be gone");
}
#[tokio::test]
async fn rename_and_fetch_failure_leaves_no_target_dir() {
let base = tempfile::tempdir().unwrap();
let from = base.path().join("localhost").join("prev");
init_repo_with_commit(&from);
let target = base.path().join("localhost").join("a".repeat(40));
let folder = GitFolder {
reuse: ReuseOptions::RenameAndFetch {
from,
target_dir: target.clone(),
commit_hash: "deadbeef".repeat(5),
},
_claim: None,
};
assert!(folder.prepare(&mut TestLogger).await.is_err());
assert!(
!target.exists(),
"a failed rename+fetch must leave no target"
);
assert!(!has_partial_leftover(&target), "temp must be cleaned up");
}
}