use crate::{
cache::Cache,
config::HttpConfig,
messages::{GitMessage, MessageReporter},
};
use backon::{BlockingRetryable, ExponentialBuilder};
use gix::{ObjectId, bstr::BString, protocol::transport::IsSpuriousError, remote::Direction};
use serde::{Deserialize, Serialize};
use snafu::{IntoError, ResultExt, prelude::*};
use std::{
fs,
path::{Path, PathBuf},
sync::atomic::AtomicBool,
};
#[derive(Debug, Snafu)]
#[snafu(visibility(pub(crate)))]
pub(crate) enum Error {
#[snafu(display("Git commit hash is invalid: {hash}"))]
InvalidCommitHash {
hash: String,
#[snafu(source(from(gix::hash::decode::Error, Box::new)))]
source: Box<gix::hash::decode::Error>,
},
#[snafu(display("Failed to initialize bare repository at {}", path.display()))]
InitBareRepo {
path: PathBuf,
#[snafu(source(from(gix::init::Error, Box::new)))]
source: Box<gix::init::Error>,
},
#[snafu(display("Failed to open git repository at {}", path.display()))]
OpenRepo {
path: PathBuf,
#[snafu(source(from(gix::open::Error, Box::new)))]
source: Box<gix::open::Error>,
},
#[snafu(display("Failed to resolve git selector: {message}"))]
ResolveSelector {
message: String,
source: Box<dyn std::error::Error + Send + Sync>,
},
#[snafu(display("Failed to fetch ref from '{url}'"))]
FetchRef {
url: String,
source: Box<dyn std::error::Error + Send + Sync>,
},
#[snafu(display("Failed to checkout from database to {}", path.display()))]
CheckoutFromDb {
path: PathBuf,
source: Box<dyn std::error::Error + Send + Sync>,
},
#[snafu(display("Failed to create git directory at {}", path.display()))]
CreateDirectory { path: PathBuf, source: std::io::Error },
#[snafu(display("Failed to write marker file at {}", path.display()))]
WriteMarkerFile { path: PathBuf, source: std::io::Error },
}
pub(crate) type Result<T> = std::result::Result<T, Error>;
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub enum GitSelector {
DefaultBranch,
Branch(String),
Tag(String),
Commit(String),
}
#[derive(Clone, Debug)]
pub(crate) struct GitClient {
cache: Cache,
reporter: MessageReporter,
http_config: HttpConfig,
}
impl GitClient {
pub(crate) fn new(cache: Cache, reporter: MessageReporter, http_config: HttpConfig) -> Self {
Self {
cache,
reporter,
http_config,
}
}
pub(crate) fn checkout_ref(&self, url: &str, selector: GitSelector) -> Result<(PathBuf, String)> {
let db_path = self.ensure_db(url)?;
self.reporter.report(|| GitMessage::resolving_ref(url, &selector));
let commit_str = if let Ok(oid) = resolve_selector(&db_path, &selector) {
let commit_str = oid.to_string();
self.reporter
.report(|| GitMessage::ref_found_locally(url, &selector, &commit_str));
commit_str
} else {
self.reporter.report(|| GitMessage::fetching_repo(url, &selector));
fetch_ref(&db_path, url, &selector, &self.http_config)?;
let oid = resolve_selector(&db_path, &selector)?;
let commit_str = oid.to_string();
self.reporter.report(|| GitMessage::resolved_ref(&commit_str));
commit_str
};
let checkout_path = self.ensure_checkout(&db_path, url, &commit_str)?;
Ok((checkout_path, commit_str))
}
fn ensure_db(&self, url: &str) -> Result<PathBuf> {
let db_path = self.cache.git_db_path(url);
if !db_path.exists() {
fs::create_dir_all(&db_path).with_context(|_| CreateDirectorySnafu {
path: db_path.clone(),
})?;
init_bare_repo(&db_path)?;
}
Ok(db_path)
}
fn ensure_checkout(&self, db_path: &Path, url: &str, commit: &str) -> Result<PathBuf> {
let checkout_path = self.cache.git_checkout_path(url, commit);
if checkout_path.exists() && checkout_path.join(".cgx-ok").exists() {
self.reporter
.report(|| GitMessage::checkout_exists(commit, &checkout_path));
return Ok(checkout_path);
}
self.reporter
.report(|| GitMessage::checking_out(commit, &checkout_path));
fs::create_dir_all(&checkout_path).with_context(|_| CreateDirectorySnafu {
path: checkout_path.clone(),
})?;
let _ = fs::remove_file(checkout_path.join(".cgx-ok"));
let commit_oid = ObjectId::from_hex(commit.as_bytes())
.map_err(|e| InvalidCommitHashSnafu { hash: commit }.into_error(e))?;
checkout_from_db(db_path, commit_oid, &checkout_path)?;
let marker_path = checkout_path.join(".cgx-ok");
fs::write(&marker_path, "").with_context(|_| WriteMarkerFileSnafu {
path: marker_path.clone(),
})?;
self.reporter
.report(|| GitMessage::checkout_complete(&checkout_path));
Ok(checkout_path)
}
}
fn init_bare_repo(path: &Path) -> Result<()> {
gix::init_bare(path)
.map_err(|e| {
InitBareRepoSnafu {
path: path.to_path_buf(),
}
.into_error(e)
})
.map(|_| ())
}
fn fetch_ref(db_path: &Path, url: &str, selector: &GitSelector, http_config: &HttpConfig) -> Result<()> {
let backoff = ExponentialBuilder::default()
.with_min_delay(http_config.backoff_base)
.with_max_delay(http_config.backoff_max)
.with_max_times(http_config.retries)
.with_jitter();
(|| fetch_ref_impl(db_path, url, selector, http_config))
.retry(backoff)
.when(is_retryable_error)
.sleep(std::thread::sleep)
.call()
}
fn is_retryable_error(e: &Error) -> bool {
let Error::FetchRef { source, .. } = e else {
return false;
};
let err = source.as_ref();
let spurious = if let Some(e) = err.downcast_ref::<gix::remote::connect::Error>() {
e.is_spurious()
} else if let Some(e) = err.downcast_ref::<gix::remote::fetch::prepare::Error>() {
e.is_spurious()
} else if let Some(e) = err.downcast_ref::<gix::remote::fetch::Error>() {
e.is_spurious()
} else {
false
};
if spurious {
return true;
}
let mut source: Option<&(dyn std::error::Error)> = Some(err);
while let Some(current) = source {
if let Some(io_err) = current.downcast_ref::<std::io::Error>() {
if io_err.to_string().contains("Received HTTP status 429") {
return true;
}
}
source = current.source();
}
false
}
fn http_config_overrides(http_config: &HttpConfig) -> Vec<BString> {
let ua = crate::http::user_agent();
let low_speed_time_secs = http_config.timeout.as_secs().max(1);
let mut overrides = vec![
format!("gitoxide.userAgent={ua}").into(),
format!("http.userAgent={ua}").into(),
format!("http.extraHeader=User-Agent: {ua}").into(),
format!("gitoxide.http.connectTimeout={}", http_config.timeout.as_millis()).into(),
"http.lowSpeedLimit=1".into(),
format!("http.lowSpeedTime={low_speed_time_secs}").into(),
];
if let Some(ref proxy) = http_config.proxy {
overrides.push(format!("http.proxy={proxy}").into());
}
overrides
}
fn http_open_options(http_config: &HttpConfig) -> gix::open::Options {
let overrides = http_config_overrides(http_config);
gix::open::Options::default().config_overrides(overrides)
}
fn fetch_ref_impl(db_path: &Path, url: &str, selector: &GitSelector, http_config: &HttpConfig) -> Result<()> {
let repo = gix::open_opts(db_path, http_open_options(http_config)).map_err(|e| {
OpenRepoSnafu {
path: db_path.to_path_buf(),
}
.into_error(e)
})?;
let refspec = match selector {
GitSelector::DefaultBranch => "+HEAD:refs/remotes/origin/HEAD".to_string(),
GitSelector::Branch(b) => format!("+refs/heads/{b}:refs/remotes/origin/{b}"),
GitSelector::Tag(t) => format!("+refs/tags/{t}:refs/remotes/origin/tags/{t}"),
GitSelector::Commit(c) if c.len() == 40 => {
format!("+{c}:refs/commit/{c}")
}
GitSelector::Commit(_) => {
"+HEAD:refs/remotes/origin/HEAD".to_string()
}
};
let remote = repo
.remote_at(url)
.map_err(|e| FetchRefSnafu { url: url.to_string() }.into_error(Box::new(e)))?
.with_refspecs([refspec.as_str()], Direction::Fetch)
.map_err(|e| FetchRefSnafu { url: url.to_string() }.into_error(Box::new(e)))?;
let connection = remote
.connect(Direction::Fetch)
.map_err(|e| FetchRefSnafu { url: url.to_string() }.into_error(Box::new(e)))?;
connection
.prepare_fetch(&mut gix::progress::Discard, Default::default())
.map_err(|e| FetchRefSnafu { url: url.to_string() }.into_error(Box::new(e)))?
.receive(&mut gix::progress::Discard, &AtomicBool::new(false))
.map_err(|e| FetchRefSnafu { url: url.to_string() }.into_error(Box::new(e)))?;
Ok(())
}
fn resolve_selector(db_path: &Path, selector: &GitSelector) -> Result<ObjectId> {
let repo = gix::open(db_path).map_err(|e| {
OpenRepoSnafu {
path: db_path.to_path_buf(),
}
.into_error(e)
})?;
let oid = match selector {
GitSelector::DefaultBranch => {
let ref_name = "refs/remotes/origin/HEAD";
let reference = repo.find_reference(ref_name).map_err(|e| {
ResolveSelectorSnafu {
message: format!("Failed to find {}", ref_name),
}
.into_error(Box::new(e))
})?;
reference
.into_fully_peeled_id()
.map_err(|e| {
ResolveSelectorSnafu {
message: "Failed to peel reference".to_string(),
}
.into_error(Box::new(e))
})?
.detach()
}
GitSelector::Branch(b) => {
let ref_name = format!("refs/remotes/origin/{}", b);
let reference = repo.find_reference(&ref_name).map_err(|e| {
ResolveSelectorSnafu {
message: format!("Branch '{}' not found", b),
}
.into_error(Box::new(e))
})?;
reference
.into_fully_peeled_id()
.map_err(|e| {
ResolveSelectorSnafu {
message: format!("Failed to peel branch '{}'", b),
}
.into_error(Box::new(e))
})?
.detach()
}
GitSelector::Tag(t) => {
let ref_name = format!("refs/remotes/origin/tags/{}", t);
let reference = repo.find_reference(&ref_name).map_err(|e| {
ResolveSelectorSnafu {
message: format!("Tag '{}' not found", t),
}
.into_error(Box::new(e))
})?;
reference
.into_fully_peeled_id()
.map_err(|e| {
ResolveSelectorSnafu {
message: format!("Failed to peel tag '{}'", t),
}
.into_error(Box::new(e))
})?
.detach()
}
GitSelector::Commit(c) => {
let spec = repo.rev_parse_single(c.as_bytes()).map_err(|e| {
ResolveSelectorSnafu {
message: format!("Failed to resolve commit '{}'", c),
}
.into_error(Box::new(e))
})?;
spec.object()
.map_err(|e| {
ResolveSelectorSnafu {
message: format!("Failed to get object for commit '{}'", c),
}
.into_error(Box::new(e))
})?
.id
}
};
Ok(oid)
}
fn checkout_from_db(db_path: &Path, commit_oid: ObjectId, dest: &Path) -> Result<()> {
let repo = gix::open(db_path).map_err(|e| {
OpenRepoSnafu {
path: db_path.to_path_buf(),
}
.into_error(e)
})?;
let commit = repo.find_commit(commit_oid).map_err(|e| {
CheckoutFromDbSnafu {
path: dest.to_path_buf(),
}
.into_error(Box::new(e))
})?;
let tree_id = commit.tree_id().map_err(|e| {
CheckoutFromDbSnafu {
path: dest.to_path_buf(),
}
.into_error(Box::new(e))
})?;
let mut index = repo.index_from_tree(&tree_id).map_err(|e| {
CheckoutFromDbSnafu {
path: dest.to_path_buf(),
}
.into_error(Box::new(e))
})?;
let options = repo
.checkout_options(gix::worktree::stack::state::attributes::Source::IdMapping)
.map_err(|e| {
CheckoutFromDbSnafu {
path: dest.to_path_buf(),
}
.into_error(Box::new(e))
})?;
gix::worktree::state::checkout(
&mut index,
dest,
repo.objects.clone(),
&gix::progress::Discard,
&gix::progress::Discard,
&AtomicBool::new(false),
options,
)
.map_err(|e| {
CheckoutFromDbSnafu {
path: dest.to_path_buf(),
}
.into_error(Box::new(e))
})?;
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use assert_matches::assert_matches;
use tempfile::TempDir;
fn test_git_client() -> (GitClient, TempDir) {
let (temp_dir, config) = crate::config::create_test_env();
let reporter = MessageReporter::null();
let cache = Cache::new(config.clone(), reporter.clone());
let git_client = GitClient::new(cache, reporter, config.http);
(git_client, temp_dir)
}
mod http_config_overrides {
use super::*;
fn overrides_to_strings(overrides: Vec<BString>) -> Vec<String> {
overrides
.into_iter()
.map(|override_value| String::from_utf8_lossy(override_value.as_ref()).into_owned())
.collect()
}
#[test]
fn includes_user_agent_and_timeout_settings() {
let (_temp_dir, config) = crate::config::create_test_env();
let overrides = overrides_to_strings(http_config_overrides(&config.http));
assert!(overrides.iter().any(|o| o.starts_with("gitoxide.userAgent=")));
assert!(overrides.iter().any(|o| o.starts_with("http.userAgent=")));
assert!(
overrides
.iter()
.any(|o| o.starts_with("gitoxide.http.connectTimeout="))
);
assert!(overrides.iter().any(|o| o == "http.lowSpeedLimit=1"));
}
#[test]
fn includes_proxy_when_configured() {
let (_temp_dir, mut config) = crate::config::create_test_env();
config.http.proxy = Some("http://proxy.example:8080".to_string());
let overrides = overrides_to_strings(http_config_overrides(&config.http));
assert!(
overrides
.iter()
.any(|o| o == "http.proxy=http://proxy.example:8080")
);
}
#[test]
fn omits_proxy_when_not_configured() {
let (_temp_dir, config) = crate::config::create_test_env();
let overrides = overrides_to_strings(http_config_overrides(&config.http));
assert!(!overrides.iter().any(|o| o.starts_with("http.proxy=")));
}
}
mod checkout_ref {
use super::*;
#[test]
fn checkout_default_branch() {
let (git_client, _temp) = test_git_client();
let url = "https://github.com/rust-lang/rustlings.git";
let (checkout_path, _commit_hash) =
git_client.checkout_ref(url, GitSelector::DefaultBranch).unwrap();
assert!(checkout_path.exists());
assert!(checkout_path.join(".cgx-ok").exists());
}
#[test]
fn checkout_specific_branch() {
let (git_client, _temp) = test_git_client();
let url = "https://github.com/rust-lang/rustlings.git";
let (checkout_path, _commit_hash) = git_client
.checkout_ref(url, GitSelector::Branch("main".to_string()))
.unwrap();
assert!(checkout_path.exists());
assert!(checkout_path.join("Cargo.toml").exists());
}
#[test]
fn checkout_specific_tag() {
let (git_client, _temp) = test_git_client();
let url = "https://github.com/rust-lang/rustlings.git";
let (checkout_path, commit_hash) = git_client
.checkout_ref(url, GitSelector::Tag("v6.0.0".to_string()))
.unwrap();
assert!(checkout_path.exists());
assert_eq!("28d2bb04326d7036514245d73f10fb72b9ed108c", &commit_hash);
}
#[test]
fn checkout_specific_advertised_commit() {
let (git_client, _temp) = test_git_client();
let url = "https://github.com/rust-lang/rustlings.git";
let commit = "28d2bb04326d7036514245d73f10fb72b9ed108c";
let (checkout_path, commit_hash) = git_client
.checkout_ref(url, GitSelector::Commit(commit.to_string()))
.unwrap();
assert!(checkout_path.exists());
assert!(checkout_path.join(".cgx-ok").exists());
assert_eq!(commit, &commit_hash);
drop(_temp);
let (git_client, _temp) = test_git_client();
let short_commit = &commit[..7];
let (checkout_path, commit_hash) = git_client
.checkout_ref(url, GitSelector::Commit(short_commit.to_string()))
.unwrap();
assert!(checkout_path.exists());
assert!(checkout_path.join(".cgx-ok").exists());
assert_eq!(commit, &commit_hash);
}
#[test]
fn checkout_specific_non_advertised_commit() {
let (git_client, _temp) = test_git_client();
let url = "https://github.com/rust-lang/rustlings.git";
let commit = "6cf75d569bd0dd33a041e37c59cb75d28664bd7b";
let (checkout_path, commit_hash) = git_client
.checkout_ref(url, GitSelector::Commit(commit.to_string()))
.unwrap();
assert!(checkout_path.exists());
assert!(checkout_path.join(".cgx-ok").exists());
assert_eq!(commit, &commit_hash);
drop(_temp);
let (git_client, _temp) = test_git_client();
let short_commit = &commit[..7];
let (checkout_path, commit_hash) = git_client
.checkout_ref(url, GitSelector::Commit(short_commit.to_string()))
.unwrap();
assert!(checkout_path.exists());
assert!(checkout_path.join(".cgx-ok").exists());
assert_eq!(commit, &commit_hash);
}
#[test]
fn cache_reuse_same_commit() {
let (git_client, _temp) = test_git_client();
let url = "https://github.com/rust-lang/rustlings.git";
let commit = "28d2bb04326d7036514245d73f10fb72b9ed108c";
let (first_checkout_path, first_checkout_hash) = git_client
.checkout_ref(url, GitSelector::Commit(commit.to_string()))
.unwrap();
let (second_checkout_path, second_checkout_hash) = git_client
.checkout_ref(url, GitSelector::Commit(commit.to_string()))
.unwrap();
assert_eq!(commit, &first_checkout_hash);
assert_eq!(commit, &second_checkout_hash);
assert_eq!(first_checkout_path, second_checkout_path);
}
#[test]
fn nonexistent_branch() {
let (git_client, _temp) = test_git_client();
let url = "https://github.com/rust-lang/rustlings.git";
let result = git_client.checkout_ref(
url,
GitSelector::Branch("this-branch-does-not-exist-xyzzy".to_string()),
);
assert_matches!(result, Err(Error::ResolveSelector { .. }));
}
#[test]
fn nonexistent_tag() {
let (git_client, _temp) = test_git_client();
let url = "https://github.com/rust-lang/rustlings.git";
let result = git_client.checkout_ref(url, GitSelector::Tag("v999.999.999".to_string()));
assert_matches!(result, Err(Error::ResolveSelector { .. }));
}
#[test]
fn nonexistent_commit() {
let (git_client, _temp) = test_git_client();
let url = "https://github.com/rust-lang/rustlings.git";
let result = git_client.checkout_ref(
url,
GitSelector::Commit("0000000000000000000000000000000000000000".to_string()),
);
assert_matches!(result, Err(Error::FetchRef { .. }));
}
}
mod integration {
use super::*;
use httpmock::prelude::*;
use std::time::Duration;
fn fast_retry_config() -> HttpConfig {
HttpConfig {
retries: 2,
backoff_base: Duration::from_millis(1),
backoff_max: Duration::from_millis(10),
timeout: Duration::from_secs(30),
..Default::default()
}
}
fn no_retry_config() -> HttpConfig {
HttpConfig {
retries: 0,
backoff_base: Duration::from_millis(1),
backoff_max: Duration::from_millis(1),
timeout: Duration::from_secs(5),
..Default::default()
}
}
fn test_bare_repo() -> (TempDir, PathBuf) {
let temp_dir = TempDir::new().unwrap();
let repo_path = temp_dir.path().join("bare.git");
fs::create_dir_all(&repo_path).unwrap();
init_bare_repo(&repo_path).unwrap();
(temp_dir, repo_path)
}
#[test]
fn server_503_is_retried() {
let server = MockServer::start();
let mock = server.mock(|_when, then| {
then.status(503);
});
let (_temp, db_path) = test_bare_repo();
let config = fast_retry_config();
let result = fetch_ref(
&db_path,
&server.url("/repo.git"),
&GitSelector::DefaultBranch,
&config,
);
assert_matches!(result, Err(Error::FetchRef { .. }));
mock.assert_calls(3);
}
#[test]
fn server_500_is_retried() {
let server = MockServer::start();
let mock = server.mock(|_when, then| {
then.status(500);
});
let (_temp, db_path) = test_bare_repo();
let config = fast_retry_config();
let result = fetch_ref(
&db_path,
&server.url("/repo.git"),
&GitSelector::DefaultBranch,
&config,
);
assert_matches!(result, Err(Error::FetchRef { .. }));
mock.assert_calls(3);
}
#[test]
fn server_429_is_retried() {
let server = MockServer::start();
let mock = server.mock(|_when, then| {
then.status(429);
});
let (_temp, db_path) = test_bare_repo();
let config = fast_retry_config();
let result = fetch_ref(
&db_path,
&server.url("/repo.git"),
&GitSelector::DefaultBranch,
&config,
);
assert_matches!(result, Err(Error::FetchRef { .. }));
mock.assert_calls(3);
}
#[test]
fn server_403_is_not_retried() {
let server = MockServer::start();
let mock = server.mock(|_when, then| {
then.status(403);
});
let (_temp, db_path) = test_bare_repo();
let config = fast_retry_config();
let result = fetch_ref(
&db_path,
&server.url("/repo.git"),
&GitSelector::DefaultBranch,
&config,
);
assert_matches!(result, Err(Error::FetchRef { .. }));
mock.assert_calls(1);
}
#[test]
fn server_404_is_not_retried() {
let server = MockServer::start();
let mock = server.mock(|_when, then| {
then.status(404);
});
let (_temp, db_path) = test_bare_repo();
let config = fast_retry_config();
let result = fetch_ref(
&db_path,
&server.url("/repo.git"),
&GitSelector::DefaultBranch,
&config,
);
assert_matches!(result, Err(Error::FetchRef { .. }));
mock.assert_calls(1);
}
#[test]
fn connection_timeout_is_retried() {
let server = MockServer::start();
let mock = server.mock(|_when, then| {
then.status(200).delay(Duration::from_secs(3));
});
let (_temp, db_path) = test_bare_repo();
let config = HttpConfig {
retries: 2,
backoff_base: Duration::from_millis(1),
backoff_max: Duration::from_millis(10),
timeout: Duration::from_secs(1),
..Default::default()
};
let result = fetch_ref(
&db_path,
&server.url("/repo.git"),
&GitSelector::DefaultBranch,
&config,
);
assert_matches!(result, Err(Error::FetchRef { .. }));
mock.assert_calls(3);
}
#[test]
fn user_agent_is_applied_to_git_http_requests() {
let server = MockServer::start();
let expected_ua = crate::http::user_agent();
let mock = server.mock(|when, then| {
when.method(GET)
.path("/repo.git/info/refs")
.query_param("service", "git-upload-pack")
.header("User-Agent", expected_ua.as_str());
then.status(500);
});
let (_temp, db_path) = test_bare_repo();
let config = no_retry_config();
let result = fetch_ref(
&db_path,
&server.url("/repo.git"),
&GitSelector::DefaultBranch,
&config,
);
assert_matches!(result, Err(Error::FetchRef { .. }));
mock.assert_calls(1);
}
#[test]
fn proxy_setting_is_used_for_git_http_requests() {
let server = MockServer::start();
let expected_ua = crate::http::user_agent();
let mock = server.mock(|when, then| {
when.method(GET)
.host("example.invalid")
.path("/repo.git/info/refs")
.query_param("service", "git-upload-pack")
.header("User-Agent", expected_ua.as_str());
then.status(502);
});
let (_temp, db_path) = test_bare_repo();
let config = HttpConfig {
proxy: Some(server.base_url()),
..no_retry_config()
};
let result = fetch_ref(
&db_path,
"http://example.invalid/repo.git",
&GitSelector::DefaultBranch,
&config,
);
assert_matches!(result, Err(Error::FetchRef { .. }));
mock.assert_calls(1);
}
}
}