use std::num::NonZeroU32;
use std::path::Path;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Condvar, Mutex};
use std::time::Duration;
use dynamic_config::Error;
use dynamic_config_store_core::documents;
use gix::remote::Direction;
use crate::auth::Auth;
use crate::url::redacted;
use crate::{check_path, Keys, Reference};
pub(crate) enum Failure {
Refused(Error),
Other(Error),
}
impl Failure {
pub(crate) fn into_error(self) -> Error {
match self {
Self::Refused(error) | Self::Other(error) => error,
}
}
}
pub(crate) fn open(dir: &Path, ssh_command: Option<String>) -> Result<gix::Repository, Failure> {
let overrides: Vec<String> = ssh_command
.map(|command| format!("core.sshCommand={command}"))
.into_iter()
.collect();
let options = gix::open::Options::default().cli_overrides(overrides.iter().map(String::as_str));
match gix::open_opts(dir, options.clone()) {
Ok(repository) => Ok(repository),
Err(_) => {
gix::init_bare(dir).map_err(|error| {
Failure::Other(Error::remote(format!(
"git: cannot prepare the working directory {}: {error}",
dir.display()
)))
})?;
crate::working::mark(dir).map_err(Failure::Other)?;
gix::open_opts(dir, options).map_err(|error| {
Failure::Other(Error::remote(format!(
"git: cannot open the working directory {}: {error}",
dir.display()
)))
})
}
}
}
pub(crate) struct Plan<'a> {
pub(crate) url: &'a str,
pub(crate) reference: &'a Reference,
pub(crate) auth: &'a Auth,
pub(crate) tls: &'a dynamic_config_store_core::tls::TlsConfig,
pub(crate) timeout: Duration,
pub(crate) described: &'a str,
}
#[allow(
clippy::result_large_err,
reason = "the error is `gix_credentials::protocol::Error`, which is not \
ours to box; the credential closure's signature is `gix`'s"
)]
pub(crate) fn fetch(
repository: &gix::Repository,
plan: &Plan<'_>,
want_objects: bool,
) -> Result<gix::ObjectId, Failure> {
let Plan {
url,
reference,
tls,
timeout,
described,
..
} = plan;
let (url, timeout) = (*url, *timeout);
let remote = repository
.remote_at(url)
.map_err(|error| other(url, format_args!("the url is not usable: {error}")))?
.with_fetch_tags(gix::remote::fetch::Tags::None)
.with_refspecs([reference.refspec().as_str()], Direction::Fetch)
.map_err(|error| {
other(
url,
format_args!("{} is not a valid ref: {error}", reference),
)
})?;
if tls.is_empty() {
let connection = remote
.connect(Direction::Fetch)
.map_err(|error| classify(url, &error, format_args!("cannot connect")))?;
return negotiate(repository, connection, plan, want_objects);
}
let (sanitized, version) = remote
.sanitized_url_and_version(Direction::Fetch)
.map_err(|error| classify(url, &error, format_args!("cannot connect")))?;
let transport = crate::tls::transport(
&sanitized,
version,
tls,
timeout,
repository
.config_snapshot()
.boolean("gitoxide.trace.packet")
== Some(true),
described,
)
.map_err(Failure::Other)?;
negotiate(
repository,
remote.to_connection_with_transport(transport),
plan,
want_objects,
)
}
#[allow(
clippy::result_large_err,
reason = "the error is `gix_credentials::protocol::Error`, which is not \
ours to box; the credential closure's signature is `gix`'s"
)]
fn negotiate<T>(
repository: &gix::Repository,
connection: gix::remote::Connection<'_, '_, '_, T>,
plan: &Plan<'_>,
want_objects: bool,
) -> Result<gix::ObjectId, Failure>
where
T: gix::protocol::transport::client::blocking_io::Transport,
{
let Plan {
url,
reference,
auth,
timeout,
..
} = plan;
let (url, timeout) = (*url, *timeout);
let identity = match auth {
Auth::Https { username, password } => Some(gix::sec::identity::Account {
username: username.clone(),
password: password.clone(),
oauth_refresh_token: None,
}),
Auth::Anonymous | Auth::Ssh(_) => None,
};
let connection = connection.with_credentials(move |action| match action {
gix::credentials::helper::Action::Get(context) => {
Ok(identity
.clone()
.map(|identity| gix::credentials::protocol::Outcome {
identity,
next: context.into(),
}))
}
gix::credentials::helper::Action::Store(_) | gix::credentials::helper::Action::Erase(_) => {
Ok(None)
}
});
let deadline = Deadline::start(timeout);
let prepared = connection
.prepare_fetch(
gix::progress::Discard,
gix::remote::ref_map::Options::default(),
)
.map_err(|error| classify(url, &error, format_args!("cannot list {reference}")))?;
let commit = reference.resolve(prepared.ref_map(), url)?;
if !want_objects || repository.has_object(commit) {
return Ok(commit);
}
prepared
.with_shallow(gix::remote::fetch::Shallow::DepthAtRemote(DEPTH))
.receive(gix::progress::Discard, deadline.interrupt())
.map_err(|error| {
if deadline.expired() {
return Failure::Other(Error::remote(format!(
"git {}: fetching {reference} took longer than {timeout:?}",
redacted(url)
)));
}
classify(url, &error, format_args!("cannot fetch {reference}"))
})?;
Ok(commit)
}
const DEPTH: NonZeroU32 = match NonZeroU32::new(1) {
Some(depth) => depth,
None => unreachable!(),
};
pub(crate) fn read_documents(
repository: &gix::Repository,
commit: gix::ObjectId,
keys: &Keys,
max_bytes: u64,
url: &str,
) -> Result<Vec<(String, String)>, Failure> {
let at = |path: &str, what: std::fmt::Arguments<'_>| {
Failure::Other(Error::remote(format!(
"git {} {}:{path}: {what}",
redacted(url),
commit.to_hex_with_len(12),
)))
};
let tree = repository
.find_object(commit)
.map_err(|error| at("", format_args!("the commit is not readable: {error}")))?
.peel_to_tree()
.map_err(|error| at("", format_args!("the commit has no tree: {error}")))?;
let paths = match keys {
Keys::One(path) => vec![path.clone()],
Keys::Several(paths) => paths.clone(),
Keys::Prefix(prefix) => {
let described = format!("git {} {}", redacted(url), commit.to_hex_with_len(12));
under(&tree, prefix, &described).map_err(Failure::Other)?
}
};
if paths.is_empty() {
return Err(at(
&keys.describe(),
format_args!("nothing is there at this commit, so there is nothing to load"),
));
}
let mut documents = Vec::with_capacity(paths.len());
let mut budget = Budget::of(max_bytes);
for path in paths {
let text = read_blob(repository, &tree, &path, &mut budget, &|what| {
at(&path, what)
})?;
documents.push((path, text));
}
Ok(documents)
}
struct Budget {
max_bytes: u64,
left: u64,
}
impl Budget {
fn of(max_bytes: u64) -> Self {
Self {
max_bytes,
left: max_bytes,
}
}
}
fn read_blob(
repository: &gix::Repository,
tree: &gix::Tree<'_>,
path: &str,
budget: &mut Budget,
at: &dyn Fn(std::fmt::Arguments<'_>) -> Failure,
) -> Result<String, Failure> {
let entry = tree
.lookup_entry(path.split('/').map(str::as_bytes))
.map_err(|error| at(format_args!("the tree is not readable: {error}")))?
.ok_or_else(|| at(format_args!("there is no such file at this commit")))?;
match entry.mode().kind() {
gix::object::tree::EntryKind::Blob | gix::object::tree::EntryKind::BlobExecutable => {}
gix::object::tree::EntryKind::Tree => {
return Err(at(format_args!(
"that is a directory; a source reads a document, and a whole \
directory of them is `Keys::prefix`"
)))
}
gix::object::tree::EntryKind::Link => {
return Err(at(format_args!(
"that is a symbolic link; this crate will not follow one, \
because where it points is the remote repository's choice \
and not this program's"
)))
}
gix::object::tree::EntryKind::Commit => {
return Err(at(format_args!(
"that is a submodule; its objects are in another repository"
)))
}
}
let size = repository
.find_header(entry.object_id())
.map_err(|error| at(format_args!("the blob is not readable: {error}")))?
.size();
let max_bytes = budget.max_bytes;
if size > max_bytes {
return Err(at(format_args!(
"the file is {size} bytes, over the {max_bytes}-byte limit; \
raise it with `max_bytes` if that is really a configuration file"
)));
}
if size > budget.left {
return Err(at(format_args!(
"the files read so far and this one come to more than the \
{max_bytes}-byte limit, which is what one read may cost in \
total; name fewer files, or raise `max_bytes`"
)));
}
budget.left -= size;
let object = repository
.find_object(entry.object_id())
.map_err(|error| at(format_args!("the blob is not readable: {error}")))?;
String::from_utf8(object.data.clone())
.map_err(|_| at(format_args!("the file is not valid UTF-8")))
}
fn under(tree: &gix::Tree<'_>, directory: &str, described: &str) -> Result<Vec<String>, Error> {
let root = directory.trim_end_matches('/');
let mut start = tree.clone();
if !root.is_empty() {
let entry = start
.lookup_entry(root.split('/').map(str::as_bytes))
.map_err(|error| {
Error::remote(format!(
"{described}: {root}: the tree is not readable: {error}"
))
})?
.ok_or_else(|| {
Error::remote(format!(
"{described}: {root}: there is no such directory at this commit"
))
})?;
if entry.mode().kind() != gix::object::tree::EntryKind::Tree {
return Err(Error::remote(format!(
"{described}: {root}: that is a file rather than a directory; \
name it as one path instead of as a prefix"
)));
}
start = entry
.object()
.map_err(|error| {
Error::remote(format!("{described}: {root}: it is not readable: {error}"))
})?
.peel_to_tree()
.map_err(|error| {
Error::remote(format!(
"{described}: {root}: it is not a directory: {error}"
))
})?;
}
let mut found: Vec<String> = Vec::new();
let mut pending = vec![(root.to_owned(), start)];
let mut descended = 0_usize;
while let Some((at, tree)) = pending.pop() {
for entry in tree.iter() {
let entry = entry.map_err(|error| {
Error::remote(format!(
"{described}: {at}: the tree is not readable: {error}"
))
})?;
let name = std::str::from_utf8(entry.filename()).map_err(|_| {
Error::remote(format!(
"{described}: {at}: a file here has a name that is not valid UTF-8"
))
})?;
let path = if at.is_empty() {
name.to_owned()
} else {
format!("{at}/{name}")
};
check_path(&path).map_err(|error| Error::remote(format!("{described}: {error}")))?;
if !root.is_empty() {
documents::under_prefix(&path, &format!("{root}/"), described)?;
}
if entry.mode().kind() == gix::object::tree::EntryKind::Tree {
descended += 1;
documents::within_key_budget(descended, described)?;
pending.push((
path,
entry
.object()
.map_err(|error| {
Error::remote(format!("{described}: the tree is not readable: {error}"))
})?
.peel_to_tree()
.map_err(|error| {
Error::remote(format!("{described}: the tree is not readable: {error}"))
})?,
));
continue;
}
found.push(path);
documents::within_key_budget(found.len(), described)?;
}
}
found.sort();
Ok(found)
}
fn other(url: &str, what: std::fmt::Arguments<'_>) -> Failure {
Failure::Other(Error::remote(format!("git {}: {what}", redacted(url))))
}
fn classify(
url: &str,
error: &(dyn std::error::Error + 'static),
what: std::fmt::Arguments<'_>,
) -> Failure {
let described = format!("git {}: {what}: {}", redacted(url), chain(error));
if refused(error) {
Failure::Refused(Error::auth(described))
} else {
Failure::Other(Error::remote(described))
}
}
pub(crate) fn chain(error: &(dyn std::error::Error + 'static)) -> String {
let mut rendered: Vec<String> = Vec::new();
let mut current = Some(error);
while let Some(error) = current {
let text = error.to_string();
if !rendered.contains(&text) {
rendered.push(text);
}
current = error.source();
}
rendered.join(": ")
}
fn refused(error: &(dyn std::error::Error + 'static)) -> bool {
let mut current = Some(error);
while let Some(error) = current {
if let Some(io) = error.downcast_ref::<std::io::Error>() {
if io.kind() == std::io::ErrorKind::PermissionDenied {
return true;
}
}
if let Some(gix::protocol::transport::client::Error::AuthenticationRefused(_)) =
error.downcast_ref::<gix::protocol::transport::client::Error>()
{
return true;
}
current = error.source();
}
false
}
struct Deadline {
interrupt: Arc<AtomicBool>,
expired: Arc<AtomicBool>,
finished: Arc<(Mutex<bool>, Condvar)>,
}
impl Deadline {
fn start(after: Duration) -> Self {
let deadline = Self {
interrupt: Arc::new(AtomicBool::new(false)),
expired: Arc::new(AtomicBool::new(false)),
finished: Arc::new((Mutex::new(false), Condvar::new())),
};
let interrupt = Arc::clone(&deadline.interrupt);
let expired = Arc::clone(&deadline.expired);
let finished = Arc::clone(&deadline.finished);
std::thread::spawn(move || {
let (lock, condition) = &*finished;
let done = lock
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let (done, timed_out) = condition
.wait_timeout_while(done, after, |done| !*done)
.unwrap_or_else(std::sync::PoisonError::into_inner);
if timed_out.timed_out() && !*done {
expired.store(true, Ordering::SeqCst);
interrupt.store(true, Ordering::SeqCst);
}
});
deadline
}
fn interrupt(&self) -> &AtomicBool {
&self.interrupt
}
fn expired(&self) -> bool {
self.expired.load(Ordering::SeqCst)
}
}
impl Drop for Deadline {
fn drop(&mut self) {
let (lock, condition) = &*self.finished;
*lock
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) = true;
condition.notify_all();
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_refused_credential_is_recognised_through_the_error_chain() {
#[derive(Debug)]
struct Wrapper(std::io::Error);
impl std::fmt::Display for Wrapper {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str("could not fetch")
}
}
impl std::error::Error for Wrapper {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
Some(&self.0)
}
}
let refusal = Wrapper(std::io::Error::new(
std::io::ErrorKind::PermissionDenied,
"Received HTTP status 401",
));
let outage = Wrapper(std::io::Error::new(
std::io::ErrorKind::ConnectionRefused,
"connection refused",
));
assert!(matches!(
classify("https://host/r.git", &refusal, format_args!("x")),
Failure::Refused(_)
));
assert!(matches!(
classify("https://host/r.git", &outage, format_args!("x")),
Failure::Other(_)
));
}
#[test]
fn a_deadline_that_expires_says_so() {
let deadline = Deadline::start(Duration::from_millis(20));
assert!(!deadline.expired());
std::thread::sleep(Duration::from_millis(200));
assert!(deadline.expired(), "the watchdog must fire");
assert!(deadline.interrupt().load(Ordering::SeqCst));
}
#[test]
fn a_deadline_that_is_dropped_first_never_fires() {
let deadline = Deadline::start(Duration::from_secs(3600));
let interrupt = Arc::clone(&deadline.interrupt);
drop(deadline);
std::thread::sleep(Duration::from_millis(50));
assert!(
!interrupt.load(Ordering::SeqCst),
"a fetch that finished must not leave a watchdog parked for an hour"
);
}
}