use git2::{self, Error as GitError, Config as GitConfig, Cred as GitCred, RemoteCallbacks, CredentialType, FetchOptions, ProxyOptions, Repository, Tree, Oid};
use semver::{VersionReq as SemverReq, Version as Semver};
use std::fs::{self, DirEntry, File};
use std::path::{PathBuf, Path};
use std::io::{Write, Read};
use std::time::SystemTime;
use std::{cmp, env, mem};
use std::borrow::Cow;
use regex::Regex;
use url::Url;
use toml;
use json;
mod config;
pub use self::config::*;
lazy_static! {
static ref MAIN_PACKAGE_RGX: Regex = Regex::new(r"([^\s]+) ([^\s]+) \(registry+\+([^\s]+)\)").unwrap();
static ref GIT_PACKAGE_RGX: Regex = Regex::new(r"([^\s]+) ([^\s]+) \(git+\+([^#\s]+)#([^\s]{40})\)").unwrap();
}
#[derive(Debug, Clone, Hash, PartialEq, Eq)]
pub struct MainRepoPackage {
pub name: String,
pub version: Option<Semver>,
pub newest_version: Option<Semver>,
pub alternative_version: Option<Semver>,
pub max_version: Option<Semver>,
}
#[derive(Debug, Clone, Hash, PartialEq, Eq)]
pub struct GitRepoPackage {
pub name: String,
pub url: String,
pub branch: Option<String>,
pub id: Oid,
pub newest_id: Option<Oid>,
}
impl MainRepoPackage {
pub fn parse(what: &str) -> Option<MainRepoPackage> {
MAIN_PACKAGE_RGX.captures(what).map(|c| {
MainRepoPackage {
name: c.get(1).unwrap().as_str().to_string(),
version: Some(Semver::parse(c.get(2).unwrap().as_str()).unwrap()),
newest_version: None,
alternative_version: None,
max_version: None,
}
})
}
pub fn pull_version<'t>(&mut self, registry: &Tree<'t>, registry_parent: &'t Repository, install_prereleases: Option<bool>) {
let mut vers =
crate_versions(&mut &find_package_data(&self.name, registry, registry_parent).ok_or_else(|| format!("package {} not found", self.name)).unwrap()
[..]);
vers.sort();
self.newest_version = None;
self.alternative_version = None;
let mut vers = vers.into_iter().rev();
if let Some(newest) = vers.next() {
self.newest_version = Some(newest);
if self.newest_version.as_ref().unwrap().is_prerelease() && !install_prereleases.unwrap_or(false) {
if let Some(newest_nonpre) = vers.find(|v| !v.is_prerelease()) {
mem::swap(&mut self.alternative_version, &mut self.newest_version);
self.newest_version = Some(newest_nonpre);
}
}
}
}
pub fn needs_update(&self, req: Option<&SemverReq>, install_prereleases: Option<bool>) -> bool {
let update_to_version = self.update_to_version();
(req.into_iter().zip(self.version.as_ref()).map(|(sr, cv)| !sr.matches(cv)).next().unwrap_or(true) ||
req.into_iter().zip(update_to_version).map(|(sr, uv)| sr.matches(uv)).next().unwrap_or(true)) &&
update_to_version.map(|upd_v| {
(!upd_v.is_prerelease() || install_prereleases.unwrap_or(false)) && (self.version.is_none() || (*self.version.as_ref().unwrap() < *upd_v))
})
.unwrap_or(false)
}
pub fn update_to_version(&self) -> Option<&Semver> {
self.newest_version.as_ref().map(|new_v| cmp::min(new_v, self.max_version.as_ref().unwrap_or(new_v)))
}
}
impl GitRepoPackage {
pub fn parse(what: &str) -> Option<GitRepoPackage> {
GIT_PACKAGE_RGX.captures(what).map(|c| {
let mut url = Url::parse(c.get(3).unwrap().as_str()).unwrap();
let branch = url.query_pairs().find(|&(ref name, _)| name == "branch").map(|(_, value)| value.to_string());
url.set_query(None);
GitRepoPackage {
name: c.get(1).unwrap().as_str().to_string(),
url: url.into_string(),
branch: branch,
id: Oid::from_str(c.get(4).unwrap().as_str()).unwrap(),
newest_id: None,
}
})
}
pub fn pull_version<Pt: AsRef<Path>, Pg: AsRef<Path>>(&mut self, temp_dir: Pt, git_db_dir: Pg, http_proxy: Option<&str>) {
self.pull_version_impl(temp_dir.as_ref(), git_db_dir.as_ref(), http_proxy)
}
fn pull_version_impl(&mut self, temp_dir: &Path, git_db_dir: &Path, http_proxy: Option<&str>) {
let clone_dir = find_git_db_repo(git_db_dir, &self.name).unwrap_or_else(|| {
fs::create_dir_all(temp_dir).unwrap();
temp_dir.join(&self.name)
});
let repo = if let Ok(r) = Repository::open(&clone_dir) {
r.find_remote("origin")
.or_else(|_| r.remote_anonymous(&self.url))
.and_then(|mut rm| {
with_authentication(&self.url, |creds| {
let mut cb = RemoteCallbacks::new();
cb.credentials(|a, b, c| creds(a, b, c));
rm.fetch(&[self.branch.as_ref().map(String::as_str).unwrap_or("master")],
Some(&mut fetch_options_from_proxy_url_and_callbacks(http_proxy, cb)),
None)
})
})
.unwrap();
r.set_head("FETCH_HEAD").unwrap();
Ok(r)
} else {
if clone_dir.exists() {
fs::remove_dir_all(&clone_dir).unwrap();
}
with_authentication(&self.url, |creds| {
let mut bldr = git2::build::RepoBuilder::new();
let mut cb = RemoteCallbacks::new();
cb.credentials(|a, b, c| creds(a, b, c));
bldr.fetch_options(fetch_options_from_proxy_url_and_callbacks(http_proxy, cb));
if let Some(ref b) = self.branch.as_ref() {
bldr.branch(b);
}
bldr.bare(true);
bldr.clone(&self.url, &clone_dir)
})
};
self.newest_id = Some(repo.and_then(|r| r.head().and_then(|h| h.target().ok_or_else(|| GitError::from_str("HEAD not a direct reference")))).unwrap());
}
pub fn needs_update(&self) -> bool {
self.newest_id.is_some() && self.id != *self.newest_id.as_ref().unwrap()
}
}
#[derive(Debug, Clone, Hash, PartialEq, Eq, PartialOrd, Ord)]
pub enum PackageFilterElement {
Toolchain(String),
}
impl PackageFilterElement {
pub fn parse(from: &str) -> Result<PackageFilterElement, String> {
let (key, value) = from.split_at(from.find('=').ok_or_else(|| format!(r#"Filter string "{}" does not contain the key/value separator "=""#, from))?);
let value = &value[1..];
Ok(match key {
"toolchain" => PackageFilterElement::Toolchain(value.to_string()),
_ => return Err(format!(r#"Unrecognised filter key "{}""#, key)),
})
}
pub fn matches(&self, cfg: &PackageConfig) -> bool {
match *self {
PackageFilterElement::Toolchain(ref chain) => Some(chain) == cfg.toolchain.as_ref(),
}
}
}
pub fn resolve_crates_file(crates_file: PathBuf) -> PathBuf {
let config_file = crates_file.with_file_name("config");
if config_file.exists() {
let mut crates = String::new();
File::open(&config_file).unwrap().read_to_string(&mut crates).unwrap();
if let Some(idir) = toml::from_str::<toml::Value>(&crates)
.unwrap()
.get("install")
.and_then(|t| t.as_table())
.and_then(|t| t.get("root"))
.and_then(|t| t.as_str()) {
return resolve_crates_file(Path::new(idir).join(".crates.toml"));
}
}
crates_file
}
pub fn installed_main_repo_packages(crates_file: &Path) -> Vec<MainRepoPackage> {
if crates_file.exists() {
let mut crates = String::new();
File::open(crates_file).unwrap().read_to_string(&mut crates).unwrap();
let mut res = Vec::<MainRepoPackage>::new();
for pkg in toml::from_str::<toml::Value>(&crates).unwrap()["v1"].as_table().unwrap().keys().flat_map(|s| MainRepoPackage::parse(s)) {
if let Some(saved) = res.iter_mut().find(|p| p.name == pkg.name) {
if saved.version.is_none() || saved.version.as_ref().unwrap() < pkg.version.as_ref().unwrap() {
saved.version = pkg.version;
}
continue;
}
res.push(pkg);
}
res
} else {
Vec::new()
}
}
pub fn installed_git_repo_packages(crates_file: &Path) -> Vec<GitRepoPackage> {
if crates_file.exists() {
let mut crates = String::new();
File::open(crates_file).unwrap().read_to_string(&mut crates).unwrap();
let mut res = Vec::<GitRepoPackage>::new();
for pkg in toml::from_str::<toml::Value>(&crates).unwrap()["v1"].as_table().unwrap().keys().flat_map(|s| GitRepoPackage::parse(s)) {
if let Some(saved) = res.iter_mut().find(|p| p.name == pkg.name) {
saved.id = pkg.id;
continue;
}
res.push(pkg);
}
res
} else {
Vec::new()
}
}
pub fn intersect_packages(installed: &[MainRepoPackage], to_update: &[(String, Option<Semver>)], allow_installs: bool, installed_git: &[GitRepoPackage])
-> Vec<MainRepoPackage> {
installed.iter()
.filter(|p| to_update.iter().any(|u| p.name == u.0))
.cloned()
.map(|p| MainRepoPackage { max_version: to_update.iter().find(|u| p.name == u.0).and_then(|u| u.1.clone()), ..p })
.chain(to_update.iter()
.filter(|p| allow_installs && installed.iter().find(|i| i.name == p.0).is_none() && installed_git.iter().find(|i| i.name == p.0).is_none())
.map(|p| {
MainRepoPackage {
name: p.0.clone(),
version: None,
newest_version: None,
alternative_version: None,
max_version: p.1.clone(),
}
}))
.collect()
}
pub fn crate_versions<R: Read>(package_desc: &mut R) -> Vec<Semver> {
let mut buf = String::new();
package_desc.read_to_string(&mut buf).unwrap();
crate_versions_impl(buf)
}
fn crate_versions_impl(buf: String) -> Vec<Semver> {
buf.lines()
.map(|p| json::parse(p).unwrap())
.filter(|j| !j["yanked"].as_bool().unwrap())
.map(|j| Semver::parse(j["vers"].as_str().unwrap()).unwrap())
.collect()
}
pub fn get_index_path(cargo_dir: &Path, registry_url: Option<&str>) -> Result<PathBuf, &'static str> {
let registry_url = registry_url.map(|u| Url::parse(u).map_err(|_| "registry URL not an URL")).transpose()?;
let registry_host = registry_url.as_ref().and_then(|u| u.host_str());
Ok(fs::read_dir(cargo_dir.join("registry").join("index"))
.map_err(|_| "index directory nonexistant")?
.map(Result::unwrap)
.filter(|i| i.file_type().unwrap().is_dir())
.filter(|i| registry_host.map(|rh| i.file_name().to_string_lossy().starts_with(rh)).unwrap_or(true))
.max_by_key(latest_modified)
.ok_or("empty index directory")?
.path())
}
fn latest_modified(ent: &DirEntry) -> SystemTime {
let meta = ent.metadata().unwrap();
let mut latest = meta.modified().unwrap();
if meta.is_dir() {
for ent in fs::read_dir(ent.path()).unwrap() {
latest = cmp::max(latest, latest_modified(&ent.unwrap()));
}
}
latest
}
pub fn update_index<W: Write>(index_repo: &mut Repository, repo_url: &str, http_proxy: Option<&str>, out: &mut W) -> Result<(), String> {
writeln!(out, " Updating registry '{}'", repo_url).map_err(|_| "failed to write updating message".to_string())?;
index_repo.remote_anonymous(repo_url)
.and_then(|mut r| {
with_authentication(repo_url, |creds| {
let mut cb = RemoteCallbacks::new();
cb.credentials(|a, b, c| creds(a, b, c));
r.fetch(&["refs/heads/master:refs/remotes/origin/master"],
Some(&mut fetch_options_from_proxy_url_and_callbacks(http_proxy, cb)),
None)
})
})
.map_err(|e| e.message().to_string())?;
writeln!(out).map_err(|_| "failed to write post-update newline".to_string())?;
Ok(())
}
fn fetch_options_from_proxy_url_and_callbacks<'a>(proxy_url: Option<&str>, callbacks: RemoteCallbacks<'a>) -> FetchOptions<'a> {
let mut ret = FetchOptions::new();
if let Some(proxy_url) = proxy_url {
ret.proxy_options({
let mut prx = ProxyOptions::new();
prx.url(proxy_url);
prx
});
}
ret.remote_callbacks(callbacks);
ret
}
pub fn get_index_url(crates_file: &Path) -> Cow<'static, str> {
get_index_url_impl(crates_file).map(Cow::from).unwrap_or(Cow::from("https://github.com/rust-lang/crates.io-index"))
}
fn get_index_url_impl(crates_file: &Path) -> Option<String> {
let config = fs::read_to_string(crates_file.with_file_name("config")).ok()?;
let config = toml::from_str::<toml::Value>(&config).ok()?;
let sources = config.get("source")?;
let mut cur_source = sources.get("crates-io")?;
loop {
match cur_source.get("replace-with") {
Some(redir) => cur_source = sources.get(redir.as_str()?)?,
None => return Some(cur_source.get("registry")?.as_str()?.to_string()),
}
}
}
fn with_authentication<T, F>(url: &str, mut f: F) -> Result<T, GitError>
where F: FnMut(&mut git2::Credentials) -> Result<T, GitError>
{
let cfg = GitConfig::open_default().unwrap();
let mut cred_helper = git2::CredentialHelper::new(url);
cred_helper.config(&cfg);
let mut ssh_username_requested = false;
let mut cred_helper_bad = None;
let mut ssh_agent_attempts = Vec::new();
let mut any_attempts = false;
let mut tried_ssh_key = false;
let mut res = f(&mut |url, username, allowed| {
any_attempts = true;
if allowed.contains(CredentialType::USERNAME) {
ssh_username_requested = true;
Err(GitError::from_str("username to be tried later"))
} else if allowed.contains(CredentialType::SSH_KEY) && !tried_ssh_key {
tried_ssh_key = true;
let username = username.unwrap();
ssh_agent_attempts.push(username.to_string());
GitCred::ssh_key_from_agent(username)
} else if allowed.contains(CredentialType::USER_PASS_PLAINTEXT) && cred_helper_bad.is_none() {
let ret = GitCred::credential_helper(&cfg, url, username);
cred_helper_bad = Some(ret.is_err());
ret
} else if allowed.contains(CredentialType::DEFAULT) {
GitCred::default()
} else {
Err(GitError::from_str("no authentication available"))
}
});
if ssh_username_requested {
for uname in cred_helper.username
.into_iter()
.chain(cfg.get_string("user.name"))
.chain(["USERNAME", "USER"].into_iter().flat_map(env::var))
.chain(Some("git").into_iter().map(str::to_string)) {
let mut ssh_attempts = 0;
res = f(&mut |_, _, allowed| {
if allowed.contains(CredentialType::USERNAME) {
return GitCred::username(&uname);
} else if allowed.contains(CredentialType::SSH_KEY) {
ssh_attempts += 1;
if ssh_attempts == 1 {
ssh_agent_attempts.push(uname.to_string());
return GitCred::ssh_key_from_agent(&uname);
}
}
Err(GitError::from_str("no authentication available"))
});
if ssh_attempts != 2 {
break;
}
}
}
if res.is_ok() || !any_attempts {
res
} else {
let err = res.err().map(|e| format!("{}: ", e)).unwrap_or(String::new());
let mut msg = format!("{}failed to authenticate when downloading repository {}", err, url);
if !ssh_agent_attempts.is_empty() {
msg.push_str(" (tried ssh-agent, but none of the following usernames worked: ");
for (i, uname) in ssh_agent_attempts.into_iter().enumerate() {
if i != 0 {
msg.push_str(", ");
}
msg.push('\"');
msg.push_str(&uname);
msg.push('\"');
}
msg.push(')');
}
if let Some(failed_cred_helper) = cred_helper_bad {
msg.push_str(" (tried to find username+password via ");
if failed_cred_helper {
msg.push_str("git's credential.helper support, but failed)");
} else {
msg.push_str("credential.helper, but found credentials were incorrect)");
}
}
Err(GitError::from_str(&msg)).unwrap()
}
}
pub fn find_package_data<'t>(cratename: &str, registry: &Tree<'t>, registry_parent: &'t Repository) -> Option<Vec<u8>> {
let clen = cratename.len().to_string();
let mut elems = Vec::new();
if cratename.len() <= 3 {
elems.push(&clen[..]);
}
match cratename.len() {
0 => panic!("0-length cratename"),
1 | 2 => {}
3 => elems.push(&cratename[0..1]),
_ => {
elems.push(&cratename[0..2]);
elems.push(&cratename[2..4]);
}
}
elems.push(cratename);
let ent = registry.get_name(elems[0])?;
let obj = ent.to_object(registry_parent).ok()?;
let ent = obj.as_tree()?.get_name(elems[1])?;
let obj = ent.to_object(registry_parent).ok()?;
if elems.len() == 3 {
let ent = obj.as_tree()?.get_name(elems[2])?;
let obj = ent.to_object(registry_parent).ok()?;
Some(obj.as_blob()?.content().into())
} else {
Some(obj.as_blob()?.content().into())
}
}
pub fn find_proxy(crates_file: &Path) -> Option<String> {
let config_file = crates_file.with_file_name("config");
if config_file.exists() {
let mut crates = String::new();
File::open(&config_file).unwrap().read_to_string(&mut crates).unwrap();
if let Some(proxy) = toml::from_str::<toml::Value>(&crates)
.unwrap()
.get("http")
.and_then(|t| t.as_table())
.and_then(|t| t.get("proxy"))
.and_then(|t| t.as_str()) {
if !proxy.is_empty() {
return Some(proxy.to_string());
}
}
}
if let Ok(cfg) = GitConfig::open_default() {
if let Ok(proxy) = cfg.get_str("http.proxy") {
if !proxy.is_empty() {
return Some(proxy.to_string());
}
}
}
["http_proxy", "HTTP_PROXY", "https_proxy", "HTTPS_PROXY"].iter().flat_map(env::var).filter(|proxy| !proxy.is_empty()).next()
}
pub fn find_git_db_repo(git_db_dir: &Path, cratename: &str) -> Option<PathBuf> {
fs::read_dir(git_db_dir).ok()?.flatten().find(|de| de.file_name().to_str().map(|n| n.starts_with(cratename)).unwrap_or(false)).map(|de| de.path())
}