use crate::{
build::{context::BuildContext, Targets},
package::{manifest::Manifest, PackageId, Spec},
remote::{
resolution::{DirectRes, Resolution},
Index, Indices,
},
util::{
clear_dir, copy_dir,
errors::Res,
graph::Graph,
hexify_hash,
lock::DirLock,
shell::{Shell, Verbosity},
valid_file,
},
};
use console::style;
use failure::{bail, format_err, Error, ResultExt};
use indexmap::{IndexMap, IndexSet};
use itertools::Itertools;
use reqwest::Client;
use sha2::{Digest, Sha256};
use slog::{debug, o, Logger};
use std::{
collections::VecDeque,
fs,
io::{prelude::*, BufReader},
path::{Path, PathBuf},
str::FromStr,
sync::Arc,
time::Duration,
};
use toml;
use walkdir::WalkDir;
#[derive(Debug, Clone)]
pub struct Cache {
pub layout: Layout,
client: Client,
pub logger: Logger,
pub shell: Shell,
}
impl Cache {
pub fn from_disk(plog: &Logger, layout: Layout, shell: Shell) -> Res<Self> {
layout.init()?;
let client = Client::builder().timeout(Duration::from_secs(10)).build()?;
let logger = plog.new(o!("phase" => "cache"));
Ok(Cache {
layout,
client,
logger,
shell,
})
}
pub fn checkout_source(
&self,
pkg: &PackageId,
loc: &DirectRes,
eager: bool,
offline: bool,
dl_f: impl Fn(),
) -> Result<(Option<DirectRes>, Source), Error> {
let p = self.load_source(pkg, loc, eager, offline, dl_f)?;
Ok((p.0, Source::from_folder(pkg, p.1, loc.clone())?))
}
fn load_source(
&self,
pkg: &PackageId,
loc: &DirectRes,
eager: bool,
offline: bool,
dl_f: impl Fn(),
) -> Result<(Option<DirectRes>, DirLock), Error> {
if let DirectRes::Dir { path } = loc {
debug!(self.logger, "loaded source"; "cause" => "dir", "pkg" => pkg.to_string());
return Ok((None, DirLock::acquire(&path)?));
}
let eager = if offline { false } else { eager };
let new_dir = self.layout.src.join(Self::get_source_dir(loc, true));
if loc.is_tar() && new_dir.exists() {
debug!(
self.logger, "loaded source";
"cause" => "exists",
"pkg" => pkg.to_string(),
"dir" => new_dir.display()
);
return Ok((None, DirLock::acquire(&new_dir)?));
}
let new_f = |dl_online| {
if offline && dl_online {
return Err(format_err!("Can't download package in offline mode"));
}
dl_f();
Ok(())
};
let dir = DirLock::acquire(&self.layout.src.join(Self::get_source_dir(loc, true)))?;
let res = if let Resolution::Direct(g) = pkg.resolution() {
if g.is_git() && g != loc {
debug_assert!(loc.is_git());
loc.retrieve(&self.client, &dir, eager, new_f)
.and_then(|_| {
g.retrieve(&self.client, &dir, false, |dl_online| {
if offline && dl_online {
Err(format_err!("Can't download package in offline mode"))
} else {
Ok(())
}
})
})
} else {
loc.retrieve(&self.client, &dir, eager, new_f)
}
} else {
loc.retrieve(&self.client, &dir, eager, new_f)
}?;
let new_dir = self.layout.src.join(&Self::get_source_dir(
if let Some(r) = res.as_ref() { r } else { &loc },
true,
));
let dir = if new_dir != dir.path() {
if !new_dir.exists() {
copy_dir(dir.path(), &new_dir, true)?;
}
DirLock::acquire(&new_dir)?
} else {
dir
};
debug!(
self.logger, "loaded source";
"cause" => "retrieved_new",
"pkg" => pkg.to_string(),
"loc" => loc.to_string(),
"dir" => dir.path().display()
);
Ok((res, dir))
}
pub fn get_source_dir(loc: &DirectRes, include_tag: bool) -> String {
let mut hasher = Sha256::default();
if !include_tag {
if let DirectRes::Git { repo, .. } = loc {
hasher.input(repo.to_string().as_bytes());
} else {
hasher.input(loc.to_string().as_bytes());
}
} else {
hasher.input(loc.to_string().as_bytes());
}
hexify_hash(hasher.result().as_slice())
}
pub fn checkout_build(&self, hash: &BuildHash) -> Res<Option<Binary>> {
if let Some(path) = self.check_build(&hash) {
Ok(Some(Binary {
target: DirLock::acquire(&path)?,
}))
} else {
Ok(None)
}
}
pub fn checkout_tmp(&self, hash: &BuildHash) -> Res<OutputLayout> {
let path = self.layout.tmp.join(&hash.0);
let lock = DirLock::acquire(&path)?;
if lock.path().exists() {
clear_dir(&lock.path()).context(format_err!("couldn't remove existing output path"))?;
}
OutputLayout::new(lock)
}
pub fn store_bins(&self, bins: &[(PathBuf, String)], force: bool) -> Res<()> {
let mut dot_f = fs::OpenOptions::new()
.create(true)
.write(true)
.read(true)
.open(self.layout.bin.join(".bins"))
.with_context(|e| format_err!("could not open .bins file:\n{}", e))?;
let mut dot_c = String::new();
dot_f
.read_to_string(&mut dot_c)
.with_context(|e| format_err!("could not read .bins file:\n{}", e))?;
let mut dot: IndexMap<String, String> = toml::from_str(&dot_c)
.with_context(|e| format_err!("could not deserialize .bins file:\n{}", e))?;
for (path, sum) in bins {
self.shell.println(
style("Installing").cyan(),
path.file_name().unwrap().to_string_lossy().as_ref(),
Verbosity::Normal,
);
self.store_bin(path, force)?;
dot.insert(
path.file_name()
.unwrap()
.to_string_lossy()
.to_owned()
.to_string(),
sum.to_string(),
);
}
drop(dot_f);
fs::remove_file(self.layout.bin.join(".bins"))
.with_context(|e| format_err!("could not clear existing .bins file:\n{}", e))?;
fs::write(
self.layout.bin.join(".bins"),
toml::to_string(&dot).unwrap().as_bytes(),
)
.with_context(|e| format_err!("could not write to .bins file:\n{}", e))?;
Ok(())
}
fn store_bin(&self, from: &Path, force: bool) -> Res<()> {
let bin_name = from
.file_name()
.ok_or_else(|| format_err!("{} isn't a path to a binary", from.display()))?;
let to = self.layout.bin.join(bin_name);
if !force && to.exists() {
bail!(
"binary {} already exists in the global bin directory",
bin_name.to_string_lossy().as_ref()
)
} else if to.exists() {
fs::remove_file(&to).with_context(|e| {
format!("could not remove existing binary {}:\n{}", to.display(), e)
})?;
}
fs::File::create(&to)
.with_context(|e| format_err!("couldn't create file {}:\n{}", to.display(), e))?;
let _ = fs::copy(&from, &to).with_context(|e| {
format_err!(
"couldn't copy {} to {}:\n{}",
from.display(),
to.display(),
e
)
})?;
Ok(())
}
pub fn remove_bins(&self, query: &Spec, bins: &[&str]) -> Res<u32> {
fn contains(sum: &str, query: &Spec) -> bool {
match (
&query.name,
query.resolution.as_ref(),
query.version.as_ref(),
) {
(_, Some(_), Some(_)) => sum == query.to_string(),
(name, None, Some(ver)) => {
sum.starts_with(&name.to_string()) && sum.ends_with(&ver.to_string())
}
_ => sum.starts_with(&query.to_string()),
}
};
let mut c = 0;
if self.layout.bin.join(".bins").exists() {
let mut s = String::new();
let mut f = fs::OpenOptions::new()
.read(true)
.open(self.layout.bin.join(".bins"))
.with_context(|e| format_err!("could not open .bins file:\n{}", e))?;
f.read_to_string(&mut s)
.with_context(|e| format_err!("could not read from .bins file:\n{}", e))?;
let dot: IndexMap<String, String> = toml::from_str(&s)
.with_context(|e| format_err!("could not deserialize .bins file:\n{}", e))?;
let (discard, dot): (IndexMap<_, _>, IndexMap<_, _>) =
dot.into_iter().partition(|(bin, sum)| {
(bins.is_empty() || bins.contains(&bin.as_str())) && contains(sum, query)
});
let sums = discard.iter().dedup().collect::<Vec<_>>();
if sums.len() > 1 {
return Err(format_err!(
"spec `{}` is ambiguous between {:?}",
query,
sums
));
}
for (bin, _) in discard {
fs::remove_file(self.layout.bin.join(&bin))
.with_context(|e| format_err!("couldn't remove binary {}:\n{}", bin, e))?;
c += 1;
}
drop(f);
fs::remove_file(self.layout.bin.join(".bins"))
.with_context(|e| format_err!("could not clear existing .bins file:\n{}", e))?;
fs::write(
self.layout.bin.join(".bins"),
toml::to_string(&dot).unwrap().as_bytes(),
)
.with_context(|e| format_err!("could not write to .bins file:\n{}", e))?;
}
Ok(c)
}
pub fn store_build(&self, from: &Path, hash: &BuildHash) -> Res<Binary> {
let dest = self.layout.build.join(&hash.0);
if !dest.exists() {
fs::create_dir_all(&dest)?;
}
let dest = DirLock::acquire(&dest)?;
clear_dir(dest.path())?;
copy_dir(from, dest.path(), false)?;
Ok(Binary { target: dest })
}
fn check_build(&self, hash: &BuildHash) -> Option<PathBuf> {
let path = self.layout.build.join(&hash.0);
if path.exists() {
Some(path)
} else {
None
}
}
pub fn get_indices(&self, index_reses: &[DirectRes], eager: bool, offline: bool) -> Indices {
let mut indices = vec![];
let mut seen = vec![];
let mut q: VecDeque<DirectRes> = index_reses.iter().cloned().collect();
while let Some(index) = q.pop_front() {
if seen.contains(&index) {
continue;
}
if let DirectRes::Registry { .. } = &index {
self.shell.println(
style("[warn]").yellow().bold(),
format!("Ignoring registry resolution: {}", index),
Verbosity::Quiet,
);
continue;
}
if let DirectRes::Dir { path } = &index {
let lock = match DirLock::acquire(path) {
Ok(dir) => dir,
Err(e) => {
self.shell.println(
style("[warn]").yellow().bold(),
format!("Couldn't lock dir index {}: {}", path.display(), e),
Verbosity::Quiet,
);
continue;
}
};
let ix = Index::from_disk(index.clone(), lock);
if let Ok(ix) = ix {
for dependent in ix.depends().cloned().map(|i| i.res) {
q.push_back(dependent);
}
seen.push(index);
indices.push(ix);
}
continue;
}
let index_path = self.layout.indices.join(Self::get_index_dir(&index));
let dir = match DirLock::acquire(&index_path) {
Ok(dir) => dir,
Err(e) => {
self.shell.println(
style("[warn]").yellow().bold(),
format!("Couldn't lock cached index {}: {}", index, e),
Verbosity::Quiet,
);
continue;
}
};
let res = index.retrieve(&self.client, &dir, eager, |dl_online| {
if offline && dl_online {
return Err(format_err!("Offline mode; can't update indices"));
}
self.shell.println(
style("Retrieving").cyan(),
format!("index {}", &index),
Verbosity::Normal,
);
Ok(())
});
match res {
Ok(_) => {
let ix = Index::from_disk(index.clone(), dir);
match ix {
Ok(ix) => {
for dependent in ix.depends().cloned().map(|i| i.res) {
q.push_back(dependent);
}
seen.push(index);
indices.push(ix);
}
Err(e) => {
self.shell.println(
style("[warn]").yellow().bold(),
format!("Invalid/corrupt index {}: {}", index, e),
Verbosity::Quiet,
);
}
}
}
Err(e) => {
self.shell.println(
style("[warn]").yellow().bold(),
format!("Couldn't retrieve cache {}: {}", index, e),
Verbosity::Quiet,
);
}
}
}
Indices::new(indices)
}
fn get_index_dir(loc: &DirectRes) -> String {
Self::get_source_dir(loc, false)
}
pub fn cached_packages(&self) -> IndexSet<String> {
let walker = WalkDir::new(&self.layout.src)
.min_depth(1)
.into_iter()
.filter_map(|e| e.ok());
let mut res = IndexSet::new();
for dir in walker {
let fname = dir
.path()
.file_name()
.unwrap()
.to_string_lossy()
.to_string();
if dir.path().join("elba.toml").exists() {
res.insert(fname);
}
}
res
}
}
#[derive(Debug, Clone)]
pub struct Layout {
pub bin: PathBuf,
pub src: PathBuf,
pub build: PathBuf,
pub tmp: PathBuf,
pub indices: PathBuf,
}
impl Layout {
pub fn init(&self) -> Res<()> {
fs::create_dir_all(&self.bin)?;
fs::create_dir_all(&self.src)?;
fs::create_dir_all(&self.build)?;
fs::create_dir_all(&self.indices)?;
fs::create_dir_all(&self.tmp)?;
Ok(())
}
}
#[derive(Debug)]
pub struct OutputLayout {
lock: DirLock,
pub root: PathBuf,
pub artifacts: PathBuf,
pub bin: PathBuf,
pub docs: PathBuf,
pub lib: PathBuf,
pub build: PathBuf,
pub hash: Option<BuildHash>,
}
impl OutputLayout {
pub fn new(lock: DirLock) -> Res<Self> {
let root = lock.path().to_path_buf();
let layout = OutputLayout {
lock,
root: root.clone(),
artifacts: root.join("artifacts"),
bin: root.join("bin"),
docs: root.join("docs"),
lib: root.join("lib"),
build: root.join("build"),
hash: fs::read(root.join("hash"))
.map(|x| BuildHash(String::from_utf8_lossy(&x).to_string()))
.ok(),
};
fs::create_dir_all(&layout.root)?;
fs::create_dir_all(&layout.artifacts)?;
fs::create_dir_all(&layout.bin)?;
fs::create_dir_all(&layout.docs)?;
fs::create_dir_all(&layout.lib)?;
fs::create_dir_all(&layout.build)?;
Ok(layout)
}
pub fn write_hash(&self, hash: &BuildHash) -> Res<()> {
fs::write(self.root.join("hash"), hash.0.as_bytes())
.context(format_err!("couldn't write hash"))?;
Ok(())
}
pub fn is_built(&self, hash: &BuildHash) -> bool {
self.hash.as_ref() == Some(hash)
}
}
#[derive(Debug, Clone)]
pub struct Source {
inner: Arc<SourceInner>,
}
#[derive(Debug)]
struct SourceInner {
meta: Manifest,
res: Resolution,
location: DirectRes,
path: DirLock,
hash: String,
}
impl Source {
pub fn from_folder(pkg: &PackageId, path: DirLock, location: DirectRes) -> Res<Self> {
let mf_path = path.path().join("elba.toml");
let file = fs::File::open(mf_path).context(format_err!(
"package {} at {} is missing manifest",
pkg,
path.path().display()
))?;
let mut file = BufReader::new(file);
let mut contents = String::new();
file.read_to_string(&mut contents)?;
if let Some(x) = Manifest::workspace(&contents) {
if let Some(p) = x.get(pkg.name()) {
let lock = DirLock::acquire(&path.path().join(&p.0))?;
drop(path);
return Source::from_folder(pkg, lock, location);
}
}
let manifest = Manifest::from_str(&contents)?;
if manifest.name() != pkg.name() {
bail!(
"names don't match: {} was declared, but {} was found in elba.toml",
pkg.name(),
manifest.name()
)
}
let walker = manifest
.list_files(path.path(), path.path(), |entry| {
entry.file_name() != ".git" && entry.file_name() != "target"
})?
.filter(valid_file);
let mut hash = Sha256::new();
for f in walker {
let mut file = fs::File::open(f.path())?;
let fh = Sha256::digest_reader(&mut file)?;
hash.input(&fh);
}
let hash = hexify_hash(hash.result().as_slice());
Ok(Source {
inner: Arc::new(SourceInner {
meta: manifest,
res: if pkg.resolution().direct().is_some() {
location.clone().into()
} else {
pkg.resolution().clone()
},
location,
path,
hash,
}),
})
}
pub fn pretty_summary(&self) -> String {
format!(
"{} {} ({})",
self.meta().package.name,
self.meta().version(),
self.inner.res,
)
}
pub fn summary(&self) -> String {
format!(
"{}@{}|{}",
self.meta().package.name,
self.inner.res,
self.meta().version(),
)
}
pub fn meta(&self) -> &Manifest {
&self.inner.meta
}
pub fn location(&self) -> &DirectRes {
&self.inner.location
}
pub fn hash(&self) -> &str {
&self.inner.hash
}
pub fn path(&self) -> &Path {
self.inner.path.path()
}
}
impl PartialEq for Source {
fn eq(&self, other: &Self) -> bool {
self.hash() == other.hash()
}
}
impl Eq for Source {}
#[derive(Debug, PartialEq, Eq)]
pub struct Binary {
pub target: DirLock,
}
#[derive(Debug, PartialEq, Eq, Clone)]
pub struct BuildHash(pub String);
impl BuildHash {
pub fn new(
root: &Source,
sources: &Graph<Source>,
targets: &Targets,
ctx: &BuildContext,
codegen: bool,
) -> Self {
let mut hasher = Sha256::default();
for (_, src) in sources.sub_tree(sources.find_id(root).unwrap()) {
hasher.input(&src.hash().as_bytes());
}
if let Ok(ver) = ctx.compiler.version() {
hasher.input(ver.as_bytes());
}
for opt in ctx.opts {
hasher.input(opt.as_bytes());
}
if codegen {
hasher.input(ctx.backend.name.as_bytes());
for opt in &ctx.backend.opts {
hasher.input(opt.as_bytes());
}
hasher.input(
ctx.backend
.extension
.as_ref()
.map(|x| x.as_bytes())
.unwrap_or(&[]),
);
}
for t in &targets.0 {
let bytes: [u8; 5] = t.as_bytes();
hasher.input(&bytes);
}
let hash = hexify_hash(hasher.result().as_slice());
BuildHash(hash)
}
}