use semver::Version as SemverVersion;
use serde_derive::{Deserialize, Serialize};
use smol_str::SmolStr;
use std::{
collections::HashMap,
io, iter,
path::{Path, PathBuf},
};
mod bare_index;
mod error;
pub use bare_index::{BareIndex, BareIndexRepo};
pub use error::Error;
static INDEX_GIT_URL: &str = "https://github.com/rust-lang/crates.io-index";
#[derive(Serialize, Deserialize, Clone, Debug)]
pub struct Version {
name: SmolStr,
vers: SmolStr,
deps: Box<[Dependency]>,
#[serde(with = "hex")]
cksum: [u8; 32],
features: HashMap<String, Vec<String>>,
yanked: bool,
}
impl Version {
#[inline]
pub fn name(&self) -> &str {
&self.name
}
#[inline]
pub fn version(&self) -> &str {
&self.vers
}
#[inline]
pub fn dependencies(&self) -> &[Dependency] {
&self.deps
}
#[inline]
pub fn checksum(&self) -> &[u8; 32] {
&self.cksum
}
#[inline]
pub fn features(&self) -> &HashMap<String, Vec<String>> {
&self.features
}
#[inline]
pub fn is_yanked(&self) -> bool {
self.yanked
}
}
#[derive(Serialize, Deserialize, Clone, Debug)]
pub struct Dependency {
name: SmolStr,
req: SmolStr,
features: Box<[String]>,
optional: bool,
default_features: bool,
target: Option<Box<str>>,
#[serde(skip_serializing_if = "Option::is_none")]
kind: Option<DependencyKind>,
#[serde(skip_serializing_if = "Option::is_none")]
package: Option<Box<str>>,
}
impl Dependency {
#[inline]
pub fn name(&self) -> &str {
&self.name
}
#[inline]
pub fn requirement(&self) -> &str {
&self.req
}
#[inline]
pub fn features(&self) -> &[String] {
&self.features
}
#[inline]
pub fn is_optional(&self) -> bool {
self.optional
}
#[inline]
pub fn has_default_features(&self) -> bool {
self.default_features
}
#[inline]
pub fn target(&self) -> Option<&str> {
self.target.as_deref()
}
#[inline]
pub fn kind(&self) -> DependencyKind {
self.kind.unwrap_or_default()
}
#[inline]
pub fn package(&self) -> Option<&str> {
self.package.as_deref()
}
#[inline]
pub fn crate_name(&self) -> &str {
match self.package {
Some(ref s) => s,
None => self.name(),
}
}
}
#[derive(Debug, Copy, Clone, Serialize, Deserialize, Eq, PartialEq, Hash)]
#[serde(rename_all = "lowercase")]
pub enum DependencyKind {
Normal,
Dev,
Build,
}
impl Default for DependencyKind {
fn default() -> Self {
Self::Normal
}
}
pub struct Crates(CrateIndexPaths);
impl Iterator for Crates {
type Item = Crate;
fn next(&mut self) -> Option<Self::Item> {
while let Some(p) = self.0.next() {
if let Ok(c) = Crate::new(&p) {
return Some(c);
}
}
None
}
}
pub struct CrateIndexPaths(iter::Chain<iter::Chain<glob::Paths, glob::Paths>, glob::Paths>);
impl CrateIndexPaths {
fn new<P: AsRef<Path>>(path: P) -> CrateIndexPaths {
let mut match_options = glob::MatchOptions::new();
match_options.require_literal_leading_dot = true;
let path = path.as_ref();
let index_paths1 =
glob::glob_with(&format!("{}/*/*/*", path.to_str().unwrap()), match_options).unwrap();
let index_paths2 =
glob::glob_with(&format!("{}/[12]/*", path.to_str().unwrap()), match_options).unwrap();
let index_paths3 =
glob::glob_with(&format!("{}/3/*/*", path.to_str().unwrap()), match_options).unwrap();
CrateIndexPaths(index_paths1.chain(index_paths2).chain(index_paths3))
}
}
impl Iterator for CrateIndexPaths {
type Item = PathBuf;
fn next(&mut self) -> Option<Self::Item> {
self.0.next().map(|glob_result| glob_result.unwrap())
}
}
fn fetch_opts<'cb>() -> git2::FetchOptions<'cb> {
let mut proxy_opts = git2::ProxyOptions::new();
proxy_opts.auto();
let mut fetch_opts = git2::FetchOptions::new();
fetch_opts.proxy_options(proxy_opts);
fetch_opts
}
#[derive(Debug, Clone, PartialEq)]
pub struct Index {
path: PathBuf,
}
impl Index {
pub fn new<P: Into<PathBuf>>(path: P) -> Index {
Index { path: path.into() }
}
pub fn new_cargo_default() -> Index {
let cargo_home = home::cargo_home().unwrap_or_default();
Self::new(
cargo_home
.join("registry")
.join("index")
.join("github.com-1ecc6299db9ec823"),
)
}
pub fn exists(&self) -> bool {
git2::Repository::discover(&self.path)
.map(|repository| {
repository
.find_remote("origin")
.ok()
.map_or(true, |remote| {
remote.url().map_or(true, |url| url == INDEX_GIT_URL)
})
})
.unwrap_or(false)
}
pub fn retrieve(&self) -> Result<(), Error> {
git2::build::RepoBuilder::new()
.fetch_options(fetch_opts())
.clone(INDEX_GIT_URL, &self.path)?;
Ok(())
}
pub fn update(&self) -> Result<(), Error> {
debug_assert!(self.exists());
let repo = git2::Repository::discover(&self.path)?;
let mut origin_remote = repo
.find_remote("origin")
.or_else(|_| repo.remote_anonymous(INDEX_GIT_URL))?;
origin_remote.fetch(&["master"], Some(&mut fetch_opts()), None)?;
let oid = repo.refname_to_id("FETCH_HEAD")?;
let object = repo.find_object(oid, None).unwrap();
repo.reset(&object, git2::ResetType::Hard, None)?;
Ok(())
}
pub fn retrieve_or_update(&self) -> Result<(), Error> {
if self.exists() {
self.update()
} else {
self.retrieve()
}
}
pub fn crate_(&self, crate_name: &str) -> Option<Crate> {
match crate_name_to_relative_path(crate_name) {
Some(rel_path) => {
let path = self.path.join(rel_path);
if path.exists() {
Crate::new(path.as_path()).ok()
} else {
None
}
}
None => None,
}
}
pub fn crates(&self) -> Crates {
Crates(self.crate_index_paths())
}
pub fn crate_index_paths(&self) -> CrateIndexPaths {
CrateIndexPaths::new(&self.path)
}
#[inline]
pub fn path(&self) -> &Path {
&self.path
}
}
fn crate_name_to_relative_path(crate_name: &str) -> Option<String> {
if !crate_name.is_ascii() {
return None;
}
let name_lower = crate_name.to_ascii_lowercase();
let mut rel_path = String::with_capacity(crate_name.len() + 6);
match name_lower.len() {
0 => return None,
1 => rel_path.push('1'),
2 => rel_path.push('2'),
3 => {
rel_path.push('3');
rel_path.push(std::path::MAIN_SEPARATOR);
rel_path.push_str(&name_lower[0..1]);
}
_ => {
rel_path.push_str(&name_lower[0..2]);
rel_path.push(std::path::MAIN_SEPARATOR);
rel_path.push_str(&name_lower[2..4]);
}
};
rel_path.push(std::path::MAIN_SEPARATOR);
rel_path.push_str(&name_lower);
Some(rel_path)
}
#[derive(Serialize, Deserialize, Clone, Debug)]
pub struct Crate {
versions: Box<[Version]>,
}
impl Crate {
#[inline]
pub fn new<P: AsRef<Path>>(index_path: P) -> io::Result<Crate> {
let lines = std::fs::read(index_path)?;
Self::from_slice(&lines)
}
#[doc(hidden)]
#[deprecated(note = "new_checked() is no longer needed, you can use new() now")]
pub fn new_checked<P: AsRef<Path>>(index_path: P) -> io::Result<Crate> {
Self::new(index_path)
}
pub fn from_slice(mut bytes: &[u8]) -> io::Result<Crate> {
while bytes.last() == Some(&b'\n') {
bytes = &bytes[..bytes.len() - 1];
}
#[inline(always)]
fn is_newline(&c: &u8) -> bool {
c == b'\n'
}
let mut versions = Vec::with_capacity(bytes.split(is_newline).count());
for line in bytes.split(is_newline) {
let version: Version = serde_json::from_slice(line)
.map_err(|e| io::Error::new(io::ErrorKind::Other, e))?;
versions.push(version);
}
if versions.is_empty() {
return Err(io::Error::new(
io::ErrorKind::Other,
"crate must have versions",
));
}
debug_assert_eq!(versions.len(), versions.capacity());
Ok(Crate {
versions: versions.into_boxed_slice(),
})
}
pub fn from_cache_slice(bytes: &[u8], index_version: &str) -> io::Result<Crate> {
const CURRENT_CACHE_VERSION: u8 = 1;
let (first_byte, rest) = bytes
.split_first()
.ok_or_else(|| io::Error::new(io::ErrorKind::Other, "malformed cache"))?;
if *first_byte != CURRENT_CACHE_VERSION {
return Err(io::Error::new(
io::ErrorKind::Other,
"looks like a different Cargo's cache, bailing out",
));
}
fn split<'a>(haystack: &'a [u8], needle: u8) -> impl Iterator<Item = &'a [u8]> + 'a {
struct Split<'a> {
haystack: &'a [u8],
needle: u8,
}
impl<'a> Iterator for Split<'a> {
type Item = &'a [u8];
fn next(&mut self) -> Option<&'a [u8]> {
if self.haystack.is_empty() {
return None;
}
let (ret, remaining) = match memchr::memchr(self.needle, self.haystack) {
Some(pos) => (&self.haystack[..pos], &self.haystack[pos + 1..]),
None => (self.haystack, &[][..]),
};
self.haystack = remaining;
Some(ret)
}
}
Split { haystack, needle }
}
let mut iter = split(rest, 0);
if let Some(update) = iter.next() {
if update != index_version.as_bytes() {
return Err(io::Error::new(
io::ErrorKind::Other,
format!(
"cache out of date: current index ({}) != cache ({})",
index_version,
std::str::from_utf8(update)
.map_err(|e| io::Error::new(io::ErrorKind::Other, e))?,
),
));
}
} else {
return Err(io::Error::new(io::ErrorKind::Other, "malformed file"));
}
let mut versions = Vec::new();
while let Some(_version) = iter.next() {
let version_slice = iter
.next()
.ok_or_else(|| io::Error::new(io::ErrorKind::Other, "malformed file"))?;
let version: Version = serde_json::from_slice(version_slice)
.map_err(|e| io::Error::new(io::ErrorKind::Other, e))?;
versions.push(version);
}
Ok(Self {
versions: versions.into_boxed_slice(),
})
}
#[inline]
pub fn versions(&self) -> &[Version] {
&self.versions
}
#[inline]
pub fn earliest_version(&self) -> &Version {
&self.versions[0]
}
#[inline]
pub fn latest_version(&self) -> &Version {
&self.versions[self.versions.len() - 1]
}
#[inline]
pub fn highest_version(&self) -> SemverVersion {
self.versions
.iter()
.map(|v| SemverVersion::parse(&v.vers).ok())
.flatten()
.max()
.unwrap()
}
#[inline]
pub fn highest_stable_version(&self) -> Option<SemverVersion> {
self.versions
.iter()
.map(|v| SemverVersion::parse(&v.vers).ok())
.flatten()
.filter(|v| !v.is_prerelease())
.max()
}
#[inline]
pub fn name(&self) -> &str {
self.latest_version().name()
}
}
#[cfg(test)]
mod test {
use super::{Crate, Index};
use tempdir::TempDir;
#[test]
fn test_dependencies() {
let tmp_dir = TempDir::new("test1").unwrap();
let index = Index::new(tmp_dir.path());
index
.retrieve_or_update()
.expect("could not fetch crates io index");
let crate_ = index
.crate_("sval")
.expect("Could not find the crate libnotify in the index");
let _ = format!("supports debug {:?}", crate_);
let version = crate_
.versions()
.iter()
.find(|v| v.version() == "0.0.1")
.expect("Version 0.0.1 of sval does not exist?");
let dep_with_package_name = version
.dependencies()
.iter()
.find(|d| d.name() == "serde_lib")
.expect("sval does not have expected dependency?");
assert_ne!(
dep_with_package_name.name(),
dep_with_package_name.package().unwrap()
);
assert_eq!(
dep_with_package_name.crate_name(),
dep_with_package_name.package().unwrap()
);
}
#[test]
fn test_retrieve_or_update() {
let tmp_dir = TempDir::new("test2").unwrap();
let index = Index::new(tmp_dir.path());
index
.retrieve_or_update()
.expect("could not fetch crates io index");
assert!(index.exists());
index
.retrieve_or_update()
.expect("could not fetch crates io index");
assert!(index.exists());
}
#[test]
fn test_cargo_default_updates() {
let index = Index::new_cargo_default();
index
.update()
.map_err(|e| {
format!(
"could not fetch cargo's index in {}: {}",
index.path().display(),
e
)
})
.unwrap();
assert!(index.crate_("crates-index").is_some());
assert!(index.crate_("toml").is_some());
assert!(index.crate_("gcc").is_some());
assert!(index.crate_("cc").is_some());
assert!(index.crate_("CC").is_some());
assert!(index.crate_("無").is_none());
}
#[test]
fn test_can_parse_all() {
let tmp_dir = TempDir::new("test3").unwrap();
let mut found_gcc_crate = false;
let index = Index::new(tmp_dir.path());
assert!(!index.exists());
index.retrieve().unwrap();
assert!(index.exists());
for path in index.crate_index_paths() {
match Crate::new(&path) {
Ok(c) => {
if c.name() == "gcc" {
found_gcc_crate = true;
}
}
Err(e) => {
let _ = tmp_dir.into_path();
panic!("{} {}", e, path.display());
}
}
}
assert!(found_gcc_crate);
}
}