cargo_clone/
lib.rs

1//! A library to create a local clone of a Cargo package.
2//!
3//! This fetches the repository for a package as defined in the `repository`
4//! field of a Cargo package. If there is no repository, it can also fetch the
5//! `.crate` file from crates.io.
6
7#![warn(missing_docs)]
8use anyhow::{anyhow, bail, Context, Error};
9use flate2::read::GzDecoder;
10use regex::Regex;
11use reqwest::StatusCode;
12use semver;
13use serde_json::Value;
14use std::env;
15use std::path::PathBuf;
16use std::process::Command;
17use tar::Archive;
18
19#[macro_use]
20extern crate log;
21
22static APP_USER_AGENT: &str = concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION"));
23
24/// https://api.bitbucket.org/2.0/repositories
25pub const DEFAULT_BITBUCKET_URL: &'static str = "https://api.bitbucket.org/2.0/repositories";
26/// https://github.com
27pub const DEFAULT_GITHUB_URL: &'static str = "https://github.com";
28/// https://gitlab.com
29pub const DEFAULT_GITLAB_URL: &'static str = "https://gitlab.com";
30/// https://crates.io
31pub const DEFAULT_REGISTRY_URL: &'static str = "https://crates.io";
32
33/// An enum representation of supported cloning methods.
34#[derive(Debug, Clone)]
35pub enum CloneMethodKind {
36    /// Downloads the `.crate` file from the registry and extracts it.
37    Crate,
38    /// Clones using `git`.
39    Git,
40    /// Clones using `hg`.
41    Mercurial,
42    /// Clones using `pijul`.
43    Pijul,
44    /// Clones using `fossil`.
45    Fossil,
46    /// Attempts to automatically detect which method to use using heuristics.
47    Auto,
48}
49
50impl CloneMethodKind {
51    /// Returns the underlying command line command for the method.
52    pub fn command(&self) -> &str {
53        match *self {
54            CloneMethodKind::Crate => "crate",
55            CloneMethodKind::Git => "git",
56            CloneMethodKind::Mercurial => "hg",
57            CloneMethodKind::Pijul => "pijul",
58            CloneMethodKind::Fossil => "fossil",
59            CloneMethodKind::Auto => "auto",
60        }
61    }
62
63    /// Creates a `CloneMethodKind` from a method name. If no name matches then None is returned.
64    /// Current options are `crate`, `git`, `hg`, `mercurial`, `pijul`, `fossil`, and `auto`
65    pub fn from(method_name: &str) -> Option<CloneMethodKind> {
66        match method_name {
67            "crate" => Some(CloneMethodKind::Crate),
68            "git" => Some(CloneMethodKind::Git),
69            "hg" => Some(CloneMethodKind::Mercurial),
70            "mercurial" => Some(CloneMethodKind::Mercurial),
71            "pijul" => Some(CloneMethodKind::Pijul),
72            "fossil" => Some(CloneMethodKind::Fossil),
73            "auto" => Some(CloneMethodKind::Auto),
74            _ => None,
75        }
76    }
77}
78
79/// A struct containg all url and workspace information necessary to clone a crate.
80#[derive(Debug, Clone)]
81pub struct Cloner {
82    /// Defaults to https://crates.io
83    registry_url: String,
84
85    /// Defaults to https://github.com
86    github_url: String,
87
88    /// Defaults to https://gitlab.com
89    gitlab_url: String,
90
91    /// Defaults to https://api.bitbucket.org/2.0/repositories
92    bitbutcket_url: String,
93
94    /// Output directory of the Crate source code.
95    ///
96    /// Uses `std::env::current_dir()` if `None`.
97    out_dir: Option<PathBuf>,
98}
99
100fn check_semver_req(version: &str) -> Result<String, Error> {
101    let first = version
102        .chars()
103        .nth(0)
104        .ok_or_else(|| anyhow!("version is empty"))?;
105
106    let is_req = "<>=^~".contains(first) || version.contains('*');
107    if is_req {
108        Ok(version.parse::<semver::VersionReq>()?.to_string())
109    } else {
110        match semver::Version::parse(version) {
111            Ok(v) => Ok(format!("={}", v)),
112            Err(e) => Err(e).context(anyhow!(
113                "`{}` is not a valid semver version.\n\
114                 Use an exact version like 1.2.3 or a version requirement expression.",
115                version
116            ))?,
117        }
118    }
119}
120
121/// Determine the repo path from the package info.
122fn get_repo(pkg_info: &Value) -> Result<Option<String>, Error> {
123    let krate = pkg_info
124        .get("crate")
125        .ok_or_else(|| anyhow!("`crate` expected in pkg info"))?;
126    let repo = &krate["repository"];
127    if repo.is_string() {
128        return Ok(Some(repo.as_str().unwrap().to_string()));
129    }
130    let home = &krate["homepage"];
131    if home.is_string() {
132        return Ok(Some(home.as_str().unwrap().to_string()));
133    }
134    Ok(None)
135}
136
137/// A wrapper around `reqwest::blocking::get` that provides a User Agent. This
138/// is required by crates.io
139fn reqwest_get(url: &str) -> reqwest::Result<reqwest::blocking::Response> {
140    let client = reqwest::blocking::Client::builder()
141        .user_agent(APP_USER_AGENT)
142        .build()?;
143
144    client.get(url).send()
145}
146
147impl Cloner {
148    /// Create a Crate Cloner using all the default settings
149    pub fn new() -> Cloner {
150        Cloner {
151            registry_url: DEFAULT_REGISTRY_URL.to_string(),
152            github_url: DEFAULT_GITHUB_URL.to_string(),
153            gitlab_url: DEFAULT_GITLAB_URL.to_string(),
154            bitbutcket_url: DEFAULT_BITBUCKET_URL.to_string(),
155            out_dir: None,
156        }
157    }
158
159    /// Sets the URL to use for downloading `.crate` files from crates.io.
160    pub fn set_registry_url(&mut self, value: impl Into<String>) -> &mut Self {
161        self.registry_url = value.into();
162        self
163    }
164
165    /// Sets the URL to use for downloading GitHub repositories.
166    pub fn set_github_url(&mut self, value: impl Into<String>) -> &mut Self {
167        self.github_url = value.into();
168        self
169    }
170
171    /// Sets the URL to use for downloading GitLab repositories.
172    pub fn set_gitlab_url(&mut self, value: impl Into<String>) -> &mut Self {
173        self.gitlab_url = value.into();
174        self
175    }
176
177    /// Sets the URL to use for downloading Bitbucket repositories.
178    pub fn set_bitbucket_url(&mut self, value: impl Into<String>) -> &mut Self {
179        self.bitbutcket_url = value.into();
180        self
181    }
182
183    /// Sets the directory where the clone will be done.
184    ///
185    /// The package will appear as a directory underneath the given path.
186    pub fn set_out_dir(&mut self, value: impl Into<PathBuf>) -> &mut Self {
187        self.out_dir = Some(value.into());
188        self
189    }
190
191    /// Returns the output directory.
192    fn out_dir(&self) -> Result<PathBuf, Error> {
193        Ok(self
194            .out_dir
195            .as_ref()
196            .map_or_else(|| env::current_dir(), |v| Ok(v.to_path_buf()))?)
197    }
198
199    /// Clones a crate using the provided method.
200    ///
201    /// - `method_kind` - Method to fetch crate.
202    /// - `spec` - The name of the crate to clone
203    /// - `version` - The semantic version (semver) of the spec crate to clone
204    /// - `extra` - Additional arguments passed to clone command.
205    ///
206    pub fn clone(
207        &self,
208        method_kind: CloneMethodKind,
209        spec: &str,
210        version: Option<&str>,
211        extra: &[&str],
212    ) -> Result<(), Error> {
213        let mut parts = spec.splitn(2, &[':', '@']);
214        let name = parts.next().unwrap();
215        let spec_version_req = parts.next();
216        if spec_version_req.is_some() && version.is_some() {
217            bail!("Cannot specify both a :version and --version.");
218        }
219        let version_req = version
220            .or(spec_version_req)
221            .map(check_semver_req)
222            .transpose()?;
223        let pkg_info = self.get_pkg_info(name)?;
224        let repo = get_repo(&pkg_info)?;
225        let (method, repo) = match method_kind {
226            CloneMethodKind::Auto => {
227                if version_req.is_some() {
228                    (CloneMethodKind::Crate, "".to_string())
229                } else if let Some(repo) = repo {
230                    self.detect_repo(&repo)?
231                } else {
232                    (CloneMethodKind::Crate, "".to_string())
233                }
234            }
235            CloneMethodKind::Crate => (method_kind, "".to_string()),
236            _ => {
237                if repo.is_none() {
238                    bail!("Could not find repository path in crates.io.");
239                }
240                (method_kind, repo.unwrap())
241            }
242        };
243        match method {
244            CloneMethodKind::Crate => {
245                if !extra.is_empty() {
246                    bail!("Got extra arguments, crate downloads take no extra arguments.");
247                }
248                self.clone_crate(name, version_req, &pkg_info)?;
249            }
250            CloneMethodKind::Git
251            | CloneMethodKind::Mercurial
252            | CloneMethodKind::Pijul
253            | CloneMethodKind::Fossil => {
254                if let Some(version_req) = version_req {
255                    bail!(
256                        "Specifying a version `{}` only works with the `crate` method.",
257                        version_req
258                    );
259                }
260                self.run_clone(method.command(), &repo, extra)?;
261            }
262            CloneMethodKind::Auto => unreachable!(),
263        }
264
265        Ok(())
266    }
267
268    fn detect_repo(&self, repo: &str) -> Result<(CloneMethodKind, String), Error> {
269        if repo.ends_with(".git") {
270            return Ok((CloneMethodKind::Git, repo.to_string()));
271        }
272        if let Some(c) = Regex::new(r"https?://(?:www\.)?github\.com/([^/]+)/([^/]+)")
273            .unwrap()
274            .captures(repo)
275        {
276            return Ok((
277                CloneMethodKind::Git,
278                format!(
279                    "{}/{}/{}.git",
280                    self.github_url,
281                    c.get(1).unwrap().as_str(),
282                    c.get(2).unwrap().as_str()
283                ),
284            ));
285        }
286        if let Some(c) = Regex::new(r"https?://(?:www\.)?gitlab\.com/([^/]+)/([^/]+)")
287            .unwrap()
288            .captures(repo)
289        {
290            return Ok((
291                CloneMethodKind::Git,
292                format!(
293                    "{}/{}/{}.git",
294                    self.gitlab_url,
295                    c.get(1).unwrap().as_str(),
296                    c.get(2).unwrap().as_str()
297                ),
298            ));
299        }
300        if let Some(c) = Regex::new(r"https?://(?:www\.)?bitbucket\.(?:org|com)/([^/]+)/([^/]+)")
301            .unwrap()
302            .captures(repo)
303        {
304            let user = c.get(1).unwrap().as_str();
305            let name = c.get(2).unwrap().as_str();
306            return self.bitbucket(user, name);
307        }
308        if repo.starts_with("https://nest.pijul.com/") {
309            return Ok((CloneMethodKind::Pijul, repo.to_string()));
310        }
311        bail!(
312            "Could not determine the VCS from repo `{}`, \
313             use the `--method` option to specify how to download.",
314            repo
315        );
316    }
317
318    fn bitbucket(&self, user: &str, name: &str) -> Result<(CloneMethodKind, String), Error> {
319        // Determine if it is git or hg.
320        let api_url = &format!("{}/{}/{}", self.bitbutcket_url, user, name);
321        let repo_info =
322            reqwest_get(api_url).context("Failed to fetch repo info from bitbucket.")?;
323        let code = repo_info.status();
324        if !code.is_success() {
325            bail!(
326                "Failed to get repo info from bitbucket API `{}`: `{}`",
327                api_url,
328                code
329            );
330        }
331        let repo_info: Value = repo_info
332            .json()
333            .context("Failed to convert to bitbucket json.")?;
334        let method = repo_info["scm"]
335            .as_str()
336            .expect("Could not get `scm` from bitbucket.");
337        let method = match method {
338            "git" => CloneMethodKind::Git,
339            "hg" => CloneMethodKind::Mercurial,
340            _ => bail!("Unexpected bitbucket scm: `{}`", method),
341        };
342        let clones = repo_info["links"]["clone"]
343            .as_array()
344            .expect("Could not get `clone` from bitbucket.");
345        let href = clones
346            .iter()
347            .find(|c| {
348                c["name"]
349                    .as_str()
350                    .expect("Could not get clone `name` from bitbucket.")
351                    == "https"
352            })
353            .expect("Could not find `https` clone in bitbucket.")["href"]
354            .as_str()
355            .expect("Could not get clone `href` from bitbucket.");
356        Ok((method, href.to_string()))
357    }
358
359    /// Grab package info from crates.io.
360    fn get_pkg_info(&self, name: &str) -> Result<Value, Error> {
361        let url = format!("{}/api/v1/crates/{}", self.registry_url, name);
362        debug!("GET {url}");
363        let pkg_info = reqwest_get(&url).context("Failed to fetch package info from crates.io.")?;
364        let code = pkg_info.status();
365        match code {
366            StatusCode::OK => {}
367            StatusCode::NOT_FOUND => bail!("Package `{}` not found on crates.io.", name),
368            _ => bail!("Failed to get package info from crates.io: `{}`", code),
369        }
370        let pkg_info: Value = pkg_info.json().context("Failed to convert to json.")?;
371        Ok(pkg_info)
372    }
373
374    /// Download a crate from crates.io.
375    fn clone_crate(
376        &self,
377        name: &str,
378        version_req: Option<String>,
379        pkg_info: &Value,
380    ) -> Result<(), Error> {
381        // Determine which version to download.
382        let versions = pkg_info["versions"]
383            .as_array()
384            .expect("Could not find `versions` array on crates.io.");
385        let versions = versions.iter().map(|crate_version| {
386            let num = crate_version["num"]
387                .as_str()
388                .expect("Could not get `num` from version.");
389            let v = semver::Version::parse(num).expect("Could not parse crate `num`.");
390            (crate_version, v)
391        });
392        let mut versions: Vec<_> = if let Some(version_req) = version_req {
393            let req = semver::VersionReq::parse(&version_req)?;
394            versions
395                .filter(|(_crate_version, ver)| req.matches(ver))
396                .collect()
397        } else {
398            versions.collect()
399        };
400        // Find the largest version.
401        if versions.is_empty() {
402            bail!("Could not find any matching versions.");
403        }
404        versions.sort_unstable_by_key(|x| x.1.clone());
405        let last = versions.last().unwrap().0;
406        let dl_path = last["dl_path"]
407            .as_str()
408            .expect("Could not find `dl_path` in crate version info.");
409        let dl_path = format!("{}{}", self.registry_url, dl_path);
410        let version = last["num"]
411            .as_str()
412            .expect("Could not find `num` in crate version info.");
413        info!("Downloading `{}`", dl_path);
414        let mut response =
415            reqwest_get(&dl_path).context(format!("Failed to download `{}`", dl_path))?;
416        // TODO: This could be much better.
417        let mut body = Vec::new();
418        response.copy_to(&mut body)?;
419        let gz = GzDecoder::new(body.as_slice());
420        let mut tar = Archive::new(gz);
421        let base = format!("{}-{}", name.to_lowercase(), version);
422
423        for entry in tar.entries()? {
424            let mut entry = entry.context("Failed to get tar entry.")?;
425            let entry_path = entry
426                .path()
427                .context("Failed to read entry path.")?
428                .into_owned();
429            info!("{}", entry_path.display());
430
431            // Sanity check.
432            if !entry_path.starts_with(&base) {
433                bail!(
434                    "Expected path `{}` in tarball, got `{}`.",
435                    base,
436                    entry_path.display()
437                );
438            }
439
440            entry.unpack_in(&self.out_dir()?).context(format!(
441                "failed to unpack entry at `{}`",
442                entry_path.display()
443            ))?;
444        }
445        Ok(())
446    }
447
448    /// Runs the clone process.
449    fn run_clone(&self, method: &str, repo: &str, extra: &[&str]) -> Result<(), Error> {
450        info!("Running: {} clone {} {}", method, repo, extra.join(" "));
451        let status = Command::new(method)
452            .arg("clone")
453            .arg(repo)
454            .args(extra)
455            .current_dir(&self.out_dir()?)
456            .status()
457            .context(format!("Failed to run `{}`.", method))?;
458        if !status.success() {
459            bail!("`{} clone` did not finish successfully.", method);
460        }
461        Ok(())
462    }
463}
464
465/// A helper function for cloning a crate into the current working directory.
466///
467/// - `method_name` - Method to fetch crate. Options are "crate", "git", "hg", "pijul", "fossil", "auto"
468/// - `spec` - The name of the crate to clone
469/// - `version` - The semantic version (semver) of the spec crate to clone
470/// - `extra` - Additional arguments passed to clone command.
471///
472pub fn clone(
473    method_name: &str,
474    spec: &str,
475    version: Option<&str>,
476    extra: &[&str],
477) -> Result<(), Error> {
478    Cloner::new().clone(
479        CloneMethodKind::from(method_name).unwrap(),
480        spec,
481        version,
482        extra,
483    )
484}
cargo_clone/lib.rs

cargo_clone/
lib.rs
