cargo_depot/
lib.rs

1use std::collections::{BTreeMap, HashSet};
2use std::io::Write;
3use std::path::{Path, PathBuf};
4
5use itertools::Itertools;
6use log::*;
7use serde::{Deserialize, Serialize};
8use sha2::Digest;
9
10const INDEX: &str = "index";
11const CRATES: &str = "crates";
12
13#[derive(clap::Parser)]
14pub struct FeaturesFlags {
15    #[clap(long, conflicts_with = "features")]
16    all_features: bool,
17    #[clap(long)]
18    features: Option<String>,
19}
20impl FeaturesFlags {
21    fn flags(&self) -> String {
22        if self.all_features {
23            "--all-features".into()
24        } else {
25            format!("--features={}", self.features.clone().unwrap_or_default())
26        }
27    }
28}
29
30/// config.json at the root of the index
31#[derive(serde::Serialize)]
32pub struct IndexConfig {
33    dl: String,
34}
35impl IndexConfig {
36    pub fn from_url(url: &str) -> Self {
37        Self {
38            dl: format!(
39                "{}/{}/{{crate}}/{{crate}}-{{version}}.crate",
40                url.trim_end_matches('/'),
41                CRATES,
42            ),
43        }
44    }
45    pub fn write(&self, index: &Path) -> anyhow::Result<()> {
46        std::fs::create_dir_all(index)?;
47        Ok(std::fs::write(
48            index.join("config.json"),
49            serde_json::to_string_pretty(&self)?,
50        )?)
51    }
52}
53
54// See reg_index/src/util.rs and https://doc.rust-lang.org/cargo/reference/registry-index.html#index-files
55pub fn pkg_path(name: &str) -> PathBuf {
56    let name = name.to_lowercase();
57    match name.len() {
58        1 => PathBuf::from("1"),
59        2 => PathBuf::from("2"),
60        3 => Path::new("3").join(&name[..1]),
61        _ => Path::new(&name[0..2]).join(&name[2..4]),
62    }
63}
64
65#[derive(Serialize, Deserialize)]
66struct Dependency {
67    name: String,
68    req: cargo_metadata::semver::VersionReq,
69    features: Vec<String>,
70    optional: bool,
71    default_features: bool,
72    target: Option<cargo_platform::Platform>,
73    kind: cargo_metadata::DependencyKind,
74    registry: Option<String>,
75    package: Option<String>,
76}
77impl From<cargo_metadata::Dependency> for Dependency {
78    fn from(s: cargo_metadata::Dependency) -> Self {
79        Self {
80            name: s.name,
81            req: s.req,
82            features: s.features,
83            optional: s.optional,
84            default_features: s.uses_default_features,
85            target: s.target,
86            kind: s.kind,
87            // Note source -> registry
88            registry: s.source.clone(),
89            package: None,
90        }
91    }
92}
93
94fn check_dirty(repository: &Path) -> anyhow::Result<()> {
95    let out = std::process::Command::new("git")
96        .args(["status", "--porcelain"])
97        .current_dir(repository)
98        .output()?;
99    if !out.status.success() {
100        // Likely not a git repository
101        return Ok(());
102    }
103
104    let out = String::from_utf8_lossy(&out.stdout);
105    let out = out
106        .lines()
107        .filter(|l| !l.trim().is_empty())
108        // This gets filtered by cargo package anyway
109        .filter(|l| !l.contains("Cargo.lock"))
110        .collect_vec();
111    anyhow::ensure!(out.is_empty(), "Repository not clean: {}. These files would be embedded in the package. Stash them with `git stash -u` or add them to gitignore", out.join(" "));
112    Ok(())
113}
114
115// https://doc.rust-lang.org/cargo/reference/registry-index.html#json-schema
116#[derive(Serialize, Deserialize)]
117pub struct IndexMeta {
118    name: String,
119    vers: cargo_metadata::semver::Version,
120    deps: Vec<Dependency>,
121    features: BTreeMap<String, Vec<String>>,
122    license: Option<String>,
123    license_file: Option<cargo_metadata::camino::Utf8PathBuf>,
124    cksum: String,
125    v: u8,
126    yanked: bool,
127}
128impl IndexMeta {
129    pub fn from_package(p: &cargo_metadata::Package, checksum: String) -> Self {
130        let mut deps: Vec<Dependency> = vec![];
131        for dep_meta in &p.dependencies {
132            let mut dep = Dependency::from(dep_meta.clone());
133            if dep.registry.as_ref().map_or(false, |s| {
134                s != "registry+https://github.com/rust-lang/crates.io-index"
135            }) || dep_meta.path.is_some()
136            {
137                // Use our registry when the package is a path, a git repository, or another
138                // registry.
139                dep.registry = None;
140            }
141            // Renames
142            if let Some(original) = dep_meta.rename.clone() {
143                dep.package = Some(dep.name);
144                dep.name = original;
145            }
146            deps.push(dep);
147        }
148        // Remove features referencing removed dependencies
149        let deps_names: HashSet<_> = deps.iter().map(|d| d.name.clone()).collect();
150        let mut features = p.features.clone();
151        features.iter_mut().for_each(|(_, v)| {
152            v.retain(|f| {
153                if deps_names.contains(f) {
154                    true
155                } else if let Some(dep) = f.strip_prefix("dep:") {
156                    deps_names.contains(dep)
157                } else if let Some((dep, _)) = f.split_once("/") {
158                    deps_names.contains(dep.trim_end_matches('?'))
159                } else {
160                    true
161                }
162            })
163        });
164        Self {
165            deps,
166            name: p.name.clone(),
167            vers: p.version.clone(),
168            features,
169            license: p.license.clone(),
170            license_file: p.license_file.clone(),
171            cksum: checksum,
172            v: 2,
173            yanked: false,
174        }
175    }
176}
177
178pub struct Registry(pub PathBuf);
179impl Registry {
180    pub fn package_index(&self, name: &str) -> PathBuf {
181        self.0.join(INDEX).join(pkg_path(name)).join(name)
182    }
183    pub fn read_package(&self, name: &str) -> anyhow::Result<Vec<IndexMeta>> {
184        let filename = self.package_index(name);
185        if !filename.exists() {
186            return Ok(vec![]);
187        }
188        let mut res = vec![];
189        for line in std::fs::read_to_string(&filename)?
190            .lines()
191            .filter(|l| !l.trim().is_empty())
192        {
193            res.push(serde_json::from_str(line)?);
194        }
195        Ok(res)
196    }
197    pub fn add_package(
198        &self,
199        p: &cargo_metadata::Package,
200        workspace_metadata: &cargo_metadata::Metadata,
201        features: &FeaturesFlags,
202    ) -> anyhow::Result<()> {
203        if !p
204            .targets
205            .iter()
206            .any(|t| t.is_lib() || t.kind.contains(&"proc-macro".into()))
207        {
208            warn!("Skipping non-library package");
209            return Ok(());
210        }
211        // Check if already in the index
212        if self
213            .read_package(&p.name)?
214            .into_iter()
215            .any(|p_index| p_index.vers == p.version)
216        {
217            warn!("Package already in the index, skipping");
218            return Ok(());
219        }
220
221        check_dirty(workspace_metadata.workspace_root.as_std_path())?;
222        // Edit manifest
223        info!("Editing manifest");
224        let manifest = std::fs::read_to_string(&p.manifest_path)?;
225        let mut manifest: cargo_util_schemas::manifest::TomlManifest = toml::from_str(&manifest)?;
226        if let Some(package) = &mut manifest.package {
227            package.autoexamples = Some(false);
228        }
229        manifest.bin = None;
230        manifest.example = None;
231        let manifest_orig = p.manifest_path.with_extension("toml.pre-edit");
232        std::fs::rename(&p.manifest_path, &manifest_orig)?;
233        std::fs::write(&p.manifest_path, toml::to_string_pretty(&manifest)?)?;
234
235        info!("Building package");
236        let parent = self.0.join(CRATES).join(&p.name);
237        std::fs::create_dir_all(&parent)?;
238        // Do not use .with_extension due to the . in the name.
239        let crate_dest = parent.join(format!("{}-{}.crate", p.name, p.version));
240
241        let out = std::process::Command::new("cargo")
242            .args([
243                "package",
244                "-p",
245                &p.name,
246                "--no-verify",
247                &features.flags(),
248                "--allow-dirty",
249            ])
250            .current_dir(p.manifest_path.parent().unwrap())
251            .spawn()?
252            .wait()?;
253        std::fs::rename(manifest_orig, &p.manifest_path)?;
254        anyhow::ensure!(out.success(), "Failed to build package");
255        // Hash .crate
256        let crate_src = workspace_metadata
257            .target_directory
258            .as_std_path()
259            .join("package")
260            .join(crate_dest.file_name().unwrap());
261        let mut hasher = sha2::Sha256::new();
262        let mut file = std::fs::File::open(&crate_src)?;
263        std::io::copy(&mut file, &mut hasher)?;
264        let hash = format!("{:x}", hasher.finalize());
265        // Copy .crate
266        anyhow::ensure!(!crate_dest.exists(), "{:?} already exists", crate_dest);
267        std::fs::copy(crate_src, crate_dest)?;
268
269        // Compute metadata
270        let metadata = IndexMeta::from_package(p, hash);
271
272        // Write to index
273        let index = self.package_index(&p.name);
274        std::fs::create_dir_all(index.parent().unwrap())?;
275        let mut f = std::fs::OpenOptions::new()
276            .create(true)
277            .append(true)
278            .open(index)?;
279        writeln!(f, "{}", serde_json::to_string(&metadata)?)?;
280        Ok(())
281    }
282    pub fn open(root: &Path, url: Option<&str>) -> anyhow::Result<Self> {
283        std::fs::create_dir_all(root)?;
284
285        let index = root.join(INDEX);
286        if !index.join("config.json").exists() {
287            info!("Initializing registry at {:?}", root);
288            let Some(url) = &url else {
289                anyhow::bail!(
290                    "Provide the URL where the registry will be hosted with the --url flag"
291                );
292            };
293            IndexConfig::from_url(url).write(&index)?;
294        }
295        Ok(Self(root.into()))
296    }
297}
cargo_depot/lib.rs

cargo_depot/
lib.rs
