use std::path::Path;
#[derive(Debug, Clone)]
pub struct CrateEntry {
pub name: String,
pub version: String,
pub cksum: String,
pub bytes: Vec<u8>,
}
pub fn index_path(name: &str) -> String {
let n = name.to_lowercase();
let take =
|from: usize, to: usize| -> String { n.chars().skip(from).take(to - from).collect() };
match n.chars().count() {
1 => format!("1/{n}"),
2 => format!("2/{n}"),
3 => format!("3/{}/{n}", take(0, 1)),
_ => format!("{}/{}/{n}", take(0, 2), take(2, 4)),
}
}
pub fn validate_crate_name(name: &str) -> Result<(), CrateExportError> {
if name.is_empty() {
return Err(CrateExportError::UnrepresentableName {
name: name.to_string(),
why: "empty".into(),
});
}
if let Some(bad) = name
.chars()
.find(|c| !(c.is_ascii_alphanumeric() || *c == '-' || *c == '_'))
{
return Err(CrateExportError::UnrepresentableName {
name: name.to_string(),
why: format!("contains {bad:?}; Cargo names are ASCII alphanumeric, '-' or '_'"),
});
}
Ok(())
}
pub fn validate_crate_version(version: &str) -> Result<(), CrateExportError> {
if version.is_empty() {
return Err(CrateExportError::UnrepresentableVersion {
version: version.to_string(),
why: "empty".into(),
});
}
if let Some(bad) = version
.chars()
.find(|c| !(c.is_ascii_alphanumeric() || matches!(c, '.' | '-' | '+')))
{
return Err(CrateExportError::UnrepresentableVersion {
version: version.to_string(),
why: format!("contains {bad:?}; semver is ASCII alphanumeric, '.', '-' or '+'"),
});
}
Ok(())
}
fn validate_entries(crates: &[CrateEntry]) -> Result<(), CrateExportError> {
for e in crates {
validate_crate_name(&e.name)?;
validate_crate_version(&e.version)?;
if e.cksum.len() != 64 || !e.cksum.chars().all(|c| c.is_ascii_hexdigit()) {
return Err(CrateExportError::UnrepresentableCksum {
name: e.name.clone(),
cksum: e.cksum.clone(),
});
}
}
Ok(())
}
pub fn index_line(entry: &CrateEntry) -> String {
format!(
r#"{{"name":"{}","vers":"{}","deps":[],"cksum":"{}","features":{{}},"yanked":false}}"#,
entry.name, entry.version, entry.cksum
)
}
pub fn cargo_config_toml(registry_dir: &Path) -> String {
format!(
"# Generated by `varve export-cargo` (REQ-CRATE-001).\n\
# Redirects crates.io to a varve-verified local registry; build --offline.\n\
[source.crates-io]\n\
replace-with = \"varve\"\n\n\
[source.varve]\n\
local-registry = \"{}\"\n",
registry_dir.display()
)
}
#[derive(Debug, thiserror::Error)]
pub enum CrateExportError {
#[error("io error at {path}: {source}")]
Io {
path: String,
#[source]
source: std::io::Error,
},
#[error("crate name {name:?} cannot be exported: {why}")]
UnrepresentableName { name: String, why: String },
#[error("crate version {version:?} cannot be exported: {why}")]
UnrepresentableVersion { version: String, why: String },
#[error("crate {name:?} has a cksum that is not a bare sha256 hex digest: {cksum:?}")]
UnrepresentableCksum { name: String, cksum: String },
}
pub fn cargo_checksum_json(cksum: &str) -> String {
format!(r#"{{"files":{{}},"package":"{cksum}"}}"#)
}
pub fn vendored_config_toml(vendor_dir: &Path) -> String {
format!(
"# Generated by `varve export-crates-vendor` (REQ-VENDOR-001).\n\
[source.crates-io]\n\
replace-with = \"vendored-sources\"\n\n\
[source.vendored-sources]\n\
directory = \"{}\"\n",
vendor_dir.display()
)
}
pub fn export_vendor_dir(
crates: &[CrateEntry],
vendor_dir: &Path,
) -> Result<usize, CrateExportError> {
validate_entries(crates)?;
let io = |path: &Path, source: std::io::Error| CrateExportError::Io {
path: path.display().to_string(),
source,
};
std::fs::create_dir_all(vendor_dir).map_err(|e| io(vendor_dir, e))?;
for entry in crates {
let mut archive = tar::Archive::new(flate2::read::GzDecoder::new(entry.bytes.as_slice()));
archive.unpack(vendor_dir).map_err(|e| io(vendor_dir, e))?;
let crate_dir = vendor_dir.join(format!("{}-{}", entry.name, entry.version));
let checksum = crate_dir.join(".cargo-checksum.json");
std::fs::write(&checksum, cargo_checksum_json(&entry.cksum))
.map_err(|e| io(&checksum, e))?;
}
Ok(crates.len())
}
pub fn export_local_registry(
crates: &[CrateEntry],
registry_dir: &Path,
) -> Result<usize, CrateExportError> {
validate_entries(crates)?;
let io = |path: &Path, source: std::io::Error| CrateExportError::Io {
path: path.display().to_string(),
source,
};
std::fs::create_dir_all(registry_dir).map_err(|e| io(registry_dir, e))?;
for entry in crates {
let crate_file = registry_dir.join(format!("{}-{}.crate", entry.name, entry.version));
std::fs::write(&crate_file, &entry.bytes).map_err(|e| io(&crate_file, e))?;
let idx = registry_dir.join("index").join(index_path(&entry.name));
if let Some(parent) = idx.parent() {
std::fs::create_dir_all(parent).map_err(|e| io(parent, e))?;
}
let mut line = index_line(entry);
line.push('\n');
let existing = std::fs::read_to_string(&idx).unwrap_or_default();
let prefix = format!(r#"{{"name":"{}","vers":"{}""#, entry.name, entry.version);
let mut kept: Vec<String> = existing
.lines()
.filter(|l| !l.starts_with(&prefix))
.map(str::to_string)
.collect();
kept.push(line.trim_end().to_string());
std::fs::write(&idx, kept.join("\n") + "\n").map_err(|e| io(&idx, e))?;
}
Ok(crates.len())
}
pub fn export_distdir(crates: &[CrateEntry], distdir: &Path) -> Result<usize, CrateExportError> {
validate_entries(crates)?;
let io = |path: &Path, source: std::io::Error| CrateExportError::Io {
path: path.display().to_string(),
source,
};
std::fs::create_dir_all(distdir).map_err(|e| io(distdir, e))?;
for entry in crates {
let file = distdir.join(format!("{}-{}.crate", entry.name, entry.version));
std::fs::write(&file, &entry.bytes).map_err(|e| io(&file, e))?;
}
Ok(crates.len())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn index_paths_follow_cargos_layout() {
assert_eq!(index_path("a"), "1/a");
assert_eq!(index_path("ab"), "2/ab");
assert_eq!(index_path("abc"), "3/a/abc");
assert_eq!(index_path("serde"), "se/rd/serde");
assert_eq!(index_path("Varve-SDK"), "va/rv/varve-sdk"); }
#[test]
fn a_non_ascii_crate_name_is_an_error_not_a_panic() {
for bad in ["日本語", "ααα", "café-utils"] {
assert!(
validate_crate_name(bad).is_err(),
"{bad} must be refused, not sliced"
);
let _ = index_path(bad);
}
}
#[test]
fn a_name_or_version_that_would_corrupt_the_index_json_is_refused() {
assert!(validate_crate_name("evil\"name").is_err());
assert!(validate_crate_name("back\\slash").is_err());
assert!(validate_crate_name("").is_err());
assert!(validate_crate_version("1.0.0\"").is_err());
assert!(validate_crate_name("serde_json").is_ok());
assert!(validate_crate_name("varve-core").is_ok());
assert!(validate_crate_version("0.1.0-alpha.1+build.2").is_ok());
}
#[test]
fn export_refuses_an_unrepresentable_crate_name() {
let dir = tempfile::tempdir().unwrap();
let bad = [CrateEntry {
name: "café-utils".into(),
version: "0.1.0".into(),
cksum: "a".repeat(64),
bytes: vec![],
}];
assert!(export_local_registry(&bad, dir.path()).is_err());
assert!(export_vendor_dir(&bad, dir.path()).is_err());
assert!(export_distdir(&bad, dir.path()).is_err());
}
#[test]
fn an_index_line_carries_the_cksum_cargo_will_verify() {
let e = CrateEntry {
name: "demo".into(),
version: "0.1.0".into(),
cksum: "b".repeat(64),
bytes: vec![],
};
let line = index_line(&e);
assert!(line.contains(r#""name":"demo""#));
assert!(line.contains(r#""vers":"0.1.0""#));
assert!(line.contains(&format!(r#""cksum":"{}""#, "b".repeat(64))));
assert!(line.contains(r#""yanked":false"#));
}
#[test]
fn vendoring_never_writes_outside_the_vendor_directory() {
use std::io::Write;
let dir = tempfile::tempdir().unwrap();
let outside = dir.path().join("OUTSIDE");
std::fs::create_dir_all(&outside).unwrap();
let vendor = dir.path().join("vendor");
let mut tar_bytes = Vec::new();
{
let mut b = tar::Builder::new(&mut tar_bytes);
let mut link = tar::Header::new_gnu();
link.set_entry_type(tar::EntryType::Symlink);
link.set_size(0);
link.set_mode(0o777);
b.append_link(&mut link, "escape-0.1.0/link", &outside)
.unwrap();
let payload = b"PWNED";
let mut f = tar::Header::new_gnu();
f.set_size(payload.len() as u64);
f.set_mode(0o644);
b.append_data(&mut f, "escape-0.1.0/link/pwned.txt", &payload[..])
.unwrap();
b.finish().unwrap();
}
let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast());
gz.write_all(&tar_bytes).unwrap();
let evil = gz.finish().unwrap();
let entries = [CrateEntry {
name: "escape".into(),
version: "0.1.0".into(),
cksum: "c".repeat(64),
bytes: evil,
}];
let _ = export_vendor_dir(&entries, &vendor);
assert!(
!outside.join("pwned.txt").exists(),
"a crate tarball escaped the vendor directory"
);
assert!(
std::fs::read_dir(&outside).unwrap().next().is_none(),
"nothing may be written outside the vendor directory"
);
}
#[test]
fn vendoring_unpacks_the_crate_and_preserves_the_upstream_hash() {
let mut builder = tar::Builder::new(flate2::write::GzEncoder::new(
Vec::new(),
flate2::Compression::default(),
));
for (name, body) in [
("demo-0.1.0/Cargo.toml", "[package]\nname=\"demo\"\n"),
("demo-0.1.0/src/lib.rs", "pub fn f() {}\n"),
] {
let mut h = tar::Header::new_gnu();
h.set_size(body.len() as u64);
h.set_mode(0o644);
h.set_cksum();
builder.append_data(&mut h, name, body.as_bytes()).unwrap();
}
let targz = builder.into_inner().unwrap().finish().unwrap();
let tmp = tempfile::tempdir().unwrap();
let vendor = tmp.path().join("vendor");
let e = CrateEntry {
name: "demo".into(),
version: "0.1.0".into(),
cksum: "d".repeat(64),
bytes: targz,
};
assert_eq!(
export_vendor_dir(std::slice::from_ref(&e), &vendor).unwrap(),
1
);
assert!(vendor.join("demo-0.1.0/Cargo.toml").is_file());
assert!(vendor.join("demo-0.1.0/src/lib.rs").is_file());
let checksum =
std::fs::read_to_string(vendor.join("demo-0.1.0/.cargo-checksum.json")).unwrap();
assert_eq!(
checksum,
format!(r#"{{"files":{{}},"package":"{}"}}"#, "d".repeat(64))
);
}
#[test]
fn a_distdir_holds_the_verified_crate_bytes_keyed_for_bazel() {
let tmp = tempfile::tempdir().unwrap();
let dd = tmp.path().join("distdir");
let bytes = b"the-verified-crate-tarball-bytes".to_vec();
let cksum = {
use sha2::{Digest, Sha256};
hex::encode(Sha256::digest(&bytes))
};
let e = CrateEntry {
name: "cfg-if".into(),
version: "1.0.0".into(),
cksum: cksum.clone(),
bytes: bytes.clone(),
};
assert_eq!(export_distdir(std::slice::from_ref(&e), &dd).unwrap(), 1);
let file = dd.join("cfg-if-1.0.0.crate");
assert_eq!(std::fs::read(&file).unwrap(), bytes);
let on_disk = {
use sha2::{Digest, Sha256};
hex::encode(Sha256::digest(std::fs::read(&file).unwrap()))
};
assert_eq!(
on_disk, cksum,
"distdir file sha256 must equal the crate_universe pin"
);
}
#[test]
fn the_vendored_config_replaces_with_a_directory_source() {
let cfg = vendored_config_toml(Path::new("/v/dir"));
assert!(cfg.contains(r#"replace-with = "vendored-sources""#));
assert!(cfg.contains(r#"directory = "/v/dir""#));
}
#[test]
fn config_redirects_crates_io_to_the_local_registry() {
let cfg = cargo_config_toml(Path::new("/verified/reg"));
assert!(cfg.contains(r#"replace-with = "varve""#));
assert!(cfg.contains(r#"local-registry = "/verified/reg""#));
}
#[test]
fn materialising_writes_the_crate_and_a_matching_index_entry() {
let tmp = tempfile::tempdir().unwrap();
let reg = tmp.path().join("registry");
let e = CrateEntry {
name: "demo".into(),
version: "0.1.0".into(),
cksum: "e".repeat(64),
bytes: b"crate-tarball-bytes".to_vec(),
};
assert_eq!(
export_local_registry(std::slice::from_ref(&e), ®).unwrap(),
1
);
assert_eq!(
std::fs::read(reg.join("demo-0.1.0.crate")).unwrap(),
b"crate-tarball-bytes"
);
let idx = std::fs::read_to_string(reg.join("index/de/mo/demo")).unwrap();
assert!(idx.contains(&format!(r#""cksum":"{}""#, "e".repeat(64))));
export_local_registry(std::slice::from_ref(&e), ®).unwrap();
let idx2 = std::fs::read_to_string(reg.join("index/de/mo/demo")).unwrap();
assert_eq!(idx2.lines().count(), 1, "one line per (name, version)");
}
#[test]
fn a_registry_exported_from_a_layer_offers_every_version_it_pins() {
use sha2::{Digest, Sha256};
let tmp = tempfile::tempdir().unwrap();
let reg = tmp.path().join("registry");
let bytes = |v: &str| format!("serde-{v}-crate-tarball").into_bytes();
let entry = |v: &str| CrateEntry {
name: "serde".into(),
version: v.into(),
cksum: hex::encode(Sha256::digest(bytes(v))),
bytes: bytes(v),
};
let crates = [entry("1.0.200"), entry("1.0.210")];
export_local_registry(&crates, ®).unwrap();
for v in ["1.0.200", "1.0.210"] {
assert_eq!(
std::fs::read(reg.join(format!("serde-{v}.crate"))).unwrap(),
bytes(v),
"version {v} must export its own bytes"
);
}
let idx = std::fs::read_to_string(reg.join("index/se/rd/serde")).unwrap();
let lines: Vec<serde_json::Value> = idx
.lines()
.filter(|l| !l.trim().is_empty())
.map(|l| serde_json::from_str(l).expect("each index line is JSON Cargo can parse"))
.collect();
let mut offered: Vec<(String, String)> = lines
.iter()
.map(|l| {
(
l["vers"].as_str().unwrap().to_string(),
l["cksum"].as_str().unwrap().to_string(),
)
})
.collect();
offered.sort();
assert_eq!(
offered,
vec![
(
"1.0.200".to_string(),
hex::encode(Sha256::digest(bytes("1.0.200")))
),
(
"1.0.210".to_string(),
hex::encode(Sha256::digest(bytes("1.0.210")))
),
],
"the index must offer BOTH versions, each bound to its own bytes"
);
assert!(lines.iter().all(|l| l["name"] == "serde"));
let dd = tmp.path().join("distdir");
export_distdir(&crates, &dd).unwrap();
assert_eq!(
std::fs::read(dd.join("serde-1.0.200.crate")).unwrap(),
bytes("1.0.200")
);
assert_eq!(
std::fs::read(dd.join("serde-1.0.210.crate")).unwrap(),
bytes("1.0.210")
);
}
}