use std::collections::{BTreeMap, BTreeSet};
use std::io::Read;
use std::path::Path;
use std::process::Command;
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum DriftVerdict {
Unchanged,
Drifted { differing: Vec<String> },
Unknown(String),
}
const MAX_CRATE_BYTES: u64 = 64 * 1024 * 1024;
const MAX_DIFFERING_SHOWN: usize = 12;
fn is_excluded_entry(rel: &str) -> bool {
matches!(
rel,
"Cargo.toml" | "Cargo.lock" | ".cargo_vcs_info.json" | ".gitignore" | ".gitattributes"
)
}
fn canonical_manifest(bytes: &[u8]) -> Vec<u8> {
match std::str::from_utf8(bytes).ok().and_then(|s| s.parse::<toml::Value>().ok()) {
Some(v) => toml::to_string(&v).map(String::into_bytes).unwrap_or_else(|_| bytes.to_vec()),
None => bytes.to_vec(),
}
}
pub fn diff_source_maps(
local: &BTreeMap<String, Vec<u8>>,
published: &BTreeMap<String, Vec<u8>>,
) -> DriftVerdict {
let mut differing: Vec<String> = Vec::new();
let keys: BTreeSet<&String> = local.keys().chain(published.keys()).collect();
for k in keys {
match (local.get(k), published.get(k)) {
(Some(a), Some(b)) if a == b => {}
(Some(_), Some(_)) => differing.push(format!("{k} (changed)")),
(Some(_), None) => differing.push(format!("{k} (added locally)")),
(None, Some(_)) => differing.push(format!("{k} (removed locally)")),
(None, None) => {}
}
}
if differing.is_empty() {
DriftVerdict::Unchanged
} else {
DriftVerdict::Drifted { differing }
}
}
pub fn summarize_differing(differing: &[String]) -> String {
if differing.len() <= MAX_DIFFERING_SHOWN {
differing.join(", ")
} else {
format!(
"{}, (+{} more)",
differing[..MAX_DIFFERING_SHOWN].join(", "),
differing.len() - MAX_DIFFERING_SHOWN
)
}
}
pub fn can_skip_republish(published_max: Option<&str>, verdict: &DriftVerdict) -> bool {
published_max.is_some() && matches!(verdict, DriftVerdict::Unchanged)
}
pub trait PublishedSourceOracle {
fn published_sources(&self, krate: &str, version: &str) -> Result<BTreeMap<String, Vec<u8>>, String>;
}
pub struct CratesIoSources;
impl PublishedSourceOracle for CratesIoSources {
fn published_sources(&self, krate: &str, version: &str) -> Result<BTreeMap<String, Vec<u8>>, String> {
let bytes = download_crate(krate, version)?;
extract_published_map(&bytes)
}
}
fn download_crate(krate: &str, version: &str) -> Result<Vec<u8>, String> {
let url = format!("https://static.crates.io/crates/{krate}/{krate}-{version}.crate");
let resp = ureq::get(&url)
.set("User-Agent", "nornir-release (rickard.lundin@ubm.se)")
.timeout(std::time::Duration::from_secs(30))
.call()
.map_err(|e| format!("download {url}: {e}"))?;
let mut buf = Vec::new();
resp.into_reader()
.take(MAX_CRATE_BYTES)
.read_to_end(&mut buf)
.map_err(|e| format!("read {url}: {e}"))?;
if buf.is_empty() {
return Err(format!("empty tarball at {url}"));
}
Ok(buf)
}
pub fn extract_published_map(crate_bytes: &[u8]) -> Result<BTreeMap<String, Vec<u8>>, String> {
let gz = flate2::read::GzDecoder::new(crate_bytes);
let mut ar = tar::Archive::new(gz);
let mut map: BTreeMap<String, Vec<u8>> = BTreeMap::new();
for entry in ar.entries().map_err(|e| format!("tar entries: {e}"))? {
let mut entry = entry.map_err(|e| format!("tar entry: {e}"))?;
if !entry.header().entry_type().is_file() {
continue;
}
let path = entry.path().map_err(|e| format!("tar path: {e}"))?.to_path_buf();
let rel: String = {
let mut comps = path.components();
comps.next(); comps.as_path().to_string_lossy().replace('\\', "/")
};
if rel.is_empty() {
continue;
}
let mut bytes = Vec::new();
entry.read_to_end(&mut bytes).map_err(|e| format!("read {rel}: {e}"))?;
if rel == "Cargo.toml.orig" {
map.insert("Cargo.toml".to_string(), canonical_manifest(&bytes));
} else if !is_excluded_entry(&rel) {
map.insert(rel, bytes);
}
}
Ok(map)
}
pub fn local_shipped_sources(crate_dir: &Path) -> Result<BTreeMap<String, Vec<u8>>, String> {
let manifest = crate_dir.join("Cargo.toml");
let cargo = crate::release::cargo::cargo_bin();
let out = Command::new(&cargo)
.args(["package", "--list", "--allow-dirty", "--manifest-path"])
.arg(&manifest)
.output()
.map_err(|e| format!("spawn `{} package --list`: {e}", cargo.to_string_lossy()))?;
if !out.status.success() {
return Err(format!(
"`cargo package --list` failed: {}",
String::from_utf8_lossy(&out.stderr).lines().last().unwrap_or("").trim()
));
}
let listing = String::from_utf8_lossy(&out.stdout);
let mut map: BTreeMap<String, Vec<u8>> = BTreeMap::new();
for line in listing.lines().map(str::trim).filter(|l| !l.is_empty()) {
let rel = line.replace('\\', "/");
if rel == "Cargo.toml" {
let disk = crate_dir.join(&rel);
let bytes = std::fs::read(&disk).map_err(|e| format!("read {}: {e}", disk.display()))?;
map.insert("Cargo.toml".to_string(), canonical_manifest(&bytes));
continue;
}
if rel == "Cargo.toml.orig" || is_excluded_entry(&rel) {
continue;
}
let disk = crate_dir.join(&rel);
let bytes = std::fs::read(&disk).map_err(|e| format!("read {}: {e}", disk.display()))?;
map.insert(rel, bytes);
}
if map.is_empty() {
return Err("`cargo package --list` produced no files".to_string());
}
Ok(map)
}
pub fn assess(
oracle: &dyn PublishedSourceOracle,
crate_dir: &Path,
krate: &str,
version: &str,
) -> DriftVerdict {
let published = match oracle.published_sources(krate, version) {
Ok(m) => m,
Err(e) => return DriftVerdict::Unknown(e),
};
let local = match local_shipped_sources(crate_dir) {
Ok(m) => m,
Err(e) => return DriftVerdict::Unknown(e),
};
diff_source_maps(&local, &published)
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum SelectReason {
Changed,
CascadedFrom(String),
Forced,
OnlyNamed,
SkippedUnchanged,
Excluded,
}
#[derive(Debug, Clone, Default)]
pub struct ReleaseSelection {
pub publish: Vec<String>,
pub skipped: Vec<(String, SelectReason)>,
}
impl ReleaseSelection {
pub fn will_publish(&self, krate: &str) -> bool {
self.publish.iter().any(|c| c == krate)
}
}
pub fn select_republish(
candidates_topo: &[String],
unchanged: &BTreeSet<String>,
deps: &BTreeMap<String, BTreeSet<String>>,
only: &[String],
exclude: &[String],
force_all: bool,
) -> ReleaseSelection {
let exclude_set: BTreeSet<&str> = exclude.iter().map(String::as_str).collect();
let only_set: BTreeSet<&str> = only.iter().map(String::as_str).collect();
let base: Vec<String> = candidates_topo
.iter()
.filter(|c| only_set.is_empty() || only_set.contains(c.as_str()))
.filter(|c| !exclude_set.contains(c.as_str()))
.cloned()
.collect();
let base_set: BTreeSet<&str> = base.iter().map(String::as_str).collect();
let mut sel = ReleaseSelection::default();
for c in candidates_topo {
if exclude_set.contains(c.as_str()) {
sel.skipped.push((c.clone(), SelectReason::Excluded));
}
}
if force_all || !only.is_empty() {
let reason = if !only.is_empty() { SelectReason::OnlyNamed } else { SelectReason::Forced };
for c in &base {
sel.publish.push(c.clone());
let _ = &reason; }
return sel;
}
let mut cause: BTreeMap<&str, SelectReason> = BTreeMap::new();
for c in &base {
if !unchanged.contains(c.as_str()) {
cause.insert(c.as_str(), SelectReason::Changed);
}
}
loop {
let mut added = false;
for c in &base {
if cause.contains_key(c.as_str()) {
continue;
}
if let Some(cdeps) = deps.get(c) {
if let Some(dep) = cdeps
.iter()
.find(|d| base_set.contains(d.as_str()) && cause.contains_key(d.as_str()))
{
cause.insert(c.as_str(), SelectReason::CascadedFrom(dep.clone()));
added = true;
}
}
}
if !added {
break;
}
}
for c in &base {
if cause.contains_key(c.as_str()) {
sel.publish.push(c.clone());
} else {
sel.skipped.push((c.clone(), SelectReason::SkippedUnchanged));
}
}
sel
}
#[cfg(test)]
mod tests {
use super::*;
fn m(pairs: &[(&str, &[u8])]) -> BTreeMap<String, Vec<u8>> {
pairs.iter().map(|(k, v)| (k.to_string(), v.to_vec())).collect()
}
fn names(v: &[&str]) -> Vec<String> {
v.iter().map(|s| s.to_string()).collect()
}
fn set(v: &[&str]) -> BTreeSet<String> {
v.iter().map(|s| s.to_string()).collect()
}
fn deps(pairs: &[(&str, &[&str])]) -> BTreeMap<String, BTreeSet<String>> {
pairs.iter().map(|(k, ds)| (k.to_string(), set(ds))).collect()
}
#[test]
fn smart_default_skips_unchanged_and_cascades_reverse_deps() {
let cands = names(&["a", "b", "c", "d"]);
let dep_edges = deps(&[("b", &["a"]), ("c", &["b"]), ("d", &[])]);
let unchanged = set(&["b", "c", "d"]);
let s = select_republish(&cands, &unchanged, &dep_edges, &[], &[], false);
assert_eq!(s.publish, names(&["a", "b", "c"]), "changed a + its rev-dep cascade");
assert_eq!(s.skipped.len(), 1);
assert_eq!(s.skipped[0].0, "d");
assert_eq!(s.skipped[0].1, SelectReason::SkippedUnchanged);
}
#[test]
fn all_unchanged_publishes_nothing() {
let cands = names(&["a", "b"]);
let s = select_republish(&cands, &set(&["a", "b"]), &deps(&[]), &[], &[], false);
assert!(s.publish.is_empty(), "nothing drifted ⇒ nothing to release");
assert_eq!(s.skipped.len(), 2);
}
#[test]
fn force_all_ignores_the_skip() {
let cands = names(&["a", "b"]);
let s = select_republish(&cands, &set(&["a", "b"]), &deps(&[]), &[], &[], true);
assert_eq!(s.publish, names(&["a", "b"]), "--all forces every crate");
}
#[test]
fn only_and_exclude_filter_the_base() {
let cands = names(&["a", "b", "c"]);
let s = select_republish(&cands, &set(&[]), &deps(&[]), &names(&["b", "c"]), &[], false);
assert_eq!(s.publish, names(&["b", "c"]));
let s2 = select_republish(&cands, &set(&[]), &deps(&[]), &[], &names(&["b"]), false);
assert_eq!(s2.publish, names(&["a", "c"]));
assert!(s2.skipped.iter().any(|(c, r)| c == "b" && *r == SelectReason::Excluded));
}
#[test]
fn only_one_crate_yields_a_single_crate_publish_set() {
let cands = names(&[
"funnel-core",
"draupnir",
"znippy-zoomies",
"facett-core",
"nornir",
"edda",
]);
let s = select_republish(&cands, &set(&[]), &deps(&[]), &names(&["nornir"]), &[], false);
assert_eq!(s.publish, names(&["nornir"]), "--only nornir ⇒ exactly nornir");
assert_eq!(s.publish.len(), 1, "a 1-crate publish set");
assert!(s.will_publish("nornir"));
assert!(!s.will_publish("facett-core"));
assert!(!s.will_publish("znippy-zoomies"));
assert!(!s.will_publish("draupnir"));
}
#[test]
fn cascade_is_order_independent() {
let cands = names(&["c", "b", "a"]);
let dep_edges = deps(&[("b", &["a"]), ("c", &["b"])]);
let s = select_republish(&cands, &set(&["b", "c"]), &dep_edges, &[], &[], false);
assert_eq!(s.publish, names(&["c", "b", "a"]));
assert!(s.skipped.is_empty());
}
#[test]
fn identical_maps_are_unchanged() {
let a = m(&[("src/lib.rs", b"fn x(){}"), ("README.md", b"hi")]);
let b = m(&[("src/lib.rs", b"fn x(){}"), ("README.md", b"hi")]);
assert_eq!(diff_source_maps(&a, &b), DriftVerdict::Unchanged);
}
#[test]
fn changed_file_is_drift() {
let local = m(&[("src/native.rs", b"lzip_parallel::decompress_zip_filter()")]);
let published = m(&[("src/native.rs", b"lzip_parallel::thread_pool()")]);
match diff_source_maps(&local, &published) {
DriftVerdict::Drifted { differing } => {
assert_eq!(differing, vec!["src/native.rs (changed)".to_string()]);
}
other => panic!("expected drift, got {other:?}"),
}
}
#[test]
fn added_and_removed_files_are_drift() {
let local = m(&[("src/lib.rs", b"x"), ("src/new.rs", b"y")]);
let published = m(&[("src/lib.rs", b"x"), ("src/gone.rs", b"z")]);
match diff_source_maps(&local, &published) {
DriftVerdict::Drifted { differing } => {
assert!(differing.contains(&"src/new.rs (added locally)".to_string()));
assert!(differing.contains(&"src/gone.rs (removed locally)".to_string()));
}
other => panic!("expected drift, got {other:?}"),
}
}
#[test]
fn manifest_reformatting_is_not_drift() {
let raw_a = b"[package]\nname = \"c\"\nversion = \"0.1.0\"\n\n[dependencies]\nserde = \"1\"\n";
let raw_b = b"[dependencies]\nserde = \"1\"\n[package]\nversion=\"0.1.0\"\nname=\"c\"\n";
let local = m(&[("Cargo.toml", &canonical_manifest(raw_a))]);
let published = m(&[("Cargo.toml", &canonical_manifest(raw_b))]);
assert_eq!(diff_source_maps(&local, &published), DriftVerdict::Unchanged);
}
#[test]
fn skip_only_when_published_and_unchanged() {
assert!(can_skip_republish(Some("1.2.3"), &DriftVerdict::Unchanged));
assert!(!can_skip_republish(None, &DriftVerdict::Unchanged));
assert!(!can_skip_republish(
Some("1.2.3"),
&DriftVerdict::Drifted { differing: vec!["src/lib.rs (changed)".into()] }
));
assert!(!can_skip_republish(Some("1.2.3"), &DriftVerdict::Unknown("network".into())));
}
#[test]
fn manifest_dependency_change_is_drift() {
let raw_a = b"[package]\nname=\"c\"\nversion=\"0.1.0\"\n[dependencies]\nserde=\"1\"\n";
let raw_b = b"[package]\nname=\"c\"\nversion=\"0.1.0\"\n[dependencies]\nserde=\"2\"\n";
let local = m(&[("Cargo.toml", &canonical_manifest(raw_a))]);
let published = m(&[("Cargo.toml", &canonical_manifest(raw_b))]);
assert!(matches!(diff_source_maps(&local, &published), DriftVerdict::Drifted { .. }));
}
#[test]
#[ignore = "network + local checkout; run explicitly with the env vars set"]
fn e2e_assess_against_published_crate() {
let (Ok(dir), Ok(krate), Ok(ver)) = (
std::env::var("NORNIR_DRIFT_SMOKE_DIR"),
std::env::var("NORNIR_DRIFT_SMOKE_KRATE"),
std::env::var("NORNIR_DRIFT_SMOKE_VER"),
) else {
eprintln!("skipped: set NORNIR_DRIFT_SMOKE_{{DIR,KRATE,VER}}");
return;
};
let v = assess(&CratesIoSources, Path::new(&dir), &krate, &ver);
assert_eq!(v, DriftVerdict::Unchanged, "expected consistent tree to be Unchanged, got {v:?}");
}
#[test]
fn extract_strips_prefix_and_drops_synthesized() {
let mut tar_buf = Vec::new();
{
let mut builder = tar::Builder::new(&mut tar_buf);
let mut add = |name: &str, data: &[u8]| {
let mut h = tar::Header::new_gnu();
h.set_size(data.len() as u64);
h.set_mode(0o644);
h.set_cksum();
builder.append_data(&mut h, name, data).unwrap();
};
add("foo-1.0.0/src/lib.rs", b"pub fn f() {}");
add("foo-1.0.0/Cargo.toml", b"normalized junk = true"); add("foo-1.0.0/Cargo.toml.orig", b"[package]\nname=\"foo\"\nversion=\"1.0.0\"\n");
add("foo-1.0.0/.cargo_vcs_info.json", b"{\"git\":{\"sha1\":\"abc\"}}"); builder.finish().unwrap();
}
let mut gz = Vec::new();
{
use std::io::Write;
let mut enc = flate2::write::GzEncoder::new(&mut gz, flate2::Compression::default());
enc.write_all(&tar_buf).unwrap();
enc.finish().unwrap();
}
let map = extract_published_map(&gz).unwrap();
assert!(map.contains_key("src/lib.rs"));
assert!(map.contains_key("Cargo.toml")); assert!(!map.contains_key("Cargo.toml.orig"));
assert!(!map.contains_key(".cargo_vcs_info.json"));
assert_eq!(map.get("src/lib.rs").unwrap(), b"pub fn f() {}");
}
}