debcargo/

crates.rs

use anyhow::{format_err, Context, Error};
use cargo::{
    core::{
        manifest::ManifestMetadata, registry::PackageRegistry, resolver::features::CliFeatures,
        Dependency, EitherManifest, FeatureMap, FeatureValue, Manifest, Package, PackageId,
        Registry, SourceId, Summary, Target, TargetKind, Workspace,
    },
    ops::{self, PackageMessageFormat, PackageOpts, Packages},
    sources::{
        source::{MaybePackage, QueryKind, Source},
        IndexSummary, RegistrySource, SourceConfigMap,
    },
    util::{
        cache_lock::CacheLockMode, interning::InternedString, toml::read_manifest, FileLock,
        GlobalContext,
    },
};
use cargo_toml_edit::{self, DependencyFilter, FeatureFilter};
use cfg_expr::targets::get_builtin_target_by_triple;
use filetime::{set_file_times, FileTime};
use flate2::read::GzDecoder;
use glob::Pattern;
use indexmap::{IndexMap, IndexSet};
use itertools::Itertools;
use regex::Regex;
use semver::Version;
use tar::Archive;
use tempfile;

use crate::config::{testing_ignore_debpolv, FilterTarget, TargetType};
use crate::errors::Result;
use crate::util::diff_files;
use cargo::util::Filesystem;
use std::fs;
use std::io::{self, Read};
use std::path::Path;
use std::{self, ffi::OsStr};
use std::{
    collections::{BTreeMap, HashMap, HashSet},
    str::FromStr,
};

pub struct CrateInfo {
    // only used for to_registry_toml in extract_crate. DO NOT USE ELSEWHERE
    package: Package,
    // allows overriding package.manifest() e.g. via patches
    manifest: Manifest,
    crate_file: FileLock,
    context: GlobalContext,
    source_id: SourceId,
    excludes: Vec<String>,
    includes: Vec<String>,
}

pub type AutoPatches = IndexMap<String, Vec<u8>>;

pub type CrateDepInfo = BTreeMap<
    &'static str, // name of feature / optional dependency,
    // or "" for the base package w/ no default features, guaranteed to be in the map
    (
        Vec<&'static str>, // dependencies: other features (of the current package)
        Vec<Dependency>,
    ),
>;

fn fetch_candidates(registry: &mut PackageRegistry, dep: &Dependency) -> Result<Vec<IndexSummary>> {
    let mut summaries = match registry.query_vec(dep, QueryKind::Exact) {
        std::task::Poll::Ready(res) => res?,
        std::task::Poll::Pending => {
            registry.block_until_ready()?;
            return fetch_candidates(registry, dep);
        }
    };
    summaries.sort_by(|a, b| b.package_id().partial_cmp(&a.package_id()).unwrap());
    Ok(summaries)
}

pub fn invalidate_crates_io_cache() -> Result<()> {
    let context = GlobalContext::default()?;
    let _lock = context.acquire_package_cache_lock(CacheLockMode::DownloadExclusive)?;
    let source_id = SourceId::crates_io_maybe_sparse_http(&context)?;
    let yanked_whitelist = HashSet::new();
    let mut r = RegistrySource::remote(source_id, &yanked_whitelist, &context)?;
    r.invalidate_cache();
    Ok(())
}

pub fn crate_name_ver_to_dep(crate_name: &str, version: Option<&str>) -> Result<Dependency> {
    // note: this forces a network call
    let context = GlobalContext::default()?;
    let source_id = SourceId::crates_io_maybe_sparse_http(&context)?;
    let version = version.and_then(|v| {
        if v.is_empty() {
            None
        } else if v.starts_with(|c: char| c.is_ascii_digit()) {
            Some(["=", v].concat())
        } else {
            Some(v.to_string())
        }
    });
    Dependency::parse(crate_name, version.as_deref(), source_id)
}

// attempt to map back a version requirement to a version that can be used as last resort
// fallback in case all versions satisfying the requirement are yanked
fn ver_req_to_ver(dep: &Dependency) -> Option<Version> {
    match dep.version_req() {
        // if any version would satisfy the dep, but all of them are yanked, we probably don't want
        // to use this crate anyway ;)
        cargo::util::OptVersionReq::Any => None,
        // a regular version requirement is only useful if it has a single element
        cargo::util::OptVersionReq::Req(req) if req.comparators.len() == 1 => {
            let comp = req.comparators.first().unwrap();
            match comp.op {
                // these are all satisfied by the full version, if one is set
                semver::Op::Exact
                | semver::Op::GreaterEq
                | semver::Op::LessEq
                | semver::Op::Tilde
                | semver::Op::Caret => {
                    if let Some(minor) = comp.minor {
                        // if the requirement comes with a full version extract it
                        comp.patch.map(|patch| Version {
                            major: comp.major,
                            minor,
                            patch,
                            pre: comp.pre.clone(),
                            build: semver::BuildMetadata::default(),
                        })
                    } else {
                        // we'd have to guess here.
                        None
                    }
                }
                // Greater, Less or Wildcard all require guessing as well
                _ => None,
            }
        }
        // requirements with multiple constraints
        cargo::util::OptVersionReq::Req(_) => None,
        // locked requirements contain an exact version
        cargo::util::OptVersionReq::Locked(ver, _) => Some(ver.clone()),
        // precise requirements contain an exact version
        cargo::util::OptVersionReq::Precise(ver, _) => Some(ver.clone()),
    }
}

pub fn show_dep(dep: &Dependency) -> String {
    format!("{} {}", dep.package_name(), dep.version_req())
}

impl CrateInfo {
    pub fn new(crate_name: &str, version: Option<&str>) -> Result<CrateInfo> {
        CrateInfo::new_with_update(crate_name, version, true)
    }

    pub fn new_with_local_crate(
        crate_name: &str,
        version: Option<&str>,
        crate_path: &Path,
    ) -> Result<CrateInfo> {
        let context = GlobalContext::default()?;
        let crate_path = crate_path.canonicalize()?;
        let source_id = SourceId::for_path(&crate_path)?;

        let (package, crate_file) = {
            let yanked_whitelist = HashSet::new();

            let mut source = source_id.load(&context, &yanked_whitelist)?;

            let package_id = match version {
                None => {
                    let dep = Dependency::parse(crate_name, None, source_id)?;
                    let mut package_id: Option<PackageId> = None;
                    loop {
                        match source.query(&dep, QueryKind::Exact, &mut |p| {
                            package_id = Some(p.package_id());
                        }) {
                            std::task::Poll::Ready(res) => {
                                res?;
                                break;
                            }
                            std::task::Poll::Pending => {
                                source.block_until_ready()?;
                            }
                        }
                    }
                    package_id.unwrap()
                }
                Some(version) => {
                    let version = version.parse::<Version>()?;
                    PackageId::new(crate_name.into(), version, source_id)
                }
            };

            let maybe_package = source.download(package_id)?;
            let package = match maybe_package {
                MaybePackage::Ready(p) => Ok(p),
                _ => Err(format_err!(
                    "Failed to 'download' local crate {} from {}",
                    crate_name,
                    crate_path.display()
                )),
            }?;

            let crate_file = {
                let workspace = Workspace::ephemeral(package.clone(), &context, None, true)?;

                let opts = PackageOpts {
                    gctx: &context,
                    verify: false,
                    list: false,
                    check_metadata: true,
                    allow_dirty: true,
                    cli_features: CliFeatures::from_command_line(&[], true, false)?,
                    jobs: None,
                    targets: Vec::new(),
                    to_package: Packages::Default,
                    keep_going: false,
                    reg_or_index: None,
                    dry_run: false,
                    include_lockfile: false,
                    fmt: PackageMessageFormat::Human,
                };
                ops::package(&workspace, &opts)?;

                let filename = format!("{}-{}.crate", crate_name, package_id.version());
                workspace
                    .target_dir()
                    .join("package")
                    .open_rw_exclusive_create(filename, &context, "crate file")?
            };

            (package, crate_file)
        };

        let manifest = package.manifest().clone();

        Ok(CrateInfo {
            package,
            manifest,
            crate_file,
            context,
            source_id,
            excludes: vec![],
            includes: vec![],
        })
    }

    pub fn new_with_update(
        crate_name: &str,
        version: Option<&str>,
        update: bool,
    ) -> Result<CrateInfo> {
        let dep = crate_name_ver_to_dep(crate_name, version)?;
        Self::new_from_dependency(&dep, update)
    }

    pub fn new_from_dependency(dependency: &Dependency, update: bool) -> Result<CrateInfo> {
        let mut context = GlobalContext::default()?;
        if !update {
            // unfriendly API from cargo; we'll have to make do with it for
            // now as there is no other alternative
            context.configure(
                0,
                false,
                None,
                false,
                false,
                true, // offline
                &context.target_dir()?.map(Filesystem::into_path_unlocked),
                &[],
                &[],
            )?;
        }

        let source_id = SourceId::crates_io_maybe_sparse_http(&context)?;
        let registry_name = format!(
            "{}-{}",
            source_id.url().host_str().unwrap_or(""),
            cargo::util::hex::short_hash(&source_id)
        );
        let get_package_info = |context: &GlobalContext,
                                possibly_yanked_ver: Option<&Version>|
         -> Result<_> {
            let lock = context.acquire_package_cache_lock(CacheLockMode::DownloadExclusive)?;
            let mut registry =
                PackageRegistry::new_with_source_config(context, SourceConfigMap::new(context)?)?;

            // if we have some exact version that should satisfy our dependency requirements, and
            // we are on our third attempt of retrieving the crate info from the registry, add this
            // version to the yanked whitelist as a last resort
            if let Some(ver) = possibly_yanked_ver {
                let pkgids = vec![PackageId::new(
                    dependency.package_name(),
                    ver.clone(),
                    dependency.source_id(),
                )];
                debcargo_warn!(
                    "Adding {} to yanked whitelist as last resort..",
                    pkgids.first().unwrap()
                );
                registry.add_to_yanked_whitelist(pkgids.into_iter());
            }
            registry.lock_patches();
            let summaries = fetch_candidates(&mut registry, dependency)?;
            drop(lock);
            let pkgids = summaries.into_iter().map(|s| s.package_id()).collect_vec();
            let pkgid = pkgids.iter().max().ok_or_else(|| {
                format_err!(
                    concat!(
                        "Couldn't find any crate matching {}\n",
                        "Try `debcargo update` to update the crates.io index."
                    ),
                    show_dep(dependency)
                )
            })?;
            let pkgset = registry.get(pkgids.as_slice())?;
            let package = pkgset.get_one(*pkgid)?;

            let manifest = package.manifest();
            for f in dependency.features() {
                // apparently, if offline is set then cargo sometimes selects
                // an offline-available version that doesn't satisfy the
                // requested features. this is dumb. if it happens, then we
                // retry with online allowed.
                if !manifest.summary().features().contains_key(f) {
                    debcargo_bail!(
                        "resolve ({} {}) -> ({}) failed to pick up required feature ({})\n\
                        This can happen with very old or yanked crates. Try patching one of \
                        its dependants, to drop or update the offending dependency.",
                        dependency.package_name(),
                        dependency.version_req(),
                        pkgid,
                        f,
                    )
                }
            }
            let filename = format!("{}-{}.crate", pkgid.name(), pkgid.version());
            let crate_file = context
                .registry_cache_path()
                .join(&registry_name)
                .open_ro_shared(&filename, context, &filename)?;
            Ok((package.clone(), manifest.clone(), crate_file))
        };
        // if update is false but the user never downloaded the crate then the
        // first call will error; re-try with online in that case
        let (package, manifest, crate_file) = get_package_info(&context, None)
            .or_else(|_| get_package_info(&GlobalContext::default()?, None))
            .or_else(|err| {
                let ver = ver_req_to_ver(dependency);
                if ver.is_some() {
                    get_package_info(&GlobalContext::default()?, ver.as_ref())
                } else {
                    Err(err)
                }
            })?;

        Ok(CrateInfo {
            package,
            manifest,
            crate_file,
            context,
            source_id,
            excludes: vec![],
            includes: vec![],
        })
    }

    pub fn crate_name(&self) -> &'static str {
        self.package_id().name().as_str()
    }

    pub fn version(&self) -> &Version {
        self.package_id().version()
    }

    pub fn semver(&self) -> String {
        match *self.package_id().version() {
            Version {
                major: 0, minor, ..
            } => format!("0.{minor}"),
            Version { major, .. } => format!("{major}"),
        }
    }

    pub fn manifest(&self) -> &Manifest {
        &self.manifest
    }

    pub fn replace_manifest(&mut self, path: &Path) -> Result<&Self> {
        if let EitherManifest::Real(v) = read_manifest(path, self.source_id, &self.context)? {
            self.manifest = v;
        }
        Ok(self)
    }

    pub fn metadata(&self) -> &ManifestMetadata {
        self.manifest.metadata()
    }

    pub fn manifest_path(&self) -> &Path {
        self.package.manifest_path()
    }

    pub fn rust_version(&self) -> Option<String> {
        self.manifest.rust_version().map(ToString::to_string)
    }

    pub fn targets(&self) -> &[Target] {
        self.manifest.targets()
    }

    pub fn is_lib(&self) -> bool {
        let mut lib = false;
        for target in self.manifest.targets() {
            match *target.kind() {
                TargetKind::Lib(_) => {
                    lib = true;
                    break;
                }
                _ => continue,
            }
        }
        lib
    }

    pub fn get_binary_targets(&self) -> Vec<&str> {
        let mut bins = Vec::new();
        for target in self.manifest.targets() {
            match *target.kind() {
                TargetKind::Bin => {
                    bins.push(target.name());
                }
                _ => continue,
            }
        }
        bins.sort_unstable();
        bins
    }

    pub fn summary(&self) -> &Summary {
        self.manifest.summary()
    }

    pub fn checksum(&self) -> Option<&str> {
        self.manifest.summary().checksum()
    }

    pub fn package_id(&self) -> PackageId {
        self.manifest.summary().package_id()
    }

    pub fn crate_file(&self) -> &FileLock {
        &self.crate_file
    }

    pub fn dependencies(&self) -> &[Dependency] {
        self.manifest.dependencies()
    }

    pub fn dev_dependencies(&self) -> Vec<Dependency> {
        use cargo::core::dependency::DepKind;
        let mut deps = vec![];
        for dep in self.dependencies() {
            if dep.kind() == DepKind::Development {
                deps.push(dep.clone());
            }
        }
        deps
    }

    pub fn get_summary_description(&self) -> (Option<String>, Option<String>) {
        let (summary, description) = if let Some(ref description) = self.metadata().description {
            // Convention these days seems to be to do manual text
            // wrapping in crate descriptions, boo. \n\n is a real line break.
            let mut description = description
                .replace("\n\n", "\r")
                .replace('\n', " ")
                .replace('\r', "\n")
                .trim()
                .to_string();
            // Trim off common prefixes
            let re = Regex::new(&format!(
                r"^(?i)({}|This(\s+\w+)?)(\s*,|\s+is|\s+provides)\s+",
                self.package_id().name()
            ))
            .unwrap();
            description = re.replace(&description, "").to_string();
            let re = Regex::new(r"^(?i)(a|an|the)\s+").unwrap();
            description = re.replace(&description, "").to_string();
            let re =
                Regex::new(r"^(?i)(rust\s+)?(implementation|library|tool|crate)\s+(of|to|for)\s+")
                    .unwrap();
            description = re.replace(&description, "").to_string();

            // https://stackoverflow.com/questions/38406793/why-is-capitalizing-the-first-letter-of-a-string-so-convoluted-in-rust
            description = {
                let mut d = description.chars();
                match d.next() {
                    None => String::new(),
                    Some(f) => f.to_uppercase().chain(d).collect::<String>(),
                }
            };

            // Use the first sentence or first line, whichever comes first, as the summary.
            let p1 = description.find('\n');
            let p2 = description.find(". ");
            match p1.into_iter().chain(p2).min() {
                Some(p) => {
                    let s = description[..p].trim_end_matches('.').to_string();
                    let d = description[p + 1..].trim();
                    if d.is_empty() {
                        (Some(s), None)
                    } else {
                        (Some(s), Some(d.to_string()))
                    }
                }
                None => (Some(description.trim_end_matches('.').to_string()), None),
            }
        } else {
            (None, None)
        };

        (summary, description)
    }

    /// To be called before `extract_crate`.
    pub fn set_includes_excludes(
        &mut self,
        excludes: Option<&Vec<String>>,
        includes: Option<&Vec<String>>,
    ) {
        self.excludes = excludes.cloned().unwrap_or_default();
        self.includes = includes.cloned().unwrap_or_default();
    }

    pub fn filter_path(&self, path: &Path) -> std::result::Result<bool, String> {
        let top_level = path
            .ancestors()
            .find(|p| p.parent() == Some(Path::new("")))
            .unwrap()
            .to_str()
            .ok_or_else(|| format!("Failed to get top-level element of {path:?}"))?;

        let matches = move |pattern: &String| -> bool {
            Pattern::new(&format!("{top_level}/{pattern}"))
                .unwrap()
                .matches_path(path)
        };

        if self.excludes.iter().any(matches) {
            return Ok(true);
        }
        let suspicious = match path.extension().map(OsStr::to_ascii_lowercase) {
            Some(ext) => {
                ext == "c"
                    || ext == "a"
                    || ext == "s"
                    || ext == "o"
                    || ext == "so"
                    || ext == "cpp"
                    || ext == "cc"
                    || ext == "cxx"
            }
            _ => false,
        };

        if suspicious {
            if self.includes.iter().any(matches) {
                debcargo_info!("Suspicious file, on whitelist so ignored: {:?}", path);
                Ok(false)
            } else if testing_ignore_debpolv() {
                debcargo_warn!("Suspicious file, ignoring as per override: {:?}", path);
                Ok(false)
            } else {
                Err(format!(
                    "Suspicious file, should probably be excluded in debcargo.toml: {path:?}"
                ))
            }
        } else {
            Ok(false)
        }
    }

    pub fn extract_crate(&mut self, path: &Path) -> Result<(bool, Option<String>)> {
        let mut archive = Archive::new(GzDecoder::new(self.crate_file.file()));
        let tempdir = tempfile::Builder::new()
            .prefix("debcargo")
            .tempdir_in(".")
            .context("Failed to create tempdir for extracting crate")?;
        let mut source_modified = false;
        let mut manifest_normalization_patch = None;
        let mut last_mtime = 0;
        let mut err = vec![];

        for entry in archive
            .entries()
            .context("Failed iterating crate tarball")?
        {
            let mut entry = entry.context("Failed iterating crate tarball entry")?;
            let entry_path = entry
                .path()
                .with_context(|| format!("Failed to convert entry path {:?}", entry.path_bytes()))?
                .into_owned();
            match self.filter_path(&entry_path) {
                Err(e) => err.push(e),
                Ok(r) => {
                    if r {
                        source_modified = true;
                        continue;
                    }
                }
            }

            if !entry.unpack_in(tempdir.path()).with_context(|| {
                format!("Failed to unpack {entry_path:?} entry of crate tarball")
            })? {
                debcargo_bail!("Crate contained path traversals via '..'");
            }

            if let Ok(mtime) = entry.header().mtime() {
                if mtime > last_mtime {
                    last_mtime = mtime;
                }
            }
        }
        if !err.is_empty() {
            for e in err {
                debcargo_warn!("{}", e);
            }
            debcargo_bail!(
                "Suspicious files detected, aborting. Ask on #debian-rust if you are stuck."
            )
        }

        let entries = tempdir
            .path()
            .read_dir()
            .context("Failed reading extracted tarball tempdir")?
            .collect::<io::Result<Vec<_>>>()
            .context("Failed collecting extracted tarball tempdir contents")?;
        if entries.len() != 1
            || !entries[0]
                .file_type()
                .context(
                    "Failed to determine file type of first dir entry of extracted tarball tempdir",
                )?
                .is_dir()
        {
            let pkgid = self.package_id();
            debcargo_bail!(
                "{}-{}.crate did not unpack to a single top-level directory",
                pkgid.name(),
                pkgid.version()
            );
        }

        if let Err(e) = fs::rename(entries[0].path(), path) {
            return Err(Error::from(e).context(format!(
                concat!(
                    "Could not create source directory {0}\n",
                    "To regenerate, move or remove {0}"
                ),
                path.display()
            )));
        }

        // Ensure an upstream debian/ dir is stripped, just like dpkg-source does
        let _ = fs::remove_dir_all(path.join("debian"));

        // Ensure that Cargo.toml is in standard form, e.g. does not contain
        // path dependencies, so can be built standalone (see #4030).
        let toml_path = path
            .join("Cargo.toml")
            .canonicalize()
            .context("Failed to canonicalize Cargo.toml path in temporary crate unpack dir")?;
        let mut actual_toml = String::new();
        fs::File::open(&toml_path)
            .context("Failed to open crate manifest in extracted crate tarball tempdir")?
            .read_to_string(&mut actual_toml)
            .context("Failed to read crate manifest in extracted crate tarball tempdir")?;

        if !actual_toml.contains("AUTOMATICALLY GENERATED BY CARGO") {
            // This logic should only fire for old crates, and that's what the
            // if-conditional is supposed to check; modern versions of cargo
            // already do this before uploading the crate, and we shouldn't need
            // to handle it specially.
            log::debug!("Cargo.toml not canonicalized..");
            let orig_toml = actual_toml.clone();

            // Some old uncanonicalized Cargo.toml files contain invalid relative references to
            // license or readme files that newer cargo doesn't accept anymore. attempt to monkey
            // patch them if possible
            let mut check_reference = |name, value: Option<&str>| {
                if let Some(referenced_path) = value {
                    let full_referenced_path = path.join(referenced_path);
                    if !full_referenced_path.exists() {
                        debcargo_warn!(
                            "Cargo.toml references non-existing license_file path: {:?}",
                            full_referenced_path
                        );
                        if let Some((_prefix, file)) = referenced_path.rsplit_once('/') {
                            if path.join(file).exists() {
                                debcargo_info!("Replacing reference with '{}'", file);
                                actual_toml = actual_toml.replace(referenced_path, file);
                            } else {
                                debcargo_info!("Removing reference");
                                actual_toml = actual_toml
                                    .replace(&format!("{name} = \"{referenced_path}\""), "");
                            }
                        } else {
                            debcargo_info!("Removing reference");
                            actual_toml =
                                actual_toml.replace(&format!("{name} = \"{referenced_path}\""), "");
                        }
                        return true;
                    }
                }

                false
            };

            if check_reference("license-file", self.metadata().license_file.as_deref())
                || check_reference("readme", self.metadata().readme.as_deref())
            {
                fs::write(&toml_path, actual_toml.as_bytes())
                    .context("Failed to overwrite Cargo.toml during normalization")?;
            }
            let updated = self
                .replace_manifest(&toml_path)
                .context("Failed to reload manifest after canonicalization")?;

            let ws = Workspace::new(&toml_path, &updated.context)
                .context("Failed to instantiate workspace after reloading manifest")?;
            let opts = PackageOpts {
                gctx: &updated.context,
                list: false,
                check_metadata: false,
                allow_dirty: true,
                verify: false,
                jobs: None,
                keep_going: false,
                to_package: Packages::Default,
                targets: Vec::new(),
                cli_features: CliFeatures::new_all(true),
                reg_or_index: None,
                dry_run: false,
                include_lockfile: false,
                fmt: PackageMessageFormat::Human,
            };
            let files = ops::package(&ws, &opts)
                .context("Failed to run `cargo package` for normalization")?;
            let file = files
                .first()
                .ok_or_else(|| format_err!("No canonicalized archives found.."))?;
            log::trace!("canonical crate tarball: {file:?}");
            let file = file.path();
            let file = fs::File::open(file)?;
            log::trace!("{file:?}");
            let mut archive = Archive::new(GzDecoder::new(file));

            for entry in archive.entries()? {
                let mut entry =
                    entry.context("Failed to parse tar entry in normalized crate tarball")?;
                let entry_path = entry.path().with_context(|| {
                    format!(
                        "Failed to convert crate tarball entry path to UTF8: {:?}",
                        entry.path_bytes(),
                    )
                })?;
                log::trace!("normalized .crate archive entry: {entry_path:?}");
                let mut components = entry_path.iter();
                if components.clone().count() == 2
                    && components.next_back() == Some(OsStr::new("Cargo.toml"))
                {
                    entry
                        .unpack(&toml_path)
                        .context("Failed to extract normalized Cargo.toml")?;
                    break;
                }
            }

            log::debug!("generating Cargo.toml normalization patch");
            fs::rename(path.join("Cargo.toml"), path.join("Cargo.toml.normalized"))
                .context("Faield to rename normalized Cargo.toml")?;
            fs::write(path.join("Cargo.toml"), orig_toml.as_bytes())
                .context("Failed to write original Cargo.toml contents")?;
            let diff = diff_files(
                "Cargo.toml",
                &path.join("Cargo.toml"),
                &path.join("Cargo.toml.normalized"),
            )?;
            manifest_normalization_patch = Some({
                let mut patch = String::from(
                    "This patch was autogenerated using debcargo. Do not edit manually!\n",
                );
                patch.push_str(&diff);
                patch
            });
            fs::remove_file(path.join("Cargo.toml.normalized"))
                .context("Failed to remove temporary normalized Cargo.toml")?;

            // avoid lintian errors about package-contains-ancient-file
            // TODO: do we want to do this for unmodified tarballs? it would
            // force us to modify them, but otherwise we get that ugly warning
            let last_mtime = FileTime::from_unix_time(i64::try_from(last_mtime)?, 0);
            set_file_times(toml_path, last_mtime, last_mtime)?;
            debcargo_info!("Cargo.toml manually canonicalized!");
        }
        Ok((source_modified, manifest_normalization_patch))
    }
}

/// Collect information about the dependency structure of features and
/// their external crate dependencies, in a simple output format.
pub fn all_dependencies_and_features(manifest: &Manifest) -> CrateDepInfo {
    all_dependencies_and_features_filtered(manifest, false)
}

fn check_cycle_in_feature<'a>(feature_map: &'a FeatureMap, feature: &'a str) -> bool {
    fn check_cycle_in_feature_impl<'a>(
        feature_map: &'a FeatureMap,
        current: &'a str,
        processing: &mut Vec<&'a str>,
    ) -> bool {
        if processing.contains(&current) {
            return true;
        }

        processing.push(current);

        if let Some(deps) = feature_map.get(current) {
            for dep in deps {
                if let FeatureValue::Feature(dep_feature) = dep {
                    if check_cycle_in_feature_impl(feature_map, dep_feature, processing) {
                        return true;
                    }
                }
            }
        }

        processing.pop();
        false
    }

    let mut processing = Vec::new();
    check_cycle_in_feature_impl(feature_map, feature, &mut processing)
}

/// Collect information about the dependency structure of features and
/// their external crate dependencies, in a simple output format.
/// If `include_dev_dependencies` is true, dev-dependencies will be included.
pub fn all_dependencies_and_features_filtered(
    manifest: &Manifest,
    include_dev_dependencies: bool,
) -> CrateDepInfo {
    use cargo::core::dependency::DepKind;

    let mut deps_by_name: BTreeMap<&str, Vec<&Dependency>> = BTreeMap::new();
    for dep in manifest.dependencies() {
        // we treat build-dependencies also as dependencies in Debian
        if include_dev_dependencies || dep.kind() != DepKind::Development {
            let s = dep.name_in_toml().as_str();
            deps_by_name.entry(s).or_default().push(dep);
        }
    }
    let deps_by_name = deps_by_name;

    let mut features_with_deps = BTreeMap::new();

    // calculate dependencies of this crate's features
    let feature_map = manifest.summary().features();
    for (feature, deps) in feature_map {
        let mut feature_deps: Vec<&'static str> = vec![];
        let mut other_deps: Vec<Dependency> = Vec::new();
        for dep in deps {
            use self::FeatureValue::{Dep, DepFeature, Feature};
            match dep {
                // another feature is a dependency
                Feature(dep_feature) => {
                    feature_deps.push(InternedString::new(dep_feature).as_str());
                }
                // another package is a dependency
                Dep { dep_name } => {
                    // unwrap is ok, valid Cargo.toml files must have this
                    for &dep in deps_by_name.get(dep_name.as_str()).unwrap() {
                        other_deps.push(dep.clone());
                    }
                }
                // another package is a dependency
                DepFeature {
                    dep_name,
                    dep_feature,
                    ..
                } => {
                    if let Some(dd) = deps_by_name.get(dep_name.as_str()) {
                        for &dep in dd {
                            let mut dep = dep.clone();
                            let mut features: Vec<InternedString> =
                                vec![InternedString::new(dep_feature)];
                            features.extend(dep.features());
                            dep.set_features(features);
                            dep.set_default_features(false);
                            other_deps.push(dep);
                        }
                    } else {
                        let mut expected = false;
                        for dep in manifest.dependencies() {
                            if dep.kind() == DepKind::Development {
                                let s = dep.name_in_toml().as_str();
                                if s == dep_name.as_str() {
                                    expected = true;
                                }
                            }
                        }
                        if expected {
                            debcargo_warn!(
                                "Ignoring \"{}\" feature \"{}\" as it depends on a \
                                     dev-dependency \"{}\"",
                                manifest.package_id(),
                                feature,
                                dep_name
                            );
                        } else {
                            panic!(
                                "failed to account for dependency \"{}\" of \"{}\" feature \"{}\"",
                                dep_name,
                                manifest.package_id(),
                                feature
                            );
                        }
                    }
                }
            }
        }

        if feature_deps.is_empty()
            || check_cycle_in_feature(feature_map, feature) && !feature_deps.contains(&"")
        {
            // everything depends on bare library
            feature_deps.push("");
        }

        features_with_deps.insert(feature.as_str(), (feature_deps, other_deps));
    }

    // calculate required dependencies for implicit no-default-features
    let mut deps_required: Vec<Dependency> = Vec::new();
    for deps in deps_by_name.values() {
        for &dep in deps {
            if !dep.is_optional() {
                deps_required.push(dep.clone());
            }
        }
    }

    // implicit no-default-features
    features_with_deps.insert("", (vec![], deps_required));

    // implicit default feature
    if !features_with_deps.contains_key("default") {
        features_with_deps.insert("default", (vec![""], vec![]));
    }

    features_with_deps
}

/// Add `[lints.rust.unexpected_cfgs]` `check-cfg` entries for `features`, so
/// rustc doesn't warn about `#[cfg(feature = "name")]` guards that remain in
/// the source. Existing entries are preserved and not duplicated. A malformed
/// `[lints]` section (where one of the expected keys is not a table/array) is
/// left untouched.
fn add_unexpected_cfgs(doc: &mut toml_edit::DocumentMut, features: &[String]) {
    use toml_edit::{Item, Table};

    let cfg_value = |f: &str| format!("cfg(feature, values(\"{f}\"))");

    let new_table = || {
        let mut t = Table::new();
        t.set_implicit(true);
        Item::Table(t)
    };
    let unexpected = doc
        .entry("lints")
        .or_insert_with(new_table)
        .as_table_mut()
        .and_then(|t| t.entry("rust").or_insert_with(new_table).as_table_mut())
        .map(|t| t.entry("unexpected_cfgs").or_insert_with(new_table));
    // If lints / lints.rust / unexpected_cfgs already exist but aren't tables,
    // the manifest is malformed; leave it untouched.
    let Some(unexpected) = unexpected.and_then(Item::as_table_mut) else {
        debcargo_warn!("Cargo.toml: [lints.rust.unexpected_cfs] is not a TOML table, not injecting removed features!");
        return;
    };
    // This table is written explicitly, so don't let it stay implicit.
    unexpected.set_implicit(false);
    unexpected
        .entry("level")
        .or_insert_with(|| toml_edit::value("warn"));
    let Some(check_cfg) = unexpected
        .entry("check-cfg")
        .or_insert_with(|| toml_edit::value(toml_edit::Array::new()))
        .as_array_mut()
    else {
        return;
    };

    let existing: HashSet<&str> = check_cfg
        .iter()
        .filter_map(toml_edit::Value::as_str)
        .collect();
    let additions: Vec<String> = features
        .iter()
        .map(|f| cfg_value(f))
        .filter(|v| !existing.contains(v.as_str()))
        .collect();
    for entry in additions {
        // Cargo.toml should be normalized, this is the format used there
        let formatted = toml_edit::Value::from(entry).decorated("\n    ", "");
        check_cfg.push_formatted(formatted);
    }
}

/// Remove named features from the `[features]` section of a parsed Cargo.toml,
/// also removing any references to those features in the dependency lists of
/// remaining features, and removing optional dependencies that are exclusively
/// enabled by the removed features. Returns true if the document was modified.
fn apply_remove_features(
    doc: toml_edit::DocumentMut,
    features: &[String],
) -> Result<Option<String>> {
    let mut manifest = cargo_toml_edit::Manifest::from(doc);

    let mut filter = FeatureFilter::empty();
    filter.features = features.to_vec();
    filter.unsafe_mode = true;
    log::debug!("Feature filters: {filter:?}");

    let filtered = manifest.filter_features(&filter)?;
    if filtered.removed_features.is_empty() {
        Ok(None)
    } else {
        debcargo_info!("Removed features:");
        for (feature, _item) in &filtered.removed_features {
            debcargo_info!(format!("- {feature}"));
        }
        debcargo_info!("Dependencies:");
        for (dependency, _opt) in &filtered.removed_dependencies {
            debcargo_info!(format!("- {dependency}"));
        }

        let mut doc = toml_edit::DocumentMut::from_str(&manifest.to_string())?;
        add_unexpected_cfgs(&mut doc, features);
        Ok(Some(doc.to_string()))
    }
}

/// Generate an auto-patch that removes the named features from a Cargo.toml,
/// appending it to `auto_patches`.
///
/// `path` is the extracted (original) manifest, with any earlier auto-patches
/// already collected in `auto_patches` (e.g. Cargo.toml normalization). The
/// feature-removal patch is generated on top of those, so the whole set stacks
/// cleanly in the quilt series. The new patch is inserted into `auto_patches`
/// under `auto-remove-features.patch`; nothing is inserted if removing the
/// features did not change anything.
pub fn remove_features_patch(
    path: &Path,
    features: &[String],
    auto_patches: &mut AutoPatches,
) -> Result<()> {
    let dir = path
        .parent()
        .ok_or_else(|| format_err!("{path:?} has no parent directory"))?;

    // Bring the on-disk manifest up to date with the earlier auto-patches, edit
    // it, capture the diff, then restore it. The orig manifest is left untouched
    // once we are done.
    apply_auto_patches(path, auto_patches)?;
    let base_content = fs::read_to_string(path)?;

    // Edit the manifest with toml_edit so the diff only touches the lines we
    // actually change, rather than reformatting the whole file.
    let doc: toml_edit::DocumentMut = base_content.parse()?;
    let modified = apply_remove_features(doc, features)?;

    let result = if let Some(modified) = modified {
        let base_path = dir.join("Cargo.toml.featured");
        let modified_path = dir.join("Cargo.toml.stripped");
        fs::write(&base_path, &base_content)?;
        fs::write(&modified_path, modified)?;
        let diff = diff_files("Cargo.toml", &base_path, &modified_path)?;
        fs::remove_file(&base_path)?;
        fs::remove_file(&modified_path)?;
        let mut patch =
            String::from("This patch was autogenerated using debcargo via `remove_features`. Do not edit manually!\n");
        patch.push_str(&diff);
        Some(patch.into_bytes())
    } else {
        None
    };

    unapply_auto_patches(path, auto_patches)?;

    if let Some(patch) = result {
        auto_patches.insert("auto-remove-features.patch".to_string(), patch);
    }
    Ok(())
}

fn apply_remove_target_type(
    mut doc: toml_edit::DocumentMut,
    target_type: &TargetType,
) -> Result<Option<String>> {
    log::debug!("Removing target type {target_type}");
    let mut modified = false;
    let Some(package_item) = doc.get_mut("package") else {
        debcargo_bail!("Cargo.toml must contain 'package' section");
    };
    let Some(package_table) = package_item.as_table_mut() else {
        debcargo_bail!("Cargo.toml must contain 'package' table");
    };

    let autokey = match target_type {
        TargetType::Bench => "autobenches",
        TargetType::Example => "autoexamples",
        TargetType::Test => "autotests",
    };

    if let Some(value) = package_table.get_mut(autokey) {
        let Some(enabled) = value.as_bool() else {
            debcargo_bail!("Malformed manifest: package.{autokey} is not a boolean value: {item}");
        };
        if enabled {
            modified = true;
            *value = false.into();
        }
    } else {
        modified = true;
        package_table.insert(autokey, false.into());
    }

    let section_key = match target_type {
        TargetType::Bench => "bench",
        TargetType::Example => "example",
        TargetType::Test => "test",
    };
    let removed = doc.remove(section_key);
    if removed.is_some() {
        modified = true;
    }

    if modified {
        return Ok(Some(doc.to_string()));
    }

    Ok(None)
}

pub fn remove_target_types_patch(
    path: &Path,
    target_types: &[TargetType],
    auto_patches: &mut AutoPatches,
) -> Result<()> {
    let dir = path
        .parent()
        .ok_or_else(|| format_err!("{path:?} has no parent directory"))?;

    // Bring the on-disk manifest up to date with the earlier auto-patches, edit
    // it, capture the diff, then restore it. The orig manifest is left untouched
    // once we are done.
    apply_auto_patches(path, auto_patches)?;
    for target_type in target_types {
        let base_content = fs::read_to_string(path)?;

        // Edit the manifest with toml_edit so the diff only touches the lines we
        // actually change, rather than reformatting the whole file.
        let doc: toml_edit::DocumentMut = base_content.parse()?;
        let modified = apply_remove_target_type(doc, target_type)?;

        let result = if let Some(modified) = modified {
            let base_path = dir.join("Cargo.toml.base");
            let modified_path = dir.join(format!("Cargo.toml.stripped_{target_type}"));
            fs::write(&base_path, &base_content)?;
            fs::write(&modified_path, &modified)?;
            let diff = diff_files("Cargo.toml", &base_path, &modified_path)?;
            fs::remove_file(&base_path)?;
            fs::remove_file(&modified_path)?;
            let mut patch = String::from(
                "This patch was autogenerated using debcargo via `remove_target_types`. Do not edit manually!\n",
            );
            patch.push_str(&diff);
            Some(patch.into_bytes())
        } else {
            None
        };
        if let Some(patch) = result {
            run_patch(path, &patch, false)?;
            auto_patches.insert(format!("auto-remove-{target_type}.patch"), patch);
        }
    }

    unapply_auto_patches(path, auto_patches)?;

    Ok(())
}

fn apply_remove_targets(
    doc: toml_edit::DocumentMut,
    filter_targets: &[FilterTarget],
) -> Result<Option<String>> {
    let mut manifest = cargo_toml_edit::Manifest::from(doc);

    let targets: Result<Vec<_>> = filter_targets
        .iter()
        .flat_map(FilterTarget::target_list)
        .map(|t| {
            get_builtin_target_by_triple(t)
                .ok_or_else(|| format_err!("Triple '{t}' is not known to cfg-expr"))
        })
        .collect();
    let targets = targets?;
    let mut filter = DependencyFilter::empty();
    filter.targets = targets;
    log::debug!("Target filters: {filter:?}");

    let filtered = manifest.filter_dependencies(&filter)?;
    if filtered.removed_targets.is_empty() {
        Ok(None)
    } else {
        debcargo_info!("Removed dependencies of following targets:");
        for (target, _item) in &filtered.removed_targets {
            debcargo_info!(format!("- {target}"));
        }
        debcargo_info!("Dependencies:");
        for (dependency, _opt) in &filtered.removed_dependencies {
            debcargo_info!(format!("- {dependency}"));
        }
        Ok(Some(manifest.to_string()))
    }
}

/// Generate an auto-patch that removes all dependencies (and references to them in features) that
/// do not match our desired target triples.
///
/// `path` is the extracted (original) manifest, with any earlier auto-patches already collected in
/// `auto_patches`. The target-removal patch is generated on top of those, so the whole set stacks
/// cleanly in the quilt series. The new patch is inserted into `auto_patches` under
/// `auto-remove-targets.patch`; nothing is inserted if removing the features did not change
/// anything.
pub fn remove_targets_patch(
    path: &Path,
    filter_targets: &[FilterTarget],
    auto_patches: &mut AutoPatches,
) -> Result<()> {
    let dir = path
        .parent()
        .ok_or_else(|| format_err!("{path:?} has no parent directory"))?;

    // Bring the on-disk manifest up to date with the earlier auto-patches, edit
    // it, capture the diff, then restore it. The orig manifest is left untouched
    // once we are done.
    apply_auto_patches(path, auto_patches)?;
    let base_content = fs::read_to_string(path)?;

    // Edit the manifest with toml_edit so the diff only touches the lines we
    // actually change, rather than reformatting the whole file.
    let doc: toml_edit::DocumentMut = base_content.parse()?;
    let modified = apply_remove_targets(doc, filter_targets)?;

    let result = if let Some(modified) = modified {
        let base_path = dir.join("Cargo.toml.all_targets");
        let modified_path = dir.join("Cargo.toml.stripped_targets");
        fs::write(&base_path, &base_content)?;
        fs::write(&modified_path, &modified)?;
        let diff = diff_files("Cargo.toml", &base_path, &modified_path)?;
        fs::remove_file(&base_path)?;
        fs::remove_file(&modified_path)?;
        let mut patch =
            String::from("This patch was autogenerated using debcargo via `remove_targets`. Do not edit manually!\n");
        patch.push_str(&diff);
        Some(patch.into_bytes())
    } else {
        None
    };

    unapply_auto_patches(path, auto_patches)?;

    if let Some(patch) = result {
        auto_patches.insert("auto-remove-targets.patch".to_string(), patch);
    }
    Ok(())
}

/// Apply every auto-patch in `auto_patches`, in series, to `path` in place.
fn apply_auto_patches(path: &Path, auto_patches: &AutoPatches) -> Result<()> {
    for patch in auto_patches.values() {
        run_patch(path, patch, false)?;
    }
    Ok(())
}

/// Reverse every auto-patch in `auto_patches`, in reverse order, restoring
/// `path` to the state it had before `apply_auto_patches`.
fn unapply_auto_patches(path: &Path, auto_patches: &AutoPatches) -> Result<()> {
    for patch in auto_patches.values().rev() {
        run_patch(path, patch, true)?;
    }
    Ok(())
}

/// Apply (or, with `reverse`, un-apply) a unified-diff patch to `path` in
/// place, using the `patch` tool.
fn run_patch(path: &Path, patch: &[u8], reverse: bool) -> Result<()> {
    use std::io::Write;
    use std::process::Stdio;

    let mut cmd = std::process::Command::new("patch");
    cmd.arg("--quiet").arg(path);
    if reverse {
        cmd.arg("--reverse");
    }
    let mut child = cmd.stdin(Stdio::piped()).spawn()?;
    child
        .stdin
        .take()
        .ok_or_else(|| format_err!("failed to open stdin for patch"))?
        .write_all(patch)?;
    let status = child.wait()?;
    if !status.success() {
        debcargo_bail!("failed to apply auto-patch to {:?}", path);
    }
    Ok(())
}

/// Calculate all feature-dependencies and external-dependencies of a given
/// feature, using the information previously generated by
/// `all_dependencies_and_features`.
pub fn transitive_deps<'a>(
    features_with_deps: &'a CrateDepInfo,
    feature: &'a str,
) -> Result<(Vec<&'a str>, Vec<Dependency>)> {
    fn transitive_deps_impl<'a>(
        features_with_deps: &'a CrateDepInfo,
        feature: &'a str,
        cache: &mut HashMap<&'a str, (IndexSet<&'a str>, IndexSet<Dependency>)>,
        processing: &mut Vec<&'a str>,
    ) -> Result<(IndexSet<&'a str>, IndexSet<Dependency>)> {
        let mut all_features = IndexSet::new();
        let mut all_deps = IndexSet::new();
        processing.push(feature);
        if let Some((ff, dd)) = features_with_deps.get(feature) {
            all_features.extend(ff.clone());
            all_deps.extend(dd.clone());

            for f in ff {
                if processing.contains(f) {
                    continue;
                }

                let (ff1, dd1) = if let Some(cached) = cache.get(f) {
                    cached.clone()
                } else {
                    transitive_deps_impl(features_with_deps, f, cache, processing)?
                };

                let ff1: Vec<_> = ff1
                    .iter()
                    .filter(|f| **f != feature && !processing.contains(f))
                    .collect();

                all_features.extend(ff1);
                all_deps.extend(dd1);
            }
        } else {
            log::trace!("{features_with_deps:?}");
            debcargo_bail!(
                "requested feature '{}' not found in resolved map of features+dependencies",
                feature
            );
        }
        let result = (all_features, all_deps);
        cache.insert(feature, result.clone());
        processing.pop();
        Ok(result)
    }

    let mut cache = HashMap::new();
    let mut processing = Vec::new();
    let (all_features, all_deps) =
        transitive_deps_impl(features_with_deps, feature, &mut cache, &mut processing)?;

    Ok((
        all_features.into_iter().collect_vec(),
        all_deps.into_iter().collect_vec(),
    ))
}

#[cfg(test)]
mod tests {
    use super::*;

    fn doc_from(s: &str) -> toml_edit::DocumentMut {
        s.parse().unwrap()
    }

    #[test]
    fn apply_remove_features_keeps_dep_referenced_only_via_slash_syntax() {
        let doc = doc_from(
            r#"
[package]
name = "test-crate"
version = "0.1.0"
edition = "2021"

[features]
default = []
unstable = ["bar/some-feature"]
stable = ["bar?/other-feature"]

[dependencies]
bar = { version = "1", optional = true }
"#,
        );
        let modified =
            apply_remove_features(doc.clone(), &["unstable".to_string(), "stable".to_string()])
                .expect("Removing features should not return error")
                .expect("Removing features should return modified Cargo.toml");
        let doc = doc_from(&modified);
        // bar is never named via `dep:bar`, so cargo synthesizes an implicit
        // `bar` feature; the dependency stays enablable on its own and is kept.
        let deps = doc["dependencies"].as_table().unwrap();
        assert!(deps.contains_key("bar"));
    }

    #[test]
    fn apply_remove_features_noop_when_feature_absent() {
        let doc = doc_from(
            r#"
[package]
name = "test-crate"
version = "0.1.0"
edition = "2021"

[features]
stable = []
"#,
        );
        assert!(
            apply_remove_features(doc.clone(), &["nonexistent".to_string()])
                .expect("Removing features should not return error")
                .is_none()
        );
    }

    #[test]
    fn apply_remove_features_removes_named_feature() {
        let doc = doc_from(
            r#"
[features]
default = ["stable"]
stable = []
unstable = []
"#,
        );
        let modified = apply_remove_features(doc.clone(), &["unstable".to_string()])
            .expect("Removing features should not return error")
            .expect("Removing features should return modified Cargo.toml");
        let doc = doc_from(&modified);
        let feats = doc["features"].as_table().unwrap();
        assert!(!feats.contains_key("unstable"));
        assert!(feats.contains_key("stable"));
    }

    #[test]
    fn apply_remove_features_minimal_diff_preserves_formatting() {
        // A multi-line array in an unrelated feature must stay untouched.
        let doc = doc_from(
            r#"[features]
default = []
keep = [
    "a",
    "b",
]
drop = []
"#,
        );
        let out = apply_remove_features(doc.clone(), &["drop".to_string()])
            .expect("Removing features should not return error")
            .expect("Removing features should return modified Cargo.toml");
        assert!(out.contains("keep = [\n    \"a\",\n    \"b\",\n]"));
        assert!(!out.contains("drop = []"));
    }
}