use crate::pyproject_toml::{Format, SdistGenerator};
use crate::{BuildContext, ModuleWriter, PyProjectToml, SDistWriter, VirtualWriter};
use anyhow::{Context, Result, bail};
use cargo_metadata::camino::{self, Utf8Path};
use cargo_metadata::{Metadata, MetadataCommand, PackageId};
use fs_err as fs;
use ignore::overrides::Override;
use normpath::PathExt as _;
use path_slash::PathExt as _;
use pyproject_toml::check_pep639_glob;
use std::collections::{HashMap, HashSet};
use std::env;
use std::ffi::OsStr;
use std::io::Write;
use std::path::{Path, PathBuf};
use std::process::Command;
use std::str;
use toml_edit::DocumentMut;
use tracing::{debug, trace, warn};
#[derive(Debug, Clone)]
pub struct PathDependency {
manifest_path: PathBuf,
workspace_root: PathBuf,
readme: Option<PathBuf>,
license_file: Option<PathBuf>,
}
fn is_compiled_artifact(path: &Path) -> bool {
matches!(path.extension(), Some(ext) if ext == "pyc" || ext == "pyd" || ext == "so")
}
fn resolve_and_add_file(
writer: &mut VirtualWriter<SDistWriter>,
file: &Path,
manifest_dir: &Path,
target_dir: &Path,
kind: &str,
allowed_root: Option<&Path>,
) -> Result<PathBuf> {
let file = manifest_dir.join(file);
let abs_file = file
.normalize()
.with_context(|| {
format!(
"{kind} path `{}` does not exist or is invalid",
file.display()
)
})?
.into_path_buf();
if let Some(allowed_root) = allowed_root {
let allowed_root = allowed_root
.normalize()
.with_context(|| {
format!(
"allowed root `{}` does not exist or is invalid",
allowed_root.display()
)
})?
.into_path_buf();
if !abs_file.starts_with(&allowed_root) {
bail!(
"{kind} path `{}` resolves outside allowed root `{}`",
file.display(),
allowed_root.display()
);
}
}
let filename = file
.file_name()
.with_context(|| format!("{kind} path `{}` has no filename", file.display()))?;
writer.add_file(target_dir.join(filename), &abs_file, false)?;
Ok(abs_file)
}
fn resolve_and_add_readme(
writer: &mut VirtualWriter<SDistWriter>,
readme: &Path,
manifest_dir: &Path,
target_dir: &Path,
) -> Result<PathBuf> {
resolve_and_add_file(writer, readme, manifest_dir, target_dir, "readme", None)
}
fn parse_toml_file(path: &Path, kind: &str) -> Result<toml_edit::DocumentMut> {
let text = fs::read_to_string(path)?;
let document = text.parse::<toml_edit::DocumentMut>().context(format!(
"Failed to parse {} at {}",
kind,
path.display()
))?;
Ok(document)
}
fn rewrite_cargo_toml(
document: &mut DocumentMut,
manifest_path: &Path,
known_path_deps: &HashMap<String, PathDependency>,
) -> Result<()> {
debug!(
"Rewriting Cargo.toml `workspace.members` at {}",
manifest_path.display()
);
if let Some(workspace) = document.get_mut("workspace").and_then(|x| x.as_table_mut())
&& let Some(members) = workspace.get_mut("members").and_then(|x| x.as_array())
{
if known_path_deps.is_empty() {
workspace.remove("members");
if workspace.is_empty() {
document.remove("workspace");
}
} else {
let relative_dep_dirs: HashSet<String> = known_path_deps
.values()
.filter_map(|path_dep| {
let manifest_rel = path_dep
.manifest_path
.strip_prefix(&path_dep.workspace_root)
.ok()?;
manifest_rel.parent().and_then(|p| p.to_slash()).map(|s| {
if s.is_empty() {
".".into()
} else {
s.into_owned()
}
})
})
.collect();
let mut new_members = toml_edit::Array::new();
for member in members {
if let toml_edit::Value::String(s) = member {
let member_path = s.value();
let is_glob_pattern = member_path.contains(['*', '?', '[', ']']);
if is_glob_pattern {
let pattern = glob::Pattern::new(member_path).with_context(|| {
format!(
"Invalid `workspace.members` glob pattern: {} in {}",
member_path,
manifest_path.display()
)
})?;
if relative_dep_dirs.iter().any(|dir| pattern.matches(dir)) {
new_members.push(member_path);
}
} else if relative_dep_dirs.contains(member_path) {
new_members.push(member_path);
}
}
}
if !new_members.is_empty() {
workspace["members"] = toml_edit::value(new_members);
} else {
workspace.remove("members");
}
}
}
if let Some(workspace) = document.get_mut("workspace").and_then(|x| x.as_table_mut()) {
workspace.remove("default-members");
}
Ok(())
}
fn normalize_path(path: &Path) -> PathBuf {
use std::path::Component;
let mut components = Vec::new();
for component in path.components() {
match component {
Component::CurDir => {}
Component::ParentDir => {
match components.last() {
Some(Component::Normal(_)) => {
components.pop();
}
_ => {
components.push(component);
}
}
}
_ => components.push(component),
}
}
components.iter().collect()
}
fn rewrite_cargo_toml_targets(
document: &mut DocumentMut,
manifest_path: &Path,
packaged_files: &HashSet<PathBuf>,
) -> Result<()> {
debug!(
"Rewriting Cargo.toml build targets at {}",
manifest_path.display()
);
let manifest_dir = manifest_path.parent().unwrap();
let package_name = document
.get("package")
.and_then(|item| item.as_table())
.and_then(|table| table.get("name"))
.and_then(|item| item.as_str())
.map(str::to_string);
let normalize = |path: &Path| -> PathBuf {
let path = if path.is_absolute() {
path.strip_prefix(manifest_dir).unwrap_or(path)
} else {
path
};
normalize_path(path)
};
let has_packaged_path =
|paths: &[PathBuf]| -> bool { paths.iter().any(|path| packaged_files.contains(path)) };
let candidate_paths_for_target =
|kind: &str, name: Option<&str>, path: Option<&str>, package_name: Option<&str>| {
if let Some(path) = path {
return vec![normalize(Path::new(path))];
}
let name = name.or(package_name);
match (kind, name) {
("lib", _) => vec![normalize(Path::new("src/lib.rs"))],
("bin", Some(name)) => {
vec![
normalize(Path::new(&format!("src/bin/{name}.rs"))),
normalize(Path::new(&format!("src/bin/{name}/main.rs"))),
]
}
("bin", None) => vec![normalize(Path::new("src/main.rs"))],
("example", Some(name)) => vec![
normalize(Path::new(&format!("examples/{name}.rs"))),
normalize(Path::new(&format!("examples/{name}/main.rs"))),
],
("test", Some(name)) => vec![
normalize(Path::new(&format!("tests/{name}.rs"))),
normalize(Path::new(&format!("tests/{name}/main.rs"))),
],
("bench", Some(name)) => vec![
normalize(Path::new(&format!("benches/{name}.rs"))),
normalize(Path::new(&format!("benches/{name}/main.rs"))),
],
_ => Vec::new(),
}
};
let package_name = package_name.as_deref();
let mut drop_lib = false;
if let Some(lib) = document.get("lib").and_then(|item| item.as_table()) {
let name = lib.get("name").and_then(|item| item.as_str());
let path = lib.get("path").and_then(|item| item.as_str());
let candidates = candidate_paths_for_target("lib", name, path, package_name);
if !candidates.is_empty() && !has_packaged_path(&candidates) {
debug!(
"Stripping [lib] target {:?} from {}",
name.or(path),
manifest_path.display()
);
drop_lib = true;
}
}
if drop_lib {
document.remove("lib");
}
let mut removed_bins = Vec::new();
for (key, kind) in [
("bin", "bin"),
("example", "example"),
("test", "test"),
("bench", "bench"),
] {
if let Some(targets) = document
.get_mut(key)
.and_then(|item| item.as_array_of_tables_mut())
{
let mut idx = 0;
while idx < targets.len() {
let target = targets.get(idx).unwrap();
let name = target.get("name").and_then(|item| item.as_str());
let path = target.get("path").and_then(|item| item.as_str());
let candidates = candidate_paths_for_target(kind, name, path, package_name);
if !candidates.is_empty() && !has_packaged_path(&candidates) {
debug!(
"Stripping {key} target {:?} from {}",
name.or(path),
manifest_path.display()
);
if kind == "bin"
&& let Some(name) = name
{
removed_bins.push(name.to_string());
}
targets.remove(idx);
} else {
idx += 1;
}
}
if targets.is_empty() {
document.remove(key);
}
}
}
if !removed_bins.is_empty()
&& let Some(package) = document
.get_mut("package")
.and_then(|item| item.as_table_mut())
&& let Some(default_run) = package.get("default-run").and_then(|item| item.as_str())
&& removed_bins.iter().any(|name| name == default_run)
{
debug!(
"Stripping [package.default-run] target {:?} from {}",
default_run,
manifest_path.display()
);
package.remove("default-run");
}
Ok(())
}
fn rewrite_cargo_toml_readme(
document: &mut DocumentMut,
manifest_path: &Path,
readme_name: Option<&str>,
) -> Result<()> {
debug!(
"Rewriting Cargo.toml `package.readme` at {}",
manifest_path.display()
);
if let Some(readme_name) = readme_name {
let project = document.get_mut("package").with_context(|| {
format!(
"Missing `[package]` table in Cargo.toml with readme at {}",
manifest_path.display()
)
})?;
project["readme"] = toml_edit::value(readme_name);
}
Ok(())
}
fn rewrite_cargo_toml_license_file(
document: &mut DocumentMut,
manifest_path: &Path,
license_file_name: Option<&str>,
) -> Result<()> {
debug!(
"Rewriting Cargo.toml `package.license-file` at {}",
manifest_path.display()
);
if let Some(license_file_name) = license_file_name {
let project = document.get_mut("package").with_context(|| {
format!(
"Missing `[package]` table in Cargo.toml with license-file at {}",
manifest_path.display()
)
})?;
project["license-file"] = toml_edit::value(license_file_name);
}
Ok(())
}
fn rewrite_pyproject_toml(
pyproject_toml_path: &Path,
relative_manifest_path: &Path,
relative_python_source: Option<&Path>,
) -> Result<String> {
let mut data = parse_toml_file(pyproject_toml_path, "pyproject.toml")?;
let tool = data
.entry("tool")
.or_insert_with(|| toml_edit::Item::Table(toml_edit::Table::new()))
.as_table_like_mut()
.with_context(|| {
format!(
"`[tool]` must be a table in {}",
pyproject_toml_path.display()
)
})?;
let maturin = tool
.entry("maturin")
.or_insert_with(|| toml_edit::Item::Table(toml_edit::Table::new()))
.as_table_like_mut()
.with_context(|| {
format!(
"`[tool.maturin]` must be a table in {}",
pyproject_toml_path.display()
)
})?;
maturin.remove("manifest-path");
let manifest_path_str = relative_manifest_path.to_slash().with_context(|| {
format!(
"manifest-path `{}` is not valid UTF-8",
relative_manifest_path.display()
)
})?;
maturin.insert(
"manifest-path",
toml_edit::value(manifest_path_str.as_ref()),
);
if let Some(python_source) = relative_python_source {
maturin.remove("python-source");
let python_source_str = python_source.to_slash().with_context(|| {
format!(
"python-source path `{}` is not valid UTF-8",
python_source.display()
)
})?;
maturin.insert(
"python-source",
toml_edit::value(python_source_str.as_ref()),
);
}
Ok(data.to_string())
}
enum CrateRole<'a> {
Root {
known_path_deps: &'a HashMap<String, PathDependency>,
skip_prefixes: Vec<PathBuf>,
},
PathDependency { skip_cargo_toml: bool },
}
fn add_crate_to_source_distribution(
writer: &mut VirtualWriter<SDistWriter>,
manifest_path: impl AsRef<Path>,
prefix: impl AsRef<Path>,
readme: Option<&Path>,
license_file: Option<&Path>,
role: CrateRole<'_>,
) -> Result<()> {
debug!(
"Getting cargo package file list for {}",
manifest_path.as_ref().display()
);
let prefix = prefix.as_ref();
let manifest_path = manifest_path.as_ref();
let args = ["package", "--list", "--allow-dirty", "--manifest-path"];
let output = Command::new("cargo")
.args(args)
.arg(manifest_path)
.output()
.with_context(|| {
format!(
"Failed to run `cargo package --list --allow-dirty --manifest-path {}`",
manifest_path.display()
)
})?;
if !output.status.success() {
bail!(
"Failed to query file list from cargo: {}\n--- Manifest path: {}\n--- Stdout:\n{}\n--- Stderr:\n{}",
output.status,
manifest_path.display(),
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr),
);
}
if !output.stderr.is_empty() {
eprintln!(
"From `cargo {} {}`:",
args.join(" "),
manifest_path.display()
);
std::io::stderr().write_all(&output.stderr)?;
}
let file_list: Vec<&str> = str::from_utf8(&output.stdout)
.context("Cargo printed invalid utf-8 ಠ_ಠ")?
.lines()
.collect();
trace!("File list: {}", file_list.join(", "));
let manifest_dir = manifest_path.parent().unwrap();
let skip_prefixes = match &role {
CrateRole::Root { skip_prefixes, .. } => skip_prefixes.as_slice(),
CrateRole::PathDependency { .. } => &[],
};
let target_source: Vec<_> = file_list
.into_iter()
.map(|relative_to_manifests| {
let relative_to_cwd = manifest_dir.join(relative_to_manifests);
(relative_to_manifests, relative_to_cwd)
})
.filter(|(target, source)| {
if *target == "Cargo.toml.orig" {
false
} else if *target == "Cargo.toml" {
false
} else if matches!(role, CrateRole::Root { .. }) && *target == "pyproject.toml" {
false
} else if prefix.components().count() == 1 && *target == "pyproject.toml" {
debug!(
"Skipping potentially non-main {}",
prefix.join(target).display()
);
false
} else if !skip_prefixes.is_empty()
&& skip_prefixes
.iter()
.any(|p| Path::new(target).starts_with(p))
{
debug!(
"Skipping {} (will be added separately)",
prefix.join(target).display()
);
false
} else if is_compiled_artifact(Path::new(target)) {
debug!("Ignoring {}", target);
false
} else {
source.is_file() && !writer.exclude(source) && !writer.exclude(prefix.join(target))
}
})
.collect();
let packaged_files: HashSet<PathBuf> = target_source
.iter()
.map(|(target, _)| normalize_path(Path::new(target)))
.collect();
let cargo_toml_path = prefix.join(manifest_path.file_name().unwrap());
let readme_name = readme
.as_ref()
.map(|readme| {
readme
.file_name()
.and_then(OsStr::to_str)
.with_context(|| format!("Missing readme filename for {}", manifest_path.display()))
})
.transpose()?;
let license_file_name = license_file
.as_ref()
.map(|lf| {
lf.file_name().and_then(OsStr::to_str).with_context(|| {
format!(
"Missing license-file filename for {}",
manifest_path.display()
)
})
})
.transpose()?;
let target_source: Vec<_> = target_source
.into_iter()
.filter(|(target, _)| !writer.contains_target(prefix.join(target)))
.collect();
if !matches!(
role,
CrateRole::PathDependency {
skip_cargo_toml: true
}
) {
let mut document = parse_toml_file(manifest_path, "Cargo.toml")?;
rewrite_cargo_toml_readme(&mut document, manifest_path, readme_name)?;
rewrite_cargo_toml_license_file(&mut document, manifest_path, license_file_name)?;
if let CrateRole::Root {
known_path_deps, ..
} = &role
{
rewrite_cargo_toml(&mut document, manifest_path, known_path_deps)?;
}
rewrite_cargo_toml_targets(&mut document, manifest_path, &packaged_files)?;
writer.add_bytes(
cargo_toml_path,
Some(manifest_path),
document.to_string().as_bytes(),
false,
)?;
}
for (target, source) in target_source {
writer.add_file(prefix.join(target), source, false)?;
}
Ok(())
}
pub fn find_path_deps(cargo_metadata: &Metadata) -> Result<HashMap<String, PathDependency>> {
let root = cargo_metadata
.root_package()
.context("Expected the dependency graph to have a root package")?;
let packages_by_id: HashMap<&PackageId, &cargo_metadata::Package> =
cargo_metadata.packages.iter().map(|p| (&p.id, p)).collect();
let pkg_readmes: HashMap<&PackageId, PathBuf> = cargo_metadata
.packages
.iter()
.filter_map(|package| {
package
.readme
.as_ref()
.map(|readme| (&package.id, readme.clone().into_std_path_buf()))
})
.collect();
let pkg_license_files: HashMap<&PackageId, PathBuf> = cargo_metadata
.packages
.iter()
.filter_map(|package| {
package
.license_file
.as_ref()
.map(|license_file| (&package.id, license_file.clone().into_std_path_buf()))
})
.collect();
let resolve_nodes: HashMap<&PackageId, &[cargo_metadata::NodeDep]> = cargo_metadata
.resolve
.as_ref()
.context("cargo metadata is missing dependency resolve information")?
.nodes
.iter()
.map(|node| (&node.id, node.deps.as_slice()))
.collect();
let mut path_deps = HashMap::new();
let mut stack: Vec<&cargo_metadata::Package> = vec![root];
while let Some(top) = stack.pop() {
let node_deps = resolve_nodes
.get(&top.id)
.with_context(|| format!("missing resolve node for package {}", top.id))?;
for node_dep in *node_deps {
let dep_pkg = packages_by_id
.get(&node_dep.pkg)
.with_context(|| format!("missing package metadata for {}", node_dep.pkg))?;
let dependency = top
.dependencies
.iter()
.find(|d| d.name == dep_pkg.name.as_ref())
.with_context(|| {
format!(
"could not find dependency {} in package {}",
dep_pkg.name, top.id
)
})?;
if let Some(path) = &dependency.path {
let dep_name = dependency.rename.as_ref().unwrap_or(&dependency.name);
if path_deps.contains_key(dep_name) {
continue;
}
let dep_manifest_path = path.join("Cargo.toml");
let path_dep_metadata = MetadataCommand::new()
.manifest_path(&dep_manifest_path)
.verbose(true)
.no_deps()
.exec()
.with_context(|| {
format!(
"Failed to resolve workspace root for {} at '{dep_manifest_path}'",
node_dep.pkg
)
})?;
path_deps.insert(
dep_name.clone(),
PathDependency {
manifest_path: PathBuf::from(dep_manifest_path.clone()),
workspace_root: path_dep_metadata
.workspace_root
.clone()
.into_std_path_buf(),
readme: pkg_readmes.get(&node_dep.pkg).cloned(),
license_file: pkg_license_files.get(&node_dep.pkg).cloned(),
},
);
if let Some(&dep_package) = packages_by_id.get(&node_dep.pkg) {
stack.push(dep_package)
}
}
}
}
Ok(path_deps)
}
fn add_git_tracked_files_to_sdist(
pyproject_toml_path: &Path,
writer: &mut VirtualWriter<SDistWriter>,
prefix: impl AsRef<Path>,
) -> Result<()> {
let pyproject_dir = pyproject_toml_path.parent().unwrap();
let output = Command::new("git")
.args(["ls-files", "-z"])
.current_dir(pyproject_dir)
.output()
.context("Failed to run `git ls-files -z`")?;
if !output.status.success() {
bail!(
"Failed to query file list from git: {}\n--- Project Path: {}\n--- Stdout:\n{}\n--- Stderr:\n{}",
output.status,
pyproject_dir.display(),
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr),
);
}
let prefix = prefix.as_ref();
let file_paths = str::from_utf8(&output.stdout)
.context("git printed invalid utf-8 ಠ_ಠ")?
.split('\0')
.filter(|s| !s.is_empty())
.map(Path::new);
for source in file_paths {
writer.add_file(prefix.join(source), pyproject_dir.join(source), false)?;
}
Ok(())
}
struct SdistContext<'a> {
root_dir: &'a Path,
workspace_root: &'a Utf8Path,
workspace_manifest_path: camino::Utf8PathBuf,
known_path_deps: HashMap<String, PathDependency>,
sdist_root: PathBuf,
}
fn add_cargo_package_files_to_sdist(
build_context: &BuildContext,
pyproject_toml_path: &Path,
writer: &mut VirtualWriter<SDistWriter>,
root_dir: &Path,
) -> Result<()> {
let manifest_path = &build_context.manifest_path;
let workspace_root = &build_context.cargo_metadata.workspace_root;
let workspace_manifest_path = workspace_root.join("Cargo.toml");
let known_path_deps = find_path_deps(&build_context.cargo_metadata)?;
debug!(
"Found path dependencies: {:?}",
known_path_deps.keys().collect::<Vec<_>>()
);
let mut sdist_root =
common_path_prefix(workspace_root.as_std_path(), pyproject_toml_path).unwrap();
for path_dep in known_path_deps.values() {
if let Some(prefix) =
common_path_prefix(&sdist_root, path_dep.manifest_path.parent().unwrap())
{
sdist_root = prefix;
} else {
bail!("Failed to determine common path prefix of path dependencies");
}
}
let python_dir = &build_context.project_layout.python_dir;
if !python_dir.starts_with(&sdist_root)
&& let Some(prefix) = common_path_prefix(&sdist_root, python_dir)
{
sdist_root = prefix;
}
debug!("Found sdist root: {}", sdist_root.display());
let ctx = SdistContext {
root_dir,
workspace_root,
workspace_manifest_path,
known_path_deps,
sdist_root,
};
for (name, path_dep) in ctx.known_path_deps.iter() {
add_path_dep(writer, &ctx, name, path_dep)
.with_context(|| format!("Failed to add path dependency {name}"))?;
}
debug!("Adding the main crate {}", manifest_path.display());
let abs_manifest_path = manifest_path
.normalize()
.with_context(|| {
format!(
"manifest path `{}` does not exist or is invalid",
manifest_path.display()
)
})?
.into_path_buf();
let abs_manifest_dir = abs_manifest_path.parent().unwrap();
let main_crate = build_context.cargo_metadata.root_package().unwrap();
let relative_main_crate_manifest_dir = manifest_path
.parent()
.unwrap()
.strip_prefix(&ctx.sdist_root)
.unwrap();
let readme_path = if let Some(readme) = main_crate.readme.as_ref() {
let target_dir = root_dir.join(relative_main_crate_manifest_dir);
Some(resolve_and_add_readme(
writer,
readme.as_std_path(),
abs_manifest_dir,
&target_dir,
)?)
} else {
None
};
let license_file_path = if let Some(license_file) = main_crate.license_file.as_ref() {
let target_dir = root_dir.join(relative_main_crate_manifest_dir);
Some(resolve_and_add_file(
writer,
license_file.as_std_path(),
abs_manifest_dir,
&target_dir,
"license-file",
Some(workspace_root.as_std_path()),
)?)
} else {
None
};
let skip_prefixes: Vec<PathBuf> = if !relative_main_crate_manifest_dir.as_os_str().is_empty() {
let mut prefixes = Vec::new();
if let Some(python_module) = build_context.project_layout.python_module.as_ref()
&& let Ok(rel) = python_module.strip_prefix(abs_manifest_dir)
{
prefixes.push(rel.to_path_buf());
}
for package in &build_context.project_layout.python_packages {
let package_path = build_context.project_layout.python_dir.join(package);
if let Ok(rel) = package_path.strip_prefix(abs_manifest_dir)
&& !prefixes.contains(&rel.to_path_buf())
{
prefixes.push(rel.to_path_buf());
}
}
prefixes
} else {
Vec::new()
};
add_crate_to_source_distribution(
writer,
manifest_path,
root_dir.join(relative_main_crate_manifest_dir),
readme_path.as_deref(),
license_file_path.as_deref(),
CrateRole::Root {
known_path_deps: &ctx.known_path_deps,
skip_prefixes,
},
)?;
let manifest_cargo_lock_path = abs_manifest_dir.join("Cargo.lock");
let workspace_cargo_lock = ctx.workspace_root.join("Cargo.lock").into_std_path_buf();
let cargo_lock_path = if manifest_cargo_lock_path.exists() {
Some(manifest_cargo_lock_path.clone())
} else if workspace_cargo_lock.exists() {
Some(workspace_cargo_lock)
} else {
None
};
let cargo_lock_required =
build_context.cargo_options.locked || build_context.cargo_options.frozen;
let pyproject_root = pyproject_toml_path.parent().unwrap();
let project_root =
if pyproject_root == ctx.sdist_root || pyproject_root.starts_with(&ctx.sdist_root) {
&ctx.sdist_root
} else {
assert!(ctx.sdist_root.starts_with(pyproject_root));
pyproject_root
};
if let Some(cargo_lock_path) = cargo_lock_path {
let relative_cargo_lock = cargo_lock_path.strip_prefix(project_root).unwrap();
writer.add_file(root_dir.join(relative_cargo_lock), &cargo_lock_path, false)?;
} else if cargo_lock_required {
bail!("Cargo.lock is required by `--locked`/`--frozen` but it's not found.");
} else {
eprintln!(
"⚠️ Warning: Cargo.lock is not found, it is recommended \
to include it in the source distribution"
);
}
let normalized_workspace_root = ctx
.workspace_root
.as_std_path()
.normalize()
.map(|p| p.into_path_buf())
.unwrap_or_else(|_| ctx.workspace_root.as_std_path().to_path_buf());
let is_in_workspace = normalized_workspace_root != abs_manifest_dir;
if is_in_workspace {
let relative_workspace_cargo_toml = ctx
.workspace_manifest_path
.as_std_path()
.strip_prefix(project_root)
.unwrap();
let mut deps_to_keep = ctx.known_path_deps.clone();
let main_member_name = abs_manifest_dir
.strip_prefix(ctx.workspace_root)
.unwrap()
.to_slash()
.unwrap()
.to_string();
deps_to_keep.insert(
main_member_name,
PathDependency {
manifest_path: manifest_path.clone(),
workspace_root: ctx.workspace_root.as_std_path().to_path_buf(),
readme: None,
license_file: None,
},
);
let mut document =
parse_toml_file(ctx.workspace_manifest_path.as_std_path(), "Cargo.toml")?;
rewrite_cargo_toml(
&mut document,
ctx.workspace_manifest_path.as_std_path(),
&deps_to_keep,
)?;
let workspace_root_is_path_dep = ctx
.known_path_deps
.values()
.any(|dep| dep.manifest_path.as_path() == ctx.workspace_manifest_path.as_std_path());
if !workspace_root_is_path_dep && document.contains_key("package") {
debug!(
"Stripping [package] from workspace Cargo.toml at {} (source files not in sdist)",
ctx.workspace_manifest_path
);
let package_level_keys: Vec<String> = document
.as_table()
.iter()
.filter(|(key, _)| !matches!(&**key, "workspace" | "profile" | "patch" | "replace"))
.map(|(key, _)| key.to_string())
.collect();
for key in &package_level_keys {
document.remove(key);
}
}
writer.add_bytes(
root_dir.join(relative_workspace_cargo_toml),
Some(ctx.workspace_manifest_path.as_std_path()),
document.to_string().as_bytes(),
false,
)?;
}
let pyproject_dir = pyproject_toml_path.parent().unwrap();
if pyproject_dir != ctx.sdist_root {
let python_dir = &build_context.project_layout.python_dir;
let relative_python_source = if python_dir != pyproject_dir {
python_dir
.strip_prefix(pyproject_dir)
.or_else(|_| python_dir.strip_prefix(project_root))
.ok()
.map(|p| p.to_path_buf())
} else {
None
};
let rewritten_pyproject_toml = rewrite_pyproject_toml(
pyproject_toml_path,
&relative_main_crate_manifest_dir.join("Cargo.toml"),
relative_python_source.as_deref(),
)?;
writer.add_bytes(
root_dir.join("pyproject.toml"),
Some(pyproject_toml_path),
rewritten_pyproject_toml.as_bytes(),
false,
)?;
} else {
writer.add_file(root_dir.join("pyproject.toml"), pyproject_toml_path, false)?;
}
let mut python_packages = Vec::new();
if let Some(python_module) = build_context.project_layout.python_module.as_ref() {
trace!("Resolved python module: {}", python_module.display());
python_packages.push(python_module.to_path_buf());
}
for package in &build_context.project_layout.python_packages {
let package_path = build_context.project_layout.python_dir.join(package);
if python_packages.contains(&package_path) {
continue;
}
trace!("Resolved python package: {}", package_path.display());
python_packages.push(package_path);
}
for package in python_packages {
for entry in ignore::Walk::new(package) {
let source = entry?.into_path();
if is_compiled_artifact(&source) {
debug!("Ignoring {}", source.display());
continue;
}
let relative = source
.strip_prefix(pyproject_dir)
.or_else(|_| source.strip_prefix(project_root))
.with_context(|| {
format!(
"Python source file `{}` is outside both pyproject dir `{}` and project root `{}`",
source.display(),
pyproject_dir.display(),
project_root.display(),
)
})?;
let target = root_dir.join(relative);
if !source.is_dir() {
writer.add_file(target, &source, false)?;
}
}
}
Ok(())
}
fn add_path_dep(
writer: &mut VirtualWriter<SDistWriter>,
ctx: &SdistContext<'_>,
name: &str,
path_dep: &PathDependency,
) -> Result<()> {
debug!(
"Adding path dependency: {} at {}",
name,
path_dep.manifest_path.display()
);
let path_dep_manifest_dir = path_dep.manifest_path.parent().unwrap();
let relative_path_dep_manifest_dir =
path_dep_manifest_dir.strip_prefix(&ctx.sdist_root).unwrap();
let skip_cargo_toml =
ctx.workspace_manifest_path.as_std_path() == path_dep.manifest_path.as_path();
let readme_path = if let Some(readme) = path_dep.readme.as_ref() {
let target_dir = ctx.root_dir.join(relative_path_dep_manifest_dir);
Some(resolve_and_add_readme(
writer,
readme,
path_dep_manifest_dir,
&target_dir,
)?)
} else {
None
};
let license_file_path = if let Some(license_file) = path_dep.license_file.as_ref() {
let target_dir = ctx.root_dir.join(relative_path_dep_manifest_dir);
Some(resolve_and_add_file(
writer,
license_file,
path_dep_manifest_dir,
&target_dir,
"license-file",
Some(&path_dep.workspace_root),
)?)
} else {
None
};
add_crate_to_source_distribution(
writer,
&path_dep.manifest_path,
ctx.root_dir.join(relative_path_dep_manifest_dir),
readme_path.as_deref(),
license_file_path.as_deref(),
CrateRole::PathDependency { skip_cargo_toml },
)
.with_context(|| {
format!(
"Failed to add local dependency {} at {} to the source distribution",
name,
path_dep.manifest_path.display()
)
})?;
if path_dep.workspace_root.as_path() != ctx.workspace_root.as_std_path() {
let path_dep_workspace_manifest = path_dep.workspace_root.join("Cargo.toml");
let relative_path_dep_workspace_manifest = path_dep_workspace_manifest
.strip_prefix(&ctx.sdist_root)
.unwrap();
writer.add_file(
ctx.root_dir.join(relative_path_dep_workspace_manifest),
&path_dep_workspace_manifest,
false,
)?;
}
Ok(())
}
pub fn source_distribution(
build_context: &BuildContext,
pyproject: &PyProjectToml,
excludes: Override,
) -> Result<PathBuf> {
let pyproject_toml_path = build_context
.pyproject_toml_path
.normalize()
.with_context(|| {
format!(
"pyproject.toml path `{}` does not exist or is invalid",
build_context.pyproject_toml_path.display()
)
})?
.into_path_buf();
let source_date_epoch: Option<u64> =
env::var("SOURCE_DATE_EPOCH")
.ok()
.and_then(|var| match var.parse() {
Err(_) => {
warn!("SOURCE_DATE_EPOCH is malformed, ignoring");
None
}
Ok(val) => Some(val),
});
let metadata24 = &build_context.metadata24;
let writer = SDistWriter::new(&build_context.out, metadata24, source_date_epoch)?;
let mut writer = VirtualWriter::new(writer, excludes);
let root_dir = PathBuf::from(format!(
"{}-{}",
&metadata24.get_distribution_escaped(),
&metadata24.get_version_escaped()
));
match pyproject.sdist_generator() {
SdistGenerator::Cargo => add_cargo_package_files_to_sdist(
build_context,
&pyproject_toml_path,
&mut writer,
&root_dir,
)?,
SdistGenerator::Git => {
add_git_tracked_files_to_sdist(&pyproject_toml_path, &mut writer, &root_dir)?
}
}
let pyproject_dir = pyproject_toml_path.parent().unwrap();
if let Some(project) = pyproject.project.as_ref() {
if let Some(pyproject_toml::ReadMe::RelativePath(readme)) = project.readme.as_ref() {
let target = root_dir.join(readme);
if !writer.contains_target(&target) {
writer.add_file(target, pyproject_dir.join(readme), false)?;
}
}
if let Some(pyproject_toml::License::File { file }) = project.license.as_ref() {
let target = root_dir.join(file);
if !writer.contains_target(&target) {
writer.add_file(target, pyproject_dir.join(file), false)?;
}
}
if let Some(license_files) = &project.license_files {
let escaped_pyproject_dir =
PathBuf::from(glob::Pattern::escape(pyproject_dir.to_str().unwrap()));
let mut seen = HashSet::new();
for license_glob in license_files {
check_pep639_glob(license_glob)?;
for license_path in
glob::glob(&escaped_pyproject_dir.join(license_glob).to_string_lossy())?
{
let license_path = license_path?;
if !license_path.is_file() {
continue;
}
let license_path = license_path
.strip_prefix(pyproject_dir)
.expect("matched path starts with glob root")
.to_path_buf();
if seen.insert(license_path.clone()) {
debug!("Including license file `{}`", license_path.display());
writer.add_file(
root_dir.join(&license_path),
pyproject_dir.join(&license_path),
false,
)?;
}
}
}
}
}
let escaped_pyproject_dir = PathBuf::from(glob::Pattern::escape(
pyproject_dir.to_string_lossy().as_ref(),
));
let mut include = |pattern| -> Result<()> {
eprintln!("📦 Including files matching \"{pattern}\"");
for source in glob::glob(&escaped_pyproject_dir.join(pattern).to_string_lossy())
.with_context(|| format!("Invalid glob pattern: {pattern}"))?
.filter_map(Result::ok)
{
let target = root_dir.join(source.strip_prefix(pyproject_dir).unwrap());
if !source.is_dir() {
writer.add_file(target, source, false)?;
}
}
Ok(())
};
if let Some(glob_patterns) = pyproject.include() {
for pattern in glob_patterns
.iter()
.filter_map(|glob_pattern| glob_pattern.targets(Format::Sdist))
{
include(pattern)?;
}
}
let pkg_info = root_dir.join("PKG-INFO");
writer.add_bytes(
&pkg_info,
None,
metadata24.to_file_contents()?.as_bytes(),
false,
)?;
let source_distribution_path = writer.finish(&pkg_info)?;
eprintln!(
"📦 Built source distribution to {}",
source_distribution_path.display()
);
Ok(source_distribution_path)
}
fn common_path_prefix<P, Q>(one: P, two: Q) -> Option<PathBuf>
where
P: AsRef<Path>,
Q: AsRef<Path>,
{
let one = one.as_ref();
let two = two.as_ref();
let one = one.components();
let two = two.components();
let mut final_path = PathBuf::new();
let mut found = false;
let paths = one.zip(two);
for (l, r) in paths {
if l == r {
final_path.push(l.as_os_str());
found = true;
} else {
break;
}
}
if found { Some(final_path) } else { None }
}
#[cfg(test)]
mod tests {
use super::*;
use cargo_metadata::MetadataCommand;
use fs_err as fs;
use ignore::overrides::Override;
use pep440_rs::Version;
use std::str::FromStr;
use tempfile::TempDir;
use crate::Metadata24;
#[test]
fn test_normalize_path() {
let test_cases = vec![
("foo/bar", "foo/bar"),
("src/lib.rs", "src/lib.rs"),
("./foo/bar", "foo/bar"),
("foo/./bar", "foo/bar"),
(".", ""),
("foo/../bar", "bar"),
("foo/bar/..", "foo"),
("foo/bar/../baz", "foo/baz"),
("../foo", "../foo"),
("../../foo", "../../foo"),
("../../../foo/bar", "../../../foo/bar"),
("a/../../b", "../b"),
("a/b/../../../c", "../c"),
("foo/bar/baz/../../..", ""),
("./foo/../bar", "bar"),
("foo/./bar/../baz", "foo/baz"),
("./../foo/./bar", "../foo/bar"),
("", ""),
("foo", "foo"),
("..", ".."),
];
for (input, expected) in test_cases {
assert_eq!(
normalize_path(Path::new(input)),
PathBuf::from(expected),
"normalize_path({:?}) should equal {:?}",
input,
expected
);
}
}
#[test]
fn test_copy_manifest_sidecar_file_rejects_license_outside_allowed_root() {
let temp_dir = TempDir::new().unwrap();
let workspace_root = temp_dir.path().join("workspace");
let manifest_dir = workspace_root.join("crate");
let out_dir = temp_dir.path().join("out");
fs::create_dir_all(manifest_dir.join("src")).unwrap();
fs::create_dir_all(&out_dir).unwrap();
fs::write(manifest_dir.join("src/lib.rs"), "").unwrap();
fs::write(temp_dir.path().join("SECRET_LICENSE"), "secret").unwrap();
let metadata = Metadata24::new("test-pkg".to_string(), Version::from_str("1.0.0").unwrap());
let sdist_writer = SDistWriter::new(&out_dir, &metadata, None).unwrap();
let mut writer = VirtualWriter::new(sdist_writer, Override::empty());
let err = resolve_and_add_file(
&mut writer,
Path::new("../../SECRET_LICENSE"),
&manifest_dir,
&Path::new("pkg-1.0.0").join("crate"),
"license-file",
Some(&workspace_root),
)
.unwrap_err();
assert!(
err.to_string().contains("outside allowed root"),
"unexpected error: {err:#}"
);
}
#[test]
fn test_find_path_deps_captures_workspace_license_file() {
let temp_dir = TempDir::new().unwrap();
let workspace_root = temp_dir.path();
let py_dir = workspace_root.join("py");
let dep_dir = workspace_root.join("dep");
fs::create_dir_all(py_dir.join("src")).unwrap();
fs::create_dir_all(dep_dir.join("src")).unwrap();
fs::write(py_dir.join("src/lib.rs"), "").unwrap();
fs::write(dep_dir.join("src/lib.rs"), "").unwrap();
fs::write(workspace_root.join("LICENSE"), "MIT").unwrap();
fs::write(
workspace_root.join("Cargo.toml"),
indoc::indoc!(
r#"
[workspace]
resolver = "2"
members = ["py", "dep"]
[workspace.package]
license-file = "LICENSE"
"#
),
)
.unwrap();
fs::write(
dep_dir.join("Cargo.toml"),
indoc::indoc!(
r#"
[package]
name = "dep"
version = "0.1.0"
edition = "2021"
license-file.workspace = true
"#
),
)
.unwrap();
fs::write(
py_dir.join("Cargo.toml"),
indoc::indoc!(
r#"
[package]
name = "py"
version = "0.1.0"
edition = "2021"
[dependencies]
dep = { path = "../dep" }
"#
),
)
.unwrap();
let cargo_metadata = MetadataCommand::new()
.manifest_path(py_dir.join("Cargo.toml"))
.exec()
.unwrap();
let path_deps = find_path_deps(&cargo_metadata).unwrap();
let dep = path_deps.get("dep").expect("missing path dependency");
assert_eq!(dep.license_file.as_deref(), Some(Path::new("../LICENSE")));
}
#[test]
fn test_rewrite_cargo_toml_license_file() {
let manifest_path = Path::new("Cargo.toml");
let toml_str = r#"
[package]
name = "test"
version = "0.1.0"
license-file = "../../LICENSE"
"#;
let mut document = toml_str.parse::<DocumentMut>().unwrap();
rewrite_cargo_toml_license_file(&mut document, manifest_path, Some("LICENSE")).unwrap();
let result = document.to_string();
assert!(
result.contains(r#"license-file = "LICENSE""#),
"expected rewritten license-file, got: {result}"
);
let mut document2 = toml_str.parse::<DocumentMut>().unwrap();
rewrite_cargo_toml_license_file(&mut document2, manifest_path, None).unwrap();
let result2 = document2.to_string();
assert!(
result2.contains(r#"license-file = "../../LICENSE""#),
"expected unchanged license-file, got: {result2}"
);
}
#[test]
fn test_rewrite_cargo_toml_removes_default_members() {
let manifest_path = Path::new("Cargo.toml");
let toml_str = r#"
[workspace]
members = ["crate-a", "crate-b"]
default-members = ["crate-a", "crate-c"]
"#;
let mut document = toml_str.parse::<DocumentMut>().unwrap();
let mut known_path_deps = HashMap::new();
known_path_deps.insert(
"crate-a".to_string(),
PathDependency {
manifest_path: PathBuf::from("crate-a/Cargo.toml"),
workspace_root: PathBuf::from(""),
readme: None,
license_file: None,
},
);
rewrite_cargo_toml(&mut document, manifest_path, &known_path_deps).unwrap();
let result = document.to_string();
assert!(
result.contains(r#"members = ["crate-a"]"#),
"expected filtered members, got: {result}"
);
assert!(
!result.contains("default-members"),
"expected default-members to be removed, got: {result}"
);
}
}