use {
crate::{
module_util::{is_package_from_path, PythonModuleSuffixes},
package_metadata::PythonPackageMetadata,
resource::{
BytecodeOptimizationLevel, PythonEggFile, PythonExtensionModule, PythonModuleBytecode,
PythonModuleSource, PythonPackageDistributionResource,
PythonPackageDistributionResourceFlavor, PythonPackageResource, PythonPathExtension,
PythonResource,
},
},
anyhow::{Context, Result},
simple_file_manifest::{File, FileData, FileEntry, FileManifest},
std::{
collections::HashSet,
ffi::OsStr,
path::{Path, PathBuf},
},
};
#[cfg(unix)]
use std::os::unix::fs::PermissionsExt;
#[cfg(unix)]
fn is_executable(metadata: &std::fs::Metadata) -> bool {
let permissions = metadata.permissions();
permissions.mode() & 0o111 != 0
}
#[cfg(windows)]
fn is_executable(_metadata: &std::fs::Metadata) -> bool {
false
}
pub fn walk_tree_files(path: &Path) -> Box<dyn Iterator<Item = walkdir::DirEntry>> {
let res = walkdir::WalkDir::new(path).sort_by(|a, b| a.file_name().cmp(b.file_name()));
let filtered = res.into_iter().filter_map(|entry| {
let entry = entry.expect("unable to get directory entry");
let path = entry.path();
if path.is_dir() {
None
} else {
Some(entry)
}
});
Box::new(filtered)
}
#[derive(Debug, PartialEq)]
struct ResourceFile {
pub full_path: PathBuf,
pub relative_path: PathBuf,
}
#[derive(Debug, PartialEq)]
enum PathItem<'a> {
PythonResource(Box<PythonResource<'a>>),
ResourceFile(ResourceFile),
}
#[derive(Debug, PartialEq)]
struct PathEntry {
path: PathBuf,
file_emitted: bool,
non_file_emitted: bool,
}
pub struct PythonResourceIterator<'a> {
root_path: PathBuf,
cache_tag: String,
suffixes: PythonModuleSuffixes,
paths: Vec<PathEntry>,
path_content_overrides: FileManifest,
seen_packages: HashSet<String>,
resources: Vec<ResourceFile>,
emit_files: bool,
emit_non_files: bool,
_phantom: std::marker::PhantomData<&'a ()>,
}
impl<'a> PythonResourceIterator<'a> {
fn new(
path: &Path,
cache_tag: &str,
suffixes: &PythonModuleSuffixes,
emit_files: bool,
emit_non_files: bool,
) -> Result<PythonResourceIterator<'a>> {
let res = walkdir::WalkDir::new(path).sort_by(|a, b| a.file_name().cmp(b.file_name()));
let filtered = res
.into_iter()
.map(|entry| {
let entry = entry.context("resolving directory entry")?;
let path = entry.path();
Ok(if path.is_dir() {
None
} else {
Some(PathEntry {
path: path.to_path_buf(),
file_emitted: false,
non_file_emitted: false,
})
})
})
.collect::<Result<Vec<_>>>()?
.into_iter()
.flatten()
.collect::<Vec<_>>();
Ok(PythonResourceIterator {
root_path: path.to_path_buf(),
cache_tag: cache_tag.to_string(),
suffixes: suffixes.clone(),
paths: filtered,
path_content_overrides: FileManifest::default(),
seen_packages: HashSet::new(),
resources: Vec::new(),
emit_files,
emit_non_files,
_phantom: std::marker::PhantomData,
})
}
pub fn from_data_locations(
resources: &[File],
cache_tag: &str,
suffixes: &PythonModuleSuffixes,
emit_files: bool,
emit_non_files: bool,
) -> Result<PythonResourceIterator<'a>> {
let mut paths = resources
.iter()
.map(|file| PathEntry {
path: file.path().to_path_buf(),
file_emitted: false,
non_file_emitted: false,
})
.collect::<Vec<_>>();
paths.sort_by(|a, b| a.path.cmp(&b.path));
let mut path_content_overrides = FileManifest::default();
for resource in resources {
path_content_overrides.add_file_entry(resource.path(), resource.entry().clone())?;
}
Ok(PythonResourceIterator {
root_path: PathBuf::new(),
cache_tag: cache_tag.to_string(),
suffixes: suffixes.clone(),
paths,
path_content_overrides,
seen_packages: HashSet::new(),
resources: Vec::new(),
emit_files,
emit_non_files,
_phantom: std::marker::PhantomData,
})
}
fn resolve_is_executable(&self, path: &Path) -> bool {
match self.path_content_overrides.get(path) {
Some(file) => file.is_executable(),
None => {
if let Ok(metadata) = path.metadata() {
is_executable(&metadata)
} else {
false
}
}
}
}
fn resolve_file_data(&self, path: &Path) -> FileData {
match self.path_content_overrides.get(path) {
Some(file) => file.file_data().clone(),
None => FileData::Path(path.to_path_buf()),
}
}
fn resolve_path(&mut self, path: &Path) -> Option<PathItem<'a>> {
let mut rel_path = path
.strip_prefix(&self.root_path)
.expect("unable to strip path prefix");
let mut rel_str = rel_path.to_str().expect("could not convert path to str");
let mut components = rel_path
.iter()
.map(|p| p.to_str().expect("unable to get path as str"))
.collect::<Vec<_>>();
let distribution_info = if components[0].ends_with(".dist-info") {
Some((
self.root_path.join(components[0]).join("METADATA"),
PythonPackageDistributionResourceFlavor::DistInfo,
))
} else if components[0].ends_with(".egg-info") {
Some((
self.root_path.join(components[0]).join("PKG-INFO"),
PythonPackageDistributionResourceFlavor::EggInfo,
))
} else {
None
};
if let Some((metadata_path, location)) = distribution_info {
let data = if let Some(file) = self.path_content_overrides.get(&metadata_path) {
file.resolve_content().ok()?
} else {
std::fs::read(&metadata_path).ok()?
};
let metadata = PythonPackageMetadata::from_metadata(&data).ok()?;
let package = metadata.name()?;
let version = metadata.version()?;
let name = components[1..components.len()].join("/");
return Some(PathItem::PythonResource(Box::new(
PythonPackageDistributionResource {
location,
package: package.to_string(),
version: version.to_string(),
name,
data: self.resolve_file_data(path),
}
.into(),
)));
}
let in_site_packages = if components[0] == "site-packages" {
let sp_path = self.root_path.join("site-packages");
rel_path = path
.strip_prefix(sp_path)
.expect("unable to strip site-packages prefix");
rel_str = rel_path.to_str().expect("could not convert path to str");
components = rel_path
.iter()
.map(|p| p.to_str().expect("unable to get path as str"))
.collect::<Vec<_>>();
true
} else {
false
};
if components[0..components.len() - 1]
.iter()
.any(|p| p.ends_with(".egg"))
{
let mut egg_root_path = self.root_path.clone();
if in_site_packages {
egg_root_path = egg_root_path.join("site-packages");
}
for p in &components[0..components.len() - 1] {
egg_root_path = egg_root_path.join(p);
if p.ends_with(".egg") {
break;
}
}
rel_path = path
.strip_prefix(egg_root_path)
.expect("unable to strip egg prefix");
components = rel_path
.iter()
.map(|p| p.to_str().expect("unable to get path as str"))
.collect::<Vec<_>>();
if components[0] == "EGG-INFO" {
return None;
}
}
let file_name = rel_path.file_name().unwrap().to_string_lossy();
for ext_suffix in &self.suffixes.extension {
if file_name.ends_with(ext_suffix) {
let package_parts = &components[0..components.len() - 1];
let mut package = itertools::join(package_parts, ".");
let module_name = &file_name[0..file_name.len() - ext_suffix.len()];
let mut full_module_name: Vec<&str> = package_parts.to_vec();
if module_name != "__init__" {
full_module_name.push(module_name);
}
let full_module_name = itertools::join(full_module_name, ".");
if package.is_empty() {
package = full_module_name.clone();
}
self.seen_packages.insert(package);
let module_components = full_module_name.split('.').collect::<Vec<_>>();
let final_name = module_components[module_components.len() - 1];
let init_fn = Some(format!("PyInit_{final_name}"));
return Some(PathItem::PythonResource(Box::new(
PythonExtensionModule {
name: full_module_name,
init_fn,
extension_file_suffix: ext_suffix.clone(),
shared_library: Some(self.resolve_file_data(path)),
object_file_data: vec![],
is_package: is_package_from_path(path),
link_libraries: vec![],
is_stdlib: false,
builtin_default: false,
required: false,
variant: None,
license: None,
}
.into(),
)));
}
}
if self
.suffixes
.source
.iter()
.any(|ext| rel_str.ends_with(ext))
{
let package_parts = &components[0..components.len() - 1];
let mut package = itertools::join(package_parts, ".");
let module_name = rel_path
.file_stem()
.expect("unable to get file stem")
.to_str()
.expect("unable to convert path to str");
let mut full_module_name: Vec<&str> = package_parts.to_vec();
if module_name != "__init__" {
full_module_name.push(module_name);
}
let full_module_name = itertools::join(full_module_name, ".");
if package.is_empty() {
package = full_module_name.clone();
}
self.seen_packages.insert(package);
return Some(PathItem::PythonResource(Box::new(
PythonModuleSource {
name: full_module_name,
source: self.resolve_file_data(path),
is_package: is_package_from_path(path),
cache_tag: self.cache_tag.clone(),
is_stdlib: false,
is_test: false,
}
.into(),
)));
}
if self
.suffixes
.bytecode
.iter()
.any(|ext| rel_str.ends_with(ext))
{
if components.len() < 2 {
return None;
}
if components[components.len() - 2] != "__pycache__" {
return None;
}
let package_parts = &components[0..components.len() - 2];
let mut package = itertools::join(package_parts, ".");
let filename = rel_path
.file_name()
.expect("unable to get file name")
.to_string_lossy()
.to_string();
let filename_parts = filename.split('.').collect::<Vec<&str>>();
if filename_parts.len() < 3 {
return None;
}
let mut remaining_filename = filename.clone();
let module_name = filename_parts[0];
remaining_filename = remaining_filename[module_name.len() + 1..].to_string();
if filename_parts[1] != self.cache_tag {
return None;
}
remaining_filename = remaining_filename[self.cache_tag.len()..].to_string();
let optimization_level = if filename_parts[2] == "opt-1" {
remaining_filename = remaining_filename[6..].to_string();
BytecodeOptimizationLevel::One
} else if filename_parts[2] == "opt-2" {
remaining_filename = remaining_filename[6..].to_string();
BytecodeOptimizationLevel::Two
} else {
BytecodeOptimizationLevel::Zero
};
if !self.suffixes.bytecode.contains(&remaining_filename) {
return None;
}
let mut full_module_name: Vec<&str> = package_parts.to_vec();
if module_name != "__init__" {
full_module_name.push(module_name);
}
let full_module_name = itertools::join(full_module_name, ".");
if package.is_empty() {
package = full_module_name.clone();
}
self.seen_packages.insert(package);
return Some(PathItem::PythonResource(Box::new(
PythonModuleBytecode::from_path(
&full_module_name,
optimization_level,
&self.cache_tag,
path,
)
.into(),
)));
}
let resource = match rel_path.extension().and_then(OsStr::to_str) {
Some("egg") => PathItem::PythonResource(Box::new(
PythonEggFile {
data: self.resolve_file_data(path),
}
.into(),
)),
Some("pth") => PathItem::PythonResource(Box::new(
PythonPathExtension {
data: self.resolve_file_data(path),
}
.into(),
)),
_ => {
PathItem::ResourceFile(ResourceFile {
full_path: path.to_path_buf(),
relative_path: rel_path.to_path_buf(),
})
}
};
Some(resource)
}
}
impl<'a> Iterator for PythonResourceIterator<'a> {
type Item = Result<PythonResource<'a>>;
fn next(&mut self) -> Option<Result<PythonResource<'a>>> {
loop {
if self.paths.is_empty() {
break;
}
if self.emit_files && !self.paths[0].file_emitted {
self.paths[0].file_emitted = true;
let rel_path = self.paths[0]
.path
.strip_prefix(&self.root_path)
.expect("unable to strip path prefix")
.to_path_buf();
let f = File::new(
rel_path,
FileEntry::new_from_data(
self.resolve_file_data(&self.paths[0].path),
self.resolve_is_executable(&self.paths[0].path),
),
);
return Some(Ok(f.into()));
}
if self.emit_non_files && !self.paths[0].non_file_emitted {
self.paths[0].non_file_emitted = true;
let path_temp = self.paths[0].path.clone();
if let Some(entry) = self.resolve_path(&path_temp) {
match entry {
PathItem::ResourceFile(resource) => {
self.resources.push(resource);
}
PathItem::PythonResource(resource) => {
return Some(Ok(*resource));
}
}
}
}
self.paths.remove(0);
continue;
}
loop {
if self.resources.is_empty() {
return None;
}
let resource = self.resources.remove(0);
let basename = resource
.relative_path
.file_name()
.unwrap()
.to_string_lossy();
let (leaf_package, relative_name) =
if let Some(relative_directory) = resource.relative_path.parent() {
let mut components = relative_directory
.iter()
.map(|p| p.to_string_lossy())
.collect::<Vec<_>>();
let mut relative_components = vec![basename];
let mut package = None;
let mut relative_name = None;
while !components.is_empty() {
let candidate_package = itertools::join(&components, ".");
if self.seen_packages.contains(&candidate_package) {
package = Some(candidate_package);
relative_components.reverse();
relative_name = Some(itertools::join(&relative_components, "/"));
break;
}
let popped = components.pop().unwrap();
relative_components.push(popped);
}
(package, relative_name)
} else {
(None, None)
};
if leaf_package.is_none() {
continue;
}
let leaf_package = leaf_package.unwrap();
let relative_name = relative_name.unwrap();
return Some(Ok(PythonPackageResource {
leaf_package,
relative_name,
data: self.resolve_file_data(&resource.full_path),
is_stdlib: false,
is_test: false,
}
.into()));
}
}
}
pub fn find_python_resources<'a>(
root_path: &Path,
cache_tag: &str,
suffixes: &PythonModuleSuffixes,
emit_files: bool,
emit_non_files: bool,
) -> Result<PythonResourceIterator<'a>> {
PythonResourceIterator::new(root_path, cache_tag, suffixes, emit_files, emit_non_files)
}
#[cfg(test)]
mod tests {
use {
super::*,
once_cell::sync::Lazy,
std::fs::{create_dir_all, write},
};
const DEFAULT_CACHE_TAG: &str = "cpython-37";
static DEFAULT_SUFFIXES: Lazy<PythonModuleSuffixes> = Lazy::new(|| PythonModuleSuffixes {
source: vec![".py".to_string()],
bytecode: vec![".pyc".to_string()],
debug_bytecode: vec![],
optimized_bytecode: vec![],
extension: vec![],
});
#[test]
fn test_source_resolution() -> Result<()> {
let td = tempfile::Builder::new()
.prefix("python-packaging-test")
.tempdir()?;
let tp = td.path();
let acme_path = tp.join("acme");
let acme_a_path = acme_path.join("a");
let acme_bar_path = acme_path.join("bar");
create_dir_all(&acme_a_path).unwrap();
create_dir_all(&acme_bar_path).unwrap();
write(acme_path.join("__init__.py"), "")?;
write(acme_a_path.join("__init__.py"), "")?;
write(acme_bar_path.join("__init__.py"), "")?;
write(acme_a_path.join("foo.py"), "# acme.foo")?;
let resources =
PythonResourceIterator::new(tp, DEFAULT_CACHE_TAG, &DEFAULT_SUFFIXES, true, true)?
.collect::<Result<Vec<_>>>()?;
assert_eq!(resources.len(), 8);
assert_eq!(
resources[0],
File::new(
"acme/__init__.py",
FileEntry::try_from(acme_path.join("__init__.py"))?
)
.into()
);
assert_eq!(
resources[1],
PythonModuleSource {
name: "acme".to_string(),
source: FileData::Path(acme_path.join("__init__.py")),
is_package: true,
cache_tag: DEFAULT_CACHE_TAG.to_string(),
is_stdlib: false,
is_test: false,
}
.into()
);
assert_eq!(
resources[2],
File::new(
"acme/a/__init__.py",
FileEntry::try_from(acme_a_path.join("__init__.py"))?
)
.into()
);
assert_eq!(
resources[3],
PythonModuleSource {
name: "acme.a".to_string(),
source: FileData::Path(acme_a_path.join("__init__.py")),
is_package: true,
cache_tag: DEFAULT_CACHE_TAG.to_string(),
is_stdlib: false,
is_test: false,
}
.into()
);
assert_eq!(
resources[4],
File::new(
"acme/a/foo.py",
FileEntry::try_from(acme_a_path.join("foo.py"))?
)
.into()
);
assert_eq!(
resources[5],
PythonModuleSource {
name: "acme.a.foo".to_string(),
source: FileData::Path(acme_a_path.join("foo.py")),
is_package: false,
cache_tag: DEFAULT_CACHE_TAG.to_string(),
is_stdlib: false,
is_test: false,
}
.into()
);
assert_eq!(
resources[6],
File::new(
"acme/bar/__init__.py",
FileEntry::try_from(acme_bar_path.join("__init__.py"))?
)
.into()
);
assert_eq!(
resources[7],
PythonModuleSource {
name: "acme.bar".to_string(),
source: FileData::Path(acme_bar_path.join("__init__.py")),
is_package: true,
cache_tag: DEFAULT_CACHE_TAG.to_string(),
is_stdlib: false,
is_test: false,
}
.into()
);
Ok(())
}
#[test]
fn test_bytecode_resolution() -> Result<()> {
let td = tempfile::Builder::new()
.prefix("python-packaging-test")
.tempdir()?;
let tp = td.path();
let acme_path = tp.join("acme");
let acme_a_path = acme_path.join("a");
let acme_bar_path = acme_path.join("bar");
create_dir_all(&acme_a_path)?;
create_dir_all(&acme_bar_path)?;
let acme_pycache_path = acme_path.join("__pycache__");
let acme_a_pycache_path = acme_a_path.join("__pycache__");
let acme_bar_pycache_path = acme_bar_path.join("__pycache__");
create_dir_all(&acme_pycache_path)?;
create_dir_all(&acme_a_pycache_path)?;
create_dir_all(&acme_bar_pycache_path)?;
write(acme_pycache_path.join("__init__.pyc"), "")?;
write(acme_pycache_path.join("__init__.cpython-37.foo.pyc"), "")?;
write(acme_pycache_path.join("__init__.cpython-37.pyc"), "")?;
write(acme_pycache_path.join("__init__.cpython-37.opt-1.pyc"), "")?;
write(acme_pycache_path.join("__init__.cpython-37.opt-2.pyc"), "")?;
write(acme_pycache_path.join("__init__.cpython-38.pyc"), "")?;
write(acme_pycache_path.join("__init__.cpython-38.opt-1.pyc"), "")?;
write(acme_pycache_path.join("__init__.cpython-38.opt-2.pyc"), "")?;
write(acme_pycache_path.join("foo.cpython-37.pyc"), "")?;
write(acme_pycache_path.join("foo.cpython-37.opt-1.pyc"), "")?;
write(acme_pycache_path.join("foo.cpython-37.opt-2.pyc"), "")?;
write(acme_pycache_path.join("foo.cpython-38.pyc"), "")?;
write(acme_pycache_path.join("foo.cpython-38.opt-1.pyc"), "")?;
write(acme_pycache_path.join("foo.cpython-38.opt-2.pyc"), "")?;
write(acme_a_pycache_path.join("__init__.cpython-37.pyc"), "")?;
write(
acme_a_pycache_path.join("__init__.cpython-37.opt-1.pyc"),
"",
)?;
write(
acme_a_pycache_path.join("__init__.cpython-37.opt-2.pyc"),
"",
)?;
write(acme_a_pycache_path.join("__init__.cpython-38.pyc"), "")?;
write(
acme_a_pycache_path.join("__init__.cpython-38.opt-1.pyc"),
"",
)?;
write(
acme_a_pycache_path.join("__init__.cpython-38.opt-2.pyc"),
"",
)?;
write(acme_a_pycache_path.join("foo.cpython-37.pyc"), "")?;
write(acme_a_pycache_path.join("foo.cpython-37.opt-1.pyc"), "")?;
write(acme_a_pycache_path.join("foo.cpython-37.opt-2.pyc"), "")?;
write(acme_a_pycache_path.join("foo.cpython-38.pyc"), "")?;
write(acme_a_pycache_path.join("foo.cpython-38.opt-1.pyc"), "")?;
write(acme_a_pycache_path.join("foo.cpython-38.opt-2.pyc"), "")?;
write(acme_bar_pycache_path.join("__init__.cpython-37.pyc"), "")?;
write(
acme_bar_pycache_path.join("__init__.cpython-37.opt-1.pyc"),
"",
)?;
write(
acme_bar_pycache_path.join("__init__.cpython-37.opt-2.pyc"),
"",
)?;
write(acme_bar_pycache_path.join("__init__.cpython-38.pyc"), "")?;
write(
acme_bar_pycache_path.join("__init__.cpython-38.opt-1.pyc"),
"",
)?;
write(
acme_bar_pycache_path.join("__init__.cpython-38.opt-2.pyc"),
"",
)?;
write(acme_bar_pycache_path.join("foo.cpython-37.pyc"), "")?;
write(acme_bar_pycache_path.join("foo.cpython-37.opt-1.pyc"), "")?;
write(acme_bar_pycache_path.join("foo.cpython-37.opt-2.pyc"), "")?;
write(acme_bar_pycache_path.join("foo.cpython-38.pyc"), "")?;
write(acme_bar_pycache_path.join("foo.cpython-38.opt-1.pyc"), "")?;
write(acme_bar_pycache_path.join("foo.cpython-38.opt-2.pyc"), "")?;
let resources =
PythonResourceIterator::new(tp, "cpython-38", &DEFAULT_SUFFIXES, false, true)?
.collect::<Result<Vec<_>>>()?;
assert_eq!(resources.len(), 18);
assert_eq!(
resources[0],
PythonModuleBytecode::from_path(
"acme",
BytecodeOptimizationLevel::One,
"cpython-38",
&acme_pycache_path.join("__init__.cpython-38.opt-1.pyc")
)
.into()
);
assert_eq!(
resources[1],
PythonModuleBytecode::from_path(
"acme",
BytecodeOptimizationLevel::Two,
"cpython-38",
&acme_pycache_path.join("__init__.cpython-38.opt-2.pyc")
)
.into()
);
assert_eq!(
resources[2],
PythonModuleBytecode::from_path(
"acme",
BytecodeOptimizationLevel::Zero,
"cpython-38",
&acme_pycache_path.join("__init__.cpython-38.pyc")
)
.into()
);
assert_eq!(
resources[3],
PythonModuleBytecode::from_path(
"acme.foo",
BytecodeOptimizationLevel::One,
"cpython-38",
&acme_pycache_path.join("foo.cpython-38.opt-1.pyc")
)
.into()
);
assert_eq!(
resources[4],
PythonModuleBytecode::from_path(
"acme.foo",
BytecodeOptimizationLevel::Two,
"cpython-38",
&acme_pycache_path.join("foo.cpython-38.opt-2.pyc")
)
.into()
);
assert_eq!(
resources[5],
PythonModuleBytecode::from_path(
"acme.foo",
BytecodeOptimizationLevel::Zero,
"cpython-38",
&acme_pycache_path.join("foo.cpython-38.pyc")
)
.into()
);
assert_eq!(
resources[6],
PythonModuleBytecode::from_path(
"acme.a",
BytecodeOptimizationLevel::One,
"cpython-38",
&acme_a_pycache_path.join("__init__.cpython-38.opt-1.pyc")
)
.into()
);
assert_eq!(
resources[7],
PythonModuleBytecode::from_path(
"acme.a",
BytecodeOptimizationLevel::Two,
"cpython-38",
&acme_a_pycache_path.join("__init__.cpython-38.opt-2.pyc")
)
.into()
);
assert_eq!(
resources[8],
PythonModuleBytecode::from_path(
"acme.a",
BytecodeOptimizationLevel::Zero,
"cpython-38",
&acme_a_pycache_path.join("__init__.cpython-38.pyc")
)
.into()
);
assert_eq!(
resources[9],
PythonModuleBytecode::from_path(
"acme.a.foo",
BytecodeOptimizationLevel::One,
"cpython-38",
&acme_a_pycache_path.join("foo.cpython-38.opt-1.pyc")
)
.into()
);
assert_eq!(
resources[10],
PythonModuleBytecode::from_path(
"acme.a.foo",
BytecodeOptimizationLevel::Two,
"cpython-38",
&acme_a_pycache_path.join("foo.cpython-38.opt-2.pyc")
)
.into()
);
assert_eq!(
resources[11],
PythonModuleBytecode::from_path(
"acme.a.foo",
BytecodeOptimizationLevel::Zero,
"cpython-38",
&acme_a_pycache_path.join("foo.cpython-38.pyc")
)
.into()
);
assert_eq!(
resources[12],
PythonModuleBytecode::from_path(
"acme.bar",
BytecodeOptimizationLevel::One,
"cpython-38",
&acme_bar_pycache_path.join("__init__.cpython-38.opt-1.pyc")
)
.into()
);
assert_eq!(
resources[13],
PythonModuleBytecode::from_path(
"acme.bar",
BytecodeOptimizationLevel::Two,
"cpython-38",
&acme_bar_pycache_path.join("__init__.cpython-38.opt-2.pyc")
)
.into()
);
assert_eq!(
resources[14],
PythonModuleBytecode::from_path(
"acme.bar",
BytecodeOptimizationLevel::Zero,
"cpython-38",
&acme_bar_pycache_path.join("__init__.cpython-38.pyc")
)
.into()
);
assert_eq!(
resources[15],
PythonModuleBytecode::from_path(
"acme.bar.foo",
BytecodeOptimizationLevel::One,
"cpython-38",
&acme_bar_pycache_path.join("foo.cpython-38.opt-1.pyc")
)
.into()
);
assert_eq!(
resources[16],
PythonModuleBytecode::from_path(
"acme.bar.foo",
BytecodeOptimizationLevel::Two,
"cpython-38",
&acme_bar_pycache_path.join("foo.cpython-38.opt-2.pyc")
)
.into()
);
assert_eq!(
resources[17],
PythonModuleBytecode::from_path(
"acme.bar.foo",
BytecodeOptimizationLevel::Zero,
"cpython-38",
&acme_bar_pycache_path.join("foo.cpython-38.pyc")
)
.into()
);
Ok(())
}
#[test]
fn test_site_packages() -> Result<()> {
let td = tempfile::Builder::new()
.prefix("python-packaging-test")
.tempdir()?;
let tp = td.path();
let sp_path = tp.join("site-packages");
let acme_path = sp_path.join("acme");
create_dir_all(&acme_path).unwrap();
write(acme_path.join("__init__.py"), "")?;
write(acme_path.join("bar.py"), "")?;
let resources =
PythonResourceIterator::new(tp, DEFAULT_CACHE_TAG, &DEFAULT_SUFFIXES, false, true)?
.collect::<Result<Vec<_>>>()?;
assert_eq!(resources.len(), 2);
assert_eq!(
resources[0],
PythonModuleSource {
name: "acme".to_string(),
source: FileData::Path(acme_path.join("__init__.py")),
is_package: true,
cache_tag: DEFAULT_CACHE_TAG.to_string(),
is_stdlib: false,
is_test: false,
}
.into()
);
assert_eq!(
resources[1],
PythonModuleSource {
name: "acme.bar".to_string(),
source: FileData::Path(acme_path.join("bar.py")),
is_package: false,
cache_tag: DEFAULT_CACHE_TAG.to_string(),
is_stdlib: false,
is_test: false,
}
.into()
);
Ok(())
}
#[test]
fn test_extension_module() -> Result<()> {
let td = tempfile::Builder::new()
.prefix("python-packaging-test")
.tempdir()?;
let tp = td.path();
create_dir_all(tp.join("markupsafe"))?;
let pyd_path = tp.join("foo.pyd");
let so_path = tp.join("bar.so");
let cffi_path = tp.join("_cffi_backend.cp37-win_amd64.pyd");
let markupsafe_speedups_path = tp
.join("markupsafe")
.join("_speedups.cpython-37m-x86_64-linux-gnu.so");
let zstd_path = tp.join("zstd.cpython-37m-x86_64-linux-gnu.so");
write(&pyd_path, "")?;
write(&so_path, "")?;
write(&cffi_path, "")?;
write(&markupsafe_speedups_path, "")?;
write(&zstd_path, "")?;
let suffixes = PythonModuleSuffixes {
source: vec![],
bytecode: vec![],
debug_bytecode: vec![],
optimized_bytecode: vec![],
extension: vec![
".cp37-win_amd64.pyd".to_string(),
".cp37-win32.pyd".to_string(),
".cpython-37m-x86_64-linux-gnu.so".to_string(),
".pyd".to_string(),
".so".to_string(),
],
};
let resources = PythonResourceIterator::new(tp, "cpython-37", &suffixes, false, true)?
.collect::<Result<Vec<_>>>()?;
assert_eq!(resources.len(), 5);
assert_eq!(
resources[0],
PythonExtensionModule {
name: "_cffi_backend".to_string(),
init_fn: Some("PyInit__cffi_backend".to_string()),
extension_file_suffix: ".cp37-win_amd64.pyd".to_string(),
shared_library: Some(FileData::Path(cffi_path)),
object_file_data: vec![],
is_package: false,
link_libraries: vec![],
is_stdlib: false,
builtin_default: false,
required: false,
variant: None,
license: None,
}
.into()
);
assert_eq!(
resources[1],
PythonExtensionModule {
name: "bar".to_string(),
init_fn: Some("PyInit_bar".to_string()),
extension_file_suffix: ".so".to_string(),
shared_library: Some(FileData::Path(so_path)),
object_file_data: vec![],
is_package: false,
link_libraries: vec![],
is_stdlib: false,
builtin_default: false,
required: false,
variant: None,
license: None,
}
.into(),
);
assert_eq!(
resources[2],
PythonExtensionModule {
name: "foo".to_string(),
init_fn: Some("PyInit_foo".to_string()),
extension_file_suffix: ".pyd".to_string(),
shared_library: Some(FileData::Path(pyd_path)),
object_file_data: vec![],
is_package: false,
link_libraries: vec![],
is_stdlib: false,
builtin_default: false,
required: false,
variant: None,
license: None,
}
.into(),
);
assert_eq!(
resources[3],
PythonExtensionModule {
name: "markupsafe._speedups".to_string(),
init_fn: Some("PyInit__speedups".to_string()),
extension_file_suffix: ".cpython-37m-x86_64-linux-gnu.so".to_string(),
shared_library: Some(FileData::Path(markupsafe_speedups_path)),
object_file_data: vec![],
is_package: false,
link_libraries: vec![],
is_stdlib: false,
builtin_default: false,
required: false,
variant: None,
license: None,
}
.into(),
);
assert_eq!(
resources[4],
PythonExtensionModule {
name: "zstd".to_string(),
init_fn: Some("PyInit_zstd".to_string()),
extension_file_suffix: ".cpython-37m-x86_64-linux-gnu.so".to_string(),
shared_library: Some(FileData::Path(zstd_path)),
object_file_data: vec![],
is_package: false,
link_libraries: vec![],
is_stdlib: false,
builtin_default: false,
required: false,
variant: None,
license: None,
}
.into(),
);
Ok(())
}
#[test]
fn test_egg_file() -> Result<()> {
let td = tempfile::Builder::new()
.prefix("python-packaging-test")
.tempdir()?;
let tp = td.path();
create_dir_all(tp)?;
let egg_path = tp.join("foo-1.0-py3.7.egg");
write(&egg_path, "")?;
let resources =
PythonResourceIterator::new(tp, DEFAULT_CACHE_TAG, &DEFAULT_SUFFIXES, false, true)?
.collect::<Result<Vec<_>>>()?;
assert_eq!(resources.len(), 1);
assert_eq!(
resources[0],
PythonEggFile {
data: FileData::Path(egg_path)
}
.into()
);
Ok(())
}
#[test]
fn test_egg_dir() -> Result<()> {
let td = tempfile::Builder::new()
.prefix("python-packaging-test")
.tempdir()?;
let tp = td.path();
create_dir_all(tp)?;
let egg_path = tp.join("site-packages").join("foo-1.0-py3.7.egg");
let egg_info_path = egg_path.join("EGG-INFO");
let package_path = egg_path.join("foo");
create_dir_all(&egg_info_path)?;
create_dir_all(&package_path)?;
write(egg_info_path.join("PKG-INFO"), "")?;
write(package_path.join("__init__.py"), "")?;
write(package_path.join("bar.py"), "")?;
let resources =
PythonResourceIterator::new(tp, DEFAULT_CACHE_TAG, &DEFAULT_SUFFIXES, false, true)?
.collect::<Result<Vec<_>>>()?;
assert_eq!(resources.len(), 2);
assert_eq!(
resources[0],
PythonModuleSource {
name: "foo".to_string(),
source: FileData::Path(package_path.join("__init__.py")),
is_package: true,
cache_tag: DEFAULT_CACHE_TAG.to_string(),
is_stdlib: false,
is_test: false,
}
.into()
);
assert_eq!(
resources[1],
PythonModuleSource {
name: "foo.bar".to_string(),
source: FileData::Path(package_path.join("bar.py")),
is_package: false,
cache_tag: DEFAULT_CACHE_TAG.to_string(),
is_stdlib: false,
is_test: false,
}
.into()
);
Ok(())
}
#[test]
fn test_pth_file() -> Result<()> {
let td = tempfile::Builder::new()
.prefix("python-packaging-test")
.tempdir()?;
let tp = td.path();
create_dir_all(tp)?;
let pth_path = tp.join("foo.pth");
write(&pth_path, "")?;
let resources =
PythonResourceIterator::new(tp, DEFAULT_CACHE_TAG, &DEFAULT_SUFFIXES, false, true)?
.collect::<Result<Vec<_>>>()?;
assert_eq!(resources.len(), 1);
assert_eq!(
resources[0],
PythonPathExtension {
data: FileData::Path(pth_path)
}
.into()
);
Ok(())
}
#[test]
fn test_root_resource_file() -> Result<()> {
let td = tempfile::Builder::new()
.prefix("python-packaging-test")
.tempdir()?;
let tp = td.path();
let resource_path = tp.join("resource.txt");
write(resource_path, "content")?;
assert!(PythonResourceIterator::new(
tp,
DEFAULT_CACHE_TAG,
&DEFAULT_SUFFIXES,
false,
true
)?
.next()
.is_none());
Ok(())
}
#[test]
fn test_relative_resource_no_package() -> Result<()> {
let td = tempfile::Builder::new()
.prefix("python-packaging-test")
.tempdir()?;
let tp = td.path();
write(tp.join("foo.py"), "")?;
let resource_dir = tp.join("resources");
create_dir_all(&resource_dir)?;
let resource_path = resource_dir.join("resource.txt");
write(resource_path, "content")?;
let resources =
PythonResourceIterator::new(tp, DEFAULT_CACHE_TAG, &DEFAULT_SUFFIXES, false, true)?
.collect::<Result<Vec<_>>>()?;
assert_eq!(resources.len(), 1);
assert_eq!(
resources[0],
PythonModuleSource {
name: "foo".to_string(),
source: FileData::Path(tp.join("foo.py")),
is_package: false,
cache_tag: DEFAULT_CACHE_TAG.to_string(),
is_stdlib: false,
is_test: false,
}
.into()
);
Ok(())
}
#[test]
fn test_relative_package_resource() -> Result<()> {
let td = tempfile::Builder::new()
.prefix("python-packaging-test")
.tempdir()?;
let tp = td.path();
let package_dir = tp.join("foo");
create_dir_all(&package_dir)?;
let module_path = package_dir.join("__init__.py");
write(&module_path, "")?;
let resource_path = package_dir.join("resource.txt");
write(&resource_path, "content")?;
let resources =
PythonResourceIterator::new(tp, DEFAULT_CACHE_TAG, &DEFAULT_SUFFIXES, false, true)?
.collect::<Result<Vec<_>>>()?;
assert_eq!(resources.len(), 2);
assert_eq!(
resources[0],
PythonModuleSource {
name: "foo".to_string(),
source: FileData::Path(module_path),
is_package: true,
cache_tag: DEFAULT_CACHE_TAG.to_string(),
is_stdlib: false,
is_test: false,
}
.into()
);
assert_eq!(
resources[1],
PythonPackageResource {
leaf_package: "foo".to_string(),
relative_name: "resource.txt".to_string(),
data: FileData::Path(resource_path),
is_stdlib: false,
is_test: false,
}
.into()
);
Ok(())
}
#[test]
fn test_subdirectory_resource() -> Result<()> {
let td = tempfile::Builder::new()
.prefix("python-packaging-test")
.tempdir()?;
let tp = td.path();
let package_dir = tp.join("foo");
let subdir = package_dir.join("resources");
create_dir_all(&subdir)?;
let module_path = package_dir.join("__init__.py");
write(&module_path, "")?;
let resource_path = subdir.join("resource.txt");
write(&resource_path, "content")?;
let resources =
PythonResourceIterator::new(tp, DEFAULT_CACHE_TAG, &DEFAULT_SUFFIXES, false, true)?
.collect::<Result<Vec<_>>>()?;
assert_eq!(resources.len(), 2);
assert_eq!(
resources[0],
PythonModuleSource {
name: "foo".to_string(),
source: FileData::Path(module_path),
is_package: true,
cache_tag: DEFAULT_CACHE_TAG.to_string(),
is_stdlib: false,
is_test: false,
}
.into(),
);
assert_eq!(
resources[1],
PythonPackageResource {
leaf_package: "foo".to_string(),
relative_name: "resources/resource.txt".to_string(),
data: FileData::Path(resource_path),
is_stdlib: false,
is_test: false,
}
.into()
);
Ok(())
}
#[test]
fn test_distinfo_missing_metadata() -> Result<()> {
let td = tempfile::Builder::new()
.prefix("python-packaging-test")
.tempdir()?;
let tp = td.path();
let dist_path = tp.join("foo-1.2.dist-info");
create_dir_all(&dist_path)?;
let resource = dist_path.join("file.txt");
write(resource, "content")?;
let resources =
PythonResourceIterator::new(tp, DEFAULT_CACHE_TAG, &DEFAULT_SUFFIXES, false, true)?
.collect::<Result<Vec<_>>>()?;
assert!(resources.is_empty());
Ok(())
}
#[test]
fn test_distinfo_bad_metadata() -> Result<()> {
let td = tempfile::Builder::new()
.prefix("python-packaging-test")
.tempdir()?;
let tp = td.path();
let dist_path = tp.join("foo-1.2.dist-info");
create_dir_all(&dist_path)?;
let metadata = dist_path.join("METADATA");
write(metadata, "bad content")?;
let resource = dist_path.join("file.txt");
write(resource, "content")?;
let resources =
PythonResourceIterator::new(tp, DEFAULT_CACHE_TAG, &DEFAULT_SUFFIXES, false, true)?
.collect::<Result<Vec<_>>>()?;
assert!(resources.is_empty());
Ok(())
}
#[test]
fn test_distinfo_partial_metadata() -> Result<()> {
let td = tempfile::Builder::new()
.prefix("python-packaging-test")
.tempdir()?;
let tp = td.path();
let dist_path = tp.join("black-1.2.3.dist-info");
create_dir_all(&dist_path)?;
let metadata = dist_path.join("METADATA");
write(metadata, "Name: black\n")?;
let resource = dist_path.join("file.txt");
write(resource, "content")?;
let resources =
PythonResourceIterator::new(tp, DEFAULT_CACHE_TAG, &DEFAULT_SUFFIXES, false, true)?
.collect::<Result<Vec<_>>>()?;
assert!(resources.is_empty());
Ok(())
}
#[test]
fn test_distinfo_valid_metadata() -> Result<()> {
let td = tempfile::Builder::new()
.prefix("python-packaging-test")
.tempdir()?;
let tp = td.path();
let dist_path = tp.join("black-1.2.3.dist-info");
create_dir_all(&dist_path)?;
let metadata_path = dist_path.join("METADATA");
write(&metadata_path, "Name: black\nVersion: 1.2.3\n")?;
let resource_path = dist_path.join("file.txt");
write(&resource_path, "content")?;
let subdir = dist_path.join("subdir");
create_dir_all(&subdir)?;
let subdir_resource_path = subdir.join("sub.txt");
write(&subdir_resource_path, "content")?;
let resources =
PythonResourceIterator::new(tp, DEFAULT_CACHE_TAG, &DEFAULT_SUFFIXES, false, true)?
.collect::<Result<Vec<_>>>()?;
assert_eq!(resources.len(), 3);
assert_eq!(
resources[0],
PythonPackageDistributionResource {
location: PythonPackageDistributionResourceFlavor::DistInfo,
package: "black".to_string(),
version: "1.2.3".to_string(),
name: "METADATA".to_string(),
data: FileData::Path(metadata_path),
}
.into()
);
assert_eq!(
resources[1],
PythonPackageDistributionResource {
location: PythonPackageDistributionResourceFlavor::DistInfo,
package: "black".to_string(),
version: "1.2.3".to_string(),
name: "file.txt".to_string(),
data: FileData::Path(resource_path),
}
.into()
);
assert_eq!(
resources[2],
PythonPackageDistributionResource {
location: PythonPackageDistributionResourceFlavor::DistInfo,
package: "black".to_string(),
version: "1.2.3".to_string(),
name: "subdir/sub.txt".to_string(),
data: FileData::Path(subdir_resource_path),
}
.into()
);
Ok(())
}
#[test]
fn distinfo_package_name_normalization() -> Result<()> {
let td = tempfile::Builder::new()
.prefix("python-packaging-test")
.tempdir()?;
let tp = td.path();
let dist_path = tp.join("foo_bar-1.0.dist-info");
create_dir_all(&dist_path)?;
let metadata_path = dist_path.join("METADATA");
write(&metadata_path, "Name: Foo-BAR\nVersion: 1.0\n")?;
let resource_path = dist_path.join("resource.txt");
write(&resource_path, "content")?;
let resources =
PythonResourceIterator::new(tp, DEFAULT_CACHE_TAG, &DEFAULT_SUFFIXES, false, true)?
.collect::<Result<Vec<_>>>()?;
assert_eq!(resources.len(), 2);
assert_eq!(
resources[0],
PythonPackageDistributionResource {
location: PythonPackageDistributionResourceFlavor::DistInfo,
package: "Foo-BAR".into(),
version: "1.0".into(),
name: "METADATA".into(),
data: FileData::Path(metadata_path),
}
.into()
);
assert_eq!(
resources[1],
PythonPackageDistributionResource {
location: PythonPackageDistributionResourceFlavor::DistInfo,
package: "Foo-BAR".into(),
version: "1.0".into(),
name: "resource.txt".into(),
data: FileData::Path(resource_path),
}
.into()
);
Ok(())
}
#[test]
fn test_egginfo_valid_metadata() -> Result<()> {
let td = tempfile::Builder::new()
.prefix("python-packaging-test")
.tempdir()?;
let tp = td.path();
let egg_path = tp.join("black-1.2.3.egg-info");
create_dir_all(&egg_path)?;
let metadata_path = egg_path.join("PKG-INFO");
write(&metadata_path, "Name: black\nVersion: 1.2.3\n")?;
let resource_path = egg_path.join("file.txt");
write(&resource_path, "content")?;
let subdir = egg_path.join("subdir");
create_dir_all(&subdir)?;
let subdir_resource_path = subdir.join("sub.txt");
write(&subdir_resource_path, "content")?;
let resources =
PythonResourceIterator::new(tp, DEFAULT_CACHE_TAG, &DEFAULT_SUFFIXES, false, true)?
.collect::<Result<Vec<_>>>()?;
assert_eq!(resources.len(), 3);
assert_eq!(
resources[0],
PythonPackageDistributionResource {
location: PythonPackageDistributionResourceFlavor::EggInfo,
package: "black".to_string(),
version: "1.2.3".to_string(),
name: "PKG-INFO".to_string(),
data: FileData::Path(metadata_path),
}
.into()
);
assert_eq!(
resources[1],
PythonPackageDistributionResource {
location: PythonPackageDistributionResourceFlavor::EggInfo,
package: "black".to_string(),
version: "1.2.3".to_string(),
name: "file.txt".to_string(),
data: FileData::Path(resource_path),
}
.into()
);
assert_eq!(
resources[2],
PythonPackageDistributionResource {
location: PythonPackageDistributionResourceFlavor::EggInfo,
package: "black".to_string(),
version: "1.2.3".to_string(),
name: "subdir/sub.txt".to_string(),
data: FileData::Path(subdir_resource_path),
}
.into()
);
Ok(())
}
#[test]
fn egginfo_package_name_normalization() -> Result<()> {
let td = tempfile::Builder::new()
.prefix("python-packaging-test")
.tempdir()?;
let tp = td.path();
let dist_path = tp.join("foo_bar-1.0.egg-info");
create_dir_all(&dist_path)?;
let metadata_path = dist_path.join("PKG-INFO");
write(&metadata_path, "Name: Foo-BAR\nVersion: 1.0\n")?;
let resource_path = dist_path.join("resource.txt");
write(&resource_path, "content")?;
let resources =
PythonResourceIterator::new(tp, DEFAULT_CACHE_TAG, &DEFAULT_SUFFIXES, false, true)?
.collect::<Result<Vec<_>>>()?;
assert_eq!(resources.len(), 2);
assert_eq!(
resources[0],
PythonPackageDistributionResource {
location: PythonPackageDistributionResourceFlavor::EggInfo,
package: "Foo-BAR".into(),
version: "1.0".into(),
name: "PKG-INFO".into(),
data: FileData::Path(metadata_path),
}
.into()
);
assert_eq!(
resources[1],
PythonPackageDistributionResource {
location: PythonPackageDistributionResourceFlavor::EggInfo,
package: "Foo-BAR".into(),
version: "1.0".into(),
name: "resource.txt".into(),
data: FileData::Path(resource_path),
}
.into()
);
Ok(())
}
#[test]
fn test_memory_resources() -> Result<()> {
let inputs = vec![
File::new("foo/__init__.py", vec![0]),
File::new("foo/bar.py", FileEntry::new_from_data(vec![1], true)),
];
let resources = PythonResourceIterator::from_data_locations(
&inputs,
DEFAULT_CACHE_TAG,
&DEFAULT_SUFFIXES,
true,
true,
)?
.collect::<Result<Vec<_>>>()?;
assert_eq!(resources.len(), 4);
assert_eq!(resources[0], File::new("foo/__init__.py", vec![0]).into());
assert_eq!(
resources[1],
PythonModuleSource {
name: "foo".to_string(),
source: FileData::Memory(vec![0]),
is_package: true,
cache_tag: DEFAULT_CACHE_TAG.to_string(),
is_stdlib: false,
is_test: false,
}
.into()
);
assert_eq!(
resources[2],
File::new("foo/bar.py", FileEntry::new_from_data(vec![1], true)).into()
);
assert_eq!(
resources[3],
PythonModuleSource {
name: "foo.bar".to_string(),
source: FileData::Memory(vec![1]),
is_package: false,
cache_tag: DEFAULT_CACHE_TAG.to_string(),
is_stdlib: false,
is_test: false,
}
.into()
);
Ok(())
}
}