use {
crate::{
module_util::{is_package_from_path, PythonModuleSuffixes},
package_metadata::PythonPackageMetadata,
resource::{
BytecodeOptimizationLevel, DataLocation, PythonEggFile, PythonExtensionModule,
PythonModuleBytecode, PythonModuleSource, PythonPackageDistributionResource,
PythonPackageDistributionResourceFlavor, PythonPackageResource, PythonPathExtension,
PythonResource,
},
},
anyhow::Result,
std::{
collections::{HashMap, HashSet},
ffi::OsStr,
iter::FromIterator,
path::{Path, PathBuf},
},
};
pub fn walk_tree_files(path: &Path) -> Box<dyn Iterator<Item = walkdir::DirEntry>> {
let res = walkdir::WalkDir::new(path).sort_by(|a, b| a.file_name().cmp(b.file_name()));
let filtered = res.into_iter().filter_map(|entry| {
let entry = entry.expect("unable to get directory entry");
let path = entry.path();
if path.is_dir() {
None
} else {
Some(entry)
}
});
Box::new(filtered)
}
#[derive(Debug, PartialEq)]
struct ResourceFile {
pub full_path: PathBuf,
pub relative_path: PathBuf,
}
#[derive(Debug, PartialEq)]
enum PathItem<'a> {
PythonResource(PythonResource<'a>),
ResourceFile(ResourceFile),
}
pub struct PythonResourceIterator<'a> {
root_path: PathBuf,
cache_tag: String,
suffixes: PythonModuleSuffixes,
paths: Vec<PathBuf>,
path_content_overrides: HashMap<PathBuf, DataLocation>,
seen_packages: HashSet<String>,
resources: Vec<ResourceFile>,
_phantom: std::marker::PhantomData<&'a ()>,
}
impl<'a> PythonResourceIterator<'a> {
fn new(
path: &Path,
cache_tag: &str,
suffixes: &PythonModuleSuffixes,
) -> PythonResourceIterator<'a> {
let res = walkdir::WalkDir::new(path).sort_by(|a, b| a.file_name().cmp(b.file_name()));
let filtered = res
.into_iter()
.filter_map(|entry| {
let entry = entry.expect("unable to get directory entry");
let path = entry.path();
if path.is_dir() {
None
} else {
Some(path.to_path_buf())
}
})
.collect::<Vec<_>>();
PythonResourceIterator {
root_path: path.to_path_buf(),
cache_tag: cache_tag.to_string(),
suffixes: suffixes.clone(),
paths: filtered,
path_content_overrides: HashMap::new(),
seen_packages: HashSet::new(),
resources: Vec::new(),
_phantom: std::marker::PhantomData,
}
}
pub fn from_data_locations(
resources: &[(PathBuf, DataLocation)],
cache_tag: &str,
suffixes: &PythonModuleSuffixes,
) -> PythonResourceIterator<'a> {
PythonResourceIterator {
root_path: PathBuf::new(),
cache_tag: cache_tag.to_string(),
suffixes: suffixes.clone(),
paths: resources.iter().map(|(k, _)| k.clone()).collect::<Vec<_>>(),
path_content_overrides: HashMap::from_iter(resources.iter().cloned()),
seen_packages: HashSet::new(),
resources: Vec::new(),
_phantom: std::marker::PhantomData,
}
}
fn resolve_data_location(&self, path: &Path) -> DataLocation {
match self.path_content_overrides.get(path) {
Some(location) => location.clone(),
None => DataLocation::Path(path.to_path_buf()),
}
}
fn resolve_path(&mut self, path: &Path) -> Option<PathItem<'a>> {
let mut rel_path = path
.strip_prefix(&self.root_path)
.expect("unable to strip path prefix");
let mut rel_str = rel_path.to_str().expect("could not convert path to str");
let mut components = rel_path
.iter()
.map(|p| p.to_str().expect("unable to get path as str"))
.collect::<Vec<_>>();
let distribution_info = if components[0].ends_with(".dist-info") {
Some((
self.root_path.join(components[0]).join("METADATA"),
PythonPackageDistributionResourceFlavor::DistInfo,
))
} else if components[0].ends_with(".egg-info") {
Some((
self.root_path.join(components[0]).join("PKG-INFO"),
PythonPackageDistributionResourceFlavor::EggInfo,
))
} else {
None
};
if let Some((metadata_path, location)) = distribution_info {
let data = if let Some(location) = self.path_content_overrides.get(&metadata_path) {
if let Ok(data) = location.resolve() {
data
} else {
return None;
}
} else if let Ok(data) = std::fs::read(&metadata_path) {
data
} else {
return None;
};
let metadata = if let Ok(metadata) = PythonPackageMetadata::from_metadata(&data) {
metadata
} else {
return None;
};
let package = metadata.name()?;
let version = metadata.version()?;
let name = components[1..components.len()].join("/");
return Some(PathItem::PythonResource(
PythonPackageDistributionResource {
location,
package: package.to_string(),
version: version.to_string(),
name,
data: self.resolve_data_location(path),
}
.into(),
));
}
let in_site_packages = if components[0] == "site-packages" {
let sp_path = self.root_path.join("site-packages");
rel_path = path
.strip_prefix(sp_path)
.expect("unable to strip site-packages prefix");
rel_str = rel_path.to_str().expect("could not convert path to str");
components = rel_path
.iter()
.map(|p| p.to_str().expect("unable to get path as str"))
.collect::<Vec<_>>();
true
} else {
false
};
if (&components[0..components.len() - 1])
.iter()
.any(|p| p.ends_with(".egg"))
{
let mut egg_root_path = self.root_path.clone();
if in_site_packages {
egg_root_path = egg_root_path.join("site-packages");
}
for p in &components[0..components.len() - 1] {
egg_root_path = egg_root_path.join(p);
if p.ends_with(".egg") {
break;
}
}
rel_path = path
.strip_prefix(egg_root_path)
.expect("unable to strip egg prefix");
components = rel_path
.iter()
.map(|p| p.to_str().expect("unable to get path as str"))
.collect::<Vec<_>>();
if components[0] == "EGG-INFO" {
return None;
}
}
let file_name = rel_path.file_name().unwrap().to_string_lossy();
for ext_suffix in &self.suffixes.extension {
if file_name.ends_with(ext_suffix) {
let package_parts = &components[0..components.len() - 1];
let mut package = itertools::join(package_parts, ".");
let module_name = &file_name[0..file_name.len() - ext_suffix.len()];
let mut full_module_name: Vec<&str> = package_parts.to_vec();
if module_name != "__init__" {
full_module_name.push(module_name);
}
let full_module_name = itertools::join(full_module_name, ".");
if package.is_empty() {
package = full_module_name.clone();
}
self.seen_packages.insert(package);
let module_components = full_module_name.split('.').collect::<Vec<_>>();
let final_name = module_components[module_components.len() - 1];
let init_fn = Some(format!("PyInit_{}", final_name));
return Some(PathItem::PythonResource(
PythonExtensionModule {
name: full_module_name,
init_fn,
extension_file_suffix: ext_suffix.clone(),
shared_library: Some(self.resolve_data_location(path)),
object_file_data: vec![],
is_package: is_package_from_path(path),
link_libraries: vec![],
is_stdlib: false,
builtin_default: false,
required: false,
variant: None,
licenses: None,
license_public_domain: None,
}
.into(),
));
}
}
if self
.suffixes
.source
.iter()
.any(|ext| rel_str.ends_with(ext))
{
let package_parts = &components[0..components.len() - 1];
let mut package = itertools::join(package_parts, ".");
let module_name = rel_path
.file_stem()
.expect("unable to get file stem")
.to_str()
.expect("unable to convert path to str");
let mut full_module_name: Vec<&str> = package_parts.to_vec();
if module_name != "__init__" {
full_module_name.push(module_name);
}
let full_module_name = itertools::join(full_module_name, ".");
if package.is_empty() {
package = full_module_name.clone();
}
self.seen_packages.insert(package);
return Some(PathItem::PythonResource(
PythonModuleSource {
name: full_module_name,
source: self.resolve_data_location(path),
is_package: is_package_from_path(&path),
cache_tag: self.cache_tag.clone(),
is_stdlib: false,
is_test: false,
}
.into(),
));
}
if self
.suffixes
.bytecode
.iter()
.any(|ext| rel_str.ends_with(ext))
{
if components.len() < 2 {
return None;
}
if components[components.len() - 2] != "__pycache__" {
return None;
}
let package_parts = &components[0..components.len() - 2];
let mut package = itertools::join(package_parts, ".");
let filename = rel_path
.file_name()
.expect("unable to get file name")
.to_string_lossy()
.to_string();
let filename_parts = filename.split('.').collect::<Vec<&str>>();
if filename_parts.len() < 3 {
return None;
}
let mut remaining_filename = filename.clone();
let module_name = filename_parts[0];
remaining_filename = remaining_filename[module_name.len() + 1..].to_string();
if filename_parts[1] != self.cache_tag {
return None;
}
remaining_filename = remaining_filename[self.cache_tag.len()..].to_string();
let optimization_level = if filename_parts[2] == "opt-1" {
remaining_filename = remaining_filename[6..].to_string();
BytecodeOptimizationLevel::One
} else if filename_parts[2] == "opt-2" {
remaining_filename = remaining_filename[6..].to_string();
BytecodeOptimizationLevel::Two
} else {
BytecodeOptimizationLevel::Zero
};
if !self.suffixes.bytecode.contains(&remaining_filename) {
return None;
}
let mut full_module_name: Vec<&str> = package_parts.to_vec();
if module_name != "__init__" {
full_module_name.push(&module_name);
}
let full_module_name = itertools::join(full_module_name, ".");
if package.is_empty() {
package = full_module_name.clone();
}
self.seen_packages.insert(package);
return Some(PathItem::PythonResource(
PythonModuleBytecode::from_path(
&full_module_name,
optimization_level,
&self.cache_tag,
path,
)
.into(),
));
}
let resource = match rel_path.extension().and_then(OsStr::to_str) {
Some("egg") => PathItem::PythonResource(
PythonEggFile {
data: self.resolve_data_location(path),
}
.into(),
),
Some("pth") => PathItem::PythonResource(
PythonPathExtension {
data: self.resolve_data_location(path),
}
.into(),
),
_ => {
PathItem::ResourceFile(ResourceFile {
full_path: path.to_path_buf(),
relative_path: rel_path.to_path_buf(),
})
}
};
Some(resource)
}
}
impl<'a> Iterator for PythonResourceIterator<'a> {
type Item = Result<PythonResource<'a>>;
fn next(&mut self) -> Option<Result<PythonResource<'a>>> {
loop {
if self.paths.is_empty() {
break;
}
let path = self.paths.remove(0);
let entry = self.resolve_path(&path);
if entry.is_none() {
continue;
}
let entry = entry?;
match entry {
PathItem::ResourceFile(resource) => {
self.resources.push(resource);
}
PathItem::PythonResource(resource) => {
return Some(Ok(resource));
}
}
}
loop {
if self.resources.is_empty() {
return None;
}
let resource = self.resources.remove(0);
let basename = resource
.relative_path
.file_name()
.unwrap()
.to_string_lossy();
let (leaf_package, relative_name) =
if let Some(relative_directory) = resource.relative_path.parent() {
let mut components = relative_directory
.iter()
.map(|p| p.to_string_lossy())
.collect::<Vec<_>>();
let mut relative_components = vec![basename];
let mut package = None;
let mut relative_name = None;
while !components.is_empty() {
let candidate_package = itertools::join(&components, ".");
if self.seen_packages.contains(&candidate_package) {
package = Some(candidate_package);
relative_components.reverse();
relative_name = Some(itertools::join(&relative_components, "/"));
break;
}
let popped = components.pop().unwrap();
relative_components.push(popped);
}
(package, relative_name)
} else {
(None, None)
};
if leaf_package.is_none() {
continue;
}
let leaf_package = leaf_package.unwrap();
let relative_name = relative_name.unwrap();
return Some(Ok(PythonPackageResource {
leaf_package,
relative_name,
data: self.resolve_data_location(&resource.full_path),
is_stdlib: false,
is_test: false,
}
.into()));
}
}
}
pub fn find_python_resources<'a>(
root_path: &Path,
cache_tag: &str,
suffixes: &PythonModuleSuffixes,
) -> PythonResourceIterator<'a> {
PythonResourceIterator::new(root_path, cache_tag, suffixes)
}
#[cfg(test)]
mod tests {
use {
super::*,
lazy_static::lazy_static,
std::fs::{create_dir_all, write},
};
const DEFAULT_CACHE_TAG: &str = "cpython-37";
lazy_static! {
static ref DEFAULT_SUFFIXES: PythonModuleSuffixes = PythonModuleSuffixes {
source: vec![".py".to_string()],
bytecode: vec![".pyc".to_string()],
debug_bytecode: vec![],
optimized_bytecode: vec![],
extension: vec![],
};
}
#[test]
fn test_source_resolution() -> Result<()> {
let td = tempdir::TempDir::new("pyoxidizer-test")?;
let tp = td.path();
let acme_path = tp.join("acme");
let acme_a_path = acme_path.join("a");
let acme_bar_path = acme_path.join("bar");
create_dir_all(&acme_a_path).unwrap();
create_dir_all(&acme_bar_path).unwrap();
write(acme_path.join("__init__.py"), "")?;
write(acme_a_path.join("__init__.py"), "")?;
write(acme_bar_path.join("__init__.py"), "")?;
write(acme_a_path.join("foo.py"), "# acme.foo")?;
let resources = PythonResourceIterator::new(tp, DEFAULT_CACHE_TAG, &DEFAULT_SUFFIXES)
.collect::<Result<Vec<_>>>()?;
assert_eq!(resources.len(), 4);
assert_eq!(
resources[0],
PythonModuleSource {
name: "acme".to_string(),
source: DataLocation::Path(acme_path.join("__init__.py")),
is_package: true,
cache_tag: DEFAULT_CACHE_TAG.to_string(),
is_stdlib: false,
is_test: false,
}
.into()
);
assert_eq!(
resources[1],
PythonModuleSource {
name: "acme.a".to_string(),
source: DataLocation::Path(acme_a_path.join("__init__.py")),
is_package: true,
cache_tag: DEFAULT_CACHE_TAG.to_string(),
is_stdlib: false,
is_test: false,
}
.into()
);
assert_eq!(
resources[2],
PythonModuleSource {
name: "acme.a.foo".to_string(),
source: DataLocation::Path(acme_a_path.join("foo.py")),
is_package: false,
cache_tag: DEFAULT_CACHE_TAG.to_string(),
is_stdlib: false,
is_test: false,
}
.into()
);
assert_eq!(
resources[3],
PythonModuleSource {
name: "acme.bar".to_string(),
source: DataLocation::Path(acme_bar_path.join("__init__.py")),
is_package: true,
cache_tag: DEFAULT_CACHE_TAG.to_string(),
is_stdlib: false,
is_test: false,
}
.into()
);
Ok(())
}
#[test]
fn test_bytecode_resolution() -> Result<()> {
let td = tempdir::TempDir::new("pyoxidizer-test")?;
let tp = td.path();
let acme_path = tp.join("acme");
let acme_a_path = acme_path.join("a");
let acme_bar_path = acme_path.join("bar");
create_dir_all(&acme_a_path)?;
create_dir_all(&acme_bar_path)?;
let acme_pycache_path = acme_path.join("__pycache__");
let acme_a_pycache_path = acme_a_path.join("__pycache__");
let acme_bar_pycache_path = acme_bar_path.join("__pycache__");
create_dir_all(&acme_pycache_path)?;
create_dir_all(&acme_a_pycache_path)?;
create_dir_all(&acme_bar_pycache_path)?;
write(acme_pycache_path.join("__init__.pyc"), "")?;
write(acme_pycache_path.join("__init__.cpython-37.foo.pyc"), "")?;
write(acme_pycache_path.join("__init__.cpython-37.pyc"), "")?;
write(acme_pycache_path.join("__init__.cpython-37.opt-1.pyc"), "")?;
write(acme_pycache_path.join("__init__.cpython-37.opt-2.pyc"), "")?;
write(acme_pycache_path.join("__init__.cpython-38.pyc"), "")?;
write(acme_pycache_path.join("__init__.cpython-38.opt-1.pyc"), "")?;
write(acme_pycache_path.join("__init__.cpython-38.opt-2.pyc"), "")?;
write(acme_pycache_path.join("foo.cpython-37.pyc"), "")?;
write(acme_pycache_path.join("foo.cpython-37.opt-1.pyc"), "")?;
write(acme_pycache_path.join("foo.cpython-37.opt-2.pyc"), "")?;
write(acme_pycache_path.join("foo.cpython-38.pyc"), "")?;
write(acme_pycache_path.join("foo.cpython-38.opt-1.pyc"), "")?;
write(acme_pycache_path.join("foo.cpython-38.opt-2.pyc"), "")?;
write(acme_a_pycache_path.join("__init__.cpython-37.pyc"), "")?;
write(
acme_a_pycache_path.join("__init__.cpython-37.opt-1.pyc"),
"",
)?;
write(
acme_a_pycache_path.join("__init__.cpython-37.opt-2.pyc"),
"",
)?;
write(acme_a_pycache_path.join("__init__.cpython-38.pyc"), "")?;
write(
acme_a_pycache_path.join("__init__.cpython-38.opt-1.pyc"),
"",
)?;
write(
acme_a_pycache_path.join("__init__.cpython-38.opt-2.pyc"),
"",
)?;
write(acme_a_pycache_path.join("foo.cpython-37.pyc"), "")?;
write(acme_a_pycache_path.join("foo.cpython-37.opt-1.pyc"), "")?;
write(acme_a_pycache_path.join("foo.cpython-37.opt-2.pyc"), "")?;
write(acme_a_pycache_path.join("foo.cpython-38.pyc"), "")?;
write(acme_a_pycache_path.join("foo.cpython-38.opt-1.pyc"), "")?;
write(acme_a_pycache_path.join("foo.cpython-38.opt-2.pyc"), "")?;
write(acme_bar_pycache_path.join("__init__.cpython-37.pyc"), "")?;
write(
acme_bar_pycache_path.join("__init__.cpython-37.opt-1.pyc"),
"",
)?;
write(
acme_bar_pycache_path.join("__init__.cpython-37.opt-2.pyc"),
"",
)?;
write(acme_bar_pycache_path.join("__init__.cpython-38.pyc"), "")?;
write(
acme_bar_pycache_path.join("__init__.cpython-38.opt-1.pyc"),
"",
)?;
write(
acme_bar_pycache_path.join("__init__.cpython-38.opt-2.pyc"),
"",
)?;
write(acme_bar_pycache_path.join("foo.cpython-37.pyc"), "")?;
write(acme_bar_pycache_path.join("foo.cpython-37.opt-1.pyc"), "")?;
write(acme_bar_pycache_path.join("foo.cpython-37.opt-2.pyc"), "")?;
write(acme_bar_pycache_path.join("foo.cpython-38.pyc"), "")?;
write(acme_bar_pycache_path.join("foo.cpython-38.opt-1.pyc"), "")?;
write(acme_bar_pycache_path.join("foo.cpython-38.opt-2.pyc"), "")?;
let resources = PythonResourceIterator::new(tp, "cpython-38", &DEFAULT_SUFFIXES)
.collect::<Result<Vec<_>>>()?;
assert_eq!(resources.len(), 18);
assert_eq!(
resources[0],
PythonModuleBytecode::from_path(
"acme",
BytecodeOptimizationLevel::One,
"cpython-38",
&acme_pycache_path.join("__init__.cpython-38.opt-1.pyc")
)
.into()
);
assert_eq!(
resources[1],
PythonModuleBytecode::from_path(
"acme",
BytecodeOptimizationLevel::Two,
"cpython-38",
&acme_pycache_path.join("__init__.cpython-38.opt-2.pyc")
)
.into()
);
assert_eq!(
resources[2],
PythonModuleBytecode::from_path(
"acme",
BytecodeOptimizationLevel::Zero,
"cpython-38",
&acme_pycache_path.join("__init__.cpython-38.pyc")
)
.into()
);
assert_eq!(
resources[3],
PythonModuleBytecode::from_path(
"acme.foo",
BytecodeOptimizationLevel::One,
"cpython-38",
&acme_pycache_path.join("foo.cpython-38.opt-1.pyc")
)
.into()
);
assert_eq!(
resources[4],
PythonModuleBytecode::from_path(
"acme.foo",
BytecodeOptimizationLevel::Two,
"cpython-38",
&acme_pycache_path.join("foo.cpython-38.opt-2.pyc")
)
.into()
);
assert_eq!(
resources[5],
PythonModuleBytecode::from_path(
"acme.foo",
BytecodeOptimizationLevel::Zero,
"cpython-38",
&acme_pycache_path.join("foo.cpython-38.pyc")
)
.into()
);
assert_eq!(
resources[6],
PythonModuleBytecode::from_path(
"acme.a",
BytecodeOptimizationLevel::One,
"cpython-38",
&acme_a_pycache_path.join("__init__.cpython-38.opt-1.pyc")
)
.into()
);
assert_eq!(
resources[7],
PythonModuleBytecode::from_path(
"acme.a",
BytecodeOptimizationLevel::Two,
"cpython-38",
&acme_a_pycache_path.join("__init__.cpython-38.opt-2.pyc")
)
.into()
);
assert_eq!(
resources[8],
PythonModuleBytecode::from_path(
"acme.a",
BytecodeOptimizationLevel::Zero,
"cpython-38",
&acme_a_pycache_path.join("__init__.cpython-38.pyc")
)
.into()
);
assert_eq!(
resources[9],
PythonModuleBytecode::from_path(
"acme.a.foo",
BytecodeOptimizationLevel::One,
"cpython-38",
&acme_a_pycache_path.join("foo.cpython-38.opt-1.pyc")
)
.into()
);
assert_eq!(
resources[10],
PythonModuleBytecode::from_path(
"acme.a.foo",
BytecodeOptimizationLevel::Two,
"cpython-38",
&acme_a_pycache_path.join("foo.cpython-38.opt-2.pyc")
)
.into()
);
assert_eq!(
resources[11],
PythonModuleBytecode::from_path(
"acme.a.foo",
BytecodeOptimizationLevel::Zero,
"cpython-38",
&acme_a_pycache_path.join("foo.cpython-38.pyc")
)
.into()
);
assert_eq!(
resources[12],
PythonModuleBytecode::from_path(
"acme.bar",
BytecodeOptimizationLevel::One,
"cpython-38",
&acme_bar_pycache_path.join("__init__.cpython-38.opt-1.pyc")
)
.into()
);
assert_eq!(
resources[13],
PythonModuleBytecode::from_path(
"acme.bar",
BytecodeOptimizationLevel::Two,
"cpython-38",
&acme_bar_pycache_path.join("__init__.cpython-38.opt-2.pyc")
)
.into()
);
assert_eq!(
resources[14],
PythonModuleBytecode::from_path(
"acme.bar",
BytecodeOptimizationLevel::Zero,
"cpython-38",
&acme_bar_pycache_path.join("__init__.cpython-38.pyc")
)
.into()
);
assert_eq!(
resources[15],
PythonModuleBytecode::from_path(
"acme.bar.foo",
BytecodeOptimizationLevel::One,
"cpython-38",
&acme_bar_pycache_path.join("foo.cpython-38.opt-1.pyc")
)
.into()
);
assert_eq!(
resources[16],
PythonModuleBytecode::from_path(
"acme.bar.foo",
BytecodeOptimizationLevel::Two,
"cpython-38",
&acme_bar_pycache_path.join("foo.cpython-38.opt-2.pyc")
)
.into()
);
assert_eq!(
resources[17],
PythonModuleBytecode::from_path(
"acme.bar.foo",
BytecodeOptimizationLevel::Zero,
"cpython-38",
&acme_bar_pycache_path.join("foo.cpython-38.pyc")
)
.into()
);
Ok(())
}
#[test]
fn test_site_packages() -> Result<()> {
let td = tempdir::TempDir::new("pyoxidizer-test")?;
let tp = td.path();
let sp_path = tp.join("site-packages");
let acme_path = sp_path.join("acme");
create_dir_all(&acme_path).unwrap();
write(acme_path.join("__init__.py"), "")?;
write(acme_path.join("bar.py"), "")?;
let resources = PythonResourceIterator::new(tp, DEFAULT_CACHE_TAG, &DEFAULT_SUFFIXES)
.collect::<Result<Vec<_>>>()?;
assert_eq!(resources.len(), 2);
assert_eq!(
resources[0],
PythonModuleSource {
name: "acme".to_string(),
source: DataLocation::Path(acme_path.join("__init__.py")),
is_package: true,
cache_tag: DEFAULT_CACHE_TAG.to_string(),
is_stdlib: false,
is_test: false,
}
.into()
);
assert_eq!(
resources[1],
PythonModuleSource {
name: "acme.bar".to_string(),
source: DataLocation::Path(acme_path.join("bar.py")),
is_package: false,
cache_tag: DEFAULT_CACHE_TAG.to_string(),
is_stdlib: false,
is_test: false,
}
.into()
);
Ok(())
}
#[test]
fn test_extension_module() -> Result<()> {
let td = tempdir::TempDir::new("pyoxidizer-test")?;
let tp = td.path();
create_dir_all(&tp.join("markupsafe"))?;
let pyd_path = tp.join("foo.pyd");
let so_path = tp.join("bar.so");
let cffi_path = tp.join("_cffi_backend.cp37-win_amd64.pyd");
let markupsafe_speedups_path = tp
.join("markupsafe")
.join("_speedups.cpython-37m-x86_64-linux-gnu.so");
let zstd_path = tp.join("zstd.cpython-37m-x86_64-linux-gnu.so");
write(&pyd_path, "")?;
write(&so_path, "")?;
write(&cffi_path, "")?;
write(&markupsafe_speedups_path, "")?;
write(&zstd_path, "")?;
let suffixes = PythonModuleSuffixes {
source: vec![],
bytecode: vec![],
debug_bytecode: vec![],
optimized_bytecode: vec![],
extension: vec![
".cp37-win_amd64.pyd".to_string(),
".cp37-win32.pyd".to_string(),
".cpython-37m-x86_64-linux-gnu.so".to_string(),
".pyd".to_string(),
".so".to_string(),
],
};
let resources =
PythonResourceIterator::new(tp, "cpython-37", &suffixes).collect::<Result<Vec<_>>>()?;
assert_eq!(resources.len(), 5);
assert_eq!(
resources[0],
PythonExtensionModule {
name: "_cffi_backend".to_string(),
init_fn: Some("PyInit__cffi_backend".to_string()),
extension_file_suffix: ".cp37-win_amd64.pyd".to_string(),
shared_library: Some(DataLocation::Path(cffi_path)),
object_file_data: vec![],
is_package: false,
link_libraries: vec![],
is_stdlib: false,
builtin_default: false,
required: false,
variant: None,
licenses: None,
license_public_domain: None,
}
.into()
);
assert_eq!(
resources[1],
PythonExtensionModule {
name: "bar".to_string(),
init_fn: Some("PyInit_bar".to_string()),
extension_file_suffix: ".so".to_string(),
shared_library: Some(DataLocation::Path(so_path)),
object_file_data: vec![],
is_package: false,
link_libraries: vec![],
is_stdlib: false,
builtin_default: false,
required: false,
variant: None,
licenses: None,
license_public_domain: None,
}
.into(),
);
assert_eq!(
resources[2],
PythonExtensionModule {
name: "foo".to_string(),
init_fn: Some("PyInit_foo".to_string()),
extension_file_suffix: ".pyd".to_string(),
shared_library: Some(DataLocation::Path(pyd_path)),
object_file_data: vec![],
is_package: false,
link_libraries: vec![],
is_stdlib: false,
builtin_default: false,
required: false,
variant: None,
licenses: None,
license_public_domain: None,
}
.into(),
);
assert_eq!(
resources[3],
PythonExtensionModule {
name: "markupsafe._speedups".to_string(),
init_fn: Some("PyInit__speedups".to_string()),
extension_file_suffix: ".cpython-37m-x86_64-linux-gnu.so".to_string(),
shared_library: Some(DataLocation::Path(markupsafe_speedups_path)),
object_file_data: vec![],
is_package: false,
link_libraries: vec![],
is_stdlib: false,
builtin_default: false,
required: false,
variant: None,
licenses: None,
license_public_domain: None,
}
.into(),
);
assert_eq!(
resources[4],
PythonExtensionModule {
name: "zstd".to_string(),
init_fn: Some("PyInit_zstd".to_string()),
extension_file_suffix: ".cpython-37m-x86_64-linux-gnu.so".to_string(),
shared_library: Some(DataLocation::Path(zstd_path)),
object_file_data: vec![],
is_package: false,
link_libraries: vec![],
is_stdlib: false,
builtin_default: false,
required: false,
variant: None,
licenses: None,
license_public_domain: None,
}
.into(),
);
Ok(())
}
#[test]
fn test_egg_file() -> Result<()> {
let td = tempdir::TempDir::new("pyoxidizer-test")?;
let tp = td.path();
create_dir_all(&tp)?;
let egg_path = tp.join("foo-1.0-py3.7.egg");
write(&egg_path, "")?;
let resources = PythonResourceIterator::new(tp, DEFAULT_CACHE_TAG, &DEFAULT_SUFFIXES)
.collect::<Result<Vec<_>>>()?;
assert_eq!(resources.len(), 1);
assert_eq!(
resources[0],
PythonEggFile {
data: DataLocation::Path(egg_path)
}
.into()
);
Ok(())
}
#[test]
fn test_egg_dir() -> Result<()> {
let td = tempdir::TempDir::new("pyoxidizer-test")?;
let tp = td.path();
create_dir_all(&tp)?;
let egg_path = tp.join("site-packages").join("foo-1.0-py3.7.egg");
let egg_info_path = egg_path.join("EGG-INFO");
let package_path = egg_path.join("foo");
create_dir_all(&egg_info_path)?;
create_dir_all(&package_path)?;
write(egg_info_path.join("PKG-INFO"), "")?;
write(package_path.join("__init__.py"), "")?;
write(package_path.join("bar.py"), "")?;
let resources = PythonResourceIterator::new(tp, DEFAULT_CACHE_TAG, &DEFAULT_SUFFIXES)
.collect::<Result<Vec<_>>>()?;
assert_eq!(resources.len(), 2);
assert_eq!(
resources[0],
PythonModuleSource {
name: "foo".to_string(),
source: DataLocation::Path(package_path.join("__init__.py")),
is_package: true,
cache_tag: DEFAULT_CACHE_TAG.to_string(),
is_stdlib: false,
is_test: false,
}
.into()
);
assert_eq!(
resources[1],
PythonModuleSource {
name: "foo.bar".to_string(),
source: DataLocation::Path(package_path.join("bar.py")),
is_package: false,
cache_tag: DEFAULT_CACHE_TAG.to_string(),
is_stdlib: false,
is_test: false,
}
.into()
);
Ok(())
}
#[test]
fn test_pth_file() -> Result<()> {
let td = tempdir::TempDir::new("pyoxidizer-test")?;
let tp = td.path();
create_dir_all(&tp)?;
let pth_path = tp.join("foo.pth");
write(&pth_path, "")?;
let resources = PythonResourceIterator::new(tp, DEFAULT_CACHE_TAG, &DEFAULT_SUFFIXES)
.collect::<Result<Vec<_>>>()?;
assert_eq!(resources.len(), 1);
assert_eq!(
resources[0],
PythonPathExtension {
data: DataLocation::Path(pth_path)
}
.into()
);
Ok(())
}
#[test]
fn test_root_resource_file() -> Result<()> {
let td = tempdir::TempDir::new("pyoxidizer-test")?;
let tp = td.path();
let resource_path = tp.join("resource.txt");
write(&resource_path, "content")?;
let resources = PythonResourceIterator::new(tp, DEFAULT_CACHE_TAG, &DEFAULT_SUFFIXES)
.collect::<Vec<_>>();
assert!(resources.is_empty());
Ok(())
}
#[test]
fn test_relative_resource_no_package() -> Result<()> {
let td = tempdir::TempDir::new("pyoxidizer-test")?;
let tp = td.path();
write(&tp.join("foo.py"), "")?;
let resource_dir = tp.join("resources");
create_dir_all(&resource_dir)?;
let resource_path = resource_dir.join("resource.txt");
write(&resource_path, "content")?;
let resources = PythonResourceIterator::new(tp, DEFAULT_CACHE_TAG, &DEFAULT_SUFFIXES)
.collect::<Result<Vec<_>>>()?;
assert_eq!(resources.len(), 1);
assert_eq!(
resources[0],
PythonModuleSource {
name: "foo".to_string(),
source: DataLocation::Path(tp.join("foo.py")),
is_package: false,
cache_tag: DEFAULT_CACHE_TAG.to_string(),
is_stdlib: false,
is_test: false,
}
.into()
);
Ok(())
}
#[test]
fn test_relative_package_resource() -> Result<()> {
let td = tempdir::TempDir::new("pyoxidizer-test")?;
let tp = td.path();
let package_dir = tp.join("foo");
create_dir_all(&package_dir)?;
let module_path = package_dir.join("__init__.py");
write(&module_path, "")?;
let resource_path = package_dir.join("resource.txt");
write(&resource_path, "content")?;
let resources = PythonResourceIterator::new(tp, DEFAULT_CACHE_TAG, &DEFAULT_SUFFIXES)
.collect::<Result<Vec<_>>>()?;
assert_eq!(resources.len(), 2);
assert_eq!(
resources[0],
PythonModuleSource {
name: "foo".to_string(),
source: DataLocation::Path(module_path),
is_package: true,
cache_tag: DEFAULT_CACHE_TAG.to_string(),
is_stdlib: false,
is_test: false,
}
.into()
);
assert_eq!(
resources[1],
PythonPackageResource {
leaf_package: "foo".to_string(),
relative_name: "resource.txt".to_string(),
data: DataLocation::Path(resource_path),
is_stdlib: false,
is_test: false,
}
.into()
);
Ok(())
}
#[test]
fn test_subdirectory_resource() -> Result<()> {
let td = tempdir::TempDir::new("pyoxidizer-test")?;
let tp = td.path();
let package_dir = tp.join("foo");
let subdir = package_dir.join("resources");
create_dir_all(&subdir)?;
let module_path = package_dir.join("__init__.py");
write(&module_path, "")?;
let resource_path = subdir.join("resource.txt");
write(&resource_path, "content")?;
let resources = PythonResourceIterator::new(tp, DEFAULT_CACHE_TAG, &DEFAULT_SUFFIXES)
.collect::<Result<Vec<_>>>()?;
assert_eq!(resources.len(), 2);
assert_eq!(
resources[0],
PythonModuleSource {
name: "foo".to_string(),
source: DataLocation::Path(module_path),
is_package: true,
cache_tag: DEFAULT_CACHE_TAG.to_string(),
is_stdlib: false,
is_test: false,
}
.into(),
);
assert_eq!(
resources[1],
PythonPackageResource {
leaf_package: "foo".to_string(),
relative_name: "resources/resource.txt".to_string(),
data: DataLocation::Path(resource_path),
is_stdlib: false,
is_test: false,
}
.into()
);
Ok(())
}
#[test]
fn test_distinfo_missing_metadata() -> Result<()> {
let td = tempdir::TempDir::new("pyoxidizer-test")?;
let tp = td.path();
let dist_path = tp.join("foo-1.2.dist-info");
create_dir_all(&dist_path)?;
let resource = dist_path.join("file.txt");
write(&resource, "content")?;
let resources = PythonResourceIterator::new(tp, DEFAULT_CACHE_TAG, &DEFAULT_SUFFIXES)
.collect::<Result<Vec<_>>>()?;
assert!(resources.is_empty());
Ok(())
}
#[test]
fn test_distinfo_bad_metadata() -> Result<()> {
let td = tempdir::TempDir::new("pyoxidizer-test")?;
let tp = td.path();
let dist_path = tp.join("foo-1.2.dist-info");
create_dir_all(&dist_path)?;
let metadata = dist_path.join("METADATA");
write(&metadata, "bad content")?;
let resource = dist_path.join("file.txt");
write(&resource, "content")?;
let resources = PythonResourceIterator::new(tp, DEFAULT_CACHE_TAG, &DEFAULT_SUFFIXES)
.collect::<Result<Vec<_>>>()?;
assert!(resources.is_empty());
Ok(())
}
#[test]
fn test_distinfo_partial_metadata() -> Result<()> {
let td = tempdir::TempDir::new("pyoxidizer-test")?;
let tp = td.path();
let dist_path = tp.join("black-1.2.3.dist-info");
create_dir_all(&dist_path)?;
let metadata = dist_path.join("METADATA");
write(&metadata, "Name: black\n")?;
let resource = dist_path.join("file.txt");
write(&resource, "content")?;
let resources = PythonResourceIterator::new(tp, DEFAULT_CACHE_TAG, &DEFAULT_SUFFIXES)
.collect::<Result<Vec<_>>>()?;
assert!(resources.is_empty());
Ok(())
}
#[test]
fn test_distinfo_valid_metadata() -> Result<()> {
let td = tempdir::TempDir::new("pyoxidizer-test")?;
let tp = td.path();
let dist_path = tp.join("black-1.2.3.dist-info");
create_dir_all(&dist_path)?;
let metadata_path = dist_path.join("METADATA");
write(&metadata_path, "Name: black\nVersion: 1.2.3\n")?;
let resource_path = dist_path.join("file.txt");
write(&resource_path, "content")?;
let subdir = dist_path.join("subdir");
create_dir_all(&subdir)?;
let subdir_resource_path = subdir.join("sub.txt");
write(&subdir_resource_path, "content")?;
let resources = PythonResourceIterator::new(tp, DEFAULT_CACHE_TAG, &DEFAULT_SUFFIXES)
.collect::<Result<Vec<_>>>()?;
assert_eq!(resources.len(), 3);
assert_eq!(
resources[0],
PythonPackageDistributionResource {
location: PythonPackageDistributionResourceFlavor::DistInfo,
package: "black".to_string(),
version: "1.2.3".to_string(),
name: "METADATA".to_string(),
data: DataLocation::Path(metadata_path),
}
.into()
);
assert_eq!(
resources[1],
PythonPackageDistributionResource {
location: PythonPackageDistributionResourceFlavor::DistInfo,
package: "black".to_string(),
version: "1.2.3".to_string(),
name: "file.txt".to_string(),
data: DataLocation::Path(resource_path),
}
.into()
);
assert_eq!(
resources[2],
PythonPackageDistributionResource {
location: PythonPackageDistributionResourceFlavor::DistInfo,
package: "black".to_string(),
version: "1.2.3".to_string(),
name: "subdir/sub.txt".to_string(),
data: DataLocation::Path(subdir_resource_path),
}
.into()
);
Ok(())
}
#[test]
fn test_egginfo_valid_metadata() -> Result<()> {
let td = tempdir::TempDir::new("pyoxidizer-test")?;
let tp = td.path();
let egg_path = tp.join("black-1.2.3.egg-info");
create_dir_all(&egg_path)?;
let metadata_path = egg_path.join("PKG-INFO");
write(&metadata_path, "Name: black\nVersion: 1.2.3\n")?;
let resource_path = egg_path.join("file.txt");
write(&resource_path, "content")?;
let subdir = egg_path.join("subdir");
create_dir_all(&subdir)?;
let subdir_resource_path = subdir.join("sub.txt");
write(&subdir_resource_path, "content")?;
let resources = PythonResourceIterator::new(tp, DEFAULT_CACHE_TAG, &DEFAULT_SUFFIXES)
.collect::<Result<Vec<_>>>()?;
assert_eq!(resources.len(), 3);
assert_eq!(
resources[0],
PythonPackageDistributionResource {
location: PythonPackageDistributionResourceFlavor::EggInfo,
package: "black".to_string(),
version: "1.2.3".to_string(),
name: "PKG-INFO".to_string(),
data: DataLocation::Path(metadata_path),
}
.into()
);
assert_eq!(
resources[1],
PythonPackageDistributionResource {
location: PythonPackageDistributionResourceFlavor::EggInfo,
package: "black".to_string(),
version: "1.2.3".to_string(),
name: "file.txt".to_string(),
data: DataLocation::Path(resource_path),
}
.into()
);
assert_eq!(
resources[2],
PythonPackageDistributionResource {
location: PythonPackageDistributionResourceFlavor::EggInfo,
package: "black".to_string(),
version: "1.2.3".to_string(),
name: "subdir/sub.txt".to_string(),
data: DataLocation::Path(subdir_resource_path),
}
.into()
);
Ok(())
}
#[test]
fn test_memory_resources() -> Result<()> {
let inputs = vec![
(
PathBuf::from("foo/__init__.py"),
DataLocation::Memory(vec![0]),
),
(PathBuf::from("foo/bar.py"), DataLocation::Memory(vec![1])),
];
let resources = PythonResourceIterator::from_data_locations(
&inputs,
DEFAULT_CACHE_TAG,
&DEFAULT_SUFFIXES,
)
.collect::<Result<Vec<_>>>()?;
assert_eq!(resources.len(), 2);
assert_eq!(
resources[0],
PythonModuleSource {
name: "foo".to_string(),
source: DataLocation::Memory(vec![0]),
is_package: true,
cache_tag: DEFAULT_CACHE_TAG.to_string(),
is_stdlib: false,
is_test: false,
}
.into()
);
assert_eq!(
resources[1],
PythonModuleSource {
name: "foo.bar".to_string(),
source: DataLocation::Memory(vec![1]),
is_package: false,
cache_tag: DEFAULT_CACHE_TAG.to_string(),
is_stdlib: false,
is_test: false,
}
.into()
);
Ok(())
}
}