use {
crate::{
environment::{canonicalize_path, Environment, RustEnvironment},
licensing::{licenses_from_cargo_manifest, log_licensing_info},
project_layout::initialize_project,
py_packaging::{
binary::{LibpythonLinkMode, PythonBinaryBuilder},
distribution::AppleSdkInfo,
embedding::{EmbeddedPythonContext, DEFAULT_PYTHON_CONFIG_FILENAME},
},
starlark::eval::{EvaluationContext, EvaluationContextBuilder},
},
anyhow::{anyhow, Context, Result},
apple_sdk::AppleSdk,
duct::cmd,
log::warn,
starlark_dialect_build_targets::ResolvedTarget,
std::{
collections::{BTreeMap, HashMap},
fs::create_dir_all,
io::{BufRead, BufReader},
path::{Path, PathBuf},
},
};
pub fn find_pyoxidizer_config_file(start_dir: &Path) -> Option<PathBuf> {
for test_dir in start_dir.ancestors() {
let candidate = test_dir.to_path_buf().join("pyoxidizer.bzl");
if candidate.exists() {
return Some(candidate);
}
}
None
}
pub fn find_pyoxidizer_config_file_env(start_dir: &Path) -> Option<PathBuf> {
if let Ok(path) = std::env::var("PYOXIDIZER_CONFIG") {
warn!(
"using PyOxidizer config file from PYOXIDIZER_CONFIG: {}",
path
);
return Some(PathBuf::from(path));
}
if let Ok(path) = std::env::var("OUT_DIR") {
warn!("looking for config file in ancestry of {}", path);
let res = find_pyoxidizer_config_file(Path::new(&path));
if res.is_some() {
return res;
}
}
find_pyoxidizer_config_file(start_dir)
}
pub struct BuildEnvironment {
pub rust_environment: RustEnvironment,
pub extra_environment_vars: BTreeMap<String, String>,
}
impl BuildEnvironment {
#[allow(clippy::too_many_arguments)]
pub fn new(
env: &Environment,
target_triple: &str,
artifacts_path: &Path,
pyo3_config_path: impl AsRef<Path>,
libpython_link_mode: LibpythonLinkMode,
apple_sdk_info: Option<&AppleSdkInfo>,
) -> Result<Self> {
let rust_environment = env
.ensure_rust_toolchain(Some(target_triple))
.context("ensuring Rust toolchain available")?;
let mut envs = BTreeMap::default();
envs.insert(
"PYOXIDIZER_ARTIFACT_DIR".to_string(),
artifacts_path.display().to_string(),
);
envs.insert("PYOXIDIZER_REUSE_ARTIFACTS".to_string(), "1".to_string());
envs.insert(
"PYO3_CONFIG_FILE".to_string(),
pyo3_config_path.as_ref().display().to_string(),
);
if target_triple.contains("-apple-") {
let sdk_info = apple_sdk_info.ok_or_else(|| {
anyhow!("targeting Apple platform but Apple SDK info not available")
})?;
let sdk = env
.resolve_apple_sdk(sdk_info)
.context("resolving Apple SDK")?;
let deployment_target_name = sdk.supported_targets.get(&sdk_info.platform).ok_or_else(|| {
anyhow!("could not find settings for target {} (this shouldn't happen)", &sdk_info.platform)
})?.deployment_target_setting_name.clone().unwrap_or_else(|| {
warn!("Apple SDK does not define deployment target name; assuming MACOSX_DEPLOYMENT_TARGET");
warn!("(If you see this message, the SDK you are attempting to use may be too old and build failures may occur.)");
"MACOSX_DEPLOYMENT_TARGET".to_string()
});
envs.insert("SDKROOT".to_string(), sdk.path().display().to_string());
if envs.get(&deployment_target_name).is_none() {
envs.insert(deployment_target_name, sdk_info.deployment_target.clone());
}
}
let mut rust_flags = vec![];
if target_triple.contains("-windows-") && libpython_link_mode == LibpythonLinkMode::Static {
rust_flags.extend(
[
"-C".to_string(),
"target-feature=+crt-static".to_string(),
"-C".to_string(),
"link-args=/FORCE:MULTIPLE".to_string(),
]
.iter()
.map(|x| x.to_string()),
);
}
if !rust_flags.is_empty() {
let extra_flags = rust_flags.join(" ");
envs.insert(
"RUSTFLAGS".to_string(),
if let Some(value) = envs.get("RUSTFLAGS") {
format!("{} {}", extra_flags, value)
} else {
extra_flags
},
);
}
envs.insert(
"RUSTC".to_string(),
format!("{}", rust_environment.rustc_exe.display()),
);
Ok(Self {
rust_environment,
extra_environment_vars: envs,
})
}
pub fn environment_variables(&self) -> HashMap<String, String> {
let mut envs = std::env::vars().collect::<HashMap<_, _>>();
for (k, v) in &self.extra_environment_vars {
envs.insert(k.clone(), v.clone());
}
envs
}
}
pub fn cargo_features(exe: &dyn PythonBinaryBuilder) -> Vec<&str> {
let mut res = vec!["build-mode-prebuilt-artifacts"];
if exe.requires_jemalloc() {
res.push("global-allocator-jemalloc");
res.push("allocator-jemalloc");
}
if exe.requires_mimalloc() {
res.push("global-allocator-mimalloc");
res.push("allocator-mimalloc");
}
if exe.requires_snmalloc() {
res.push("global-allocator-snmalloc");
res.push("allocator-snmalloc");
}
res
}
pub struct BuiltExecutable<'a> {
pub exe_path: Option<PathBuf>,
pub exe_name: String,
pub exe_data: Vec<u8>,
pub binary_data: EmbeddedPythonContext<'a>,
}
#[allow(clippy::too_many_arguments)]
pub fn build_executable_with_rust_project<'a>(
env: &Environment,
project_path: &Path,
bin_name: &str,
exe: &'a (dyn PythonBinaryBuilder + 'a),
build_path: &Path,
artifacts_path: &Path,
target_triple: &str,
opt_level: &str,
release: bool,
locked: bool,
include_self_license: bool,
) -> Result<BuiltExecutable<'a>> {
create_dir_all(artifacts_path).context("creating directory for PyOxidizer build artifacts")?;
let mut embedded_data = exe
.to_embedded_python_context(env, opt_level)
.context("obtaining embedded python context")?;
embedded_data
.write_files(artifacts_path)
.context("writing embedded python context files")?;
let build_env = BuildEnvironment::new(
env,
exe.target_triple(),
artifacts_path,
embedded_data.pyo3_config_path(artifacts_path),
exe.libpython_link_mode(),
exe.apple_sdk_info(),
)
.context("resolving build environment")?;
warn!(
"building with Rust {}",
build_env.rust_environment.rust_version.semver
);
let target_base_path = build_path.join("target");
let target_triple_base_path =
target_base_path
.join(target_triple)
.join(if release { "release" } else { "debug" });
let mut args = vec!["build", "--target", target_triple];
let target_dir = target_base_path.display().to_string();
args.push("--target-dir");
args.push(&target_dir);
args.push("--bin");
args.push(bin_name);
if locked {
args.push("--locked");
}
if release {
args.push("--release");
}
args.push("--no-default-features");
let features = cargo_features(exe).join(" ");
if !features.is_empty() {
args.push("--features");
args.push(&features);
}
let mut log_args = vec![];
for (k, v) in &build_env.extra_environment_vars {
log_args.push(format!("{}={}", k, v));
}
log_args.push(build_env.rust_environment.cargo_exe.display().to_string());
log_args.extend(args.iter().map(|x| x.to_string()));
warn!(
"build command: {}",
shlex::join(log_args.iter().map(|x| x.as_str()))
);
let command = cmd(&build_env.rust_environment.cargo_exe, &args)
.dir(project_path)
.full_env(build_env.environment_variables())
.stderr_to_stdout()
.unchecked()
.reader()
.context("invoking cargo command")?;
{
let reader = BufReader::new(&command);
for line in reader.lines() {
warn!("{}", line.context("reading cargo output")?);
}
}
let output = command
.try_wait()
.context("waiting on cargo process")?
.ok_or_else(|| anyhow!("unable to wait on command"))?;
if !output.status.success() {
return Err(anyhow!("cargo build failed"));
}
let exe_name = if target_triple.contains("pc-windows") {
format!("{}.exe", bin_name)
} else {
bin_name.to_string()
};
let exe_path = target_triple_base_path.join(&exe_name);
if !exe_path.exists() {
return Err(anyhow!("{} does not exist", exe_path.display()));
}
let exe_data =
std::fs::read(&exe_path).with_context(|| format!("reading {}", exe_path.display()))?;
let exe_name = exe_path.file_name().unwrap().to_string_lossy().to_string();
for component in licenses_from_cargo_manifest(
project_path.join("Cargo.toml"),
false,
cargo_features(exe),
Some(target_triple),
&build_env.rust_environment,
include_self_license,
)?
.into_components()
{
embedded_data.add_licensed_component(component)?;
}
log_licensing_info(embedded_data.licensing());
Ok(BuiltExecutable {
exe_path: Some(exe_path),
exe_name,
exe_data,
binary_data: embedded_data,
})
}
pub fn build_python_executable<'a>(
env: &Environment,
bin_name: &str,
exe: &'a (dyn PythonBinaryBuilder + 'a),
target_triple: &str,
opt_level: &str,
release: bool,
) -> Result<BuiltExecutable<'a>> {
let cargo_exe = env
.ensure_rust_toolchain(Some(target_triple))
.context("resolving Rust toolchain")?
.cargo_exe;
let temp_dir = env.temporary_directory("pyoxidizer")?;
let project_path = temp_dir.path().join(bin_name);
let build_path = temp_dir.path().join("build");
let artifacts_path = temp_dir.path().join("artifacts");
initialize_project(
&env.pyoxidizer_source,
&project_path,
&cargo_exe,
None,
&[],
exe.windows_subsystem(),
)
.context("initializing project")?;
let mut build = build_executable_with_rust_project(
env,
&project_path,
bin_name,
exe,
&build_path,
&artifacts_path,
target_triple,
opt_level,
release,
true,
false,
)
.context("building executable with Rust project")?;
build.exe_path = None;
temp_dir.close().context("closing temporary directory")?;
Ok(build)
}
#[allow(clippy::too_many_arguments)]
pub fn build_pyembed_artifacts(
env: &Environment,
config_path: &Path,
artifacts_path: &Path,
resolve_target: Option<&str>,
extra_vars: HashMap<String, Option<String>>,
target_triple: &str,
release: bool,
verbose: bool,
) -> Result<()> {
create_dir_all(artifacts_path)?;
let artifacts_path = canonicalize_path(artifacts_path)?;
if artifacts_current(config_path, &artifacts_path) {
return Ok(());
}
let mut context: EvaluationContext =
EvaluationContextBuilder::new(env, config_path, target_triple.to_string())
.extra_vars(extra_vars)
.release(release)
.verbose(verbose)
.resolve_target_optional(resolve_target)
.build_script_mode(true)
.try_into()?;
context.evaluate_file(config_path)?;
for target in context.targets_to_resolve()? {
let resolved: ResolvedTarget = context.build_resolved_target(&target)?;
let default_python_config = resolved.output_path.join(DEFAULT_PYTHON_CONFIG_FILENAME);
if !default_python_config.exists() {
continue;
}
for p in std::fs::read_dir(&resolved.output_path).context(format!(
"reading directory {}",
&resolved.output_path.display()
))? {
let p = p?;
let dest_path = artifacts_path.join(p.file_name());
std::fs::copy(&p.path(), &dest_path).context(format!(
"copying {} to {}",
p.path().display(),
dest_path.display()
))?;
}
return Ok(());
}
Err(anyhow!(
"unable to find generated {}; did you specify the correct target to resolve?",
DEFAULT_PYTHON_CONFIG_FILENAME
))
}
pub fn run_from_build(
env: &Environment,
build_script: &str,
resolve_target: Option<&str>,
extra_vars: HashMap<String, Option<String>>,
) -> Result<()> {
println!("cargo:rerun-if-changed={}", build_script);
println!("cargo:rerun-if-env-changed=PYOXIDIZER_CONFIG");
let target = std::env::var("TARGET").context("TARGET")?;
let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").context("CARGO_MANIFEST_DIR")?;
let profile = std::env::var("PROFILE").context("PROFILE")?;
let config_path = match find_pyoxidizer_config_file_env(&PathBuf::from(manifest_dir)) {
Some(v) => v,
None => panic!("Could not find PyOxidizer config file"),
};
if !config_path.exists() {
panic!("PyOxidizer config file does not exist");
}
println!("cargo:rerun-if-changed={}", config_path.display());
let dest_dir = match std::env::var("PYOXIDIZER_ARTIFACT_DIR") {
Ok(ref v) => PathBuf::from(v),
Err(_) => PathBuf::from(std::env::var("OUT_DIR").context("OUT_DIR")?),
};
build_pyembed_artifacts(
env,
&config_path,
&dest_dir,
resolve_target,
extra_vars,
&target,
profile == "release",
false,
)?;
let default_python_config_path = dest_dir.join(DEFAULT_PYTHON_CONFIG_FILENAME);
println!(
"cargo:rustc-env=DEFAULT_PYTHON_CONFIG_RS={}",
default_python_config_path.display()
);
Ok(())
}
fn dependency_current(path: &Path, built_time: std::time::SystemTime) -> bool {
match path.metadata() {
Ok(md) => match md.modified() {
Ok(t) => {
if t > built_time {
warn!("building artifacts because {} changed", path.display());
false
} else {
true
}
}
Err(_) => {
warn!("error resolving mtime of {}", path.display());
false
}
},
Err(_) => {
warn!("error resolving metadata of {}", path.display());
false
}
}
}
fn artifacts_current(config_path: &Path, artifacts_path: &Path) -> bool {
let python_config_path = artifacts_path.join(DEFAULT_PYTHON_CONFIG_FILENAME);
if !python_config_path.exists() {
warn!("no existing PyOxidizer artifacts found");
return false;
}
let built_time = match python_config_path.metadata() {
Ok(md) => match md.modified() {
Ok(t) => t,
Err(_) => {
warn!(
"error determining mtime of {}",
python_config_path.display()
);
return false;
}
},
Err(_) => {
warn!(
"error resolving metadata of {}",
python_config_path.display()
);
return false;
}
};
let current_exe = std::env::current_exe().expect("unable to determine current exe");
if !dependency_current(¤t_exe, built_time) {
return false;
}
if !dependency_current(config_path, built_time) {
return false;
}
true
}
#[cfg(test)]
mod tests {
use {
super::*,
crate::{
environment::default_target_triple,
py_packaging::standalone_builder::tests::StandalonePythonExecutableBuilderOptions,
testutil::*,
},
python_packaging::interpreter::MemoryAllocatorBackend,
};
#[cfg(target_env = "msvc")]
use crate::py_packaging::distribution::DistributionFlavor;
#[test]
fn test_empty_project() -> Result<()> {
let env = get_env()?;
let options = StandalonePythonExecutableBuilderOptions::default();
let pre_built = options.new_builder()?;
build_python_executable(
&env,
"myapp",
pre_built.as_ref(),
default_target_triple(),
"0",
false,
)?;
Ok(())
}
#[cfg(not(all(target_os = "macos", target_arch = "aarch64")))]
#[test]
fn test_empty_project_python_38() -> Result<()> {
let env = get_env()?;
let options = StandalonePythonExecutableBuilderOptions {
distribution_version: Some("3.8".to_string()),
..Default::default()
};
let pre_built = options.new_builder()?;
build_python_executable(
&env,
"myapp",
pre_built.as_ref(),
default_target_triple(),
"0",
false,
)?;
Ok(())
}
#[test]
fn test_empty_project_python_310() -> Result<()> {
let env = get_env()?;
let options = StandalonePythonExecutableBuilderOptions {
distribution_version: Some("3.10".to_string()),
..Default::default()
};
let pre_built = options.new_builder()?;
build_python_executable(
&env,
"myapp",
pre_built.as_ref(),
default_target_triple(),
"0",
false,
)?;
Ok(())
}
#[test]
fn test_empty_project_system_rust() -> Result<()> {
let mut env = get_env()?;
env.unmanage_rust()?;
let options = StandalonePythonExecutableBuilderOptions::default();
let pre_built = options.new_builder()?;
build_python_executable(
&env,
"myapp",
pre_built.as_ref(),
default_target_triple(),
"0",
false,
)?;
Ok(())
}
#[test]
#[cfg(target_env = "msvc")]
fn test_empty_project_standalone_static() -> Result<()> {
let env = get_env()?;
let options = StandalonePythonExecutableBuilderOptions {
distribution_flavor: DistributionFlavor::StandaloneStatic,
..Default::default()
};
let pre_built = options.new_builder()?;
build_python_executable(
&env,
"myapp",
pre_built.as_ref(),
default_target_triple(),
"0",
false,
)?;
Ok(())
}
#[test]
#[cfg(target_env = "msvc")]
fn test_empty_project_standalone_static_38() -> Result<()> {
let env = get_env()?;
let options = StandalonePythonExecutableBuilderOptions {
distribution_version: Some("3.8".to_string()),
distribution_flavor: DistributionFlavor::StandaloneStatic,
..Default::default()
};
let pre_built = options.new_builder()?;
build_python_executable(
&env,
"myapp",
pre_built.as_ref(),
default_target_triple(),
"0",
false,
)?;
Ok(())
}
#[test]
#[cfg(target_env = "msvc")]
fn test_empty_project_standalone_static_310() -> Result<()> {
let env = get_env()?;
let options = StandalonePythonExecutableBuilderOptions {
distribution_version: Some("3.10".to_string()),
distribution_flavor: DistributionFlavor::StandaloneStatic,
..Default::default()
};
let pre_built = options.new_builder()?;
build_python_executable(
&env,
"myapp",
pre_built.as_ref(),
default_target_triple(),
"0",
false,
)?;
Ok(())
}
#[test]
#[cfg(not(target_env = "msvc"))]
fn test_allocator_jemalloc() -> Result<()> {
let env = get_env()?;
let mut options = StandalonePythonExecutableBuilderOptions::default();
options.config.allocator_backend = MemoryAllocatorBackend::Jemalloc;
let pre_built = options.new_builder()?;
build_python_executable(
&env,
"myapp",
pre_built.as_ref(),
default_target_triple(),
"0",
false,
)?;
Ok(())
}
#[test]
fn test_allocator_mimalloc() -> Result<()> {
if cfg!(windows) {
eprintln!("skipping on Windows due to build sensitivity");
return Ok(());
}
let env = get_env()?;
let mut options = StandalonePythonExecutableBuilderOptions::default();
options.config.allocator_backend = MemoryAllocatorBackend::Mimalloc;
let pre_built = options.new_builder()?;
build_python_executable(
&env,
"myapp",
pre_built.as_ref(),
default_target_triple(),
"0",
false,
)?;
Ok(())
}
#[test]
fn test_allocator_snmalloc() -> Result<()> {
if cfg!(windows) {
eprintln!("skipping on Windows due to build sensitivity");
return Ok(());
}
let env = get_env()?;
let mut options = StandalonePythonExecutableBuilderOptions::default();
options.config.allocator_backend = MemoryAllocatorBackend::Snmalloc;
let pre_built = options.new_builder()?;
build_python_executable(
&env,
"myapp",
pre_built.as_ref(),
default_target_triple(),
"0",
false,
)?;
Ok(())
}
}