1use {
6 crate::{
7 environment::{canonicalize_path, Environment, RustEnvironment},
8 licensing::{licenses_from_cargo_manifest, log_licensing_info},
9 project_layout::initialize_project,
10 py_packaging::{
11 binary::{LibpythonLinkMode, PythonBinaryBuilder},
12 distribution::AppleSdkInfo,
13 embedding::{EmbeddedPythonContext, DEFAULT_PYTHON_CONFIG_FILENAME},
14 },
15 starlark::eval::{EvaluationContext, EvaluationContextBuilder},
16 },
17 anyhow::{anyhow, Context, Result},
18 apple_sdk::AppleSdk,
19 duct::cmd,
20 log::warn,
21 starlark_dialect_build_targets::ResolvedTarget,
22 std::{
23 collections::{BTreeMap, HashMap},
24 fs::create_dir_all,
25 io::{BufRead, BufReader},
26 path::{Path, PathBuf},
27 },
28};
29
30pub fn find_pyoxidizer_config_file(start_dir: &Path) -> Option<PathBuf> {
32 for test_dir in start_dir.ancestors() {
33 let candidate = test_dir.to_path_buf().join("pyoxidizer.bzl");
34
35 if candidate.exists() {
36 return Some(candidate);
37 }
38 }
39
40 None
41}
42
43pub fn find_pyoxidizer_config_file_env(start_dir: &Path) -> Option<PathBuf> {
62 if let Ok(path) = std::env::var("PYOXIDIZER_CONFIG") {
63 warn!(
64 "using PyOxidizer config file from PYOXIDIZER_CONFIG: {}",
65 path
66 );
67 return Some(PathBuf::from(path));
68 }
69
70 if let Ok(path) = std::env::var("OUT_DIR") {
71 warn!("looking for config file in ancestry of {}", path);
72 let res = find_pyoxidizer_config_file(Path::new(&path));
73 if res.is_some() {
74 return res;
75 }
76 }
77
78 find_pyoxidizer_config_file(start_dir)
79}
80
81pub struct BuildEnvironment {
83 pub rust_environment: RustEnvironment,
85
86 pub extra_environment_vars: BTreeMap<String, String>,
88}
89
90impl BuildEnvironment {
91 #[allow(clippy::too_many_arguments)]
93 pub fn new(
94 env: &Environment,
95 target_triple: &str,
96 artifacts_path: &Path,
97 pyo3_config_path: impl AsRef<Path>,
98 libpython_link_mode: LibpythonLinkMode,
99 apple_sdk_info: Option<&AppleSdkInfo>,
100 ) -> Result<Self> {
101 let rust_environment = env
102 .ensure_rust_toolchain(Some(target_triple))
103 .context("ensuring Rust toolchain available")?;
104
105 let mut envs = BTreeMap::default();
106
107 envs.insert(
109 "PYOXIDIZER_ARTIFACT_DIR".to_string(),
110 artifacts_path.display().to_string(),
111 );
112
113 envs.insert("PYOXIDIZER_REUSE_ARTIFACTS".to_string(), "1".to_string());
115
116 envs.insert(
119 "PYO3_CONFIG_FILE".to_string(),
120 pyo3_config_path.as_ref().display().to_string(),
121 );
122
123 if target_triple.contains("-apple-") {
139 let sdk_info = apple_sdk_info.ok_or_else(|| {
140 anyhow!("targeting Apple platform but Apple SDK info not available")
141 })?;
142
143 let sdk = env
144 .resolve_apple_sdk(sdk_info)
145 .context("resolving Apple SDK")?;
146
147 let deployment_target_name = sdk.supported_targets.get(&sdk_info.platform).ok_or_else(|| {
148 anyhow!("could not find settings for target {} (this shouldn't happen)", &sdk_info.platform)
149 })?.deployment_target_setting_name.clone().unwrap_or_else(|| {
150 warn!("Apple SDK does not define deployment target name; assuming MACOSX_DEPLOYMENT_TARGET");
151 warn!("(If you see this message, the SDK you are attempting to use may be too old and build failures may occur.)");
152 "MACOSX_DEPLOYMENT_TARGET".to_string()
153 });
154
155 envs.insert("SDKROOT".to_string(), sdk.path().display().to_string());
157
158 if envs.get(&deployment_target_name).is_none() {
162 envs.insert(deployment_target_name, sdk_info.deployment_target.clone());
163 }
164 }
165
166 let mut rust_flags = vec![];
167
168 if target_triple.contains("-windows-") && libpython_link_mode == LibpythonLinkMode::Static {
178 rust_flags.extend(
179 [
180 "-C".to_string(),
181 "target-feature=+crt-static".to_string(),
182 "-C".to_string(),
183 "link-args=/FORCE:MULTIPLE".to_string(),
184 ]
185 .iter()
186 .map(|x| x.to_string()),
187 );
188 }
189
190 if !rust_flags.is_empty() {
191 let extra_flags = rust_flags.join(" ");
192
193 envs.insert(
194 "RUSTFLAGS".to_string(),
195 if let Some(value) = envs.get("RUSTFLAGS") {
196 format!("{} {}", extra_flags, value)
197 } else {
198 extra_flags
199 },
200 );
201 }
202
203 envs.insert(
206 "RUSTC".to_string(),
207 format!("{}", rust_environment.rustc_exe.display()),
208 );
209
210 Ok(Self {
211 rust_environment,
212 extra_environment_vars: envs,
213 })
214 }
215
216 pub fn environment_variables(&self) -> HashMap<String, String> {
218 let mut envs = std::env::vars().collect::<HashMap<_, _>>();
219
220 for (k, v) in &self.extra_environment_vars {
221 envs.insert(k.clone(), v.clone());
222 }
223
224 envs
225 }
226}
227
228pub fn cargo_features(exe: &dyn PythonBinaryBuilder) -> Vec<&str> {
230 let mut res = vec!["build-mode-prebuilt-artifacts"];
231
232 if exe.requires_jemalloc() {
233 res.push("global-allocator-jemalloc");
234 res.push("allocator-jemalloc");
235 }
236 if exe.requires_mimalloc() {
237 res.push("global-allocator-mimalloc");
238 res.push("allocator-mimalloc");
239 }
240 if exe.requires_snmalloc() {
241 res.push("global-allocator-snmalloc");
242 res.push("allocator-snmalloc");
243 }
244
245 res
246}
247
248pub struct BuiltExecutable<'a> {
250 pub exe_path: Option<PathBuf>,
252
253 pub exe_name: String,
255
256 pub exe_data: Vec<u8>,
258
259 pub binary_data: EmbeddedPythonContext<'a>,
261}
262
263#[allow(clippy::too_many_arguments)]
267pub fn build_executable_with_rust_project<'a>(
268 env: &Environment,
269 project_path: &Path,
270 bin_name: &str,
271 exe: &'a (dyn PythonBinaryBuilder + 'a),
272 build_path: &Path,
273 artifacts_path: &Path,
274 target_triple: &str,
275 opt_level: &str,
276 release: bool,
277 locked: bool,
278 include_self_license: bool,
279) -> Result<BuiltExecutable<'a>> {
280 create_dir_all(artifacts_path).context("creating directory for PyOxidizer build artifacts")?;
281
282 let mut embedded_data = exe
284 .to_embedded_python_context(env, opt_level)
285 .context("obtaining embedded python context")?;
286 embedded_data
287 .write_files(artifacts_path)
288 .context("writing embedded python context files")?;
289
290 let build_env = BuildEnvironment::new(
291 env,
292 exe.target_triple(),
293 artifacts_path,
294 embedded_data.pyo3_config_path(artifacts_path),
295 exe.libpython_link_mode(),
296 exe.apple_sdk_info(),
297 )
298 .context("resolving build environment")?;
299
300 warn!(
301 "building with Rust {}",
302 build_env.rust_environment.rust_version.semver
303 );
304
305 let target_base_path = build_path.join("target");
306 let target_triple_base_path =
307 target_base_path
308 .join(target_triple)
309 .join(if release { "release" } else { "debug" });
310
311 let mut args = vec!["build", "--target", target_triple];
312
313 let target_dir = target_base_path.display().to_string();
314 args.push("--target-dir");
315 args.push(&target_dir);
316
317 args.push("--bin");
318 args.push(bin_name);
319
320 if locked {
321 args.push("--locked");
322 }
323
324 if release {
325 args.push("--release");
326 }
327
328 args.push("--no-default-features");
329
330 let features = cargo_features(exe).join(" ");
331
332 if !features.is_empty() {
333 args.push("--features");
334 args.push(&features);
335 }
336
337 let mut log_args = vec![];
338
339 for (k, v) in &build_env.extra_environment_vars {
340 log_args.push(format!("{}={}", k, v));
341 }
342 log_args.push(build_env.rust_environment.cargo_exe.display().to_string());
343 log_args.extend(args.iter().map(|x| x.to_string()));
344
345 warn!(
346 "build command: {}",
347 shlex::join(log_args.iter().map(|x| x.as_str()))
348 );
349
350 let command = cmd(&build_env.rust_environment.cargo_exe, &args)
352 .dir(project_path)
353 .full_env(build_env.environment_variables())
354 .stderr_to_stdout()
355 .unchecked()
356 .reader()
357 .context("invoking cargo command")?;
358 {
359 let reader = BufReader::new(&command);
360 for line in reader.lines() {
361 warn!("{}", line.context("reading cargo output")?);
362 }
363 }
364 let output = command
365 .try_wait()
366 .context("waiting on cargo process")?
367 .ok_or_else(|| anyhow!("unable to wait on command"))?;
368 if !output.status.success() {
369 return Err(anyhow!("cargo build failed"));
370 }
371
372 let exe_name = if target_triple.contains("pc-windows") {
373 format!("{}.exe", bin_name)
374 } else {
375 bin_name.to_string()
376 };
377
378 let exe_path = target_triple_base_path.join(&exe_name);
379
380 if !exe_path.exists() {
381 return Err(anyhow!("{} does not exist", exe_path.display()));
382 }
383
384 let exe_data =
385 std::fs::read(&exe_path).with_context(|| format!("reading {}", exe_path.display()))?;
386 let exe_name = exe_path.file_name().unwrap().to_string_lossy().to_string();
387
388 for component in licenses_from_cargo_manifest(
391 project_path.join("Cargo.toml"),
392 false,
393 cargo_features(exe),
394 Some(target_triple),
395 &build_env.rust_environment,
396 include_self_license,
397 )?
398 .into_components()
399 {
400 embedded_data.add_licensed_component(component)?;
401 }
402
403 log_licensing_info(embedded_data.licensing());
405
406 Ok(BuiltExecutable {
407 exe_path: Some(exe_path),
408 exe_name,
409 exe_data,
410 binary_data: embedded_data,
411 })
412}
413
414pub fn build_python_executable<'a>(
418 env: &Environment,
419 bin_name: &str,
420 exe: &'a (dyn PythonBinaryBuilder + 'a),
421 target_triple: &str,
422 opt_level: &str,
423 release: bool,
424) -> Result<BuiltExecutable<'a>> {
425 let cargo_exe = env
426 .ensure_rust_toolchain(Some(target_triple))
427 .context("resolving Rust toolchain")?
428 .cargo_exe;
429
430 let temp_dir = env.temporary_directory("pyoxidizer")?;
431
432 let project_path = temp_dir.path().join(bin_name);
434 let build_path = temp_dir.path().join("build");
435 let artifacts_path = temp_dir.path().join("artifacts");
436
437 initialize_project(
438 &env.pyoxidizer_source,
439 &project_path,
440 &cargo_exe,
441 None,
442 &[],
443 exe.windows_subsystem(),
444 )
445 .context("initializing project")?;
446
447 let mut build = build_executable_with_rust_project(
448 env,
449 &project_path,
450 bin_name,
451 exe,
452 &build_path,
453 &artifacts_path,
454 target_triple,
455 opt_level,
456 release,
457 true,
460 false,
463 )
464 .context("building executable with Rust project")?;
465
466 build.exe_path = None;
468
469 temp_dir.close().context("closing temporary directory")?;
470
471 Ok(build)
472}
473
474#[allow(clippy::too_many_arguments)]
479pub fn build_pyembed_artifacts(
480 env: &Environment,
481 config_path: &Path,
482 artifacts_path: &Path,
483 resolve_target: Option<&str>,
484 extra_vars: HashMap<String, Option<String>>,
485 target_triple: &str,
486 release: bool,
487 verbose: bool,
488) -> Result<()> {
489 create_dir_all(artifacts_path)?;
490
491 let artifacts_path = canonicalize_path(artifacts_path)?;
492
493 if artifacts_current(config_path, &artifacts_path) {
494 return Ok(());
495 }
496
497 let mut context: EvaluationContext =
498 EvaluationContextBuilder::new(env, config_path, target_triple.to_string())
499 .extra_vars(extra_vars)
500 .release(release)
501 .verbose(verbose)
502 .resolve_target_optional(resolve_target)
503 .build_script_mode(true)
504 .try_into()?;
505
506 context.evaluate_file(config_path)?;
507
508 for target in context.targets_to_resolve()? {
510 let resolved: ResolvedTarget = context.build_resolved_target(&target)?;
511
512 let default_python_config = resolved.output_path.join(DEFAULT_PYTHON_CONFIG_FILENAME);
515 if !default_python_config.exists() {
516 continue;
517 }
518
519 for p in std::fs::read_dir(&resolved.output_path).context(format!(
520 "reading directory {}",
521 &resolved.output_path.display()
522 ))? {
523 let p = p?;
524
525 let dest_path = artifacts_path.join(p.file_name());
526 std::fs::copy(&p.path(), &dest_path).context(format!(
527 "copying {} to {}",
528 p.path().display(),
529 dest_path.display()
530 ))?;
531 }
532
533 return Ok(());
534 }
535
536 Err(anyhow!(
537 "unable to find generated {}; did you specify the correct target to resolve?",
538 DEFAULT_PYTHON_CONFIG_FILENAME
539 ))
540}
541
542pub fn run_from_build(
564 env: &Environment,
565 build_script: &str,
566 resolve_target: Option<&str>,
567 extra_vars: HashMap<String, Option<String>>,
568) -> Result<()> {
569 println!("cargo:rerun-if-changed={}", build_script);
572
573 println!("cargo:rerun-if-env-changed=PYOXIDIZER_CONFIG");
574
575 let target = std::env::var("TARGET").context("TARGET")?;
578 let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").context("CARGO_MANIFEST_DIR")?;
580 let profile = std::env::var("PROFILE").context("PROFILE")?;
581
582 let config_path = match find_pyoxidizer_config_file_env(&PathBuf::from(manifest_dir)) {
585 Some(v) => v,
586 None => panic!("Could not find PyOxidizer config file"),
587 };
588
589 if !config_path.exists() {
590 panic!("PyOxidizer config file does not exist");
591 }
592
593 println!("cargo:rerun-if-changed={}", config_path.display());
594
595 let dest_dir = match std::env::var("PYOXIDIZER_ARTIFACT_DIR") {
596 Ok(ref v) => PathBuf::from(v),
597 Err(_) => PathBuf::from(std::env::var("OUT_DIR").context("OUT_DIR")?),
598 };
599
600 build_pyembed_artifacts(
601 env,
602 &config_path,
603 &dest_dir,
604 resolve_target,
605 extra_vars,
606 &target,
607 profile == "release",
608 false,
609 )?;
610
611 let default_python_config_path = dest_dir.join(DEFAULT_PYTHON_CONFIG_FILENAME);
612 println!(
613 "cargo:rustc-env=DEFAULT_PYTHON_CONFIG_RS={}",
614 default_python_config_path.display()
615 );
616
617 Ok(())
618}
619
620fn dependency_current(path: &Path, built_time: std::time::SystemTime) -> bool {
621 match path.metadata() {
622 Ok(md) => match md.modified() {
623 Ok(t) => {
624 if t > built_time {
625 warn!("building artifacts because {} changed", path.display());
626 false
627 } else {
628 true
629 }
630 }
631 Err(_) => {
632 warn!("error resolving mtime of {}", path.display());
633 false
634 }
635 },
636 Err(_) => {
637 warn!("error resolving metadata of {}", path.display());
638 false
639 }
640 }
641}
642
643fn artifacts_current(config_path: &Path, artifacts_path: &Path) -> bool {
645 let python_config_path = artifacts_path.join(DEFAULT_PYTHON_CONFIG_FILENAME);
646
647 if !python_config_path.exists() {
648 warn!("no existing PyOxidizer artifacts found");
649 return false;
650 }
651
652 let built_time = match python_config_path.metadata() {
655 Ok(md) => match md.modified() {
656 Ok(t) => t,
657 Err(_) => {
658 warn!(
659 "error determining mtime of {}",
660 python_config_path.display()
661 );
662 return false;
663 }
664 },
665 Err(_) => {
666 warn!(
667 "error resolving metadata of {}",
668 python_config_path.display()
669 );
670 return false;
671 }
672 };
673
674 let current_exe = std::env::current_exe().expect("unable to determine current exe");
675 if !dependency_current(¤t_exe, built_time) {
676 return false;
677 }
678
679 if !dependency_current(config_path, built_time) {
680 return false;
681 }
682
683 true
685}
686
687#[cfg(test)]
688mod tests {
689 use {
690 super::*,
691 crate::{
692 environment::default_target_triple,
693 py_packaging::standalone_builder::tests::StandalonePythonExecutableBuilderOptions,
694 testutil::*,
695 },
696 python_packaging::interpreter::MemoryAllocatorBackend,
697 };
698
699 #[cfg(target_env = "msvc")]
700 use crate::py_packaging::distribution::DistributionFlavor;
701
702 #[test]
703 fn test_empty_project() -> Result<()> {
704 let env = get_env()?;
705 let options = StandalonePythonExecutableBuilderOptions::default();
706 let pre_built = options.new_builder()?;
707
708 build_python_executable(
709 &env,
710 "myapp",
711 pre_built.as_ref(),
712 default_target_triple(),
713 "0",
714 false,
715 )?;
716
717 Ok(())
718 }
719
720 #[cfg(not(all(target_os = "macos", target_arch = "aarch64")))]
722 #[test]
723 fn test_empty_project_python_38() -> Result<()> {
724 let env = get_env()?;
725 let options = StandalonePythonExecutableBuilderOptions {
726 distribution_version: Some("3.8".to_string()),
727 ..Default::default()
728 };
729 let pre_built = options.new_builder()?;
730
731 build_python_executable(
732 &env,
733 "myapp",
734 pre_built.as_ref(),
735 default_target_triple(),
736 "0",
737 false,
738 )?;
739
740 Ok(())
741 }
742
743 #[test]
744 fn test_empty_project_python_310() -> Result<()> {
745 let env = get_env()?;
746 let options = StandalonePythonExecutableBuilderOptions {
747 distribution_version: Some("3.10".to_string()),
748 ..Default::default()
749 };
750 let pre_built = options.new_builder()?;
751
752 build_python_executable(
753 &env,
754 "myapp",
755 pre_built.as_ref(),
756 default_target_triple(),
757 "0",
758 false,
759 )?;
760
761 Ok(())
762 }
763
764 #[test]
765 fn test_empty_project_system_rust() -> Result<()> {
766 let mut env = get_env()?;
767 env.unmanage_rust()?;
768 let options = StandalonePythonExecutableBuilderOptions::default();
769 let pre_built = options.new_builder()?;
770
771 build_python_executable(
772 &env,
773 "myapp",
774 pre_built.as_ref(),
775 default_target_triple(),
776 "0",
777 false,
778 )?;
779
780 Ok(())
781 }
782
783 #[test]
784 #[cfg(target_env = "msvc")]
785 fn test_empty_project_standalone_static() -> Result<()> {
786 let env = get_env()?;
787 let options = StandalonePythonExecutableBuilderOptions {
788 distribution_flavor: DistributionFlavor::StandaloneStatic,
789 ..Default::default()
790 };
791 let pre_built = options.new_builder()?;
792
793 build_python_executable(
794 &env,
795 "myapp",
796 pre_built.as_ref(),
797 default_target_triple(),
798 "0",
799 false,
800 )?;
801
802 Ok(())
803 }
804
805 #[test]
806 #[cfg(target_env = "msvc")]
807 fn test_empty_project_standalone_static_38() -> Result<()> {
808 let env = get_env()?;
809 let options = StandalonePythonExecutableBuilderOptions {
810 distribution_version: Some("3.8".to_string()),
811 distribution_flavor: DistributionFlavor::StandaloneStatic,
812 ..Default::default()
813 };
814 let pre_built = options.new_builder()?;
815
816 build_python_executable(
817 &env,
818 "myapp",
819 pre_built.as_ref(),
820 default_target_triple(),
821 "0",
822 false,
823 )?;
824
825 Ok(())
826 }
827
828 #[test]
829 #[cfg(target_env = "msvc")]
830 fn test_empty_project_standalone_static_310() -> Result<()> {
831 let env = get_env()?;
832 let options = StandalonePythonExecutableBuilderOptions {
833 distribution_version: Some("3.10".to_string()),
834 distribution_flavor: DistributionFlavor::StandaloneStatic,
835 ..Default::default()
836 };
837 let pre_built = options.new_builder()?;
838
839 build_python_executable(
840 &env,
841 "myapp",
842 pre_built.as_ref(),
843 default_target_triple(),
844 "0",
845 false,
846 )?;
847
848 Ok(())
849 }
850
851 #[test]
852 #[cfg(not(target_env = "msvc"))]
854 fn test_allocator_jemalloc() -> Result<()> {
855 let env = get_env()?;
856
857 let mut options = StandalonePythonExecutableBuilderOptions::default();
858 options.config.allocator_backend = MemoryAllocatorBackend::Jemalloc;
859
860 let pre_built = options.new_builder()?;
861
862 build_python_executable(
863 &env,
864 "myapp",
865 pre_built.as_ref(),
866 default_target_triple(),
867 "0",
868 false,
869 )?;
870
871 Ok(())
872 }
873
874 #[test]
875 fn test_allocator_mimalloc() -> Result<()> {
876 if cfg!(windows) {
878 eprintln!("skipping on Windows due to build sensitivity");
879 return Ok(());
880 }
881
882 let env = get_env()?;
883
884 let mut options = StandalonePythonExecutableBuilderOptions::default();
885 options.config.allocator_backend = MemoryAllocatorBackend::Mimalloc;
886
887 let pre_built = options.new_builder()?;
888
889 build_python_executable(
890 &env,
891 "myapp",
892 pre_built.as_ref(),
893 default_target_triple(),
894 "0",
895 false,
896 )?;
897
898 Ok(())
899 }
900
901 #[test]
902 fn test_allocator_snmalloc() -> Result<()> {
903 if cfg!(windows) {
905 eprintln!("skipping on Windows due to build sensitivity");
906 return Ok(());
907 }
908
909 let env = get_env()?;
910
911 let mut options = StandalonePythonExecutableBuilderOptions::default();
912 options.config.allocator_backend = MemoryAllocatorBackend::Snmalloc;
913
914 let pre_built = options.new_builder()?;
915
916 build_python_executable(
917 &env,
918 "myapp",
919 pre_built.as_ref(),
920 default_target_triple(),
921 "0",
922 false,
923 )?;
924
925 Ok(())
926 }
927}