1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at https://mozilla.org/MPL/2.0/.

//! Handle file layout of PyOxidizer projects.

use {
    crate::environment::{PyOxidizerSource, BUILD_GIT_COMMIT, PYOXIDIZER_VERSION},
    anyhow::{anyhow, Context, Result},
    handlebars::Handlebars,
    once_cell::sync::Lazy,
    python_packaging::filesystem_scanning::walk_tree_files,
    serde::Serialize,
    std::{
        collections::BTreeMap,
        io::Write,
        path::{Path, PathBuf},
        str::FromStr,
    },
};

static HANDLEBARS: Lazy<Handlebars<'static>> = Lazy::new(|| {
    let mut handlebars = Handlebars::new();

    handlebars
        .register_template_string(
            "application-manifest.rc",
            include_str!("templates/application-manifest.rc.hbs"),
        )
        .unwrap();
    handlebars
        .register_template_string(
            "cargo-extra.toml",
            include_str!("templates/cargo-extra.toml.hbs"),
        )
        .unwrap();
    handlebars
        .register_template_string("exe.manifest", include_str!("templates/exe.manifest.hbs"))
        .unwrap();
    handlebars
        .register_template_string("new-build.rs", include_str!("templates/new-build.rs.hbs"))
        .unwrap();
    handlebars
        .register_template_string(
            "new-cargo-config",
            include_str!("templates/new-cargo-config.hbs"),
        )
        .unwrap();
    handlebars
        .register_template_string("new-main.rs", include_str!("templates/new-main.rs.hbs"))
        .unwrap();
    handlebars
        .register_template_string(
            "new-pyoxidizer.bzl",
            include_str!("templates/new-pyoxidizer.bzl.hbs"),
        )
        .unwrap();

    handlebars
});

/// Contents of Cargo.lock file for the new Rust projects.
const NEW_PROJECT_CARGO_LOCK: &str = include_str!("new-project-cargo.lock");

/// Package dependencies of new Rust projects to be recorded in the Cargo.lock.
const NEW_PROJECT_DEPENDENCIES: &[&str] = &[
    "embed-resource",
    "jemallocator",
    "mimalloc",
    "pyembed",
    "snmalloc-rs",
];

#[derive(Serialize)]
struct PythonDistribution {
    build_target: String,
    url: String,
    sha256: String,
}

#[derive(Serialize)]
struct TemplateData {
    pyoxidizer_version: Option<String>,
    pyoxidizer_commit: Option<String>,
    pyoxidizer_local_repo_path: Option<String>,
    pyoxidizer_git_url: Option<String>,
    pyoxidizer_git_commit: Option<String>,
    pyoxidizer_git_tag: Option<String>,

    python_distributions: Vec<PythonDistribution>,
    program_name: Option<String>,
    code: Option<String>,
    pip_install_simple: Vec<String>,
}

impl TemplateData {
    fn new() -> TemplateData {
        TemplateData {
            pyoxidizer_version: None,
            pyoxidizer_commit: None,
            pyoxidizer_local_repo_path: None,
            pyoxidizer_git_url: None,
            pyoxidizer_git_commit: None,
            pyoxidizer_git_tag: None,
            python_distributions: Vec::new(),
            program_name: None,
            code: None,
            pip_install_simple: Vec::new(),
        }
    }
}

fn populate_template_data(source: &PyOxidizerSource, data: &mut TemplateData) {
    data.pyoxidizer_version = Some(PYOXIDIZER_VERSION.to_string());
    data.pyoxidizer_commit = Some(
        BUILD_GIT_COMMIT
            .clone()
            .unwrap_or_else(|| "UNKNOWN".to_string()),
    );

    match source {
        PyOxidizerSource::LocalPath { path } => {
            data.pyoxidizer_local_repo_path = Some(path.display().to_string());
        }
        PyOxidizerSource::GitUrl { url, commit, tag } => {
            data.pyoxidizer_git_url = Some(url.clone());

            if let Some(commit) = commit {
                data.pyoxidizer_git_commit = Some(commit.clone());
            }
            if let Some(tag) = tag {
                data.pyoxidizer_git_tag = Some(tag.clone());
            }
        }
    }
}

/// Find existing PyOxidizer files in a project directory.
pub fn find_pyoxidizer_files(root: &Path) -> Vec<PathBuf> {
    let mut res: Vec<PathBuf> = Vec::new();

    for f in walk_tree_files(&root) {
        let path = f.path().strip_prefix(root).expect("unable to strip prefix");
        let path_s = path.to_str().expect("unable to convert path to str");

        if path_s.contains("pyoxidizer") || path_s.contains("pyembed") {
            res.push(path.to_path_buf());
        }
    }

    res
}

/// Write a new .cargo/config file for a project path.
pub fn write_new_cargo_config(project_path: &Path) -> Result<()> {
    let cargo_path = project_path.join(".cargo");

    if !cargo_path.is_dir() {
        std::fs::create_dir(&cargo_path)?;
    }

    let data: BTreeMap<String, String> = BTreeMap::new();
    let t = HANDLEBARS.render("new-cargo-config", &data)?;

    let config_path = cargo_path.join("config");
    println!("writing {}", config_path.display());
    std::fs::write(&config_path, t)?;

    Ok(())
}

/// Write a Cargo.lock file for a project path.
///
/// The Cargo.lock content is under version control and is automatically
/// updated as part of the release automation. The file is generated in
/// `--offline` mode and the contents of this `Cargo.lock` should closely
/// resemble those in this repository's `Cargo.lock`. This helps ensure that
/// the crate versions used by generated Rust projects match those of the
/// build/commit of PyOxidizer used to generate the project.
pub fn write_new_cargo_lock(project_path: &Path, project_name: &str) -> Result<()> {
    // Add this project's entry to the lock file contents, otherwise the
    // lock file will need updating on first use.
    let mut lock_file = cargo_lock::Lockfile::from_str(NEW_PROJECT_CARGO_LOCK)?;

    let dependencies = NEW_PROJECT_DEPENDENCIES
        .iter()
        .map(|dep| cargo_lock::Dependency {
            name: cargo_lock::Name::from_str(dep)
                .expect("could not convert dependency name to Name"),
            version: lock_file
                .packages
                .iter()
                .filter_map(|package| {
                    if package.name.as_str() == *dep {
                        Some(package.version.clone())
                    } else {
                        None
                    }
                })
                .next()
                .expect("unable to find dependency in frozen Cargo lock; something is out of sync"),
            source: None,
        })
        .collect::<Vec<_>>();

    lock_file.packages.push(cargo_lock::Package {
        name: cargo_lock::Name::from_str(project_name)?,
        version: semver::Version::new(0, 1, 0),
        source: None,
        checksum: None,
        dependencies,
        replace: None,
    });

    // cargo_lock isn't smart enough to sort the packages. So do that here.
    lock_file
        .packages
        .sort_by(|a, b| a.name.as_str().cmp(&b.name.as_str()));

    let lock_path = project_path.join("Cargo.lock");
    println!("writing {}", lock_path.display());
    std::fs::write(&lock_path, &lock_file.to_string())?;

    Ok(())
}

pub fn write_new_build_rs(path: &Path, program_name: &str) -> Result<()> {
    let mut data = TemplateData::new();
    data.program_name = Some(program_name.to_string());
    let t = HANDLEBARS.render("new-build.rs", &data)?;

    println!("writing {}", path.display());
    std::fs::write(path, t)?;

    Ok(())
}

/// Write a new main.rs file that runs the embedded Python interpreter.
///
/// `windows_subsystem` is the value of the `windows_subsystem` Rust attribute.
pub fn write_new_main_rs(path: &Path, windows_subsystem: &str) -> Result<()> {
    let mut data: BTreeMap<String, String> = BTreeMap::new();
    data.insert(
        "windows_subsystem".to_string(),
        windows_subsystem.to_string(),
    );
    let t = HANDLEBARS.render("new-main.rs", &data)?;

    println!("writing {}", path.to_str().unwrap());
    let mut fh = std::fs::File::create(path)?;
    fh.write_all(t.as_bytes())?;

    Ok(())
}

/// Writes default PyOxidizer config files into a project directory.
pub fn write_new_pyoxidizer_config_file(
    source: &PyOxidizerSource,
    project_dir: &Path,
    name: &str,
    code: Option<&str>,
    pip_install: &[&str],
) -> Result<()> {
    let path = project_dir.join("pyoxidizer.bzl");

    let mut data = TemplateData::new();
    populate_template_data(source, &mut data);
    data.program_name = Some(name.to_string());

    if let Some(code) = code {
        // Replace " with \" to work around
        // https://github.com/google/starlark-rust/issues/230.
        data.code = Some(code.replace("\"", "\\\""));
    }

    data.pip_install_simple = pip_install.iter().map(|v| (*v).to_string()).collect();

    let t = HANDLEBARS.render("new-pyoxidizer.bzl", &data)?;

    println!("writing {}", path.to_str().unwrap());
    let mut fh = std::fs::File::create(path)?;
    fh.write_all(t.as_bytes())?;

    Ok(())
}

/// Write an application manifest and corresponding resource file.
///
/// This is used on Windows to allow the built executable to use long paths.
///
/// Windows 10 version 1607 and above enable long paths by default. So we
/// might be able to remove this someday. It isn't clear if you get long
/// paths support if using that version of the Windows SDK or if you have
/// to be running on a modern Windows version as well.
pub fn write_application_manifest(project_dir: &Path, program_name: &str) -> Result<()> {
    let mut data = TemplateData::new();
    data.program_name = Some(program_name.to_string());

    let manifest_path = project_dir.join(format!("{}.exe.manifest", program_name));
    let manifest_data = HANDLEBARS.render("exe.manifest", &data)?;
    println!("writing {}", manifest_path.display());
    let mut fh = std::fs::File::create(&manifest_path)?;
    fh.write_all(manifest_data.as_bytes())?;

    let rc_path = project_dir.join(format!("{}-manifest.rc", program_name));
    let rc_data = HANDLEBARS.render("application-manifest.rc", &data)?;
    println!("writing {}", rc_path.display());
    let mut fh = std::fs::File::create(&rc_path)?;
    fh.write_all(rc_data.as_bytes())?;

    Ok(())
}

/// Add PyOxidizer to an existing Rust project on the filesystem.
///
/// The target directory must not already have PyOxidizer files. This
/// will be verified during execution.
///
/// When called, various Rust source files required to embed Python
/// are created at the target directory. Instructions for finalizing the
/// configuration are also printed to stdout.
///
/// The Rust source files added to the target project are installed into
/// a sub-directory defined by ``module_name``.
pub fn add_pyoxidizer(project_dir: &Path, _suppress_help: bool) -> Result<()> {
    let existing_files = find_pyoxidizer_files(&project_dir);

    if !existing_files.is_empty() {
        return Err(anyhow!("existing PyOxidizer files found; cannot add"));
    }

    let cargo_toml = project_dir.to_path_buf().join("Cargo.toml");

    if !cargo_toml.exists() {
        return Err(anyhow!("Cargo.toml does not exist at destination"));
    }

    let cargo_toml_data = std::fs::read(cargo_toml)?;
    let manifest = cargo_toml::Manifest::from_slice(&cargo_toml_data)?;

    let _package = match &manifest.package {
        Some(package) => package,
        None => panic!("no [package]; that's weird"),
    };

    // TODO look for pyembed dependency and print message about adding it.

    Ok(())
}

/// How to define the ``pyembed`` crate dependency.
pub enum PyembedLocation {
    /// Use a specific version, installed from the crate registry.
    ///
    /// (This is how most Rust dependencies are defined.)
    Version(String),

    /// Use a local filesystem path.
    Path(PathBuf),

    /// A git repository URL and revision hash.
    Git(String, String),
}

impl PyembedLocation {
    /// Convert the location to a string holding Cargo manifest location info.
    pub fn cargo_manifest_fields(&self) -> String {
        match self {
            Self::Version(version) => format!("version = \"{}\"", version),
            Self::Path(path) => format!("path = \"{}\"", path.display()),
            Self::Git(url, commit) => format!("git = \"{}\", rev = \"{}\"", url, commit),
        }
    }
}

/// Update the Cargo.toml of a new Rust project to use pyembed.
pub fn update_new_cargo_toml(path: &Path, pyembed_location: &PyembedLocation) -> Result<()> {
    let content = std::fs::read_to_string(path)?;

    // Insert a `build = build.rs` line after the `version = *\n` line. We key off
    // version because it should always be present.
    let version_start = match content.find("version =") {
        Some(off) => off,
        None => return Err(anyhow!("could not find version line in Cargo.toml")),
    };

    let nl_off = match &content[version_start..content.len()].find('\n') {
        Some(off) => version_start + off + 1,
        None => return Err(anyhow!("could not find newline after version line")),
    };

    let (before, after) = content.split_at(nl_off);

    let mut content = before.to_string();
    content.push_str("build = \"build.rs\"\n");
    content.push_str(after);

    content.push_str(&format!(
        "pyembed = {{ {}, default-features = false }}\n",
        pyembed_location.cargo_manifest_fields()
    ));
    content.push('\n');

    let data = TemplateData::new();
    content.push_str(
        &HANDLEBARS
            .render("cargo-extra.toml", &data)
            .context("rendering cargo-extra.toml template")?,
    );

    std::fs::write(path, content)?;

    Ok(())
}

/// Initialize a new Rust project using PyOxidizer.
///
/// The created binary application will have the name of the final
/// path component.
///
/// `windows_subsystem` is the value of the `windows_subsystem` compiler
/// attribute.
pub fn initialize_project(
    source: &PyOxidizerSource,
    project_path: &Path,
    cargo_exe: &Path,
    code: Option<&str>,
    pip_install: &[&str],
    windows_subsystem: &str,
) -> Result<()> {
    let status = std::process::Command::new(cargo_exe)
        .arg("init")
        .arg("--bin")
        .arg(project_path)
        .status()
        .context("invoking cargo init")?;

    if !status.success() {
        return Err(anyhow!("cargo init failed"));
    }

    let path = PathBuf::from(project_path);
    let name = path.iter().last().unwrap().to_str().unwrap();
    add_pyoxidizer(&path, true).context("adding PyOxidizer to Rust project")?;
    update_new_cargo_toml(&path.join("Cargo.toml"), &source.as_pyembed_location())
        .context("updating Cargo.toml")?;
    write_new_cargo_config(&path).context("writing cargo config")?;
    write_new_cargo_lock(&path, name).context("writing Cargo.lock")?;
    write_new_build_rs(&path.join("build.rs"), name).context("writing build.rs")?;
    write_new_main_rs(&path.join("src").join("main.rs"), windows_subsystem)
        .context("writing main.rs")?;
    write_new_pyoxidizer_config_file(source, &path, &name, code, pip_install)
        .context("writing PyOxidizer config file")?;
    write_application_manifest(&path, &name).context("writing application manifest")?;

    Ok(())
}